From 244ab44ccf1a3feb2fc0fcc8d038e055bf13078e Mon Sep 17 00:00:00 2001 From: Dillen Meijboom Date: Mon, 9 Nov 2026 20:48:22 +0000 Subject: [PATCH 01/10] feat: recognize projects with prioritized declarative rules --- README.md | 39 +++ src/cmd/mod.rs | 10 +- src/describe.rs | 9 +- src/detect/mod.rs | 474 +++++++++++++++++++++++++++++ src/detect/parsers.rs | 571 +++++++++++++++++++++++++++++++++++ src/detect/rules.rs | 304 +++++++++++++++++++ src/detect/rules.toml | 335 +++++++++++++++++++++ src/detect/tests.rs | 686 ++++++++++++++++++++++++++++++++++++++++++ src/main.rs | 5 + src/output.rs | 2 + 10 files changed, 2433 insertions(+), 2 deletions(-) create mode 100644 src/detect/mod.rs create mode 100644 src/detect/parsers.rs create mode 100644 src/detect/rules.rs create mode 100644 src/detect/rules.toml create mode 100644 src/detect/tests.rs diff --git a/README.md b/README.md index 68d09d1..2bc3d3b 100644 --- a/README.md +++ b/README.md @@ -168,6 +168,7 @@ Errors go to stderr and return a non-zero exit code. With `--json`, errors also ## Commands ```text +brainpod detect [] [--rules ] brainpod describe [...] brainpod describe resource [] brainpod login @@ -263,6 +264,44 @@ Pod-scoped commands use `--pod`, `BRAINPOD_POD`, or the configured default pod. `revision wait` polls revision details until every resource reports `healthy: true`. `deploy --wait` deploys first, then waits on the returned revision in the same way. Both use a 90-second timeout by default; pass `--timeout ` to override it. Interactive waits report each unhealthy-to-healthy transition on stderr. Progress is suppressed when stderr is redirected or `--json` is used. Failed or canceled revisions stop the wait immediately, and timeouts report the resources that are still unhealthy. +## Project detection + +Inspect a directory without credentials, network access, subprocesses, or executing project code: + +```sh +brainpod detect . +brainpod detect ./services --json +brainpod detect . --rules ./detection.toml --json +``` + +The result contains sorted project paths, manifest names, languages and version declarations, priority-ranked detections with evidence, workspace members, diagnostics, and scan statistics. Framework versions are attached to dependency evidence. JavaScript versions describe Node.js; TypeScript versions describe the compiler. Requirements and pins retain their source and kind: these are declarations, not resolved or installed versions. Missing versions remain unknown. + +Built-in coverage includes Laravel, Symfony, CakePHP, CodeIgniter, Composer; Go modules/workspaces and Gin/Echo/Fiber; Cargo workspaces, axum, Rocket, Actix; Bundler, Rails, Sinatra, Hanami; Python requirements, PEP 621, Poetry, Pipenv, uv workspaces, Django, FastAPI, Flask, Starlette, Celery; npm workspaces, Next.js, Nuxt, Astro, SvelteKit, Remix, Angular, Express, React, Vue, Vite; and standalone source files in all seven languages. Rust workspace dependency aliases and inherited `rust-version` are understood. Ancestor runtime pin/toolchain files are included with provenance. + +Rules live in [`src/detect/rules.toml`](src/detect/rules.toml), independently of manifest parsers. Additional rules use unique IDs: + +```toml +[[rules]] +id = "company-next" +language = "javascript" +kind = "framework" +priority = 200 +all = ["dep:npm:next", "file:company.config.*"] +none = ["file:legacy.config"] +``` + +`all` requires every pattern, `any` requires at least one when supplied, and `none` rejects any match. Higher priorities sort first; ties sort by ID. Matches coexist rather than hiding other frameworks or languages. Patterns support `file:` (`*`, `?`), `source:`, `dep::`, `workspace:`, and `package:cargo`. Workspace paths additionally support `**`. Ecosystems are `composer`, `go`, `cargo`, `gem`, `pypi`, and `npm`. + +The scanner walks once, caches indexed rules, and never reads source bodies or lockfiles. Each manifest/version file is capped at 1 MiB. Symlinks are not followed; traversal stops at depth 128 with a diagnostic. Dependency/build directories such as `.git`, `.jj`, `node_modules`, `vendor`, `target`, `.venv`, `.next`, `dist`, and `build` are pruned. It does not evaluate `.gitignore`. Source-only directories roll up to a same-language project ancestor when available; sibling services keep their own manifest boundaries. + +Detection uses declared dependencies, not arbitrary substring matches or inferred framework usage from source. Composer/Cargo development dependencies are excluded; npm development dependencies are included because tools such as Astro commonly live there. Dynamic Ruby declarations and unsupported requirements options produce diagnostics instead of executing code. Invalid manifests are diagnosed without stopping discovery of other projects. + +Run the generated large-tree performance fixture with: + +```sh +cargo test --release detect::tests::benchmark_polyglot_scan -- --ignored --nocapture +``` + ## Image discovery Image commands require a pod and API token with `registry:pull` permission. List returns active public images and images in the selected pod: diff --git a/src/cmd/mod.rs b/src/cmd/mod.rs index 6541d08..ae6c968 100644 --- a/src/cmd/mod.rs +++ b/src/cmd/mod.rs @@ -17,6 +17,8 @@ use crate::output::{CommandOutput, View}; pub enum Command { /// Describe CLI commands and their machine-readable contract Describe(DescribeArgs), + /// Detect local projects, frameworks, languages, and version constraints + Detect(crate::detect::DetectArgs), /// Drive the session console the user watches while a workflow runs Agent(crate::agent::AgentArgs), /// Authenticate through the Brainpod dashboard @@ -493,7 +495,11 @@ fn is_event_resource_urn(value: &str) -> bool { pub fn needs_api_token(command: &Command) -> bool { !matches!( command, - Command::Describe(_) | Command::Agent(_) | Command::Login(_) | Command::Config(_) + Command::Describe(_) + | Command::Detect(_) + | Command::Agent(_) + | Command::Login(_) + | Command::Config(_) ) } @@ -502,6 +508,7 @@ pub fn needs_client(command: &Command) -> bool { command, Command::S3(_) | Command::Describe(_) + | Command::Detect(_) | Command::Agent(_) | Command::Login(_) | Command::Config(_) @@ -543,6 +550,7 @@ pub async fn handle(command: Command, context: CommandContext<'_>) -> Result crate::detect::handle(args), Command::Agent(args) => crate::agent::handle(args, pod, dashboard_endpoint, json).await, Command::Login(args) => Ok(CommandOutput::new( crate::auth::login( diff --git a/src/describe.rs b/src/describe.rs index f5b0405..fd451cd 100644 --- a/src/describe.rs +++ b/src/describe.rs @@ -246,7 +246,9 @@ fn argument_syntax(argument: &Arg, value_names: &[String]) -> String { fn requirements(path: &[&str]) -> (Option, Option) { match path { [] => (None, None), - ["describe"] | ["login"] | ["config"] | ["config", _] => (Some(false), Some(false)), + ["describe"] | ["detect"] | ["login"] | ["config"] | ["config", _] => { + (Some(false), Some(false)) + } ["blueprint"] => (Some(true), None), ["blueprint", "install"] => (Some(true), Some(true)), ["blueprint", _] | ["whoami"] | ["cluster"] | ["cluster", _] | ["pod"] | ["pod", _] => { @@ -269,6 +271,10 @@ fn requirements(path: &[&str]) -> (Option, Option) { fn effect(path: &[&str]) -> (&'static str, &'static str) { match path { + ["detect"] => ( + "read", + "Reads local project manifests without executing project code or accessing the network.", + ), ["describe"] | ["config", "show"] | ["config", "path"] => ( "read", "Reads local CLI information without changing state.", @@ -376,6 +382,7 @@ fn next_steps(path: &[&str]) -> Vec<&'static str> { fn examples(path: &[&str]) -> Vec<&'static str> { match path { + ["detect"] => vec!["brainpod detect .", "brainpod detect ./services --json"], ["describe"] => vec![ "brainpod describe", "brainpod describe resource create", diff --git a/src/detect/mod.rs b/src/detect/mod.rs new file mode 100644 index 0000000..b6d781d --- /dev/null +++ b/src/detect/mod.rs @@ -0,0 +1,474 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::fs::{self, File}; +use std::io::Read; +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result, bail}; +use clap::Args; +use serde::Serialize; + +mod parsers; +mod rules; +#[cfg(test)] +mod tests; + +use rules::{Kind, Rules}; + +const MAX_FILE_BYTES: u64 = 1024 * 1024; +const MAX_DEPTH: usize = 128; +const IGNORED: &[&str] = &[ + ".git", + ".jj", + ".svn", + "node_modules", + "vendor", + "target", + ".venv", + "venv", + "__pycache__", + ".next", + ".nuxt", + ".output", + "dist", + "build", + "coverage", + ".bundle", + ".brainpod", + ".cache", + "site-packages", +]; + +#[derive(Debug, Args)] +pub struct DetectArgs { + /// Repository or project directory to inspect without executing project code + #[arg(default_value = ".")] + pub path: PathBuf, + /// Additional TOML detection rules; IDs must be unique + #[arg(long, value_name = "PATH")] + pub rules: Option, +} + +#[derive(Debug, Serialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct Report { + pub schema_version: u32, + pub projects: Vec, + pub diagnostics: Vec, + pub stats: Stats, +} + +#[derive(Debug, Serialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct Project { + pub path: String, + pub names: Vec, + pub languages: Vec, + pub detections: Vec, + pub workspace_members: Vec, +} + +#[derive(Debug, Serialize, PartialEq, Eq)] +pub struct Language { + pub name: String, + pub versions: Vec, +} + +#[derive(Clone, Debug, Serialize, PartialEq, Eq, PartialOrd, Ord)] +pub struct Version { + pub value: String, + pub source: String, + pub kind: String, +} + +#[derive(Debug, Serialize, PartialEq, Eq)] +pub struct Detection { + pub id: String, + pub language: String, + pub kind: Kind, + pub priority: i32, + pub evidence: Vec, +} + +#[derive(Clone, Debug, Serialize, PartialEq, Eq, PartialOrd, Ord)] +pub struct Evidence { + pub source: String, + pub pattern: String, + pub value: Option, +} + +#[derive(Debug, Serialize, PartialEq, Eq)] +pub struct Diagnostic { + pub path: String, + pub message: String, +} + +#[derive(Debug, Default, Serialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct Stats { + pub directories: usize, + pub files: usize, + pub files_read: usize, + pub bytes_read: usize, + pub skipped_directories: usize, +} + +#[derive(Default)] +struct Node { + path: PathBuf, + files: BTreeSet, + contents: BTreeMap, + sources: BTreeMap, + dependencies: BTreeMap, + packages: BTreeSet, + versions: BTreeMap>, + names: BTreeSet, + workspaces: Vec, +} + +struct Dependency { + version: Option, + source: String, +} + +struct Workspace { + ecosystem: String, + source: String, + members: Vec, + exclude: Vec, +} + +impl Node { + fn dependency(&mut self, ecosystem: &str, name: &str, version: Option<&str>, file: &str) { + self.dependencies.insert( + format!("{ecosystem}:{name}"), + Dependency { + version: version.map(str::to_owned), + source: relative_file(&self.path, file), + }, + ); + } + + fn version(&mut self, language: &str, value: &str, file: &str, kind: &str) { + if !value.trim().is_empty() { + self.versions + .entry(language.to_owned()) + .or_default() + .push(Version { + value: value.trim().to_owned(), + source: relative_file(&self.path, file), + kind: kind.to_owned(), + }); + } + } +} + +pub fn handle(args: DetectArgs) -> Result { + let extra = args + .rules + .as_deref() + .map(fs::read_to_string) + .transpose() + .context("could not read detection rules")?; + let report = scan(&args.path, extra.as_deref())?; + Ok(crate::output::CommandOutput::new( + serde_json::to_value(report)?, + crate::output::View::Detect, + )) +} + +pub fn scan(root: &Path, extra_rules: Option<&str>) -> Result { + let custom = extra_rules + .map(|extra| Rules::new(Some(extra))) + .transpose()?; + let rules = match custom.as_ref() { + Some(rules) => rules, + None => Rules::builtin()?, + }; + let root = root + .canonicalize() + .with_context(|| format!("could not open {}", root.display()))?; + if !root.is_dir() { + bail!("detection root is not a directory: {}", root.display()); + } + let mut report = Report { + schema_version: 1, + projects: Vec::new(), + diagnostics: Vec::new(), + stats: Stats::default(), + }; + let mut nodes = Vec::new(); + let mut pending = vec![(PathBuf::new(), 0)]; + while let Some((path, depth)) = pending.pop() { + let mut node = Node { + path: path.clone(), + ..Node::default() + }; + report.stats.directories += 1; + let entries = fs::read_dir(root.join(&path)) + .with_context(|| format!("could not scan {}", root.join(&path).display()))?; + let mut children = Vec::new(); + for entry in entries { + let entry = entry?; + let ty = entry.file_type()?; + let Some(name) = entry.file_name().to_str().map(str::to_owned) else { + continue; + }; + if ty.is_dir() { + if IGNORED.contains(&name.as_str()) { + report.stats.skipped_directories += 1; + } else if depth == MAX_DEPTH { + report.diagnostics.push(Diagnostic { + path: relative_file(&path, &name), + message: "directory depth limit reached".to_owned(), + }); + } else { + children.push((path.join(&name), depth + 1)); + } + } else if ty.is_file() { + report.stats.files += 1; + if let Some(ext) = Path::new(&name).extension().and_then(|e| e.to_str()) { + node.sources + .entry(ext.to_owned()) + .and_modify(|existing| { + if name < *existing { + *existing = name.clone(); + } + }) + .or_insert_with(|| name.clone()); + } + if parsers::readable(&name) { + let file = File::open(entry.path())?; + let mut bytes = Vec::new(); + file.take(MAX_FILE_BYTES + 1).read_to_end(&mut bytes)?; + report.stats.files_read += 1; + report.stats.bytes_read += bytes.len(); + let content = if bytes.len() > MAX_FILE_BYTES as usize { + Err("manifest exceeds 1 MiB read limit".to_owned()) + } else { + String::from_utf8(bytes).map_err(|_| "manifest is not UTF-8".to_owned()) + }; + match content { + Ok(content) => { + node.contents.insert(name.clone(), content); + } + Err(message) => report.diagnostics.push(Diagnostic { + path: relative_file(&path, &name), + message, + }), + } + } + if parsers::readable(&name) || rules.needs_filename(&name) { + node.files.insert(name); + } + } + } + children.sort(); + pending.extend(children.into_iter().rev()); + nodes.push(node); + } + parsers::parse(&mut nodes, &mut report.diagnostics); + + let mut projects = BTreeMap::::new(); + let mut source_detections = Vec::new(); + for node in &nodes { + let mut detections = Vec::new(); + for detection in rules.evaluate(node) { + if detection.kind == Kind::Source { + source_detections.push((&node.path, detection)); + } else { + detections.push(detection); + } + } + if !detections.is_empty() { + projects.insert( + node.path.clone(), + Project { + path: display_path(&node.path), + names: node.names.iter().cloned().collect(), + languages: Vec::new(), + detections, + workspace_members: Vec::new(), + }, + ); + } + } + for (path, detection) in source_detections { + let owner = path + .ancestors() + .find(|p| { + projects.get(*p).is_some_and(|project| { + project.detections.iter().any(|d| { + d.kind != Kind::Workspace + && family(&d.language) == family(&detection.language) + }) + }) + }) + .map(Path::to_path_buf) + .unwrap_or_else(|| path.clone()); + let project = projects.entry(owner.clone()).or_insert_with(|| Project { + path: display_path(&owner), + names: Vec::new(), + languages: Vec::new(), + detections: Vec::new(), + workspace_members: Vec::new(), + }); + if !project + .detections + .iter() + .any(|d| d.language == detection.language) + { + project.detections.push(detection); + } + } + for (path, project) in &mut projects { + let languages = project + .detections + .iter() + .map(|d| d.language.clone()) + .collect::>(); + for language in languages { + let mut versions = Vec::new(); + for ancestor in path.ancestors() { + if let Ok(i) = nodes.binary_search_by(|n| n.path.as_path().cmp(ancestor)) { + let node = &nodes[i]; + if let Some(found) = node.versions.get(&language) { + versions.extend( + found + .iter() + .filter(|v| { + ancestor == path || v.kind == "pin" || v.kind == "toolchain" + }) + .cloned(), + ); + } + } + } + let mut seen = BTreeSet::new(); + versions.retain(|v| seen.insert(v.clone())); + project.languages.push(Language { + name: language, + versions, + }); + } + project + .detections + .sort_by(|a, b| b.priority.cmp(&a.priority).then(a.id.cmp(&b.id))); + } + let paths = projects.keys().cloned().collect::>(); + for node in &nodes { + let Some(project) = projects.get_mut(&node.path) else { + continue; + }; + for workspace in &node.workspaces { + for member in &paths { + if member == &node.path { + continue; + } + let Ok(relative) = member.strip_prefix(&node.path) else { + continue; + }; + let relative = display_path(relative); + let is_member = workspace + .members + .iter() + .any(|p| rules::path_glob(p, &relative)); + let excluded = workspace + .exclude + .iter() + .any(|p| rules::path_glob(p, &relative)); + let same_language = projects_language(&nodes, member, &workspace.ecosystem); + if is_member && !excluded && same_language { + project.workspace_members.push(display_path(member)); + } + } + } + project.workspace_members.sort(); + project.workspace_members.dedup(); + } + report.projects = projects.into_values().collect(); + report + .diagnostics + .sort_by(|a, b| a.path.cmp(&b.path).then(a.message.cmp(&b.message))); + Ok(report) +} + +fn projects_language(nodes: &[Node], path: &Path, ecosystem: &str) -> bool { + let file = match ecosystem { + "cargo" => "Cargo.toml", + "go" => "go.mod", + "node" => "package.json", + "python" => "pyproject.toml", + _ => return false, + }; + nodes + .binary_search_by(|n| n.path.as_path().cmp(path)) + .ok() + .is_some_and(|i| nodes[i].files.contains(file)) +} + +fn family(language: &str) -> &str { + match language { + "typescript" | "javascript" => "node", + other => other, + } +} + +fn display_path(path: &Path) -> String { + if path.as_os_str().is_empty() { + ".".to_owned() + } else { + path.to_string_lossy().replace('\\', "/") + } +} + +fn relative_file(path: &Path, name: &str) -> String { + display_path(&path.join(name)) +} + +pub fn render(value: &serde_json::Value) -> Vec { + let mut lines = Vec::new(); + if let Some(projects) = value["projects"].as_array() { + for project in projects { + let languages = project["languages"] + .as_array() + .into_iter() + .flatten() + .filter_map(|l| l["name"].as_str()) + .collect::>(); + let detections = project["detections"] + .as_array() + .into_iter() + .flatten() + .filter_map(|d| d["id"].as_str()) + .collect::>(); + lines.push(format!( + "{}\t{}\t{}", + project["path"].as_str().unwrap_or("."), + languages.join(", "), + detections.join(", ") + )); + for language in project["languages"].as_array().into_iter().flatten() { + for version in language["versions"].as_array().into_iter().flatten() { + lines.push(format!( + " {} {} ({}, {})", + language["name"].as_str().unwrap_or(""), + version["value"].as_str().unwrap_or(""), + version["kind"].as_str().unwrap_or(""), + version["source"].as_str().unwrap_or("") + )); + } + } + } + } + for diagnostic in value["diagnostics"].as_array().into_iter().flatten() { + lines.push(format!( + "warning: {}: {}", + diagnostic["path"].as_str().unwrap_or(""), + diagnostic["message"].as_str().unwrap_or("") + )); + } + if lines.is_empty() { + lines.push("No projects detected".to_owned()); + } + lines +} diff --git a/src/detect/parsers.rs b/src/detect/parsers.rs new file mode 100644 index 0000000..b58f755 --- /dev/null +++ b/src/detect/parsers.rs @@ -0,0 +1,571 @@ +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; + +use serde_json::Value; + +use super::{Diagnostic, Node, Workspace, relative_file}; + +pub fn readable(name: &str) -> bool { + matches!( + name, + "composer.json" + | "package.json" + | "Cargo.toml" + | "go.mod" + | "go.work" + | "Gemfile" + | "pyproject.toml" + | "Pipfile" + | ".go-version" + | ".php-version" + | ".ruby-version" + | ".python-version" + | ".node-version" + | ".nvmrc" + | ".tool-versions" + | "rust-toolchain" + | "rust-toolchain.toml" + ) || (name.starts_with("requirements") && name.ends_with(".txt")) +} + +pub fn parse(nodes: &mut [Node], diagnostics: &mut Vec) { + let mut documents = BTreeMap::new(); + for node in nodes.iter() { + for (file, content) in &node.contents { + let parsed = match file.as_str() { + "composer.json" | "package.json" => { + Some(serde_json::from_str::(content).map_err(|e| e.to_string())) + } + "Cargo.toml" | "pyproject.toml" | "Pipfile" | "rust-toolchain.toml" => { + Some(toml::from_str::(content).map_err(|e| e.to_string())) + } + _ => None, + }; + if let Some(parsed) = parsed { + match parsed { + Ok(doc) if doc.is_object() => { + documents.insert((node.path.clone(), file.clone()), doc); + } + Ok(_) => diagnostics.push(Diagnostic { + path: relative_file(&node.path, file), + message: "manifest must be an object".to_owned(), + }), + Err(message) => diagnostics.push(Diagnostic { + path: relative_file(&node.path, file), + message, + }), + } + } + } + } + for node in nodes { + for file in [ + "composer.json", + "package.json", + "Cargo.toml", + "pyproject.toml", + "Pipfile", + ] { + let Some(doc) = documents.get(&(node.path.clone(), file.to_owned())) else { + continue; + }; + if let Some(name) = doc + .get("name") + .or_else(|| doc.pointer("/package/name")) + .or_else(|| doc.pointer("/project/name")) + .and_then(Value::as_str) + { + node.names.insert(name.to_owned()); + } + match file { + "composer.json" => composer(node, doc), + "package.json" => javascript(node, doc), + "Cargo.toml" => cargo(node, doc, &documents), + "pyproject.toml" => python(node, doc), + "Pipfile" => { + dependencies(node, doc.get("packages"), "pypi", file); + if let Some(v) = doc + .pointer("/requires/python_full_version") + .or_else(|| doc.pointer("/requires/python_version")) + .and_then(Value::as_str) + { + node.version("python", v, file, "requirement"); + } + } + _ => (), + } + } + let contents = std::mem::take(&mut node.contents); + for (file, text) in &contents { + match file.as_str() { + "go.mod" | "go.work" => go(node, file, text), + "Gemfile" => ruby(node, text, diagnostics), + ".go-version" => pin(node, "go", file, text), + ".php-version" => pin(node, "php", file, text), + ".ruby-version" => pin(node, "ruby", file, text), + ".python-version" => pin(node, "python", file, text), + ".node-version" | ".nvmrc" => pin(node, "javascript", file, text), + "rust-toolchain" => pin(node, "rust", file, text), + "rust-toolchain.toml" => { + if let Some(v) = documents + .get(&(node.path.clone(), file.clone())) + .and_then(|d| d.pointer("/toolchain/channel")) + .and_then(Value::as_str) + { + node.version("rust", v, file, "toolchain"); + } + } + ".tool-versions" => { + for line in text.lines() { + let mut words = line.split('#').next().unwrap_or("").split_whitespace(); + let language = match words.next() { + Some("nodejs") => "javascript", + Some("golang") => "go", + Some("python") => "python", + Some("ruby") => "ruby", + Some("rust") => "rust", + Some("php") => "php", + _ => continue, + }; + let value = words.collect::>().join(" "); + node.version(language, &value, file, "pin"); + } + } + f if f.starts_with("requirements") && f.ends_with(".txt") => { + for line in logical_lines(text) { + let line = line.trim(); + if line.starts_with('-') { + if !line.starts_with("--hash=") { + diagnostics.push(Diagnostic { + path: relative_file(&node.path, file), + message: format!("requirements option not evaluated: {line}"), + }); + } + continue; + } + requirement(node, line, file); + } + } + _ => (), + } + } + } +} + +fn dependencies(node: &mut Node, object: Option<&Value>, ecosystem: &str, file: &str) { + if let Some(deps) = object.and_then(Value::as_object) { + for (name, value) in deps { + let name = if ecosystem == "pypi" { + normalize_python(name) + } else { + name.clone() + }; + let version = value + .as_str() + .or_else(|| value.get("version").and_then(Value::as_str)); + node.dependency(ecosystem, &name, version, file); + } + } +} + +fn composer(node: &mut Node, doc: &Value) { + dependencies(node, doc.get("require"), "composer", "composer.json"); + if let Some(v) = doc.pointer("/require/php").and_then(Value::as_str) { + node.version("php", v, "composer.json", "requirement"); + } + if let Some(v) = doc.pointer("/config/platform/php").and_then(Value::as_str) { + node.version("php", v, "composer.json", "platform"); + } +} + +fn javascript(node: &mut Node, doc: &Value) { + dependencies(node, doc.get("devDependencies"), "npm", "package.json"); + dependencies(node, doc.get("dependencies"), "npm", "package.json"); + if let Some(v) = doc.pointer("/engines/node").and_then(Value::as_str) { + node.version("javascript", v, "package.json", "requirement"); + } + if let Some(v) = doc.pointer("/volta/node").and_then(Value::as_str) { + node.version("javascript", v, "package.json", "pin"); + } + if let Some(v) = doc + .pointer("/devDependencies/typescript") + .or_else(|| doc.pointer("/dependencies/typescript")) + .and_then(Value::as_str) + { + node.version("typescript", v, "package.json", "requirement"); + } + if let Some(workspaces) = doc.get("workspaces") { + let members = strings(workspaces.get("packages").unwrap_or(workspaces)); + node.workspaces.push(Workspace { + ecosystem: "node".to_owned(), + source: relative_file(&node.path, "package.json"), + members, + exclude: Vec::new(), + }); + } +} + +type Documents = BTreeMap<(PathBuf, String), Value>; + +fn cargo(node: &mut Node, doc: &Value, documents: &Documents) { + let file = "Cargo.toml"; + if doc.get("package").is_some_and(Value::is_object) { + node.packages.insert("cargo".to_owned()); + } + let workspace = + if let Some(explicit) = doc.pointer("/package/workspace").and_then(Value::as_str) { + normalize_path(&node.path.join(explicit)).and_then(|path| { + documents + .get(&(path.clone(), file.to_owned())) + .filter(|d| d.get("workspace").is_some()) + .map(|d| (path, d)) + }) + } else { + node.path.ancestors().find_map(|path| { + documents + .get(&(path.to_path_buf(), file.to_owned())) + .filter(|d| d.get("workspace").is_some()) + .map(|d| (path.to_path_buf(), d)) + }) + }; + if let Some(version) = doc.pointer("/package/rust-version") { + if let Some(v) = version.as_str() { + node.version("rust", v, file, "requirement"); + } else if version.get("workspace").and_then(Value::as_bool) == Some(true) + && let Some((path, ws)) = &workspace + && let Some(v) = ws + .pointer("/workspace/package/rust-version") + .and_then(Value::as_str) + { + node.versions + .entry("rust".to_owned()) + .or_default() + .push(super::Version { + value: v.to_owned(), + source: relative_file(path, file), + kind: "requirement".to_owned(), + }); + } + } + let mut dep_tables = Vec::new(); + if let Some(d) = doc.get("dependencies") { + dep_tables.push(d); + } + if let Some(targets) = doc.get("target").and_then(Value::as_object) { + dep_tables.extend(targets.values().filter_map(|v| v.get("dependencies"))); + } + for table in dep_tables { + if let Some(deps) = table.as_object() { + for (alias, declaration) in deps { + let inherited = declaration.get("workspace").and_then(Value::as_bool) == Some(true); + let (value, source) = if inherited { + let Some((path, ws)) = &workspace else { + continue; + }; + let Some(value) = ws + .pointer("/workspace/dependencies") + .and_then(|d| d.get(alias)) + else { + continue; + }; + (value, relative_file(path, file)) + } else { + (declaration, relative_file(&node.path, file)) + }; + let name = value + .get("package") + .and_then(Value::as_str) + .unwrap_or(alias); + let version = value + .as_str() + .or_else(|| value.get("version").and_then(Value::as_str)) + .map(str::to_owned); + node.dependencies.insert( + format!("cargo:{name}"), + super::Dependency { version, source }, + ); + } + } + } + if let Some(ws) = doc.get("workspace") { + node.workspaces.push(Workspace { + ecosystem: "cargo".to_owned(), + source: relative_file(&node.path, file), + members: strings(&ws["members"]), + exclude: strings(&ws["exclude"]), + }); + } +} + +fn python(node: &mut Node, doc: &Value) { + let file = "pyproject.toml"; + if let Some(v) = doc + .pointer("/project/requires-python") + .and_then(Value::as_str) + { + node.version("python", v, file, "requirement"); + } + for dep in doc + .pointer("/project/dependencies") + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(Value::as_str) + { + requirement(node, dep, file); + } + if let Some(deps) = doc + .pointer("/tool/poetry/dependencies") + .and_then(Value::as_object) + { + for (name, v) in deps { + let version = v + .as_str() + .or_else(|| v.get("version").and_then(Value::as_str)); + if name == "python" { + if let Some(v) = version { + node.version("python", v, file, "requirement"); + } + } else { + node.dependency("pypi", &normalize_python(name), version, file); + } + } + } + if let Some(ws) = doc.pointer("/tool/uv/workspace") { + node.workspaces.push(Workspace { + ecosystem: "python".to_owned(), + source: relative_file(&node.path, file), + members: strings(&ws["members"]), + exclude: strings(&ws["exclude"]), + }); + } +} + +fn normalize_python(name: &str) -> String { + let mut normalized = String::new(); + for ch in name.chars() { + if matches!(ch, '-' | '_' | '.') { + if !normalized.ends_with('-') { + normalized.push('-'); + } + } else { + normalized.push(ch.to_ascii_lowercase()); + } + } + normalized +} + +fn requirement(node: &mut Node, line: &str, file: &str) { + let line = line.split('#').next().unwrap_or("").trim(); + if line.is_empty() { + return; + } + let end = line + .find(|c: char| !c.is_ascii_alphanumeric() && !matches!(c, '-' | '_' | '.')) + .unwrap_or(line.len()); + if end == 0 { + return; + } + let name = normalize_python(&line[..end]); + let mut rest = line[end..].trim(); + if rest.starts_with('[') { + let Some(end) = rest.find(']') else { + return; + }; + rest = rest[end + 1..].trim(); + } + let constraint = rest.split(';').next().unwrap_or("").trim(); + let constraint = constraint.split(" --hash=").next().unwrap_or("").trim(); + let version = if constraint.is_empty() || constraint.starts_with('@') { + None + } else { + Some(constraint) + }; + node.dependency("pypi", &name, version, file); +} + +fn logical_lines(text: &str) -> Vec { + let mut lines = Vec::new(); + let mut current = String::new(); + for line in text.lines() { + let line = line.trim_end(); + if let Some(part) = line.strip_suffix('\\') { + current.push_str(part); + current.push(' '); + } else { + current.push_str(line); + lines.push(std::mem::take(&mut current)); + } + } + if !current.is_empty() { + lines.push(current); + } + lines +} + +fn go(node: &mut Node, file: &str, text: &str) { + let mut block = ""; + let mut members = Vec::new(); + for line in text.lines() { + if line + .split_once("//") + .is_some_and(|(_, comment)| comment.trim() == "indirect") + { + continue; + } + let line = line.split("//").next().unwrap_or("").trim(); + if line.is_empty() { + continue; + } + if line == ")" { + block = ""; + continue; + } + let words = line.split_whitespace().collect::>(); + let (directive, values) = if block.is_empty() { + (words[0], &words[1..]) + } else { + (block, words.as_slice()) + }; + if values == ["("] { + block = directive; + continue; + } + match directive { + "go" => { + if let Some(v) = values.first() { + node.version("go", v, file, "requirement"); + } + } + "toolchain" => { + if let Some(v) = values.first() { + node.version("go", v.trim_start_matches("go"), file, "toolchain"); + } + } + "module" => { + if let Some(name) = values.first() { + node.names.insert(name.trim_matches('"').to_owned()); + } + } + "require" => { + if let Some(name) = values.first() { + node.dependency("go", name, values.get(1).copied(), file); + } + } + "use" => { + if let Some(path) = values.first() { + members.push(path.trim_matches('"').trim_start_matches("./").to_owned()); + } + } + _ => (), + } + } + if file == "go.work" { + node.workspaces.push(Workspace { + ecosystem: "go".to_owned(), + source: relative_file(&node.path, file), + members, + exclude: Vec::new(), + }); + } +} + +fn quoted(input: &str) -> Option<(&str, &str)> { + let input = input.trim_start(); + let quote = input.as_bytes().first().copied()?; + if quote != b'\'' && quote != b'"' { + return None; + } + let end = input[1..].find(char::from(quote))? + 1; + let value = &input[1..end]; + if value.contains("#{") || value.contains('\\') { + return None; + } + Some((value, &input[end + 1..])) +} + +fn ruby(node: &mut Node, text: &str, diagnostics: &mut Vec) { + for line in text.lines() { + let line = line.trim(); + let (directive, rest) = if let Some(rest) = line + .strip_prefix("gem ") + .or_else(|| line.strip_prefix("gem(")) + .or_else(|| line.strip_prefix("gem\t")) + { + ("gem", rest) + } else if let Some(rest) = line + .strip_prefix("ruby ") + .or_else(|| line.strip_prefix("ruby(")) + .or_else(|| line.strip_prefix("ruby\t")) + { + ("ruby", rest) + } else { + continue; + }; + let Some((name, rest)) = quoted(rest) else { + diagnostics.push(Diagnostic { + path: relative_file(&node.path, "Gemfile"), + message: format!("dynamic {directive} declaration not evaluated"), + }); + continue; + }; + if directive == "ruby" { + node.version("ruby", name, "Gemfile", "requirement"); + } else { + let mut versions = Vec::new(); + let mut rest = rest.trim_start(); + while let Some(next) = rest.strip_prefix(',') { + let Some((version, tail)) = quoted(next) else { + break; + }; + versions.push(version); + rest = tail.trim_start(); + } + let version = versions.join(", "); + node.dependency( + "gem", + name, + (!version.is_empty()).then_some(version.as_str()), + "Gemfile", + ); + } + } +} + +fn strings(value: &Value) -> Vec { + value + .as_array() + .into_iter() + .flatten() + .filter_map(Value::as_str) + .map(str::to_owned) + .collect() +} + +fn pin(node: &mut Node, language: &str, file: &str, text: &str) { + let value = text + .lines() + .filter(|line| !line.trim().starts_with('#')) + .collect::>() + .join(" "); + node.version(language, &value, file, "pin"); +} + +fn normalize_path(path: &Path) -> Option { + if path.is_absolute() { + return None; + } + let mut normalized = PathBuf::new(); + for component in path.components() { + match component { + std::path::Component::ParentDir => { + if !normalized.pop() { + return None; + } + } + std::path::Component::CurDir => (), + other => normalized.push(other), + } + } + Some(normalized) +} diff --git a/src/detect/rules.rs b/src/detect/rules.rs new file mode 100644 index 0000000..f2955ff --- /dev/null +++ b/src/detect/rules.rs @@ -0,0 +1,304 @@ +use std::collections::{BTreeSet, HashMap}; +use std::sync::OnceLock; + +use anyhow::{Result, bail}; +use serde::Deserialize; + +use super::{Detection, Evidence, Node}; + +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Rule { + pub id: String, + pub language: String, + pub kind: Kind, + pub priority: i32, + #[serde(default)] + pub any: Vec, + #[serde(default)] + pub all: Vec, + #[serde(default)] + pub none: Vec, +} + +#[derive(Clone, Copy, Debug, Deserialize, serde::Serialize, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +pub enum Kind { + Framework, + Project, + Workspace, + Source, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Document { + rules: Vec, +} + +pub struct Rules { + rules: Vec, + files: HashMap>, + sources: HashMap>, + dependencies: HashMap>, + workspaces: HashMap>, + packages: HashMap>, + wildcard_files: Vec<(String, usize)>, + file_patterns: BTreeSet, + wildcard_interest: BTreeSet, +} + +impl Rules { + pub fn builtin() -> Result<&'static Self> { + static RULES: OnceLock> = OnceLock::new(); + RULES + .get_or_init(|| Self::new(None).map_err(|e| e.to_string())) + .as_ref() + .map_err(|e| anyhow::anyhow!("{e}")) + } + + pub fn new(extra: Option<&str>) -> Result { + let mut document: Document = toml::from_str(include_str!("rules.toml"))?; + if let Some(extra) = extra { + document + .rules + .extend(toml::from_str::(extra)?.rules); + } + let mut ids = BTreeSet::new(); + for rule in &document.rules { + if rule.id.is_empty() || rule.language.is_empty() || !ids.insert(&rule.id) { + bail!("empty or duplicate detection rule: {}", rule.id); + } + if rule.any.is_empty() && rule.all.is_empty() { + bail!("rule {} needs an any or all pattern", rule.id); + } + for pattern in rule.any.iter().chain(&rule.all).chain(&rule.none) { + let valid = match pattern.split_once(':') { + Some(("file" | "source", value)) => !value.is_empty() && !value.contains('/'), + Some(("dep", value)) => value + .split_once(':') + .is_some_and(|(ecosystem, name)| !ecosystem.is_empty() && !name.is_empty()), + Some(("package", "cargo")) => true, + Some(("workspace", value)) => { + matches!(value, "cargo" | "node" | "go" | "python") + } + _ => false, + }; + if !valid { + bail!("invalid pattern {pattern:?} in rule {}", rule.id); + } + } + } + document + .rules + .sort_by(|a, b| b.priority.cmp(&a.priority).then(a.id.cmp(&b.id))); + let mut rules = Self { + rules: document.rules, + files: HashMap::new(), + sources: HashMap::new(), + dependencies: HashMap::new(), + workspaces: HashMap::new(), + packages: HashMap::new(), + wildcard_files: Vec::new(), + file_patterns: BTreeSet::new(), + wildcard_interest: BTreeSet::new(), + }; + for (i, rule) in rules.rules.iter().enumerate() { + for pattern in rule.all.iter().chain(&rule.any).chain(&rule.none) { + if let Some(value) = pattern.strip_prefix("file:") { + if value.contains(['*', '?']) { + rules.wildcard_interest.insert(value.to_owned()); + } else { + rules.file_patterns.insert(value.to_owned()); + } + } + } + for pattern in rule.all.iter().chain(&rule.any) { + let Some((kind, value)) = pattern.split_once(':') else { + continue; + }; + let index = match kind { + "file" if value.contains(['*', '?']) => { + rules.wildcard_files.push((value.to_owned(), i)); + continue; + } + "file" => &mut rules.files, + "source" => &mut rules.sources, + "dep" => &mut rules.dependencies, + "workspace" => &mut rules.workspaces, + "package" => &mut rules.packages, + _ => continue, + }; + index.entry(value.to_owned()).or_default().push(i); + } + } + Ok(rules) + } + pub fn needs_filename(&self, name: &str) -> bool { + self.file_patterns.contains(name) || self.wildcard_interest.iter().any(|p| glob(p, name)) + } + + pub fn evaluate(&self, node: &Node) -> Vec { + let mut candidates = Vec::new(); + for file in &node.files { + if let Some(ids) = self.files.get(file) { + candidates.extend(ids); + } + } + for source in node.sources.keys() { + if let Some(ids) = self.sources.get(source) { + candidates.extend(ids); + } + } + for dependency in node.dependencies.keys() { + if let Some(ids) = self.dependencies.get(dependency) { + candidates.extend(ids); + } + } + for package in &node.packages { + if let Some(ids) = self.packages.get(package) { + candidates.extend(ids); + } + } + for workspace in &node.workspaces { + if let Some(ids) = self.workspaces.get(&workspace.ecosystem) { + candidates.extend(ids); + } + } + for (pattern, id) in &self.wildcard_files { + if node.files.iter().any(|file| glob(pattern, file)) { + candidates.push(id); + } + } + candidates.sort_unstable(); + candidates.dedup(); + candidates + .into_iter() + .filter_map(|&i| { + let rule = &self.rules[i]; + rule.evaluate(node).map(|evidence| Detection { + id: rule.id.clone(), + language: rule.language.clone(), + kind: rule.kind, + priority: rule.priority, + evidence, + }) + }) + .collect() + } +} + +impl Rule { + pub fn evaluate(&self, node: &Node) -> Option> { + if !self.all.iter().all(|p| matched(node, p).is_some()) + || (!self.any.is_empty() && !self.any.iter().any(|p| matched(node, p).is_some())) + || self.none.iter().any(|p| matched(node, p).is_some()) + { + return None; + } + let mut evidence: Vec<_> = self + .all + .iter() + .chain(&self.any) + .filter_map(|p| matched(node, p).map(|found| found.evidence(node, p))) + .collect(); + evidence.sort(); + evidence.dedup(); + Some(evidence) + } +} + +enum Matched<'a> { + File(&'a str), + Dependency(&'a super::Dependency), + Workspace(&'a super::Workspace), +} + +impl Matched<'_> { + fn evidence(self, node: &Node, pattern: &str) -> Evidence { + let (source, value) = match self { + Self::File(name) => (super::relative_file(&node.path, name), None), + Self::Dependency(dep) => (dep.source.clone(), dep.version.clone()), + Self::Workspace(ws) => (ws.source.clone(), None), + }; + Evidence { + source, + pattern: pattern.to_owned(), + value, + } + } +} + +fn matched<'a>(node: &'a Node, pattern: &str) -> Option> { + let (kind, value) = pattern.split_once(':')?; + match kind { + "file" => { + let name = if value.contains(['*', '?']) { + node.files.iter().find(|name| glob(value, name)) + } else { + node.files.get(value) + }; + name.map(|name| Matched::File(name)) + } + "source" => node.sources.get(value).map(|name| Matched::File(name)), + "dep" => node.dependencies.get(value).map(Matched::Dependency), + "package" => node + .packages + .get(value) + .map(|_| Matched::File("Cargo.toml")), + "workspace" => node + .workspaces + .iter() + .find(|w| w.ecosystem == value) + .map(Matched::Workspace), + _ => None, + } +} + +pub fn glob(pattern: &str, value: &str) -> bool { + let (p, v) = (pattern.as_bytes(), value.as_bytes()); + let (mut i, mut j, mut star, mut retry) = (0, 0, None, 0); + while j < v.len() { + if i < p.len() && (p[i] == b'?' || p[i] == v[j]) { + i += 1; + j += 1; + } else if i < p.len() && p[i] == b'*' { + star = Some(i); + i += 1; + retry = j; + } else if let Some(s) = star { + retry += 1; + j = retry; + i = s + 1; + } else { + return false; + } + } + while i < p.len() && p[i] == b'*' { + i += 1; + } + i == p.len() +} + +pub fn path_glob(pattern: &str, value: &str) -> bool { + let pattern = pattern.strip_prefix("./").unwrap_or(pattern); + let (head, tail) = pattern.split_once('/').unwrap_or((pattern, "")); + if head == "**" { + if tail.is_empty() { + return true; + } + if path_glob(tail, value) { + return true; + } + return value + .split_once('/') + .is_some_and(|(_, rest)| path_glob(pattern, rest)); + } + let (part, rest) = value.split_once('/').unwrap_or((value, "")); + glob(head, part) + && if tail.is_empty() { + rest.is_empty() + } else { + !rest.is_empty() && path_glob(tail, rest) + } +} diff --git a/src/detect/rules.toml b/src/detect/rules.toml new file mode 100644 index 0000000..383422e --- /dev/null +++ b/src/detect/rules.toml @@ -0,0 +1,335 @@ +[[rules]] +id = "composer" +language = "php" +kind = "project" +priority = 10 +any = ["file:composer.json"] + +[[rules]] +id = "php-source" +language = "php" +kind = "source" +priority = 0 +any = ["source:php"] + +[[rules]] +id = "go-module" +language = "go" +kind = "project" +priority = 10 +any = ["file:go.mod"] + +[[rules]] +id = "go-source" +language = "go" +kind = "source" +priority = 0 +any = ["source:go"] + +[[rules]] +id = "cargo" +language = "rust" +kind = "project" +priority = 10 +any = ["package:cargo"] + +[[rules]] +id = "rust-source" +language = "rust" +kind = "source" +priority = 0 +any = ["source:rs"] + +[[rules]] +id = "bundler" +language = "ruby" +kind = "project" +priority = 10 +any = ["file:Gemfile"] + +[[rules]] +id = "ruby-source" +language = "ruby" +kind = "source" +priority = 0 +any = ["source:rb"] + +[[rules]] +id = "python" +language = "python" +kind = "project" +priority = 10 +any = ["file:pyproject.toml"] + +[[rules]] +id = "python-source" +language = "python" +kind = "source" +priority = 0 +any = ["source:py"] + +[[rules]] +id = "node" +language = "javascript" +kind = "project" +priority = 10 +any = ["file:package.json"] + +[[rules]] +id = "javascript-source" +language = "javascript" +kind = "source" +priority = 0 +any = ["source:js", "source:mjs", "source:cjs", "source:jsx"] + +[[rules]] +id = "typescript" +language = "typescript" +kind = "project" +priority = 10 +any = ["file:tsconfig.json", "dep:npm:typescript"] + +[[rules]] +id = "typescript-source" +language = "typescript" +kind = "source" +priority = 0 +any = ["source:ts", "source:mts", "source:cts", "source:tsx"] + +[[rules]] +id = "python-requirements" +language = "python" +kind = "project" +priority = 10 +any = ["file:requirements.txt", "file:Pipfile", "file:setup.py", "file:setup.cfg"] + +[[rules]] +id = "go-workspace" +language = "go" +kind = "workspace" +priority = 20 +any = ["workspace:go"] + +[[rules]] +id = "cargo-workspace" +language = "rust" +kind = "workspace" +priority = 20 +any = ["workspace:cargo"] + +[[rules]] +id = "node-workspace" +language = "javascript" +kind = "workspace" +priority = 20 +any = ["workspace:node"] + +[[rules]] +id = "python-workspace" +language = "python" +kind = "workspace" +priority = 20 +any = ["workspace:python"] + +[[rules]] +id = "laravel" +language = "php" +kind = "framework" +priority = 100 +any = ["dep:composer:laravel/framework"] + +[[rules]] +id = "symfony" +language = "php" +kind = "framework" +priority = 100 +any = ["dep:composer:symfony/framework-bundle", "dep:composer:symfony/symfony"] + +[[rules]] +id = "cakephp" +language = "php" +kind = "framework" +priority = 100 +any = ["dep:composer:cakephp/cakephp"] + +[[rules]] +id = "codeigniter" +language = "php" +kind = "framework" +priority = 100 +any = ["dep:composer:codeigniter4/framework", "dep:composer:codeigniter/framework"] + +[[rules]] +id = "slim" +language = "php" +kind = "framework" +priority = 100 +any = ["dep:composer:slim/slim"] + +[[rules]] +id = "axum" +language = "rust" +kind = "framework" +priority = 100 +any = ["dep:cargo:axum"] + +[[rules]] +id = "rocket" +language = "rust" +kind = "framework" +priority = 100 +any = ["dep:cargo:rocket"] + +[[rules]] +id = "actix-web" +language = "rust" +kind = "framework" +priority = 100 +any = ["dep:cargo:actix-web"] + +[[rules]] +id = "rails" +language = "ruby" +kind = "framework" +priority = 100 +any = ["dep:gem:rails"] + +[[rules]] +id = "sinatra" +language = "ruby" +kind = "framework" +priority = 100 +any = ["dep:gem:sinatra"] + +[[rules]] +id = "hanami" +language = "ruby" +kind = "framework" +priority = 100 +any = ["dep:gem:hanami"] + +[[rules]] +id = "django" +language = "python" +kind = "framework" +priority = 100 +any = ["dep:pypi:django"] + +[[rules]] +id = "fastapi" +language = "python" +kind = "framework" +priority = 100 +any = ["dep:pypi:fastapi"] + +[[rules]] +id = "flask" +language = "python" +kind = "framework" +priority = 100 +any = ["dep:pypi:flask"] + +[[rules]] +id = "starlette" +language = "python" +kind = "framework" +priority = 50 +any = ["dep:pypi:starlette"] + +[[rules]] +id = "celery" +language = "python" +kind = "framework" +priority = 100 +any = ["dep:pypi:celery"] + +[[rules]] +id = "gin" +language = "go" +kind = "framework" +priority = 100 +any = ["dep:go:github.com/gin-gonic/gin"] + +[[rules]] +id = "echo" +language = "go" +kind = "framework" +priority = 100 +any = ["dep:go:github.com/labstack/echo/v4"] + +[[rules]] +id = "fiber" +language = "go" +kind = "framework" +priority = 100 +any = ["dep:go:github.com/gofiber/fiber/v2"] + +[[rules]] +id = "nextjs" +language = "javascript" +kind = "framework" +priority = 100 +any = ["dep:npm:next"] + +[[rules]] +id = "nuxtjs" +language = "javascript" +kind = "framework" +priority = 100 +any = ["dep:npm:nuxt", "dep:npm:nuxt3"] + +[[rules]] +id = "astro" +language = "javascript" +kind = "framework" +priority = 100 +any = ["dep:npm:astro"] + +[[rules]] +id = "sveltekit" +language = "javascript" +kind = "framework" +priority = 100 +any = ["dep:npm:@sveltejs/kit"] + +[[rules]] +id = "remix" +language = "javascript" +kind = "framework" +priority = 100 +any = ["dep:npm:@remix-run/react"] + +[[rules]] +id = "angular" +language = "javascript" +kind = "framework" +priority = 100 +any = ["dep:npm:@angular/core"] + +[[rules]] +id = "express" +language = "javascript" +kind = "framework" +priority = 100 +any = ["dep:npm:express"] + +[[rules]] +id = "react" +language = "javascript" +kind = "framework" +priority = 50 +any = ["dep:npm:react"] + +[[rules]] +id = "vue" +language = "javascript" +kind = "framework" +priority = 50 +any = ["dep:npm:vue"] + +[[rules]] +id = "vite" +language = "javascript" +kind = "framework" +priority = 50 +any = ["dep:npm:vite"] diff --git a/src/detect/tests.rs b/src/detect/tests.rs new file mode 100644 index 0000000..55cf73d --- /dev/null +++ b/src/detect/tests.rs @@ -0,0 +1,686 @@ +use std::fs; +use std::path::Path; +use std::time::Instant; + +use super::{Project, Report, scan}; + +fn fixture(files: &[(&str, &str)]) -> tempfile::TempDir { + let dir = tempfile::tempdir().unwrap(); + for (path, text) in files { + let path = dir.path().join(path); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, text).unwrap(); + } + dir +} + +fn project<'a>(report: &'a Report, path: &str) -> &'a Project { + report + .projects + .iter() + .find(|p| p.path == path) + .unwrap_or_else(|| panic!("missing {path}: {report:#?}")) +} + +fn detects(project: &Project, id: &str) -> bool { + project.detections.iter().any(|d| d.id == id) +} + +fn version(project: &Project, language: &str, value: &str, source: &str) -> bool { + project.languages.iter().any(|l| { + l.name == language + && l.versions + .iter() + .any(|v| v.value == value && v.source == source) + }) +} + +#[test] +fn frameworks_and_plain_projects() { + let cases = [ + ( + "laravel", + "composer.json", + r#"{"require":{"php":"^8.3","laravel/framework":"^12","symfony/console":"^7"}}"#, + "php", + ), + ( + "symfony", + "composer.json", + r#"{"require":{"symfony/framework-bundle":"^7.2"}}"#, + "php", + ), + ( + "symfony", + "composer.json", + r#"{"require":{"symfony/symfony":"^6.4"}}"#, + "php", + ), + ( + "cakephp", + "composer.json", + r#"{"require":{"cakephp/cakephp":"^5"}}"#, + "php", + ), + ( + "codeigniter", + "composer.json", + r#"{"require":{"codeigniter4/framework":"^4"}}"#, + "php", + ), + ( + "composer", + "composer.json", + r#"{"require":{"symfony/console":"^7"}}"#, + "php", + ), + ( + "go-module", + "go.mod", + "module example.com/service\ngo 1.24\n", + "go", + ), + ( + "gin", + "go.mod", + "module example.com/api\ngo 1.24\nrequire (\n github.com/gin-gonic/gin v1.10.0\n)\n", + "go", + ), + ( + "axum", + "Cargo.toml", + "[package]\nname='api'\nrust-version='1.85'\n[dependencies]\naxum='0.8'\n", + "rust", + ), + ( + "rocket", + "Cargo.toml", + "[package]\nname='api'\n[dependencies]\nrocket={version='0.5',features=['json']}\n", + "rust", + ), + ( + "cargo", + "Cargo.toml", + "[package]\nname='plain'\nversion='0.1.0'\nedition='2024'\n", + "rust", + ), + ( + "rails", + "Gemfile", + "source 'https://rubygems.org'\nruby '3.3.1'\ngem 'rails', '~> 8.0'\n", + "ruby", + ), + ( + "sinatra", + "Gemfile", + "gem(\"sinatra\", \"~> 4.0\")\n", + "ruby", + ), + ("hanami", "Gemfile", "gem 'hanami', '~> 2.2'\n", "ruby"), + ( + "django", + "requirements.txt", + "# FastAPI\nDjango>=5.1,<6\n", + "python", + ), + ( + "fastapi", + "pyproject.toml", + "[project]\nname='api'\nrequires-python='>=3.12'\ndependencies=['FastAPI[standard]>=0.115; python_version >= \"3.12\"']\n", + "python", + ), + ( + "flask", + "Pipfile", + "[requires]\npython_version='3.12'\n[packages]\nflask='>=3'\n", + "python", + ), + ( + "django", + "pyproject.toml", + "[tool.poetry.dependencies]\npython='^3.12'\ndjango={version='^5.1'}\n", + "python", + ), + ( + "nextjs", + "package.json", + r#"{"dependencies":{"next":"^15","react":"^19"},"engines":{"node":">=22"}}"#, + "javascript", + ), + ( + "nuxtjs", + "package.json", + r#"{"dependencies":{"nuxt":"^3","vue":"^3"}}"#, + "javascript", + ), + ( + "astro", + "package.json", + r#"{"devDependencies":{"astro":"^5"}}"#, + "javascript", + ), + ( + "sveltekit", + "package.json", + r#"{"devDependencies":{"@sveltejs/kit":"^2"}}"#, + "javascript", + ), + ( + "express", + "package.json", + r#"{"dependencies":{"express":"^5"}}"#, + "javascript", + ), + ]; + for (expected, file, text, language) in cases { + let dir = fixture(&[(file, text)]); + let report = scan(dir.path(), None).unwrap(); + assert!( + report.diagnostics.is_empty(), + "{expected}: {:?}", + report.diagnostics + ); + assert_eq!(report.projects.len(), 1, "{expected}: {report:#?}"); + let p = &report.projects[0]; + assert_eq!(p.detections[0].id, expected, "{p:#?}"); + assert!(p.languages.iter().any(|l| l.name == language)); + } +} + +#[test] +fn bare_sources_never_need_to_be_read() { + for (extension, language) in [ + ("php", "php"), + ("go", "go"), + ("rs", "rust"), + ("rb", "ruby"), + ("py", "python"), + ("js", "javascript"), + ("tsx", "typescript"), + ] { + let dir = fixture(&[(&format!("main.{extension}"), "not even valid source")]); + let report = scan(dir.path(), None).unwrap(); + assert_eq!(report.stats.files_read, 0); + assert_eq!(report.projects.len(), 1); + assert_eq!(report.projects[0].languages[0].name, language); + assert!(report.projects[0].languages[0].versions.is_empty()); + } +} + +#[test] +fn complex_polyglot_monorepo() { + let dir = fixture(&[ + ( + "package.json", + r#"{"name":"platform","private":true,"workspaces":["apps/*","packages/*"]}"#, + ), + (".nvmrc", "22.14.0\n"), + (".tool-versions", "python 3.13.2\nruby 3.4.1\n"), + ( + "go.work", + "go 1.24.0\nuse (\n ./services/gateway\n ./services/jobs\n)\n", + ), + ( + "Cargo.toml", + "[workspace]\nmembers=['crates/*']\nexclude=['crates/excluded']\n[workspace.package]\nrust-version='1.85'\n[workspace.dependencies]\nweb={package='axum',version='0.8'}\nrocket='0.5'\n", + ), + ("rust-toolchain.toml", "[toolchain]\nchannel='1.86.0'\n"), + ( + "apps/web/package.json", + r#"{"name":"web","dependencies":{"next":"^15","react":"^19"},"devDependencies":{"typescript":"^5.7"},"engines":{"node":">=20"}}"#, + ), + ("apps/web/tsconfig.json", "{}"), + ("apps/web/src/app/page.tsx", "export default () => null"), + ("apps/web/src/lib/helper.ts", "export const x = 1"), + ( + "apps/docs/package.json", + r#"{"dependencies":{"astro":"^5"}}"#, + ), + ( + "apps/store/package.json", + r#"{"dependencies":{"nuxt":"^3"}}"#, + ), + ( + "packages/ui/package.json", + r#"{"name":"ui","dependencies":{"react":"^19"}}"#, + ), + ( + "services/gateway/go.mod", + "module example.com/gateway\ngo 1.23.0\ntoolchain go1.24.1\n", + ), + ("services/gateway/cmd/api/main.go", "package main"), + ( + "services/jobs/go.mod", + "module example.com/jobs\ngo 1.24.0\n", + ), + ("services/jobs/main.go", "package main"), + ( + "crates/api/Cargo.toml", + "[package]\nname='api'\nrust-version.workspace=true\n[dependencies]\nweb.workspace=true\n", + ), + ("crates/api/src/main.rs", "fn main() {}"), + ( + "crates/rocket/Cargo.toml", + "[package]\nname='rocket-api'\n[dependencies]\nrocket.workspace=true\n", + ), + ("crates/excluded/Cargo.toml", "[package]\nname='excluded'\n"), + ( + "services/php/composer.json", + r#"{"require":{"php":"^8.3","laravel/framework":"^12","symfony/console":"^7"}}"#, + ), + ( + "services/php/package.json", + r#"{"devDependencies":{"vite":"^6"},"engines":{"node":">=22"}}"#, + ), + ("services/php/app/Models/User.php", "= 3.3'\ngem 'rails', '~> 8.0'\n", + ), + ("services/rails/app/models/user.rb", "class User; end"), + ("services/django/requirements.txt", "Django==5.1.6\n"), + ("services/django/project/settings.py", "DEBUG = False"), + ( + "services/fastapi/pyproject.toml", + "[project]\nname='fastapi-service'\nrequires-python='>=3.12'\ndependencies=['fastapi>=0.115']\n", + ), + ("services/fastapi/app/main.py", "# service"), + ("tools/plain/main.go", "package main"), + ("tools/script/task.py", "print('hello')"), + ( + "node_modules/fake/package.json", + r#"{"dependencies":{"next":"1"}}"#, + ), + ( + "services/php/vendor/fake/composer.json", + r#"{"require":{"symfony/framework-bundle":"1"}}"#, + ), + ("target/fake/Cargo.toml", "invalid"), + (".venv/fake/requirements.txt", "flask==1"), + ]); + let report = scan(dir.path(), None).unwrap(); + assert!(report.diagnostics.is_empty(), "{report:#?}"); + assert_eq!(report.projects.len(), 19, "{report:#?}"); + let root = project(&report, "."); + for detection in ["node-workspace", "go-workspace", "cargo-workspace"] { + assert!(detects(root, detection)); + } + assert_eq!( + root.workspace_members, + [ + "apps/docs", + "apps/store", + "apps/web", + "crates/api", + "crates/rocket", + "packages/ui", + "services/gateway", + "services/jobs" + ] + ); + for (path, detection) in [ + ("apps/web", "nextjs"), + ("apps/docs", "astro"), + ("apps/store", "nuxtjs"), + ("crates/api", "axum"), + ("crates/rocket", "rocket"), + ("services/php", "laravel"), + ("services/symfony", "symfony"), + ("services/cake", "cakephp"), + ("services/codeigniter", "codeigniter"), + ("services/rails", "rails"), + ("services/django", "django"), + ("services/fastapi", "fastapi"), + ] { + assert!(detects(project(&report, path), detection), "{path}"); + } + assert!(!detects(project(&report, "services/php"), "symfony")); + assert_eq!( + project(&report, "services/php") + .languages + .iter() + .map(|l| l.name.as_str()) + .collect::>(), + ["javascript", "php"] + ); + assert!(version( + project(&report, "apps/web"), + "javascript", + "22.14.0", + ".nvmrc" + )); + assert!(version( + project(&report, "apps/web"), + "javascript", + ">=20", + "apps/web/package.json" + )); + assert!(version( + project(&report, "apps/web"), + "typescript", + "^5.7", + "apps/web/package.json" + )); + assert!(version( + project(&report, "crates/api"), + "rust", + "1.85", + "Cargo.toml" + )); + assert!(version( + project(&report, "crates/api"), + "rust", + "1.86.0", + "rust-toolchain.toml" + )); + assert!(version( + project(&report, "services/django"), + "python", + "3.13.2", + ".tool-versions" + )); + assert!(version( + project(&report, "services/rails"), + "ruby", + ">= 3.3", + "services/rails/Gemfile" + )); + assert_eq!(report.stats.skipped_directories, 4); + assert_eq!(scan(dir.path(), None).unwrap(), report); +} + +#[test] +fn avoids_false_framework_matches() { + let dir = fixture(&[ + ( + "composer.json", + r#"{"description":"laravel/framework","require":{"symfony/console":"^7"},"require-dev":{"laravel/framework":"^12"}}"#, + ), + ( + "Cargo.toml", + "[package]\nname='axum'\n[dev-dependencies]\nrocket='0.5'\n[workspace.dependencies]\naxum='0.8'\n", + ), + ( + "Gemfile", + "# gem 'rails'\ngem 'rails-html-sanitizer'\ngem 'sinatra-contrib'\n", + ), + ( + "requirements.txt", + "# django==5\ndjango-filter>=24\nfastapi-utils==0.8\n", + ), + ( + "package.json", + r#"{"scripts":{"start":"next start"},"dependencies":{"next-themes":"*","@astrojs/check":"*"}}"#, + ), + ]); + let report = scan(dir.path(), None).unwrap(); + assert!( + report + .projects + .iter() + .flat_map(|p| &p.detections) + .all(|d| d.kind != super::Kind::Framework), + "{report:#?}" + ); +} + +#[test] +fn custom_patterns_and_priority() { + let dir = fixture(&[ + ("package.json", r#"{"dependencies":{"next":"15"}}"#), + ("special.config", ""), + ("disabled", ""), + ]); + let extra = r#" +[[rules]] +id = 'company-next' +language = 'javascript' +kind = 'framework' +priority = 999 +all = ['dep:npm:next', 'file:special.*'] +none = ['file:blocked'] +[[rules]] +id = 'must-not-match' +language = 'javascript' +kind = 'framework' +priority = 1000 +any = ['file:special.config'] +none = ['file:disabled'] +"#; + let report = scan(dir.path(), Some(extra)).unwrap(); + assert_eq!(report.projects[0].detections[0].id, "company-next"); + assert!(!detects(&report.projects[0], "must-not-match")); + let json = serde_json::to_value(&report).unwrap(); + assert_eq!(json["schemaVersion"], 1); + assert!(super::render(&json)[0].contains("company-next")); + for invalid in [ + extra.replace("company-next", "nextjs"), + extra.replace("file:blocked", "unsupported:x"), + extra.replace("priority = 999", "prio = 999"), + ] { + assert!(scan(dir.path(), Some(&invalid)).is_err()); + } +} + +#[test] +fn malformed_and_oversized_manifests_are_diagnostic_not_fatal() { + let dir = fixture(&[ + ("broken/package.json", "["), + ("valid/composer.json", "{}"), + ("invalid/Cargo.toml", "["), + ]); + fs::write(dir.path().join("broken/composer.json"), [0xff, 0xfe]).unwrap(); + fs::write( + dir.path().join("broken/Gemfile"), + vec![b'a'; super::MAX_FILE_BYTES as usize + 20], + ) + .unwrap(); + let report = scan(dir.path(), None).unwrap(); + assert_eq!(report.diagnostics.len(), 4, "{report:#?}"); + assert!(detects(project(&report, "valid"), "composer")); + assert!(report.stats.bytes_read <= super::MAX_FILE_BYTES as usize + 100); +} + +#[test] +fn version_constraints_keep_provenance_and_do_not_invent_versions() { + let dir = fixture(&[ + ( + "composer.json", + r#"{"require":{"php":"^8.2","laravel/framework":"^12"},"config":{"platform":{"php":"8.3.14"}}}"#, + ), + ( + "Cargo.toml", + "[package]\nname='plain'\nversion='9.9.9'\nedition='2024'\n", + ), + ("Gemfile", "gem 'rails', '~> 8.0', '>= 8.0.1'\n"), + (".python-version", "3.12.8\n"), + ( + "app/pyproject.toml", + "[project]\nrequires-python='>=3.11,<4'\n", + ), + ]); + let report = scan(dir.path(), None).unwrap(); + let p = project(&report, "."); + assert!(version(p, "php", "^8.2", "composer.json")); + assert!(version(p, "php", "8.3.14", "composer.json")); + assert!( + p.languages + .iter() + .find(|l| l.name == "rust") + .unwrap() + .versions + .is_empty() + ); + let rails = p.detections.iter().find(|d| d.id == "rails").unwrap(); + assert_eq!(rails.evidence[0].value.as_deref(), Some("~> 8.0, >= 8.0.1")); + assert!(version( + project(&report, "app"), + "python", + "3.12.8", + ".python-version" + )); +} + +#[test] +fn python_uv_workspace_and_workspace_exclusions() { + let dir = fixture(&[ + ( + "pyproject.toml", + "[tool.uv.workspace]\nmembers=['services/*']\nexclude=['services/ignored']\n", + ), + ( + "services/api/pyproject.toml", + "[project]\nname='api'\ndependencies=['fastapi==0.115']\n", + ), + ( + "services/ignored/pyproject.toml", + "[project]\nname='ignored'\n", + ), + ]); + let report = scan(dir.path(), None).unwrap(); + assert_eq!(project(&report, ".").workspace_members, ["services/api"]); +} + +#[test] +fn glob_matching_respects_directory_boundaries() { + for (pattern, value, expected) in [ + ("services/*", "services/api", true), + ("services/*", "services/api/nested", false), + ("services/**/api", "services/a/b/api", true), + ("services/**/api", "different/api", false), + ("services/**/api", "services/api", true), + ("./crates/*", "crates/api", true), + ] { + assert_eq!( + super::rules::path_glob(pattern, value), + expected, + "{pattern}: {value}" + ); + } +} + +#[cfg(unix)] +#[test] +fn symlinks_are_not_followed() { + let dir = fixture(&[("real/go.mod", "module example.com/api\ngo 1.24\n")]); + std::os::unix::fs::symlink(dir.path(), dir.path().join("real/loop")).unwrap(); + std::os::unix::fs::symlink(dir.path().join("real/go.mod"), dir.path().join("go.mod")).unwrap(); + let report = scan(dir.path(), None).unwrap(); + assert_eq!(report.projects.len(), 1); + assert_eq!(report.stats.files_read, 1); + assert_eq!(report.stats.directories, 2); +} + +#[test] +fn source_volume_does_not_increase_manifest_reads() { + let dir = fixture(&[("go.mod", "module example.com/api\ngo 1.24\n")]); + for i in 0..200 { + let path = dir.path().join(format!("pkg/p{i}")); + fs::create_dir_all(&path).unwrap(); + for j in 0..10 { + fs::write(path.join(format!("file{j}.go")), "package p").unwrap(); + } + } + let report = scan(dir.path(), None).unwrap(); + assert_eq!(report.projects.len(), 1); + assert_eq!(report.stats.files_read, 1); + assert_eq!(report.stats.files, 2001); +} + +#[test] +fn cli_is_local_and_authentication_free() { + use clap::Parser; + let opts = crate::Opts::try_parse_from([ + "brainpod", + "detect", + "./services", + "--json", + "--rules", + "custom.toml", + ]) + .unwrap(); + assert!(!crate::cmd::needs_client(&opts.command)); + assert!(!crate::cmd::needs_api_token(&opts.command)); + assert!(opts.json); + let crate::cmd::Command::Detect(args) = opts.command else { + panic!("expected detect"); + }; + assert_eq!(args.path, Path::new("./services")); + assert_eq!(args.rules.as_deref(), Some(Path::new("custom.toml"))); +} + +#[test] +#[ignore = "manual release-mode performance measurement"] +fn benchmark_polyglot_scan() { + let dir = tempfile::tempdir().unwrap(); + let manifests = [ + ( + "composer.json", + r#"{"require":{"php":"^8.3","laravel/framework":"^12"}}"#, + "php", + ), + ("go.mod", "module example.com/api\ngo 1.24\n", "go"), + ( + "Cargo.toml", + "[package]\nname='api'\n[dependencies]\naxum='0.8'\n", + "rs", + ), + ("Gemfile", "ruby '3.4'\ngem 'rails', '~> 8.0'\n", "rb"), + ( + "pyproject.toml", + "[project]\nrequires-python='>=3.12'\ndependencies=['fastapi>=0.115']\n", + "py", + ), + ( + "package.json", + r#"{"dependencies":{"next":"^15","react":"^19"},"engines":{"node":">=22"}}"#, + "js", + ), + ( + "package.json", + r#"{"dependencies":{"nuxt":"^3"},"devDependencies":{"typescript":"^5.7"}}"#, + "ts", + ), + ]; + for i in 0..200 { + let path = dir.path().join(format!("services/s{i}")); + fs::create_dir_all(path.join("src")).unwrap(); + let (file, manifest, extension) = manifests[i % manifests.len()]; + fs::write(path.join(file), manifest).unwrap(); + for j in 0..100 { + fs::write( + path.join(format!("src/file{j}.{extension}")), + vec![b'x'; 1024], + ) + .unwrap(); + } + } + let mut samples = Vec::new(); + for _ in 0..20 { + let start = Instant::now(); + let report = scan(dir.path(), None).unwrap(); + samples.push(start.elapsed().as_secs_f64() * 1000.0); + assert_eq!(report.projects.len(), 200); + assert_eq!(report.stats.files_read, 200); + assert_eq!(report.stats.files, 20200); + std::hint::black_box(report); + } + let first = samples[0]; + samples.sort_by(f64::total_cmp); + eprintln!( + "200 polyglot services, 20,200 files (20 warm filesystem runs): first {first:.3} ms, median {:.3} ms, p95 {:.3} ms; 200 manifest reads, zero source reads", + samples[10], samples[18] + ); +} diff --git a/src/main.rs b/src/main.rs index 5488e46..cf8d604 100644 --- a/src/main.rs +++ b/src/main.rs @@ -12,6 +12,7 @@ mod client; mod cmd; mod config; mod describe; +mod detect; mod image; mod openapi; mod output; @@ -130,6 +131,10 @@ async fn run(opts: Opts) -> Result { )); } + if let Command::Detect(args) = opts.command { + return detect::handle(args); + } + let config_path = Config::path()?; let mut config = Config::load(&config_path)?; diff --git a/src/output.rs b/src/output.rs index b864d56..297589a 100644 --- a/src/output.rs +++ b/src/output.rs @@ -25,6 +25,7 @@ impl CommandOutput { pub enum View { S3, Describe, + Detect, ResourceSchema, AgentStart, AgentServe, @@ -164,6 +165,7 @@ fn render(value: &Value, view: View, color: bool) -> Vec { match view { View::S3 => crate::s3::render(value), View::Describe => render_describe(value), + View::Detect => crate::detect::render(value), View::ResourceSchema => render_resource_schema(value), View::AgentStart => crate::agent::render_start(value), View::AgentServe => vec![format!("Session console: {}", field(value, "url"))], From c6f75eb5de0fa29572d48e3eb0ab9126ebdca0c9 Mon Sep 17 00:00:00 2001 From: Dillen Meijboom Date: Mon, 9 Nov 2026 21:22:49 +0000 Subject: [PATCH 02/10] test: validate detection against popular GitHub web projects --- README.md | 2 + tests/audit_detect.py | 252 ++++++ tests/detect-corpus.json | 1480 ++++++++++++++++++++++++++++++++++++ tests/detect-corpus.md | 100 +++ tests/test_audit_detect.py | 155 ++++ 5 files changed, 1989 insertions(+) create mode 100644 tests/audit_detect.py create mode 100644 tests/detect-corpus.json create mode 100644 tests/detect-corpus.md create mode 100644 tests/test_audit_detect.py diff --git a/README.md b/README.md index 2bc3d3b..134555a 100644 --- a/README.md +++ b/README.md @@ -302,6 +302,8 @@ Run the generated large-tree performance fixture with: cargo test --release detect::tests::benchmark_polyglot_scan -- --ignored --nocapture ``` +A [35-repository GitHub audit](tests/detect-corpus.md) records pinned real-world coverage, timings, known gaps, and an opt-in reproducible runner. + ## Image discovery Image commands require a pod and API token with `registry:pull` permission. List returns active public images and images in the selected pod: diff --git a/tests/audit_detect.py b/tests/audit_detect.py new file mode 100644 index 0000000..c9ea0ae --- /dev/null +++ b/tests/audit_detect.py @@ -0,0 +1,252 @@ +import argparse +import collections +import hashlib +import json +import re +import shutil +import statistics +import subprocess +import tarfile +import tempfile +import time +import urllib.request +from pathlib import Path, PurePosixPath + +MAX_ARCHIVE_BYTES = 1024 * 1024 * 1024 + + +def snapshot(entry, cache, fetch): + repository, revision = entry["repository"], entry["revision"] + if ( + not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9-]*/[\w.-]+", repository) + or repository.split("/")[-1] in {".", ".."} + or not re.fullmatch(r"[0-9a-f]{40}", revision) + ): + raise ValueError("invalid repository or pinned revision") + name = repository.replace("/", "--") + destination = cache / "repos" / name + metadata = cache / "metadata" / f"{name}.json" + if destination.is_dir() and metadata.is_file(): + data = json.loads(metadata.read_text()) + if data.get("revision", data.get("sha")) == revision: + return destination + if not fetch: + raise ValueError( + f"missing pinned snapshot: {repository}; pass --fetch to download" + ) + if destination.exists(): + raise ValueError( + f"unverified snapshot exists: {destination}; use a fresh cache" + ) + destination.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix="fetch-", dir=cache) as temporary: + temporary = Path(temporary) + archive = temporary / "snapshot.tar.gz" + with ( + urllib.request.urlopen( + f"https://codeload.github.com/{repository}/tar.gz/{revision}", + timeout=90, + ) as response, + archive.open("wb") as output, + ): + size = 0 + while chunk := response.read(1024 * 1024): + size += len(chunk) + if size > MAX_ARCHIVE_BYTES: + raise ValueError("archive exceeds 1 GiB") + output.write(chunk) + extracted = temporary / "snapshot" + extracted.mkdir() + size = 0 + with tarfile.open(archive, "r:gz") as source: + for member in source: + parts = PurePosixPath(member.name).parts + if not parts or member.name.startswith("/") or ".." in parts: + raise ValueError("unsafe archive path") + if len(parts) == 1: + continue + path = extracted.joinpath(*parts[1:]) + if member.isdir(): + path.mkdir(parents=True, exist_ok=True) + elif member.isfile(): + size += member.size + if size > 2 * MAX_ARCHIVE_BYTES: + raise ValueError("expanded snapshot exceeds 2 GiB") + path.parent.mkdir(parents=True, exist_ok=True) + with ( + source.extractfile(member) as content, + path.open("wb") as output, + ): + shutil.copyfileobj(content, output) + extracted.rename(destination) + metadata.parent.mkdir(parents=True, exist_ok=True) + metadata.write_text( + json.dumps({"repository": repository, "revision": revision}) + "\n" + ) + return destination + + +def check(projects, expected): + project = projects.get(expected["path"]) + if project is None: + return False, "project missing" + kind = expected["kind"] + ids = [d["id"] for d in project["detections"]] + if kind == "language": + actual = [language["name"] for language in project["languages"]] + return expected["language"] in actual, actual + if kind == "detection": + return expected["id"] in ids, ids + if kind == "name": + return expected["name"] in project["names"], project["names"] + if kind == "workspace": + actual = project["workspaceMembers"] + return expected["id"] in ids and expected["member"] in actual, { + "ids": ids, + "members": actual, + } + if kind == "version": + actual = [ + v + for language in project["languages"] + if language["name"] == expected["language"] + for v in language["versions"] + ] + return any( + v["value"] == expected["value"] and v["source"] == expected["source"] + for v in actual + ), actual + raise ValueError(f"unknown check kind: {kind}") + + +def audit(entry, args): + path = snapshot(entry, args.cache, args.fetch) + samples, first = [], None + deterministic = True + for _ in range(args.runs): + start = time.perf_counter() + process = subprocess.run( + [str(args.binary), "detect", str(path), "--json"], + capture_output=True, + timeout=90, + check=False, + ) + samples.append((time.perf_counter() - start) * 1000) + if process.returncode: + raise ValueError(process.stderr.decode(errors="replace")) + if first is None: + first = process.stdout + else: + deterministic = deterministic and first == process.stdout + report = json.loads(first) + if report["schemaVersion"] != 1: + raise ValueError("unexpected detector schema version") + projects = {project["path"]: project for project in report["projects"]} + checks = [] + for expected in entry["checks"]: + passed, actual = check(projects, expected) + checks.append({"expected": expected, "passed": passed, "actual": actual}) + return { + "repository": entry["repository"], + "language": entry["language"], + "revision": entry["revision"], + "firstMs": samples[0], + "medianMs": statistics.median(samples[1:]), + "samplesMs": samples, + "deterministic": deterministic, + "stats": report["stats"], + "projectCount": len(projects), + "diagnostics": report["diagnostics"], + "checks": checks, + } + + +def main(): + parser = argparse.ArgumentParser( + description="Audit local project detection against pinned public GitHub snapshots without installing or executing project code." + ) + parser.add_argument( + "--corpus", type=Path, default=Path(__file__).with_name("detect-corpus.json") + ) + parser.add_argument( + "--binary", + type=Path, + default=Path(__file__).resolve().parents[1] / "target/release/brainpod", + ) + parser.add_argument("--cache", type=Path, required=True) + parser.add_argument( + "--fetch", + action="store_true", + help="Explicitly allow downloading missing pinned GitHub snapshots", + ) + parser.add_argument( + "--runs", + type=int, + default=5, + help="CLI invocations per repository; the first is excluded from the warm median", + ) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + if args.runs < 2: + parser.error("--runs must be at least 2") + args.binary = args.binary.resolve(strict=True) + args.cache = args.cache.resolve() + args.cache.mkdir(parents=True, exist_ok=True) + corpus = json.loads(args.corpus.read_text()) + if corpus["schemaVersion"] != 1: + parser.error("unsupported corpus schema version") + with args.binary.open("rb") as binary: + binary_sha = hashlib.file_digest(binary, "sha256").hexdigest() + results = [] + for entry in corpus["repositories"]: + try: + result = audit(entry, args) + passed = sum(c["passed"] for c in result["checks"]) + print( + f"{entry['repository']}: {passed}/{len(result['checks'])} checks; {result['medianMs']:.2f} ms", + flush=True, + ) + except ( + OSError, + ValueError, + KeyError, + tarfile.TarError, + subprocess.TimeoutExpired, + ) as error: + result = { + "repository": entry["repository"], + "revision": entry["revision"], + "error": str(error), + } + print(f"{entry['repository']}: ERROR {error}", flush=True) + results.append(result) + counts = collections.defaultdict(lambda: {"passed": 0, "total": 0}) + for result in results: + for item in result.get("checks", []): + category = counts[item["expected"]["category"]] + category["total"] += 1 + category["passed"] += item["passed"] + output = { + "schemaVersion": 1, + "selectedAt": corpus["selectedAt"], + "binarySha256": binary_sha, + "categories": dict(counts), + "results": results, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(output, indent=2) + "\n") + for category, count in sorted(counts.items()): + print(f"{category}: {count['passed']}/{count['total']}") + print(f"Results: {args.output}") + return int( + any( + "error" in r + or not r["deterministic"] + or any(not c["passed"] for c in r["checks"]) + for r in results + ) + ) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/detect-corpus.json b/tests/detect-corpus.json new file mode 100644 index 0000000..9c6a4ce --- /dev/null +++ b/tests/detect-corpus.json @@ -0,0 +1,1480 @@ +{ + "schemaVersion": 1, + "selectedAt": "2026-11-09T21:24:33.642341+00:00", + "ranking": "GitHub stars descending; non-fork, non-archived; primary language per GitHub", + "eligibility": "Web applications (including production learning platforms), HTTP servers/frameworks/libraries, browser UI libraries, and web-specific site/build tools. Exclude tutorial/list repositories, general runtimes, coding agents, desktop-only apps, ML engines without a primary web UI, and generic infrastructure/networking tools.", + "expectations": "Reviewed primary project paths and static manifests. Framework checks cover consumers; identity checks separately probe framework implementation packages. Catalog checks require a locally declared version rather than a catalog reference. Workspace checks use representative declared members. Not an exhaustive precision/recall oracle.", + "repositories": [ + { + "repository": "infiniflow/ragflow", + "language": "Go", + "stars": 93887, + "languageRank": 8, + "webRank": 1, + "revision": "ed5a84fee9abc8091aea111d1195cf7768541ff1", + "checks": [ + { + "category": "language", + "kind": "language", + "path": ".", + "language": "go" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "go-module" + }, + { + "category": "name", + "kind": "name", + "path": ".", + "name": "ragflow" + }, + { + "category": "version", + "kind": "version", + "path": ".", + "language": "go", + "value": "1.27", + "source": "go.mod" + }, + { + "category": "framework", + "kind": "detection", + "path": ".", + "id": "gin" + } + ] + }, + { + "repository": "gohugoio/hugo", + "language": "Go", + "stars": 90425, + "languageRank": 9, + "webRank": 2, + "revision": "d2010fcc9cd30c5b4e9274ded4485f4222326b53", + "checks": [ + { + "category": "language", + "kind": "language", + "path": ".", + "language": "go" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "go-module" + }, + { + "category": "name", + "kind": "name", + "path": ".", + "name": "github.com/gohugoio/hugo" + }, + { + "category": "version", + "kind": "version", + "path": ".", + "language": "go", + "value": "1.27.0", + "source": "go.mod" + } + ] + }, + { + "repository": "gin-gonic/gin", + "language": "Go", + "stars": 89609, + "languageRank": 11, + "webRank": 3, + "revision": "0f09c3a9b4626d9fe9979cf4d0521d6ae32ab646", + "checks": [ + { + "category": "language", + "kind": "language", + "path": ".", + "language": "go" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "go-module" + }, + { + "category": "name", + "kind": "name", + "path": ".", + "name": "github.com/gin-gonic/gin" + }, + { + "category": "version", + "kind": "version", + "path": ".", + "language": "go", + "value": "1.26.0", + "source": "go.mod" + }, + { + "category": "identity", + "kind": "detection", + "path": ".", + "id": "gin" + } + ] + }, + { + "repository": "netdata/netdata", + "language": "Go", + "stars": 81330, + "languageRank": 14, + "webRank": 4, + "revision": "0d54b4424ea6f40c344738b9d566e1a26d160da3", + "checks": [ + { + "category": "language", + "kind": "language", + "path": "src/go", + "language": "go" + }, + { + "category": "project", + "kind": "detection", + "path": "src/go", + "id": "go-module" + }, + { + "category": "name", + "kind": "name", + "path": "src/go", + "name": "github.com/netdata/netdata/go/plugins" + }, + { + "category": "version", + "kind": "version", + "path": "src/go", + "language": "go", + "value": "1.27.0", + "source": "src/go/go.mod" + } + ] + }, + { + "repository": "caddyserver/caddy", + "language": "Go", + "stars": 78927, + "languageRank": 15, + "webRank": 5, + "revision": "9ea3108377d2abec102866808133c40ac9582162", + "checks": [ + { + "category": "language", + "kind": "language", + "path": ".", + "language": "go" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "go-module" + }, + { + "category": "name", + "kind": "name", + "path": ".", + "name": "github.com/caddyserver/caddy/v2" + }, + { + "category": "version", + "kind": "version", + "path": ".", + "language": "go", + "value": "1.26.0", + "source": "go.mod" + } + ] + }, + { + "repository": "react/react", + "language": "JavaScript", + "stars": 252455, + "languageRank": 2, + "webRank": 1, + "revision": "b05c0b75fd9778536152eead30814c9ca90c810a", + "checks": [ + { + "category": "language", + "kind": "language", + "path": "packages/react", + "language": "javascript" + }, + { + "category": "project", + "kind": "detection", + "path": "packages/react", + "id": "node" + }, + { + "category": "name", + "kind": "name", + "path": "packages/react", + "name": "react" + }, + { + "category": "version", + "kind": "version", + "path": "packages/react", + "language": "javascript", + "value": ">=0.10.0", + "source": "packages/react/package.json" + }, + { + "category": "identity", + "kind": "detection", + "path": "packages/react", + "id": "react" + }, + { + "category": "workspace", + "kind": "workspace", + "path": ".", + "id": "node-workspace", + "member": "packages/react" + } + ] + }, + { + "repository": "vercel/next.js", + "language": "JavaScript", + "stars": 143437, + "languageRank": 8, + "webRank": 2, + "revision": "5368e013e74e4e4ed5be51c051c989545c4450b9", + "checks": [ + { + "category": "language", + "kind": "language", + "path": "packages/next", + "language": "javascript" + }, + { + "category": "project", + "kind": "detection", + "path": "packages/next", + "id": "node" + }, + { + "category": "name", + "kind": "name", + "path": "packages/next", + "name": "next" + }, + { + "category": "version", + "kind": "version", + "path": "packages/next", + "language": "javascript", + "value": ">=20.9.0", + "source": "packages/next/package.json" + }, + { + "category": "version", + "kind": "version", + "path": "packages/next", + "language": "typescript", + "value": "6.0.2", + "source": "packages/next/package.json" + }, + { + "category": "identity", + "kind": "detection", + "path": "packages/next", + "id": "nextjs" + }, + { + "category": "workspace", + "kind": "workspace", + "path": ".", + "id": "node-workspace", + "member": "packages/next" + } + ] + }, + { + "repository": "mrdoob/three.js", + "language": "JavaScript", + "stars": 117953, + "languageRank": 11, + "webRank": 3, + "revision": "3f7edc3d75e054bc755dad56c1c95c62f63a3561", + "checks": [ + { + "category": "language", + "kind": "language", + "path": ".", + "language": "javascript" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "node" + }, + { + "category": "name", + "kind": "name", + "path": ".", + "name": "three" + } + ] + }, + { + "repository": "axios/axios", + "language": "JavaScript", + "stars": 109295, + "languageRank": 14, + "webRank": 4, + "revision": "335e2f8464f58a3ab485486b8d6964fea3e3f30a", + "checks": [ + { + "category": "language", + "kind": "language", + "path": ".", + "language": "javascript" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "node" + }, + { + "category": "name", + "kind": "name", + "path": ".", + "name": "axios" + }, + { + "category": "version", + "kind": "version", + "path": ".", + "language": "typescript", + "value": "^7.0.2", + "source": "package.json" + } + ] + }, + { + "repository": "react/create-react-app", + "language": "JavaScript", + "stars": 103230, + "languageRank": 15, + "webRank": 5, + "revision": "6254386531d263688ccfa542d0e628fbc0de0b28", + "checks": [ + { + "category": "language", + "kind": "language", + "path": "packages/react-scripts", + "language": "javascript" + }, + { + "category": "project", + "kind": "detection", + "path": "packages/react-scripts", + "id": "node" + }, + { + "category": "name", + "kind": "name", + "path": "packages/react-scripts", + "name": "react-scripts" + }, + { + "category": "version", + "kind": "version", + "path": "packages/react-scripts", + "language": "javascript", + "value": ">=14.0.0", + "source": "packages/react-scripts/package.json" + }, + { + "category": "framework", + "kind": "detection", + "path": "packages/react-scripts", + "id": "react" + }, + { + "category": "workspace", + "kind": "workspace", + "path": ".", + "id": "node-workspace", + "member": "packages/react-scripts" + } + ] + }, + { + "repository": "coollabsio/coolify", + "language": "PHP", + "stars": 65849, + "languageRank": 2, + "webRank": 1, + "revision": "30335d80866ff3391d899cf4ccb599b696f47b7e", + "checks": [ + { + "category": "language", + "kind": "language", + "path": ".", + "language": "php" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "composer" + }, + { + "category": "name", + "kind": "name", + "path": ".", + "name": "coollabsio/coolify" + }, + { + "category": "version", + "kind": "version", + "path": ".", + "language": "php", + "value": "^8.4", + "source": "composer.json" + }, + { + "category": "framework", + "kind": "detection", + "path": ".", + "id": "laravel" + } + ] + }, + { + "repository": "nextcloud/server", + "language": "PHP", + "stars": 37456, + "languageRank": 3, + "webRank": 2, + "revision": "d655757fb306aca893651958030b5dae69ed0f39", + "checks": [ + { + "category": "language", + "kind": "language", + "path": ".", + "language": "php" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "composer" + }, + { + "category": "version", + "kind": "version", + "path": ".", + "language": "php", + "value": "^8.3", + "source": "composer.json" + } + ] + }, + { + "repository": "laravel/framework", + "language": "PHP", + "stars": 34995, + "languageRank": 4, + "webRank": 3, + "revision": "338e44065c0a5b18f1f290605b6253a1e69d8f56", + "checks": [ + { + "category": "language", + "kind": "language", + "path": ".", + "language": "php" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "composer" + }, + { + "category": "name", + "kind": "name", + "path": ".", + "name": "laravel/framework" + }, + { + "category": "version", + "kind": "version", + "path": ".", + "language": "php", + "value": "^8.3", + "source": "composer.json" + }, + { + "category": "identity", + "kind": "detection", + "path": ".", + "id": "laravel" + } + ] + }, + { + "repository": "filamentphp/filament", + "language": "PHP", + "stars": 32462, + "languageRank": 5, + "webRank": 4, + "revision": "9d78093cfebdc96c077b1d8dbf94b41738f28992", + "checks": [ + { + "category": "language", + "kind": "language", + "path": ".", + "language": "php" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "composer" + }, + { + "category": "name", + "kind": "name", + "path": ".", + "name": "filament/filament" + }, + { + "category": "version", + "kind": "version", + "path": ".", + "language": "php", + "value": "^8.2", + "source": "composer.json" + } + ] + }, + { + "repository": "symfony/symfony", + "language": "PHP", + "stars": 31523, + "languageRank": 6, + "webRank": 5, + "revision": "f07928484c9e96646964deb7b9570d7885492512", + "checks": [ + { + "category": "language", + "kind": "language", + "path": ".", + "language": "php" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "composer" + }, + { + "category": "name", + "kind": "name", + "path": ".", + "name": "symfony/symfony" + }, + { + "category": "version", + "kind": "version", + "path": ".", + "language": "php", + "value": ">=8.4.1", + "source": "composer.json" + }, + { + "category": "identity", + "kind": "detection", + "path": ".", + "id": "symfony" + } + ] + }, + { + "repository": "Significant-Gravitas/AutoGPT", + "language": "Python", + "stars": 187972, + "languageRank": 11, + "webRank": 1, + "revision": "9cb76f9026d388c9bf2e098af2beae1cc864f055", + "checks": [ + { + "category": "language", + "kind": "language", + "path": "autogpt_platform/backend", + "language": "python" + }, + { + "category": "project", + "kind": "detection", + "path": "autogpt_platform/backend", + "id": "python" + }, + { + "category": "name", + "kind": "name", + "path": "autogpt_platform/backend", + "name": "autogpt-platform-backend" + }, + { + "category": "version", + "kind": "version", + "path": "autogpt_platform/backend", + "language": "python", + "value": ">=3.11,<3.14", + "source": "autogpt_platform/backend/pyproject.toml" + }, + { + "category": "framework", + "kind": "detection", + "path": "autogpt_platform/backend", + "id": "fastapi" + }, + { + "category": "framework", + "kind": "detection", + "path": "autogpt_platform/frontend", + "id": "nextjs" + } + ] + }, + { + "repository": "AUTOMATIC1111/stable-diffusion-webui", + "language": "Python", + "stars": 165269, + "languageRank": 14, + "webRank": 2, + "revision": "82a973c04367123ae98bd9abdf80d9eda9b910e2", + "checks": [ + { + "category": "language", + "kind": "language", + "path": ".", + "language": "python" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "python" + }, + { + "category": "framework", + "kind": "detection", + "path": ".", + "id": "fastapi" + } + ] + }, + { + "repository": "langgenius/dify", + "language": "Python", + "stars": 160774, + "languageRank": 15, + "webRank": 3, + "revision": "fdb09a9e76dc8cbd052fabd72cfff4071328d9bc", + "checks": [ + { + "category": "language", + "kind": "language", + "path": "api", + "language": "python" + }, + { + "category": "project", + "kind": "detection", + "path": "api", + "id": "python" + }, + { + "category": "name", + "kind": "name", + "path": "api", + "name": "dify-api" + }, + { + "category": "version", + "kind": "version", + "path": "api", + "language": "python", + "value": "~=3.13.0", + "source": "api/pyproject.toml" + }, + { + "category": "framework", + "kind": "detection", + "path": "api", + "id": "flask" + }, + { + "category": "framework", + "kind": "detection", + "path": "api", + "id": "celery" + }, + { + "category": "framework", + "kind": "detection", + "path": "web", + "id": "nextjs" + }, + { + "category": "workspace", + "kind": "workspace", + "path": ".", + "id": "node-workspace", + "member": "web" + }, + { + "category": "workspace", + "kind": "workspace", + "path": "api", + "id": "python-workspace", + "member": "api/providers/vdb/vdb-alibabacloud-mysql" + }, + { + "category": "catalog", + "kind": "version", + "path": "web", + "language": "typescript", + "value": "npm:@typescript/typescript6@6.0.2", + "source": "pnpm-workspace.yaml" + } + ] + }, + { + "repository": "langflow-ai/langflow", + "language": "Python", + "stars": 156479, + "languageRank": 16, + "webRank": 4, + "revision": "60999cc9a7350e44dde3ef641baae864e10861f0", + "checks": [ + { + "category": "language", + "kind": "language", + "path": ".", + "language": "python" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "python" + }, + { + "category": "name", + "kind": "name", + "path": ".", + "name": "langflow" + }, + { + "category": "version", + "kind": "version", + "path": ".", + "language": "python", + "value": ">=3.10,<3.15", + "source": "pyproject.toml" + }, + { + "category": "framework", + "kind": "detection", + "path": "src/backend/base", + "id": "fastapi" + }, + { + "category": "framework", + "kind": "detection", + "path": "src/backend/base", + "id": "starlette" + } + ] + }, + { + "repository": "Comfy-Org/ComfyUI", + "language": "Python", + "stars": 145983, + "languageRank": 20, + "webRank": 5, + "revision": "540a82ee722bda1f32a1fbd97fcadadeb22d548b", + "checks": [ + { + "category": "language", + "kind": "language", + "path": ".", + "language": "python" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "python" + }, + { + "category": "name", + "kind": "name", + "path": ".", + "name": "ComfyUI" + }, + { + "category": "version", + "kind": "version", + "path": ".", + "language": "python", + "value": ">=3.10", + "source": "pyproject.toml" + } + ] + }, + { + "repository": "rails/rails", + "language": "Ruby", + "stars": 59087, + "languageRank": 1, + "webRank": 1, + "revision": "3a0aff317bc037f43f168e3772028f1033f1e810", + "checks": [ + { + "category": "language", + "kind": "language", + "path": ".", + "language": "ruby" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "bundler" + }, + { + "category": "identity", + "kind": "detection", + "path": ".", + "id": "rails" + } + ] + }, + { + "repository": "jekyll/jekyll", + "language": "Ruby", + "stars": 51756, + "languageRank": 2, + "webRank": 2, + "revision": "97c3bdc822df9d5bea4c269b29a28de5a5e29d70", + "checks": [ + { + "category": "language", + "kind": "language", + "path": ".", + "language": "ruby" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "bundler" + } + ] + }, + { + "repository": "mastodon/mastodon", + "language": "Ruby", + "stars": 50502, + "languageRank": 3, + "webRank": 3, + "revision": "5a207bc8f5046e27cffc640a3c64f96504f6afcf", + "checks": [ + { + "category": "language", + "kind": "language", + "path": ".", + "language": "ruby" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "bundler" + }, + { + "category": "version", + "kind": "version", + "path": ".", + "language": "ruby", + "value": "4.0.9", + "source": ".ruby-version" + }, + { + "category": "framework", + "kind": "detection", + "path": ".", + "id": "rails" + } + ] + }, + { + "repository": "huginn/huginn", + "language": "Ruby", + "stars": 50454, + "languageRank": 4, + "webRank": 4, + "revision": "639c050b2bf4e14ef9983012af99dfc1dc68b334", + "checks": [ + { + "category": "language", + "kind": "language", + "path": ".", + "language": "ruby" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "bundler" + }, + { + "category": "version", + "kind": "version", + "path": ".", + "language": "ruby", + "value": ">=3.4.0", + "source": "Gemfile" + }, + { + "category": "framework", + "kind": "detection", + "path": ".", + "id": "rails" + } + ] + }, + { + "repository": "discourse/discourse", + "language": "Ruby", + "stars": 48066, + "languageRank": 6, + "webRank": 5, + "revision": "899c68d783e7808c50144752369a685220250d7c", + "checks": [ + { + "category": "language", + "kind": "language", + "path": ".", + "language": "ruby" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "bundler" + }, + { + "category": "version", + "kind": "version", + "path": ".", + "language": "ruby", + "value": "~> 4.0.0", + "source": "Gemfile" + }, + { + "category": "framework", + "kind": "detection", + "path": ".", + "id": "rails" + }, + { + "category": "workspace", + "kind": "workspace", + "path": ".", + "id": "node-workspace", + "member": "frontend/discourse" + } + ] + }, + { + "repository": "dani-garcia/vaultwarden", + "language": "Rust", + "stars": 70490, + "languageRank": 17, + "webRank": 1, + "revision": "17b159beff4b892f07f61f38ce2fe452e55251c1", + "checks": [ + { + "category": "language", + "kind": "language", + "path": ".", + "language": "rust" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "cargo" + }, + { + "category": "name", + "kind": "name", + "path": ".", + "name": "vaultwarden" + }, + { + "category": "version", + "kind": "version", + "path": ".", + "language": "rust", + "value": "1.97.1", + "source": "Cargo.toml" + }, + { + "category": "framework", + "kind": "detection", + "path": ".", + "id": "rocket" + }, + { + "category": "workspace", + "kind": "workspace", + "path": ".", + "id": "cargo-workspace", + "member": "macros" + } + ] + }, + { + "repository": "meilisearch/meilisearch", + "language": "Rust", + "stars": 59822, + "languageRank": 29, + "webRank": 2, + "revision": "8c7bfcfa9bcff7083bcb5d184fa2a523a23884d8", + "checks": [ + { + "category": "language", + "kind": "language", + "path": "crates/meilisearch", + "language": "rust" + }, + { + "category": "project", + "kind": "detection", + "path": "crates/meilisearch", + "id": "cargo" + }, + { + "category": "name", + "kind": "name", + "path": "crates/meilisearch", + "name": "meilisearch" + }, + { + "category": "framework", + "kind": "detection", + "path": "crates/meilisearch", + "id": "actix-web" + }, + { + "category": "workspace", + "kind": "workspace", + "path": ".", + "id": "cargo-workspace", + "member": "crates/meilisearch" + } + ] + }, + { + "repository": "juspay/hyperswitch", + "language": "Rust", + "stars": 45760, + "languageRank": 39, + "webRank": 3, + "revision": "5786b738c088b8452296ca19922fcdc0dc74e129", + "checks": [ + { + "category": "language", + "kind": "language", + "path": "crates/router", + "language": "rust" + }, + { + "category": "project", + "kind": "detection", + "path": "crates/router", + "id": "cargo" + }, + { + "category": "name", + "kind": "name", + "path": "crates/router", + "name": "router" + }, + { + "category": "version", + "kind": "version", + "path": "crates/router", + "language": "rust", + "value": "1.94.1", + "source": "Cargo.toml" + }, + { + "category": "framework", + "kind": "detection", + "path": "crates/router", + "id": "actix-web" + }, + { + "category": "workspace", + "kind": "workspace", + "path": ".", + "id": "cargo-workspace", + "member": "crates/router" + } + ] + }, + { + "repository": "parcel-bundler/parcel", + "language": "Rust", + "stars": 44012, + "languageRank": 42, + "webRank": 4, + "revision": "20c0f2ede4c1044092d44288169cce589c5d5c14", + "checks": [ + { + "category": "language", + "kind": "language", + "path": ".", + "language": "rust" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "cargo-workspace" + }, + { + "category": "workspace", + "kind": "workspace", + "path": ".", + "id": "cargo-workspace", + "member": "crates/browserslist-compat" + } + ] + }, + { + "repository": "DioxusLabs/dioxus", + "language": "Rust", + "stars": 39705, + "languageRank": 52, + "webRank": 5, + "revision": "17d9a5924e845e4f37deca3fada6c35d1d836af0", + "checks": [ + { + "category": "language", + "kind": "language", + "path": ".", + "language": "rust" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "cargo" + }, + { + "category": "name", + "kind": "name", + "path": ".", + "name": "dioxus-examples" + }, + { + "category": "version", + "kind": "version", + "path": ".", + "language": "rust", + "value": "1.85.0", + "source": "Cargo.toml" + }, + { + "category": "workspace", + "kind": "workspace", + "path": ".", + "id": "cargo-workspace", + "member": "packages/dioxus" + } + ] + }, + { + "repository": "freeCodeCamp/freeCodeCamp", + "language": "TypeScript", + "stars": 458826, + "languageRank": 1, + "webRank": 1, + "revision": "11442d66944785ac72a2a2b9c955fae8b9f33083", + "checks": [ + { + "category": "language", + "kind": "language", + "path": ".", + "language": "typescript" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "node" + }, + { + "category": "name", + "kind": "name", + "path": ".", + "name": "@freecodecamp/freecodecamp" + }, + { + "category": "version", + "kind": "version", + "path": ".", + "language": "javascript", + "value": ">=24", + "source": "package.json" + }, + { + "category": "version", + "kind": "version", + "path": ".", + "language": "typescript", + "value": "7.0.2", + "source": "package.json" + }, + { + "category": "workspace", + "kind": "workspace", + "path": ".", + "id": "node-workspace", + "member": "api" + } + ] + }, + { + "repository": "vuejs/vue", + "language": "TypeScript", + "stars": 212624, + "languageRank": 7, + "webRank": 2, + "revision": "9e88707940088cb1f4cd7dd210c9168a50dc347c", + "checks": [ + { + "category": "language", + "kind": "language", + "path": ".", + "language": "typescript" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "node" + }, + { + "category": "name", + "kind": "name", + "path": ".", + "name": "vue" + }, + { + "category": "version", + "kind": "version", + "path": ".", + "language": "typescript", + "value": "^4.8.4", + "source": "package.json" + }, + { + "category": "identity", + "kind": "detection", + "path": ".", + "id": "vue" + }, + { + "category": "workspace", + "kind": "workspace", + "path": ".", + "id": "node-workspace", + "member": "packages/compiler-sfc" + } + ] + }, + { + "repository": "n8n-io/n8n", + "language": "TypeScript", + "stars": 209927, + "languageRank": 8, + "webRank": 3, + "revision": "93243edc00b288a840fbb562a5e9f64177f2337f", + "checks": [ + { + "category": "language", + "kind": "language", + "path": "packages/cli", + "language": "typescript" + }, + { + "category": "project", + "kind": "detection", + "path": "packages/cli", + "id": "node" + }, + { + "category": "name", + "kind": "name", + "path": "packages/cli", + "name": "n8n" + }, + { + "category": "version", + "kind": "version", + "path": "packages/cli", + "language": "javascript", + "value": ">=24.0.0", + "source": "packages/cli/package.json" + }, + { + "category": "framework", + "kind": "detection", + "path": "packages/cli", + "id": "express" + }, + { + "category": "workspace", + "kind": "workspace", + "path": ".", + "id": "node-workspace", + "member": "packages/cli" + }, + { + "category": "catalog", + "kind": "version", + "path": "packages/cli", + "language": "typescript", + "value": "npm:@typescript/typescript6@6.0.2", + "source": "pnpm-workspace.yaml" + } + ] + }, + { + "repository": "firecrawl/firecrawl", + "language": "TypeScript", + "stars": 200129, + "languageRank": 9, + "webRank": 4, + "revision": "40c8ff95d719d2acbd8fd6273bd19b14a1c2bbb8", + "checks": [ + { + "category": "language", + "kind": "language", + "path": "apps/api", + "language": "typescript" + }, + { + "category": "project", + "kind": "detection", + "path": "apps/api", + "id": "node" + }, + { + "category": "name", + "kind": "name", + "path": "apps/api", + "name": "firecrawl-scraper-js" + }, + { + "category": "version", + "kind": "version", + "path": "apps/api", + "language": "typescript", + "value": "npm:@typescript/typescript6@6.0.2", + "source": "apps/api/package.json" + }, + { + "category": "framework", + "kind": "detection", + "path": "apps/api", + "id": "express" + } + ] + }, + { + "repository": "excalidraw/excalidraw", + "language": "TypeScript", + "stars": 135400, + "languageRank": 15, + "webRank": 5, + "revision": "701b901a0f1ba3d0b390d9cdf63d112e25abac4b", + "checks": [ + { + "category": "language", + "kind": "language", + "path": ".", + "language": "typescript" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "node" + }, + { + "category": "name", + "kind": "name", + "path": ".", + "name": "excalidraw-monorepo" + }, + { + "category": "version", + "kind": "version", + "path": ".", + "language": "javascript", + "value": ">=18.0.0", + "source": "package.json" + }, + { + "category": "version", + "kind": "version", + "path": ".", + "language": "typescript", + "value": "5.9.3", + "source": "package.json" + }, + { + "category": "framework", + "kind": "detection", + "path": "excalidraw-app", + "id": "react" + }, + { + "category": "workspace", + "kind": "workspace", + "path": ".", + "id": "node-workspace", + "member": "packages/excalidraw" + } + ] + } + ] +} diff --git a/tests/detect-corpus.md b/tests/detect-corpus.md new file mode 100644 index 0000000..a7219c9 --- /dev/null +++ b/tests/detect-corpus.md @@ -0,0 +1,100 @@ +# GitHub project detection audit + +Selected 2026-11-09; detector commit `244ab44ccf1a` (PR #20 baseline). + +## Method + +Five manually qualified web projects per supported language, ordered by GitHub stars among non-fork, non-archived repositories. Language means GitHub’s primary-language classification, not a measured ranking of the languages themselves. Production learning platforms qualify; tutorial collections, lists, coding agents, general runtimes, native desktop apps, and generic infrastructure tools do not. Eligibility is a judgment call, not an objective GitHub category. + +Examples excluded above selected projects: Node.js (general runtime), Ollama (model runtime), Kubernetes (infrastructure), frp (generic networking), and developer-roadmap (educational roadmap collection). Stars and language ranks are frozen with full commit SHAs in [`detect-corpus.json`](detect-corpus.json). + +Checks target reviewed project paths and static manifest declarations, not a matching framework somewhere in the repository’s examples. Identity checks separately ask whether framework implementations identify themselves. Catalog checks ask for the local catalog declaration rather than an unresolved reference. These last two categories probe capabilities beyond the current dependency-only behavior. + +GitHub source archives were downloaded into a temporary cache. No dependencies were installed and no repository code was executed. Submodules and LFS targets were not fetched; symlinks were not extracted. The corpus occupies approximately 4.1 GiB. This is a sampled assertion suite, not an exhaustive precision/recall or buildability evaluation. + +## Results + +All **35/35** scans succeeded and produced identical output across five runs. **156/173 assertions passed**, and 23/35 repositories passed every sampled assertion. + +| Category | Passed | +|---|---:| +| language | 35/35 | +| project | 35/35 | +| name | 26/27 | +| version | 32/32 | +| framework | 19/20 | +| workspace | 9/15 | +| identity | 0/7 | +| catalog | 0/2 | + +β€œVersion” checks cover selected literal constraints and inherited declarations, not installed versions. Framework checks cover selected consumer projects, while identity checks cover framework implementation packages. + +### Gaps found + +- **pnpm workspaces:** Next.js, freeCodeCamp, Vue, n8n, Dify, and Discourse lack Node workspace labels/member relationships. Their component projects are discovered individually. Cargo, npm/Yarn, and uv representatives passed. +- **Rails component dependencies:** Discourse declares `actionpack`, `railties`, and the other Rails components instead of `gem "rails"`; its root is reported as Bundler, not Rails. +- **Poetry names:** AutoGPT’s `autogpt_platform/backend/pyproject.toml` declares `tool.poetry.name = "autogpt-platform-backend"`, but the project’s names array is empty. +- **Framework identities:** React (`packages/react`), Next.js (`packages/next`), Vue, Laravel, Symfony, Rails, and Gin are not framework-labeled at their implementation package paths. Consumers/examples elsewhere must not mask this. A future identity rule should distinguish β€œprovides” from β€œuses” rather than assume these packages are deployable applications. +- **pnpm catalogs:** n8n and Dify expose `catalog:typescript-tooling` / `catalog:` as TypeScript requirements without resolving the local `npm:@typescript/typescript6@6.0.2` declaration. + +Additional unscored coverage gaps include Jekyll, Hugo, Dioxus, Ember, Gradio, and aiohttp. Their generic language/project fallback is not counted as a failed framework assertion. Test fixtures and example projects are intentionally scanned, so the report is not a list of deployable services. + +### Timing + +Each repository was scanned through the release CLI five times, serially. The table reports the median of runs 2–5 with a warm filesystem, including process startup, traversal, JSON serialization, and stdout transfer. Python JSON parsing and snapshot downloads are excluded. These are local measurements, not portable timing budgets or controlled cold-cache benchmarks. + +Across the corpus, the scanner counted 259,818 files and read only 2,204 manifest/version files (1.77 MiB). The median repository time was 7.9 ms; the slowest was Next.js at 168.4 ms across 16,954 directories and 853 reported projects. + +| Language | Repository | Stars | Language rank | Revision | Assertions | Median ms | +|---|---|---:|---:|---|---:|---:| +| JavaScript | [react/react](https://github.com/react/react) | 252,455 | 2 | [`b05c0b75fd97`](https://github.com/react/react/tree/b05c0b75fd9778536152eead30814c9ca90c810a) | 5/6 | 14.47 | +| JavaScript | [vercel/next.js](https://github.com/vercel/next.js) | 143,437 | 8 | [`5368e013e74e`](https://github.com/vercel/next.js/tree/5368e013e74e4e4ed5be51c051c989545c4450b9) | 5/7 | 168.41 | +| JavaScript | [mrdoob/three.js](https://github.com/mrdoob/three.js) | 117,953 | 11 | [`3f7edc3d75e0`](https://github.com/mrdoob/three.js/tree/3f7edc3d75e054bc755dad56c1c95c62f63a3561) | 3/3 | 4.31 | +| JavaScript | [axios/axios](https://github.com/axios/axios) | 109,295 | 14 | [`335e2f8464f5`](https://github.com/axios/axios/tree/335e2f8464f58a3ab485486b8d6964fea3e3f30a) | 4/4 | 2.18 | +| JavaScript | [react/create-react-app](https://github.com/react/create-react-app) | 103,230 | 15 | [`6254386531d2`](https://github.com/react/create-react-app/tree/6254386531d263688ccfa542d0e628fbc0de0b28) | 6/6 | 2.30 | +| TypeScript | [freeCodeCamp/freeCodeCamp](https://github.com/freeCodeCamp/freeCodeCamp) | 458,826 | 1 | [`11442d669447`](https://github.com/freeCodeCamp/freeCodeCamp/tree/11442d66944785ac72a2a2b9c955fae8b9f33083) | 5/6 | 14.84 | +| TypeScript | [vuejs/vue](https://github.com/vuejs/vue) | 212,624 | 7 | [`9e8870794008`](https://github.com/vuejs/vue/tree/9e88707940088cb1f4cd7dd210c9168a50dc347c) | 4/6 | 2.58 | +| TypeScript | [n8n-io/n8n](https://github.com/n8n-io/n8n) | 209,927 | 8 | [`93243edc00b2`](https://github.com/n8n-io/n8n/tree/93243edc00b288a840fbb562a5e9f64177f2337f) | 5/7 | 60.35 | +| TypeScript | [firecrawl/firecrawl](https://github.com/firecrawl/firecrawl) | 200,129 | 9 | [`40c8ff95d719`](https://github.com/firecrawl/firecrawl/tree/40c8ff95d719d2acbd8fd6273bd19b14a1c2bbb8) | 5/5 | 4.86 | +| TypeScript | [excalidraw/excalidraw](https://github.com/excalidraw/excalidraw) | 135,400 | 15 | [`701b901a0f1b`](https://github.com/excalidraw/excalidraw/tree/701b901a0f1ba3d0b390d9cdf63d112e25abac4b) | 7/7 | 3.94 | +| Python | [Significant-Gravitas/AutoGPT](https://github.com/Significant-Gravitas/AutoGPT) | 187,972 | 11 | [`9cb76f9026d3`](https://github.com/Significant-Gravitas/AutoGPT/tree/9cb76f9026d388c9bf2e098af2beae1cc864f055) | 5/6 | 14.63 | +| Python | [AUTOMATIC1111/stable-diffusion-webui](https://github.com/AUTOMATIC1111/stable-diffusion-webui) | 165,269 | 14 | [`82a973c04367`](https://github.com/AUTOMATIC1111/stable-diffusion-webui/tree/82a973c04367123ae98bd9abdf80d9eda9b910e2) | 3/3 | 1.60 | +| Python | [langgenius/dify](https://github.com/langgenius/dify) | 160,774 | 15 | [`fdb09a9e76dc`](https://github.com/langgenius/dify/tree/fdb09a9e76dc8cbd052fabd72cfff4071328d9bc) | 8/10 | 30.46 | +| Python | [langflow-ai/langflow](https://github.com/langflow-ai/langflow) | 156,479 | 16 | [`60999cc9a735`](https://github.com/langflow-ai/langflow/tree/60999cc9a7350e44dde3ef641baae864e10861f0) | 6/6 | 20.32 | +| Python | [Comfy-Org/ComfyUI](https://github.com/Comfy-Org/ComfyUI) | 145,983 | 20 | [`540a82ee722b`](https://github.com/Comfy-Org/ComfyUI/tree/540a82ee722bda1f32a1fbd97fcadadeb22d548b) | 4/4 | 3.49 | +| PHP | [coollabsio/coolify](https://github.com/coollabsio/coolify) | 65,849 | 2 | [`30335d80866f`](https://github.com/coollabsio/coolify/tree/30335d80866ff3391d899cf4ccb599b696f47b7e) | 5/5 | 5.30 | +| PHP | [nextcloud/server](https://github.com/nextcloud/server) | 37,456 | 3 | [`d655757fb306`](https://github.com/nextcloud/server/tree/d655757fb306aca893651958030b5dae69ed0f39) | 3/3 | 14.62 | +| PHP | [laravel/framework](https://github.com/laravel/framework) | 34,995 | 4 | [`338e44065c0a`](https://github.com/laravel/framework/tree/338e44065c0a5b18f1f290605b6253a1e69d8f56) | 4/5 | 6.33 | +| PHP | [filamentphp/filament](https://github.com/filamentphp/filament) | 32,462 | 5 | [`9d78093cfebd`](https://github.com/filamentphp/filament/tree/9d78093cfebdc96c077b1d8dbf94b41738f28992) | 4/4 | 16.63 | +| PHP | [symfony/symfony](https://github.com/symfony/symfony) | 31,523 | 6 | [`f07928484c9e`](https://github.com/symfony/symfony/tree/f07928484c9e96646964deb7b9570d7885492512) | 4/5 | 10.10 | +| Ruby | [rails/rails](https://github.com/rails/rails) | 59,087 | 1 | [`3a0aff317bc0`](https://github.com/rails/rails/tree/3a0aff317bc037f43f168e3772028f1033f1e810) | 2/3 | 7.90 | +| Ruby | [jekyll/jekyll](https://github.com/jekyll/jekyll) | 51,756 | 2 | [`97c3bdc822df`](https://github.com/jekyll/jekyll/tree/97c3bdc822df9d5bea4c269b29a28de5a5e29d70) | 2/2 | 2.42 | +| Ruby | [mastodon/mastodon](https://github.com/mastodon/mastodon) | 50,502 | 3 | [`5a207bc8f504`](https://github.com/mastodon/mastodon/tree/5a207bc8f5046e27cffc640a3c64f96504f6afcf) | 4/4 | 8.68 | +| Ruby | [huginn/huginn](https://github.com/huginn/huginn) | 50,454 | 4 | [`639c050b2bf4`](https://github.com/huginn/huginn/tree/639c050b2bf4e14ef9983012af99dfc1dc68b334) | 4/4 | 2.35 | +| Ruby | [discourse/discourse](https://github.com/discourse/discourse) | 48,066 | 6 | [`899c68d783e7`](https://github.com/discourse/discourse/tree/899c68d783e7808c50144752369a685220250d7c) | 3/5 | 35.98 | +| Go | [infiniflow/ragflow](https://github.com/infiniflow/ragflow) | 93,887 | 8 | [`ed5a84fee9ab`](https://github.com/infiniflow/ragflow/tree/ed5a84fee9abc8091aea111d1195cf7768541ff1) | 5/5 | 10.20 | +| Go | [gohugoio/hugo](https://github.com/gohugoio/hugo) | 90,425 | 9 | [`d2010fcc9cd3`](https://github.com/gohugoio/hugo/tree/d2010fcc9cd30c5b4e9274ded4485f4222326b53) | 4/4 | 4.85 | +| Go | [gin-gonic/gin](https://github.com/gin-gonic/gin) | 89,609 | 11 | [`0f09c3a9b462`](https://github.com/gin-gonic/gin/tree/0f09c3a9b4626d9fe9979cf4d0521d6ae32ab646) | 4/5 | 2.11 | +| Go | [netdata/netdata](https://github.com/netdata/netdata) | 81,330 | 14 | [`0d54b4424ea6`](https://github.com/netdata/netdata/tree/0d54b4424ea6f40c344738b9d566e1a26d160da3) | 4/4 | 23.42 | +| Go | [caddyserver/caddy](https://github.com/caddyserver/caddy) | 78,927 | 15 | [`9ea3108377d2`](https://github.com/caddyserver/caddy/tree/9ea3108377d2abec102866808133c40ac9582162) | 4/4 | 2.56 | +| Rust | [dani-garcia/vaultwarden](https://github.com/dani-garcia/vaultwarden) | 70,490 | 17 | [`17b159beff4b`](https://github.com/dani-garcia/vaultwarden/tree/17b159beff4b892f07f61f38ce2fe452e55251c1) | 6/6 | 2.51 | +| Rust | [meilisearch/meilisearch](https://github.com/meilisearch/meilisearch) | 59,822 | 29 | [`8c7bfcfa9bcf`](https://github.com/meilisearch/meilisearch/tree/8c7bfcfa9bcff7083bcb5d184fa2a523a23884d8) | 5/5 | 4.99 | +| Rust | [juspay/hyperswitch](https://github.com/juspay/hyperswitch) | 45,760 | 39 | [`5786b738c088`](https://github.com/juspay/hyperswitch/tree/5786b738c088b8452296ca19922fcdc0dc74e129) | 6/6 | 12.07 | +| Rust | [parcel-bundler/parcel](https://github.com/parcel-bundler/parcel) | 44,012 | 42 | [`20c0f2ede4c1`](https://github.com/parcel-bundler/parcel/tree/20c0f2ede4c1044092d44288169cce589c5d5c14) | 3/3 | 18.92 | +| Rust | [DioxusLabs/dioxus](https://github.com/DioxusLabs/dioxus) | 39,705 | 52 | [`17d9a5924e84`](https://github.com/DioxusLabs/dioxus/tree/17d9a5924e845e4f37deca3fada6c35d1d836af0) | 5/5 | 8.71 | + +## Reproduce + +```sh +cargo build --release --locked +cache=$(mktemp -d) +python3 tests/audit_detect.py --fetch --cache "$cache" --output "$cache/results.json" +``` + +Downloads require the explicit `--fetch` flag; omit it to rerun against an existing pinned cache entirely offline. Downloads are capped at 1 GiB compressed / 2 GiB expanded per repository. Existing unverified cache entries are not overwritten. The runner exits nonzero for missing assertions, command failures, or nondeterministic output; **exit 1 is expected for this baseline**. + +```sh +PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s tests -p 'test_*.py' -v +``` + +The runner uses Python 3.11+ and the standard library only. Full per-check results and repeat timings are written to the specified JSON output; downloaded repositories and generated reports are not committed. diff --git a/tests/test_audit_detect.py b/tests/test_audit_detect.py new file mode 100644 index 0000000..35dc673 --- /dev/null +++ b/tests/test_audit_detect.py @@ -0,0 +1,155 @@ +import io +import json +import tarfile +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +import audit_detect + + +class DetectionAuditTests(unittest.TestCase): + def setUp(self): + self.project = { + "path": ".", + "names": ["app"], + "languages": [ + { + "name": "javascript", + "versions": [{"value": ">=22", "source": "package.json"}], + } + ], + "detections": [{"id": "node"}, {"id": "node-workspace"}], + "workspaceMembers": ["packages/api"], + } + self.entry = {"repository": "owner/repo", "revision": "a" * 40} + + def test_checks_are_scoped_to_the_expected_project(self): + cases = [ + {"kind": "language", "language": "javascript"}, + {"kind": "detection", "id": "node"}, + {"kind": "name", "name": "app"}, + {"kind": "workspace", "id": "node-workspace", "member": "packages/api"}, + { + "kind": "version", + "language": "javascript", + "value": ">=22", + "source": "package.json", + }, + ] + for case in cases: + with self.subTest(case=case): + self.assertTrue( + audit_detect.check({".": self.project}, {**case, "path": "."})[0] + ) + self.assertFalse( + audit_detect.check( + {".": self.project}, {**case, "path": "missing"} + )[0] + ) + wrong = { + "kind": "version", + "path": ".", + "language": "javascript", + "value": ">=22", + "source": "wrong.json", + } + self.assertFalse(audit_detect.check({".": self.project}, wrong)[0]) + with self.assertRaises(ValueError): + audit_detect.check({".": self.project}, {"kind": "unknown", "path": "."}) + + def archive(self, path="repo/package.json", size=None): + data = b"{}" if size is None else b"x" * size + output = io.BytesIO() + with tarfile.open(fileobj=output, mode="w:gz") as archive: + member = tarfile.TarInfo(path) + member.size = len(data) + archive.addfile(member, io.BytesIO(data)) + link = tarfile.TarInfo("repo/linked.json") + link.type = tarfile.SYMTYPE + link.linkname = "/etc/passwd" + archive.addfile(link) + return output.getvalue() + + def test_missing_snapshots_never_download_without_fetch(self): + with ( + tempfile.TemporaryDirectory() as directory, + patch("urllib.request.urlopen") as request, + ): + with self.assertRaisesRegex(ValueError, "pass --fetch"): + audit_detect.snapshot(self.entry, Path(directory), False) + request.assert_not_called() + + def test_cached_snapshot_requires_the_pinned_revision(self): + with ( + tempfile.TemporaryDirectory() as directory, + patch("urllib.request.urlopen") as request, + ): + cache = Path(directory) + destination = cache / "repos/owner--repo" + destination.mkdir(parents=True) + metadata = cache / "metadata/owner--repo.json" + metadata.parent.mkdir() + metadata.write_text(json.dumps(self.entry)) + self.assertEqual( + audit_detect.snapshot(self.entry, cache, False), destination + ) + metadata.write_text(json.dumps({**self.entry, "revision": "b" * 40})) + with self.assertRaisesRegex(ValueError, "unverified snapshot exists"): + audit_detect.snapshot(self.entry, cache, True) + request.assert_not_called() + + def test_fetch_extracts_regular_files_but_not_symlinks(self): + with ( + tempfile.TemporaryDirectory() as directory, + patch("urllib.request.urlopen", return_value=io.BytesIO(self.archive())), + ): + destination = audit_detect.snapshot(self.entry, Path(directory), True) + self.assertEqual((destination / "package.json").read_text(), "{}") + self.assertFalse((destination / "linked.json").exists()) + self.assertFalse((destination / "linked.json").is_symlink()) + + def test_unsafe_archives_do_not_escape_the_cache(self): + for path in ["repo/../../escape", "/absolute/path"]: + with ( + self.subTest(path=path), + tempfile.TemporaryDirectory() as directory, + patch( + "urllib.request.urlopen", + return_value=io.BytesIO(self.archive(path)), + ), + ): + with self.assertRaisesRegex(ValueError, "unsafe archive path"): + audit_detect.snapshot(self.entry, Path(directory), True) + self.assertFalse((Path(directory) / "repos/owner--repo").exists()) + + def test_archive_and_expansion_limits_are_enforced(self): + for limit, archive, message in [ + (1, self.archive(), "archive exceeds"), + (512, self.archive(size=4096), "expanded snapshot exceeds"), + ]: + with ( + self.subTest(limit=limit), + tempfile.TemporaryDirectory() as directory, + patch("urllib.request.urlopen", return_value=io.BytesIO(archive)), + patch.object(audit_detect, "MAX_ARCHIVE_BYTES", limit), + self.assertRaisesRegex(ValueError, message), + ): + audit_detect.snapshot(self.entry, Path(directory), True) + + def test_invalid_repository_metadata_is_rejected(self): + for entry in [ + {**self.entry, "repository": "../repo"}, + {**self.entry, "revision": "main"}, + ]: + with ( + self.subTest(entry=entry), + tempfile.TemporaryDirectory() as directory, + self.assertRaisesRegex(ValueError, "invalid repository"), + ): + audit_detect.snapshot(entry, Path(directory), False) + + +if __name__ == "__main__": + unittest.main() From 1a0ac3daf745e156d99bd2160d86d2b1abf8ae71 Mon Sep 17 00:00:00 2001 From: Dillen Meijboom Date: Mon, 9 Nov 2026 21:38:50 +0000 Subject: [PATCH 03/10] fix: complete project detection and accelerate large monorepo scans --- Cargo.lock | 59 ++++++- Cargo.toml | 1 + README.md | 10 +- src/detect/mod.rs | 370 ++++++++++++++++++++++++++------------- src/detect/parsers.rs | 271 ++++++++++++++++++++++++++--- src/detect/rules.rs | 160 +++++++++++++---- src/detect/rules.toml | 65 +++---- src/detect/tests.rs | 380 ++++++++++++++++++++++++++++++++++++++++- src/main.rs | 49 +++++- src/output.rs | 6 +- tests/detect-corpus.md | 126 +++++++------- 11 files changed, 1224 insertions(+), 273 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d5a438a..771bd68 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -23,6 +23,17 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "annotate-snippets" +version = "0.12.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a5b0a5ab8561c603f16c3b78c24a386b1f70e7fc1b1b1e8c15738828e4a176" +dependencies = [ + "anstyle", + "memchr", + "unicode-width 0.2.3", +] + [[package]] name = "anstream" version = "1.0.0" @@ -79,6 +90,12 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "arraydeque" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d902e3d592a523def97af8f317b08ce16b7ab854c1985a0c671e6f15cebc236" + [[package]] name = "async-stream" version = "0.3.6" @@ -572,6 +589,7 @@ dependencies = [ "reqwest", "rustls", "serde", + "serde-saphyr", "serde_json", "sha2 0.10.9", "tar", @@ -881,6 +899,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "encoding_rs_io" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fba3fe847045ecff794b9c138293a80db914678c453ad63fbf0c6a9eb6e00b22" +dependencies = [ + "encoding_rs", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -1060,6 +1087,16 @@ dependencies = [ "r-efi 6.0.0", ] +[[package]] +name = "granit-parser" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f05586d4d4df3d47123207826c40b252728b438a1a07bad3c01ee6bea401df" +dependencies = [ + "arraydeque", + "smallvec", +] + [[package]] name = "h2" version = "0.4.15" @@ -1626,7 +1663,7 @@ checksum = "5f98efec8807c63c752b5bd61f862c165c115b0a35685bdcfd9238c7aeb592b7" dependencies = [ "cfg-if", "miette-derive", - "unicode-width", + "unicode-width 0.1.14", ] [[package]] @@ -2269,6 +2306,20 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-saphyr" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d43fb82d550b07e6f929a58d7d0935182602f48ddead875f5ee313c41c8f07fe" +dependencies = [ + "annotate-snippets", + "encoding_rs_io", + "granit-parser", + "num-traits", + "serde_core", + "smallvec", +] + [[package]] name = "serde_core" version = "1.0.228" @@ -2897,6 +2948,12 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" +[[package]] +name = "unicode-width" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca718471876992059f251e33aafc334346ada08042b7661dd1fe313a9bced557" + [[package]] name = "untrusted" version = "0.9.0" diff --git a/Cargo.toml b/Cargo.toml index 8cce05e..9cd8ff3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,7 @@ prost-types = "0.13.5" reqwest = { version = "0.13.4", default-features = false, features = ["charset", "http2", "json", "query", "rustls-no-provider", "stream", "system-proxy"] } rustls = { version = "0.23.40", default-features = false, features = ["ring", "std", "tls12"] } serde = { version = "1.0.228", features = ["derive"] } +serde-saphyr = { version = "2.1.1", default-features = false, features = ["deserialize"] } serde_json = "1.0.150" sha2 = "0.10.9" tar = "0.4.44" diff --git a/README.md b/README.md index 134555a..ed51bb3 100644 --- a/README.md +++ b/README.md @@ -274,9 +274,9 @@ brainpod detect ./services --json brainpod detect . --rules ./detection.toml --json ``` -The result contains sorted project paths, manifest names, languages and version declarations, priority-ranked detections with evidence, workspace members, diagnostics, and scan statistics. Framework versions are attached to dependency evidence. JavaScript versions describe Node.js; TypeScript versions describe the compiler. Requirements and pins retain their source and kind: these are declarations, not resolved or installed versions. Missing versions remain unknown. +The result contains sorted project paths, manifest names, languages and version declarations, priority-ranked detections with evidence and `uses`/`provides` framework roles, workspace members, diagnostics, and scan statistics. Framework versions are attached to dependency evidence. JavaScript versions describe Node.js; TypeScript versions describe the compiler. Requirements and pins retain their source and kind: these are declarations, not resolved or installed versions. Missing versions remain unknown. -Built-in coverage includes Laravel, Symfony, CakePHP, CodeIgniter, Composer; Go modules/workspaces and Gin/Echo/Fiber; Cargo workspaces, axum, Rocket, Actix; Bundler, Rails, Sinatra, Hanami; Python requirements, PEP 621, Poetry, Pipenv, uv workspaces, Django, FastAPI, Flask, Starlette, Celery; npm workspaces, Next.js, Nuxt, Astro, SvelteKit, Remix, Angular, Express, React, Vue, Vite; and standalone source files in all seven languages. Rust workspace dependency aliases and inherited `rust-version` are understood. Ancestor runtime pin/toolchain files are included with provenance. +Built-in coverage includes Laravel, Symfony, CakePHP, CodeIgniter, Composer; Go modules/workspaces and Gin/Echo/Fiber; Cargo workspaces, axum, Rocket, Actix; Bundler, Rails, Sinatra, Hanami; Python requirements, PEP 621, Poetry, Pipenv, uv workspaces, Django, FastAPI, Flask, Starlette, Celery; npm/pnpm workspaces and catalogs, Next.js, Nuxt, Astro, SvelteKit, Remix, Angular, Express, React, Vue, Vite; and standalone source files in all seven languages. Rust workspace dependency aliases and inherited `rust-version` are understood. Ancestor runtime pin/toolchain files are included with provenance. Poetry names, literal gemspec metadata, and Rails component dependencies are recognized. pnpm catalog references retain the original declaration and add the locally resolved constraint with its YAML source; unresolved references produce diagnostics. Rules live in [`src/detect/rules.toml`](src/detect/rules.toml), independently of manifest parsers. Additional rules use unique IDs: @@ -290,9 +290,9 @@ all = ["dep:npm:next", "file:company.config.*"] none = ["file:legacy.config"] ``` -`all` requires every pattern, `any` requires at least one when supplied, and `none` rejects any match. Higher priorities sort first; ties sort by ID. Matches coexist rather than hiding other frameworks or languages. Patterns support `file:` (`*`, `?`), `source:`, `dep::`, `workspace:`, and `package:cargo`. Workspace paths additionally support `**`. Ecosystems are `composer`, `go`, `cargo`, `gem`, `pypi`, and `npm`. +`all` requires every pattern, `any` requires at least one when supplied, and `none` rejects any match. Higher priorities sort first; ties sort by ID. Matches coexist rather than hiding other frameworks or languages. Patterns also accept conjunction groups, for example `any = ["dep:gem:rails", { all = ["dep:gem:actionpack", "dep:gem:railties"] }]`. Patterns support `file:` (`*`, `?`), `source:`, `dep::`, `identity::`, `workspace:`, and `package:cargo`. Workspace paths additionally support `**`. Ecosystems are `composer`, `go`, `cargo`, `gem`, `pypi`, and `npm`. -The scanner walks once, caches indexed rules, and never reads source bodies or lockfiles. Each manifest/version file is capped at 1 MiB. Symlinks are not followed; traversal stops at depth 128 with a diagnostic. Dependency/build directories such as `.git`, `.jj`, `node_modules`, `vendor`, `target`, `.venv`, `.next`, `dist`, and `build` are pruned. It does not evaluate `.gitignore`. Source-only directories roll up to a same-language project ancestor when available; sibling services keep their own manifest boundaries. +The scanner walks once, caches indexed rules, and never reads application source bodies or lockfiles. Wide trees use at most four directory workers; small trees stay serial. Directory facts are indexed and irrelevant empty directories are discarded, while output stays deterministic. pnpm YAML is deserialized directly into typed fields without building a syntax tree; unused fields/comments are not retained. Each manifest/version file is capped at 1 MiB. Symlinks are not followed; traversal stops at depth 128 with a diagnostic. Dependency/build directories such as `.git`, `.jj`, `node_modules`, `vendor`, `target`, `.venv`, `.next`, `dist`, and `build` are pruned. It does not evaluate `.gitignore`. Source-only directories roll up to a same-language project ancestor when available; sibling services keep their own manifest boundaries. Detection uses declared dependencies, not arbitrary substring matches or inferred framework usage from source. Composer/Cargo development dependencies are excluded; npm development dependencies are included because tools such as Astro commonly live there. Dynamic Ruby declarations and unsupported requirements options produce diagnostics instead of executing code. Invalid manifests are diagnosed without stopping discovery of other projects. @@ -302,7 +302,7 @@ Run the generated large-tree performance fixture with: cargo test --release detect::tests::benchmark_polyglot_scan -- --ignored --nocapture ``` -A [35-repository GitHub audit](tests/detect-corpus.md) records pinned real-world coverage, timings, known gaps, and an opt-in reproducible runner. +A [35-repository GitHub audit](tests/detect-corpus.md) records pinned real-world coverage, before/after timings, regression fixes, and an opt-in reproducible runner. ## Image discovery diff --git a/src/detect/mod.rs b/src/detect/mod.rs index b6d781d..5007d58 100644 --- a/src/detect/mod.rs +++ b/src/detect/mod.rs @@ -2,6 +2,8 @@ use std::collections::{BTreeMap, BTreeSet}; use std::fs::{self, File}; use std::io::Read; use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::thread; use anyhow::{Context, Result, bail}; use clap::Args; @@ -86,9 +88,18 @@ pub struct Detection { pub language: String, pub kind: Kind, pub priority: i32, + #[serde(skip_serializing_if = "Vec::is_empty")] + pub roles: Vec, pub evidence: Vec, } +#[derive(Clone, Copy, Debug, Serialize, PartialEq, Eq, PartialOrd, Ord)] +#[serde(rename_all = "kebab-case")] +pub enum FrameworkRole { + Uses, + Provides, +} + #[derive(Clone, Debug, Serialize, PartialEq, Eq, PartialOrd, Ord)] pub struct Evidence { pub source: String, @@ -115,10 +126,12 @@ pub struct Stats { #[derive(Default)] struct Node { path: PathBuf, + parent: Option, files: BTreeSet, contents: BTreeMap, sources: BTreeMap, dependencies: BTreeMap, + identities: BTreeMap, packages: BTreeSet, versions: BTreeMap>, names: BTreeSet, @@ -128,8 +141,10 @@ struct Node { struct Dependency { version: Option, source: String, + resolved: Option, } +#[derive(Clone)] struct Workspace { ecosystem: String, source: String, @@ -137,6 +152,13 @@ struct Workspace { exclude: Vec, } +impl Workspace { + fn contains(&self, path: &str) -> bool { + self.members.iter().any(|p| rules::path_glob(p, path)) + && !self.exclude.iter().any(|p| rules::path_glob(p, path)) + } +} + impl Node { fn dependency(&mut self, ecosystem: &str, name: &str, version: Option<&str>, file: &str) { self.dependencies.insert( @@ -144,6 +166,18 @@ impl Node { Dependency { version: version.map(str::to_owned), source: relative_file(&self.path, file), + resolved: None, + }, + ); + } + + fn identity(&mut self, ecosystem: &str, name: &str, version: Option<&str>, file: &str) { + self.identities.insert( + format!("{ecosystem}:{name}"), + Dependency { + version: version.map(str::to_owned), + source: relative_file(&self.path, file), + resolved: None, }, ); } @@ -176,36 +210,25 @@ pub fn handle(args: DetectArgs) -> Result { )) } -pub fn scan(root: &Path, extra_rules: Option<&str>) -> Result { - let custom = extra_rules - .map(|extra| Rules::new(Some(extra))) - .transpose()?; - let rules = match custom.as_ref() { - Some(rules) => rules, - None => Rules::builtin()?, - }; - let root = root - .canonicalize() - .with_context(|| format!("could not open {}", root.display()))?; - if !root.is_dir() { - bail!("detection root is not a directory: {}", root.display()); - } - let mut report = Report { - schema_version: 1, - projects: Vec::new(), - diagnostics: Vec::new(), - stats: Stats::default(), - }; - let mut nodes = Vec::new(); - let mut pending = vec![(PathBuf::new(), 0)]; - while let Some((path, depth)) = pending.pop() { - let mut node = Node { - path: path.clone(), - ..Node::default() - }; - report.stats.directories += 1; - let entries = fs::read_dir(root.join(&path)) - .with_context(|| format!("could not scan {}", root.join(&path).display()))?; +#[derive(Default)] +struct Inventory { + nodes: Vec, + diagnostics: Vec, + stats: Stats, +} + +impl Inventory { + fn read_directory( + &mut self, + root: &Path, + path: &Path, + depth: usize, + rules: &Rules, + ) -> Result> { + let mut node = Node::default(); + self.stats.directories += 1; + let entries = fs::read_dir(root.join(path)) + .with_context(|| format!("could not scan {}", root.join(path).display()))?; let mut children = Vec::new(); for entry in entries { let entry = entry?; @@ -215,18 +238,20 @@ pub fn scan(root: &Path, extra_rules: Option<&str>) -> Result { }; if ty.is_dir() { if IGNORED.contains(&name.as_str()) { - report.stats.skipped_directories += 1; + self.stats.skipped_directories += 1; } else if depth == MAX_DEPTH { - report.diagnostics.push(Diagnostic { - path: relative_file(&path, &name), + self.diagnostics.push(Diagnostic { + path: relative_file(path, &name), message: "directory depth limit reached".to_owned(), }); } else { children.push((path.join(&name), depth + 1)); } } else if ty.is_file() { - report.stats.files += 1; - if let Some(ext) = Path::new(&name).extension().and_then(|e| e.to_str()) { + self.stats.files += 1; + if let Some(ext) = Path::new(&name).extension().and_then(|e| e.to_str()) + && rules.needs_source(ext) + { node.sources .entry(ext.to_owned()) .and_modify(|existing| { @@ -240,8 +265,8 @@ pub fn scan(root: &Path, extra_rules: Option<&str>) -> Result { let file = File::open(entry.path())?; let mut bytes = Vec::new(); file.take(MAX_FILE_BYTES + 1).read_to_end(&mut bytes)?; - report.stats.files_read += 1; - report.stats.bytes_read += bytes.len(); + self.stats.files_read += 1; + self.stats.bytes_read += bytes.len(); let content = if bytes.len() > MAX_FILE_BYTES as usize { Err("manifest exceeds 1 MiB read limit".to_owned()) } else { @@ -251,8 +276,8 @@ pub fn scan(root: &Path, extra_rules: Option<&str>) -> Result { Ok(content) => { node.contents.insert(name.clone(), content); } - Err(message) => report.diagnostics.push(Diagnostic { - path: relative_file(&path, &name), + Err(message) => self.diagnostics.push(Diagnostic { + path: relative_file(path, &name), message, }), } @@ -262,26 +287,140 @@ pub fn scan(root: &Path, extra_rules: Option<&str>) -> Result { } } } - children.sort(); - pending.extend(children.into_iter().rev()); - nodes.push(node); + if !node.files.is_empty() || !node.sources.is_empty() { + node.path = path.to_path_buf(); + self.nodes.push(node); + } + Ok(children) } + + fn walk( + &mut self, + root: &Path, + rules: &Rules, + mut pending: Vec<(PathBuf, usize)>, + ) -> Result<()> { + while let Some((path, depth)) = pending.pop() { + pending.extend(self.read_directory(root, &path, depth, rules)?); + } + Ok(()) + } + + fn merge(&mut self, other: Self) { + self.nodes.extend(other.nodes); + self.diagnostics.extend(other.diagnostics); + self.stats.directories += other.stats.directories; + self.stats.files += other.stats.files; + self.stats.files_read += other.stats.files_read; + self.stats.bytes_read += other.stats.bytes_read; + self.stats.skipped_directories += other.stats.skipped_directories; + } +} + +fn inventory(root: &Path, rules: &Rules, workers: usize) -> Result { + let mut inventory = Inventory::default(); + let mut pending = vec![(PathBuf::new(), 0)]; + for _ in 0..3 { + let mut next = Vec::new(); + for (path, depth) in pending { + next.extend(inventory.read_directory(root, &path, depth, rules)?); + } + pending = next; + if pending.is_empty() { + break; + } + } + if workers > 1 && pending.len() >= 64 { + let next = AtomicUsize::new(0); + let results = thread::scope(|scope| { + let handles = (0..workers) + .map(|_| { + scope.spawn(|| -> Result { + let mut inventory = Inventory::default(); + while let Some(task) = pending.get(next.fetch_add(1, Ordering::Relaxed)) { + inventory.walk(root, rules, vec![task.clone()])?; + } + Ok(inventory) + }) + }) + .collect::>(); + handles + .into_iter() + .map(|handle| { + handle + .join() + .map_err(|_| anyhow::anyhow!("directory scan worker panicked"))? + }) + .collect::>>() + })?; + for result in results { + inventory.merge(result); + } + } else { + inventory.walk(root, rules, pending)?; + } + inventory.nodes.sort_by(|a, b| a.path.cmp(&b.path)); + let mut ancestors = Vec::::new(); + for index in 0..inventory.nodes.len() { + while ancestors.last().is_some_and(|&parent| { + !inventory.nodes[index] + .path + .starts_with(&inventory.nodes[parent].path) + }) { + ancestors.pop(); + } + inventory.nodes[index].parent = ancestors.last().copied(); + ancestors.push(index); + } + Ok(inventory) +} + +pub fn scan(root: &Path, extra_rules: Option<&str>) -> Result { + let custom = extra_rules + .map(|extra| Rules::new(Some(extra))) + .transpose()?; + let rules = match custom.as_ref() { + Some(rules) => rules, + None => Rules::builtin()?, + }; + let root = root + .canonicalize() + .with_context(|| format!("could not open {}", root.display()))?; + if !root.is_dir() { + bail!("detection root is not a directory: {}", root.display()); + } + let mut report = Report { + schema_version: 1, + projects: Vec::new(), + diagnostics: Vec::new(), + stats: Stats::default(), + }; + let workers = thread::available_parallelism().map_or(1, |n| n.get().min(4)); + let Inventory { + mut nodes, + diagnostics, + stats, + } = inventory(&root, rules, workers)?; + report.diagnostics = diagnostics; + report.stats = stats; parsers::parse(&mut nodes, &mut report.diagnostics); - let mut projects = BTreeMap::::new(); + let mut projects = Vec::<(usize, Project)>::new(); + let mut project_indices = vec![None; nodes.len()]; let mut source_detections = Vec::new(); - for node in &nodes { + for (index, node) in nodes.iter().enumerate() { let mut detections = Vec::new(); for detection in rules.evaluate(node) { if detection.kind == Kind::Source { - source_detections.push((&node.path, detection)); + source_detections.push((index, detection)); } else { detections.push(detection); } } if !detections.is_empty() { - projects.insert( - node.path.clone(), + project_indices[index] = Some(projects.len()); + projects.push(( + index, Project { path: display_path(&node.path), names: node.names.iter().cloned().collect(), @@ -289,29 +428,39 @@ pub fn scan(root: &Path, extra_rules: Option<&str>) -> Result { detections, workspace_members: Vec::new(), }, - ); + )); } } - for (path, detection) in source_detections { - let owner = path - .ancestors() - .find(|p| { - projects.get(*p).is_some_and(|project| { - project.detections.iter().any(|d| { - d.kind != Kind::Workspace - && family(&d.language) == family(&detection.language) - }) + for (index, detection) in source_detections { + let mut ancestor = Some(index); + let mut owner = None; + while let Some(i) = ancestor { + if let Some(project_index) = project_indices[i] + && projects[project_index].1.detections.iter().any(|d| { + d.kind != Kind::Workspace && family(&d.language) == family(&detection.language) }) - }) - .map(Path::to_path_buf) - .unwrap_or_else(|| path.clone()); - let project = projects.entry(owner.clone()).or_insert_with(|| Project { - path: display_path(&owner), - names: Vec::new(), - languages: Vec::new(), - detections: Vec::new(), - workspace_members: Vec::new(), + { + owner = Some(project_index); + break; + } + ancestor = nodes[i].parent; + } + let owner = owner.or(project_indices[index]).unwrap_or_else(|| { + let project_index = projects.len(); + project_indices[index] = Some(project_index); + projects.push(( + index, + Project { + path: display_path(&nodes[index].path), + names: Vec::new(), + languages: Vec::new(), + detections: Vec::new(), + workspace_members: Vec::new(), + }, + )); + project_index }); + let project = &mut projects[owner].1; if !project .detections .iter() @@ -320,7 +469,7 @@ pub fn scan(root: &Path, extra_rules: Option<&str>) -> Result { project.detections.push(detection); } } - for (path, project) in &mut projects { + for (index, project) in &mut projects { let languages = project .detections .iter() @@ -328,20 +477,17 @@ pub fn scan(root: &Path, extra_rules: Option<&str>) -> Result { .collect::>(); for language in languages { let mut versions = Vec::new(); - for ancestor in path.ancestors() { - if let Ok(i) = nodes.binary_search_by(|n| n.path.as_path().cmp(ancestor)) { - let node = &nodes[i]; - if let Some(found) = node.versions.get(&language) { - versions.extend( - found - .iter() - .filter(|v| { - ancestor == path || v.kind == "pin" || v.kind == "toolchain" - }) - .cloned(), - ); - } + let mut ancestor = Some(*index); + while let Some(i) = ancestor { + if let Some(found) = nodes[i].versions.get(&language) { + versions.extend( + found + .iter() + .filter(|v| i == *index || v.kind == "pin" || v.kind == "toolchain") + .cloned(), + ); } + ancestor = nodes[i].parent; } let mut seen = BTreeSet::new(); versions.retain(|v| seen.insert(v.clone())); @@ -354,58 +500,54 @@ pub fn scan(root: &Path, extra_rules: Option<&str>) -> Result { .detections .sort_by(|a, b| b.priority.cmp(&a.priority).then(a.id.cmp(&b.id))); } - let paths = projects.keys().cloned().collect::>(); - for node in &nodes { - let Some(project) = projects.get_mut(&node.path) else { + let mut candidates = BTreeMap::<&str, Vec>::new(); + for (index, node) in nodes.iter().enumerate() { + if project_indices[index].is_none() { continue; - }; + } + for (ecosystem, file) in [ + ("cargo", "Cargo.toml"), + ("go", "go.mod"), + ("node", "package.json"), + ("python", "pyproject.toml"), + ] { + if node.files.contains(file) { + candidates.entry(ecosystem).or_default().push(index); + } + } + } + for (index, project) in &mut projects { + let node = &nodes[*index]; for workspace in &node.workspaces { - for member in &paths { - if member == &node.path { + for &member in candidates + .get(workspace.ecosystem.as_str()) + .into_iter() + .flatten() + { + if member == *index { continue; } - let Ok(relative) = member.strip_prefix(&node.path) else { + let Ok(relative) = nodes[member].path.strip_prefix(&node.path) else { continue; }; - let relative = display_path(relative); - let is_member = workspace - .members - .iter() - .any(|p| rules::path_glob(p, &relative)); - let excluded = workspace - .exclude - .iter() - .any(|p| rules::path_glob(p, &relative)); - let same_language = projects_language(&nodes, member, &workspace.ecosystem); - if is_member && !excluded && same_language { - project.workspace_members.push(display_path(member)); + if workspace.contains(&display_path(relative)) { + project + .workspace_members + .push(display_path(&nodes[member].path)); } } } project.workspace_members.sort(); project.workspace_members.dedup(); } - report.projects = projects.into_values().collect(); + projects.sort_by(|a, b| nodes[a.0].path.cmp(&nodes[b.0].path)); + report.projects = projects.into_iter().map(|(_, project)| project).collect(); report .diagnostics .sort_by(|a, b| a.path.cmp(&b.path).then(a.message.cmp(&b.message))); Ok(report) } -fn projects_language(nodes: &[Node], path: &Path, ecosystem: &str) -> bool { - let file = match ecosystem { - "cargo" => "Cargo.toml", - "go" => "go.mod", - "node" => "package.json", - "python" => "pyproject.toml", - _ => return false, - }; - nodes - .binary_search_by(|n| n.path.as_path().cmp(path)) - .ok() - .is_some_and(|i| nodes[i].files.contains(file)) -} - fn family(language: &str) -> &str { match language { "typescript" | "javascript" => "node", diff --git a/src/detect/parsers.rs b/src/detect/parsers.rs index b58f755..c8c33a8 100644 --- a/src/detect/parsers.rs +++ b/src/detect/parsers.rs @@ -1,6 +1,7 @@ -use std::collections::BTreeMap; +use std::collections::{BTreeMap, HashMap}; use std::path::{Path, PathBuf}; +use serde::Deserialize; use serde_json::Value; use super::{Diagnostic, Node, Workspace, relative_file}; @@ -10,6 +11,7 @@ pub fn readable(name: &str) -> bool { name, "composer.json" | "package.json" + | "pnpm-workspace.yaml" | "Cargo.toml" | "go.mod" | "go.work" @@ -25,13 +27,45 @@ pub fn readable(name: &str) -> bool { | ".tool-versions" | "rust-toolchain" | "rust-toolchain.toml" - ) || (name.starts_with("requirements") && name.ends_with(".txt")) + ) || name.ends_with(".gemspec") + || (name.starts_with("requirements") && name.ends_with(".txt")) } pub fn parse(nodes: &mut [Node], diagnostics: &mut Vec) { - let mut documents = BTreeMap::new(); + let mut documents = Documents::new(); + let mut pnpm = HashMap::>::new(); for node in nodes.iter() { + if node.files.contains("pnpm-workspace.yaml") { + pnpm.insert(node.path.clone(), None); + } for (file, content) in &node.contents { + if file == "pnpm-workspace.yaml" { + match serde_saphyr::from_str::(content) { + Ok(manifest) => { + let (members, exclude) = workspace_patterns( + manifest.packages.unwrap_or_else(|| vec!["**".to_owned()]), + ); + pnpm.insert( + node.path.clone(), + Some(Pnpm { + workspace: Workspace { + ecosystem: "node".to_owned(), + source: relative_file(&node.path, file), + members, + exclude, + }, + catalog: manifest.catalog.unwrap_or_default(), + catalogs: manifest.catalogs.unwrap_or_default(), + }), + ); + } + Err(error) => diagnostics.push(Diagnostic { + path: relative_file(&node.path, file), + message: error.to_string(), + }), + } + continue; + } let parsed = match file.as_str() { "composer.json" | "package.json" => { Some(serde_json::from_str::(content).map_err(|e| e.to_string())) @@ -44,7 +78,10 @@ pub fn parse(nodes: &mut [Node], diagnostics: &mut Vec) { if let Some(parsed) = parsed { match parsed { Ok(doc) if doc.is_object() => { - documents.insert((node.path.clone(), file.clone()), doc); + documents + .entry(node.path.clone()) + .or_default() + .insert(file.clone(), doc); } Ok(_) => diagnostics.push(Diagnostic { path: relative_file(&node.path, file), @@ -59,16 +96,10 @@ pub fn parse(nodes: &mut [Node], diagnostics: &mut Vec) { } } for node in nodes { - for file in [ - "composer.json", - "package.json", - "Cargo.toml", - "pyproject.toml", - "Pipfile", - ] { - let Some(doc) = documents.get(&(node.path.clone(), file.to_owned())) else { - continue; - }; + if let Some(Some(manifest)) = pnpm.get(&node.path) { + node.workspaces.push(manifest.workspace.clone()); + } + for (file, doc) in documents.get(&node.path).into_iter().flatten() { if let Some(name) = doc .get("name") .or_else(|| doc.pointer("/package/name")) @@ -77,7 +108,7 @@ pub fn parse(nodes: &mut [Node], diagnostics: &mut Vec) { { node.names.insert(name.to_owned()); } - match file { + match file.as_str() { "composer.json" => composer(node, doc), "package.json" => javascript(node, doc), "Cargo.toml" => cargo(node, doc, &documents), @@ -100,6 +131,7 @@ pub fn parse(nodes: &mut [Node], diagnostics: &mut Vec) { match file.as_str() { "go.mod" | "go.work" => go(node, file, text), "Gemfile" => ruby(node, text, diagnostics), + f if f.ends_with(".gemspec") => gemspec(node, file, text), ".go-version" => pin(node, "go", file, text), ".php-version" => pin(node, "php", file, text), ".ruby-version" => pin(node, "ruby", file, text), @@ -108,7 +140,8 @@ pub fn parse(nodes: &mut [Node], diagnostics: &mut Vec) { "rust-toolchain" => pin(node, "rust", file, text), "rust-toolchain.toml" => { if let Some(v) = documents - .get(&(node.path.clone(), file.clone())) + .get(&node.path) + .and_then(|docs| docs.get(file)) .and_then(|d| d.pointer("/toolchain/channel")) .and_then(Value::as_str) { @@ -149,6 +182,92 @@ pub fn parse(nodes: &mut [Node], diagnostics: &mut Vec) { _ => (), } } + resolve_catalogs(node, &pnpm, diagnostics); + } +} + +#[derive(Deserialize)] +struct PnpmDocument { + packages: Option>, + catalog: Option>, + catalogs: Option>>, +} + +struct Pnpm { + workspace: Workspace, + catalog: BTreeMap, + catalogs: BTreeMap>, +} + +fn workspace_patterns(patterns: Vec) -> (Vec, Vec) { + let (mut members, mut exclude) = (Vec::new(), Vec::new()); + for pattern in patterns { + if let Some(pattern) = pattern.strip_prefix('!') { + exclude.push(pattern.to_owned()); + } else { + members.push(pattern); + } + } + (members, exclude) +} + +fn resolve_catalogs( + node: &mut Node, + manifests: &HashMap>, + diagnostics: &mut Vec, +) { + let workspace = node + .path + .ancestors() + .find_map(|path| manifests.get(path).map(|manifest| (path, manifest))) + .and_then(|(path, manifest)| manifest.as_ref().map(|manifest| (path, manifest))) + .filter(|(path, manifest)| { + *path == node.path + || node.path.strip_prefix(path).ok().is_some_and(|relative| { + manifest.workspace.contains(&super::display_path(relative)) + }) + }) + .map(|(_, manifest)| manifest); + for (key, dependency) in &mut node.dependencies { + let Some(package) = key.strip_prefix("npm:") else { + continue; + }; + let Some(reference) = dependency + .version + .as_deref() + .and_then(|v| v.strip_prefix("catalog:")) + else { + continue; + }; + let value = workspace.and_then(|manifest| { + let catalog = if reference.is_empty() || reference == "default" { + Some(&manifest.catalog) + } else { + manifest.catalogs.get(reference) + }; + catalog + .and_then(|catalog| catalog.get(package)) + .filter(|v| !v.starts_with("catalog:")) + .map(|value| super::Version { + value: value.clone(), + source: manifest.workspace.source.clone(), + kind: "requirement".to_owned(), + }) + }); + if let Some(resolved) = value { + if package == "typescript" { + node.versions + .entry("typescript".to_owned()) + .or_default() + .push(resolved.clone()); + } + dependency.resolved = Some(resolved); + } else { + diagnostics.push(Diagnostic { + path: dependency.source.clone(), + message: format!("could not resolve catalog:{reference} for {package}"), + }); + } } } @@ -169,6 +288,14 @@ fn dependencies(node: &mut Node, object: Option<&Value>, ecosystem: &str, file: } fn composer(node: &mut Node, doc: &Value) { + if let Some(name) = doc.get("name").and_then(Value::as_str) { + node.identity( + "composer", + name, + doc.get("version").and_then(Value::as_str), + "composer.json", + ); + } dependencies(node, doc.get("require"), "composer", "composer.json"); if let Some(v) = doc.pointer("/require/php").and_then(Value::as_str) { node.version("php", v, "composer.json", "requirement"); @@ -179,6 +306,14 @@ fn composer(node: &mut Node, doc: &Value) { } fn javascript(node: &mut Node, doc: &Value) { + if let Some(name) = doc.get("name").and_then(Value::as_str) { + node.identity( + "npm", + name, + doc.get("version").and_then(Value::as_str), + "package.json", + ); + } dependencies(node, doc.get("devDependencies"), "npm", "package.json"); dependencies(node, doc.get("dependencies"), "npm", "package.json"); if let Some(v) = doc.pointer("/engines/node").and_then(Value::as_str) { @@ -188,24 +323,34 @@ fn javascript(node: &mut Node, doc: &Value) { node.version("javascript", v, "package.json", "pin"); } if let Some(v) = doc - .pointer("/devDependencies/typescript") - .or_else(|| doc.pointer("/dependencies/typescript")) + .pointer("/dependencies/typescript") + .or_else(|| doc.pointer("/devDependencies/typescript")) .and_then(Value::as_str) { - node.version("typescript", v, "package.json", "requirement"); + node.version( + "typescript", + v, + "package.json", + if v.starts_with("catalog:") { + "reference" + } else { + "requirement" + }, + ); } if let Some(workspaces) = doc.get("workspaces") { - let members = strings(workspaces.get("packages").unwrap_or(workspaces)); + let (members, exclude) = + workspace_patterns(strings(workspaces.get("packages").unwrap_or(workspaces))); node.workspaces.push(Workspace { ecosystem: "node".to_owned(), source: relative_file(&node.path, "package.json"), members, - exclude: Vec::new(), + exclude, }); } } -type Documents = BTreeMap<(PathBuf, String), Value>; +type Documents = HashMap>; fn cargo(node: &mut Node, doc: &Value, documents: &Documents) { let file = "Cargo.toml"; @@ -216,18 +361,44 @@ fn cargo(node: &mut Node, doc: &Value, documents: &Documents) { if let Some(explicit) = doc.pointer("/package/workspace").and_then(Value::as_str) { normalize_path(&node.path.join(explicit)).and_then(|path| { documents - .get(&(path.clone(), file.to_owned())) + .get(&path) + .and_then(|docs| docs.get(file)) .filter(|d| d.get("workspace").is_some()) .map(|d| (path, d)) }) } else { node.path.ancestors().find_map(|path| { documents - .get(&(path.to_path_buf(), file.to_owned())) + .get(path) + .and_then(|docs| docs.get(file)) .filter(|d| d.get("workspace").is_some()) .map(|d| (path.to_path_buf(), d)) }) }; + if let Some(name) = doc.pointer("/package/name").and_then(Value::as_str) { + node.identity( + "cargo", + name, + doc.pointer("/package/version").and_then(Value::as_str), + file, + ); + if doc + .pointer("/package/version/workspace") + .and_then(Value::as_bool) + == Some(true) + && let Some((path, workspace)) = &workspace + && let Some(version) = workspace + .pointer("/workspace/package/version") + .and_then(Value::as_str) + && let Some(identity) = node.identities.get_mut(&format!("cargo:{name}")) + { + identity.resolved = Some(super::Version { + value: version.to_owned(), + source: relative_file(path, file), + kind: "package".to_owned(), + }); + } + } if let Some(version) = doc.pointer("/package/rust-version") { if let Some(v) = version.as_str() { node.version("rust", v, file, "requirement"); @@ -282,7 +453,11 @@ fn cargo(node: &mut Node, doc: &Value, documents: &Documents) { .map(str::to_owned); node.dependencies.insert( format!("cargo:{name}"), - super::Dependency { version, source }, + super::Dependency { + version, + source, + resolved: None, + }, ); } } @@ -299,6 +474,18 @@ fn cargo(node: &mut Node, doc: &Value, documents: &Documents) { fn python(node: &mut Node, doc: &Value) { let file = "pyproject.toml"; + if let Some(name) = doc + .pointer("/project/name") + .or_else(|| doc.pointer("/tool/poetry/name")) + .and_then(Value::as_str) + { + let version = doc + .pointer("/project/version") + .or_else(|| doc.pointer("/tool/poetry/version")) + .and_then(Value::as_str); + node.identity("pypi", &normalize_python(name), version, file); + node.names.insert(name.to_owned()); + } if let Some(v) = doc .pointer("/project/requires-python") .and_then(Value::as_str) @@ -445,6 +632,7 @@ fn go(node: &mut Node, file: &str, text: &str) { "module" => { if let Some(name) = values.first() { node.names.insert(name.trim_matches('"').to_owned()); + node.identity("go", name.trim_matches('"'), None, file); } } "require" => { @@ -532,6 +720,39 @@ fn ruby(node: &mut Node, text: &str, diagnostics: &mut Vec) { } } +fn gemspec(node: &mut Node, file: &str, text: &str) { + let mut name = None; + let mut version = None; + for line in text.lines() { + let Some((receiver, assignment)) = line.trim().split_once('.') else { + continue; + }; + if receiver.is_empty() + || !receiver + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '_') + { + continue; + } + let Some((field, value)) = assignment.split_once('=') else { + continue; + }; + let Some((value, _)) = quoted(value) else { + continue; + }; + match field.trim() { + "name" => name = Some(value), + "version" => version = Some(value), + "required_ruby_version" => node.version("ruby", value, file, "requirement"), + _ => (), + } + } + if let Some(name) = name { + node.names.insert(name.to_owned()); + node.identity("gem", name, version, file); + } +} + fn strings(value: &Value) -> Vec { value .as_array() diff --git a/src/detect/rules.rs b/src/detect/rules.rs index f2955ff..b947a37 100644 --- a/src/detect/rules.rs +++ b/src/detect/rules.rs @@ -4,7 +4,7 @@ use std::sync::OnceLock; use anyhow::{Result, bail}; use serde::Deserialize; -use super::{Detection, Evidence, Node}; +use super::{Detection, Evidence, FrameworkRole, Node}; #[derive(Clone, Debug, Deserialize)] #[serde(deny_unknown_fields)] @@ -14,11 +14,39 @@ pub struct Rule { pub kind: Kind, pub priority: i32, #[serde(default)] - pub any: Vec, + pub any: Vec, #[serde(default)] - pub all: Vec, + pub all: Vec, #[serde(default)] - pub none: Vec, + pub none: Vec, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(untagged)] +pub enum Pattern { + Atom(String), + All(AllPattern), +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct AllPattern { + all: Vec, +} + +impl Pattern { + fn atoms(&self) -> &[String] { + match self { + Self::Atom(atom) => std::slice::from_ref(atom), + Self::All(group) => &group.all, + } + } + + fn matches(&self, node: &Node) -> bool { + self.atoms() + .iter() + .all(|atom| matched(node, atom).is_some()) + } } #[derive(Clone, Copy, Debug, Deserialize, serde::Serialize, PartialEq, Eq)] @@ -41,11 +69,13 @@ pub struct Rules { files: HashMap>, sources: HashMap>, dependencies: HashMap>, + identities: HashMap>, workspaces: HashMap>, packages: HashMap>, wildcard_files: Vec<(String, usize)>, file_patterns: BTreeSet, wildcard_interest: BTreeSet, + source_interest: BTreeSet, } impl Rules { @@ -73,19 +103,28 @@ impl Rules { bail!("rule {} needs an any or all pattern", rule.id); } for pattern in rule.any.iter().chain(&rule.all).chain(&rule.none) { - let valid = match pattern.split_once(':') { - Some(("file" | "source", value)) => !value.is_empty() && !value.contains('/'), - Some(("dep", value)) => value - .split_once(':') - .is_some_and(|(ecosystem, name)| !ecosystem.is_empty() && !name.is_empty()), - Some(("package", "cargo")) => true, - Some(("workspace", value)) => { - matches!(value, "cargo" | "node" | "go" | "python") + if pattern.atoms().is_empty() { + bail!("empty pattern group in rule {}", rule.id); + } + for pattern in pattern.atoms() { + let valid = match pattern.split_once(':') { + Some(("file" | "source", value)) => { + !value.is_empty() && !value.contains('/') + } + Some(("dep" | "identity", value)) => { + value.split_once(':').is_some_and(|(ecosystem, name)| { + !ecosystem.is_empty() && !name.is_empty() + }) + } + Some(("package", "cargo")) => true, + Some(("workspace", value)) => { + matches!(value, "cargo" | "node" | "go" | "python") + } + _ => false, + }; + if !valid { + bail!("invalid pattern {pattern:?} in rule {}", rule.id); } - _ => false, - }; - if !valid { - bail!("invalid pattern {pattern:?} in rule {}", rule.id); } } } @@ -97,14 +136,25 @@ impl Rules { files: HashMap::new(), sources: HashMap::new(), dependencies: HashMap::new(), + identities: HashMap::new(), workspaces: HashMap::new(), packages: HashMap::new(), wildcard_files: Vec::new(), file_patterns: BTreeSet::new(), wildcard_interest: BTreeSet::new(), + source_interest: BTreeSet::new(), }; for (i, rule) in rules.rules.iter().enumerate() { - for pattern in rule.all.iter().chain(&rule.any).chain(&rule.none) { + for pattern in rule + .all + .iter() + .chain(&rule.any) + .chain(&rule.none) + .flat_map(Pattern::atoms) + { + if let Some(value) = pattern.strip_prefix("source:") { + rules.source_interest.insert(value.to_owned()); + } if let Some(value) = pattern.strip_prefix("file:") { if value.contains(['*', '?']) { rules.wildcard_interest.insert(value.to_owned()); @@ -113,7 +163,7 @@ impl Rules { } } } - for pattern in rule.all.iter().chain(&rule.any) { + for pattern in rule.all.iter().chain(&rule.any).flat_map(Pattern::atoms) { let Some((kind, value)) = pattern.split_once(':') else { continue; }; @@ -125,6 +175,7 @@ impl Rules { "file" => &mut rules.files, "source" => &mut rules.sources, "dep" => &mut rules.dependencies, + "identity" => &mut rules.identities, "workspace" => &mut rules.workspaces, "package" => &mut rules.packages, _ => continue, @@ -134,6 +185,10 @@ impl Rules { } Ok(rules) } + pub fn needs_source(&self, extension: &str) -> bool { + self.source_interest.contains(extension) + } + pub fn needs_filename(&self, name: &str) -> bool { self.file_patterns.contains(name) || self.wildcard_interest.iter().any(|p| glob(p, name)) } @@ -155,6 +210,11 @@ impl Rules { candidates.extend(ids); } } + for identity in node.identities.keys() { + if let Some(ids) = self.identities.get(identity) { + candidates.extend(ids); + } + } for package in &node.packages { if let Some(ids) = self.packages.get(package) { candidates.extend(ids); @@ -181,6 +241,28 @@ impl Rules { language: rule.language.clone(), kind: rule.kind, priority: rule.priority, + roles: if rule.kind == Kind::Framework { + let mut roles = evidence + .iter() + .filter_map(|e| { + if e.pattern.starts_with("identity:") { + Some(FrameworkRole::Provides) + } else if e.pattern.starts_with("dep:") { + Some(FrameworkRole::Uses) + } else { + None + } + }) + .collect::>(); + if roles.is_empty() { + roles.push(FrameworkRole::Uses); + } + roles.sort(); + roles.dedup(); + roles + } else { + Vec::new() + }, evidence, }) }) @@ -190,18 +272,20 @@ impl Rules { impl Rule { pub fn evaluate(&self, node: &Node) -> Option> { - if !self.all.iter().all(|p| matched(node, p).is_some()) - || (!self.any.is_empty() && !self.any.iter().any(|p| matched(node, p).is_some())) - || self.none.iter().any(|p| matched(node, p).is_some()) + if !self.all.iter().all(|p| p.matches(node)) + || (!self.any.is_empty() && !self.any.iter().any(|p| p.matches(node))) + || self.none.iter().any(|p| p.matches(node)) { return None; } - let mut evidence: Vec<_> = self - .all - .iter() - .chain(&self.any) - .filter_map(|p| matched(node, p).map(|found| found.evidence(node, p))) - .collect(); + let mut evidence = Vec::new(); + for pattern in self.all.iter().chain(&self.any).filter(|p| p.matches(node)) { + for atom in pattern.atoms() { + if let Some(found) = matched(node, atom) { + found.evidence(node, atom, &mut evidence); + } + } + } evidence.sort(); evidence.dedup(); Some(evidence) @@ -215,17 +299,26 @@ enum Matched<'a> { } impl Matched<'_> { - fn evidence(self, node: &Node, pattern: &str) -> Evidence { + fn evidence(self, node: &Node, pattern: &str, evidence: &mut Vec) { let (source, value) = match self { Self::File(name) => (super::relative_file(&node.path, name), None), - Self::Dependency(dep) => (dep.source.clone(), dep.version.clone()), + Self::Dependency(dep) => { + if let Some(resolved) = &dep.resolved { + evidence.push(Evidence { + source: resolved.source.clone(), + pattern: pattern.to_owned(), + value: Some(resolved.value.clone()), + }); + } + (dep.source.clone(), dep.version.clone()) + } Self::Workspace(ws) => (ws.source.clone(), None), }; - Evidence { + evidence.push(Evidence { source, pattern: pattern.to_owned(), value, - } + }); } } @@ -242,6 +335,7 @@ fn matched<'a>(node: &'a Node, pattern: &str) -> Option> { } "source" => node.sources.get(value).map(|name| Matched::File(name)), "dep" => node.dependencies.get(value).map(Matched::Dependency), + "identity" => node.identities.get(value).map(Matched::Dependency), "package" => node .packages .get(value) @@ -298,7 +392,9 @@ pub fn path_glob(pattern: &str, value: &str) -> bool { glob(head, part) && if tail.is_empty() { rest.is_empty() + } else if rest.is_empty() { + tail.split('/').all(|part| part == "**") } else { - !rest.is_empty() && path_glob(tail, rest) + path_glob(tail, rest) } } diff --git a/src/detect/rules.toml b/src/detect/rules.toml index 383422e..8064222 100644 --- a/src/detect/rules.toml +++ b/src/detect/rules.toml @@ -136,200 +136,207 @@ id = "laravel" language = "php" kind = "framework" priority = 100 -any = ["dep:composer:laravel/framework"] +any = ["dep:composer:laravel/framework", "identity:composer:laravel/framework"] [[rules]] id = "symfony" language = "php" kind = "framework" priority = 100 -any = ["dep:composer:symfony/framework-bundle", "dep:composer:symfony/symfony"] +any = ["dep:composer:symfony/framework-bundle", "dep:composer:symfony/symfony", "identity:composer:symfony/framework-bundle", "identity:composer:symfony/symfony"] [[rules]] id = "cakephp" language = "php" kind = "framework" priority = 100 -any = ["dep:composer:cakephp/cakephp"] +any = ["dep:composer:cakephp/cakephp", "identity:composer:cakephp/cakephp"] [[rules]] id = "codeigniter" language = "php" kind = "framework" priority = 100 -any = ["dep:composer:codeigniter4/framework", "dep:composer:codeigniter/framework"] +any = ["dep:composer:codeigniter4/framework", "dep:composer:codeigniter/framework", "identity:composer:codeigniter4/framework", "identity:composer:codeigniter/framework"] [[rules]] id = "slim" language = "php" kind = "framework" priority = 100 -any = ["dep:composer:slim/slim"] +any = ["dep:composer:slim/slim", "identity:composer:slim/slim"] [[rules]] id = "axum" language = "rust" kind = "framework" priority = 100 -any = ["dep:cargo:axum"] +any = ["dep:cargo:axum", "identity:cargo:axum"] [[rules]] id = "rocket" language = "rust" kind = "framework" priority = 100 -any = ["dep:cargo:rocket"] +any = ["dep:cargo:rocket", "identity:cargo:rocket"] [[rules]] id = "actix-web" language = "rust" kind = "framework" priority = 100 -any = ["dep:cargo:actix-web"] +any = ["dep:cargo:actix-web", "identity:cargo:actix-web"] [[rules]] id = "rails" language = "ruby" kind = "framework" priority = 100 -any = ["dep:gem:rails"] +any = ["dep:gem:rails", "identity:gem:rails", { all = ["dep:gem:actionpack", "dep:gem:railties"] }] [[rules]] id = "sinatra" language = "ruby" kind = "framework" priority = 100 -any = ["dep:gem:sinatra"] +any = ["dep:gem:sinatra", "identity:gem:sinatra"] [[rules]] id = "hanami" language = "ruby" kind = "framework" priority = 100 -any = ["dep:gem:hanami"] +any = ["dep:gem:hanami", "identity:gem:hanami"] [[rules]] id = "django" language = "python" kind = "framework" priority = 100 -any = ["dep:pypi:django"] +any = ["dep:pypi:django", "identity:pypi:django"] [[rules]] id = "fastapi" language = "python" kind = "framework" priority = 100 -any = ["dep:pypi:fastapi"] +any = ["dep:pypi:fastapi", "identity:pypi:fastapi"] [[rules]] id = "flask" language = "python" kind = "framework" priority = 100 -any = ["dep:pypi:flask"] +any = ["dep:pypi:flask", "identity:pypi:flask"] [[rules]] id = "starlette" language = "python" kind = "framework" priority = 50 -any = ["dep:pypi:starlette"] +any = ["dep:pypi:starlette", "identity:pypi:starlette"] [[rules]] id = "celery" language = "python" kind = "framework" priority = 100 -any = ["dep:pypi:celery"] +any = ["dep:pypi:celery", "identity:pypi:celery"] [[rules]] id = "gin" language = "go" kind = "framework" priority = 100 -any = ["dep:go:github.com/gin-gonic/gin"] +any = ["dep:go:github.com/gin-gonic/gin", "identity:go:github.com/gin-gonic/gin"] [[rules]] id = "echo" language = "go" kind = "framework" priority = 100 -any = ["dep:go:github.com/labstack/echo/v4"] +any = ["dep:go:github.com/labstack/echo/v4", "identity:go:github.com/labstack/echo/v4"] [[rules]] id = "fiber" language = "go" kind = "framework" priority = 100 -any = ["dep:go:github.com/gofiber/fiber/v2"] +any = ["dep:go:github.com/gofiber/fiber/v2", "identity:go:github.com/gofiber/fiber/v2"] [[rules]] id = "nextjs" language = "javascript" kind = "framework" priority = 100 -any = ["dep:npm:next"] +any = ["dep:npm:next", "identity:npm:next"] [[rules]] id = "nuxtjs" language = "javascript" kind = "framework" priority = 100 -any = ["dep:npm:nuxt", "dep:npm:nuxt3"] +any = ["dep:npm:nuxt", "dep:npm:nuxt3", "identity:npm:nuxt", "identity:npm:nuxt3"] [[rules]] id = "astro" language = "javascript" kind = "framework" priority = 100 -any = ["dep:npm:astro"] +any = ["dep:npm:astro", "identity:npm:astro"] [[rules]] id = "sveltekit" language = "javascript" kind = "framework" priority = 100 -any = ["dep:npm:@sveltejs/kit"] +any = ["dep:npm:@sveltejs/kit", "identity:npm:@sveltejs/kit"] [[rules]] id = "remix" language = "javascript" kind = "framework" priority = 100 -any = ["dep:npm:@remix-run/react"] +any = ["dep:npm:@remix-run/react", "identity:npm:@remix-run/react"] [[rules]] id = "angular" language = "javascript" kind = "framework" priority = 100 -any = ["dep:npm:@angular/core"] +any = ["dep:npm:@angular/core", "identity:npm:@angular/core"] [[rules]] id = "express" language = "javascript" kind = "framework" priority = 100 -any = ["dep:npm:express"] +any = ["dep:npm:express", "identity:npm:express"] [[rules]] id = "react" language = "javascript" kind = "framework" priority = 50 -any = ["dep:npm:react"] +any = ["dep:npm:react", "identity:npm:react"] [[rules]] id = "vue" language = "javascript" kind = "framework" priority = 50 -any = ["dep:npm:vue"] +any = ["dep:npm:vue", "identity:npm:vue"] [[rules]] id = "vite" language = "javascript" kind = "framework" priority = 50 -any = ["dep:npm:vite"] +any = ["dep:npm:vite", "identity:npm:vite"] + +[[rules]] +id = "rubygems" +language = "ruby" +kind = "project" +priority = 10 +any = ["file:*.gemspec"] diff --git a/src/detect/tests.rs b/src/detect/tests.rs index 55cf73d..0fecb1e 100644 --- a/src/detect/tests.rs +++ b/src/detect/tests.rs @@ -410,7 +410,7 @@ fn avoids_false_framework_matches() { ), ( "Cargo.toml", - "[package]\nname='axum'\n[dev-dependencies]\nrocket='0.5'\n[workspace.dependencies]\naxum='0.8'\n", + "[package]\nname='not-axum'\n[dev-dependencies]\nrocket='0.5'\n[workspace.dependencies]\naxum='0.8'\n", ), ( "Gemfile", @@ -684,3 +684,381 @@ fn benchmark_polyglot_scan() { samples[10], samples[18] ); } + +#[test] +fn framework_definitions_are_not_confused_with_consumers() { + let cases = [ + ( + "react", + "package.json", + r#"{"name":"react","version":"19.3.0"}"#, + ), + ( + "nextjs", + "package.json", + r#"{"name":"next","version":"16.5.0-canary.45"}"#, + ), + ( + "vue", + "package.json", + r#"{"name":"vue","version":"2.7.16"}"#, + ), + ( + "laravel", + "composer.json", + r#"{"name":"laravel/framework"}"#, + ), + ("symfony", "composer.json", r#"{"name":"symfony/symfony"}"#), + ( + "axum", + "Cargo.toml", + "[package]\nname='axum'\nversion='0.8.0'\n", + ), + ( + "gin", + "go.mod", + "module github.com/gin-gonic/gin\ngo 1.26.0\n", + ), + ( + "django", + "pyproject.toml", + "[project]\nname='Django'\nversion='5.1'\n", + ), + ( + "rails", + "rails.gemspec", + "Gem::Specification.new do |s|\n s.name = 'rails'\n s.version = version\n s.required_ruby_version = '>= 3.3.5'\nend\n", + ), + ]; + for (id, file, content) in cases { + let dir = fixture(&[(file, content)]); + let report = scan(dir.path(), None).unwrap(); + assert!(report.diagnostics.is_empty(), "{id}: {report:#?}"); + let p = project(&report, "."); + let detection = p.detections.iter().find(|d| d.id == id).unwrap(); + assert_eq!(detection.roles, [super::FrameworkRole::Provides], "{id}"); + assert!( + detection + .evidence + .iter() + .all(|e| e.pattern.starts_with("identity:")) + ); + if id == "django" { + assert_eq!(p.names, ["Django"]); + } + if id == "rails" { + assert!(version(p, "ruby", ">= 3.3.5", "rails.gemspec")); + } + } + let dir = fixture(&[( + "package.json", + r#"{"name":"react","version":"19","dependencies":{"react":"18"}}"#, + )]); + let report = scan(dir.path(), None).unwrap(); + assert_eq!( + project(&report, ".") + .detections + .iter() + .find(|d| d.id == "react") + .unwrap() + .roles, + [super::FrameworkRole::Uses, super::FrameworkRole::Provides] + ); +} + +#[test] +fn rails_component_detection_requires_both_web_components() { + for (text, expected) in [ + ("gem 'activesupport', '~> 8.1'\n", false), + ("gem 'actionpack', '~> 8.1'\n", false), + ("gem 'railties', '~> 8.1'\n", false), + ( + "gem 'actionpack', '~> 8.1'\ngem 'railties', '~> 8.1'\n", + true, + ), + ] { + let dir = fixture(&[("Gemfile", text)]); + let report = scan(dir.path(), None).unwrap(); + let p = project(&report, "."); + assert_eq!(detects(p, "rails"), expected, "{text}"); + if expected { + let detection = p.detections.iter().find(|d| d.id == "rails").unwrap(); + assert_eq!(detection.roles, [super::FrameworkRole::Uses]); + assert_eq!(detection.evidence.len(), 2); + } + } +} + +#[test] +fn poetry_names_preserve_the_declared_spelling() { + let dir = fixture(&[( + "pyproject.toml", + "[tool.poetry]\nname='AutoGPT.Platform.Backend'\nversion='0.6.22'\n[tool.poetry.dependencies]\npython='>=3.11,<3.14'\nfastapi='^0.128.6'\n", + )]); + let report = scan(dir.path(), None).unwrap(); + let p = project(&report, "."); + assert_eq!(p.names, ["AutoGPT.Platform.Backend"]); + assert!(detects(p, "fastapi")); + assert!(version(p, "python", ">=3.11,<3.14", "pyproject.toml")); +} + +#[test] +fn pnpm_catalogs_preserve_references_and_resolve_local_versions() { + let dir = fixture(&[ + ( + "pnpm-workspace.yaml", + "packages: ['apps/*', '!apps/excluded']\ncatalog: &default\n next: '16.0.0'\n typescript: 'npm:@typescript/typescript6@6.0.2'\ncatalogs:\n tooling:\n <<: *default\n typescript: '7.0.2'\n", + ), + ( + "apps/web/package.json", + r#"{"name":"web","dependencies":{"next":"catalog:"},"devDependencies":{"typescript":"catalog:tooling"}}"#, + ), + ("apps/excluded/package.json", "{}"), + ("apps/web/src/page.tsx", "source must not be read"), + ]); + let report = scan(dir.path(), None).unwrap(); + assert!(report.diagnostics.is_empty(), "{report:#?}"); + let root = project(&report, "."); + assert!(detects(root, "node-workspace")); + assert_eq!(root.workspace_members, ["apps/web"]); + let web = project(&report, "apps/web"); + assert!(version(web, "typescript", "7.0.2", "pnpm-workspace.yaml")); + assert!( + web.languages + .iter() + .flat_map(|l| &l.versions) + .any(|v| v.kind == "reference" && v.value == "catalog:tooling") + ); + let next = web.detections.iter().find(|d| d.id == "nextjs").unwrap(); + assert_eq!(next.roles, [super::FrameworkRole::Uses]); + assert!( + next.evidence + .iter() + .any(|e| e.source == "apps/web/package.json" && e.value.as_deref() == Some("catalog:")) + ); + assert!( + next.evidence + .iter() + .any(|e| e.source == "pnpm-workspace.yaml" && e.value.as_deref() == Some("16.0.0")) + ); + assert_eq!(report.stats.files_read, 3); +} + +#[test] +fn nested_pnpm_catalogs_do_not_leak_across_workspace_boundaries() { + let dir = fixture(&[ + ( + "pnpm-workspace.yaml", + "packages: ['apps/**']\ncatalog:\n typescript: '6.0.2'\n", + ), + ( + "apps/nested/pnpm-workspace.yaml", + "packages: ['packages/*']\ncatalog:\n typescript: '7.0.2'\n", + ), + ( + "apps/nested/packages/web/package.json", + r#"{"devDependencies":{"typescript":"catalog:"}}"#, + ), + ( + "apps/nested/excluded/package.json", + r#"{"devDependencies":{"typescript":"catalog:"}}"#, + ), + ]); + let report = scan(dir.path(), None).unwrap(); + let nested = project(&report, "apps/nested/packages/web"); + assert!(version( + nested, + "typescript", + "7.0.2", + "apps/nested/pnpm-workspace.yaml" + )); + assert!(!version( + nested, + "typescript", + "6.0.2", + "pnpm-workspace.yaml" + )); + let excluded = project(&report, "apps/nested/excluded"); + assert!( + excluded + .languages + .iter() + .flat_map(|l| &l.versions) + .all(|v| v.kind == "reference") + ); + assert_eq!(report.diagnostics.len(), 1); + assert_eq!( + report.diagnostics[0].path, + "apps/nested/excluded/package.json" + ); +} + +#[test] +fn pnpm_defaults_and_missing_catalog_entries_are_explicit() { + let dir = fixture(&[ + ("pnpm-workspace.yaml", "catalog:\n typescript: '6.0.2'\n"), + ( + "packages/web/package.json", + r#"{"devDependencies":{"typescript":"catalog:default"}}"#, + ), + ( + "packages/missing/package.json", + r#"{"devDependencies":{"typescript":"catalog:missing"}}"#, + ), + ]); + let report = scan(dir.path(), None).unwrap(); + assert_eq!( + project(&report, ".").workspace_members, + ["packages/missing", "packages/web"] + ); + assert!(version( + project(&report, "packages/web"), + "typescript", + "6.0.2", + "pnpm-workspace.yaml" + )); + assert_eq!(report.diagnostics.len(), 1); + assert!(report.diagnostics[0].message.contains("catalog:missing")); + let broken = fixture(&[ + ("pnpm-workspace.yaml", "packages: [broken"), + ("package.json", "{}"), + ]); + let report = scan(broken.path(), None).unwrap(); + assert_eq!(report.diagnostics.len(), 1); + assert!(detects(project(&report, "."), "node")); +} + +#[test] +fn conjunction_patterns_require_complete_groups_and_validate_atoms() { + let dir = fixture(&[("a.config", ""), ("b.config", "")]); + let rules = "[[rules]]\nid='grouped'\nlanguage='javascript'\nkind='framework'\npriority=999\nany=[{all=['file:a.config','file:b.config']}]\nnone=[{all=['file:a.config','file:blocked']}]\n"; + let report = scan(dir.path(), Some(rules)).unwrap(); + assert!(detects(project(&report, "."), "grouped")); + assert!( + scan( + dir.path(), + Some(&rules.replace("file:b.config", "file:missing")) + ) + .unwrap() + .projects + .is_empty() + ); + for invalid in [ + rules.replace("file:b.config", "invalid:atom"), + rules.replace("['file:a.config','file:b.config']", "[]"), + ] { + assert!(scan(dir.path(), Some(&invalid)).is_err()); + } +} + +#[test] +fn parallel_inventory_matches_serial_inventory() { + let dir = tempfile::tempdir().unwrap(); + for i in 0..128 { + let path = dir.path().join(format!("services/s{i}/src/deep")); + fs::create_dir_all(&path).unwrap(); + fs::write(path.join("main.ts"), "not read").unwrap(); + fs::write(path.join("irrelevant.md"), "not read").unwrap(); + fs::write(dir.path().join(format!("services/s{i}/package.json")), "{}").unwrap(); + } + let rules = super::Rules::builtin().unwrap(); + let serial = super::inventory(dir.path(), rules, 1).unwrap(); + let parallel = super::inventory(dir.path(), rules, 4).unwrap(); + assert_eq!(serial.stats, parallel.stats); + assert_eq!(serial.nodes.len(), parallel.nodes.len()); + for (a, b) in serial.nodes.iter().zip(¶llel.nodes) { + assert_eq!( + (&a.path, a.parent, &a.files, &a.sources, &a.contents), + (&b.path, b.parent, &b.files, &b.sources, &b.contents) + ); + } + assert_eq!(serial.stats.files_read, 128); + let report = scan(dir.path(), None).unwrap(); + assert_eq!(report.projects.len(), 128); + assert_eq!(scan(dir.path(), None).unwrap(), report); +} + +#[test] +fn trailing_globstars_match_zero_or_more_directories() { + for (pattern, path, expected) in [ + ("packages/**", "packages", true), + ("packages/**/**", "packages", true), + ("packages/*", "packages", false), + ("packages/**", "packages/web/src", true), + ] { + assert_eq!( + super::rules::path_glob(pattern, path), + expected, + "{pattern}: {path}" + ); + } +} + +#[test] +fn inherited_package_identity_versions_keep_workspace_provenance() { + let dir = fixture(&[ + ( + "Cargo.toml", + "[workspace]\nmembers=['crates/*']\n[workspace.package]\nversion='0.8.1'\n", + ), + ( + "crates/axum/Cargo.toml", + "[package]\nname='axum'\nversion.workspace=true\n", + ), + ]); + let report = scan(dir.path(), None).unwrap(); + let axum = project(&report, "crates/axum") + .detections + .iter() + .find(|d| d.id == "axum") + .unwrap(); + assert_eq!(axum.roles, [super::FrameworkRole::Provides]); + assert!( + axum.evidence + .iter() + .any(|e| e.source == "Cargo.toml" && e.value.as_deref() == Some("0.8.1")) + ); + assert!( + axum.evidence + .iter() + .all(|e| e.source != "crates/axum/Cargo.toml" || e.value.is_none()) + ); +} + +#[test] +fn malformed_nested_pnpm_files_block_outer_catalog_resolution() { + let dir = fixture(&[ + ( + "pnpm-workspace.yaml", + "packages: ['apps/**']\ncatalog:\n typescript: '6.0.2'\n", + ), + ("apps/nested/pnpm-workspace.yaml", "packages: [broken"), + ( + "apps/nested/web/package.json", + r#"{"devDependencies":{"typescript":"catalog:"}}"#, + ), + ]); + let report = scan(dir.path(), None).unwrap(); + assert_eq!(report.diagnostics.len(), 2); + let web = project(&report, "apps/nested/web"); + assert!( + web.languages + .iter() + .flat_map(|l| &l.versions) + .all(|v| v.kind == "reference") + ); +} + +#[test] +fn provider_file_guards_do_not_imply_framework_consumption() { + let dir = fixture(&[ + ("package.json", r#"{"name":"react","version":"19"}"#), + ("provider.config", ""), + ]); + let rules = "[[rules]]\nid='guarded-provider'\nlanguage='javascript'\nkind='framework'\npriority=200\nall=['identity:npm:react','file:provider.config']\n"; + let report = scan(dir.path(), Some(rules)).unwrap(); + assert_eq!( + project(&report, ".").detections[0].roles, + [super::FrameworkRole::Provides] + ); +} diff --git a/src/main.rs b/src/main.rs index cf8d604..b285be6 100644 --- a/src/main.rs +++ b/src/main.rs @@ -296,10 +296,17 @@ fn api_error_json(api_error: &ApiError) -> Value { } fn is_broken_pipe(error: &anyhow::Error) -> bool { - error - .chain() - .filter_map(|cause| cause.downcast_ref::()) - .any(|error| error.kind() == std::io::ErrorKind::BrokenPipe) + error.chain().any(|cause| { + cause + .downcast_ref::() + .map(std::io::Error::kind) + .or_else(|| { + cause + .downcast_ref::() + .and_then(serde_json::Error::io_error_kind) + }) + == Some(std::io::ErrorKind::BrokenPipe) + }) } #[cfg(test)] @@ -308,6 +315,40 @@ mod tests { use super::{Command, Opts}; + #[test] + fn handles_broken_pipes_wrapped_by_json_serialization() { + struct FailedWriter(std::io::ErrorKind); + + impl std::io::Write for FailedWriter { + fn write(&mut self, _: &[u8]) -> std::io::Result { + Err(self.0.into()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + for kind in [ + std::io::ErrorKind::BrokenPipe, + std::io::ErrorKind::PermissionDenied, + ] { + let expected = kind == std::io::ErrorKind::BrokenPipe; + let error = serde_json::to_writer(FailedWriter(kind), &serde_json::json!({"value": 1})) + .unwrap_err(); + assert_eq!( + super::is_broken_pipe(&anyhow::Error::new(error).context("write output")), + expected + ); + assert_eq!( + super::is_broken_pipe(&anyhow::Error::new(std::io::Error::from(kind))), + expected + ); + } + let error = serde_json::from_str::("invalid").unwrap_err(); + assert!(!super::is_broken_pipe(&error.into())); + } + #[test] fn parses_describe_path() { let opts = Opts::try_parse_from(["brainpod", "describe", "resource", "create"]).unwrap(); diff --git a/src/output.rs b/src/output.rs index 297589a..30854eb 100644 --- a/src/output.rs +++ b/src/output.rs @@ -81,8 +81,10 @@ fn write_buffered(value: &Value, view: View, json: bool) -> Result<()> { serde_json::to_writer(&mut stdout, value)?; writeln!(stdout)?; } else { - serde_json::to_writer_pretty(&mut stdout, value)?; - writeln!(stdout)?; + let mut buffered = io::BufWriter::new(&mut stdout); + serde_json::to_writer_pretty(&mut buffered, value)?; + writeln!(buffered)?; + buffered.flush()?; } return Ok(()); } diff --git a/tests/detect-corpus.md b/tests/detect-corpus.md index a7219c9..18d2b6d 100644 --- a/tests/detect-corpus.md +++ b/tests/detect-corpus.md @@ -1,6 +1,6 @@ # GitHub project detection audit -Selected 2026-11-09; detector commit `244ab44ccf1a` (PR #20 baseline). +Selected 2026-11-09. Baseline: detector commit `244ab44ccf1a` (PR #20). Current: the follow-up fixes in this revision. ## Method @@ -8,34 +8,34 @@ Five manually qualified web projects per supported language, ordered by GitHub s Examples excluded above selected projects: Node.js (general runtime), Ollama (model runtime), Kubernetes (infrastructure), frp (generic networking), and developer-roadmap (educational roadmap collection). Stars and language ranks are frozen with full commit SHAs in [`detect-corpus.json`](detect-corpus.json). -Checks target reviewed project paths and static manifest declarations, not a matching framework somewhere in the repository’s examples. Identity checks separately ask whether framework implementations identify themselves. Catalog checks ask for the local catalog declaration rather than an unresolved reference. These last two categories probe capabilities beyond the current dependency-only behavior. +Checks target reviewed project paths and static manifest declarations, not a matching framework somewhere in the repository’s examples. Identity checks separately ask whether framework implementations identify themselves. Catalog checks ask for the local catalog declaration rather than an unresolved reference. These last two categories exposed capabilities missing from the baseline dependency-only behavior. -GitHub source archives were downloaded into a temporary cache. No dependencies were installed and no repository code was executed. Submodules and LFS targets were not fetched; symlinks were not extracted. The corpus occupies approximately 4.1 GiB. This is a sampled assertion suite, not an exhaustive precision/recall or buildability evaluation. +GitHub source archives were downloaded into a temporary cache. No application dependencies were installed and no downloaded repository code was executed. Submodules and LFS targets were not fetched; symlinks were not extracted. The corpus occupies approximately 4.1 GiB. This is a sampled assertion suite, not an exhaustive precision/recall or buildability evaluation. ## Results -All **35/35** scans succeeded and produced identical output across five runs. **156/173 assertions passed**, and 23/35 repositories passed every sampled assertion. +All **35/35** scans succeed and produce identical output across five runs. **173/173 sampled assertions now pass**, up from 156/173. Every repository passes its sampled assertions; this is not a claim of exhaustive detection accuracy. -| Category | Passed | -|---|---:| -| language | 35/35 | -| project | 35/35 | -| name | 26/27 | -| version | 32/32 | -| framework | 19/20 | -| workspace | 9/15 | -| identity | 0/7 | -| catalog | 0/2 | +| Category | Baseline | Current | +|---|---:|---:| +| language | 35/35 | 35/35 | +| project | 35/35 | 35/35 | +| name | 26/27 | 27/27 | +| version | 32/32 | 32/32 | +| framework | 19/20 | 20/20 | +| workspace | 9/15 | 15/15 | +| identity | 0/7 | 7/7 | +| catalog | 0/2 | 2/2 | β€œVersion” checks cover selected literal constraints and inherited declarations, not installed versions. Framework checks cover selected consumer projects, while identity checks cover framework implementation packages. -### Gaps found +### Misses fixed -- **pnpm workspaces:** Next.js, freeCodeCamp, Vue, n8n, Dify, and Discourse lack Node workspace labels/member relationships. Their component projects are discovered individually. Cargo, npm/Yarn, and uv representatives passed. -- **Rails component dependencies:** Discourse declares `actionpack`, `railties`, and the other Rails components instead of `gem "rails"`; its root is reported as Bundler, not Rails. -- **Poetry names:** AutoGPT’s `autogpt_platform/backend/pyproject.toml` declares `tool.poetry.name = "autogpt-platform-backend"`, but the project’s names array is empty. -- **Framework identities:** React (`packages/react`), Next.js (`packages/next`), Vue, Laravel, Symfony, Rails, and Gin are not framework-labeled at their implementation package paths. Consumers/examples elsewhere must not mask this. A future identity rule should distinguish β€œprovides” from β€œuses” rather than assume these packages are deployable applications. -- **pnpm catalogs:** n8n and Dify expose `catalog:typescript-tooling` / `catalog:` as TypeScript requirements without resolving the local `npm:@typescript/typescript6@6.0.2` declaration. +- **pnpm workspaces:** Next.js, freeCodeCamp, Vue, n8n, Dify, and Discourse now report their Node workspace labels and members, including negative patterns. A separate directory-glob cross-check found all 225 package members across these six snapshots, with no extra members. Cargo, npm/Yarn, and uv representatives still pass. +- **Rails components:** Discourse is recognized from the conjunction of `actionpack` and `railties`. `activesupport` or either web component alone does not label a project as Rails. +- **Poetry names:** AutoGPT’s backend retains its declared `autogpt-platform-backend` name. Names preserve declared spelling; dependency identity matching uses ecosystem normalization. +- **Framework identities:** React, Next.js, Vue, Laravel, Symfony, Rails, and Gin now identify their implementation packages. Framework detections distinguish `provides` from `uses`, rather than implying that a framework library is a deployable application. Ruby gemspec metadata is read statically, never evaluated. +- **pnpm catalogs:** Default and named catalogs resolve local version declarations while preserving both the reference and its YAML provenance. Missing entries, excluded packages, and malformed nested workspace files do not silently inherit unrelated catalog values. Additional unscored coverage gaps include Jekyll, Hugo, Dioxus, Ember, Gradio, and aiohttp. Their generic language/project fallback is not counted as a failed framework assertion. Test fixtures and example projects are intentionally scanned, so the report is not a list of deployable services. @@ -43,45 +43,51 @@ Additional unscored coverage gaps include Jekyll, Hugo, Dioxus, Ember, Gradio, a Each repository was scanned through the release CLI five times, serially. The table reports the median of runs 2–5 with a warm filesystem, including process startup, traversal, JSON serialization, and stdout transfer. Python JSON parsing and snapshot downloads are excluded. These are local measurements, not portable timing budgets or controlled cold-cache benchmarks. -Across the corpus, the scanner counted 259,818 files and read only 2,204 manifest/version files (1.77 MiB). The median repository time was 7.9 ms; the slowest was Next.js at 168.4 ms across 16,954 directories and 853 reported projects. - -| Language | Repository | Stars | Language rank | Revision | Assertions | Median ms | -|---|---|---:|---:|---|---:|---:| -| JavaScript | [react/react](https://github.com/react/react) | 252,455 | 2 | [`b05c0b75fd97`](https://github.com/react/react/tree/b05c0b75fd9778536152eead30814c9ca90c810a) | 5/6 | 14.47 | -| JavaScript | [vercel/next.js](https://github.com/vercel/next.js) | 143,437 | 8 | [`5368e013e74e`](https://github.com/vercel/next.js/tree/5368e013e74e4e4ed5be51c051c989545c4450b9) | 5/7 | 168.41 | -| JavaScript | [mrdoob/three.js](https://github.com/mrdoob/three.js) | 117,953 | 11 | [`3f7edc3d75e0`](https://github.com/mrdoob/three.js/tree/3f7edc3d75e054bc755dad56c1c95c62f63a3561) | 3/3 | 4.31 | -| JavaScript | [axios/axios](https://github.com/axios/axios) | 109,295 | 14 | [`335e2f8464f5`](https://github.com/axios/axios/tree/335e2f8464f58a3ab485486b8d6964fea3e3f30a) | 4/4 | 2.18 | -| JavaScript | [react/create-react-app](https://github.com/react/create-react-app) | 103,230 | 15 | [`6254386531d2`](https://github.com/react/create-react-app/tree/6254386531d263688ccfa542d0e628fbc0de0b28) | 6/6 | 2.30 | -| TypeScript | [freeCodeCamp/freeCodeCamp](https://github.com/freeCodeCamp/freeCodeCamp) | 458,826 | 1 | [`11442d669447`](https://github.com/freeCodeCamp/freeCodeCamp/tree/11442d66944785ac72a2a2b9c955fae8b9f33083) | 5/6 | 14.84 | -| TypeScript | [vuejs/vue](https://github.com/vuejs/vue) | 212,624 | 7 | [`9e8870794008`](https://github.com/vuejs/vue/tree/9e88707940088cb1f4cd7dd210c9168a50dc347c) | 4/6 | 2.58 | -| TypeScript | [n8n-io/n8n](https://github.com/n8n-io/n8n) | 209,927 | 8 | [`93243edc00b2`](https://github.com/n8n-io/n8n/tree/93243edc00b288a840fbb562a5e9f64177f2337f) | 5/7 | 60.35 | -| TypeScript | [firecrawl/firecrawl](https://github.com/firecrawl/firecrawl) | 200,129 | 9 | [`40c8ff95d719`](https://github.com/firecrawl/firecrawl/tree/40c8ff95d719d2acbd8fd6273bd19b14a1c2bbb8) | 5/5 | 4.86 | -| TypeScript | [excalidraw/excalidraw](https://github.com/excalidraw/excalidraw) | 135,400 | 15 | [`701b901a0f1b`](https://github.com/excalidraw/excalidraw/tree/701b901a0f1ba3d0b390d9cdf63d112e25abac4b) | 7/7 | 3.94 | -| Python | [Significant-Gravitas/AutoGPT](https://github.com/Significant-Gravitas/AutoGPT) | 187,972 | 11 | [`9cb76f9026d3`](https://github.com/Significant-Gravitas/AutoGPT/tree/9cb76f9026d388c9bf2e098af2beae1cc864f055) | 5/6 | 14.63 | -| Python | [AUTOMATIC1111/stable-diffusion-webui](https://github.com/AUTOMATIC1111/stable-diffusion-webui) | 165,269 | 14 | [`82a973c04367`](https://github.com/AUTOMATIC1111/stable-diffusion-webui/tree/82a973c04367123ae98bd9abdf80d9eda9b910e2) | 3/3 | 1.60 | -| Python | [langgenius/dify](https://github.com/langgenius/dify) | 160,774 | 15 | [`fdb09a9e76dc`](https://github.com/langgenius/dify/tree/fdb09a9e76dc8cbd052fabd72cfff4071328d9bc) | 8/10 | 30.46 | -| Python | [langflow-ai/langflow](https://github.com/langflow-ai/langflow) | 156,479 | 16 | [`60999cc9a735`](https://github.com/langflow-ai/langflow/tree/60999cc9a7350e44dde3ef641baae864e10861f0) | 6/6 | 20.32 | -| Python | [Comfy-Org/ComfyUI](https://github.com/Comfy-Org/ComfyUI) | 145,983 | 20 | [`540a82ee722b`](https://github.com/Comfy-Org/ComfyUI/tree/540a82ee722bda1f32a1fbd97fcadadeb22d548b) | 4/4 | 3.49 | -| PHP | [coollabsio/coolify](https://github.com/coollabsio/coolify) | 65,849 | 2 | [`30335d80866f`](https://github.com/coollabsio/coolify/tree/30335d80866ff3391d899cf4ccb599b696f47b7e) | 5/5 | 5.30 | -| PHP | [nextcloud/server](https://github.com/nextcloud/server) | 37,456 | 3 | [`d655757fb306`](https://github.com/nextcloud/server/tree/d655757fb306aca893651958030b5dae69ed0f39) | 3/3 | 14.62 | -| PHP | [laravel/framework](https://github.com/laravel/framework) | 34,995 | 4 | [`338e44065c0a`](https://github.com/laravel/framework/tree/338e44065c0a5b18f1f290605b6253a1e69d8f56) | 4/5 | 6.33 | -| PHP | [filamentphp/filament](https://github.com/filamentphp/filament) | 32,462 | 5 | [`9d78093cfebd`](https://github.com/filamentphp/filament/tree/9d78093cfebdc96c077b1d8dbf94b41738f28992) | 4/4 | 16.63 | -| PHP | [symfony/symfony](https://github.com/symfony/symfony) | 31,523 | 6 | [`f07928484c9e`](https://github.com/symfony/symfony/tree/f07928484c9e96646964deb7b9570d7885492512) | 4/5 | 10.10 | -| Ruby | [rails/rails](https://github.com/rails/rails) | 59,087 | 1 | [`3a0aff317bc0`](https://github.com/rails/rails/tree/3a0aff317bc037f43f168e3772028f1033f1e810) | 2/3 | 7.90 | -| Ruby | [jekyll/jekyll](https://github.com/jekyll/jekyll) | 51,756 | 2 | [`97c3bdc822df`](https://github.com/jekyll/jekyll/tree/97c3bdc822df9d5bea4c269b29a28de5a5e29d70) | 2/2 | 2.42 | -| Ruby | [mastodon/mastodon](https://github.com/mastodon/mastodon) | 50,502 | 3 | [`5a207bc8f504`](https://github.com/mastodon/mastodon/tree/5a207bc8f5046e27cffc640a3c64f96504f6afcf) | 4/4 | 8.68 | -| Ruby | [huginn/huginn](https://github.com/huginn/huginn) | 50,454 | 4 | [`639c050b2bf4`](https://github.com/huginn/huginn/tree/639c050b2bf4e14ef9983012af99dfc1dc68b334) | 4/4 | 2.35 | -| Ruby | [discourse/discourse](https://github.com/discourse/discourse) | 48,066 | 6 | [`899c68d783e7`](https://github.com/discourse/discourse/tree/899c68d783e7808c50144752369a685220250d7c) | 3/5 | 35.98 | -| Go | [infiniflow/ragflow](https://github.com/infiniflow/ragflow) | 93,887 | 8 | [`ed5a84fee9ab`](https://github.com/infiniflow/ragflow/tree/ed5a84fee9abc8091aea111d1195cf7768541ff1) | 5/5 | 10.20 | -| Go | [gohugoio/hugo](https://github.com/gohugoio/hugo) | 90,425 | 9 | [`d2010fcc9cd3`](https://github.com/gohugoio/hugo/tree/d2010fcc9cd30c5b4e9274ded4485f4222326b53) | 4/4 | 4.85 | -| Go | [gin-gonic/gin](https://github.com/gin-gonic/gin) | 89,609 | 11 | [`0f09c3a9b462`](https://github.com/gin-gonic/gin/tree/0f09c3a9b4626d9fe9979cf4d0521d6ae32ab646) | 4/5 | 2.11 | -| Go | [netdata/netdata](https://github.com/netdata/netdata) | 81,330 | 14 | [`0d54b4424ea6`](https://github.com/netdata/netdata/tree/0d54b4424ea6f40c344738b9d566e1a26d160da3) | 4/4 | 23.42 | -| Go | [caddyserver/caddy](https://github.com/caddyserver/caddy) | 78,927 | 15 | [`9ea3108377d2`](https://github.com/caddyserver/caddy/tree/9ea3108377d2abec102866808133c40ac9582162) | 4/4 | 2.56 | -| Rust | [dani-garcia/vaultwarden](https://github.com/dani-garcia/vaultwarden) | 70,490 | 17 | [`17b159beff4b`](https://github.com/dani-garcia/vaultwarden/tree/17b159beff4b892f07f61f38ce2fe452e55251c1) | 6/6 | 2.51 | -| Rust | [meilisearch/meilisearch](https://github.com/meilisearch/meilisearch) | 59,822 | 29 | [`8c7bfcfa9bcf`](https://github.com/meilisearch/meilisearch/tree/8c7bfcfa9bcff7083bcb5d184fa2a523a23884d8) | 5/5 | 4.99 | -| Rust | [juspay/hyperswitch](https://github.com/juspay/hyperswitch) | 45,760 | 39 | [`5786b738c088`](https://github.com/juspay/hyperswitch/tree/5786b738c088b8452296ca19922fcdc0dc74e129) | 6/6 | 12.07 | -| Rust | [parcel-bundler/parcel](https://github.com/parcel-bundler/parcel) | 44,012 | 42 | [`20c0f2ede4c1`](https://github.com/parcel-bundler/parcel/tree/20c0f2ede4c1044092d44288169cce589c5d5c14) | 3/3 | 18.92 | -| Rust | [DioxusLabs/dioxus](https://github.com/DioxusLabs/dioxus) | 39,705 | 52 | [`17d9a5924e84`](https://github.com/DioxusLabs/dioxus/tree/17d9a5924e845e4f37deca3fada6c35d1d836af0) | 5/5 | 8.71 | +Across the corpus, the scanner counted 259,818 files and read only 2,270 manifest/version files (1.85 MiB). The current median repository time is 5.4 ms. + +An interleaved, same-machine A/B comparison ran the baseline and current binaries 12 times each, discarding the first two pairs: **Next.js improved from 170.8 ms to 86.8 ms** (1.97Γ— faster), with the same full-tree scope. The generated 200-service / 20,200-file benchmark improved from 5.8 ms to **3.5 ms median**, still reading only 200 manifests. + +Profiling identified directory traversal and repeated path/ancestor lookups as the main costs. Changes index document lookups, retain relevant directory facts only, use parent indices for source ownership/version inheritance, and prefilter workspace candidates by manifest type. Wide directory trees use at most four workers; small frontiers remain serial. Buffered JSON output avoids per-line stdout flushes. Before adding the detection fixes, the indexing/ownership optimizations alone produced byte-identical results on all 35 repositories. + +pnpm uses `serde-saphyr` with deserialization only: typed fields are populated directly without constructing a syntax tree, retaining comments, enabling includes, or evaluating application code. A tree-sitter syntax tree is unnecessary for this manifest-only scan. + +| Language | Repository | Stars | Language rank | Revision | Baseline assertions | Current assertions | Before ms | After ms | +|---|---|---:|---:|---|---:|---:|---:|---:| +| JavaScript | [react/react](https://github.com/react/react) | 252,455 | 2 | [`b05c0b75fd97`](https://github.com/react/react/tree/b05c0b75fd9778536152eead30814c9ca90c810a) | 5/6 | 6/6 | 14.47 | 7.29 | +| JavaScript | [vercel/next.js](https://github.com/vercel/next.js) | 143,437 | 8 | [`5368e013e74e`](https://github.com/vercel/next.js/tree/5368e013e74e4e4ed5be51c051c989545c4450b9) | 5/7 | 7/7 | 168.41 | 85.82 | +| JavaScript | [mrdoob/three.js](https://github.com/mrdoob/three.js) | 117,953 | 11 | [`3f7edc3d75e0`](https://github.com/mrdoob/three.js/tree/3f7edc3d75e054bc755dad56c1c95c62f63a3561) | 3/3 | 3/3 | 4.31 | 3.74 | +| JavaScript | [axios/axios](https://github.com/axios/axios) | 109,295 | 14 | [`335e2f8464f5`](https://github.com/axios/axios/tree/335e2f8464f58a3ab485486b8d6964fea3e3f30a) | 4/4 | 4/4 | 2.18 | 2.61 | +| JavaScript | [react/create-react-app](https://github.com/react/create-react-app) | 103,230 | 15 | [`6254386531d2`](https://github.com/react/create-react-app/tree/6254386531d263688ccfa542d0e628fbc0de0b28) | 6/6 | 6/6 | 2.30 | 2.26 | +| TypeScript | [freeCodeCamp/freeCodeCamp](https://github.com/freeCodeCamp/freeCodeCamp) | 458,826 | 1 | [`11442d669447`](https://github.com/freeCodeCamp/freeCodeCamp/tree/11442d66944785ac72a2a2b9c955fae8b9f33083) | 5/6 | 6/6 | 14.84 | 10.73 | +| TypeScript | [vuejs/vue](https://github.com/vuejs/vue) | 212,624 | 7 | [`9e8870794008`](https://github.com/vuejs/vue/tree/9e88707940088cb1f4cd7dd210c9168a50dc347c) | 4/6 | 6/6 | 2.58 | 2.70 | +| TypeScript | [n8n-io/n8n](https://github.com/n8n-io/n8n) | 209,927 | 8 | [`93243edc00b2`](https://github.com/n8n-io/n8n/tree/93243edc00b288a840fbb562a5e9f64177f2337f) | 5/7 | 7/7 | 60.35 | 32.57 | +| TypeScript | [firecrawl/firecrawl](https://github.com/firecrawl/firecrawl) | 200,129 | 9 | [`40c8ff95d719`](https://github.com/firecrawl/firecrawl/tree/40c8ff95d719d2acbd8fd6273bd19b14a1c2bbb8) | 5/5 | 5/5 | 4.86 | 4.54 | +| TypeScript | [excalidraw/excalidraw](https://github.com/excalidraw/excalidraw) | 135,400 | 15 | [`701b901a0f1b`](https://github.com/excalidraw/excalidraw/tree/701b901a0f1ba3d0b390d9cdf63d112e25abac4b) | 7/7 | 7/7 | 3.94 | 2.66 | +| Python | [Significant-Gravitas/AutoGPT](https://github.com/Significant-Gravitas/AutoGPT) | 187,972 | 11 | [`9cb76f9026d3`](https://github.com/Significant-Gravitas/AutoGPT/tree/9cb76f9026d388c9bf2e098af2beae1cc864f055) | 5/6 | 6/6 | 14.63 | 11.53 | +| Python | [AUTOMATIC1111/stable-diffusion-webui](https://github.com/AUTOMATIC1111/stable-diffusion-webui) | 165,269 | 14 | [`82a973c04367`](https://github.com/AUTOMATIC1111/stable-diffusion-webui/tree/82a973c04367123ae98bd9abdf80d9eda9b910e2) | 3/3 | 3/3 | 1.60 | 1.93 | +| Python | [langgenius/dify](https://github.com/langgenius/dify) | 160,774 | 15 | [`fdb09a9e76dc`](https://github.com/langgenius/dify/tree/fdb09a9e76dc8cbd052fabd72cfff4071328d9bc) | 8/10 | 10/10 | 30.46 | 18.70 | +| Python | [langflow-ai/langflow](https://github.com/langflow-ai/langflow) | 156,479 | 16 | [`60999cc9a735`](https://github.com/langflow-ai/langflow/tree/60999cc9a7350e44dde3ef641baae864e10861f0) | 6/6 | 6/6 | 20.32 | 12.02 | +| Python | [Comfy-Org/ComfyUI](https://github.com/Comfy-Org/ComfyUI) | 145,983 | 20 | [`540a82ee722b`](https://github.com/Comfy-Org/ComfyUI/tree/540a82ee722bda1f32a1fbd97fcadadeb22d548b) | 4/4 | 4/4 | 3.49 | 3.34 | +| PHP | [coollabsio/coolify](https://github.com/coollabsio/coolify) | 65,849 | 2 | [`30335d80866f`](https://github.com/coollabsio/coolify/tree/30335d80866ff3391d899cf4ccb599b696f47b7e) | 5/5 | 5/5 | 5.30 | 4.41 | +| PHP | [nextcloud/server](https://github.com/nextcloud/server) | 37,456 | 3 | [`d655757fb306`](https://github.com/nextcloud/server/tree/d655757fb306aca893651958030b5dae69ed0f39) | 3/3 | 3/3 | 14.62 | 7.52 | +| PHP | [laravel/framework](https://github.com/laravel/framework) | 34,995 | 4 | [`338e44065c0a`](https://github.com/laravel/framework/tree/338e44065c0a5b18f1f290605b6253a1e69d8f56) | 4/5 | 5/5 | 6.33 | 5.06 | +| PHP | [filamentphp/filament](https://github.com/filamentphp/filament) | 32,462 | 5 | [`9d78093cfebd`](https://github.com/filamentphp/filament/tree/9d78093cfebdc96c077b1d8dbf94b41738f28992) | 4/4 | 4/4 | 16.63 | 15.30 | +| PHP | [symfony/symfony](https://github.com/symfony/symfony) | 31,523 | 6 | [`f07928484c9e`](https://github.com/symfony/symfony/tree/f07928484c9e96646964deb7b9570d7885492512) | 4/5 | 5/5 | 10.10 | 9.20 | +| Ruby | [rails/rails](https://github.com/rails/rails) | 59,087 | 1 | [`3a0aff317bc0`](https://github.com/rails/rails/tree/3a0aff317bc037f43f168e3772028f1033f1e810) | 2/3 | 3/3 | 7.90 | 5.62 | +| Ruby | [jekyll/jekyll](https://github.com/jekyll/jekyll) | 51,756 | 2 | [`97c3bdc822df`](https://github.com/jekyll/jekyll/tree/97c3bdc822df9d5bea4c269b29a28de5a5e29d70) | 2/2 | 2/2 | 2.42 | 2.12 | +| Ruby | [mastodon/mastodon](https://github.com/mastodon/mastodon) | 50,502 | 3 | [`5a207bc8f504`](https://github.com/mastodon/mastodon/tree/5a207bc8f5046e27cffc640a3c64f96504f6afcf) | 4/4 | 4/4 | 8.68 | 7.09 | +| Ruby | [huginn/huginn](https://github.com/huginn/huginn) | 50,454 | 4 | [`639c050b2bf4`](https://github.com/huginn/huginn/tree/639c050b2bf4e14ef9983012af99dfc1dc68b334) | 4/4 | 4/4 | 2.35 | 2.06 | +| Ruby | [discourse/discourse](https://github.com/discourse/discourse) | 48,066 | 6 | [`899c68d783e7`](https://github.com/discourse/discourse/tree/899c68d783e7808c50144752369a685220250d7c) | 3/5 | 5/5 | 35.98 | 21.07 | +| Go | [infiniflow/ragflow](https://github.com/infiniflow/ragflow) | 93,887 | 8 | [`ed5a84fee9ab`](https://github.com/infiniflow/ragflow/tree/ed5a84fee9abc8091aea111d1195cf7768541ff1) | 5/5 | 5/5 | 10.20 | 5.38 | +| Go | [gohugoio/hugo](https://github.com/gohugoio/hugo) | 90,425 | 9 | [`d2010fcc9cd3`](https://github.com/gohugoio/hugo/tree/d2010fcc9cd30c5b4e9274ded4485f4222326b53) | 4/4 | 4/4 | 4.85 | 3.15 | +| Go | [gin-gonic/gin](https://github.com/gin-gonic/gin) | 89,609 | 11 | [`0f09c3a9b462`](https://github.com/gin-gonic/gin/tree/0f09c3a9b4626d9fe9979cf4d0521d6ae32ab646) | 4/5 | 5/5 | 2.11 | 1.70 | +| Go | [netdata/netdata](https://github.com/netdata/netdata) | 81,330 | 14 | [`0d54b4424ea6`](https://github.com/netdata/netdata/tree/0d54b4424ea6f40c344738b9d566e1a26d160da3) | 4/4 | 4/4 | 23.42 | 11.05 | +| Go | [caddyserver/caddy](https://github.com/caddyserver/caddy) | 78,927 | 15 | [`9ea3108377d2`](https://github.com/caddyserver/caddy/tree/9ea3108377d2abec102866808133c40ac9582162) | 4/4 | 4/4 | 2.56 | 2.33 | +| Rust | [dani-garcia/vaultwarden](https://github.com/dani-garcia/vaultwarden) | 70,490 | 17 | [`17b159beff4b`](https://github.com/dani-garcia/vaultwarden/tree/17b159beff4b892f07f61f38ce2fe452e55251c1) | 6/6 | 6/6 | 2.51 | 2.76 | +| Rust | [meilisearch/meilisearch](https://github.com/meilisearch/meilisearch) | 59,822 | 29 | [`8c7bfcfa9bcf`](https://github.com/meilisearch/meilisearch/tree/8c7bfcfa9bcff7083bcb5d184fa2a523a23884d8) | 5/5 | 5/5 | 4.99 | 5.17 | +| Rust | [juspay/hyperswitch](https://github.com/juspay/hyperswitch) | 45,760 | 39 | [`5786b738c088`](https://github.com/juspay/hyperswitch/tree/5786b738c088b8452296ca19922fcdc0dc74e129) | 6/6 | 6/6 | 12.07 | 7.95 | +| Rust | [parcel-bundler/parcel](https://github.com/parcel-bundler/parcel) | 44,012 | 42 | [`20c0f2ede4c1`](https://github.com/parcel-bundler/parcel/tree/20c0f2ede4c1044092d44288169cce589c5d5c14) | 3/3 | 3/3 | 18.92 | 11.77 | +| Rust | [DioxusLabs/dioxus](https://github.com/DioxusLabs/dioxus) | 39,705 | 52 | [`17d9a5924e84`](https://github.com/DioxusLabs/dioxus/tree/17d9a5924e845e4f37deca3fada6c35d1d836af0) | 5/5 | 5/5 | 8.71 | 7.17 | ## Reproduce @@ -91,7 +97,7 @@ cache=$(mktemp -d) python3 tests/audit_detect.py --fetch --cache "$cache" --output "$cache/results.json" ``` -Downloads require the explicit `--fetch` flag; omit it to rerun against an existing pinned cache entirely offline. Downloads are capped at 1 GiB compressed / 2 GiB expanded per repository. Existing unverified cache entries are not overwritten. The runner exits nonzero for missing assertions, command failures, or nondeterministic output; **exit 1 is expected for this baseline**. +Downloads require the explicit `--fetch` flag; omit it to rerun against an existing pinned cache entirely offline. Downloads are capped at 1 GiB compressed / 2 GiB expanded per repository. Existing unverified cache entries are not overwritten. The runner exits nonzero for missing assertions, command failures, or nondeterministic output; **the current corpus is expected to exit 0**. ```sh PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s tests -p 'test_*.py' -v From ad4102213bfdda6098fa726a32fd4cbf74a9fc4a Mon Sep 17 00:00:00 2001 From: Dillen Meijboom Date: Mon, 9 Nov 2026 22:27:27 +0000 Subject: [PATCH 04/10] docs: record detection validation against Brainpod starters --- tests/detect-corpus.md | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/tests/detect-corpus.md b/tests/detect-corpus.md index 18d2b6d..73febd6 100644 --- a/tests/detect-corpus.md +++ b/tests/detect-corpus.md @@ -14,7 +14,7 @@ GitHub source archives were downloaded into a temporary cache. No application de ## Results -All **35/35** scans succeed and produce identical output across five runs. **173/173 sampled assertions now pass**, up from 156/173. Every repository passes its sampled assertions; this is not a claim of exhaustive detection accuracy. +All **35/35** scans succeed and produce identical output across five runs. **173/173 sampled assertions now pass**, up from 156/173. Each of the 35 ranked repositories passes its sampled assertions; this is not a claim of exhaustive detection accuracy. | Category | Baseline | Current | |---|---:|---:| @@ -89,6 +89,29 @@ pnpm uses `serde-saphyr` with deserialization only: typed fields are populated d | Rust | [parcel-bundler/parcel](https://github.com/parcel-bundler/parcel) | 44,012 | 42 | [`20c0f2ede4c1`](https://github.com/parcel-bundler/parcel/tree/20c0f2ede4c1044092d44288169cce589c5d5c14) | 3/3 | 3/3 | 18.92 | 11.77 | | Rust | [DioxusLabs/dioxus](https://github.com/DioxusLabs/dioxus) | 39,705 | 52 | [`17d9a5924e84`](https://github.com/DioxusLabs/dioxus/tree/17d9a5924e845e4f37deca3fada6c35d1d836af0) | 5/5 | 5/5 | 8.71 | 7.17 | +## Additional validation: Brainpod starters + +Ran the release detector against all ten `case/*` branches of [`brainpodnl/starters`](https://github.com/brainpodnl/starters), pinning each commit below. The default branch is `case/bookstack`. This is a separate starter smoke test, not an expansion of the 173 ranked-corpus assertions above. + +Each branch was scanned five times without installing application dependencies or executing downloaded code. All ten scans succeeded and repeated byte-identically. Seven expected main stacks were recognized; Outline is partial because Koa has no rule, and eShop / Spring Petclinic are outside the supported language/manifest set. Auxiliary docs/assets must not be counted as detection of eShop’s main application. + +Timings are warm-filesystem, end-to-end CLI medians of runs 2–5. Empty or partial output is not treated as an accuracy pass merely because it is fast. + +| Branch | Revision | Result | Median ms | +|---|---|---|---:| +| `case/bookstack` | [`6265d5667dd6`](https://github.com/brainpodnl/starters/tree/6265d5667dd6904b7aa0d05a8c60795cf104011b) | Laravel, Composer, JavaScript/TypeScript. PHP `^8.2.0` and Laravel `^v12.26.4` declarations retained. | 4.07 | +| `case/eshop` | [`f21d6233b851`](https://github.com/brainpodnl/starters/tree/f21d6233b851897371c25c5e7ffa67bb8552dba8) | **Unsupported primary stack:** .NET 10/C# not detected. Only the docs Gemfile and JavaScript assets appear. | 2.54 | +| `case/fastapi-fullstack` | [`84c5a9e11a9e`](https://github.com/brainpodnl/starters/tree/84c5a9e11a9e58262599e3d772f5386924e02ca7) | FastAPI backend, React/Vite frontend, npm and uv workspaces. Python `3.14` pin plus `>=3.14,<4.0` constraint retained; all three workspace members found. | 2.44 | +| `case/linkding` | [`24b5ad6cc9bd`](https://github.com/brainpodnl/starters/tree/24b5ad6cc9bde497b5d1b1e86aed5a2fb7d25c2f) | Django backend (`python >=3.13`) and Astro documentation project detected. | 2.52 | +| `case/maybe` | [`224f21354a91`](https://github.com/brainpodnl/starters/tree/224f21354a916dfbec29e9435db1045b3d61a94b) | Rails and Ruby `3.4.4` pin detected. Non-fatal `dynamic ruby declaration not evaluated` warning for `ruby file: ".ruby-version"`. | 3.09 | +| `case/miniflux` | [`faecfa2fac61`](https://github.com/brainpodnl/starters/tree/faecfa2fac61a91f4f69550bb1afd506c4b51749) | Go module `miniflux.app/v2`, Go `1.26.0` requirement detected; auxiliary scripts/assets are separate source projects. | 2.47 | +| `case/outline` | [`09f11188b87b`](https://github.com/brainpodnl/starters/tree/09f11188b87b0220fa318dfa6fdd0fa9772195dd) | **Partial:** Node/React/Vite, TypeScript, Node engine constraint and `.nvmrc` pin detected. Declared Koa `^3.0.3` backend is not labeled. | 4.06 | +| `case/saas-starter` | [`6e33e58b1e55`](https://github.com/brainpodnl/starters/tree/6e33e58b1e553a41fe22e6b941a7229a002de361) | Next.js/React and TypeScript detected. No package name or Node version is invented when absent from supported manifest fields. | 2.05 | +| `case/spring-petclinic` | [`500158f73241`](https://github.com/brainpodnl/starters/tree/500158f732419217507c7656904b8e6aa1bcc0d6) | **Unsupported primary stack:** Java 17 / Spring Boot `4.1.0` declared in Maven, but the detector returns no projects. | 2.16 | +| `case/uptime-kuma` | [`df601c58d920`](https://github.com/brainpodnl/starters/tree/df601c58d9205562f34cfe5ef35264b341019d19) | Express/Vue/Vite and Node `>=20.4.0` constraint detected, alongside auxiliary example projects. | 2.60 | + +Koa needs an additional npm framework rule. C#/.NET and Java/Spring require new language and manifest support (`global.json` / MSBuild and Maven, respectively). The Maybe warning is redundant with the already detected Ruby pin. None of these observations was silently converted into a passing assertion or hidden by skipping a branch. The detector was unchanged during this validation. + ## Reproduce ```sh From 0a81c2561a1b8c106ea4e3d23427498ff76f7cd4 Mon Sep 17 00:00:00 2001 From: Dillen Meijboom Date: Mon, 9 Nov 2026 22:49:27 +0000 Subject: [PATCH 05/10] fix: repair project detection CI builds --- src/s3/tests.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/s3/tests.rs b/src/s3/tests.rs index 093769a..9eed01f 100644 --- a/src/s3/tests.rs +++ b/src/s3/tests.rs @@ -447,7 +447,8 @@ fn trust_provider(gateway: &mut Gateway<'_>) { .no_proxy() .tls_certs_only([certificate]) .redirect(reqwest::redirect::Policy::none()) - .timeout(Duration::from_secs(5)) + .connect_timeout(Duration::from_secs(5)) + .read_timeout(Duration::from_secs(5)) .build() .unwrap(); } From 1b7ae5beb8af743fbb8521a15d2bf11232f909ad Mon Sep 17 00:00:00 2001 From: Dillen Meijboom Date: Mon, 9 Nov 2026 22:49:27 +0000 Subject: [PATCH 06/10] feat: detect C# and Java projects and framework versions --- Cargo.lock | 1 + Cargo.toml | 1 + README.md | 10 +- src/detect/dotnet.rs | 700 ++++++++++++++++++++++++++++++++ src/detect/gradle.rs | 718 +++++++++++++++++++++++++++++++++ src/detect/java.rs | 359 +++++++++++++++++ src/detect/jvm_dotnet_tests.rs | 564 ++++++++++++++++++++++++++ src/detect/mod.rs | 67 ++- src/detect/parsers.rs | 33 +- src/detect/rules.rs | 13 +- src/detect/rules.toml | 112 +++++ src/detect/tests.rs | 8 +- src/detect/values.rs | 199 +++++++++ src/detect/xml.rs | 171 ++++++++ tests/audit_detect.py | 15 +- tests/detect-corpus.json | 512 +++++++++++++++++++++++ tests/detect-corpus.md | 126 +++--- tests/test_audit_detect.py | 24 ++ 18 files changed, 3552 insertions(+), 81 deletions(-) create mode 100644 src/detect/dotnet.rs create mode 100644 src/detect/gradle.rs create mode 100644 src/detect/java.rs create mode 100644 src/detect/jvm_dotnet_tests.rs create mode 100644 src/detect/values.rs create mode 100644 src/detect/xml.rs diff --git a/Cargo.lock b/Cargo.lock index 771bd68..555f15e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -602,6 +602,7 @@ dependencies = [ "tonic-build", "uuid", "webbrowser", + "xmlparser", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 9cd8ff3..6649d4a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,6 +35,7 @@ toml = "0.9.5" tonic = { version = "0.12.3", features = ["tls", "tls-webpki-roots"] } uuid = { version = "1.16.0", features = ["v4"] } webbrowser = "=1.0.6" +xmlparser = "0.13.6" [build-dependencies] protox = "0.7.1" diff --git a/README.md b/README.md index ed51bb3..95868f5 100644 --- a/README.md +++ b/README.md @@ -274,9 +274,9 @@ brainpod detect ./services --json brainpod detect . --rules ./detection.toml --json ``` -The result contains sorted project paths, manifest names, languages and version declarations, priority-ranked detections with evidence and `uses`/`provides` framework roles, workspace members, diagnostics, and scan statistics. Framework versions are attached to dependency evidence. JavaScript versions describe Node.js; TypeScript versions describe the compiler. Requirements and pins retain their source and kind: these are declarations, not resolved or installed versions. Missing versions remain unknown. +The result contains sorted project paths, manifest names, languages and version declarations, priority-ranked detections with evidence and `uses`/`provides` framework roles, workspace members, diagnostics, and scan statistics. Framework versions are attached to dependency evidence. JavaScript versions describe Node.js; TypeScript versions describe the compiler. C# versions distinguish `language` (LangVersion), `target` (TFM), `runtime`, and `sdk`; Java distinguishes compiler `source`/`target`/`release`, toolchains, build-JDK metadata, and pins. Requirements and pins retain their source and kind: these are declarations, not a claim about installed versions. Missing versions remain unknown. -Built-in coverage includes Laravel, Symfony, CakePHP, CodeIgniter, Composer; Go modules/workspaces and Gin/Echo/Fiber; Cargo workspaces, axum, Rocket, Actix; Bundler, Rails, Sinatra, Hanami; Python requirements, PEP 621, Poetry, Pipenv, uv workspaces, Django, FastAPI, Flask, Starlette, Celery; npm/pnpm workspaces and catalogs, Next.js, Nuxt, Astro, SvelteKit, Remix, Angular, Express, React, Vue, Vite; and standalone source files in all seven languages. Rust workspace dependency aliases and inherited `rust-version` are understood. Ancestor runtime pin/toolchain files are included with provenance. Poetry names, literal gemspec metadata, and Rails component dependencies are recognized. pnpm catalog references retain the original declaration and add the locally resolved constraint with its YAML source; unresolved references produce diagnostics. +Built-in coverage includes Laravel, Symfony, CakePHP, CodeIgniter, Composer; Go modules/workspaces and Gin/Echo/Fiber; Cargo workspaces, axum, Rocket, Actix; Bundler, Rails, Sinatra, Hanami; Python requirements, PEP 621, Poetry, Pipenv, uv workspaces, Django, FastAPI, Flask, Starlette, Celery; npm/pnpm workspaces and catalogs, Next.js, Nuxt, Astro, SvelteKit, Remix, Angular, Express, React, Vue, Vite; C#/.NET projects and solutions, ASP.NET Core/classic, Aspire, FastEndpoints; Java/JVM Maven and Gradle projects, Spring/Spring Boot, Quarkus, Micronaut; and standalone source files in all nine languages. Rust workspace dependency aliases and inherited `rust-version` are understood. Ancestor runtime pin/toolchain files are included with provenance. Poetry names, literal gemspec metadata, and Rails component dependencies are recognized. pnpm catalog references retain the original declaration and add the locally resolved constraint with its YAML source; unresolved references produce diagnostics. Rules live in [`src/detect/rules.toml`](src/detect/rules.toml), independently of manifest parsers. Additional rules use unique IDs: @@ -290,9 +290,9 @@ all = ["dep:npm:next", "file:company.config.*"] none = ["file:legacy.config"] ``` -`all` requires every pattern, `any` requires at least one when supplied, and `none` rejects any match. Higher priorities sort first; ties sort by ID. Matches coexist rather than hiding other frameworks or languages. Patterns also accept conjunction groups, for example `any = ["dep:gem:rails", { all = ["dep:gem:actionpack", "dep:gem:railties"] }]`. Patterns support `file:` (`*`, `?`), `source:`, `dep::`, `identity::`, `workspace:`, and `package:cargo`. Workspace paths additionally support `**`. Ecosystems are `composer`, `go`, `cargo`, `gem`, `pypi`, and `npm`. +`all` requires every pattern, `any` requires at least one when supplied, and `none` rejects any match. Higher priorities sort first; ties sort by ID. Matches coexist rather than hiding other frameworks or languages. Patterns also accept conjunction groups, for example `any = ["dep:gem:rails", { all = ["dep:gem:actionpack", "dep:gem:railties"] }]`. Patterns support `file:` (`*`, `?`), `source:`, `dep::`, `identity::`, `workspace:`, and `package:`. Workspace paths additionally support `**`. Ecosystems include `composer`, `go`, `cargo`, `gem`, `pypi`, `npm`, `nuget`, `dotnet-sdk`, `dotnet-framework`, `dotnet-reference`, `maven`, `maven-parent`, `maven-plugin`, and `gradle-plugin`. -The scanner walks once, caches indexed rules, and never reads application source bodies or lockfiles. Wide trees use at most four directory workers; small trees stay serial. Directory facts are indexed and irrelevant empty directories are discarded, while output stays deterministic. pnpm YAML is deserialized directly into typed fields without building a syntax tree; unused fields/comments are not retained. Each manifest/version file is capped at 1 MiB. Symlinks are not followed; traversal stops at depth 128 with a diagnostic. Dependency/build directories such as `.git`, `.jj`, `node_modules`, `vendor`, `target`, `.venv`, `.next`, `dist`, and `build` are pruned. It does not evaluate `.gitignore`. Source-only directories roll up to a same-language project ancestor when available; sibling services keep their own manifest boundaries. +The scanner walks once, caches indexed rules, and never reads application source bodies or lockfiles. Wide trees use at most four directory workers; small trees stay serial. Directory facts are indexed and irrelevant empty directories are discarded, while output stays deterministic. pnpm YAML is deserialized directly into typed fields without building a syntax tree; unused fields/comments are not retained. XML uses the existing zero-allocation pull tokenizer, with structure/namespace checks and DTD/custom-entity rejection. Maven local parents and managed versions, MSBuild directory props/targets, central package versions, and static Gradle plugin/dependency/module declarations are resolved without running build tools. Arbitrary MSBuild functions, remote Maven parents/BOMs, profile activation, and dynamic Gradle build logic are not executed or guessed; unresolved version references remain explicit. Each manifest/version file is capped at 1 MiB. Symlinks are not followed; traversal stops at depth 128 with a diagnostic. Dependency/build directories such as `.git`, `.jj`, `node_modules`, `vendor`, `target`, `.venv`, `.next`, `dist`, `build`, `.gradle`, and `.vs` are pruned. `bin`/`obj` are additionally pruned beneath C# project roots without skipping Rust `src/bin` sources. It does not evaluate `.gitignore`. Source-only directories roll up to a same-language project ancestor when available; sibling services keep their own manifest boundaries. Detection uses declared dependencies, not arbitrary substring matches or inferred framework usage from source. Composer/Cargo development dependencies are excluded; npm development dependencies are included because tools such as Astro commonly live there. Dynamic Ruby declarations and unsupported requirements options produce diagnostics instead of executing code. Invalid manifests are diagnosed without stopping discovery of other projects. @@ -302,7 +302,7 @@ Run the generated large-tree performance fixture with: cargo test --release detect::tests::benchmark_polyglot_scan -- --ignored --nocapture ``` -A [35-repository GitHub audit](tests/detect-corpus.md) records pinned real-world coverage, before/after timings, regression fixes, and an opt-in reproducible runner. +A [45-repository GitHub audit](tests/detect-corpus.md) records pinned real-world coverage, before/after timings, regression fixes, and an opt-in reproducible runner. ## Image discovery diff --git a/src/detect/dotnet.rs b/src/detect/dotnet.rs new file mode 100644 index 0000000..a3aa397 --- /dev/null +++ b/src/detect/dotnet.rs @@ -0,0 +1,700 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Path, PathBuf}; + +use anyhow::{Result, bail}; +use serde_json::Value; + +use super::values::{self, Located, Properties, Syntax}; +use super::{Diagnostic, Node, Version, Workspace, display_path, relative_file, xml}; + +#[derive(Default)] +struct Document { + properties: Vec<(String, Located, Vec)>, + steps: Vec, + references: Vec, + imports: Vec<(String, Vec)>, + projects: Vec, + sdk: Vec<(String, Option)>, + solution: bool, +} + +enum Step { + Property(usize), + Import(usize), +} + +struct Reference { + kind: String, + name: String, + version: Option, +} + +fn document(text: &str, source: &str) -> Result { + let mut doc = Document::default(); + let mut nested_versions = BTreeMap::new(); + xml::read(text, |stack| { + let Some(element) = stack.last() else { + return Ok(()); + }; + let root = stack.first().map(|e| e.name); + if root == Some("Solution") { + doc.solution = true; + if element.name == "Project" + && let Some(path) = element.attributes.get("Path") + { + doc.projects.push(path.clone()); + } + return Ok(()); + } + if root != Some("Project") { + bail!("expected an MSBuild Project or Solution root"); + } + let conditions = stack + .iter() + .filter_map(|e| { + e.attributes + .get("Condition") + .filter(|v| !v.trim().is_empty()) + .cloned() + }) + .collect::>(); + let conditional = !conditions.is_empty(); + let located = |value: &str| Located { + value: value.trim().to_owned(), + source: source.to_owned(), + conditional, + }; + let parent = stack.get(stack.len().saturating_sub(2)); + if parent.is_some_and(|p| p.name == "PropertyGroup") && !element.text.trim().is_empty() { + doc.steps.push(Step::Property(doc.properties.len())); + doc.properties.push(( + element.name.to_ascii_lowercase(), + located(&element.text), + conditions.clone(), + )); + } + if element.name == "Version" + && let Some(parent) = parent + { + nested_versions.insert( + (stack.len() - 1, parent.name.to_owned()), + located(&element.text), + ); + } + if matches!( + element.name, + "PackageReference" | "PackageVersion" | "FrameworkReference" | "Reference" + ) && let Some(name) = element + .attributes + .get("Include") + .or_else(|| element.attributes.get("Update")) + { + let version = element + .attributes + .get("VersionOverride") + .or_else(|| element.attributes.get("Version")) + .map(|v| located(v)) + .or_else(|| nested_versions.remove(&(stack.len(), element.name.to_owned()))); + doc.references.push(Reference { + kind: element.name.to_owned(), + name: name.clone(), + version, + }); + } + if element.name == "ProjectReference" + && let Some(path) = element.attributes.get("Include") + { + doc.projects.push(path.clone()); + } + if element.name == "Import" { + if let Some(sdk) = element.attributes.get("Sdk") { + doc.sdk + .push((sdk.clone(), element.attributes.get("Version").cloned())); + } else if let Some(path) = element.attributes.get("Project") { + doc.steps.push(Step::Import(doc.imports.len())); + doc.imports.push((path.clone(), conditions.clone())); + } + } + if element.name == "Sdk" + && let Some(name) = element.attributes.get("Name") + { + doc.sdk + .push((name.clone(), element.attributes.get("Version").cloned())); + } + if stack.len() == 1 + && let Some(sdk) = element.attributes.get("Sdk") + { + for sdk in sdk.split(';') { + let (name, version) = sdk + .trim() + .split_once('/') + .map_or((sdk.trim(), None), |(name, version)| { + (name, Some(version.to_owned())) + }); + if !name.is_empty() { + doc.sdk.push((name.to_owned(), version)); + } + } + } + Ok(()) + })?; + Ok(doc) +} + +fn nearest( + path: &Path, + file: &str, + documents: &BTreeMap>, +) -> Option { + path.ancestors() + .map(|p| p.join(file)) + .find(|p| documents.contains_key(p)) +} + +fn split_condition<'a>(text: &'a str, operator: &str) -> Option<(&'a str, &'a str)> { + let upper = text.to_ascii_uppercase(); + let mut depth = 0; + let mut quote = None; + for (index, ch) in text.char_indices() { + if quote == Some(ch) { + quote = None; + continue; + } + if quote.is_some() { + continue; + } + if ch == '\'' || ch == '"' { + quote = Some(ch); + continue; + } + if ch == '(' { + depth += 1; + } else if ch == ')' { + depth -= 1; + } + if depth == 0 && upper[index..].starts_with(operator) { + return Some((&text[..index], &text[index + operator.len()..])); + } + } + None +} + +fn condition(text: &str, properties: &Properties, depth: usize) -> Option { + if depth == 64 { + return None; + } + let text = text.trim(); + if text.is_empty() { + return Some(true); + } + for (operator, or) in [(" OR ", true), (" AND ", false)] { + if let Some((left, right)) = split_condition(text, operator) { + let a = condition(left, properties, depth + 1); + let b = condition(right, properties, depth + 1); + return if or { + if a == Some(true) || b == Some(true) { + Some(true) + } else if a == Some(false) && b == Some(false) { + Some(false) + } else { + None + } + } else if a == Some(false) || b == Some(false) { + Some(false) + } else if a == Some(true) && b == Some(true) { + Some(true) + } else { + None + }; + } + } + if text.starts_with('(') && text.ends_with(')') { + return condition(&text[1..text.len() - 1], properties, depth + 1); + } + let mut expanded = text.to_owned(); + for _ in 0..64 { + let Some(start) = expanded.find("$(") else { + break; + }; + let end = expanded[start + 2..].find(')')? + start + 2; + let name = &expanded[start + 2..end]; + if !name + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '_' | '.')) + { + return None; + } + let key = name.to_ascii_lowercase(); + let value = if let Some(value) = properties.get(&key) { + value.value.as_str() + } else if key.starts_with("msbuild") + || matches!( + key.as_str(), + "configuration" + | "platform" + | "platformtarget" + | "runtimeidentifier" + | "os" + | "designtimebuild" + | "buildinginsidevisualstudio" + ) + { + return None; + } else { + "" + }; + if value.contains("$(") { + return None; + } + expanded.replace_range(start..end + 1, value); + } + for operator in ["==", "!="] { + if let Some((left, right)) = split_condition(&expanded, operator) { + let literal = |value: &str| -> Option { + let value = value.trim(); + if value.len() >= 2 + && ((value.starts_with('\'') && value.ends_with('\'')) + || (value.starts_with('"') && value.ends_with('"'))) + { + return Some(value[1..value.len() - 1].to_owned()); + } + None + }; + let equal = literal(left)?.eq_ignore_ascii_case(&literal(right)?); + return Some(if operator == "==" { equal } else { !equal }); + } + } + match expanded.trim().to_ascii_lowercase().as_str() { + "true" => Some(true), + "false" => Some(false), + _ => None, + } +} + +fn conditions(values: &[String], properties: &Properties) -> Option { + let mut unknown = false; + for value in values { + match condition(value, properties, 0) { + Some(false) => return Some(false), + Some(true) => (), + None => unknown = true, + } + } + if unknown { None } else { Some(true) } +} + +fn apply( + path: &Path, + documents: &BTreeMap>, + properties: &mut Properties, + active: &mut BTreeSet, + sources: &mut Vec, + conditional: &mut Vec<(String, Located)>, +) { + let Some(Some(doc)) = documents.get(path) else { + return; + }; + if active.len() == 64 || !active.insert(path.to_path_buf()) { + return; + } + if !sources.iter().any(|p| p == path) { + sources.push(path.to_path_buf()); + } + let directory = path.parent().unwrap_or(Path::new("")); + let this_directory = Located { + value: format!("{}/", display_path(directory)), + source: display_path(path), + conditional: false, + }; + let previous = properties.insert("msbuildthisfiledirectory".to_owned(), this_directory); + for step in &doc.steps { + match step { + Step::Property(index) => { + let (name, value, tests) = &doc.properties[*index]; + match conditions(tests, properties) { + Some(true) => { + let mut value = value.clone(); + value.conditional = false; + let value = + values::resolve(&value, properties, Syntax::MsBuild).unwrap_or(value); + properties.insert(name.clone(), value); + } + None => conditional.push((name.clone(), value.clone())), + Some(false) => (), + } + } + Step::Import(index) => { + let (import, tests) = &doc.imports[*index]; + if conditions(tests, properties) != Some(true) { + continue; + } + let raw = Located { + value: import.clone(), + source: display_path(path), + conditional: false, + }; + if let Some(resolved) = values::resolve(&raw, properties, Syntax::MsBuild) + && let Some(target) = values::child_file(directory, &resolved.value) + { + apply(&target, documents, properties, active, sources, conditional); + } + } + } + } + if let Some(previous) = previous { + properties.insert("msbuildthisfiledirectory".to_owned(), previous); + } else { + properties.remove("msbuildthisfiledirectory"); + } + active.remove(path); +} + +pub(super) fn parse(nodes: &mut [Node], diagnostics: &mut Vec) { + if !nodes.iter().any(|node| { + node.files.iter().any(|file| { + file == "global.json" + || file.ends_with(".csproj") + || file.ends_with(".sln") + || file.ends_with(".slnx") + }) + }) { + return; + } + let mut documents = BTreeMap::>::new(); + let mut sdk_versions = BTreeMap::>::new(); + for node in nodes.iter_mut() { + for file in &node.files { + if file == "global.json" { + sdk_versions.insert(node.path.clone(), BTreeMap::new()); + } + if file.ends_with(".csproj") + || file.ends_with(".slnx") + || file.ends_with(".props") + || file.ends_with(".targets") + { + documents.insert(node.path.join(file), None); + } + } + for (file, text) in &node.contents { + let source = relative_file(&node.path, file); + if file == "global.json" { + sdk_versions.insert(node.path.clone(), BTreeMap::new()); + match serde_json::from_str::(text) { + Ok(doc) => { + if let Some(version) = doc.pointer("/sdk/version").and_then(Value::as_str) { + node.versions + .entry("csharp".to_owned()) + .or_default() + .push(Version { + value: version.to_owned(), + source: source.clone(), + kind: "sdk".to_owned(), + }); + } + if let Some(versions) = doc.get("msbuild-sdks").and_then(Value::as_object) { + sdk_versions.insert( + node.path.clone(), + versions + .iter() + .filter_map(|(name, version)| { + version.as_str().map(|value| { + ( + name.to_ascii_lowercase(), + Located { + value: value.to_owned(), + source: source.clone(), + conditional: false, + }, + ) + }) + }) + .collect(), + ); + } + } + Err(error) => diagnostics.push(Diagnostic { + path: source, + message: error.to_string(), + }), + } + } else if let std::collections::btree_map::Entry::Occupied(mut entry) = + documents.entry(node.path.join(file)) + { + match document(text, &source) { + Ok(doc) if file.ends_with(".slnx") == doc.solution => { + entry.insert(Some(doc)); + } + Ok(_) => diagnostics.push(Diagnostic { + path: source, + message: "XML root does not match the MSBuild manifest type".to_owned(), + }), + Err(error) => diagnostics.push(Diagnostic { + path: source, + message: error.to_string(), + }), + } + } + } + } + for node in nodes { + for file in node + .files + .iter() + .filter(|file| file.ends_with(".sln") || file.ends_with(".slnx")) + .cloned() + .collect::>() + { + let projects = if file.ends_with(".slnx") { + documents + .get(&node.path.join(&file)) + .and_then(Option::as_ref) + .map(|d| d.projects.clone()) + .unwrap_or_default() + } else { + node.contents + .get(&file) + .into_iter() + .flat_map(|text| text.lines()) + .filter(|line| line.trim_start().starts_with("Project(")) + .filter_map(|line| line.split_once('=')?.1.split('"').nth(3)) + .map(str::to_owned) + .collect() + }; + let members = projects + .iter() + .filter(|p| p.to_ascii_lowercase().ends_with(".csproj")) + .filter_map(|p| values::child_file(&node.path, p)) + .filter_map(|p| p.parent().map(Path::to_path_buf)) + .filter_map(|p| p.strip_prefix(&node.path).ok().map(display_path)) + .collect::>(); + if !members.is_empty() { + node.names.insert( + Path::new(&file) + .file_stem() + .map_or_else(String::new, |s| s.to_string_lossy().into_owned()), + ); + node.workspaces.push(Workspace { + ecosystem: "dotnet".to_owned(), + source: relative_file(&node.path, &file), + members, + exclude: Vec::new(), + }); + } + } + let projects = node + .files + .iter() + .filter(|file| file.ends_with(".csproj")) + .cloned() + .collect::>(); + for file in projects { + let path = node.path.join(&file); + let source = display_path(&path); + let Some(Some(_)) = documents.get(&path) else { + continue; + }; + let name = Path::new(&file) + .file_stem() + .map_or_else(String::new, |s| s.to_string_lossy().into_owned()); + node.packages.insert("dotnet".to_owned(), source.clone()); + node.names.insert(name.clone()); + let mut properties = Properties::new(); + for (key, value) in [ + ("msbuildprojectname", name.as_str()), + ("assemblyname", name.as_str()), + ] { + properties.insert( + key.to_owned(), + Located { + value: value.to_owned(), + source: source.clone(), + conditional: false, + }, + ); + } + let mut sources = Vec::new(); + let mut conditional = Vec::new(); + properties.insert( + "msbuildprojectextension".to_owned(), + Located { + value: ".csproj".to_owned(), + source: source.clone(), + conditional: false, + }, + ); + if let Some(path) = nearest(&node.path, "Directory.Build.props", &documents) { + apply( + &path, + &documents, + &mut properties, + &mut BTreeSet::new(), + &mut sources, + &mut conditional, + ); + } + apply( + &path, + &documents, + &mut properties, + &mut BTreeSet::new(), + &mut sources, + &mut conditional, + ); + if let Some(path) = nearest(&node.path, "Directory.Build.targets", &documents) { + apply( + &path, + &documents, + &mut properties, + &mut BTreeSet::new(), + &mut sources, + &mut conditional, + ); + } + let mut central = BTreeMap::new(); + if let Some(path) = nearest(&node.path, "Directory.Packages.props", &documents) + && !properties + .get("managepackageversionscentrally") + .is_some_and(|value| value.value.eq_ignore_ascii_case("false")) + { + let mut central_sources = Vec::new(); + apply( + &path, + &documents, + &mut properties, + &mut BTreeSet::new(), + &mut central_sources, + &mut conditional, + ); + if properties + .get("managepackageversionscentrally") + .is_some_and(|value| value.value.eq_ignore_ascii_case("false")) + { + central_sources.clear(); + } + for path in central_sources { + if let Some(Some(doc)) = documents.get(&path) { + for reference in &doc.references { + if reference.kind == "PackageVersion" { + central.insert( + reference.name.to_ascii_lowercase(), + reference.version.as_ref(), + ); + } + } + } + } + } + for (key, kind) in [ + ("targetframework", "target"), + ("targetframeworks", "target"), + ("targetframeworkversion", "target"), + ("langversion", "language"), + ("runtimeframeworkversion", "runtime"), + ] { + if let Some(raw) = properties.get(key) { + for value in raw.value.split(';').filter(|v| !v.trim().is_empty()) { + values::version( + node, + "csharp", + &Located { + value: value.trim().to_owned(), + ..raw.clone() + }, + kind, + &properties, + Syntax::MsBuild, + ); + } + } + } + for (key, raw) in &conditional { + let kind = match key.as_str() { + "targetframework" | "targetframeworks" | "targetframeworkversion" => "target", + "langversion" => "language", + "runtimeframeworkversion" => "runtime", + _ => continue, + }; + values::version(node, "csharp", raw, kind, &properties, Syntax::MsBuild); + } + for location in &sources { + let Some(Some(document)) = documents.get(location) else { + continue; + }; + for (sdk, version) in &document.sdk { + let raw = Located { + value: sdk.clone(), + source: display_path(location), + conditional: false, + }; + let Some(sdk) = values::resolve(&raw, &properties, Syntax::MsBuild) else { + continue; + }; + let sdk = sdk.value.to_ascii_lowercase(); + let version = version.as_ref().map(|v| Located { + value: v.clone(), + source: raw.source.clone(), + conditional: false, + }); + let inherited = node + .path + .ancestors() + .find_map(|p| sdk_versions.get(p)) + .and_then(|v| v.get(&sdk)); + values::dependency( + node, + "dotnet-sdk", + &sdk, + &raw.source, + version.as_ref().or(inherited), + &properties, + Syntax::MsBuild, + ); + } + for reference in &document.references { + let ecosystem = match reference.kind.as_str() { + "PackageReference" => "nuget", + "FrameworkReference" => "dotnet-framework", + "Reference" => "dotnet-reference", + _ => continue, + }; + let name = reference + .name + .split(',') + .next() + .unwrap_or("") + .trim() + .to_ascii_lowercase(); + let version = reference + .version + .as_ref() + .or_else(|| central.get(&name).copied().flatten()); + values::dependency( + node, + ecosystem, + &name, + &display_path(location), + version, + &properties, + Syntax::MsBuild, + ); + } + } + if let Some(raw) = properties + .get("packageid") + .or_else(|| properties.get("assemblyname")) + && let Some(package) = values::resolve(raw, &properties, Syntax::MsBuild) + { + node.names.insert(package.value.clone()); + values::identity( + node, + "nuget", + &package.value.to_ascii_lowercase(), + &source, + properties.get("version"), + &properties, + Syntax::MsBuild, + ); + } + } + } +} diff --git a/src/detect/gradle.rs b/src/detect/gradle.rs new file mode 100644 index 0000000..2f47bb0 --- /dev/null +++ b/src/detect/gradle.rs @@ -0,0 +1,718 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Path, PathBuf}; + +use super::values::{self, Located, Properties, Syntax}; +use super::{Diagnostic, Node, Workspace, display_path, relative_file}; + +#[derive(Debug)] +enum Token<'a> { + Word(&'a str), + Text(String), + Symbol(char), + Break, +} + +fn tokens(text: &str) -> Result>, String> { + let bytes = text.as_bytes(); + let mut result = Vec::new(); + let mut i = 0; + while i < bytes.len() { + if bytes[i] == b'\n' { + result.push(Token::Break); + i += 1; + } else if bytes[i].is_ascii_whitespace() { + i += 1; + } else if bytes[i..].starts_with(b"//") { + while i < bytes.len() && bytes[i] != b'\n' { + i += 1; + } + } else if bytes[i..].starts_with(b"/*") { + i += 2; + let mut depth = 1; + while i < bytes.len() && depth > 0 { + if bytes[i..].starts_with(b"/*") { + depth += 1; + i += 2; + } else if bytes[i..].starts_with(b"*/") { + depth -= 1; + i += 2; + } else { + i += 1; + } + } + if depth != 0 { + return Err("unterminated Gradle comment".to_owned()); + } + } else if bytes[i..].starts_with(b"$/") { + let start = i + 2; + let Some(end) = text[start..].find("/$") else { + return Err("unterminated Gradle dollar-slashy string".to_owned()); + }; + result.push(Token::Text(text[start..start + end].to_owned())); + i = start + end + 2; + } else if bytes[i] == b'/' + && result.last().is_some_and(|t| { + matches!(t, Token::Symbol('~' | '=' | '(' | '[' | ',' | ':' | '!')) + || word(t) == Some("return") + }) + { + i += 1; + let start = i; + while i < bytes.len() && bytes[i] != b'/' { + if bytes[i] == b'\\' { + i += 1; + } + i += 1; + } + if i >= bytes.len() { + return Err("unterminated Gradle slashy string".to_owned()); + } + result.push(Token::Text(text[start..i].to_owned())); + i += 1; + } else if bytes[i] == b'\'' || bytes[i] == b'"' { + let quote = bytes[i]; + let triple = bytes[i..].starts_with(&[quote, quote, quote]); + i += if triple { 3 } else { 1 }; + let start = i; + while i < bytes.len() { + if triple && bytes[i..].starts_with(&[quote, quote, quote]) { + break; + } + if !triple && bytes[i] == quote { + break; + } + if !triple && bytes[i] == b'\\' { + i += 1; + } + i += 1; + } + if i >= bytes.len() { + return Err("unterminated Gradle string".to_owned()); + } + let value = &text[start..i]; + result.push(Token::Text( + value + .replace("\\'", "'") + .replace("\\\"", "\"") + .replace("\\\\", "\\"), + )); + i += if triple { 3 } else { 1 }; + } else if bytes[i].is_ascii_alphanumeric() || matches!(bytes[i], b'_' | b'$') { + let start = i; + while i < bytes.len() + && (bytes[i].is_ascii_alphanumeric() || matches!(bytes[i], b'_' | b'.' | b'$')) + { + i += 1; + } + result.push(Token::Word(&text[start..i])); + } else { + let Some(ch) = text[i..].chars().next() else { + break; + }; + result.push(Token::Symbol(ch)); + i += ch.len_utf8(); + } + } + Ok(result) +} + +fn word<'a>(token: &'a Token<'_>) -> Option<&'a str> { + match token { + Token::Word(word) => Some(word), + _ => None, + } +} + +fn argument(tokens: &[Token<'_>], mut index: usize) -> Option<(String, usize)> { + while matches!(tokens.get(index), Some(Token::Symbol('(' | '=' | ':'))) { + index += 1; + } + match tokens.get(index)? { + Token::Text(text) => Some((text.clone(), index + 1)), + Token::Word(word) if word.chars().all(|c| c.is_ascii_digit() || c == '.') => { + Some(((*word).to_owned(), index + 1)) + } + Token::Word(word) if word.starts_with("JavaVersion.VERSION_") => Some(( + word.trim_start_matches("JavaVersion.VERSION_") + .replace('_', "."), + index + 1, + )), + Token::Word(word) + if word.ends_with("JavaLanguageVersion.of") || *word == "JavaVersion.toVersion" => + { + argument(tokens, index + 1) + } + Token::Word(word) => Some((format!("${{{word}}}"), index + 1)), + _ => None, + } +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Scope { + Other, + All, + Subprojects, + Plugins, + Dependencies, + Management, + Buildscript, + Java, + Extra, +} + +struct Declaration { + value: Located, + shared: bool, + root: bool, +} +struct Plugin { + name: String, + version: Option, + active: bool, + shared: bool, + root: bool, +} +struct Dependency { + value: Located, + shared: bool, + root: bool, +} + +#[derive(Default)] +struct Script { + properties: BTreeMap, + plugins: Vec, + dependencies: Vec, + java: Vec<(String, Declaration)>, + modules: Vec, + directories: BTreeMap, + project_names: BTreeMap, + name: Option, + dynamic_includes: bool, +} + +fn script(text: &str, source: &str) -> Result { + let tokens = tokens(text)?; + let mut script = Script::default(); + let mut scopes = Vec::new(); + let mut lists = BTreeMap::>::new(); + for (index, token) in tokens.iter().enumerate() { + if matches!(token, Token::Symbol('{')) { + let previous = tokens[..index] + .iter() + .rev() + .take(32) + .take_while(|t| !matches!(t, Token::Break | Token::Symbol('{' | '}' | ';'))) + .filter_map(word) + .collect::>(); + let scope = if previous.contains(&"allprojects") { + Scope::All + } else if previous.contains(&"subprojects") { + Scope::Subprojects + } else if previous.contains(&"pluginManagement") + || previous.contains(&"dependencyManagement") + { + Scope::Management + } else if previous.contains(&"buildscript") { + Scope::Buildscript + } else if previous.contains(&"plugins") { + Scope::Plugins + } else if previous.contains(&"dependencies") { + Scope::Dependencies + } else if previous + .iter() + .any(|w| matches!(*w, "java" | "toolchain" | "JavaCompile")) + { + Scope::Java + } else if previous.contains(&"ext") { + Scope::Extra + } else { + Scope::Other + }; + scopes.push(scope); + continue; + } + if matches!(token, Token::Symbol('}')) { + scopes.pop(); + continue; + } + let Some(key) = word(token) else { + continue; + }; + let shared = scopes + .iter() + .any(|s| matches!(s, Scope::All | Scope::Subprojects)); + let root = !scopes.contains(&Scope::Subprojects); + let located = |value| Located { + value, + source: source.to_owned(), + conditional: false, + }; + let scalar_scope = scopes.is_empty() + || scopes.last().is_some_and(|s| { + matches!( + s, + Scope::All | Scope::Subprojects | Scope::Extra | Scope::Java + ) + }); + if matches!(tokens.get(index + 1), Some(Token::Symbol('='))) && scalar_scope { + if matches!(tokens.get(index + 2), Some(Token::Symbol('['))) { + let list = tokens[index + 3..] + .iter() + .take_while(|t| !matches!(t, Token::Symbol(']'))) + .filter_map(|t| { + if let Token::Text(v) = t { + Some(v.clone()) + } else { + None + } + }) + .collect(); + lists.insert(key.to_owned(), list); + } else if let Some((value, _)) = argument(&tokens, index + 1) { + if key == "rootProject.name" { + script.name = Some(value); + } else { + let declaration = Declaration { + value: located(value), + shared, + root, + }; + if matches!( + key, + "sourceCompatibility" + | "targetCompatibility" + | "languageVersion" + | "options.release" + ) { + let kind = if key == "languageVersion" { + "toolchain" + } else if key == "sourceCompatibility" { + "source" + } else if key == "targetCompatibility" { + "target" + } else { + "release" + }; + script.java.push((kind.to_owned(), declaration)); + } else { + script.properties.insert(key.to_owned(), declaration); + } + } + } + } + if matches!(key, "languageVersion.set" | "options.release.set") + && let Some((value, _)) = argument(&tokens, index + 1) + { + script.java.push(( + if key == "languageVersion.set" { + "toolchain" + } else { + "release" + } + .to_owned(), + Declaration { + value: located(value), + shared, + root, + }, + )); + } + if key == "include" || key == "includeBuild" { + let first = if matches!(tokens.get(index + 1), Some(Token::Symbol('('))) { + index + 2 + } else { + index + 1 + }; + if let Some(name) = tokens + .get(first) + .and_then(word) + .map(|name| name.strip_suffix(".toArray").unwrap_or(name)) + && let Some(list) = lists.get(name) + { + script.modules.extend( + list.iter() + .map(|v| v.trim_start_matches(':').replace(':', "/")), + ); + continue; + } + let mut cursor = index + 1; + while let Some((value, next)) = argument(&tokens, cursor) { + if value.contains('$') { + script.dynamic_includes = true; + break; + } + script + .modules + .push(value.trim_start_matches(':').replace(':', "/")); + cursor = next; + if matches!(tokens.get(cursor), Some(Token::Symbol(','))) { + cursor += 1; + } else { + break; + } + } + } + if (key.ends_with(".each") || key.ends_with(".forEach")) + && let Some(list) = + lists.get(key.trim_end_matches(".each").trim_end_matches(".forEach")) + && tokens[index + 1..] + .iter() + .take(40) + .filter_map(word) + .any(|w| w == "include") + { + script.modules.extend( + list.iter() + .map(|v| v.trim_start_matches(':').replace(':', "/")), + ); + } + if key == "project" + && let Some((project, end)) = argument(&tokens, index + 1) + { + let tail = &tokens[end..tokens.len().min(end + 12)]; + if let Some(name) = tail.iter().position(|t| word(t) == Some("name")) + && let Some((value, _)) = argument(tail, name + 1) + && !value.contains('$') + { + script + .project_names + .insert(project.trim_start_matches(':').replace(':', "/"), value); + } + if tail + .iter() + .filter_map(word) + .any(|w| w == ".projectDir" || w == "projectDir") + && let Some(file) = tail.iter().position(|t| word(t) == Some("file")) + && let Some((path, _)) = argument(tail, file + 1) + { + script + .directories + .insert(project.trim_start_matches(':').replace(':', "/"), path); + } + } + let managed = scopes.contains(&Scope::Management) || scopes.contains(&Scope::Buildscript); + if key == "id" + && scopes.last() == Some(&Scope::Plugins) + && let Some((name, end)) = argument(&tokens, index + 1) + { + let tail = &tokens[end..tokens.len().min(end + 8)]; + let version = tail + .iter() + .position(|t| word(t) == Some("version")) + .and_then(|i| argument(tail, i + 1)) + .map(|(value, _)| located(value)); + let inactive = tail + .windows(2) + .any(|w| word(&w[0]) == Some("apply") && word(&w[1]) == Some("false")); + script.plugins.push(Plugin { + name, + version, + active: !managed && !inactive, + shared, + root, + }); + } + if key == "apply" && !managed { + let mut cursor = index + 1; + if matches!(tokens.get(cursor), Some(Token::Symbol('('))) { + cursor += 1; + } + if word(tokens.get(cursor).unwrap_or(token)) == Some("plugin") + && let Some((name, _)) = argument(&tokens, cursor + 1) + { + script.plugins.push(Plugin { + name, + version: None, + active: true, + shared, + root, + }); + } + } + if scopes.contains(&Scope::Dependencies) + && !managed + && matches!( + key, + "implementation" + | "api" + | "compile" + | "compileOnly" + | "runtimeOnly" + | "runtime" + | "annotationProcessor" + ) + && let Some((value, _)) = argument(&tokens, index + 1) + { + script.dependencies.push(Dependency { + value: located(value), + shared, + root, + }); + } + } + Ok(script) +} + +fn build_file(path: &Path) -> bool { + let Some(file) = path.file_name().and_then(|f| f.to_str()) else { + return false; + }; + let directory = path + .parent() + .and_then(Path::file_name) + .and_then(|f| f.to_str()) + .unwrap_or(""); + matches!(file, "build.gradle" | "build.gradle.kts") + || file == format!("{directory}.gradle") + || file == format!("{directory}.gradle.kts") +} + +pub(super) fn parse(nodes: &mut Vec, diagnostics: &mut Vec) { + let mut scripts = BTreeMap::::new(); + let mut property_files = BTreeMap::::new(); + let mut daemon_versions = BTreeMap::::new(); + for node in nodes.iter() { + for (file, text) in &node.contents { + let source = relative_file(&node.path, file); + if file.ends_with(".gradle") || file.ends_with(".gradle.kts") { + match script(text, &source) { + Ok(script) => { + if matches!(file.as_str(), "settings.gradle" | "settings.gradle.kts") + && script.dynamic_includes + { + diagnostics.push(Diagnostic { path: source.clone(), message: "dynamic Gradle include declarations are not evaluated; projects with manifests are still discovered".to_owned() }); + } + scripts.insert(node.path.join(file), script); + } + Err(message) => diagnostics.push(Diagnostic { + path: source, + message, + }), + } + } else if file == "gradle-daemon-jvm.properties" { + if let Some(version) = values::properties(text, &source).get("toolchainVersion") { + daemon_versions.insert( + node.path.parent().unwrap_or(Path::new("")).to_path_buf(), + version.clone(), + ); + } + } else if file == "gradle.properties" { + property_files.insert(node.path.clone(), values::properties(text, &source)); + } + } + } + if scripts.is_empty() { + return; + } + let settings = scripts + .iter() + .filter(|(path, _)| { + path.file_name() + .is_some_and(|f| f == "settings.gradle" || f == "settings.gradle.kts") + }) + .map(|(path, script)| { + ( + path.parent().unwrap_or(Path::new("")).to_path_buf(), + (path, script), + ) + }) + .collect::>(); + let mut builds = BTreeMap::>::new(); + for (path, script) in &scripts { + if build_file(path) { + builds + .entry(path.parent().unwrap_or(Path::new("")).to_path_buf()) + .or_default() + .push((path, script)); + } + } + let mut declared = BTreeMap::::new(); + let mut declared_names = BTreeMap::::new(); + for (path, script) in settings.values() { + let base = path.parent().unwrap_or(Path::new("")); + for module in script.modules.iter().chain(script.project_names.keys()) { + let directory = script.directories.get(module).unwrap_or(module); + if let Some(directory) = values::child_file(base, directory) { + let name = script + .project_names + .get(module) + .cloned() + .unwrap_or_else(|| module.rsplit('/').next().unwrap_or(module).to_owned()); + declared_names.insert(directory.clone(), name); + if script.modules.contains(module) { + declared.insert(directory, display_path(path)); + } + } + } + } + let existing = nodes + .iter() + .map(|n| n.path.clone()) + .collect::>(); + let synthetic = declared + .keys() + .filter(|path| { + !existing.contains(*path) + && existing + .range((*path).clone()..) + .next() + .is_some_and(|candidate| candidate.starts_with(path)) + }) + .cloned() + .collect::>(); + for path in synthetic { + nodes.push(Node { + path, + ..Node::default() + }); + } + for node in nodes.iter_mut() { + let build = builds.get(&node.path).map(Vec::as_slice).unwrap_or(&[]); + let settings_here = settings.get(&node.path); + let module_source = declared.get(&node.path); + if build.is_empty() && settings_here.is_none() && module_source.is_none() { + continue; + } + let source = build + .first() + .map(|(path, _)| display_path(path)) + .or_else(|| module_source.cloned()) + .or_else(|| settings_here.map(|(path, _)| display_path(path))) + .unwrap_or_default(); + node.packages.insert("gradle".to_owned(), source.clone()); + let root = node + .path + .ancestors() + .find_map(|parent| settings.get(parent)); + let base = root + .map(|(path, _)| path.parent().unwrap_or(Path::new(""))) + .map(Path::to_path_buf) + .unwrap_or_else(|| node.path.clone()); + if let Some(raw) = daemon_versions.get(&base) { + values::version( + node, + "java", + raw, + "build-toolchain", + &Properties::new(), + Syntax::Maven, + ); + } + let mut properties = Properties::new(); + for parent in node.path.ancestors().collect::>().into_iter().rev() { + if let Some(found) = property_files.get(parent) { + properties.extend(found.iter().map(|(k, v)| (k.clone(), v.clone()))); + } + } + let root_build = builds.get(&base).map(Vec::as_slice).unwrap_or(&[]); + let mut declarations = root_build + .iter() + .map(|(path, script)| (*path, *script, base != node.path)) + .collect::>(); + if base != node.path { + declarations.extend(build.iter().map(|(path, script)| (*path, *script, false))); + } + for (_, script, inherited) in &declarations { + for (name, declaration) in &script.properties { + if (!*inherited && declaration.root) || (*inherited && declaration.shared) { + properties.insert(name.clone(), declaration.value.clone()); + } + } + } + let name = settings_here + .and_then(|(_, script)| script.name.clone()) + .or_else(|| declared_names.get(&node.path).cloned()) + .or_else(|| { + node.path + .file_name() + .map(|s| s.to_string_lossy().into_owned()) + }); + if let Some(name) = &name { + node.names.insert(name.clone()); + } + for (path, script, inherited) in declarations { + for (kind, declaration) in &script.java { + if (!inherited && declaration.root) || (inherited && declaration.shared) { + values::version( + node, + "java", + &declaration.value, + kind, + &properties, + Syntax::Maven, + ); + } + } + for plugin in &script.plugins { + if plugin.active && ((!inherited && plugin.root) || (inherited && plugin.shared)) { + values::dependency( + node, + "gradle-plugin", + &plugin.name, + &display_path(path), + plugin.version.as_ref(), + &properties, + Syntax::Maven, + ); + } + } + for dependency in &script.dependencies { + if !((!inherited && dependency.root) || (inherited && dependency.shared)) { + continue; + } + let Some(coordinate) = + values::resolve(&dependency.value, &properties, Syntax::Maven) + else { + continue; + }; + let parts = coordinate.value.split(':').collect::>(); + if parts.len() < 2 || parts[0].is_empty() || parts[1].is_empty() { + continue; + } + let version = parts.get(2).map(|version| Located { + value: (*version).to_owned(), + source: coordinate.source.clone(), + conditional: false, + }); + values::dependency( + node, + "maven", + &format!("{}:{}", parts[0], parts[1]), + &display_path(path), + version.as_ref(), + &properties, + Syntax::Maven, + ); + } + } + if let Some(name) = name + && let Some(group) = properties + .get("group") + .and_then(|v| values::resolve(v, &properties, Syntax::Maven)) + { + values::identity( + node, + "maven", + &format!("{}:{name}", group.value), + &source, + properties.get("version"), + &properties, + Syntax::Maven, + ); + } + if let Some((path, settings)) = settings_here { + let members = settings + .modules + .iter() + .map(|module| settings.directories.get(module).unwrap_or(module).clone()) + .collect(); + node.workspaces.push(Workspace { + ecosystem: "gradle".to_owned(), + source: display_path(path), + members, + exclude: Vec::new(), + }); + } + } +} diff --git a/src/detect/java.rs b/src/detect/java.rs new file mode 100644 index 0000000..05b5c11 --- /dev/null +++ b/src/detect/java.rs @@ -0,0 +1,359 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Path, PathBuf}; + +use anyhow::{Result, bail}; + +use super::values::{self, Located, Properties, Syntax}; +use super::{Diagnostic, Node, Workspace, display_path, relative_file, xml}; + +#[derive(Default)] +struct Artifact { + group: String, + name: String, + version: Option, + scope: String, + kind: String, + compiler: Vec<(String, Located)>, +} + +#[derive(Default)] +struct Pom { + group: String, + name: String, + version: String, + parent: Option, + parent_path: Option, + properties: Properties, + dependencies: Vec, + managed: Vec, + plugins: Vec, + modules: Vec, +} + +fn pom(text: &str, source: &str) -> Result { + let mut doc = Pom::default(); + let mut artifacts = BTreeMap::::new(); + xml::read(text, |stack| { + if stack.first().is_none_or(|e| e.name != "project") { + bail!("expected a Maven project root"); + } + let Some(element) = stack.last() else { + return Ok(()); + }; + let is = |path: &[&str]| stack.iter().map(|e| e.name).eq(path.iter().copied()); + let located = || Located { + value: element.text.trim().to_owned(), + source: source.to_owned(), + conditional: false, + }; + if stack.len() == 2 { + match element.name { + "groupId" => doc.group = element.text.trim().to_owned(), + "artifactId" => doc.name = element.text.trim().to_owned(), + "version" => doc.version = element.text.trim().to_owned(), + _ => (), + } + } + if stack.len() == 3 && stack[1].name == "parent" { + let parent = doc.parent.get_or_insert_with(Artifact::default); + match element.name { + "groupId" => parent.group = element.text.trim().to_owned(), + "artifactId" => parent.name = element.text.trim().to_owned(), + "version" => parent.version = Some(located()), + "relativePath" => doc.parent_path = Some(element.text.trim().to_owned()), + _ => (), + } + } + if stack.len() == 3 && stack[1].name == "properties" { + doc.properties.insert(element.name.to_owned(), located()); + } + if is(&["project", "modules", "module"]) { + doc.modules.push(element.text.trim().to_owned()); + } + if let Some(parent) = stack.get(stack.len().saturating_sub(2)) + && matches!(parent.name, "dependency" | "plugin") + { + let artifact = artifacts.entry(stack.len() - 1).or_default(); + match element.name { + "groupId" => artifact.group = element.text.trim().to_owned(), + "artifactId" => artifact.name = element.text.trim().to_owned(), + "version" => artifact.version = Some(located()), + "scope" => artifact.scope = element.text.trim().to_owned(), + "type" => artifact.kind = element.text.trim().to_owned(), + _ => (), + } + } + if let Some(plugin) = stack.iter().position(|e| e.name == "plugin") + && matches!(element.name, "release" | "source" | "target") + && stack + .get(stack.len().saturating_sub(2)) + .is_some_and(|e| e.name == "configuration") + { + artifacts + .entry(plugin + 1) + .or_default() + .compiler + .push((element.name.to_owned(), located())); + } + if element.name == "dependency" { + let artifact = artifacts.remove(&stack.len()).unwrap_or_default(); + if is(&["project", "dependencies", "dependency"]) && artifact.scope != "test" { + doc.dependencies.push(artifact); + } else if is(&[ + "project", + "dependencyManagement", + "dependencies", + "dependency", + ]) { + doc.managed.push(artifact); + } + } + if element.name == "plugin" { + let mut artifact = artifacts.remove(&stack.len()).unwrap_or_default(); + if artifact.group.is_empty() { + artifact.group = "org.apache.maven.plugins".to_owned(); + } + if is(&["project", "build", "plugins", "plugin"]) { + doc.plugins.push(artifact); + } + } + Ok(()) + })?; + if doc.name.is_empty() { + bail!("Maven project artifactId is missing"); + } + Ok(doc) +} + +fn chain<'a>( + path: &Path, + documents: &'a BTreeMap>, + seen: &mut BTreeSet, + output: &mut Vec<(PathBuf, &'a Pom)>, +) { + let Some(Some(doc)) = documents.get(path) else { + return; + }; + if seen.len() == 64 || !seen.insert(path.to_path_buf()) { + return; + } + if let Some(parent) = &doc.parent { + let relative = doc.parent_path.as_deref().unwrap_or("../pom.xml"); + if !relative.is_empty() + && let Some(target) = + values::child_file(path.parent().unwrap_or(Path::new("")), relative) + && let Some(Some(local)) = documents.get(&target) + && local.name == parent.name + && (if local.group.is_empty() { + local.parent.as_ref().map(|p| p.group.as_str()) + } else { + Some(local.group.as_str()) + }) == Some(parent.group.as_str()) + && parent.version.as_ref().is_none_or(|requested| { + let actual = if local.version.is_empty() { + local + .parent + .as_ref() + .and_then(|p| p.version.as_ref()) + .map(|v| v.value.as_str()) + } else { + Some(local.version.as_str()) + }; + actual == Some(requested.value.as_str()) + }) + { + chain(&target, documents, seen, output); + } + } + output.push((path.to_path_buf(), doc)); +} + +fn literal(value: &str, source: &str) -> Located { + Located { + value: value.to_owned(), + source: source.to_owned(), + conditional: false, + } +} + +fn resolved(value: &str, source: &str, properties: &Properties) -> Option { + values::resolve(&literal(value, source), properties, Syntax::Maven).map(|v| v.value) +} + +pub(super) fn parse(nodes: &mut Vec, diagnostics: &mut Vec) { + let mut documents = BTreeMap::>::new(); + for node in nodes.iter() { + if node.files.contains("pom.xml") { + documents.insert(node.path.join("pom.xml"), None); + } + if let Some(text) = node.contents.get("pom.xml") { + let source = relative_file(&node.path, "pom.xml"); + match pom(text, &source) { + Ok(doc) => { + documents.insert(node.path.join("pom.xml"), Some(doc)); + } + Err(error) => diagnostics.push(Diagnostic { + path: source, + message: error.to_string(), + }), + } + } + } + if documents.is_empty() { + super::gradle::parse(nodes, diagnostics); + return; + } + for node in nodes.iter_mut() { + let path = node.path.join("pom.xml"); + let Some(Some(doc)) = documents.get(&path) else { + continue; + }; + node.packages + .insert("maven".to_owned(), display_path(&path)); + let mut inherited = Vec::new(); + chain(&path, &documents, &mut BTreeSet::new(), &mut inherited); + let mut properties = Properties::new(); + let mut managed = BTreeMap::new(); + for (path, parent) in &inherited { + let source = display_path(path); + properties.extend( + parent + .properties + .iter() + .map(|(k, v)| (k.clone(), v.clone())), + ); + let group = if parent.group.is_empty() { + parent.parent.as_ref().map_or("", |p| p.group.as_str()) + } else { + &parent.group + }; + let version = if parent.version.is_empty() { + parent + .parent + .as_ref() + .and_then(|p| p.version.as_ref()) + .map_or("", |v| v.value.as_str()) + } else { + &parent.version + }; + for (key, value) in [ + ("project.groupId", group), + ("pom.groupId", group), + ("project.artifactId", parent.name.as_str()), + ("pom.artifactId", parent.name.as_str()), + ("project.version", version), + ("pom.version", version), + ] { + properties.insert(key.to_owned(), literal(value, &source)); + } + for dependency in &parent.managed { + if let (Some(group), Some(name)) = ( + resolved(&dependency.group, &source, &properties), + resolved(&dependency.name, &source, &properties), + ) { + managed.insert(format!("{group}:{name}"), dependency.version.as_ref()); + } + } + } + let source = display_path(&path); + let group = if doc.group.is_empty() { + doc.parent.as_ref().map_or("", |p| p.group.as_str()) + } else { + &doc.group + }; + let group = resolved(group, &source, &properties).unwrap_or_else(|| group.to_owned()); + node.names.insert(doc.name.clone()); + if !group.is_empty() { + node.names.insert(format!("{group}:{}", doc.name)); + let version = if doc.version.is_empty() { + doc.parent + .as_ref() + .and_then(|p| p.version.as_ref()) + .cloned() + } else { + Some(literal(&doc.version, &source)) + }; + values::identity( + node, + "maven", + &format!("{group}:{}", doc.name), + &source, + version.as_ref(), + &properties, + Syntax::Maven, + ); + } + for (key, kind) in [ + ("java.version", "requirement"), + ("maven.compiler.release", "release"), + ("maven.compiler.source", "source"), + ("maven.compiler.target", "target"), + ] { + if let Some(raw) = properties.get(key) { + values::version(node, "java", raw, kind, &properties, Syntax::Maven); + } + } + for (path, parent) in &inherited { + let source = display_path(path); + if let Some(parent) = &parent.parent { + values::dependency( + node, + "maven-parent", + &format!("{}:{}", parent.group, parent.name), + &source, + parent.version.as_ref(), + &properties, + Syntax::Maven, + ); + } + for (ecosystem, dependencies) in [ + ("maven", &parent.dependencies), + ("maven-plugin", &parent.plugins), + ] { + for artifact in dependencies { + let (Some(group), Some(name)) = ( + resolved(&artifact.group, &source, &properties), + resolved(&artifact.name, &source, &properties), + ) else { + continue; + }; + let key = format!("{group}:{name}"); + let version = artifact + .version + .as_ref() + .or_else(|| managed.get(&key).copied().flatten()); + values::dependency( + node, + ecosystem, + &key, + &source, + version, + &properties, + Syntax::Maven, + ); + if artifact.name == "maven-compiler-plugin" { + for (kind, raw) in &artifact.compiler { + values::version(node, "java", raw, kind, &properties, Syntax::Maven); + } + } + } + } + } + if !doc.modules.is_empty() { + let members = doc + .modules + .iter() + .filter_map(|module| resolved(module, &source, &properties)) + .filter_map(|module| values::child_file(Path::new(""), &module)) + .map(|path| display_path(&path)) + .collect(); + node.workspaces.push(Workspace { + ecosystem: "maven".to_owned(), + source, + members, + exclude: Vec::new(), + }); + } + } + super::gradle::parse(nodes, diagnostics); +} diff --git a/src/detect/jvm_dotnet_tests.rs b/src/detect/jvm_dotnet_tests.rs new file mode 100644 index 0000000..c1b9992 --- /dev/null +++ b/src/detect/jvm_dotnet_tests.rs @@ -0,0 +1,564 @@ +use super::tests::{detects, fixture, project, version}; +use super::{FrameworkRole, scan}; + +#[test] +fn bare_java_and_csharp_sources_are_never_read() { + for (file, language) in [ + ("Main.java", "java"), + ("Program.cs", "csharp"), + ("script.csx", "csharp"), + ] { + let dir = fixture(&[(file, "not valid source")]); + let report = scan(dir.path(), None).unwrap(); + assert_eq!(report.stats.files_read, 0); + assert_eq!(report.projects[0].languages[0].name, language); + assert!(report.projects[0].languages[0].versions.is_empty()); + } +} + +#[test] +fn dotnet_web_targets_language_and_sdk_versions_are_distinct() { + let dir = fixture(&[ + ( + "global.json", + r#"{"sdk":{"version":"10.0.403","rollForward":"latestFeature"}}"#, + ), + ( + "Directory.Build.props", + "net10.014.0", + ), + ( + "src/Web/Web.csproj", + "Shop.Web", + ), + ("src/Web/Program.cs", "source must not be read"), + ]); + let report = scan(dir.path(), None).unwrap(); + assert!(report.diagnostics.is_empty(), "{report:#?}"); + let web = project(&report, "src/Web"); + assert!(detects(web, "aspnetcore")); + assert!(version(web, "csharp", "net10.0", "Directory.Build.props")); + assert!(version(web, "csharp", "14.0", "Directory.Build.props")); + assert!(version(web, "csharp", "10.0.403", "global.json")); + let versions = &web + .languages + .iter() + .find(|l| l.name == "csharp") + .unwrap() + .versions; + assert!( + versions + .iter() + .any(|v| v.value == "net10.0" && v.kind == "target") + ); + assert!( + versions + .iter() + .any(|v| v.value == "14.0" && v.kind == "language") + ); + assert!( + versions + .iter() + .any(|v| v.value == "10.0.403" && v.kind == "sdk") + ); + assert_eq!(report.stats.files_read, 3); +} + +#[test] +fn msbuild_import_order_defaults_and_central_package_versions() { + let dir = fixture(&[ + ( + "Directory.Build.props", + "net10.014.0", + ), + ( + "eng/base.props", + "13.0", + ), + ( + "Directory.Packages.props", + "8.3.0", + ), + ( + "src/Web/Web.csproj", + "2.0", + ), + ]); + let report = scan(dir.path(), None).unwrap(); + assert!(report.diagnostics.is_empty(), "{report:#?}"); + let web = project(&report, "src/Web"); + assert!(version(web, "csharp", "net10.0", "Directory.Build.props")); + assert!(version(web, "csharp", "14.0", "Directory.Build.props")); + assert!(!version(web, "csharp", "13.0", "eng/base.props")); + let detection = web + .detections + .iter() + .find(|d| d.id == "fastendpoints") + .unwrap(); + assert_eq!(detection.roles, [FrameworkRole::Uses]); + assert!( + detection + .evidence + .iter() + .any(|e| e.source == "Directory.Packages.props" && e.value.as_deref() == Some("8.3.0")) + ); + assert!( + detection + .evidence + .iter() + .any(|e| e.source == "src/Web/Web.csproj") + ); +} + +#[test] +fn dotnet_solutions_find_members_without_counting_solution_folders() { + for (file, content) in [ + ( + "App.sln", + "Project(\"TYPE\") = \"Web, API\", \"src\\Web\\Web.csproj\", \"ID\"\nEndProject\nProject(\"FOLDER\") = \"src\", \"src\", \"ID2\"\nEndProject\n", + ), + ( + "App.slnx", + "", + ), + ] { + let dir = fixture(&[ + (file, content), + ( + "src/Web/Web.csproj", + "", + ), + ]); + let report = scan(dir.path(), None).unwrap(); + assert!(report.diagnostics.is_empty(), "{report:#?}"); + assert!(detects(project(&report, "."), "dotnet-workspace")); + assert_eq!(project(&report, ".").workspace_members, ["src/Web"]); + } +} + +#[test] +fn dotnet_target_frameworks_do_not_invent_a_csharp_language_version() { + let dir = fixture(&[( + "App.csproj", + "net8.0;net10.0", + )]); + let report = scan(dir.path(), None).unwrap(); + let p = project(&report, "."); + assert!(version(p, "csharp", "net8.0", "App.csproj")); + assert!(version(p, "csharp", "net10.0", "App.csproj")); + assert!(p.languages[0].versions.iter().all(|v| v.kind != "language")); + assert!(!detects(p, "aspnetcore")); +} + +#[test] +fn msbuild_unknown_conditions_and_property_cycles_are_not_executed() { + let dir = fixture(&[( + "App.csproj", + "$(B)$(A)$(A)14.0", + )]); + let report = scan(dir.path(), None).unwrap(); + let p = project(&report, "."); + assert!( + p.languages[0] + .versions + .iter() + .any(|v| v.kind == "reference") + ); + assert!( + p.languages[0] + .versions + .iter() + .any(|v| v.kind == "conditional-language") + ); + assert!(p.languages[0].versions.iter().all(|v| v.kind != "target")); +} + +#[test] +fn maven_spring_boot_parent_and_java_release_are_recognized() { + let dir = fixture(&[( + "pom.xml", + "org.springframework.bootspring-boot-starter-parent4.1.0petclinic17${java.version}", + )]); + let report = scan(dir.path(), None).unwrap(); + assert!(report.diagnostics.is_empty(), "{report:#?}"); + let p = project(&report, "."); + assert!(detects(p, "maven")); + assert!(detects(p, "spring-boot")); + assert!(version(p, "java", "17", "pom.xml")); + assert!( + p.languages[0] + .versions + .iter() + .any(|v| v.value == "17" && v.kind == "release") + ); +} + +#[test] +fn maven_modules_inherit_local_properties_and_managed_versions() { + let dir = fixture(&[ + ( + "pom.xml", + "exampleplatform1214.0.0apiorg.springframework.bootspring-boot-starter-web${boot.version}", + ), + ( + "api/pom.xml", + "exampleplatform1apiorg.springframework.bootspring-boot-starter-web", + ), + ]); + let report = scan(dir.path(), None).unwrap(); + assert!(report.diagnostics.is_empty(), "{report:#?}"); + assert_eq!(project(&report, ".").workspace_members, ["api"]); + assert!(!detects(project(&report, "."), "spring-boot")); + let api = project(&report, "api"); + assert!(detects(api, "spring-boot")); + assert!(version(api, "java", "21", "pom.xml")); + let boot = api + .detections + .iter() + .find(|d| d.id == "spring-boot") + .unwrap(); + assert!( + boot.evidence + .iter() + .any(|e| e.source == "pom.xml" && e.value.as_deref() == Some("4.0.0")) + ); +} + +#[test] +fn maven_test_dependencies_and_plugin_management_are_not_runtime_frameworks() { + let dir = fixture(&[( + "pom.xml", + "plainorg.springframework.bootspring-boot-starter-webtestorg.springframework.bootspring-boot-maven-plugin", + )]); + let report = scan(dir.path(), None).unwrap(); + assert!(!detects(project(&report, "."), "spring-boot")); +} + +#[test] +fn xml_entities_cdata_namespaces_and_structure_are_checked() { + let dir = fixture(&[( + "App.csproj", + "net10.0A&B", + )]); + let report = scan(dir.path(), None).unwrap(); + assert!(report.diagnostics.is_empty(), "{report:#?}"); + assert!(version( + project(&report, "."), + "csharp", + "net10.0", + "App.csproj" + )); + assert!(project(&report, ".").names.contains(&"A&B".to_owned())); + for broken in [ + "", + "", + "", + "]>&x;", + ] { + let dir = fixture(&[("App.csproj", broken)]); + let report = scan(dir.path(), None).unwrap(); + assert_eq!(report.diagnostics.len(), 1, "{broken}: {report:#?}"); + assert_eq!(report.stats.files_read, 1); + } +} + +#[test] +fn gradle_groovy_and_kotlin_plugins_and_toolchains_are_static_facts() { + for (file, content) in [ + ( + "build.gradle", + "plugins { id 'org.springframework.boot' version '4.0.0' }\ngroup = 'example'\njava { toolchain { languageVersion = JavaLanguageVersion.of(21) } }\ndependencies { implementation 'org.springframework:spring-web:7.0.0' }\n", + ), + ( + "build.gradle.kts", + "plugins { id(\"org.springframework.boot\") version \"4.0.0\" }\ngroup = \"example\"\njava { toolchain { languageVersion.set(JavaLanguageVersion.of(21)) } }\ndependencies { implementation(\"org.springframework:spring-web:7.0.0\") }\n", + ), + ] { + let dir = fixture(&[(file, content)]); + let report = scan(dir.path(), None).unwrap(); + assert!(report.diagnostics.is_empty(), "{file}: {report:#?}"); + let p = project(&report, "."); + assert!(detects(p, "spring-boot")); + assert!(detects(p, "spring")); + assert!(version(p, "java", "21", file)); + } +} + +#[test] +fn gradle_inactive_plugins_comments_and_test_dependencies_do_not_match() { + let dir = fixture(&[( + "build.gradle", + "plugins { id 'org.springframework.boot' version '4.0.0' apply false }\n// plugins { id 'org.springframework.boot' }\ndependencies { testImplementation 'org.springframework:spring-web:7.0.0' }\ndef description = \"org.springframework.boot\"\n", + )]); + let report = scan(dir.path(), None).unwrap(); + assert!(!detects(project(&report, "."), "spring-boot")); + assert!(!detects(project(&report, "."), "spring")); +} + +#[test] +fn gradle_settings_literal_lists_and_source_only_modules_form_workspaces() { + let dir = fixture(&[ + ( + "settings.gradle", + "rootProject.name='platform'\nList projects = ['api', 'worker']\ninclude projects.toArray(new String[0])\n", + ), + ("build.gradle", "allprojects { group = 'example' }\n"), + ("api/src/main/java/Main.java", "not read"), + ("worker/build.gradle", "plugins { id 'java' }\n"), + ]); + let report = scan(dir.path(), None).unwrap(); + assert!(report.diagnostics.is_empty(), "{report:#?}"); + assert_eq!(project(&report, ".").workspace_members, ["api", "worker"]); + assert!(detects(project(&report, "api"), "gradle")); + assert!( + report + .projects + .iter() + .all(|p| p.path != "api/src/main/java") + ); +} + +#[test] +fn gradle_regex_literals_do_not_create_fake_strings_or_plugins() { + let dir = fixture(&[( + "build.gradle", + r#"plugins { id 'java' } +def pattern = /"preemptedAt"\s*:\s*"([^"]+)"/ +def other = $/"an "embedded" quote"/$ +"#, + )]); + let report = scan(dir.path(), None).unwrap(); + assert!(report.diagnostics.is_empty(), "{report:#?}"); + assert!(detects(project(&report, "."), "gradle")); +} + +#[test] +fn mixed_java_csharp_monorepo_keeps_runtime_versions_in_the_right_services() { + let dir = fixture(&[ + ( + "services/java/pom.xml", + "api21", + ), + ( + "services/dotnet/global.json", + r#"{"sdk":{"version":"10.0.403"}}"#, + ), + ( + "services/dotnet/Web.csproj", + "net10.0", + ), + ("services/dotnet/Program.cs", "not read"), + ( + "services/frontend/package.json", + r#"{"dependencies":{"next":"16"},"engines":{"node":">=24"}}"#, + ), + ]); + let report = scan(dir.path(), None).unwrap(); + assert_eq!(report.projects.len(), 3); + assert!(version( + project(&report, "services/java"), + "java", + "21", + "services/java/pom.xml" + )); + assert!( + project(&report, "services/java") + .languages + .iter() + .all(|l| l.name != "csharp") + ); + assert!(version( + project(&report, "services/dotnet"), + "csharp", + "10.0.403", + "services/dotnet/global.json" + )); + assert!( + project(&report, "services/frontend") + .languages + .iter() + .all(|l| l.name != "java" && l.name != "csharp") + ); +} + +#[test] +fn closest_global_json_selects_the_sdk_without_inheriting_sdk_package_catalogs() { + let dir = fixture(&[ + ( + "global.json", + r#"{"sdk":{"version":"10.0.100"},"msbuild-sdks":{"Aspire.AppHost.Sdk":"13.0.0"}}"#, + ), + ("nested/global.json", r#"{"sdk":{"version":"11.0.100"}}"#), + ( + "nested/App.csproj", + "net11.0", + ), + ]); + let report = scan(dir.path(), None).unwrap(); + let app = project(&report, "nested"); + assert!(version(app, "csharp", "11.0.100", "nested/global.json")); + assert!(!version(app, "csharp", "10.0.100", "global.json")); + let aspire = app.detections.iter().find(|d| d.id == "aspire").unwrap(); + assert!(aspire.evidence.iter().all(|e| e.value.is_none())); +} + +#[test] +fn java_pin_and_gradle_daemon_jdk_are_not_reported_as_target_release() { + let dir = fixture(&[ + ( + "build.gradle", + "plugins { id 'java' }\njava { sourceCompatibility = JavaVersion.VERSION_17 }\n", + ), + ( + "gradle/gradle-daemon-jvm.properties", + "toolchainVersion=25\n", + ), + (".sdkmanrc", "java=25.0.1-tem\n"), + ]); + let report = scan(dir.path(), None).unwrap(); + let java = project(&report, ".") + .languages + .iter() + .find(|l| l.name == "java") + .unwrap(); + assert!( + java.versions + .iter() + .any(|v| v.value == "17" && v.kind == "source") + ); + assert!( + java.versions + .iter() + .any(|v| v.value == "25" && v.kind == "build-toolchain") + ); + assert!( + java.versions + .iter() + .any(|v| v.value == "25.0.1-tem" && v.kind == "pin") + ); + assert!( + !java + .versions + .iter() + .any(|v| v.value == "25" && v.kind == "target") + ); +} + +#[test] +fn maven_relative_parents_with_different_coordinates_are_not_inherited() { + let dir = fixture(&[ + ( + "pom.xml", + "exampleplatform225", + ), + ( + "api/pom.xml", + "exampleplatform1api", + ), + ]); + let report = scan(dir.path(), None).unwrap(); + assert!(!version(project(&report, "api"), "java", "25", "pom.xml")); +} + +#[test] +fn classic_aspnet_and_aspire_sdk_declarations_are_separate_framework_facts() { + let dir = fixture(&[ + ( + "legacy/Web.csproj", + "v4.8", + ), + ( + "apphost/AppHost.csproj", + "net10.0", + ), + ]); + let report = scan(dir.path(), None).unwrap(); + assert!(report.diagnostics.is_empty(), "{report:#?}"); + assert!(detects(project(&report, "legacy"), "aspnet")); + assert!(!detects(project(&report, "legacy"), "aspnetcore")); + assert!(version( + project(&report, "legacy"), + "csharp", + "v4.8", + "legacy/Web.csproj" + )); + assert!(detects(project(&report, "apphost"), "aspire")); +} + +#[test] +fn gradle_project_directory_and_name_overrides_remain_distinct() { + let dir = fixture(&[ + ( + "settings.gradle", + "rootProject.name='platform'\ninclude ':web'\nproject(':web').projectDir = file('services/api')\nproject(':web').name = 'web-service'\n", + ), + ("build.gradle", "allprojects { group = 'example' }\n"), + ("services/api/build.gradle", "plugins { id 'java' }\n"), + ]); + let report = scan(dir.path(), None).unwrap(); + assert_eq!(project(&report, ".").workspace_members, ["services/api"]); + assert_eq!(project(&report, "services/api").names, ["web-service"]); +} + +#[test] +fn dotnet_build_outputs_are_pruned_without_skipping_rust_source_bins() { + let dir = fixture(&[ + ("Web/Web.csproj", ""), + ("Web/obj/generated/Fake.csproj", "net10.013.014.0", + )]); + let report = scan(dir.path(), None).unwrap(); + let versions = &project(&report, ".").languages[0].versions; + assert!( + versions + .iter() + .any(|v| v.value == "13.0" && v.kind == "conditional-language") + ); + assert!( + versions + .iter() + .any(|v| v.value == "14.0" && v.kind == "conditional-language") + ); + assert!(versions.iter().all(|v| v.kind != "language")); +} + +#[test] +fn disabled_central_package_management_does_not_supply_a_version() { + let dir = fixture(&[ + ( + "Directory.Packages.props", + "", + ), + ( + "App.csproj", + "false", + ), + ]); + let report = scan(dir.path(), None).unwrap(); + let framework = project(&report, ".") + .detections + .iter() + .find(|d| d.id == "fastendpoints") + .unwrap(); + assert!(framework.evidence.iter().all(|e| e.value.is_none())); +} diff --git a/src/detect/mod.rs b/src/detect/mod.rs index 5007d58..a3b6d69 100644 --- a/src/detect/mod.rs +++ b/src/detect/mod.rs @@ -9,10 +9,17 @@ use anyhow::{Context, Result, bail}; use clap::Args; use serde::Serialize; +mod dotnet; +mod gradle; +mod java; +#[cfg(test)] +mod jvm_dotnet_tests; mod parsers; mod rules; #[cfg(test)] mod tests; +mod values; +mod xml; use rules::{Kind, Rules}; @@ -26,6 +33,8 @@ const IGNORED: &[&str] = &[ "vendor", "target", ".venv", + ".gradle", + ".vs", "venv", "__pycache__", ".next", @@ -132,7 +141,7 @@ struct Node { sources: BTreeMap, dependencies: BTreeMap, identities: BTreeMap, - packages: BTreeSet, + packages: BTreeMap, versions: BTreeMap>, names: BTreeSet, workspaces: Vec, @@ -287,6 +296,20 @@ impl Inventory { } } } + if node.files.iter().any(|file| file.ends_with(".csproj")) { + children.retain(|(path, _)| { + let output = path + .file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| { + name.eq_ignore_ascii_case("bin") || name.eq_ignore_ascii_case("obj") + }); + if output { + self.stats.skipped_directories += 1; + } + !output + }); + } if !node.files.is_empty() || !node.sources.is_empty() { node.path = path.to_path_buf(); self.nodes.push(node); @@ -359,20 +382,23 @@ fn inventory(root: &Path, rules: &Rules, workers: usize) -> Result { } else { inventory.walk(root, rules, pending)?; } - inventory.nodes.sort_by(|a, b| a.path.cmp(&b.path)); + index_parents(&mut inventory.nodes); + Ok(inventory) +} + +fn index_parents(nodes: &mut [Node]) { + nodes.sort_by(|a, b| a.path.cmp(&b.path)); let mut ancestors = Vec::::new(); - for index in 0..inventory.nodes.len() { - while ancestors.last().is_some_and(|&parent| { - !inventory.nodes[index] - .path - .starts_with(&inventory.nodes[parent].path) - }) { + for index in 0..nodes.len() { + while ancestors + .last() + .is_some_and(|&parent| !nodes[index].path.starts_with(&nodes[parent].path)) + { ancestors.pop(); } - inventory.nodes[index].parent = ancestors.last().copied(); + nodes[index].parent = ancestors.last().copied(); ancestors.push(index); } - Ok(inventory) } pub fn scan(root: &Path, extra_rules: Option<&str>) -> Result { @@ -403,7 +429,11 @@ pub fn scan(root: &Path, extra_rules: Option<&str>) -> Result { } = inventory(&root, rules, workers)?; report.diagnostics = diagnostics; report.stats = stats; + let original_nodes = nodes.len(); parsers::parse(&mut nodes, &mut report.diagnostics); + if nodes.len() != original_nodes { + index_parents(&mut nodes); + } let mut projects = Vec::<(usize, Project)>::new(); let mut project_indices = vec![None; nodes.len()]; @@ -478,15 +508,24 @@ pub fn scan(root: &Path, extra_rules: Option<&str>) -> Result { for language in languages { let mut versions = Vec::new(); let mut ancestor = Some(*index); + let mut sdk_scope_seen = false; while let Some(i) = ancestor { if let Some(found) = nodes[i].versions.get(&language) { versions.extend( found .iter() - .filter(|v| i == *index || v.kind == "pin" || v.kind == "toolchain") + .filter(|v| { + if v.kind == "sdk" { + return !sdk_scope_seen; + } + i == *index || v.kind == "pin" || v.kind == "toolchain" + }) .cloned(), ); } + if nodes[i].files.contains("global.json") { + sdk_scope_seen = true; + } ancestor = nodes[i].parent; } let mut seen = BTreeSet::new(); @@ -505,11 +544,17 @@ pub fn scan(root: &Path, extra_rules: Option<&str>) -> Result { if project_indices[index].is_none() { continue; } + for ecosystem in ["dotnet", "gradle"] { + if node.packages.contains_key(ecosystem) { + candidates.entry(ecosystem).or_default().push(index); + } + } for (ecosystem, file) in [ ("cargo", "Cargo.toml"), ("go", "go.mod"), ("node", "package.json"), ("python", "pyproject.toml"), + ("maven", "pom.xml"), ] { if node.files.contains(file) { candidates.entry(ecosystem).or_default().push(index); diff --git a/src/detect/parsers.rs b/src/detect/parsers.rs index c8c33a8..927f364 100644 --- a/src/detect/parsers.rs +++ b/src/detect/parsers.rs @@ -27,11 +27,25 @@ pub fn readable(name: &str) -> bool { | ".tool-versions" | "rust-toolchain" | "rust-toolchain.toml" - ) || name.ends_with(".gemspec") + | "global.json" + | "pom.xml" + | "gradle.properties" + | "gradle-daemon-jvm.properties" + | ".java-version" + | ".sdkmanrc" + ) || name.ends_with(".csproj") + || name.ends_with(".sln") + || name.ends_with(".slnx") + || name.ends_with(".props") + || name.ends_with(".targets") + || name.ends_with(".gradle") + || name.ends_with(".gradle.kts") + || name.ends_with(".versions.toml") + || name.ends_with(".gemspec") || (name.starts_with("requirements") && name.ends_with(".txt")) } -pub fn parse(nodes: &mut [Node], diagnostics: &mut Vec) { +pub fn parse(nodes: &mut Vec, diagnostics: &mut Vec) { let mut documents = Documents::new(); let mut pnpm = HashMap::>::new(); for node in nodes.iter() { @@ -95,6 +109,8 @@ pub fn parse(nodes: &mut [Node], diagnostics: &mut Vec) { } } } + super::dotnet::parse(nodes, diagnostics); + super::java::parse(nodes, diagnostics); for node in nodes { if let Some(Some(manifest)) = pnpm.get(&node.path) { node.workspaces.push(manifest.workspace.clone()); @@ -132,6 +148,15 @@ pub fn parse(nodes: &mut [Node], diagnostics: &mut Vec) { "go.mod" | "go.work" => go(node, file, text), "Gemfile" => ruby(node, text, diagnostics), f if f.ends_with(".gemspec") => gemspec(node, file, text), + ".java-version" => pin(node, "java", file, text), + ".sdkmanrc" => { + if let Some(value) = + super::values::properties(text, &relative_file(&node.path, file)) + .get("java") + { + node.version("java", &value.value, file, "pin"); + } + } ".go-version" => pin(node, "go", file, text), ".php-version" => pin(node, "php", file, text), ".ruby-version" => pin(node, "ruby", file, text), @@ -154,6 +179,7 @@ pub fn parse(nodes: &mut [Node], diagnostics: &mut Vec) { let language = match words.next() { Some("nodejs") => "javascript", Some("golang") => "go", + Some("java") => "java", Some("python") => "python", Some("ruby") => "ruby", Some("rust") => "rust", @@ -355,7 +381,8 @@ type Documents = HashMap>; fn cargo(node: &mut Node, doc: &Value, documents: &Documents) { let file = "Cargo.toml"; if doc.get("package").is_some_and(Value::is_object) { - node.packages.insert("cargo".to_owned()); + node.packages + .insert("cargo".to_owned(), relative_file(&node.path, file)); } let workspace = if let Some(explicit) = doc.pointer("/package/workspace").and_then(Value::as_str) { diff --git a/src/detect/rules.rs b/src/detect/rules.rs index b947a37..1f4c2de 100644 --- a/src/detect/rules.rs +++ b/src/detect/rules.rs @@ -116,9 +116,12 @@ impl Rules { !ecosystem.is_empty() && !name.is_empty() }) } - Some(("package", "cargo")) => true, + Some(("package", "cargo" | "dotnet" | "maven" | "gradle")) => true, Some(("workspace", value)) => { - matches!(value, "cargo" | "node" | "go" | "python") + matches!( + value, + "cargo" | "node" | "go" | "python" | "dotnet" | "maven" | "gradle" + ) } _ => false, }; @@ -215,7 +218,7 @@ impl Rules { candidates.extend(ids); } } - for package in &node.packages { + for package in node.packages.keys() { if let Some(ids) = self.packages.get(package) { candidates.extend(ids); } @@ -296,11 +299,13 @@ enum Matched<'a> { File(&'a str), Dependency(&'a super::Dependency), Workspace(&'a super::Workspace), + Package(&'a str), } impl Matched<'_> { fn evidence(self, node: &Node, pattern: &str, evidence: &mut Vec) { let (source, value) = match self { + Self::Package(source) => (source.to_owned(), None), Self::File(name) => (super::relative_file(&node.path, name), None), Self::Dependency(dep) => { if let Some(resolved) = &dep.resolved { @@ -339,7 +344,7 @@ fn matched<'a>(node: &'a Node, pattern: &str) -> Option> { "package" => node .packages .get(value) - .map(|_| Matched::File("Cargo.toml")), + .map(|source| Matched::Package(source)), "workspace" => node .workspaces .iter() diff --git a/src/detect/rules.toml b/src/detect/rules.toml index 8064222..e74d78b 100644 --- a/src/detect/rules.toml +++ b/src/detect/rules.toml @@ -340,3 +340,115 @@ language = "ruby" kind = "project" priority = 10 any = ["file:*.gemspec"] + +[[rules]] +id = "dotnet" +language = "csharp" +kind = "project" +priority = 10 +any = ["package:dotnet", "file:*.csproj"] + +[[rules]] +id = "csharp-source" +language = "csharp" +kind = "source" +priority = 0 +any = ["source:cs", "source:csx"] + +[[rules]] +id = "dotnet-workspace" +language = "csharp" +kind = "workspace" +priority = 20 +any = ["workspace:dotnet"] + +[[rules]] +id = "aspnetcore" +language = "csharp" +kind = "framework" +priority = 100 +any = ["dep:dotnet-sdk:microsoft.net.sdk.web", "dep:dotnet-sdk:microsoft.net.sdk.razor", "dep:dotnet-framework:microsoft.aspnetcore.app", "dep:nuget:microsoft.aspnetcore.app", "dep:nuget:microsoft.aspnetcore", "identity:nuget:microsoft.aspnetcore.app", "identity:nuget:microsoft.aspnetcore.app.ref", "identity:nuget:microsoft.aspnetcore.mvc"] + +[[rules]] +id = "aspnet" +language = "csharp" +kind = "framework" +priority = 100 +any = ["dep:nuget:microsoft.aspnet.mvc", "dep:nuget:microsoft.aspnet.webapi", "dep:dotnet-reference:system.web", "dep:dotnet-reference:system.web.mvc"] + +[[rules]] +id = "aspire" +language = "csharp" +kind = "framework" +priority = 110 +any = ["dep:dotnet-sdk:aspire.apphost.sdk", "dep:nuget:aspire.hosting.apphost"] + +[[rules]] +id = "fastendpoints" +language = "csharp" +kind = "framework" +priority = 110 +any = ["dep:nuget:fastendpoints", "identity:nuget:fastendpoints"] + +[[rules]] +id = "maven" +language = "java" +kind = "project" +priority = 10 +any = ["package:maven", "file:pom.xml"] + +[[rules]] +id = "gradle" +language = "java" +kind = "project" +priority = 10 +any = ["package:gradle", "file:build.gradle", "file:build.gradle.kts"] + +[[rules]] +id = "java-source" +language = "java" +kind = "source" +priority = 0 +any = ["source:java"] + +[[rules]] +id = "maven-workspace" +language = "java" +kind = "workspace" +priority = 20 +any = ["workspace:maven"] + +[[rules]] +id = "gradle-workspace" +language = "java" +kind = "workspace" +priority = 20 +any = ["workspace:gradle"] + +[[rules]] +id = "spring-boot" +language = "java" +kind = "framework" +priority = 110 +any = ["dep:maven-parent:org.springframework.boot:spring-boot-starter-parent", "dep:maven:org.springframework.boot:spring-boot", "dep:maven:org.springframework.boot:spring-boot-starter", "dep:maven:org.springframework.boot:spring-boot-starter-web", "dep:maven:org.springframework.boot:spring-boot-starter-webmvc", "dep:maven:org.springframework.boot:spring-boot-starter-webflux", "dep:maven-plugin:org.springframework.boot:spring-boot-maven-plugin", "dep:gradle-plugin:org.springframework.boot", "identity:maven:org.springframework.boot:spring-boot"] + +[[rules]] +id = "spring" +language = "java" +kind = "framework" +priority = 100 +any = ["dep:maven:org.springframework:spring-context", "dep:maven:org.springframework:spring-web", "dep:maven:org.springframework:spring-webmvc", "dep:maven:org.springframework:spring-webflux", "identity:maven:org.springframework:spring-core", "identity:maven:org.springframework:spring-context", "identity:maven:org.springframework:spring-web"] + +[[rules]] +id = "quarkus" +language = "java" +kind = "framework" +priority = 110 +any = ["dep:maven:io.quarkus:quarkus-core", "dep:maven:io.quarkus:quarkus-rest", "dep:maven-plugin:io.quarkus:quarkus-maven-plugin", "dep:gradle-plugin:io.quarkus", "identity:maven:io.quarkus:quarkus-core"] + +[[rules]] +id = "micronaut" +language = "java" +kind = "framework" +priority = 110 +any = ["dep:maven:io.micronaut:micronaut-runtime", "dep:gradle-plugin:io.micronaut.application", "identity:maven:io.micronaut:micronaut-runtime"] diff --git a/src/detect/tests.rs b/src/detect/tests.rs index 0fecb1e..99838b8 100644 --- a/src/detect/tests.rs +++ b/src/detect/tests.rs @@ -4,7 +4,7 @@ use std::time::Instant; use super::{Project, Report, scan}; -fn fixture(files: &[(&str, &str)]) -> tempfile::TempDir { +pub(super) fn fixture(files: &[(&str, &str)]) -> tempfile::TempDir { let dir = tempfile::tempdir().unwrap(); for (path, text) in files { let path = dir.path().join(path); @@ -14,7 +14,7 @@ fn fixture(files: &[(&str, &str)]) -> tempfile::TempDir { dir } -fn project<'a>(report: &'a Report, path: &str) -> &'a Project { +pub(super) fn project<'a>(report: &'a Report, path: &str) -> &'a Project { report .projects .iter() @@ -22,11 +22,11 @@ fn project<'a>(report: &'a Report, path: &str) -> &'a Project { .unwrap_or_else(|| panic!("missing {path}: {report:#?}")) } -fn detects(project: &Project, id: &str) -> bool { +pub(super) fn detects(project: &Project, id: &str) -> bool { project.detections.iter().any(|d| d.id == id) } -fn version(project: &Project, language: &str, value: &str, source: &str) -> bool { +pub(super) fn version(project: &Project, language: &str, value: &str, source: &str) -> bool { project.languages.iter().any(|l| { l.name == language && l.versions diff --git a/src/detect/values.rs b/src/detect/values.rs new file mode 100644 index 0000000..bba039e --- /dev/null +++ b/src/detect/values.rs @@ -0,0 +1,199 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Component, Path, PathBuf}; + +use super::{Dependency, Node, Version}; + +#[derive(Clone, Debug)] +pub(super) struct Located { + pub value: String, + pub source: String, + pub conditional: bool, +} + +pub(super) type Properties = BTreeMap; + +#[derive(Clone, Copy)] +pub(super) enum Syntax { + Maven, + MsBuild, +} + +pub(super) fn resolve(raw: &Located, properties: &Properties, syntax: Syntax) -> Option { + let (prefix, suffix) = match syntax { + Syntax::Maven => ("${", '}'), + Syntax::MsBuild => ("$(", ')'), + }; + let mut result = raw.clone(); + let mut seen = BTreeSet::new(); + for _ in 0..64 { + let Some(start) = result.value.find(prefix) else { + return Some(result); + }; + if !seen.insert(result.value.clone()) { + return None; + } + let end = result.value[start + 2..].find(suffix)? + start + 2; + let name = &result.value[start + 2..end]; + let name = match syntax { + Syntax::Maven => name.to_owned(), + Syntax::MsBuild => name.to_ascii_lowercase(), + }; + let replacement = properties.get(&name)?; + if replacement.conditional { + return None; + } + let entire = start == 0 && end + 1 == result.value.len(); + let length = result + .value + .len() + .checked_sub(end + 1 - start)? + .checked_add(replacement.value.len())?; + if length > super::MAX_FILE_BYTES as usize { + return None; + } + result + .value + .replace_range(start..end + 1, &replacement.value); + if entire { + result.source = replacement.source.clone(); + } + } + None +} + +pub(super) fn version( + node: &mut Node, + language: &str, + raw: &Located, + kind: &str, + properties: &Properties, + syntax: Syntax, +) { + let resolved = resolve(raw, properties, syntax); + let reference = match syntax { + Syntax::Maven => raw.value.contains("${"), + Syntax::MsBuild => raw.value.contains("$("), + }; + let kind = if raw.conditional { + format!("conditional-{kind}") + } else { + kind.to_owned() + }; + if reference { + node.versions + .entry(language.to_owned()) + .or_default() + .push(Version { + value: raw.value.clone(), + source: raw.source.clone(), + kind: "reference".to_owned(), + }); + } + if let Some(value) = resolved.filter(|v| !v.value.trim().is_empty()) { + node.versions + .entry(language.to_owned()) + .or_default() + .push(Version { + value: value.value, + source: value.source, + kind, + }); + } +} + +fn declaration( + source: &str, + raw: Option<&Located>, + properties: &Properties, + syntax: Syntax, +) -> Dependency { + let mut dependency = Dependency { + version: raw.filter(|v| v.source == source).map(|v| v.value.clone()), + source: source.to_owned(), + resolved: None, + }; + if let Some(raw) = raw { + let resolved = resolve(raw, properties, syntax).unwrap_or_else(|| raw.clone()); + if raw.source != source || raw.value != resolved.value { + dependency.resolved = Some(Version { + value: resolved.value, + source: resolved.source, + kind: "declaration".to_owned(), + }); + } + } + dependency +} + +pub(super) fn dependency( + node: &mut Node, + ecosystem: &str, + name: &str, + source: &str, + raw: Option<&Located>, + properties: &Properties, + syntax: Syntax, +) { + node.dependencies.insert( + format!("{ecosystem}:{name}"), + declaration(source, raw, properties, syntax), + ); +} + +pub(super) fn identity( + node: &mut Node, + ecosystem: &str, + name: &str, + source: &str, + raw: Option<&Located>, + properties: &Properties, + syntax: Syntax, +) { + node.identities.insert( + format!("{ecosystem}:{name}"), + declaration(source, raw, properties, syntax), + ); +} + +pub(super) fn relative(path: &Path) -> Option { + if path.is_absolute() { + return None; + } + let mut normalized = PathBuf::new(); + for component in path.components() { + match component { + Component::ParentDir => { + if !normalized.pop() { + return None; + } + } + Component::CurDir => (), + other => normalized.push(other), + } + } + Some(normalized) +} + +pub(super) fn child_file(base: &Path, value: &str) -> Option { + relative(&base.join(value.replace('\\', "/"))) +} + +pub(super) fn properties(text: &str, source: &str) -> Properties { + text.lines() + .filter_map(|line| { + let line = line.trim(); + if line.starts_with(['#', '!']) { + return None; + } + let (name, value) = line.split_once('=')?; + Some(( + name.trim().to_owned(), + Located { + value: value.trim().to_owned(), + source: source.to_owned(), + conditional: false, + }, + )) + }) + .collect() +} diff --git a/src/detect/xml.rs b/src/detect/xml.rs new file mode 100644 index 0000000..02a8ecc --- /dev/null +++ b/src/detect/xml.rs @@ -0,0 +1,171 @@ +use std::collections::BTreeMap; + +use anyhow::{Result, bail}; +use xmlparser::{ElementEnd, Token, Tokenizer}; + +pub(super) struct Element<'a> { + pub name: &'a str, + pub attributes: BTreeMap, + pub text: String, + prefix: &'a str, +} + +pub(super) fn read(input: &str, mut visit: impl FnMut(&[Element<'_>]) -> Result<()>) -> Result<()> { + let mut stack = Vec::>::new(); + let mut roots = 0; + for token in Tokenizer::from(input) { + match token? { + Token::ElementStart { prefix, local, .. } => { + if stack.is_empty() { + roots += 1; + } + if stack.len() == 128 || roots > 1 { + bail!("invalid XML root or nesting exceeds 128 elements"); + } + stack.push(Element { + name: local.as_str(), + prefix: prefix.as_str(), + attributes: BTreeMap::new(), + text: String::new(), + }); + } + Token::Attribute { + prefix, + local, + value, + .. + } => { + let Some(element) = stack.last_mut() else { + bail!("XML attribute outside an element"); + }; + let name = if prefix.is_empty() { + local.as_str().to_owned() + } else { + format!("{}:{}", prefix.as_str(), local.as_str()) + }; + if element + .attributes + .insert(name, unescape(value.as_str())?) + .is_some() + { + bail!("duplicate XML attribute"); + } + } + Token::ElementEnd { end, .. } => match end { + ElementEnd::Open => validate_namespaces(&stack)?, + ElementEnd::Empty => { + validate_namespaces(&stack)?; + visit(&stack)?; + stack.pop(); + } + ElementEnd::Close(prefix, name) => { + if !stack + .last() + .is_some_and(|e| e.name == name.as_str() && e.prefix == prefix.as_str()) + { + bail!("mismatched XML closing element"); + } + visit(&stack)?; + stack.pop(); + } + }, + Token::Text { text } => { + if let Some(element) = stack.last_mut() { + if !text.as_str().trim().is_empty() || !element.text.is_empty() { + element.text.push_str(&unescape(text.as_str())?); + } + } else if !text.as_str().trim().is_empty() { + bail!("XML text outside the root element"); + } + } + Token::Cdata { text, .. } => { + let Some(element) = stack.last_mut() else { + bail!("CDATA outside an XML element"); + }; + element.text.push_str(text.as_str()); + } + Token::DtdStart { .. } | Token::EmptyDtd { .. } | Token::EntityDeclaration { .. } => { + bail!("XML DTDs and custom entities are not evaluated") + } + _ => (), + } + } + if roots != 1 || !stack.is_empty() { + bail!("XML document has a missing root or unclosed element"); + } + Ok(()) +} + +fn validate_namespaces(stack: &[Element<'_>]) -> Result<()> { + let Some(element) = stack.last() else { + bail!("XML element missing"); + }; + let namespace = |prefix: &str| -> Option<&str> { + if prefix == "xml" { + return Some("http://www.w3.org/XML/1998/namespace"); + } + let name = format!("xmlns:{prefix}"); + stack + .iter() + .rev() + .find_map(|e| e.attributes.get(&name).map(String::as_str)) + .filter(|value| !value.is_empty()) + }; + if !element.prefix.is_empty() && namespace(element.prefix).is_none() { + bail!("undeclared XML namespace prefix"); + } + let mut expanded = Vec::new(); + for name in element.attributes.keys() { + if name == "xmlns" || name.starts_with("xmlns:") { + continue; + } + let (uri, local) = if let Some((prefix, local)) = name.split_once(':') { + let Some(uri) = namespace(prefix) else { + bail!("undeclared XML attribute namespace"); + }; + (uri, local) + } else { + ("", name.as_str()) + }; + if expanded.contains(&(uri, local)) { + bail!("duplicate expanded XML attribute"); + } + expanded.push((uri, local)); + } + Ok(()) +} + +fn unescape(input: &str) -> Result { + let mut rest = input; + let mut output = String::with_capacity(input.len()); + while let Some(index) = rest.find('&') { + output.push_str(&rest[..index]); + rest = &rest[index + 1..]; + let Some(end) = rest.find(';') else { + bail!("unterminated XML entity reference"); + }; + let entity = &rest[..end]; + let ch = match entity { + "amp" => '&', + "lt" => '<', + "gt" => '>', + "quot" => '"', + "apos" => '\'', + _ => { + let code = if let Some(hex) = entity.strip_prefix("#x") { + u32::from_str_radix(hex, 16).ok() + } else { + entity + .strip_prefix('#') + .and_then(|digits| digits.parse::().ok()) + }; + let Some(ch) = code.and_then(char::from_u32).filter(|ch| matches!(u32::from(*ch), 9 | 10 | 13 | 0x20..=0xD7FF | 0xE000..=0xFFFD | 0x10000..=0x10FFFF)) else { bail!("unsupported XML entity reference: {entity}"); }; + ch + } + }; + output.push(ch); + rest = &rest[end + 1..]; + } + output.push_str(rest); + Ok(output) +} diff --git a/tests/audit_detect.py b/tests/audit_detect.py index c9ea0ae..f4e5e28 100644 --- a/tests/audit_detect.py +++ b/tests/audit_detect.py @@ -96,7 +96,13 @@ def check(projects, expected): actual = [language["name"] for language in project["languages"]] return expected["language"] in actual, actual if kind == "detection": - return expected["id"] in ids, ids + found = next( + (d for d in project["detections"] if d["id"] == expected["id"]), None + ) + roles = expected.get("roles", []) + return found is not None and set(roles).issubset( + found.get("roles", []) + ), project["detections"] if roles else ids if kind == "name": return expected["name"] in project["names"], project["names"] if kind == "workspace": @@ -113,7 +119,12 @@ def check(projects, expected): for v in language["versions"] ] return any( - v["value"] == expected["value"] and v["source"] == expected["source"] + v["value"] == expected["value"] + and v["source"] == expected["source"] + and ( + "versionKind" not in expected + or v.get("kind") == expected["versionKind"] + ) for v in actual ), actual raise ValueError(f"unknown check kind: {kind}") diff --git a/tests/detect-corpus.json b/tests/detect-corpus.json index 9c6a4ce..088f90b 100644 --- a/tests/detect-corpus.json +++ b/tests/detect-corpus.json @@ -1475,6 +1475,518 @@ "member": "packages/excalidraw" } ] + }, + { + "repository": "jellyfin/jellyfin", + "language": "C#", + "stars": 59164, + "languageRank": 3, + "webRank": 1, + "revision": "797999fc9bd49a9f0853916e2933e3ad06173503", + "checks": [ + { + "category": "language", + "kind": "language", + "path": "Jellyfin.Server", + "language": "csharp" + }, + { + "category": "project", + "kind": "detection", + "path": "Jellyfin.Server", + "id": "dotnet" + }, + { + "category": "framework", + "kind": "detection", + "path": "Jellyfin.Server", + "id": "aspnetcore", + "roles": [ + "uses" + ] + }, + { + "category": "version", + "kind": "version", + "path": "Jellyfin.Server", + "language": "csharp", + "value": "10.0.0", + "source": "global.json", + "versionKind": "sdk" + }, + { + "category": "version", + "kind": "version", + "path": "Jellyfin.Server", + "language": "csharp", + "value": "net10.0", + "source": "Jellyfin.Server/Jellyfin.Server.csproj", + "versionKind": "target" + }, + { + "category": "workspace", + "kind": "workspace", + "path": ".", + "id": "dotnet-workspace", + "member": "Jellyfin.Server" + } + ] + }, + { + "repository": "dotnet/aspnetcore", + "language": "C#", + "stars": 38507, + "languageRank": 8, + "webRank": 2, + "revision": "eeceb660128d3d00ea9b35effe4fad3fac3d42e3", + "checks": [ + { + "category": "language", + "kind": "language", + "path": "src/Mvc/Mvc/src", + "language": "csharp" + }, + { + "category": "project", + "kind": "detection", + "path": "src/Mvc/Mvc/src", + "id": "dotnet" + }, + { + "category": "framework", + "kind": "detection", + "path": "src/Mvc/Mvc/src", + "id": "aspnetcore", + "roles": [ + "provides" + ] + }, + { + "category": "version", + "kind": "version", + "path": "src/Mvc/Mvc/src", + "language": "csharp", + "value": "12.0.100-alpha.1.26555.106", + "source": "global.json", + "versionKind": "sdk" + }, + { + "category": "workspace", + "kind": "workspace", + "path": ".", + "id": "dotnet-workspace", + "member": "src/Mvc/Mvc/src" + } + ] + }, + { + "repository": "jasontaylordev/CleanArchitecture", + "language": "C#", + "stars": 20711, + "languageRank": 29, + "webRank": 3, + "revision": "e1ab4cc4d5d7b4e25e57bd8b0514b766573b8c04", + "checks": [ + { + "category": "language", + "kind": "language", + "path": "src/Web", + "language": "csharp" + }, + { + "category": "project", + "kind": "detection", + "path": "src/Web", + "id": "dotnet" + }, + { + "category": "framework", + "kind": "detection", + "path": "src/Web", + "id": "aspnetcore", + "roles": [ + "uses" + ] + }, + { + "category": "version", + "kind": "version", + "path": "src/Web", + "language": "csharp", + "value": "10.0.403", + "source": "global.json", + "versionKind": "sdk" + }, + { + "category": "version", + "kind": "version", + "path": "src/Web", + "language": "csharp", + "value": "net10.0", + "source": "Directory.Build.props", + "versionKind": "target" + }, + { + "category": "workspace", + "kind": "workspace", + "path": ".", + "id": "dotnet-workspace", + "member": "src/Web" + } + ] + }, + { + "repository": "bitwarden/server", + "language": "C#", + "stars": 20610, + "languageRank": 30, + "webRank": 4, + "revision": "04a4d7548b07d18ab7327ddaae81a05fd893a820", + "checks": [ + { + "category": "language", + "kind": "language", + "path": "src/Api", + "language": "csharp" + }, + { + "category": "project", + "kind": "detection", + "path": "src/Api", + "id": "dotnet" + }, + { + "category": "framework", + "kind": "detection", + "path": "src/Api", + "id": "aspnetcore", + "roles": [ + "uses" + ] + }, + { + "category": "version", + "kind": "version", + "path": "src/Api", + "language": "csharp", + "value": "10.0.103", + "source": "global.json", + "versionKind": "sdk" + }, + { + "category": "version", + "kind": "version", + "path": "src/Api", + "language": "csharp", + "value": "net10.0", + "source": "Directory.Build.props", + "versionKind": "target" + }, + { + "category": "workspace", + "kind": "workspace", + "path": ".", + "id": "dotnet-workspace", + "member": "src/Api" + } + ] + }, + { + "repository": "ardalis/CleanArchitecture", + "language": "C#", + "stars": 18543, + "languageRank": 36, + "webRank": 5, + "revision": "da5491f1641f3d388e6cfc2c0ddd29f763746762", + "checks": [ + { + "category": "language", + "kind": "language", + "path": "src/Clean.Architecture.Web", + "language": "csharp" + }, + { + "category": "project", + "kind": "detection", + "path": "src/Clean.Architecture.Web", + "id": "dotnet" + }, + { + "category": "framework", + "kind": "detection", + "path": "src/Clean.Architecture.Web", + "id": "aspnetcore", + "roles": [ + "uses" + ] + }, + { + "category": "version", + "kind": "version", + "path": "src/Clean.Architecture.Web", + "language": "csharp", + "value": "10.0.100", + "source": "global.json", + "versionKind": "sdk" + }, + { + "category": "version", + "kind": "version", + "path": "src/Clean.Architecture.Web", + "language": "csharp", + "value": "net10.0", + "source": "Directory.Build.props", + "versionKind": "target" + }, + { + "category": "workspace", + "kind": "workspace", + "path": ".", + "id": "dotnet-workspace", + "member": "src/Clean.Architecture.Web" + }, + { + "category": "framework", + "kind": "detection", + "path": "src/Clean.Architecture.Web", + "id": "fastendpoints" + } + ] + }, + { + "repository": "macrozheng/mall", + "language": "Java", + "stars": 85023, + "languageRank": 3, + "webRank": 1, + "revision": "dcaa93b3150352e5044708d7211b5bed0af4509f", + "checks": [ + { + "category": "language", + "kind": "language", + "path": ".", + "language": "java" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "maven" + }, + { + "category": "framework", + "kind": "detection", + "path": ".", + "id": "spring-boot", + "roles": [ + "uses" + ] + }, + { + "category": "version", + "kind": "version", + "path": ".", + "language": "java", + "value": "17", + "source": "pom.xml" + }, + { + "category": "workspace", + "kind": "workspace", + "path": ".", + "id": "maven-workspace", + "member": "mall-admin" + } + ] + }, + { + "repository": "spring-projects/spring-boot", + "language": "Java", + "stars": 81788, + "languageRank": 4, + "webRank": 2, + "revision": "52dba7773c16cc5d145e59ad69de1b99e87896bb", + "checks": [ + { + "category": "language", + "kind": "language", + "path": "core/spring-boot", + "language": "java" + }, + { + "category": "project", + "kind": "detection", + "path": "core/spring-boot", + "id": "gradle" + }, + { + "category": "identity", + "kind": "detection", + "path": "core/spring-boot", + "id": "spring-boot", + "roles": [ + "provides" + ] + }, + { + "category": "version", + "kind": "version", + "path": "core/spring-boot", + "language": "java", + "value": "25.0.5.0+11-librca", + "source": ".sdkmanrc", + "versionKind": "pin" + }, + { + "category": "workspace", + "kind": "workspace", + "path": ".", + "id": "gradle-workspace", + "member": "core/spring-boot" + } + ] + }, + { + "repository": "elastic/elasticsearch", + "language": "Java", + "stars": 78546, + "languageRank": 6, + "webRank": 3, + "revision": "97fdd221f5919be472b60d29241f673774c19c51", + "checks": [ + { + "category": "language", + "kind": "language", + "path": ".", + "language": "java" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "gradle" + }, + { + "category": "version", + "kind": "version", + "path": ".", + "language": "java", + "value": "21", + "source": "gradle/gradle-daemon-jvm.properties", + "versionKind": "build-toolchain" + }, + { + "category": "workspace", + "kind": "workspace", + "path": ".", + "id": "gradle-workspace", + "member": "server" + } + ] + }, + { + "repository": "spring-projects/spring-framework", + "language": "Java", + "stars": 60307, + "languageRank": 11, + "webRank": 4, + "revision": "cc8008c301bc57b69c7c226254b720a2239741bf", + "checks": [ + { + "category": "language", + "kind": "language", + "path": "spring-core", + "language": "java" + }, + { + "category": "project", + "kind": "detection", + "path": "spring-core", + "id": "gradle" + }, + { + "category": "identity", + "kind": "detection", + "path": "spring-core", + "id": "spring", + "roles": [ + "provides" + ] + }, + { + "category": "version", + "kind": "version", + "path": "spring-core", + "language": "java", + "value": "25.0.5.0+11-librca", + "source": ".sdkmanrc", + "versionKind": "pin" + }, + { + "category": "workspace", + "kind": "workspace", + "path": ".", + "id": "gradle-workspace", + "member": "spring-core" + } + ] + }, + { + "repository": "jeecgboot/JeecgBoot", + "language": "Java", + "stars": 48413, + "languageRank": 15, + "webRank": 5, + "revision": "49c29032e18df784f1a75123d90647bc8f74cedb", + "checks": [ + { + "category": "language", + "kind": "language", + "path": "jeecg-boot/jeecg-module-system/jeecg-system-start", + "language": "java" + }, + { + "category": "project", + "kind": "detection", + "path": "jeecg-boot/jeecg-module-system/jeecg-system-start", + "id": "maven" + }, + { + "category": "framework", + "kind": "detection", + "path": "jeecg-boot/jeecg-module-system/jeecg-system-start", + "id": "spring-boot", + "roles": [ + "uses" + ] + }, + { + "category": "version", + "kind": "version", + "path": "jeecg-boot/jeecg-module-system/jeecg-system-start", + "language": "java", + "value": "17", + "source": "jeecg-boot/pom.xml" + }, + { + "category": "workspace", + "kind": "workspace", + "path": "jeecg-boot", + "id": "maven-workspace", + "member": "jeecg-boot/jeecg-module-system" + } + ] + } + ], + "extensions": [ + { + "selectedAt": "2026-11-09T22:54:00.932983+00:00", + "languages": [ + "C#", + "Java" + ], + "repositoriesPerLanguage": 5, + "eligibility": "Same web-project criteria as existing corpus, including production web solution templates (as with create-react-app), excluding native/desktop-first apps, general runtimes, lists, and tutorial collections." } ] } diff --git a/tests/detect-corpus.md b/tests/detect-corpus.md index 73febd6..c20df03 100644 --- a/tests/detect-corpus.md +++ b/tests/detect-corpus.md @@ -1,10 +1,10 @@ # GitHub project detection audit -Selected 2026-11-09. Baseline: detector commit `244ab44ccf1a` (PR #20). Current: the follow-up fixes in this revision. +Selected 2026-11-09; C#/Java added later that day. Baseline: detector commit `244ab44ccf1a` (PR #20). Current: the follow-up fixes in this revision. ## Method -Five manually qualified web projects per supported language, ordered by GitHub stars among non-fork, non-archived repositories. Language means GitHub’s primary-language classification, not a measured ranking of the languages themselves. Production learning platforms qualify; tutorial collections, lists, coding agents, general runtimes, native desktop apps, and generic infrastructure tools do not. Eligibility is a judgment call, not an objective GitHub category. +Five manually qualified web projects per supported language, ordered by GitHub stars among non-fork, non-archived repositories. Language means GitHub’s primary-language classification, not a measured ranking of the languages themselves. Production learning platforms qualify; tutorial collections, lists, coding agents, general runtimes, native desktop apps, and generic infrastructure tools do not. Production web solution templates qualify, consistently with create-react-app in the original sample. Eligibility is a judgment call, not an objective GitHub category. Examples excluded above selected projects: Node.js (general runtime), Ollama (model runtime), Kubernetes (infrastructure), frp (generic networking), and developer-roadmap (educational roadmap collection). Stars and language ranks are frozen with full commit SHAs in [`detect-corpus.json`](detect-corpus.json). @@ -14,17 +14,17 @@ GitHub source archives were downloaded into a temporary cache. No application de ## Results -All **35/35** scans succeed and produce identical output across five runs. **173/173 sampled assertions now pass**, up from 156/173. Each of the 35 ranked repositories passes its sampled assertions; this is not a claim of exhaustive detection accuracy. +The expanded corpus covers **45 repositories across nine languages**: the original seven-language sample plus five C# and five Java projects. All **45/45** scans succeed and produce identical output across five runs. **227/227 sampled assertions pass** (the original baseline passed 156/173). This is not a claim of exhaustive detection accuracy. -| Category | Baseline | Current | +| Category | Original 35 baseline | Current 45 | |---|---:|---:| -| language | 35/35 | 35/35 | -| project | 35/35 | 35/35 | +| language | 35/35 | 45/45 | +| project | 35/35 | 45/45 | | name | 26/27 | 27/27 | -| version | 32/32 | 32/32 | -| framework | 19/20 | 20/20 | -| workspace | 9/15 | 15/15 | -| identity | 0/7 | 7/7 | +| version | 32/32 | 46/46 | +| framework | 19/20 | 28/28 | +| workspace | 9/15 | 25/25 | +| identity | 0/7 | 9/9 | | catalog | 0/2 | 2/2 | β€œVersion” checks cover selected literal constraints and inherited declarations, not installed versions. Framework checks cover selected consumer projects, while identity checks cover framework implementation packages. @@ -43,7 +43,7 @@ Additional unscored coverage gaps include Jekyll, Hugo, Dioxus, Ember, Gradio, a Each repository was scanned through the release CLI five times, serially. The table reports the median of runs 2–5 with a warm filesystem, including process startup, traversal, JSON serialization, and stdout transfer. Python JSON parsing and snapshot downloads are excluded. These are local measurements, not portable timing budgets or controlled cold-cache benchmarks. -Across the corpus, the scanner counted 259,818 files and read only 2,270 manifest/version files (1.85 MiB). The current median repository time is 5.4 ms. +Across the expanded corpus, the scanner counted 368,237 files and read 4,943 bounded manifest/version files (6.64 MiB). The current median repository time is 5.6 ms. Ordinary C#/Java source bodies are not read. An interleaved, same-machine A/B comparison ran the baseline and current binaries 12 times each, discarding the first two pairs: **Next.js improved from 170.8 ms to 86.8 ms** (1.97Γ— faster), with the same full-tree scope. The generated 200-service / 20,200-file benchmark improved from 5.8 ms to **3.5 ms median**, still reading only 200 manifests. @@ -51,66 +51,88 @@ Profiling identified directory traversal and repeated path/ancestor lookups as t pnpm uses `serde-saphyr` with deserialization only: typed fields are populated directly without constructing a syntax tree, retaining comments, enabling includes, or evaluating application code. A tree-sitter syntax tree is unnecessary for this manifest-only scan. -| Language | Repository | Stars | Language rank | Revision | Baseline assertions | Current assertions | Before ms | After ms | +| Language | Repository | Stars | Language rank | Revision | Original baseline assertions | Current assertions | Before ms | After ms | |---|---|---:|---:|---|---:|---:|---:|---:| -| JavaScript | [react/react](https://github.com/react/react) | 252,455 | 2 | [`b05c0b75fd97`](https://github.com/react/react/tree/b05c0b75fd9778536152eead30814c9ca90c810a) | 5/6 | 6/6 | 14.47 | 7.29 | -| JavaScript | [vercel/next.js](https://github.com/vercel/next.js) | 143,437 | 8 | [`5368e013e74e`](https://github.com/vercel/next.js/tree/5368e013e74e4e4ed5be51c051c989545c4450b9) | 5/7 | 7/7 | 168.41 | 85.82 | -| JavaScript | [mrdoob/three.js](https://github.com/mrdoob/three.js) | 117,953 | 11 | [`3f7edc3d75e0`](https://github.com/mrdoob/three.js/tree/3f7edc3d75e054bc755dad56c1c95c62f63a3561) | 3/3 | 3/3 | 4.31 | 3.74 | -| JavaScript | [axios/axios](https://github.com/axios/axios) | 109,295 | 14 | [`335e2f8464f5`](https://github.com/axios/axios/tree/335e2f8464f58a3ab485486b8d6964fea3e3f30a) | 4/4 | 4/4 | 2.18 | 2.61 | -| JavaScript | [react/create-react-app](https://github.com/react/create-react-app) | 103,230 | 15 | [`6254386531d2`](https://github.com/react/create-react-app/tree/6254386531d263688ccfa542d0e628fbc0de0b28) | 6/6 | 6/6 | 2.30 | 2.26 | -| TypeScript | [freeCodeCamp/freeCodeCamp](https://github.com/freeCodeCamp/freeCodeCamp) | 458,826 | 1 | [`11442d669447`](https://github.com/freeCodeCamp/freeCodeCamp/tree/11442d66944785ac72a2a2b9c955fae8b9f33083) | 5/6 | 6/6 | 14.84 | 10.73 | -| TypeScript | [vuejs/vue](https://github.com/vuejs/vue) | 212,624 | 7 | [`9e8870794008`](https://github.com/vuejs/vue/tree/9e88707940088cb1f4cd7dd210c9168a50dc347c) | 4/6 | 6/6 | 2.58 | 2.70 | -| TypeScript | [n8n-io/n8n](https://github.com/n8n-io/n8n) | 209,927 | 8 | [`93243edc00b2`](https://github.com/n8n-io/n8n/tree/93243edc00b288a840fbb562a5e9f64177f2337f) | 5/7 | 7/7 | 60.35 | 32.57 | -| TypeScript | [firecrawl/firecrawl](https://github.com/firecrawl/firecrawl) | 200,129 | 9 | [`40c8ff95d719`](https://github.com/firecrawl/firecrawl/tree/40c8ff95d719d2acbd8fd6273bd19b14a1c2bbb8) | 5/5 | 5/5 | 4.86 | 4.54 | -| TypeScript | [excalidraw/excalidraw](https://github.com/excalidraw/excalidraw) | 135,400 | 15 | [`701b901a0f1b`](https://github.com/excalidraw/excalidraw/tree/701b901a0f1ba3d0b390d9cdf63d112e25abac4b) | 7/7 | 7/7 | 3.94 | 2.66 | -| Python | [Significant-Gravitas/AutoGPT](https://github.com/Significant-Gravitas/AutoGPT) | 187,972 | 11 | [`9cb76f9026d3`](https://github.com/Significant-Gravitas/AutoGPT/tree/9cb76f9026d388c9bf2e098af2beae1cc864f055) | 5/6 | 6/6 | 14.63 | 11.53 | -| Python | [AUTOMATIC1111/stable-diffusion-webui](https://github.com/AUTOMATIC1111/stable-diffusion-webui) | 165,269 | 14 | [`82a973c04367`](https://github.com/AUTOMATIC1111/stable-diffusion-webui/tree/82a973c04367123ae98bd9abdf80d9eda9b910e2) | 3/3 | 3/3 | 1.60 | 1.93 | -| Python | [langgenius/dify](https://github.com/langgenius/dify) | 160,774 | 15 | [`fdb09a9e76dc`](https://github.com/langgenius/dify/tree/fdb09a9e76dc8cbd052fabd72cfff4071328d9bc) | 8/10 | 10/10 | 30.46 | 18.70 | -| Python | [langflow-ai/langflow](https://github.com/langflow-ai/langflow) | 156,479 | 16 | [`60999cc9a735`](https://github.com/langflow-ai/langflow/tree/60999cc9a7350e44dde3ef641baae864e10861f0) | 6/6 | 6/6 | 20.32 | 12.02 | -| Python | [Comfy-Org/ComfyUI](https://github.com/Comfy-Org/ComfyUI) | 145,983 | 20 | [`540a82ee722b`](https://github.com/Comfy-Org/ComfyUI/tree/540a82ee722bda1f32a1fbd97fcadadeb22d548b) | 4/4 | 4/4 | 3.49 | 3.34 | -| PHP | [coollabsio/coolify](https://github.com/coollabsio/coolify) | 65,849 | 2 | [`30335d80866f`](https://github.com/coollabsio/coolify/tree/30335d80866ff3391d899cf4ccb599b696f47b7e) | 5/5 | 5/5 | 5.30 | 4.41 | -| PHP | [nextcloud/server](https://github.com/nextcloud/server) | 37,456 | 3 | [`d655757fb306`](https://github.com/nextcloud/server/tree/d655757fb306aca893651958030b5dae69ed0f39) | 3/3 | 3/3 | 14.62 | 7.52 | -| PHP | [laravel/framework](https://github.com/laravel/framework) | 34,995 | 4 | [`338e44065c0a`](https://github.com/laravel/framework/tree/338e44065c0a5b18f1f290605b6253a1e69d8f56) | 4/5 | 5/5 | 6.33 | 5.06 | -| PHP | [filamentphp/filament](https://github.com/filamentphp/filament) | 32,462 | 5 | [`9d78093cfebd`](https://github.com/filamentphp/filament/tree/9d78093cfebdc96c077b1d8dbf94b41738f28992) | 4/4 | 4/4 | 16.63 | 15.30 | -| PHP | [symfony/symfony](https://github.com/symfony/symfony) | 31,523 | 6 | [`f07928484c9e`](https://github.com/symfony/symfony/tree/f07928484c9e96646964deb7b9570d7885492512) | 4/5 | 5/5 | 10.10 | 9.20 | -| Ruby | [rails/rails](https://github.com/rails/rails) | 59,087 | 1 | [`3a0aff317bc0`](https://github.com/rails/rails/tree/3a0aff317bc037f43f168e3772028f1033f1e810) | 2/3 | 3/3 | 7.90 | 5.62 | -| Ruby | [jekyll/jekyll](https://github.com/jekyll/jekyll) | 51,756 | 2 | [`97c3bdc822df`](https://github.com/jekyll/jekyll/tree/97c3bdc822df9d5bea4c269b29a28de5a5e29d70) | 2/2 | 2/2 | 2.42 | 2.12 | -| Ruby | [mastodon/mastodon](https://github.com/mastodon/mastodon) | 50,502 | 3 | [`5a207bc8f504`](https://github.com/mastodon/mastodon/tree/5a207bc8f5046e27cffc640a3c64f96504f6afcf) | 4/4 | 4/4 | 8.68 | 7.09 | -| Ruby | [huginn/huginn](https://github.com/huginn/huginn) | 50,454 | 4 | [`639c050b2bf4`](https://github.com/huginn/huginn/tree/639c050b2bf4e14ef9983012af99dfc1dc68b334) | 4/4 | 4/4 | 2.35 | 2.06 | -| Ruby | [discourse/discourse](https://github.com/discourse/discourse) | 48,066 | 6 | [`899c68d783e7`](https://github.com/discourse/discourse/tree/899c68d783e7808c50144752369a685220250d7c) | 3/5 | 5/5 | 35.98 | 21.07 | -| Go | [infiniflow/ragflow](https://github.com/infiniflow/ragflow) | 93,887 | 8 | [`ed5a84fee9ab`](https://github.com/infiniflow/ragflow/tree/ed5a84fee9abc8091aea111d1195cf7768541ff1) | 5/5 | 5/5 | 10.20 | 5.38 | -| Go | [gohugoio/hugo](https://github.com/gohugoio/hugo) | 90,425 | 9 | [`d2010fcc9cd3`](https://github.com/gohugoio/hugo/tree/d2010fcc9cd30c5b4e9274ded4485f4222326b53) | 4/4 | 4/4 | 4.85 | 3.15 | -| Go | [gin-gonic/gin](https://github.com/gin-gonic/gin) | 89,609 | 11 | [`0f09c3a9b462`](https://github.com/gin-gonic/gin/tree/0f09c3a9b4626d9fe9979cf4d0521d6ae32ab646) | 4/5 | 5/5 | 2.11 | 1.70 | -| Go | [netdata/netdata](https://github.com/netdata/netdata) | 81,330 | 14 | [`0d54b4424ea6`](https://github.com/netdata/netdata/tree/0d54b4424ea6f40c344738b9d566e1a26d160da3) | 4/4 | 4/4 | 23.42 | 11.05 | -| Go | [caddyserver/caddy](https://github.com/caddyserver/caddy) | 78,927 | 15 | [`9ea3108377d2`](https://github.com/caddyserver/caddy/tree/9ea3108377d2abec102866808133c40ac9582162) | 4/4 | 4/4 | 2.56 | 2.33 | -| Rust | [dani-garcia/vaultwarden](https://github.com/dani-garcia/vaultwarden) | 70,490 | 17 | [`17b159beff4b`](https://github.com/dani-garcia/vaultwarden/tree/17b159beff4b892f07f61f38ce2fe452e55251c1) | 6/6 | 6/6 | 2.51 | 2.76 | -| Rust | [meilisearch/meilisearch](https://github.com/meilisearch/meilisearch) | 59,822 | 29 | [`8c7bfcfa9bcf`](https://github.com/meilisearch/meilisearch/tree/8c7bfcfa9bcff7083bcb5d184fa2a523a23884d8) | 5/5 | 5/5 | 4.99 | 5.17 | -| Rust | [juspay/hyperswitch](https://github.com/juspay/hyperswitch) | 45,760 | 39 | [`5786b738c088`](https://github.com/juspay/hyperswitch/tree/5786b738c088b8452296ca19922fcdc0dc74e129) | 6/6 | 6/6 | 12.07 | 7.95 | -| Rust | [parcel-bundler/parcel](https://github.com/parcel-bundler/parcel) | 44,012 | 42 | [`20c0f2ede4c1`](https://github.com/parcel-bundler/parcel/tree/20c0f2ede4c1044092d44288169cce589c5d5c14) | 3/3 | 3/3 | 18.92 | 11.77 | -| Rust | [DioxusLabs/dioxus](https://github.com/DioxusLabs/dioxus) | 39,705 | 52 | [`17d9a5924e84`](https://github.com/DioxusLabs/dioxus/tree/17d9a5924e845e4f37deca3fada6c35d1d836af0) | 5/5 | 5/5 | 8.71 | 7.17 | +| JavaScript | [react/react](https://github.com/react/react) | 252,455 | 2 | [`b05c0b75fd97`](https://github.com/react/react/tree/b05c0b75fd9778536152eead30814c9ca90c810a) | 5/6 | 6/6 | 14.47 | 8.69 | +| JavaScript | [vercel/next.js](https://github.com/vercel/next.js) | 143,437 | 8 | [`5368e013e74e`](https://github.com/vercel/next.js/tree/5368e013e74e4e4ed5be51c051c989545c4450b9) | 5/7 | 7/7 | 168.41 | 86.46 | +| JavaScript | [mrdoob/three.js](https://github.com/mrdoob/three.js) | 117,953 | 11 | [`3f7edc3d75e0`](https://github.com/mrdoob/three.js/tree/3f7edc3d75e054bc755dad56c1c95c62f63a3561) | 3/3 | 3/3 | 4.31 | 3.77 | +| JavaScript | [axios/axios](https://github.com/axios/axios) | 109,295 | 14 | [`335e2f8464f5`](https://github.com/axios/axios/tree/335e2f8464f58a3ab485486b8d6964fea3e3f30a) | 4/4 | 4/4 | 2.18 | 2.07 | +| JavaScript | [react/create-react-app](https://github.com/react/create-react-app) | 103,230 | 15 | [`6254386531d2`](https://github.com/react/create-react-app/tree/6254386531d263688ccfa542d0e628fbc0de0b28) | 6/6 | 6/6 | 2.30 | 2.41 | +| TypeScript | [freeCodeCamp/freeCodeCamp](https://github.com/freeCodeCamp/freeCodeCamp) | 458,826 | 1 | [`11442d669447`](https://github.com/freeCodeCamp/freeCodeCamp/tree/11442d66944785ac72a2a2b9c955fae8b9f33083) | 5/6 | 6/6 | 14.84 | 11.84 | +| TypeScript | [vuejs/vue](https://github.com/vuejs/vue) | 212,624 | 7 | [`9e8870794008`](https://github.com/vuejs/vue/tree/9e88707940088cb1f4cd7dd210c9168a50dc347c) | 4/6 | 6/6 | 2.58 | 3.65 | +| TypeScript | [n8n-io/n8n](https://github.com/n8n-io/n8n) | 209,927 | 8 | [`93243edc00b2`](https://github.com/n8n-io/n8n/tree/93243edc00b288a840fbb562a5e9f64177f2337f) | 5/7 | 7/7 | 60.35 | 34.09 | +| TypeScript | [firecrawl/firecrawl](https://github.com/firecrawl/firecrawl) | 200,129 | 9 | [`40c8ff95d719`](https://github.com/firecrawl/firecrawl/tree/40c8ff95d719d2acbd8fd6273bd19b14a1c2bbb8) | 5/5 | 5/5 | 4.86 | 5.07 | +| TypeScript | [excalidraw/excalidraw](https://github.com/excalidraw/excalidraw) | 135,400 | 15 | [`701b901a0f1b`](https://github.com/excalidraw/excalidraw/tree/701b901a0f1ba3d0b390d9cdf63d112e25abac4b) | 7/7 | 7/7 | 3.94 | 3.51 | +| Python | [Significant-Gravitas/AutoGPT](https://github.com/Significant-Gravitas/AutoGPT) | 187,972 | 11 | [`9cb76f9026d3`](https://github.com/Significant-Gravitas/AutoGPT/tree/9cb76f9026d388c9bf2e098af2beae1cc864f055) | 5/6 | 6/6 | 14.63 | 12.74 | +| Python | [AUTOMATIC1111/stable-diffusion-webui](https://github.com/AUTOMATIC1111/stable-diffusion-webui) | 165,269 | 14 | [`82a973c04367`](https://github.com/AUTOMATIC1111/stable-diffusion-webui/tree/82a973c04367123ae98bd9abdf80d9eda9b910e2) | 3/3 | 3/3 | 1.60 | 2.37 | +| Python | [langgenius/dify](https://github.com/langgenius/dify) | 160,774 | 15 | [`fdb09a9e76dc`](https://github.com/langgenius/dify/tree/fdb09a9e76dc8cbd052fabd72cfff4071328d9bc) | 8/10 | 10/10 | 30.46 | 20.48 | +| Python | [langflow-ai/langflow](https://github.com/langflow-ai/langflow) | 156,479 | 16 | [`60999cc9a735`](https://github.com/langflow-ai/langflow/tree/60999cc9a7350e44dde3ef641baae864e10861f0) | 6/6 | 6/6 | 20.32 | 12.47 | +| Python | [Comfy-Org/ComfyUI](https://github.com/Comfy-Org/ComfyUI) | 145,983 | 20 | [`540a82ee722b`](https://github.com/Comfy-Org/ComfyUI/tree/540a82ee722bda1f32a1fbd97fcadadeb22d548b) | 4/4 | 4/4 | 3.49 | 2.72 | +| PHP | [coollabsio/coolify](https://github.com/coollabsio/coolify) | 65,849 | 2 | [`30335d80866f`](https://github.com/coollabsio/coolify/tree/30335d80866ff3391d899cf4ccb599b696f47b7e) | 5/5 | 5/5 | 5.30 | 4.69 | +| PHP | [nextcloud/server](https://github.com/nextcloud/server) | 37,456 | 3 | [`d655757fb306`](https://github.com/nextcloud/server/tree/d655757fb306aca893651958030b5dae69ed0f39) | 3/3 | 3/3 | 14.62 | 8.06 | +| PHP | [laravel/framework](https://github.com/laravel/framework) | 34,995 | 4 | [`338e44065c0a`](https://github.com/laravel/framework/tree/338e44065c0a5b18f1f290605b6253a1e69d8f56) | 4/5 | 5/5 | 6.33 | 4.31 | +| PHP | [filamentphp/filament](https://github.com/filamentphp/filament) | 32,462 | 5 | [`9d78093cfebd`](https://github.com/filamentphp/filament/tree/9d78093cfebdc96c077b1d8dbf94b41738f28992) | 4/4 | 4/4 | 16.63 | 14.29 | +| PHP | [symfony/symfony](https://github.com/symfony/symfony) | 31,523 | 6 | [`f07928484c9e`](https://github.com/symfony/symfony/tree/f07928484c9e96646964deb7b9570d7885492512) | 4/5 | 5/5 | 10.10 | 8.76 | +| Ruby | [rails/rails](https://github.com/rails/rails) | 59,087 | 1 | [`3a0aff317bc0`](https://github.com/rails/rails/tree/3a0aff317bc037f43f168e3772028f1033f1e810) | 2/3 | 3/3 | 7.90 | 5.01 | +| Ruby | [jekyll/jekyll](https://github.com/jekyll/jekyll) | 51,756 | 2 | [`97c3bdc822df`](https://github.com/jekyll/jekyll/tree/97c3bdc822df9d5bea4c269b29a28de5a5e29d70) | 2/2 | 2/2 | 2.42 | 2.31 | +| Ruby | [mastodon/mastodon](https://github.com/mastodon/mastodon) | 50,502 | 3 | [`5a207bc8f504`](https://github.com/mastodon/mastodon/tree/5a207bc8f5046e27cffc640a3c64f96504f6afcf) | 4/4 | 4/4 | 8.68 | 6.27 | +| Ruby | [huginn/huginn](https://github.com/huginn/huginn) | 50,454 | 4 | [`639c050b2bf4`](https://github.com/huginn/huginn/tree/639c050b2bf4e14ef9983012af99dfc1dc68b334) | 4/4 | 4/4 | 2.35 | 2.21 | +| Ruby | [discourse/discourse](https://github.com/discourse/discourse) | 48,066 | 6 | [`899c68d783e7`](https://github.com/discourse/discourse/tree/899c68d783e7808c50144752369a685220250d7c) | 3/5 | 5/5 | 35.98 | 20.48 | +| Go | [infiniflow/ragflow](https://github.com/infiniflow/ragflow) | 93,887 | 8 | [`ed5a84fee9ab`](https://github.com/infiniflow/ragflow/tree/ed5a84fee9abc8091aea111d1195cf7768541ff1) | 5/5 | 5/5 | 10.20 | 5.06 | +| Go | [gohugoio/hugo](https://github.com/gohugoio/hugo) | 90,425 | 9 | [`d2010fcc9cd3`](https://github.com/gohugoio/hugo/tree/d2010fcc9cd30c5b4e9274ded4485f4222326b53) | 4/4 | 4/4 | 4.85 | 3.63 | +| Go | [gin-gonic/gin](https://github.com/gin-gonic/gin) | 89,609 | 11 | [`0f09c3a9b462`](https://github.com/gin-gonic/gin/tree/0f09c3a9b4626d9fe9979cf4d0521d6ae32ab646) | 4/5 | 5/5 | 2.11 | 1.89 | +| Go | [netdata/netdata](https://github.com/netdata/netdata) | 81,330 | 14 | [`0d54b4424ea6`](https://github.com/netdata/netdata/tree/0d54b4424ea6f40c344738b9d566e1a26d160da3) | 4/4 | 4/4 | 23.42 | 10.38 | +| Go | [caddyserver/caddy](https://github.com/caddyserver/caddy) | 78,927 | 15 | [`9ea3108377d2`](https://github.com/caddyserver/caddy/tree/9ea3108377d2abec102866808133c40ac9582162) | 4/4 | 4/4 | 2.56 | 2.31 | +| Rust | [dani-garcia/vaultwarden](https://github.com/dani-garcia/vaultwarden) | 70,490 | 17 | [`17b159beff4b`](https://github.com/dani-garcia/vaultwarden/tree/17b159beff4b892f07f61f38ce2fe452e55251c1) | 6/6 | 6/6 | 2.51 | 3.03 | +| Rust | [meilisearch/meilisearch](https://github.com/meilisearch/meilisearch) | 59,822 | 29 | [`8c7bfcfa9bcf`](https://github.com/meilisearch/meilisearch/tree/8c7bfcfa9bcff7083bcb5d184fa2a523a23884d8) | 5/5 | 5/5 | 4.99 | 4.70 | +| Rust | [juspay/hyperswitch](https://github.com/juspay/hyperswitch) | 45,760 | 39 | [`5786b738c088`](https://github.com/juspay/hyperswitch/tree/5786b738c088b8452296ca19922fcdc0dc74e129) | 6/6 | 6/6 | 12.07 | 7.99 | +| Rust | [parcel-bundler/parcel](https://github.com/parcel-bundler/parcel) | 44,012 | 42 | [`20c0f2ede4c1`](https://github.com/parcel-bundler/parcel/tree/20c0f2ede4c1044092d44288169cce589c5d5c14) | 3/3 | 3/3 | 18.92 | 12.36 | +| Rust | [DioxusLabs/dioxus](https://github.com/DioxusLabs/dioxus) | 39,705 | 52 | [`17d9a5924e84`](https://github.com/DioxusLabs/dioxus/tree/17d9a5924e845e4f37deca3fada6c35d1d836af0) | 5/5 | 5/5 | 8.71 | 7.18 | +| C# | [jellyfin/jellyfin](https://github.com/jellyfin/jellyfin) | 59,164 | 3 | [`797999fc9bd4`](https://github.com/jellyfin/jellyfin/tree/797999fc9bd49a9f0853916e2933e3ad06173503) | β€” | 6/6 | β€” | 5.57 | +| C# | [dotnet/aspnetcore](https://github.com/dotnet/aspnetcore) | 38,507 | 8 | [`eeceb660128d`](https://github.com/dotnet/aspnetcore/tree/eeceb660128d3d00ea9b35effe4fad3fac3d42e3) | β€” | 5/5 | β€” | 135.12 | +| C# | [jasontaylordev/CleanArchitecture](https://github.com/jasontaylordev/CleanArchitecture) | 20,711 | 29 | [`e1ab4cc4d5d7`](https://github.com/jasontaylordev/CleanArchitecture/tree/e1ab4cc4d5d7b4e25e57bd8b0514b766573b8c04) | β€” | 6/6 | β€” | 2.65 | +| C# | [bitwarden/server](https://github.com/bitwarden/server) | 20,610 | 30 | [`04a4d7548b07`](https://github.com/bitwarden/server/tree/04a4d7548b07d18ab7327ddaae81a05fd893a820) | β€” | 6/6 | β€” | 10.03 | +| C# | [ardalis/CleanArchitecture](https://github.com/ardalis/CleanArchitecture) | 18,543 | 36 | [`da5491f1641f`](https://github.com/ardalis/CleanArchitecture/tree/da5491f1641f3d388e6cfc2c0ddd29f763746762) | β€” | 7/7 | β€” | 3.70 | +| Java | [macrozheng/mall](https://github.com/macrozheng/mall) | 85,023 | 3 | [`dcaa93b31503`](https://github.com/macrozheng/mall/tree/dcaa93b3150352e5044708d7211b5bed0af4509f) | β€” | 5/5 | β€” | 2.67 | +| Java | [spring-projects/spring-boot](https://github.com/spring-projects/spring-boot) | 81,788 | 4 | [`52dba7773c16`](https://github.com/spring-projects/spring-boot/tree/52dba7773c16cc5d145e59ad69de1b99e87896bb) | β€” | 5/5 | β€” | 44.52 | +| Java | [elastic/elasticsearch](https://github.com/elastic/elasticsearch) | 78,546 | 6 | [`97fdd221f591`](https://github.com/elastic/elasticsearch/tree/97fdd221f5919be472b60d29241f673774c19c51) | β€” | 4/4 | β€” | 52.31 | +| Java | [spring-projects/spring-framework](https://github.com/spring-projects/spring-framework) | 60,307 | 11 | [`cc8008c301bc`](https://github.com/spring-projects/spring-framework/tree/cc8008c301bc57b69c7c226254b720a2239741bf) | β€” | 5/5 | β€” | 9.90 | +| Java | [jeecgboot/JeecgBoot](https://github.com/jeecgboot/JeecgBoot) | 48,413 | 15 | [`49c29032e18d`](https://github.com/jeecgboot/JeecgBoot/tree/49c29032e18df784f1a75123d90647bc8f74cedb) | β€” | 5/5 | β€” | 10.83 | + +### C#/Java coverage and limits + +The new C# sample is Jellyfin, ASP.NET Core, both production Clean Architecture templates, and Bitwarden. The Java sample is mall, Spring Boot, Elasticsearch, Spring Framework, and JeecgBoot. SDK/TFM/language assertions are distinct; provider roles are checked on framework implementation packages rather than examples elsewhere in those repositories. + +MSBuild reads `.csproj`, `.sln`/`.slnx`, `global.json`, directory props/targets, in-repo imports and central package versions. Simple literal conditions and property substitutions are handled; arbitrary property functions and external SDK imports are not evaluated. Maven reads local matching parent coordinates, properties, managed versions, direct non-test dependencies, compiler configuration and modules. Remote parents/BOMs and profile activation are not fetched or evaluated. Gradle Groovy/Kotlin DSL support is static: literal plugins/dependencies, toolchains, module lists, directory/name overrides, conventional build filenames, SDKMAN/JDK pins and daemon JVM declarations. Custom/dynamic Gradle include logic remains diagnosed rather than executed; physically discovered project manifests are still reported. No JDK, .NET SDK, Maven, Gradle, or application packages were installed. + +Elasticsearch demonstrates that limit: its static server module, root workspace, source projects and build JDK are recognized, but arbitrary code-driven module expansion is not claimed complete. These sampled assertions do not certify every build condition or inferred deployment topology. ## Additional validation: Brainpod starters -Ran the release detector against all ten `case/*` branches of [`brainpodnl/starters`](https://github.com/brainpodnl/starters), pinning each commit below. The default branch is `case/bookstack`. This is a separate starter smoke test, not an expansion of the 173 ranked-corpus assertions above. +Ran the release detector against all ten `case/*` branches of [`brainpodnl/starters`](https://github.com/brainpodnl/starters), pinning each commit below. The default branch is `case/bookstack`. This is a separate starter smoke test, not part of the ranked-corpus assertions above. -Each branch was scanned five times without installing application dependencies or executing downloaded code. All ten scans succeeded and repeated byte-identically. Seven expected main stacks were recognized; Outline is partial because Koa has no rule, and eShop / Spring Petclinic are outside the supported language/manifest set. Auxiliary docs/assets must not be counted as detection of eShop’s main application. +Each branch was scanned five times without installing application dependencies or executing downloaded code. All ten scans succeeded and repeated byte-identically. The initial run recognized seven expected main stacks. After the C#/Java follow-up, nine main stacks are recognized; Outline remains partial because Koa has no rule. Auxiliary docs/assets must not be counted as detection of eShop’s main application. Timings are warm-filesystem, end-to-end CLI medians of runs 2–5. Empty or partial output is not treated as an accuracy pass merely because it is fast. | Branch | Revision | Result | Median ms | |---|---|---|---:| | `case/bookstack` | [`6265d5667dd6`](https://github.com/brainpodnl/starters/tree/6265d5667dd6904b7aa0d05a8c60795cf104011b) | Laravel, Composer, JavaScript/TypeScript. PHP `^8.2.0` and Laravel `^v12.26.4` declarations retained. | 4.07 | -| `case/eshop` | [`f21d6233b851`](https://github.com/brainpodnl/starters/tree/f21d6233b851897371c25c5e7ffa67bb8552dba8) | **Unsupported primary stack:** .NET 10/C# not detected. Only the docs Gemfile and JavaScript assets appear. | 2.54 | +| `case/eshop` | [`f21d6233b851`](https://github.com/brainpodnl/starters/tree/f21d6233b851897371c25c5e7ffa67bb8552dba8) | **Now detected:** .NET solution, ASP.NET Core projects, declared SDK `10.0.0` and `net10.0` targets; docs/assets remain separate. | 3.14 | | `case/fastapi-fullstack` | [`84c5a9e11a9e`](https://github.com/brainpodnl/starters/tree/84c5a9e11a9e58262599e3d772f5386924e02ca7) | FastAPI backend, React/Vite frontend, npm and uv workspaces. Python `3.14` pin plus `>=3.14,<4.0` constraint retained; all three workspace members found. | 2.44 | | `case/linkding` | [`24b5ad6cc9bd`](https://github.com/brainpodnl/starters/tree/24b5ad6cc9bde497b5d1b1e86aed5a2fb7d25c2f) | Django backend (`python >=3.13`) and Astro documentation project detected. | 2.52 | | `case/maybe` | [`224f21354a91`](https://github.com/brainpodnl/starters/tree/224f21354a916dfbec29e9435db1045b3d61a94b) | Rails and Ruby `3.4.4` pin detected. Non-fatal `dynamic ruby declaration not evaluated` warning for `ruby file: ".ruby-version"`. | 3.09 | | `case/miniflux` | [`faecfa2fac61`](https://github.com/brainpodnl/starters/tree/faecfa2fac61a91f4f69550bb1afd506c4b51749) | Go module `miniflux.app/v2`, Go `1.26.0` requirement detected; auxiliary scripts/assets are separate source projects. | 2.47 | | `case/outline` | [`09f11188b87b`](https://github.com/brainpodnl/starters/tree/09f11188b87b0220fa318dfa6fdd0fa9772195dd) | **Partial:** Node/React/Vite, TypeScript, Node engine constraint and `.nvmrc` pin detected. Declared Koa `^3.0.3` backend is not labeled. | 4.06 | | `case/saas-starter` | [`6e33e58b1e55`](https://github.com/brainpodnl/starters/tree/6e33e58b1e553a41fe22e6b941a7229a002de361) | Next.js/React and TypeScript detected. No package name or Node version is invented when absent from supported manifest fields. | 2.05 | -| `case/spring-petclinic` | [`500158f73241`](https://github.com/brainpodnl/starters/tree/500158f732419217507c7656904b8e6aa1bcc0d6) | **Unsupported primary stack:** Java 17 / Spring Boot `4.1.0` declared in Maven, but the detector returns no projects. | 2.16 | +| `case/spring-petclinic` | [`500158f73241`](https://github.com/brainpodnl/starters/tree/500158f732419217507c7656904b8e6aa1bcc0d6) | **Now detected:** Maven, Spring Boot parent `4.1.0`, and Java `17` declaration. | 1.89 | | `case/uptime-kuma` | [`df601c58d920`](https://github.com/brainpodnl/starters/tree/df601c58d9205562f34cfe5ef35264b341019d19) | Express/Vue/Vite and Node `>=20.4.0` constraint detected, alongside auxiliary example projects. | 2.60 | -Koa needs an additional npm framework rule. C#/.NET and Java/Spring require new language and manifest support (`global.json` / MSBuild and Maven, respectively). The Maybe warning is redundant with the already detected Ruby pin. None of these observations was silently converted into a passing assertion or hidden by skipping a branch. The detector was unchanged during this validation. +Koa needs an additional npm framework rule. C#/.NET and Java/Spring are now supported and verified by the follow-up run. The Maybe warning is redundant with the already detected Ruby pin. None of these observations was silently converted into a passing assertion or hidden by skipping a branch. The initial validation was read-only; the follow-up uses the new language support. + +## CI regression repaired in a separate session + +The Intel macOS failure was `s3::tests::streaming_download_exceeds_small_copy_limit_without_credentials_or_summary`, not a detector or build-workflow defect. The fixture imposed a five-second total deadline on a 26 MiB localhost TLS transfer. A controlled 60 ms / 256 KiB throttled reproduction failed at 5.02 s with `Body TimedOut`; replacing that total deadline with five-second connect and read-inactivity limits let the same transfer pass in 6.33 s. A separately stalled response still failed after five seconds. Only the test fixture client changed; size, redirect, and credential assertions remain intact, and no CI checks were disabled. Linux release tests/Clippy passed; native macOS verification depends on the new GitHub Actions run. ## Reproduce diff --git a/tests/test_audit_detect.py b/tests/test_audit_detect.py index 35dc673..ff6dbd7 100644 --- a/tests/test_audit_detect.py +++ b/tests/test_audit_detect.py @@ -59,6 +59,30 @@ def test_checks_are_scoped_to_the_expected_project(self): with self.assertRaises(ValueError): audit_detect.check({".": self.project}, {"kind": "unknown", "path": "."}) + def test_role_and_version_kind_expectations_are_enforced(self): + self.project["detections"][0]["roles"] = ["uses"] + expected = { + "kind": "detection", + "path": ".", + "id": "node", + "roles": ["provides"], + } + self.assertFalse(audit_detect.check({".": self.project}, expected)[0]) + expected["roles"] = ["uses"] + self.assertTrue(audit_detect.check({".": self.project}, expected)[0]) + self.project["languages"][0]["versions"][0]["kind"] = "sdk" + expected = { + "kind": "version", + "path": ".", + "language": "javascript", + "value": ">=22", + "source": "package.json", + "versionKind": "language", + } + self.assertFalse(audit_detect.check({".": self.project}, expected)[0]) + expected["versionKind"] = "sdk" + self.assertTrue(audit_detect.check({".": self.project}, expected)[0]) + def archive(self, path="repo/package.json", size=None): data = b"{}" if size is None else b"x" * size output = io.BytesIO() From 02219a518d1a48170814949ba024c2929d9f2e3c Mon Sep 17 00:00:00 2001 From: Dillen Meijboom Date: Tue, 10 Nov 2026 00:24:02 +0000 Subject: [PATCH 07/10] fix: stabilize TLS streaming fixture on macOS --- .github/workflows/build.yml | 2 +- src/s3/multipart.rs | 17 ++++++++++++----- src/s3/tests.rs | 20 +++++++++++--------- tests/detect-corpus.md | 8 ++++++-- 4 files changed, 30 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9e8f464..5542884 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -58,7 +58,7 @@ jobs: gc-max-store-size: 2G - name: Build - run: nix build ".#packages.${{ matrix.system }}.${{ matrix.package }}" + run: nix build --print-build-logs ".#packages.${{ matrix.system }}.${{ matrix.package }}" - name: Package Unix binary if: matrix.package == 'default' diff --git a/src/s3/multipart.rs b/src/s3/multipart.rs index 9c2a74e..40f5208 100644 --- a/src/s3/multipart.rs +++ b/src/s3/multipart.rs @@ -269,11 +269,18 @@ impl Gateway<'_> { ) -> Result { let mut response = self.provider_response(remote, true).await?; let mut size = 0u64; - while let Some(chunk) = response - .chunk() - .await - .map_err(|_| anyhow!("provider download interrupted; output may be partial"))? - { + while let Some(chunk) = response.chunk().await.map_err(|error| { + #[cfg(test)] + eprintln!( + "provider body failed after {size} bytes: timeout={}, body={}, error={:?}", + error.is_timeout(), + error.is_body(), + error.without_url() + ); + #[cfg(not(test))] + let _ = error; + anyhow!("provider download interrupted; output may be partial") + })? { output .write_all(&chunk) .await diff --git a/src/s3/tests.rs b/src/s3/tests.rs index 9eed01f..e0ef7d7 100644 --- a/src/s3/tests.rs +++ b/src/s3/tests.rs @@ -432,7 +432,9 @@ fn provider(body: &[u8], declared_length: usize) -> (String, std::thread::JoinHa "HTTP/1.1 200 OK\r\nContent-Length: {declared_length}\r\nConnection: close\r\n\r\n" ) .unwrap(); - stream.write_all(&body).unwrap(); + for chunk in body.chunks(16 * 1024) { + stream.write_all(chunk).unwrap(); + } stream.flush().unwrap(); String::from_utf8(request).unwrap() }); @@ -861,14 +863,14 @@ async fn streaming_download_exceeds_small_copy_limit_without_credentials_or_summ mock(vec![(307, vec![("location", url)], String::new())]).await; trust_provider(&mut gateway); let mut output = tokio::io::sink(); - assert_eq!( - gateway - .download_stream(&streaming_remote(), &mut output) - .await - .unwrap(), - body.len() as u64 - ); - let request = provider.join().unwrap().to_ascii_lowercase(); + let result = gateway + .download_stream(&streaming_remote(), &mut output) + .await; + let provider_result = tokio::task::spawn_blocking(move || provider.join()) + .await + .unwrap(); + assert_eq!(result.unwrap(), body.len() as u64); + let request = provider_result.unwrap().to_ascii_lowercase(); assert!(!request.contains("authorization:")); assert!(!request.contains("test-key")); assert_eq!(requests.lock().unwrap().len(), 1); diff --git a/tests/detect-corpus.md b/tests/detect-corpus.md index c20df03..6c763de 100644 --- a/tests/detect-corpus.md +++ b/tests/detect-corpus.md @@ -130,9 +130,13 @@ Timings are warm-filesystem, end-to-end CLI medians of runs 2–5. Empty or part Koa needs an additional npm framework rule. C#/.NET and Java/Spring are now supported and verified by the follow-up run. The Maybe warning is redundant with the already detected Ruby pin. None of these observations was silently converted into a passing assertion or hidden by skipping a branch. The initial validation was read-only; the follow-up uses the new language support. -## CI regression repaired in a separate session +## CI regression investigation -The Intel macOS failure was `s3::tests::streaming_download_exceeds_small_copy_limit_without_credentials_or_summary`, not a detector or build-workflow defect. The fixture imposed a five-second total deadline on a 26 MiB localhost TLS transfer. A controlled 60 ms / 256 KiB throttled reproduction failed at 5.02 s with `Body TimedOut`; replacing that total deadline with five-second connect and read-inactivity limits let the same transfer pass in 6.33 s. A separately stalled response still failed after five seconds. Only the test fixture client changed; size, redirect, and credential assertions remain intact, and no CI checks were disabled. Linux release tests/Clippy passed; native macOS verification depends on the new GitHub Actions run. +The Intel macOS failure is `s3::tests::streaming_download_exceeds_small_copy_limit_without_credentials_or_summary`, not a detector compilation failure. Replacing the fixture's five-second total deadline with connect/read-inactivity limits fixes a controlled slow-transfer failure, but native run `42725208393` still failed the same test after a 7.18-second suite. The original timeout diagnosis was therefore insufficient to establish the macOS cause; all other platform jobs passed. + +Follow-up controls show a progressing transfer can exceed five seconds while retaining the five-second read-inactivity bound. Record-sized 16 KiB TLS writes are substantially faster than the original 64 KiB write-all progression locally, but Linux cannot prove the native macOS cause. The fixture now uses those record-sized writes and joins its provider asynchronously before unwrapping the download result, so server-side I/O panics are captured without starving Tokio shutdown. Transfer-size, redirect, and credential assertions are unchanged. No timeouts were increased and no checks were disabled. + +Nix now streams full build logs, and test-only body failures report received bytes and URL-stripped error details. Production error messages remain sanitized and unchanged. Native macOS confirmation is still required for this follow-up. ## Reproduce From 40829ade521edd5d25335f4911f087a0a50a3655 Mon Sep 17 00:00:00 2001 From: Dillen Meijboom Date: Tue, 10 Nov 2026 07:23:13 +0000 Subject: [PATCH 08/10] test: measure XML detection performance --- src/detect/xml.rs | 178 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 178 insertions(+) diff --git a/src/detect/xml.rs b/src/detect/xml.rs index 02a8ecc..e099e85 100644 --- a/src/detect/xml.rs +++ b/src/detect/xml.rs @@ -169,3 +169,181 @@ fn unescape(input: &str) -> Result { output.push_str(rest); Ok(output) } + +#[cfg(test)] +mod tests { + use std::fs::{self, File}; + use std::hint::black_box; + use std::io::Read; + use std::path::{Path, PathBuf}; + use std::time::{Duration, Instant}; + + use anyhow::{Result, ensure}; + + use super::read; + use crate::detect::{MAX_FILE_BYTES, inventory, rules::Rules}; + + fn kind(name: &str) -> Option<&'static str> { + if name == "pom.xml" { + Some("pom") + } else { + ["csproj", "props", "targets", "slnx"] + .into_iter() + .find(|ext| name.ends_with(&format!(".{ext}"))) + } + } + + fn files(root: &Path) -> Result> { + let inventory = inventory(root, Rules::builtin()?, 1)?; + for diagnostic in &inventory.diagnostics { + eprintln!("inventory: {}: {}", diagnostic.path, diagnostic.message); + } + Ok(inventory + .nodes + .into_iter() + .flat_map(|node| { + node.contents.into_iter().filter_map(move |(name, text)| { + kind(&name).map(|kind| (root.join(&node.path).join(name), text, kind)) + }) + }) + .collect()) + } + + fn bounded_read(path: &Path) -> Result { + let mut bytes = Vec::new(); + File::open(path)? + .take(MAX_FILE_BYTES + 1) + .read_to_end(&mut bytes)?; + ensure!(bytes.len() <= MAX_FILE_BYTES as usize, "manifest too large"); + Ok(String::from_utf8(bytes)?) + } + + fn timed(mut work: impl FnMut() -> Result<()>) -> Result<(usize, Duration)> { + let start = Instant::now(); + let mut iterations = 0; + loop { + work()?; + iterations += 1; + if start.elapsed() >= Duration::from_millis(200) { + return Ok((iterations, start.elapsed())); + } + } + } + + fn benchmark(label: &str, files: &[(PathBuf, String, &'static str)]) -> Result<()> { + for kind in ["csproj", "props", "targets", "slnx", "pom", "all"] { + let selected: Vec<_> = files + .iter() + .filter(|(_, _, ext)| kind == "all" || *ext == kind) + .collect(); + if selected.is_empty() { + continue; + } + let bytes: usize = selected.iter().map(|(_, text, _)| text.len()).sum(); + let mut errors = 0; + let mut elements = 0; + for (path, text, _) in &selected { + if let Err(error) = read(text, |stack| { + elements += 1; + black_box(stack); + Ok(()) + }) { + errors += 1; + if kind != "all" { + eprintln!("parse: {}: {error}", path.display()); + } + } + } + let (reads, read_elapsed) = timed(|| { + for (path, _, _) in &selected { + black_box(bounded_read(path)?); + } + Ok(()) + })?; + let (parses, parse_elapsed) = timed(|| { + for (_, text, _) in &selected { + let result = read(black_box(text), |stack| { + black_box(stack); + Ok(()) + }); + black_box(result.is_ok()); + } + Ok(()) + })?; + eprintln!( + "{label} {kind}: files={} bytes={bytes} elements={elements} errors={errors} read_us={:.2} parse_us={:.2} read_iters={reads} parse_iters={parses}", + selected.len(), + read_elapsed.as_secs_f64() * 1e6 / reads as f64, + parse_elapsed.as_secs_f64() * 1e6 / parses as f64, + ); + } + Ok(()) + } + + #[test] + #[ignore = "local XML I/O and parser microbenchmark; run release with --ignored --nocapture"] + fn benchmark_xml_local() -> Result<()> { + let root = tempfile::tempdir()?; + let item = "a & b"; + let project = format!( + "{}", + item.repeat(128) + ); + for name in [ + "app.csproj", + "Directory.Build.props", + "Directory.Build.targets", + ] { + fs::write(root.path().join(name), &project)?; + } + fs::write( + root.path().join("app.slnx"), + "", + )?; + fs::write( + root.path().join("pom.xml"), + format!( + "4.0.0{}", + "21".repeat(128) + ), + )?; + for dir in ["target", "bin", "obj"] { + fs::create_dir(root.path().join(dir))?; + fs::write(root.path().join(dir).join("ignored.csproj"), &project)?; + } + fs::write( + root.path().join("oversized.props"), + vec![b' '; MAX_FILE_BYTES as usize + 1], + )?; + let files = files(root.path())?; + assert_eq!(files.len(), 5); + benchmark("local-synthetic", &files) + } + + #[test] + #[ignore = "set XML_BENCH_CACHE to pinned detect-corpus cache; run release with --ignored --nocapture"] + fn benchmark_xml_corpus() -> Result<()> { + let cache = PathBuf::from(std::env::var("XML_BENCH_CACHE")?); + let manifest: serde_json::Value = + serde_json::from_str(include_str!("../../tests/detect-corpus.json"))?; + let mut all = Vec::new(); + let entries = manifest["repositories"].as_array().unwrap(); + for entry in entries { + let repository = entry["repository"].as_str().unwrap(); + let name = repository.replace('/', "--"); + let metadata: serde_json::Value = serde_json::from_str(&fs::read_to_string( + cache.join("metadata").join(format!("{name}.json")), + )?)?; + let revision = metadata.get("revision").or_else(|| metadata.get("sha")); + ensure!( + revision == Some(&entry["revision"]), + "unpinned {repository}" + ); + let selected = files(&cache.join("repos").join(name))?; + benchmark(repository, &selected)?; + all.extend(selected); + } + eprintln!("corpus repositories={}", entries.len()); + benchmark("corpus-total", &all) + } +} From 7e17a8859f7cd972179d06d2eb6a3c8e72b2dc77 Mon Sep 17 00:00:00 2001 From: Dillen Meijboom Date: Tue, 10 Nov 2026 07:16:12 +0000 Subject: [PATCH 09/10] refactor: drive manifest discovery and fact extraction from declarative rules --- README.md | 68 +++- src/detect/catalog.rs | 717 +++++++++++++++++++++++++++++++++++++ src/detect/dotnet.rs | 179 ++++----- src/detect/gradle.rs | 55 +-- src/detect/java.rs | 32 +- src/detect/mod.rs | 99 ++--- src/detect/parsers.rs | 484 ++++++------------------- src/detect/rules.rs | 519 ++++++++++++++++----------- src/detect/rules.toml | 469 ++++++++++++++++++++++++ src/detect/tests.rs | 158 ++++++++ src/detect/xml.rs | 5 +- tests/audit_detect.py | 25 +- tests/detect-corpus.md | 40 ++- tests/test_audit_detect.py | 38 ++ 14 files changed, 2041 insertions(+), 847 deletions(-) create mode 100644 src/detect/catalog.rs diff --git a/README.md b/README.md index 95868f5..1363b66 100644 --- a/README.md +++ b/README.md @@ -278,7 +278,9 @@ The result contains sorted project paths, manifest names, languages and version Built-in coverage includes Laravel, Symfony, CakePHP, CodeIgniter, Composer; Go modules/workspaces and Gin/Echo/Fiber; Cargo workspaces, axum, Rocket, Actix; Bundler, Rails, Sinatra, Hanami; Python requirements, PEP 621, Poetry, Pipenv, uv workspaces, Django, FastAPI, Flask, Starlette, Celery; npm/pnpm workspaces and catalogs, Next.js, Nuxt, Astro, SvelteKit, Remix, Angular, Express, React, Vue, Vite; C#/.NET projects and solutions, ASP.NET Core/classic, Aspire, FastEndpoints; Java/JVM Maven and Gradle projects, Spring/Spring Boot, Quarkus, Micronaut; and standalone source files in all nine languages. Rust workspace dependency aliases and inherited `rust-version` are understood. Ancestor runtime pin/toolchain files are included with provenance. Poetry names, literal gemspec metadata, and Rails component dependencies are recognized. pnpm catalog references retain the original declaration and add the locally resolved constraint with its YAML source; unresolved references produce diagnostics. -Rules live in [`src/detect/rules.toml`](src/detect/rules.toml), independently of manifest parsers. Additional rules use unique IDs: +The engine has three layers: bounded discovery, format readers that emit facts, and compiled detection rules. [`src/detect/rules.toml`](src/detect/rules.toml) owns manifest filenames/readers, JSON/TOML field projections, version/property mappings, workspace candidates, source-language families, inheritance boundaries, and directory exclusions. Format-specific syntax and local-reference resolution stay in adapters; framework decisions do not. + +Additional framework rules use unique IDs: ```toml [[rules]] @@ -290,9 +292,34 @@ all = ["dep:npm:next", "file:company.config.*"] none = ["file:legacy.config"] ``` -`all` requires every pattern, `any` requires at least one when supplied, and `none` rejects any match. Higher priorities sort first; ties sort by ID. Matches coexist rather than hiding other frameworks or languages. Patterns also accept conjunction groups, for example `any = ["dep:gem:rails", { all = ["dep:gem:actionpack", "dep:gem:railties"] }]`. Patterns support `file:` (`*`, `?`), `source:`, `dep::`, `identity::`, `workspace:`, and `package:`. Workspace paths additionally support `**`. Ecosystems include `composer`, `go`, `cargo`, `gem`, `pypi`, `npm`, `nuget`, `dotnet-sdk`, `dotnet-framework`, `dotnet-reference`, `maven`, `maven-parent`, `maven-plugin`, and `gradle-plugin`. +A new JSON/TOML manifest can also be registered without another Rust parser or filename branch. For example, `--rules custom.toml` can add a project format and runtime declaration: -The scanner walks once, caches indexed rules, and never reads application source bodies or lockfiles. Wide trees use at most four directory workers; small trees stay serial. Directory facts are indexed and irrelevant empty directories are discarded, while output stays deterministic. pnpm YAML is deserialized directly into typed fields without building a syntax tree; unused fields/comments are not retained. XML uses the existing zero-allocation pull tokenizer, with structure/namespace checks and DTD/custom-entity rejection. Maven local parents and managed versions, MSBuild directory props/targets, central package versions, and static Gradle plugin/dependency/module declarations are resolved without running build tools. Arbitrary MSBuild functions, remote Maven parents/BOMs, profile activation, and dynamic Gradle build logic are not executed or guessed; unresolved version references remain explicit. Each manifest/version file is capped at 1 MiB. Symlinks are not followed; traversal stops at depth 128 with a diagnostic. Dependency/build directories such as `.git`, `.jj`, `node_modules`, `vendor`, `target`, `.venv`, `.next`, `dist`, `build`, `.gradle`, and `.vs` are pruned. `bin`/`obj` are additionally pruned beneath C# project roots without skipping Rust `src/bin` sources. It does not evaluate `.gitignore`. Source-only directories roll up to a same-language project ancestor when available; sibling services keep their own manifest boundaries. +```toml +[[catalog.manifests]] +id = "service-meta" +files = ["service.meta.json"] +reader = "json" +package = "service-meta" +names = ["/name"] + +[[catalog.manifests.versions]] +language = "javascript" +kind = "requirement" +paths = ["/runtime/node"] + +[[rules]] +id = "service-meta" +language = "javascript" +kind = "project" +priority = 10 +any = ["package:service-meta"] +``` + +The same catalog supports identity/dependency extraction and workspace members. JSON pointers select fields; schemas reject unknown options/readers and conflicting definitions. MSBuild/Maven/Gradle property-to-version mappings and MSBuild reference item kinds are declarative too. + +`all` requires every pattern, `any` requires at least one when supplied, and `none` rejects any match. Higher priorities sort first; ties sort by ID. Matches coexist rather than hiding other frameworks or languages. Patterns also accept conjunction groups, for example `any = ["dep:gem:rails", { all = ["dep:gem:actionpack", "dep:gem:railties"] }]`. Patterns support `file:` (`*`, `?`), `source:`, `dep::`, `identity::`, `workspace:`, and `package:`. Workspace paths additionally support `**`. Ecosystems include `composer`, `go`, `cargo`, `gem`, `pypi`, `npm`, `nuget`, `dotnet-sdk`, `dotnet-framework`, `dotnet-reference`, `maven`, `maven-parent`, `maven-plugin`, and `gradle-plugin`. + +Rule atoms and basename patterns are compiled once into typed fact indexes and fast exact/prefix/suffix matchers. The scanner walks once, caches the catalog, and never reads application source bodies or lockfiles. Wide trees use at most four directory workers; small trees stay serial. Directory facts are indexed and irrelevant empty directories are discarded, while output stays deterministic. pnpm YAML is deserialized directly into typed fields without building a syntax tree; unused fields/comments are not retained. XML uses the existing zero-allocation pull tokenizer, with structure/namespace checks and DTD/custom-entity rejection. Maven local parents and managed versions, MSBuild directory props/targets, central package versions, and static Gradle plugin/dependency/module declarations are resolved without running build tools. Arbitrary MSBuild functions, remote Maven parents/BOMs, profile activation, and dynamic Gradle build logic are not executed or guessed; unresolved version references remain explicit. Each manifest/version file is capped at 1 MiB. Symlinks are not followed; traversal stops at depth 128 with a diagnostic. Dependency/build directories such as `.git`, `.jj`, `node_modules`, `vendor`, `target`, `.venv`, `.next`, `dist`, `build`, `.gradle`, and `.vs` are pruned. `bin`/`obj` are additionally pruned beneath C# project roots without skipping Rust `src/bin` sources. It does not evaluate `.gitignore`. Source-only directories roll up to a same-language project ancestor when available; sibling services keep their own manifest boundaries. Detection uses declared dependencies, not arbitrary substring matches or inferred framework usage from source. Composer/Cargo development dependencies are excluded; npm development dependencies are included because tools such as Astro commonly live there. Dynamic Ruby declarations and unsupported requirements options produce diagnostics instead of executing code. Invalid manifests are diagnosed without stopping discovery of other projects. @@ -302,7 +329,40 @@ Run the generated large-tree performance fixture with: cargo test --release detect::tests::benchmark_polyglot_scan -- --ignored --nocapture ``` -A [45-repository GitHub audit](tests/detect-corpus.md) records pinned real-world coverage, before/after timings, regression fixes, and an opt-in reproducible runner. +### Reproduce the repository checks + +The corpus is public and pinned to commit SHAs in [`tests/detect-corpus.json`](tests/detect-corpus.json). The [audit report](tests/detect-corpus.md) contains assertions, per-repository timings, known limitations, and the additional Brainpod starter branches. + +| Language | Repositories | +|---|---| +| JavaScript | [react/react](https://github.com/react/react), [vercel/next.js](https://github.com/vercel/next.js), [mrdoob/three.js](https://github.com/mrdoob/three.js), [axios/axios](https://github.com/axios/axios), [react/create-react-app](https://github.com/react/create-react-app) | +| TypeScript | [freeCodeCamp/freeCodeCamp](https://github.com/freeCodeCamp/freeCodeCamp), [vuejs/vue](https://github.com/vuejs/vue), [n8n-io/n8n](https://github.com/n8n-io/n8n), [firecrawl/firecrawl](https://github.com/firecrawl/firecrawl), [excalidraw/excalidraw](https://github.com/excalidraw/excalidraw) | +| Python | [Significant-Gravitas/AutoGPT](https://github.com/Significant-Gravitas/AutoGPT), [AUTOMATIC1111/stable-diffusion-webui](https://github.com/AUTOMATIC1111/stable-diffusion-webui), [langgenius/dify](https://github.com/langgenius/dify), [langflow-ai/langflow](https://github.com/langflow-ai/langflow), [Comfy-Org/ComfyUI](https://github.com/Comfy-Org/ComfyUI) | +| PHP | [coollabsio/coolify](https://github.com/coollabsio/coolify), [nextcloud/server](https://github.com/nextcloud/server), [laravel/framework](https://github.com/laravel/framework), [filamentphp/filament](https://github.com/filamentphp/filament), [symfony/symfony](https://github.com/symfony/symfony) | +| Ruby | [rails/rails](https://github.com/rails/rails), [jekyll/jekyll](https://github.com/jekyll/jekyll), [mastodon/mastodon](https://github.com/mastodon/mastodon), [huginn/huginn](https://github.com/huginn/huginn), [discourse/discourse](https://github.com/discourse/discourse) | +| Go | [infiniflow/ragflow](https://github.com/infiniflow/ragflow), [gohugoio/hugo](https://github.com/gohugoio/hugo), [gin-gonic/gin](https://github.com/gin-gonic/gin), [netdata/netdata](https://github.com/netdata/netdata), [caddyserver/caddy](https://github.com/caddyserver/caddy) | +| Rust | [dani-garcia/vaultwarden](https://github.com/dani-garcia/vaultwarden), [meilisearch/meilisearch](https://github.com/meilisearch/meilisearch), [juspay/hyperswitch](https://github.com/juspay/hyperswitch), [parcel-bundler/parcel](https://github.com/parcel-bundler/parcel), [DioxusLabs/dioxus](https://github.com/DioxusLabs/dioxus) | +| C# | [jellyfin/jellyfin](https://github.com/jellyfin/jellyfin), [dotnet/aspnetcore](https://github.com/dotnet/aspnetcore), [jasontaylordev/CleanArchitecture](https://github.com/jasontaylordev/CleanArchitecture), [bitwarden/server](https://github.com/bitwarden/server), [ardalis/CleanArchitecture](https://github.com/ardalis/CleanArchitecture) | +| Java | [macrozheng/mall](https://github.com/macrozheng/mall), [spring-projects/spring-boot](https://github.com/spring-projects/spring-boot), [elastic/elasticsearch](https://github.com/elastic/elasticsearch), [spring-projects/spring-framework](https://github.com/spring-projects/spring-framework), [jeecgboot/JeecgBoot](https://github.com/jeecgboot/JeecgBoot) | + +```sh +python3 tests/audit_detect.py --list +cargo build --release --locked +cache=$(mktemp -d) +python3 tests/audit_detect.py --fetch --cache "$cache" --output "$cache/results.json" +``` + +Repeat offline by reusing the cache and omitting `--fetch`. + +XML timing is independently reproducible (warm bounded reads/UTF-8 versus preloaded pull parsing; excludes discovery, fact extraction, and CLI output): + +```sh +XML_BENCH_CACHE="$cache" cargo test --release detect::xml::tests::benchmark_xml_corpus \ + -- --ignored --nocapture --test-threads=1 +cargo test --release detect::xml::tests::benchmark_xml_local -- --ignored --nocapture +``` + +On the pinned corpus, 1,192 XML manifests (2.7 MB) took about **2.7 ms to read** and **7.4 ms to parse** in aggregate on the measured warm-cache machine. This is not a cold-disk or whole-engine latency claim. No XML parser replacement was justified by those measurements. ## Image discovery diff --git a/src/detect/catalog.rs b/src/detect/catalog.rs new file mode 100644 index 0000000..b9fe551 --- /dev/null +++ b/src/detect/catalog.rs @@ -0,0 +1,717 @@ +use std::collections::{BTreeMap, BTreeSet, HashMap}; + +use anyhow::{Result, bail}; +use serde::Deserialize; +use serde_json::Value; + +use super::rules::{FilePattern, glob}; +use super::values::{self, Located, Properties, Syntax}; +use super::{Node, Version, Workspace, relative_file}; + +#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +pub(super) enum Reader { + Json, + Toml, + Pnpm, + Go, + Gemfile, + Gemspec, + Requirements, + Pin, + Tools, + Properties, + Msbuild, + Solution, + Maven, + Gradle, + GradleProperties, + GradleJvm, + VersionCatalog, +} + +#[derive(Clone, Copy, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +pub(super) enum Resolver { + Cargo, +} + +#[derive(Default, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub(super) struct CatalogSpec { + manifests: Vec, + scan: Scan, + families: BTreeMap, + inheritance: Vec, + workspaces: Vec, + property_versions: Vec, + reference_kinds: BTreeMap, +} + +#[derive(Default, Deserialize)] +#[serde(default, deny_unknown_fields)] +struct Scan { + ignore: Vec, + prune: Vec, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Prune { + when_files: Vec, + directories: Vec, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Inheritance { + kind: String, + #[serde(default)] + boundary_files: Vec, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct ReferenceKind { + pub ecosystem: String, + #[serde(default)] + pub catalog: bool, + #[serde(default)] + normalize: Normalize, + separator: Option, +} + +impl ReferenceKind { + pub fn name(&self, value: &str) -> String { + self.normalize.name( + self.separator + .and_then(|separator| value.split(separator).next()) + .unwrap_or(value) + .trim(), + ) + } +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct WorkspaceMembers { + ecosystem: String, + #[serde(default)] + files: Vec, + package: Option, +} + +#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq, Hash)] +#[serde(rename_all = "kebab-case")] +pub(super) enum PropertyReader { + Msbuild, + Maven, + MavenCompiler, + Gradle, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct PropertyVersion { + reader: PropertyReader, + fields: Vec, + pub language: String, + pub kind: String, + split: Option, +} + +impl PropertyVersion { + pub fn emit(&self, node: &mut Node, raw: &Located, properties: &Properties, syntax: Syntax) { + for value in self.split.filter(|_| !raw.conditional).map_or_else( + || vec![raw.value.as_str()], + |split| raw.value.split(split).collect(), + ) { + if !value.trim().is_empty() { + values::version( + node, + &self.language, + &Located { + value: value.trim().to_owned(), + ..raw.clone() + }, + &self.kind, + properties, + syntax, + ); + } + } + } +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Manifest { + id: String, + files: Vec, + pub reader: Reader, + pub resolver: Option, + pub package: Option, + package_path: Option, + #[serde(default)] + names: Vec, + identity: Option, + #[serde(default)] + dependencies: Vec, + #[serde(default)] + versions: Vec, + #[serde(default)] + workspaces: Vec, + pub workspace: Option, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Identity { + ecosystem: String, + paths: Vec, + #[serde(default)] + versions: Vec, + #[serde(default)] + normalize: Normalize, +} + +#[derive(Clone, Copy, Default, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(super) enum Normalize { + #[default] + None, + Python, + Lowercase, +} + +impl Normalize { + pub fn name(self, name: &str) -> String { + match self { + Self::None => name.to_owned(), + Self::Lowercase => name.to_ascii_lowercase(), + Self::Python => { + let mut result = String::new(); + for ch in name.chars() { + if matches!(ch, '-' | '_' | '.') { + if !result.ends_with('-') { + result.push('-'); + } + } else { + result.push(ch.to_ascii_lowercase()); + } + } + result + } + } + } +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Dependencies { + ecosystem: String, + paths: Vec, + #[serde(default)] + exclude: Vec, + #[serde(default)] + normalize: Normalize, + #[serde(default)] + requirements: bool, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct VersionField { + language: String, + kind: String, + #[serde(default)] + paths: Vec, + dependency: Option, + key: Option, + reference_prefix: Option, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct WorkspaceField { + ecosystem: String, + present: String, + members: Vec, + #[serde(default)] + exclude: Vec, +} + +pub(super) struct Catalog { + spec: CatalogSpec, + exact: HashMap>, + wildcard: Vec<(FilePattern, usize)>, + extensions: HashMap>, + fields: HashMap>, + ignored: BTreeSet, + readable_extensions: BTreeSet, +} + +impl CatalogSpec { + pub fn extend(&mut self, extra: Self) { + self.manifests.extend(extra.manifests); + self.scan.ignore.extend(extra.scan.ignore); + self.scan.prune.extend(extra.scan.prune); + self.families.extend(extra.families); + self.inheritance.extend(extra.inheritance); + self.workspaces.extend(extra.workspaces); + self.property_versions.extend(extra.property_versions); + self.reference_kinds.extend(extra.reference_kinds); + } +} + +impl Catalog { + pub fn new(spec: CatalogSpec) -> Result { + let mut catalog = Self { + spec, + exact: HashMap::new(), + wildcard: Vec::new(), + extensions: HashMap::new(), + fields: HashMap::new(), + ignored: BTreeSet::new(), + readable_extensions: BTreeSet::new(), + }; + let mut ids = BTreeSet::new(); + for (index, manifest) in catalog.spec.manifests.iter().enumerate() { + if manifest.id.is_empty() || !ids.insert(&manifest.id) || manifest.files.is_empty() { + bail!("empty or duplicate manifest definition: {}", manifest.id); + } + for pattern in &manifest.files { + if pattern.is_empty() || pattern.contains(['/', '\\']) { + bail!("manifest patterns must be basenames: {pattern}"); + } + if pattern.contains(['*', '?']) { + let compiled = (FilePattern::new(pattern), index); + if let Some((_, extension)) = pattern.rsplit_once('.') + && !extension.is_empty() + && !extension.contains(['*', '?']) + { + catalog + .extensions + .entry(extension.to_owned()) + .or_default() + .push(compiled); + } else { + catalog.wildcard.push(compiled); + } + } else { + catalog + .exact + .entry(pattern.clone()) + .or_default() + .push(index); + } + } + let pointers = manifest + .names + .iter() + .chain( + manifest + .identity + .iter() + .flat_map(|v| v.paths.iter().chain(&v.versions)), + ) + .chain(manifest.dependencies.iter().flat_map(|v| &v.paths)) + .chain(manifest.versions.iter().flat_map(|v| &v.paths)) + .chain(manifest.workspaces.iter().flat_map(|v| { + std::iter::once(&v.present) + .chain(&v.members) + .chain(&v.exclude) + })) + .chain(manifest.package_path.iter()); + for path in pointers { + if !path.is_empty() && !path.starts_with('/') { + bail!("manifest selectors must be JSON pointers: {path}"); + } + } + for version in &manifest.versions { + if version.language.is_empty() || version.kind.is_empty() { + bail!("manifest version fields need a language and kind"); + } + let valid_source = match manifest.reader { + Reader::Json | Reader::Toml => { + !version.paths.is_empty() || version.dependency.is_some() + } + Reader::Pin => { + version.key.is_none() + && version.dependency.is_none() + && version.paths.is_empty() + } + Reader::Tools | Reader::Properties => { + version.key.as_ref().is_some_and(|key| !key.is_empty()) + } + _ => false, + }; + if !valid_source { + bail!( + "version field has no applicable value source in {}", + manifest.id + ); + } + } + } + for (file, indexes) in &catalog.exact { + let first = catalog.spec.manifests[indexes[0]].reader; + if indexes + .iter() + .any(|index| catalog.spec.manifests[*index].reader != first) + || catalog + .matching(file) + .any(|manifest| manifest.reader != first) + { + bail!("conflicting manifest readers for {file}"); + } + } + for (index, field) in catalog.spec.property_versions.iter().enumerate() { + if field.fields.is_empty() || field.language.is_empty() || field.kind.is_empty() { + bail!("incomplete property version definition"); + } + for name in &field.fields { + if catalog + .fields + .entry(field.reader) + .or_default() + .insert(name.clone(), index) + .is_some() + { + bail!( + "duplicate property version field: {:?}:{name}", + field.reader + ); + } + } + } + catalog + .readable_extensions + .extend(catalog.exact.keys().map(|file| { + file.rsplit_once('.') + .map_or("", |(_, extension)| extension) + .to_owned() + })); + catalog + .readable_extensions + .extend(catalog.extensions.keys().cloned()); + catalog + .ignored + .extend(catalog.spec.scan.ignore.iter().cloned()); + Ok(catalog) + } + + pub fn matching<'a>(&'a self, file: &'a str) -> impl Iterator + 'a { + let extension = file + .rsplit_once('.') + .map(|(_, extension)| extension) + .unwrap_or(""); + self.exact + .get(file) + .into_iter() + .flatten() + .copied() + .chain( + self.extensions + .get(extension) + .into_iter() + .flatten() + .chain(&self.wildcard) + .filter(move |(pattern, _)| pattern.matches(file)) + .map(|(_, index)| *index), + ) + .map(|index| &self.spec.manifests[index]) + } + + pub fn readable(&self, file: &str) -> bool { + let extension = file.rsplit_once('.').map_or("", |(_, extension)| extension); + (!self.wildcard.is_empty() || self.readable_extensions.contains(extension)) + && self.matching(file).next().is_some() + } + pub fn reader(&self, file: &str, reader: Reader) -> bool { + self.matching(file) + .any(|manifest| manifest.reader == reader) + } + pub fn ignored(&self, directory: &str) -> bool { + self.ignored.contains(directory) + } + pub fn pruned(&self, node: &Node, directory: &str) -> bool { + self.spec.scan.prune.iter().any(|rule| { + rule.directories + .iter() + .any(|name| name.eq_ignore_ascii_case(directory)) + && node + .files + .iter() + .any(|file| rule.when_files.iter().any(|pattern| glob(pattern, file))) + }) + } + pub fn family<'a>(&'a self, language: &'a str) -> &'a str { + self.spec + .families + .get(language) + .map_or(language, String::as_str) + } + pub fn inherited(&self, kind: &str) -> bool { + self.spec.inheritance.iter().any(|rule| rule.kind == kind) + } + pub fn boundaries<'a>(&'a self, node: &'a Node) -> impl Iterator { + self.spec + .inheritance + .iter() + .filter(|rule| { + node.files.iter().any(|file| { + rule.boundary_files + .iter() + .any(|pattern| glob(pattern, file)) + }) + }) + .map(|rule| rule.kind.as_str()) + } + pub fn workspace_candidates<'a>(&'a self, node: &'a Node) -> impl Iterator { + self.spec + .workspaces + .iter() + .filter(|rule| { + rule.package + .as_ref() + .is_some_and(|package| node.packages.contains_key(package)) + || node + .files + .iter() + .any(|file| rule.files.iter().any(|pattern| glob(pattern, file))) + }) + .map(|rule| rule.ecosystem.as_str()) + } + pub fn property(&self, reader: PropertyReader, name: &str) -> Option<&PropertyVersion> { + self.fields + .get(&reader) + .and_then(|fields| fields.get(name)) + .map(|index| &self.spec.property_versions[*index]) + } + pub fn properties<'a>( + &'a self, + reader: PropertyReader, + ) -> impl Iterator + 'a { + self.spec + .property_versions + .iter() + .filter(move |field| field.reader == reader) + } + pub fn property_values<'a>( + &'a self, + reader: PropertyReader, + values: &'a Properties, + ) -> impl Iterator { + self.properties(reader).flat_map(move |field| { + field + .fields + .iter() + .filter_map(move |key| values.get(key).map(|value| (field, value))) + }) + } + pub fn reference(&self, kind: &str) -> Option<&ReferenceKind> { + self.spec.reference_kinds.get(kind) + } +} + +fn select<'a>(document: &'a Value, paths: &[String]) -> Option<&'a Value> { + paths.iter().find_map(|path| document.pointer(path)) +} +fn scalar(value: &Value) -> Option<&str> { + value + .as_str() + .or_else(|| value.get("version").and_then(Value::as_str)) +} +fn selected_scalar<'a>(document: &'a Value, paths: &[String]) -> Option<&'a str> { + select(document, paths).and_then(scalar) +} +fn strings(value: Option<&Value>) -> Vec { + value + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(Value::as_str) + .map(str::to_owned) + .collect() +} + +pub(super) fn workspace_patterns(patterns: Vec) -> (Vec, Vec) { + let mut members = Vec::new(); + let mut exclude = Vec::new(); + for pattern in patterns { + if let Some(pattern) = pattern.strip_prefix('!') { + exclude.push(pattern.to_owned()); + } else { + members.push(pattern); + } + } + (members, exclude) +} + +impl Manifest { + pub fn apply(&self, node: &mut Node, file: &str, document: &Value) { + if let Some(name) = selected_scalar(document, &self.names) { + node.names.insert(name.to_owned()); + } + if let Some(identity) = &self.identity + && let Some(name) = selected_scalar(document, &identity.paths) + { + node.names.insert(name.to_owned()); + node.identity( + &identity.ecosystem, + &identity.normalize.name(name), + selected_scalar(document, &identity.versions), + file, + ); + } + if let Some(package) = &self.package + && self + .package_path + .as_ref() + .is_none_or(|path| document.pointer(path).is_some_and(Value::is_object)) + { + node.packages + .insert(package.clone(), relative_file(&node.path, file)); + } + for dependencies in &self.dependencies { + for path in &dependencies.paths { + if dependencies.requirements { + for requirement in document + .pointer(path) + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(Value::as_str) + { + if let Some((name, version)) = + super::parsers::requirement_parts(requirement) + { + node.dependency( + &dependencies.ecosystem, + &dependencies.normalize.name(name), + version, + file, + ); + } + } + } else if let Some(entries) = document.pointer(path).and_then(Value::as_object) { + for (name, value) in entries { + if !dependencies.exclude.contains(name) { + node.dependency( + &dependencies.ecosystem, + &dependencies.normalize.name(name), + scalar(value), + file, + ); + } + } + } + } + } + for workspace in &self.workspaces { + if document.pointer(&workspace.present).is_none() { + continue; + } + let (members, mut exclude) = + workspace_patterns(strings(select(document, &workspace.members))); + exclude.extend(strings(select(document, &workspace.exclude))); + node.workspaces.push(Workspace { + ecosystem: workspace.ecosystem.clone(), + source: relative_file(&node.path, file), + members, + exclude, + }); + } + self.versions(node, file, Some(document), None, false); + } + + pub fn versions( + &self, + node: &mut Node, + file: &str, + document: Option<&Value>, + text: Option<&str>, + dependencies: bool, + ) { + for field in &self.versions { + if dependencies != field.dependency.is_some() { + continue; + } + if let Some(key) = &field.dependency { + if let Some(dependency) = node.dependencies.get(key) { + let mut versions = Vec::new(); + if let Some(raw) = &dependency.version { + let kind = if field + .reference_prefix + .as_ref() + .is_some_and(|prefix| raw.starts_with(prefix)) + { + "reference" + } else { + &field.kind + }; + versions.push(Version { + value: raw.clone(), + source: dependency.source.clone(), + kind: kind.to_owned(), + }); + } + if let Some(resolved) = &dependency.resolved { + versions.push(Version { + kind: field.kind.clone(), + ..resolved.clone() + }); + } + node.versions + .entry(field.language.clone()) + .or_default() + .extend(versions); + } + } else if let Some(value) = document + .and_then(|doc| selected_scalar(doc, &field.paths)) + .or(text) + { + node.version(&field.language, value, file, &field.kind); + } + } + } + + pub fn text_versions(&self, node: &mut Node, file: &str, text: &str) { + match self.reader { + Reader::Pin => { + let value = text + .lines() + .filter(|line| !line.trim().starts_with('#')) + .collect::>() + .join(" "); + self.versions(node, file, None, Some(&value), false); + } + Reader::Tools => { + for line in text.lines() { + let mut words = line.split('#').next().unwrap_or("").split_whitespace(); + let Some(key) = words.next() else { + continue; + }; + let value = words.collect::>().join(" "); + for field in self + .versions + .iter() + .filter(|field| field.key.as_deref() == Some(key)) + { + node.version(&field.language, &value, file, &field.kind); + } + } + } + Reader::Properties => { + let properties = values::properties(text, &relative_file(&node.path, file)); + for field in &self.versions { + if let Some(value) = field.key.as_ref().and_then(|key| properties.get(key)) { + node.version(&field.language, &value.value, file, &field.kind); + } + } + } + _ => (), + } + } +} diff --git a/src/detect/dotnet.rs b/src/detect/dotnet.rs index a3aa397..b23bce7 100644 --- a/src/detect/dotnet.rs +++ b/src/detect/dotnet.rs @@ -1,11 +1,13 @@ use std::collections::{BTreeMap, BTreeSet}; use std::path::{Path, PathBuf}; +use super::catalog::{Catalog, PropertyReader, Reader}; +use super::parsers::Documents; use anyhow::{Result, bail}; use serde_json::Value; use super::values::{self, Located, Properties, Syntax}; -use super::{Diagnostic, Node, Version, Workspace, display_path, relative_file, xml}; +use super::{Diagnostic, Node, Workspace, display_path, relative_file, xml}; #[derive(Default)] struct Document { @@ -29,7 +31,7 @@ struct Reference { version: Option, } -fn document(text: &str, source: &str) -> Result { +fn document(text: &str, source: &str, catalog: &Catalog) -> Result { let mut doc = Document::default(); let mut nested_versions = BTreeMap::new(); xml::read(text, |stack| { @@ -81,13 +83,11 @@ fn document(text: &str, source: &str) -> Result { located(&element.text), ); } - if matches!( - element.name, - "PackageReference" | "PackageVersion" | "FrameworkReference" | "Reference" - ) && let Some(name) = element - .attributes - .get("Include") - .or_else(|| element.attributes.get("Update")) + if catalog.reference(element.name).is_some() + && let Some(name) = element + .attributes + .get("Include") + .or_else(|| element.attributes.get("Update")) { let version = element .attributes @@ -349,13 +349,18 @@ fn apply( active.remove(path); } -pub(super) fn parse(nodes: &mut [Node], diagnostics: &mut Vec) { +pub(super) fn parse( + nodes: &mut [Node], + catalog: &Catalog, + parsed: &Documents, + diagnostics: &mut Vec, +) { if !nodes.iter().any(|node| { node.files.iter().any(|file| { - file == "global.json" - || file.ends_with(".csproj") - || file.ends_with(".sln") - || file.ends_with(".slnx") + catalog.matching(file).any(|manifest| { + manifest.package.as_deref() == Some("dotnet") + || manifest.workspace.as_deref() == Some("dotnet") + }) }) }) { return; @@ -367,11 +372,7 @@ pub(super) fn parse(nodes: &mut [Node], diagnostics: &mut Vec) { if file == "global.json" { sdk_versions.insert(node.path.clone(), BTreeMap::new()); } - if file.ends_with(".csproj") - || file.ends_with(".slnx") - || file.ends_with(".props") - || file.ends_with(".targets") - { + if catalog.reader(file, Reader::Msbuild) { documents.insert(node.path.join(file), None); } } @@ -379,49 +380,38 @@ pub(super) fn parse(nodes: &mut [Node], diagnostics: &mut Vec) { let source = relative_file(&node.path, file); if file == "global.json" { sdk_versions.insert(node.path.clone(), BTreeMap::new()); - match serde_json::from_str::(text) { - Ok(doc) => { - if let Some(version) = doc.pointer("/sdk/version").and_then(Value::as_str) { - node.versions - .entry("csharp".to_owned()) - .or_default() - .push(Version { - value: version.to_owned(), - source: source.clone(), - kind: "sdk".to_owned(), - }); - } - if let Some(versions) = doc.get("msbuild-sdks").and_then(Value::as_object) { - sdk_versions.insert( - node.path.clone(), - versions - .iter() - .filter_map(|(name, version)| { - version.as_str().map(|value| { - ( - name.to_ascii_lowercase(), - Located { - value: value.to_owned(), - source: source.clone(), - conditional: false, - }, - ) - }) - }) - .collect(), - ); - } - } - Err(error) => diagnostics.push(Diagnostic { - path: source, - message: error.to_string(), - }), + if let Some(doc) = parsed.get(&node.path).and_then(|docs| docs.get(file)) + && let Some(versions) = doc.get("msbuild-sdks").and_then(Value::as_object) + { + sdk_versions.insert( + node.path.clone(), + versions + .iter() + .filter_map(|(name, version)| { + version.as_str().map(|value| { + ( + name.to_ascii_lowercase(), + Located { + value: value.to_owned(), + source: source.clone(), + conditional: false, + }, + ) + }) + }) + .collect(), + ); } } else if let std::collections::btree_map::Entry::Occupied(mut entry) = documents.entry(node.path.join(file)) { - match document(text, &source) { - Ok(doc) if file.ends_with(".slnx") == doc.solution => { + match document(text, &source, catalog) { + Ok(doc) + if catalog + .matching(file) + .any(|manifest| manifest.workspace.as_deref() == Some("dotnet")) + == doc.solution => + { entry.insert(Some(doc)); } Ok(_) => diagnostics.push(Diagnostic { @@ -440,11 +430,15 @@ pub(super) fn parse(nodes: &mut [Node], diagnostics: &mut Vec) { for file in node .files .iter() - .filter(|file| file.ends_with(".sln") || file.ends_with(".slnx")) + .filter(|file| { + catalog + .matching(file) + .any(|manifest| manifest.workspace.as_deref() == Some("dotnet")) + }) .cloned() .collect::>() { - let projects = if file.ends_with(".slnx") { + let projects = if catalog.reader(&file, Reader::Msbuild) { documents .get(&node.path.join(&file)) .and_then(Option::as_ref) @@ -484,7 +478,11 @@ pub(super) fn parse(nodes: &mut [Node], diagnostics: &mut Vec) { let projects = node .files .iter() - .filter(|file| file.ends_with(".csproj")) + .filter(|file| { + catalog + .matching(file) + .any(|manifest| manifest.package.as_deref() == Some("dotnet")) + }) .cloned() .collect::>(); for file in projects { @@ -574,7 +572,10 @@ pub(super) fn parse(nodes: &mut [Node], diagnostics: &mut Vec) { for path in central_sources { if let Some(Some(doc)) = documents.get(&path) { for reference in &doc.references { - if reference.kind == "PackageVersion" { + if catalog + .reference(&reference.kind) + .is_some_and(|reference| reference.catalog) + { central.insert( reference.name.to_ascii_lowercase(), reference.version.as_ref(), @@ -584,37 +585,13 @@ pub(super) fn parse(nodes: &mut [Node], diagnostics: &mut Vec) { } } } - for (key, kind) in [ - ("targetframework", "target"), - ("targetframeworks", "target"), - ("targetframeworkversion", "target"), - ("langversion", "language"), - ("runtimeframeworkversion", "runtime"), - ] { - if let Some(raw) = properties.get(key) { - for value in raw.value.split(';').filter(|v| !v.trim().is_empty()) { - values::version( - node, - "csharp", - &Located { - value: value.trim().to_owned(), - ..raw.clone() - }, - kind, - &properties, - Syntax::MsBuild, - ); - } - } + for (field, raw) in catalog.property_values(PropertyReader::Msbuild, &properties) { + field.emit(node, raw, &properties, Syntax::MsBuild); } for (key, raw) in &conditional { - let kind = match key.as_str() { - "targetframework" | "targetframeworks" | "targetframeworkversion" => "target", - "langversion" => "language", - "runtimeframeworkversion" => "runtime", - _ => continue, - }; - values::version(node, "csharp", raw, kind, &properties, Syntax::MsBuild); + if let Some(field) = catalog.property(PropertyReader::Msbuild, key) { + field.emit(node, raw, &properties, Syntax::MsBuild); + } } for location in &sources { let Some(Some(document)) = documents.get(location) else { @@ -651,26 +628,20 @@ pub(super) fn parse(nodes: &mut [Node], diagnostics: &mut Vec) { ); } for reference in &document.references { - let ecosystem = match reference.kind.as_str() { - "PackageReference" => "nuget", - "FrameworkReference" => "dotnet-framework", - "Reference" => "dotnet-reference", - _ => continue, + let Some(policy) = catalog + .reference(&reference.kind) + .filter(|policy| !policy.catalog) + else { + continue; }; - let name = reference - .name - .split(',') - .next() - .unwrap_or("") - .trim() - .to_ascii_lowercase(); + let name = policy.name(&reference.name); let version = reference .version .as_ref() .or_else(|| central.get(&name).copied().flatten()); values::dependency( node, - ecosystem, + &policy.ecosystem, &name, &display_path(location), version, diff --git a/src/detect/gradle.rs b/src/detect/gradle.rs index 2f47bb0..6777dc3 100644 --- a/src/detect/gradle.rs +++ b/src/detect/gradle.rs @@ -1,6 +1,7 @@ use std::collections::{BTreeMap, BTreeSet}; use std::path::{Path, PathBuf}; +use super::catalog::{Catalog, PropertyReader, Reader}; use super::values::{self, Located, Properties, Syntax}; use super::{Diagnostic, Node, Workspace, display_path, relative_file}; @@ -191,7 +192,7 @@ struct Script { dynamic_includes: bool, } -fn script(text: &str, source: &str) -> Result { +fn script(text: &str, source: &str, catalog: &Catalog) -> Result { let tokens = tokens(text)?; let mut script = Script::default(); let mut scopes = Vec::new(); @@ -278,39 +279,20 @@ fn script(text: &str, source: &str) -> Result { shared, root, }; - if matches!( - key, - "sourceCompatibility" - | "targetCompatibility" - | "languageVersion" - | "options.release" - ) { - let kind = if key == "languageVersion" { - "toolchain" - } else if key == "sourceCompatibility" { - "source" - } else if key == "targetCompatibility" { - "target" - } else { - "release" - }; - script.java.push((kind.to_owned(), declaration)); + if catalog.property(PropertyReader::Gradle, key).is_some() { + script.java.push((key.to_owned(), declaration)); } else { script.properties.insert(key.to_owned(), declaration); } } } } - if matches!(key, "languageVersion.set" | "options.release.set") + if catalog.property(PropertyReader::Gradle, key).is_some() + && matches!(tokens.get(index + 1), Some(Token::Symbol('('))) && let Some((value, _)) = argument(&tokens, index + 1) { script.java.push(( - if key == "languageVersion.set" { - "toolchain" - } else { - "release" - } - .to_owned(), + key.to_owned(), Declaration { value: located(value), shared, @@ -468,15 +450,15 @@ fn build_file(path: &Path) -> bool { || file == format!("{directory}.gradle.kts") } -pub(super) fn parse(nodes: &mut Vec, diagnostics: &mut Vec) { +pub(super) fn parse(nodes: &mut Vec, catalog: &Catalog, diagnostics: &mut Vec) { let mut scripts = BTreeMap::::new(); let mut property_files = BTreeMap::::new(); let mut daemon_versions = BTreeMap::::new(); for node in nodes.iter() { for (file, text) in &node.contents { let source = relative_file(&node.path, file); - if file.ends_with(".gradle") || file.ends_with(".gradle.kts") { - match script(text, &source) { + if catalog.reader(file, Reader::Gradle) { + match script(text, &source, catalog) { Ok(script) => { if matches!(file.as_str(), "settings.gradle" | "settings.gradle.kts") && script.dynamic_includes @@ -490,14 +472,14 @@ pub(super) fn parse(nodes: &mut Vec, diagnostics: &mut Vec) { message, }), } - } else if file == "gradle-daemon-jvm.properties" { + } else if catalog.reader(file, Reader::GradleJvm) { if let Some(version) = values::properties(text, &source).get("toolchainVersion") { daemon_versions.insert( node.path.parent().unwrap_or(Path::new("")).to_path_buf(), version.clone(), ); } - } else if file == "gradle.properties" { + } else if catalog.reader(file, Reader::GradleProperties) { property_files.insert(node.path.clone(), values::properties(text, &source)); } } @@ -633,15 +615,10 @@ pub(super) fn parse(nodes: &mut Vec, diagnostics: &mut Vec) { } for (path, script, inherited) in declarations { for (kind, declaration) in &script.java { - if (!inherited && declaration.root) || (inherited && declaration.shared) { - values::version( - node, - "java", - &declaration.value, - kind, - &properties, - Syntax::Maven, - ); + if ((!inherited && declaration.root) || (inherited && declaration.shared)) + && let Some(field) = catalog.property(PropertyReader::Gradle, kind) + { + field.emit(node, &declaration.value, &properties, Syntax::Maven); } } for plugin in &script.plugins { diff --git a/src/detect/java.rs b/src/detect/java.rs index 05b5c11..76e7f2c 100644 --- a/src/detect/java.rs +++ b/src/detect/java.rs @@ -3,6 +3,7 @@ use std::path::{Path, PathBuf}; use anyhow::{Result, bail}; +use super::catalog::{Catalog, PropertyReader}; use super::values::{self, Located, Properties, Syntax}; use super::{Diagnostic, Node, Workspace, display_path, relative_file, xml}; @@ -30,7 +31,7 @@ struct Pom { modules: Vec, } -fn pom(text: &str, source: &str) -> Result { +fn pom(text: &str, source: &str, catalog: &Catalog) -> Result { let mut doc = Pom::default(); let mut artifacts = BTreeMap::::new(); xml::read(text, |stack| { @@ -84,7 +85,9 @@ fn pom(text: &str, source: &str) -> Result { } } if let Some(plugin) = stack.iter().position(|e| e.name == "plugin") - && matches!(element.name, "release" | "source" | "target") + && catalog + .property(PropertyReader::MavenCompiler, element.name) + .is_some() && stack .get(stack.len().saturating_sub(2)) .is_some_and(|e| e.name == "configuration") @@ -180,7 +183,7 @@ fn resolved(value: &str, source: &str, properties: &Properties) -> Option, diagnostics: &mut Vec) { +pub(super) fn parse(nodes: &mut Vec, catalog: &Catalog, diagnostics: &mut Vec) { let mut documents = BTreeMap::>::new(); for node in nodes.iter() { if node.files.contains("pom.xml") { @@ -188,7 +191,7 @@ pub(super) fn parse(nodes: &mut Vec, diagnostics: &mut Vec) { } if let Some(text) = node.contents.get("pom.xml") { let source = relative_file(&node.path, "pom.xml"); - match pom(text, &source) { + match pom(text, &source, catalog) { Ok(doc) => { documents.insert(node.path.join("pom.xml"), Some(doc)); } @@ -200,7 +203,7 @@ pub(super) fn parse(nodes: &mut Vec, diagnostics: &mut Vec) { } } if documents.is_empty() { - super::gradle::parse(nodes, diagnostics); + super::gradle::parse(nodes, catalog, diagnostics); return; } for node in nodes.iter_mut() { @@ -283,15 +286,8 @@ pub(super) fn parse(nodes: &mut Vec, diagnostics: &mut Vec) { Syntax::Maven, ); } - for (key, kind) in [ - ("java.version", "requirement"), - ("maven.compiler.release", "release"), - ("maven.compiler.source", "source"), - ("maven.compiler.target", "target"), - ] { - if let Some(raw) = properties.get(key) { - values::version(node, "java", raw, kind, &properties, Syntax::Maven); - } + for (field, raw) in catalog.property_values(PropertyReader::Maven, &properties) { + field.emit(node, raw, &properties, Syntax::Maven); } for (path, parent) in &inherited { let source = display_path(path); @@ -333,7 +329,11 @@ pub(super) fn parse(nodes: &mut Vec, diagnostics: &mut Vec) { ); if artifact.name == "maven-compiler-plugin" { for (kind, raw) in &artifact.compiler { - values::version(node, "java", raw, kind, &properties, Syntax::Maven); + if let Some(field) = + catalog.property(PropertyReader::MavenCompiler, kind) + { + field.emit(node, raw, &properties, Syntax::Maven); + } } } } @@ -355,5 +355,5 @@ pub(super) fn parse(nodes: &mut Vec, diagnostics: &mut Vec) { }); } } - super::gradle::parse(nodes, diagnostics); + super::gradle::parse(nodes, catalog, diagnostics); } diff --git a/src/detect/mod.rs b/src/detect/mod.rs index a3b6d69..1dfbb86 100644 --- a/src/detect/mod.rs +++ b/src/detect/mod.rs @@ -9,6 +9,7 @@ use anyhow::{Context, Result, bail}; use clap::Args; use serde::Serialize; +mod catalog; mod dotnet; mod gradle; mod java; @@ -25,30 +26,6 @@ use rules::{Kind, Rules}; const MAX_FILE_BYTES: u64 = 1024 * 1024; const MAX_DEPTH: usize = 128; -const IGNORED: &[&str] = &[ - ".git", - ".jj", - ".svn", - "node_modules", - "vendor", - "target", - ".venv", - ".gradle", - ".vs", - "venv", - "__pycache__", - ".next", - ".nuxt", - ".output", - "dist", - "build", - "coverage", - ".bundle", - ".brainpod", - ".cache", - "site-packages", -]; - #[derive(Debug, Args)] pub struct DetectArgs { /// Repository or project directory to inspect without executing project code @@ -246,7 +223,7 @@ impl Inventory { continue; }; if ty.is_dir() { - if IGNORED.contains(&name.as_str()) { + if rules.catalog.ignored(&name) { self.stats.skipped_directories += 1; } else if depth == MAX_DEPTH { self.diagnostics.push(Diagnostic { @@ -270,7 +247,8 @@ impl Inventory { }) .or_insert_with(|| name.clone()); } - if parsers::readable(&name) { + let readable = rules.catalog.readable(&name); + if readable { let file = File::open(entry.path())?; let mut bytes = Vec::new(); file.take(MAX_FILE_BYTES + 1).read_to_end(&mut bytes)?; @@ -291,25 +269,21 @@ impl Inventory { }), } } - if parsers::readable(&name) || rules.needs_filename(&name) { + if readable || rules.needs_filename(&name) { node.files.insert(name); } } } - if node.files.iter().any(|file| file.ends_with(".csproj")) { - children.retain(|(path, _)| { - let output = path - .file_name() - .and_then(|name| name.to_str()) - .is_some_and(|name| { - name.eq_ignore_ascii_case("bin") || name.eq_ignore_ascii_case("obj") - }); - if output { - self.stats.skipped_directories += 1; - } - !output - }); - } + children.retain(|(path, _)| { + let prune = path + .file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| rules.catalog.pruned(&node, name)); + if prune { + self.stats.skipped_directories += 1; + } + !prune + }); if !node.files.is_empty() || !node.sources.is_empty() { node.path = path.to_path_buf(); self.nodes.push(node); @@ -430,7 +404,7 @@ pub fn scan(root: &Path, extra_rules: Option<&str>) -> Result { report.diagnostics = diagnostics; report.stats = stats; let original_nodes = nodes.len(); - parsers::parse(&mut nodes, &mut report.diagnostics); + parsers::parse(&mut nodes, &rules.catalog, &mut report.diagnostics); if nodes.len() != original_nodes { index_parents(&mut nodes); } @@ -467,7 +441,9 @@ pub fn scan(root: &Path, extra_rules: Option<&str>) -> Result { while let Some(i) = ancestor { if let Some(project_index) = project_indices[i] && projects[project_index].1.detections.iter().any(|d| { - d.kind != Kind::Workspace && family(&d.language) == family(&detection.language) + d.kind != Kind::Workspace + && rules.catalog.family(&d.language) + == rules.catalog.family(&detection.language) }) { owner = Some(project_index); @@ -508,28 +484,25 @@ pub fn scan(root: &Path, extra_rules: Option<&str>) -> Result { for language in languages { let mut versions = Vec::new(); let mut ancestor = Some(*index); - let mut sdk_scope_seen = false; + let mut boundaries = BTreeSet::new(); while let Some(i) = ancestor { if let Some(found) = nodes[i].versions.get(&language) { versions.extend( found .iter() .filter(|v| { - if v.kind == "sdk" { - return !sdk_scope_seen; - } - i == *index || v.kind == "pin" || v.kind == "toolchain" + (i == *index || rules.catalog.inherited(&v.kind)) + && !boundaries.contains(v.kind.as_str()) }) .cloned(), ); } - if nodes[i].files.contains("global.json") { - sdk_scope_seen = true; - } + boundaries.extend(rules.catalog.boundaries(&nodes[i])); ancestor = nodes[i].parent; } let mut seen = BTreeSet::new(); versions.retain(|v| seen.insert(v.clone())); + versions.sort(); project.languages.push(Language { name: language, versions, @@ -544,21 +517,8 @@ pub fn scan(root: &Path, extra_rules: Option<&str>) -> Result { if project_indices[index].is_none() { continue; } - for ecosystem in ["dotnet", "gradle"] { - if node.packages.contains_key(ecosystem) { - candidates.entry(ecosystem).or_default().push(index); - } - } - for (ecosystem, file) in [ - ("cargo", "Cargo.toml"), - ("go", "go.mod"), - ("node", "package.json"), - ("python", "pyproject.toml"), - ("maven", "pom.xml"), - ] { - if node.files.contains(file) { - candidates.entry(ecosystem).or_default().push(index); - } + for ecosystem in rules.catalog.workspace_candidates(node) { + candidates.entry(ecosystem).or_default().push(index); } } for (index, project) in &mut projects { @@ -593,13 +553,6 @@ pub fn scan(root: &Path, extra_rules: Option<&str>) -> Result { Ok(report) } -fn family(language: &str) -> &str { - match language { - "typescript" | "javascript" => "node", - other => other, - } -} - fn display_path(path: &Path) -> String { if path.as_os_str().is_empty() { ".".to_owned() diff --git a/src/detect/parsers.rs b/src/detect/parsers.rs index 927f364..6f5fa9e 100644 --- a/src/detect/parsers.rs +++ b/src/detect/parsers.rs @@ -4,56 +4,28 @@ use std::path::{Path, PathBuf}; use serde::Deserialize; use serde_json::Value; +use super::catalog::{Catalog, Normalize, Reader, Resolver, workspace_patterns}; use super::{Diagnostic, Node, Workspace, relative_file}; -pub fn readable(name: &str) -> bool { - matches!( - name, - "composer.json" - | "package.json" - | "pnpm-workspace.yaml" - | "Cargo.toml" - | "go.mod" - | "go.work" - | "Gemfile" - | "pyproject.toml" - | "Pipfile" - | ".go-version" - | ".php-version" - | ".ruby-version" - | ".python-version" - | ".node-version" - | ".nvmrc" - | ".tool-versions" - | "rust-toolchain" - | "rust-toolchain.toml" - | "global.json" - | "pom.xml" - | "gradle.properties" - | "gradle-daemon-jvm.properties" - | ".java-version" - | ".sdkmanrc" - ) || name.ends_with(".csproj") - || name.ends_with(".sln") - || name.ends_with(".slnx") - || name.ends_with(".props") - || name.ends_with(".targets") - || name.ends_with(".gradle") - || name.ends_with(".gradle.kts") - || name.ends_with(".versions.toml") - || name.ends_with(".gemspec") - || (name.starts_with("requirements") && name.ends_with(".txt")) -} +pub(super) type Documents = HashMap>; -pub fn parse(nodes: &mut Vec, diagnostics: &mut Vec) { +pub fn parse(nodes: &mut Vec, catalog: &Catalog, diagnostics: &mut Vec) { let mut documents = Documents::new(); let mut pnpm = HashMap::>::new(); for node in nodes.iter() { - if node.files.contains("pnpm-workspace.yaml") { + if node + .files + .iter() + .any(|file| catalog.reader(file, Reader::Pnpm)) + { pnpm.insert(node.path.clone(), None); } for (file, content) in &node.contents { - if file == "pnpm-workspace.yaml" { + let manifests = catalog.matching(file).collect::>(); + let Some(manifest) = manifests.first() else { + continue; + }; + if manifest.reader == Reader::Pnpm { match serde_saphyr::from_str::(content) { Ok(manifest) => { let (members, exclude) = workspace_patterns( @@ -80,13 +52,11 @@ pub fn parse(nodes: &mut Vec, diagnostics: &mut Vec) { } continue; } - let parsed = match file.as_str() { - "composer.json" | "package.json" => { + let parsed = match manifest.reader { + Reader::Json => { Some(serde_json::from_str::(content).map_err(|e| e.to_string())) } - "Cargo.toml" | "pyproject.toml" | "Pipfile" | "rust-toolchain.toml" => { - Some(toml::from_str::(content).map_err(|e| e.to_string())) - } + Reader::Toml => Some(toml::from_str::(content).map_err(|e| e.to_string())), _ => None, }; if let Some(parsed) = parsed { @@ -109,106 +79,64 @@ pub fn parse(nodes: &mut Vec, diagnostics: &mut Vec) { } } } - super::dotnet::parse(nodes, diagnostics); - super::java::parse(nodes, diagnostics); - for node in nodes { + for node in nodes.iter_mut() { if let Some(Some(manifest)) = pnpm.get(&node.path) { node.workspaces.push(manifest.workspace.clone()); } for (file, doc) in documents.get(&node.path).into_iter().flatten() { - if let Some(name) = doc - .get("name") - .or_else(|| doc.pointer("/package/name")) - .or_else(|| doc.pointer("/project/name")) - .and_then(Value::as_str) - { - node.names.insert(name.to_owned()); + for manifest in catalog.matching(file) { + manifest.apply(node, file, doc); } - match file.as_str() { - "composer.json" => composer(node, doc), - "package.json" => javascript(node, doc), - "Cargo.toml" => cargo(node, doc, &documents), - "pyproject.toml" => python(node, doc), - "Pipfile" => { - dependencies(node, doc.get("packages"), "pypi", file); - if let Some(v) = doc - .pointer("/requires/python_full_version") - .or_else(|| doc.pointer("/requires/python_version")) - .and_then(Value::as_str) - { - node.version("python", v, file, "requirement"); - } - } - _ => (), + } + } + super::dotnet::parse(nodes, catalog, &documents, diagnostics); + super::java::parse(nodes, catalog, diagnostics); + for node in nodes { + for (file, doc) in documents.get(&node.path).into_iter().flatten() { + if catalog + .matching(file) + .any(|manifest| manifest.resolver == Some(Resolver::Cargo)) + { + cargo(node, doc, &documents); } } let contents = std::mem::take(&mut node.contents); for (file, text) in &contents { - match file.as_str() { - "go.mod" | "go.work" => go(node, file, text), - "Gemfile" => ruby(node, text, diagnostics), - f if f.ends_with(".gemspec") => gemspec(node, file, text), - ".java-version" => pin(node, "java", file, text), - ".sdkmanrc" => { - if let Some(value) = - super::values::properties(text, &relative_file(&node.path, file)) - .get("java") - { - node.version("java", &value.value, file, "pin"); - } - } - ".go-version" => pin(node, "go", file, text), - ".php-version" => pin(node, "php", file, text), - ".ruby-version" => pin(node, "ruby", file, text), - ".python-version" => pin(node, "python", file, text), - ".node-version" | ".nvmrc" => pin(node, "javascript", file, text), - "rust-toolchain" => pin(node, "rust", file, text), - "rust-toolchain.toml" => { - if let Some(v) = documents - .get(&node.path) - .and_then(|docs| docs.get(file)) - .and_then(|d| d.pointer("/toolchain/channel")) - .and_then(Value::as_str) - { - node.version("rust", v, file, "toolchain"); + for manifest in catalog.matching(file) { + match manifest.reader { + Reader::Go => go(node, file, text, manifest.workspace.as_deref()), + Reader::Gemfile => ruby(node, file, text, diagnostics), + Reader::Gemspec => gemspec(node, file, text), + Reader::Pin | Reader::Properties | Reader::Tools => { + manifest.text_versions(node, file, text) } - } - ".tool-versions" => { - for line in text.lines() { - let mut words = line.split('#').next().unwrap_or("").split_whitespace(); - let language = match words.next() { - Some("nodejs") => "javascript", - Some("golang") => "go", - Some("java") => "java", - Some("python") => "python", - Some("ruby") => "ruby", - Some("rust") => "rust", - Some("php") => "php", - _ => continue, - }; - let value = words.collect::>().join(" "); - node.version(language, &value, file, "pin"); - } - } - f if f.starts_with("requirements") && f.ends_with(".txt") => { - for line in logical_lines(text) { - let line = line.trim(); - if line.starts_with('-') { - if !line.starts_with("--hash=") { - diagnostics.push(Diagnostic { - path: relative_file(&node.path, file), - message: format!("requirements option not evaluated: {line}"), - }); + Reader::Requirements => { + for line in logical_lines(text) { + let line = line.trim(); + if line.starts_with('-') { + if !line.starts_with("--hash=") { + diagnostics.push(Diagnostic { + path: relative_file(&node.path, file), + message: format!( + "requirements option not evaluated: {line}" + ), + }); + } + } else { + requirement(node, line, file); } - continue; } - requirement(node, line, file); } + _ => (), } - _ => (), } } resolve_catalogs(node, &pnpm, diagnostics); + for (file, doc) in documents.get(&node.path).into_iter().flatten() { + for manifest in catalog.matching(file) { + manifest.versions(node, file, Some(doc), None, true); + } + } } } @@ -225,18 +153,6 @@ struct Pnpm { catalogs: BTreeMap>, } -fn workspace_patterns(patterns: Vec) -> (Vec, Vec) { - let (mut members, mut exclude) = (Vec::new(), Vec::new()); - for pattern in patterns { - if let Some(pattern) = pattern.strip_prefix('!') { - exclude.push(pattern.to_owned()); - } else { - members.push(pattern); - } - } - (members, exclude) -} - fn resolve_catalogs( node: &mut Node, manifests: &HashMap>, @@ -281,12 +197,6 @@ fn resolve_catalogs( }) }); if let Some(resolved) = value { - if package == "typescript" { - node.versions - .entry("typescript".to_owned()) - .or_default() - .push(resolved.clone()); - } dependency.resolved = Some(resolved); } else { diagnostics.push(Diagnostic { @@ -297,93 +207,8 @@ fn resolve_catalogs( } } -fn dependencies(node: &mut Node, object: Option<&Value>, ecosystem: &str, file: &str) { - if let Some(deps) = object.and_then(Value::as_object) { - for (name, value) in deps { - let name = if ecosystem == "pypi" { - normalize_python(name) - } else { - name.clone() - }; - let version = value - .as_str() - .or_else(|| value.get("version").and_then(Value::as_str)); - node.dependency(ecosystem, &name, version, file); - } - } -} - -fn composer(node: &mut Node, doc: &Value) { - if let Some(name) = doc.get("name").and_then(Value::as_str) { - node.identity( - "composer", - name, - doc.get("version").and_then(Value::as_str), - "composer.json", - ); - } - dependencies(node, doc.get("require"), "composer", "composer.json"); - if let Some(v) = doc.pointer("/require/php").and_then(Value::as_str) { - node.version("php", v, "composer.json", "requirement"); - } - if let Some(v) = doc.pointer("/config/platform/php").and_then(Value::as_str) { - node.version("php", v, "composer.json", "platform"); - } -} - -fn javascript(node: &mut Node, doc: &Value) { - if let Some(name) = doc.get("name").and_then(Value::as_str) { - node.identity( - "npm", - name, - doc.get("version").and_then(Value::as_str), - "package.json", - ); - } - dependencies(node, doc.get("devDependencies"), "npm", "package.json"); - dependencies(node, doc.get("dependencies"), "npm", "package.json"); - if let Some(v) = doc.pointer("/engines/node").and_then(Value::as_str) { - node.version("javascript", v, "package.json", "requirement"); - } - if let Some(v) = doc.pointer("/volta/node").and_then(Value::as_str) { - node.version("javascript", v, "package.json", "pin"); - } - if let Some(v) = doc - .pointer("/dependencies/typescript") - .or_else(|| doc.pointer("/devDependencies/typescript")) - .and_then(Value::as_str) - { - node.version( - "typescript", - v, - "package.json", - if v.starts_with("catalog:") { - "reference" - } else { - "requirement" - }, - ); - } - if let Some(workspaces) = doc.get("workspaces") { - let (members, exclude) = - workspace_patterns(strings(workspaces.get("packages").unwrap_or(workspaces))); - node.workspaces.push(Workspace { - ecosystem: "node".to_owned(), - source: relative_file(&node.path, "package.json"), - members, - exclude, - }); - } -} - -type Documents = HashMap>; - fn cargo(node: &mut Node, doc: &Value, documents: &Documents) { let file = "Cargo.toml"; - if doc.get("package").is_some_and(Value::is_object) { - node.packages - .insert("cargo".to_owned(), relative_file(&node.path, file)); - } let workspace = if let Some(explicit) = doc.pointer("/package/workspace").and_then(Value::as_str) { normalize_path(&node.path.join(explicit)).and_then(|path| { @@ -402,48 +227,40 @@ fn cargo(node: &mut Node, doc: &Value, documents: &Documents) { .map(|d| (path.to_path_buf(), d)) }) }; - if let Some(name) = doc.pointer("/package/name").and_then(Value::as_str) { - node.identity( - "cargo", - name, - doc.pointer("/package/version").and_then(Value::as_str), - file, - ); - if doc + if let Some(name) = doc.pointer("/package/name").and_then(Value::as_str) + && doc .pointer("/package/version/workspace") .and_then(Value::as_bool) == Some(true) - && let Some((path, workspace)) = &workspace - && let Some(version) = workspace - .pointer("/workspace/package/version") - .and_then(Value::as_str) - && let Some(identity) = node.identities.get_mut(&format!("cargo:{name}")) - { - identity.resolved = Some(super::Version { + && let Some((path, workspace)) = &workspace + && let Some(version) = workspace + .pointer("/workspace/package/version") + .and_then(Value::as_str) + && let Some(identity) = node.identities.get_mut(&format!("cargo:{name}")) + { + identity.resolved = Some(super::Version { + value: version.to_owned(), + source: relative_file(path, file), + kind: "package".to_owned(), + }); + } + if doc + .pointer("/package/rust-version/workspace") + .and_then(Value::as_bool) + == Some(true) + && let Some((path, ws)) = &workspace + && let Some(version) = ws + .pointer("/workspace/package/rust-version") + .and_then(Value::as_str) + { + node.versions + .entry("rust".to_owned()) + .or_default() + .push(super::Version { value: version.to_owned(), source: relative_file(path, file), - kind: "package".to_owned(), + kind: "requirement".to_owned(), }); - } - } - if let Some(version) = doc.pointer("/package/rust-version") { - if let Some(v) = version.as_str() { - node.version("rust", v, file, "requirement"); - } else if version.get("workspace").and_then(Value::as_bool) == Some(true) - && let Some((path, ws)) = &workspace - && let Some(v) = ws - .pointer("/workspace/package/rust-version") - .and_then(Value::as_str) - { - node.versions - .entry("rust".to_owned()) - .or_default() - .push(super::Version { - value: v.to_owned(), - source: relative_file(path, file), - kind: "requirement".to_owned(), - }); - } } let mut dep_tables = Vec::new(); if let Some(d) = doc.get("dependencies") { @@ -489,103 +306,23 @@ fn cargo(node: &mut Node, doc: &Value, documents: &Documents) { } } } - if let Some(ws) = doc.get("workspace") { - node.workspaces.push(Workspace { - ecosystem: "cargo".to_owned(), - source: relative_file(&node.path, file), - members: strings(&ws["members"]), - exclude: strings(&ws["exclude"]), - }); - } -} - -fn python(node: &mut Node, doc: &Value) { - let file = "pyproject.toml"; - if let Some(name) = doc - .pointer("/project/name") - .or_else(|| doc.pointer("/tool/poetry/name")) - .and_then(Value::as_str) - { - let version = doc - .pointer("/project/version") - .or_else(|| doc.pointer("/tool/poetry/version")) - .and_then(Value::as_str); - node.identity("pypi", &normalize_python(name), version, file); - node.names.insert(name.to_owned()); - } - if let Some(v) = doc - .pointer("/project/requires-python") - .and_then(Value::as_str) - { - node.version("python", v, file, "requirement"); - } - for dep in doc - .pointer("/project/dependencies") - .and_then(Value::as_array) - .into_iter() - .flatten() - .filter_map(Value::as_str) - { - requirement(node, dep, file); - } - if let Some(deps) = doc - .pointer("/tool/poetry/dependencies") - .and_then(Value::as_object) - { - for (name, v) in deps { - let version = v - .as_str() - .or_else(|| v.get("version").and_then(Value::as_str)); - if name == "python" { - if let Some(v) = version { - node.version("python", v, file, "requirement"); - } - } else { - node.dependency("pypi", &normalize_python(name), version, file); - } - } - } - if let Some(ws) = doc.pointer("/tool/uv/workspace") { - node.workspaces.push(Workspace { - ecosystem: "python".to_owned(), - source: relative_file(&node.path, file), - members: strings(&ws["members"]), - exclude: strings(&ws["exclude"]), - }); - } -} - -fn normalize_python(name: &str) -> String { - let mut normalized = String::new(); - for ch in name.chars() { - if matches!(ch, '-' | '_' | '.') { - if !normalized.ends_with('-') { - normalized.push('-'); - } - } else { - normalized.push(ch.to_ascii_lowercase()); - } - } - normalized } -fn requirement(node: &mut Node, line: &str, file: &str) { +pub(super) fn requirement_parts(line: &str) -> Option<(&str, Option<&str>)> { let line = line.split('#').next().unwrap_or("").trim(); if line.is_empty() { - return; + return None; } let end = line .find(|c: char| !c.is_ascii_alphanumeric() && !matches!(c, '-' | '_' | '.')) .unwrap_or(line.len()); if end == 0 { - return; + return None; } - let name = normalize_python(&line[..end]); + let name = &line[..end]; let mut rest = line[end..].trim(); if rest.starts_with('[') { - let Some(end) = rest.find(']') else { - return; - }; + let end = rest.find(']')?; rest = rest[end + 1..].trim(); } let constraint = rest.split(';').next().unwrap_or("").trim(); @@ -595,7 +332,13 @@ fn requirement(node: &mut Node, line: &str, file: &str) { } else { Some(constraint) }; - node.dependency("pypi", &name, version, file); + Some((name, version)) +} + +fn requirement(node: &mut Node, line: &str, file: &str) { + if let Some((name, version)) = requirement_parts(line) { + node.dependency("pypi", &Normalize::Python.name(name), version, file); + } } fn logical_lines(text: &str) -> Vec { @@ -617,7 +360,7 @@ fn logical_lines(text: &str) -> Vec { lines } -fn go(node: &mut Node, file: &str, text: &str) { +fn go(node: &mut Node, file: &str, text: &str, workspace: Option<&str>) { let mut block = ""; let mut members = Vec::new(); for line in text.lines() { @@ -675,9 +418,9 @@ fn go(node: &mut Node, file: &str, text: &str) { _ => (), } } - if file == "go.work" { + if let Some(ecosystem) = workspace { node.workspaces.push(Workspace { - ecosystem: "go".to_owned(), + ecosystem: ecosystem.to_owned(), source: relative_file(&node.path, file), members, exclude: Vec::new(), @@ -699,7 +442,7 @@ fn quoted(input: &str) -> Option<(&str, &str)> { Some((value, &input[end + 1..])) } -fn ruby(node: &mut Node, text: &str, diagnostics: &mut Vec) { +fn ruby(node: &mut Node, file: &str, text: &str, diagnostics: &mut Vec) { for line in text.lines() { let line = line.trim(); let (directive, rest) = if let Some(rest) = line @@ -719,13 +462,13 @@ fn ruby(node: &mut Node, text: &str, diagnostics: &mut Vec) { }; let Some((name, rest)) = quoted(rest) else { diagnostics.push(Diagnostic { - path: relative_file(&node.path, "Gemfile"), + path: relative_file(&node.path, file), message: format!("dynamic {directive} declaration not evaluated"), }); continue; }; if directive == "ruby" { - node.version("ruby", name, "Gemfile", "requirement"); + node.version("ruby", name, file, "requirement"); } else { let mut versions = Vec::new(); let mut rest = rest.trim_start(); @@ -741,7 +484,7 @@ fn ruby(node: &mut Node, text: &str, diagnostics: &mut Vec) { "gem", name, (!version.is_empty()).then_some(version.as_str()), - "Gemfile", + file, ); } } @@ -780,25 +523,6 @@ fn gemspec(node: &mut Node, file: &str, text: &str) { } } -fn strings(value: &Value) -> Vec { - value - .as_array() - .into_iter() - .flatten() - .filter_map(Value::as_str) - .map(str::to_owned) - .collect() -} - -fn pin(node: &mut Node, language: &str, file: &str, text: &str) { - let value = text - .lines() - .filter(|line| !line.trim().starts_with('#')) - .collect::>() - .join(" "); - node.version(language, &value, file, "pin"); -} - fn normalize_path(path: &Path) -> Option { if path.is_absolute() { return None; diff --git a/src/detect/rules.rs b/src/detect/rules.rs index 1f4c2de..fe3a89e 100644 --- a/src/detect/rules.rs +++ b/src/detect/rules.rs @@ -4,77 +4,184 @@ use std::sync::OnceLock; use anyhow::{Result, bail}; use serde::Deserialize; +use super::catalog::{Catalog, CatalogSpec}; use super::{Detection, Evidence, FrameworkRole, Node}; -#[derive(Clone, Debug, Deserialize)] +#[derive(Clone, Copy, Debug, Deserialize, serde::Serialize, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +pub enum Kind { + Framework, + Project, + Workspace, + Source, +} + +#[derive(Deserialize)] #[serde(deny_unknown_fields)] -pub struct Rule { - pub id: String, - pub language: String, - pub kind: Kind, - pub priority: i32, +struct RuleSpec { + id: String, + language: String, + kind: Kind, + priority: i32, #[serde(default)] - pub any: Vec, + any: Vec, #[serde(default)] - pub all: Vec, + all: Vec, #[serde(default)] - pub none: Vec, + none: Vec, } -#[derive(Clone, Debug, Deserialize)] +#[derive(Deserialize)] #[serde(untagged)] -pub enum Pattern { +enum PatternSpec { Atom(String), All(AllPattern), } -#[derive(Clone, Debug, Deserialize)] +#[derive(Deserialize)] #[serde(deny_unknown_fields)] -pub struct AllPattern { +struct AllPattern { all: Vec, } -impl Pattern { - fn atoms(&self) -> &[String] { - match self { - Self::Atom(atom) => std::slice::from_ref(atom), - Self::All(group) => &group.all, +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Document { + #[serde(default)] + rules: Vec, + #[serde(default)] + catalog: CatalogSpec, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +#[repr(usize)] +enum FactKind { + File, + Source, + Dependency, + Identity, + Package, + Workspace, +} + +impl FactKind { + fn parse(name: &str) -> Result { + match name { + "file" => Ok(Self::File), + "source" => Ok(Self::Source), + "dep" => Ok(Self::Dependency), + "identity" => Ok(Self::Identity), + "package" => Ok(Self::Package), + "workspace" => Ok(Self::Workspace), + _ => bail!("unknown detection fact: {name}"), } } +} - fn matches(&self, node: &Node) -> bool { - self.atoms() - .iter() - .all(|atom| matched(node, atom).is_some()) +pub(super) enum FilePattern { + Exact(String), + Star { prefix: String, suffix: String }, + Glob(String), +} + +impl FilePattern { + pub fn new(value: &str) -> Self { + if !value.contains(['*', '?']) { + Self::Exact(value.to_owned()) + } else if !value.contains('?') && value.bytes().filter(|b| *b == b'*').count() == 1 { + let (prefix, suffix) = value.split_once('*').unwrap_or((value, "")); + Self::Star { + prefix: prefix.to_owned(), + suffix: suffix.to_owned(), + } + } else { + Self::Glob(value.to_owned()) + } + } + + pub fn matches(&self, value: &str) -> bool { + match self { + Self::Exact(exact) => exact == value, + Self::Star { prefix, suffix } => { + value.len() >= prefix.len() + suffix.len() + && value.starts_with(prefix) + && value.ends_with(suffix) + } + Self::Glob(pattern) => glob(pattern, value), + } } } -#[derive(Clone, Copy, Debug, Deserialize, serde::Serialize, PartialEq, Eq)] -#[serde(rename_all = "kebab-case")] -pub enum Kind { - Framework, - Project, - Workspace, - Source, +struct Atom { + kind: FactKind, + value: String, + text: String, + file: FilePattern, +} +struct Pattern { + atoms: Vec, +} +struct Rule { + id: String, + language: String, + kind: Kind, + priority: i32, + any: Vec, + all: Vec, + none: Vec, } -#[derive(Deserialize)] -#[serde(deny_unknown_fields)] -struct Document { - rules: Vec, +impl PatternSpec { + fn compile(self) -> Result { + let atoms = match self { + Self::Atom(atom) => vec![atom], + Self::All(group) => group.all, + }; + if atoms.is_empty() { + bail!("empty detection pattern group"); + } + let atoms = atoms + .into_iter() + .map(|text| { + let Some((kind, value)) = text.split_once(':') else { + bail!("invalid detection pattern: {text}"); + }; + let kind = FactKind::parse(kind)?; + let valid = match kind { + FactKind::File | FactKind::Source => !value.is_empty() && !value.contains('/'), + FactKind::Dependency | FactKind::Identity => value + .split_once(':') + .is_some_and(|(ecosystem, name)| !ecosystem.is_empty() && !name.is_empty()), + FactKind::Package | FactKind::Workspace => !value.is_empty(), + }; + if !valid { + bail!("invalid detection pattern: {text}"); + } + Ok(Atom { + kind, + value: value.to_owned(), + file: FilePattern::new(value), + text, + }) + }) + .collect::>>()?; + Ok(Pattern { atoms }) + } +} + +impl Pattern { + fn matches(&self, node: &Node) -> bool { + self.atoms.iter().all(|atom| atom.matched(node).is_some()) + } } pub struct Rules { + pub catalog: Catalog, rules: Vec, - files: HashMap>, - sources: HashMap>, - dependencies: HashMap>, - identities: HashMap>, - workspaces: HashMap>, - packages: HashMap>, - wildcard_files: Vec<(String, usize)>, - file_patterns: BTreeSet, - wildcard_interest: BTreeSet, + indexes: [HashMap>; 6], + wildcard_files: Vec<(FilePattern, usize)>, + file_interest: BTreeSet, + wildcard_interest: Vec, source_interest: BTreeSet, } @@ -90,191 +197,147 @@ impl Rules { pub fn new(extra: Option<&str>) -> Result { let mut document: Document = toml::from_str(include_str!("rules.toml"))?; if let Some(extra) = extra { - document - .rules - .extend(toml::from_str::(extra)?.rules); + let extra: Document = toml::from_str(extra)?; + document.rules.extend(extra.rules); + document.catalog.extend(extra.catalog); } let mut ids = BTreeSet::new(); - for rule in &document.rules { - if rule.id.is_empty() || rule.language.is_empty() || !ids.insert(&rule.id) { - bail!("empty or duplicate detection rule: {}", rule.id); - } - if rule.any.is_empty() && rule.all.is_empty() { - bail!("rule {} needs an any or all pattern", rule.id); + let mut rules = Self { + catalog: Catalog::new(document.catalog)?, + rules: Vec::new(), + indexes: std::array::from_fn(|_| HashMap::new()), + wildcard_files: Vec::new(), + file_interest: BTreeSet::new(), + wildcard_interest: Vec::new(), + source_interest: BTreeSet::new(), + }; + for spec in document.rules { + if spec.id.is_empty() || spec.language.is_empty() || !ids.insert(spec.id.clone()) { + bail!("empty or duplicate detection rule: {}", spec.id); } - for pattern in rule.any.iter().chain(&rule.all).chain(&rule.none) { - if pattern.atoms().is_empty() { - bail!("empty pattern group in rule {}", rule.id); - } - for pattern in pattern.atoms() { - let valid = match pattern.split_once(':') { - Some(("file" | "source", value)) => { - !value.is_empty() && !value.contains('/') - } - Some(("dep" | "identity", value)) => { - value.split_once(':').is_some_and(|(ecosystem, name)| { - !ecosystem.is_empty() && !name.is_empty() - }) - } - Some(("package", "cargo" | "dotnet" | "maven" | "gradle")) => true, - Some(("workspace", value)) => { - matches!( - value, - "cargo" | "node" | "go" | "python" | "dotnet" | "maven" | "gradle" - ) - } - _ => false, - }; - if !valid { - bail!("invalid pattern {pattern:?} in rule {}", rule.id); - } - } + if spec.any.is_empty() && spec.all.is_empty() { + bail!("rule {} needs an any or all pattern", spec.id); } + let compile = |patterns: Vec| { + patterns + .into_iter() + .map(PatternSpec::compile) + .collect::>>() + }; + rules.rules.push(Rule { + id: spec.id, + language: spec.language, + kind: spec.kind, + priority: spec.priority, + any: compile(spec.any)?, + all: compile(spec.all)?, + none: compile(spec.none)?, + }); } - document + rules .rules .sort_by(|a, b| b.priority.cmp(&a.priority).then(a.id.cmp(&b.id))); - let mut rules = Self { - rules: document.rules, - files: HashMap::new(), - sources: HashMap::new(), - dependencies: HashMap::new(), - identities: HashMap::new(), - workspaces: HashMap::new(), - packages: HashMap::new(), - wildcard_files: Vec::new(), - file_patterns: BTreeSet::new(), - wildcard_interest: BTreeSet::new(), - source_interest: BTreeSet::new(), - }; - for (i, rule) in rules.rules.iter().enumerate() { - for pattern in rule + let mut wildcards = BTreeSet::new(); + for (index, rule) in rules.rules.iter().enumerate() { + for atom in rule .all .iter() .chain(&rule.any) .chain(&rule.none) - .flat_map(Pattern::atoms) + .flat_map(|p| &p.atoms) { - if let Some(value) = pattern.strip_prefix("source:") { - rules.source_interest.insert(value.to_owned()); - } - if let Some(value) = pattern.strip_prefix("file:") { - if value.contains(['*', '?']) { - rules.wildcard_interest.insert(value.to_owned()); - } else { - rules.file_patterns.insert(value.to_owned()); + match atom.kind { + FactKind::Source => { + rules.source_interest.insert(atom.value.clone()); + } + FactKind::File if matches!(atom.file, FilePattern::Exact(_)) => { + rules.file_interest.insert(atom.value.clone()); } + FactKind::File => { + wildcards.insert(atom.value.clone()); + } + _ => (), } } - for pattern in rule.all.iter().chain(&rule.any).flat_map(Pattern::atoms) { - let Some((kind, value)) = pattern.split_once(':') else { - continue; - }; - let index = match kind { - "file" if value.contains(['*', '?']) => { - rules.wildcard_files.push((value.to_owned(), i)); - continue; - } - "file" => &mut rules.files, - "source" => &mut rules.sources, - "dep" => &mut rules.dependencies, - "identity" => &mut rules.identities, - "workspace" => &mut rules.workspaces, - "package" => &mut rules.packages, - _ => continue, - }; - index.entry(value.to_owned()).or_default().push(i); + for atom in rule.all.iter().chain(&rule.any).flat_map(|p| &p.atoms) { + if atom.kind == FactKind::File && !matches!(atom.file, FilePattern::Exact(_)) { + rules + .wildcard_files + .push((FilePattern::new(&atom.value), index)); + } else { + rules.indexes[atom.kind as usize] + .entry(atom.value.clone()) + .or_default() + .push(index); + } } } + rules.wildcard_interest = wildcards + .iter() + .map(|pattern| FilePattern::new(pattern)) + .collect(); Ok(rules) } + pub fn needs_source(&self, extension: &str) -> bool { self.source_interest.contains(extension) } - pub fn needs_filename(&self, name: &str) -> bool { - self.file_patterns.contains(name) || self.wildcard_interest.iter().any(|p| glob(p, name)) + self.file_interest.contains(name) + || self + .wildcard_interest + .iter() + .any(|pattern| pattern.matches(name)) } pub fn evaluate(&self, node: &Node) -> Vec { let mut candidates = Vec::new(); - for file in &node.files { - if let Some(ids) = self.files.get(file) { - candidates.extend(ids); - } - } - for source in node.sources.keys() { - if let Some(ids) = self.sources.get(source) { - candidates.extend(ids); - } - } - for dependency in node.dependencies.keys() { - if let Some(ids) = self.dependencies.get(dependency) { - candidates.extend(ids); - } - } - for identity in node.identities.keys() { - if let Some(ids) = self.identities.get(identity) { - candidates.extend(ids); - } - } - for package in node.packages.keys() { - if let Some(ids) = self.packages.get(package) { - candidates.extend(ids); - } - } - for workspace in &node.workspaces { - if let Some(ids) = self.workspaces.get(&workspace.ecosystem) { + let facts = node + .files + .iter() + .map(|s| (FactKind::File, s.as_str())) + .chain(node.sources.keys().map(|s| (FactKind::Source, s.as_str()))) + .chain( + node.dependencies + .keys() + .map(|s| (FactKind::Dependency, s.as_str())), + ) + .chain( + node.identities + .keys() + .map(|s| (FactKind::Identity, s.as_str())), + ) + .chain( + node.packages + .keys() + .map(|s| (FactKind::Package, s.as_str())), + ) + .chain( + node.workspaces + .iter() + .map(|w| (FactKind::Workspace, w.ecosystem.as_str())), + ); + for (kind, value) in facts { + if let Some(ids) = self.indexes[kind as usize].get(value) { candidates.extend(ids); } } - for (pattern, id) in &self.wildcard_files { - if node.files.iter().any(|file| glob(pattern, file)) { - candidates.push(id); + for (pattern, index) in &self.wildcard_files { + if node.files.iter().any(|file| pattern.matches(file)) { + candidates.push(index); } } candidates.sort_unstable(); candidates.dedup(); candidates .into_iter() - .filter_map(|&i| { - let rule = &self.rules[i]; - rule.evaluate(node).map(|evidence| Detection { - id: rule.id.clone(), - language: rule.language.clone(), - kind: rule.kind, - priority: rule.priority, - roles: if rule.kind == Kind::Framework { - let mut roles = evidence - .iter() - .filter_map(|e| { - if e.pattern.starts_with("identity:") { - Some(FrameworkRole::Provides) - } else if e.pattern.starts_with("dep:") { - Some(FrameworkRole::Uses) - } else { - None - } - }) - .collect::>(); - if roles.is_empty() { - roles.push(FrameworkRole::Uses); - } - roles.sort(); - roles.dedup(); - roles - } else { - Vec::new() - }, - evidence, - }) - }) + .filter_map(|&index| self.rules[index].evaluate(node)) .collect() } } impl Rule { - pub fn evaluate(&self, node: &Node) -> Option> { + fn evaluate(&self, node: &Node) -> Option { if !self.all.iter().all(|p| p.matches(node)) || (!self.any.is_empty() && !self.any.iter().any(|p| p.matches(node))) || self.none.iter().any(|p| p.matches(node)) @@ -282,16 +345,40 @@ impl Rule { return None; } let mut evidence = Vec::new(); - for pattern in self.all.iter().chain(&self.any).filter(|p| p.matches(node)) { - for atom in pattern.atoms() { - if let Some(found) = matched(node, atom) { - found.evidence(node, atom, &mut evidence); + let mut roles = Vec::new(); + for atom in self + .all + .iter() + .chain(&self.any) + .filter(|p| p.matches(node)) + .flat_map(|p| &p.atoms) + { + if let Some(found) = atom.matched(node) { + found.evidence(node, &atom.text, &mut evidence); + match atom.kind { + FactKind::Identity => roles.push(FrameworkRole::Provides), + FactKind::Dependency => roles.push(FrameworkRole::Uses), + _ => (), } } } evidence.sort(); evidence.dedup(); - Some(evidence) + roles.sort(); + roles.dedup(); + if self.kind != Kind::Framework { + roles.clear(); + } else if roles.is_empty() { + roles.push(FrameworkRole::Uses); + } + Some(Detection { + id: self.id.clone(), + language: self.language.clone(), + kind: self.kind, + priority: self.priority, + roles, + evidence, + }) } } @@ -327,33 +414,35 @@ impl Matched<'_> { } } -fn matched<'a>(node: &'a Node, pattern: &str) -> Option> { - let (kind, value) = pattern.split_once(':')?; - match kind { - "file" => { - let name = if value.contains(['*', '?']) { - node.files.iter().find(|name| glob(value, name)) - } else { - node.files.get(value) - }; - name.map(|name| Matched::File(name)) +impl Atom { + fn matched<'a>(&self, node: &'a Node) -> Option> { + match self.kind { + FactKind::File => { + let file = if matches!(self.file, FilePattern::Exact(_)) { + node.files.get(&self.value) + } else { + node.files.iter().find(|name| self.file.matches(name)) + }; + file.map(|name| Matched::File(name)) + } + FactKind::Source => node + .sources + .get(&self.value) + .map(|name| Matched::File(name)), + FactKind::Dependency => node.dependencies.get(&self.value).map(Matched::Dependency), + FactKind::Identity => node.identities.get(&self.value).map(Matched::Dependency), + FactKind::Package => node + .packages + .get(&self.value) + .map(|source| Matched::Package(source)), + FactKind::Workspace => node + .workspaces + .iter() + .find(|w| w.ecosystem == self.value) + .map(Matched::Workspace), } - "source" => node.sources.get(value).map(|name| Matched::File(name)), - "dep" => node.dependencies.get(value).map(Matched::Dependency), - "identity" => node.identities.get(value).map(Matched::Dependency), - "package" => node - .packages - .get(value) - .map(|source| Matched::Package(source)), - "workspace" => node - .workspaces - .iter() - .find(|w| w.ecosystem == value) - .map(Matched::Workspace), - _ => None, } } - pub fn glob(pattern: &str, value: &str) -> bool { let (p, v) = (pattern.as_bytes(), value.as_bytes()); let (mut i, mut j, mut star, mut retry) = (0, 0, None, 0); diff --git a/src/detect/rules.toml b/src/detect/rules.toml index e74d78b..97d4fd2 100644 --- a/src/detect/rules.toml +++ b/src/detect/rules.toml @@ -1,3 +1,472 @@ +[catalog.scan] +ignore = [".git", ".jj", ".svn", "node_modules", "vendor", "target", ".venv", ".gradle", ".vs", "venv", "__pycache__", ".next", ".nuxt", ".output", "dist", "build", "coverage", ".bundle", ".brainpod", ".cache", "site-packages"] + +[[catalog.scan.prune]] +when_files = ["*.csproj"] +directories = ["bin", "obj"] + +[catalog.families] +javascript = "node" +typescript = "node" + +[[catalog.inheritance]] +kind = "pin" + +[[catalog.inheritance]] +kind = "toolchain" + +[[catalog.inheritance]] +kind = "sdk" +boundary_files = ["global.json"] + +[[catalog.workspaces]] +ecosystem = "cargo" +files = ["Cargo.toml"] + +[[catalog.workspaces]] +ecosystem = "go" +files = ["go.mod"] + +[[catalog.workspaces]] +ecosystem = "node" +files = ["package.json"] + +[[catalog.workspaces]] +ecosystem = "python" +files = ["pyproject.toml"] + +[[catalog.workspaces]] +ecosystem = "maven" +files = ["pom.xml"] + +[[catalog.workspaces]] +ecosystem = "dotnet" +package = "dotnet" + +[[catalog.workspaces]] +ecosystem = "gradle" +package = "gradle" + +[catalog.reference_kinds.PackageReference] +ecosystem = "nuget" +normalize = "lowercase" + +[catalog.reference_kinds.PackageVersion] +ecosystem = "nuget" +normalize = "lowercase" +catalog = true + +[catalog.reference_kinds.FrameworkReference] +ecosystem = "dotnet-framework" +normalize = "lowercase" + +[catalog.reference_kinds.Reference] +ecosystem = "dotnet-reference" +normalize = "lowercase" +separator = "," + +[[catalog.property_versions]] +reader = "msbuild" +fields = ["targetframework", "targetframeworks", "targetframeworkversion"] +language = "csharp" +kind = "target" +split = ";" + +[[catalog.property_versions]] +reader = "msbuild" +fields = ["langversion"] +language = "csharp" +kind = "language" +split = ";" + +[[catalog.property_versions]] +reader = "msbuild" +fields = ["runtimeframeworkversion"] +language = "csharp" +kind = "runtime" +split = ";" + +[[catalog.property_versions]] +reader = "maven" +fields = ["java.version"] +language = "java" +kind = "requirement" + +[[catalog.property_versions]] +reader = "maven" +fields = ["maven.compiler.release"] +language = "java" +kind = "release" + +[[catalog.property_versions]] +reader = "maven" +fields = ["maven.compiler.source"] +language = "java" +kind = "source" + +[[catalog.property_versions]] +reader = "maven" +fields = ["maven.compiler.target"] +language = "java" +kind = "target" + +[[catalog.property_versions]] +reader = "maven-compiler" +fields = ["release"] +language = "java" +kind = "release" + +[[catalog.property_versions]] +reader = "maven-compiler" +fields = ["source"] +language = "java" +kind = "source" + +[[catalog.property_versions]] +reader = "maven-compiler" +fields = ["target"] +language = "java" +kind = "target" + +[[catalog.property_versions]] +reader = "gradle" +fields = ["sourceCompatibility"] +language = "java" +kind = "source" + +[[catalog.property_versions]] +reader = "gradle" +fields = ["targetCompatibility"] +language = "java" +kind = "target" + +[[catalog.property_versions]] +reader = "gradle" +fields = ["languageVersion", "languageVersion.set"] +language = "java" +kind = "toolchain" + +[[catalog.property_versions]] +reader = "gradle" +fields = ["options.release", "options.release.set"] +language = "java" +kind = "release" + +[[catalog.manifests]] +id = "composer" +files = ["composer.json"] +reader = "json" +names = ["/name"] +[catalog.manifests.identity] +ecosystem = "composer" +paths = ["/name"] +versions = ["/version"] +[[catalog.manifests.dependencies]] +ecosystem = "composer" +paths = ["/require"] +[[catalog.manifests.versions]] +language = "php" +kind = "requirement" +paths = ["/require/php"] +[[catalog.manifests.versions]] +language = "php" +kind = "platform" +paths = ["/config/platform/php"] + +[[catalog.manifests]] +id = "npm" +files = ["package.json"] +reader = "json" +names = ["/name"] +[catalog.manifests.identity] +ecosystem = "npm" +paths = ["/name"] +versions = ["/version"] +[[catalog.manifests.dependencies]] +ecosystem = "npm" +paths = ["/devDependencies", "/dependencies"] +[[catalog.manifests.versions]] +language = "javascript" +kind = "requirement" +paths = ["/engines/node"] +[[catalog.manifests.versions]] +language = "javascript" +kind = "pin" +paths = ["/volta/node"] +[[catalog.manifests.versions]] +language = "typescript" +kind = "requirement" +dependency = "npm:typescript" +reference_prefix = "catalog:" +[[catalog.manifests.workspaces]] +ecosystem = "node" +present = "/workspaces" +members = ["/workspaces/packages", "/workspaces"] + +[[catalog.manifests]] +id = "cargo" +files = ["Cargo.toml"] +reader = "toml" +resolver = "cargo" +package = "cargo" +package_path = "/package" +names = ["/package/name"] +[catalog.manifests.identity] +ecosystem = "cargo" +paths = ["/package/name"] +versions = ["/package/version"] +[[catalog.manifests.versions]] +language = "rust" +kind = "requirement" +paths = ["/package/rust-version"] +[[catalog.manifests.workspaces]] +ecosystem = "cargo" +present = "/workspace" +members = ["/workspace/members"] +exclude = ["/workspace/exclude"] + +[[catalog.manifests]] +id = "python" +files = ["pyproject.toml"] +reader = "toml" +names = ["/project/name", "/tool/poetry/name"] +[catalog.manifests.identity] +ecosystem = "pypi" +paths = ["/project/name", "/tool/poetry/name"] +versions = ["/project/version", "/tool/poetry/version"] +normalize = "python" +[[catalog.manifests.dependencies]] +ecosystem = "pypi" +paths = ["/project/dependencies"] +requirements = true +normalize = "python" +[[catalog.manifests.dependencies]] +ecosystem = "pypi" +paths = ["/tool/poetry/dependencies"] +exclude = ["python"] +normalize = "python" +[[catalog.manifests.versions]] +language = "python" +kind = "requirement" +paths = ["/project/requires-python"] +[[catalog.manifests.versions]] +language = "python" +kind = "requirement" +paths = ["/tool/poetry/dependencies/python"] +[[catalog.manifests.workspaces]] +ecosystem = "python" +present = "/tool/uv/workspace" +members = ["/tool/uv/workspace/members"] +exclude = ["/tool/uv/workspace/exclude"] + +[[catalog.manifests]] +id = "pipenv" +files = ["Pipfile"] +reader = "toml" +[[catalog.manifests.dependencies]] +ecosystem = "pypi" +paths = ["/packages"] +normalize = "python" +[[catalog.manifests.versions]] +language = "python" +kind = "requirement" +paths = ["/requires/python_full_version", "/requires/python_version"] + +[[catalog.manifests]] +id = "pnpm" +files = ["pnpm-workspace.yaml"] +reader = "pnpm" + +[[catalog.manifests]] +id = "go-module" +files = ["go.mod"] +reader = "go" + +[[catalog.manifests]] +id = "go-workspace" +files = ["go.work"] +reader = "go" +workspace = "go" + +[[catalog.manifests]] +id = "bundler" +files = ["Gemfile"] +reader = "gemfile" + +[[catalog.manifests]] +id = "rubygems" +files = ["*.gemspec"] +reader = "gemspec" + +[[catalog.manifests]] +id = "requirements" +files = ["requirements*.txt"] +reader = "requirements" + +[[catalog.manifests]] +id = "ruby-pin" +files = [".ruby-version"] +reader = "pin" +[[catalog.manifests.versions]] +language = "ruby" +kind = "pin" + +[[catalog.manifests]] +id = "python-pin" +files = [".python-version"] +reader = "pin" +[[catalog.manifests.versions]] +language = "python" +kind = "pin" + +[[catalog.manifests]] +id = "node-pin" +files = [".node-version", ".nvmrc"] +reader = "pin" +[[catalog.manifests.versions]] +language = "javascript" +kind = "pin" + +[[catalog.manifests]] +id = "rust-pin" +files = ["rust-toolchain"] +reader = "pin" +[[catalog.manifests.versions]] +language = "rust" +kind = "pin" + +[[catalog.manifests]] +id = "go-pin" +files = [".go-version"] +reader = "pin" +[[catalog.manifests.versions]] +language = "go" +kind = "pin" + +[[catalog.manifests]] +id = "php-pin" +files = [".php-version"] +reader = "pin" +[[catalog.manifests.versions]] +language = "php" +kind = "pin" + +[[catalog.manifests]] +id = "java-pin" +files = [".java-version"] +reader = "pin" +[[catalog.manifests.versions]] +language = "java" +kind = "pin" + +[[catalog.manifests]] +id = "rust-toolchain" +files = ["rust-toolchain.toml"] +reader = "toml" +[[catalog.manifests.versions]] +language = "rust" +kind = "toolchain" +paths = ["/toolchain/channel"] + +[[catalog.manifests]] +id = "tool-versions" +files = [".tool-versions"] +reader = "tools" +[[catalog.manifests.versions]] +language = "javascript" +kind = "pin" +key = "nodejs" +[[catalog.manifests.versions]] +language = "go" +kind = "pin" +key = "golang" +[[catalog.manifests.versions]] +language = "java" +kind = "pin" +key = "java" +[[catalog.manifests.versions]] +language = "python" +kind = "pin" +key = "python" +[[catalog.manifests.versions]] +language = "ruby" +kind = "pin" +key = "ruby" +[[catalog.manifests.versions]] +language = "rust" +kind = "pin" +key = "rust" +[[catalog.manifests.versions]] +language = "php" +kind = "pin" +key = "php" + +[[catalog.manifests]] +id = "sdkman" +files = [".sdkmanrc"] +reader = "properties" +[[catalog.manifests.versions]] +language = "java" +kind = "pin" +key = "java" + +[[catalog.manifests]] +id = "dotnet-sdk" +files = ["global.json"] +reader = "json" +[[catalog.manifests.versions]] +language = "csharp" +kind = "sdk" +paths = ["/sdk/version"] + +[[catalog.manifests]] +id = "dotnet-project" +files = ["*.csproj"] +reader = "msbuild" +package = "dotnet" + +[[catalog.manifests]] +id = "solution-xml" +files = ["*.slnx"] +reader = "msbuild" +workspace = "dotnet" + +[[catalog.manifests]] +id = "msbuild-metadata" +files = ["*.props", "*.targets"] +reader = "msbuild" + +[[catalog.manifests]] +id = "solution" +files = ["*.sln"] +reader = "solution" +workspace = "dotnet" + +[[catalog.manifests]] +id = "maven" +files = ["pom.xml"] +reader = "maven" + +[[catalog.manifests]] +id = "gradle" +files = ["*.gradle", "*.gradle.kts"] +reader = "gradle" + +[[catalog.manifests]] +id = "gradle-properties" +files = ["gradle.properties"] +reader = "gradle-properties" + +[[catalog.manifests]] +id = "gradle-jvm" +files = ["gradle-daemon-jvm.properties"] +reader = "gradle-jvm" + +[[catalog.manifests]] +id = "version-catalog" +files = ["*.versions.toml"] +reader = "version-catalog" + [[rules]] id = "composer" language = "php" diff --git a/src/detect/tests.rs b/src/detect/tests.rs index 99838b8..b509fd2 100644 --- a/src/detect/tests.rs +++ b/src/detect/tests.rs @@ -1062,3 +1062,161 @@ fn provider_file_guards_do_not_imply_framework_consumption() { [super::FrameworkRole::Provides] ); } + +#[test] +fn custom_manifest_registration_emits_facts_without_a_new_parser() { + let dir = fixture(&[ + ( + "service.meta.json", + r#"{"project":{"name":"platform","version":"1.0"},"uses":{"next":"16"},"runtime":{"node":">=26"},"modules":["services/*"]}"#, + ), + ( + "services/api/service.meta.json", + r#"{"project":{"name":"api"},"runtime":{"node":">=24"}}"#, + ), + ("unrelated.json", "not a manifest and must not be parsed"), + ]); + let config = r#" +[[catalog.manifests]] +id = "service-meta" +files = ["service.meta.json"] +reader = "json" +package = "service-meta" +names = ["/project/name"] +[catalog.manifests.identity] +ecosystem = "company" +paths = ["/project/name"] +versions = ["/project/version"] +[[catalog.manifests.dependencies]] +ecosystem = "npm" +paths = ["/uses"] +[[catalog.manifests.versions]] +language = "javascript" +kind = "requirement" +paths = ["/runtime/node"] +[[catalog.manifests.workspaces]] +ecosystem = "service-meta" +present = "/modules" +members = ["/modules"] +[[catalog.workspaces]] +ecosystem = "service-meta" +package = "service-meta" +[[rules]] +id = "company-project" +language = "javascript" +kind = "project" +priority = 10 +any = ["package:service-meta"] +[[rules]] +id = "company-workspace" +language = "javascript" +kind = "workspace" +priority = 20 +any = ["workspace:service-meta"] +"#; + let report = scan(dir.path(), Some(config)).unwrap(); + assert!(report.diagnostics.is_empty(), "{report:#?}"); + assert_eq!(report.stats.files_read, 2); + let root = project(&report, "."); + assert_eq!(root.names, ["platform"]); + assert!(detects(root, "nextjs")); + assert!(detects(root, "company-workspace")); + assert_eq!(root.workspace_members, ["services/api"]); + assert!(version(root, "javascript", ">=26", "service.meta.json")); + assert!(version( + project(&report, "services/api"), + "javascript", + ">=24", + "services/api/service.meta.json" + )); +} + +#[test] +fn xml_property_and_reference_policies_are_extensible_in_toml() { + let dir = fixture(&[( + "App.csproj", + "15.0", + )]); + let config = r#" +[[catalog.property_versions]] +reader = "msbuild" +fields = ["companycompilerlevel"] +language = "csharp" +kind = "language" +[catalog.reference_kinds.CompanyReference] +ecosystem = "company" +normalize = "lowercase" +[[rules]] +id = "company-web" +language = "csharp" +kind = "framework" +priority = 100 +any = ["dep:company:company.web"] +"#; + let report = scan(dir.path(), Some(config)).unwrap(); + assert!(report.diagnostics.is_empty(), "{report:#?}"); + let app = project(&report, "."); + assert!(version(app, "csharp", "15.0", "App.csproj")); + assert!(detects(app, "company-web")); + assert_eq!( + app.detections + .iter() + .find(|d| d.id == "company-web") + .unwrap() + .roles, + [super::FrameworkRole::Uses] + ); +} + +#[test] +fn manifest_schema_errors_are_rejected_before_scanning() { + let config = "[[catalog.manifests]]\nid='custom'\nfiles=['custom.json']\nreader='json'\n"; + for invalid in [ + config.replace("reader='json'", "reader='command'"), + config.replace("custom.json", "../custom.json"), + format!("{config}{config}"), + format!("{config}names=['not-a-pointer']\n"), + format!("{config}unknown_option=true\n"), + "[[catalog.manifests]]\nid='conflict'\nfiles=['package.json']\nreader='toml'\n".to_owned(), + "[[catalog.property_versions]]\nreader='msbuid'\nfields=['newfield']\nlanguage='csharp'\nkind='language'\n".to_owned(), + ] { + let error = scan( + std::path::Path::new("/nonexistent-detection-fixture"), + Some(&invalid), + ) + .unwrap_err(); + assert!(!error.to_string().contains("could not open"), "{error}"); + } +} + +#[test] +fn compiled_basename_patterns_preserve_glob_matching() { + for pattern in [ + "*.csproj", + "requirements*.txt", + "a*b", + "a?b", + "a**b", + "Cargo.toml", + "ab*bc", + ] { + let compiled = super::rules::FilePattern::new(pattern); + for value in [ + "App.csproj", + "requirements.txt", + "requirements-dev.txt", + "ab", + "a123b", + "Cargo.toml", + "abc", + "abbc", + "", + ] { + assert_eq!( + compiled.matches(value), + super::rules::glob(pattern, value), + "{pattern}: {value}" + ); + } + } +} diff --git a/src/detect/xml.rs b/src/detect/xml.rs index e099e85..c91f42e 100644 --- a/src/detect/xml.rs +++ b/src/detect/xml.rs @@ -324,8 +324,9 @@ mod tests { #[ignore = "set XML_BENCH_CACHE to pinned detect-corpus cache; run release with --ignored --nocapture"] fn benchmark_xml_corpus() -> Result<()> { let cache = PathBuf::from(std::env::var("XML_BENCH_CACHE")?); - let manifest: serde_json::Value = - serde_json::from_str(include_str!("../../tests/detect-corpus.json"))?; + let manifest: serde_json::Value = serde_json::from_str(&fs::read_to_string( + Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/detect-corpus.json"), + )?)?; let mut all = Vec::new(); let entries = manifest["repositories"].as_array().unwrap(); for entry in entries { diff --git a/tests/audit_detect.py b/tests/audit_detect.py index f4e5e28..5c9b123 100644 --- a/tests/audit_detect.py +++ b/tests/audit_detect.py @@ -184,7 +184,9 @@ def main(): type=Path, default=Path(__file__).resolve().parents[1] / "target/release/brainpod", ) - parser.add_argument("--cache", type=Path, required=True) + parser.add_argument( + "--cache", type=Path, help="Snapshot cache; required unless --list" + ) parser.add_argument( "--fetch", action="store_true", @@ -196,16 +198,29 @@ def main(): default=5, help="CLI invocations per repository; the first is excluded from the warm median", ) - parser.add_argument("--output", type=Path, required=True) + parser.add_argument( + "--output", type=Path, help="Results JSON; required unless --list" + ) + parser.add_argument( + "--list", + action="store_true", + help="List pinned repositories without downloading or running the detector", + ) args = parser.parse_args() + corpus = json.loads(args.corpus.read_text()) + if corpus["schemaVersion"] != 1: + parser.error("unsupported corpus schema version") + if args.list: + for entry in corpus["repositories"]: + print(f"{entry['language']}\t{entry['repository']}\t{entry['revision']}") + return 0 + if args.cache is None or args.output is None: + parser.error("--cache and --output are required unless --list") if args.runs < 2: parser.error("--runs must be at least 2") args.binary = args.binary.resolve(strict=True) args.cache = args.cache.resolve() args.cache.mkdir(parents=True, exist_ok=True) - corpus = json.loads(args.corpus.read_text()) - if corpus["schemaVersion"] != 1: - parser.error("unsupported corpus schema version") with args.binary.open("rb") as binary: binary_sha = hashlib.file_digest(binary, "sha256").hexdigest() results = [] diff --git a/tests/detect-corpus.md b/tests/detect-corpus.md index 6c763de..23c4c2c 100644 --- a/tests/detect-corpus.md +++ b/tests/detect-corpus.md @@ -39,6 +39,29 @@ The expanded corpus covers **45 repositories across nine languages**: the origin Additional unscored coverage gaps include Jekyll, Hugo, Dioxus, Ember, Gradio, and aiohttp. Their generic language/project fallback is not counted as a failed framework assertion. Test fixtures and example projects are intentionally scanned, so the report is not a list of deployable services. +### Engine refactor validation + +The manifest catalog now declares filename-to-reader dispatch, JSON/TOML field extraction, workspace candidate selection, language families, inheritance boundaries, exclusions, property/version mappings, and MSBuild reference kinds. Detection predicates compile into typed fact indexes. Tests add an entire JSON manifest format and an XML property/reference policy through TOML only, rather than adding another language-specific Rust branch. + +All 45 complete reports match the pre-refactor binary after canonicalizing version-declaration ordering. The 227 corpus assertions still pass, including framework provider/consumer roles and version kinds. The ten starter branches are rerun too; Outline's unsupported Koa label and the redundant Ruby file-directive warning remain documented rather than counted as passing main-stack assertions. Version declarations are now sorted consistently, independent of reader execution order. + +The final local rerun passed 179 Rust tests and nine audit-runner tests. The XML corpus and portable benchmarks were run explicitly (they are ignored in ordinary test runs). The declarative filename router indexes known extensions and uses compiled basename matchers; this avoids paying for every manifest wildcard on every ordinary source filename. The 200-service/20,200-file warm synthetic scan remains roughly 4–5 ms on this host, reading only 200 manifests. The configuration compiler adds roughly half a millisecond of cold CLI startup in the local empty-directory A/B; these measurements are not cross-machine guarantees. + +### XML I/O and parser measurement + +The pinned cache contains 1,192 selected XML manifests, totaling 2,714,415 bytes and 31,157 closed/empty elements. Three independent release benchmark invocations measured the following aggregate warm-cache medians. Read timing includes bounded `open`/`read_to_end` plus UTF-8 ownership/validation; parse timing uses preloaded strings and a black-box no-op visitor. Discovery, .NET/Maven fact extraction, rules, and CLI output are excluded. These are not cold-disk measurements or timing budgets for CI. + +| XML kind | Files | Bytes | Warm read + UTF-8, ms | Preloaded parse, ms | +|---|---:|---:|---:|---:| +| csproj | 803 | 1,781,367 | 1.795 | 4.387 | +| props | 107 | 328,021 | 0.212 | 0.835 | +| targets | 67 | 178,848 | 0.131 | 0.539 | +| slnx | 7 | 105,053 | 0.019 | 0.249 | +| pom | 208 | 321,126 | 0.438 | 1.290 | +| **All XML** | **1,192** | **2,714,415** | **2.668** | **7.414** | + +XML is not demonstrated to be the whole-engine bottleneck. A text-allocation fast path reduced allocation calls by 9.5%, but paired CPU runs showed only a noisy 0.7% difference; separate invocations were sometimes slower. It was rejected, and production XML parsing/validation is unchanged. The ignored corpus and portable XML microbenchmarks are retained so future parser changes can be measured rather than guessed. + ### Timing Each repository was scanned through the release CLI five times, serially. The table reports the median of runs 2–5 with a warm filesystem, including process startup, traversal, JSON serialization, and stdout transfer. Python JSON parsing and snapshot downloads are excluded. These are local measurements, not portable timing budgets or controlled cold-cache benchmarks. @@ -130,17 +153,10 @@ Timings are warm-filesystem, end-to-end CLI medians of runs 2–5. Empty or part Koa needs an additional npm framework rule. C#/.NET and Java/Spring are now supported and verified by the follow-up run. The Maybe warning is redundant with the already detected Ruby pin. None of these observations was silently converted into a passing assertion or hidden by skipping a branch. The initial validation was read-only; the follow-up uses the new language support. -## CI regression investigation - -The Intel macOS failure is `s3::tests::streaming_download_exceeds_small_copy_limit_without_credentials_or_summary`, not a detector compilation failure. Replacing the fixture's five-second total deadline with connect/read-inactivity limits fixes a controlled slow-transfer failure, but native run `42725208393` still failed the same test after a 7.18-second suite. The original timeout diagnosis was therefore insufficient to establish the macOS cause; all other platform jobs passed. - -Follow-up controls show a progressing transfer can exceed five seconds while retaining the five-second read-inactivity bound. Record-sized 16 KiB TLS writes are substantially faster than the original 64 KiB write-all progression locally, but Linux cannot prove the native macOS cause. The fixture now uses those record-sized writes and joins its provider asynchronously before unwrapping the download result, so server-side I/O panics are captured without starving Tokio shutdown. Transfer-size, redirect, and credential assertions are unchanged. No timeouts were increased and no checks were disabled. - -Nix now streams full build logs, and test-only body failures report received bytes and URL-stripped error details. Production error messages remain sanitized and unchanged. Native macOS confirmation is still required for this follow-up. - ## Reproduce ```sh +python3 tests/audit_detect.py --list cargo build --release --locked cache=$(mktemp -d) python3 tests/audit_detect.py --fetch --cache "$cache" --output "$cache/results.json" @@ -152,4 +168,10 @@ Downloads require the explicit `--fetch` flag; omit it to rerun against an exist PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s tests -p 'test_*.py' -v ``` -The runner uses Python 3.11+ and the standard library only. Full per-check results and repeat timings are written to the specified JSON output; downloaded repositories and generated reports are not committed. +Use `--list` to print all repository names and pinned revisions without a binary, cache, downloads, or network access. The complete language-grouped list is also in the main README. The runner uses Python 3.11+ and the standard library only. Full per-check results and repeat timings are written to the specified JSON output; downloaded repositories and generated reports are not committed. + +```sh +XML_BENCH_CACHE="$cache" cargo test --release detect::xml::tests::benchmark_xml_corpus \ + -- --ignored --nocapture --test-threads=1 +cargo test --release detect::xml::tests::benchmark_xml_local -- --ignored --nocapture +``` diff --git a/tests/test_audit_detect.py b/tests/test_audit_detect.py index ff6dbd7..8012b85 100644 --- a/tests/test_audit_detect.py +++ b/tests/test_audit_detect.py @@ -83,6 +83,44 @@ def test_role_and_version_kind_expectations_are_enforced(self): expected["versionKind"] = "sdk" self.assertTrue(audit_detect.check({".": self.project}, expected)[0]) + def test_listing_needs_no_binary_cache_or_network(self): + with ( + tempfile.TemporaryDirectory() as directory, + patch("urllib.request.urlopen") as request, + ): + corpus = Path(directory) / "corpus.json" + corpus.write_text( + json.dumps( + { + "schemaVersion": 1, + "repositories": [ + { + "language": "Java", + "repository": "owner/app", + "revision": "a" * 40, + } + ], + } + ) + ) + with ( + patch( + "sys.argv", + [ + "audit_detect.py", + "--corpus", + str(corpus), + "--binary", + "/missing/binary", + "--list", + ], + ), + patch("sys.stdout", new_callable=io.StringIO) as output, + ): + self.assertEqual(audit_detect.main(), 0) + self.assertIn("Java\towner/app\t" + "a" * 40, output.getvalue()) + request.assert_not_called() + def archive(self, path="repo/package.json", size=None): data = b"{}" if size is None else b"x" * size output = io.BytesIO() From 124014e7793baba392d3eb31cae3341ca9562c0c Mon Sep 17 00:00:00 2001 From: Dillen Meijboom Date: Tue, 10 Nov 2026 08:37:09 +0000 Subject: [PATCH 10/10] feat: detect Deno projects and mixed workspaces --- README.md | 10 +- src/detect/catalog.rs | 88 +++++- src/detect/deno.rs | 306 ++++++++++++++++++++ src/detect/deno_tests.rs | 306 ++++++++++++++++++++ src/detect/jsonc.rs | 200 +++++++++++++ src/detect/mod.rs | 116 +++++++- src/detect/parsers.rs | 45 ++- src/detect/rules.rs | 7 + src/detect/rules.toml | 80 +++++- src/detect/values.rs | 1 + tests/audit_detect.py | 46 +++ tests/detect-corpus.json | 558 +++++++++++++++++++++++++++++++++++++ tests/detect-corpus.md | 25 +- tests/test_audit_detect.py | 101 +++++++ 14 files changed, 1855 insertions(+), 34 deletions(-) create mode 100644 src/detect/deno.rs create mode 100644 src/detect/deno_tests.rs create mode 100644 src/detect/jsonc.rs diff --git a/README.md b/README.md index 1363b66..f0b5175 100644 --- a/README.md +++ b/README.md @@ -274,9 +274,11 @@ brainpod detect ./services --json brainpod detect . --rules ./detection.toml --json ``` -The result contains sorted project paths, manifest names, languages and version declarations, priority-ranked detections with evidence and `uses`/`provides` framework roles, workspace members, diagnostics, and scan statistics. Framework versions are attached to dependency evidence. JavaScript versions describe Node.js; TypeScript versions describe the compiler. C# versions distinguish `language` (LangVersion), `target` (TFM), `runtime`, and `sdk`; Java distinguishes compiler `source`/`target`/`release`, toolchains, build-JDK metadata, and pins. Requirements and pins retain their source and kind: these are declarations, not a claim about installed versions. Missing versions remain unknown. +The result contains sorted project paths, manifest names, languages and version declarations, priority-ranked detections with evidence and `uses`/`provides` framework roles, workspace members, diagnostics, and scan statistics. Optional `runtimes` records Deno declarations separately from JavaScript/TypeScript versions; `workspaces` records each manager’s ecosystem, source, and members, while `workspaceMembers` remains their union. Framework versions are attached to dependency evidence. JavaScript versions describe Node.js; TypeScript versions describe the compiler. C# versions distinguish `language` (LangVersion), `target` (TFM), `runtime`, and `sdk`; Java distinguishes compiler `source`/`target`/`release`, toolchains, build-JDK metadata, and pins. Requirements and pins retain their source and kind: these are declarations, not a claim about installed versions. Missing versions remain unknown. -Built-in coverage includes Laravel, Symfony, CakePHP, CodeIgniter, Composer; Go modules/workspaces and Gin/Echo/Fiber; Cargo workspaces, axum, Rocket, Actix; Bundler, Rails, Sinatra, Hanami; Python requirements, PEP 621, Poetry, Pipenv, uv workspaces, Django, FastAPI, Flask, Starlette, Celery; npm/pnpm workspaces and catalogs, Next.js, Nuxt, Astro, SvelteKit, Remix, Angular, Express, React, Vue, Vite; C#/.NET projects and solutions, ASP.NET Core/classic, Aspire, FastEndpoints; Java/JVM Maven and Gradle projects, Spring/Spring Boot, Quarkus, Micronaut; and standalone source files in all nine languages. Rust workspace dependency aliases and inherited `rust-version` are understood. Ancestor runtime pin/toolchain files are included with provenance. Poetry names, literal gemspec metadata, and Rails component dependencies are recognized. pnpm catalog references retain the original declaration and add the locally resolved constraint with its YAML source; unresolved references produce diagnostics. +Built-in coverage includes Laravel, Symfony, CakePHP, CodeIgniter, Composer; Go modules/workspaces and Gin/Echo/Fiber; Cargo workspaces, axum, Rocket, Actix; Bundler, Rails, Sinatra, Hanami; Python requirements, PEP 621, Poetry, Pipenv, uv workspaces, Django, FastAPI, Flask, Starlette, Celery; npm/pnpm and Deno workspaces and catalogs, Fresh, Oak, Hono, Next.js, Nuxt, Astro, SvelteKit, Remix, Angular, Express, React, Vue, Vite; C#/.NET projects and solutions, ASP.NET Core/classic, Aspire, FastEndpoints; Java/JVM Maven and Gradle projects, Spring/Spring Boot, Quarkus, Micronaut; and standalone source files in all nine languages. Rust workspace dependency aliases and inherited `rust-version` are understood. Ancestor runtime pin/toolchain files are included with provenance. Poetry names, literal gemspec metadata, and Rails component dependencies are recognized. pnpm catalog references retain the original declaration and add the locally resolved constraint with its YAML source; unresolved references produce diagnostics. + +Deno supports `deno.json`/`deno.jsonc` (comments and trailing commas), Deno-only, hybrid, and npm-only workspace members, globs/exclusions, and shared imports with member alias overrides. Registry targets (`npm:`, `jsr:`) and supported `deno.land/x`/`esm.sh` URLs supply dependency evidence without downloading modules. Local relative import maps are bounded and cannot escape the scan root or follow symlinks. `.deno-version`, `.tool-versions` and npm `engines.deno` retain explicit runtime version provenance; a JSR package’s `version` is never treated as a Deno runtime version. Peer and development dependency constraints remain separate evidence. pnpm-only membership is not inferred as Deno membership. Scoped import-map overrides and non-relative/remote import-map files are diagnosed, not fetched or guessed; source imports, lockfiles, Docker/CI version hints, and tasks are not evaluated. The engine has three layers: bounded discovery, format readers that emit facts, and compiled detection rules. [`src/detect/rules.toml`](src/detect/rules.toml) owns manifest filenames/readers, JSON/TOML field projections, version/property mappings, workspace candidates, source-language families, inheritance boundaries, and directory exclusions. Format-specific syntax and local-reference resolution stay in adapters; framework decisions do not. @@ -331,7 +333,7 @@ cargo test --release detect::tests::benchmark_polyglot_scan -- --ignored --nocap ### Reproduce the repository checks -The corpus is public and pinned to commit SHAs in [`tests/detect-corpus.json`](tests/detect-corpus.json). The [audit report](tests/detect-corpus.md) contains assertions, per-repository timings, known limitations, and the additional Brainpod starter branches. +The corpus contains 51 repositories: the original 45, five ranked Deno web projects, and one supplemental mixed-workspace repository. It is public and pinned to commit SHAs in [`tests/detect-corpus.json`](tests/detect-corpus.json). The [audit report](tests/detect-corpus.md) contains assertions, per-repository timings, known limitations, and the additional Brainpod starter branches. | Language | Repositories | |---|---| @@ -344,6 +346,8 @@ The corpus is public and pinned to commit SHAs in [`tests/detect-corpus.json`](t | Rust | [dani-garcia/vaultwarden](https://github.com/dani-garcia/vaultwarden), [meilisearch/meilisearch](https://github.com/meilisearch/meilisearch), [juspay/hyperswitch](https://github.com/juspay/hyperswitch), [parcel-bundler/parcel](https://github.com/parcel-bundler/parcel), [DioxusLabs/dioxus](https://github.com/DioxusLabs/dioxus) | | C# | [jellyfin/jellyfin](https://github.com/jellyfin/jellyfin), [dotnet/aspnetcore](https://github.com/dotnet/aspnetcore), [jasontaylordev/CleanArchitecture](https://github.com/jasontaylordev/CleanArchitecture), [bitwarden/server](https://github.com/bitwarden/server), [ardalis/CleanArchitecture](https://github.com/ardalis/CleanArchitecture) | | Java | [macrozheng/mall](https://github.com/macrozheng/mall), [spring-projects/spring-boot](https://github.com/spring-projects/spring-boot), [elastic/elasticsearch](https://github.com/elastic/elasticsearch), [spring-projects/spring-framework](https://github.com/spring-projects/spring-framework), [jeecgboot/JeecgBoot](https://github.com/jeecgboot/JeecgBoot) | +| Deno ecosystem (JS/TS) | [supabase/supabase](https://github.com/supabase/supabase), [honojs/hono](https://github.com/honojs/hono), [freshframework/fresh](https://github.com/freshframework/fresh), [InsForge/InsForge](https://github.com/InsForge/InsForge), [ryo-ma/github-profile-trophy](https://github.com/ryo-ma/github-profile-trophy) | +| Supplemental mixed Deno/npm workspace | [honojs/middleware](https://github.com/honojs/middleware) | ```sh python3 tests/audit_detect.py --list diff --git a/src/detect/catalog.rs b/src/detect/catalog.rs index b9fe551..423b1d2 100644 --- a/src/detect/catalog.rs +++ b/src/detect/catalog.rs @@ -12,6 +12,7 @@ use super::{Node, Version, Workspace, relative_file}; #[serde(rename_all = "kebab-case")] pub(super) enum Reader { Json, + Jsonc, Toml, Pnpm, Go, @@ -148,6 +149,8 @@ impl PropertyVersion { pub(super) struct Manifest { id: String, files: Vec, + #[serde(default)] + prefer_first: bool, pub reader: Reader, pub resolver: Option, pub package: Option, @@ -162,6 +165,7 @@ pub(super) struct Manifest { #[serde(default)] workspaces: Vec, pub workspace: Option, + pub runtime: Option, } #[derive(Deserialize)] @@ -222,7 +226,9 @@ struct Dependencies { #[derive(Deserialize)] #[serde(deny_unknown_fields)] struct VersionField { + #[serde(default)] language: String, + runtime: Option, kind: String, #[serde(default)] paths: Vec, @@ -329,11 +335,19 @@ impl Catalog { } } for version in &manifest.versions { - if version.language.is_empty() || version.kind.is_empty() { - bail!("manifest version fields need a language and kind"); + if version.language.is_empty() + == version + .runtime + .as_ref() + .is_none_or(|runtime| runtime.is_empty()) + || version.kind.is_empty() + { + bail!( + "manifest version fields need exactly one language/runtime target and a kind" + ); } let valid_source = match manifest.reader { - Reader::Json | Reader::Toml => { + Reader::Json | Reader::Jsonc | Reader::Toml => { !version.paths.is_empty() || version.dependency.is_some() } Reader::Pin => { @@ -550,7 +564,21 @@ pub(super) fn workspace_patterns(patterns: Vec) -> (Vec, Vec) -> bool { + if self.prefer_first { + self.files + .iter() + .find(|name| files.contains(*name)) + .is_none_or(|name| name == file) + } else { + true + } + } pub fn apply(&self, node: &mut Node, file: &str, document: &Value) { + if let Some(runtime) = &self.runtime { + node.runtimes + .insert(runtime.clone(), relative_file(&node.path, file)); + } if let Some(name) = selected_scalar(document, &self.names) { node.names.insert(name.to_owned()); } @@ -640,7 +668,22 @@ impl Manifest { } if let Some(key) = &field.dependency { if let Some(dependency) = node.dependencies.get(key) { - let mut versions = Vec::new(); + let mut versions = dependency + .declarations + .iter() + .map(|version| Version { + kind: if field + .reference_prefix + .as_ref() + .is_some_and(|prefix| version.value.starts_with(prefix)) + { + "reference".to_owned() + } else { + field.kind.clone() + }, + ..version.clone() + }) + .collect::>(); if let Some(raw) = &dependency.version { let kind = if field .reference_prefix @@ -663,8 +706,9 @@ impl Manifest { ..resolved.clone() }); } - node.versions - .entry(field.language.clone()) + field + .target(node) + .entry(field.name().to_owned()) .or_default() .extend(versions); } @@ -672,7 +716,7 @@ impl Manifest { .and_then(|doc| selected_scalar(doc, &field.paths)) .or(text) { - node.version(&field.language, value, file, &field.kind); + field.emit(node, value, file); } } } @@ -699,7 +743,7 @@ impl Manifest { .iter() .filter(|field| field.key.as_deref() == Some(key)) { - node.version(&field.language, &value, file, &field.kind); + field.emit(node, &value, file); } } } @@ -707,7 +751,7 @@ impl Manifest { let properties = values::properties(text, &relative_file(&node.path, file)); for field in &self.versions { if let Some(value) = field.key.as_ref().and_then(|key| properties.get(key)) { - node.version(&field.language, &value.value, file, &field.kind); + field.emit(node, &value.value, file); } } } @@ -715,3 +759,29 @@ impl Manifest { } } } + +impl VersionField { + fn name(&self) -> &str { + self.runtime.as_deref().unwrap_or(&self.language) + } + fn target<'a>(&self, node: &'a mut Node) -> &'a mut BTreeMap> { + if self.runtime.is_some() { + &mut node.runtime_versions + } else { + &mut node.versions + } + } + fn emit(&self, node: &mut Node, value: &str, file: &str) { + if !value.trim().is_empty() { + let version = Version { + value: value.trim().to_owned(), + source: relative_file(&node.path, file), + kind: self.kind.clone(), + }; + self.target(node) + .entry(self.name().to_owned()) + .or_default() + .push(version); + } + } +} diff --git a/src/detect/deno.rs b/src/detect/deno.rs new file mode 100644 index 0000000..83b703a --- /dev/null +++ b/src/detect/deno.rs @@ -0,0 +1,306 @@ +use std::collections::{BTreeMap, HashMap}; +use std::fs::{self, File}; +use std::io::Read; +use std::path::{Component, Path, PathBuf}; + +use serde_json::Value; + +use super::parsers::{Catalogs, Documents}; +use super::{Dependency, Diagnostic, Node, Stats, Workspace, display_path, relative_file}; + +type Bindings = BTreeMap>; + +struct Context { + source: String, + workspaces: Vec, + bindings: Bindings, +} + +pub(super) fn resolve( + root: &Path, + nodes: &mut [Node], + documents: &Documents, + catalogs: &mut HashMap>, + stats: &mut Stats, + diagnostics: &mut Vec, +) { + let mut contexts = HashMap::new(); + for node in nodes.iter() { + let Some(source) = node.packages.get("deno") else { + continue; + }; + let Some(file) = Path::new(source).file_name().and_then(|file| file.to_str()) else { + continue; + }; + let Some(document) = documents.get(&node.path).and_then(|docs| docs.get(file)) else { + continue; + }; + let mut bindings = Bindings::new(); + if let Some(reference) = document.get("importMap").and_then(Value::as_str) { + if document.get("imports").is_some() || document.get("scopes").is_some() { + diagnostics.push(Diagnostic { + path: source.clone(), + message: "importMap is exclusive with imports/scopes".to_owned(), + }); + } else if let Some((source, map)) = + import_map(root, &node.path, reference, stats, diagnostics) + { + imports(&map, &source, &mut bindings, diagnostics); + } + } else { + imports(document, source, &mut bindings, diagnostics); + } + let workspaces = node + .workspaces + .iter() + .filter(|workspace| { + workspace.ecosystem == "deno" + || (workspace.ecosystem == "node" + && workspace.source == relative_file(&node.path, "package.json")) + }) + .cloned() + .collect::>(); + let package = documents + .get(&node.path) + .and_then(|docs| docs.get("package.json")); + let definitions = package + .filter(|doc| doc.get("catalog").is_some() || doc.get("catalogs").is_some()) + .unwrap_or(document); + if definitions.get("catalog").is_some() || definitions.get("catalogs").is_some() { + let source = if std::ptr::eq(definitions, document) { + source.clone() + } else { + relative_file(&node.path, "package.json") + }; + catalogs.entry(node.path.clone()).or_insert_with(|| { + Some(Catalogs { + workspaces: workspaces.clone(), + source, + catalog: definitions + .get("catalog") + .and_then(|v| serde_json::from_value(v.clone()).ok()) + .unwrap_or_default(), + catalogs: definitions + .get("catalogs") + .and_then(|v| serde_json::from_value(v.clone()).ok()) + .unwrap_or_default(), + }) + }); + } + contexts.insert( + node.path.clone(), + Context { + source: source.clone(), + workspaces, + bindings, + }, + ); + } + if contexts.is_empty() { + return; + } + for node in nodes { + let local = contexts.get(&node.path); + let inherited = node.path.ancestors().skip(1).find_map(|path| { + contexts.get(path).filter(|context| { + node.files.iter().any(|file| { + matches!(file.as_str(), "deno.json" | "deno.jsonc" | "package.json") + }) && node.path.strip_prefix(path).ok().is_some_and(|relative| { + context + .workspaces + .iter() + .any(|workspace| workspace.contains(&display_path(relative))) + }) + }) + }); + if let Some(context) = local.or(inherited) { + node.runtimes + .entry("deno".to_owned()) + .or_insert_with(|| context.source.clone()); + } + let mut bindings = inherited.map_or_else(Bindings::new, |context| context.bindings.clone()); + if let Some(local) = local { + bindings.extend(local.bindings.clone()); + } + for (key, dependency) in bindings.into_values().flatten() { + node.merge_dependency(key, dependency); + } + } +} + +fn imports( + document: &Value, + source: &str, + bindings: &mut Bindings, + diagnostics: &mut Vec, +) { + if let Some(entries) = document.get("imports").and_then(Value::as_object) { + for (alias, value) in entries { + bindings.insert( + alias.clone(), + value + .as_str() + .and_then(|value| { + specifier(value).or_else(|| { + value.starts_with("catalog:").then_some(( + "npm", + alias.as_str(), + Some(value), + )) + }) + }) + .map(|(ecosystem, name, version)| { + ( + format!("{ecosystem}:{name}"), + Dependency { + version: version.map(str::to_owned), + source: source.to_owned(), + resolved: None, + declarations: Vec::new(), + }, + ) + }), + ); + } + } + if document + .get("scopes") + .is_some_and(|scopes| scopes.as_object().is_none_or(|scopes| !scopes.is_empty())) + { + diagnostics.push(Diagnostic { + path: source.to_owned(), + message: "scoped import-map overrides are not evaluated".to_owned(), + }); + } +} + +fn specifier(value: &str) -> Option<(&str, &str, Option<&str>)> { + let (ecosystem, package) = if let Some(package) = value.strip_prefix("npm:") { + ("npm", package) + } else if let Some(package) = value + .strip_prefix("jsr:") + .or_else(|| value.strip_prefix("https://jsr.io/")) + { + ("jsr", package) + } else if let Some(package) = value.strip_prefix("https://deno.land/x/") { + ("deno", package) + } else if let Some(package) = value.strip_prefix("https://esm.sh/") { + ("npm", package) + } else { + return None; + }; + let start = if package.starts_with('@') { + package.find('/')? + 1 + } else { + 0 + }; + let tail = &package[start..]; + let end = tail + .find(['@', '/', '?', '#']) + .map_or(package.len(), |index| start + index); + let name = &package[..end]; + if name.is_empty() || name.ends_with('/') { + return None; + } + let version = package[end..] + .strip_prefix('@') + .map(|value| value.split(['/', '?', '#']).next().unwrap_or("")); + if version == Some("") { + return None; + } + Some((ecosystem, name, version)) +} + +fn import_map( + root: &Path, + base: &Path, + reference: &str, + stats: &mut Stats, + diagnostics: &mut Vec, +) -> Option<(String, Value)> { + let loaded = (|| -> anyhow::Result<(String, Value)> { + if reference.contains(':') || reference.contains('\\') || Path::new(reference).is_absolute() + { + anyhow::bail!("only local relative import maps are inspected"); + } + let mut relative = PathBuf::new(); + for part in base.join(reference).components() { + match part { + Component::Normal(value) => relative.push(value), + Component::CurDir => (), + Component::ParentDir if relative.pop() => (), + _ => anyhow::bail!("import map escapes detection root"), + } + } + let mut path = root.to_path_buf(); + for part in &relative { + path.push(part); + if fs::symlink_metadata(&path)?.file_type().is_symlink() { + anyhow::bail!("import-map symlinks are not followed"); + } + } + if !fs::metadata(&path)?.is_file() { + anyhow::bail!("import map must be a regular file"); + } + let file = File::open(&path)?; + let mut bytes = Vec::new(); + file.take(super::MAX_FILE_BYTES + 1) + .read_to_end(&mut bytes)?; + stats.files_read += 1; + stats.bytes_read += bytes.len(); + if bytes.len() > super::MAX_FILE_BYTES as usize { + anyhow::bail!("import map exceeds 1 MiB read limit"); + } + let text = String::from_utf8(bytes)?; + let document = super::jsonc::parse(&text).map_err(anyhow::Error::msg)?; + if !document.is_object() { + anyhow::bail!("import map must be an object"); + } + Ok((display_path(&relative), document)) + })(); + match loaded { + Ok(result) => Some(result), + Err(error) => { + diagnostics.push(Diagnostic { + path: relative_file(base, reference), + message: format!("could not inspect import map: {error}"), + }); + None + } + } +} + +#[cfg(test)] +mod tests { + use super::specifier; + #[test] + fn registry_and_url_specifiers_use_targets_not_alias_names() { + for (input, expected) in [ + ("npm:hono@^4/subpath", Some(("npm", "hono", Some("^4")))), + ( + "npm:@scope/pkg@~2.1/sub", + Some(("npm", "@scope/pkg", Some("~2.1"))), + ), + ( + "jsr:@fresh/core@^2.0.0", + Some(("jsr", "@fresh/core", Some("^2.0.0"))), + ), + ( + "https://deno.land/x/oak@v17.0.0/mod.ts", + Some(("deno", "oak", Some("v17.0.0"))), + ), + ( + "https://esm.sh/preact@10.2.0?dev", + Some(("npm", "preact", Some("10.2.0"))), + ), + ("jsr:@oak/oak", Some(("jsr", "@oak/oak", None))), + ("https://deno.land.evil/x/fresh@1.0", None), + ("./npm:hono@4.ts", None), + ("npm:", None), + ("npm:@scope", None), + ("npm:hono@", None), + ] { + assert_eq!(specifier(input), expected, "{input}"); + } + } +} diff --git a/src/detect/deno_tests.rs b/src/detect/deno_tests.rs new file mode 100644 index 0000000..86dcc11 --- /dev/null +++ b/src/detect/deno_tests.rs @@ -0,0 +1,306 @@ +use super::scan; +use super::tests::{detects, fixture, project, version}; + +#[test] +fn deno_jsonc_detects_runtime_and_package_versions_without_conflation() { + let dir = fixture(&[ + ( + "deno.jsonc", + r#"{ // package version is not a Deno runtime pin + "name":"@acme/service", "version":"9.9.9", "imports":{"web":"jsr:@hono/hono@^4",}, + }"#, + ), + (".tool-versions", "deno 2.5.6\nnodejs 24.1.0\n"), + ("main.ts", "throw new Error('must not execute');"), + ]); + let report = scan(dir.path(), None).unwrap(); + let root = project(&report, "."); + assert!(detects(root, "deno")); + assert!(detects(root, "hono")); + assert_eq!(root.runtimes.len(), 1); + assert_eq!(root.runtimes[0].name, "deno"); + assert_eq!(root.runtimes[0].source, "deno.jsonc"); + assert_eq!(root.runtimes[0].versions[0].value, "2.5.6"); + assert_eq!(root.runtimes[0].versions[0].source, ".tool-versions"); + assert!(version(root, "javascript", "24.1.0", ".tool-versions")); + assert!(root.languages.iter().all(|language| { + language.name != "deno" + && language + .versions + .iter() + .all(|version| version.value != "9.9.9" && version.value != "2.5.6") + })); + assert_eq!(report.stats.files_read, 2); + assert!(report.diagnostics.is_empty()); +} + +#[test] +fn mixed_deno_npm_and_pnpm_workspaces_preserve_boundaries_and_member_overrides() { + let dir = fixture(&[ + ( + "deno.json", + r#"{"workspace":["./packages/*","!packages/excluded"],"imports":{"web":"jsr:@fresh/core@^2"}}"#, + ), + (".deno-version", "2.5.0"), + ("package.json", r#"{"workspaces":["node-only/*"]}"#), + ("pnpm-workspace.yaml", "packages: ['pnpm-only/*']\n"), + ("packages/deno/deno.json", "{}"), + ("packages/deno/main.ts", ""), + ( + "packages/hybrid/deno.jsonc", + r#"{"imports":{"web":"npm:hono@^4"},}"#, + ), + ( + "packages/hybrid/package.json", + r#"{"name":"hybrid","engines":{"node":">=22","deno":">=2.4"}}"#, + ), + ("packages/npm/package.json", r#"{"name":"npm-only"}"#), + ("packages/excluded/package.json", "{}"), + ("node-only/member/package.json", "{}"), + ("pnpm-only/member/package.json", "{}"), + ("outside/package.json", "{}"), + ]); + let report = scan(dir.path(), None).unwrap(); + let root = project(&report, "."); + assert!(detects(root, "deno-workspace")); + assert!(detects(root, "node-workspace")); + for name in [ + "packages/deno", + "packages/hybrid", + "packages/npm", + "node-only/member", + "pnpm-only/member", + ] { + assert!( + root.workspace_members.iter().any(|path| path == name), + "{name}" + ); + } + assert!( + !root + .workspace_members + .iter() + .any(|path| path == "packages/excluded" || path == "outside") + ); + for name in ["packages/deno", "packages/npm", "node-only/member"] { + let member = project(&report, name); + assert_eq!(member.runtimes[0].name, "deno"); + assert!( + member.runtimes[0] + .versions + .iter() + .any(|version| version.value == "2.5.0" + && version.kind == "pin" + && version.source == ".deno-version") + ); + assert!(detects(member, "fresh")); + } + assert!(project(&report, "pnpm-only/member").runtimes.is_empty()); + let deno_workspace = root + .workspaces + .iter() + .find(|workspace| workspace.ecosystem == "deno") + .unwrap(); + assert!( + !deno_workspace + .members + .iter() + .any(|member| member.starts_with("node-only/") || member.starts_with("pnpm-only/")) + ); + let hybrid = project(&report, "packages/hybrid"); + assert!(detects(hybrid, "hono")); + assert!(!detects(hybrid, "fresh")); + assert!(hybrid.runtimes[0].versions.iter().any( + |version| version.value == ">=2.4" && version.source == "packages/hybrid/package.json" + )); + assert!(version( + hybrid, + "javascript", + ">=22", + "packages/hybrid/package.json" + )); + for name in ["outside", "packages/excluded"] { + assert!(project(&report, name).runtimes.is_empty()); + assert!(!detects(project(&report, name), "fresh")); + } +} + +#[test] +fn deno_local_import_maps_keep_their_own_evidence_and_member_aliases_can_block() { + let dir = fixture(&[ + ( + "deno.json", + r#"{"workspace":["member"],"importMap":"config/custom-map.jsonc"}"#, + ), + ( + "config/custom-map.jsonc", + r#"{"imports":{"web":"https://deno.land/x/fresh@1.7.3/mod.ts",},}"#, + ), + ( + "member/deno.json", + r#"{"imports":{"web":"./not-a-framework.ts"}}"#, + ), + ]); + let report = scan(dir.path(), None).unwrap(); + let root = project(&report, "."); + let fresh = root.detections.iter().find(|d| d.id == "fresh").unwrap(); + assert!( + fresh + .evidence + .iter() + .any(|e| e.source == "config/custom-map.jsonc" && e.value.as_deref() == Some("1.7.3")) + ); + assert!(!detects(project(&report, "member"), "fresh")); + assert_eq!(report.stats.files_read, 3); + assert!(report.diagnostics.is_empty()); +} + +#[test] +fn deno_json_precedes_jsonc_and_invalid_metadata_is_diagnostic() { + let dir = fixture(&[ + ("deno.json", "{}"), + ( + "deno.jsonc", + r#"{"workspace":["foo"],"imports":{"x":"npm:hono@4"}}"#, + ), + ("foo/package.json", "{}"), + ]); + let report = scan(dir.path(), None).unwrap(); + let root = project(&report, "."); + assert!(detects(root, "deno")); + assert!(!detects(root, "hono")); + assert!(!detects(root, "deno-workspace")); + for value in ["[]", "{broken", "{ /* unterminated"] { + let dir = fixture(&[("deno.jsonc", value)]); + let report = scan(dir.path(), None).unwrap(); + assert_eq!(report.diagnostics.len(), 1); + assert!(report.projects.is_empty()); + } +} + +#[test] +fn deno_catalogs_reuse_resolution_and_package_json_takes_precedence() { + let dir = fixture(&[ + ( + "deno.json", + r#"{"workspace":["packages/*"],"catalog":{"typescript":"^5.4"},"catalogs":{"old":{"typescript":"^5.3"}}}"#, + ), + ( + "package.json", + r#"{"catalog":{"typescript":"^5.9"},"catalogs":{"old":{"typescript":"^5.8"}}}"#, + ), + ( + "packages/npm/package.json", + r#"{"devDependencies":{"typescript":"catalog:"}}"#, + ), + ( + "packages/deno/deno.json", + r#"{"imports":{"typescript":"catalog:old"}}"#, + ), + ]); + let report = scan(dir.path(), None).unwrap(); + for (path, expected) in [("packages/npm", "^5.9"), ("packages/deno", "^5.8")] { + assert!(version( + project(&report, path), + "typescript", + expected, + "package.json" + )); + } + assert!(report.diagnostics.is_empty()); +} + +#[test] +fn import_maps_do_not_escape_root_fetch_urls_or_read_oversized_files() { + for target in [ + "../outside.json", + "https://example.invalid/imports.json", + "/tmp/imports.json", + "missing.json", + ] { + let dir = fixture(&[( + "deno.json", + &format!( + r#"{{"importMap":{}}}"#, + serde_json::to_string(target).unwrap() + ), + )]); + let report = scan(dir.path(), None).unwrap(); + assert_eq!(report.stats.files_read, 1); + assert_eq!(report.diagnostics.len(), 1); + } + let dir = fixture(&[ + ("deno.json", r#"{"importMap":"huge.json"}"#), + ("huge.json", &" ".repeat(1024 * 1024 + 1)), + ]); + let report = scan(dir.path(), None).unwrap(); + assert_eq!(report.stats.files_read, 2); + assert!(report.diagnostics[0].message.contains("1 MiB")); +} + +#[cfg(unix)] +#[test] +fn import_maps_do_not_follow_file_or_parent_symlinks() { + for target in ["link.json", "link-dir/map.json"] { + let outside = fixture(&[("map.json", r#"{"imports":{"x":"npm:hono@4"}}"#)]); + let dir = fixture(&[("deno.json", &format!(r#"{{"importMap":"{target}"}}"#))]); + std::os::unix::fs::symlink( + outside.path().join("map.json"), + dir.path().join("link.json"), + ) + .unwrap(); + std::os::unix::fs::symlink(outside.path(), dir.path().join("link-dir")).unwrap(); + let report = scan(dir.path(), None).unwrap(); + assert_eq!(report.stats.files_read, 1); + assert!( + report + .diagnostics + .iter() + .any(|diagnostic| diagnostic.message.contains("symlinks")) + ); + assert!(!detects(project(&report, "."), "hono")); + } +} + +#[test] +fn hybrid_imports_preserve_peer_development_and_jsr_constraints() { + let dir = fixture(&[ + ( + "deno.json", + r#"{"imports":{"web":"jsr:@hono/hono@^4.8.3"}}"#, + ), + ( + "package.json", + r#"{"peerDependencies":{"hono":">=4"},"devDependencies":{"hono":"^4.11.5"}}"#, + ), + ]); + let report = scan(dir.path(), None).unwrap(); + let hono = project(&report, ".") + .detections + .iter() + .find(|d| d.id == "hono") + .unwrap(); + for (value, source) in [ + (">=4", "package.json"), + ("^4.11.5", "package.json"), + ("^4.8.3", "deno.json"), + ] { + assert!( + hono.evidence + .iter() + .any(|e| e.source == source && e.value.as_deref() == Some(value)) + ); + } +} + +#[test] +fn text_output_includes_an_unversioned_runtime_for_npm_only_members() { + let dir = fixture(&[ + ("deno.json", r#"{"workspace":["member"]}"#), + ("member/package.json", "{}"), + ]); + let report = scan(dir.path(), None).unwrap(); + let text = super::render(&serde_json::to_value(&report).unwrap()).join("\n"); + assert!(text.contains("runtime deno (deno.json)")); + assert!(project(&report, "member").runtimes[0].versions.is_empty()); +} diff --git a/src/detect/jsonc.rs b/src/detect/jsonc.rs new file mode 100644 index 0000000..84fec8b --- /dev/null +++ b/src/detect/jsonc.rs @@ -0,0 +1,200 @@ +pub(super) fn parse(text: &str) -> Result { + if let Ok(value) = serde_json::from_str(text) { + return Ok(value); + } + + let mut bytes = text.as_bytes().to_vec(); + let mut i = 0; + while i < bytes.len() { + match bytes[i] { + b'"' => i = string_end(&bytes, i), + b'/' if bytes.get(i + 1) == Some(&b'/') => { + while i < bytes.len() && !matches!(bytes[i], b'\r' | b'\n') { + bytes[i] = b' '; + i += 1; + } + } + b'/' if bytes.get(i + 1) == Some(&b'*') => { + let start = i; + i += 2; + while i + 1 < bytes.len() && &bytes[i..i + 2] != b"*/" { + i += 1; + } + if i + 1 >= bytes.len() { + let line = text.as_bytes()[..start] + .iter() + .filter(|&&byte| byte == b'\n') + .count() + + 1; + let column = text.as_bytes()[..start] + .iter() + .rposition(|&byte| byte == b'\n') + .map_or(start + 1, |newline| start - newline); + return Err(format!( + "unterminated block comment at line {line} column {column}" + )); + } + i += 2; + for byte in &mut bytes[start..i] { + if !matches!(*byte, b'\r' | b'\n') { + *byte = b' '; + } + } + } + _ => i += 1, + } + } + + i = 0; + let mut previous = None; + while i < bytes.len() { + match bytes[i] { + b'"' => { + i = string_end(&bytes, i); + previous = Some(b'"'); + continue; + } + b',' => { + let next = bytes[i + 1..].iter().find(|&&byte| !is_whitespace(byte)); + if matches!(next, Some(b'}' | b']')) + && !matches!(previous, None | Some(b'[' | b'{' | b',' | b':')) + { + bytes[i] = b' '; + } + previous = Some(b','); + } + byte if !is_whitespace(byte) => previous = Some(byte), + _ => {} + } + i += 1; + } + + serde_json::from_slice(&bytes).map_err(|error| error.to_string()) +} + +fn string_end(bytes: &[u8], start: usize) -> usize { + let mut i = start + 1; + while i < bytes.len() { + match bytes[i] { + b'\\' => i += 2, + b'"' => return i + 1, + _ => i += 1, + } + } + bytes.len() +} + +fn is_whitespace(byte: u8) -> bool { + matches!(byte, b' ' | b'\t' | b'\r' | b'\n') +} + +#[cfg(test)] +mod tests { + use super::parse; + use serde_json::json; + + #[test] + fn parses_nested_comments_and_trailing_commas() { + let text = r#"// config + { + /* Unicode: ζ—₯本θͺž πŸ¦€ */ + "tasks": {"dev": "deno run main.ts", /* task */}, + "values": [1, {"nested": [true, null,],},], + } // end"#; + assert_eq!( + parse(text).unwrap(), + json!({ + "tasks": {"dev": "deno run main.ts"}, + "values": [1, {"nested": [true, null]}] + }) + ); + } + + #[test] + fn preserves_strings_and_escapes() { + let expected = json!({ + "url": "https://example.com/ζ—₯本θͺž/*path*/?q=//", + "markers": "/* not a comment */ // still a string ,] ,}", + "escaped": "a\"//b\\\"/*c*/", + "backslash": "ends with \\", + }); + let plain = serde_json::to_string(&expected).unwrap(); + assert_eq!(parse(&plain).unwrap(), expected); + assert_eq!(parse(&format!("/* πŸ¦€ */{plain}// end")).unwrap(), expected); + } + + #[test] + fn accepts_json_primitives_and_empty_containers() { + for text in [ + "null", + "true", + "false", + "42", + "-1.25e2", + r#""πŸ¦€""#, + "[]", + "{}", + ] { + let expected: serde_json::Value = serde_json::from_str(text).unwrap(); + assert_eq!(parse(text).unwrap(), expected); + assert_eq!( + parse(&format!("/* before */{text}// after")).unwrap(), + expected + ); + } + assert_eq!(parse("[1,// comment\r\n]").unwrap(), json!([1])); + } + + #[test] + fn rejects_malformed_comments_and_json() { + for text in [ + "/*", + "/**", + "/* no end πŸ¦€", + "{} /* no end", + "/* outer /* inner */ */", + "[,]", + "{,}", + "[/* comment */,]", + "[1,,]", + "[1,/* c */,]", + r#"{"a":1,,}"#, + r#"{"a":,}"#, + "[1,}", + "[true false,]", + "[01,]", + "[NaN,]", + "[1\u{a0},]", + "// only comment", + "/* only */", + r#"{"a":"unterminated //}"#, + r#"["bad\q",]"#, + r#"["\uD800",]"#, + "[\"raw\nnewline\",]", + "[\"\0\",]", + "{} garbage", + ] { + assert!(parse(text).is_err(), "accepted {text:?}"); + } + } + + #[test] + fn preserves_diagnostic_byte_columns_and_newlines() { + let text = "/* πŸ¦€\r\nζ—₯本θͺž */\r\n[1, /*Γ©*/ @,]"; + let normalized = " \r\n \r\n[1, @ ]"; + assert_eq!(text.len(), normalized.len()); + let expected = serde_json::from_str::(normalized) + .unwrap_err() + .to_string(); + assert_eq!(parse(text).unwrap_err(), expected); + } + + #[test] + fn input_is_utf8_and_comment_masking_keeps_it_valid() { + assert!(String::from_utf8(vec![0xff]).is_err()); + assert_eq!( + parse("/* Γ©ζ—₯ζœ¬πŸ¦€ */[\"Γ©ζ—₯ζœ¬πŸ¦€\",]").unwrap(), + json!(["Γ©ζ—₯ζœ¬πŸ¦€"]) + ); + } +} diff --git a/src/detect/mod.rs b/src/detect/mod.rs index 1dfbb86..5f45bde 100644 --- a/src/detect/mod.rs +++ b/src/detect/mod.rs @@ -10,9 +10,13 @@ use clap::Args; use serde::Serialize; mod catalog; +mod deno; +#[cfg(test)] +mod deno_tests; mod dotnet; mod gradle; mod java; +mod jsonc; #[cfg(test)] mod jvm_dotnet_tests; mod parsers; @@ -51,8 +55,26 @@ pub struct Project { pub path: String, pub names: Vec, pub languages: Vec, + #[serde(skip_serializing_if = "Vec::is_empty")] + pub runtimes: Vec, pub detections: Vec, pub workspace_members: Vec, + #[serde(skip_serializing_if = "Vec::is_empty")] + pub workspaces: Vec, +} + +#[derive(Debug, Serialize, PartialEq, Eq)] +pub struct WorkspaceReport { + pub ecosystem: String, + pub source: String, + pub members: Vec, +} + +#[derive(Debug, Serialize, PartialEq, Eq)] +pub struct Runtime { + pub name: String, + pub source: String, + pub versions: Vec, } #[derive(Debug, Serialize, PartialEq, Eq)] @@ -120,14 +142,18 @@ struct Node { identities: BTreeMap, packages: BTreeMap, versions: BTreeMap>, + runtimes: BTreeMap, + runtime_versions: BTreeMap>, names: BTreeSet, workspaces: Vec, } +#[derive(Clone)] struct Dependency { version: Option, source: String, resolved: Option, + declarations: Vec, } #[derive(Clone)] @@ -147,16 +173,31 @@ impl Workspace { impl Node { fn dependency(&mut self, ecosystem: &str, name: &str, version: Option<&str>, file: &str) { - self.dependencies.insert( + self.merge_dependency( format!("{ecosystem}:{name}"), Dependency { version: version.map(str::to_owned), source: relative_file(&self.path, file), resolved: None, + declarations: Vec::new(), }, ); } + fn merge_dependency(&mut self, key: String, mut dependency: Dependency) { + if let Some(previous) = self.dependencies.remove(&key) { + dependency.declarations.extend(previous.declarations); + if let Some(value) = previous.version { + dependency.declarations.push(Version { + value, + source: previous.source, + kind: "requirement".to_owned(), + }); + } + } + self.dependencies.insert(key, dependency); + } + fn identity(&mut self, ecosystem: &str, name: &str, version: Option<&str>, file: &str) { self.identities.insert( format!("{ecosystem}:{name}"), @@ -164,6 +205,7 @@ impl Node { version: version.map(str::to_owned), source: relative_file(&self.path, file), resolved: None, + declarations: Vec::new(), }, ); } @@ -404,7 +446,13 @@ pub fn scan(root: &Path, extra_rules: Option<&str>) -> Result { report.diagnostics = diagnostics; report.stats = stats; let original_nodes = nodes.len(); - parsers::parse(&mut nodes, &rules.catalog, &mut report.diagnostics); + parsers::parse( + &root, + &mut nodes, + &rules.catalog, + &mut report.stats, + &mut report.diagnostics, + ); if nodes.len() != original_nodes { index_parents(&mut nodes); } @@ -429,8 +477,10 @@ pub fn scan(root: &Path, extra_rules: Option<&str>) -> Result { path: display_path(&node.path), names: node.names.iter().cloned().collect(), languages: Vec::new(), + runtimes: Vec::new(), detections, workspace_members: Vec::new(), + workspaces: Vec::new(), }, )); } @@ -460,8 +510,10 @@ pub fn scan(root: &Path, extra_rules: Option<&str>) -> Result { path: display_path(&nodes[index].path), names: Vec::new(), languages: Vec::new(), + runtimes: Vec::new(), detections: Vec::new(), workspace_members: Vec::new(), + workspaces: Vec::new(), }, )); project_index @@ -508,6 +560,36 @@ pub fn scan(root: &Path, extra_rules: Option<&str>) -> Result { versions, }); } + let mut runtime_sources = nodes[*index].runtimes.clone(); + for (name, versions) in &nodes[*index].runtime_versions { + if let Some(version) = versions.first() { + runtime_sources + .entry(name.clone()) + .or_insert_with(|| version.source.clone()); + } + } + for (name, source) in runtime_sources { + let mut versions = Vec::new(); + let mut ancestor = Some(*index); + while let Some(i) = ancestor { + if let Some(found) = nodes[i].runtime_versions.get(&name) { + versions.extend( + found + .iter() + .filter(|v| i == *index || rules.catalog.inherited(&v.kind)) + .cloned(), + ); + } + ancestor = nodes[i].parent; + } + versions.sort(); + versions.dedup(); + project.runtimes.push(Runtime { + name, + source, + versions, + }); + } project .detections .sort_by(|a, b| b.priority.cmp(&a.priority).then(a.id.cmp(&b.id))); @@ -524,6 +606,7 @@ pub fn scan(root: &Path, extra_rules: Option<&str>) -> Result { for (index, project) in &mut projects { let node = &nodes[*index]; for workspace in &node.workspaces { + let mut members = Vec::new(); for &member in candidates .get(workspace.ecosystem.as_str()) .into_iter() @@ -536,12 +619,21 @@ pub fn scan(root: &Path, extra_rules: Option<&str>) -> Result { continue; }; if workspace.contains(&display_path(relative)) { - project - .workspace_members - .push(display_path(&nodes[member].path)); + members.push(display_path(&nodes[member].path)); } } + members.sort(); + members.dedup(); + project.workspace_members.extend(members.iter().cloned()); + project.workspaces.push(WorkspaceReport { + ecosystem: workspace.ecosystem.clone(), + source: workspace.source.clone(), + members, + }); } + project + .workspaces + .sort_by(|a, b| a.ecosystem.cmp(&b.ecosystem).then(a.source.cmp(&b.source))); project.workspace_members.sort(); project.workspace_members.dedup(); } @@ -587,7 +679,19 @@ pub fn render(value: &serde_json::Value) -> Vec { languages.join(", "), detections.join(", ") )); - for language in project["languages"].as_array().into_iter().flatten() { + for runtime in project["runtimes"].as_array().into_iter().flatten() { + lines.push(format!( + " runtime {} ({})", + runtime["name"].as_str().unwrap_or(""), + runtime["source"].as_str().unwrap_or("") + )); + } + for language in project["languages"] + .as_array() + .into_iter() + .flatten() + .chain(project["runtimes"].as_array().into_iter().flatten()) + { for version in language["versions"].as_array().into_iter().flatten() { lines.push(format!( " {} {} ({}, {})", diff --git a/src/detect/parsers.rs b/src/detect/parsers.rs index 6f5fa9e..3a8f4dd 100644 --- a/src/detect/parsers.rs +++ b/src/detect/parsers.rs @@ -5,13 +5,19 @@ use serde::Deserialize; use serde_json::Value; use super::catalog::{Catalog, Normalize, Reader, Resolver, workspace_patterns}; -use super::{Diagnostic, Node, Workspace, relative_file}; +use super::{Diagnostic, Node, Stats, Workspace, relative_file}; pub(super) type Documents = HashMap>; -pub fn parse(nodes: &mut Vec, catalog: &Catalog, diagnostics: &mut Vec) { +pub fn parse( + root: &Path, + nodes: &mut Vec, + catalog: &Catalog, + stats: &mut Stats, + diagnostics: &mut Vec, +) { let mut documents = Documents::new(); - let mut pnpm = HashMap::>::new(); + let mut pnpm = HashMap::>::new(); for node in nodes.iter() { if node .files @@ -25,6 +31,9 @@ pub fn parse(nodes: &mut Vec, catalog: &Catalog, diagnostics: &mut Vec(content) { Ok(manifest) => { @@ -33,13 +42,14 @@ pub fn parse(nodes: &mut Vec, catalog: &Catalog, diagnostics: &mut Vec, catalog: &Catalog, diagnostics: &mut Vec { Some(serde_json::from_str::(content).map_err(|e| e.to_string())) } + Reader::Jsonc => Some(super::jsonc::parse(content)), Reader::Toml => Some(toml::from_str::(content).map_err(|e| e.to_string())), _ => None, }; @@ -81,7 +92,7 @@ pub fn parse(nodes: &mut Vec, catalog: &Catalog, diagnostics: &mut Vec, catalog: &Catalog, diagnostics: &mut Vec>>, } -struct Pnpm { - workspace: Workspace, - catalog: BTreeMap, - catalogs: BTreeMap>, +pub(super) struct Catalogs { + pub workspaces: Vec, + pub source: String, + pub catalog: BTreeMap, + pub catalogs: BTreeMap>, } fn resolve_catalogs( node: &mut Node, - manifests: &HashMap>, + manifests: &HashMap>, diagnostics: &mut Vec, ) { let workspace = node @@ -166,7 +179,10 @@ fn resolve_catalogs( .filter(|(path, manifest)| { *path == node.path || node.path.strip_prefix(path).ok().is_some_and(|relative| { - manifest.workspace.contains(&super::display_path(relative)) + manifest + .workspaces + .iter() + .any(|workspace| workspace.contains(&super::display_path(relative))) }) }) .map(|(_, manifest)| manifest); @@ -192,7 +208,7 @@ fn resolve_catalogs( .filter(|v| !v.starts_with("catalog:")) .map(|value| super::Version { value: value.clone(), - source: manifest.workspace.source.clone(), + source: manifest.source.clone(), kind: "requirement".to_owned(), }) }); @@ -301,6 +317,7 @@ fn cargo(node: &mut Node, doc: &Value, documents: &Documents) { version, source, resolved: None, + declarations: Vec::new(), }, ); } diff --git a/src/detect/rules.rs b/src/detect/rules.rs index fe3a89e..5091801 100644 --- a/src/detect/rules.rs +++ b/src/detect/rules.rs @@ -395,6 +395,13 @@ impl Matched<'_> { Self::Package(source) => (source.to_owned(), None), Self::File(name) => (super::relative_file(&node.path, name), None), Self::Dependency(dep) => { + for declaration in &dep.declarations { + evidence.push(Evidence { + source: declaration.source.clone(), + pattern: pattern.to_owned(), + value: Some(declaration.value.clone()), + }); + } if let Some(resolved) = &dep.resolved { evidence.push(Evidence { source: resolved.source.clone(), diff --git a/src/detect/rules.toml b/src/detect/rules.toml index 97d4fd2..e95d768 100644 --- a/src/detect/rules.toml +++ b/src/detect/rules.toml @@ -31,6 +31,10 @@ files = ["go.mod"] ecosystem = "node" files = ["package.json"] +[[catalog.workspaces]] +ecosystem = "deno" +files = ["deno.json", "deno.jsonc", "package.json"] + [[catalog.workspaces]] ecosystem = "python" files = ["pyproject.toml"] @@ -184,7 +188,7 @@ paths = ["/name"] versions = ["/version"] [[catalog.manifests.dependencies]] ecosystem = "npm" -paths = ["/devDependencies", "/dependencies"] +paths = ["/peerDependencies", "/devDependencies", "/dependencies"] [[catalog.manifests.versions]] language = "javascript" kind = "requirement" @@ -194,6 +198,10 @@ language = "javascript" kind = "pin" paths = ["/volta/node"] [[catalog.manifests.versions]] +runtime = "deno" +kind = "requirement" +paths = ["/engines/deno"] +[[catalog.manifests.versions]] language = "typescript" kind = "requirement" dependency = "npm:typescript" @@ -203,6 +211,36 @@ ecosystem = "node" present = "/workspaces" members = ["/workspaces/packages", "/workspaces"] +[[catalog.manifests]] +id = "deno" +files = ["deno.json", "deno.jsonc"] +prefer_first = true +reader = "jsonc" +package = "deno" +runtime = "deno" +names = ["/name"] +[catalog.manifests.identity] +ecosystem = "jsr" +paths = ["/name"] +versions = ["/version"] +[[catalog.manifests.versions]] +language = "typescript" +kind = "requirement" +dependency = "npm:typescript" +reference_prefix = "catalog:" +[[catalog.manifests.workspaces]] +ecosystem = "deno" +present = "/workspace" +members = ["/workspace/members", "/workspace"] + +[[catalog.manifests]] +id = "deno-pin" +files = [".deno-version"] +reader = "pin" +[[catalog.manifests.versions]] +runtime = "deno" +kind = "pin" + [[catalog.manifests]] id = "cargo" files = ["Cargo.toml"] @@ -401,6 +439,11 @@ language = "php" kind = "pin" key = "php" +[[catalog.manifests.versions]] +runtime = "deno" +kind = "pin" +key = "deno" + [[catalog.manifests]] id = "sdkman" files = [".sdkmanrc"] @@ -544,6 +587,34 @@ kind = "project" priority = 10 any = ["file:package.json"] +[[rules]] +id = "deno" +language = "javascript" +kind = "project" +priority = 10 +any = ["package:deno"] + +[[rules]] +id = "deno-workspace" +language = "javascript" +kind = "workspace" +priority = 20 +any = ["workspace:deno"] + +[[rules]] +id = "fresh" +language = "typescript" +kind = "framework" +priority = 100 +any = ["dep:jsr:@fresh/core", "identity:jsr:@fresh/core", "dep:deno:fresh"] + +[[rules]] +id = "oak" +language = "typescript" +kind = "framework" +priority = 100 +any = ["dep:jsr:@oak/oak", "identity:jsr:@oak/oak", "dep:deno:oak"] + [[rules]] id = "javascript-source" language = "javascript" @@ -921,3 +992,10 @@ language = "java" kind = "framework" priority = 110 any = ["dep:maven:io.micronaut:micronaut-runtime", "dep:gradle-plugin:io.micronaut.application", "identity:maven:io.micronaut:micronaut-runtime"] + +[[rules]] +id = "hono" +language = "typescript" +kind = "framework" +priority = 100 +any = ["dep:npm:hono", "identity:npm:hono", "dep:jsr:@hono/hono", "identity:jsr:@hono/hono", "dep:deno:hono"] diff --git a/src/detect/values.rs b/src/detect/values.rs index bba039e..337f616 100644 --- a/src/detect/values.rs +++ b/src/detect/values.rs @@ -111,6 +111,7 @@ fn declaration( version: raw.filter(|v| v.source == source).map(|v| v.value.clone()), source: source.to_owned(), resolved: None, + declarations: Vec::new(), }; if let Some(raw) = raw { let resolved = resolve(raw, properties, syntax).unwrap_or_else(|| raw.clone()); diff --git a/tests/audit_detect.py b/tests/audit_detect.py index 5c9b123..c321cc3 100644 --- a/tests/audit_detect.py +++ b/tests/audit_detect.py @@ -100,6 +100,8 @@ def check(projects, expected): (d for d in project["detections"] if d["id"] == expected["id"]), None ) roles = expected.get("roles", []) + if expected.get("absent"): + return found is None, ids return found is not None and set(roles).issubset( found.get("roles", []) ), project["detections"] if roles else ids @@ -107,10 +109,54 @@ def check(projects, expected): return expected["name"] in project["names"], project["names"] if kind == "workspace": actual = project["workspaceMembers"] + if "ecosystem" in expected: + actual = sorted( + { + member + for workspace in project.get("workspaces", []) + if workspace["ecosystem"] == expected["ecosystem"] + and ( + "source" not in expected + or workspace["source"] == expected["source"] + ) + for member in workspace["members"] + } + ) return expected["id"] in ids and expected["member"] in actual, { "ids": ids, "members": actual, } + if kind in {"runtime", "runtime-version"}: + runtime = next( + ( + runtime + for runtime in project.get("runtimes", []) + if runtime["name"] == expected["runtime"] + ), + None, + ) + if kind == "runtime": + if expected.get("absent"): + return runtime is None, project.get("runtimes", []) + if ( + "versionCount" in expected + and runtime is not None + and len(runtime["versions"]) != expected["versionCount"] + ): + return False, runtime + return runtime is not None and ( + "source" not in expected or runtime["source"] == expected["source"] + ), project.get("runtimes", []) + actual = runtime["versions"] if runtime else [] + return any( + v["value"] == expected["value"] + and v["source"] == expected["source"] + and ( + "versionKind" not in expected + or v.get("kind") == expected["versionKind"] + ) + for v in actual + ), actual if kind == "version": actual = [ v diff --git a/tests/detect-corpus.json b/tests/detect-corpus.json index 088f90b..ff59a6a 100644 --- a/tests/detect-corpus.json +++ b/tests/detect-corpus.json @@ -1976,6 +1976,551 @@ "member": "jeecg-boot/jeecg-module-system" } ] + }, + { + "repository": "supabase/supabase", + "language": "Deno", + "stars": 114734, + "webRank": 1, + "revision": "797a43c7d023777af25ba4621bdf87e66362e7bb", + "checks": [ + { + "category": "project", + "kind": "detection", + "path": "docker/volumes/functions", + "id": "deno" + }, + { + "category": "language", + "kind": "language", + "path": "docker/volumes/functions", + "language": "typescript" + }, + { + "category": "project", + "kind": "detection", + "path": "examples/edge-functions/supabase/functions/drizzle", + "id": "deno" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "node" + }, + { + "category": "name", + "kind": "name", + "path": ".", + "name": "supabase" + }, + { + "category": "runtime", + "kind": "runtime", + "path": "docker/volumes/functions", + "runtime": "deno", + "versionCount": 0 + }, + { + "category": "runtime", + "kind": "runtime", + "path": "examples/edge-functions/supabase/functions/drizzle", + "runtime": "deno", + "versionCount": 0 + }, + { + "category": "boundary", + "kind": "runtime", + "path": ".", + "runtime": "deno", + "absent": true + }, + { + "category": "boundary", + "kind": "detection", + "path": ".", + "id": "deno-workspace", + "absent": true + } + ], + "topicRank": 1, + "selectionQuery": "topic:deno archived:false fork:false" + }, + { + "repository": "honojs/hono", + "language": "Deno", + "stars": 32889, + "webRank": 2, + "revision": "61f7c59e50244ab4d2dd83c151928cf7ea448510", + "checks": [ + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "node" + }, + { + "category": "identity", + "kind": "detection", + "path": ".", + "id": "hono" + }, + { + "category": "project", + "kind": "detection", + "path": "adapters/deno", + "id": "deno" + }, + { + "category": "language", + "kind": "language", + "path": "adapters/deno", + "language": "typescript" + }, + { + "category": "name", + "kind": "name", + "path": "adapters/deno", + "name": "@hono/deno" + }, + { + "category": "framework", + "kind": "detection", + "path": "adapters/deno", + "id": "hono", + "roles": [ + "uses" + ] + }, + { + "category": "project", + "kind": "detection", + "path": "runtime-tests/deno", + "id": "deno" + }, + { + "category": "runtime", + "kind": "runtime", + "path": ".", + "runtime": "deno", + "versionCount": 1 + }, + { + "category": "runtime", + "kind": "runtime", + "path": "adapters/deno", + "runtime": "deno", + "versionCount": 1 + }, + { + "category": "runtime", + "kind": "runtime", + "path": "runtime-tests/deno", + "runtime": "deno", + "versionCount": 1 + }, + { + "category": "runtime-version", + "kind": "runtime-version", + "path": ".", + "runtime": "deno", + "value": "2.4.5", + "source": ".tool-versions", + "versionKind": "pin" + } + ], + "topicRank": 4, + "selectionQuery": "topic:deno archived:false fork:false" + }, + { + "repository": "freshframework/fresh", + "language": "Deno", + "stars": 13789, + "webRank": 3, + "revision": "86d6cdeb331a719cf8b1c85bf5e43c8ffa889b3b", + "checks": [ + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "deno" + }, + { + "category": "workspace", + "kind": "detection", + "path": ".", + "id": "deno-workspace" + }, + { + "category": "workspace", + "kind": "workspace", + "path": ".", + "id": "deno-workspace", + "member": "packages/fresh", + "ecosystem": "deno" + }, + { + "category": "workspace", + "kind": "workspace", + "path": ".", + "id": "deno-workspace", + "member": "packages/plugin-vite", + "ecosystem": "deno" + }, + { + "category": "workspace", + "kind": "workspace", + "path": ".", + "id": "deno-workspace", + "member": "www", + "ecosystem": "deno" + }, + { + "category": "project", + "kind": "detection", + "path": "packages/fresh", + "id": "deno" + }, + { + "category": "language", + "kind": "language", + "path": "packages/fresh", + "language": "typescript" + }, + { + "category": "name", + "kind": "name", + "path": "packages/fresh", + "name": "@fresh/core" + }, + { + "category": "project", + "kind": "detection", + "path": "www", + "id": "deno" + }, + { + "category": "runtime", + "kind": "runtime", + "path": ".", + "runtime": "deno", + "versionCount": 0 + }, + { + "category": "runtime", + "kind": "runtime", + "path": "packages/fresh", + "runtime": "deno", + "versionCount": 0 + }, + { + "category": "runtime", + "kind": "runtime", + "path": "www", + "runtime": "deno", + "versionCount": 0 + }, + { + "category": "identity", + "kind": "detection", + "path": "packages/fresh", + "id": "fresh", + "roles": [ + "provides" + ] + }, + { + "category": "project", + "kind": "detection", + "path": "packages/examples", + "id": "deno" + }, + { + "category": "framework", + "kind": "detection", + "path": "packages/examples", + "id": "fresh", + "roles": [ + "uses" + ] + }, + { + "category": "runtime", + "kind": "runtime", + "path": "packages/examples", + "runtime": "deno", + "versionCount": 0 + } + ], + "topicRank": 8, + "selectionQuery": "topic:deno archived:false fork:false" + }, + { + "repository": "InsForge/InsForge", + "language": "Deno", + "stars": 13316, + "webRank": 4, + "revision": "375f4bd7449a46481bb7e57bf872def52b9f9f68", + "checks": [ + { + "category": "project", + "kind": "detection", + "path": "functions", + "id": "deno" + }, + { + "category": "language", + "kind": "language", + "path": "functions", + "language": "typescript" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "node" + }, + { + "category": "name", + "kind": "name", + "path": ".", + "name": "insforge" + }, + { + "category": "framework", + "kind": "detection", + "path": "backend", + "id": "express", + "roles": [ + "uses" + ] + }, + { + "category": "framework", + "kind": "detection", + "path": "frontend", + "id": "react", + "roles": [ + "uses" + ] + }, + { + "category": "runtime", + "kind": "runtime", + "path": "functions", + "runtime": "deno", + "versionCount": 0 + } + ], + "topicRank": 9, + "selectionQuery": "topic:deno archived:false fork:false" + }, + { + "repository": "ryo-ma/github-profile-trophy", + "language": "Deno", + "stars": 6684, + "webRank": 5, + "revision": "43736b50a82fca73c8df1c81abb40aa03519975c", + "checks": [ + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "deno" + }, + { + "category": "language", + "kind": "language", + "path": ".", + "language": "typescript" + }, + { + "category": "runtime", + "kind": "runtime", + "path": ".", + "runtime": "deno", + "versionCount": 0 + } + ], + "topicRank": 15, + "selectionQuery": "topic:deno archived:false fork:false" + }, + { + "repository": "honojs/middleware", + "language": "Deno", + "stars": 1002, + "webRank": null, + "revision": "d813df31d07dfe1959558a1a25935ec3eddb9d02", + "checks": [ + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "deno" + }, + { + "category": "workspace", + "kind": "detection", + "path": ".", + "id": "deno-workspace" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "node" + }, + { + "category": "workspace", + "kind": "workspace", + "path": ".", + "id": "deno-workspace", + "member": "packages/hello", + "ecosystem": "deno" + }, + { + "category": "workspace", + "kind": "workspace", + "path": ".", + "id": "deno-workspace", + "member": "packages/eslint-config", + "ecosystem": "deno" + }, + { + "category": "workspace", + "kind": "workspace", + "path": ".", + "id": "deno-workspace", + "member": "packages/ssg-plugins-essential", + "ecosystem": "deno" + }, + { + "category": "name", + "kind": "name", + "path": "packages/eslint-config", + "name": "@hono/eslint-config" + }, + { + "category": "project", + "kind": "detection", + "path": "packages/eslint-config", + "id": "node" + }, + { + "category": "project", + "kind": "detection", + "path": "packages/hello", + "id": "deno" + }, + { + "category": "project", + "kind": "detection", + "path": "packages/hello", + "id": "node" + }, + { + "category": "framework", + "kind": "detection", + "path": "packages/hello", + "id": "hono", + "roles": [ + "uses" + ] + }, + { + "category": "framework", + "kind": "detection", + "path": "packages/ssg-plugins-essential", + "id": "hono", + "roles": [ + "uses" + ] + }, + { + "category": "project", + "kind": "detection", + "path": "packages/graphql-server", + "id": "node" + }, + { + "category": "runtime", + "kind": "runtime", + "path": ".", + "runtime": "deno", + "versionCount": 1 + }, + { + "category": "runtime", + "kind": "runtime", + "path": "packages/hello", + "runtime": "deno", + "versionCount": 1 + }, + { + "category": "runtime", + "kind": "runtime", + "path": "packages/eslint-config", + "runtime": "deno", + "source": "deno.jsonc", + "versionCount": 1 + }, + { + "category": "runtime", + "kind": "runtime", + "path": "packages/ssg-plugins-essential", + "runtime": "deno", + "source": "deno.jsonc", + "versionCount": 1 + }, + { + "category": "framework", + "kind": "detection", + "path": "packages/ssg-plugins-essential", + "id": "hono", + "roles": [ + "uses" + ] + }, + { + "category": "runtime-version", + "kind": "runtime-version", + "path": ".", + "runtime": "deno", + "value": "2.3.7", + "source": ".tool-versions", + "versionKind": "pin" + }, + { + "category": "runtime-version", + "kind": "runtime-version", + "path": "packages/hello", + "runtime": "deno", + "value": "2.3.7", + "source": ".tool-versions", + "versionKind": "pin" + }, + { + "category": "runtime-version", + "kind": "runtime-version", + "path": "packages/eslint-config", + "runtime": "deno", + "value": "2.3.7", + "source": ".tool-versions", + "versionKind": "pin" + }, + { + "category": "runtime-version", + "kind": "runtime-version", + "path": "packages/ssg-plugins-essential", + "runtime": "deno", + "value": "2.3.7", + "source": ".tool-versions", + "versionKind": "pin" + } + ], + "topicRank": 60, + "selectionQuery": "topic:deno archived:false fork:false", + "supplemental": true } ], "extensions": [ @@ -1987,6 +2532,19 @@ ], "repositoriesPerLanguage": 5, "eligibility": "Same web-project criteria as existing corpus, including production web solution templates (as with create-react-app), excluding native/desktop-first apps, general runtimes, lists, and tutorial collections." + }, + { + "selectedAt": "2026-11-10T08:37:50.845853+00:00", + "ecosystems": [ + "Deno" + ], + "repositoriesPerEcosystem": 5, + "query": "GitHub query topic:deno archived:false fork:false; sort=stars; order=desc. languageRank stores raw topic-query rank (historical field name), webRank stores eligible web rank. API first 100 results retained in research/github-search.json.", + "eligibility": "Same web-first scope as prior45: applications, HTTP frameworks/libraries, browser UI and web site/build tools; require actual Deno config. Exclude runtimes, lists/tutorials, generic SQL/security/translation/CLI libraries, and optional-Deno tools without Deno config.", + "supplemental": [ + "honojs/middleware" + ], + "note": "Deno is a runtime ecosystem group, not a new language. Primary-language checks stay javascript/typescript." } ] } diff --git a/tests/detect-corpus.md b/tests/detect-corpus.md index 23c4c2c..c826f56 100644 --- a/tests/detect-corpus.md +++ b/tests/detect-corpus.md @@ -1,6 +1,6 @@ # GitHub project detection audit -Selected 2026-11-09; C#/Java added later that day. Baseline: detector commit `244ab44ccf1a` (PR #20). Current: the follow-up fixes in this revision. +Selected 2026-11-09; C#/Java added later that day; Deno added 2026-11-10. Baseline: detector commit `244ab44ccf1a` (PR #20). Current: the follow-up fixes in this revision. ## Method @@ -153,6 +153,29 @@ Timings are warm-filesystem, end-to-end CLI medians of runs 2–5. Empty or part Koa needs an additional npm framework rule. C#/.NET and Java/Spring are now supported and verified by the follow-up run. The Maybe warning is redundant with the already detected Ruby pin. None of these observations was silently converted into a passing assertion or hidden by skipping a branch. The initial validation was read-only; the follow-up uses the new language support. +## Deno extension (2026-11-10) + +The default corpus now includes **51 pinned repositories**: all original 45 plus five Deno web projects and one explicit mixed-workspace supplement. The runtime group is selected with GitHub query `topic:deno archived:false fork:false`, sorted descending by stars; it is not a tenth language or a primary-language ranking. `topicRank`, star counts, full SHAs, and independently reviewed assertions are frozen in `detect-corpus.json`. + +| Eligible rank | Topic rank | Repository | Stars | +|---|---:|---|---:| +| 1 | 1 | supabase/supabase | 114,734 | +| 2 | 4 | honojs/hono | 32,889 | +| 3 | 8 | freshframework/fresh | 13,789 | +| 4 | 9 | InsForge/InsForge | 13,316 | +| 5 | 15 | ryo-ma/github-profile-trophy | 6,684 | +| Supplemental | 60 | honojs/middleware | 1,002 | + +Higher-ranked exclusions follow the existing web-first criteria: denoland/deno is a runtime; rust-course is educational; pretty-ts-errors is an editor extension; Kysely is a general SQL builder; SheetJS and several other optional-Deno libraries have no Deno config in the inspected trees. This is a qualified sample, not β€œthe five most-starred repositories that happen to mention Deno.” + +Supabase and InsForge cover nested Deno functions inside npm/pnpm applications. Hono preserves framework-provider identity, its Deno adapter, and an explicit `.tool-versions` **2.4.5 pin**. Fresh covers a **nine-member glob workspace**, inherited imports, member overrides, and JSR package identity without confusing package version `2.3.3` with a runtime version. The supplemental middleware repository covers JSONC, npm-only members, hybrid packages, manager-specific workspace membership, distinct npm peer/development and JSR constraints, and its `.tool-versions` **2.3.7 pin**. github-profile-trophy is a Deno HTTP application with no static runtime version declaration; that absence remains explicit. + +All **295/295 assertions across 51 repositories** pass on the extended engine, with five deterministic runs each. This includes all original 227 assertions plus 68 new checks, including runtime/version provenance, manager-qualified membership, and negative workspace-boundary checks. All ten pinned Brainpod starter branches are rerun as well; their previously documented Koa label/Ruby warning limitations remain. Synthetic fixtures cover simultaneous config preference, comments/trailing commas, excluded/non-member packages, local import maps, missing/oversized/escaping/symlinked references, and runtime/package version separation. The scanner still executes no inspected application code or dependency installs. + +`runtimes` is additive JSON metadata (Deno is not a language). `workspaces` exposes `{ecosystem, source, members}` separately per manager; `workspaceMembers` is still their sorted union. A pnpm-only workspace member does not become a Deno member merely because both managers coexist at the root. Scoped import-map overrides and absolute/remote import-map file locations remain explicit unsupported diagnostics. Registry/URL module targets are metadata only and are never fetched. + +Official behavior references: [configuration](https://docs.deno.com/runtime/fundamentals/configuration/), [workspaces](https://docs.deno.com/runtime/fundamentals/workspaces/), and [config discovery implementation](https://github.com/denoland/deno_config/blob/main/src/deno_json/mod.rs) (`deno.json` precedes `deno.jsonc`). These document current behavior; pinned repository SHAs, not moving documentation or CI/Docker versions, define the assertion evidence. + ## Reproduce ```sh diff --git a/tests/test_audit_detect.py b/tests/test_audit_detect.py index 8012b85..7471dec 100644 --- a/tests/test_audit_detect.py +++ b/tests/test_audit_detect.py @@ -83,6 +83,107 @@ def test_role_and_version_kind_expectations_are_enforced(self): expected["versionKind"] = "sdk" self.assertTrue(audit_detect.check({".": self.project}, expected)[0]) + def test_workspace_checks_can_distinguish_managers(self): + self.project["workspaceMembers"] = ["deno-member", "pnpm-member"] + self.project["detections"].append({"id": "deno-workspace"}) + self.project["workspaces"] = [ + {"ecosystem": "deno", "source": "deno.jsonc", "members": ["deno-member"]}, + { + "ecosystem": "node", + "source": "pnpm-workspace.yaml", + "members": ["pnpm-member"], + }, + ] + check = { + "kind": "workspace", + "path": ".", + "id": "deno-workspace", + "ecosystem": "deno", + "source": "deno.jsonc", + "member": "deno-member", + } + self.assertTrue(audit_detect.check({".": self.project}, check)[0]) + for wrong in [ + {"member": "pnpm-member"}, + {"source": "package.json"}, + {"ecosystem": "cargo"}, + ]: + self.assertFalse( + audit_detect.check({".": self.project}, {**check, **wrong})[0] + ) + + def test_runtime_declarations_are_not_language_or_package_versions(self): + self.project["runtimes"] = [ + { + "name": "deno", + "source": "deno.jsonc", + "versions": [ + {"value": "2.4.5", "source": ".tool-versions", "kind": "pin"} + ], + } + ] + presence = { + "kind": "runtime", + "path": ".", + "runtime": "deno", + "source": "deno.jsonc", + } + self.assertTrue(audit_detect.check({".": self.project}, presence)[0]) + self.assertFalse( + audit_detect.check( + {".": self.project}, {**presence, "source": "package.json"} + )[0] + ) + version = { + "kind": "runtime-version", + "path": ".", + "runtime": "deno", + "value": "2.4.5", + "source": ".tool-versions", + "versionKind": "pin", + } + self.assertTrue(audit_detect.check({".": self.project}, version)[0]) + for wrong in [ + {"runtime": "node"}, + {"value": "9.9.9"}, + {"source": "deno.jsonc"}, + {"versionKind": "requirement"}, + ]: + self.assertFalse( + audit_detect.check({".": self.project}, {**version, **wrong})[0] + ) + self.assertFalse( + audit_detect.check({".": self.project}, {**presence, "versionCount": 0})[0] + ) + self.assertTrue( + audit_detect.check({".": self.project}, {**presence, "versionCount": 1})[0] + ) + self.assertFalse( + audit_detect.check({".": self.project}, {**presence, "absent": True})[0] + ) + del self.project["runtimes"] + self.assertFalse(audit_detect.check({".": self.project}, presence)[0]) + self.assertTrue( + audit_detect.check({".": self.project}, {**presence, "absent": True})[0] + ) + self.assertTrue( + audit_detect.check( + {".": self.project}, + { + "path": ".", + "kind": "detection", + "id": "deno-workspace", + "absent": True, + }, + )[0] + ) + self.assertFalse( + audit_detect.check( + {".": self.project}, + {"path": ".", "kind": "detection", "id": "node", "absent": True}, + )[0] + ) + def test_listing_needs_no_binary_cache_or_network(self): with ( tempfile.TemporaryDirectory() as directory,