diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9e8f464..5542884 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -58,7 +58,7 @@ jobs: gc-max-store-size: 2G - name: Build - run: nix build ".#packages.${{ matrix.system }}.${{ matrix.package }}" + run: nix build --print-build-logs ".#packages.${{ matrix.system }}.${{ matrix.package }}" - name: Package Unix binary if: matrix.package == 'default' diff --git a/Cargo.lock b/Cargo.lock index d5a438a..555f15e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -23,6 +23,17 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "annotate-snippets" +version = "0.12.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a5b0a5ab8561c603f16c3b78c24a386b1f70e7fc1b1b1e8c15738828e4a176" +dependencies = [ + "anstyle", + "memchr", + "unicode-width 0.2.3", +] + [[package]] name = "anstream" version = "1.0.0" @@ -79,6 +90,12 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "arraydeque" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d902e3d592a523def97af8f317b08ce16b7ab854c1985a0c671e6f15cebc236" + [[package]] name = "async-stream" version = "0.3.6" @@ -572,6 +589,7 @@ dependencies = [ "reqwest", "rustls", "serde", + "serde-saphyr", "serde_json", "sha2 0.10.9", "tar", @@ -584,6 +602,7 @@ dependencies = [ "tonic-build", "uuid", "webbrowser", + "xmlparser", ] [[package]] @@ -881,6 +900,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "encoding_rs_io" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fba3fe847045ecff794b9c138293a80db914678c453ad63fbf0c6a9eb6e00b22" +dependencies = [ + "encoding_rs", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -1060,6 +1088,16 @@ dependencies = [ "r-efi 6.0.0", ] +[[package]] +name = "granit-parser" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f05586d4d4df3d47123207826c40b252728b438a1a07bad3c01ee6bea401df" +dependencies = [ + "arraydeque", + "smallvec", +] + [[package]] name = "h2" version = "0.4.15" @@ -1626,7 +1664,7 @@ checksum = "5f98efec8807c63c752b5bd61f862c165c115b0a35685bdcfd9238c7aeb592b7" dependencies = [ "cfg-if", "miette-derive", - "unicode-width", + "unicode-width 0.1.14", ] [[package]] @@ -2269,6 +2307,20 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-saphyr" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d43fb82d550b07e6f929a58d7d0935182602f48ddead875f5ee313c41c8f07fe" +dependencies = [ + "annotate-snippets", + "encoding_rs_io", + "granit-parser", + "num-traits", + "serde_core", + "smallvec", +] + [[package]] name = "serde_core" version = "1.0.228" @@ -2897,6 +2949,12 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" +[[package]] +name = "unicode-width" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca718471876992059f251e33aafc334346ada08042b7661dd1fe313a9bced557" + [[package]] name = "untrusted" version = "0.9.0" diff --git a/Cargo.toml b/Cargo.toml index 8cce05e..6649d4a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,7 @@ prost-types = "0.13.5" reqwest = { version = "0.13.4", default-features = false, features = ["charset", "http2", "json", "query", "rustls-no-provider", "stream", "system-proxy"] } rustls = { version = "0.23.40", default-features = false, features = ["ring", "std", "tls12"] } serde = { version = "1.0.228", features = ["derive"] } +serde-saphyr = { version = "2.1.1", default-features = false, features = ["deserialize"] } serde_json = "1.0.150" sha2 = "0.10.9" tar = "0.4.44" @@ -34,6 +35,7 @@ toml = "0.9.5" tonic = { version = "0.12.3", features = ["tls", "tls-webpki-roots"] } uuid = { version = "1.16.0", features = ["v4"] } webbrowser = "=1.0.6" +xmlparser = "0.13.6" [build-dependencies] protox = "0.7.1" diff --git a/README.md b/README.md index 68d09d1..f0b5175 100644 --- a/README.md +++ b/README.md @@ -168,6 +168,7 @@ Errors go to stderr and return a non-zero exit code. With `--json`, errors also ## Commands ```text +brainpod detect [] [--rules ] brainpod describe [...] brainpod describe resource [] brainpod login @@ -263,6 +264,110 @@ Pod-scoped commands use `--pod`, `BRAINPOD_POD`, or the configured default pod. `revision wait` polls revision details until every resource reports `healthy: true`. `deploy --wait` deploys first, then waits on the returned revision in the same way. Both use a 90-second timeout by default; pass `--timeout ` to override it. Interactive waits report each unhealthy-to-healthy transition on stderr. Progress is suppressed when stderr is redirected or `--json` is used. Failed or canceled revisions stop the wait immediately, and timeouts report the resources that are still unhealthy. +## Project detection + +Inspect a directory without credentials, network access, subprocesses, or executing project code: + +```sh +brainpod detect . +brainpod detect ./services --json +brainpod detect . --rules ./detection.toml --json +``` + +The result contains sorted project paths, manifest names, languages and version declarations, priority-ranked detections with evidence and `uses`/`provides` framework roles, workspace members, diagnostics, and scan statistics. Optional `runtimes` records Deno declarations separately from JavaScript/TypeScript versions; `workspaces` records each manager’s ecosystem, source, and members, while `workspaceMembers` remains their union. Framework versions are attached to dependency evidence. JavaScript versions describe Node.js; TypeScript versions describe the compiler. C# versions distinguish `language` (LangVersion), `target` (TFM), `runtime`, and `sdk`; Java distinguishes compiler `source`/`target`/`release`, toolchains, build-JDK metadata, and pins. Requirements and pins retain their source and kind: these are declarations, not a claim about installed versions. Missing versions remain unknown. + +Built-in coverage includes Laravel, Symfony, CakePHP, CodeIgniter, Composer; Go modules/workspaces and Gin/Echo/Fiber; Cargo workspaces, axum, Rocket, Actix; Bundler, Rails, Sinatra, Hanami; Python requirements, PEP 621, Poetry, Pipenv, uv workspaces, Django, FastAPI, Flask, Starlette, Celery; npm/pnpm and Deno workspaces and catalogs, Fresh, Oak, Hono, Next.js, Nuxt, Astro, SvelteKit, Remix, Angular, Express, React, Vue, Vite; C#/.NET projects and solutions, ASP.NET Core/classic, Aspire, FastEndpoints; Java/JVM Maven and Gradle projects, Spring/Spring Boot, Quarkus, Micronaut; and standalone source files in all nine languages. Rust workspace dependency aliases and inherited `rust-version` are understood. Ancestor runtime pin/toolchain files are included with provenance. Poetry names, literal gemspec metadata, and Rails component dependencies are recognized. pnpm catalog references retain the original declaration and add the locally resolved constraint with its YAML source; unresolved references produce diagnostics. + +Deno supports `deno.json`/`deno.jsonc` (comments and trailing commas), Deno-only, hybrid, and npm-only workspace members, globs/exclusions, and shared imports with member alias overrides. Registry targets (`npm:`, `jsr:`) and supported `deno.land/x`/`esm.sh` URLs supply dependency evidence without downloading modules. Local relative import maps are bounded and cannot escape the scan root or follow symlinks. `.deno-version`, `.tool-versions` and npm `engines.deno` retain explicit runtime version provenance; a JSR package’s `version` is never treated as a Deno runtime version. Peer and development dependency constraints remain separate evidence. pnpm-only membership is not inferred as Deno membership. Scoped import-map overrides and non-relative/remote import-map files are diagnosed, not fetched or guessed; source imports, lockfiles, Docker/CI version hints, and tasks are not evaluated. + +The engine has three layers: bounded discovery, format readers that emit facts, and compiled detection rules. [`src/detect/rules.toml`](src/detect/rules.toml) owns manifest filenames/readers, JSON/TOML field projections, version/property mappings, workspace candidates, source-language families, inheritance boundaries, and directory exclusions. Format-specific syntax and local-reference resolution stay in adapters; framework decisions do not. + +Additional framework rules use unique IDs: + +```toml +[[rules]] +id = "company-next" +language = "javascript" +kind = "framework" +priority = 200 +all = ["dep:npm:next", "file:company.config.*"] +none = ["file:legacy.config"] +``` + +A new JSON/TOML manifest can also be registered without another Rust parser or filename branch. For example, `--rules custom.toml` can add a project format and runtime declaration: + +```toml +[[catalog.manifests]] +id = "service-meta" +files = ["service.meta.json"] +reader = "json" +package = "service-meta" +names = ["/name"] + +[[catalog.manifests.versions]] +language = "javascript" +kind = "requirement" +paths = ["/runtime/node"] + +[[rules]] +id = "service-meta" +language = "javascript" +kind = "project" +priority = 10 +any = ["package:service-meta"] +``` + +The same catalog supports identity/dependency extraction and workspace members. JSON pointers select fields; schemas reject unknown options/readers and conflicting definitions. MSBuild/Maven/Gradle property-to-version mappings and MSBuild reference item kinds are declarative too. + +`all` requires every pattern, `any` requires at least one when supplied, and `none` rejects any match. Higher priorities sort first; ties sort by ID. Matches coexist rather than hiding other frameworks or languages. Patterns also accept conjunction groups, for example `any = ["dep:gem:rails", { all = ["dep:gem:actionpack", "dep:gem:railties"] }]`. Patterns support `file:` (`*`, `?`), `source:`, `dep::`, `identity::`, `workspace:`, and `package:`. Workspace paths additionally support `**`. Ecosystems include `composer`, `go`, `cargo`, `gem`, `pypi`, `npm`, `nuget`, `dotnet-sdk`, `dotnet-framework`, `dotnet-reference`, `maven`, `maven-parent`, `maven-plugin`, and `gradle-plugin`. + +Rule atoms and basename patterns are compiled once into typed fact indexes and fast exact/prefix/suffix matchers. The scanner walks once, caches the catalog, and never reads application source bodies or lockfiles. Wide trees use at most four directory workers; small trees stay serial. Directory facts are indexed and irrelevant empty directories are discarded, while output stays deterministic. pnpm YAML is deserialized directly into typed fields without building a syntax tree; unused fields/comments are not retained. XML uses the existing zero-allocation pull tokenizer, with structure/namespace checks and DTD/custom-entity rejection. Maven local parents and managed versions, MSBuild directory props/targets, central package versions, and static Gradle plugin/dependency/module declarations are resolved without running build tools. Arbitrary MSBuild functions, remote Maven parents/BOMs, profile activation, and dynamic Gradle build logic are not executed or guessed; unresolved version references remain explicit. Each manifest/version file is capped at 1 MiB. Symlinks are not followed; traversal stops at depth 128 with a diagnostic. Dependency/build directories such as `.git`, `.jj`, `node_modules`, `vendor`, `target`, `.venv`, `.next`, `dist`, `build`, `.gradle`, and `.vs` are pruned. `bin`/`obj` are additionally pruned beneath C# project roots without skipping Rust `src/bin` sources. It does not evaluate `.gitignore`. Source-only directories roll up to a same-language project ancestor when available; sibling services keep their own manifest boundaries. + +Detection uses declared dependencies, not arbitrary substring matches or inferred framework usage from source. Composer/Cargo development dependencies are excluded; npm development dependencies are included because tools such as Astro commonly live there. Dynamic Ruby declarations and unsupported requirements options produce diagnostics instead of executing code. Invalid manifests are diagnosed without stopping discovery of other projects. + +Run the generated large-tree performance fixture with: + +```sh +cargo test --release detect::tests::benchmark_polyglot_scan -- --ignored --nocapture +``` + +### Reproduce the repository checks + +The corpus contains 51 repositories: the original 45, five ranked Deno web projects, and one supplemental mixed-workspace repository. It is public and pinned to commit SHAs in [`tests/detect-corpus.json`](tests/detect-corpus.json). The [audit report](tests/detect-corpus.md) contains assertions, per-repository timings, known limitations, and the additional Brainpod starter branches. + +| Language | Repositories | +|---|---| +| JavaScript | [react/react](https://github.com/react/react), [vercel/next.js](https://github.com/vercel/next.js), [mrdoob/three.js](https://github.com/mrdoob/three.js), [axios/axios](https://github.com/axios/axios), [react/create-react-app](https://github.com/react/create-react-app) | +| TypeScript | [freeCodeCamp/freeCodeCamp](https://github.com/freeCodeCamp/freeCodeCamp), [vuejs/vue](https://github.com/vuejs/vue), [n8n-io/n8n](https://github.com/n8n-io/n8n), [firecrawl/firecrawl](https://github.com/firecrawl/firecrawl), [excalidraw/excalidraw](https://github.com/excalidraw/excalidraw) | +| Python | [Significant-Gravitas/AutoGPT](https://github.com/Significant-Gravitas/AutoGPT), [AUTOMATIC1111/stable-diffusion-webui](https://github.com/AUTOMATIC1111/stable-diffusion-webui), [langgenius/dify](https://github.com/langgenius/dify), [langflow-ai/langflow](https://github.com/langflow-ai/langflow), [Comfy-Org/ComfyUI](https://github.com/Comfy-Org/ComfyUI) | +| PHP | [coollabsio/coolify](https://github.com/coollabsio/coolify), [nextcloud/server](https://github.com/nextcloud/server), [laravel/framework](https://github.com/laravel/framework), [filamentphp/filament](https://github.com/filamentphp/filament), [symfony/symfony](https://github.com/symfony/symfony) | +| Ruby | [rails/rails](https://github.com/rails/rails), [jekyll/jekyll](https://github.com/jekyll/jekyll), [mastodon/mastodon](https://github.com/mastodon/mastodon), [huginn/huginn](https://github.com/huginn/huginn), [discourse/discourse](https://github.com/discourse/discourse) | +| Go | [infiniflow/ragflow](https://github.com/infiniflow/ragflow), [gohugoio/hugo](https://github.com/gohugoio/hugo), [gin-gonic/gin](https://github.com/gin-gonic/gin), [netdata/netdata](https://github.com/netdata/netdata), [caddyserver/caddy](https://github.com/caddyserver/caddy) | +| Rust | [dani-garcia/vaultwarden](https://github.com/dani-garcia/vaultwarden), [meilisearch/meilisearch](https://github.com/meilisearch/meilisearch), [juspay/hyperswitch](https://github.com/juspay/hyperswitch), [parcel-bundler/parcel](https://github.com/parcel-bundler/parcel), [DioxusLabs/dioxus](https://github.com/DioxusLabs/dioxus) | +| C# | [jellyfin/jellyfin](https://github.com/jellyfin/jellyfin), [dotnet/aspnetcore](https://github.com/dotnet/aspnetcore), [jasontaylordev/CleanArchitecture](https://github.com/jasontaylordev/CleanArchitecture), [bitwarden/server](https://github.com/bitwarden/server), [ardalis/CleanArchitecture](https://github.com/ardalis/CleanArchitecture) | +| Java | [macrozheng/mall](https://github.com/macrozheng/mall), [spring-projects/spring-boot](https://github.com/spring-projects/spring-boot), [elastic/elasticsearch](https://github.com/elastic/elasticsearch), [spring-projects/spring-framework](https://github.com/spring-projects/spring-framework), [jeecgboot/JeecgBoot](https://github.com/jeecgboot/JeecgBoot) | +| Deno ecosystem (JS/TS) | [supabase/supabase](https://github.com/supabase/supabase), [honojs/hono](https://github.com/honojs/hono), [freshframework/fresh](https://github.com/freshframework/fresh), [InsForge/InsForge](https://github.com/InsForge/InsForge), [ryo-ma/github-profile-trophy](https://github.com/ryo-ma/github-profile-trophy) | +| Supplemental mixed Deno/npm workspace | [honojs/middleware](https://github.com/honojs/middleware) | + +```sh +python3 tests/audit_detect.py --list +cargo build --release --locked +cache=$(mktemp -d) +python3 tests/audit_detect.py --fetch --cache "$cache" --output "$cache/results.json" +``` + +Repeat offline by reusing the cache and omitting `--fetch`. + +XML timing is independently reproducible (warm bounded reads/UTF-8 versus preloaded pull parsing; excludes discovery, fact extraction, and CLI output): + +```sh +XML_BENCH_CACHE="$cache" cargo test --release detect::xml::tests::benchmark_xml_corpus \ + -- --ignored --nocapture --test-threads=1 +cargo test --release detect::xml::tests::benchmark_xml_local -- --ignored --nocapture +``` + +On the pinned corpus, 1,192 XML manifests (2.7 MB) took about **2.7 ms to read** and **7.4 ms to parse** in aggregate on the measured warm-cache machine. This is not a cold-disk or whole-engine latency claim. No XML parser replacement was justified by those measurements. + ## Image discovery Image commands require a pod and API token with `registry:pull` permission. List returns active public images and images in the selected pod: diff --git a/src/cmd/mod.rs b/src/cmd/mod.rs index 6541d08..ae6c968 100644 --- a/src/cmd/mod.rs +++ b/src/cmd/mod.rs @@ -17,6 +17,8 @@ use crate::output::{CommandOutput, View}; pub enum Command { /// Describe CLI commands and their machine-readable contract Describe(DescribeArgs), + /// Detect local projects, frameworks, languages, and version constraints + Detect(crate::detect::DetectArgs), /// Drive the session console the user watches while a workflow runs Agent(crate::agent::AgentArgs), /// Authenticate through the Brainpod dashboard @@ -493,7 +495,11 @@ fn is_event_resource_urn(value: &str) -> bool { pub fn needs_api_token(command: &Command) -> bool { !matches!( command, - Command::Describe(_) | Command::Agent(_) | Command::Login(_) | Command::Config(_) + Command::Describe(_) + | Command::Detect(_) + | Command::Agent(_) + | Command::Login(_) + | Command::Config(_) ) } @@ -502,6 +508,7 @@ pub fn needs_client(command: &Command) -> bool { command, Command::S3(_) | Command::Describe(_) + | Command::Detect(_) | Command::Agent(_) | Command::Login(_) | Command::Config(_) @@ -543,6 +550,7 @@ pub async fn handle(command: Command, context: CommandContext<'_>) -> Result crate::detect::handle(args), Command::Agent(args) => crate::agent::handle(args, pod, dashboard_endpoint, json).await, Command::Login(args) => Ok(CommandOutput::new( crate::auth::login( diff --git a/src/describe.rs b/src/describe.rs index f5b0405..fd451cd 100644 --- a/src/describe.rs +++ b/src/describe.rs @@ -246,7 +246,9 @@ fn argument_syntax(argument: &Arg, value_names: &[String]) -> String { fn requirements(path: &[&str]) -> (Option, Option) { match path { [] => (None, None), - ["describe"] | ["login"] | ["config"] | ["config", _] => (Some(false), Some(false)), + ["describe"] | ["detect"] | ["login"] | ["config"] | ["config", _] => { + (Some(false), Some(false)) + } ["blueprint"] => (Some(true), None), ["blueprint", "install"] => (Some(true), Some(true)), ["blueprint", _] | ["whoami"] | ["cluster"] | ["cluster", _] | ["pod"] | ["pod", _] => { @@ -269,6 +271,10 @@ fn requirements(path: &[&str]) -> (Option, Option) { fn effect(path: &[&str]) -> (&'static str, &'static str) { match path { + ["detect"] => ( + "read", + "Reads local project manifests without executing project code or accessing the network.", + ), ["describe"] | ["config", "show"] | ["config", "path"] => ( "read", "Reads local CLI information without changing state.", @@ -376,6 +382,7 @@ fn next_steps(path: &[&str]) -> Vec<&'static str> { fn examples(path: &[&str]) -> Vec<&'static str> { match path { + ["detect"] => vec!["brainpod detect .", "brainpod detect ./services --json"], ["describe"] => vec![ "brainpod describe", "brainpod describe resource create", diff --git a/src/detect/catalog.rs b/src/detect/catalog.rs new file mode 100644 index 0000000..423b1d2 --- /dev/null +++ b/src/detect/catalog.rs @@ -0,0 +1,787 @@ +use std::collections::{BTreeMap, BTreeSet, HashMap}; + +use anyhow::{Result, bail}; +use serde::Deserialize; +use serde_json::Value; + +use super::rules::{FilePattern, glob}; +use super::values::{self, Located, Properties, Syntax}; +use super::{Node, Version, Workspace, relative_file}; + +#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +pub(super) enum Reader { + Json, + Jsonc, + Toml, + Pnpm, + Go, + Gemfile, + Gemspec, + Requirements, + Pin, + Tools, + Properties, + Msbuild, + Solution, + Maven, + Gradle, + GradleProperties, + GradleJvm, + VersionCatalog, +} + +#[derive(Clone, Copy, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +pub(super) enum Resolver { + Cargo, +} + +#[derive(Default, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub(super) struct CatalogSpec { + manifests: Vec, + scan: Scan, + families: BTreeMap, + inheritance: Vec, + workspaces: Vec, + property_versions: Vec, + reference_kinds: BTreeMap, +} + +#[derive(Default, Deserialize)] +#[serde(default, deny_unknown_fields)] +struct Scan { + ignore: Vec, + prune: Vec, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Prune { + when_files: Vec, + directories: Vec, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Inheritance { + kind: String, + #[serde(default)] + boundary_files: Vec, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct ReferenceKind { + pub ecosystem: String, + #[serde(default)] + pub catalog: bool, + #[serde(default)] + normalize: Normalize, + separator: Option, +} + +impl ReferenceKind { + pub fn name(&self, value: &str) -> String { + self.normalize.name( + self.separator + .and_then(|separator| value.split(separator).next()) + .unwrap_or(value) + .trim(), + ) + } +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct WorkspaceMembers { + ecosystem: String, + #[serde(default)] + files: Vec, + package: Option, +} + +#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq, Hash)] +#[serde(rename_all = "kebab-case")] +pub(super) enum PropertyReader { + Msbuild, + Maven, + MavenCompiler, + Gradle, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct PropertyVersion { + reader: PropertyReader, + fields: Vec, + pub language: String, + pub kind: String, + split: Option, +} + +impl PropertyVersion { + pub fn emit(&self, node: &mut Node, raw: &Located, properties: &Properties, syntax: Syntax) { + for value in self.split.filter(|_| !raw.conditional).map_or_else( + || vec![raw.value.as_str()], + |split| raw.value.split(split).collect(), + ) { + if !value.trim().is_empty() { + values::version( + node, + &self.language, + &Located { + value: value.trim().to_owned(), + ..raw.clone() + }, + &self.kind, + properties, + syntax, + ); + } + } + } +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Manifest { + id: String, + files: Vec, + #[serde(default)] + prefer_first: bool, + pub reader: Reader, + pub resolver: Option, + pub package: Option, + package_path: Option, + #[serde(default)] + names: Vec, + identity: Option, + #[serde(default)] + dependencies: Vec, + #[serde(default)] + versions: Vec, + #[serde(default)] + workspaces: Vec, + pub workspace: Option, + pub runtime: Option, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Identity { + ecosystem: String, + paths: Vec, + #[serde(default)] + versions: Vec, + #[serde(default)] + normalize: Normalize, +} + +#[derive(Clone, Copy, Default, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(super) enum Normalize { + #[default] + None, + Python, + Lowercase, +} + +impl Normalize { + pub fn name(self, name: &str) -> String { + match self { + Self::None => name.to_owned(), + Self::Lowercase => name.to_ascii_lowercase(), + Self::Python => { + let mut result = String::new(); + for ch in name.chars() { + if matches!(ch, '-' | '_' | '.') { + if !result.ends_with('-') { + result.push('-'); + } + } else { + result.push(ch.to_ascii_lowercase()); + } + } + result + } + } + } +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Dependencies { + ecosystem: String, + paths: Vec, + #[serde(default)] + exclude: Vec, + #[serde(default)] + normalize: Normalize, + #[serde(default)] + requirements: bool, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct VersionField { + #[serde(default)] + language: String, + runtime: Option, + kind: String, + #[serde(default)] + paths: Vec, + dependency: Option, + key: Option, + reference_prefix: Option, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct WorkspaceField { + ecosystem: String, + present: String, + members: Vec, + #[serde(default)] + exclude: Vec, +} + +pub(super) struct Catalog { + spec: CatalogSpec, + exact: HashMap>, + wildcard: Vec<(FilePattern, usize)>, + extensions: HashMap>, + fields: HashMap>, + ignored: BTreeSet, + readable_extensions: BTreeSet, +} + +impl CatalogSpec { + pub fn extend(&mut self, extra: Self) { + self.manifests.extend(extra.manifests); + self.scan.ignore.extend(extra.scan.ignore); + self.scan.prune.extend(extra.scan.prune); + self.families.extend(extra.families); + self.inheritance.extend(extra.inheritance); + self.workspaces.extend(extra.workspaces); + self.property_versions.extend(extra.property_versions); + self.reference_kinds.extend(extra.reference_kinds); + } +} + +impl Catalog { + pub fn new(spec: CatalogSpec) -> Result { + let mut catalog = Self { + spec, + exact: HashMap::new(), + wildcard: Vec::new(), + extensions: HashMap::new(), + fields: HashMap::new(), + ignored: BTreeSet::new(), + readable_extensions: BTreeSet::new(), + }; + let mut ids = BTreeSet::new(); + for (index, manifest) in catalog.spec.manifests.iter().enumerate() { + if manifest.id.is_empty() || !ids.insert(&manifest.id) || manifest.files.is_empty() { + bail!("empty or duplicate manifest definition: {}", manifest.id); + } + for pattern in &manifest.files { + if pattern.is_empty() || pattern.contains(['/', '\\']) { + bail!("manifest patterns must be basenames: {pattern}"); + } + if pattern.contains(['*', '?']) { + let compiled = (FilePattern::new(pattern), index); + if let Some((_, extension)) = pattern.rsplit_once('.') + && !extension.is_empty() + && !extension.contains(['*', '?']) + { + catalog + .extensions + .entry(extension.to_owned()) + .or_default() + .push(compiled); + } else { + catalog.wildcard.push(compiled); + } + } else { + catalog + .exact + .entry(pattern.clone()) + .or_default() + .push(index); + } + } + let pointers = manifest + .names + .iter() + .chain( + manifest + .identity + .iter() + .flat_map(|v| v.paths.iter().chain(&v.versions)), + ) + .chain(manifest.dependencies.iter().flat_map(|v| &v.paths)) + .chain(manifest.versions.iter().flat_map(|v| &v.paths)) + .chain(manifest.workspaces.iter().flat_map(|v| { + std::iter::once(&v.present) + .chain(&v.members) + .chain(&v.exclude) + })) + .chain(manifest.package_path.iter()); + for path in pointers { + if !path.is_empty() && !path.starts_with('/') { + bail!("manifest selectors must be JSON pointers: {path}"); + } + } + for version in &manifest.versions { + if version.language.is_empty() + == version + .runtime + .as_ref() + .is_none_or(|runtime| runtime.is_empty()) + || version.kind.is_empty() + { + bail!( + "manifest version fields need exactly one language/runtime target and a kind" + ); + } + let valid_source = match manifest.reader { + Reader::Json | Reader::Jsonc | Reader::Toml => { + !version.paths.is_empty() || version.dependency.is_some() + } + Reader::Pin => { + version.key.is_none() + && version.dependency.is_none() + && version.paths.is_empty() + } + Reader::Tools | Reader::Properties => { + version.key.as_ref().is_some_and(|key| !key.is_empty()) + } + _ => false, + }; + if !valid_source { + bail!( + "version field has no applicable value source in {}", + manifest.id + ); + } + } + } + for (file, indexes) in &catalog.exact { + let first = catalog.spec.manifests[indexes[0]].reader; + if indexes + .iter() + .any(|index| catalog.spec.manifests[*index].reader != first) + || catalog + .matching(file) + .any(|manifest| manifest.reader != first) + { + bail!("conflicting manifest readers for {file}"); + } + } + for (index, field) in catalog.spec.property_versions.iter().enumerate() { + if field.fields.is_empty() || field.language.is_empty() || field.kind.is_empty() { + bail!("incomplete property version definition"); + } + for name in &field.fields { + if catalog + .fields + .entry(field.reader) + .or_default() + .insert(name.clone(), index) + .is_some() + { + bail!( + "duplicate property version field: {:?}:{name}", + field.reader + ); + } + } + } + catalog + .readable_extensions + .extend(catalog.exact.keys().map(|file| { + file.rsplit_once('.') + .map_or("", |(_, extension)| extension) + .to_owned() + })); + catalog + .readable_extensions + .extend(catalog.extensions.keys().cloned()); + catalog + .ignored + .extend(catalog.spec.scan.ignore.iter().cloned()); + Ok(catalog) + } + + pub fn matching<'a>(&'a self, file: &'a str) -> impl Iterator + 'a { + let extension = file + .rsplit_once('.') + .map(|(_, extension)| extension) + .unwrap_or(""); + self.exact + .get(file) + .into_iter() + .flatten() + .copied() + .chain( + self.extensions + .get(extension) + .into_iter() + .flatten() + .chain(&self.wildcard) + .filter(move |(pattern, _)| pattern.matches(file)) + .map(|(_, index)| *index), + ) + .map(|index| &self.spec.manifests[index]) + } + + pub fn readable(&self, file: &str) -> bool { + let extension = file.rsplit_once('.').map_or("", |(_, extension)| extension); + (!self.wildcard.is_empty() || self.readable_extensions.contains(extension)) + && self.matching(file).next().is_some() + } + pub fn reader(&self, file: &str, reader: Reader) -> bool { + self.matching(file) + .any(|manifest| manifest.reader == reader) + } + pub fn ignored(&self, directory: &str) -> bool { + self.ignored.contains(directory) + } + pub fn pruned(&self, node: &Node, directory: &str) -> bool { + self.spec.scan.prune.iter().any(|rule| { + rule.directories + .iter() + .any(|name| name.eq_ignore_ascii_case(directory)) + && node + .files + .iter() + .any(|file| rule.when_files.iter().any(|pattern| glob(pattern, file))) + }) + } + pub fn family<'a>(&'a self, language: &'a str) -> &'a str { + self.spec + .families + .get(language) + .map_or(language, String::as_str) + } + pub fn inherited(&self, kind: &str) -> bool { + self.spec.inheritance.iter().any(|rule| rule.kind == kind) + } + pub fn boundaries<'a>(&'a self, node: &'a Node) -> impl Iterator { + self.spec + .inheritance + .iter() + .filter(|rule| { + node.files.iter().any(|file| { + rule.boundary_files + .iter() + .any(|pattern| glob(pattern, file)) + }) + }) + .map(|rule| rule.kind.as_str()) + } + pub fn workspace_candidates<'a>(&'a self, node: &'a Node) -> impl Iterator { + self.spec + .workspaces + .iter() + .filter(|rule| { + rule.package + .as_ref() + .is_some_and(|package| node.packages.contains_key(package)) + || node + .files + .iter() + .any(|file| rule.files.iter().any(|pattern| glob(pattern, file))) + }) + .map(|rule| rule.ecosystem.as_str()) + } + pub fn property(&self, reader: PropertyReader, name: &str) -> Option<&PropertyVersion> { + self.fields + .get(&reader) + .and_then(|fields| fields.get(name)) + .map(|index| &self.spec.property_versions[*index]) + } + pub fn properties<'a>( + &'a self, + reader: PropertyReader, + ) -> impl Iterator + 'a { + self.spec + .property_versions + .iter() + .filter(move |field| field.reader == reader) + } + pub fn property_values<'a>( + &'a self, + reader: PropertyReader, + values: &'a Properties, + ) -> impl Iterator { + self.properties(reader).flat_map(move |field| { + field + .fields + .iter() + .filter_map(move |key| values.get(key).map(|value| (field, value))) + }) + } + pub fn reference(&self, kind: &str) -> Option<&ReferenceKind> { + self.spec.reference_kinds.get(kind) + } +} + +fn select<'a>(document: &'a Value, paths: &[String]) -> Option<&'a Value> { + paths.iter().find_map(|path| document.pointer(path)) +} +fn scalar(value: &Value) -> Option<&str> { + value + .as_str() + .or_else(|| value.get("version").and_then(Value::as_str)) +} +fn selected_scalar<'a>(document: &'a Value, paths: &[String]) -> Option<&'a str> { + select(document, paths).and_then(scalar) +} +fn strings(value: Option<&Value>) -> Vec { + value + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(Value::as_str) + .map(str::to_owned) + .collect() +} + +pub(super) fn workspace_patterns(patterns: Vec) -> (Vec, Vec) { + let mut members = Vec::new(); + let mut exclude = Vec::new(); + for pattern in patterns { + if let Some(pattern) = pattern.strip_prefix('!') { + exclude.push(pattern.to_owned()); + } else { + members.push(pattern); + } + } + (members, exclude) +} + +impl Manifest { + pub fn preferred(&self, file: &str, files: &BTreeSet) -> bool { + if self.prefer_first { + self.files + .iter() + .find(|name| files.contains(*name)) + .is_none_or(|name| name == file) + } else { + true + } + } + pub fn apply(&self, node: &mut Node, file: &str, document: &Value) { + if let Some(runtime) = &self.runtime { + node.runtimes + .insert(runtime.clone(), relative_file(&node.path, file)); + } + if let Some(name) = selected_scalar(document, &self.names) { + node.names.insert(name.to_owned()); + } + if let Some(identity) = &self.identity + && let Some(name) = selected_scalar(document, &identity.paths) + { + node.names.insert(name.to_owned()); + node.identity( + &identity.ecosystem, + &identity.normalize.name(name), + selected_scalar(document, &identity.versions), + file, + ); + } + if let Some(package) = &self.package + && self + .package_path + .as_ref() + .is_none_or(|path| document.pointer(path).is_some_and(Value::is_object)) + { + node.packages + .insert(package.clone(), relative_file(&node.path, file)); + } + for dependencies in &self.dependencies { + for path in &dependencies.paths { + if dependencies.requirements { + for requirement in document + .pointer(path) + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(Value::as_str) + { + if let Some((name, version)) = + super::parsers::requirement_parts(requirement) + { + node.dependency( + &dependencies.ecosystem, + &dependencies.normalize.name(name), + version, + file, + ); + } + } + } else if let Some(entries) = document.pointer(path).and_then(Value::as_object) { + for (name, value) in entries { + if !dependencies.exclude.contains(name) { + node.dependency( + &dependencies.ecosystem, + &dependencies.normalize.name(name), + scalar(value), + file, + ); + } + } + } + } + } + for workspace in &self.workspaces { + if document.pointer(&workspace.present).is_none() { + continue; + } + let (members, mut exclude) = + workspace_patterns(strings(select(document, &workspace.members))); + exclude.extend(strings(select(document, &workspace.exclude))); + node.workspaces.push(Workspace { + ecosystem: workspace.ecosystem.clone(), + source: relative_file(&node.path, file), + members, + exclude, + }); + } + self.versions(node, file, Some(document), None, false); + } + + pub fn versions( + &self, + node: &mut Node, + file: &str, + document: Option<&Value>, + text: Option<&str>, + dependencies: bool, + ) { + for field in &self.versions { + if dependencies != field.dependency.is_some() { + continue; + } + if let Some(key) = &field.dependency { + if let Some(dependency) = node.dependencies.get(key) { + let mut versions = dependency + .declarations + .iter() + .map(|version| Version { + kind: if field + .reference_prefix + .as_ref() + .is_some_and(|prefix| version.value.starts_with(prefix)) + { + "reference".to_owned() + } else { + field.kind.clone() + }, + ..version.clone() + }) + .collect::>(); + if let Some(raw) = &dependency.version { + let kind = if field + .reference_prefix + .as_ref() + .is_some_and(|prefix| raw.starts_with(prefix)) + { + "reference" + } else { + &field.kind + }; + versions.push(Version { + value: raw.clone(), + source: dependency.source.clone(), + kind: kind.to_owned(), + }); + } + if let Some(resolved) = &dependency.resolved { + versions.push(Version { + kind: field.kind.clone(), + ..resolved.clone() + }); + } + field + .target(node) + .entry(field.name().to_owned()) + .or_default() + .extend(versions); + } + } else if let Some(value) = document + .and_then(|doc| selected_scalar(doc, &field.paths)) + .or(text) + { + field.emit(node, value, file); + } + } + } + + pub fn text_versions(&self, node: &mut Node, file: &str, text: &str) { + match self.reader { + Reader::Pin => { + let value = text + .lines() + .filter(|line| !line.trim().starts_with('#')) + .collect::>() + .join(" "); + self.versions(node, file, None, Some(&value), false); + } + Reader::Tools => { + for line in text.lines() { + let mut words = line.split('#').next().unwrap_or("").split_whitespace(); + let Some(key) = words.next() else { + continue; + }; + let value = words.collect::>().join(" "); + for field in self + .versions + .iter() + .filter(|field| field.key.as_deref() == Some(key)) + { + field.emit(node, &value, file); + } + } + } + Reader::Properties => { + let properties = values::properties(text, &relative_file(&node.path, file)); + for field in &self.versions { + if let Some(value) = field.key.as_ref().and_then(|key| properties.get(key)) { + field.emit(node, &value.value, file); + } + } + } + _ => (), + } + } +} + +impl VersionField { + fn name(&self) -> &str { + self.runtime.as_deref().unwrap_or(&self.language) + } + fn target<'a>(&self, node: &'a mut Node) -> &'a mut BTreeMap> { + if self.runtime.is_some() { + &mut node.runtime_versions + } else { + &mut node.versions + } + } + fn emit(&self, node: &mut Node, value: &str, file: &str) { + if !value.trim().is_empty() { + let version = Version { + value: value.trim().to_owned(), + source: relative_file(&node.path, file), + kind: self.kind.clone(), + }; + self.target(node) + .entry(self.name().to_owned()) + .or_default() + .push(version); + } + } +} diff --git a/src/detect/deno.rs b/src/detect/deno.rs new file mode 100644 index 0000000..83b703a --- /dev/null +++ b/src/detect/deno.rs @@ -0,0 +1,306 @@ +use std::collections::{BTreeMap, HashMap}; +use std::fs::{self, File}; +use std::io::Read; +use std::path::{Component, Path, PathBuf}; + +use serde_json::Value; + +use super::parsers::{Catalogs, Documents}; +use super::{Dependency, Diagnostic, Node, Stats, Workspace, display_path, relative_file}; + +type Bindings = BTreeMap>; + +struct Context { + source: String, + workspaces: Vec, + bindings: Bindings, +} + +pub(super) fn resolve( + root: &Path, + nodes: &mut [Node], + documents: &Documents, + catalogs: &mut HashMap>, + stats: &mut Stats, + diagnostics: &mut Vec, +) { + let mut contexts = HashMap::new(); + for node in nodes.iter() { + let Some(source) = node.packages.get("deno") else { + continue; + }; + let Some(file) = Path::new(source).file_name().and_then(|file| file.to_str()) else { + continue; + }; + let Some(document) = documents.get(&node.path).and_then(|docs| docs.get(file)) else { + continue; + }; + let mut bindings = Bindings::new(); + if let Some(reference) = document.get("importMap").and_then(Value::as_str) { + if document.get("imports").is_some() || document.get("scopes").is_some() { + diagnostics.push(Diagnostic { + path: source.clone(), + message: "importMap is exclusive with imports/scopes".to_owned(), + }); + } else if let Some((source, map)) = + import_map(root, &node.path, reference, stats, diagnostics) + { + imports(&map, &source, &mut bindings, diagnostics); + } + } else { + imports(document, source, &mut bindings, diagnostics); + } + let workspaces = node + .workspaces + .iter() + .filter(|workspace| { + workspace.ecosystem == "deno" + || (workspace.ecosystem == "node" + && workspace.source == relative_file(&node.path, "package.json")) + }) + .cloned() + .collect::>(); + let package = documents + .get(&node.path) + .and_then(|docs| docs.get("package.json")); + let definitions = package + .filter(|doc| doc.get("catalog").is_some() || doc.get("catalogs").is_some()) + .unwrap_or(document); + if definitions.get("catalog").is_some() || definitions.get("catalogs").is_some() { + let source = if std::ptr::eq(definitions, document) { + source.clone() + } else { + relative_file(&node.path, "package.json") + }; + catalogs.entry(node.path.clone()).or_insert_with(|| { + Some(Catalogs { + workspaces: workspaces.clone(), + source, + catalog: definitions + .get("catalog") + .and_then(|v| serde_json::from_value(v.clone()).ok()) + .unwrap_or_default(), + catalogs: definitions + .get("catalogs") + .and_then(|v| serde_json::from_value(v.clone()).ok()) + .unwrap_or_default(), + }) + }); + } + contexts.insert( + node.path.clone(), + Context { + source: source.clone(), + workspaces, + bindings, + }, + ); + } + if contexts.is_empty() { + return; + } + for node in nodes { + let local = contexts.get(&node.path); + let inherited = node.path.ancestors().skip(1).find_map(|path| { + contexts.get(path).filter(|context| { + node.files.iter().any(|file| { + matches!(file.as_str(), "deno.json" | "deno.jsonc" | "package.json") + }) && node.path.strip_prefix(path).ok().is_some_and(|relative| { + context + .workspaces + .iter() + .any(|workspace| workspace.contains(&display_path(relative))) + }) + }) + }); + if let Some(context) = local.or(inherited) { + node.runtimes + .entry("deno".to_owned()) + .or_insert_with(|| context.source.clone()); + } + let mut bindings = inherited.map_or_else(Bindings::new, |context| context.bindings.clone()); + if let Some(local) = local { + bindings.extend(local.bindings.clone()); + } + for (key, dependency) in bindings.into_values().flatten() { + node.merge_dependency(key, dependency); + } + } +} + +fn imports( + document: &Value, + source: &str, + bindings: &mut Bindings, + diagnostics: &mut Vec, +) { + if let Some(entries) = document.get("imports").and_then(Value::as_object) { + for (alias, value) in entries { + bindings.insert( + alias.clone(), + value + .as_str() + .and_then(|value| { + specifier(value).or_else(|| { + value.starts_with("catalog:").then_some(( + "npm", + alias.as_str(), + Some(value), + )) + }) + }) + .map(|(ecosystem, name, version)| { + ( + format!("{ecosystem}:{name}"), + Dependency { + version: version.map(str::to_owned), + source: source.to_owned(), + resolved: None, + declarations: Vec::new(), + }, + ) + }), + ); + } + } + if document + .get("scopes") + .is_some_and(|scopes| scopes.as_object().is_none_or(|scopes| !scopes.is_empty())) + { + diagnostics.push(Diagnostic { + path: source.to_owned(), + message: "scoped import-map overrides are not evaluated".to_owned(), + }); + } +} + +fn specifier(value: &str) -> Option<(&str, &str, Option<&str>)> { + let (ecosystem, package) = if let Some(package) = value.strip_prefix("npm:") { + ("npm", package) + } else if let Some(package) = value + .strip_prefix("jsr:") + .or_else(|| value.strip_prefix("https://jsr.io/")) + { + ("jsr", package) + } else if let Some(package) = value.strip_prefix("https://deno.land/x/") { + ("deno", package) + } else if let Some(package) = value.strip_prefix("https://esm.sh/") { + ("npm", package) + } else { + return None; + }; + let start = if package.starts_with('@') { + package.find('/')? + 1 + } else { + 0 + }; + let tail = &package[start..]; + let end = tail + .find(['@', '/', '?', '#']) + .map_or(package.len(), |index| start + index); + let name = &package[..end]; + if name.is_empty() || name.ends_with('/') { + return None; + } + let version = package[end..] + .strip_prefix('@') + .map(|value| value.split(['/', '?', '#']).next().unwrap_or("")); + if version == Some("") { + return None; + } + Some((ecosystem, name, version)) +} + +fn import_map( + root: &Path, + base: &Path, + reference: &str, + stats: &mut Stats, + diagnostics: &mut Vec, +) -> Option<(String, Value)> { + let loaded = (|| -> anyhow::Result<(String, Value)> { + if reference.contains(':') || reference.contains('\\') || Path::new(reference).is_absolute() + { + anyhow::bail!("only local relative import maps are inspected"); + } + let mut relative = PathBuf::new(); + for part in base.join(reference).components() { + match part { + Component::Normal(value) => relative.push(value), + Component::CurDir => (), + Component::ParentDir if relative.pop() => (), + _ => anyhow::bail!("import map escapes detection root"), + } + } + let mut path = root.to_path_buf(); + for part in &relative { + path.push(part); + if fs::symlink_metadata(&path)?.file_type().is_symlink() { + anyhow::bail!("import-map symlinks are not followed"); + } + } + if !fs::metadata(&path)?.is_file() { + anyhow::bail!("import map must be a regular file"); + } + let file = File::open(&path)?; + let mut bytes = Vec::new(); + file.take(super::MAX_FILE_BYTES + 1) + .read_to_end(&mut bytes)?; + stats.files_read += 1; + stats.bytes_read += bytes.len(); + if bytes.len() > super::MAX_FILE_BYTES as usize { + anyhow::bail!("import map exceeds 1 MiB read limit"); + } + let text = String::from_utf8(bytes)?; + let document = super::jsonc::parse(&text).map_err(anyhow::Error::msg)?; + if !document.is_object() { + anyhow::bail!("import map must be an object"); + } + Ok((display_path(&relative), document)) + })(); + match loaded { + Ok(result) => Some(result), + Err(error) => { + diagnostics.push(Diagnostic { + path: relative_file(base, reference), + message: format!("could not inspect import map: {error}"), + }); + None + } + } +} + +#[cfg(test)] +mod tests { + use super::specifier; + #[test] + fn registry_and_url_specifiers_use_targets_not_alias_names() { + for (input, expected) in [ + ("npm:hono@^4/subpath", Some(("npm", "hono", Some("^4")))), + ( + "npm:@scope/pkg@~2.1/sub", + Some(("npm", "@scope/pkg", Some("~2.1"))), + ), + ( + "jsr:@fresh/core@^2.0.0", + Some(("jsr", "@fresh/core", Some("^2.0.0"))), + ), + ( + "https://deno.land/x/oak@v17.0.0/mod.ts", + Some(("deno", "oak", Some("v17.0.0"))), + ), + ( + "https://esm.sh/preact@10.2.0?dev", + Some(("npm", "preact", Some("10.2.0"))), + ), + ("jsr:@oak/oak", Some(("jsr", "@oak/oak", None))), + ("https://deno.land.evil/x/fresh@1.0", None), + ("./npm:hono@4.ts", None), + ("npm:", None), + ("npm:@scope", None), + ("npm:hono@", None), + ] { + assert_eq!(specifier(input), expected, "{input}"); + } + } +} diff --git a/src/detect/deno_tests.rs b/src/detect/deno_tests.rs new file mode 100644 index 0000000..86dcc11 --- /dev/null +++ b/src/detect/deno_tests.rs @@ -0,0 +1,306 @@ +use super::scan; +use super::tests::{detects, fixture, project, version}; + +#[test] +fn deno_jsonc_detects_runtime_and_package_versions_without_conflation() { + let dir = fixture(&[ + ( + "deno.jsonc", + r#"{ // package version is not a Deno runtime pin + "name":"@acme/service", "version":"9.9.9", "imports":{"web":"jsr:@hono/hono@^4",}, + }"#, + ), + (".tool-versions", "deno 2.5.6\nnodejs 24.1.0\n"), + ("main.ts", "throw new Error('must not execute');"), + ]); + let report = scan(dir.path(), None).unwrap(); + let root = project(&report, "."); + assert!(detects(root, "deno")); + assert!(detects(root, "hono")); + assert_eq!(root.runtimes.len(), 1); + assert_eq!(root.runtimes[0].name, "deno"); + assert_eq!(root.runtimes[0].source, "deno.jsonc"); + assert_eq!(root.runtimes[0].versions[0].value, "2.5.6"); + assert_eq!(root.runtimes[0].versions[0].source, ".tool-versions"); + assert!(version(root, "javascript", "24.1.0", ".tool-versions")); + assert!(root.languages.iter().all(|language| { + language.name != "deno" + && language + .versions + .iter() + .all(|version| version.value != "9.9.9" && version.value != "2.5.6") + })); + assert_eq!(report.stats.files_read, 2); + assert!(report.diagnostics.is_empty()); +} + +#[test] +fn mixed_deno_npm_and_pnpm_workspaces_preserve_boundaries_and_member_overrides() { + let dir = fixture(&[ + ( + "deno.json", + r#"{"workspace":["./packages/*","!packages/excluded"],"imports":{"web":"jsr:@fresh/core@^2"}}"#, + ), + (".deno-version", "2.5.0"), + ("package.json", r#"{"workspaces":["node-only/*"]}"#), + ("pnpm-workspace.yaml", "packages: ['pnpm-only/*']\n"), + ("packages/deno/deno.json", "{}"), + ("packages/deno/main.ts", ""), + ( + "packages/hybrid/deno.jsonc", + r#"{"imports":{"web":"npm:hono@^4"},}"#, + ), + ( + "packages/hybrid/package.json", + r#"{"name":"hybrid","engines":{"node":">=22","deno":">=2.4"}}"#, + ), + ("packages/npm/package.json", r#"{"name":"npm-only"}"#), + ("packages/excluded/package.json", "{}"), + ("node-only/member/package.json", "{}"), + ("pnpm-only/member/package.json", "{}"), + ("outside/package.json", "{}"), + ]); + let report = scan(dir.path(), None).unwrap(); + let root = project(&report, "."); + assert!(detects(root, "deno-workspace")); + assert!(detects(root, "node-workspace")); + for name in [ + "packages/deno", + "packages/hybrid", + "packages/npm", + "node-only/member", + "pnpm-only/member", + ] { + assert!( + root.workspace_members.iter().any(|path| path == name), + "{name}" + ); + } + assert!( + !root + .workspace_members + .iter() + .any(|path| path == "packages/excluded" || path == "outside") + ); + for name in ["packages/deno", "packages/npm", "node-only/member"] { + let member = project(&report, name); + assert_eq!(member.runtimes[0].name, "deno"); + assert!( + member.runtimes[0] + .versions + .iter() + .any(|version| version.value == "2.5.0" + && version.kind == "pin" + && version.source == ".deno-version") + ); + assert!(detects(member, "fresh")); + } + assert!(project(&report, "pnpm-only/member").runtimes.is_empty()); + let deno_workspace = root + .workspaces + .iter() + .find(|workspace| workspace.ecosystem == "deno") + .unwrap(); + assert!( + !deno_workspace + .members + .iter() + .any(|member| member.starts_with("node-only/") || member.starts_with("pnpm-only/")) + ); + let hybrid = project(&report, "packages/hybrid"); + assert!(detects(hybrid, "hono")); + assert!(!detects(hybrid, "fresh")); + assert!(hybrid.runtimes[0].versions.iter().any( + |version| version.value == ">=2.4" && version.source == "packages/hybrid/package.json" + )); + assert!(version( + hybrid, + "javascript", + ">=22", + "packages/hybrid/package.json" + )); + for name in ["outside", "packages/excluded"] { + assert!(project(&report, name).runtimes.is_empty()); + assert!(!detects(project(&report, name), "fresh")); + } +} + +#[test] +fn deno_local_import_maps_keep_their_own_evidence_and_member_aliases_can_block() { + let dir = fixture(&[ + ( + "deno.json", + r#"{"workspace":["member"],"importMap":"config/custom-map.jsonc"}"#, + ), + ( + "config/custom-map.jsonc", + r#"{"imports":{"web":"https://deno.land/x/fresh@1.7.3/mod.ts",},}"#, + ), + ( + "member/deno.json", + r#"{"imports":{"web":"./not-a-framework.ts"}}"#, + ), + ]); + let report = scan(dir.path(), None).unwrap(); + let root = project(&report, "."); + let fresh = root.detections.iter().find(|d| d.id == "fresh").unwrap(); + assert!( + fresh + .evidence + .iter() + .any(|e| e.source == "config/custom-map.jsonc" && e.value.as_deref() == Some("1.7.3")) + ); + assert!(!detects(project(&report, "member"), "fresh")); + assert_eq!(report.stats.files_read, 3); + assert!(report.diagnostics.is_empty()); +} + +#[test] +fn deno_json_precedes_jsonc_and_invalid_metadata_is_diagnostic() { + let dir = fixture(&[ + ("deno.json", "{}"), + ( + "deno.jsonc", + r#"{"workspace":["foo"],"imports":{"x":"npm:hono@4"}}"#, + ), + ("foo/package.json", "{}"), + ]); + let report = scan(dir.path(), None).unwrap(); + let root = project(&report, "."); + assert!(detects(root, "deno")); + assert!(!detects(root, "hono")); + assert!(!detects(root, "deno-workspace")); + for value in ["[]", "{broken", "{ /* unterminated"] { + let dir = fixture(&[("deno.jsonc", value)]); + let report = scan(dir.path(), None).unwrap(); + assert_eq!(report.diagnostics.len(), 1); + assert!(report.projects.is_empty()); + } +} + +#[test] +fn deno_catalogs_reuse_resolution_and_package_json_takes_precedence() { + let dir = fixture(&[ + ( + "deno.json", + r#"{"workspace":["packages/*"],"catalog":{"typescript":"^5.4"},"catalogs":{"old":{"typescript":"^5.3"}}}"#, + ), + ( + "package.json", + r#"{"catalog":{"typescript":"^5.9"},"catalogs":{"old":{"typescript":"^5.8"}}}"#, + ), + ( + "packages/npm/package.json", + r#"{"devDependencies":{"typescript":"catalog:"}}"#, + ), + ( + "packages/deno/deno.json", + r#"{"imports":{"typescript":"catalog:old"}}"#, + ), + ]); + let report = scan(dir.path(), None).unwrap(); + for (path, expected) in [("packages/npm", "^5.9"), ("packages/deno", "^5.8")] { + assert!(version( + project(&report, path), + "typescript", + expected, + "package.json" + )); + } + assert!(report.diagnostics.is_empty()); +} + +#[test] +fn import_maps_do_not_escape_root_fetch_urls_or_read_oversized_files() { + for target in [ + "../outside.json", + "https://example.invalid/imports.json", + "/tmp/imports.json", + "missing.json", + ] { + let dir = fixture(&[( + "deno.json", + &format!( + r#"{{"importMap":{}}}"#, + serde_json::to_string(target).unwrap() + ), + )]); + let report = scan(dir.path(), None).unwrap(); + assert_eq!(report.stats.files_read, 1); + assert_eq!(report.diagnostics.len(), 1); + } + let dir = fixture(&[ + ("deno.json", r#"{"importMap":"huge.json"}"#), + ("huge.json", &" ".repeat(1024 * 1024 + 1)), + ]); + let report = scan(dir.path(), None).unwrap(); + assert_eq!(report.stats.files_read, 2); + assert!(report.diagnostics[0].message.contains("1 MiB")); +} + +#[cfg(unix)] +#[test] +fn import_maps_do_not_follow_file_or_parent_symlinks() { + for target in ["link.json", "link-dir/map.json"] { + let outside = fixture(&[("map.json", r#"{"imports":{"x":"npm:hono@4"}}"#)]); + let dir = fixture(&[("deno.json", &format!(r#"{{"importMap":"{target}"}}"#))]); + std::os::unix::fs::symlink( + outside.path().join("map.json"), + dir.path().join("link.json"), + ) + .unwrap(); + std::os::unix::fs::symlink(outside.path(), dir.path().join("link-dir")).unwrap(); + let report = scan(dir.path(), None).unwrap(); + assert_eq!(report.stats.files_read, 1); + assert!( + report + .diagnostics + .iter() + .any(|diagnostic| diagnostic.message.contains("symlinks")) + ); + assert!(!detects(project(&report, "."), "hono")); + } +} + +#[test] +fn hybrid_imports_preserve_peer_development_and_jsr_constraints() { + let dir = fixture(&[ + ( + "deno.json", + r#"{"imports":{"web":"jsr:@hono/hono@^4.8.3"}}"#, + ), + ( + "package.json", + r#"{"peerDependencies":{"hono":">=4"},"devDependencies":{"hono":"^4.11.5"}}"#, + ), + ]); + let report = scan(dir.path(), None).unwrap(); + let hono = project(&report, ".") + .detections + .iter() + .find(|d| d.id == "hono") + .unwrap(); + for (value, source) in [ + (">=4", "package.json"), + ("^4.11.5", "package.json"), + ("^4.8.3", "deno.json"), + ] { + assert!( + hono.evidence + .iter() + .any(|e| e.source == source && e.value.as_deref() == Some(value)) + ); + } +} + +#[test] +fn text_output_includes_an_unversioned_runtime_for_npm_only_members() { + let dir = fixture(&[ + ("deno.json", r#"{"workspace":["member"]}"#), + ("member/package.json", "{}"), + ]); + let report = scan(dir.path(), None).unwrap(); + let text = super::render(&serde_json::to_value(&report).unwrap()).join("\n"); + assert!(text.contains("runtime deno (deno.json)")); + assert!(project(&report, "member").runtimes[0].versions.is_empty()); +} diff --git a/src/detect/dotnet.rs b/src/detect/dotnet.rs new file mode 100644 index 0000000..b23bce7 --- /dev/null +++ b/src/detect/dotnet.rs @@ -0,0 +1,671 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Path, PathBuf}; + +use super::catalog::{Catalog, PropertyReader, Reader}; +use super::parsers::Documents; +use anyhow::{Result, bail}; +use serde_json::Value; + +use super::values::{self, Located, Properties, Syntax}; +use super::{Diagnostic, Node, Workspace, display_path, relative_file, xml}; + +#[derive(Default)] +struct Document { + properties: Vec<(String, Located, Vec)>, + steps: Vec, + references: Vec, + imports: Vec<(String, Vec)>, + projects: Vec, + sdk: Vec<(String, Option)>, + solution: bool, +} + +enum Step { + Property(usize), + Import(usize), +} + +struct Reference { + kind: String, + name: String, + version: Option, +} + +fn document(text: &str, source: &str, catalog: &Catalog) -> Result { + let mut doc = Document::default(); + let mut nested_versions = BTreeMap::new(); + xml::read(text, |stack| { + let Some(element) = stack.last() else { + return Ok(()); + }; + let root = stack.first().map(|e| e.name); + if root == Some("Solution") { + doc.solution = true; + if element.name == "Project" + && let Some(path) = element.attributes.get("Path") + { + doc.projects.push(path.clone()); + } + return Ok(()); + } + if root != Some("Project") { + bail!("expected an MSBuild Project or Solution root"); + } + let conditions = stack + .iter() + .filter_map(|e| { + e.attributes + .get("Condition") + .filter(|v| !v.trim().is_empty()) + .cloned() + }) + .collect::>(); + let conditional = !conditions.is_empty(); + let located = |value: &str| Located { + value: value.trim().to_owned(), + source: source.to_owned(), + conditional, + }; + let parent = stack.get(stack.len().saturating_sub(2)); + if parent.is_some_and(|p| p.name == "PropertyGroup") && !element.text.trim().is_empty() { + doc.steps.push(Step::Property(doc.properties.len())); + doc.properties.push(( + element.name.to_ascii_lowercase(), + located(&element.text), + conditions.clone(), + )); + } + if element.name == "Version" + && let Some(parent) = parent + { + nested_versions.insert( + (stack.len() - 1, parent.name.to_owned()), + located(&element.text), + ); + } + if catalog.reference(element.name).is_some() + && let Some(name) = element + .attributes + .get("Include") + .or_else(|| element.attributes.get("Update")) + { + let version = element + .attributes + .get("VersionOverride") + .or_else(|| element.attributes.get("Version")) + .map(|v| located(v)) + .or_else(|| nested_versions.remove(&(stack.len(), element.name.to_owned()))); + doc.references.push(Reference { + kind: element.name.to_owned(), + name: name.clone(), + version, + }); + } + if element.name == "ProjectReference" + && let Some(path) = element.attributes.get("Include") + { + doc.projects.push(path.clone()); + } + if element.name == "Import" { + if let Some(sdk) = element.attributes.get("Sdk") { + doc.sdk + .push((sdk.clone(), element.attributes.get("Version").cloned())); + } else if let Some(path) = element.attributes.get("Project") { + doc.steps.push(Step::Import(doc.imports.len())); + doc.imports.push((path.clone(), conditions.clone())); + } + } + if element.name == "Sdk" + && let Some(name) = element.attributes.get("Name") + { + doc.sdk + .push((name.clone(), element.attributes.get("Version").cloned())); + } + if stack.len() == 1 + && let Some(sdk) = element.attributes.get("Sdk") + { + for sdk in sdk.split(';') { + let (name, version) = sdk + .trim() + .split_once('/') + .map_or((sdk.trim(), None), |(name, version)| { + (name, Some(version.to_owned())) + }); + if !name.is_empty() { + doc.sdk.push((name.to_owned(), version)); + } + } + } + Ok(()) + })?; + Ok(doc) +} + +fn nearest( + path: &Path, + file: &str, + documents: &BTreeMap>, +) -> Option { + path.ancestors() + .map(|p| p.join(file)) + .find(|p| documents.contains_key(p)) +} + +fn split_condition<'a>(text: &'a str, operator: &str) -> Option<(&'a str, &'a str)> { + let upper = text.to_ascii_uppercase(); + let mut depth = 0; + let mut quote = None; + for (index, ch) in text.char_indices() { + if quote == Some(ch) { + quote = None; + continue; + } + if quote.is_some() { + continue; + } + if ch == '\'' || ch == '"' { + quote = Some(ch); + continue; + } + if ch == '(' { + depth += 1; + } else if ch == ')' { + depth -= 1; + } + if depth == 0 && upper[index..].starts_with(operator) { + return Some((&text[..index], &text[index + operator.len()..])); + } + } + None +} + +fn condition(text: &str, properties: &Properties, depth: usize) -> Option { + if depth == 64 { + return None; + } + let text = text.trim(); + if text.is_empty() { + return Some(true); + } + for (operator, or) in [(" OR ", true), (" AND ", false)] { + if let Some((left, right)) = split_condition(text, operator) { + let a = condition(left, properties, depth + 1); + let b = condition(right, properties, depth + 1); + return if or { + if a == Some(true) || b == Some(true) { + Some(true) + } else if a == Some(false) && b == Some(false) { + Some(false) + } else { + None + } + } else if a == Some(false) || b == Some(false) { + Some(false) + } else if a == Some(true) && b == Some(true) { + Some(true) + } else { + None + }; + } + } + if text.starts_with('(') && text.ends_with(')') { + return condition(&text[1..text.len() - 1], properties, depth + 1); + } + let mut expanded = text.to_owned(); + for _ in 0..64 { + let Some(start) = expanded.find("$(") else { + break; + }; + let end = expanded[start + 2..].find(')')? + start + 2; + let name = &expanded[start + 2..end]; + if !name + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '_' | '.')) + { + return None; + } + let key = name.to_ascii_lowercase(); + let value = if let Some(value) = properties.get(&key) { + value.value.as_str() + } else if key.starts_with("msbuild") + || matches!( + key.as_str(), + "configuration" + | "platform" + | "platformtarget" + | "runtimeidentifier" + | "os" + | "designtimebuild" + | "buildinginsidevisualstudio" + ) + { + return None; + } else { + "" + }; + if value.contains("$(") { + return None; + } + expanded.replace_range(start..end + 1, value); + } + for operator in ["==", "!="] { + if let Some((left, right)) = split_condition(&expanded, operator) { + let literal = |value: &str| -> Option { + let value = value.trim(); + if value.len() >= 2 + && ((value.starts_with('\'') && value.ends_with('\'')) + || (value.starts_with('"') && value.ends_with('"'))) + { + return Some(value[1..value.len() - 1].to_owned()); + } + None + }; + let equal = literal(left)?.eq_ignore_ascii_case(&literal(right)?); + return Some(if operator == "==" { equal } else { !equal }); + } + } + match expanded.trim().to_ascii_lowercase().as_str() { + "true" => Some(true), + "false" => Some(false), + _ => None, + } +} + +fn conditions(values: &[String], properties: &Properties) -> Option { + let mut unknown = false; + for value in values { + match condition(value, properties, 0) { + Some(false) => return Some(false), + Some(true) => (), + None => unknown = true, + } + } + if unknown { None } else { Some(true) } +} + +fn apply( + path: &Path, + documents: &BTreeMap>, + properties: &mut Properties, + active: &mut BTreeSet, + sources: &mut Vec, + conditional: &mut Vec<(String, Located)>, +) { + let Some(Some(doc)) = documents.get(path) else { + return; + }; + if active.len() == 64 || !active.insert(path.to_path_buf()) { + return; + } + if !sources.iter().any(|p| p == path) { + sources.push(path.to_path_buf()); + } + let directory = path.parent().unwrap_or(Path::new("")); + let this_directory = Located { + value: format!("{}/", display_path(directory)), + source: display_path(path), + conditional: false, + }; + let previous = properties.insert("msbuildthisfiledirectory".to_owned(), this_directory); + for step in &doc.steps { + match step { + Step::Property(index) => { + let (name, value, tests) = &doc.properties[*index]; + match conditions(tests, properties) { + Some(true) => { + let mut value = value.clone(); + value.conditional = false; + let value = + values::resolve(&value, properties, Syntax::MsBuild).unwrap_or(value); + properties.insert(name.clone(), value); + } + None => conditional.push((name.clone(), value.clone())), + Some(false) => (), + } + } + Step::Import(index) => { + let (import, tests) = &doc.imports[*index]; + if conditions(tests, properties) != Some(true) { + continue; + } + let raw = Located { + value: import.clone(), + source: display_path(path), + conditional: false, + }; + if let Some(resolved) = values::resolve(&raw, properties, Syntax::MsBuild) + && let Some(target) = values::child_file(directory, &resolved.value) + { + apply(&target, documents, properties, active, sources, conditional); + } + } + } + } + if let Some(previous) = previous { + properties.insert("msbuildthisfiledirectory".to_owned(), previous); + } else { + properties.remove("msbuildthisfiledirectory"); + } + active.remove(path); +} + +pub(super) fn parse( + nodes: &mut [Node], + catalog: &Catalog, + parsed: &Documents, + diagnostics: &mut Vec, +) { + if !nodes.iter().any(|node| { + node.files.iter().any(|file| { + catalog.matching(file).any(|manifest| { + manifest.package.as_deref() == Some("dotnet") + || manifest.workspace.as_deref() == Some("dotnet") + }) + }) + }) { + return; + } + let mut documents = BTreeMap::>::new(); + let mut sdk_versions = BTreeMap::>::new(); + for node in nodes.iter_mut() { + for file in &node.files { + if file == "global.json" { + sdk_versions.insert(node.path.clone(), BTreeMap::new()); + } + if catalog.reader(file, Reader::Msbuild) { + documents.insert(node.path.join(file), None); + } + } + for (file, text) in &node.contents { + let source = relative_file(&node.path, file); + if file == "global.json" { + sdk_versions.insert(node.path.clone(), BTreeMap::new()); + if let Some(doc) = parsed.get(&node.path).and_then(|docs| docs.get(file)) + && let Some(versions) = doc.get("msbuild-sdks").and_then(Value::as_object) + { + sdk_versions.insert( + node.path.clone(), + versions + .iter() + .filter_map(|(name, version)| { + version.as_str().map(|value| { + ( + name.to_ascii_lowercase(), + Located { + value: value.to_owned(), + source: source.clone(), + conditional: false, + }, + ) + }) + }) + .collect(), + ); + } + } else if let std::collections::btree_map::Entry::Occupied(mut entry) = + documents.entry(node.path.join(file)) + { + match document(text, &source, catalog) { + Ok(doc) + if catalog + .matching(file) + .any(|manifest| manifest.workspace.as_deref() == Some("dotnet")) + == doc.solution => + { + entry.insert(Some(doc)); + } + Ok(_) => diagnostics.push(Diagnostic { + path: source, + message: "XML root does not match the MSBuild manifest type".to_owned(), + }), + Err(error) => diagnostics.push(Diagnostic { + path: source, + message: error.to_string(), + }), + } + } + } + } + for node in nodes { + for file in node + .files + .iter() + .filter(|file| { + catalog + .matching(file) + .any(|manifest| manifest.workspace.as_deref() == Some("dotnet")) + }) + .cloned() + .collect::>() + { + let projects = if catalog.reader(&file, Reader::Msbuild) { + documents + .get(&node.path.join(&file)) + .and_then(Option::as_ref) + .map(|d| d.projects.clone()) + .unwrap_or_default() + } else { + node.contents + .get(&file) + .into_iter() + .flat_map(|text| text.lines()) + .filter(|line| line.trim_start().starts_with("Project(")) + .filter_map(|line| line.split_once('=')?.1.split('"').nth(3)) + .map(str::to_owned) + .collect() + }; + let members = projects + .iter() + .filter(|p| p.to_ascii_lowercase().ends_with(".csproj")) + .filter_map(|p| values::child_file(&node.path, p)) + .filter_map(|p| p.parent().map(Path::to_path_buf)) + .filter_map(|p| p.strip_prefix(&node.path).ok().map(display_path)) + .collect::>(); + if !members.is_empty() { + node.names.insert( + Path::new(&file) + .file_stem() + .map_or_else(String::new, |s| s.to_string_lossy().into_owned()), + ); + node.workspaces.push(Workspace { + ecosystem: "dotnet".to_owned(), + source: relative_file(&node.path, &file), + members, + exclude: Vec::new(), + }); + } + } + let projects = node + .files + .iter() + .filter(|file| { + catalog + .matching(file) + .any(|manifest| manifest.package.as_deref() == Some("dotnet")) + }) + .cloned() + .collect::>(); + for file in projects { + let path = node.path.join(&file); + let source = display_path(&path); + let Some(Some(_)) = documents.get(&path) else { + continue; + }; + let name = Path::new(&file) + .file_stem() + .map_or_else(String::new, |s| s.to_string_lossy().into_owned()); + node.packages.insert("dotnet".to_owned(), source.clone()); + node.names.insert(name.clone()); + let mut properties = Properties::new(); + for (key, value) in [ + ("msbuildprojectname", name.as_str()), + ("assemblyname", name.as_str()), + ] { + properties.insert( + key.to_owned(), + Located { + value: value.to_owned(), + source: source.clone(), + conditional: false, + }, + ); + } + let mut sources = Vec::new(); + let mut conditional = Vec::new(); + properties.insert( + "msbuildprojectextension".to_owned(), + Located { + value: ".csproj".to_owned(), + source: source.clone(), + conditional: false, + }, + ); + if let Some(path) = nearest(&node.path, "Directory.Build.props", &documents) { + apply( + &path, + &documents, + &mut properties, + &mut BTreeSet::new(), + &mut sources, + &mut conditional, + ); + } + apply( + &path, + &documents, + &mut properties, + &mut BTreeSet::new(), + &mut sources, + &mut conditional, + ); + if let Some(path) = nearest(&node.path, "Directory.Build.targets", &documents) { + apply( + &path, + &documents, + &mut properties, + &mut BTreeSet::new(), + &mut sources, + &mut conditional, + ); + } + let mut central = BTreeMap::new(); + if let Some(path) = nearest(&node.path, "Directory.Packages.props", &documents) + && !properties + .get("managepackageversionscentrally") + .is_some_and(|value| value.value.eq_ignore_ascii_case("false")) + { + let mut central_sources = Vec::new(); + apply( + &path, + &documents, + &mut properties, + &mut BTreeSet::new(), + &mut central_sources, + &mut conditional, + ); + if properties + .get("managepackageversionscentrally") + .is_some_and(|value| value.value.eq_ignore_ascii_case("false")) + { + central_sources.clear(); + } + for path in central_sources { + if let Some(Some(doc)) = documents.get(&path) { + for reference in &doc.references { + if catalog + .reference(&reference.kind) + .is_some_and(|reference| reference.catalog) + { + central.insert( + reference.name.to_ascii_lowercase(), + reference.version.as_ref(), + ); + } + } + } + } + } + for (field, raw) in catalog.property_values(PropertyReader::Msbuild, &properties) { + field.emit(node, raw, &properties, Syntax::MsBuild); + } + for (key, raw) in &conditional { + if let Some(field) = catalog.property(PropertyReader::Msbuild, key) { + field.emit(node, raw, &properties, Syntax::MsBuild); + } + } + for location in &sources { + let Some(Some(document)) = documents.get(location) else { + continue; + }; + for (sdk, version) in &document.sdk { + let raw = Located { + value: sdk.clone(), + source: display_path(location), + conditional: false, + }; + let Some(sdk) = values::resolve(&raw, &properties, Syntax::MsBuild) else { + continue; + }; + let sdk = sdk.value.to_ascii_lowercase(); + let version = version.as_ref().map(|v| Located { + value: v.clone(), + source: raw.source.clone(), + conditional: false, + }); + let inherited = node + .path + .ancestors() + .find_map(|p| sdk_versions.get(p)) + .and_then(|v| v.get(&sdk)); + values::dependency( + node, + "dotnet-sdk", + &sdk, + &raw.source, + version.as_ref().or(inherited), + &properties, + Syntax::MsBuild, + ); + } + for reference in &document.references { + let Some(policy) = catalog + .reference(&reference.kind) + .filter(|policy| !policy.catalog) + else { + continue; + }; + let name = policy.name(&reference.name); + let version = reference + .version + .as_ref() + .or_else(|| central.get(&name).copied().flatten()); + values::dependency( + node, + &policy.ecosystem, + &name, + &display_path(location), + version, + &properties, + Syntax::MsBuild, + ); + } + } + if let Some(raw) = properties + .get("packageid") + .or_else(|| properties.get("assemblyname")) + && let Some(package) = values::resolve(raw, &properties, Syntax::MsBuild) + { + node.names.insert(package.value.clone()); + values::identity( + node, + "nuget", + &package.value.to_ascii_lowercase(), + &source, + properties.get("version"), + &properties, + Syntax::MsBuild, + ); + } + } + } +} diff --git a/src/detect/gradle.rs b/src/detect/gradle.rs new file mode 100644 index 0000000..6777dc3 --- /dev/null +++ b/src/detect/gradle.rs @@ -0,0 +1,695 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Path, PathBuf}; + +use super::catalog::{Catalog, PropertyReader, Reader}; +use super::values::{self, Located, Properties, Syntax}; +use super::{Diagnostic, Node, Workspace, display_path, relative_file}; + +#[derive(Debug)] +enum Token<'a> { + Word(&'a str), + Text(String), + Symbol(char), + Break, +} + +fn tokens(text: &str) -> Result>, String> { + let bytes = text.as_bytes(); + let mut result = Vec::new(); + let mut i = 0; + while i < bytes.len() { + if bytes[i] == b'\n' { + result.push(Token::Break); + i += 1; + } else if bytes[i].is_ascii_whitespace() { + i += 1; + } else if bytes[i..].starts_with(b"//") { + while i < bytes.len() && bytes[i] != b'\n' { + i += 1; + } + } else if bytes[i..].starts_with(b"/*") { + i += 2; + let mut depth = 1; + while i < bytes.len() && depth > 0 { + if bytes[i..].starts_with(b"/*") { + depth += 1; + i += 2; + } else if bytes[i..].starts_with(b"*/") { + depth -= 1; + i += 2; + } else { + i += 1; + } + } + if depth != 0 { + return Err("unterminated Gradle comment".to_owned()); + } + } else if bytes[i..].starts_with(b"$/") { + let start = i + 2; + let Some(end) = text[start..].find("/$") else { + return Err("unterminated Gradle dollar-slashy string".to_owned()); + }; + result.push(Token::Text(text[start..start + end].to_owned())); + i = start + end + 2; + } else if bytes[i] == b'/' + && result.last().is_some_and(|t| { + matches!(t, Token::Symbol('~' | '=' | '(' | '[' | ',' | ':' | '!')) + || word(t) == Some("return") + }) + { + i += 1; + let start = i; + while i < bytes.len() && bytes[i] != b'/' { + if bytes[i] == b'\\' { + i += 1; + } + i += 1; + } + if i >= bytes.len() { + return Err("unterminated Gradle slashy string".to_owned()); + } + result.push(Token::Text(text[start..i].to_owned())); + i += 1; + } else if bytes[i] == b'\'' || bytes[i] == b'"' { + let quote = bytes[i]; + let triple = bytes[i..].starts_with(&[quote, quote, quote]); + i += if triple { 3 } else { 1 }; + let start = i; + while i < bytes.len() { + if triple && bytes[i..].starts_with(&[quote, quote, quote]) { + break; + } + if !triple && bytes[i] == quote { + break; + } + if !triple && bytes[i] == b'\\' { + i += 1; + } + i += 1; + } + if i >= bytes.len() { + return Err("unterminated Gradle string".to_owned()); + } + let value = &text[start..i]; + result.push(Token::Text( + value + .replace("\\'", "'") + .replace("\\\"", "\"") + .replace("\\\\", "\\"), + )); + i += if triple { 3 } else { 1 }; + } else if bytes[i].is_ascii_alphanumeric() || matches!(bytes[i], b'_' | b'$') { + let start = i; + while i < bytes.len() + && (bytes[i].is_ascii_alphanumeric() || matches!(bytes[i], b'_' | b'.' | b'$')) + { + i += 1; + } + result.push(Token::Word(&text[start..i])); + } else { + let Some(ch) = text[i..].chars().next() else { + break; + }; + result.push(Token::Symbol(ch)); + i += ch.len_utf8(); + } + } + Ok(result) +} + +fn word<'a>(token: &'a Token<'_>) -> Option<&'a str> { + match token { + Token::Word(word) => Some(word), + _ => None, + } +} + +fn argument(tokens: &[Token<'_>], mut index: usize) -> Option<(String, usize)> { + while matches!(tokens.get(index), Some(Token::Symbol('(' | '=' | ':'))) { + index += 1; + } + match tokens.get(index)? { + Token::Text(text) => Some((text.clone(), index + 1)), + Token::Word(word) if word.chars().all(|c| c.is_ascii_digit() || c == '.') => { + Some(((*word).to_owned(), index + 1)) + } + Token::Word(word) if word.starts_with("JavaVersion.VERSION_") => Some(( + word.trim_start_matches("JavaVersion.VERSION_") + .replace('_', "."), + index + 1, + )), + Token::Word(word) + if word.ends_with("JavaLanguageVersion.of") || *word == "JavaVersion.toVersion" => + { + argument(tokens, index + 1) + } + Token::Word(word) => Some((format!("${{{word}}}"), index + 1)), + _ => None, + } +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Scope { + Other, + All, + Subprojects, + Plugins, + Dependencies, + Management, + Buildscript, + Java, + Extra, +} + +struct Declaration { + value: Located, + shared: bool, + root: bool, +} +struct Plugin { + name: String, + version: Option, + active: bool, + shared: bool, + root: bool, +} +struct Dependency { + value: Located, + shared: bool, + root: bool, +} + +#[derive(Default)] +struct Script { + properties: BTreeMap, + plugins: Vec, + dependencies: Vec, + java: Vec<(String, Declaration)>, + modules: Vec, + directories: BTreeMap, + project_names: BTreeMap, + name: Option, + dynamic_includes: bool, +} + +fn script(text: &str, source: &str, catalog: &Catalog) -> Result { + let tokens = tokens(text)?; + let mut script = Script::default(); + let mut scopes = Vec::new(); + let mut lists = BTreeMap::>::new(); + for (index, token) in tokens.iter().enumerate() { + if matches!(token, Token::Symbol('{')) { + let previous = tokens[..index] + .iter() + .rev() + .take(32) + .take_while(|t| !matches!(t, Token::Break | Token::Symbol('{' | '}' | ';'))) + .filter_map(word) + .collect::>(); + let scope = if previous.contains(&"allprojects") { + Scope::All + } else if previous.contains(&"subprojects") { + Scope::Subprojects + } else if previous.contains(&"pluginManagement") + || previous.contains(&"dependencyManagement") + { + Scope::Management + } else if previous.contains(&"buildscript") { + Scope::Buildscript + } else if previous.contains(&"plugins") { + Scope::Plugins + } else if previous.contains(&"dependencies") { + Scope::Dependencies + } else if previous + .iter() + .any(|w| matches!(*w, "java" | "toolchain" | "JavaCompile")) + { + Scope::Java + } else if previous.contains(&"ext") { + Scope::Extra + } else { + Scope::Other + }; + scopes.push(scope); + continue; + } + if matches!(token, Token::Symbol('}')) { + scopes.pop(); + continue; + } + let Some(key) = word(token) else { + continue; + }; + let shared = scopes + .iter() + .any(|s| matches!(s, Scope::All | Scope::Subprojects)); + let root = !scopes.contains(&Scope::Subprojects); + let located = |value| Located { + value, + source: source.to_owned(), + conditional: false, + }; + let scalar_scope = scopes.is_empty() + || scopes.last().is_some_and(|s| { + matches!( + s, + Scope::All | Scope::Subprojects | Scope::Extra | Scope::Java + ) + }); + if matches!(tokens.get(index + 1), Some(Token::Symbol('='))) && scalar_scope { + if matches!(tokens.get(index + 2), Some(Token::Symbol('['))) { + let list = tokens[index + 3..] + .iter() + .take_while(|t| !matches!(t, Token::Symbol(']'))) + .filter_map(|t| { + if let Token::Text(v) = t { + Some(v.clone()) + } else { + None + } + }) + .collect(); + lists.insert(key.to_owned(), list); + } else if let Some((value, _)) = argument(&tokens, index + 1) { + if key == "rootProject.name" { + script.name = Some(value); + } else { + let declaration = Declaration { + value: located(value), + shared, + root, + }; + if catalog.property(PropertyReader::Gradle, key).is_some() { + script.java.push((key.to_owned(), declaration)); + } else { + script.properties.insert(key.to_owned(), declaration); + } + } + } + } + if catalog.property(PropertyReader::Gradle, key).is_some() + && matches!(tokens.get(index + 1), Some(Token::Symbol('('))) + && let Some((value, _)) = argument(&tokens, index + 1) + { + script.java.push(( + key.to_owned(), + Declaration { + value: located(value), + shared, + root, + }, + )); + } + if key == "include" || key == "includeBuild" { + let first = if matches!(tokens.get(index + 1), Some(Token::Symbol('('))) { + index + 2 + } else { + index + 1 + }; + if let Some(name) = tokens + .get(first) + .and_then(word) + .map(|name| name.strip_suffix(".toArray").unwrap_or(name)) + && let Some(list) = lists.get(name) + { + script.modules.extend( + list.iter() + .map(|v| v.trim_start_matches(':').replace(':', "/")), + ); + continue; + } + let mut cursor = index + 1; + while let Some((value, next)) = argument(&tokens, cursor) { + if value.contains('$') { + script.dynamic_includes = true; + break; + } + script + .modules + .push(value.trim_start_matches(':').replace(':', "/")); + cursor = next; + if matches!(tokens.get(cursor), Some(Token::Symbol(','))) { + cursor += 1; + } else { + break; + } + } + } + if (key.ends_with(".each") || key.ends_with(".forEach")) + && let Some(list) = + lists.get(key.trim_end_matches(".each").trim_end_matches(".forEach")) + && tokens[index + 1..] + .iter() + .take(40) + .filter_map(word) + .any(|w| w == "include") + { + script.modules.extend( + list.iter() + .map(|v| v.trim_start_matches(':').replace(':', "/")), + ); + } + if key == "project" + && let Some((project, end)) = argument(&tokens, index + 1) + { + let tail = &tokens[end..tokens.len().min(end + 12)]; + if let Some(name) = tail.iter().position(|t| word(t) == Some("name")) + && let Some((value, _)) = argument(tail, name + 1) + && !value.contains('$') + { + script + .project_names + .insert(project.trim_start_matches(':').replace(':', "/"), value); + } + if tail + .iter() + .filter_map(word) + .any(|w| w == ".projectDir" || w == "projectDir") + && let Some(file) = tail.iter().position(|t| word(t) == Some("file")) + && let Some((path, _)) = argument(tail, file + 1) + { + script + .directories + .insert(project.trim_start_matches(':').replace(':', "/"), path); + } + } + let managed = scopes.contains(&Scope::Management) || scopes.contains(&Scope::Buildscript); + if key == "id" + && scopes.last() == Some(&Scope::Plugins) + && let Some((name, end)) = argument(&tokens, index + 1) + { + let tail = &tokens[end..tokens.len().min(end + 8)]; + let version = tail + .iter() + .position(|t| word(t) == Some("version")) + .and_then(|i| argument(tail, i + 1)) + .map(|(value, _)| located(value)); + let inactive = tail + .windows(2) + .any(|w| word(&w[0]) == Some("apply") && word(&w[1]) == Some("false")); + script.plugins.push(Plugin { + name, + version, + active: !managed && !inactive, + shared, + root, + }); + } + if key == "apply" && !managed { + let mut cursor = index + 1; + if matches!(tokens.get(cursor), Some(Token::Symbol('('))) { + cursor += 1; + } + if word(tokens.get(cursor).unwrap_or(token)) == Some("plugin") + && let Some((name, _)) = argument(&tokens, cursor + 1) + { + script.plugins.push(Plugin { + name, + version: None, + active: true, + shared, + root, + }); + } + } + if scopes.contains(&Scope::Dependencies) + && !managed + && matches!( + key, + "implementation" + | "api" + | "compile" + | "compileOnly" + | "runtimeOnly" + | "runtime" + | "annotationProcessor" + ) + && let Some((value, _)) = argument(&tokens, index + 1) + { + script.dependencies.push(Dependency { + value: located(value), + shared, + root, + }); + } + } + Ok(script) +} + +fn build_file(path: &Path) -> bool { + let Some(file) = path.file_name().and_then(|f| f.to_str()) else { + return false; + }; + let directory = path + .parent() + .and_then(Path::file_name) + .and_then(|f| f.to_str()) + .unwrap_or(""); + matches!(file, "build.gradle" | "build.gradle.kts") + || file == format!("{directory}.gradle") + || file == format!("{directory}.gradle.kts") +} + +pub(super) fn parse(nodes: &mut Vec, catalog: &Catalog, diagnostics: &mut Vec) { + let mut scripts = BTreeMap::::new(); + let mut property_files = BTreeMap::::new(); + let mut daemon_versions = BTreeMap::::new(); + for node in nodes.iter() { + for (file, text) in &node.contents { + let source = relative_file(&node.path, file); + if catalog.reader(file, Reader::Gradle) { + match script(text, &source, catalog) { + Ok(script) => { + if matches!(file.as_str(), "settings.gradle" | "settings.gradle.kts") + && script.dynamic_includes + { + diagnostics.push(Diagnostic { path: source.clone(), message: "dynamic Gradle include declarations are not evaluated; projects with manifests are still discovered".to_owned() }); + } + scripts.insert(node.path.join(file), script); + } + Err(message) => diagnostics.push(Diagnostic { + path: source, + message, + }), + } + } else if catalog.reader(file, Reader::GradleJvm) { + if let Some(version) = values::properties(text, &source).get("toolchainVersion") { + daemon_versions.insert( + node.path.parent().unwrap_or(Path::new("")).to_path_buf(), + version.clone(), + ); + } + } else if catalog.reader(file, Reader::GradleProperties) { + property_files.insert(node.path.clone(), values::properties(text, &source)); + } + } + } + if scripts.is_empty() { + return; + } + let settings = scripts + .iter() + .filter(|(path, _)| { + path.file_name() + .is_some_and(|f| f == "settings.gradle" || f == "settings.gradle.kts") + }) + .map(|(path, script)| { + ( + path.parent().unwrap_or(Path::new("")).to_path_buf(), + (path, script), + ) + }) + .collect::>(); + let mut builds = BTreeMap::>::new(); + for (path, script) in &scripts { + if build_file(path) { + builds + .entry(path.parent().unwrap_or(Path::new("")).to_path_buf()) + .or_default() + .push((path, script)); + } + } + let mut declared = BTreeMap::::new(); + let mut declared_names = BTreeMap::::new(); + for (path, script) in settings.values() { + let base = path.parent().unwrap_or(Path::new("")); + for module in script.modules.iter().chain(script.project_names.keys()) { + let directory = script.directories.get(module).unwrap_or(module); + if let Some(directory) = values::child_file(base, directory) { + let name = script + .project_names + .get(module) + .cloned() + .unwrap_or_else(|| module.rsplit('/').next().unwrap_or(module).to_owned()); + declared_names.insert(directory.clone(), name); + if script.modules.contains(module) { + declared.insert(directory, display_path(path)); + } + } + } + } + let existing = nodes + .iter() + .map(|n| n.path.clone()) + .collect::>(); + let synthetic = declared + .keys() + .filter(|path| { + !existing.contains(*path) + && existing + .range((*path).clone()..) + .next() + .is_some_and(|candidate| candidate.starts_with(path)) + }) + .cloned() + .collect::>(); + for path in synthetic { + nodes.push(Node { + path, + ..Node::default() + }); + } + for node in nodes.iter_mut() { + let build = builds.get(&node.path).map(Vec::as_slice).unwrap_or(&[]); + let settings_here = settings.get(&node.path); + let module_source = declared.get(&node.path); + if build.is_empty() && settings_here.is_none() && module_source.is_none() { + continue; + } + let source = build + .first() + .map(|(path, _)| display_path(path)) + .or_else(|| module_source.cloned()) + .or_else(|| settings_here.map(|(path, _)| display_path(path))) + .unwrap_or_default(); + node.packages.insert("gradle".to_owned(), source.clone()); + let root = node + .path + .ancestors() + .find_map(|parent| settings.get(parent)); + let base = root + .map(|(path, _)| path.parent().unwrap_or(Path::new(""))) + .map(Path::to_path_buf) + .unwrap_or_else(|| node.path.clone()); + if let Some(raw) = daemon_versions.get(&base) { + values::version( + node, + "java", + raw, + "build-toolchain", + &Properties::new(), + Syntax::Maven, + ); + } + let mut properties = Properties::new(); + for parent in node.path.ancestors().collect::>().into_iter().rev() { + if let Some(found) = property_files.get(parent) { + properties.extend(found.iter().map(|(k, v)| (k.clone(), v.clone()))); + } + } + let root_build = builds.get(&base).map(Vec::as_slice).unwrap_or(&[]); + let mut declarations = root_build + .iter() + .map(|(path, script)| (*path, *script, base != node.path)) + .collect::>(); + if base != node.path { + declarations.extend(build.iter().map(|(path, script)| (*path, *script, false))); + } + for (_, script, inherited) in &declarations { + for (name, declaration) in &script.properties { + if (!*inherited && declaration.root) || (*inherited && declaration.shared) { + properties.insert(name.clone(), declaration.value.clone()); + } + } + } + let name = settings_here + .and_then(|(_, script)| script.name.clone()) + .or_else(|| declared_names.get(&node.path).cloned()) + .or_else(|| { + node.path + .file_name() + .map(|s| s.to_string_lossy().into_owned()) + }); + if let Some(name) = &name { + node.names.insert(name.clone()); + } + for (path, script, inherited) in declarations { + for (kind, declaration) in &script.java { + if ((!inherited && declaration.root) || (inherited && declaration.shared)) + && let Some(field) = catalog.property(PropertyReader::Gradle, kind) + { + field.emit(node, &declaration.value, &properties, Syntax::Maven); + } + } + for plugin in &script.plugins { + if plugin.active && ((!inherited && plugin.root) || (inherited && plugin.shared)) { + values::dependency( + node, + "gradle-plugin", + &plugin.name, + &display_path(path), + plugin.version.as_ref(), + &properties, + Syntax::Maven, + ); + } + } + for dependency in &script.dependencies { + if !((!inherited && dependency.root) || (inherited && dependency.shared)) { + continue; + } + let Some(coordinate) = + values::resolve(&dependency.value, &properties, Syntax::Maven) + else { + continue; + }; + let parts = coordinate.value.split(':').collect::>(); + if parts.len() < 2 || parts[0].is_empty() || parts[1].is_empty() { + continue; + } + let version = parts.get(2).map(|version| Located { + value: (*version).to_owned(), + source: coordinate.source.clone(), + conditional: false, + }); + values::dependency( + node, + "maven", + &format!("{}:{}", parts[0], parts[1]), + &display_path(path), + version.as_ref(), + &properties, + Syntax::Maven, + ); + } + } + if let Some(name) = name + && let Some(group) = properties + .get("group") + .and_then(|v| values::resolve(v, &properties, Syntax::Maven)) + { + values::identity( + node, + "maven", + &format!("{}:{name}", group.value), + &source, + properties.get("version"), + &properties, + Syntax::Maven, + ); + } + if let Some((path, settings)) = settings_here { + let members = settings + .modules + .iter() + .map(|module| settings.directories.get(module).unwrap_or(module).clone()) + .collect(); + node.workspaces.push(Workspace { + ecosystem: "gradle".to_owned(), + source: display_path(path), + members, + exclude: Vec::new(), + }); + } + } +} diff --git a/src/detect/java.rs b/src/detect/java.rs new file mode 100644 index 0000000..76e7f2c --- /dev/null +++ b/src/detect/java.rs @@ -0,0 +1,359 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Path, PathBuf}; + +use anyhow::{Result, bail}; + +use super::catalog::{Catalog, PropertyReader}; +use super::values::{self, Located, Properties, Syntax}; +use super::{Diagnostic, Node, Workspace, display_path, relative_file, xml}; + +#[derive(Default)] +struct Artifact { + group: String, + name: String, + version: Option, + scope: String, + kind: String, + compiler: Vec<(String, Located)>, +} + +#[derive(Default)] +struct Pom { + group: String, + name: String, + version: String, + parent: Option, + parent_path: Option, + properties: Properties, + dependencies: Vec, + managed: Vec, + plugins: Vec, + modules: Vec, +} + +fn pom(text: &str, source: &str, catalog: &Catalog) -> Result { + let mut doc = Pom::default(); + let mut artifacts = BTreeMap::::new(); + xml::read(text, |stack| { + if stack.first().is_none_or(|e| e.name != "project") { + bail!("expected a Maven project root"); + } + let Some(element) = stack.last() else { + return Ok(()); + }; + let is = |path: &[&str]| stack.iter().map(|e| e.name).eq(path.iter().copied()); + let located = || Located { + value: element.text.trim().to_owned(), + source: source.to_owned(), + conditional: false, + }; + if stack.len() == 2 { + match element.name { + "groupId" => doc.group = element.text.trim().to_owned(), + "artifactId" => doc.name = element.text.trim().to_owned(), + "version" => doc.version = element.text.trim().to_owned(), + _ => (), + } + } + if stack.len() == 3 && stack[1].name == "parent" { + let parent = doc.parent.get_or_insert_with(Artifact::default); + match element.name { + "groupId" => parent.group = element.text.trim().to_owned(), + "artifactId" => parent.name = element.text.trim().to_owned(), + "version" => parent.version = Some(located()), + "relativePath" => doc.parent_path = Some(element.text.trim().to_owned()), + _ => (), + } + } + if stack.len() == 3 && stack[1].name == "properties" { + doc.properties.insert(element.name.to_owned(), located()); + } + if is(&["project", "modules", "module"]) { + doc.modules.push(element.text.trim().to_owned()); + } + if let Some(parent) = stack.get(stack.len().saturating_sub(2)) + && matches!(parent.name, "dependency" | "plugin") + { + let artifact = artifacts.entry(stack.len() - 1).or_default(); + match element.name { + "groupId" => artifact.group = element.text.trim().to_owned(), + "artifactId" => artifact.name = element.text.trim().to_owned(), + "version" => artifact.version = Some(located()), + "scope" => artifact.scope = element.text.trim().to_owned(), + "type" => artifact.kind = element.text.trim().to_owned(), + _ => (), + } + } + if let Some(plugin) = stack.iter().position(|e| e.name == "plugin") + && catalog + .property(PropertyReader::MavenCompiler, element.name) + .is_some() + && stack + .get(stack.len().saturating_sub(2)) + .is_some_and(|e| e.name == "configuration") + { + artifacts + .entry(plugin + 1) + .or_default() + .compiler + .push((element.name.to_owned(), located())); + } + if element.name == "dependency" { + let artifact = artifacts.remove(&stack.len()).unwrap_or_default(); + if is(&["project", "dependencies", "dependency"]) && artifact.scope != "test" { + doc.dependencies.push(artifact); + } else if is(&[ + "project", + "dependencyManagement", + "dependencies", + "dependency", + ]) { + doc.managed.push(artifact); + } + } + if element.name == "plugin" { + let mut artifact = artifacts.remove(&stack.len()).unwrap_or_default(); + if artifact.group.is_empty() { + artifact.group = "org.apache.maven.plugins".to_owned(); + } + if is(&["project", "build", "plugins", "plugin"]) { + doc.plugins.push(artifact); + } + } + Ok(()) + })?; + if doc.name.is_empty() { + bail!("Maven project artifactId is missing"); + } + Ok(doc) +} + +fn chain<'a>( + path: &Path, + documents: &'a BTreeMap>, + seen: &mut BTreeSet, + output: &mut Vec<(PathBuf, &'a Pom)>, +) { + let Some(Some(doc)) = documents.get(path) else { + return; + }; + if seen.len() == 64 || !seen.insert(path.to_path_buf()) { + return; + } + if let Some(parent) = &doc.parent { + let relative = doc.parent_path.as_deref().unwrap_or("../pom.xml"); + if !relative.is_empty() + && let Some(target) = + values::child_file(path.parent().unwrap_or(Path::new("")), relative) + && let Some(Some(local)) = documents.get(&target) + && local.name == parent.name + && (if local.group.is_empty() { + local.parent.as_ref().map(|p| p.group.as_str()) + } else { + Some(local.group.as_str()) + }) == Some(parent.group.as_str()) + && parent.version.as_ref().is_none_or(|requested| { + let actual = if local.version.is_empty() { + local + .parent + .as_ref() + .and_then(|p| p.version.as_ref()) + .map(|v| v.value.as_str()) + } else { + Some(local.version.as_str()) + }; + actual == Some(requested.value.as_str()) + }) + { + chain(&target, documents, seen, output); + } + } + output.push((path.to_path_buf(), doc)); +} + +fn literal(value: &str, source: &str) -> Located { + Located { + value: value.to_owned(), + source: source.to_owned(), + conditional: false, + } +} + +fn resolved(value: &str, source: &str, properties: &Properties) -> Option { + values::resolve(&literal(value, source), properties, Syntax::Maven).map(|v| v.value) +} + +pub(super) fn parse(nodes: &mut Vec, catalog: &Catalog, diagnostics: &mut Vec) { + let mut documents = BTreeMap::>::new(); + for node in nodes.iter() { + if node.files.contains("pom.xml") { + documents.insert(node.path.join("pom.xml"), None); + } + if let Some(text) = node.contents.get("pom.xml") { + let source = relative_file(&node.path, "pom.xml"); + match pom(text, &source, catalog) { + Ok(doc) => { + documents.insert(node.path.join("pom.xml"), Some(doc)); + } + Err(error) => diagnostics.push(Diagnostic { + path: source, + message: error.to_string(), + }), + } + } + } + if documents.is_empty() { + super::gradle::parse(nodes, catalog, diagnostics); + return; + } + for node in nodes.iter_mut() { + let path = node.path.join("pom.xml"); + let Some(Some(doc)) = documents.get(&path) else { + continue; + }; + node.packages + .insert("maven".to_owned(), display_path(&path)); + let mut inherited = Vec::new(); + chain(&path, &documents, &mut BTreeSet::new(), &mut inherited); + let mut properties = Properties::new(); + let mut managed = BTreeMap::new(); + for (path, parent) in &inherited { + let source = display_path(path); + properties.extend( + parent + .properties + .iter() + .map(|(k, v)| (k.clone(), v.clone())), + ); + let group = if parent.group.is_empty() { + parent.parent.as_ref().map_or("", |p| p.group.as_str()) + } else { + &parent.group + }; + let version = if parent.version.is_empty() { + parent + .parent + .as_ref() + .and_then(|p| p.version.as_ref()) + .map_or("", |v| v.value.as_str()) + } else { + &parent.version + }; + for (key, value) in [ + ("project.groupId", group), + ("pom.groupId", group), + ("project.artifactId", parent.name.as_str()), + ("pom.artifactId", parent.name.as_str()), + ("project.version", version), + ("pom.version", version), + ] { + properties.insert(key.to_owned(), literal(value, &source)); + } + for dependency in &parent.managed { + if let (Some(group), Some(name)) = ( + resolved(&dependency.group, &source, &properties), + resolved(&dependency.name, &source, &properties), + ) { + managed.insert(format!("{group}:{name}"), dependency.version.as_ref()); + } + } + } + let source = display_path(&path); + let group = if doc.group.is_empty() { + doc.parent.as_ref().map_or("", |p| p.group.as_str()) + } else { + &doc.group + }; + let group = resolved(group, &source, &properties).unwrap_or_else(|| group.to_owned()); + node.names.insert(doc.name.clone()); + if !group.is_empty() { + node.names.insert(format!("{group}:{}", doc.name)); + let version = if doc.version.is_empty() { + doc.parent + .as_ref() + .and_then(|p| p.version.as_ref()) + .cloned() + } else { + Some(literal(&doc.version, &source)) + }; + values::identity( + node, + "maven", + &format!("{group}:{}", doc.name), + &source, + version.as_ref(), + &properties, + Syntax::Maven, + ); + } + for (field, raw) in catalog.property_values(PropertyReader::Maven, &properties) { + field.emit(node, raw, &properties, Syntax::Maven); + } + for (path, parent) in &inherited { + let source = display_path(path); + if let Some(parent) = &parent.parent { + values::dependency( + node, + "maven-parent", + &format!("{}:{}", parent.group, parent.name), + &source, + parent.version.as_ref(), + &properties, + Syntax::Maven, + ); + } + for (ecosystem, dependencies) in [ + ("maven", &parent.dependencies), + ("maven-plugin", &parent.plugins), + ] { + for artifact in dependencies { + let (Some(group), Some(name)) = ( + resolved(&artifact.group, &source, &properties), + resolved(&artifact.name, &source, &properties), + ) else { + continue; + }; + let key = format!("{group}:{name}"); + let version = artifact + .version + .as_ref() + .or_else(|| managed.get(&key).copied().flatten()); + values::dependency( + node, + ecosystem, + &key, + &source, + version, + &properties, + Syntax::Maven, + ); + if artifact.name == "maven-compiler-plugin" { + for (kind, raw) in &artifact.compiler { + if let Some(field) = + catalog.property(PropertyReader::MavenCompiler, kind) + { + field.emit(node, raw, &properties, Syntax::Maven); + } + } + } + } + } + } + if !doc.modules.is_empty() { + let members = doc + .modules + .iter() + .filter_map(|module| resolved(module, &source, &properties)) + .filter_map(|module| values::child_file(Path::new(""), &module)) + .map(|path| display_path(&path)) + .collect(); + node.workspaces.push(Workspace { + ecosystem: "maven".to_owned(), + source, + members, + exclude: Vec::new(), + }); + } + } + super::gradle::parse(nodes, catalog, diagnostics); +} diff --git a/src/detect/jsonc.rs b/src/detect/jsonc.rs new file mode 100644 index 0000000..84fec8b --- /dev/null +++ b/src/detect/jsonc.rs @@ -0,0 +1,200 @@ +pub(super) fn parse(text: &str) -> Result { + if let Ok(value) = serde_json::from_str(text) { + return Ok(value); + } + + let mut bytes = text.as_bytes().to_vec(); + let mut i = 0; + while i < bytes.len() { + match bytes[i] { + b'"' => i = string_end(&bytes, i), + b'/' if bytes.get(i + 1) == Some(&b'/') => { + while i < bytes.len() && !matches!(bytes[i], b'\r' | b'\n') { + bytes[i] = b' '; + i += 1; + } + } + b'/' if bytes.get(i + 1) == Some(&b'*') => { + let start = i; + i += 2; + while i + 1 < bytes.len() && &bytes[i..i + 2] != b"*/" { + i += 1; + } + if i + 1 >= bytes.len() { + let line = text.as_bytes()[..start] + .iter() + .filter(|&&byte| byte == b'\n') + .count() + + 1; + let column = text.as_bytes()[..start] + .iter() + .rposition(|&byte| byte == b'\n') + .map_or(start + 1, |newline| start - newline); + return Err(format!( + "unterminated block comment at line {line} column {column}" + )); + } + i += 2; + for byte in &mut bytes[start..i] { + if !matches!(*byte, b'\r' | b'\n') { + *byte = b' '; + } + } + } + _ => i += 1, + } + } + + i = 0; + let mut previous = None; + while i < bytes.len() { + match bytes[i] { + b'"' => { + i = string_end(&bytes, i); + previous = Some(b'"'); + continue; + } + b',' => { + let next = bytes[i + 1..].iter().find(|&&byte| !is_whitespace(byte)); + if matches!(next, Some(b'}' | b']')) + && !matches!(previous, None | Some(b'[' | b'{' | b',' | b':')) + { + bytes[i] = b' '; + } + previous = Some(b','); + } + byte if !is_whitespace(byte) => previous = Some(byte), + _ => {} + } + i += 1; + } + + serde_json::from_slice(&bytes).map_err(|error| error.to_string()) +} + +fn string_end(bytes: &[u8], start: usize) -> usize { + let mut i = start + 1; + while i < bytes.len() { + match bytes[i] { + b'\\' => i += 2, + b'"' => return i + 1, + _ => i += 1, + } + } + bytes.len() +} + +fn is_whitespace(byte: u8) -> bool { + matches!(byte, b' ' | b'\t' | b'\r' | b'\n') +} + +#[cfg(test)] +mod tests { + use super::parse; + use serde_json::json; + + #[test] + fn parses_nested_comments_and_trailing_commas() { + let text = r#"// config + { + /* Unicode: 日本語 🦀 */ + "tasks": {"dev": "deno run main.ts", /* task */}, + "values": [1, {"nested": [true, null,],},], + } // end"#; + assert_eq!( + parse(text).unwrap(), + json!({ + "tasks": {"dev": "deno run main.ts"}, + "values": [1, {"nested": [true, null]}] + }) + ); + } + + #[test] + fn preserves_strings_and_escapes() { + let expected = json!({ + "url": "https://example.com/日本語/*path*/?q=//", + "markers": "/* not a comment */ // still a string ,] ,}", + "escaped": "a\"//b\\\"/*c*/", + "backslash": "ends with \\", + }); + let plain = serde_json::to_string(&expected).unwrap(); + assert_eq!(parse(&plain).unwrap(), expected); + assert_eq!(parse(&format!("/* 🦀 */{plain}// end")).unwrap(), expected); + } + + #[test] + fn accepts_json_primitives_and_empty_containers() { + for text in [ + "null", + "true", + "false", + "42", + "-1.25e2", + r#""🦀""#, + "[]", + "{}", + ] { + let expected: serde_json::Value = serde_json::from_str(text).unwrap(); + assert_eq!(parse(text).unwrap(), expected); + assert_eq!( + parse(&format!("/* before */{text}// after")).unwrap(), + expected + ); + } + assert_eq!(parse("[1,// comment\r\n]").unwrap(), json!([1])); + } + + #[test] + fn rejects_malformed_comments_and_json() { + for text in [ + "/*", + "/**", + "/* no end 🦀", + "{} /* no end", + "/* outer /* inner */ */", + "[,]", + "{,}", + "[/* comment */,]", + "[1,,]", + "[1,/* c */,]", + r#"{"a":1,,}"#, + r#"{"a":,}"#, + "[1,}", + "[true false,]", + "[01,]", + "[NaN,]", + "[1\u{a0},]", + "// only comment", + "/* only */", + r#"{"a":"unterminated //}"#, + r#"["bad\q",]"#, + r#"["\uD800",]"#, + "[\"raw\nnewline\",]", + "[\"\0\",]", + "{} garbage", + ] { + assert!(parse(text).is_err(), "accepted {text:?}"); + } + } + + #[test] + fn preserves_diagnostic_byte_columns_and_newlines() { + let text = "/* 🦀\r\n日本語 */\r\n[1, /*é*/ @,]"; + let normalized = " \r\n \r\n[1, @ ]"; + assert_eq!(text.len(), normalized.len()); + let expected = serde_json::from_str::(normalized) + .unwrap_err() + .to_string(); + assert_eq!(parse(text).unwrap_err(), expected); + } + + #[test] + fn input_is_utf8_and_comment_masking_keeps_it_valid() { + assert!(String::from_utf8(vec![0xff]).is_err()); + assert_eq!( + parse("/* é日本🦀 */[\"é日本🦀\",]").unwrap(), + json!(["é日本🦀"]) + ); + } +} diff --git a/src/detect/jvm_dotnet_tests.rs b/src/detect/jvm_dotnet_tests.rs new file mode 100644 index 0000000..c1b9992 --- /dev/null +++ b/src/detect/jvm_dotnet_tests.rs @@ -0,0 +1,564 @@ +use super::tests::{detects, fixture, project, version}; +use super::{FrameworkRole, scan}; + +#[test] +fn bare_java_and_csharp_sources_are_never_read() { + for (file, language) in [ + ("Main.java", "java"), + ("Program.cs", "csharp"), + ("script.csx", "csharp"), + ] { + let dir = fixture(&[(file, "not valid source")]); + let report = scan(dir.path(), None).unwrap(); + assert_eq!(report.stats.files_read, 0); + assert_eq!(report.projects[0].languages[0].name, language); + assert!(report.projects[0].languages[0].versions.is_empty()); + } +} + +#[test] +fn dotnet_web_targets_language_and_sdk_versions_are_distinct() { + let dir = fixture(&[ + ( + "global.json", + r#"{"sdk":{"version":"10.0.403","rollForward":"latestFeature"}}"#, + ), + ( + "Directory.Build.props", + "net10.014.0", + ), + ( + "src/Web/Web.csproj", + "Shop.Web", + ), + ("src/Web/Program.cs", "source must not be read"), + ]); + let report = scan(dir.path(), None).unwrap(); + assert!(report.diagnostics.is_empty(), "{report:#?}"); + let web = project(&report, "src/Web"); + assert!(detects(web, "aspnetcore")); + assert!(version(web, "csharp", "net10.0", "Directory.Build.props")); + assert!(version(web, "csharp", "14.0", "Directory.Build.props")); + assert!(version(web, "csharp", "10.0.403", "global.json")); + let versions = &web + .languages + .iter() + .find(|l| l.name == "csharp") + .unwrap() + .versions; + assert!( + versions + .iter() + .any(|v| v.value == "net10.0" && v.kind == "target") + ); + assert!( + versions + .iter() + .any(|v| v.value == "14.0" && v.kind == "language") + ); + assert!( + versions + .iter() + .any(|v| v.value == "10.0.403" && v.kind == "sdk") + ); + assert_eq!(report.stats.files_read, 3); +} + +#[test] +fn msbuild_import_order_defaults_and_central_package_versions() { + let dir = fixture(&[ + ( + "Directory.Build.props", + "net10.014.0", + ), + ( + "eng/base.props", + "13.0", + ), + ( + "Directory.Packages.props", + "8.3.0", + ), + ( + "src/Web/Web.csproj", + "2.0", + ), + ]); + let report = scan(dir.path(), None).unwrap(); + assert!(report.diagnostics.is_empty(), "{report:#?}"); + let web = project(&report, "src/Web"); + assert!(version(web, "csharp", "net10.0", "Directory.Build.props")); + assert!(version(web, "csharp", "14.0", "Directory.Build.props")); + assert!(!version(web, "csharp", "13.0", "eng/base.props")); + let detection = web + .detections + .iter() + .find(|d| d.id == "fastendpoints") + .unwrap(); + assert_eq!(detection.roles, [FrameworkRole::Uses]); + assert!( + detection + .evidence + .iter() + .any(|e| e.source == "Directory.Packages.props" && e.value.as_deref() == Some("8.3.0")) + ); + assert!( + detection + .evidence + .iter() + .any(|e| e.source == "src/Web/Web.csproj") + ); +} + +#[test] +fn dotnet_solutions_find_members_without_counting_solution_folders() { + for (file, content) in [ + ( + "App.sln", + "Project(\"TYPE\") = \"Web, API\", \"src\\Web\\Web.csproj\", \"ID\"\nEndProject\nProject(\"FOLDER\") = \"src\", \"src\", \"ID2\"\nEndProject\n", + ), + ( + "App.slnx", + "", + ), + ] { + let dir = fixture(&[ + (file, content), + ( + "src/Web/Web.csproj", + "", + ), + ]); + let report = scan(dir.path(), None).unwrap(); + assert!(report.diagnostics.is_empty(), "{report:#?}"); + assert!(detects(project(&report, "."), "dotnet-workspace")); + assert_eq!(project(&report, ".").workspace_members, ["src/Web"]); + } +} + +#[test] +fn dotnet_target_frameworks_do_not_invent_a_csharp_language_version() { + let dir = fixture(&[( + "App.csproj", + "net8.0;net10.0", + )]); + let report = scan(dir.path(), None).unwrap(); + let p = project(&report, "."); + assert!(version(p, "csharp", "net8.0", "App.csproj")); + assert!(version(p, "csharp", "net10.0", "App.csproj")); + assert!(p.languages[0].versions.iter().all(|v| v.kind != "language")); + assert!(!detects(p, "aspnetcore")); +} + +#[test] +fn msbuild_unknown_conditions_and_property_cycles_are_not_executed() { + let dir = fixture(&[( + "App.csproj", + "$(B)$(A)$(A)14.0", + )]); + let report = scan(dir.path(), None).unwrap(); + let p = project(&report, "."); + assert!( + p.languages[0] + .versions + .iter() + .any(|v| v.kind == "reference") + ); + assert!( + p.languages[0] + .versions + .iter() + .any(|v| v.kind == "conditional-language") + ); + assert!(p.languages[0].versions.iter().all(|v| v.kind != "target")); +} + +#[test] +fn maven_spring_boot_parent_and_java_release_are_recognized() { + let dir = fixture(&[( + "pom.xml", + "org.springframework.bootspring-boot-starter-parent4.1.0petclinic17${java.version}", + )]); + let report = scan(dir.path(), None).unwrap(); + assert!(report.diagnostics.is_empty(), "{report:#?}"); + let p = project(&report, "."); + assert!(detects(p, "maven")); + assert!(detects(p, "spring-boot")); + assert!(version(p, "java", "17", "pom.xml")); + assert!( + p.languages[0] + .versions + .iter() + .any(|v| v.value == "17" && v.kind == "release") + ); +} + +#[test] +fn maven_modules_inherit_local_properties_and_managed_versions() { + let dir = fixture(&[ + ( + "pom.xml", + "exampleplatform1214.0.0apiorg.springframework.bootspring-boot-starter-web${boot.version}", + ), + ( + "api/pom.xml", + "exampleplatform1apiorg.springframework.bootspring-boot-starter-web", + ), + ]); + let report = scan(dir.path(), None).unwrap(); + assert!(report.diagnostics.is_empty(), "{report:#?}"); + assert_eq!(project(&report, ".").workspace_members, ["api"]); + assert!(!detects(project(&report, "."), "spring-boot")); + let api = project(&report, "api"); + assert!(detects(api, "spring-boot")); + assert!(version(api, "java", "21", "pom.xml")); + let boot = api + .detections + .iter() + .find(|d| d.id == "spring-boot") + .unwrap(); + assert!( + boot.evidence + .iter() + .any(|e| e.source == "pom.xml" && e.value.as_deref() == Some("4.0.0")) + ); +} + +#[test] +fn maven_test_dependencies_and_plugin_management_are_not_runtime_frameworks() { + let dir = fixture(&[( + "pom.xml", + "plainorg.springframework.bootspring-boot-starter-webtestorg.springframework.bootspring-boot-maven-plugin", + )]); + let report = scan(dir.path(), None).unwrap(); + assert!(!detects(project(&report, "."), "spring-boot")); +} + +#[test] +fn xml_entities_cdata_namespaces_and_structure_are_checked() { + let dir = fixture(&[( + "App.csproj", + "net10.0A&B", + )]); + let report = scan(dir.path(), None).unwrap(); + assert!(report.diagnostics.is_empty(), "{report:#?}"); + assert!(version( + project(&report, "."), + "csharp", + "net10.0", + "App.csproj" + )); + assert!(project(&report, ".").names.contains(&"A&B".to_owned())); + for broken in [ + "", + "", + "", + "]>&x;", + ] { + let dir = fixture(&[("App.csproj", broken)]); + let report = scan(dir.path(), None).unwrap(); + assert_eq!(report.diagnostics.len(), 1, "{broken}: {report:#?}"); + assert_eq!(report.stats.files_read, 1); + } +} + +#[test] +fn gradle_groovy_and_kotlin_plugins_and_toolchains_are_static_facts() { + for (file, content) in [ + ( + "build.gradle", + "plugins { id 'org.springframework.boot' version '4.0.0' }\ngroup = 'example'\njava { toolchain { languageVersion = JavaLanguageVersion.of(21) } }\ndependencies { implementation 'org.springframework:spring-web:7.0.0' }\n", + ), + ( + "build.gradle.kts", + "plugins { id(\"org.springframework.boot\") version \"4.0.0\" }\ngroup = \"example\"\njava { toolchain { languageVersion.set(JavaLanguageVersion.of(21)) } }\ndependencies { implementation(\"org.springframework:spring-web:7.0.0\") }\n", + ), + ] { + let dir = fixture(&[(file, content)]); + let report = scan(dir.path(), None).unwrap(); + assert!(report.diagnostics.is_empty(), "{file}: {report:#?}"); + let p = project(&report, "."); + assert!(detects(p, "spring-boot")); + assert!(detects(p, "spring")); + assert!(version(p, "java", "21", file)); + } +} + +#[test] +fn gradle_inactive_plugins_comments_and_test_dependencies_do_not_match() { + let dir = fixture(&[( + "build.gradle", + "plugins { id 'org.springframework.boot' version '4.0.0' apply false }\n// plugins { id 'org.springframework.boot' }\ndependencies { testImplementation 'org.springframework:spring-web:7.0.0' }\ndef description = \"org.springframework.boot\"\n", + )]); + let report = scan(dir.path(), None).unwrap(); + assert!(!detects(project(&report, "."), "spring-boot")); + assert!(!detects(project(&report, "."), "spring")); +} + +#[test] +fn gradle_settings_literal_lists_and_source_only_modules_form_workspaces() { + let dir = fixture(&[ + ( + "settings.gradle", + "rootProject.name='platform'\nList projects = ['api', 'worker']\ninclude projects.toArray(new String[0])\n", + ), + ("build.gradle", "allprojects { group = 'example' }\n"), + ("api/src/main/java/Main.java", "not read"), + ("worker/build.gradle", "plugins { id 'java' }\n"), + ]); + let report = scan(dir.path(), None).unwrap(); + assert!(report.diagnostics.is_empty(), "{report:#?}"); + assert_eq!(project(&report, ".").workspace_members, ["api", "worker"]); + assert!(detects(project(&report, "api"), "gradle")); + assert!( + report + .projects + .iter() + .all(|p| p.path != "api/src/main/java") + ); +} + +#[test] +fn gradle_regex_literals_do_not_create_fake_strings_or_plugins() { + let dir = fixture(&[( + "build.gradle", + r#"plugins { id 'java' } +def pattern = /"preemptedAt"\s*:\s*"([^"]+)"/ +def other = $/"an "embedded" quote"/$ +"#, + )]); + let report = scan(dir.path(), None).unwrap(); + assert!(report.diagnostics.is_empty(), "{report:#?}"); + assert!(detects(project(&report, "."), "gradle")); +} + +#[test] +fn mixed_java_csharp_monorepo_keeps_runtime_versions_in_the_right_services() { + let dir = fixture(&[ + ( + "services/java/pom.xml", + "api21", + ), + ( + "services/dotnet/global.json", + r#"{"sdk":{"version":"10.0.403"}}"#, + ), + ( + "services/dotnet/Web.csproj", + "net10.0", + ), + ("services/dotnet/Program.cs", "not read"), + ( + "services/frontend/package.json", + r#"{"dependencies":{"next":"16"},"engines":{"node":">=24"}}"#, + ), + ]); + let report = scan(dir.path(), None).unwrap(); + assert_eq!(report.projects.len(), 3); + assert!(version( + project(&report, "services/java"), + "java", + "21", + "services/java/pom.xml" + )); + assert!( + project(&report, "services/java") + .languages + .iter() + .all(|l| l.name != "csharp") + ); + assert!(version( + project(&report, "services/dotnet"), + "csharp", + "10.0.403", + "services/dotnet/global.json" + )); + assert!( + project(&report, "services/frontend") + .languages + .iter() + .all(|l| l.name != "java" && l.name != "csharp") + ); +} + +#[test] +fn closest_global_json_selects_the_sdk_without_inheriting_sdk_package_catalogs() { + let dir = fixture(&[ + ( + "global.json", + r#"{"sdk":{"version":"10.0.100"},"msbuild-sdks":{"Aspire.AppHost.Sdk":"13.0.0"}}"#, + ), + ("nested/global.json", r#"{"sdk":{"version":"11.0.100"}}"#), + ( + "nested/App.csproj", + "net11.0", + ), + ]); + let report = scan(dir.path(), None).unwrap(); + let app = project(&report, "nested"); + assert!(version(app, "csharp", "11.0.100", "nested/global.json")); + assert!(!version(app, "csharp", "10.0.100", "global.json")); + let aspire = app.detections.iter().find(|d| d.id == "aspire").unwrap(); + assert!(aspire.evidence.iter().all(|e| e.value.is_none())); +} + +#[test] +fn java_pin_and_gradle_daemon_jdk_are_not_reported_as_target_release() { + let dir = fixture(&[ + ( + "build.gradle", + "plugins { id 'java' }\njava { sourceCompatibility = JavaVersion.VERSION_17 }\n", + ), + ( + "gradle/gradle-daemon-jvm.properties", + "toolchainVersion=25\n", + ), + (".sdkmanrc", "java=25.0.1-tem\n"), + ]); + let report = scan(dir.path(), None).unwrap(); + let java = project(&report, ".") + .languages + .iter() + .find(|l| l.name == "java") + .unwrap(); + assert!( + java.versions + .iter() + .any(|v| v.value == "17" && v.kind == "source") + ); + assert!( + java.versions + .iter() + .any(|v| v.value == "25" && v.kind == "build-toolchain") + ); + assert!( + java.versions + .iter() + .any(|v| v.value == "25.0.1-tem" && v.kind == "pin") + ); + assert!( + !java + .versions + .iter() + .any(|v| v.value == "25" && v.kind == "target") + ); +} + +#[test] +fn maven_relative_parents_with_different_coordinates_are_not_inherited() { + let dir = fixture(&[ + ( + "pom.xml", + "exampleplatform225", + ), + ( + "api/pom.xml", + "exampleplatform1api", + ), + ]); + let report = scan(dir.path(), None).unwrap(); + assert!(!version(project(&report, "api"), "java", "25", "pom.xml")); +} + +#[test] +fn classic_aspnet_and_aspire_sdk_declarations_are_separate_framework_facts() { + let dir = fixture(&[ + ( + "legacy/Web.csproj", + "v4.8", + ), + ( + "apphost/AppHost.csproj", + "net10.0", + ), + ]); + let report = scan(dir.path(), None).unwrap(); + assert!(report.diagnostics.is_empty(), "{report:#?}"); + assert!(detects(project(&report, "legacy"), "aspnet")); + assert!(!detects(project(&report, "legacy"), "aspnetcore")); + assert!(version( + project(&report, "legacy"), + "csharp", + "v4.8", + "legacy/Web.csproj" + )); + assert!(detects(project(&report, "apphost"), "aspire")); +} + +#[test] +fn gradle_project_directory_and_name_overrides_remain_distinct() { + let dir = fixture(&[ + ( + "settings.gradle", + "rootProject.name='platform'\ninclude ':web'\nproject(':web').projectDir = file('services/api')\nproject(':web').name = 'web-service'\n", + ), + ("build.gradle", "allprojects { group = 'example' }\n"), + ("services/api/build.gradle", "plugins { id 'java' }\n"), + ]); + let report = scan(dir.path(), None).unwrap(); + assert_eq!(project(&report, ".").workspace_members, ["services/api"]); + assert_eq!(project(&report, "services/api").names, ["web-service"]); +} + +#[test] +fn dotnet_build_outputs_are_pruned_without_skipping_rust_source_bins() { + let dir = fixture(&[ + ("Web/Web.csproj", ""), + ("Web/obj/generated/Fake.csproj", "net10.013.014.0", + )]); + let report = scan(dir.path(), None).unwrap(); + let versions = &project(&report, ".").languages[0].versions; + assert!( + versions + .iter() + .any(|v| v.value == "13.0" && v.kind == "conditional-language") + ); + assert!( + versions + .iter() + .any(|v| v.value == "14.0" && v.kind == "conditional-language") + ); + assert!(versions.iter().all(|v| v.kind != "language")); +} + +#[test] +fn disabled_central_package_management_does_not_supply_a_version() { + let dir = fixture(&[ + ( + "Directory.Packages.props", + "", + ), + ( + "App.csproj", + "false", + ), + ]); + let report = scan(dir.path(), None).unwrap(); + let framework = project(&report, ".") + .detections + .iter() + .find(|d| d.id == "fastendpoints") + .unwrap(); + assert!(framework.evidence.iter().all(|e| e.value.is_none())); +} diff --git a/src/detect/mod.rs b/src/detect/mod.rs new file mode 100644 index 0000000..5f45bde --- /dev/null +++ b/src/detect/mod.rs @@ -0,0 +1,718 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::fs::{self, File}; +use std::io::Read; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::thread; + +use anyhow::{Context, Result, bail}; +use clap::Args; +use serde::Serialize; + +mod catalog; +mod deno; +#[cfg(test)] +mod deno_tests; +mod dotnet; +mod gradle; +mod java; +mod jsonc; +#[cfg(test)] +mod jvm_dotnet_tests; +mod parsers; +mod rules; +#[cfg(test)] +mod tests; +mod values; +mod xml; + +use rules::{Kind, Rules}; + +const MAX_FILE_BYTES: u64 = 1024 * 1024; +const MAX_DEPTH: usize = 128; +#[derive(Debug, Args)] +pub struct DetectArgs { + /// Repository or project directory to inspect without executing project code + #[arg(default_value = ".")] + pub path: PathBuf, + /// Additional TOML detection rules; IDs must be unique + #[arg(long, value_name = "PATH")] + pub rules: Option, +} + +#[derive(Debug, Serialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct Report { + pub schema_version: u32, + pub projects: Vec, + pub diagnostics: Vec, + pub stats: Stats, +} + +#[derive(Debug, Serialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct Project { + pub path: String, + pub names: Vec, + pub languages: Vec, + #[serde(skip_serializing_if = "Vec::is_empty")] + pub runtimes: Vec, + pub detections: Vec, + pub workspace_members: Vec, + #[serde(skip_serializing_if = "Vec::is_empty")] + pub workspaces: Vec, +} + +#[derive(Debug, Serialize, PartialEq, Eq)] +pub struct WorkspaceReport { + pub ecosystem: String, + pub source: String, + pub members: Vec, +} + +#[derive(Debug, Serialize, PartialEq, Eq)] +pub struct Runtime { + pub name: String, + pub source: String, + pub versions: Vec, +} + +#[derive(Debug, Serialize, PartialEq, Eq)] +pub struct Language { + pub name: String, + pub versions: Vec, +} + +#[derive(Clone, Debug, Serialize, PartialEq, Eq, PartialOrd, Ord)] +pub struct Version { + pub value: String, + pub source: String, + pub kind: String, +} + +#[derive(Debug, Serialize, PartialEq, Eq)] +pub struct Detection { + pub id: String, + pub language: String, + pub kind: Kind, + pub priority: i32, + #[serde(skip_serializing_if = "Vec::is_empty")] + pub roles: Vec, + pub evidence: Vec, +} + +#[derive(Clone, Copy, Debug, Serialize, PartialEq, Eq, PartialOrd, Ord)] +#[serde(rename_all = "kebab-case")] +pub enum FrameworkRole { + Uses, + Provides, +} + +#[derive(Clone, Debug, Serialize, PartialEq, Eq, PartialOrd, Ord)] +pub struct Evidence { + pub source: String, + pub pattern: String, + pub value: Option, +} + +#[derive(Debug, Serialize, PartialEq, Eq)] +pub struct Diagnostic { + pub path: String, + pub message: String, +} + +#[derive(Debug, Default, Serialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct Stats { + pub directories: usize, + pub files: usize, + pub files_read: usize, + pub bytes_read: usize, + pub skipped_directories: usize, +} + +#[derive(Default)] +struct Node { + path: PathBuf, + parent: Option, + files: BTreeSet, + contents: BTreeMap, + sources: BTreeMap, + dependencies: BTreeMap, + identities: BTreeMap, + packages: BTreeMap, + versions: BTreeMap>, + runtimes: BTreeMap, + runtime_versions: BTreeMap>, + names: BTreeSet, + workspaces: Vec, +} + +#[derive(Clone)] +struct Dependency { + version: Option, + source: String, + resolved: Option, + declarations: Vec, +} + +#[derive(Clone)] +struct Workspace { + ecosystem: String, + source: String, + members: Vec, + exclude: Vec, +} + +impl Workspace { + fn contains(&self, path: &str) -> bool { + self.members.iter().any(|p| rules::path_glob(p, path)) + && !self.exclude.iter().any(|p| rules::path_glob(p, path)) + } +} + +impl Node { + fn dependency(&mut self, ecosystem: &str, name: &str, version: Option<&str>, file: &str) { + self.merge_dependency( + format!("{ecosystem}:{name}"), + Dependency { + version: version.map(str::to_owned), + source: relative_file(&self.path, file), + resolved: None, + declarations: Vec::new(), + }, + ); + } + + fn merge_dependency(&mut self, key: String, mut dependency: Dependency) { + if let Some(previous) = self.dependencies.remove(&key) { + dependency.declarations.extend(previous.declarations); + if let Some(value) = previous.version { + dependency.declarations.push(Version { + value, + source: previous.source, + kind: "requirement".to_owned(), + }); + } + } + self.dependencies.insert(key, dependency); + } + + fn identity(&mut self, ecosystem: &str, name: &str, version: Option<&str>, file: &str) { + self.identities.insert( + format!("{ecosystem}:{name}"), + Dependency { + version: version.map(str::to_owned), + source: relative_file(&self.path, file), + resolved: None, + declarations: Vec::new(), + }, + ); + } + + fn version(&mut self, language: &str, value: &str, file: &str, kind: &str) { + if !value.trim().is_empty() { + self.versions + .entry(language.to_owned()) + .or_default() + .push(Version { + value: value.trim().to_owned(), + source: relative_file(&self.path, file), + kind: kind.to_owned(), + }); + } + } +} + +pub fn handle(args: DetectArgs) -> Result { + let extra = args + .rules + .as_deref() + .map(fs::read_to_string) + .transpose() + .context("could not read detection rules")?; + let report = scan(&args.path, extra.as_deref())?; + Ok(crate::output::CommandOutput::new( + serde_json::to_value(report)?, + crate::output::View::Detect, + )) +} + +#[derive(Default)] +struct Inventory { + nodes: Vec, + diagnostics: Vec, + stats: Stats, +} + +impl Inventory { + fn read_directory( + &mut self, + root: &Path, + path: &Path, + depth: usize, + rules: &Rules, + ) -> Result> { + let mut node = Node::default(); + self.stats.directories += 1; + let entries = fs::read_dir(root.join(path)) + .with_context(|| format!("could not scan {}", root.join(path).display()))?; + let mut children = Vec::new(); + for entry in entries { + let entry = entry?; + let ty = entry.file_type()?; + let Some(name) = entry.file_name().to_str().map(str::to_owned) else { + continue; + }; + if ty.is_dir() { + if rules.catalog.ignored(&name) { + self.stats.skipped_directories += 1; + } else if depth == MAX_DEPTH { + self.diagnostics.push(Diagnostic { + path: relative_file(path, &name), + message: "directory depth limit reached".to_owned(), + }); + } else { + children.push((path.join(&name), depth + 1)); + } + } else if ty.is_file() { + self.stats.files += 1; + if let Some(ext) = Path::new(&name).extension().and_then(|e| e.to_str()) + && rules.needs_source(ext) + { + node.sources + .entry(ext.to_owned()) + .and_modify(|existing| { + if name < *existing { + *existing = name.clone(); + } + }) + .or_insert_with(|| name.clone()); + } + let readable = rules.catalog.readable(&name); + if readable { + let file = File::open(entry.path())?; + let mut bytes = Vec::new(); + file.take(MAX_FILE_BYTES + 1).read_to_end(&mut bytes)?; + self.stats.files_read += 1; + self.stats.bytes_read += bytes.len(); + let content = if bytes.len() > MAX_FILE_BYTES as usize { + Err("manifest exceeds 1 MiB read limit".to_owned()) + } else { + String::from_utf8(bytes).map_err(|_| "manifest is not UTF-8".to_owned()) + }; + match content { + Ok(content) => { + node.contents.insert(name.clone(), content); + } + Err(message) => self.diagnostics.push(Diagnostic { + path: relative_file(path, &name), + message, + }), + } + } + if readable || rules.needs_filename(&name) { + node.files.insert(name); + } + } + } + children.retain(|(path, _)| { + let prune = path + .file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| rules.catalog.pruned(&node, name)); + if prune { + self.stats.skipped_directories += 1; + } + !prune + }); + if !node.files.is_empty() || !node.sources.is_empty() { + node.path = path.to_path_buf(); + self.nodes.push(node); + } + Ok(children) + } + + fn walk( + &mut self, + root: &Path, + rules: &Rules, + mut pending: Vec<(PathBuf, usize)>, + ) -> Result<()> { + while let Some((path, depth)) = pending.pop() { + pending.extend(self.read_directory(root, &path, depth, rules)?); + } + Ok(()) + } + + fn merge(&mut self, other: Self) { + self.nodes.extend(other.nodes); + self.diagnostics.extend(other.diagnostics); + self.stats.directories += other.stats.directories; + self.stats.files += other.stats.files; + self.stats.files_read += other.stats.files_read; + self.stats.bytes_read += other.stats.bytes_read; + self.stats.skipped_directories += other.stats.skipped_directories; + } +} + +fn inventory(root: &Path, rules: &Rules, workers: usize) -> Result { + let mut inventory = Inventory::default(); + let mut pending = vec![(PathBuf::new(), 0)]; + for _ in 0..3 { + let mut next = Vec::new(); + for (path, depth) in pending { + next.extend(inventory.read_directory(root, &path, depth, rules)?); + } + pending = next; + if pending.is_empty() { + break; + } + } + if workers > 1 && pending.len() >= 64 { + let next = AtomicUsize::new(0); + let results = thread::scope(|scope| { + let handles = (0..workers) + .map(|_| { + scope.spawn(|| -> Result { + let mut inventory = Inventory::default(); + while let Some(task) = pending.get(next.fetch_add(1, Ordering::Relaxed)) { + inventory.walk(root, rules, vec![task.clone()])?; + } + Ok(inventory) + }) + }) + .collect::>(); + handles + .into_iter() + .map(|handle| { + handle + .join() + .map_err(|_| anyhow::anyhow!("directory scan worker panicked"))? + }) + .collect::>>() + })?; + for result in results { + inventory.merge(result); + } + } else { + inventory.walk(root, rules, pending)?; + } + index_parents(&mut inventory.nodes); + Ok(inventory) +} + +fn index_parents(nodes: &mut [Node]) { + nodes.sort_by(|a, b| a.path.cmp(&b.path)); + let mut ancestors = Vec::::new(); + for index in 0..nodes.len() { + while ancestors + .last() + .is_some_and(|&parent| !nodes[index].path.starts_with(&nodes[parent].path)) + { + ancestors.pop(); + } + nodes[index].parent = ancestors.last().copied(); + ancestors.push(index); + } +} + +pub fn scan(root: &Path, extra_rules: Option<&str>) -> Result { + let custom = extra_rules + .map(|extra| Rules::new(Some(extra))) + .transpose()?; + let rules = match custom.as_ref() { + Some(rules) => rules, + None => Rules::builtin()?, + }; + let root = root + .canonicalize() + .with_context(|| format!("could not open {}", root.display()))?; + if !root.is_dir() { + bail!("detection root is not a directory: {}", root.display()); + } + let mut report = Report { + schema_version: 1, + projects: Vec::new(), + diagnostics: Vec::new(), + stats: Stats::default(), + }; + let workers = thread::available_parallelism().map_or(1, |n| n.get().min(4)); + let Inventory { + mut nodes, + diagnostics, + stats, + } = inventory(&root, rules, workers)?; + report.diagnostics = diagnostics; + report.stats = stats; + let original_nodes = nodes.len(); + parsers::parse( + &root, + &mut nodes, + &rules.catalog, + &mut report.stats, + &mut report.diagnostics, + ); + if nodes.len() != original_nodes { + index_parents(&mut nodes); + } + + let mut projects = Vec::<(usize, Project)>::new(); + let mut project_indices = vec![None; nodes.len()]; + let mut source_detections = Vec::new(); + for (index, node) in nodes.iter().enumerate() { + let mut detections = Vec::new(); + for detection in rules.evaluate(node) { + if detection.kind == Kind::Source { + source_detections.push((index, detection)); + } else { + detections.push(detection); + } + } + if !detections.is_empty() { + project_indices[index] = Some(projects.len()); + projects.push(( + index, + Project { + path: display_path(&node.path), + names: node.names.iter().cloned().collect(), + languages: Vec::new(), + runtimes: Vec::new(), + detections, + workspace_members: Vec::new(), + workspaces: Vec::new(), + }, + )); + } + } + for (index, detection) in source_detections { + let mut ancestor = Some(index); + let mut owner = None; + while let Some(i) = ancestor { + if let Some(project_index) = project_indices[i] + && projects[project_index].1.detections.iter().any(|d| { + d.kind != Kind::Workspace + && rules.catalog.family(&d.language) + == rules.catalog.family(&detection.language) + }) + { + owner = Some(project_index); + break; + } + ancestor = nodes[i].parent; + } + let owner = owner.or(project_indices[index]).unwrap_or_else(|| { + let project_index = projects.len(); + project_indices[index] = Some(project_index); + projects.push(( + index, + Project { + path: display_path(&nodes[index].path), + names: Vec::new(), + languages: Vec::new(), + runtimes: Vec::new(), + detections: Vec::new(), + workspace_members: Vec::new(), + workspaces: Vec::new(), + }, + )); + project_index + }); + let project = &mut projects[owner].1; + if !project + .detections + .iter() + .any(|d| d.language == detection.language) + { + project.detections.push(detection); + } + } + for (index, project) in &mut projects { + let languages = project + .detections + .iter() + .map(|d| d.language.clone()) + .collect::>(); + for language in languages { + let mut versions = Vec::new(); + let mut ancestor = Some(*index); + let mut boundaries = BTreeSet::new(); + while let Some(i) = ancestor { + if let Some(found) = nodes[i].versions.get(&language) { + versions.extend( + found + .iter() + .filter(|v| { + (i == *index || rules.catalog.inherited(&v.kind)) + && !boundaries.contains(v.kind.as_str()) + }) + .cloned(), + ); + } + boundaries.extend(rules.catalog.boundaries(&nodes[i])); + ancestor = nodes[i].parent; + } + let mut seen = BTreeSet::new(); + versions.retain(|v| seen.insert(v.clone())); + versions.sort(); + project.languages.push(Language { + name: language, + versions, + }); + } + let mut runtime_sources = nodes[*index].runtimes.clone(); + for (name, versions) in &nodes[*index].runtime_versions { + if let Some(version) = versions.first() { + runtime_sources + .entry(name.clone()) + .or_insert_with(|| version.source.clone()); + } + } + for (name, source) in runtime_sources { + let mut versions = Vec::new(); + let mut ancestor = Some(*index); + while let Some(i) = ancestor { + if let Some(found) = nodes[i].runtime_versions.get(&name) { + versions.extend( + found + .iter() + .filter(|v| i == *index || rules.catalog.inherited(&v.kind)) + .cloned(), + ); + } + ancestor = nodes[i].parent; + } + versions.sort(); + versions.dedup(); + project.runtimes.push(Runtime { + name, + source, + versions, + }); + } + project + .detections + .sort_by(|a, b| b.priority.cmp(&a.priority).then(a.id.cmp(&b.id))); + } + let mut candidates = BTreeMap::<&str, Vec>::new(); + for (index, node) in nodes.iter().enumerate() { + if project_indices[index].is_none() { + continue; + } + for ecosystem in rules.catalog.workspace_candidates(node) { + candidates.entry(ecosystem).or_default().push(index); + } + } + for (index, project) in &mut projects { + let node = &nodes[*index]; + for workspace in &node.workspaces { + let mut members = Vec::new(); + for &member in candidates + .get(workspace.ecosystem.as_str()) + .into_iter() + .flatten() + { + if member == *index { + continue; + } + let Ok(relative) = nodes[member].path.strip_prefix(&node.path) else { + continue; + }; + if workspace.contains(&display_path(relative)) { + members.push(display_path(&nodes[member].path)); + } + } + members.sort(); + members.dedup(); + project.workspace_members.extend(members.iter().cloned()); + project.workspaces.push(WorkspaceReport { + ecosystem: workspace.ecosystem.clone(), + source: workspace.source.clone(), + members, + }); + } + project + .workspaces + .sort_by(|a, b| a.ecosystem.cmp(&b.ecosystem).then(a.source.cmp(&b.source))); + project.workspace_members.sort(); + project.workspace_members.dedup(); + } + projects.sort_by(|a, b| nodes[a.0].path.cmp(&nodes[b.0].path)); + report.projects = projects.into_iter().map(|(_, project)| project).collect(); + report + .diagnostics + .sort_by(|a, b| a.path.cmp(&b.path).then(a.message.cmp(&b.message))); + Ok(report) +} + +fn display_path(path: &Path) -> String { + if path.as_os_str().is_empty() { + ".".to_owned() + } else { + path.to_string_lossy().replace('\\', "/") + } +} + +fn relative_file(path: &Path, name: &str) -> String { + display_path(&path.join(name)) +} + +pub fn render(value: &serde_json::Value) -> Vec { + let mut lines = Vec::new(); + if let Some(projects) = value["projects"].as_array() { + for project in projects { + let languages = project["languages"] + .as_array() + .into_iter() + .flatten() + .filter_map(|l| l["name"].as_str()) + .collect::>(); + let detections = project["detections"] + .as_array() + .into_iter() + .flatten() + .filter_map(|d| d["id"].as_str()) + .collect::>(); + lines.push(format!( + "{}\t{}\t{}", + project["path"].as_str().unwrap_or("."), + languages.join(", "), + detections.join(", ") + )); + for runtime in project["runtimes"].as_array().into_iter().flatten() { + lines.push(format!( + " runtime {} ({})", + runtime["name"].as_str().unwrap_or(""), + runtime["source"].as_str().unwrap_or("") + )); + } + for language in project["languages"] + .as_array() + .into_iter() + .flatten() + .chain(project["runtimes"].as_array().into_iter().flatten()) + { + for version in language["versions"].as_array().into_iter().flatten() { + lines.push(format!( + " {} {} ({}, {})", + language["name"].as_str().unwrap_or(""), + version["value"].as_str().unwrap_or(""), + version["kind"].as_str().unwrap_or(""), + version["source"].as_str().unwrap_or("") + )); + } + } + } + } + for diagnostic in value["diagnostics"].as_array().into_iter().flatten() { + lines.push(format!( + "warning: {}: {}", + diagnostic["path"].as_str().unwrap_or(""), + diagnostic["message"].as_str().unwrap_or("") + )); + } + if lines.is_empty() { + lines.push("No projects detected".to_owned()); + } + lines +} diff --git a/src/detect/parsers.rs b/src/detect/parsers.rs new file mode 100644 index 0000000..3a8f4dd --- /dev/null +++ b/src/detect/parsers.rs @@ -0,0 +1,560 @@ +use std::collections::{BTreeMap, HashMap}; +use std::path::{Path, PathBuf}; + +use serde::Deserialize; +use serde_json::Value; + +use super::catalog::{Catalog, Normalize, Reader, Resolver, workspace_patterns}; +use super::{Diagnostic, Node, Stats, Workspace, relative_file}; + +pub(super) type Documents = HashMap>; + +pub fn parse( + root: &Path, + nodes: &mut Vec, + catalog: &Catalog, + stats: &mut Stats, + diagnostics: &mut Vec, +) { + let mut documents = Documents::new(); + let mut pnpm = HashMap::>::new(); + for node in nodes.iter() { + if node + .files + .iter() + .any(|file| catalog.reader(file, Reader::Pnpm)) + { + pnpm.insert(node.path.clone(), None); + } + for (file, content) in &node.contents { + let manifests = catalog.matching(file).collect::>(); + let Some(manifest) = manifests.first() else { + continue; + }; + if !manifest.preferred(file, &node.files) { + continue; + } + if manifest.reader == Reader::Pnpm { + match serde_saphyr::from_str::(content) { + Ok(manifest) => { + let (members, exclude) = workspace_patterns( + manifest.packages.unwrap_or_else(|| vec!["**".to_owned()]), + ); + pnpm.insert( + node.path.clone(), + Some(Catalogs { + workspaces: vec![Workspace { + ecosystem: "node".to_owned(), + source: relative_file(&node.path, file), + members, + exclude, + }], + source: relative_file(&node.path, file), + catalog: manifest.catalog.unwrap_or_default(), + catalogs: manifest.catalogs.unwrap_or_default(), + }), + ); + } + Err(error) => diagnostics.push(Diagnostic { + path: relative_file(&node.path, file), + message: error.to_string(), + }), + } + continue; + } + let parsed = match manifest.reader { + Reader::Json => { + Some(serde_json::from_str::(content).map_err(|e| e.to_string())) + } + Reader::Jsonc => Some(super::jsonc::parse(content)), + Reader::Toml => Some(toml::from_str::(content).map_err(|e| e.to_string())), + _ => None, + }; + if let Some(parsed) = parsed { + match parsed { + Ok(doc) if doc.is_object() => { + documents + .entry(node.path.clone()) + .or_default() + .insert(file.clone(), doc); + } + Ok(_) => diagnostics.push(Diagnostic { + path: relative_file(&node.path, file), + message: "manifest must be an object".to_owned(), + }), + Err(message) => diagnostics.push(Diagnostic { + path: relative_file(&node.path, file), + message, + }), + } + } + } + } + for node in nodes.iter_mut() { + if let Some(Some(manifest)) = pnpm.get(&node.path) { + node.workspaces.extend(manifest.workspaces.clone()); + } + for (file, doc) in documents.get(&node.path).into_iter().flatten() { + for manifest in catalog.matching(file) { + manifest.apply(node, file, doc); + } + } + } + super::deno::resolve(root, nodes, &documents, &mut pnpm, stats, diagnostics); + super::dotnet::parse(nodes, catalog, &documents, diagnostics); + super::java::parse(nodes, catalog, diagnostics); + for node in nodes { + for (file, doc) in documents.get(&node.path).into_iter().flatten() { + if catalog + .matching(file) + .any(|manifest| manifest.resolver == Some(Resolver::Cargo)) + { + cargo(node, doc, &documents); + } + } + let contents = std::mem::take(&mut node.contents); + for (file, text) in &contents { + for manifest in catalog.matching(file) { + match manifest.reader { + Reader::Go => go(node, file, text, manifest.workspace.as_deref()), + Reader::Gemfile => ruby(node, file, text, diagnostics), + Reader::Gemspec => gemspec(node, file, text), + Reader::Pin | Reader::Properties | Reader::Tools => { + manifest.text_versions(node, file, text) + } + Reader::Requirements => { + for line in logical_lines(text) { + let line = line.trim(); + if line.starts_with('-') { + if !line.starts_with("--hash=") { + diagnostics.push(Diagnostic { + path: relative_file(&node.path, file), + message: format!( + "requirements option not evaluated: {line}" + ), + }); + } + } else { + requirement(node, line, file); + } + } + } + _ => (), + } + } + } + resolve_catalogs(node, &pnpm, diagnostics); + for (file, doc) in documents.get(&node.path).into_iter().flatten() { + for manifest in catalog.matching(file) { + manifest.versions(node, file, Some(doc), None, true); + } + } + } +} + +#[derive(Deserialize)] +struct PnpmDocument { + packages: Option>, + catalog: Option>, + catalogs: Option>>, +} + +pub(super) struct Catalogs { + pub workspaces: Vec, + pub source: String, + pub catalog: BTreeMap, + pub catalogs: BTreeMap>, +} + +fn resolve_catalogs( + node: &mut Node, + manifests: &HashMap>, + diagnostics: &mut Vec, +) { + let workspace = node + .path + .ancestors() + .find_map(|path| manifests.get(path).map(|manifest| (path, manifest))) + .and_then(|(path, manifest)| manifest.as_ref().map(|manifest| (path, manifest))) + .filter(|(path, manifest)| { + *path == node.path + || node.path.strip_prefix(path).ok().is_some_and(|relative| { + manifest + .workspaces + .iter() + .any(|workspace| workspace.contains(&super::display_path(relative))) + }) + }) + .map(|(_, manifest)| manifest); + for (key, dependency) in &mut node.dependencies { + let Some(package) = key.strip_prefix("npm:") else { + continue; + }; + let Some(reference) = dependency + .version + .as_deref() + .and_then(|v| v.strip_prefix("catalog:")) + else { + continue; + }; + let value = workspace.and_then(|manifest| { + let catalog = if reference.is_empty() || reference == "default" { + Some(&manifest.catalog) + } else { + manifest.catalogs.get(reference) + }; + catalog + .and_then(|catalog| catalog.get(package)) + .filter(|v| !v.starts_with("catalog:")) + .map(|value| super::Version { + value: value.clone(), + source: manifest.source.clone(), + kind: "requirement".to_owned(), + }) + }); + if let Some(resolved) = value { + dependency.resolved = Some(resolved); + } else { + diagnostics.push(Diagnostic { + path: dependency.source.clone(), + message: format!("could not resolve catalog:{reference} for {package}"), + }); + } + } +} + +fn cargo(node: &mut Node, doc: &Value, documents: &Documents) { + let file = "Cargo.toml"; + let workspace = + if let Some(explicit) = doc.pointer("/package/workspace").and_then(Value::as_str) { + normalize_path(&node.path.join(explicit)).and_then(|path| { + documents + .get(&path) + .and_then(|docs| docs.get(file)) + .filter(|d| d.get("workspace").is_some()) + .map(|d| (path, d)) + }) + } else { + node.path.ancestors().find_map(|path| { + documents + .get(path) + .and_then(|docs| docs.get(file)) + .filter(|d| d.get("workspace").is_some()) + .map(|d| (path.to_path_buf(), d)) + }) + }; + if let Some(name) = doc.pointer("/package/name").and_then(Value::as_str) + && doc + .pointer("/package/version/workspace") + .and_then(Value::as_bool) + == Some(true) + && let Some((path, workspace)) = &workspace + && let Some(version) = workspace + .pointer("/workspace/package/version") + .and_then(Value::as_str) + && let Some(identity) = node.identities.get_mut(&format!("cargo:{name}")) + { + identity.resolved = Some(super::Version { + value: version.to_owned(), + source: relative_file(path, file), + kind: "package".to_owned(), + }); + } + if doc + .pointer("/package/rust-version/workspace") + .and_then(Value::as_bool) + == Some(true) + && let Some((path, ws)) = &workspace + && let Some(version) = ws + .pointer("/workspace/package/rust-version") + .and_then(Value::as_str) + { + node.versions + .entry("rust".to_owned()) + .or_default() + .push(super::Version { + value: version.to_owned(), + source: relative_file(path, file), + kind: "requirement".to_owned(), + }); + } + let mut dep_tables = Vec::new(); + if let Some(d) = doc.get("dependencies") { + dep_tables.push(d); + } + if let Some(targets) = doc.get("target").and_then(Value::as_object) { + dep_tables.extend(targets.values().filter_map(|v| v.get("dependencies"))); + } + for table in dep_tables { + if let Some(deps) = table.as_object() { + for (alias, declaration) in deps { + let inherited = declaration.get("workspace").and_then(Value::as_bool) == Some(true); + let (value, source) = if inherited { + let Some((path, ws)) = &workspace else { + continue; + }; + let Some(value) = ws + .pointer("/workspace/dependencies") + .and_then(|d| d.get(alias)) + else { + continue; + }; + (value, relative_file(path, file)) + } else { + (declaration, relative_file(&node.path, file)) + }; + let name = value + .get("package") + .and_then(Value::as_str) + .unwrap_or(alias); + let version = value + .as_str() + .or_else(|| value.get("version").and_then(Value::as_str)) + .map(str::to_owned); + node.dependencies.insert( + format!("cargo:{name}"), + super::Dependency { + version, + source, + resolved: None, + declarations: Vec::new(), + }, + ); + } + } + } +} + +pub(super) fn requirement_parts(line: &str) -> Option<(&str, Option<&str>)> { + let line = line.split('#').next().unwrap_or("").trim(); + if line.is_empty() { + return None; + } + let end = line + .find(|c: char| !c.is_ascii_alphanumeric() && !matches!(c, '-' | '_' | '.')) + .unwrap_or(line.len()); + if end == 0 { + return None; + } + let name = &line[..end]; + let mut rest = line[end..].trim(); + if rest.starts_with('[') { + let end = rest.find(']')?; + rest = rest[end + 1..].trim(); + } + let constraint = rest.split(';').next().unwrap_or("").trim(); + let constraint = constraint.split(" --hash=").next().unwrap_or("").trim(); + let version = if constraint.is_empty() || constraint.starts_with('@') { + None + } else { + Some(constraint) + }; + Some((name, version)) +} + +fn requirement(node: &mut Node, line: &str, file: &str) { + if let Some((name, version)) = requirement_parts(line) { + node.dependency("pypi", &Normalize::Python.name(name), version, file); + } +} + +fn logical_lines(text: &str) -> Vec { + let mut lines = Vec::new(); + let mut current = String::new(); + for line in text.lines() { + let line = line.trim_end(); + if let Some(part) = line.strip_suffix('\\') { + current.push_str(part); + current.push(' '); + } else { + current.push_str(line); + lines.push(std::mem::take(&mut current)); + } + } + if !current.is_empty() { + lines.push(current); + } + lines +} + +fn go(node: &mut Node, file: &str, text: &str, workspace: Option<&str>) { + let mut block = ""; + let mut members = Vec::new(); + for line in text.lines() { + if line + .split_once("//") + .is_some_and(|(_, comment)| comment.trim() == "indirect") + { + continue; + } + let line = line.split("//").next().unwrap_or("").trim(); + if line.is_empty() { + continue; + } + if line == ")" { + block = ""; + continue; + } + let words = line.split_whitespace().collect::>(); + let (directive, values) = if block.is_empty() { + (words[0], &words[1..]) + } else { + (block, words.as_slice()) + }; + if values == ["("] { + block = directive; + continue; + } + match directive { + "go" => { + if let Some(v) = values.first() { + node.version("go", v, file, "requirement"); + } + } + "toolchain" => { + if let Some(v) = values.first() { + node.version("go", v.trim_start_matches("go"), file, "toolchain"); + } + } + "module" => { + if let Some(name) = values.first() { + node.names.insert(name.trim_matches('"').to_owned()); + node.identity("go", name.trim_matches('"'), None, file); + } + } + "require" => { + if let Some(name) = values.first() { + node.dependency("go", name, values.get(1).copied(), file); + } + } + "use" => { + if let Some(path) = values.first() { + members.push(path.trim_matches('"').trim_start_matches("./").to_owned()); + } + } + _ => (), + } + } + if let Some(ecosystem) = workspace { + node.workspaces.push(Workspace { + ecosystem: ecosystem.to_owned(), + source: relative_file(&node.path, file), + members, + exclude: Vec::new(), + }); + } +} + +fn quoted(input: &str) -> Option<(&str, &str)> { + let input = input.trim_start(); + let quote = input.as_bytes().first().copied()?; + if quote != b'\'' && quote != b'"' { + return None; + } + let end = input[1..].find(char::from(quote))? + 1; + let value = &input[1..end]; + if value.contains("#{") || value.contains('\\') { + return None; + } + Some((value, &input[end + 1..])) +} + +fn ruby(node: &mut Node, file: &str, text: &str, diagnostics: &mut Vec) { + for line in text.lines() { + let line = line.trim(); + let (directive, rest) = if let Some(rest) = line + .strip_prefix("gem ") + .or_else(|| line.strip_prefix("gem(")) + .or_else(|| line.strip_prefix("gem\t")) + { + ("gem", rest) + } else if let Some(rest) = line + .strip_prefix("ruby ") + .or_else(|| line.strip_prefix("ruby(")) + .or_else(|| line.strip_prefix("ruby\t")) + { + ("ruby", rest) + } else { + continue; + }; + let Some((name, rest)) = quoted(rest) else { + diagnostics.push(Diagnostic { + path: relative_file(&node.path, file), + message: format!("dynamic {directive} declaration not evaluated"), + }); + continue; + }; + if directive == "ruby" { + node.version("ruby", name, file, "requirement"); + } else { + let mut versions = Vec::new(); + let mut rest = rest.trim_start(); + while let Some(next) = rest.strip_prefix(',') { + let Some((version, tail)) = quoted(next) else { + break; + }; + versions.push(version); + rest = tail.trim_start(); + } + let version = versions.join(", "); + node.dependency( + "gem", + name, + (!version.is_empty()).then_some(version.as_str()), + file, + ); + } + } +} + +fn gemspec(node: &mut Node, file: &str, text: &str) { + let mut name = None; + let mut version = None; + for line in text.lines() { + let Some((receiver, assignment)) = line.trim().split_once('.') else { + continue; + }; + if receiver.is_empty() + || !receiver + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '_') + { + continue; + } + let Some((field, value)) = assignment.split_once('=') else { + continue; + }; + let Some((value, _)) = quoted(value) else { + continue; + }; + match field.trim() { + "name" => name = Some(value), + "version" => version = Some(value), + "required_ruby_version" => node.version("ruby", value, file, "requirement"), + _ => (), + } + } + if let Some(name) = name { + node.names.insert(name.to_owned()); + node.identity("gem", name, version, file); + } +} + +fn normalize_path(path: &Path) -> Option { + if path.is_absolute() { + return None; + } + let mut normalized = PathBuf::new(); + for component in path.components() { + match component { + std::path::Component::ParentDir => { + if !normalized.pop() { + return None; + } + } + std::path::Component::CurDir => (), + other => normalized.push(other), + } + } + Some(normalized) +} diff --git a/src/detect/rules.rs b/src/detect/rules.rs new file mode 100644 index 0000000..5091801 --- /dev/null +++ b/src/detect/rules.rs @@ -0,0 +1,501 @@ +use std::collections::{BTreeSet, HashMap}; +use std::sync::OnceLock; + +use anyhow::{Result, bail}; +use serde::Deserialize; + +use super::catalog::{Catalog, CatalogSpec}; +use super::{Detection, Evidence, FrameworkRole, Node}; + +#[derive(Clone, Copy, Debug, Deserialize, serde::Serialize, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +pub enum Kind { + Framework, + Project, + Workspace, + Source, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct RuleSpec { + id: String, + language: String, + kind: Kind, + priority: i32, + #[serde(default)] + any: Vec, + #[serde(default)] + all: Vec, + #[serde(default)] + none: Vec, +} + +#[derive(Deserialize)] +#[serde(untagged)] +enum PatternSpec { + Atom(String), + All(AllPattern), +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct AllPattern { + all: Vec, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Document { + #[serde(default)] + rules: Vec, + #[serde(default)] + catalog: CatalogSpec, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +#[repr(usize)] +enum FactKind { + File, + Source, + Dependency, + Identity, + Package, + Workspace, +} + +impl FactKind { + fn parse(name: &str) -> Result { + match name { + "file" => Ok(Self::File), + "source" => Ok(Self::Source), + "dep" => Ok(Self::Dependency), + "identity" => Ok(Self::Identity), + "package" => Ok(Self::Package), + "workspace" => Ok(Self::Workspace), + _ => bail!("unknown detection fact: {name}"), + } + } +} + +pub(super) enum FilePattern { + Exact(String), + Star { prefix: String, suffix: String }, + Glob(String), +} + +impl FilePattern { + pub fn new(value: &str) -> Self { + if !value.contains(['*', '?']) { + Self::Exact(value.to_owned()) + } else if !value.contains('?') && value.bytes().filter(|b| *b == b'*').count() == 1 { + let (prefix, suffix) = value.split_once('*').unwrap_or((value, "")); + Self::Star { + prefix: prefix.to_owned(), + suffix: suffix.to_owned(), + } + } else { + Self::Glob(value.to_owned()) + } + } + + pub fn matches(&self, value: &str) -> bool { + match self { + Self::Exact(exact) => exact == value, + Self::Star { prefix, suffix } => { + value.len() >= prefix.len() + suffix.len() + && value.starts_with(prefix) + && value.ends_with(suffix) + } + Self::Glob(pattern) => glob(pattern, value), + } + } +} + +struct Atom { + kind: FactKind, + value: String, + text: String, + file: FilePattern, +} +struct Pattern { + atoms: Vec, +} +struct Rule { + id: String, + language: String, + kind: Kind, + priority: i32, + any: Vec, + all: Vec, + none: Vec, +} + +impl PatternSpec { + fn compile(self) -> Result { + let atoms = match self { + Self::Atom(atom) => vec![atom], + Self::All(group) => group.all, + }; + if atoms.is_empty() { + bail!("empty detection pattern group"); + } + let atoms = atoms + .into_iter() + .map(|text| { + let Some((kind, value)) = text.split_once(':') else { + bail!("invalid detection pattern: {text}"); + }; + let kind = FactKind::parse(kind)?; + let valid = match kind { + FactKind::File | FactKind::Source => !value.is_empty() && !value.contains('/'), + FactKind::Dependency | FactKind::Identity => value + .split_once(':') + .is_some_and(|(ecosystem, name)| !ecosystem.is_empty() && !name.is_empty()), + FactKind::Package | FactKind::Workspace => !value.is_empty(), + }; + if !valid { + bail!("invalid detection pattern: {text}"); + } + Ok(Atom { + kind, + value: value.to_owned(), + file: FilePattern::new(value), + text, + }) + }) + .collect::>>()?; + Ok(Pattern { atoms }) + } +} + +impl Pattern { + fn matches(&self, node: &Node) -> bool { + self.atoms.iter().all(|atom| atom.matched(node).is_some()) + } +} + +pub struct Rules { + pub catalog: Catalog, + rules: Vec, + indexes: [HashMap>; 6], + wildcard_files: Vec<(FilePattern, usize)>, + file_interest: BTreeSet, + wildcard_interest: Vec, + source_interest: BTreeSet, +} + +impl Rules { + pub fn builtin() -> Result<&'static Self> { + static RULES: OnceLock> = OnceLock::new(); + RULES + .get_or_init(|| Self::new(None).map_err(|e| e.to_string())) + .as_ref() + .map_err(|e| anyhow::anyhow!("{e}")) + } + + pub fn new(extra: Option<&str>) -> Result { + let mut document: Document = toml::from_str(include_str!("rules.toml"))?; + if let Some(extra) = extra { + let extra: Document = toml::from_str(extra)?; + document.rules.extend(extra.rules); + document.catalog.extend(extra.catalog); + } + let mut ids = BTreeSet::new(); + let mut rules = Self { + catalog: Catalog::new(document.catalog)?, + rules: Vec::new(), + indexes: std::array::from_fn(|_| HashMap::new()), + wildcard_files: Vec::new(), + file_interest: BTreeSet::new(), + wildcard_interest: Vec::new(), + source_interest: BTreeSet::new(), + }; + for spec in document.rules { + if spec.id.is_empty() || spec.language.is_empty() || !ids.insert(spec.id.clone()) { + bail!("empty or duplicate detection rule: {}", spec.id); + } + if spec.any.is_empty() && spec.all.is_empty() { + bail!("rule {} needs an any or all pattern", spec.id); + } + let compile = |patterns: Vec| { + patterns + .into_iter() + .map(PatternSpec::compile) + .collect::>>() + }; + rules.rules.push(Rule { + id: spec.id, + language: spec.language, + kind: spec.kind, + priority: spec.priority, + any: compile(spec.any)?, + all: compile(spec.all)?, + none: compile(spec.none)?, + }); + } + rules + .rules + .sort_by(|a, b| b.priority.cmp(&a.priority).then(a.id.cmp(&b.id))); + let mut wildcards = BTreeSet::new(); + for (index, rule) in rules.rules.iter().enumerate() { + for atom in rule + .all + .iter() + .chain(&rule.any) + .chain(&rule.none) + .flat_map(|p| &p.atoms) + { + match atom.kind { + FactKind::Source => { + rules.source_interest.insert(atom.value.clone()); + } + FactKind::File if matches!(atom.file, FilePattern::Exact(_)) => { + rules.file_interest.insert(atom.value.clone()); + } + FactKind::File => { + wildcards.insert(atom.value.clone()); + } + _ => (), + } + } + for atom in rule.all.iter().chain(&rule.any).flat_map(|p| &p.atoms) { + if atom.kind == FactKind::File && !matches!(atom.file, FilePattern::Exact(_)) { + rules + .wildcard_files + .push((FilePattern::new(&atom.value), index)); + } else { + rules.indexes[atom.kind as usize] + .entry(atom.value.clone()) + .or_default() + .push(index); + } + } + } + rules.wildcard_interest = wildcards + .iter() + .map(|pattern| FilePattern::new(pattern)) + .collect(); + Ok(rules) + } + + pub fn needs_source(&self, extension: &str) -> bool { + self.source_interest.contains(extension) + } + pub fn needs_filename(&self, name: &str) -> bool { + self.file_interest.contains(name) + || self + .wildcard_interest + .iter() + .any(|pattern| pattern.matches(name)) + } + + pub fn evaluate(&self, node: &Node) -> Vec { + let mut candidates = Vec::new(); + let facts = node + .files + .iter() + .map(|s| (FactKind::File, s.as_str())) + .chain(node.sources.keys().map(|s| (FactKind::Source, s.as_str()))) + .chain( + node.dependencies + .keys() + .map(|s| (FactKind::Dependency, s.as_str())), + ) + .chain( + node.identities + .keys() + .map(|s| (FactKind::Identity, s.as_str())), + ) + .chain( + node.packages + .keys() + .map(|s| (FactKind::Package, s.as_str())), + ) + .chain( + node.workspaces + .iter() + .map(|w| (FactKind::Workspace, w.ecosystem.as_str())), + ); + for (kind, value) in facts { + if let Some(ids) = self.indexes[kind as usize].get(value) { + candidates.extend(ids); + } + } + for (pattern, index) in &self.wildcard_files { + if node.files.iter().any(|file| pattern.matches(file)) { + candidates.push(index); + } + } + candidates.sort_unstable(); + candidates.dedup(); + candidates + .into_iter() + .filter_map(|&index| self.rules[index].evaluate(node)) + .collect() + } +} + +impl Rule { + fn evaluate(&self, node: &Node) -> Option { + if !self.all.iter().all(|p| p.matches(node)) + || (!self.any.is_empty() && !self.any.iter().any(|p| p.matches(node))) + || self.none.iter().any(|p| p.matches(node)) + { + return None; + } + let mut evidence = Vec::new(); + let mut roles = Vec::new(); + for atom in self + .all + .iter() + .chain(&self.any) + .filter(|p| p.matches(node)) + .flat_map(|p| &p.atoms) + { + if let Some(found) = atom.matched(node) { + found.evidence(node, &atom.text, &mut evidence); + match atom.kind { + FactKind::Identity => roles.push(FrameworkRole::Provides), + FactKind::Dependency => roles.push(FrameworkRole::Uses), + _ => (), + } + } + } + evidence.sort(); + evidence.dedup(); + roles.sort(); + roles.dedup(); + if self.kind != Kind::Framework { + roles.clear(); + } else if roles.is_empty() { + roles.push(FrameworkRole::Uses); + } + Some(Detection { + id: self.id.clone(), + language: self.language.clone(), + kind: self.kind, + priority: self.priority, + roles, + evidence, + }) + } +} + +enum Matched<'a> { + File(&'a str), + Dependency(&'a super::Dependency), + Workspace(&'a super::Workspace), + Package(&'a str), +} + +impl Matched<'_> { + fn evidence(self, node: &Node, pattern: &str, evidence: &mut Vec) { + let (source, value) = match self { + Self::Package(source) => (source.to_owned(), None), + Self::File(name) => (super::relative_file(&node.path, name), None), + Self::Dependency(dep) => { + for declaration in &dep.declarations { + evidence.push(Evidence { + source: declaration.source.clone(), + pattern: pattern.to_owned(), + value: Some(declaration.value.clone()), + }); + } + if let Some(resolved) = &dep.resolved { + evidence.push(Evidence { + source: resolved.source.clone(), + pattern: pattern.to_owned(), + value: Some(resolved.value.clone()), + }); + } + (dep.source.clone(), dep.version.clone()) + } + Self::Workspace(ws) => (ws.source.clone(), None), + }; + evidence.push(Evidence { + source, + pattern: pattern.to_owned(), + value, + }); + } +} + +impl Atom { + fn matched<'a>(&self, node: &'a Node) -> Option> { + match self.kind { + FactKind::File => { + let file = if matches!(self.file, FilePattern::Exact(_)) { + node.files.get(&self.value) + } else { + node.files.iter().find(|name| self.file.matches(name)) + }; + file.map(|name| Matched::File(name)) + } + FactKind::Source => node + .sources + .get(&self.value) + .map(|name| Matched::File(name)), + FactKind::Dependency => node.dependencies.get(&self.value).map(Matched::Dependency), + FactKind::Identity => node.identities.get(&self.value).map(Matched::Dependency), + FactKind::Package => node + .packages + .get(&self.value) + .map(|source| Matched::Package(source)), + FactKind::Workspace => node + .workspaces + .iter() + .find(|w| w.ecosystem == self.value) + .map(Matched::Workspace), + } + } +} +pub fn glob(pattern: &str, value: &str) -> bool { + let (p, v) = (pattern.as_bytes(), value.as_bytes()); + let (mut i, mut j, mut star, mut retry) = (0, 0, None, 0); + while j < v.len() { + if i < p.len() && (p[i] == b'?' || p[i] == v[j]) { + i += 1; + j += 1; + } else if i < p.len() && p[i] == b'*' { + star = Some(i); + i += 1; + retry = j; + } else if let Some(s) = star { + retry += 1; + j = retry; + i = s + 1; + } else { + return false; + } + } + while i < p.len() && p[i] == b'*' { + i += 1; + } + i == p.len() +} + +pub fn path_glob(pattern: &str, value: &str) -> bool { + let pattern = pattern.strip_prefix("./").unwrap_or(pattern); + let (head, tail) = pattern.split_once('/').unwrap_or((pattern, "")); + if head == "**" { + if tail.is_empty() { + return true; + } + if path_glob(tail, value) { + return true; + } + return value + .split_once('/') + .is_some_and(|(_, rest)| path_glob(pattern, rest)); + } + let (part, rest) = value.split_once('/').unwrap_or((value, "")); + glob(head, part) + && if tail.is_empty() { + rest.is_empty() + } else if rest.is_empty() { + tail.split('/').all(|part| part == "**") + } else { + path_glob(tail, rest) + } +} diff --git a/src/detect/rules.toml b/src/detect/rules.toml new file mode 100644 index 0000000..e95d768 --- /dev/null +++ b/src/detect/rules.toml @@ -0,0 +1,1001 @@ +[catalog.scan] +ignore = [".git", ".jj", ".svn", "node_modules", "vendor", "target", ".venv", ".gradle", ".vs", "venv", "__pycache__", ".next", ".nuxt", ".output", "dist", "build", "coverage", ".bundle", ".brainpod", ".cache", "site-packages"] + +[[catalog.scan.prune]] +when_files = ["*.csproj"] +directories = ["bin", "obj"] + +[catalog.families] +javascript = "node" +typescript = "node" + +[[catalog.inheritance]] +kind = "pin" + +[[catalog.inheritance]] +kind = "toolchain" + +[[catalog.inheritance]] +kind = "sdk" +boundary_files = ["global.json"] + +[[catalog.workspaces]] +ecosystem = "cargo" +files = ["Cargo.toml"] + +[[catalog.workspaces]] +ecosystem = "go" +files = ["go.mod"] + +[[catalog.workspaces]] +ecosystem = "node" +files = ["package.json"] + +[[catalog.workspaces]] +ecosystem = "deno" +files = ["deno.json", "deno.jsonc", "package.json"] + +[[catalog.workspaces]] +ecosystem = "python" +files = ["pyproject.toml"] + +[[catalog.workspaces]] +ecosystem = "maven" +files = ["pom.xml"] + +[[catalog.workspaces]] +ecosystem = "dotnet" +package = "dotnet" + +[[catalog.workspaces]] +ecosystem = "gradle" +package = "gradle" + +[catalog.reference_kinds.PackageReference] +ecosystem = "nuget" +normalize = "lowercase" + +[catalog.reference_kinds.PackageVersion] +ecosystem = "nuget" +normalize = "lowercase" +catalog = true + +[catalog.reference_kinds.FrameworkReference] +ecosystem = "dotnet-framework" +normalize = "lowercase" + +[catalog.reference_kinds.Reference] +ecosystem = "dotnet-reference" +normalize = "lowercase" +separator = "," + +[[catalog.property_versions]] +reader = "msbuild" +fields = ["targetframework", "targetframeworks", "targetframeworkversion"] +language = "csharp" +kind = "target" +split = ";" + +[[catalog.property_versions]] +reader = "msbuild" +fields = ["langversion"] +language = "csharp" +kind = "language" +split = ";" + +[[catalog.property_versions]] +reader = "msbuild" +fields = ["runtimeframeworkversion"] +language = "csharp" +kind = "runtime" +split = ";" + +[[catalog.property_versions]] +reader = "maven" +fields = ["java.version"] +language = "java" +kind = "requirement" + +[[catalog.property_versions]] +reader = "maven" +fields = ["maven.compiler.release"] +language = "java" +kind = "release" + +[[catalog.property_versions]] +reader = "maven" +fields = ["maven.compiler.source"] +language = "java" +kind = "source" + +[[catalog.property_versions]] +reader = "maven" +fields = ["maven.compiler.target"] +language = "java" +kind = "target" + +[[catalog.property_versions]] +reader = "maven-compiler" +fields = ["release"] +language = "java" +kind = "release" + +[[catalog.property_versions]] +reader = "maven-compiler" +fields = ["source"] +language = "java" +kind = "source" + +[[catalog.property_versions]] +reader = "maven-compiler" +fields = ["target"] +language = "java" +kind = "target" + +[[catalog.property_versions]] +reader = "gradle" +fields = ["sourceCompatibility"] +language = "java" +kind = "source" + +[[catalog.property_versions]] +reader = "gradle" +fields = ["targetCompatibility"] +language = "java" +kind = "target" + +[[catalog.property_versions]] +reader = "gradle" +fields = ["languageVersion", "languageVersion.set"] +language = "java" +kind = "toolchain" + +[[catalog.property_versions]] +reader = "gradle" +fields = ["options.release", "options.release.set"] +language = "java" +kind = "release" + +[[catalog.manifests]] +id = "composer" +files = ["composer.json"] +reader = "json" +names = ["/name"] +[catalog.manifests.identity] +ecosystem = "composer" +paths = ["/name"] +versions = ["/version"] +[[catalog.manifests.dependencies]] +ecosystem = "composer" +paths = ["/require"] +[[catalog.manifests.versions]] +language = "php" +kind = "requirement" +paths = ["/require/php"] +[[catalog.manifests.versions]] +language = "php" +kind = "platform" +paths = ["/config/platform/php"] + +[[catalog.manifests]] +id = "npm" +files = ["package.json"] +reader = "json" +names = ["/name"] +[catalog.manifests.identity] +ecosystem = "npm" +paths = ["/name"] +versions = ["/version"] +[[catalog.manifests.dependencies]] +ecosystem = "npm" +paths = ["/peerDependencies", "/devDependencies", "/dependencies"] +[[catalog.manifests.versions]] +language = "javascript" +kind = "requirement" +paths = ["/engines/node"] +[[catalog.manifests.versions]] +language = "javascript" +kind = "pin" +paths = ["/volta/node"] +[[catalog.manifests.versions]] +runtime = "deno" +kind = "requirement" +paths = ["/engines/deno"] +[[catalog.manifests.versions]] +language = "typescript" +kind = "requirement" +dependency = "npm:typescript" +reference_prefix = "catalog:" +[[catalog.manifests.workspaces]] +ecosystem = "node" +present = "/workspaces" +members = ["/workspaces/packages", "/workspaces"] + +[[catalog.manifests]] +id = "deno" +files = ["deno.json", "deno.jsonc"] +prefer_first = true +reader = "jsonc" +package = "deno" +runtime = "deno" +names = ["/name"] +[catalog.manifests.identity] +ecosystem = "jsr" +paths = ["/name"] +versions = ["/version"] +[[catalog.manifests.versions]] +language = "typescript" +kind = "requirement" +dependency = "npm:typescript" +reference_prefix = "catalog:" +[[catalog.manifests.workspaces]] +ecosystem = "deno" +present = "/workspace" +members = ["/workspace/members", "/workspace"] + +[[catalog.manifests]] +id = "deno-pin" +files = [".deno-version"] +reader = "pin" +[[catalog.manifests.versions]] +runtime = "deno" +kind = "pin" + +[[catalog.manifests]] +id = "cargo" +files = ["Cargo.toml"] +reader = "toml" +resolver = "cargo" +package = "cargo" +package_path = "/package" +names = ["/package/name"] +[catalog.manifests.identity] +ecosystem = "cargo" +paths = ["/package/name"] +versions = ["/package/version"] +[[catalog.manifests.versions]] +language = "rust" +kind = "requirement" +paths = ["/package/rust-version"] +[[catalog.manifests.workspaces]] +ecosystem = "cargo" +present = "/workspace" +members = ["/workspace/members"] +exclude = ["/workspace/exclude"] + +[[catalog.manifests]] +id = "python" +files = ["pyproject.toml"] +reader = "toml" +names = ["/project/name", "/tool/poetry/name"] +[catalog.manifests.identity] +ecosystem = "pypi" +paths = ["/project/name", "/tool/poetry/name"] +versions = ["/project/version", "/tool/poetry/version"] +normalize = "python" +[[catalog.manifests.dependencies]] +ecosystem = "pypi" +paths = ["/project/dependencies"] +requirements = true +normalize = "python" +[[catalog.manifests.dependencies]] +ecosystem = "pypi" +paths = ["/tool/poetry/dependencies"] +exclude = ["python"] +normalize = "python" +[[catalog.manifests.versions]] +language = "python" +kind = "requirement" +paths = ["/project/requires-python"] +[[catalog.manifests.versions]] +language = "python" +kind = "requirement" +paths = ["/tool/poetry/dependencies/python"] +[[catalog.manifests.workspaces]] +ecosystem = "python" +present = "/tool/uv/workspace" +members = ["/tool/uv/workspace/members"] +exclude = ["/tool/uv/workspace/exclude"] + +[[catalog.manifests]] +id = "pipenv" +files = ["Pipfile"] +reader = "toml" +[[catalog.manifests.dependencies]] +ecosystem = "pypi" +paths = ["/packages"] +normalize = "python" +[[catalog.manifests.versions]] +language = "python" +kind = "requirement" +paths = ["/requires/python_full_version", "/requires/python_version"] + +[[catalog.manifests]] +id = "pnpm" +files = ["pnpm-workspace.yaml"] +reader = "pnpm" + +[[catalog.manifests]] +id = "go-module" +files = ["go.mod"] +reader = "go" + +[[catalog.manifests]] +id = "go-workspace" +files = ["go.work"] +reader = "go" +workspace = "go" + +[[catalog.manifests]] +id = "bundler" +files = ["Gemfile"] +reader = "gemfile" + +[[catalog.manifests]] +id = "rubygems" +files = ["*.gemspec"] +reader = "gemspec" + +[[catalog.manifests]] +id = "requirements" +files = ["requirements*.txt"] +reader = "requirements" + +[[catalog.manifests]] +id = "ruby-pin" +files = [".ruby-version"] +reader = "pin" +[[catalog.manifests.versions]] +language = "ruby" +kind = "pin" + +[[catalog.manifests]] +id = "python-pin" +files = [".python-version"] +reader = "pin" +[[catalog.manifests.versions]] +language = "python" +kind = "pin" + +[[catalog.manifests]] +id = "node-pin" +files = [".node-version", ".nvmrc"] +reader = "pin" +[[catalog.manifests.versions]] +language = "javascript" +kind = "pin" + +[[catalog.manifests]] +id = "rust-pin" +files = ["rust-toolchain"] +reader = "pin" +[[catalog.manifests.versions]] +language = "rust" +kind = "pin" + +[[catalog.manifests]] +id = "go-pin" +files = [".go-version"] +reader = "pin" +[[catalog.manifests.versions]] +language = "go" +kind = "pin" + +[[catalog.manifests]] +id = "php-pin" +files = [".php-version"] +reader = "pin" +[[catalog.manifests.versions]] +language = "php" +kind = "pin" + +[[catalog.manifests]] +id = "java-pin" +files = [".java-version"] +reader = "pin" +[[catalog.manifests.versions]] +language = "java" +kind = "pin" + +[[catalog.manifests]] +id = "rust-toolchain" +files = ["rust-toolchain.toml"] +reader = "toml" +[[catalog.manifests.versions]] +language = "rust" +kind = "toolchain" +paths = ["/toolchain/channel"] + +[[catalog.manifests]] +id = "tool-versions" +files = [".tool-versions"] +reader = "tools" +[[catalog.manifests.versions]] +language = "javascript" +kind = "pin" +key = "nodejs" +[[catalog.manifests.versions]] +language = "go" +kind = "pin" +key = "golang" +[[catalog.manifests.versions]] +language = "java" +kind = "pin" +key = "java" +[[catalog.manifests.versions]] +language = "python" +kind = "pin" +key = "python" +[[catalog.manifests.versions]] +language = "ruby" +kind = "pin" +key = "ruby" +[[catalog.manifests.versions]] +language = "rust" +kind = "pin" +key = "rust" +[[catalog.manifests.versions]] +language = "php" +kind = "pin" +key = "php" + +[[catalog.manifests.versions]] +runtime = "deno" +kind = "pin" +key = "deno" + +[[catalog.manifests]] +id = "sdkman" +files = [".sdkmanrc"] +reader = "properties" +[[catalog.manifests.versions]] +language = "java" +kind = "pin" +key = "java" + +[[catalog.manifests]] +id = "dotnet-sdk" +files = ["global.json"] +reader = "json" +[[catalog.manifests.versions]] +language = "csharp" +kind = "sdk" +paths = ["/sdk/version"] + +[[catalog.manifests]] +id = "dotnet-project" +files = ["*.csproj"] +reader = "msbuild" +package = "dotnet" + +[[catalog.manifests]] +id = "solution-xml" +files = ["*.slnx"] +reader = "msbuild" +workspace = "dotnet" + +[[catalog.manifests]] +id = "msbuild-metadata" +files = ["*.props", "*.targets"] +reader = "msbuild" + +[[catalog.manifests]] +id = "solution" +files = ["*.sln"] +reader = "solution" +workspace = "dotnet" + +[[catalog.manifests]] +id = "maven" +files = ["pom.xml"] +reader = "maven" + +[[catalog.manifests]] +id = "gradle" +files = ["*.gradle", "*.gradle.kts"] +reader = "gradle" + +[[catalog.manifests]] +id = "gradle-properties" +files = ["gradle.properties"] +reader = "gradle-properties" + +[[catalog.manifests]] +id = "gradle-jvm" +files = ["gradle-daemon-jvm.properties"] +reader = "gradle-jvm" + +[[catalog.manifests]] +id = "version-catalog" +files = ["*.versions.toml"] +reader = "version-catalog" + +[[rules]] +id = "composer" +language = "php" +kind = "project" +priority = 10 +any = ["file:composer.json"] + +[[rules]] +id = "php-source" +language = "php" +kind = "source" +priority = 0 +any = ["source:php"] + +[[rules]] +id = "go-module" +language = "go" +kind = "project" +priority = 10 +any = ["file:go.mod"] + +[[rules]] +id = "go-source" +language = "go" +kind = "source" +priority = 0 +any = ["source:go"] + +[[rules]] +id = "cargo" +language = "rust" +kind = "project" +priority = 10 +any = ["package:cargo"] + +[[rules]] +id = "rust-source" +language = "rust" +kind = "source" +priority = 0 +any = ["source:rs"] + +[[rules]] +id = "bundler" +language = "ruby" +kind = "project" +priority = 10 +any = ["file:Gemfile"] + +[[rules]] +id = "ruby-source" +language = "ruby" +kind = "source" +priority = 0 +any = ["source:rb"] + +[[rules]] +id = "python" +language = "python" +kind = "project" +priority = 10 +any = ["file:pyproject.toml"] + +[[rules]] +id = "python-source" +language = "python" +kind = "source" +priority = 0 +any = ["source:py"] + +[[rules]] +id = "node" +language = "javascript" +kind = "project" +priority = 10 +any = ["file:package.json"] + +[[rules]] +id = "deno" +language = "javascript" +kind = "project" +priority = 10 +any = ["package:deno"] + +[[rules]] +id = "deno-workspace" +language = "javascript" +kind = "workspace" +priority = 20 +any = ["workspace:deno"] + +[[rules]] +id = "fresh" +language = "typescript" +kind = "framework" +priority = 100 +any = ["dep:jsr:@fresh/core", "identity:jsr:@fresh/core", "dep:deno:fresh"] + +[[rules]] +id = "oak" +language = "typescript" +kind = "framework" +priority = 100 +any = ["dep:jsr:@oak/oak", "identity:jsr:@oak/oak", "dep:deno:oak"] + +[[rules]] +id = "javascript-source" +language = "javascript" +kind = "source" +priority = 0 +any = ["source:js", "source:mjs", "source:cjs", "source:jsx"] + +[[rules]] +id = "typescript" +language = "typescript" +kind = "project" +priority = 10 +any = ["file:tsconfig.json", "dep:npm:typescript"] + +[[rules]] +id = "typescript-source" +language = "typescript" +kind = "source" +priority = 0 +any = ["source:ts", "source:mts", "source:cts", "source:tsx"] + +[[rules]] +id = "python-requirements" +language = "python" +kind = "project" +priority = 10 +any = ["file:requirements.txt", "file:Pipfile", "file:setup.py", "file:setup.cfg"] + +[[rules]] +id = "go-workspace" +language = "go" +kind = "workspace" +priority = 20 +any = ["workspace:go"] + +[[rules]] +id = "cargo-workspace" +language = "rust" +kind = "workspace" +priority = 20 +any = ["workspace:cargo"] + +[[rules]] +id = "node-workspace" +language = "javascript" +kind = "workspace" +priority = 20 +any = ["workspace:node"] + +[[rules]] +id = "python-workspace" +language = "python" +kind = "workspace" +priority = 20 +any = ["workspace:python"] + +[[rules]] +id = "laravel" +language = "php" +kind = "framework" +priority = 100 +any = ["dep:composer:laravel/framework", "identity:composer:laravel/framework"] + +[[rules]] +id = "symfony" +language = "php" +kind = "framework" +priority = 100 +any = ["dep:composer:symfony/framework-bundle", "dep:composer:symfony/symfony", "identity:composer:symfony/framework-bundle", "identity:composer:symfony/symfony"] + +[[rules]] +id = "cakephp" +language = "php" +kind = "framework" +priority = 100 +any = ["dep:composer:cakephp/cakephp", "identity:composer:cakephp/cakephp"] + +[[rules]] +id = "codeigniter" +language = "php" +kind = "framework" +priority = 100 +any = ["dep:composer:codeigniter4/framework", "dep:composer:codeigniter/framework", "identity:composer:codeigniter4/framework", "identity:composer:codeigniter/framework"] + +[[rules]] +id = "slim" +language = "php" +kind = "framework" +priority = 100 +any = ["dep:composer:slim/slim", "identity:composer:slim/slim"] + +[[rules]] +id = "axum" +language = "rust" +kind = "framework" +priority = 100 +any = ["dep:cargo:axum", "identity:cargo:axum"] + +[[rules]] +id = "rocket" +language = "rust" +kind = "framework" +priority = 100 +any = ["dep:cargo:rocket", "identity:cargo:rocket"] + +[[rules]] +id = "actix-web" +language = "rust" +kind = "framework" +priority = 100 +any = ["dep:cargo:actix-web", "identity:cargo:actix-web"] + +[[rules]] +id = "rails" +language = "ruby" +kind = "framework" +priority = 100 +any = ["dep:gem:rails", "identity:gem:rails", { all = ["dep:gem:actionpack", "dep:gem:railties"] }] + +[[rules]] +id = "sinatra" +language = "ruby" +kind = "framework" +priority = 100 +any = ["dep:gem:sinatra", "identity:gem:sinatra"] + +[[rules]] +id = "hanami" +language = "ruby" +kind = "framework" +priority = 100 +any = ["dep:gem:hanami", "identity:gem:hanami"] + +[[rules]] +id = "django" +language = "python" +kind = "framework" +priority = 100 +any = ["dep:pypi:django", "identity:pypi:django"] + +[[rules]] +id = "fastapi" +language = "python" +kind = "framework" +priority = 100 +any = ["dep:pypi:fastapi", "identity:pypi:fastapi"] + +[[rules]] +id = "flask" +language = "python" +kind = "framework" +priority = 100 +any = ["dep:pypi:flask", "identity:pypi:flask"] + +[[rules]] +id = "starlette" +language = "python" +kind = "framework" +priority = 50 +any = ["dep:pypi:starlette", "identity:pypi:starlette"] + +[[rules]] +id = "celery" +language = "python" +kind = "framework" +priority = 100 +any = ["dep:pypi:celery", "identity:pypi:celery"] + +[[rules]] +id = "gin" +language = "go" +kind = "framework" +priority = 100 +any = ["dep:go:github.com/gin-gonic/gin", "identity:go:github.com/gin-gonic/gin"] + +[[rules]] +id = "echo" +language = "go" +kind = "framework" +priority = 100 +any = ["dep:go:github.com/labstack/echo/v4", "identity:go:github.com/labstack/echo/v4"] + +[[rules]] +id = "fiber" +language = "go" +kind = "framework" +priority = 100 +any = ["dep:go:github.com/gofiber/fiber/v2", "identity:go:github.com/gofiber/fiber/v2"] + +[[rules]] +id = "nextjs" +language = "javascript" +kind = "framework" +priority = 100 +any = ["dep:npm:next", "identity:npm:next"] + +[[rules]] +id = "nuxtjs" +language = "javascript" +kind = "framework" +priority = 100 +any = ["dep:npm:nuxt", "dep:npm:nuxt3", "identity:npm:nuxt", "identity:npm:nuxt3"] + +[[rules]] +id = "astro" +language = "javascript" +kind = "framework" +priority = 100 +any = ["dep:npm:astro", "identity:npm:astro"] + +[[rules]] +id = "sveltekit" +language = "javascript" +kind = "framework" +priority = 100 +any = ["dep:npm:@sveltejs/kit", "identity:npm:@sveltejs/kit"] + +[[rules]] +id = "remix" +language = "javascript" +kind = "framework" +priority = 100 +any = ["dep:npm:@remix-run/react", "identity:npm:@remix-run/react"] + +[[rules]] +id = "angular" +language = "javascript" +kind = "framework" +priority = 100 +any = ["dep:npm:@angular/core", "identity:npm:@angular/core"] + +[[rules]] +id = "express" +language = "javascript" +kind = "framework" +priority = 100 +any = ["dep:npm:express", "identity:npm:express"] + +[[rules]] +id = "react" +language = "javascript" +kind = "framework" +priority = 50 +any = ["dep:npm:react", "identity:npm:react"] + +[[rules]] +id = "vue" +language = "javascript" +kind = "framework" +priority = 50 +any = ["dep:npm:vue", "identity:npm:vue"] + +[[rules]] +id = "vite" +language = "javascript" +kind = "framework" +priority = 50 +any = ["dep:npm:vite", "identity:npm:vite"] + +[[rules]] +id = "rubygems" +language = "ruby" +kind = "project" +priority = 10 +any = ["file:*.gemspec"] + +[[rules]] +id = "dotnet" +language = "csharp" +kind = "project" +priority = 10 +any = ["package:dotnet", "file:*.csproj"] + +[[rules]] +id = "csharp-source" +language = "csharp" +kind = "source" +priority = 0 +any = ["source:cs", "source:csx"] + +[[rules]] +id = "dotnet-workspace" +language = "csharp" +kind = "workspace" +priority = 20 +any = ["workspace:dotnet"] + +[[rules]] +id = "aspnetcore" +language = "csharp" +kind = "framework" +priority = 100 +any = ["dep:dotnet-sdk:microsoft.net.sdk.web", "dep:dotnet-sdk:microsoft.net.sdk.razor", "dep:dotnet-framework:microsoft.aspnetcore.app", "dep:nuget:microsoft.aspnetcore.app", "dep:nuget:microsoft.aspnetcore", "identity:nuget:microsoft.aspnetcore.app", "identity:nuget:microsoft.aspnetcore.app.ref", "identity:nuget:microsoft.aspnetcore.mvc"] + +[[rules]] +id = "aspnet" +language = "csharp" +kind = "framework" +priority = 100 +any = ["dep:nuget:microsoft.aspnet.mvc", "dep:nuget:microsoft.aspnet.webapi", "dep:dotnet-reference:system.web", "dep:dotnet-reference:system.web.mvc"] + +[[rules]] +id = "aspire" +language = "csharp" +kind = "framework" +priority = 110 +any = ["dep:dotnet-sdk:aspire.apphost.sdk", "dep:nuget:aspire.hosting.apphost"] + +[[rules]] +id = "fastendpoints" +language = "csharp" +kind = "framework" +priority = 110 +any = ["dep:nuget:fastendpoints", "identity:nuget:fastendpoints"] + +[[rules]] +id = "maven" +language = "java" +kind = "project" +priority = 10 +any = ["package:maven", "file:pom.xml"] + +[[rules]] +id = "gradle" +language = "java" +kind = "project" +priority = 10 +any = ["package:gradle", "file:build.gradle", "file:build.gradle.kts"] + +[[rules]] +id = "java-source" +language = "java" +kind = "source" +priority = 0 +any = ["source:java"] + +[[rules]] +id = "maven-workspace" +language = "java" +kind = "workspace" +priority = 20 +any = ["workspace:maven"] + +[[rules]] +id = "gradle-workspace" +language = "java" +kind = "workspace" +priority = 20 +any = ["workspace:gradle"] + +[[rules]] +id = "spring-boot" +language = "java" +kind = "framework" +priority = 110 +any = ["dep:maven-parent:org.springframework.boot:spring-boot-starter-parent", "dep:maven:org.springframework.boot:spring-boot", "dep:maven:org.springframework.boot:spring-boot-starter", "dep:maven:org.springframework.boot:spring-boot-starter-web", "dep:maven:org.springframework.boot:spring-boot-starter-webmvc", "dep:maven:org.springframework.boot:spring-boot-starter-webflux", "dep:maven-plugin:org.springframework.boot:spring-boot-maven-plugin", "dep:gradle-plugin:org.springframework.boot", "identity:maven:org.springframework.boot:spring-boot"] + +[[rules]] +id = "spring" +language = "java" +kind = "framework" +priority = 100 +any = ["dep:maven:org.springframework:spring-context", "dep:maven:org.springframework:spring-web", "dep:maven:org.springframework:spring-webmvc", "dep:maven:org.springframework:spring-webflux", "identity:maven:org.springframework:spring-core", "identity:maven:org.springframework:spring-context", "identity:maven:org.springframework:spring-web"] + +[[rules]] +id = "quarkus" +language = "java" +kind = "framework" +priority = 110 +any = ["dep:maven:io.quarkus:quarkus-core", "dep:maven:io.quarkus:quarkus-rest", "dep:maven-plugin:io.quarkus:quarkus-maven-plugin", "dep:gradle-plugin:io.quarkus", "identity:maven:io.quarkus:quarkus-core"] + +[[rules]] +id = "micronaut" +language = "java" +kind = "framework" +priority = 110 +any = ["dep:maven:io.micronaut:micronaut-runtime", "dep:gradle-plugin:io.micronaut.application", "identity:maven:io.micronaut:micronaut-runtime"] + +[[rules]] +id = "hono" +language = "typescript" +kind = "framework" +priority = 100 +any = ["dep:npm:hono", "identity:npm:hono", "dep:jsr:@hono/hono", "identity:jsr:@hono/hono", "dep:deno:hono"] diff --git a/src/detect/tests.rs b/src/detect/tests.rs new file mode 100644 index 0000000..b509fd2 --- /dev/null +++ b/src/detect/tests.rs @@ -0,0 +1,1222 @@ +use std::fs; +use std::path::Path; +use std::time::Instant; + +use super::{Project, Report, scan}; + +pub(super) fn fixture(files: &[(&str, &str)]) -> tempfile::TempDir { + let dir = tempfile::tempdir().unwrap(); + for (path, text) in files { + let path = dir.path().join(path); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, text).unwrap(); + } + dir +} + +pub(super) fn project<'a>(report: &'a Report, path: &str) -> &'a Project { + report + .projects + .iter() + .find(|p| p.path == path) + .unwrap_or_else(|| panic!("missing {path}: {report:#?}")) +} + +pub(super) fn detects(project: &Project, id: &str) -> bool { + project.detections.iter().any(|d| d.id == id) +} + +pub(super) fn version(project: &Project, language: &str, value: &str, source: &str) -> bool { + project.languages.iter().any(|l| { + l.name == language + && l.versions + .iter() + .any(|v| v.value == value && v.source == source) + }) +} + +#[test] +fn frameworks_and_plain_projects() { + let cases = [ + ( + "laravel", + "composer.json", + r#"{"require":{"php":"^8.3","laravel/framework":"^12","symfony/console":"^7"}}"#, + "php", + ), + ( + "symfony", + "composer.json", + r#"{"require":{"symfony/framework-bundle":"^7.2"}}"#, + "php", + ), + ( + "symfony", + "composer.json", + r#"{"require":{"symfony/symfony":"^6.4"}}"#, + "php", + ), + ( + "cakephp", + "composer.json", + r#"{"require":{"cakephp/cakephp":"^5"}}"#, + "php", + ), + ( + "codeigniter", + "composer.json", + r#"{"require":{"codeigniter4/framework":"^4"}}"#, + "php", + ), + ( + "composer", + "composer.json", + r#"{"require":{"symfony/console":"^7"}}"#, + "php", + ), + ( + "go-module", + "go.mod", + "module example.com/service\ngo 1.24\n", + "go", + ), + ( + "gin", + "go.mod", + "module example.com/api\ngo 1.24\nrequire (\n github.com/gin-gonic/gin v1.10.0\n)\n", + "go", + ), + ( + "axum", + "Cargo.toml", + "[package]\nname='api'\nrust-version='1.85'\n[dependencies]\naxum='0.8'\n", + "rust", + ), + ( + "rocket", + "Cargo.toml", + "[package]\nname='api'\n[dependencies]\nrocket={version='0.5',features=['json']}\n", + "rust", + ), + ( + "cargo", + "Cargo.toml", + "[package]\nname='plain'\nversion='0.1.0'\nedition='2024'\n", + "rust", + ), + ( + "rails", + "Gemfile", + "source 'https://rubygems.org'\nruby '3.3.1'\ngem 'rails', '~> 8.0'\n", + "ruby", + ), + ( + "sinatra", + "Gemfile", + "gem(\"sinatra\", \"~> 4.0\")\n", + "ruby", + ), + ("hanami", "Gemfile", "gem 'hanami', '~> 2.2'\n", "ruby"), + ( + "django", + "requirements.txt", + "# FastAPI\nDjango>=5.1,<6\n", + "python", + ), + ( + "fastapi", + "pyproject.toml", + "[project]\nname='api'\nrequires-python='>=3.12'\ndependencies=['FastAPI[standard]>=0.115; python_version >= \"3.12\"']\n", + "python", + ), + ( + "flask", + "Pipfile", + "[requires]\npython_version='3.12'\n[packages]\nflask='>=3'\n", + "python", + ), + ( + "django", + "pyproject.toml", + "[tool.poetry.dependencies]\npython='^3.12'\ndjango={version='^5.1'}\n", + "python", + ), + ( + "nextjs", + "package.json", + r#"{"dependencies":{"next":"^15","react":"^19"},"engines":{"node":">=22"}}"#, + "javascript", + ), + ( + "nuxtjs", + "package.json", + r#"{"dependencies":{"nuxt":"^3","vue":"^3"}}"#, + "javascript", + ), + ( + "astro", + "package.json", + r#"{"devDependencies":{"astro":"^5"}}"#, + "javascript", + ), + ( + "sveltekit", + "package.json", + r#"{"devDependencies":{"@sveltejs/kit":"^2"}}"#, + "javascript", + ), + ( + "express", + "package.json", + r#"{"dependencies":{"express":"^5"}}"#, + "javascript", + ), + ]; + for (expected, file, text, language) in cases { + let dir = fixture(&[(file, text)]); + let report = scan(dir.path(), None).unwrap(); + assert!( + report.diagnostics.is_empty(), + "{expected}: {:?}", + report.diagnostics + ); + assert_eq!(report.projects.len(), 1, "{expected}: {report:#?}"); + let p = &report.projects[0]; + assert_eq!(p.detections[0].id, expected, "{p:#?}"); + assert!(p.languages.iter().any(|l| l.name == language)); + } +} + +#[test] +fn bare_sources_never_need_to_be_read() { + for (extension, language) in [ + ("php", "php"), + ("go", "go"), + ("rs", "rust"), + ("rb", "ruby"), + ("py", "python"), + ("js", "javascript"), + ("tsx", "typescript"), + ] { + let dir = fixture(&[(&format!("main.{extension}"), "not even valid source")]); + let report = scan(dir.path(), None).unwrap(); + assert_eq!(report.stats.files_read, 0); + assert_eq!(report.projects.len(), 1); + assert_eq!(report.projects[0].languages[0].name, language); + assert!(report.projects[0].languages[0].versions.is_empty()); + } +} + +#[test] +fn complex_polyglot_monorepo() { + let dir = fixture(&[ + ( + "package.json", + r#"{"name":"platform","private":true,"workspaces":["apps/*","packages/*"]}"#, + ), + (".nvmrc", "22.14.0\n"), + (".tool-versions", "python 3.13.2\nruby 3.4.1\n"), + ( + "go.work", + "go 1.24.0\nuse (\n ./services/gateway\n ./services/jobs\n)\n", + ), + ( + "Cargo.toml", + "[workspace]\nmembers=['crates/*']\nexclude=['crates/excluded']\n[workspace.package]\nrust-version='1.85'\n[workspace.dependencies]\nweb={package='axum',version='0.8'}\nrocket='0.5'\n", + ), + ("rust-toolchain.toml", "[toolchain]\nchannel='1.86.0'\n"), + ( + "apps/web/package.json", + r#"{"name":"web","dependencies":{"next":"^15","react":"^19"},"devDependencies":{"typescript":"^5.7"},"engines":{"node":">=20"}}"#, + ), + ("apps/web/tsconfig.json", "{}"), + ("apps/web/src/app/page.tsx", "export default () => null"), + ("apps/web/src/lib/helper.ts", "export const x = 1"), + ( + "apps/docs/package.json", + r#"{"dependencies":{"astro":"^5"}}"#, + ), + ( + "apps/store/package.json", + r#"{"dependencies":{"nuxt":"^3"}}"#, + ), + ( + "packages/ui/package.json", + r#"{"name":"ui","dependencies":{"react":"^19"}}"#, + ), + ( + "services/gateway/go.mod", + "module example.com/gateway\ngo 1.23.0\ntoolchain go1.24.1\n", + ), + ("services/gateway/cmd/api/main.go", "package main"), + ( + "services/jobs/go.mod", + "module example.com/jobs\ngo 1.24.0\n", + ), + ("services/jobs/main.go", "package main"), + ( + "crates/api/Cargo.toml", + "[package]\nname='api'\nrust-version.workspace=true\n[dependencies]\nweb.workspace=true\n", + ), + ("crates/api/src/main.rs", "fn main() {}"), + ( + "crates/rocket/Cargo.toml", + "[package]\nname='rocket-api'\n[dependencies]\nrocket.workspace=true\n", + ), + ("crates/excluded/Cargo.toml", "[package]\nname='excluded'\n"), + ( + "services/php/composer.json", + r#"{"require":{"php":"^8.3","laravel/framework":"^12","symfony/console":"^7"}}"#, + ), + ( + "services/php/package.json", + r#"{"devDependencies":{"vite":"^6"},"engines":{"node":">=22"}}"#, + ), + ("services/php/app/Models/User.php", "= 3.3'\ngem 'rails', '~> 8.0'\n", + ), + ("services/rails/app/models/user.rb", "class User; end"), + ("services/django/requirements.txt", "Django==5.1.6\n"), + ("services/django/project/settings.py", "DEBUG = False"), + ( + "services/fastapi/pyproject.toml", + "[project]\nname='fastapi-service'\nrequires-python='>=3.12'\ndependencies=['fastapi>=0.115']\n", + ), + ("services/fastapi/app/main.py", "# service"), + ("tools/plain/main.go", "package main"), + ("tools/script/task.py", "print('hello')"), + ( + "node_modules/fake/package.json", + r#"{"dependencies":{"next":"1"}}"#, + ), + ( + "services/php/vendor/fake/composer.json", + r#"{"require":{"symfony/framework-bundle":"1"}}"#, + ), + ("target/fake/Cargo.toml", "invalid"), + (".venv/fake/requirements.txt", "flask==1"), + ]); + let report = scan(dir.path(), None).unwrap(); + assert!(report.diagnostics.is_empty(), "{report:#?}"); + assert_eq!(report.projects.len(), 19, "{report:#?}"); + let root = project(&report, "."); + for detection in ["node-workspace", "go-workspace", "cargo-workspace"] { + assert!(detects(root, detection)); + } + assert_eq!( + root.workspace_members, + [ + "apps/docs", + "apps/store", + "apps/web", + "crates/api", + "crates/rocket", + "packages/ui", + "services/gateway", + "services/jobs" + ] + ); + for (path, detection) in [ + ("apps/web", "nextjs"), + ("apps/docs", "astro"), + ("apps/store", "nuxtjs"), + ("crates/api", "axum"), + ("crates/rocket", "rocket"), + ("services/php", "laravel"), + ("services/symfony", "symfony"), + ("services/cake", "cakephp"), + ("services/codeigniter", "codeigniter"), + ("services/rails", "rails"), + ("services/django", "django"), + ("services/fastapi", "fastapi"), + ] { + assert!(detects(project(&report, path), detection), "{path}"); + } + assert!(!detects(project(&report, "services/php"), "symfony")); + assert_eq!( + project(&report, "services/php") + .languages + .iter() + .map(|l| l.name.as_str()) + .collect::>(), + ["javascript", "php"] + ); + assert!(version( + project(&report, "apps/web"), + "javascript", + "22.14.0", + ".nvmrc" + )); + assert!(version( + project(&report, "apps/web"), + "javascript", + ">=20", + "apps/web/package.json" + )); + assert!(version( + project(&report, "apps/web"), + "typescript", + "^5.7", + "apps/web/package.json" + )); + assert!(version( + project(&report, "crates/api"), + "rust", + "1.85", + "Cargo.toml" + )); + assert!(version( + project(&report, "crates/api"), + "rust", + "1.86.0", + "rust-toolchain.toml" + )); + assert!(version( + project(&report, "services/django"), + "python", + "3.13.2", + ".tool-versions" + )); + assert!(version( + project(&report, "services/rails"), + "ruby", + ">= 3.3", + "services/rails/Gemfile" + )); + assert_eq!(report.stats.skipped_directories, 4); + assert_eq!(scan(dir.path(), None).unwrap(), report); +} + +#[test] +fn avoids_false_framework_matches() { + let dir = fixture(&[ + ( + "composer.json", + r#"{"description":"laravel/framework","require":{"symfony/console":"^7"},"require-dev":{"laravel/framework":"^12"}}"#, + ), + ( + "Cargo.toml", + "[package]\nname='not-axum'\n[dev-dependencies]\nrocket='0.5'\n[workspace.dependencies]\naxum='0.8'\n", + ), + ( + "Gemfile", + "# gem 'rails'\ngem 'rails-html-sanitizer'\ngem 'sinatra-contrib'\n", + ), + ( + "requirements.txt", + "# django==5\ndjango-filter>=24\nfastapi-utils==0.8\n", + ), + ( + "package.json", + r#"{"scripts":{"start":"next start"},"dependencies":{"next-themes":"*","@astrojs/check":"*"}}"#, + ), + ]); + let report = scan(dir.path(), None).unwrap(); + assert!( + report + .projects + .iter() + .flat_map(|p| &p.detections) + .all(|d| d.kind != super::Kind::Framework), + "{report:#?}" + ); +} + +#[test] +fn custom_patterns_and_priority() { + let dir = fixture(&[ + ("package.json", r#"{"dependencies":{"next":"15"}}"#), + ("special.config", ""), + ("disabled", ""), + ]); + let extra = r#" +[[rules]] +id = 'company-next' +language = 'javascript' +kind = 'framework' +priority = 999 +all = ['dep:npm:next', 'file:special.*'] +none = ['file:blocked'] +[[rules]] +id = 'must-not-match' +language = 'javascript' +kind = 'framework' +priority = 1000 +any = ['file:special.config'] +none = ['file:disabled'] +"#; + let report = scan(dir.path(), Some(extra)).unwrap(); + assert_eq!(report.projects[0].detections[0].id, "company-next"); + assert!(!detects(&report.projects[0], "must-not-match")); + let json = serde_json::to_value(&report).unwrap(); + assert_eq!(json["schemaVersion"], 1); + assert!(super::render(&json)[0].contains("company-next")); + for invalid in [ + extra.replace("company-next", "nextjs"), + extra.replace("file:blocked", "unsupported:x"), + extra.replace("priority = 999", "prio = 999"), + ] { + assert!(scan(dir.path(), Some(&invalid)).is_err()); + } +} + +#[test] +fn malformed_and_oversized_manifests_are_diagnostic_not_fatal() { + let dir = fixture(&[ + ("broken/package.json", "["), + ("valid/composer.json", "{}"), + ("invalid/Cargo.toml", "["), + ]); + fs::write(dir.path().join("broken/composer.json"), [0xff, 0xfe]).unwrap(); + fs::write( + dir.path().join("broken/Gemfile"), + vec![b'a'; super::MAX_FILE_BYTES as usize + 20], + ) + .unwrap(); + let report = scan(dir.path(), None).unwrap(); + assert_eq!(report.diagnostics.len(), 4, "{report:#?}"); + assert!(detects(project(&report, "valid"), "composer")); + assert!(report.stats.bytes_read <= super::MAX_FILE_BYTES as usize + 100); +} + +#[test] +fn version_constraints_keep_provenance_and_do_not_invent_versions() { + let dir = fixture(&[ + ( + "composer.json", + r#"{"require":{"php":"^8.2","laravel/framework":"^12"},"config":{"platform":{"php":"8.3.14"}}}"#, + ), + ( + "Cargo.toml", + "[package]\nname='plain'\nversion='9.9.9'\nedition='2024'\n", + ), + ("Gemfile", "gem 'rails', '~> 8.0', '>= 8.0.1'\n"), + (".python-version", "3.12.8\n"), + ( + "app/pyproject.toml", + "[project]\nrequires-python='>=3.11,<4'\n", + ), + ]); + let report = scan(dir.path(), None).unwrap(); + let p = project(&report, "."); + assert!(version(p, "php", "^8.2", "composer.json")); + assert!(version(p, "php", "8.3.14", "composer.json")); + assert!( + p.languages + .iter() + .find(|l| l.name == "rust") + .unwrap() + .versions + .is_empty() + ); + let rails = p.detections.iter().find(|d| d.id == "rails").unwrap(); + assert_eq!(rails.evidence[0].value.as_deref(), Some("~> 8.0, >= 8.0.1")); + assert!(version( + project(&report, "app"), + "python", + "3.12.8", + ".python-version" + )); +} + +#[test] +fn python_uv_workspace_and_workspace_exclusions() { + let dir = fixture(&[ + ( + "pyproject.toml", + "[tool.uv.workspace]\nmembers=['services/*']\nexclude=['services/ignored']\n", + ), + ( + "services/api/pyproject.toml", + "[project]\nname='api'\ndependencies=['fastapi==0.115']\n", + ), + ( + "services/ignored/pyproject.toml", + "[project]\nname='ignored'\n", + ), + ]); + let report = scan(dir.path(), None).unwrap(); + assert_eq!(project(&report, ".").workspace_members, ["services/api"]); +} + +#[test] +fn glob_matching_respects_directory_boundaries() { + for (pattern, value, expected) in [ + ("services/*", "services/api", true), + ("services/*", "services/api/nested", false), + ("services/**/api", "services/a/b/api", true), + ("services/**/api", "different/api", false), + ("services/**/api", "services/api", true), + ("./crates/*", "crates/api", true), + ] { + assert_eq!( + super::rules::path_glob(pattern, value), + expected, + "{pattern}: {value}" + ); + } +} + +#[cfg(unix)] +#[test] +fn symlinks_are_not_followed() { + let dir = fixture(&[("real/go.mod", "module example.com/api\ngo 1.24\n")]); + std::os::unix::fs::symlink(dir.path(), dir.path().join("real/loop")).unwrap(); + std::os::unix::fs::symlink(dir.path().join("real/go.mod"), dir.path().join("go.mod")).unwrap(); + let report = scan(dir.path(), None).unwrap(); + assert_eq!(report.projects.len(), 1); + assert_eq!(report.stats.files_read, 1); + assert_eq!(report.stats.directories, 2); +} + +#[test] +fn source_volume_does_not_increase_manifest_reads() { + let dir = fixture(&[("go.mod", "module example.com/api\ngo 1.24\n")]); + for i in 0..200 { + let path = dir.path().join(format!("pkg/p{i}")); + fs::create_dir_all(&path).unwrap(); + for j in 0..10 { + fs::write(path.join(format!("file{j}.go")), "package p").unwrap(); + } + } + let report = scan(dir.path(), None).unwrap(); + assert_eq!(report.projects.len(), 1); + assert_eq!(report.stats.files_read, 1); + assert_eq!(report.stats.files, 2001); +} + +#[test] +fn cli_is_local_and_authentication_free() { + use clap::Parser; + let opts = crate::Opts::try_parse_from([ + "brainpod", + "detect", + "./services", + "--json", + "--rules", + "custom.toml", + ]) + .unwrap(); + assert!(!crate::cmd::needs_client(&opts.command)); + assert!(!crate::cmd::needs_api_token(&opts.command)); + assert!(opts.json); + let crate::cmd::Command::Detect(args) = opts.command else { + panic!("expected detect"); + }; + assert_eq!(args.path, Path::new("./services")); + assert_eq!(args.rules.as_deref(), Some(Path::new("custom.toml"))); +} + +#[test] +#[ignore = "manual release-mode performance measurement"] +fn benchmark_polyglot_scan() { + let dir = tempfile::tempdir().unwrap(); + let manifests = [ + ( + "composer.json", + r#"{"require":{"php":"^8.3","laravel/framework":"^12"}}"#, + "php", + ), + ("go.mod", "module example.com/api\ngo 1.24\n", "go"), + ( + "Cargo.toml", + "[package]\nname='api'\n[dependencies]\naxum='0.8'\n", + "rs", + ), + ("Gemfile", "ruby '3.4'\ngem 'rails', '~> 8.0'\n", "rb"), + ( + "pyproject.toml", + "[project]\nrequires-python='>=3.12'\ndependencies=['fastapi>=0.115']\n", + "py", + ), + ( + "package.json", + r#"{"dependencies":{"next":"^15","react":"^19"},"engines":{"node":">=22"}}"#, + "js", + ), + ( + "package.json", + r#"{"dependencies":{"nuxt":"^3"},"devDependencies":{"typescript":"^5.7"}}"#, + "ts", + ), + ]; + for i in 0..200 { + let path = dir.path().join(format!("services/s{i}")); + fs::create_dir_all(path.join("src")).unwrap(); + let (file, manifest, extension) = manifests[i % manifests.len()]; + fs::write(path.join(file), manifest).unwrap(); + for j in 0..100 { + fs::write( + path.join(format!("src/file{j}.{extension}")), + vec![b'x'; 1024], + ) + .unwrap(); + } + } + let mut samples = Vec::new(); + for _ in 0..20 { + let start = Instant::now(); + let report = scan(dir.path(), None).unwrap(); + samples.push(start.elapsed().as_secs_f64() * 1000.0); + assert_eq!(report.projects.len(), 200); + assert_eq!(report.stats.files_read, 200); + assert_eq!(report.stats.files, 20200); + std::hint::black_box(report); + } + let first = samples[0]; + samples.sort_by(f64::total_cmp); + eprintln!( + "200 polyglot services, 20,200 files (20 warm filesystem runs): first {first:.3} ms, median {:.3} ms, p95 {:.3} ms; 200 manifest reads, zero source reads", + samples[10], samples[18] + ); +} + +#[test] +fn framework_definitions_are_not_confused_with_consumers() { + let cases = [ + ( + "react", + "package.json", + r#"{"name":"react","version":"19.3.0"}"#, + ), + ( + "nextjs", + "package.json", + r#"{"name":"next","version":"16.5.0-canary.45"}"#, + ), + ( + "vue", + "package.json", + r#"{"name":"vue","version":"2.7.16"}"#, + ), + ( + "laravel", + "composer.json", + r#"{"name":"laravel/framework"}"#, + ), + ("symfony", "composer.json", r#"{"name":"symfony/symfony"}"#), + ( + "axum", + "Cargo.toml", + "[package]\nname='axum'\nversion='0.8.0'\n", + ), + ( + "gin", + "go.mod", + "module github.com/gin-gonic/gin\ngo 1.26.0\n", + ), + ( + "django", + "pyproject.toml", + "[project]\nname='Django'\nversion='5.1'\n", + ), + ( + "rails", + "rails.gemspec", + "Gem::Specification.new do |s|\n s.name = 'rails'\n s.version = version\n s.required_ruby_version = '>= 3.3.5'\nend\n", + ), + ]; + for (id, file, content) in cases { + let dir = fixture(&[(file, content)]); + let report = scan(dir.path(), None).unwrap(); + assert!(report.diagnostics.is_empty(), "{id}: {report:#?}"); + let p = project(&report, "."); + let detection = p.detections.iter().find(|d| d.id == id).unwrap(); + assert_eq!(detection.roles, [super::FrameworkRole::Provides], "{id}"); + assert!( + detection + .evidence + .iter() + .all(|e| e.pattern.starts_with("identity:")) + ); + if id == "django" { + assert_eq!(p.names, ["Django"]); + } + if id == "rails" { + assert!(version(p, "ruby", ">= 3.3.5", "rails.gemspec")); + } + } + let dir = fixture(&[( + "package.json", + r#"{"name":"react","version":"19","dependencies":{"react":"18"}}"#, + )]); + let report = scan(dir.path(), None).unwrap(); + assert_eq!( + project(&report, ".") + .detections + .iter() + .find(|d| d.id == "react") + .unwrap() + .roles, + [super::FrameworkRole::Uses, super::FrameworkRole::Provides] + ); +} + +#[test] +fn rails_component_detection_requires_both_web_components() { + for (text, expected) in [ + ("gem 'activesupport', '~> 8.1'\n", false), + ("gem 'actionpack', '~> 8.1'\n", false), + ("gem 'railties', '~> 8.1'\n", false), + ( + "gem 'actionpack', '~> 8.1'\ngem 'railties', '~> 8.1'\n", + true, + ), + ] { + let dir = fixture(&[("Gemfile", text)]); + let report = scan(dir.path(), None).unwrap(); + let p = project(&report, "."); + assert_eq!(detects(p, "rails"), expected, "{text}"); + if expected { + let detection = p.detections.iter().find(|d| d.id == "rails").unwrap(); + assert_eq!(detection.roles, [super::FrameworkRole::Uses]); + assert_eq!(detection.evidence.len(), 2); + } + } +} + +#[test] +fn poetry_names_preserve_the_declared_spelling() { + let dir = fixture(&[( + "pyproject.toml", + "[tool.poetry]\nname='AutoGPT.Platform.Backend'\nversion='0.6.22'\n[tool.poetry.dependencies]\npython='>=3.11,<3.14'\nfastapi='^0.128.6'\n", + )]); + let report = scan(dir.path(), None).unwrap(); + let p = project(&report, "."); + assert_eq!(p.names, ["AutoGPT.Platform.Backend"]); + assert!(detects(p, "fastapi")); + assert!(version(p, "python", ">=3.11,<3.14", "pyproject.toml")); +} + +#[test] +fn pnpm_catalogs_preserve_references_and_resolve_local_versions() { + let dir = fixture(&[ + ( + "pnpm-workspace.yaml", + "packages: ['apps/*', '!apps/excluded']\ncatalog: &default\n next: '16.0.0'\n typescript: 'npm:@typescript/typescript6@6.0.2'\ncatalogs:\n tooling:\n <<: *default\n typescript: '7.0.2'\n", + ), + ( + "apps/web/package.json", + r#"{"name":"web","dependencies":{"next":"catalog:"},"devDependencies":{"typescript":"catalog:tooling"}}"#, + ), + ("apps/excluded/package.json", "{}"), + ("apps/web/src/page.tsx", "source must not be read"), + ]); + let report = scan(dir.path(), None).unwrap(); + assert!(report.diagnostics.is_empty(), "{report:#?}"); + let root = project(&report, "."); + assert!(detects(root, "node-workspace")); + assert_eq!(root.workspace_members, ["apps/web"]); + let web = project(&report, "apps/web"); + assert!(version(web, "typescript", "7.0.2", "pnpm-workspace.yaml")); + assert!( + web.languages + .iter() + .flat_map(|l| &l.versions) + .any(|v| v.kind == "reference" && v.value == "catalog:tooling") + ); + let next = web.detections.iter().find(|d| d.id == "nextjs").unwrap(); + assert_eq!(next.roles, [super::FrameworkRole::Uses]); + assert!( + next.evidence + .iter() + .any(|e| e.source == "apps/web/package.json" && e.value.as_deref() == Some("catalog:")) + ); + assert!( + next.evidence + .iter() + .any(|e| e.source == "pnpm-workspace.yaml" && e.value.as_deref() == Some("16.0.0")) + ); + assert_eq!(report.stats.files_read, 3); +} + +#[test] +fn nested_pnpm_catalogs_do_not_leak_across_workspace_boundaries() { + let dir = fixture(&[ + ( + "pnpm-workspace.yaml", + "packages: ['apps/**']\ncatalog:\n typescript: '6.0.2'\n", + ), + ( + "apps/nested/pnpm-workspace.yaml", + "packages: ['packages/*']\ncatalog:\n typescript: '7.0.2'\n", + ), + ( + "apps/nested/packages/web/package.json", + r#"{"devDependencies":{"typescript":"catalog:"}}"#, + ), + ( + "apps/nested/excluded/package.json", + r#"{"devDependencies":{"typescript":"catalog:"}}"#, + ), + ]); + let report = scan(dir.path(), None).unwrap(); + let nested = project(&report, "apps/nested/packages/web"); + assert!(version( + nested, + "typescript", + "7.0.2", + "apps/nested/pnpm-workspace.yaml" + )); + assert!(!version( + nested, + "typescript", + "6.0.2", + "pnpm-workspace.yaml" + )); + let excluded = project(&report, "apps/nested/excluded"); + assert!( + excluded + .languages + .iter() + .flat_map(|l| &l.versions) + .all(|v| v.kind == "reference") + ); + assert_eq!(report.diagnostics.len(), 1); + assert_eq!( + report.diagnostics[0].path, + "apps/nested/excluded/package.json" + ); +} + +#[test] +fn pnpm_defaults_and_missing_catalog_entries_are_explicit() { + let dir = fixture(&[ + ("pnpm-workspace.yaml", "catalog:\n typescript: '6.0.2'\n"), + ( + "packages/web/package.json", + r#"{"devDependencies":{"typescript":"catalog:default"}}"#, + ), + ( + "packages/missing/package.json", + r#"{"devDependencies":{"typescript":"catalog:missing"}}"#, + ), + ]); + let report = scan(dir.path(), None).unwrap(); + assert_eq!( + project(&report, ".").workspace_members, + ["packages/missing", "packages/web"] + ); + assert!(version( + project(&report, "packages/web"), + "typescript", + "6.0.2", + "pnpm-workspace.yaml" + )); + assert_eq!(report.diagnostics.len(), 1); + assert!(report.diagnostics[0].message.contains("catalog:missing")); + let broken = fixture(&[ + ("pnpm-workspace.yaml", "packages: [broken"), + ("package.json", "{}"), + ]); + let report = scan(broken.path(), None).unwrap(); + assert_eq!(report.diagnostics.len(), 1); + assert!(detects(project(&report, "."), "node")); +} + +#[test] +fn conjunction_patterns_require_complete_groups_and_validate_atoms() { + let dir = fixture(&[("a.config", ""), ("b.config", "")]); + let rules = "[[rules]]\nid='grouped'\nlanguage='javascript'\nkind='framework'\npriority=999\nany=[{all=['file:a.config','file:b.config']}]\nnone=[{all=['file:a.config','file:blocked']}]\n"; + let report = scan(dir.path(), Some(rules)).unwrap(); + assert!(detects(project(&report, "."), "grouped")); + assert!( + scan( + dir.path(), + Some(&rules.replace("file:b.config", "file:missing")) + ) + .unwrap() + .projects + .is_empty() + ); + for invalid in [ + rules.replace("file:b.config", "invalid:atom"), + rules.replace("['file:a.config','file:b.config']", "[]"), + ] { + assert!(scan(dir.path(), Some(&invalid)).is_err()); + } +} + +#[test] +fn parallel_inventory_matches_serial_inventory() { + let dir = tempfile::tempdir().unwrap(); + for i in 0..128 { + let path = dir.path().join(format!("services/s{i}/src/deep")); + fs::create_dir_all(&path).unwrap(); + fs::write(path.join("main.ts"), "not read").unwrap(); + fs::write(path.join("irrelevant.md"), "not read").unwrap(); + fs::write(dir.path().join(format!("services/s{i}/package.json")), "{}").unwrap(); + } + let rules = super::Rules::builtin().unwrap(); + let serial = super::inventory(dir.path(), rules, 1).unwrap(); + let parallel = super::inventory(dir.path(), rules, 4).unwrap(); + assert_eq!(serial.stats, parallel.stats); + assert_eq!(serial.nodes.len(), parallel.nodes.len()); + for (a, b) in serial.nodes.iter().zip(¶llel.nodes) { + assert_eq!( + (&a.path, a.parent, &a.files, &a.sources, &a.contents), + (&b.path, b.parent, &b.files, &b.sources, &b.contents) + ); + } + assert_eq!(serial.stats.files_read, 128); + let report = scan(dir.path(), None).unwrap(); + assert_eq!(report.projects.len(), 128); + assert_eq!(scan(dir.path(), None).unwrap(), report); +} + +#[test] +fn trailing_globstars_match_zero_or_more_directories() { + for (pattern, path, expected) in [ + ("packages/**", "packages", true), + ("packages/**/**", "packages", true), + ("packages/*", "packages", false), + ("packages/**", "packages/web/src", true), + ] { + assert_eq!( + super::rules::path_glob(pattern, path), + expected, + "{pattern}: {path}" + ); + } +} + +#[test] +fn inherited_package_identity_versions_keep_workspace_provenance() { + let dir = fixture(&[ + ( + "Cargo.toml", + "[workspace]\nmembers=['crates/*']\n[workspace.package]\nversion='0.8.1'\n", + ), + ( + "crates/axum/Cargo.toml", + "[package]\nname='axum'\nversion.workspace=true\n", + ), + ]); + let report = scan(dir.path(), None).unwrap(); + let axum = project(&report, "crates/axum") + .detections + .iter() + .find(|d| d.id == "axum") + .unwrap(); + assert_eq!(axum.roles, [super::FrameworkRole::Provides]); + assert!( + axum.evidence + .iter() + .any(|e| e.source == "Cargo.toml" && e.value.as_deref() == Some("0.8.1")) + ); + assert!( + axum.evidence + .iter() + .all(|e| e.source != "crates/axum/Cargo.toml" || e.value.is_none()) + ); +} + +#[test] +fn malformed_nested_pnpm_files_block_outer_catalog_resolution() { + let dir = fixture(&[ + ( + "pnpm-workspace.yaml", + "packages: ['apps/**']\ncatalog:\n typescript: '6.0.2'\n", + ), + ("apps/nested/pnpm-workspace.yaml", "packages: [broken"), + ( + "apps/nested/web/package.json", + r#"{"devDependencies":{"typescript":"catalog:"}}"#, + ), + ]); + let report = scan(dir.path(), None).unwrap(); + assert_eq!(report.diagnostics.len(), 2); + let web = project(&report, "apps/nested/web"); + assert!( + web.languages + .iter() + .flat_map(|l| &l.versions) + .all(|v| v.kind == "reference") + ); +} + +#[test] +fn provider_file_guards_do_not_imply_framework_consumption() { + let dir = fixture(&[ + ("package.json", r#"{"name":"react","version":"19"}"#), + ("provider.config", ""), + ]); + let rules = "[[rules]]\nid='guarded-provider'\nlanguage='javascript'\nkind='framework'\npriority=200\nall=['identity:npm:react','file:provider.config']\n"; + let report = scan(dir.path(), Some(rules)).unwrap(); + assert_eq!( + project(&report, ".").detections[0].roles, + [super::FrameworkRole::Provides] + ); +} + +#[test] +fn custom_manifest_registration_emits_facts_without_a_new_parser() { + let dir = fixture(&[ + ( + "service.meta.json", + r#"{"project":{"name":"platform","version":"1.0"},"uses":{"next":"16"},"runtime":{"node":">=26"},"modules":["services/*"]}"#, + ), + ( + "services/api/service.meta.json", + r#"{"project":{"name":"api"},"runtime":{"node":">=24"}}"#, + ), + ("unrelated.json", "not a manifest and must not be parsed"), + ]); + let config = r#" +[[catalog.manifests]] +id = "service-meta" +files = ["service.meta.json"] +reader = "json" +package = "service-meta" +names = ["/project/name"] +[catalog.manifests.identity] +ecosystem = "company" +paths = ["/project/name"] +versions = ["/project/version"] +[[catalog.manifests.dependencies]] +ecosystem = "npm" +paths = ["/uses"] +[[catalog.manifests.versions]] +language = "javascript" +kind = "requirement" +paths = ["/runtime/node"] +[[catalog.manifests.workspaces]] +ecosystem = "service-meta" +present = "/modules" +members = ["/modules"] +[[catalog.workspaces]] +ecosystem = "service-meta" +package = "service-meta" +[[rules]] +id = "company-project" +language = "javascript" +kind = "project" +priority = 10 +any = ["package:service-meta"] +[[rules]] +id = "company-workspace" +language = "javascript" +kind = "workspace" +priority = 20 +any = ["workspace:service-meta"] +"#; + let report = scan(dir.path(), Some(config)).unwrap(); + assert!(report.diagnostics.is_empty(), "{report:#?}"); + assert_eq!(report.stats.files_read, 2); + let root = project(&report, "."); + assert_eq!(root.names, ["platform"]); + assert!(detects(root, "nextjs")); + assert!(detects(root, "company-workspace")); + assert_eq!(root.workspace_members, ["services/api"]); + assert!(version(root, "javascript", ">=26", "service.meta.json")); + assert!(version( + project(&report, "services/api"), + "javascript", + ">=24", + "services/api/service.meta.json" + )); +} + +#[test] +fn xml_property_and_reference_policies_are_extensible_in_toml() { + let dir = fixture(&[( + "App.csproj", + "15.0", + )]); + let config = r#" +[[catalog.property_versions]] +reader = "msbuild" +fields = ["companycompilerlevel"] +language = "csharp" +kind = "language" +[catalog.reference_kinds.CompanyReference] +ecosystem = "company" +normalize = "lowercase" +[[rules]] +id = "company-web" +language = "csharp" +kind = "framework" +priority = 100 +any = ["dep:company:company.web"] +"#; + let report = scan(dir.path(), Some(config)).unwrap(); + assert!(report.diagnostics.is_empty(), "{report:#?}"); + let app = project(&report, "."); + assert!(version(app, "csharp", "15.0", "App.csproj")); + assert!(detects(app, "company-web")); + assert_eq!( + app.detections + .iter() + .find(|d| d.id == "company-web") + .unwrap() + .roles, + [super::FrameworkRole::Uses] + ); +} + +#[test] +fn manifest_schema_errors_are_rejected_before_scanning() { + let config = "[[catalog.manifests]]\nid='custom'\nfiles=['custom.json']\nreader='json'\n"; + for invalid in [ + config.replace("reader='json'", "reader='command'"), + config.replace("custom.json", "../custom.json"), + format!("{config}{config}"), + format!("{config}names=['not-a-pointer']\n"), + format!("{config}unknown_option=true\n"), + "[[catalog.manifests]]\nid='conflict'\nfiles=['package.json']\nreader='toml'\n".to_owned(), + "[[catalog.property_versions]]\nreader='msbuid'\nfields=['newfield']\nlanguage='csharp'\nkind='language'\n".to_owned(), + ] { + let error = scan( + std::path::Path::new("/nonexistent-detection-fixture"), + Some(&invalid), + ) + .unwrap_err(); + assert!(!error.to_string().contains("could not open"), "{error}"); + } +} + +#[test] +fn compiled_basename_patterns_preserve_glob_matching() { + for pattern in [ + "*.csproj", + "requirements*.txt", + "a*b", + "a?b", + "a**b", + "Cargo.toml", + "ab*bc", + ] { + let compiled = super::rules::FilePattern::new(pattern); + for value in [ + "App.csproj", + "requirements.txt", + "requirements-dev.txt", + "ab", + "a123b", + "Cargo.toml", + "abc", + "abbc", + "", + ] { + assert_eq!( + compiled.matches(value), + super::rules::glob(pattern, value), + "{pattern}: {value}" + ); + } + } +} diff --git a/src/detect/values.rs b/src/detect/values.rs new file mode 100644 index 0000000..337f616 --- /dev/null +++ b/src/detect/values.rs @@ -0,0 +1,200 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Component, Path, PathBuf}; + +use super::{Dependency, Node, Version}; + +#[derive(Clone, Debug)] +pub(super) struct Located { + pub value: String, + pub source: String, + pub conditional: bool, +} + +pub(super) type Properties = BTreeMap; + +#[derive(Clone, Copy)] +pub(super) enum Syntax { + Maven, + MsBuild, +} + +pub(super) fn resolve(raw: &Located, properties: &Properties, syntax: Syntax) -> Option { + let (prefix, suffix) = match syntax { + Syntax::Maven => ("${", '}'), + Syntax::MsBuild => ("$(", ')'), + }; + let mut result = raw.clone(); + let mut seen = BTreeSet::new(); + for _ in 0..64 { + let Some(start) = result.value.find(prefix) else { + return Some(result); + }; + if !seen.insert(result.value.clone()) { + return None; + } + let end = result.value[start + 2..].find(suffix)? + start + 2; + let name = &result.value[start + 2..end]; + let name = match syntax { + Syntax::Maven => name.to_owned(), + Syntax::MsBuild => name.to_ascii_lowercase(), + }; + let replacement = properties.get(&name)?; + if replacement.conditional { + return None; + } + let entire = start == 0 && end + 1 == result.value.len(); + let length = result + .value + .len() + .checked_sub(end + 1 - start)? + .checked_add(replacement.value.len())?; + if length > super::MAX_FILE_BYTES as usize { + return None; + } + result + .value + .replace_range(start..end + 1, &replacement.value); + if entire { + result.source = replacement.source.clone(); + } + } + None +} + +pub(super) fn version( + node: &mut Node, + language: &str, + raw: &Located, + kind: &str, + properties: &Properties, + syntax: Syntax, +) { + let resolved = resolve(raw, properties, syntax); + let reference = match syntax { + Syntax::Maven => raw.value.contains("${"), + Syntax::MsBuild => raw.value.contains("$("), + }; + let kind = if raw.conditional { + format!("conditional-{kind}") + } else { + kind.to_owned() + }; + if reference { + node.versions + .entry(language.to_owned()) + .or_default() + .push(Version { + value: raw.value.clone(), + source: raw.source.clone(), + kind: "reference".to_owned(), + }); + } + if let Some(value) = resolved.filter(|v| !v.value.trim().is_empty()) { + node.versions + .entry(language.to_owned()) + .or_default() + .push(Version { + value: value.value, + source: value.source, + kind, + }); + } +} + +fn declaration( + source: &str, + raw: Option<&Located>, + properties: &Properties, + syntax: Syntax, +) -> Dependency { + let mut dependency = Dependency { + version: raw.filter(|v| v.source == source).map(|v| v.value.clone()), + source: source.to_owned(), + resolved: None, + declarations: Vec::new(), + }; + if let Some(raw) = raw { + let resolved = resolve(raw, properties, syntax).unwrap_or_else(|| raw.clone()); + if raw.source != source || raw.value != resolved.value { + dependency.resolved = Some(Version { + value: resolved.value, + source: resolved.source, + kind: "declaration".to_owned(), + }); + } + } + dependency +} + +pub(super) fn dependency( + node: &mut Node, + ecosystem: &str, + name: &str, + source: &str, + raw: Option<&Located>, + properties: &Properties, + syntax: Syntax, +) { + node.dependencies.insert( + format!("{ecosystem}:{name}"), + declaration(source, raw, properties, syntax), + ); +} + +pub(super) fn identity( + node: &mut Node, + ecosystem: &str, + name: &str, + source: &str, + raw: Option<&Located>, + properties: &Properties, + syntax: Syntax, +) { + node.identities.insert( + format!("{ecosystem}:{name}"), + declaration(source, raw, properties, syntax), + ); +} + +pub(super) fn relative(path: &Path) -> Option { + if path.is_absolute() { + return None; + } + let mut normalized = PathBuf::new(); + for component in path.components() { + match component { + Component::ParentDir => { + if !normalized.pop() { + return None; + } + } + Component::CurDir => (), + other => normalized.push(other), + } + } + Some(normalized) +} + +pub(super) fn child_file(base: &Path, value: &str) -> Option { + relative(&base.join(value.replace('\\', "/"))) +} + +pub(super) fn properties(text: &str, source: &str) -> Properties { + text.lines() + .filter_map(|line| { + let line = line.trim(); + if line.starts_with(['#', '!']) { + return None; + } + let (name, value) = line.split_once('=')?; + Some(( + name.trim().to_owned(), + Located { + value: value.trim().to_owned(), + source: source.to_owned(), + conditional: false, + }, + )) + }) + .collect() +} diff --git a/src/detect/xml.rs b/src/detect/xml.rs new file mode 100644 index 0000000..c91f42e --- /dev/null +++ b/src/detect/xml.rs @@ -0,0 +1,350 @@ +use std::collections::BTreeMap; + +use anyhow::{Result, bail}; +use xmlparser::{ElementEnd, Token, Tokenizer}; + +pub(super) struct Element<'a> { + pub name: &'a str, + pub attributes: BTreeMap, + pub text: String, + prefix: &'a str, +} + +pub(super) fn read(input: &str, mut visit: impl FnMut(&[Element<'_>]) -> Result<()>) -> Result<()> { + let mut stack = Vec::>::new(); + let mut roots = 0; + for token in Tokenizer::from(input) { + match token? { + Token::ElementStart { prefix, local, .. } => { + if stack.is_empty() { + roots += 1; + } + if stack.len() == 128 || roots > 1 { + bail!("invalid XML root or nesting exceeds 128 elements"); + } + stack.push(Element { + name: local.as_str(), + prefix: prefix.as_str(), + attributes: BTreeMap::new(), + text: String::new(), + }); + } + Token::Attribute { + prefix, + local, + value, + .. + } => { + let Some(element) = stack.last_mut() else { + bail!("XML attribute outside an element"); + }; + let name = if prefix.is_empty() { + local.as_str().to_owned() + } else { + format!("{}:{}", prefix.as_str(), local.as_str()) + }; + if element + .attributes + .insert(name, unescape(value.as_str())?) + .is_some() + { + bail!("duplicate XML attribute"); + } + } + Token::ElementEnd { end, .. } => match end { + ElementEnd::Open => validate_namespaces(&stack)?, + ElementEnd::Empty => { + validate_namespaces(&stack)?; + visit(&stack)?; + stack.pop(); + } + ElementEnd::Close(prefix, name) => { + if !stack + .last() + .is_some_and(|e| e.name == name.as_str() && e.prefix == prefix.as_str()) + { + bail!("mismatched XML closing element"); + } + visit(&stack)?; + stack.pop(); + } + }, + Token::Text { text } => { + if let Some(element) = stack.last_mut() { + if !text.as_str().trim().is_empty() || !element.text.is_empty() { + element.text.push_str(&unescape(text.as_str())?); + } + } else if !text.as_str().trim().is_empty() { + bail!("XML text outside the root element"); + } + } + Token::Cdata { text, .. } => { + let Some(element) = stack.last_mut() else { + bail!("CDATA outside an XML element"); + }; + element.text.push_str(text.as_str()); + } + Token::DtdStart { .. } | Token::EmptyDtd { .. } | Token::EntityDeclaration { .. } => { + bail!("XML DTDs and custom entities are not evaluated") + } + _ => (), + } + } + if roots != 1 || !stack.is_empty() { + bail!("XML document has a missing root or unclosed element"); + } + Ok(()) +} + +fn validate_namespaces(stack: &[Element<'_>]) -> Result<()> { + let Some(element) = stack.last() else { + bail!("XML element missing"); + }; + let namespace = |prefix: &str| -> Option<&str> { + if prefix == "xml" { + return Some("http://www.w3.org/XML/1998/namespace"); + } + let name = format!("xmlns:{prefix}"); + stack + .iter() + .rev() + .find_map(|e| e.attributes.get(&name).map(String::as_str)) + .filter(|value| !value.is_empty()) + }; + if !element.prefix.is_empty() && namespace(element.prefix).is_none() { + bail!("undeclared XML namespace prefix"); + } + let mut expanded = Vec::new(); + for name in element.attributes.keys() { + if name == "xmlns" || name.starts_with("xmlns:") { + continue; + } + let (uri, local) = if let Some((prefix, local)) = name.split_once(':') { + let Some(uri) = namespace(prefix) else { + bail!("undeclared XML attribute namespace"); + }; + (uri, local) + } else { + ("", name.as_str()) + }; + if expanded.contains(&(uri, local)) { + bail!("duplicate expanded XML attribute"); + } + expanded.push((uri, local)); + } + Ok(()) +} + +fn unescape(input: &str) -> Result { + let mut rest = input; + let mut output = String::with_capacity(input.len()); + while let Some(index) = rest.find('&') { + output.push_str(&rest[..index]); + rest = &rest[index + 1..]; + let Some(end) = rest.find(';') else { + bail!("unterminated XML entity reference"); + }; + let entity = &rest[..end]; + let ch = match entity { + "amp" => '&', + "lt" => '<', + "gt" => '>', + "quot" => '"', + "apos" => '\'', + _ => { + let code = if let Some(hex) = entity.strip_prefix("#x") { + u32::from_str_radix(hex, 16).ok() + } else { + entity + .strip_prefix('#') + .and_then(|digits| digits.parse::().ok()) + }; + let Some(ch) = code.and_then(char::from_u32).filter(|ch| matches!(u32::from(*ch), 9 | 10 | 13 | 0x20..=0xD7FF | 0xE000..=0xFFFD | 0x10000..=0x10FFFF)) else { bail!("unsupported XML entity reference: {entity}"); }; + ch + } + }; + output.push(ch); + rest = &rest[end + 1..]; + } + output.push_str(rest); + Ok(output) +} + +#[cfg(test)] +mod tests { + use std::fs::{self, File}; + use std::hint::black_box; + use std::io::Read; + use std::path::{Path, PathBuf}; + use std::time::{Duration, Instant}; + + use anyhow::{Result, ensure}; + + use super::read; + use crate::detect::{MAX_FILE_BYTES, inventory, rules::Rules}; + + fn kind(name: &str) -> Option<&'static str> { + if name == "pom.xml" { + Some("pom") + } else { + ["csproj", "props", "targets", "slnx"] + .into_iter() + .find(|ext| name.ends_with(&format!(".{ext}"))) + } + } + + fn files(root: &Path) -> Result> { + let inventory = inventory(root, Rules::builtin()?, 1)?; + for diagnostic in &inventory.diagnostics { + eprintln!("inventory: {}: {}", diagnostic.path, diagnostic.message); + } + Ok(inventory + .nodes + .into_iter() + .flat_map(|node| { + node.contents.into_iter().filter_map(move |(name, text)| { + kind(&name).map(|kind| (root.join(&node.path).join(name), text, kind)) + }) + }) + .collect()) + } + + fn bounded_read(path: &Path) -> Result { + let mut bytes = Vec::new(); + File::open(path)? + .take(MAX_FILE_BYTES + 1) + .read_to_end(&mut bytes)?; + ensure!(bytes.len() <= MAX_FILE_BYTES as usize, "manifest too large"); + Ok(String::from_utf8(bytes)?) + } + + fn timed(mut work: impl FnMut() -> Result<()>) -> Result<(usize, Duration)> { + let start = Instant::now(); + let mut iterations = 0; + loop { + work()?; + iterations += 1; + if start.elapsed() >= Duration::from_millis(200) { + return Ok((iterations, start.elapsed())); + } + } + } + + fn benchmark(label: &str, files: &[(PathBuf, String, &'static str)]) -> Result<()> { + for kind in ["csproj", "props", "targets", "slnx", "pom", "all"] { + let selected: Vec<_> = files + .iter() + .filter(|(_, _, ext)| kind == "all" || *ext == kind) + .collect(); + if selected.is_empty() { + continue; + } + let bytes: usize = selected.iter().map(|(_, text, _)| text.len()).sum(); + let mut errors = 0; + let mut elements = 0; + for (path, text, _) in &selected { + if let Err(error) = read(text, |stack| { + elements += 1; + black_box(stack); + Ok(()) + }) { + errors += 1; + if kind != "all" { + eprintln!("parse: {}: {error}", path.display()); + } + } + } + let (reads, read_elapsed) = timed(|| { + for (path, _, _) in &selected { + black_box(bounded_read(path)?); + } + Ok(()) + })?; + let (parses, parse_elapsed) = timed(|| { + for (_, text, _) in &selected { + let result = read(black_box(text), |stack| { + black_box(stack); + Ok(()) + }); + black_box(result.is_ok()); + } + Ok(()) + })?; + eprintln!( + "{label} {kind}: files={} bytes={bytes} elements={elements} errors={errors} read_us={:.2} parse_us={:.2} read_iters={reads} parse_iters={parses}", + selected.len(), + read_elapsed.as_secs_f64() * 1e6 / reads as f64, + parse_elapsed.as_secs_f64() * 1e6 / parses as f64, + ); + } + Ok(()) + } + + #[test] + #[ignore = "local XML I/O and parser microbenchmark; run release with --ignored --nocapture"] + fn benchmark_xml_local() -> Result<()> { + let root = tempfile::tempdir()?; + let item = "a & b"; + let project = format!( + "{}", + item.repeat(128) + ); + for name in [ + "app.csproj", + "Directory.Build.props", + "Directory.Build.targets", + ] { + fs::write(root.path().join(name), &project)?; + } + fs::write( + root.path().join("app.slnx"), + "", + )?; + fs::write( + root.path().join("pom.xml"), + format!( + "4.0.0{}", + "21".repeat(128) + ), + )?; + for dir in ["target", "bin", "obj"] { + fs::create_dir(root.path().join(dir))?; + fs::write(root.path().join(dir).join("ignored.csproj"), &project)?; + } + fs::write( + root.path().join("oversized.props"), + vec![b' '; MAX_FILE_BYTES as usize + 1], + )?; + let files = files(root.path())?; + assert_eq!(files.len(), 5); + benchmark("local-synthetic", &files) + } + + #[test] + #[ignore = "set XML_BENCH_CACHE to pinned detect-corpus cache; run release with --ignored --nocapture"] + fn benchmark_xml_corpus() -> Result<()> { + let cache = PathBuf::from(std::env::var("XML_BENCH_CACHE")?); + let manifest: serde_json::Value = serde_json::from_str(&fs::read_to_string( + Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/detect-corpus.json"), + )?)?; + let mut all = Vec::new(); + let entries = manifest["repositories"].as_array().unwrap(); + for entry in entries { + let repository = entry["repository"].as_str().unwrap(); + let name = repository.replace('/', "--"); + let metadata: serde_json::Value = serde_json::from_str(&fs::read_to_string( + cache.join("metadata").join(format!("{name}.json")), + )?)?; + let revision = metadata.get("revision").or_else(|| metadata.get("sha")); + ensure!( + revision == Some(&entry["revision"]), + "unpinned {repository}" + ); + let selected = files(&cache.join("repos").join(name))?; + benchmark(repository, &selected)?; + all.extend(selected); + } + eprintln!("corpus repositories={}", entries.len()); + benchmark("corpus-total", &all) + } +} diff --git a/src/main.rs b/src/main.rs index 5488e46..b285be6 100644 --- a/src/main.rs +++ b/src/main.rs @@ -12,6 +12,7 @@ mod client; mod cmd; mod config; mod describe; +mod detect; mod image; mod openapi; mod output; @@ -130,6 +131,10 @@ async fn run(opts: Opts) -> Result { )); } + if let Command::Detect(args) = opts.command { + return detect::handle(args); + } + let config_path = Config::path()?; let mut config = Config::load(&config_path)?; @@ -291,10 +296,17 @@ fn api_error_json(api_error: &ApiError) -> Value { } fn is_broken_pipe(error: &anyhow::Error) -> bool { - error - .chain() - .filter_map(|cause| cause.downcast_ref::()) - .any(|error| error.kind() == std::io::ErrorKind::BrokenPipe) + error.chain().any(|cause| { + cause + .downcast_ref::() + .map(std::io::Error::kind) + .or_else(|| { + cause + .downcast_ref::() + .and_then(serde_json::Error::io_error_kind) + }) + == Some(std::io::ErrorKind::BrokenPipe) + }) } #[cfg(test)] @@ -303,6 +315,40 @@ mod tests { use super::{Command, Opts}; + #[test] + fn handles_broken_pipes_wrapped_by_json_serialization() { + struct FailedWriter(std::io::ErrorKind); + + impl std::io::Write for FailedWriter { + fn write(&mut self, _: &[u8]) -> std::io::Result { + Err(self.0.into()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + for kind in [ + std::io::ErrorKind::BrokenPipe, + std::io::ErrorKind::PermissionDenied, + ] { + let expected = kind == std::io::ErrorKind::BrokenPipe; + let error = serde_json::to_writer(FailedWriter(kind), &serde_json::json!({"value": 1})) + .unwrap_err(); + assert_eq!( + super::is_broken_pipe(&anyhow::Error::new(error).context("write output")), + expected + ); + assert_eq!( + super::is_broken_pipe(&anyhow::Error::new(std::io::Error::from(kind))), + expected + ); + } + let error = serde_json::from_str::("invalid").unwrap_err(); + assert!(!super::is_broken_pipe(&error.into())); + } + #[test] fn parses_describe_path() { let opts = Opts::try_parse_from(["brainpod", "describe", "resource", "create"]).unwrap(); diff --git a/src/output.rs b/src/output.rs index b864d56..30854eb 100644 --- a/src/output.rs +++ b/src/output.rs @@ -25,6 +25,7 @@ impl CommandOutput { pub enum View { S3, Describe, + Detect, ResourceSchema, AgentStart, AgentServe, @@ -80,8 +81,10 @@ fn write_buffered(value: &Value, view: View, json: bool) -> Result<()> { serde_json::to_writer(&mut stdout, value)?; writeln!(stdout)?; } else { - serde_json::to_writer_pretty(&mut stdout, value)?; - writeln!(stdout)?; + let mut buffered = io::BufWriter::new(&mut stdout); + serde_json::to_writer_pretty(&mut buffered, value)?; + writeln!(buffered)?; + buffered.flush()?; } return Ok(()); } @@ -164,6 +167,7 @@ fn render(value: &Value, view: View, color: bool) -> Vec { match view { View::S3 => crate::s3::render(value), View::Describe => render_describe(value), + View::Detect => crate::detect::render(value), View::ResourceSchema => render_resource_schema(value), View::AgentStart => crate::agent::render_start(value), View::AgentServe => vec![format!("Session console: {}", field(value, "url"))], diff --git a/src/s3/multipart.rs b/src/s3/multipart.rs index 9c2a74e..40f5208 100644 --- a/src/s3/multipart.rs +++ b/src/s3/multipart.rs @@ -269,11 +269,18 @@ impl Gateway<'_> { ) -> Result { let mut response = self.provider_response(remote, true).await?; let mut size = 0u64; - while let Some(chunk) = response - .chunk() - .await - .map_err(|_| anyhow!("provider download interrupted; output may be partial"))? - { + while let Some(chunk) = response.chunk().await.map_err(|error| { + #[cfg(test)] + eprintln!( + "provider body failed after {size} bytes: timeout={}, body={}, error={:?}", + error.is_timeout(), + error.is_body(), + error.without_url() + ); + #[cfg(not(test))] + let _ = error; + anyhow!("provider download interrupted; output may be partial") + })? { output .write_all(&chunk) .await diff --git a/src/s3/tests.rs b/src/s3/tests.rs index 093769a..e0ef7d7 100644 --- a/src/s3/tests.rs +++ b/src/s3/tests.rs @@ -432,7 +432,9 @@ fn provider(body: &[u8], declared_length: usize) -> (String, std::thread::JoinHa "HTTP/1.1 200 OK\r\nContent-Length: {declared_length}\r\nConnection: close\r\n\r\n" ) .unwrap(); - stream.write_all(&body).unwrap(); + for chunk in body.chunks(16 * 1024) { + stream.write_all(chunk).unwrap(); + } stream.flush().unwrap(); String::from_utf8(request).unwrap() }); @@ -447,7 +449,8 @@ fn trust_provider(gateway: &mut Gateway<'_>) { .no_proxy() .tls_certs_only([certificate]) .redirect(reqwest::redirect::Policy::none()) - .timeout(Duration::from_secs(5)) + .connect_timeout(Duration::from_secs(5)) + .read_timeout(Duration::from_secs(5)) .build() .unwrap(); } @@ -860,14 +863,14 @@ async fn streaming_download_exceeds_small_copy_limit_without_credentials_or_summ mock(vec![(307, vec![("location", url)], String::new())]).await; trust_provider(&mut gateway); let mut output = tokio::io::sink(); - assert_eq!( - gateway - .download_stream(&streaming_remote(), &mut output) - .await - .unwrap(), - body.len() as u64 - ); - let request = provider.join().unwrap().to_ascii_lowercase(); + let result = gateway + .download_stream(&streaming_remote(), &mut output) + .await; + let provider_result = tokio::task::spawn_blocking(move || provider.join()) + .await + .unwrap(); + assert_eq!(result.unwrap(), body.len() as u64); + let request = provider_result.unwrap().to_ascii_lowercase(); assert!(!request.contains("authorization:")); assert!(!request.contains("test-key")); assert_eq!(requests.lock().unwrap().len(), 1); diff --git a/tests/audit_detect.py b/tests/audit_detect.py new file mode 100644 index 0000000..c321cc3 --- /dev/null +++ b/tests/audit_detect.py @@ -0,0 +1,324 @@ +import argparse +import collections +import hashlib +import json +import re +import shutil +import statistics +import subprocess +import tarfile +import tempfile +import time +import urllib.request +from pathlib import Path, PurePosixPath + +MAX_ARCHIVE_BYTES = 1024 * 1024 * 1024 + + +def snapshot(entry, cache, fetch): + repository, revision = entry["repository"], entry["revision"] + if ( + not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9-]*/[\w.-]+", repository) + or repository.split("/")[-1] in {".", ".."} + or not re.fullmatch(r"[0-9a-f]{40}", revision) + ): + raise ValueError("invalid repository or pinned revision") + name = repository.replace("/", "--") + destination = cache / "repos" / name + metadata = cache / "metadata" / f"{name}.json" + if destination.is_dir() and metadata.is_file(): + data = json.loads(metadata.read_text()) + if data.get("revision", data.get("sha")) == revision: + return destination + if not fetch: + raise ValueError( + f"missing pinned snapshot: {repository}; pass --fetch to download" + ) + if destination.exists(): + raise ValueError( + f"unverified snapshot exists: {destination}; use a fresh cache" + ) + destination.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix="fetch-", dir=cache) as temporary: + temporary = Path(temporary) + archive = temporary / "snapshot.tar.gz" + with ( + urllib.request.urlopen( + f"https://codeload.github.com/{repository}/tar.gz/{revision}", + timeout=90, + ) as response, + archive.open("wb") as output, + ): + size = 0 + while chunk := response.read(1024 * 1024): + size += len(chunk) + if size > MAX_ARCHIVE_BYTES: + raise ValueError("archive exceeds 1 GiB") + output.write(chunk) + extracted = temporary / "snapshot" + extracted.mkdir() + size = 0 + with tarfile.open(archive, "r:gz") as source: + for member in source: + parts = PurePosixPath(member.name).parts + if not parts or member.name.startswith("/") or ".." in parts: + raise ValueError("unsafe archive path") + if len(parts) == 1: + continue + path = extracted.joinpath(*parts[1:]) + if member.isdir(): + path.mkdir(parents=True, exist_ok=True) + elif member.isfile(): + size += member.size + if size > 2 * MAX_ARCHIVE_BYTES: + raise ValueError("expanded snapshot exceeds 2 GiB") + path.parent.mkdir(parents=True, exist_ok=True) + with ( + source.extractfile(member) as content, + path.open("wb") as output, + ): + shutil.copyfileobj(content, output) + extracted.rename(destination) + metadata.parent.mkdir(parents=True, exist_ok=True) + metadata.write_text( + json.dumps({"repository": repository, "revision": revision}) + "\n" + ) + return destination + + +def check(projects, expected): + project = projects.get(expected["path"]) + if project is None: + return False, "project missing" + kind = expected["kind"] + ids = [d["id"] for d in project["detections"]] + if kind == "language": + actual = [language["name"] for language in project["languages"]] + return expected["language"] in actual, actual + if kind == "detection": + found = next( + (d for d in project["detections"] if d["id"] == expected["id"]), None + ) + roles = expected.get("roles", []) + if expected.get("absent"): + return found is None, ids + return found is not None and set(roles).issubset( + found.get("roles", []) + ), project["detections"] if roles else ids + if kind == "name": + return expected["name"] in project["names"], project["names"] + if kind == "workspace": + actual = project["workspaceMembers"] + if "ecosystem" in expected: + actual = sorted( + { + member + for workspace in project.get("workspaces", []) + if workspace["ecosystem"] == expected["ecosystem"] + and ( + "source" not in expected + or workspace["source"] == expected["source"] + ) + for member in workspace["members"] + } + ) + return expected["id"] in ids and expected["member"] in actual, { + "ids": ids, + "members": actual, + } + if kind in {"runtime", "runtime-version"}: + runtime = next( + ( + runtime + for runtime in project.get("runtimes", []) + if runtime["name"] == expected["runtime"] + ), + None, + ) + if kind == "runtime": + if expected.get("absent"): + return runtime is None, project.get("runtimes", []) + if ( + "versionCount" in expected + and runtime is not None + and len(runtime["versions"]) != expected["versionCount"] + ): + return False, runtime + return runtime is not None and ( + "source" not in expected or runtime["source"] == expected["source"] + ), project.get("runtimes", []) + actual = runtime["versions"] if runtime else [] + return any( + v["value"] == expected["value"] + and v["source"] == expected["source"] + and ( + "versionKind" not in expected + or v.get("kind") == expected["versionKind"] + ) + for v in actual + ), actual + if kind == "version": + actual = [ + v + for language in project["languages"] + if language["name"] == expected["language"] + for v in language["versions"] + ] + return any( + v["value"] == expected["value"] + and v["source"] == expected["source"] + and ( + "versionKind" not in expected + or v.get("kind") == expected["versionKind"] + ) + for v in actual + ), actual + raise ValueError(f"unknown check kind: {kind}") + + +def audit(entry, args): + path = snapshot(entry, args.cache, args.fetch) + samples, first = [], None + deterministic = True + for _ in range(args.runs): + start = time.perf_counter() + process = subprocess.run( + [str(args.binary), "detect", str(path), "--json"], + capture_output=True, + timeout=90, + check=False, + ) + samples.append((time.perf_counter() - start) * 1000) + if process.returncode: + raise ValueError(process.stderr.decode(errors="replace")) + if first is None: + first = process.stdout + else: + deterministic = deterministic and first == process.stdout + report = json.loads(first) + if report["schemaVersion"] != 1: + raise ValueError("unexpected detector schema version") + projects = {project["path"]: project for project in report["projects"]} + checks = [] + for expected in entry["checks"]: + passed, actual = check(projects, expected) + checks.append({"expected": expected, "passed": passed, "actual": actual}) + return { + "repository": entry["repository"], + "language": entry["language"], + "revision": entry["revision"], + "firstMs": samples[0], + "medianMs": statistics.median(samples[1:]), + "samplesMs": samples, + "deterministic": deterministic, + "stats": report["stats"], + "projectCount": len(projects), + "diagnostics": report["diagnostics"], + "checks": checks, + } + + +def main(): + parser = argparse.ArgumentParser( + description="Audit local project detection against pinned public GitHub snapshots without installing or executing project code." + ) + parser.add_argument( + "--corpus", type=Path, default=Path(__file__).with_name("detect-corpus.json") + ) + parser.add_argument( + "--binary", + type=Path, + default=Path(__file__).resolve().parents[1] / "target/release/brainpod", + ) + parser.add_argument( + "--cache", type=Path, help="Snapshot cache; required unless --list" + ) + parser.add_argument( + "--fetch", + action="store_true", + help="Explicitly allow downloading missing pinned GitHub snapshots", + ) + parser.add_argument( + "--runs", + type=int, + default=5, + help="CLI invocations per repository; the first is excluded from the warm median", + ) + parser.add_argument( + "--output", type=Path, help="Results JSON; required unless --list" + ) + parser.add_argument( + "--list", + action="store_true", + help="List pinned repositories without downloading or running the detector", + ) + args = parser.parse_args() + corpus = json.loads(args.corpus.read_text()) + if corpus["schemaVersion"] != 1: + parser.error("unsupported corpus schema version") + if args.list: + for entry in corpus["repositories"]: + print(f"{entry['language']}\t{entry['repository']}\t{entry['revision']}") + return 0 + if args.cache is None or args.output is None: + parser.error("--cache and --output are required unless --list") + if args.runs < 2: + parser.error("--runs must be at least 2") + args.binary = args.binary.resolve(strict=True) + args.cache = args.cache.resolve() + args.cache.mkdir(parents=True, exist_ok=True) + with args.binary.open("rb") as binary: + binary_sha = hashlib.file_digest(binary, "sha256").hexdigest() + results = [] + for entry in corpus["repositories"]: + try: + result = audit(entry, args) + passed = sum(c["passed"] for c in result["checks"]) + print( + f"{entry['repository']}: {passed}/{len(result['checks'])} checks; {result['medianMs']:.2f} ms", + flush=True, + ) + except ( + OSError, + ValueError, + KeyError, + tarfile.TarError, + subprocess.TimeoutExpired, + ) as error: + result = { + "repository": entry["repository"], + "revision": entry["revision"], + "error": str(error), + } + print(f"{entry['repository']}: ERROR {error}", flush=True) + results.append(result) + counts = collections.defaultdict(lambda: {"passed": 0, "total": 0}) + for result in results: + for item in result.get("checks", []): + category = counts[item["expected"]["category"]] + category["total"] += 1 + category["passed"] += item["passed"] + output = { + "schemaVersion": 1, + "selectedAt": corpus["selectedAt"], + "binarySha256": binary_sha, + "categories": dict(counts), + "results": results, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(output, indent=2) + "\n") + for category, count in sorted(counts.items()): + print(f"{category}: {count['passed']}/{count['total']}") + print(f"Results: {args.output}") + return int( + any( + "error" in r + or not r["deterministic"] + or any(not c["passed"] for c in r["checks"]) + for r in results + ) + ) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/detect-corpus.json b/tests/detect-corpus.json new file mode 100644 index 0000000..ff59a6a --- /dev/null +++ b/tests/detect-corpus.json @@ -0,0 +1,2550 @@ +{ + "schemaVersion": 1, + "selectedAt": "2026-11-09T21:24:33.642341+00:00", + "ranking": "GitHub stars descending; non-fork, non-archived; primary language per GitHub", + "eligibility": "Web applications (including production learning platforms), HTTP servers/frameworks/libraries, browser UI libraries, and web-specific site/build tools. Exclude tutorial/list repositories, general runtimes, coding agents, desktop-only apps, ML engines without a primary web UI, and generic infrastructure/networking tools.", + "expectations": "Reviewed primary project paths and static manifests. Framework checks cover consumers; identity checks separately probe framework implementation packages. Catalog checks require a locally declared version rather than a catalog reference. Workspace checks use representative declared members. Not an exhaustive precision/recall oracle.", + "repositories": [ + { + "repository": "infiniflow/ragflow", + "language": "Go", + "stars": 93887, + "languageRank": 8, + "webRank": 1, + "revision": "ed5a84fee9abc8091aea111d1195cf7768541ff1", + "checks": [ + { + "category": "language", + "kind": "language", + "path": ".", + "language": "go" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "go-module" + }, + { + "category": "name", + "kind": "name", + "path": ".", + "name": "ragflow" + }, + { + "category": "version", + "kind": "version", + "path": ".", + "language": "go", + "value": "1.27", + "source": "go.mod" + }, + { + "category": "framework", + "kind": "detection", + "path": ".", + "id": "gin" + } + ] + }, + { + "repository": "gohugoio/hugo", + "language": "Go", + "stars": 90425, + "languageRank": 9, + "webRank": 2, + "revision": "d2010fcc9cd30c5b4e9274ded4485f4222326b53", + "checks": [ + { + "category": "language", + "kind": "language", + "path": ".", + "language": "go" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "go-module" + }, + { + "category": "name", + "kind": "name", + "path": ".", + "name": "github.com/gohugoio/hugo" + }, + { + "category": "version", + "kind": "version", + "path": ".", + "language": "go", + "value": "1.27.0", + "source": "go.mod" + } + ] + }, + { + "repository": "gin-gonic/gin", + "language": "Go", + "stars": 89609, + "languageRank": 11, + "webRank": 3, + "revision": "0f09c3a9b4626d9fe9979cf4d0521d6ae32ab646", + "checks": [ + { + "category": "language", + "kind": "language", + "path": ".", + "language": "go" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "go-module" + }, + { + "category": "name", + "kind": "name", + "path": ".", + "name": "github.com/gin-gonic/gin" + }, + { + "category": "version", + "kind": "version", + "path": ".", + "language": "go", + "value": "1.26.0", + "source": "go.mod" + }, + { + "category": "identity", + "kind": "detection", + "path": ".", + "id": "gin" + } + ] + }, + { + "repository": "netdata/netdata", + "language": "Go", + "stars": 81330, + "languageRank": 14, + "webRank": 4, + "revision": "0d54b4424ea6f40c344738b9d566e1a26d160da3", + "checks": [ + { + "category": "language", + "kind": "language", + "path": "src/go", + "language": "go" + }, + { + "category": "project", + "kind": "detection", + "path": "src/go", + "id": "go-module" + }, + { + "category": "name", + "kind": "name", + "path": "src/go", + "name": "github.com/netdata/netdata/go/plugins" + }, + { + "category": "version", + "kind": "version", + "path": "src/go", + "language": "go", + "value": "1.27.0", + "source": "src/go/go.mod" + } + ] + }, + { + "repository": "caddyserver/caddy", + "language": "Go", + "stars": 78927, + "languageRank": 15, + "webRank": 5, + "revision": "9ea3108377d2abec102866808133c40ac9582162", + "checks": [ + { + "category": "language", + "kind": "language", + "path": ".", + "language": "go" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "go-module" + }, + { + "category": "name", + "kind": "name", + "path": ".", + "name": "github.com/caddyserver/caddy/v2" + }, + { + "category": "version", + "kind": "version", + "path": ".", + "language": "go", + "value": "1.26.0", + "source": "go.mod" + } + ] + }, + { + "repository": "react/react", + "language": "JavaScript", + "stars": 252455, + "languageRank": 2, + "webRank": 1, + "revision": "b05c0b75fd9778536152eead30814c9ca90c810a", + "checks": [ + { + "category": "language", + "kind": "language", + "path": "packages/react", + "language": "javascript" + }, + { + "category": "project", + "kind": "detection", + "path": "packages/react", + "id": "node" + }, + { + "category": "name", + "kind": "name", + "path": "packages/react", + "name": "react" + }, + { + "category": "version", + "kind": "version", + "path": "packages/react", + "language": "javascript", + "value": ">=0.10.0", + "source": "packages/react/package.json" + }, + { + "category": "identity", + "kind": "detection", + "path": "packages/react", + "id": "react" + }, + { + "category": "workspace", + "kind": "workspace", + "path": ".", + "id": "node-workspace", + "member": "packages/react" + } + ] + }, + { + "repository": "vercel/next.js", + "language": "JavaScript", + "stars": 143437, + "languageRank": 8, + "webRank": 2, + "revision": "5368e013e74e4e4ed5be51c051c989545c4450b9", + "checks": [ + { + "category": "language", + "kind": "language", + "path": "packages/next", + "language": "javascript" + }, + { + "category": "project", + "kind": "detection", + "path": "packages/next", + "id": "node" + }, + { + "category": "name", + "kind": "name", + "path": "packages/next", + "name": "next" + }, + { + "category": "version", + "kind": "version", + "path": "packages/next", + "language": "javascript", + "value": ">=20.9.0", + "source": "packages/next/package.json" + }, + { + "category": "version", + "kind": "version", + "path": "packages/next", + "language": "typescript", + "value": "6.0.2", + "source": "packages/next/package.json" + }, + { + "category": "identity", + "kind": "detection", + "path": "packages/next", + "id": "nextjs" + }, + { + "category": "workspace", + "kind": "workspace", + "path": ".", + "id": "node-workspace", + "member": "packages/next" + } + ] + }, + { + "repository": "mrdoob/three.js", + "language": "JavaScript", + "stars": 117953, + "languageRank": 11, + "webRank": 3, + "revision": "3f7edc3d75e054bc755dad56c1c95c62f63a3561", + "checks": [ + { + "category": "language", + "kind": "language", + "path": ".", + "language": "javascript" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "node" + }, + { + "category": "name", + "kind": "name", + "path": ".", + "name": "three" + } + ] + }, + { + "repository": "axios/axios", + "language": "JavaScript", + "stars": 109295, + "languageRank": 14, + "webRank": 4, + "revision": "335e2f8464f58a3ab485486b8d6964fea3e3f30a", + "checks": [ + { + "category": "language", + "kind": "language", + "path": ".", + "language": "javascript" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "node" + }, + { + "category": "name", + "kind": "name", + "path": ".", + "name": "axios" + }, + { + "category": "version", + "kind": "version", + "path": ".", + "language": "typescript", + "value": "^7.0.2", + "source": "package.json" + } + ] + }, + { + "repository": "react/create-react-app", + "language": "JavaScript", + "stars": 103230, + "languageRank": 15, + "webRank": 5, + "revision": "6254386531d263688ccfa542d0e628fbc0de0b28", + "checks": [ + { + "category": "language", + "kind": "language", + "path": "packages/react-scripts", + "language": "javascript" + }, + { + "category": "project", + "kind": "detection", + "path": "packages/react-scripts", + "id": "node" + }, + { + "category": "name", + "kind": "name", + "path": "packages/react-scripts", + "name": "react-scripts" + }, + { + "category": "version", + "kind": "version", + "path": "packages/react-scripts", + "language": "javascript", + "value": ">=14.0.0", + "source": "packages/react-scripts/package.json" + }, + { + "category": "framework", + "kind": "detection", + "path": "packages/react-scripts", + "id": "react" + }, + { + "category": "workspace", + "kind": "workspace", + "path": ".", + "id": "node-workspace", + "member": "packages/react-scripts" + } + ] + }, + { + "repository": "coollabsio/coolify", + "language": "PHP", + "stars": 65849, + "languageRank": 2, + "webRank": 1, + "revision": "30335d80866ff3391d899cf4ccb599b696f47b7e", + "checks": [ + { + "category": "language", + "kind": "language", + "path": ".", + "language": "php" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "composer" + }, + { + "category": "name", + "kind": "name", + "path": ".", + "name": "coollabsio/coolify" + }, + { + "category": "version", + "kind": "version", + "path": ".", + "language": "php", + "value": "^8.4", + "source": "composer.json" + }, + { + "category": "framework", + "kind": "detection", + "path": ".", + "id": "laravel" + } + ] + }, + { + "repository": "nextcloud/server", + "language": "PHP", + "stars": 37456, + "languageRank": 3, + "webRank": 2, + "revision": "d655757fb306aca893651958030b5dae69ed0f39", + "checks": [ + { + "category": "language", + "kind": "language", + "path": ".", + "language": "php" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "composer" + }, + { + "category": "version", + "kind": "version", + "path": ".", + "language": "php", + "value": "^8.3", + "source": "composer.json" + } + ] + }, + { + "repository": "laravel/framework", + "language": "PHP", + "stars": 34995, + "languageRank": 4, + "webRank": 3, + "revision": "338e44065c0a5b18f1f290605b6253a1e69d8f56", + "checks": [ + { + "category": "language", + "kind": "language", + "path": ".", + "language": "php" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "composer" + }, + { + "category": "name", + "kind": "name", + "path": ".", + "name": "laravel/framework" + }, + { + "category": "version", + "kind": "version", + "path": ".", + "language": "php", + "value": "^8.3", + "source": "composer.json" + }, + { + "category": "identity", + "kind": "detection", + "path": ".", + "id": "laravel" + } + ] + }, + { + "repository": "filamentphp/filament", + "language": "PHP", + "stars": 32462, + "languageRank": 5, + "webRank": 4, + "revision": "9d78093cfebdc96c077b1d8dbf94b41738f28992", + "checks": [ + { + "category": "language", + "kind": "language", + "path": ".", + "language": "php" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "composer" + }, + { + "category": "name", + "kind": "name", + "path": ".", + "name": "filament/filament" + }, + { + "category": "version", + "kind": "version", + "path": ".", + "language": "php", + "value": "^8.2", + "source": "composer.json" + } + ] + }, + { + "repository": "symfony/symfony", + "language": "PHP", + "stars": 31523, + "languageRank": 6, + "webRank": 5, + "revision": "f07928484c9e96646964deb7b9570d7885492512", + "checks": [ + { + "category": "language", + "kind": "language", + "path": ".", + "language": "php" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "composer" + }, + { + "category": "name", + "kind": "name", + "path": ".", + "name": "symfony/symfony" + }, + { + "category": "version", + "kind": "version", + "path": ".", + "language": "php", + "value": ">=8.4.1", + "source": "composer.json" + }, + { + "category": "identity", + "kind": "detection", + "path": ".", + "id": "symfony" + } + ] + }, + { + "repository": "Significant-Gravitas/AutoGPT", + "language": "Python", + "stars": 187972, + "languageRank": 11, + "webRank": 1, + "revision": "9cb76f9026d388c9bf2e098af2beae1cc864f055", + "checks": [ + { + "category": "language", + "kind": "language", + "path": "autogpt_platform/backend", + "language": "python" + }, + { + "category": "project", + "kind": "detection", + "path": "autogpt_platform/backend", + "id": "python" + }, + { + "category": "name", + "kind": "name", + "path": "autogpt_platform/backend", + "name": "autogpt-platform-backend" + }, + { + "category": "version", + "kind": "version", + "path": "autogpt_platform/backend", + "language": "python", + "value": ">=3.11,<3.14", + "source": "autogpt_platform/backend/pyproject.toml" + }, + { + "category": "framework", + "kind": "detection", + "path": "autogpt_platform/backend", + "id": "fastapi" + }, + { + "category": "framework", + "kind": "detection", + "path": "autogpt_platform/frontend", + "id": "nextjs" + } + ] + }, + { + "repository": "AUTOMATIC1111/stable-diffusion-webui", + "language": "Python", + "stars": 165269, + "languageRank": 14, + "webRank": 2, + "revision": "82a973c04367123ae98bd9abdf80d9eda9b910e2", + "checks": [ + { + "category": "language", + "kind": "language", + "path": ".", + "language": "python" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "python" + }, + { + "category": "framework", + "kind": "detection", + "path": ".", + "id": "fastapi" + } + ] + }, + { + "repository": "langgenius/dify", + "language": "Python", + "stars": 160774, + "languageRank": 15, + "webRank": 3, + "revision": "fdb09a9e76dc8cbd052fabd72cfff4071328d9bc", + "checks": [ + { + "category": "language", + "kind": "language", + "path": "api", + "language": "python" + }, + { + "category": "project", + "kind": "detection", + "path": "api", + "id": "python" + }, + { + "category": "name", + "kind": "name", + "path": "api", + "name": "dify-api" + }, + { + "category": "version", + "kind": "version", + "path": "api", + "language": "python", + "value": "~=3.13.0", + "source": "api/pyproject.toml" + }, + { + "category": "framework", + "kind": "detection", + "path": "api", + "id": "flask" + }, + { + "category": "framework", + "kind": "detection", + "path": "api", + "id": "celery" + }, + { + "category": "framework", + "kind": "detection", + "path": "web", + "id": "nextjs" + }, + { + "category": "workspace", + "kind": "workspace", + "path": ".", + "id": "node-workspace", + "member": "web" + }, + { + "category": "workspace", + "kind": "workspace", + "path": "api", + "id": "python-workspace", + "member": "api/providers/vdb/vdb-alibabacloud-mysql" + }, + { + "category": "catalog", + "kind": "version", + "path": "web", + "language": "typescript", + "value": "npm:@typescript/typescript6@6.0.2", + "source": "pnpm-workspace.yaml" + } + ] + }, + { + "repository": "langflow-ai/langflow", + "language": "Python", + "stars": 156479, + "languageRank": 16, + "webRank": 4, + "revision": "60999cc9a7350e44dde3ef641baae864e10861f0", + "checks": [ + { + "category": "language", + "kind": "language", + "path": ".", + "language": "python" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "python" + }, + { + "category": "name", + "kind": "name", + "path": ".", + "name": "langflow" + }, + { + "category": "version", + "kind": "version", + "path": ".", + "language": "python", + "value": ">=3.10,<3.15", + "source": "pyproject.toml" + }, + { + "category": "framework", + "kind": "detection", + "path": "src/backend/base", + "id": "fastapi" + }, + { + "category": "framework", + "kind": "detection", + "path": "src/backend/base", + "id": "starlette" + } + ] + }, + { + "repository": "Comfy-Org/ComfyUI", + "language": "Python", + "stars": 145983, + "languageRank": 20, + "webRank": 5, + "revision": "540a82ee722bda1f32a1fbd97fcadadeb22d548b", + "checks": [ + { + "category": "language", + "kind": "language", + "path": ".", + "language": "python" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "python" + }, + { + "category": "name", + "kind": "name", + "path": ".", + "name": "ComfyUI" + }, + { + "category": "version", + "kind": "version", + "path": ".", + "language": "python", + "value": ">=3.10", + "source": "pyproject.toml" + } + ] + }, + { + "repository": "rails/rails", + "language": "Ruby", + "stars": 59087, + "languageRank": 1, + "webRank": 1, + "revision": "3a0aff317bc037f43f168e3772028f1033f1e810", + "checks": [ + { + "category": "language", + "kind": "language", + "path": ".", + "language": "ruby" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "bundler" + }, + { + "category": "identity", + "kind": "detection", + "path": ".", + "id": "rails" + } + ] + }, + { + "repository": "jekyll/jekyll", + "language": "Ruby", + "stars": 51756, + "languageRank": 2, + "webRank": 2, + "revision": "97c3bdc822df9d5bea4c269b29a28de5a5e29d70", + "checks": [ + { + "category": "language", + "kind": "language", + "path": ".", + "language": "ruby" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "bundler" + } + ] + }, + { + "repository": "mastodon/mastodon", + "language": "Ruby", + "stars": 50502, + "languageRank": 3, + "webRank": 3, + "revision": "5a207bc8f5046e27cffc640a3c64f96504f6afcf", + "checks": [ + { + "category": "language", + "kind": "language", + "path": ".", + "language": "ruby" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "bundler" + }, + { + "category": "version", + "kind": "version", + "path": ".", + "language": "ruby", + "value": "4.0.9", + "source": ".ruby-version" + }, + { + "category": "framework", + "kind": "detection", + "path": ".", + "id": "rails" + } + ] + }, + { + "repository": "huginn/huginn", + "language": "Ruby", + "stars": 50454, + "languageRank": 4, + "webRank": 4, + "revision": "639c050b2bf4e14ef9983012af99dfc1dc68b334", + "checks": [ + { + "category": "language", + "kind": "language", + "path": ".", + "language": "ruby" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "bundler" + }, + { + "category": "version", + "kind": "version", + "path": ".", + "language": "ruby", + "value": ">=3.4.0", + "source": "Gemfile" + }, + { + "category": "framework", + "kind": "detection", + "path": ".", + "id": "rails" + } + ] + }, + { + "repository": "discourse/discourse", + "language": "Ruby", + "stars": 48066, + "languageRank": 6, + "webRank": 5, + "revision": "899c68d783e7808c50144752369a685220250d7c", + "checks": [ + { + "category": "language", + "kind": "language", + "path": ".", + "language": "ruby" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "bundler" + }, + { + "category": "version", + "kind": "version", + "path": ".", + "language": "ruby", + "value": "~> 4.0.0", + "source": "Gemfile" + }, + { + "category": "framework", + "kind": "detection", + "path": ".", + "id": "rails" + }, + { + "category": "workspace", + "kind": "workspace", + "path": ".", + "id": "node-workspace", + "member": "frontend/discourse" + } + ] + }, + { + "repository": "dani-garcia/vaultwarden", + "language": "Rust", + "stars": 70490, + "languageRank": 17, + "webRank": 1, + "revision": "17b159beff4b892f07f61f38ce2fe452e55251c1", + "checks": [ + { + "category": "language", + "kind": "language", + "path": ".", + "language": "rust" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "cargo" + }, + { + "category": "name", + "kind": "name", + "path": ".", + "name": "vaultwarden" + }, + { + "category": "version", + "kind": "version", + "path": ".", + "language": "rust", + "value": "1.97.1", + "source": "Cargo.toml" + }, + { + "category": "framework", + "kind": "detection", + "path": ".", + "id": "rocket" + }, + { + "category": "workspace", + "kind": "workspace", + "path": ".", + "id": "cargo-workspace", + "member": "macros" + } + ] + }, + { + "repository": "meilisearch/meilisearch", + "language": "Rust", + "stars": 59822, + "languageRank": 29, + "webRank": 2, + "revision": "8c7bfcfa9bcff7083bcb5d184fa2a523a23884d8", + "checks": [ + { + "category": "language", + "kind": "language", + "path": "crates/meilisearch", + "language": "rust" + }, + { + "category": "project", + "kind": "detection", + "path": "crates/meilisearch", + "id": "cargo" + }, + { + "category": "name", + "kind": "name", + "path": "crates/meilisearch", + "name": "meilisearch" + }, + { + "category": "framework", + "kind": "detection", + "path": "crates/meilisearch", + "id": "actix-web" + }, + { + "category": "workspace", + "kind": "workspace", + "path": ".", + "id": "cargo-workspace", + "member": "crates/meilisearch" + } + ] + }, + { + "repository": "juspay/hyperswitch", + "language": "Rust", + "stars": 45760, + "languageRank": 39, + "webRank": 3, + "revision": "5786b738c088b8452296ca19922fcdc0dc74e129", + "checks": [ + { + "category": "language", + "kind": "language", + "path": "crates/router", + "language": "rust" + }, + { + "category": "project", + "kind": "detection", + "path": "crates/router", + "id": "cargo" + }, + { + "category": "name", + "kind": "name", + "path": "crates/router", + "name": "router" + }, + { + "category": "version", + "kind": "version", + "path": "crates/router", + "language": "rust", + "value": "1.94.1", + "source": "Cargo.toml" + }, + { + "category": "framework", + "kind": "detection", + "path": "crates/router", + "id": "actix-web" + }, + { + "category": "workspace", + "kind": "workspace", + "path": ".", + "id": "cargo-workspace", + "member": "crates/router" + } + ] + }, + { + "repository": "parcel-bundler/parcel", + "language": "Rust", + "stars": 44012, + "languageRank": 42, + "webRank": 4, + "revision": "20c0f2ede4c1044092d44288169cce589c5d5c14", + "checks": [ + { + "category": "language", + "kind": "language", + "path": ".", + "language": "rust" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "cargo-workspace" + }, + { + "category": "workspace", + "kind": "workspace", + "path": ".", + "id": "cargo-workspace", + "member": "crates/browserslist-compat" + } + ] + }, + { + "repository": "DioxusLabs/dioxus", + "language": "Rust", + "stars": 39705, + "languageRank": 52, + "webRank": 5, + "revision": "17d9a5924e845e4f37deca3fada6c35d1d836af0", + "checks": [ + { + "category": "language", + "kind": "language", + "path": ".", + "language": "rust" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "cargo" + }, + { + "category": "name", + "kind": "name", + "path": ".", + "name": "dioxus-examples" + }, + { + "category": "version", + "kind": "version", + "path": ".", + "language": "rust", + "value": "1.85.0", + "source": "Cargo.toml" + }, + { + "category": "workspace", + "kind": "workspace", + "path": ".", + "id": "cargo-workspace", + "member": "packages/dioxus" + } + ] + }, + { + "repository": "freeCodeCamp/freeCodeCamp", + "language": "TypeScript", + "stars": 458826, + "languageRank": 1, + "webRank": 1, + "revision": "11442d66944785ac72a2a2b9c955fae8b9f33083", + "checks": [ + { + "category": "language", + "kind": "language", + "path": ".", + "language": "typescript" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "node" + }, + { + "category": "name", + "kind": "name", + "path": ".", + "name": "@freecodecamp/freecodecamp" + }, + { + "category": "version", + "kind": "version", + "path": ".", + "language": "javascript", + "value": ">=24", + "source": "package.json" + }, + { + "category": "version", + "kind": "version", + "path": ".", + "language": "typescript", + "value": "7.0.2", + "source": "package.json" + }, + { + "category": "workspace", + "kind": "workspace", + "path": ".", + "id": "node-workspace", + "member": "api" + } + ] + }, + { + "repository": "vuejs/vue", + "language": "TypeScript", + "stars": 212624, + "languageRank": 7, + "webRank": 2, + "revision": "9e88707940088cb1f4cd7dd210c9168a50dc347c", + "checks": [ + { + "category": "language", + "kind": "language", + "path": ".", + "language": "typescript" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "node" + }, + { + "category": "name", + "kind": "name", + "path": ".", + "name": "vue" + }, + { + "category": "version", + "kind": "version", + "path": ".", + "language": "typescript", + "value": "^4.8.4", + "source": "package.json" + }, + { + "category": "identity", + "kind": "detection", + "path": ".", + "id": "vue" + }, + { + "category": "workspace", + "kind": "workspace", + "path": ".", + "id": "node-workspace", + "member": "packages/compiler-sfc" + } + ] + }, + { + "repository": "n8n-io/n8n", + "language": "TypeScript", + "stars": 209927, + "languageRank": 8, + "webRank": 3, + "revision": "93243edc00b288a840fbb562a5e9f64177f2337f", + "checks": [ + { + "category": "language", + "kind": "language", + "path": "packages/cli", + "language": "typescript" + }, + { + "category": "project", + "kind": "detection", + "path": "packages/cli", + "id": "node" + }, + { + "category": "name", + "kind": "name", + "path": "packages/cli", + "name": "n8n" + }, + { + "category": "version", + "kind": "version", + "path": "packages/cli", + "language": "javascript", + "value": ">=24.0.0", + "source": "packages/cli/package.json" + }, + { + "category": "framework", + "kind": "detection", + "path": "packages/cli", + "id": "express" + }, + { + "category": "workspace", + "kind": "workspace", + "path": ".", + "id": "node-workspace", + "member": "packages/cli" + }, + { + "category": "catalog", + "kind": "version", + "path": "packages/cli", + "language": "typescript", + "value": "npm:@typescript/typescript6@6.0.2", + "source": "pnpm-workspace.yaml" + } + ] + }, + { + "repository": "firecrawl/firecrawl", + "language": "TypeScript", + "stars": 200129, + "languageRank": 9, + "webRank": 4, + "revision": "40c8ff95d719d2acbd8fd6273bd19b14a1c2bbb8", + "checks": [ + { + "category": "language", + "kind": "language", + "path": "apps/api", + "language": "typescript" + }, + { + "category": "project", + "kind": "detection", + "path": "apps/api", + "id": "node" + }, + { + "category": "name", + "kind": "name", + "path": "apps/api", + "name": "firecrawl-scraper-js" + }, + { + "category": "version", + "kind": "version", + "path": "apps/api", + "language": "typescript", + "value": "npm:@typescript/typescript6@6.0.2", + "source": "apps/api/package.json" + }, + { + "category": "framework", + "kind": "detection", + "path": "apps/api", + "id": "express" + } + ] + }, + { + "repository": "excalidraw/excalidraw", + "language": "TypeScript", + "stars": 135400, + "languageRank": 15, + "webRank": 5, + "revision": "701b901a0f1ba3d0b390d9cdf63d112e25abac4b", + "checks": [ + { + "category": "language", + "kind": "language", + "path": ".", + "language": "typescript" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "node" + }, + { + "category": "name", + "kind": "name", + "path": ".", + "name": "excalidraw-monorepo" + }, + { + "category": "version", + "kind": "version", + "path": ".", + "language": "javascript", + "value": ">=18.0.0", + "source": "package.json" + }, + { + "category": "version", + "kind": "version", + "path": ".", + "language": "typescript", + "value": "5.9.3", + "source": "package.json" + }, + { + "category": "framework", + "kind": "detection", + "path": "excalidraw-app", + "id": "react" + }, + { + "category": "workspace", + "kind": "workspace", + "path": ".", + "id": "node-workspace", + "member": "packages/excalidraw" + } + ] + }, + { + "repository": "jellyfin/jellyfin", + "language": "C#", + "stars": 59164, + "languageRank": 3, + "webRank": 1, + "revision": "797999fc9bd49a9f0853916e2933e3ad06173503", + "checks": [ + { + "category": "language", + "kind": "language", + "path": "Jellyfin.Server", + "language": "csharp" + }, + { + "category": "project", + "kind": "detection", + "path": "Jellyfin.Server", + "id": "dotnet" + }, + { + "category": "framework", + "kind": "detection", + "path": "Jellyfin.Server", + "id": "aspnetcore", + "roles": [ + "uses" + ] + }, + { + "category": "version", + "kind": "version", + "path": "Jellyfin.Server", + "language": "csharp", + "value": "10.0.0", + "source": "global.json", + "versionKind": "sdk" + }, + { + "category": "version", + "kind": "version", + "path": "Jellyfin.Server", + "language": "csharp", + "value": "net10.0", + "source": "Jellyfin.Server/Jellyfin.Server.csproj", + "versionKind": "target" + }, + { + "category": "workspace", + "kind": "workspace", + "path": ".", + "id": "dotnet-workspace", + "member": "Jellyfin.Server" + } + ] + }, + { + "repository": "dotnet/aspnetcore", + "language": "C#", + "stars": 38507, + "languageRank": 8, + "webRank": 2, + "revision": "eeceb660128d3d00ea9b35effe4fad3fac3d42e3", + "checks": [ + { + "category": "language", + "kind": "language", + "path": "src/Mvc/Mvc/src", + "language": "csharp" + }, + { + "category": "project", + "kind": "detection", + "path": "src/Mvc/Mvc/src", + "id": "dotnet" + }, + { + "category": "framework", + "kind": "detection", + "path": "src/Mvc/Mvc/src", + "id": "aspnetcore", + "roles": [ + "provides" + ] + }, + { + "category": "version", + "kind": "version", + "path": "src/Mvc/Mvc/src", + "language": "csharp", + "value": "12.0.100-alpha.1.26555.106", + "source": "global.json", + "versionKind": "sdk" + }, + { + "category": "workspace", + "kind": "workspace", + "path": ".", + "id": "dotnet-workspace", + "member": "src/Mvc/Mvc/src" + } + ] + }, + { + "repository": "jasontaylordev/CleanArchitecture", + "language": "C#", + "stars": 20711, + "languageRank": 29, + "webRank": 3, + "revision": "e1ab4cc4d5d7b4e25e57bd8b0514b766573b8c04", + "checks": [ + { + "category": "language", + "kind": "language", + "path": "src/Web", + "language": "csharp" + }, + { + "category": "project", + "kind": "detection", + "path": "src/Web", + "id": "dotnet" + }, + { + "category": "framework", + "kind": "detection", + "path": "src/Web", + "id": "aspnetcore", + "roles": [ + "uses" + ] + }, + { + "category": "version", + "kind": "version", + "path": "src/Web", + "language": "csharp", + "value": "10.0.403", + "source": "global.json", + "versionKind": "sdk" + }, + { + "category": "version", + "kind": "version", + "path": "src/Web", + "language": "csharp", + "value": "net10.0", + "source": "Directory.Build.props", + "versionKind": "target" + }, + { + "category": "workspace", + "kind": "workspace", + "path": ".", + "id": "dotnet-workspace", + "member": "src/Web" + } + ] + }, + { + "repository": "bitwarden/server", + "language": "C#", + "stars": 20610, + "languageRank": 30, + "webRank": 4, + "revision": "04a4d7548b07d18ab7327ddaae81a05fd893a820", + "checks": [ + { + "category": "language", + "kind": "language", + "path": "src/Api", + "language": "csharp" + }, + { + "category": "project", + "kind": "detection", + "path": "src/Api", + "id": "dotnet" + }, + { + "category": "framework", + "kind": "detection", + "path": "src/Api", + "id": "aspnetcore", + "roles": [ + "uses" + ] + }, + { + "category": "version", + "kind": "version", + "path": "src/Api", + "language": "csharp", + "value": "10.0.103", + "source": "global.json", + "versionKind": "sdk" + }, + { + "category": "version", + "kind": "version", + "path": "src/Api", + "language": "csharp", + "value": "net10.0", + "source": "Directory.Build.props", + "versionKind": "target" + }, + { + "category": "workspace", + "kind": "workspace", + "path": ".", + "id": "dotnet-workspace", + "member": "src/Api" + } + ] + }, + { + "repository": "ardalis/CleanArchitecture", + "language": "C#", + "stars": 18543, + "languageRank": 36, + "webRank": 5, + "revision": "da5491f1641f3d388e6cfc2c0ddd29f763746762", + "checks": [ + { + "category": "language", + "kind": "language", + "path": "src/Clean.Architecture.Web", + "language": "csharp" + }, + { + "category": "project", + "kind": "detection", + "path": "src/Clean.Architecture.Web", + "id": "dotnet" + }, + { + "category": "framework", + "kind": "detection", + "path": "src/Clean.Architecture.Web", + "id": "aspnetcore", + "roles": [ + "uses" + ] + }, + { + "category": "version", + "kind": "version", + "path": "src/Clean.Architecture.Web", + "language": "csharp", + "value": "10.0.100", + "source": "global.json", + "versionKind": "sdk" + }, + { + "category": "version", + "kind": "version", + "path": "src/Clean.Architecture.Web", + "language": "csharp", + "value": "net10.0", + "source": "Directory.Build.props", + "versionKind": "target" + }, + { + "category": "workspace", + "kind": "workspace", + "path": ".", + "id": "dotnet-workspace", + "member": "src/Clean.Architecture.Web" + }, + { + "category": "framework", + "kind": "detection", + "path": "src/Clean.Architecture.Web", + "id": "fastendpoints" + } + ] + }, + { + "repository": "macrozheng/mall", + "language": "Java", + "stars": 85023, + "languageRank": 3, + "webRank": 1, + "revision": "dcaa93b3150352e5044708d7211b5bed0af4509f", + "checks": [ + { + "category": "language", + "kind": "language", + "path": ".", + "language": "java" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "maven" + }, + { + "category": "framework", + "kind": "detection", + "path": ".", + "id": "spring-boot", + "roles": [ + "uses" + ] + }, + { + "category": "version", + "kind": "version", + "path": ".", + "language": "java", + "value": "17", + "source": "pom.xml" + }, + { + "category": "workspace", + "kind": "workspace", + "path": ".", + "id": "maven-workspace", + "member": "mall-admin" + } + ] + }, + { + "repository": "spring-projects/spring-boot", + "language": "Java", + "stars": 81788, + "languageRank": 4, + "webRank": 2, + "revision": "52dba7773c16cc5d145e59ad69de1b99e87896bb", + "checks": [ + { + "category": "language", + "kind": "language", + "path": "core/spring-boot", + "language": "java" + }, + { + "category": "project", + "kind": "detection", + "path": "core/spring-boot", + "id": "gradle" + }, + { + "category": "identity", + "kind": "detection", + "path": "core/spring-boot", + "id": "spring-boot", + "roles": [ + "provides" + ] + }, + { + "category": "version", + "kind": "version", + "path": "core/spring-boot", + "language": "java", + "value": "25.0.5.0+11-librca", + "source": ".sdkmanrc", + "versionKind": "pin" + }, + { + "category": "workspace", + "kind": "workspace", + "path": ".", + "id": "gradle-workspace", + "member": "core/spring-boot" + } + ] + }, + { + "repository": "elastic/elasticsearch", + "language": "Java", + "stars": 78546, + "languageRank": 6, + "webRank": 3, + "revision": "97fdd221f5919be472b60d29241f673774c19c51", + "checks": [ + { + "category": "language", + "kind": "language", + "path": ".", + "language": "java" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "gradle" + }, + { + "category": "version", + "kind": "version", + "path": ".", + "language": "java", + "value": "21", + "source": "gradle/gradle-daemon-jvm.properties", + "versionKind": "build-toolchain" + }, + { + "category": "workspace", + "kind": "workspace", + "path": ".", + "id": "gradle-workspace", + "member": "server" + } + ] + }, + { + "repository": "spring-projects/spring-framework", + "language": "Java", + "stars": 60307, + "languageRank": 11, + "webRank": 4, + "revision": "cc8008c301bc57b69c7c226254b720a2239741bf", + "checks": [ + { + "category": "language", + "kind": "language", + "path": "spring-core", + "language": "java" + }, + { + "category": "project", + "kind": "detection", + "path": "spring-core", + "id": "gradle" + }, + { + "category": "identity", + "kind": "detection", + "path": "spring-core", + "id": "spring", + "roles": [ + "provides" + ] + }, + { + "category": "version", + "kind": "version", + "path": "spring-core", + "language": "java", + "value": "25.0.5.0+11-librca", + "source": ".sdkmanrc", + "versionKind": "pin" + }, + { + "category": "workspace", + "kind": "workspace", + "path": ".", + "id": "gradle-workspace", + "member": "spring-core" + } + ] + }, + { + "repository": "jeecgboot/JeecgBoot", + "language": "Java", + "stars": 48413, + "languageRank": 15, + "webRank": 5, + "revision": "49c29032e18df784f1a75123d90647bc8f74cedb", + "checks": [ + { + "category": "language", + "kind": "language", + "path": "jeecg-boot/jeecg-module-system/jeecg-system-start", + "language": "java" + }, + { + "category": "project", + "kind": "detection", + "path": "jeecg-boot/jeecg-module-system/jeecg-system-start", + "id": "maven" + }, + { + "category": "framework", + "kind": "detection", + "path": "jeecg-boot/jeecg-module-system/jeecg-system-start", + "id": "spring-boot", + "roles": [ + "uses" + ] + }, + { + "category": "version", + "kind": "version", + "path": "jeecg-boot/jeecg-module-system/jeecg-system-start", + "language": "java", + "value": "17", + "source": "jeecg-boot/pom.xml" + }, + { + "category": "workspace", + "kind": "workspace", + "path": "jeecg-boot", + "id": "maven-workspace", + "member": "jeecg-boot/jeecg-module-system" + } + ] + }, + { + "repository": "supabase/supabase", + "language": "Deno", + "stars": 114734, + "webRank": 1, + "revision": "797a43c7d023777af25ba4621bdf87e66362e7bb", + "checks": [ + { + "category": "project", + "kind": "detection", + "path": "docker/volumes/functions", + "id": "deno" + }, + { + "category": "language", + "kind": "language", + "path": "docker/volumes/functions", + "language": "typescript" + }, + { + "category": "project", + "kind": "detection", + "path": "examples/edge-functions/supabase/functions/drizzle", + "id": "deno" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "node" + }, + { + "category": "name", + "kind": "name", + "path": ".", + "name": "supabase" + }, + { + "category": "runtime", + "kind": "runtime", + "path": "docker/volumes/functions", + "runtime": "deno", + "versionCount": 0 + }, + { + "category": "runtime", + "kind": "runtime", + "path": "examples/edge-functions/supabase/functions/drizzle", + "runtime": "deno", + "versionCount": 0 + }, + { + "category": "boundary", + "kind": "runtime", + "path": ".", + "runtime": "deno", + "absent": true + }, + { + "category": "boundary", + "kind": "detection", + "path": ".", + "id": "deno-workspace", + "absent": true + } + ], + "topicRank": 1, + "selectionQuery": "topic:deno archived:false fork:false" + }, + { + "repository": "honojs/hono", + "language": "Deno", + "stars": 32889, + "webRank": 2, + "revision": "61f7c59e50244ab4d2dd83c151928cf7ea448510", + "checks": [ + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "node" + }, + { + "category": "identity", + "kind": "detection", + "path": ".", + "id": "hono" + }, + { + "category": "project", + "kind": "detection", + "path": "adapters/deno", + "id": "deno" + }, + { + "category": "language", + "kind": "language", + "path": "adapters/deno", + "language": "typescript" + }, + { + "category": "name", + "kind": "name", + "path": "adapters/deno", + "name": "@hono/deno" + }, + { + "category": "framework", + "kind": "detection", + "path": "adapters/deno", + "id": "hono", + "roles": [ + "uses" + ] + }, + { + "category": "project", + "kind": "detection", + "path": "runtime-tests/deno", + "id": "deno" + }, + { + "category": "runtime", + "kind": "runtime", + "path": ".", + "runtime": "deno", + "versionCount": 1 + }, + { + "category": "runtime", + "kind": "runtime", + "path": "adapters/deno", + "runtime": "deno", + "versionCount": 1 + }, + { + "category": "runtime", + "kind": "runtime", + "path": "runtime-tests/deno", + "runtime": "deno", + "versionCount": 1 + }, + { + "category": "runtime-version", + "kind": "runtime-version", + "path": ".", + "runtime": "deno", + "value": "2.4.5", + "source": ".tool-versions", + "versionKind": "pin" + } + ], + "topicRank": 4, + "selectionQuery": "topic:deno archived:false fork:false" + }, + { + "repository": "freshframework/fresh", + "language": "Deno", + "stars": 13789, + "webRank": 3, + "revision": "86d6cdeb331a719cf8b1c85bf5e43c8ffa889b3b", + "checks": [ + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "deno" + }, + { + "category": "workspace", + "kind": "detection", + "path": ".", + "id": "deno-workspace" + }, + { + "category": "workspace", + "kind": "workspace", + "path": ".", + "id": "deno-workspace", + "member": "packages/fresh", + "ecosystem": "deno" + }, + { + "category": "workspace", + "kind": "workspace", + "path": ".", + "id": "deno-workspace", + "member": "packages/plugin-vite", + "ecosystem": "deno" + }, + { + "category": "workspace", + "kind": "workspace", + "path": ".", + "id": "deno-workspace", + "member": "www", + "ecosystem": "deno" + }, + { + "category": "project", + "kind": "detection", + "path": "packages/fresh", + "id": "deno" + }, + { + "category": "language", + "kind": "language", + "path": "packages/fresh", + "language": "typescript" + }, + { + "category": "name", + "kind": "name", + "path": "packages/fresh", + "name": "@fresh/core" + }, + { + "category": "project", + "kind": "detection", + "path": "www", + "id": "deno" + }, + { + "category": "runtime", + "kind": "runtime", + "path": ".", + "runtime": "deno", + "versionCount": 0 + }, + { + "category": "runtime", + "kind": "runtime", + "path": "packages/fresh", + "runtime": "deno", + "versionCount": 0 + }, + { + "category": "runtime", + "kind": "runtime", + "path": "www", + "runtime": "deno", + "versionCount": 0 + }, + { + "category": "identity", + "kind": "detection", + "path": "packages/fresh", + "id": "fresh", + "roles": [ + "provides" + ] + }, + { + "category": "project", + "kind": "detection", + "path": "packages/examples", + "id": "deno" + }, + { + "category": "framework", + "kind": "detection", + "path": "packages/examples", + "id": "fresh", + "roles": [ + "uses" + ] + }, + { + "category": "runtime", + "kind": "runtime", + "path": "packages/examples", + "runtime": "deno", + "versionCount": 0 + } + ], + "topicRank": 8, + "selectionQuery": "topic:deno archived:false fork:false" + }, + { + "repository": "InsForge/InsForge", + "language": "Deno", + "stars": 13316, + "webRank": 4, + "revision": "375f4bd7449a46481bb7e57bf872def52b9f9f68", + "checks": [ + { + "category": "project", + "kind": "detection", + "path": "functions", + "id": "deno" + }, + { + "category": "language", + "kind": "language", + "path": "functions", + "language": "typescript" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "node" + }, + { + "category": "name", + "kind": "name", + "path": ".", + "name": "insforge" + }, + { + "category": "framework", + "kind": "detection", + "path": "backend", + "id": "express", + "roles": [ + "uses" + ] + }, + { + "category": "framework", + "kind": "detection", + "path": "frontend", + "id": "react", + "roles": [ + "uses" + ] + }, + { + "category": "runtime", + "kind": "runtime", + "path": "functions", + "runtime": "deno", + "versionCount": 0 + } + ], + "topicRank": 9, + "selectionQuery": "topic:deno archived:false fork:false" + }, + { + "repository": "ryo-ma/github-profile-trophy", + "language": "Deno", + "stars": 6684, + "webRank": 5, + "revision": "43736b50a82fca73c8df1c81abb40aa03519975c", + "checks": [ + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "deno" + }, + { + "category": "language", + "kind": "language", + "path": ".", + "language": "typescript" + }, + { + "category": "runtime", + "kind": "runtime", + "path": ".", + "runtime": "deno", + "versionCount": 0 + } + ], + "topicRank": 15, + "selectionQuery": "topic:deno archived:false fork:false" + }, + { + "repository": "honojs/middleware", + "language": "Deno", + "stars": 1002, + "webRank": null, + "revision": "d813df31d07dfe1959558a1a25935ec3eddb9d02", + "checks": [ + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "deno" + }, + { + "category": "workspace", + "kind": "detection", + "path": ".", + "id": "deno-workspace" + }, + { + "category": "project", + "kind": "detection", + "path": ".", + "id": "node" + }, + { + "category": "workspace", + "kind": "workspace", + "path": ".", + "id": "deno-workspace", + "member": "packages/hello", + "ecosystem": "deno" + }, + { + "category": "workspace", + "kind": "workspace", + "path": ".", + "id": "deno-workspace", + "member": "packages/eslint-config", + "ecosystem": "deno" + }, + { + "category": "workspace", + "kind": "workspace", + "path": ".", + "id": "deno-workspace", + "member": "packages/ssg-plugins-essential", + "ecosystem": "deno" + }, + { + "category": "name", + "kind": "name", + "path": "packages/eslint-config", + "name": "@hono/eslint-config" + }, + { + "category": "project", + "kind": "detection", + "path": "packages/eslint-config", + "id": "node" + }, + { + "category": "project", + "kind": "detection", + "path": "packages/hello", + "id": "deno" + }, + { + "category": "project", + "kind": "detection", + "path": "packages/hello", + "id": "node" + }, + { + "category": "framework", + "kind": "detection", + "path": "packages/hello", + "id": "hono", + "roles": [ + "uses" + ] + }, + { + "category": "framework", + "kind": "detection", + "path": "packages/ssg-plugins-essential", + "id": "hono", + "roles": [ + "uses" + ] + }, + { + "category": "project", + "kind": "detection", + "path": "packages/graphql-server", + "id": "node" + }, + { + "category": "runtime", + "kind": "runtime", + "path": ".", + "runtime": "deno", + "versionCount": 1 + }, + { + "category": "runtime", + "kind": "runtime", + "path": "packages/hello", + "runtime": "deno", + "versionCount": 1 + }, + { + "category": "runtime", + "kind": "runtime", + "path": "packages/eslint-config", + "runtime": "deno", + "source": "deno.jsonc", + "versionCount": 1 + }, + { + "category": "runtime", + "kind": "runtime", + "path": "packages/ssg-plugins-essential", + "runtime": "deno", + "source": "deno.jsonc", + "versionCount": 1 + }, + { + "category": "framework", + "kind": "detection", + "path": "packages/ssg-plugins-essential", + "id": "hono", + "roles": [ + "uses" + ] + }, + { + "category": "runtime-version", + "kind": "runtime-version", + "path": ".", + "runtime": "deno", + "value": "2.3.7", + "source": ".tool-versions", + "versionKind": "pin" + }, + { + "category": "runtime-version", + "kind": "runtime-version", + "path": "packages/hello", + "runtime": "deno", + "value": "2.3.7", + "source": ".tool-versions", + "versionKind": "pin" + }, + { + "category": "runtime-version", + "kind": "runtime-version", + "path": "packages/eslint-config", + "runtime": "deno", + "value": "2.3.7", + "source": ".tool-versions", + "versionKind": "pin" + }, + { + "category": "runtime-version", + "kind": "runtime-version", + "path": "packages/ssg-plugins-essential", + "runtime": "deno", + "value": "2.3.7", + "source": ".tool-versions", + "versionKind": "pin" + } + ], + "topicRank": 60, + "selectionQuery": "topic:deno archived:false fork:false", + "supplemental": true + } + ], + "extensions": [ + { + "selectedAt": "2026-11-09T22:54:00.932983+00:00", + "languages": [ + "C#", + "Java" + ], + "repositoriesPerLanguage": 5, + "eligibility": "Same web-project criteria as existing corpus, including production web solution templates (as with create-react-app), excluding native/desktop-first apps, general runtimes, lists, and tutorial collections." + }, + { + "selectedAt": "2026-11-10T08:37:50.845853+00:00", + "ecosystems": [ + "Deno" + ], + "repositoriesPerEcosystem": 5, + "query": "GitHub query topic:deno archived:false fork:false; sort=stars; order=desc. languageRank stores raw topic-query rank (historical field name), webRank stores eligible web rank. API first 100 results retained in research/github-search.json.", + "eligibility": "Same web-first scope as prior45: applications, HTTP frameworks/libraries, browser UI and web site/build tools; require actual Deno config. Exclude runtimes, lists/tutorials, generic SQL/security/translation/CLI libraries, and optional-Deno tools without Deno config.", + "supplemental": [ + "honojs/middleware" + ], + "note": "Deno is a runtime ecosystem group, not a new language. Primary-language checks stay javascript/typescript." + } + ] +} diff --git a/tests/detect-corpus.md b/tests/detect-corpus.md new file mode 100644 index 0000000..c826f56 --- /dev/null +++ b/tests/detect-corpus.md @@ -0,0 +1,200 @@ +# GitHub project detection audit + +Selected 2026-11-09; C#/Java added later that day; Deno added 2026-11-10. Baseline: detector commit `244ab44ccf1a` (PR #20). Current: the follow-up fixes in this revision. + +## Method + +Five manually qualified web projects per supported language, ordered by GitHub stars among non-fork, non-archived repositories. Language means GitHub’s primary-language classification, not a measured ranking of the languages themselves. Production learning platforms qualify; tutorial collections, lists, coding agents, general runtimes, native desktop apps, and generic infrastructure tools do not. Production web solution templates qualify, consistently with create-react-app in the original sample. Eligibility is a judgment call, not an objective GitHub category. + +Examples excluded above selected projects: Node.js (general runtime), Ollama (model runtime), Kubernetes (infrastructure), frp (generic networking), and developer-roadmap (educational roadmap collection). Stars and language ranks are frozen with full commit SHAs in [`detect-corpus.json`](detect-corpus.json). + +Checks target reviewed project paths and static manifest declarations, not a matching framework somewhere in the repository’s examples. Identity checks separately ask whether framework implementations identify themselves. Catalog checks ask for the local catalog declaration rather than an unresolved reference. These last two categories exposed capabilities missing from the baseline dependency-only behavior. + +GitHub source archives were downloaded into a temporary cache. No application dependencies were installed and no downloaded repository code was executed. Submodules and LFS targets were not fetched; symlinks were not extracted. The corpus occupies approximately 4.1 GiB. This is a sampled assertion suite, not an exhaustive precision/recall or buildability evaluation. + +## Results + +The expanded corpus covers **45 repositories across nine languages**: the original seven-language sample plus five C# and five Java projects. All **45/45** scans succeed and produce identical output across five runs. **227/227 sampled assertions pass** (the original baseline passed 156/173). This is not a claim of exhaustive detection accuracy. + +| Category | Original 35 baseline | Current 45 | +|---|---:|---:| +| language | 35/35 | 45/45 | +| project | 35/35 | 45/45 | +| name | 26/27 | 27/27 | +| version | 32/32 | 46/46 | +| framework | 19/20 | 28/28 | +| workspace | 9/15 | 25/25 | +| identity | 0/7 | 9/9 | +| catalog | 0/2 | 2/2 | + +“Version” checks cover selected literal constraints and inherited declarations, not installed versions. Framework checks cover selected consumer projects, while identity checks cover framework implementation packages. + +### Misses fixed + +- **pnpm workspaces:** Next.js, freeCodeCamp, Vue, n8n, Dify, and Discourse now report their Node workspace labels and members, including negative patterns. A separate directory-glob cross-check found all 225 package members across these six snapshots, with no extra members. Cargo, npm/Yarn, and uv representatives still pass. +- **Rails components:** Discourse is recognized from the conjunction of `actionpack` and `railties`. `activesupport` or either web component alone does not label a project as Rails. +- **Poetry names:** AutoGPT’s backend retains its declared `autogpt-platform-backend` name. Names preserve declared spelling; dependency identity matching uses ecosystem normalization. +- **Framework identities:** React, Next.js, Vue, Laravel, Symfony, Rails, and Gin now identify their implementation packages. Framework detections distinguish `provides` from `uses`, rather than implying that a framework library is a deployable application. Ruby gemspec metadata is read statically, never evaluated. +- **pnpm catalogs:** Default and named catalogs resolve local version declarations while preserving both the reference and its YAML provenance. Missing entries, excluded packages, and malformed nested workspace files do not silently inherit unrelated catalog values. + +Additional unscored coverage gaps include Jekyll, Hugo, Dioxus, Ember, Gradio, and aiohttp. Their generic language/project fallback is not counted as a failed framework assertion. Test fixtures and example projects are intentionally scanned, so the report is not a list of deployable services. + +### Engine refactor validation + +The manifest catalog now declares filename-to-reader dispatch, JSON/TOML field extraction, workspace candidate selection, language families, inheritance boundaries, exclusions, property/version mappings, and MSBuild reference kinds. Detection predicates compile into typed fact indexes. Tests add an entire JSON manifest format and an XML property/reference policy through TOML only, rather than adding another language-specific Rust branch. + +All 45 complete reports match the pre-refactor binary after canonicalizing version-declaration ordering. The 227 corpus assertions still pass, including framework provider/consumer roles and version kinds. The ten starter branches are rerun too; Outline's unsupported Koa label and the redundant Ruby file-directive warning remain documented rather than counted as passing main-stack assertions. Version declarations are now sorted consistently, independent of reader execution order. + +The final local rerun passed 179 Rust tests and nine audit-runner tests. The XML corpus and portable benchmarks were run explicitly (they are ignored in ordinary test runs). The declarative filename router indexes known extensions and uses compiled basename matchers; this avoids paying for every manifest wildcard on every ordinary source filename. The 200-service/20,200-file warm synthetic scan remains roughly 4–5 ms on this host, reading only 200 manifests. The configuration compiler adds roughly half a millisecond of cold CLI startup in the local empty-directory A/B; these measurements are not cross-machine guarantees. + +### XML I/O and parser measurement + +The pinned cache contains 1,192 selected XML manifests, totaling 2,714,415 bytes and 31,157 closed/empty elements. Three independent release benchmark invocations measured the following aggregate warm-cache medians. Read timing includes bounded `open`/`read_to_end` plus UTF-8 ownership/validation; parse timing uses preloaded strings and a black-box no-op visitor. Discovery, .NET/Maven fact extraction, rules, and CLI output are excluded. These are not cold-disk measurements or timing budgets for CI. + +| XML kind | Files | Bytes | Warm read + UTF-8, ms | Preloaded parse, ms | +|---|---:|---:|---:|---:| +| csproj | 803 | 1,781,367 | 1.795 | 4.387 | +| props | 107 | 328,021 | 0.212 | 0.835 | +| targets | 67 | 178,848 | 0.131 | 0.539 | +| slnx | 7 | 105,053 | 0.019 | 0.249 | +| pom | 208 | 321,126 | 0.438 | 1.290 | +| **All XML** | **1,192** | **2,714,415** | **2.668** | **7.414** | + +XML is not demonstrated to be the whole-engine bottleneck. A text-allocation fast path reduced allocation calls by 9.5%, but paired CPU runs showed only a noisy 0.7% difference; separate invocations were sometimes slower. It was rejected, and production XML parsing/validation is unchanged. The ignored corpus and portable XML microbenchmarks are retained so future parser changes can be measured rather than guessed. + +### Timing + +Each repository was scanned through the release CLI five times, serially. The table reports the median of runs 2–5 with a warm filesystem, including process startup, traversal, JSON serialization, and stdout transfer. Python JSON parsing and snapshot downloads are excluded. These are local measurements, not portable timing budgets or controlled cold-cache benchmarks. + +Across the expanded corpus, the scanner counted 368,237 files and read 4,943 bounded manifest/version files (6.64 MiB). The current median repository time is 5.6 ms. Ordinary C#/Java source bodies are not read. + +An interleaved, same-machine A/B comparison ran the baseline and current binaries 12 times each, discarding the first two pairs: **Next.js improved from 170.8 ms to 86.8 ms** (1.97× faster), with the same full-tree scope. The generated 200-service / 20,200-file benchmark improved from 5.8 ms to **3.5 ms median**, still reading only 200 manifests. + +Profiling identified directory traversal and repeated path/ancestor lookups as the main costs. Changes index document lookups, retain relevant directory facts only, use parent indices for source ownership/version inheritance, and prefilter workspace candidates by manifest type. Wide directory trees use at most four workers; small frontiers remain serial. Buffered JSON output avoids per-line stdout flushes. Before adding the detection fixes, the indexing/ownership optimizations alone produced byte-identical results on all 35 repositories. + +pnpm uses `serde-saphyr` with deserialization only: typed fields are populated directly without constructing a syntax tree, retaining comments, enabling includes, or evaluating application code. A tree-sitter syntax tree is unnecessary for this manifest-only scan. + +| Language | Repository | Stars | Language rank | Revision | Original baseline assertions | Current assertions | Before ms | After ms | +|---|---|---:|---:|---|---:|---:|---:|---:| +| JavaScript | [react/react](https://github.com/react/react) | 252,455 | 2 | [`b05c0b75fd97`](https://github.com/react/react/tree/b05c0b75fd9778536152eead30814c9ca90c810a) | 5/6 | 6/6 | 14.47 | 8.69 | +| JavaScript | [vercel/next.js](https://github.com/vercel/next.js) | 143,437 | 8 | [`5368e013e74e`](https://github.com/vercel/next.js/tree/5368e013e74e4e4ed5be51c051c989545c4450b9) | 5/7 | 7/7 | 168.41 | 86.46 | +| JavaScript | [mrdoob/three.js](https://github.com/mrdoob/three.js) | 117,953 | 11 | [`3f7edc3d75e0`](https://github.com/mrdoob/three.js/tree/3f7edc3d75e054bc755dad56c1c95c62f63a3561) | 3/3 | 3/3 | 4.31 | 3.77 | +| JavaScript | [axios/axios](https://github.com/axios/axios) | 109,295 | 14 | [`335e2f8464f5`](https://github.com/axios/axios/tree/335e2f8464f58a3ab485486b8d6964fea3e3f30a) | 4/4 | 4/4 | 2.18 | 2.07 | +| JavaScript | [react/create-react-app](https://github.com/react/create-react-app) | 103,230 | 15 | [`6254386531d2`](https://github.com/react/create-react-app/tree/6254386531d263688ccfa542d0e628fbc0de0b28) | 6/6 | 6/6 | 2.30 | 2.41 | +| TypeScript | [freeCodeCamp/freeCodeCamp](https://github.com/freeCodeCamp/freeCodeCamp) | 458,826 | 1 | [`11442d669447`](https://github.com/freeCodeCamp/freeCodeCamp/tree/11442d66944785ac72a2a2b9c955fae8b9f33083) | 5/6 | 6/6 | 14.84 | 11.84 | +| TypeScript | [vuejs/vue](https://github.com/vuejs/vue) | 212,624 | 7 | [`9e8870794008`](https://github.com/vuejs/vue/tree/9e88707940088cb1f4cd7dd210c9168a50dc347c) | 4/6 | 6/6 | 2.58 | 3.65 | +| TypeScript | [n8n-io/n8n](https://github.com/n8n-io/n8n) | 209,927 | 8 | [`93243edc00b2`](https://github.com/n8n-io/n8n/tree/93243edc00b288a840fbb562a5e9f64177f2337f) | 5/7 | 7/7 | 60.35 | 34.09 | +| TypeScript | [firecrawl/firecrawl](https://github.com/firecrawl/firecrawl) | 200,129 | 9 | [`40c8ff95d719`](https://github.com/firecrawl/firecrawl/tree/40c8ff95d719d2acbd8fd6273bd19b14a1c2bbb8) | 5/5 | 5/5 | 4.86 | 5.07 | +| TypeScript | [excalidraw/excalidraw](https://github.com/excalidraw/excalidraw) | 135,400 | 15 | [`701b901a0f1b`](https://github.com/excalidraw/excalidraw/tree/701b901a0f1ba3d0b390d9cdf63d112e25abac4b) | 7/7 | 7/7 | 3.94 | 3.51 | +| Python | [Significant-Gravitas/AutoGPT](https://github.com/Significant-Gravitas/AutoGPT) | 187,972 | 11 | [`9cb76f9026d3`](https://github.com/Significant-Gravitas/AutoGPT/tree/9cb76f9026d388c9bf2e098af2beae1cc864f055) | 5/6 | 6/6 | 14.63 | 12.74 | +| Python | [AUTOMATIC1111/stable-diffusion-webui](https://github.com/AUTOMATIC1111/stable-diffusion-webui) | 165,269 | 14 | [`82a973c04367`](https://github.com/AUTOMATIC1111/stable-diffusion-webui/tree/82a973c04367123ae98bd9abdf80d9eda9b910e2) | 3/3 | 3/3 | 1.60 | 2.37 | +| Python | [langgenius/dify](https://github.com/langgenius/dify) | 160,774 | 15 | [`fdb09a9e76dc`](https://github.com/langgenius/dify/tree/fdb09a9e76dc8cbd052fabd72cfff4071328d9bc) | 8/10 | 10/10 | 30.46 | 20.48 | +| Python | [langflow-ai/langflow](https://github.com/langflow-ai/langflow) | 156,479 | 16 | [`60999cc9a735`](https://github.com/langflow-ai/langflow/tree/60999cc9a7350e44dde3ef641baae864e10861f0) | 6/6 | 6/6 | 20.32 | 12.47 | +| Python | [Comfy-Org/ComfyUI](https://github.com/Comfy-Org/ComfyUI) | 145,983 | 20 | [`540a82ee722b`](https://github.com/Comfy-Org/ComfyUI/tree/540a82ee722bda1f32a1fbd97fcadadeb22d548b) | 4/4 | 4/4 | 3.49 | 2.72 | +| PHP | [coollabsio/coolify](https://github.com/coollabsio/coolify) | 65,849 | 2 | [`30335d80866f`](https://github.com/coollabsio/coolify/tree/30335d80866ff3391d899cf4ccb599b696f47b7e) | 5/5 | 5/5 | 5.30 | 4.69 | +| PHP | [nextcloud/server](https://github.com/nextcloud/server) | 37,456 | 3 | [`d655757fb306`](https://github.com/nextcloud/server/tree/d655757fb306aca893651958030b5dae69ed0f39) | 3/3 | 3/3 | 14.62 | 8.06 | +| PHP | [laravel/framework](https://github.com/laravel/framework) | 34,995 | 4 | [`338e44065c0a`](https://github.com/laravel/framework/tree/338e44065c0a5b18f1f290605b6253a1e69d8f56) | 4/5 | 5/5 | 6.33 | 4.31 | +| PHP | [filamentphp/filament](https://github.com/filamentphp/filament) | 32,462 | 5 | [`9d78093cfebd`](https://github.com/filamentphp/filament/tree/9d78093cfebdc96c077b1d8dbf94b41738f28992) | 4/4 | 4/4 | 16.63 | 14.29 | +| PHP | [symfony/symfony](https://github.com/symfony/symfony) | 31,523 | 6 | [`f07928484c9e`](https://github.com/symfony/symfony/tree/f07928484c9e96646964deb7b9570d7885492512) | 4/5 | 5/5 | 10.10 | 8.76 | +| Ruby | [rails/rails](https://github.com/rails/rails) | 59,087 | 1 | [`3a0aff317bc0`](https://github.com/rails/rails/tree/3a0aff317bc037f43f168e3772028f1033f1e810) | 2/3 | 3/3 | 7.90 | 5.01 | +| Ruby | [jekyll/jekyll](https://github.com/jekyll/jekyll) | 51,756 | 2 | [`97c3bdc822df`](https://github.com/jekyll/jekyll/tree/97c3bdc822df9d5bea4c269b29a28de5a5e29d70) | 2/2 | 2/2 | 2.42 | 2.31 | +| Ruby | [mastodon/mastodon](https://github.com/mastodon/mastodon) | 50,502 | 3 | [`5a207bc8f504`](https://github.com/mastodon/mastodon/tree/5a207bc8f5046e27cffc640a3c64f96504f6afcf) | 4/4 | 4/4 | 8.68 | 6.27 | +| Ruby | [huginn/huginn](https://github.com/huginn/huginn) | 50,454 | 4 | [`639c050b2bf4`](https://github.com/huginn/huginn/tree/639c050b2bf4e14ef9983012af99dfc1dc68b334) | 4/4 | 4/4 | 2.35 | 2.21 | +| Ruby | [discourse/discourse](https://github.com/discourse/discourse) | 48,066 | 6 | [`899c68d783e7`](https://github.com/discourse/discourse/tree/899c68d783e7808c50144752369a685220250d7c) | 3/5 | 5/5 | 35.98 | 20.48 | +| Go | [infiniflow/ragflow](https://github.com/infiniflow/ragflow) | 93,887 | 8 | [`ed5a84fee9ab`](https://github.com/infiniflow/ragflow/tree/ed5a84fee9abc8091aea111d1195cf7768541ff1) | 5/5 | 5/5 | 10.20 | 5.06 | +| Go | [gohugoio/hugo](https://github.com/gohugoio/hugo) | 90,425 | 9 | [`d2010fcc9cd3`](https://github.com/gohugoio/hugo/tree/d2010fcc9cd30c5b4e9274ded4485f4222326b53) | 4/4 | 4/4 | 4.85 | 3.63 | +| Go | [gin-gonic/gin](https://github.com/gin-gonic/gin) | 89,609 | 11 | [`0f09c3a9b462`](https://github.com/gin-gonic/gin/tree/0f09c3a9b4626d9fe9979cf4d0521d6ae32ab646) | 4/5 | 5/5 | 2.11 | 1.89 | +| Go | [netdata/netdata](https://github.com/netdata/netdata) | 81,330 | 14 | [`0d54b4424ea6`](https://github.com/netdata/netdata/tree/0d54b4424ea6f40c344738b9d566e1a26d160da3) | 4/4 | 4/4 | 23.42 | 10.38 | +| Go | [caddyserver/caddy](https://github.com/caddyserver/caddy) | 78,927 | 15 | [`9ea3108377d2`](https://github.com/caddyserver/caddy/tree/9ea3108377d2abec102866808133c40ac9582162) | 4/4 | 4/4 | 2.56 | 2.31 | +| Rust | [dani-garcia/vaultwarden](https://github.com/dani-garcia/vaultwarden) | 70,490 | 17 | [`17b159beff4b`](https://github.com/dani-garcia/vaultwarden/tree/17b159beff4b892f07f61f38ce2fe452e55251c1) | 6/6 | 6/6 | 2.51 | 3.03 | +| Rust | [meilisearch/meilisearch](https://github.com/meilisearch/meilisearch) | 59,822 | 29 | [`8c7bfcfa9bcf`](https://github.com/meilisearch/meilisearch/tree/8c7bfcfa9bcff7083bcb5d184fa2a523a23884d8) | 5/5 | 5/5 | 4.99 | 4.70 | +| Rust | [juspay/hyperswitch](https://github.com/juspay/hyperswitch) | 45,760 | 39 | [`5786b738c088`](https://github.com/juspay/hyperswitch/tree/5786b738c088b8452296ca19922fcdc0dc74e129) | 6/6 | 6/6 | 12.07 | 7.99 | +| Rust | [parcel-bundler/parcel](https://github.com/parcel-bundler/parcel) | 44,012 | 42 | [`20c0f2ede4c1`](https://github.com/parcel-bundler/parcel/tree/20c0f2ede4c1044092d44288169cce589c5d5c14) | 3/3 | 3/3 | 18.92 | 12.36 | +| Rust | [DioxusLabs/dioxus](https://github.com/DioxusLabs/dioxus) | 39,705 | 52 | [`17d9a5924e84`](https://github.com/DioxusLabs/dioxus/tree/17d9a5924e845e4f37deca3fada6c35d1d836af0) | 5/5 | 5/5 | 8.71 | 7.18 | +| C# | [jellyfin/jellyfin](https://github.com/jellyfin/jellyfin) | 59,164 | 3 | [`797999fc9bd4`](https://github.com/jellyfin/jellyfin/tree/797999fc9bd49a9f0853916e2933e3ad06173503) | — | 6/6 | — | 5.57 | +| C# | [dotnet/aspnetcore](https://github.com/dotnet/aspnetcore) | 38,507 | 8 | [`eeceb660128d`](https://github.com/dotnet/aspnetcore/tree/eeceb660128d3d00ea9b35effe4fad3fac3d42e3) | — | 5/5 | — | 135.12 | +| C# | [jasontaylordev/CleanArchitecture](https://github.com/jasontaylordev/CleanArchitecture) | 20,711 | 29 | [`e1ab4cc4d5d7`](https://github.com/jasontaylordev/CleanArchitecture/tree/e1ab4cc4d5d7b4e25e57bd8b0514b766573b8c04) | — | 6/6 | — | 2.65 | +| C# | [bitwarden/server](https://github.com/bitwarden/server) | 20,610 | 30 | [`04a4d7548b07`](https://github.com/bitwarden/server/tree/04a4d7548b07d18ab7327ddaae81a05fd893a820) | — | 6/6 | — | 10.03 | +| C# | [ardalis/CleanArchitecture](https://github.com/ardalis/CleanArchitecture) | 18,543 | 36 | [`da5491f1641f`](https://github.com/ardalis/CleanArchitecture/tree/da5491f1641f3d388e6cfc2c0ddd29f763746762) | — | 7/7 | — | 3.70 | +| Java | [macrozheng/mall](https://github.com/macrozheng/mall) | 85,023 | 3 | [`dcaa93b31503`](https://github.com/macrozheng/mall/tree/dcaa93b3150352e5044708d7211b5bed0af4509f) | — | 5/5 | — | 2.67 | +| Java | [spring-projects/spring-boot](https://github.com/spring-projects/spring-boot) | 81,788 | 4 | [`52dba7773c16`](https://github.com/spring-projects/spring-boot/tree/52dba7773c16cc5d145e59ad69de1b99e87896bb) | — | 5/5 | — | 44.52 | +| Java | [elastic/elasticsearch](https://github.com/elastic/elasticsearch) | 78,546 | 6 | [`97fdd221f591`](https://github.com/elastic/elasticsearch/tree/97fdd221f5919be472b60d29241f673774c19c51) | — | 4/4 | — | 52.31 | +| Java | [spring-projects/spring-framework](https://github.com/spring-projects/spring-framework) | 60,307 | 11 | [`cc8008c301bc`](https://github.com/spring-projects/spring-framework/tree/cc8008c301bc57b69c7c226254b720a2239741bf) | — | 5/5 | — | 9.90 | +| Java | [jeecgboot/JeecgBoot](https://github.com/jeecgboot/JeecgBoot) | 48,413 | 15 | [`49c29032e18d`](https://github.com/jeecgboot/JeecgBoot/tree/49c29032e18df784f1a75123d90647bc8f74cedb) | — | 5/5 | — | 10.83 | + +### C#/Java coverage and limits + +The new C# sample is Jellyfin, ASP.NET Core, both production Clean Architecture templates, and Bitwarden. The Java sample is mall, Spring Boot, Elasticsearch, Spring Framework, and JeecgBoot. SDK/TFM/language assertions are distinct; provider roles are checked on framework implementation packages rather than examples elsewhere in those repositories. + +MSBuild reads `.csproj`, `.sln`/`.slnx`, `global.json`, directory props/targets, in-repo imports and central package versions. Simple literal conditions and property substitutions are handled; arbitrary property functions and external SDK imports are not evaluated. Maven reads local matching parent coordinates, properties, managed versions, direct non-test dependencies, compiler configuration and modules. Remote parents/BOMs and profile activation are not fetched or evaluated. Gradle Groovy/Kotlin DSL support is static: literal plugins/dependencies, toolchains, module lists, directory/name overrides, conventional build filenames, SDKMAN/JDK pins and daemon JVM declarations. Custom/dynamic Gradle include logic remains diagnosed rather than executed; physically discovered project manifests are still reported. No JDK, .NET SDK, Maven, Gradle, or application packages were installed. + +Elasticsearch demonstrates that limit: its static server module, root workspace, source projects and build JDK are recognized, but arbitrary code-driven module expansion is not claimed complete. These sampled assertions do not certify every build condition or inferred deployment topology. + +## Additional validation: Brainpod starters + +Ran the release detector against all ten `case/*` branches of [`brainpodnl/starters`](https://github.com/brainpodnl/starters), pinning each commit below. The default branch is `case/bookstack`. This is a separate starter smoke test, not part of the ranked-corpus assertions above. + +Each branch was scanned five times without installing application dependencies or executing downloaded code. All ten scans succeeded and repeated byte-identically. The initial run recognized seven expected main stacks. After the C#/Java follow-up, nine main stacks are recognized; Outline remains partial because Koa has no rule. Auxiliary docs/assets must not be counted as detection of eShop’s main application. + +Timings are warm-filesystem, end-to-end CLI medians of runs 2–5. Empty or partial output is not treated as an accuracy pass merely because it is fast. + +| Branch | Revision | Result | Median ms | +|---|---|---|---:| +| `case/bookstack` | [`6265d5667dd6`](https://github.com/brainpodnl/starters/tree/6265d5667dd6904b7aa0d05a8c60795cf104011b) | Laravel, Composer, JavaScript/TypeScript. PHP `^8.2.0` and Laravel `^v12.26.4` declarations retained. | 4.07 | +| `case/eshop` | [`f21d6233b851`](https://github.com/brainpodnl/starters/tree/f21d6233b851897371c25c5e7ffa67bb8552dba8) | **Now detected:** .NET solution, ASP.NET Core projects, declared SDK `10.0.0` and `net10.0` targets; docs/assets remain separate. | 3.14 | +| `case/fastapi-fullstack` | [`84c5a9e11a9e`](https://github.com/brainpodnl/starters/tree/84c5a9e11a9e58262599e3d772f5386924e02ca7) | FastAPI backend, React/Vite frontend, npm and uv workspaces. Python `3.14` pin plus `>=3.14,<4.0` constraint retained; all three workspace members found. | 2.44 | +| `case/linkding` | [`24b5ad6cc9bd`](https://github.com/brainpodnl/starters/tree/24b5ad6cc9bde497b5d1b1e86aed5a2fb7d25c2f) | Django backend (`python >=3.13`) and Astro documentation project detected. | 2.52 | +| `case/maybe` | [`224f21354a91`](https://github.com/brainpodnl/starters/tree/224f21354a916dfbec29e9435db1045b3d61a94b) | Rails and Ruby `3.4.4` pin detected. Non-fatal `dynamic ruby declaration not evaluated` warning for `ruby file: ".ruby-version"`. | 3.09 | +| `case/miniflux` | [`faecfa2fac61`](https://github.com/brainpodnl/starters/tree/faecfa2fac61a91f4f69550bb1afd506c4b51749) | Go module `miniflux.app/v2`, Go `1.26.0` requirement detected; auxiliary scripts/assets are separate source projects. | 2.47 | +| `case/outline` | [`09f11188b87b`](https://github.com/brainpodnl/starters/tree/09f11188b87b0220fa318dfa6fdd0fa9772195dd) | **Partial:** Node/React/Vite, TypeScript, Node engine constraint and `.nvmrc` pin detected. Declared Koa `^3.0.3` backend is not labeled. | 4.06 | +| `case/saas-starter` | [`6e33e58b1e55`](https://github.com/brainpodnl/starters/tree/6e33e58b1e553a41fe22e6b941a7229a002de361) | Next.js/React and TypeScript detected. No package name or Node version is invented when absent from supported manifest fields. | 2.05 | +| `case/spring-petclinic` | [`500158f73241`](https://github.com/brainpodnl/starters/tree/500158f732419217507c7656904b8e6aa1bcc0d6) | **Now detected:** Maven, Spring Boot parent `4.1.0`, and Java `17` declaration. | 1.89 | +| `case/uptime-kuma` | [`df601c58d920`](https://github.com/brainpodnl/starters/tree/df601c58d9205562f34cfe5ef35264b341019d19) | Express/Vue/Vite and Node `>=20.4.0` constraint detected, alongside auxiliary example projects. | 2.60 | + +Koa needs an additional npm framework rule. C#/.NET and Java/Spring are now supported and verified by the follow-up run. The Maybe warning is redundant with the already detected Ruby pin. None of these observations was silently converted into a passing assertion or hidden by skipping a branch. The initial validation was read-only; the follow-up uses the new language support. + +## Deno extension (2026-11-10) + +The default corpus now includes **51 pinned repositories**: all original 45 plus five Deno web projects and one explicit mixed-workspace supplement. The runtime group is selected with GitHub query `topic:deno archived:false fork:false`, sorted descending by stars; it is not a tenth language or a primary-language ranking. `topicRank`, star counts, full SHAs, and independently reviewed assertions are frozen in `detect-corpus.json`. + +| Eligible rank | Topic rank | Repository | Stars | +|---|---:|---|---:| +| 1 | 1 | supabase/supabase | 114,734 | +| 2 | 4 | honojs/hono | 32,889 | +| 3 | 8 | freshframework/fresh | 13,789 | +| 4 | 9 | InsForge/InsForge | 13,316 | +| 5 | 15 | ryo-ma/github-profile-trophy | 6,684 | +| Supplemental | 60 | honojs/middleware | 1,002 | + +Higher-ranked exclusions follow the existing web-first criteria: denoland/deno is a runtime; rust-course is educational; pretty-ts-errors is an editor extension; Kysely is a general SQL builder; SheetJS and several other optional-Deno libraries have no Deno config in the inspected trees. This is a qualified sample, not “the five most-starred repositories that happen to mention Deno.” + +Supabase and InsForge cover nested Deno functions inside npm/pnpm applications. Hono preserves framework-provider identity, its Deno adapter, and an explicit `.tool-versions` **2.4.5 pin**. Fresh covers a **nine-member glob workspace**, inherited imports, member overrides, and JSR package identity without confusing package version `2.3.3` with a runtime version. The supplemental middleware repository covers JSONC, npm-only members, hybrid packages, manager-specific workspace membership, distinct npm peer/development and JSR constraints, and its `.tool-versions` **2.3.7 pin**. github-profile-trophy is a Deno HTTP application with no static runtime version declaration; that absence remains explicit. + +All **295/295 assertions across 51 repositories** pass on the extended engine, with five deterministic runs each. This includes all original 227 assertions plus 68 new checks, including runtime/version provenance, manager-qualified membership, and negative workspace-boundary checks. All ten pinned Brainpod starter branches are rerun as well; their previously documented Koa label/Ruby warning limitations remain. Synthetic fixtures cover simultaneous config preference, comments/trailing commas, excluded/non-member packages, local import maps, missing/oversized/escaping/symlinked references, and runtime/package version separation. The scanner still executes no inspected application code or dependency installs. + +`runtimes` is additive JSON metadata (Deno is not a language). `workspaces` exposes `{ecosystem, source, members}` separately per manager; `workspaceMembers` is still their sorted union. A pnpm-only workspace member does not become a Deno member merely because both managers coexist at the root. Scoped import-map overrides and absolute/remote import-map file locations remain explicit unsupported diagnostics. Registry/URL module targets are metadata only and are never fetched. + +Official behavior references: [configuration](https://docs.deno.com/runtime/fundamentals/configuration/), [workspaces](https://docs.deno.com/runtime/fundamentals/workspaces/), and [config discovery implementation](https://github.com/denoland/deno_config/blob/main/src/deno_json/mod.rs) (`deno.json` precedes `deno.jsonc`). These document current behavior; pinned repository SHAs, not moving documentation or CI/Docker versions, define the assertion evidence. + +## Reproduce + +```sh +python3 tests/audit_detect.py --list +cargo build --release --locked +cache=$(mktemp -d) +python3 tests/audit_detect.py --fetch --cache "$cache" --output "$cache/results.json" +``` + +Downloads require the explicit `--fetch` flag; omit it to rerun against an existing pinned cache entirely offline. Downloads are capped at 1 GiB compressed / 2 GiB expanded per repository. Existing unverified cache entries are not overwritten. The runner exits nonzero for missing assertions, command failures, or nondeterministic output; **the current corpus is expected to exit 0**. + +```sh +PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s tests -p 'test_*.py' -v +``` + +Use `--list` to print all repository names and pinned revisions without a binary, cache, downloads, or network access. The complete language-grouped list is also in the main README. The runner uses Python 3.11+ and the standard library only. Full per-check results and repeat timings are written to the specified JSON output; downloaded repositories and generated reports are not committed. + +```sh +XML_BENCH_CACHE="$cache" cargo test --release detect::xml::tests::benchmark_xml_corpus \ + -- --ignored --nocapture --test-threads=1 +cargo test --release detect::xml::tests::benchmark_xml_local -- --ignored --nocapture +``` diff --git a/tests/test_audit_detect.py b/tests/test_audit_detect.py new file mode 100644 index 0000000..7471dec --- /dev/null +++ b/tests/test_audit_detect.py @@ -0,0 +1,318 @@ +import io +import json +import tarfile +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +import audit_detect + + +class DetectionAuditTests(unittest.TestCase): + def setUp(self): + self.project = { + "path": ".", + "names": ["app"], + "languages": [ + { + "name": "javascript", + "versions": [{"value": ">=22", "source": "package.json"}], + } + ], + "detections": [{"id": "node"}, {"id": "node-workspace"}], + "workspaceMembers": ["packages/api"], + } + self.entry = {"repository": "owner/repo", "revision": "a" * 40} + + def test_checks_are_scoped_to_the_expected_project(self): + cases = [ + {"kind": "language", "language": "javascript"}, + {"kind": "detection", "id": "node"}, + {"kind": "name", "name": "app"}, + {"kind": "workspace", "id": "node-workspace", "member": "packages/api"}, + { + "kind": "version", + "language": "javascript", + "value": ">=22", + "source": "package.json", + }, + ] + for case in cases: + with self.subTest(case=case): + self.assertTrue( + audit_detect.check({".": self.project}, {**case, "path": "."})[0] + ) + self.assertFalse( + audit_detect.check( + {".": self.project}, {**case, "path": "missing"} + )[0] + ) + wrong = { + "kind": "version", + "path": ".", + "language": "javascript", + "value": ">=22", + "source": "wrong.json", + } + self.assertFalse(audit_detect.check({".": self.project}, wrong)[0]) + with self.assertRaises(ValueError): + audit_detect.check({".": self.project}, {"kind": "unknown", "path": "."}) + + def test_role_and_version_kind_expectations_are_enforced(self): + self.project["detections"][0]["roles"] = ["uses"] + expected = { + "kind": "detection", + "path": ".", + "id": "node", + "roles": ["provides"], + } + self.assertFalse(audit_detect.check({".": self.project}, expected)[0]) + expected["roles"] = ["uses"] + self.assertTrue(audit_detect.check({".": self.project}, expected)[0]) + self.project["languages"][0]["versions"][0]["kind"] = "sdk" + expected = { + "kind": "version", + "path": ".", + "language": "javascript", + "value": ">=22", + "source": "package.json", + "versionKind": "language", + } + self.assertFalse(audit_detect.check({".": self.project}, expected)[0]) + expected["versionKind"] = "sdk" + self.assertTrue(audit_detect.check({".": self.project}, expected)[0]) + + def test_workspace_checks_can_distinguish_managers(self): + self.project["workspaceMembers"] = ["deno-member", "pnpm-member"] + self.project["detections"].append({"id": "deno-workspace"}) + self.project["workspaces"] = [ + {"ecosystem": "deno", "source": "deno.jsonc", "members": ["deno-member"]}, + { + "ecosystem": "node", + "source": "pnpm-workspace.yaml", + "members": ["pnpm-member"], + }, + ] + check = { + "kind": "workspace", + "path": ".", + "id": "deno-workspace", + "ecosystem": "deno", + "source": "deno.jsonc", + "member": "deno-member", + } + self.assertTrue(audit_detect.check({".": self.project}, check)[0]) + for wrong in [ + {"member": "pnpm-member"}, + {"source": "package.json"}, + {"ecosystem": "cargo"}, + ]: + self.assertFalse( + audit_detect.check({".": self.project}, {**check, **wrong})[0] + ) + + def test_runtime_declarations_are_not_language_or_package_versions(self): + self.project["runtimes"] = [ + { + "name": "deno", + "source": "deno.jsonc", + "versions": [ + {"value": "2.4.5", "source": ".tool-versions", "kind": "pin"} + ], + } + ] + presence = { + "kind": "runtime", + "path": ".", + "runtime": "deno", + "source": "deno.jsonc", + } + self.assertTrue(audit_detect.check({".": self.project}, presence)[0]) + self.assertFalse( + audit_detect.check( + {".": self.project}, {**presence, "source": "package.json"} + )[0] + ) + version = { + "kind": "runtime-version", + "path": ".", + "runtime": "deno", + "value": "2.4.5", + "source": ".tool-versions", + "versionKind": "pin", + } + self.assertTrue(audit_detect.check({".": self.project}, version)[0]) + for wrong in [ + {"runtime": "node"}, + {"value": "9.9.9"}, + {"source": "deno.jsonc"}, + {"versionKind": "requirement"}, + ]: + self.assertFalse( + audit_detect.check({".": self.project}, {**version, **wrong})[0] + ) + self.assertFalse( + audit_detect.check({".": self.project}, {**presence, "versionCount": 0})[0] + ) + self.assertTrue( + audit_detect.check({".": self.project}, {**presence, "versionCount": 1})[0] + ) + self.assertFalse( + audit_detect.check({".": self.project}, {**presence, "absent": True})[0] + ) + del self.project["runtimes"] + self.assertFalse(audit_detect.check({".": self.project}, presence)[0]) + self.assertTrue( + audit_detect.check({".": self.project}, {**presence, "absent": True})[0] + ) + self.assertTrue( + audit_detect.check( + {".": self.project}, + { + "path": ".", + "kind": "detection", + "id": "deno-workspace", + "absent": True, + }, + )[0] + ) + self.assertFalse( + audit_detect.check( + {".": self.project}, + {"path": ".", "kind": "detection", "id": "node", "absent": True}, + )[0] + ) + + def test_listing_needs_no_binary_cache_or_network(self): + with ( + tempfile.TemporaryDirectory() as directory, + patch("urllib.request.urlopen") as request, + ): + corpus = Path(directory) / "corpus.json" + corpus.write_text( + json.dumps( + { + "schemaVersion": 1, + "repositories": [ + { + "language": "Java", + "repository": "owner/app", + "revision": "a" * 40, + } + ], + } + ) + ) + with ( + patch( + "sys.argv", + [ + "audit_detect.py", + "--corpus", + str(corpus), + "--binary", + "/missing/binary", + "--list", + ], + ), + patch("sys.stdout", new_callable=io.StringIO) as output, + ): + self.assertEqual(audit_detect.main(), 0) + self.assertIn("Java\towner/app\t" + "a" * 40, output.getvalue()) + request.assert_not_called() + + def archive(self, path="repo/package.json", size=None): + data = b"{}" if size is None else b"x" * size + output = io.BytesIO() + with tarfile.open(fileobj=output, mode="w:gz") as archive: + member = tarfile.TarInfo(path) + member.size = len(data) + archive.addfile(member, io.BytesIO(data)) + link = tarfile.TarInfo("repo/linked.json") + link.type = tarfile.SYMTYPE + link.linkname = "/etc/passwd" + archive.addfile(link) + return output.getvalue() + + def test_missing_snapshots_never_download_without_fetch(self): + with ( + tempfile.TemporaryDirectory() as directory, + patch("urllib.request.urlopen") as request, + ): + with self.assertRaisesRegex(ValueError, "pass --fetch"): + audit_detect.snapshot(self.entry, Path(directory), False) + request.assert_not_called() + + def test_cached_snapshot_requires_the_pinned_revision(self): + with ( + tempfile.TemporaryDirectory() as directory, + patch("urllib.request.urlopen") as request, + ): + cache = Path(directory) + destination = cache / "repos/owner--repo" + destination.mkdir(parents=True) + metadata = cache / "metadata/owner--repo.json" + metadata.parent.mkdir() + metadata.write_text(json.dumps(self.entry)) + self.assertEqual( + audit_detect.snapshot(self.entry, cache, False), destination + ) + metadata.write_text(json.dumps({**self.entry, "revision": "b" * 40})) + with self.assertRaisesRegex(ValueError, "unverified snapshot exists"): + audit_detect.snapshot(self.entry, cache, True) + request.assert_not_called() + + def test_fetch_extracts_regular_files_but_not_symlinks(self): + with ( + tempfile.TemporaryDirectory() as directory, + patch("urllib.request.urlopen", return_value=io.BytesIO(self.archive())), + ): + destination = audit_detect.snapshot(self.entry, Path(directory), True) + self.assertEqual((destination / "package.json").read_text(), "{}") + self.assertFalse((destination / "linked.json").exists()) + self.assertFalse((destination / "linked.json").is_symlink()) + + def test_unsafe_archives_do_not_escape_the_cache(self): + for path in ["repo/../../escape", "/absolute/path"]: + with ( + self.subTest(path=path), + tempfile.TemporaryDirectory() as directory, + patch( + "urllib.request.urlopen", + return_value=io.BytesIO(self.archive(path)), + ), + ): + with self.assertRaisesRegex(ValueError, "unsafe archive path"): + audit_detect.snapshot(self.entry, Path(directory), True) + self.assertFalse((Path(directory) / "repos/owner--repo").exists()) + + def test_archive_and_expansion_limits_are_enforced(self): + for limit, archive, message in [ + (1, self.archive(), "archive exceeds"), + (512, self.archive(size=4096), "expanded snapshot exceeds"), + ]: + with ( + self.subTest(limit=limit), + tempfile.TemporaryDirectory() as directory, + patch("urllib.request.urlopen", return_value=io.BytesIO(archive)), + patch.object(audit_detect, "MAX_ARCHIVE_BYTES", limit), + self.assertRaisesRegex(ValueError, message), + ): + audit_detect.snapshot(self.entry, Path(directory), True) + + def test_invalid_repository_metadata_is_rejected(self): + for entry in [ + {**self.entry, "repository": "../repo"}, + {**self.entry, "revision": "main"}, + ]: + with ( + self.subTest(entry=entry), + tempfile.TemporaryDirectory() as directory, + self.assertRaisesRegex(ValueError, "invalid repository"), + ): + audit_detect.snapshot(entry, Path(directory), False) + + +if __name__ == "__main__": + unittest.main()