Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
195 changes: 113 additions & 82 deletions crates/codeguard-adapters/src/grammar_asset_manifest.rs
Original file line number Diff line number Diff line change
Expand Up @@ -93,10 +93,39 @@ pub struct GrammarAsset {
/// 读取仓内固定清单,并检查来源、许可证及候选资产字节。
pub fn bundled_grammar_candidates() -> Result<GrammarAssetManifest, String> {
let manifest = parse_grammar_asset_manifest(include_bytes!("../../../grammars/manifest.json"))?;
let codegraph_license = include_bytes!("../../../grammars/LICENSE.codegraph");
if digest(codegraph_license) != CODEGRAPH_LICENSE_SHA256 {
verify_codegraph_license()?;
verify_dependency_license(&manifest)?;
for asset in &manifest.assets {
verify_bundled_asset(asset)?;
}
Ok(manifest)
}

/// 只核对指定内置 grammar 的字节与许可,供按需加载的语法工作进程使用。
/// 参数为固定语言 ID;返回清单身份和静态 WASM 字节,未知语言返回错误。
pub fn bundled_grammar_candidate(language: &str) -> Result<(GrammarAsset, &'static [u8]), String> {
let manifest = parse_grammar_asset_manifest(include_bytes!("../../../grammars/manifest.json"))?;
let asset = manifest
.assets
.iter()
.find(|asset| asset.language == language)
.ok_or("不支持的 grammar 语种")?;
verify_codegraph_license()?;
if matches!(language, "objc" | "solidity") {
verify_dependency_license(&manifest)?;
}
let wasm = verify_bundled_asset(asset)?;
Ok((asset.clone(), wasm))
}

fn verify_codegraph_license() -> Result<(), String> {
if digest(include_bytes!("../../../grammars/LICENSE.codegraph")) != CODEGRAPH_LICENSE_SHA256 {
return Err("CodeGraph 许可证字节与固定来源不符".into());
}
Ok(())
}

fn verify_dependency_license(manifest: &GrammarAssetManifest) -> Result<(), String> {
if digest(include_bytes!(
"../../../grammars/LICENSE.tree-sitter-wasms"
)) != manifest
Expand All @@ -106,90 +135,92 @@ pub fn bundled_grammar_candidates() -> Result<GrammarAssetManifest, String> {
{
return Err("依赖包许可证字节与固定来源不符".into());
}
for asset in &manifest.assets {
let (wasm, license) = match asset.language.as_str() {
"c" => (
include_bytes!("../../../grammars/c/parser.wasm").as_slice(),
include_bytes!("../../../grammars/c/LICENSE").as_slice(),
),
"cpp" => (
include_bytes!("../../../grammars/cpp/parser.wasm").as_slice(),
include_bytes!("../../../grammars/cpp/LICENSE").as_slice(),
),
"csharp" => (
include_bytes!("../../../grammars/csharp/parser.wasm").as_slice(),
include_bytes!("../../../grammars/csharp/LICENSE").as_slice(),
),
"go" => (
include_bytes!("../../../grammars/go/parser.wasm").as_slice(),
include_bytes!("../../../grammars/go/LICENSE").as_slice(),
),
"javascript" => (
include_bytes!("../../../grammars/javascript/parser.wasm").as_slice(),
include_bytes!("../../../grammars/javascript/LICENSE").as_slice(),
),
"lua" => (
include_bytes!("../../../grammars/lua/parser.wasm").as_slice(),
include_bytes!("../../../grammars/lua/LICENSE").as_slice(),
),
"luau" => (
include_bytes!("../../../grammars/luau/parser.wasm").as_slice(),
include_bytes!("../../../grammars/luau/LICENSE").as_slice(),
),
"java" => (
include_bytes!("../../../grammars/java/parser.wasm").as_slice(),
include_bytes!("../../../grammars/java/LICENSE").as_slice(),
),
"typescript" => (
include_bytes!("../../../grammars/typescript/parser.wasm").as_slice(),
include_bytes!("../../../grammars/typescript/LICENSE").as_slice(),
),
"python" => (
include_bytes!("../../../grammars/python/parser.wasm").as_slice(),
include_bytes!("../../../grammars/python/LICENSE").as_slice(),
),
"tsx" => (
include_bytes!("../../../grammars/tsx/parser.wasm").as_slice(),
include_bytes!("../../../grammars/typescript/LICENSE").as_slice(),
),
"zig" => (
include_bytes!("../../../grammars/zig/parser.wasm").as_slice(),
include_bytes!("../../../grammars/zig/LICENSE").as_slice(),
),
"objc" => (
include_bytes!("../../../grammars/objc/parser.wasm").as_slice(),
include_bytes!("../../../grammars/objc/LICENSE").as_slice(),
),
"solidity" => (
include_bytes!("../../../grammars/solidity/parser.wasm").as_slice(),
include_bytes!("../../../grammars/solidity/LICENSE").as_slice(),
),
"rust" => (
include_bytes!("../../../grammars/rust/parser.wasm").as_slice(),
include_bytes!("../../../grammars/rust/LICENSE").as_slice(),
),
_ => return Err("候选语言未知".into()),
Ok(())
}

fn verify_bundled_asset(asset: &GrammarAsset) -> Result<&'static [u8], String> {
let (wasm, license) = match asset.language.as_str() {
"c" => (
include_bytes!("../../../grammars/c/parser.wasm").as_slice(),
include_bytes!("../../../grammars/c/LICENSE").as_slice(),
),
"cpp" => (
include_bytes!("../../../grammars/cpp/parser.wasm").as_slice(),
include_bytes!("../../../grammars/cpp/LICENSE").as_slice(),
),
"csharp" => (
include_bytes!("../../../grammars/csharp/parser.wasm").as_slice(),
include_bytes!("../../../grammars/csharp/LICENSE").as_slice(),
),
"go" => (
include_bytes!("../../../grammars/go/parser.wasm").as_slice(),
include_bytes!("../../../grammars/go/LICENSE").as_slice(),
),
"javascript" => (
include_bytes!("../../../grammars/javascript/parser.wasm").as_slice(),
include_bytes!("../../../grammars/javascript/LICENSE").as_slice(),
),
"lua" => (
include_bytes!("../../../grammars/lua/parser.wasm").as_slice(),
include_bytes!("../../../grammars/lua/LICENSE").as_slice(),
),
"luau" => (
include_bytes!("../../../grammars/luau/parser.wasm").as_slice(),
include_bytes!("../../../grammars/luau/LICENSE").as_slice(),
),
"java" => (
include_bytes!("../../../grammars/java/parser.wasm").as_slice(),
include_bytes!("../../../grammars/java/LICENSE").as_slice(),
),
"typescript" => (
include_bytes!("../../../grammars/typescript/parser.wasm").as_slice(),
include_bytes!("../../../grammars/typescript/LICENSE").as_slice(),
),
"python" => (
include_bytes!("../../../grammars/python/parser.wasm").as_slice(),
include_bytes!("../../../grammars/python/LICENSE").as_slice(),
),
"tsx" => (
include_bytes!("../../../grammars/tsx/parser.wasm").as_slice(),
include_bytes!("../../../grammars/typescript/LICENSE").as_slice(),
),
"zig" => (
include_bytes!("../../../grammars/zig/parser.wasm").as_slice(),
include_bytes!("../../../grammars/zig/LICENSE").as_slice(),
),
"objc" => (
include_bytes!("../../../grammars/objc/parser.wasm").as_slice(),
include_bytes!("../../../grammars/objc/LICENSE").as_slice(),
),
"solidity" => (
include_bytes!("../../../grammars/solidity/parser.wasm").as_slice(),
include_bytes!("../../../grammars/solidity/LICENSE").as_slice(),
),
"rust" => (
include_bytes!("../../../grammars/rust/parser.wasm").as_slice(),
include_bytes!("../../../grammars/rust/LICENSE").as_slice(),
),
_ => return Err("候选语言未知".into()),
};
verify_grammar_asset(asset, wasm, license)?;
if asset.language == "zig"
&& crate::adapt_zig_wasm(include_bytes!("../../../grammars/zig/source.wasm"))? != wasm
{
return Err("Zig grammar 来源适配关系不符".into());
}
if matches!(asset.language.as_str(), "objc" | "solidity") {
let source = if asset.language == "objc" {
include_bytes!("../../../grammars/objc/source.wasm").as_slice()
} else {
include_bytes!("../../../grammars/solidity/source.wasm").as_slice()
};
verify_grammar_asset(asset, wasm, license)?;
if asset.language == "zig"
&& crate::adapt_zig_wasm(include_bytes!("../../../grammars/zig/source.wasm"))? != wasm
if asset.source_sha256.as_deref() != Some(digest(source).as_str())
|| crate::adapt_legacy_dylink(&asset.language, source)? != wasm
{
return Err("Zig grammar 来源适配关系不符".into());
}
if matches!(asset.language.as_str(), "objc" | "solidity") {
let source = if asset.language == "objc" {
include_bytes!("../../../grammars/objc/source.wasm").as_slice()
} else {
include_bytes!("../../../grammars/solidity/source.wasm").as_slice()
};
if asset.source_sha256.as_deref() != Some(digest(source).as_str())
|| crate::adapt_legacy_dylink(&asset.language, source)? != wasm
{
return Err("依赖 grammar 来源适配关系不符".into());
}
return Err("依赖 grammar 来源适配关系不符".into());
}
}
Ok(manifest)
Ok(wasm)
}

/// 解析来源固定的候选清单;拒绝重复字段、漂移来源和伪造的已验收状态。
Expand Down
4 changes: 2 additions & 2 deletions crates/codeguard-adapters/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -67,8 +67,8 @@ mod grammar_asset_manifest;
mod legacy_dylink_compat;
mod zig_wasm_compat;
pub use grammar_asset_manifest::{
GrammarAsset, GrammarAssetManifest, bundled_grammar_candidates, parse_grammar_asset_manifest,
verify_grammar_asset,
GrammarAsset, GrammarAssetManifest, bundled_grammar_candidate, bundled_grammar_candidates,
parse_grammar_asset_manifest, verify_grammar_asset,
};
pub use legacy_dylink_compat::adapt_legacy_dylink;
pub use zig_wasm_compat::adapt_zig_wasm;
Expand Down
15 changes: 13 additions & 2 deletions crates/codeguard-adapters/tests/grammar_asset_manifest.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,18 @@
use codeguard_adapters::{
adapt_legacy_dylink, adapt_zig_wasm, bundled_grammar_candidates, parse_grammar_asset_manifest,
verify_grammar_asset,
adapt_legacy_dylink, adapt_zig_wasm, bundled_grammar_candidate, bundled_grammar_candidates,
parse_grammar_asset_manifest, verify_grammar_asset,
};

#[test]
fn selected_candidate_matches_full_manifest_without_promoting_unknown_language() {
let full = bundled_grammar_candidates().expect("all bundled assets");
for asset in &full.assets {
let (selected, wasm) = bundled_grammar_candidate(&asset.language).expect("selected asset");
assert_eq!(&selected, asset);
assert_eq!(wasm.len(), asset.bytes);
}
assert!(bundled_grammar_candidate("unknown").is_err());
}
use serde_json::Value;

#[test]
Expand Down
27 changes: 2 additions & 25 deletions crates/codeguard-cli/src/syntax_worker_command.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

use crate::syntax_worker_envelope::SyntaxWorkerEnvelope;
use crate::syntax_worker_recovery::SyntaxWorkerRecovery;
use codeguard_adapters::bundled_grammar_candidates;
use codeguard_adapters::bundled_grammar_candidate;
use codeguard_runtime::{WasmGrammar, scan_wasm_recoveries};
use sha2::{Digest, Sha256};
use std::io::Read;
Expand Down Expand Up @@ -54,33 +54,10 @@ pub fn run(args: &[String]) -> ExitCode {
}

fn observe(language: &str, source: &[u8]) -> Result<SyntaxWorkerEnvelope, String> {
let manifest = bundled_grammar_candidates()?;
let asset = manifest
.assets
.iter()
.find(|asset| asset.language == language)
.ok_or("不支持的 grammar 语种")?;
let (asset, wasm) = bundled_grammar_candidate(language)?;
if asset.codeguard_runtime_validation != "rust_loader_smoke_passed" {
return Err("grammar 与当前 Rust WASM 运行时不兼容".into());
}
let wasm = match language {
"c" => include_bytes!("../../../grammars/c/parser.wasm").as_slice(),
"cpp" => include_bytes!("../../../grammars/cpp/parser.wasm").as_slice(),
"csharp" => include_bytes!("../../../grammars/csharp/parser.wasm").as_slice(),
"go" => include_bytes!("../../../grammars/go/parser.wasm").as_slice(),
"javascript" => include_bytes!("../../../grammars/javascript/parser.wasm").as_slice(),
"lua" => include_bytes!("../../../grammars/lua/parser.wasm").as_slice(),
"luau" => include_bytes!("../../../grammars/luau/parser.wasm").as_slice(),
"java" => include_bytes!("../../../grammars/java/parser.wasm").as_slice(),
"python" => include_bytes!("../../../grammars/python/parser.wasm").as_slice(),
"typescript" => include_bytes!("../../../grammars/typescript/parser.wasm").as_slice(),
"tsx" => include_bytes!("../../../grammars/tsx/parser.wasm").as_slice(),
"zig" => include_bytes!("../../../grammars/zig/parser.wasm").as_slice(),
"objc" => include_bytes!("../../../grammars/objc/parser.wasm").as_slice(),
"solidity" => include_bytes!("../../../grammars/solidity/parser.wasm").as_slice(),
"rust" => include_bytes!("../../../grammars/rust/parser.wasm").as_slice(),
_ => return Err("不支持的 grammar 语种".into()),
};
let mut grammar = WasmGrammar::load(
asset.loader_symbol.as_deref().unwrap_or(language),
wasm,
Expand Down
2 changes: 2 additions & 0 deletions openspec/changes/introduce-rust-codeguard-cli/tasks.md
Original file line number Diff line number Diff line change
Expand Up @@ -1047,3 +1047,5 @@ TDD缺自动同步先RED。相关普通:build9、work sync15、next6,共30
- [ ] 14.17 逐语言误报与性能评测。责任:评测/adapters。规格:SP06,SP12。依赖:14.3,14.4,14.11。验收:独立合法/非法 oracle、版本/方言语料、原生对照;分别报告误报、漏报、未知覆盖及冷暖启动/包体/内存;先实测再定预算。
- [ ] 14.18 打包兼容与分阶段发布验收。责任:发布/CLI。规格:SP04,SP11,SP12。依赖:14.12-14.17。验收:每个平台验证内置资产/许可证/散列和离线 npx;保持退出码、统一入口及旧报告读取;同步中英文示例;未验收语言/宿主仍标 gap。
- [ ] 14.19 CodeGraph 32 种 grammar 全量接入与 Zig 误报回归。责任:adapters/runtime/CLI/发布。规格:SP01,SP04,SP06,SP11,SP12。依赖:14.1-14.18。验收:逐语言固定可再分发来源、许可证、原始和发行字节、真实 ABI、Rust worker 离线加载、合法/非法版本语料、原生工具对照、原生优先统一命令、任务与对话反馈、资源预算及发行包实装;覆盖 arkts/c/cfml/cfquery/cfscript/cobol/cpp/csharp/dart/erlang/go/java/javascript/kotlin/lua/luau/nix/objc/pascal/php/python/r/ruby/rust/scala/solidity/swift/terraform/tsx/typescript/vbnet/zig。当前十五份候选(C/C++/C#/Go/Java/JavaScript/Lua/Luau/Objective-C/Python/Rust/Solidity/TypeScript/TSX/Zig),已验收发行 0;COBOL 超出 8 MiB 上限,其他 17 种仍无资产清单和执行路由。C++/C#/Lua/Luau 的固定来源、许可证、真实 ABI、离线加载及基础正反例见[局部验收](../../../tests/acceptance/cpp-csharp-lua-luau-grammar-candidates.md)。C/Go/JavaScript/Rust 的固定来源、许可证、离线加载及基础正反例见[局部验收](../../../tests/acceptance/mainstream-grammar-candidates.md)。不得以 `grammar status` 库存、WASM 可加载或未验收候选替代原生 lint 或语言验收。

14.19 性能子项进展:候选 worker 已改为仅校验所选固定 grammar 及相关许可证,避免每次单文件初检遍历全部 WASM;全量库存命令保持完整校验。选择接口先 RED 后通过 15 语种身份一致性测试,单次调试构建启动观察见[局部验收](../../../tests/acceptance/syntax-worker-selected-asset.md)。这不是正式性能评测,也不改变 15/32 候选和 0 份发行验收的状态。
11 changes: 11 additions & 0 deletions tests/acceptance/syntax-worker-selected-asset.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
# 语法 worker 按需资产校验局部验收

日期:2026-09-29。范围:已固定的 15 份候选 grammar,仍非语言能力或发行验收。

原实现的每次 `__syntax-worker <language>` 都先调用全量 `bundled_grammar_candidates()`,对所有已打包 WASM 和许可证重新计算摘要,再选择一份资产。随着语言数增加,单文件反馈会承担无关语言的校验开销。

现在 worker 通过 `bundled_grammar_candidate(language)` 仅选择固定清单中的目标语言,校验 CodeGraph 许可证、该语言的 WASM 与许可证,以及该语言适用的来源适配关系;Objective-C/Solidity 额外校验依赖包许可证。未知语言拒绝。全量 `grammar status` 仍校验全部资产。测试先因缺少选择接口而编译失败,随后证明 15 份选择结果与全量清单身份及字节长度一致,未知语言拒绝。

调试构建、同一主机上的一次前后观察:Lua worker 从约 2.14 秒到 1.63 秒,C# 从约 3.25 秒到 1.62 秒。此观察包含进程启动和 Rust WASM 解析开销,不能分离摘要计算贡献,也不能当作正式冷/暖启动性能基准。

未完成:32 份资产覆盖、逐语言语料与原生对照、公开原生优先路由、宿主实装及发行验收。候选仍不能代替原生 lint;结果仍为 `incomplete`。
Loading