diff --git a/.cargo/vendor-config.toml b/.cargo/vendor-config.toml new file mode 100644 index 0000000..631aa00 --- /dev/null +++ b/.cargo/vendor-config.toml @@ -0,0 +1,24 @@ +# 폐쇄형(Air-Gapped) 빌드용 cargo source-replacement 템플릿. +# +# scripts/vendor.sh 가 이 파일을 .cargo/config.toml 로 복사하여 활성화합니다. +# 일반 (온라인) 빌드는 이 파일을 무시하므로 영향이 없습니다. +# +# 활성화 후: +# - cargo build --offline --workspace +# - cargo test --offline --workspace +# - cargo clippy --offline --workspace --all-targets -- -D warnings +# - cargo fmt --all -- --check +# +# vendor/ 디렉토리는 .gitignore 에 등록되어 있고, scripts/vendor.sh 의 +# `cargo vendor` 실행 산출물입니다 (~500MB–1GB). + +[source.crates-io] +replace-with = "vendored-sources" + +[source.vendored-sources] +directory = "vendor" + +# 결정론적 빌드를 위해 net.offline 강제. 활성화 후 외부 fetch 시도가 있으면 +# 빌드가 즉시 실패하여 폐쇄망 정책 위반을 컴파일 시점에 차단합니다. +[net] +offline = true diff --git a/.github/actions/checkout-deps/action.yml b/.github/actions/checkout-deps/action.yml new file mode 100644 index 0000000..9cdcd4d --- /dev/null +++ b/.github/actions/checkout-deps/action.yml @@ -0,0 +1,59 @@ +# 워크스페이스 `Cargo.toml`의 path dependency (`../elib-k0-nt/*`)가 가리키는 +# 시블링 디렉토리를 CI 환경에 재현합니다. `actions/checkout`는 `path` 입력에 +# 워크스페이스 외부(`..`)를 허용하지 않으므로 `git clone`으로 직접 시블링 +# 위치에 배치합니다. +# +# 핀: 의존성 그래프(특히 zeroize 엣지)가 `Cargo.lock`과 일치해야 `--locked` +# 빌드가 통과합니다. elib-k0-nt 태그 `1.0.0`은 zeroize 엣지가 없는 옛 +# 스냅샷이므로 사용 불가. 현재 lock은 master HEAD에서 생성되었기에 +# 동일 커밋 SHA에 고정합니다. lock을 갱신할 때 본 SHA도 함께 +# 갱신해야 CI가 재현 가능 상태로 유지됩니다. + +name: checkout-deps +description: Lumen 워크스페이스의 시블링 path dependency(elib-k0-nt) 체크아웃 + +inputs: + ref: + description: elib-k0-nt에서 체크아웃할 git ref(태그/브랜치/커밋 SHA) + required: false + # `Cargo.lock` 과 정합하는 master HEAD; 변경 시 lock 도 함께 재생성 + default: "618ac0dc8200ce740cbd698d0e93adb98ddc75e7" + token: + description: 비공개 elib-k0-nt 접근용 토큰; 비워두면 익명 HTTPS clone 시도 + required: false + default: "" + +runs: + using: composite + steps: + - name: clone elib-k0-nt as sibling + shell: bash + run: | + set -euo pipefail + TARGET="$(dirname "$GITHUB_WORKSPACE")/elib-k0-nt" + REF="${{ inputs.ref }}" + TOKEN="${{ inputs.token }}" + + if [ -n "$TOKEN" ]; then + URL="https://x-access-token:${TOKEN}@github.com/Quant-Off/elib-k0-nt.git" + else + URL="https://github.com/Quant-Off/elib-k0-nt.git" + fi + + # init + fetch 패턴: `git clone --branch`는 SHA를 받지 못하므로 + # 임의 ref(태그/브랜치/커밋)를 모두 처리하기 위해 빈 저장소에서 + # 시작해 단일 ref를 얕게(depth=1) 가져옵니다. + if [ ! -d "$TARGET/.git" ]; then + echo "elib-k0-nt 를 '$TARGET' 에 초기화합니다." + mkdir -p "$TARGET" + git -C "$TARGET" init -q + git -C "$TARGET" remote add origin "$URL" + else + echo "elib-k0-nt 이미 존재: origin URL 갱신 후 재페치합니다." + git -C "$TARGET" remote set-url origin "$URL" + fi + + git -C "$TARGET" fetch --depth=1 origin "$REF" + git -C "$TARGET" checkout --detach FETCH_HEAD + + echo "elib-k0-nt HEAD: $(git -C "$TARGET" rev-parse HEAD)" diff --git a/.github/docker/Dockerfile.ci b/.github/docker/Dockerfile.ci index fdb7ec3..35ebc67 100644 --- a/.github/docker/Dockerfile.ci +++ b/.github/docker/Dockerfile.ci @@ -2,20 +2,23 @@ # # Lumen CI 베이스 이미지. # -# 모든 GitHub Actions 잡(`build-test`, `features`, `lint`, `docs`, `deny`, +# 모든 GitHub Actions 잡(`build-test`, `air-gapped`, `lint`, `docs`, `deny`, # `audit`)이 이 컨테이너 안에서 실행되어 toolchain·시스템 의존성·사전 설치된 # cargo 도구를 동일하게 사용합니다. 이 파일·`rust-toolchain.toml`이 변경되면 # `.github/workflows/ci.yml` 의 `image` 잡이 자동으로 새 태그를 빌드/푸시합니다. # 베이스: Debian Bookworm 위의 Rust stable 고정. # `rust-toolchain.toml` 의 채널과 일치시켜 로컬-CI 동작을 동기화합니다. -FROM rust:1.93-bookworm +FROM rust:1.95-bookworm # 워크스페이스 빌드에 필요한 시스템 패키지. -# - protobuf-compiler : `lumen-inference` 의 candle / candle-onnx 빌드 의존성 # - cmake / build-essential / pkg-config / libssl-dev : -# 일부 native crate (wasmtime / cranelift / 향후 dep) 의 보편적 빌드 요구 +# 일부 native crate (wasmtime / cranelift / 향후 llama.cpp 등) 의 +# 보편적 빌드 요구 # - git : cargo-deny advisory DB fetch 및 일부 build script 가 사용 +# +# 참고: 과거에는 `protobuf-compiler` 도 설치했으나 candle / candle-onnx +# 제거(v0.4)로 더 이상 필요하지 않습니다. RUN apt-get update \ && apt-get install -y --no-install-recommends \ ca-certificates \ @@ -23,7 +26,6 @@ RUN apt-get update \ cmake \ pkg-config \ libssl-dev \ - protobuf-compiler \ git \ && rm -rf /var/lib/apt/lists/* diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9de1ce9..340151a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,15 @@ permissions: env: CARGO_TERM_COLOR: always - RUSTFLAGS: -D warnings + # Q. T. Felix NOTE: 과거에는 `RUSTFLAGS: -D warnings` 를 워크플로우 전역으로 + # 걸었으나, 이 환경변수는 워크스페이스 외부 path dep + # (`../elib-k0-nt/*`) 컴파일에도 그대로 적용되어 외부 + # 레포의 사소한 dead_code 경고가 lumen CI 를 깨뜨리는 + # 문제가 있었습니다. 동등한 게이트는 이제 워크스페이스 + # `[workspace.lints.rust]` 의 `warnings = "deny"` 로 + # lumen 멤버에만 한정해 적용됩니다. clippy 게이트는 + # 여전히 `cargo clippy ... -- -D warnings` 로 수행하며, + # 이 인자는 primary package (lumen 멤버) 에만 적용됩니다. jobs: # --------------------------------------------------------------------------- @@ -77,6 +85,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} steps: - uses: actions/checkout@v4 + - uses: ./.github/actions/checkout-deps - uses: Swatinem/rust-cache@v2 - name: cargo build run: cargo build --workspace --all-targets --locked @@ -98,8 +107,7 @@ jobs: runs-on: macos-latest steps: - uses: actions/checkout@v4 - - name: install protoc - run: brew install protobuf + - uses: ./.github/actions/checkout-deps - uses: Swatinem/rust-cache@v2 - name: cargo build run: cargo build --workspace --all-targets --locked @@ -112,10 +120,18 @@ jobs: run: cargo build --release --target wasm32-unknown-unknown --locked # --------------------------------------------------------------------------- - # 무거운 optional features (halo2, candle). - # --------------------------------------------------------------------------- - features: - name: optional features (halo2, candle) + # 폐쇄 환경(Air-Gapped) 호환성: `--offline` 빌드로 네트워크 fetch 없이도 + # 워크스페이스가 빌드되는지 검증합니다. 의존성은 사전에 한 번만 받아오고, + # 그 이후의 `--offline` 단계가 통과해야 진짜 폐쇄망 배포 가능 상태입니다. + # + # 참고: + # - `lumen-zkml` 의 `halo2` feature 는 v0.4 에서 제거되었습니다. + # - `lumen-inference` 의 `candle` feature 도 v0.4 에서 제거되었습니다 + # (`tokenizers` / `rand_core` 등 외부 의존 트리 정리). 도구 라우팅용 + # 토크나이저는 자체 구현 `lumen_inference::BpeTokenizer` 를 사용합니다. + # --------------------------------------------------------------------------- + air-gapped: + name: air-gapped (--offline) needs: image runs-on: ubuntu-latest container: @@ -125,13 +141,16 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} steps: - uses: actions/checkout@v4 + - uses: ./.github/actions/checkout-deps - uses: Swatinem/rust-cache@v2 - - name: cargo build --features halo2 - run: cargo build -p lumen-zkml --features halo2 --locked - - name: cargo test --features halo2 - run: cargo test -p lumen-zkml --features halo2 --locked - - name: cargo build --features candle - run: cargo build -p lumen-inference --features candle --locked + - name: cargo fetch (online, one-shot) + run: cargo fetch --locked + - name: cargo build --offline (default features) + run: cargo build --workspace --all-targets --locked --offline + - name: cargo build --offline (no default features) + run: cargo build --workspace --no-default-features --locked --offline + - name: cargo test --offline + run: cargo test --workspace --locked --offline # --------------------------------------------------------------------------- # Lint: clippy + rustfmt. @@ -147,6 +166,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} steps: - uses: actions/checkout@v4 + - uses: ./.github/actions/checkout-deps - uses: Swatinem/rust-cache@v2 - run: cargo fmt --all -- --check - run: cargo clippy --workspace --all-targets -- -D warnings @@ -167,6 +187,7 @@ jobs: RUSTDOCFLAGS: -D warnings steps: - uses: actions/checkout@v4 + - uses: ./.github/actions/checkout-deps - uses: Swatinem/rust-cache@v2 - run: cargo doc --workspace --no-deps --document-private-items @@ -184,6 +205,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} steps: - uses: actions/checkout@v4 + - uses: ./.github/actions/checkout-deps - run: cargo deny check # --------------------------------------------------------------------------- @@ -200,4 +222,5 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} steps: - uses: actions/checkout@v4 + - uses: ./.github/actions/checkout-deps - run: cargo audit diff --git a/.gitignore b/.gitignore index 56317c9..627afe4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,7 @@ target/ .idea/ *.swp -.DS_Store \ No newline at end of file +.DS_Store +/vendor/ +/.cargo/config.toml +/.cargo/config.toml.backup-* \ No newline at end of file diff --git a/.planning/codebase/ARCHITECTURE.md b/.planning/codebase/ARCHITECTURE.md new file mode 100644 index 0000000..13b88e8 --- /dev/null +++ b/.planning/codebase/ARCHITECTURE.md @@ -0,0 +1,346 @@ +--- +last_mapped_commit: 7c6fc8c +analysis_date: 2026-05-07 +focus: architecture +--- + +# Architecture + +**Analysis Date:** 2026-05-07 + +## System Overview + +Lumen is a zero-trust, verifiable AI-agent framework that isolates agent logic in WASM sandboxes while enforcing security through capability-based access control, cryptographic provenance, and optional zero-knowledge proofs of tool routing decisions. + +```text +┌────────────────────────────────────────────────────────────────────┐ +│ Agent Runtime │ +│ `crates/lumen-agent/src/runtime.rs` │ +│ - step() pipeline: defense → infer → policy → tool → prove │ +└────────────┬────────────────────────────────┬─────────────────────┘ + │ │ + ┌────────▼──────────┐ ┌──────────▼─────────────┐ + │ Defense Engine │ │ Inference Engine │ + │ `lumen-defense/` │ │ `lumen-inference/` │ + │ - Aho-Corasick │ │ - DummyEngine │ + │ - RegexSet │ │ - CandleEngine (ONNX) │ + │ - Heuristics │ │ - LlamaCppEngine │ + └───────────────────┘ │ - ChannelEngine (TEE) │ + └──────────┬────────────┘ + │ + ┌────────────────────────────────────┼─────────────────────────┐ + │ │ │ + ┌────▼──────────────┐ ┌─────────────────▼────┐ ┌───────────────▼──┐ + │ Policy Engine │ │ Tool Registry │ │ Proving System │ + │ `lumen-capability`│ │ `lumen-agent/tool.rs`│ │ `lumen-zkml/` │ + │ - Verify Cap │ │ - Tool handlers │ │ - MockProver │ + │ - Replay defense │ │ - JSON args │ │ - ezkl stub │ + │ - Audit logging │ │ │ │ - BLAKE3 commit │ + └───────────────────┘ └──────────────────────┘ └──────────────────┘ +``` + +## Component Responsibilities + +| Component | Responsibility | File | +|-----------|----------------|------| +| **AgentRuntime** | Orchestrates defense→infer→policy→tool→prove pipeline | `crates/lumen-agent/src/runtime.rs` | +| **DefenseEngine** | Real-time prompt injection / jailbreak detection | `crates/lumen-defense/src/engine.rs` | +| **InferenceEngine** | Backend-agnostic completion generation (trait) | `crates/lumen-inference/src/backend.rs` | +| **PolicyEngine** | Capability verification, replay prevention, audit logging | `crates/lumen-capability/src/policy.rs` | +| **Sandbox** | WASM isolation via wasmtime with deterministic config | `crates/lumen-sandbox/src/runner.rs` | +| **ProvingSystem** | Abstract proof generation (mock BLAKE3 commitment) | `crates/lumen-zkml/src/mock.rs` | +| **SecureChannel** | Typed async message transport (inproc / encrypted / attested) | `crates/lumen-channel/src/lib.rs` | +| **ModelManifest** | Provenance: BLAKE3 + Ed25519 signature verification | `crates/lumen-provenance/src/manifest.rs` | +| **Orchestrator** | Multi-agent supervisor with capability-gated inter-agent messaging | `crates/lumen-orchestrator/src/supervisor.rs` | + +## Pattern Overview + +**Overall:** Multi-crate workspace with **strict layering** via capability-based security and WASM sandboxing. Core pattern: + +1. **Entry point** → Agent step invoked by host (CLI, SDK) +2. **Defense filter** → Prompt analyzed for jailbreak patterns +3. **Inference** → LLM or reasoning engine produces completion + optional tool call +4. **Policy gate** → Capability checked before tool execution +5. **Tool exec** → WASM agent calls host tool via explicit capability +6. **ZK bind** → Routing decision committed to BLAKE3 (mock) or ZK proof +7. **Audit trail** → All decisions logged with deterministic field names + +**Security boundary:** Host (LLM inference, TEE trust) ↔ WASM guest (agent logic) = **capability-only**. + +## Layers + +**Core Primitives:** +- Purpose: Lightweight, zero-IO types used by all crates +- Location: `crates/lumen-core/src/` +- Contains: `AgentId`, `CapabilityId`, `Blake3Hash`, `Ed25519` wrappers, `Timestamp`, error types +- Depends on: `elib-k0-nt/*` crypto (BLAKE3, ED25519, X25519, AES, RNG) +- Used by: All other crates + +**Fixed-Point Arithmetic:** +- Purpose: Deterministic quantization (no floating-point nondeterminism for ZK) +- Location: `crates/lumen-fixed/src/` +- Contains: `Q16_16`, `Q8_24` fixed-point types, quantization utilities +- Depends on: `lumen-core` +- Used by: `lumen-inference` (weight quantization), `lumen-zkml` (deterministic arithmetic) + +**Cryptography & Capabilities:** +- Purpose: Signed capability tokens, policy enforcement, audit +- Location: `crates/lumen-capability/src/` +- Contains: `Capability` (signed token), `PolicyEngine` (verify + replay defense), `Resource` (file / net / tool / inference) +- Depends on: `lumen-core` +- Used by: `lumen-sandbox`, `lumen-orchestrator`, `lumen-agent` + +**Secure Channels:** +- Purpose: Typed message transport between sandbox and host (or inter-agent) +- Location: `crates/lumen-channel/src/` +- Contains: `SecureChannel` trait, `InProcChannel`, `AttestedChannel`, `EncryptedChannel` (feature-gated), `TeeChannel` (stub) +- Depends on: `lumen-core` +- Used by: `lumen-sandbox`, `lumen-orchestrator`, `lumen-inference::ChannelEngine` + +**Defense Engine:** +- Purpose: Sub-µs prompt-injection / jailbreak detection +- Location: `crates/lumen-defense/src/` +- Contains: `DefenseEngine` (Aho-Corasick lexicon + RegexSet + heuristics), `Verdict` +- Depends on: `lumen-core`, `aho-corasick`, `regex` +- Used by: `lumen-agent::runtime` + +**Model Provenance:** +- Purpose: BLAKE3 hash verification, signature validation, SBOM emission +- Location: `crates/lumen-provenance/src/` +- Contains: `ModelManifest`, `verify_model()`, ONNX/Safetensors/GGUF sniffing +- Depends on: `lumen-core`, `safetensors` +- Used by: `lumen-inference::loader`, CLI commands + +**Proving System:** +- Purpose: Pluggable ZK backend abstraction (mock BLAKE3, ezkl stub) +- Location: `crates/lumen-zkml/src/` +- Contains: `ProvingSystem` trait, `MockCommitmentProver` (BLAKE3), `Verification` enum +- Depends on: `lumen-core` +- Used by: `lumen-agent::runtime` + +**Inference Engine:** +- Purpose: Backend-agnostic completion generation with streaming support +- Location: `crates/lumen-inference/src/` +- Contains: `InferenceEngine` trait, `DummyEngine` (deterministic), `CandleEngine` (ONNX), `ChannelEngine` (TEE forward) +- Depends on: `lumen-core`, `lumen-provenance`, `lumen-fixed` +- Optional deps: `candle-*` (feature: `candle`) +- Used by: `lumen-agent::runtime` + +**WASM Sandbox:** +- Purpose: Capability-gated wasmtime isolation for agent code +- Location: `crates/lumen-sandbox/src/` +- Contains: `Sandbox`, `SandboxConfig::deterministic_engine()`, `HostState`, capability-gated imports (`lumen_log`, `lumen_call_tool`) +- Depends on: `lumen-core`, `lumen-capability`, `wasmtime` +- Used by: Agent runtime for untrusted WASM execution + +**Agent Runtime:** +- Purpose: Orchestrates full step pipeline (defense → infer → policy → tool → prove) +- Location: `crates/lumen-agent/src/` +- Contains: `AgentRuntime`, `AgentRuntimeBuilder`, `StepResult`, `Tool` / `ToolRegistry`, routing decision types +- Depends on: `lumen-core`, `lumen-defense`, `lumen-inference`, `lumen-capability`, `lumen-zkml` +- Used by: `lumen-orchestrator`, CLI commands, SDK agents + +**Orchestrator (Multi-agent):** +- Purpose: Supervisor for isolated multi-agent systems with capability-gated inter-agent messaging +- Location: `crates/lumen-orchestrator/src/` +- Contains: `Orchestrator`, `AgentHandle`, `AgentInbox` (bounded mpsc), `InterAgentMessage` +- Depends on: `lumen-core`, `lumen-agent`, `lumen-capability`, `lumen-channel` +- Used by: Multi-agent deployments + +**SDK & Macros:** +- Purpose: WASM-targeted SDK for agent code (`no_std` compatible, `#[lumen_agent]` proc-macro) +- Location: `crates/lumen-sdk/src/`, `crates/lumen-sdk-macros/src/` +- Contains: `log()`, `call_tool()` host imports, `LogLevel`, `ToolError` +- Depends on: None (SDK is `no_std`) +- Used by: WASM agents compiled to `wasm32-unknown-unknown` + +**Attestation (TEE):** +- Purpose: Parse and format-verify Intel TDX and AMD SEV-SNP attestation documents +- Location: `crates/lumen-attestation/src/` +- Contains: `AttestationDoc`, `TdxQuote`, `SevSnpReport`, format parsing (no crypto verification yet) +- Depends on: `lumen-core` +- Used by: TEE channel backends (future), host trust decisions + +**On-Chain Verifiers:** +- Purpose: Emit contract scaffolds (EVM Solidity + Mina o1js) for on-chain proof verification +- Location: `crates/lumen-onchain/src/` +- Contains: `emit_artifacts()`, `deploy_evm()`, routing verifier contracts +- Depends on: `lumen-core` +- Used by: CLI verifier commands + +**CLI:** +- Purpose: User-facing command-line interface (init, verify-model, sbom, defend, prove, run, verifier) +- Location: `crates/lumen-cli/src/` +- Contains: Subcommands, policy file loading, end-to-end agent execution +- Depends on: All policy/inference/zkml/provenance crates +- Used by: `lumen` binary + +## Data Flow + +### Primary Request Path (Agent Step) + +1. **Input:** Host calls `AgentRuntime::step(prompt: &str)` (`crates/lumen-agent/src/runtime.rs:99`) +2. **Defense:** `DefenseEngine::analyze(prompt)` → `Verdict` (block or pass) (`crates/lumen-defense/src/engine.rs`) +3. **Inference:** `InferenceEngine::complete(prompt, params)` → `Completion { text, tool_call }` (`crates/lumen-inference/src/backend.rs`) +4. **Policy Check:** For each `tool_call`, `PolicyEngine::check(cap, agent, Action::CallTool(tool_id), now)` verifies capability (`crates/lumen-capability/src/policy.rs:61`) +5. **Tool Execution:** If policy passes, invoke tool via `ToolHandler::call(args_json)` → JSON result (`crates/lumen-agent/src/tool.rs`) +6. **ZK Bind:** `RoutingDecision` (public inputs + witness) created (`crates/lumen-agent/src/route.rs`), passed to `ProvingSystem::prove()` → `MockProof` (`crates/lumen-zkml/src/mock.rs`) +7. **Verification:** `ProvingSystem::verify(vk, public, proof)` → `Verification::CommitmentOnly` or `Invalid` (mock never `ZkVerified`) (`crates/lumen-zkml/src/verification.rs`) +8. **Output:** `StepResult { completion, tool_output, routing, proof, verification, defense_verdict }` (`crates/lumen-agent/src/runtime.rs:22`) + +### Streaming Path + +1. **Input:** Host calls `AgentRuntime::stream_step(prompt: &str)` +2. **Defense:** Same as step path +3. **Stream Init:** `StreamingEngine::stream_complete(prompt, params)` → `TokenStream` yields `Token` events +4. **Tee Stream:** Accumulate tokens to full `Completion`, then proceed as step path +5. **Event Output:** Stream yields `StreamEvent::Token(token)` in real-time, finally `StreamEvent::Complete(StepResult)` + +### WASM Sandbox Execution + +1. **Guest:** WASM agent calls `lumen_call_tool(tool_ptr, tool_len, args_ptr, args_len)` (imported from host) +2. **Host Memcpy:** `lumen_sandbox::imports::lumen_call_tool()` copies `tool_id` and `args_json` out of guest linear memory (`crates/lumen-sandbox/src/imports.rs`) +3. **Policy Gate:** `HostState::policy.check(cap, agent, Action::CallTool(tool_id), now)` blocks if capability missing +4. **Tool Lookup:** `ToolRegistry::call(tool_id, args_json)` executes registered handler +5. **Result Buffer:** Result (or error code) placed in shared memory; guest reads via future `lumen_recv` import (v0.4) + +### Multi-Agent Inter-Communication + +1. **Send:** Agent A calls `Orchestrator::send_agent_message(recipient, msg)` +2. **Policy Check:** `PolicyEngine::check(cap, agent, Action::SendAgentMessage(recipient_id), now)` +3. **Routing:** Message queued in `AgentInbox` (bounded mpsc) of agent B +4. **Receive:** Agent B polls `AgentInbox::recv()` → typed message via `SecureChannel::recv()` + +**State Management:** +- Each agent in orchestrator owns **separate tokio task** with no shared memory access +- `Orchestrator` maintains agent registry (`HashMap`) and inbox map +- All inter-agent data passed through `SecureChannel` (postcard-encoded) +- No global mutable state (enforced by Rust ownership model) + +## Key Abstractions + +**InferenceEngine:** +- Purpose: Pluggable completion backend selection +- Examples: `DummyEngine` (deterministic test), `CandleEngine` (ONNX routing), `ChannelEngine` (TEE forward) +- Pattern: Trait object with builder factory (`backend::create_engine()`) selects concrete type at runtime +- File: `crates/lumen-inference/src/backend.rs` + +**ProvingSystem:** +- Purpose: Pluggable ZK proof backend (mock BLAKE3 or ezkl) +- Pattern: Generic trait with associated types (`Witness`, `PublicInputs`, `Proof`, `Vk`) +- Enables boxed trait objects in hot path without backend knowledge +- File: `crates/lumen-zkml/src/lib.rs:48` + +**SecureChannel:** +- Purpose: Abstract typed message transport (inproc, encrypted, attested, TEE) +- Pattern: Trait with `send_bytes()` / `recv_bytes()`, plus convenience `send()` / `recv()` +- Implementations hidden behind feature flags (e.g., `crypto-channel` for encrypted) +- File: `crates/lumen-channel/src/lib.rs:38` + +**BackendConfig:** +- Purpose: Runtime selection of inference backend + sampling parameters +- Pattern: Serializable enum with per-backend config fields +- Used by CLI to instantiate correct `InferenceEngine` type +- File: `crates/lumen-inference/src/backend.rs` + +**Capability:** +- Purpose: Unforgeable signed token binding audience, resource, nonce, expiry, and issuer signature +- Pattern: Serializable struct with Ed25519 signature; `PolicyEngine::check()` validates before action +- Design: No implicit permissions — every meaningful action requires explicit capability +- File: `crates/lumen-capability/src/capability.rs` + +**RoutingDecision:** +- Purpose: Record of tool selection with public inputs (prompt hash, policy hash, tool ID) and witness (args hash, defense corpus) +- Pattern: Opaque witness never leaves proving system in true ZK; mock prover exposes it for testing +- Enables on-chain re-verification of tool routing logic +- File: `crates/lumen-agent/src/route.rs:39` + +## Entry Points + +**CLI Binary:** +- Location: `crates/lumen-cli/src/main.rs` +- Triggers: `lumen init|verify-model|sbom|defend|prove|run|verifier` +- Responsibilities: Policy loading, model verification, defense testing, ZK proof demo, full agent step execution, contract generation + +**SDK Agent (WASM):** +- Location: `crates/lumen-sdk/src/lib.rs` +- Triggers: Agent code compiled to `wasm32-unknown-unknown`, linked against SDK +- Responsibilities: Expose `log()` and `call_tool()` wrappers that marshal arguments to host imports +- Pattern: Optional `#[lumen_agent]` proc-macro generates `_start()` entry point + +**Agent Runtime API (Library):** +- Location: `crates/lumen-agent/src/runtime.rs:99` +- Triggers: `AgentRuntime::step(prompt)` or `AgentRuntime::stream_step(prompt)` +- Responsibilities: Execute full defense→infer→policy→tool→prove pipeline, return `StepResult` + +## Architectural Constraints + +- **Threading:** Tokio async runtime; single-threaded or multi-threaded depending on `tokio` feature setup. Agent tasks isolated per tokio task, no shared memory. +- **Global state:** Orchestrator maintains shared `Arc`, `Arc`, agent registry. No module-level singletons beyond these. +- **Circular imports:** None detected. Dependency graph is DAG: `lumen-core` ← all others; `lumen-inference` ← `lumen-agent`; `lumen-zkml` ← `lumen-agent`. +- **Determinism:** WASM engine config disables SIMD, threads, relaxed ops, forces Cranelift NaN canonicalization. Fixed-point arithmetic enforced for ZK-bound code. RNG seed pinned in tests. +- **Memory isolation:** WASM linear memory separate from host; capability-gated imports prevent host data leak. +- **Namespace collision:** Feature flags (`candle`, `llama-cpp`, `crypto-channel`, `ezkl`) ensure only enabled code is compiled. + +## Anti-Patterns + +### Implicit Permissions + +**What happens:** Code assumes a tool can be called without checking `PolicyEngine` first. + +**Why it's wrong:** Enables privilege escalation in multi-agent deployments. A malicious tool could execute without capability checks, bypassing audit. + +**Do this instead:** Always wrap tool invocation in `policy.check(cap, agent, Action::CallTool(tool_id), now)` before calling `ToolHandler::call()`. See `crates/lumen-agent/src/runtime.rs:111` for the correct pattern. + +### Floating-Point in Proof-Bound Code + +**What happens:** LLM weight quantization or routing scores computed with `f32` operations. + +**Why it's wrong:** Floating-point is nondeterministic across CPU architectures (denormals, FMA contraction, x87 extended precision). ZK proof fails reproducibility. + +**Do this instead:** Use `lumen-fixed::Q16_16` or `Q8_24` for all computation in proof paths. Calibration (f32↔Qn_m conversion) permitted only offline, behind the `calibration` feature flag disabled in ZK-bound binaries. See `crates/lumen-fixed/src/lib.rs`. + +### Direct WASM Host Function Calls + +**What happens:** Agent WASM code calls `lumen_log` or `lumen_call_tool` directly instead of via SDK wrapper. + +**Why it's wrong:** Raw FFI pointers bypass SDK safety checks (linear memory bounds validation, panic handling). + +**Do this instead:** Use `lumen_sdk::log(level, msg)` and `lumen_sdk::call_tool(tool_id, args_json)`. These handle pointer/length marshalling safely. See `crates/lumen-sdk/src/lib.rs:81`. + +### Unbounded Nonce Cache + +**What happens:** `PolicyEngine::seen_nonces` allowed to grow without limit, consuming unbounded memory under reuse attacks. + +**Why it's wrong:** Attacker repeatedly issues capabilities with unique nonces, exhausting host memory. + +**Do this instead:** Cache enforces `MAX_NONCES = 65_536` limit with LRU eviction. See `crates/lumen-capability/src/policy.rs:35`. + +## Error Handling + +**Strategy:** Result-based with `lumen_core::Error` enum. No panic-on-error; all failable operations return `Result`. + +**Patterns:** +- **Defense block:** `Error::Defense(reason)` stops execution immediately, returns blocked verdict to caller +- **Policy denial:** `Error::Capability(reason)` logged to audit trail, blocks action +- **Inference failure:** `Error::Inference(reason)` returned; step fails (no tool call attempted) +- **Proof failure:** `Error::Proving(reason)` returned; step result includes `Verification::Invalid` +- **Provenance mismatch:** `Error::Provenance(reason)` halts model load; invalid model cannot reach inference engine + +## Cross-Cutting Concerns + +**Logging:** Via `tracing` crate with `tracing-subscriber` in CLI. All audit events use deterministic field names for external SIEM ingestion. See `crates/lumen-capability/src/audit.rs`. + +**Validation:** +- **Capabilities:** `PolicyEngine::check()` validates signature, replay, audience, expiry, resource matching +- **Models:** `ModelManifest::verify_model()` checks BLAKE3 hash and Ed25519 signature before load +- **Attestation:** `AttestationDoc::parse()` validates wire format (v0.4 will add crypto verification) + +**Authentication:** +- **Agent identity:** `AgentId` (16-byte random) identifies sender; no human username involved +- **Capability issuer:** `VerifyingKey` (Ed25519 public) identifies issuer; must be in `PolicyEngine::trusted` list +- **Policy enforcement:** Per-action capability requirement; no role-based access control (purely capability-based) + +--- + +*Architecture analysis: 2026-05-07* diff --git a/.planning/codebase/CONCERNS.md b/.planning/codebase/CONCERNS.md new file mode 100644 index 0000000..cb16811 --- /dev/null +++ b/.planning/codebase/CONCERNS.md @@ -0,0 +1,430 @@ +--- +analysis_date: 2026-05-07 +focus: concerns +last_mapped_commit: dfba311 +--- + +# Codebase Concerns + +**Analysis Date:** 2026-05-07 + +## Air-Gap Migration Debt (v0.4→v0.5) + +### Stale Documentation (PRACTICE_LLM.md / PRACTICE_LLM_EN.md) + +**Issue:** Documentation still references removed feature `candle-llm` and deleted type `CandleLlmEngine`. + +**Files affected:** +- `PRACTICE_LLM.md` (~660 lines) — lines 31–41, 141, 171, 277–281, 341, 356, 419, 503–507, 564, 590, 614 mention `candle-llm` feature, `CandleLlmEngine::from_gguf()`, and GGUF-based text generation workflow +- `PRACTICE_LLM_EN.md` (~660 lines) — English translation with identical references + +**Impact:** Users following these guides will encounter broken imports and missing types. Builds with `features = ["candle-llm"]` will fail. + +**Root cause:** kernel-comp.md (lines 99–106) documents removal of `candle-transformers` + `tokenizers` for air-gap compliance, but docs were not updated. Feature gate was removed but documentation remains. + +**Fix approach:** +1. Remove or relocate PRACTICE_LLM*.md to a `docs/deprecated/` directory with a deprecation notice. +2. Create new guide `PRACTICE_ONNX.md` for `CandleEngine` (ONNX routing only). +3. Create future `PRACTICE_LLAMA_CPP.md` once `llama-cpp-2` backend is implemented (v0.5). + +**Priority:** High — blocks users from following written examples. + +--- + +### Stub Inference Backends (Not Yet Implemented) + +#### LlamaCppEngine — v0.5 milestone + +**Issue:** `lumen-inference/src/llama_cpp.rs` is a stub returning `Error::NotImplemented`. + +**Files:** +- `crates/lumen-inference/src/llama_cpp.rs:54–56` — `InferenceEngine::complete()` returns error +- `crates/lumen-inference/src/llama_cpp.rs:67–69` — `StreamingEngine::stream_complete()` returns error + +**Symptoms:** +- Code compiles with `--features llama-cpp` but runtime fails with "llama-cpp-2 연동이 v0.5 에서 구현됩니다" +- No actual llama.cpp bindings present + +**Workaround:** Disable `llama-cpp` feature; use `CandleEngine` (ONNX routing) until v0.5. + +**Fix approach:** Implement `llama-cpp-2` crate bindings per kernel-comp.md lines 272–276. + +**Priority:** Medium — v0.5 milestone task, documented. + +--- + +#### TeeChannel (Host-TEE Secure Channel) — v0.3 stub + +**Issue:** `lumen-channel/src/tee.rs` implements `SecureChannel` trait but returns `Error::NotImplemented` for all operations. + +**Files:** +- `crates/lumen-channel/src/tee.rs:36` — `send_bytes()` stub +- `crates/lumen-channel/src/tee.rs:40` — `recv_bytes()` stub + +**Symptoms:** +- Any agent step using GPU inference via TEE channel will fail +- Current inference pipeline cannot forward GPU inference to hardware-accelerated TEE + +**Impact:** No E2E Host-WASM-TEE integration. LLM inference runs only in WASM sandbox (CPU only) or via `ChannelEngine` forward (not yet wired). + +**Workaround:** Use `CandleEngine` (CPU-bound ONNX routing) for tool selection; do not attempt TEE inference yet. + +**Fix approach:** +1. Implement attested TLS-like handshake over postcard frames (kernel-comp.md lines 5–6 envision attested handshake, but not yet implemented). +2. Wire `lumen-attestation` TDX quote validation into handshake (see `/crates/lumen-attestation/src/tdx.rs:19–25` — PCK chain validation deferred to v0.4). +3. Define `SecureChannel` framing protocol. + +**Priority:** Medium-High — blocks TEE + WASM split-execution model, a core security goal. + +--- + +#### EzklProver (ZK Backend) — v0.3 stub + +**Issue:** `lumen-zkml/src/ezkl.rs` is a feature-gated stub without real ezkl implementation. + +**Files:** +- `crates/lumen-zkml/src/ezkl.rs:31` — `setup()` returns `Error::NotImplemented("ezkl::setup")` +- `crates/lumen-zkml/src/ezkl.rs:39` — `prove()` stub +- `crates/lumen-zkml/src/ezkl.rs:47` — `verify()` stub + +**Symptoms:** +- `--features ezkl` compiles but calls fail at runtime +- No actual zero-knowledge proofs; only mock (BLAKE3 commitment) verification available + +**Impact:** ZK proof path is not wired. Agent routing decisions are *not* zero-knowledge; mock prover only commits to witness (CommitmentOnly verification). Per Cargo.toml line 100–102, halo2 was removed and no succinct ZK backend replaces it. + +**Current state:** Mock prover uses BLAKE3-based commitment (cheap, deterministic) instead of actual ZK. Per `lumen-agent/src/runtime.rs:31–32`, verification is always `CommitmentOnly` or `Invalid`, never `ZkVerified`. + +**Fix approach:** kernel-comp.md (line 246) defers to SP1 / RISC Zero selection at v0.4 milestone review. No action needed until then. + +**Priority:** Low — deferred by design; mock prover sufficient for audit trail. + +--- + +## External System Dependencies (Require Vendor Bundling) + +### Candle Build Dependency: Protocol Buffers (protoc) + +**Issue:** `candle-core` feature requires `protoc` (Protocol Buffers compiler) at build time. + +**Files:** `Cargo.toml:96` — `candle-core = { version = "0.10", default-features = false }` + +**Symptoms:** +- Building with `--features candle` fails if `protoc` is not in `$PATH` +- Air-gap environment: no network to download protoc; must be pre-bundled + +**Workaround:** Pre-install `protoc` (via `apt`, `brew`, or direct download) before building in air-gap. + +**Fix approach:** +- kernel-comp.md (line 277) recommends source vendoring. Add vendoring instructions to build docs. +- Consider alternative: `llama-cpp-2` backend does not require protoc (only cmake + llama.cpp C++ build). + +**Priority:** Medium — affects reproducible air-gap builds; documented in kernel-comp.md but not yet automated. + +--- + +### Llama.cpp Build Dependency: cmake + llama.cpp library + +**Issue:** Future `llama-cpp-2` feature will require cmake and precompiled llama.cpp library. + +**Files:** `crates/lumen-inference/src/llama_cpp.rs` (stub; no deps yet) + +**Current state:** Not yet a blocker (stub stage), but will be in v0.5. + +**Workaround:** Pre-build llama.cpp library for target OS/architecture. + +**Priority:** Low — future concern (v0.5 milestone). + +--- + +## In-House Cryptography Dependencies (Non-Portable) + +### Workspace Path Dependencies: elib-k0-nt Modules + +**Issue:** Workspace imports cryptographic functions from sibling `../elib-k0-nt/*` path dependencies. + +**Files:** +- `Cargo.toml:55–61` — workspace dependencies declared: + - `elib-blake` (BLAKE3 KDF + hashing) + - `elib-ed25519` (Ed25519 signing/verification) + - `elib-x25519` (X25519 ECDH) + - `elib-aes` (AES-256-GCM) + - `elib-rng` (Hash-DRBG-SHA256) + - `elib-constant-time` (constant-time comparison) + - `elib-zeroize` (memory zeroization) + +**Impact:** +- **Downstream consumers cannot use Lumen as a library** — crates importing Lumen must also have `../elib-k0-nt/` in their workspace. +- Not published on crates.io (path-only). +- Violates Rust crate ecosystem conventions. + +**Root cause:** Intentional per kernel-comp.md (air-gap strategy). elib-k0-nt replaces standard crates (blake3, ed25519-dalek, x25519-dalek, aes-gcm, rand) for FIPS 140-3 compliance path. + +**Fix approach (v1.0 milestone):** +1. Publish elib-k0-nt modules to crates.io with stable semver. +2. Switch path dependencies to `crates.io` versions with version bounds. +3. Implement FIPS 140-3 audit trail (currently deferred per CLAUDE.md line 28). + +**Priority:** Low — accepted trade-off for air-gap compliance; v1.0 concern. + +--- + +## Determinism Risk: Floating-Point in Candle Path + +### F32 Inference Non-Determinism + +**Issue:** `CandleEngine` uses f32 operations which may differ across hardware/compiler. + +**Files:** +- `crates/lumen-inference/src/candle.rs:6` — documents f32 concern +- `crates/lumen-inference/src/candle.rs:12` — "candle의 f32 연산은 미세하게 비결정적일 수 있으므로" +- `crates/lumen-inference/src/candle.rs:66–77` — `encode_prompt()` converts BLAKE3 hash to f32 vector +- `crates/lumen-inference/src/candle.rs:99–104` — inference output to f32, then argmax + +**Impact:** +- ZK proofs are NOT bound to Candle's floating-point outputs (by design). +- Per runtime.rs:16, proofs bind only to "integer routing index" (argmax result), not float values. +- This breaks reproducibility if Candle's f32 argmax differs between runs. + +**Current mitigation:** +- `lumen-fixed/src/q16_16.rs` (233 lines) implements fixed-point arithmetic. +- Per candle.rs:14–17, ZK witnesses use integer decision only. + +**Residual risk:** If two runs produce different argmax due to f32 rounding, the tool routing changes without proof notification. No automated test yet checks cross-platform f32 determinism. + +**Fix approach:** +- Add CI test comparing Candle f32 outputs across x86-64 and ARM64. +- Migrate Candle inference to fixed-point engine (`lumen-fixed`) for tooling (v0.5 milestone). + +**Priority:** Medium — affects audit trail integrity; partially mitigated by integer binding. + +--- + +## Missing TEE Attestation Chain Integration + +### Incomplete Policy Enforcement + TEE Attestation Wiring + +**Issue:** `lumen-agent/src/runtime.rs` does not enforce TEE attestation policy in agent step. + +**Files:** +- `crates/lumen-agent/src/runtime.rs:98–112` — `step()` orchestrates defense → inference → policy → tool → prove, but no attestation checkpoint +- `crates/lumen-attestation/src/tdx.rs:19–25` — TDX Quote v4 parser exists, but X.509 chain + signature verification deferred to v0.4 +- `crates/lumen-channel/src/tee.rs:1–7` — TeeChannel stub; no actual attested handshake + +**Symptoms:** +- TEE hardware attestation is parsed but not enforced. +- No check that GPU inference host is inside a TEE with valid quote. +- Agent step proceeds even if TEE attestation fails. + +**Impact:** E2E zero-trust guarantee is incomplete. WASM sandbox is isolated, but GPU inference (TEE) can silently fail attestation and agent does not know. + +**Fix approach:** +1. Implement X.509 PCK chain verification in TDX parser (kernel-comp.md lines 25–26 flag as v0.4 work). +2. Wire quote validation into `TeeChannel::new()` handshake. +3. Fail agent step if TEE attestation invalid. + +**Priority:** High — core security property (zero-trust) is not fully enforced. + +--- + +## Secret Scanning & SBOM Automation + +### No Automated Secret Detection + +**Issue:** No pre-commit secret scanner (e.g., git-secrets, truffleHog) in CI/CD. + +**Risk:** Credentials (AWS keys, HF tokens, PCS API keys) may be committed to repo. + +**Current state:** Manual review only. No automated detection. + +**Fix approach:** +- Add `git-secrets` or `truffleHog` to CI pipeline. +- Run `cargo deny` for crate advisories (already done per commit eed4194). + +**Priority:** Medium — security hygiene. + +--- + +### SBOM Generation Not Automated + +**Issue:** CLAUDE.md line 20 lists "SBOM를 강제 생성" but no automation present. + +**Files:** Cargo.toml (workspace), no `cyclonedx` or `cargo-sbom` integration + +**Current state:** Manual `cargo tree` only. + +**Fix approach:** +- Add `cargo cyclonedx` or `cargo-sbom` to CI. +- Generate CycloneDX / SPDX SBOM on every release. + +**Priority:** Medium — compliance (v1.0 milestone goal). + +--- + +## Performance Concerns + +### WASM Sandbox Cold-Start Not Benchmarked + +**Issue:** No benchmark for wasmtime instance creation + compilation time. + +**Files:** No benchmark files found (`crates/*/benches/` empty) + +**Symptoms:** +- Unknown latency for agent step startup. +- Unclear if fuel-based epoch enforcement adds overhead. +- Can't track regression. + +**Current state:** DummyEngine used in tests (instant completion); no real inference + WASM overhead measured. + +**Fix approach:** +- Add criterion benchmark: `Agent step (end-to-end, ONNX routing via Candle)`. +- Measure wasmtime instantiation overhead separately. +- Set performance gates in CI. + +**Priority:** Medium — needed for production SLA planning. + +--- + +### Fixed-Point Pipeline Not Enforced + +**Issue:** `lumen-fixed/src/q16_16.rs` exists but not integrated into Candle inference path. + +**Files:** `crates/lumen-fixed/src/q16_16.rs` (233 lines) — standalone fixed-point type, no usage in lumen-inference or lumen-agent + +**Impact:** Floating-point non-determinism risk (above) persists because fixed-point is not applied. + +**Current state:** Placeholder for future quantization path. + +**Fix approach:** Integrate fixed-point routing into CandleEngine (v0.5 work). + +**Priority:** Medium — depends on llama-cpp and determinism testing. + +--- + +## Workspace Feature Fragmentation + +### Candle Feature Scattered Across Codebase + +**Issue:** `candle` feature activates candle-core + candle-onnx, but feature state unclear in dependent crates. + +**Files:** +- `Cargo.toml:96–97` — workspace-level optional deps +- `crates/lumen-inference/Cargo.toml:21` — `candle` feature gate + +**Risk:** Accidental feature activation in transitive dependencies. No enforcement that downstream crates properly gate Candle code. + +**Fix approach:** +- Document feature hygiene in CONTRIBUTING.md. +- Add CI check: `cargo build --no-default-features --features candle` to verify isolated build. + +**Priority:** Low — feature flags are documented; low risk. + +--- + +## Code Hygiene Issues + +### Dead Code Markers Without Cleanup + +**Issue:** `#[allow(dead_code)]` on struct fields in stub implementations. + +**Files:** +- `crates/lumen-inference/src/llama_cpp.rs:28–34` — three `#[allow(dead_code)]` on LlamaCppEngine fields +- `crates/lumen-capability/src/policy.rs:134` — `#[allow(dead_code)]` on policy field + +**Symptoms:** Suppresses legitimate warnings during stub phase, but masks real unused code when stubs are removed. + +**Fix approach:** +- Remove `#[allow]` when stubs are implemented. +- Use `#[cfg(feature = "...")]` instead of `#[allow]` for conditional code. + +**Priority:** Low — cleanup task; low risk. + +--- + +### Defense Pattern Library Needs Tuning + +**Issue:** `lumen-defense/src/lexicon.rs:11` flags TODO. + +**Files:** `crates/lumen-defense/src/lexicon.rs:11–46` + +**Text:** +``` +TODO: 구글링 해보니 이 부분에 대해 토큰 사용량과 적절한 패턴의 밸런스를 +중시해야 한다고 하네요. 검토 필요 있음. +``` + +**Issue:** Jailbreak lexicon is conservative (46 patterns) but may need expansion or refinement based on token budget vs false-positive rate. + +**Current patterns:** "ignore previous instructions", "dan mode", "developer mode", etc. + +**Impact:** May miss novel jailbreak attempts; false positives block legitimate prompts. + +**Fix approach:** +- Add tuning parameter (toxicity threshold) to `LexiconEngine`. +- Expand pattern set post-deployment based on telemetry. +- Implement allowlist for safe keywords (e.g., "developer" in code context). + +**Priority:** Low — current lexicon is conservative (safe); tuning is a v0.5+ task. + +--- + +## Dependency Monitoring + +### Wasmtime Security Advisory Backlog + +**Issue:** `Cargo.toml:75` pins wasmtime v44, which has had multiple security fixes. + +**Current version:** v44 (comment: "최신 line. RUSTSEC 다수 advisory (fd_renumber / wasi:http fields / ...)") + +**Risk:** +- New advisories may emerge after v44 release. +- No automated tracking (dependabot not visible in config). + +**Fix approach:** +- Enable GitHub Dependabot for Cargo.lock updates. +- Monitor RUSTSEC for wasmtime advisories. +- Establish upgrade cadence (quarterly wasmtime bump). + +**Priority:** Medium — security-critical dependency. + +--- + +## Documentation Coverage + +### Architectural Decision Records (ADRs) Missing + +**Issue:** No ADR for kernel-comp.md migration decisions. + +**Files:** `kernel-comp.md` (15K lines) documents air-gap migration but is not indexed in README or ARCHITECTURE docs. + +**Impact:** Future maintainers may not understand why elib-k0-nt is used, why halo2 was removed, etc. + +**Fix approach:** +- Extract kernel-comp.md decisions into `.adr/` directory as individual ADRs. +- Link from README.md and CLAUDE.md. + +**Priority:** Low — knowledge preservation; no code impact. + +--- + +## Summary by Risk Level + +| Issue | Severity | Category | Status | +|-------|----------|----------|--------| +| PRACTICE_LLM*.md stale docs | High | Air-gap debt | Immediate | +| Missing TEE attestation enforcement | High | Security | High priority | +| LlamaCppEngine stub (v0.5) | Medium | Feature milestone | Expected | +| TeeChannel stub (v0.3) | Medium-High | Integration | Blocks TEE model | +| Candle f32 non-determinism | Medium | Audit integrity | Mitigated by design | +| No secret scanning | Medium | CI/CD hygiene | Preventive | +| SBOM automation missing | Medium | Compliance (v1.0) | Deferred | +| Wasmtime advisory backlog | Medium | Dependency | Monitoring | +| WASM cold-start benchmark | Medium | Perf | Nice-to-have | +| Candle feature fragmentation | Low | Code hygiene | Low risk | +| Dead code markers | Low | Cleanup | Low risk | +| Defense lexicon tuning | Low | Content | v0.5+ | +| ADR documentation | Low | Preservation | Nice-to-have | + +--- + +*Concerns audit: 2026-05-07* diff --git a/.planning/codebase/CONVENTIONS.md b/.planning/codebase/CONVENTIONS.md new file mode 100644 index 0000000..eb14162 --- /dev/null +++ b/.planning/codebase/CONVENTIONS.md @@ -0,0 +1,392 @@ +--- +title: Lumen Coding Conventions +last_mapped_commit: dfba311a939388c5892574e844e73a7953796cdd +last_mapped_date: 2026-05-07 +--- + +# Coding Conventions + +**Analysis Date:** 2026-05-07 + +## Language & Edition + +**Rust 2021 Edition** +- Minimum Supported Rust Version (MSRV): `1.85` (enforced in `Cargo.toml`) +- All crates compile with Rust 2021 edition semantics + +## Safety & Lint Policy + +### Workspace-wide Lints (Cargo.toml) + +**Deny List:** +- `unsafe_code = "deny"` — Entire workspace forbids `unsafe` blocks + - Every crate enforces `#![forbid(unsafe_code)]` in `src/lib.rs` or `src/main.rs` + - Exception handling: Only elib cryptographic libraries (`elib-k0-nt`) may contain unsafe code; Lumen crates never do + +**Warning List:** +- `missing_docs = "warn"` — All public items require documentation + - Applies to all public types, functions, modules, traits + - Korean docstrings (see below) +- `unreachable_pub = "warn"` — Flag unused public visibility +- `rust_2018_idioms = "warn"` (priority -1) +- `clippy::all = "warn"` (priority -1) — Core Clippy checks required +- `clippy::undocumented_unsafe_blocks = "warn"` — Even if `unsafe` exists elsewhere, block-level docs required + +**CI Gate:** +- All Clippy warnings are treated as errors: `cargo clippy --workspace --all-targets -- -D warnings` +- `RUSTFLAGS: -D warnings` set globally in `.github/workflows/ci.yml` + +## Documentation + +**Language:** Korean (한국어) +- All docstrings and module docs written in Korean +- Example: `crates/lumen-core/src/lib.rs` — "Lumen 의 코어 프리미티브..." +- All comments explaining logic use Korean + +**Doc Comment Format:** +```rust +/// 이 함수가 무엇을 하는지 한 줄 요약. +/// +/// 더 상세한 설명이 필요하면 추가 단락으로 작성합니다. +/// +/// ## 에러 +/// 어떤 상황에서 실패하는지 설명. +/// +/// ## 예시 +/// ```rust +/// let result = my_function()?; +/// ``` +pub fn my_function() -> Result { } +``` + +**Module Header:** +```rust +//! 모듈 전체 목적과 구조. +//! +//! 주요 타입이나 패턴 설명. +``` + +## Naming Conventions + +### Files & Directories + +**Crate Names:** +- `lumen-*` prefix (kebab-case) +- Examples: `lumen-core`, `lumen-agent`, `lumen-sandbox`, `lumen-inference` +- Directory structure: `crates/lumen-*/src/` + +**Module Files:** +- Snake_case: `capability.rs`, `error.rs`, `host.rs`, `runner.rs` +- Re-exports in `lib.rs`: `pub mod capability;` + +### Types + +**PascalCase** for all public types: +```rust +pub struct CapabilityBody { } +pub enum BackendConfig { } +pub trait InferenceEngine { } +pub struct AgentRuntime { } +``` + +### Functions & Methods + +**snake_case** for all function and method names: +```rust +pub fn verify_signature(&self) -> Result<()> { } +pub async fn complete(&self, prompt: &str) -> Result { } +pub fn build() -> AgentRuntime { } +``` + +### Variables & Constants + +**snake_case** for local variables and function parameters: +```rust +let mut agent_id = AgentId::random(&mut rng); +let policy_engine = Arc::new(PolicyEngine::new(vec![])); +``` + +**SCREAMING_SNAKE_CASE** for constants: +```rust +const DEFAULT_TIMEOUT_MS: u64 = 5000; +``` + +### Error Variants + +**PascalCase** - part of enum variant naming: +```rust +#[error("crypto: {0}")] +Crypto(String), + +#[error("policy: {0}")] +Policy(String), + +#[error("not implemented: {0}")] +NotImplemented(&'static str), +``` + +## Error Handling + +### Error Type Strategy + +**Unified Error Enum:** +- Defined in `crates/lumen-core/src/error.rs` +- Single `Error` enum with module-scoped variants +- `#[derive(Debug, Error)]` from `thiserror` crate +- Marked `#[non_exhaustive]` to allow future variants + +**Common Variants:** +```rust +pub enum Error { + #[error("io: {0}")] + Io(#[from] std::io::Error), // Auto-convert from std::io::Error + + #[error("crypto: {0}")] + Crypto(String), + + #[error("policy: {0}")] + Policy(String), + + #[error("not implemented: {0}")] + NotImplemented(&'static str), + + // ... subsystem-specific variants +} + +pub type Result = std::result::Result; +``` + +### Mapping Subsystem Errors + +**Pattern:** +1. Subsystems define native errors (e.g., `CryptoError`) +2. Map into `lumen_core::Error` at public API boundaries +3. Use `.map_err()` or `?` operator to lift errors +4. Prefer reusing existing variants over creating new ones + +**Example (from `capability.rs`):** +```rust +pub fn sign(body: CapabilityBody, key: &SigningKey) -> Result { + let bytes = postcard::to_allocvec(&body) + .map_err(|e| Error::Decode(format!("capability body encode: {e}")))?; + // ... rest of logic +} +``` + +### Result Usage + +- Always return `Result` from fallible functions +- Use `?` operator for error propagation +- `NotImplemented(&'static str)` for unfinished features +- Avoid generic error strings; use contextual `Error::*` variants + +**Example:** +```rust +pub fn verify_model(path: &Path, manifest: &ModelManifest) -> Result { + let computed = Blake3Hash::of_file(path)?; + if computed != manifest.hash { + return Err(Error::Provenance("hash mismatch".into())); + } + Ok(VerificationInfo { /* ... */ }) +} +``` + +## Async & Concurrency + +### Async Runtime + +**Tokio with Selective Features:** +- `tokio` v1 with features: `["rt-multi-thread", "macros", "sync", "time", "io-util", "fs"]` +- Multi-threaded async executor for runtime +- Default is async-first design + +### Trait Bounds for Async + +**async-trait Macro:** +```rust +use async_trait::async_trait; + +#[async_trait] +pub trait InferenceEngine: Send + Sync { + /// Completion execution. + async fn complete(&self, prompt: &str, params: &SamplingParams) -> Result; +} +``` + +**Pattern:** +- All async traits use `#[async_trait]` from `async-trait` crate +- Trait methods return `Result` not `impl Future` +- Concrete implementers use regular `async fn` inside trait impl + +### Testing Async Code + +- Use `#[tokio::test]` for async tests (see TESTING.md) +- Never use blocking operations in async contexts +- Use `Arc` for shared state in multi-agent scenarios + +### Synchronization Primitives + +**Preferred:** +- `Arc` for thread-safe shared ownership +- `parking_lot::Mutex` for shared mutable state (faster than `std::sync::Mutex`) +- `tokio::sync::*` for async-aware primitives + +**Example (from `tests/determinism.rs`):** +```rust +let policy = Arc::new(PolicyEngine::new(vec![vk])); +let runtime = AgentRuntime::builder(agent, policy).build()?; +``` + +## Code Organization + +### Module Structure + +**lib.rs Pattern:** +```rust +//! Module-level docstring in Korean. + +#![forbid(unsafe_code)] +#![warn(missing_docs)] + +pub mod submodule1; +pub mod submodule2; + +pub use submodule1::{Type1, Type2}; +pub use submodule2::Type3; +``` + +**Barrel Re-exports:** +- Use re-exports in `lib.rs` for public API +- Keep internal modules private when possible +- Example: `pub use crypto::{Signature, SigningKey, VerifyingKey};` + +### Function Size Guidelines + +- Prefer functions under 50 lines +- Extracting builder patterns and setup into separate helpers +- Complex logic split into named helper functions + +**Example (Agent Runtime):** +```rust +pub struct AgentRuntime { /* ... */ } + +impl AgentRuntime { + pub fn builder(agent: AgentId, policy: Arc) -> RuntimeBuilder { } + + pub async fn step(&self, prompt: &str) -> Result { + // Step through pipeline: Defense → Inference → Policy → Exec → ZK + } +} +``` + +## Serialization + +### Preferred Format + +**postcard:** +- Binary serialization for capability tokens and wire formats +- `postcard::to_allocvec()` for encoding +- `postcard::from_bytes()` for decoding +- Map errors: `.map_err(|e| Error::Decode(format!(...)))?` + +**serde with Derive:** +```rust +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize, Debug)] +pub struct MyType { + pub field: String, +} +``` + +**JSON for config/debug:** +- `serde_json` for human-readable JSON +- Example: tool schema definitions use JSON + +## Comments & Code Style + +### When to Comment + +- **Do:** Explain non-obvious design decisions +- **Do:** Document security-critical assumptions (capability gating, ZK bindings) +- **Do:** Note workarounds and version-specific issues +- **Do Not:** Repeat function signatures or obvious logic +- **Do Not:** Comment out dead code; delete it + +### Comment Style + +```rust +// Single-line comment for brief explanations + +/* Multi-line comment rare; prefer stacked // lines instead */ + +// TODO / FIXME with context: +// TODO: Implement support for token refresh once EVM mainnet stable +``` + +### Security Comments + +```rust +// SECURITY: This capability check is mandatory before any tool execution. +// Bypassing it would allow sandbox escape. +``` + +## Import Organization + +### Import Order + +```rust +// 1. Workspace crates (lumen-*) +use lumen_core::{Result, AgentId}; +use lumen_capability::PolicyEngine; + +// 2. External crates (alphabetical) +use async_trait::async_trait; +use serde::{Deserialize, Serialize}; +use std::sync::Arc; +use tokio::sync::Mutex; + +// 3. Re-exports from current crate (pub use) +pub use some_module::{PublicType}; +``` + +### Path Aliases + +No custom path aliases are configured. Use fully qualified paths: +- `use crate::submodule::Type;` for intra-crate +- `use lumen_core::Error;` for cross-workspace + +## Formatting & Linting + +### Formatting Tool + +- **rustfmt** (implicit; no `.rustfmt.toml` — uses workspace defaults) +- Enforced in CI: `cargo fmt --all -- --check` +- Auto-format before commit: `cargo fmt --all` + +### Lint Enforcement + +- **clippy** with `-D warnings` in CI pipeline +- Run locally: `cargo clippy --workspace --all-targets -- -D warnings` +- Address all warnings before push + +### Unsafe Code + +- **Forbidden** in Lumen crates +- Only elib cryptographic dependencies (`elib-blake`, `elib-ed25519`, etc.) contain unsafe +- If unsafe is ever needed, must justify in comments and pass security review + +## Build & Test Compliance + +All code must pass the **4-gate CI pipeline** (from CLAUDE.md): + +```bash +cargo build --workspace --all-targets # No compile errors +cargo test --workspace # All tests pass +cargo clippy --workspace --all-targets -- -D warnings # No Clippy warnings +cargo fmt --all -- --check # Consistent formatting +``` + +--- + +*Convention analysis: 2026-05-07* diff --git a/.planning/codebase/INTEGRATIONS.md b/.planning/codebase/INTEGRATIONS.md new file mode 100644 index 0000000..7d1262e --- /dev/null +++ b/.planning/codebase/INTEGRATIONS.md @@ -0,0 +1,279 @@ +--- +last_mapped: dfba311a939388c5892574e844e73a7953796cdd +last_mapped_date: 2026-05-07 +--- + +# External Integrations + +**Analysis Date:** 2026-05-07 + +## WASM Sandbox & Runtime + +**wasmtime (v44.x):** +- What it's used for: Agent code isolation and deterministic execution + - Location: `lumen-sandbox` crate (enforced in `crates/lumen-sandbox/Cargo.toml`) + - Imports: `lumen-sandbox/src/runner.rs` - Module::new(), Store, Linker, async execution control +- Configuration: + - JIT: Cranelift (runtime code generation for performance) + - Fuel-based execution limits (deterministic step counting) + - Epoch deadline enforcement (time-based timeout) + - Address2line + demangle for error diagnostics +- Security Model: + - Capabilities-based host import gating (`lumen-capability`) + - Memory isolation via linear memory with explicit bounds + - Export-only functions (no host function imports by default) + - Host state passed via Store (no global mutable state) +- Production Readiness: Mature; used for production WASM execution across ecosystem + +## Model Inference & Routing + +**ONNX Routing (candle-onnx, optional feature: `candle`):** +- Service: ONNX model inference for tool routing decisions + - Crates: `lumen-inference` (feature-gated in `crates/lumen-inference/Cargo.toml`) + - Implementation: `crates/lumen-inference/src/candle.rs`, `crates/lumen-inference/src/backend.rs` + - Model formats: ONNX (.onnx files) + - Use case: Agent tool selection routing via neural network (argmax → tool ID mapping) +- SDK: candle-core (CPU tensor ops), candle-onnx (ONNX model loading) +- Deployment: Air-gapped environments require source vendoring via `cargo vendor` (see `AIR-GAPPED.md`) +- Model Loading: + - Verified by `lumen-provenance` (BLAKE3 + Ed25519 signature check before loading) + - Output: `VerifiedModelHandle` passed to inference engine + - Safetensors header inspection supported + +**LLM Inference (llama.cpp backend, optional feature: `llama-cpp`):** +- Service: Full LLM text generation via llama.cpp + - Status: Stub interface in v0.4; full implementation targeting v0.5 + - Location: `crates/lumen-inference/src/llama_cpp.rs` + - Integration: C FFI bindings (not yet materialized) +- Use case: Agent prompt reasoning when ONNX routing is insufficient +- Model support: GGUF format (built-in tokenizer, no separate tokenizer dependency) +- Note: Supersedes removed `candle-transformers` + `tokenizers` stack for text generation +- TODO: Implement C bindings and integration tests + +**Dummy Engine (test/demo):** +- Location: `crates/lumen-inference/src/lib.rs` +- Purpose: Deterministic pattern-matching inference for tests and demos (no model dependencies) + +## Secure Channel & Cryptographic Communication + +**Encrypted Channels (feature: `crypto-channel`):** +- What: End-to-end encrypted async message channels + - Location: `lumen-channel` crate + - Implementation: `crates/lumen-channel/src/encrypted.rs` + - Used by: Multi-agent orchestration via `lumen-orchestrator` +- Cryptographic Stack: + - Key Agreement: x25519 ECDH (`elib-x25519`) + - Encryption: AES-256-GCM (`elib-aes`) + - Key Derivation: BLAKE3 keyed hash from shared secret (`elib-blake`) + - Handshake Signature: Ed25519 (`elib-ed25519`) + - Nonce: Incremental counter (replay prevention) +- Transport: in-process channel (tokio channels) +- Attestation: Each channel endpoint attested via signed certificates + - Attestation data format: `lumen-attestation` wire format parsers + - Intel TDX / AMD SEV-SNP document support (wiring in v0.4+) + +## Model Provenance & Supply-Chain Security + +**Provenance Verification Module:** +- What: BLAKE3 hash + Ed25519 signature verification for model files + - Location: `crates/lumen-provenance/` + - Core API: `verify_model()` (blocks loading on any mismatch) + - Manifest format: `ModelManifest` (JSON/TOML serialized, signed variant `SignedManifest`) +- Supported Formats: + - ONNX models (`.onnx`) - header sniffed via `crates/lumen-provenance/src/onnx.rs` + - Safetensors (`.safetensors`) - header parsing via `safetensors` crate + custom inspection + - GGUF models (`.gguf`) - stub parser in `crates/lumen-provenance/src/gguf.rs` +- Signature Verification: + - Signing algorithm: Ed25519 (via `elib-ed25519`) + - Signer identity: Verifying key pinned in deployment manifest + - Certificate chain: Single-level (key → model hash) +- SBOM Generation: + - CycloneDX-compatible software bill of materials (`crates/lumen-provenance/src/sbom.rs`) + - Captured at model load time, exported via `generate_sbom()` + - Includes: model name, hash, signer identity, timestamp +- Pinset Validation: + - `PinSet` - Immutable set of approved (hash, signer) pairs + - `PinAcceptance` - Allow/Deny/Suspect result codes + - `RetiredPin` - Signature-invalid or compromised hashes can be retired + +## Agent Sandboxing & Capability-Based Access Control + +**Capability Framework:** +- What: Signed permission tokens for resource access + - Location: `crates/lumen-capability/` + - Core types: `Capability`, `CapabilityId`, capability policies +- Enforced Access: + - File operations: Glob patterns matching (via `globset`) + - Network operations: Hostname/port allowlisting (capability-gated import) + - Tool execution: Whitelisted tool names (capability token for each) + - System resources: Explicit per-capability grants +- Replay Prevention: + - Nonce-based challenge-response (per-invocation freshness) + - Timestamp validation (expiry enforcement) + - Signature: Ed25519 over capability bytes +- Integration with WASM: + - `lumen-sandbox` gates host imports to capabilities present in Store + - Capabilities re-validated on each `call_async()` invocation + +**Defense Module (prompt injection / jailbreak detection):** +- What: Real-time threat detection with minimal latency + - Location: `crates/lumen-defense/` + - Engine: Aho-Corasick multi-pattern matcher + RegexSet +- Detection Patterns: + - Prompt injection markers (e.g., "ignore previous instructions", SQL keywords) + - Known jailbreak payloads (curated set) + - Heuristic patterns (unusual control characters, excessive Unicode) +- Integration: + - Called by `lumen-agent` before inference (defense → infer → policy-check pipeline) + - Blocks malicious prompts before reaching LLM +- Performance: Sub-millisecond latency via pre-compiled DFAs + +## Deterministic Control & Fixed-Point Arithmetic + +**Fixed-Point Math (lumen-fixed):** +- What: Q-format quantization for deterministic proof-bound arithmetic + - Location: `crates/lumen-fixed/` + - Use case: ZK proof generation requires hardware-independent computation + - Feature: `calibration` - offline float-to-Q conversion (MUST NOT be used in proof path) +- Formats Supported: + - Q15 (1 sign + 15 integer bits) + - Q16 (1 sign + 16 integer bits) + - Q24 (1 sign + 24 integer bits) + - Custom precision via generic parameters +- Integration: LLM model weight quantization pipeline + - Offline: candle-onnx float32 weights → Q-format (uses `calibration` feature) + - Runtime: Quantized models loaded, deterministic inference executed + +**Deterministic Agent Runtime (lumen-agent):** +- What: State machine execution with no floating-point, no randomness in main path + - Location: `crates/lumen-agent/` + - Pipeline: defense (regex) → inference (ONNX or dummy) → policy check (ZK proof) → tool exec → routing decision binding + - State isolation: No shared mutable state; async isolation via tokio channels + - Execution guarantee: Same input + same seed → identical output (for ZK verification) + +## ZK Proving System & Circuit Logic + +**Mock Prover (lumen-zkml):** +- What: Proof generation for routing decisions (v0.4: mock-only, witness exposed) + - Location: `crates/lumen-zkml/` + - Feature: `ezkl` (optional; stub for real proving system integration) + - Implementation: `crates/lumen-zkml/src/mock.rs` +- Proof Subject: Agent tool-routing decision constraints + - Witness: Prompt embedding, tool scores, selection threshold + - Constraint: `selected_tool_score >= threshold && selected_tool_score > all_other_scores` + - Commitment: BLAKE3 hash of witness (only publicly revealed commitment) +- Removed Dependencies (v0.4): + - `halo2_proofs` - Mock prover replaced with direct assertion checking + - `ff` / `pasta_curves` - Elliptic curve ops no longer needed for mock +- Future Milestone (v1.0): + - Real succinct ZK backend (SP1 / RISC Zero / Aleo) + - Proof sizes: ~200-300 bytes (estimated) + - Verification gas cost: EVM-compatible (~200K gas for batch verification) + +## Attestation & TEE Support + +**TEE Attestation Parsers (lumen-attestation):** +- What: Intel TDX and AMD SEV-SNP attestation document wire format parsing + - Location: `crates/lumen-attestation/` + - Status: Wire format parsing only (v0.4); cryptographic validation deferred to v0.5+ + - Formats: + - Intel TDX attestation quote + token (ECDSA-based) + - AMD SEV-SNP attestation report (signed attestation) +- Use Case: + - Host-side TEE attestation: Proof that LLM inference runs in trusted execution environment + - Agent-side validation: Verify attestation before sending sensitive prompts to TEE +- Integration Path: + - Secure channel (`lumen-channel` encrypted channels) between agent (WASM) and host TEE + - Attestation certificates pinned in agent capability set + - Ongoing: Cryptographic signature validation in v0.5 + +## On-Chain Verification & Deployment + +**On-Chain Verifier Scaffolding (lumen-onchain):** +- What: Automated smart contract generation and deployment for proof verification + - Location: `crates/lumen-onchain/` + - Targets: + - EVM (Solidity): Bytecode generation for proof verification circuits + - Mina (o1js): Zero-knowledge DSL for ZK proof verification + - Use Case: + - Off-chain ZK proof generation (agent routing) + - On-chain submission + verification (immutable audit trail) + - Settlement: Confirmed proofs unlock fund transfer or policy enforcement +- Proof Format: + - Input: Proving system circuit (from `lumen-zkml`) + - Output: Solidity verifier contract OR Mina o1js verification circuit + - Integration: Manual deployment flow (v0.4; automated in v0.5+) +- Example: "Agent was authorized to access resource X based on approved ZK proof Y" + +## Logging & Audit + +**Structured Logging (tracing):** +- Framework: `tracing` + `tracing-subscriber` +- Usage: Audit trail for all agent decisions, model loads, capability grants + - Location: Used throughout workspace (e.g., `lumen-agent`, `lumen-provenance`) + - Output: Console formatter with environment filter (`fmt`, `env-filter` features) +- Filtering: `RUST_LOG` environment variable (e.g., `lumen_agent=debug,lumen_provenance=info`) + +## Build & Deployment Environment + +**Vendoring for Air-Gapped Environments:** +- Tool: `cargo vendor` (standard Cargo command) +- When Required: + - `candle` feature enabled (large dep tree, includes HF Hub examples) + - All workspace dependencies (for deployment in closed networks) +- How: `cargo vendor --locked > vendors.tar.gz` (commit sources) +- Guide: `AIR-GAPPED.md` in repository root + +**Continuous Integration:** +- CI Pipeline: `.github/docker/Dockerfile.ci` (Docker image with Rust 1.93.0) +- Build Checks: + ```bash + cargo build --workspace --all-targets + cargo test --workspace + cargo clippy --workspace --all-targets -- -D warnings + cargo fmt --all -- --check + ``` +- Enforcement: All four commands must pass for merge + +## Environment Configuration + +**Required Environment Variables:** +- None at runtime (all configs are manifest-based or CLI-provided) +- Development: + - `RUST_LOG` - Logging filter (optional, defaults to info) + - `RUST_BACKTRACE` - Backtrace verbosity (optional) + +**Configuration Files:** +- Manifest format: JSON or TOML (parsed by `serde` + `serde_json` / `toml`) + - Model manifests: `ModelManifest` type in `lumen-provenance` + - Capability policies: `Policy` serialized via `postcard` or JSON + - CLI config: `lumen.toml` (optional, parsed by `clap`) +- No secrets in environment (all cryptographic keys loaded from manifest or stored in TEE) + +## External API / Network Integrations + +**None Exposed (Design Principle):** +- No HTTP client dependencies (no `reqwest`, `hyper`) +- No cloud SDK integrations (no AWS, GCP, Azure SDKs) +- No external service calls from agent runtime +- TEE Integration: Channels only (no REST/gRPC) +- On-chain: Manual smart contract deployment (scaffolding provided, but no automated chain interaction) + +**Rationale:** Zero-trust, air-gapped deployment model requires all I/O to be explicit and capability-gated. + +## Build Artifacts & Performance + +**Binary Output:** +- CLI binary: `target/release/lumen` (~15-20 MB with symbols, ~3-5 MB stripped) +- Library artifacts: `target/release/liblumen*.rlib` / `liblumen*.so` +- WASM agent modules: Compiled with `wasm32-unknown-unknown` target + +**Optimization Profile (Release):** +- Optimization level: 3 (maximal) +- LTO: thin (balances speed + link-time cost) +- Panic: abort (smaller binary, no unwinding overhead) +- Debug symbols: stripped by default (use `release-debug` profile for debugging) + +--- + +*Integration audit: 2026-05-07* diff --git a/.planning/codebase/STACK.md b/.planning/codebase/STACK.md new file mode 100644 index 0000000..12a11db --- /dev/null +++ b/.planning/codebase/STACK.md @@ -0,0 +1,203 @@ +--- +last_mapped: dfba311a939388c5892574e844e73a7953796cdd +last_mapped_date: 2026-05-07 +--- + +# Technology Stack + +**Analysis Date:** 2026-05-07 + +## Languages + +**Primary:** +- Rust 1.85 (Edition 2021) - Entire workspace; enforced by `rust-toolchain.toml` +- WebAssembly (WASM32) - Agent sandbox bytecode (wasm32-unknown-unknown target) + +**Secondary:** +- Korean docstrings/comments - Project convention (see `CLAUDE.md`) + +## Runtime + +**Rust Toolchain:** +- Channel: 1.93.0 (synced with CI in `.github/docker/Dockerfile.ci`) +- Components: clippy, rustfmt +- Target: wasm32-unknown-unknown (for agent WASM compilation) +- Profile: minimal + +**Package Manager:** +- Cargo (bundled with Rust toolchain) +- Lockfile: `Cargo.lock` (present, must be updated when dependencies change) +- Resolver: v2 (workspace resolver) + +## Frameworks & Core Dependencies + +**Async Runtime:** +- `tokio` 1.x - Multi-threaded async runtime with macros, sync channels, time, I/O, filesystem + - Features: rt-multi-thread, macros, sync, time, io-util, fs + - Location: workspace-level dependency in `[workspace.dependencies]` + +**Serialization:** +- `serde` 1.x - Serialization framework with derive macros +- `serde_json` 1.x - JSON codec +- `postcard` 1.x - Binary serialization (no-heapless-cas, std only to avoid unmaintained atomic-polyfill) +- `toml` 0.8 - TOML parsing for manifests + +**Cryptography (In-House via elib-k0-nt):** +- `elib-blake` (path: `../elib-k0-nt/blake` v1.0.0) - BLAKE3 hashing (replaces upstream blake3) +- `elib-ed25519` (path: `../elib-k0-nt/ed25519` v1.0.0) - EdDSA signing/verification (replaces ed25519-dalek) +- `elib-x25519` (path: `../elib-k0-nt/x25519` v1.0.0) - ECDH key agreement for crypto-channel feature (replaces x25519-dalek) +- `elib-aes` (path: `../elib-k0-nt/aes` v1.0.0) - AES-256-GCM encryption for crypto-channel feature (replaces aes-gcm) +- `elib-rng` (path: `../elib-k0-nt/rng` v1.0.0) - HashDRBG-SHA256 deterministic random generation (replaces rand/rand_core) +- `elib-constant-time` (path: `../elib-k0-nt/constant-time` v1.0.0) - Timing-attack-resistant comparisons (replaces subtle) +- `elib-zeroize` (path: `../elib-k0-nt/zeroize` v1.0.0) - Secure memory clearing + +**WASM Sandbox:** +- `wasmtime` 44.x - WASM runtime with JIT compilation (Cranelift), async support, deterministic fuel/epoch control + - Features: async, cranelift, runtime, addr2line, demangle + - Security boundary: used exclusively in `lumen-sandbox` + - Rationale: No viable pure-Rust replacement (wasmi too slow, wasm3 requires C) + - Note: Unsafe code required for FFI; documented with `// SAFETY:` comments per clippy rules + +**CLI & Configuration:** +- `clap` 4.x - Command-line argument parsing with derive macros +- `tracing` 0.1.x - Structured logging framework +- `tracing-subscriber` 0.3.x - Log formatting and filtering (fmt, env-filter features) + +**Pattern Matching & Defense:** +- `aho-corasick` 1.x - Multi-pattern string matching (prompt injection detection) +- `regex` 1.x - Regular expression matching (jailbreak detection) +- `once_cell` 1.x - Lazy static initialization for compiled pattern sets +- `globset` 0.4.x - Path pattern matching for capability-gated file access + +**Concurrency Utilities:** +- `parking_lot` 0.12.x - Fast synchronization primitives (Mutex, RwLock) with fair scheduling + +**Model Provenance:** +- `safetensors` 0.7.x - Model file header inspection (read-only, no network dependencies) + - Note: aligned with candle-onnx's same version to avoid dependency duplication + +**Error Handling:** +- `thiserror` 2.x - Error type derivation +- `anyhow` 1.x - Generic error handling for CLI + +**Utilities:** +- `hex` 0.4.x - Hexadecimal encoding/decoding +- `futures` 0.3.x - Async utilities and combinator traits + +## Feature Flags + +**lumen-inference** (`crates/lumen-inference/Cargo.toml`): +- `candle` (optional) - ONNX routing engine via candle-core + candle-onnx + - Requires source vendoring in air-gapped environments (see `AIR-GAPPED.md`) + - CPU-only (no GPU acceleration) +- `llama-cpp` (optional) - llama.cpp backend for full LLM text generation (v0.5 in-progress) + +**lumen-fixed** (`crates/lumen-fixed/Cargo.toml`): +- `calibration` (optional) - Float-to-Q-format conversions for offline weight preparation + - MUST NOT be enabled in deterministic proof-bound codepath +- `serde` (optional) - Serialization support for fixed-point numbers + +**lumen-zkml** (`crates/lumen-zkml/Cargo.toml`): +- `ezkl` (optional) - ezkl proving system integration (stub; actual backend TBD at v1.0 milestone) + - Note: `halo2` feature removed in v0.4 (see kernel-comp.md) + +**lumen-channel** (`crates/lumen-channel/Cargo.toml`): +- `crypto-channel` (optional) - AES-GCM + x25519 encrypted channels + - Depends on elib-aes, elib-x25519 + +**lumen-sdk** (`crates/lumen-sdk/Cargo.toml`): +- `alloc` (optional) - Enables String/Vec helper functions (requires agent-side global allocator) +- `macros` (optional) - Re-exports `#[lumen_agent]` proc-macro from lumen-sdk-macros + +## Build Profiles + +**Release Profile:** +```toml +[profile.release] +opt-level = 3 # Full optimization +lto = "thin" # Thin Link-Time Optimization +codegen-units = 1 # Single codegen unit for maximum optimization +panic = "abort" # Abort on panic (no unwinding) +strip = "symbols" # Strip debug symbols +``` + +**Release-Debug Profile:** +- Inherits release settings but with `debug = true` and `strip = "none"` +- For debugging optimized binaries + +## Workspace Lints + +**Rust Lints (workspace-level):** +- `unsafe_code = "deny"` - Forbid unsafe blocks by default (overridden per crate: `lumen-sandbox`, `lumen-sdk`) +- `missing_docs = "warn"` - Require public item documentation +- `unreachable_pub = "warn"` - Warn on unnecessarily public items +- `rust_2018_idioms = "warn"` (priority -1) + +**Clippy Lints (workspace-level):** +- `all = "warn"` (priority -1) - Enable all correctness/suspicion/style/complexity/performance groups (basis for CI -D warnings) +- `undocumented_unsafe_blocks = "warn"` - Require `// SAFETY:` comments on all unsafe blocks +- Pedantic group NOT enabled workspace-wide (too noisy; crates can opt-in locally) + +**CI Enforcement:** +- `cargo clippy --workspace --all-targets -- -D warnings` (converts warnings to hard errors) +- `cargo fmt --all -- --check` (formatting checked; no auto-fix) + +## Critical Dependencies Removed (v0.4 Air-Gapped Compatibility) + +**Replaced:** +- `blake3` → `elib-blake` (cryptographically identical API) +- `ed25519-dalek` → `elib-ed25519` (keying strategy updated) +- `subtle` → `elib-constant-time` (constant-time comparisons) +- `rand`/`rand_core` → `elib-rng` (deterministic DRBG, requires RngCore adapter) +- `aes-gcm` → `elib-aes` (via crypto-channel feature) +- `x25519-dalek` → `elib-x25519` (via crypto-channel feature) + +**Removed Entirely:** +- `halo2_proofs` / `ff` / `pasta_curves` - ZK proof circuit library (v0.4: mock prover uses BLAKE3 commitment only; no succinct ZK yet) +- `candle-transformers` - LLM text generation via Hugging Face models (too large for air-gapped; replaced by llama-cpp stub) +- `tokenizers` - HuggingFace tokenizer library (contains C Oniguruma regex; GGUF models have built-in vocab) + +## Platform Requirements + +**Development:** +- Rust 1.85+ (enforced by rust-toolchain.toml) +- WASM compilation support: `rustup target add wasm32-unknown-unknown` +- Cargo (bundled) + +**Build Requirements:** +```bash +cargo build --workspace --all-targets +cargo test --workspace +cargo clippy --workspace --all-targets -- -D warnings +cargo fmt --all -- --check +``` + +**Production / Deployment:** +- Linux kernel 5.10+ with TEE support (Intel TDX / AMD SEV-SNP) for host-side TEE inference +- WASM sandbox runs in Ring 3 userspace (not suitable for kernel Ring 0 without separate trusted process) +- Air-gapped environments: source vendoring via `cargo vendor` (see `AIR-GAPPED.md`) + +## Workspace Members (v0.4.1) + +| Crate | Purpose | +|-------|---------| +| `lumen-core` | Shared IDs, BLAKE3 hashing, Ed25519 wrappers, error types, time utilities | +| `lumen-fixed` | Q-format fixed-point arithmetic (deterministic quantization) | +| `lumen-capability` | Capability-based access control with signed permission tokens | +| `lumen-channel` | Typed async secure channels (AES-GCM/x25519 behind feature) | +| `lumen-provenance` | Model file provenance: BLAKE3 + Ed25519 verification + SBOM (safetensors parsing) | +| `lumen-defense` | Low-latency prompt injection / jailbreak detection (Aho-Corasick + regex) | +| `lumen-zkml` | ZK proving-system trait + mock BLAKE3 commitment prover (ezkl stub) | +| `lumen-inference` | InferenceEngine trait + verified loader + quantization config (candle/llama.cpp features) | +| `lumen-sandbox` | wasmtime WASM sandbox with deterministic config and capability-gated host imports | +| `lumen-agent` | Agent runtime: defense → infer → policy-check → tool-exec → ZK-bind routing | +| `lumen-orchestrator` | Multi-agent supervisor with channel-only inter-agent communication | +| `lumen-sdk` | WASM agent SDK: safe Rust wrappers for host imports (no_std compatible) | +| `lumen-sdk-macros` | Procedural macros for `#[lumen_agent]` annotation | +| `lumen-attestation` | TEE attestation document parsers (Intel TDX, AMD SEV-SNP) | +| `lumen-onchain` | On-chain verifier scaffold (EVM Solidity / Mina o1js) + deployment automation | +| `lumen-cli` | `lumen` command-line interface | + +--- + +*Stack analysis: 2026-05-07* diff --git a/.planning/codebase/STRUCTURE.md b/.planning/codebase/STRUCTURE.md new file mode 100644 index 0000000..f810a23 --- /dev/null +++ b/.planning/codebase/STRUCTURE.md @@ -0,0 +1,443 @@ +--- +last_mapped_commit: 7c6fc8c +analysis_date: 2026-05-07 +focus: structure +--- + +# Codebase Structure + +**Analysis Date:** 2026-05-07 + +## Directory Layout + +``` +lumen/ +├── crates/ # Workspace members (15 crates) +│ ├── lumen-core/ +│ │ ├── src/ +│ │ │ ├── lib.rs # Core exports +│ │ │ ├── ids.rs # AgentId, CapabilityId, RequestId, ToolId +│ │ │ ├── crypto.rs # Ed25519 wrappers (SigningKey, VerifyingKey) +│ │ │ ├── hash.rs # Blake3Hash + constant-time comparison +│ │ │ ├── rng.rs # RNG trait + HashDrbg, OsRng adapters +│ │ │ ├── error.rs # Error enum, Result type +│ │ │ ├── time.rs # Timestamp +│ │ │ └── Cargo.toml +│ │ └── tests/ +│ │ +│ ├── lumen-fixed/ +│ │ ├── src/ +│ │ │ ├── lib.rs # Q16_16, Q8_24 exports +│ │ │ ├── q16_16.rs # Q-format fixed-point (range ±2^15) +│ │ │ ├── q8_24.rs # Q-format fixed-point (range ±2^7, precise) +│ │ │ ├── quant.rs # Quantization helpers (i8 quantize) +│ │ │ └── Cargo.toml +│ │ └── tests/ +│ │ +│ ├── lumen-capability/ +│ │ ├── src/ +│ │ │ ├── lib.rs # Capability, PolicyEngine, Action, Resource exports +│ │ │ ├── capability.rs # Capability struct, signing, verification +│ │ │ ├── policy.rs # PolicyEngine (verify, replay defense, audit) +│ │ │ ├── resource.rs # Resource enum (FsRead, FsWrite, Net, CallTool, etc) +│ │ │ ├── audit.rs # Audit logging (deterministic field names) +│ │ │ └── Cargo.toml +│ │ ├── tests/ +│ │ │ └── policy_enforcement.rs # Policy check tests +│ │ └── examples/ +│ │ +│ ├── lumen-channel/ +│ │ ├── src/ +│ │ │ ├── lib.rs # SecureChannel trait +│ │ │ ├── inproc.rs # InProcChannel (tokio mpsc) +│ │ │ ├── attested.rs # AttestedChannel (Ed25519 handshake + frame signing) +│ │ │ ├── encrypted.rs # EncryptedChannel (AES-256-GCM + x25519, feature-gated) +│ │ │ ├── tee.rs # TeeChannel stub (unimplemented) +│ │ │ ├── codec.rs # postcard encode/decode +│ │ │ └── Cargo.toml +│ │ ├── tests/ +│ │ │ └── roundtrip.rs # Serialization tests +│ │ └── examples/ +│ │ +│ ├── lumen-defense/ +│ │ ├── src/ +│ │ │ ├── lib.rs # DefenseEngine, Verdict, BlockReason exports +│ │ │ ├── engine.rs # DefenseEngine::analyze() pipeline +│ │ │ ├── lexicon.rs # Aho-Corasick jailbreak corpus (30+ triggers) +│ │ │ ├── regex_stage.rs # RegexSet for pattern matching +│ │ │ ├── heuristics.rs # Heuristic scoring (non-printable, base64, length) +│ │ │ └── Cargo.toml +│ │ ├── tests/ +│ │ │ └── verdicts.rs # Defense verdict tests +│ │ └── examples/ +│ │ +│ ├── lumen-provenance/ +│ │ ├── src/ +│ │ │ ├── lib.rs # ModelManifest, verify_model, SBOM exports +│ │ │ ├── manifest.rs # ModelManifest, SignedManifest, Format enum +│ │ │ ├── verify.rs # verify_model() hash + signature validation +│ │ │ ├── gguf.rs # GGUF header sniffing +│ │ │ ├── onnx.rs # ONNX header sniffing +│ │ │ ├── safetensors_check.rs # Safetensors header validation +│ │ │ ├── sbom.rs # CycloneDX SBOM generation +│ │ │ ├── pinset.rs # Pin acceptance lists (retired pins) +│ │ │ └── Cargo.toml +│ │ ├── tests/ +│ │ │ └── verify_safetensors.rs # Safetensors verification tests +│ │ └── examples/ +│ │ +│ ├── lumen-zkml/ +│ │ ├── src/ +│ │ │ ├── lib.rs # ProvingSystem trait, Verification exports +│ │ │ ├── mock.rs # MockCommitmentProver (BLAKE3 commitment) +│ │ │ ├── verification.rs # Verification enum (CommitmentOnly, ZkVerified, Invalid) +│ │ │ ├── ezkl.rs # ezkl stub (feature-gated) +│ │ │ └── Cargo.toml +│ │ ├── tests/ +│ │ │ └── routing_proof.rs # Mock proof generation tests +│ │ └── examples/ +│ │ +│ ├── lumen-inference/ +│ │ ├── src/ +│ │ │ ├── lib.rs # InferenceEngine trait, SamplingParams, Completion, ToolCall +│ │ │ ├── backend.rs # BackendConfig, create_engine() factory +│ │ │ ├── dummy.rs # DummyEngine (deterministic test responses) +│ │ │ ├── candle.rs # CandleEngine (ONNX routing, feature: candle) +│ │ │ ├── llama_cpp.rs # LlamaCppEngine stub (feature: llama-cpp) +│ │ │ ├── loader.rs # VerifiedModelLoader, VerifiedModelHandle +│ │ │ ├── quantize.rs # QuantizationConfig, GgufLevel, QuantizationKind +│ │ │ ├── streaming.rs # StreamingEngine trait, Token, TokenStream, FinishReason +│ │ │ ├── tee_channel.rs # ChannelEngine (TEE forward via SecureChannel) +│ │ │ └── Cargo.toml +│ │ ├── tests/ +│ │ │ └── engine_selection.rs # Backend selection tests +│ │ └── examples/ +│ │ +│ ├── lumen-sandbox/ +│ │ ├── src/ +│ │ │ ├── lib.rs # Sandbox, SandboxConfig, HostState exports +│ │ │ ├── runner.rs # Sandbox::run_module() execution +│ │ │ ├── config.rs # SandboxConfig::deterministic_engine() +│ │ │ ├── host.rs # HostState (policy, agent, audit, tool_calls) +│ │ │ ├── imports.rs # Host import implementations (lumen_log, lumen_call_tool) +│ │ │ └── Cargo.toml +│ │ ├── tests/ +│ │ │ └── wasm_echo_agent.rs # WASM agent test (determinism check) +│ │ └── examples/ +│ │ +│ ├── lumen-agent/ +│ │ ├── src/ +│ │ │ ├── lib.rs # AgentRuntime, StepResult, ToolRegistry exports +│ │ │ ├── runtime.rs # AgentRuntime::step() / stream_step() pipeline +│ │ │ ├── tool.rs # Tool trait, ToolHandler, ToolRegistry +│ │ │ ├── route.rs # RoutingDecision, RoutingPublicInputs, RoutingWitness +│ │ │ └── Cargo.toml +│ │ ├── tests/ +│ │ │ ├── determinism.rs # Determinism checks (same input → same output) +│ │ │ └── determinism_stress.rs # Stress test for determinism +│ │ └── examples/ +│ │ +│ ├── lumen-orchestrator/ +│ │ ├── src/ +│ │ │ ├── lib.rs # Orchestrator, AgentHandle, InterAgentMessage exports +│ │ │ ├── supervisor.rs # Orchestrator (registry, inboxes, message routing) +│ │ │ └── Cargo.toml +│ │ ├── tests/ +│ │ │ └── multi_agent.rs # Multi-agent execution tests +│ │ └── examples/ +│ │ +│ ├── lumen-sdk/ +│ │ ├── src/ +│ │ │ ├── lib.rs # log(), call_tool(), LogLevel, ToolError, no_std +│ │ │ ├── alloc_helpers.rs # alloc-only helpers (feature: alloc) +│ │ │ └── Cargo.toml +│ │ └── examples/ +│ │ └── hello_agent.rs # Minimal WASM agent example +│ │ +│ ├── lumen-sdk-macros/ +│ │ ├── src/ +│ │ │ ├── lib.rs # #[lumen_agent] proc-macro definition +│ │ │ └── Cargo.toml +│ │ └── tests/ +│ │ +│ ├── lumen-attestation/ +│ │ ├── src/ +│ │ │ ├── lib.rs # AttestationDoc, parse() exports +│ │ │ ├── tdx.rs # Intel TDX Quote v4 parser +│ │ │ ├── sev_snp.rs # AMD SEV-SNP Attestation Report v2 parser +│ │ │ └── Cargo.toml +│ │ └── tests/ +│ │ +│ ├── lumen-onchain/ +│ │ ├── src/ +│ │ │ ├── lib.rs # Chain enum, VerifierMeta, emit/deploy exports +│ │ │ ├── emit.rs # emit_artifacts() (Solidity + o1js) +│ │ │ ├── deploy.rs # deploy_evm(), deploy_mina() (child processes) +│ │ │ ├── evm.rs # Solidity RoutingVerifier.sol codegen +│ │ │ ├── mina.rs # o1js RoutingVerifier.ts codegen +│ │ │ └── Cargo.toml +│ │ └── examples/ +│ │ +│ └── lumen-cli/ +│ ├── src/ +│ │ ├── main.rs # CLI entry, subcommand dispatch +│ │ ├── policy_file.rs # Policy TOML loading / validation +│ │ ├── cmd/ +│ │ │ ├── init.rs # `lumen init` — emit policy template +│ │ │ ├── verify_model.rs # `lumen verify-model` — check BLAKE3 + sig +│ │ │ ├── sbom.rs # `lumen sbom` — emit CycloneDX SBOM +│ │ │ ├── defend.rs # `lumen defend` — test prompt defense +│ │ │ ├── prove.rs # `lumen prove` — generate routing proof +│ │ │ ├── run.rs # `lumen run` — end-to-end agent step +│ │ │ └── verifier.rs # `lumen verifier deploy` — emit+deploy contracts +│ │ └── Cargo.toml +│ ├── tests/ +│ │ └── e2e.rs # End-to-end CLI tests +│ └── examples/ +│ +├── agents/ # (excluded from workspace; example WASM agents) +│ └── echo-agent/ # Sample WASM agent using lumen-sdk +│ ├── src/ +│ │ └── lib.rs # Agent logic (step fn) +│ ├── Cargo.toml # target: wasm32-unknown-unknown +│ └── build.sh # wasm compilation + linking +│ +├── Cargo.toml # Workspace root, workspace dependencies, lints +├── Cargo.lock # Locked dependency versions +├── CLAUDE.md # Project-specific instructions (Korean) +├── kernel-comp.md # Air-gap compatibility audit (detailed dependency review) +├── README.md # Project overview +├── .github/ +│ └── workflows/ # CI/CD (linter, tests, clippy, fmt) +└── .planning/ + └── codebase/ + ├── ARCHITECTURE.md # This file: system design, layers, data flow + ├── STRUCTURE.md # This file: directory layout, naming conventions + └── (STACK.md, INTEGRATIONS.md, CONVENTIONS.md, TESTING.md, CONCERNS.md → on request) +``` + +## Directory Purposes + +**crates/lumen-core/:** +- Purpose: Shared primitives (IDs, hashing, crypto, errors, time) +- Contains: `AgentId`, `Blake3Hash`, `Ed25519` wrappers, `Rng` trait, `Error` enum +- Key files: `ids.rs` (newtype IDs), `crypto.rs` (signing/verification wrappers), `hash.rs` (BLAKE3), `error.rs` (Result type) + +**crates/lumen-fixed/:** +- Purpose: Deterministic fixed-point arithmetic (Q-format) for ZK-bound code +- Contains: `Q16_16`, `Q8_24` types, quantization utilities +- Key files: `q16_16.rs`, `q8_24.rs`, `quant.rs` (i8 quantizer) +- Constraint: `calibration` feature disabled in proof-bound binaries (f32 only offline) + +**crates/lumen-capability/:** +- Purpose: Signed capability tokens and policy enforcement +- Contains: `Capability` (signed token), `PolicyEngine` (verify + replay defense), `Action` (enum of gated actions), `Resource` (target of action) +- Key files: `capability.rs` (token structure), `policy.rs` (enforcement), `audit.rs` (deterministic logging) + +**crates/lumen-channel/:** +- Purpose: Typed async message transport between isolation boundaries +- Contains: `SecureChannel` trait (send/recv bytes or typed messages), implementations: `InProcChannel`, `AttestedChannel`, `EncryptedChannel` (feature-gated), `TeeChannel` (stub) +- Key files: `lib.rs` (trait), `inproc.rs`, `attested.rs`, `encrypted.rs` (crypto-channel feature), `tee.rs` (TEE stub) + +**crates/lumen-defense/:** +- Purpose: Real-time prompt-injection / jailbreak detection (sub-µs/kB) +- Contains: `DefenseEngine` (three-stage pipeline), `Verdict` enum, jailbreak lexicon (~30 triggers) +- Key files: `engine.rs` (pipeline), `lexicon.rs` (Aho-Corasick corpus), `regex_stage.rs` (RegexSet patterns), `heuristics.rs` (scoring) + +**crates/lumen-provenance/:** +- Purpose: Model file provenance (hash verification, signature, SBOM) +- Contains: `ModelManifest`, `verify_model()` function, format sniffers (ONNX, Safetensors, GGUF), SBOM generation +- Key files: `manifest.rs` (manifest struct), `verify.rs` (hash/sig check), `sbom.rs` (CycloneDX emission) + +**crates/lumen-zkml/:** +- Purpose: Pluggable ZK proof system abstraction +- Contains: `ProvingSystem` trait (generic over witness/public/proof/vk types), `MockCommitmentProver` (BLAKE3), `Verification` enum +- Key files: `mock.rs` (BLAKE3 commitment implementation), `verification.rs` (Verification::CommitmentOnly vs ZkVerified) + +**crates/lumen-inference/:** +- Purpose: Backend-agnostic completion generation with streaming support +- Contains: `InferenceEngine` trait, concrete engines: `DummyEngine`, `CandleEngine` (ONNX, feature: candle), `LlamaCppEngine` (stub, feature: llama-cpp), `ChannelEngine` (TEE forward) +- Key files: `backend.rs` (trait + factory), `dummy.rs` (deterministic test engine), `streaming.rs` (StreamingEngine trait), `loader.rs` (model verification + loading) + +**crates/lumen-sandbox/:** +- Purpose: WASM isolation via wasmtime with deterministic config and capability-gated host imports +- Contains: `Sandbox` (module runner), `SandboxConfig` (deterministic engine setup), `HostState` (policy, agent, audit), host imports (`lumen_log`, `lumen_call_tool`) +- Key files: `runner.rs` (run_module execution), `config.rs` (deterministic_engine), `imports.rs` (host FFI implementations), `host.rs` (HostState) + +**crates/lumen-agent/:** +- Purpose: Orchestrates agent step pipeline (defense → infer → policy → tool → prove) +- Contains: `AgentRuntime` (main orchestrator), `StepResult` (output), `Tool` / `ToolRegistry` (executable tools), routing decision types +- Key files: `runtime.rs` (AgentRuntime::step pipeline), `tool.rs` (Tool trait + ToolRegistry), `route.rs` (RoutingDecision types) + +**crates/lumen-orchestrator/:** +- Purpose: Multi-agent supervisor with capability-gated inter-agent messaging +- Contains: `Orchestrator` (registry + inboxes), `AgentHandle`, `InterAgentMessage`, `MessagePolicy` +- Key files: `supervisor.rs` (all logic) + +**crates/lumen-sdk/:** +- Purpose: WASM-targeted SDK for agent code (no_std compatible) +- Contains: `log()`, `call_tool()` host import wrappers, `LogLevel` enum, `ToolError` enum +- Key files: `lib.rs` (all public exports; 100% of SDK) +- Constraint: `no_std`, compiled to `wasm32-unknown-unknown` + +**crates/lumen-sdk-macros/:** +- Purpose: Proc-macro for `#[lumen_agent]` attribute +- Contains: Macro expansion for generating `_start()` WASM entry point +- Key files: `lib.rs` + +**crates/lumen-attestation/:** +- Purpose: Parse Intel TDX Quote v4 and AMD SEV-SNP Attestation Report v2 (format-only verification in v0.3) +- Contains: `AttestationDoc` enum, `TdxQuote`, `SevSnpReport`, wire format parsers +- Key files: `tdx.rs`, `sev_snp.rs`, `lib.rs` (parse dispatcher) + +**crates/lumen-onchain/:** +- Purpose: Emit and deploy on-chain verifiers (EVM Solidity + Mina o1js) for routing proof verification +- Contains: `Chain` enum, `VerifierMeta`, `emit_artifacts()`, `deploy_evm()` / `deploy_mina()` +- Key files: `emit.rs` (code generation), `deploy.rs` (deployment automation), `evm.rs` (Solidity), `mina.rs` (o1js) + +**crates/lumen-cli/:** +- Purpose: User-facing command-line interface +- Contains: Seven subcommands (init, verify-model, sbom, defend, prove, run, verifier) +- Key files: `main.rs` (entry + dispatch), `cmd/*.rs` (subcommand implementations), `policy_file.rs` (TOML loading) + +**agents/:** (Excluded from workspace) +- Purpose: Example WASM agents demonstrating SDK usage +- Contains: `echo-agent/` (minimal agent calling host tools) +- Key files: Agent `src/lib.rs`, `Cargo.toml` with `target: wasm32-unknown-unknown` + +## Naming Conventions + +**Files:** +- **Trait definitions:** `lib.rs` (e.g., `SecureChannel` trait in `lumen-channel/src/lib.rs`) +- **Concrete implementations:** `{name}.rs` matching implementation (e.g., `inproc.rs`, `encrypted.rs` in lumen-channel) +- **Type definitions:** `{plural_of_type}.rs` (e.g., `ids.rs` for `AgentId`, `CapabilityId`, etc.) +- **Entry points:** `main.rs` (CLI binary), `lib.rs` (library crates) +- **Tests:** `{module}_test.rs` or `tests/{feature}_test.rs` +- **Examples:** `examples/{demo_name}.rs` +- **Config:** `config.rs` when config struct is main export + +**Directories:** +- Crate name prefix: `lumen-{feature}` (snake_case, hyphenated) +- Module pattern: One feature = one crate (e.g., `lumen-defense`, `lumen-zkml`) +- Nested modules: Within a crate, use `.rs` files not subdirs (flat structure preferred) + +**Functions:** +- **Constructors:** `new()`, `with_*()` for builder variants (e.g., `PolicyEngine::new()`, `HostState::with_now()`) +- **Factory functions:** `create_*()` or `{type_name}_*()` (e.g., `create_engine()`, `build_stream()`) +- **Verifiers:** `verify_*()`, `check()`, `is_*()` for booleans (e.g., `verify_model()`, `is_human_readable()`) +- **Type conversions:** `as_*()`, `to_*()`, `from_*()` following Rust conventions +- **Async functions:** `_async()` suffix only if no `.await` syntax available; prefer async/await + +**Types:** +- **Newtype IDs:** `{Entity}Id` (e.g., `AgentId`, `ToolId`, `CapabilityId`) +- **Enums:** `PascalCase` with `camelCase` variants (e.g., `Verdict::Block`, `Verification::CommitmentOnly`) +- **Traits:** `{Action}Engine` or `{Concept}` (e.g., `InferenceEngine`, `SecureChannel`, `ProvingSystem`) +- **Structs:** `PascalCase` (e.g., `ModelManifest`, `ToolCall`, `SamplingParams`) +- **Constants:** `SCREAMING_SNAKE_CASE` (e.g., `MAX_NONCES`, `DEFAULT_INBOX_DEPTH`) + +**Modules:** +- **Public modules:** Exported in `lib.rs` (e.g., `pub mod policy;`, `pub mod capability;`) +- **Private modules:** File-only or nested (not re-exported) + +## Key File Locations + +**Entry Points:** +- `crates/lumen-cli/src/main.rs` — CLI binary entry +- `crates/lumen-agent/src/runtime.rs:99` — `AgentRuntime::step()` API entry +- `crates/lumen-sdk/src/lib.rs` — WASM SDK exports (`log`, `call_tool`) +- `agents/echo-agent/src/lib.rs` — Example WASM agent + +**Configuration:** +- `Cargo.toml` (workspace root) — Workspace members, lints, workspace dependencies +- `crates/lumen-sandbox/src/config.rs` — Deterministic WASM engine configuration +- `crates/lumen-inference/src/backend.rs` — Backend selection + sampling parameter defaults +- `crates/lumen-cli/src/policy_file.rs` — Policy TOML schema + loading + +**Core Logic:** +- `crates/lumen-core/src/` — All primitives (IDs, hashing, crypto, errors) +- `crates/lumen-agent/src/runtime.rs` — Main step pipeline +- `crates/lumen-capability/src/policy.rs` — Capability verification +- `crates/lumen-sandbox/src/runner.rs` — WASM execution +- `crates/lumen-defense/src/engine.rs` — Defense pipeline +- `crates/lumen-inference/src/backend.rs` — Engine trait + factories +- `crates/lumen-zkml/src/mock.rs` — Mock BLAKE3 prover + +**Testing:** +- `crates/lumen-agent/tests/determinism.rs` — Determinism validation +- `crates/lumen-sandbox/tests/wasm_echo_agent.rs` — WASM sandbox tests +- `crates/lumen-capability/tests/policy_enforcement.rs` — Policy check tests + +## Where to Add New Code + +**New inference backend:** +1. Create `crates/lumen-inference/src/{backend_name}.rs` +2. Implement `InferenceEngine` trait +3. Add variant to `BackendConfig` enum in `backend.rs` +4. Extend `create_engine()` factory function +5. Add feature flag in `lumen-inference/Cargo.toml` + +**New capability resource type:** +1. Add variant to `Resource` enum in `crates/lumen-capability/src/resource.rs` +2. Implement `matches_action()` for new `Action` variant +3. Add matching rule to `PolicyEngine::check()` in `policy.rs` +4. Update audit logging in `audit.rs` + +**New defense stage:** +1. Add function to `crates/lumen-defense/src/{stage_name}.rs` +2. Call from `DefenseEngine::analyze()` pipeline in `engine.rs` +3. Return `Verdict` or score contribution +4. Register corpus version in `DefenseEngine::corpus_version()` + +**New tool type:** +1. Create handler struct implementing `Tool` trait in `crates/lumen-agent/src/tool.rs` +2. Register in `ToolRegistry` via `register()` method +3. Include required `Capability` with matching `Resource::CallTool(tool_id)` + +**New on-chain verifier:** +1. Add variant to `Chain` enum in `crates/lumen-onchain/src/lib.rs` +2. Create `{chain}.rs` module with codegen function +3. Update `emit::emit_artifacts()` to call new codegen +4. Update `deploy::deploy()` to handle new chain + +**New WASM host import:** +1. Add FFI signature in `crates/lumen-sdk/src/lib.rs` (behind `#[cfg(target_arch = "wasm32")]`) +2. Implement panic in non-WASM in `lib.rs` +3. Add host-side handler in `crates/lumen-sandbox/src/imports.rs` +4. Register import in `crates/lumen-sandbox/src/runner.rs` linker + +**New CLI subcommand:** +1. Create `crates/lumen-cli/src/cmd/{command}.rs` +2. Define `Args` struct with clap derive macros +3. Implement `run()` function +4. Add variant to `Cmd` enum in `main.rs` +5. Dispatch in `main()` match block + +**New model format support:** +1. Create format sniff function in `crates/lumen-provenance/src/{format}.rs` +2. Add `Format` variant in `manifest.rs` +3. Call sniff from `verify_model()` in `verify.rs` + +## Special Directories + +**Generated / Non-committed:** +- `target/` — Cargo build output (in .gitignore) +- `.git/` — Git repository (not part of codebase) + +**Build artifacts (in git):** +- `Cargo.lock` — Locked dependency versions (committed for reproducibility) + +**Documentation:** +- `README.md` — Project overview +- `CLAUDE.md` — Project instructions (Korean, checked in) +- `kernel-comp.md` — Air-gap dependency audit (checked in) +- `.planning/codebase/` — GSD codebase maps (generated, committed) + +**CI/CD:** +- `.github/workflows/` — GitHub Actions (linter, tests, clippy, fmt) + +**Examples:** +- `agents/echo-agent/` — Sample WASM agent (excluded from workspace to avoid build issues) + +--- + +*Structure analysis: 2026-05-07* diff --git a/.planning/codebase/TESTING.md b/.planning/codebase/TESTING.md new file mode 100644 index 0000000..2ed821f --- /dev/null +++ b/.planning/codebase/TESTING.md @@ -0,0 +1,508 @@ +--- +title: Lumen Testing Patterns +last_mapped_commit: dfba311a939388c5892574e844e73a7953796cdd +last_mapped_date: 2026-05-07 +--- + +# Testing Patterns + +**Analysis Date:** 2026-05-07 + +## Test Framework + +### Test Runner + +**Framework:** `#[tokio::test]` macro (from `tokio` crate) +- For async functions: use `#[tokio::test]` +- For sync functions: use `#[test]` +- Config: No separate test config file; uses workspace default + +**Runtime Config:** +- `tokio` version: `1.x` with features `["macros", "rt-multi-thread", "sync", "time"]` +- Tests run with multi-threaded executor by default +- Can override per-test with `#[tokio::test(flavor = "multi_thread")]` or `flavor = "current_thread"` + +### Assertion Library + +**Built-in assertions:** +- `assert!()`, `assert_eq!()`, `assert_ne!()` +- `.expect()` / `.unwrap()` for test setup +- `.is_ok()` / `.is_err()` for Result testing + +**Custom assertions:** +- No dedicated assertion crate +- Manual string matching where needed + +### Run Commands + +```bash +# Run all tests +cargo test --workspace + +# Run tests with output (don't capture stdout) +cargo test --workspace -- --nocapture + +# Run single test +cargo test --lib agent_step_succeeds + +# Run integration tests only +cargo test --test determinism + +# Run with multiple threads (default) +cargo test --workspace -- --test-threads=4 + +# Run with single thread +cargo test --workspace -- --test-threads=1 +``` + +## Test File Organization + +### Location Pattern + +**Integration Tests (per-crate):** +- Path: `crates/[crate-name]/tests/[test-name].rs` +- Top-level `tests/` directory at crate root +- Examples: + - `crates/lumen-agent/tests/determinism.rs` + - `crates/lumen-agent/tests/determinism_stress.rs` + - `crates/lumen-sandbox/tests/wasm_echo_agent.rs` + - `crates/lumen-provenance/tests/verify_safetensors.rs` + - `crates/lumen-provenance/tests/pinset_rotation.rs` + - `crates/lumen-sdk/tests/macro_expansion.rs` + +**Unit Tests (co-located):** +- Placed in same file as code or in `#[cfg(test)]` module at bottom +- Example: `crates/lumen-core/src/crypto.rs` has `#[cfg(test)] mod tests { }` + +**Fixture Data:** +- No dedicated fixture directory; fixtures inline or in test functions +- Use `tempfile` crate for temporary directories in tests +- Example: `write_tiny_safetensors(&dir)` in `verify_safetensors.rs` + +### Naming Convention + +**Test File Names:** +- Descriptive: `determinism.rs`, `verify_safetensors.rs`, `wasm_echo_agent.rs` +- Not `test.rs` or generic names + +**Test Function Names:** +- snake_case +- Describe behavior: `step_succeeds_with_capability`, `missing_capability_rejected` +- Pattern: `[subject]_[scenario]_[outcome]` + +**Module Structure:** +``` +tests/ +├── determinism.rs # Agent step determinism + policy tests +├── determinism_stress.rs # Stress testing determinism +├── wasm_echo_agent.rs # End-to-end WASM echo agent tests +├── verify_safetensors.rs # Model provenance verification +└── pinset_rotation.rs # Pin rotation lifecycle tests +``` + +## Test Structure + +### Async Test Pattern + +```rust +#[tokio::test] +async fn step_succeeds_with_capability() { + // Arrange: Set up fixtures + let runtime = build_runtime(); + + // Act: Execute the code being tested + let result = runtime.step("echo hello").await; + + // Assert: Verify outcomes + let r = result.expect("step ok"); + assert!(r.completion.tool_call.is_some()); + let out = r.tool_output.expect("tool output"); + assert!(out.contains("hello")); +} +``` + +### Sync Test Pattern + +```rust +#[test] +fn verify_clean_file_passes() { + // Arrange + let dir = tempfile::tempdir().unwrap(); + let path = write_tiny_safetensors(&dir); + let hash = Blake3Hash::of_file(&path).unwrap(); + + let manifest = ModelManifest { /* ... */ }; + + // Act + let info = verify_model(&path, &manifest, &[]).unwrap(); + + // Assert + assert_eq!(info.name, "tiny"); +} +``` + +### Setup/Teardown Pattern + +**Setup helpers:** +- Define `fn build_*()` helper functions above tests +- Example: `fn build_runtime() -> AgentRuntime { ... }` +- Helpers initialize shared fixtures + +**Teardown:** +- Implicit via Rust ownership; `Drop` traits clean up +- `tempfile::TempDir` auto-deletes on drop +- No explicit cleanup needed in most cases + +```rust +fn build_runtime() -> AgentRuntime { + let sk = SigningKey::generate(&mut OsRng); + let vk = sk.verifying_key(); + let agent = AgentId::random(&mut OsRng); + let policy = Arc::new(PolicyEngine::new(vec![vk])); + + let echo_tool_id = ToolId::new("echo").unwrap(); + let cap = Capability::sign( + CapabilityBody { /* ... */ }, + &sk, + ).unwrap(); + + AgentRuntime::builder(agent, policy) + .inference(Arc::new(DummyEngine::new())) + .tool(Tool { /* ... */ }) + .capability(echo_tool_id, cap) + .build() + .unwrap() +} + +#[tokio::test] +async fn step_succeeds_with_capability() { + let runtime = build_runtime(); + // ... test logic +} +``` + +## Mocking Strategy + +### Mock Implementations + +**DummyEngine (from `lumen-inference`):** +- Deterministic pattern-matching inference engine for tests +- Returns predictable results based on input text +- No external dependencies or randomness + +```rust +use lumen_inference::DummyEngine; + +let engine = Arc::new(DummyEngine::new()); +``` + +**MockVk (from `lumen-zkml::mock`):** +- Mock verification key for proof validation +- Used in agent runtime tests + +```rust +use lumen_zkml::mock::MockVk; + +let proving_vk = MockVk { + circuit_id: "lumen.routing.v1".into(), +}; +``` + +### What to Mock + +**Mock these:** +- `InferenceEngine` implementations (use `DummyEngine` in tests) +- Verification keys (use `MockVk`) +- Large/slow external resources (models, services) + +**Don't Mock these:** +- Core types (`AgentId`, `CapabilityId`, `Error`) +- Cryptographic operations (use real signing/verification) +- Policy engines (use real `PolicyEngine::new()`) +- Capability tokens (use real `Capability::sign()`) + +### Fixture & Factory Pattern + +**Test Data Construction:** +```rust +// Factory helper in test file: +fn build_runtime() -> AgentRuntime { + let sk = SigningKey::generate(&mut OsRng); + let vk = sk.verifying_key(); + // ... build and return fully configured runtime +} + +// Use in multiple tests: +#[tokio::test] +async fn test_1() { + let rt = build_runtime(); + // ... +} + +#[tokio::test] +async fn test_2() { + let rt = build_runtime(); + // ... +} +``` + +**Temporary Files:** +```rust +use tempfile::TempDir; + +fn write_tiny_safetensors(dir: &tempfile::TempDir) -> PathBuf { + let data: Vec = vec![1, 2, 3, 4, 5, 6, 7, 8]; + let view = TensorView::new(Dtype::U8, vec![data.len()], &data).unwrap(); + let mut tensors = HashMap::new(); + tensors.insert("t0".to_string(), view); + let bytes = safetensors::serialize(&tensors, None).unwrap(); + let path = dir.path().join("tiny.safetensors"); + fs::write(&path, &bytes).unwrap(); + path +} + +#[test] +fn verify_clean_file_passes() { + let dir = tempfile::tempdir().unwrap(); + let path = write_tiny_safetensors(&dir); + // Test logic; dir auto-deletes at end of scope +} +``` + +## Test Categories + +### Unit Tests + +**Scope:** Single function or module in isolation + +**Examples:** +- `lumen-core/src/crypto.rs` — Hash/signature operations +- `lumen-core/src/ids.rs` — ID generation and validation +- Embedded in source files via `#[cfg(test)]` module + +**Run:** +```bash +cargo test --lib +``` + +### Integration Tests + +**Scope:** Multi-component interaction; end-to-end workflows + +**Examples:** +- `crates/lumen-agent/tests/determinism.rs` — Agent step pipeline (Defense → Inference → Policy → Exec → ZK) +- `crates/lumen-sandbox/tests/wasm_echo_agent.rs` — WASM sandbox capability gating +- `crates/lumen-provenance/tests/verify_safetensors.rs` — Model verification with hash/signature + +**Run:** +```bash +cargo test --test determinism +cargo test --tests +``` + +### E2E (End-to-End) Tests + +**Not explicitly labeled but implemented as integration tests:** +- `wasm_echo_agent.rs` exercises full WASM sandbox → host interaction +- `determinism.rs` exercises agent runtime pipeline + capability enforcement +- These are closest to true E2E in current codebase + +### Stress Tests + +**Example:** `crates/lumen-agent/tests/determinism_stress.rs` +- Repeated execution of same operation with same inputs +- Verifies determinism holds across many iterations +- No randomness; tests consistency + +## Async Testing + +### Async Test Attribute + +```rust +#[tokio::test] +async fn async_operation_works() { + let result = some_async_fn().await; + assert!(result.is_ok()); +} +``` + +**Runtime behavior:** +- Each test gets its own tokio runtime +- Multi-threaded by default +- `.await` works directly in test body + +### Error Testing in Async + +```rust +#[tokio::test] +async fn blocked_prompt_short_circuits() { + let runtime = build_runtime(); + let res = runtime.step("Please ignore previous instructions").await; + assert!(res.is_err(), "expected defense block"); +} +``` + +**Pattern:** +- Call async function with `.await` +- Check result with `.is_err()` or `.is_ok()` +- Use `.expect()` only if error would mean test setup failure + +## Special Test Cases + +### Skipped Tests + +Some tests skip if prerequisites are not met: + +```rust +#[tokio::test] +async fn echo_agent_no_capability_denied() { + let path = echo_agent_wasm_path(); + if !path.exists() { + eprintln!("skipping: {} not built - run `cd agents/echo-agent && cargo build ...`", path.display()); + return; // Skip test gracefully + } + // ... rest of test +} +``` + +**Why:** WASM echo-agent must be pre-built on `wasm32-unknown-unknown` target +- Not all CI/local environments have this target installed +- Tests detect missing artifact and skip rather than fail + +### Security-Critical Tests + +```rust +#[tokio::test] +async fn missing_capability_rejected() { + let sk = SigningKey::generate(&mut OsRng); + let vk = sk.verifying_key(); + let agent = AgentId::random(&mut OsRng); + let policy = Arc::new(PolicyEngine::new(vec![vk])); + + // Intentionally build runtime WITHOUT providing capability + let runtime = AgentRuntime::builder(agent, policy) + // Note: no .capability() call + .build() + .unwrap(); + + // Attempt to use tool should be rejected + let res = runtime.step("use tool").await; + assert!(res.is_err(), "capability missing should reject"); +} +``` + +**Pattern:** +- Test both positive (allowed) and negative (denied) paths +- Verify security checks are enforced + +## Coverage + +### Current State + +- **No explicit coverage tool configured** (no `tarpaulin`, `llvm-cov`, etc.) +- Tests exist but coverage percentage not tracked in CI +- Encouraged but not enforced + +### Test Categories by Coverage + +**Well-covered:** +- Core cryptographic primitives (`lumen-core`) +- Agent runtime determinism (`lumen-agent`) +- Policy enforcement and capability gating +- Model provenance verification (`lumen-provenance`) + +**Partially covered:** +- WASM sandbox (requires `wasm32-unknown-unknown` target) +- Inference backends (optional features) + +**Not covered:** +- Optional features (`candle`, `llama-cpp`, `halo2`) — feature-gated in CI + +## CI/CD Testing Pipeline + +### 4-Gate Validation (from `.github/workflows/ci.yml`) + +Every commit must pass: + +```bash +# 1. Build all targets +cargo build --workspace --all-targets --locked + +# 2. Run tests +cargo test --workspace --locked + +# 3. Linting (Clippy with -D warnings) +cargo clippy --workspace --all-targets -- -D warnings + +# 4. Formatting check +cargo fmt --all -- --check +``` + +**Job runs:** +- Linux container: `build-test` job +- macOS native: `build-test-macos` job +- Feature tests: `features` job (halo2, candle) +- Linting: `lint` job (Clippy + rustfmt) +- Documentation: `docs` job (cargo doc, intra-doc links) + +### WASM Target Build + +In CI (both Linux and macOS): +```bash +cd agents/echo-agent +cargo build --release --target wasm32-unknown-unknown --locked +``` + +This artifact is used by `wasm_echo_agent.rs` tests. + +## Running Tests Locally + +### Before Committing + +```bash +# Run full test suite +cargo test --workspace + +# Run with output +cargo test --workspace -- --nocapture + +# Check formatting +cargo fmt --all -- --check + +# Check Clippy +cargo clippy --workspace --all-targets -- -D warnings + +# Build all targets (including binaries, examples) +cargo build --workspace --all-targets + +# Build WASM echo-agent +cd agents/echo-agent && cargo build --release --target wasm32-unknown-unknown +``` + +### Troubleshooting + +**WASM test skipped:** +- Install wasm32 target: `rustup target add wasm32-unknown-unknown` +- Build echo-agent: `cd agents/echo-agent && cargo build --release --target wasm32-unknown-unknown` +- Retry: `cargo test --test wasm_echo_agent` + +**Test timeout:** +- Some determinism stress tests may take time +- Use `--release` profile for faster execution: `cargo test --release --workspace` +- Run single test: `cargo test stress_determinism -- --nocapture` + +## Dependencies for Testing + +**From workspace Cargo.toml:** +- `tokio` (v1, with test executor built-in) +- `tempfile` — For temporary directories in tests +- No separate testing framework dependency + +**Optional test dependencies per-crate:** +- `lumen-inference::DummyEngine` — Mock inference +- `lumen-zkml::mock::MockVk` — Mock verification key +- Elib cryptographic operations (signing, hashing) + +--- + +*Testing analysis: 2026-05-07* diff --git a/AIR-GAPPED.md b/AIR-GAPPED.md new file mode 100644 index 0000000..b9be6da --- /dev/null +++ b/AIR-GAPPED.md @@ -0,0 +1,173 @@ +# 폐쇄형(Air-Gapped) 환경 빌드 가이드 + +> **목적**: Lumen 을 외부 네트워크가 차단된 고보안 구역(High-Side)에서 +> 빌드/테스트/배포할 수 있도록 의존성을 사전 벤더링하고 오프라인 +> 4-게이트를 통과시키는 절차를 정의합니다. K0 마이크로커널의 +> Ring 3 사용자공간 컴포넌트로 통합되는 시점부터 본 가이드의 절차가 +> 빌드 시스템에 강제됩니다. + +--- + +## 1. 사전 정리 — v0.4 의존성 변경 사항 + +폐쇄망 호환을 위해 v0.4 에서 다음 의존성이 제거되었습니다: + +| 제거 의존성 | 사유 | 대체 | +|-------------|------|------| +| `blake3`, `ed25519-dalek`, `subtle`, `rand`, `rand_core`, `aes-gcm`, `x25519-dalek` | 외부 crates.io 의존, FIPS 미인증 | `elib-k0-nt/*` (in-house, 인증 경로) | +| `halo2_proofs` + `ff` + `pasta_curves` | 타원곡선 의존성 ~30 크레이트, MockProver 는 ZK 보장 없음 | mock prover 가 BLAKE3 commitment 만 사용. succinct ZK 는 SP1/RISC Zero 마일스톤에서 재검토. | +| `candle-core` / `candle-onnx` / `candle-transformers` / `tokenizers` | HuggingFace Hub 다운로드 코드 포함, `gemm`/`pulp`/`paste` 등 무거운 native 의존 | 도구 라우팅 토크나이저는 자체 `lumen_inference::BpeTokenizer` 사용. 전체 LLM 생성은 `llama-cpp` 백엔드 (v0.5) 또는 호스트 TEE 의 `ChannelEngine` forward 경로 | + +자세한 사유는 [`kernel-comp.md`](kernel-comp.md) 참고. + +--- + +## 2. 의존성 벤더링 (온라인 단계) + +폐쇄망 이전을 위해 온라인 환경에서 한 번 수행합니다. + +```bash +# 워크스페이스 루트에서: +./scripts/vendor.sh # default features 만 (가장 가벼움) +./scripts/vendor.sh --all # crypto-channel + llama-cpp 까지 포함 +``` + +> **참고:** v0.4 이후 `candle` feature 가 제거되어 `protoc` / `cmake` 가 +> 더 이상 필수 시스템 바이너리가 아닙니다. `cmake` 는 `llama-cpp` feature +> 활성화 시에만 필요합니다. + +스크립트가 수행하는 작업: + +1. `cargo vendor` 로 `Cargo.lock` 의 모든 크레이트 소스를 `vendor/` 에 복제 +2. `.cargo/vendor-config.toml` → `.cargo/config.toml` 활성화 (소스 교체 + `net.offline = true`) +3. 산출물 크기/개수 출력 + +산출물: + +| 경로 | 크기 | git | 비고 | +|------|------|-----|------| +| `vendor/` | ~500MB–1GB | ❌ ignored | `cargo vendor` 산출물 | +| `.cargo/config.toml` | ~700B | ❌ ignored | 활성화된 source-replacement 설정 | +| `.cargo/vendor-config.toml` | ~700B | ✅ committed | 템플릿 | + +--- + +## 3. 시스템 바이너리 사전 패키징 + +워크스페이스 외부의 빌드 도구는 별도로 폐쇄망에 옮겨야 합니다. + +| 바이너리 | 활성화 조건 | 용도 | +|----------|-------------|------| +| `rustc` 1.95.0 + cargo | 항상 (`rust-toolchain.toml`) | Rust 컴파일러 | +| `cmake` ≥ 3.20 | `lumen-inference/llama-cpp` feature (v0.5) | llama.cpp 네이티브 라이브러리 빌드 | + +폐쇄망 반입 시 권장: `~/.cargo/`, `cmake` 바이너리(`llama-cpp` feature +사용 시), 그리고 본 저장소(벤더링 완료 상태)를 함께 묶어 `tar.gz` 단일 +번들로 전송. + +```bash +# 온라인 측에서: +./scripts/vendor.sh --all +tar czf lumen-airgap-bundle.tar.gz \ + --exclude=target --exclude=.git \ + . + +# 폐쇄망 측에서 (예: 데이터 다이오드/CDS 통과 후): +tar xzf lumen-airgap-bundle.tar.gz +``` + +--- + +## 4. 오프라인 4-게이트 검증 + +폐쇄망에서 다음 명령이 모두 통과해야 합니다. + +```bash +cargo build --offline --workspace --all-targets +cargo test --offline --workspace +cargo clippy --offline --workspace --all-targets -- -D warnings +cargo fmt --all -- --check +``` + +`vendor-config.toml` 의 `net.offline = true` 설정 덕분에 외부 fetch 시도가 +있으면 빌드가 즉시 실패합니다 — 폐쇄망 정책 위반을 컴파일 시점에 +탐지하는 안전장치입니다. + +선택적 feature 별 검증: + +```bash +# 보안 채널 (X25519 + AES-256-GCM, elib-k0-nt 백엔드) +cargo test --offline --workspace --features lumen-channel/crypto-channel + +# llama.cpp 추론 백엔드 (v0.5 스텁, cmake 시스템 바이너리 필요) +cargo test --offline --workspace --features lumen-inference/llama-cpp +``` + +--- + +## 5. K0 마이크로커널 통합 메모 + +Lumen 은 K0 의 Ring 3 사용자공간 컴포넌트로 동작합니다. +보안 경계는 다음과 같습니다. + +``` +┌─────────────────────────── High-Side (Air-Gapped) ───────────────────────────┐ +│ │ +│ Ring 0 ┃ K0 마이크로커널 (PSK 기반 mutual auth, TLS-PSK-PQ) │ +│ ┃ - elib-k0-nt 암호 모듈 (FIPS 인증 경로) │ +│ ─────────┃───────────────────────────────────────────────────────────────── │ +│ Ring 3 ┃ Lumen 에이전트 프레임워크 (본 저장소) │ +│ ┃ - WASM 샌드박스 (wasmtime, JIT) │ +│ ┃ - 호스트 TEE 추론 forward (ChannelEngine, AES-256-GCM) │ +│ ┃ - 결정론적 상태 (tokio + lumen-fixed) │ +│ │ +└──────────────────────────────────────────────────────────────────────────────┘ +``` + +**제약 사항:** +- **Ring 0 에서 Cranelift JIT 실행 불가** — 따라서 `lumen-sandbox` 는 + 반드시 Ring 3 사용자공간 프로세스로 실행. 커널-에이전트 간 통신은 + K0 의 syscall 또는 PSK 기반 IPC 채널 사용. +- **모든 외부 fetch 금지** — `cargo build` 의 네트워크 시도조차 + `net.offline = true` 로 차단. crates.io 미러 운용 금지. +- **모델 가중치 외부 반입** — Safetensors / ONNX / GGUF 파일은 + `lumen-provenance` 의 BLAKE3 + Ed25519 매니페스트 검증을 통과해야만 + Ring 3 에서 로드 가능. + +--- + +## 6. 운영 점검 (정기) + +| 점검 항목 | 주기 | 명령 | +|-----------|------|------| +| `vendor/` 무결성 | 빌드 전 | `./scripts/vendor.sh --check` | +| 4-게이트 | 매 변경 | 위 §4 참고 | +| `Cargo.lock` 변동 | PR 검토 | `git diff Cargo.lock` | +| 시스템 바이너리 버전 | 분기 | `rustc --version`, `cmake --version` (`llama-cpp` 사용 시) | + +--- + +## 7. 트러블슈팅 + +### `error: failed to get ... from registry` + +→ `.cargo/config.toml` 의 `vendored-sources` 가 활성화되지 않았거나 + `vendor/` 가 누락. `./scripts/vendor.sh` 재실행. + +### `error: vendor/ 디렉토리가 없습니다` + +→ `./scripts/vendor.sh --check` 를 vendor 디렉토리 없이 실행한 경우. + 먼저 `./scripts/vendor.sh` 또는 `./scripts/vendor.sh --all` 실행. + +### 빌드가 오래 걸림 + +→ `wasmtime` (Cranelift JIT) 빌드가 ~2–3 분 소요. 정상. + `cargo build --offline --jobs $(nproc)` 로 병렬화 권장. + +--- + +## 참고 + +- [`kernel-comp.md`](kernel-comp.md) — 의존성 점검 보고서 +- [`CLAUDE.md`](CLAUDE.md) — 프로젝트 헌장 (4-게이트, 한국어 docstring 등) +- [K0 AIR-GAP.md](../K0/AIR-GAP.md) — 커널 측 폐쇄망 프로토콜 diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..2ed56d2 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,44 @@ +# Lumen 프로젝트 목표 + +제로 트러스트(Zero-Trust) 및 폐쇄(Air-Gapped) 환경을 위한 고보안, 검증 가능한 Rust 기반 AI 에이전트 프레임워크. 권한 통제(WASM 샌드박스)와 결과 검증(zkML)을 결합하여, 국가기관 수준의 엄격한 보안 규격을 충족하는 안전한 자율형 AI 구동을 목표. + +**AI 및 보안 오픈소스 생태계 기여를 위한 프로젝트.** + +구현 필요 기능 다음과 같음: + +1. 하이브리드 zkML 파이프라인 + - 선택적 ZK 증명: 연산 비용이 높은 LLM 전체 텍스트 생성 대신, 에이전트의 '도구 선택(Tool-use routing)' 및 '출력 필터링/정책 준수' 로직에만 ezkl 또는 ZKTorch를 적용하여 ZK Proof 생성. + - 자동화 및 검증: CI/CD 파이프라인에서 Proof를 자동 생성하고, 온체인(Mina/EVM) 또는 오프체인 검증기(Verifier)에 배포 및 확인하는 모듈. +2. 권한 분리형 Host-WASM 샌드박스 + - WASM 격리 (Agent Logic): wasmtime과 Rust의 소유권 모델을 활용해 에이전트의 도구 실행 및 통신 로직을 완벽히 격리. 파일, 네트워크, 시스템 환경 접근은 명시적인 Capability 기반으로만 최소한으로 허용. + - Host TEE 실행 (LLM Inference): 하드웨어 가속(GPU)을 온전히 활용하기 위해 무거운 모델 추론은 호스트의 신뢰 실행 환경(TEE) 내부에서 실행하고, WASM 샌드박스와는 격리된 보안 채널(Secure Channel)로만 통신. +3. 결정론적(Deterministic) 제어 모듈 + - 상태 격리: 멀티 에이전트 오케스트레이션 시 메모리 공유 없이 tokio 기반 비동기 처리로 상태 전이의 안정성 확보. + - 연산 통제: ZK 증명의 신뢰성을 보장하기 위해 부동소수점 연산의 하드웨어 의존성을 배제하는 모델 양자화(Quantization) 및 고정소수점(Fixed-point) 연산 파이프라인 구축. +4. 고속 보안 방어 및 출처 검증 엔진 + - 실시간 위협 탐지: Rust의 성능을 살려 지연 시간(Latency)을 최소화한 프롬프트 인젝션 및 제일브레이크 방어/필터링 모듈 탑재. + - 모델 Provenance (공급망 보안): ONNX, Safetensors 모델 로드 시 파일 해시 및 서명을 즉각 검증하고, SBOM을 강제 생성하여 승인되지 않은 모델 가중치 변조를 원천 차단. + +# 구현에 앞서 + +모든 설계는 보안성 우선. 단, 연산 비용 (작업 속도) 우선적 최적화 필요. + +Docstring 한국어로 작성. + +마일스톤 `v1.0`(정부 또는 규제 환경에서 production 배포, [FIPS 140-3 compliance audit](https://csrc.nist.gov/pubs/fips/140-3/final), [Kani](https://www.in-com.com/ko/blog/the-rust-developers-toolbox-best-static-code-analysis-tools/#Kani) 또는 [Prusti](https://github.com/viperproject/prusti-dev) 등을 활용한 일부 모듈의 형식 검증, 외부 보안 audit 1회 통과)은 당분간 진행 X. + +# 기능 수정 또는 추가 시 + +`Cargo.toml`이 변경된 경우 `.lock` 파일 갱신할 것. + +프로젝트는 항상 다음 네 개의 명령어를 모두 통과해야 함. + +```bash +$ cargo build --workspace --all-targets +$ cargo test --workspace +$ cargo clippy --workspace --all-targets -- -D warnings +$ cargo fmt --all -- --check +$ cargo deny check # Github CI/CD(action) 에서도 통과해야 함. +``` + +`Cargo.lock` 파일도 공개됨에 따라 동기화 여부(수정이 필요한지 등) 잘 파악. \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ea7cfa2..98128e0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,13 +1,15 @@ # 기여 가이드라인 -이 프로젝트는 **보안 우선**이므로 PR은 Team Quant의 다른 프로젝트들과 동일하게 비교적 엄격한 기준을 따릅니다. 새로운 의존성은 정당화가 필요하여 이유와 audit 결과를 PR description에 명시해주셔야 하며, 가능하면 `default-features = false` 로 추가하고 필요한 feature만 활성화해주세요. `unsafe` 사용은 `lumen-sandbox` 만 허용되며 다른 곳에서는 `#![forbid(unsafe_code)]`가 강제됩니다. `lumen-sandbox`의 `unsafe`는 wasmtime FFI가 불가피하기 때문이고, 모든 `unsafe` 블록에 영문 `// SAFETY:` 주석이 필수입니다! +이 프로젝트는 **보안 우선**이므로 PR은 Team Quant의 다른 프로젝트들과 동일하게 비교적 엄격한 기준을 따릅니다. 새로운 의존성은 정당화가 필요하여 이유와 audit 결과를 PR description에 명시해주셔야 하며, 가능하면 `default-features = false` 로 추가하고 필요한 feature만 활성화해주세요. `unsafe` 사용은 `lumen-sandbox` 만 허용되며 다른 곳에서는 `#![forbid(unsafe_code)]`가 강제됩니다. `lumen-sandbox`의 `unsafe`는 wasmtime FFI가 불가피하기 때문이고, 모든 `unsafe` 블록에 영문 `// SAFETY:` 주석이 필수입니다. -모든 PR 은 다음 네 가지 게이트를 통과해야 합니다. +모든 PR은 다음 네 가지 게이트를 통과해야 합니다. -- `cargo build --workspace --all-targets` -- `cargo test --workspace` -- `cargo clippy --workspace --all-targets -- -D warnings` -- `cargo fmt --all -- --check` +```bash +$ cargo build --workspace --all-targets +$ cargo test --workspace +$ cargo clippy --workspace --all-targets -- -D warnings +$ cargo fmt --all -- --check +``` 새로운 기능은 정확성과 보안 테스트를 동반해야 하며 적대적 입력 테스트가 포함되어야 합니다. 예를 들어, ONNX 디코더 PR에는 다음 부정적 테스트(negative-test)가 반드시 있어야 합니다. @@ -15,12 +17,8 @@ - deprecated group - oversized length -**결정성을 깨는 변경은 명시적 정당화가 필요**하여 `f32` 사용, `HashMap` 사용, 시스템 시각 의존 등은 PR description에 명시되고 결정론 경로 영향이 없음이 증명되어야 합니다. 보안 영향이 있는 PR은 위협 모델 업데이트를 동반하여 새로운 공격면을 열거나 닫는 변경이면 위 *위협 모델* 절을 함께 갱신해야 하며, 마지막으로 취약점 보고는 비공개로 **GitHub issue 가 아닌 maintainer 이메일로 직접 보고 (책임감 있는 공개)** 하기를 요청합니다. - -# 토의점 - -저희는 로드맵 마일스톤의 과정에서 암호학적 기능 제공자로 기본 [ELIB-K0-NT](https://github.com/Quant-Off/elib-k0-nt) 라이브러리 설정 후, 모듈식 또는 플래그를 통한 암호학 기능 추가(조립) 또는 변경하는 의견에 대해 어떻게 생각하시나요? ELIB-K0-NT는 암호학적 인증이 몇 부분 필요한 점이 있지만, 상수-시간 또는 소거 부문에선 확실함을 보이기도 해서 계속 고민 중에 있습니다. +**결정성을 깨는 변경은 명시적 정당화가 필요**하여 `f32` 사용, `HashMap` 사용, 시스템 시각 의존 등은 PR description에 명시되고 **결정론 경로 영향이 없음이 증명되어야 합니다.** 보안 영향이 있는 PR은 위협 모델 업데이트를 동반하여 새로운 공격면을 열거나 닫는 변경이면 위 *위협 모델* 절을 함께 갱신해야 하며, 마지막으로 취약점 보고는 비공개로 **GitHub issue 가 아닌 maintainer 이메일로 직접 보고 (책임감 있는 공개)** 하기를 요청합니다. # 라이선스 -이 프로젝트는 Apache License 2.0 OR MIT License 듀얼 라이선스이며 사용자는 둘 중 어느 쪽이든 선택할 수 있습니다. 자세한 내용은 [LICENSE-APACHE](LICENSE-APACHE) 와 [LICENSE-MIT](LICENSE-MIT) 를 참고하세요. 이 라이선스 선택은 Rust 생태계의 표준 관행을 따른 것이며 상업·연구·정부 사용 모두 명시적으로 허용합니다 - 보안 인프라가 광범위하게 채택되려면 라이선스가 장벽이 되어서는 안 된다는 판단입니다. \ No newline at end of file +이 프로젝트는 Apache License 2.0 OR MIT License 듀얼 라이선스이며 사용자는 둘 중 어느 쪽이든 선택할 수 있습니다. 자세한 내용은 [LICENSE-APACHE](LICENSE-APACHE) 와 [LICENSE-MIT](LICENSE-MIT)를 참고하세요. 이 라이선스 선택은 Rust 생태계의 표준 관행을 따른 것이며 상업·연구·정부 사용 모두 명시적으로 허용합니다. 이는 보안 인프라가 광범위하게 채택되려면 라이선스가 장벽이 되어서는 안 된다는 판단입니다. \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 77bf10e..99e824d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11,53 +11,12 @@ dependencies = [ "gimli", ] -[[package]] -name = "aead" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" -dependencies = [ - "crypto-common", - "generic-array", -] - [[package]] name = "aes" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" -dependencies = [ - "cfg-if", - "cipher", - "cpufeatures 0.2.17", -] - -[[package]] -name = "aes-gcm" -version = "0.10.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" -dependencies = [ - "aead", - "aes", - "cipher", - "ctr", - "ghash", - "subtle", -] - -[[package]] -name = "ahash" -version = "0.8.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +version = "1.0.0" dependencies = [ - "cfg-if", - "getrandom 0.3.4", - "once_cell", - "serde", - "version_check", - "zerocopy", + "constant-time", + "zeroize", ] [[package]] @@ -111,7 +70,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.61.2", + "windows-sys", ] [[package]] @@ -122,14 +81,14 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.61.2", + "windows-sys", ] [[package]] name = "anyhow" -version = "1.0.102" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "arbitrary" @@ -137,18 +96,6 @@ version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" -[[package]] -name = "arrayref" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" - -[[package]] -name = "arrayvec" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" - [[package]] name = "async-trait" version = "0.1.89" @@ -160,39 +107,6 @@ dependencies = [ "syn", ] -[[package]] -name = "autocfg" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" - -[[package]] -name = "base64" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" - -[[package]] -name = "base64ct" -version = "1.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" - -[[package]] -name = "bit-set" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" -dependencies = [ - "bit-vec", -] - -[[package]] -name = "bit-vec" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" - [[package]] name = "bitflags" version = "2.11.1" @@ -200,40 +114,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" [[package]] -name = "bitvec" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" -dependencies = [ - "funty", - "radium", - "tap", - "wyz", -] - -[[package]] -name = "blake2b_simd" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b79834656f71332577234b50bfc009996f7449e0c056884e6a02492ded0ca2f3" -dependencies = [ - "arrayref", - "arrayvec", - "constant_time_eq", -] - -[[package]] -name = "blake3" -version = "1.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce" +name = "blake" +version = "1.0.0" dependencies = [ - "arrayref", - "arrayvec", - "cc", - "cfg-if", - "constant_time_eq", - "cpufeatures 0.3.0", + "constant-time", + "zeroize", ] [[package]] @@ -264,118 +149,12 @@ dependencies = [ "allocator-api2", ] -[[package]] -name = "bytemuck" -version = "1.25.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" -dependencies = [ - "bytemuck_derive", -] - -[[package]] -name = "bytemuck_derive" -version = "1.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "byteorder" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" - [[package]] name = "bytes" version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" -[[package]] -name = "candle-core" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bd9895436c1ba5dc1037a19935d084b838db066ff4e15ef7dded020b7c12a4a" -dependencies = [ - "byteorder", - "float8", - "gemm", - "half", - "libm", - "memmap2", - "num-traits", - "num_cpus", - "rand 0.9.4", - "rand_distr", - "rayon", - "safetensors", - "thiserror 2.0.18", - "tokenizers 0.22.2", - "yoke", - "zip", -] - -[[package]] -name = "candle-nn" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9317a09d6530b758990ed7f625ac69ff43653bc9ee28b0464644ad1169ada87" -dependencies = [ - "candle-core", - "half", - "libc", - "num-traits", - "rayon", - "safetensors", - "serde", - "thiserror 2.0.18", -] - -[[package]] -name = "candle-onnx" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c26833f7747d202a47a1678c0fe11ab85c67b765a8aa99335672e4d81407a32" -dependencies = [ - "candle-core", - "candle-nn", - "prost", - "prost-build", -] - -[[package]] -name = "candle-transformers" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f59d08c89e9f4af9c464e2f3a8e16199e7cc601e6f34538c2cfbb42b623b1783" -dependencies = [ - "byteorder", - "candle-core", - "candle-nn", - "fancy-regex", - "num-traits", - "rand 0.9.4", - "rayon", - "serde", - "serde_json", - "serde_plain", - "tracing", -] - -[[package]] -name = "castaway" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" -dependencies = [ - "rustversion", -] - [[package]] name = "cc" version = "1.2.61" @@ -392,16 +171,6 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" -[[package]] -name = "cipher" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" -dependencies = [ - "crypto-common", - "inout", -] - [[package]] name = "clap" version = "4.6.1" @@ -448,7 +217,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1" dependencies = [ - "thiserror 2.0.18", + "thiserror", ] [[package]] @@ -458,50 +227,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" [[package]] -name = "compact_str" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb1325a1cece981e8a296ab8f0f9b63ae357bd0784a9faaf548cc7b480707a" -dependencies = [ - "castaway", - "cfg-if", - "itoa", - "rustversion", - "ryu", - "serde", - "static_assertions", -] - -[[package]] -name = "console" -version = "0.15.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8" -dependencies = [ - "encode_unicode", - "libc", - "once_cell", - "unicode-width", - "windows-sys 0.59.0", -] - -[[package]] -name = "const-oid" -version = "0.9.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" - -[[package]] -name = "constant_time_eq" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" +name = "constant-time" +version = "1.0.0" [[package]] name = "cpp_demangle" -version = "0.4.5" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2bb79cb74d735044c972aae58ed0aaa9a837e85b01106a54c39e42e97f62253" +checksum = "0667304c32ea56cb4cd6d2d7c0cfe9a2f8041229db8c033af7f8d69492429def" dependencies = [ "cfg-if", ] @@ -515,38 +248,29 @@ dependencies = [ "libc", ] -[[package]] -name = "cpufeatures" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" -dependencies = [ - "libc", -] - [[package]] name = "cranelift-assembler-x64" -version = "0.131.1" +version = "0.135.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8628cc4ba7f88a9205a7ee42327697abc61195a1e3d92cfae172d6a946e722e" +checksum = "42c0ca95f115597232c3c5d7286bf847d49332bee5934749f40808b7fe0c6312" dependencies = [ "cranelift-assembler-x64-meta", ] [[package]] name = "cranelift-assembler-x64-meta" -version = "0.131.1" +version = "0.135.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d582754487e6c9a065a91c42ccf1bdd8d5977af33468dac5ae9bec0ce88acb3e" +checksum = "fdd7d59ef8a52c82ea2ac2ca96aee3c52644845440f70611b573bc20ce37e6a2" dependencies = [ "cranelift-srcgen", ] [[package]] name = "cranelift-bforest" -version = "0.131.1" +version = "0.135.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb59c81ace12ee7c33074db7903d4d75d1f40b28cd3e8e6f491de57b29129eb9" +checksum = "547433a3f4204f0ed0f5a5e3ffaf086a0c005fd0efdf040098083f38572e8a49" dependencies = [ "cranelift-entity", "wasmtime-internal-core", @@ -554,9 +278,9 @@ dependencies = [ [[package]] name = "cranelift-bitset" -version = "0.131.1" +version = "0.135.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f25c06993a681be9cf3140798a3d4ac5bec955e7444416a2fdc87fda8567285d" +checksum = "61ca4f30f5f2eb3837b6e73664d9c0c9a42c4c0dce05f0027f2f2af54c1b20d7" dependencies = [ "serde", "serde_derive", @@ -565,9 +289,9 @@ dependencies = [ [[package]] name = "cranelift-codegen" -version = "0.131.1" +version = "0.135.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27b61f95c5a211918f5d336254a61a488b36a5818de47a868e8c4658dce9cccc" +checksum = "effcb11ccea65cfb2b8dd4f53b92c6449ad37745504bae036bffda597a5cfa48" dependencies = [ "bumpalo", "cranelift-assembler-x64", @@ -579,13 +303,16 @@ dependencies = [ "cranelift-entity", "cranelift-isle", "gimli", - "hashbrown 0.16.1", + "hashbrown 0.17.0", "libm", "log", + "postcard", "pulley-interpreter", "regalloc2", "rustc-hash", "serde", + "serde_derive", + "sha2 0.10.9", "smallvec", "target-lexicon", "wasmtime-internal-core", @@ -593,9 +320,9 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" -version = "0.131.1" +version = "0.135.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b85aa822fce72080d041d7c2cf7c3f5c6ecdea7afae68379ba4ef85269c4fa5" +checksum = "2cd5b4e429e24f7fd0bfe15d653ec5841fc1831ef68521b0d30182ef0d48bd01" dependencies = [ "cranelift-assembler-x64-meta", "cranelift-codegen-shared", @@ -606,24 +333,24 @@ dependencies = [ [[package]] name = "cranelift-codegen-shared" -version = "0.131.1" +version = "0.135.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "833eb9fc89326cd072cc19e96892f09b5692c0dfe17cd4da2858ba30c2cd85c0" +checksum = "a8aefe7ed05aa9a831a4696f55e020a6c93998325a09c642863f0be7845fe3a8" [[package]] name = "cranelift-control" -version = "0.131.1" +version = "0.135.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d005320f487e6e8a3edcc7f2fd4f43fcc9946d1013bf206ea649789ac1617fc" +checksum = "b3ceaa06b6ece027bb001093bf0c6984a96281e061fd40a99c60bae0ec57171d" dependencies = [ "arbitrary", ] [[package]] name = "cranelift-entity" -version = "0.131.1" +version = "0.135.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e62ef34c6e720f347a79ece043e8584e242d168911da640bac654a33a6aaaf5" +checksum = "bc2c3088c6be94765abea42eccff7ac2b6519488e66bb5159febabcf539afd6e" dependencies = [ "cranelift-bitset", "serde", @@ -633,11 +360,12 @@ dependencies = [ [[package]] name = "cranelift-frontend" -version = "0.131.1" +version = "0.135.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfa2ad00399dd47e7e7e33cb1dc23b0e39ed9dcd01e8f026fc37af91655031b8" +checksum = "0e2c4a0a90aedfa88408f33e65eda7b1f73de0af20ebec516c004b3a1cdc602b" dependencies = [ "cranelift-codegen", + "hashbrown 0.17.0", "log", "smallvec", "target-lexicon", @@ -645,15 +373,15 @@ dependencies = [ [[package]] name = "cranelift-isle" -version = "0.131.1" +version = "0.135.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02c51975ed217b4e8e5a7fd11e9ec83a96104bdff311dddcb505d1d8a9fd7fc6" +checksum = "bcb1cc85ccb60cfc53c3d55e23196fef28ae78fbcebe4e1ff3ff866a3c93d61c" [[package]] name = "cranelift-native" -version = "0.131.1" +version = "0.135.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9b1889e00da9729d8f8525f3c12998ded86ea709058ff844ebe00b97548de0e" +checksum = "5a0a8f5d008c1f5b78426e83028f7517295f6819b5cebe8193949e1bb66cc01a" dependencies = [ "cranelift-codegen", "libc", @@ -662,9 +390,9 @@ dependencies = [ [[package]] name = "cranelift-srcgen" -version = "0.131.1" +version = "0.135.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5a8f82fd5124f009f72167e60139245cd3b56cfd4b53050f22110c48c5f4da1" +checksum = "56caa8a7f1e0f2c61400996cc1567fb06dd417c49d5e673131a149cb9628625f" [[package]] name = "crc32fast" @@ -675,37 +403,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "crossbeam-deque" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" -dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.9.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-utils" -version = "0.8.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" - -[[package]] -name = "crunchy" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" - [[package]] name = "crypto-common" version = "0.1.7" @@ -713,324 +410,90 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ "generic-array", - "rand_core 0.6.4", "typenum", ] [[package]] -name = "ctr" -version = "0.9.2" +name = "digest" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "cipher", + "block-buffer", + "crypto-common", ] [[package]] -name = "curve25519-dalek" -version = "4.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +name = "ed25519" +version = "1.0.0" dependencies = [ - "cfg-if", - "cpufeatures 0.2.17", - "curve25519-dalek-derive", - "digest", - "fiat-crypto", - "rustc_version", - "subtle", + "sha2 1.0.0", "zeroize", ] [[package]] -name = "curve25519-dalek-derive" -version = "0.1.1" +name = "either" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" [[package]] -name = "darling" -version = "0.20.11" +name = "embedded-io" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" -dependencies = [ - "darling_core", - "darling_macro", -] +checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" [[package]] -name = "darling_core" -version = "0.20.11" +name = "embedded-io" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" -dependencies = [ - "fnv", - "ident_case", - "proc-macro2", - "quote", - "strsim", - "syn", -] +checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" [[package]] -name = "darling_macro" -version = "0.20.11" +name = "equivalent" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" -dependencies = [ - "darling_core", - "quote", - "syn", -] +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] -name = "dary_heap" -version = "0.3.9" +name = "errno" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b1e3a325bc115f096c8b77bbf027a7c2592230e70be2d985be950d3d5e60ebe" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ - "serde", + "libc", + "windows-sys", ] [[package]] -name = "der" -version = "0.7.10" +name = "fastrand" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" -dependencies = [ - "const-oid", - "zeroize", -] +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" [[package]] -name = "derive_builder" -version = "0.20.2" +name = "find-msvc-tools" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" -dependencies = [ - "derive_builder_macro", -] +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" [[package]] -name = "derive_builder_core" -version = "0.20.2" +name = "fnv" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" -dependencies = [ - "darling", - "proc-macro2", - "quote", - "syn", -] +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] -name = "derive_builder_macro" -version = "0.20.2" +name = "foldhash" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" -dependencies = [ - "derive_builder_core", - "syn", -] +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" [[package]] -name = "digest" -version = "0.10.7" +name = "foldhash" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" -dependencies = [ - "block-buffer", - "crypto-common", -] - -[[package]] -name = "dyn-stack" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c4713e43e2886ba72b8271aa66c93d722116acf7a75555cce11dcde84388fe8" -dependencies = [ - "bytemuck", - "dyn-stack-macros", -] - -[[package]] -name = "dyn-stack-macros" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1d926b4d407d372f141f93bb444696142c29d32962ccbd3531117cf3aa0bfa9" - -[[package]] -name = "ed25519" -version = "2.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" -dependencies = [ - "pkcs8", - "signature", -] - -[[package]] -name = "ed25519-dalek" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" -dependencies = [ - "curve25519-dalek", - "ed25519", - "rand_core 0.6.4", - "serde", - "sha2", - "subtle", - "zeroize", -] - -[[package]] -name = "either" -version = "1.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" - -[[package]] -name = "embedded-io" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" - -[[package]] -name = "embedded-io" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" - -[[package]] -name = "encode_unicode" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" - -[[package]] -name = "enum-as-inner" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1e6a265c649f3f5979b601d26f1d05ada116434c87741c9493cb56218f76cbc" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "equivalent" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" - -[[package]] -name = "errno" -version = "0.3.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" -dependencies = [ - "libc", - "windows-sys 0.61.2", -] - -[[package]] -name = "esaxx-rs" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6" -dependencies = [ - "cc", -] - -[[package]] -name = "fancy-regex" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72cf461f865c862bb7dc573f643dd6a2b6842f7c30b07882b56bd148cc2761b8" -dependencies = [ - "bit-set", - "regex-automata", - "regex-syntax", -] - -[[package]] -name = "fastrand" -version = "2.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" - -[[package]] -name = "ff" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" -dependencies = [ - "bitvec", - "rand_core 0.6.4", - "subtle", -] - -[[package]] -name = "fiat-crypto" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" - -[[package]] -name = "find-msvc-tools" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" - -[[package]] -name = "fixedbitset" -version = "0.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" - -[[package]] -name = "float8" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2d1f04709a8ac06e8e8042875a3c466cc4832d3c1a18dbcb9dba3c6e83046bc" -dependencies = [ - "half", - "num-traits", - "rand 0.9.4", - "rand_distr", -] - -[[package]] -name = "fnv" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" - -[[package]] -name = "foldhash" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" - -[[package]] -name = "foldhash" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" - -[[package]] -name = "funty" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" [[package]] name = "futures" @@ -1120,125 +583,6 @@ dependencies = [ "slab", ] -[[package]] -name = "gemm" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa0673db364b12263d103b68337a68fbecc541d6f6b61ba72fe438654709eacb" -dependencies = [ - "dyn-stack", - "gemm-c32", - "gemm-c64", - "gemm-common", - "gemm-f16", - "gemm-f32", - "gemm-f64", - "num-complex", - "num-traits", - "paste", - "raw-cpuid", - "seq-macro", -] - -[[package]] -name = "gemm-c32" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "086936dbdcb99e37aad81d320f98f670e53c1e55a98bee70573e83f95beb128c" -dependencies = [ - "dyn-stack", - "gemm-common", - "num-complex", - "num-traits", - "paste", - "raw-cpuid", - "seq-macro", -] - -[[package]] -name = "gemm-c64" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20c8aeeeec425959bda4d9827664029ba1501a90a0d1e6228e48bef741db3a3f" -dependencies = [ - "dyn-stack", - "gemm-common", - "num-complex", - "num-traits", - "paste", - "raw-cpuid", - "seq-macro", -] - -[[package]] -name = "gemm-common" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88027625910cc9b1085aaaa1c4bc46bb3a36aad323452b33c25b5e4e7c8e2a3e" -dependencies = [ - "bytemuck", - "dyn-stack", - "half", - "libm", - "num-complex", - "num-traits", - "once_cell", - "paste", - "pulp", - "raw-cpuid", - "rayon", - "seq-macro", - "sysctl", -] - -[[package]] -name = "gemm-f16" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3df7a55202e6cd6739d82ae3399c8e0c7e1402859b30e4cb780e61525d9486e" -dependencies = [ - "dyn-stack", - "gemm-common", - "gemm-f32", - "half", - "num-complex", - "num-traits", - "paste", - "raw-cpuid", - "rayon", - "seq-macro", -] - -[[package]] -name = "gemm-f32" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02e0b8c9da1fbec6e3e3ab2ce6bc259ef18eb5f6f0d3e4edf54b75f9fd41a81c" -dependencies = [ - "dyn-stack", - "gemm-common", - "num-complex", - "num-traits", - "paste", - "raw-cpuid", - "seq-macro", -] - -[[package]] -name = "gemm-f64" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "056131e8f2a521bfab322f804ccd652520c79700d81209e9d9275bbdecaadc6a" -dependencies = [ - "dyn-stack", - "gemm-common", - "num-complex", - "num-traits", - "paste", - "raw-cpuid", - "seq-macro", -] - [[package]] name = "generic-array" version = "0.14.7" @@ -1249,29 +593,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "getrandom" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" -dependencies = [ - "cfg-if", - "libc", - "wasi", -] - -[[package]] -name = "getrandom" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" -dependencies = [ - "cfg-if", - "libc", - "r-efi 5.3.0", - "wasip2", -] - [[package]] name = "getrandom" version = "0.4.2" @@ -1280,21 +601,11 @@ checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" dependencies = [ "cfg-if", "libc", - "r-efi 6.0.0", + "r-efi", "wasip2", "wasip3", ] -[[package]] -name = "ghash" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" -dependencies = [ - "opaque-debug", - "polyval", -] - [[package]] name = "gimli" version = "0.33.0" @@ -1303,7 +614,7 @@ checksum = "0bf7f043f89559805f8c7cacc432749b2fa0d0a0a9ee46ce47164ed5ba7f126c" dependencies = [ "fnv", "hashbrown 0.16.1", - "indexmap 2.14.0", + "indexmap", "stable_deref_trait", ] @@ -1320,54 +631,6 @@ dependencies = [ "regex-syntax", ] -[[package]] -name = "group" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" -dependencies = [ - "ff", - "rand_core 0.6.4", - "subtle", -] - -[[package]] -name = "half" -version = "2.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" -dependencies = [ - "bytemuck", - "cfg-if", - "crunchy", - "num-traits", - "rand 0.9.4", - "rand_distr", - "zerocopy", -] - -[[package]] -name = "halo2_proofs" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05713f117155643ce10975e0bee44a274bcda2f4bb5ef29a999ad67c1fa8d4d3" -dependencies = [ - "blake2b_simd", - "ff", - "group", - "indexmap 1.9.3", - "maybe-rayon", - "pasta_curves", - "rand_core 0.6.4", - "tracing", -] - -[[package]] -name = "hashbrown" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" - [[package]] name = "hashbrown" version = "0.15.5" @@ -1397,6 +660,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" dependencies = [ "foldhash 0.2.0", + "serde", + "serde_core", ] [[package]] @@ -1405,12 +670,6 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" -[[package]] -name = "hermit-abi" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" - [[package]] name = "hex" version = "0.4.3" @@ -1423,22 +682,6 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" -[[package]] -name = "ident_case" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" - -[[package]] -name = "indexmap" -version = "1.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" -dependencies = [ - "autocfg", - "hashbrown 0.12.3", -] - [[package]] name = "indexmap" version = "2.14.0" @@ -1451,52 +694,12 @@ dependencies = [ "serde_core", ] -[[package]] -name = "indicatif" -version = "0.17.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235" -dependencies = [ - "console", - "number_prefix", - "portable-atomic", - "unicode-width", - "web-time", -] - -[[package]] -name = "inout" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" -dependencies = [ - "generic-array", -] - [[package]] name = "is_terminal_polyfill" version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" -[[package]] -name = "itertools" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" -dependencies = [ - "either", -] - -[[package]] -name = "itertools" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.14.0" @@ -1512,26 +715,11 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" -[[package]] -name = "js-sys" -version = "0.3.97" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1840c94c045fbcf8ba2812c95db44499f7c64910a912551aaaa541decebcacf" -dependencies = [ - "cfg-if", - "futures-util", - "once_cell", - "wasm-bindgen", -] - [[package]] name = "lazy_static" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" -dependencies = [ - "spin", -] [[package]] name = "leb128fmt" @@ -1585,7 +773,6 @@ dependencies = [ "lumen-zkml", "parking_lot", "postcard", - "rand 0.8.6", "serde", "serde_json", "tokio", @@ -1600,7 +787,7 @@ dependencies = [ "lumen-core", "serde", "serde_json", - "thiserror 2.0.18", + "thiserror", ] [[package]] @@ -1611,10 +798,9 @@ dependencies = [ "lumen-core", "parking_lot", "postcard", - "rand 0.8.6", "serde", "serde_json", - "thiserror 2.0.18", + "thiserror", "toml", "tracing", ] @@ -1623,17 +809,16 @@ dependencies = [ name = "lumen-channel" version = "0.4.1" dependencies = [ - "aes-gcm", + "aes", "async-trait", - "blake3", "lumen-attestation", "lumen-core", "postcard", - "rand 0.8.6", "serde", - "thiserror 2.0.18", + "thiserror", "tokio", - "x25519-dalek", + "x25519", + "zeroize", ] [[package]] @@ -1652,7 +837,6 @@ dependencies = [ "lumen-orchestrator", "lumen-provenance", "lumen-zkml", - "rand 0.8.6", "serde", "serde_json", "tokio", @@ -1665,16 +849,18 @@ dependencies = [ name = "lumen-core" version = "0.4.1" dependencies = [ - "blake3", - "ed25519-dalek", + "aes", + "blake", + "constant-time", + "ed25519", "hex", "postcard", - "rand 0.8.6", - "rand_core 0.6.4", + "rng", "serde", "serde_json", - "subtle", - "thiserror 2.0.18", + "thiserror", + "x25519", + "zeroize", ] [[package]] @@ -1703,10 +889,6 @@ name = "lumen-inference" version = "0.4.1" dependencies = [ "async-trait", - "blake3", - "candle-core", - "candle-onnx", - "candle-transformers", "futures", "lumen-channel", "lumen-core", @@ -1714,7 +896,6 @@ dependencies = [ "lumen-provenance", "serde", "serde_json", - "tokenizers 0.19.1", "tokio", "tracing", ] @@ -1727,7 +908,7 @@ dependencies = [ "serde", "serde_json", "tempfile", - "thiserror 2.0.18", + "thiserror", "tracing", ] @@ -1744,7 +925,6 @@ dependencies = [ "lumen-inference", "lumen-zkml", "parking_lot", - "rand 0.8.6", "serde", "serde_json", "tokio", @@ -1752,482 +932,45 @@ dependencies = [ ] [[package]] -name = "lumen-provenance" -version = "0.4.1" -dependencies = [ - "lumen-core", - "postcard", - "rand 0.8.6", - "safetensors", - "serde", - "serde_json", - "tempfile", - "thiserror 2.0.18", - "toml", - "tracing", -] - -[[package]] -name = "lumen-sandbox" -version = "0.4.1" -dependencies = [ - "lumen-capability", - "lumen-core", - "parking_lot", - "rand 0.8.6", - "serde", - "thiserror 2.0.18", - "tokio", - "tracing", - "wasmtime", - "wat", -] - -[[package]] -name = "lumen-sdk" -version = "0.4.1" -dependencies = [ - "lumen-sdk-macros", -] - -[[package]] -name = "lumen-sdk-macros" -version = "0.4.1" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "lumen-zkml" -version = "0.4.1" -dependencies = [ - "blake3", - "ff", - "halo2_proofs", - "lumen-core", - "pasta_curves", - "postcard", - "serde", - "serde_json", - "thiserror 2.0.18", - "tracing", -] - -[[package]] -name = "mach2" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d640282b302c0bb0a2a8e0233ead9035e3bed871f0b7e81fe4a1ec829765db44" -dependencies = [ - "libc", -] - -[[package]] -name = "macro_rules_attribute" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65049d7923698040cd0b1ddcced9b0eb14dd22c5f86ae59c3740eab64a676520" -dependencies = [ - "macro_rules_attribute-proc_macro", - "paste", -] - -[[package]] -name = "macro_rules_attribute-proc_macro" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "670fdfda89751bc4a84ac13eaa63e205cf0fd22b4c9a5fbfa085b63c1f1d3a30" - -[[package]] -name = "matchers" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" -dependencies = [ - "regex-automata", -] - -[[package]] -name = "maybe-rayon" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ea1f30cedd69f0a2954655f7188c6a834246d2bcf1e315e2ac40c4b24dc9519" -dependencies = [ - "cfg-if", - "rayon", -] - -[[package]] -name = "memchr" -version = "2.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" - -[[package]] -name = "memfd" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad38eb12aea514a0466ea40a80fd8cc83637065948eb4a426e4aa46261175227" -dependencies = [ - "rustix", -] - -[[package]] -name = "memmap2" -version = "0.9.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3" -dependencies = [ - "libc", - "stable_deref_trait", -] - -[[package]] -name = "minimal-lexical" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" - -[[package]] -name = "monostate" -version = "0.1.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3341a273f6c9d5bef1908f17b7267bbab0e95c9bf69a0d4dcf8e9e1b2c76ef67" -dependencies = [ - "monostate-impl", - "serde", - "serde_core", -] - -[[package]] -name = "monostate-impl" -version = "0.1.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4db6d5580af57bf992f59068d4ea26fd518574ff48d7639b255a36f9de6e7e9" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "multimap" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" - -[[package]] -name = "nom" -version = "7.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" -dependencies = [ - "memchr", - "minimal-lexical", -] - -[[package]] -name = "nu-ansi-term" -version = "0.50.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" -dependencies = [ - "windows-sys 0.61.2", -] - -[[package]] -name = "num-complex" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" -dependencies = [ - "bytemuck", - "num-traits", -] - -[[package]] -name = "num-traits" -version = "0.2.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" -dependencies = [ - "autocfg", - "libm", -] - -[[package]] -name = "num_cpus" -version = "1.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" -dependencies = [ - "hermit-abi", - "libc", -] - -[[package]] -name = "number_prefix" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" - -[[package]] -name = "object" -version = "0.39.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e5a6c098c7a3b6547378093f5cc30bc54fd361ce711e05293a5cc589562739b" -dependencies = [ - "crc32fast", - "hashbrown 0.17.0", - "indexmap 2.14.0", - "memchr", -] - -[[package]] -name = "once_cell" -version = "1.21.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" - -[[package]] -name = "once_cell_polyfill" -version = "1.70.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" - -[[package]] -name = "onig" -version = "6.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cc3cbf698f9438986c11a880c90a6d04b9de27575afd28bbf45b154b6c709e2" -dependencies = [ - "bitflags", - "libc", - "once_cell", - "onig_sys", -] - -[[package]] -name = "onig_sys" -version = "69.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e68317604e77e53b85896388e1a803c1d21b74c899ec9e5e1112db90735edd7" -dependencies = [ - "cc", - "pkg-config", -] - -[[package]] -name = "opaque-debug" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" - -[[package]] -name = "parking_lot" -version = "0.12.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" -dependencies = [ - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.9.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall", - "smallvec", - "windows-link", -] - -[[package]] -name = "pasta_curves" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3e57598f73cc7e1b2ac63c79c517b31a0877cd7c402cdcaa311b5208de7a095" -dependencies = [ - "blake2b_simd", - "ff", - "group", - "lazy_static", - "rand 0.8.6", - "static_assertions", - "subtle", -] - -[[package]] -name = "paste" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" - -[[package]] -name = "petgraph" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" -dependencies = [ - "fixedbitset", - "hashbrown 0.15.5", - "indexmap 2.14.0", -] - -[[package]] -name = "pin-project-lite" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" - -[[package]] -name = "pkcs8" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" -dependencies = [ - "der", - "spki", -] - -[[package]] -name = "pkg-config" -version = "0.3.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" - -[[package]] -name = "polyval" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" -dependencies = [ - "cfg-if", - "cpufeatures 0.2.17", - "opaque-debug", - "universal-hash", -] - -[[package]] -name = "portable-atomic" -version = "1.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" - -[[package]] -name = "postcard" -version = "1.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6764c3b5dd454e283a30e6dfe78e9b31096d9e32036b5d1eaac7a6119ccb9a24" -dependencies = [ - "cobs", - "embedded-io 0.4.0", - "embedded-io 0.6.1", - "serde", -] - -[[package]] -name = "ppv-lite86" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" -dependencies = [ - "zerocopy", -] - -[[package]] -name = "prettyplease" -version = "0.2.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" -dependencies = [ - "proc-macro2", - "syn", -] - -[[package]] -name = "proc-macro2" -version = "1.0.106" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "prost" -version = "0.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" -dependencies = [ - "bytes", - "prost-derive", -] - -[[package]] -name = "prost-build" -version = "0.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" -dependencies = [ - "heck", - "itertools 0.14.0", - "log", - "multimap", - "petgraph", - "prettyplease", - "prost", - "prost-types", - "regex", - "syn", - "tempfile", -] - -[[package]] -name = "prost-derive" -version = "0.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" +name = "lumen-provenance" +version = "0.4.1" dependencies = [ - "anyhow", - "itertools 0.14.0", - "proc-macro2", - "quote", - "syn", + "lumen-core", + "postcard", + "safetensors", + "serde", + "serde_json", + "tempfile", + "thiserror", + "toml", + "tracing", ] [[package]] -name = "prost-types" -version = "0.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" +name = "lumen-sandbox" +version = "0.4.1" dependencies = [ - "prost", + "lumen-capability", + "lumen-core", + "parking_lot", + "serde", + "thiserror", + "tokio", + "tracing", + "wasmtime", + "wat", ] [[package]] -name = "pulley-interpreter" -version = "44.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9326e3a0093d170582cf64ed9e4cf253b8aac155ec4a294ff62330450bbf094" +name = "lumen-sdk" +version = "0.4.1" dependencies = [ - "cranelift-bitset", - "log", - "pulley-macros", - "wasmtime-internal-core", + "lumen-sdk-macros", ] [[package]] -name = "pulley-macros" -version = "44.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00c6433917e3789605b1f4cd2a589f637ff17212344e7fa5ba99544625ba52c7" +name = "lumen-sdk-macros" +version = "0.4.1" dependencies = [ "proc-macro2", "quote", @@ -2235,180 +978,178 @@ dependencies = [ ] [[package]] -name = "pulp" -version = "0.22.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e205bb30d5b916c55e584c22201771bcf2bad9aabd5d4127f38387140c38632" +name = "lumen-zkml" +version = "0.4.1" dependencies = [ - "bytemuck", - "cfg-if", - "libm", - "num-complex", - "paste", - "pulp-wasm-simd-flag", - "raw-cpuid", - "reborrow", - "version_check", + "blake", + "lumen-core", + "postcard", + "serde", + "serde_json", + "thiserror", + "tracing", ] [[package]] -name = "pulp-wasm-simd-flag" -version = "0.1.0" +name = "mach2" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40e24eee682d89fb193496edf918a7f407d30175b2e785fe057e4392dfd182e0" +checksum = "dae608c151f68243f2b000364e1f7b186d9c29845f7d2d85bd31b9ad77ad552b" [[package]] -name = "quote" -version = "1.0.45" +name = "matchers" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" dependencies = [ - "proc-macro2", + "regex-automata", ] [[package]] -name = "r-efi" -version = "5.3.0" +name = "memchr" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" [[package]] -name = "r-efi" -version = "6.0.0" +name = "memfd" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +checksum = "ad38eb12aea514a0466ea40a80fd8cc83637065948eb4a426e4aa46261175227" +dependencies = [ + "rustix", +] [[package]] -name = "radium" -version = "0.7.0" +name = "nu-ansi-term" +version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys", +] [[package]] -name = "rand" -version = "0.8.6" +name = "object" +version = "0.39.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" +checksum = "2e5a6c098c7a3b6547378093f5cc30bc54fd361ce711e05293a5cc589562739b" dependencies = [ - "libc", - "rand_chacha 0.3.1", - "rand_core 0.6.4", + "crc32fast", + "hashbrown 0.17.0", + "indexmap", + "memchr", ] [[package]] -name = "rand" -version = "0.9.4" +name = "once_cell" +version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" -dependencies = [ - "rand_chacha 0.9.0", - "rand_core 0.9.5", -] +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] -name = "rand_chacha" -version = "0.3.1" +name = "once_cell_polyfill" +version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", - "rand_core 0.6.4", -] +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" [[package]] -name = "rand_chacha" -version = "0.9.0" +name = "parking_lot" +version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" dependencies = [ - "ppv-lite86", - "rand_core 0.9.5", + "lock_api", + "parking_lot_core", ] [[package]] -name = "rand_core" -version = "0.6.4" +name = "parking_lot_core" +version = "0.9.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" dependencies = [ - "getrandom 0.2.17", + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", ] [[package]] -name = "rand_core" -version = "0.9.5" +name = "pin-project-lite" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" -dependencies = [ - "getrandom 0.3.4", -] +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" [[package]] -name = "rand_distr" -version = "0.5.1" +name = "postcard" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" +checksum = "6764c3b5dd454e283a30e6dfe78e9b31096d9e32036b5d1eaac7a6119ccb9a24" dependencies = [ - "num-traits", - "rand 0.9.4", + "cobs", + "embedded-io 0.4.0", + "embedded-io 0.6.1", + "serde", ] [[package]] -name = "raw-cpuid" -version = "11.6.0" +name = "prettyplease" +version = "0.2.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ - "bitflags", + "proc-macro2", + "syn", ] [[package]] -name = "rayon" -version = "1.12.0" +name = "proc-macro2" +version = "1.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" dependencies = [ - "either", - "rayon-core", + "unicode-ident", ] [[package]] -name = "rayon-cond" -version = "0.3.0" +name = "pulley-interpreter" +version = "48.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "059f538b55efd2309c9794130bc149c6a553db90e9d99c2030785c82f0bd7df9" +checksum = "6e95d890afdd44c1269bdc84e9f043c3a6c02e8bfd1781d64468e62f05880c83" dependencies = [ - "either", - "itertools 0.11.0", - "rayon", + "cranelift-bitset", + "log", + "pulley-macros", + "wasmtime-internal-core", ] [[package]] -name = "rayon-cond" -version = "0.4.0" +name = "pulley-macros" +version = "48.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f" +checksum = "550c64f4245f013664d187a37ec94b4e99bd43a00980134e0e5fd48896b1d3af" dependencies = [ - "either", - "itertools 0.14.0", - "rayon", + "proc-macro2", + "quote", + "syn", ] [[package]] -name = "rayon-core" -version = "1.13.0" +name = "quote" +version = "1.0.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" dependencies = [ - "crossbeam-deque", - "crossbeam-utils", + "proc-macro2", ] [[package]] -name = "reborrow" -version = "0.5.5" +name = "r-efi" +version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03251193000f4bd3b042892be858ee50e8b3719f2b08e5833ac4353724632430" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "redox_syscall" @@ -2421,15 +1162,16 @@ dependencies = [ [[package]] name = "regalloc2" -version = "0.15.1" +version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de2c52737737f8609e94f975dee22854a2d5c125772d4b1cf292120f4d45c186" +checksum = "757712e8e61590d6d4f5d563483755538b5aa13467837a3b41cd9832509a7f85" dependencies = [ "allocator-api2", "bumpalo", "hashbrown 0.17.0", "log", "rustc-hash", + "serde", "smallvec", ] @@ -2462,6 +1204,14 @@ version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" +[[package]] +name = "rng" +version = "1.0.0" +dependencies = [ + "sha2 1.0.0", + "zeroize", +] + [[package]] name = "rustc-demangle" version = "0.1.27" @@ -2474,15 +1224,6 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" -[[package]] -name = "rustc_version" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" -dependencies = [ - "semver", -] - [[package]] name = "rustix" version = "1.1.4" @@ -2493,21 +1234,9 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys", ] -[[package]] -name = "rustversion" -version = "1.0.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" - -[[package]] -name = "ryu" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" - [[package]] name = "safetensors" version = "0.7.0" @@ -2519,15 +1248,6 @@ dependencies = [ "serde_json", ] -[[package]] -name = "same-file" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" -dependencies = [ - "winapi-util", -] - [[package]] name = "scopeguard" version = "1.2.0" @@ -2539,12 +1259,10 @@ name = "semver" version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" - -[[package]] -name = "seq-macro" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" +dependencies = [ + "serde", + "serde_core", +] [[package]] name = "serde" @@ -2589,15 +1307,6 @@ dependencies = [ "zmij", ] -[[package]] -name = "serde_plain" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ce1fc6db65a611022b23a0dec6975d63fb80a302cb3388835ff02c097258d50" -dependencies = [ - "serde", -] - [[package]] name = "serde_spanned" version = "0.6.9" @@ -2614,10 +1323,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures 0.2.17", + "cpufeatures", "digest", ] +[[package]] +name = "sha2" +version = "1.0.0" +dependencies = [ + "constant-time", + "zeroize", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -2633,15 +1350,6 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" -[[package]] -name = "signature" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" -dependencies = [ - "rand_core 0.6.4", -] - [[package]] name = "slab" version = "0.4.12" @@ -2657,58 +1365,18 @@ dependencies = [ "serde", ] -[[package]] -name = "spin" -version = "0.9.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" - -[[package]] -name = "spki" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" -dependencies = [ - "base64ct", - "der", -] - -[[package]] -name = "spm_precompiled" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326" -dependencies = [ - "base64", - "nom", - "serde", - "unicode-segmentation", -] - [[package]] name = "stable_deref_trait" version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" -[[package]] -name = "static_assertions" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" - [[package]] name = "strsim" version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" -[[package]] -name = "subtle" -version = "2.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" - [[package]] name = "syn" version = "2.0.117" @@ -2720,37 +1388,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "synstructure" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "sysctl" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01198a2debb237c62b6826ec7081082d951f46dbb64b0e8c7649a452230d1dfc" -dependencies = [ - "bitflags", - "byteorder", - "enum-as-inner", - "libc", - "thiserror 1.0.69", - "walkdir", -] - -[[package]] -name = "tap" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" - [[package]] name = "target-lexicon" version = "0.13.5" @@ -2764,28 +1401,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.2", + "getrandom", "once_cell", "rustix", - "windows-sys 0.61.2", -] - -[[package]] -name = "termcolor" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" -dependencies = [ - "winapi-util", + "windows-sys", ] [[package]] -name = "thiserror" -version = "1.0.69" +name = "termcolor" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" dependencies = [ - "thiserror-impl 1.0.69", + "winapi-util", ] [[package]] @@ -2794,18 +1422,7 @@ version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" dependencies = [ - "thiserror-impl 2.0.18", -] - -[[package]] -name = "thiserror-impl" -version = "1.0.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" -dependencies = [ - "proc-macro2", - "quote", - "syn", + "thiserror-impl", ] [[package]] @@ -2828,71 +1445,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "tokenizers" -version = "0.19.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e500fad1dd3af3d626327e6a3fe5050e664a6eaa4708b8ca92f1794aaf73e6fd" -dependencies = [ - "aho-corasick", - "derive_builder", - "esaxx-rs", - "getrandom 0.2.17", - "indicatif", - "itertools 0.12.1", - "lazy_static", - "log", - "macro_rules_attribute", - "monostate", - "onig", - "paste", - "rand 0.8.6", - "rayon", - "rayon-cond 0.3.0", - "regex", - "regex-syntax", - "serde", - "serde_json", - "spm_precompiled", - "thiserror 1.0.69", - "unicode-normalization-alignments", - "unicode-segmentation", - "unicode_categories", -] - -[[package]] -name = "tokenizers" -version = "0.22.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b238e22d44a15349529690fb07bd645cf58149a1b1e44d6cb5bd1641ff1a6223" -dependencies = [ - "ahash", - "aho-corasick", - "compact_str", - "dary_heap", - "derive_builder", - "esaxx-rs", - "getrandom 0.3.4", - "itertools 0.14.0", - "log", - "macro_rules_attribute", - "monostate", - "onig", - "paste", - "rand 0.9.4", - "rayon", - "rayon-cond 0.4.0", - "regex", - "regex-syntax", - "serde", - "serde_json", - "spm_precompiled", - "thiserror 2.0.18", - "unicode-normalization-alignments", - "unicode-segmentation", - "unicode_categories", -] - [[package]] name = "tokio" version = "1.52.2" @@ -2942,7 +1494,7 @@ version = "0.22.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" dependencies = [ - "indexmap 2.14.0", + "indexmap", "serde", "serde_spanned", "toml_datetime", @@ -3017,12 +1569,6 @@ dependencies = [ "tracing-log", ] -[[package]] -name = "typed-path" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e" - [[package]] name = "typenum" version = "1.20.0" @@ -3035,21 +1581,6 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" -[[package]] -name = "unicode-normalization-alignments" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de" -dependencies = [ - "smallvec", -] - -[[package]] -name = "unicode-segmentation" -version = "1.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c" - [[package]] name = "unicode-width" version = "0.2.2" @@ -3062,22 +1593,6 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" -[[package]] -name = "unicode_categories" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" - -[[package]] -name = "universal-hash" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" -dependencies = [ - "crypto-common", - "subtle", -] - [[package]] name = "utf8parse" version = "0.2.2" @@ -3096,22 +1611,6 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" -[[package]] -name = "walkdir" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" -dependencies = [ - "same-file", - "winapi-util", -] - -[[package]] -name = "wasi" -version = "0.11.1+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" - [[package]] name = "wasip2" version = "1.0.3+wasi-0.2.9" @@ -3130,51 +1629,6 @@ dependencies = [ "wit-bindgen 0.51.0", ] -[[package]] -name = "wasm-bindgen" -version = "0.2.120" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df52b6d9b87e0c74c9edfa1eb2d9bf85e5d63515474513aa50fa181b3c4f5db1" -dependencies = [ - "cfg-if", - "once_cell", - "rustversion", - "wasm-bindgen-macro", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-macro" -version = "0.2.120" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78b1041f495fb322e64aca85f5756b2172e35cd459376e67f2a6c9dffcedb103" -dependencies = [ - "quote", - "wasm-bindgen-macro-support", -] - -[[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.120" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dcd0ff20416988a18ac686d4d4d0f6aae9ebf08a389ff5d29012b05af2a1b41" -dependencies = [ - "bumpalo", - "proc-macro2", - "quote", - "syn", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-shared" -version = "0.2.120" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49757b3c82ebf16c57d69365a142940b384176c24df52a087fb748e2085359ea" -dependencies = [ - "unicode-ident", -] - [[package]] name = "wasm-encoder" version = "0.244.0" @@ -3187,22 +1641,22 @@ dependencies = [ [[package]] name = "wasm-encoder" -version = "0.246.2" +version = "0.248.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61fb705ce81adde29d2a8e99d87995e39a6e927358c91398f374474746070ef7" +checksum = "ac92cf547bc18d27ecc521015c08c353b4f18b84ab388bb6d1b6b682c620d9b6" dependencies = [ "leb128fmt", - "wasmparser 0.246.2", + "wasmparser 0.248.0", ] [[package]] name = "wasm-encoder" -version = "0.248.0" +version = "0.254.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac92cf547bc18d27ecc521015c08c353b4f18b84ab388bb6d1b6b682c620d9b6" +checksum = "09480d646178e5fdd12bb06e812d0af9a3a191dbc9cd697fdc86687beade7393" dependencies = [ "leb128fmt", - "wasmparser 0.248.0", + "wasmparser 0.254.0", ] [[package]] @@ -3212,11 +1666,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" dependencies = [ "anyhow", - "indexmap 2.14.0", + "indexmap", "wasm-encoder 0.244.0", "wasmparser 0.244.0", ] +[[package]] +name = "wasm-metadata" +version = "0.254.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b01df5f3b4ca7881e843f3bc0fb8a3905d79c68692250dcb8e33e698705ccdb6" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder 0.254.0", + "wasmparser 0.254.0", +] + [[package]] name = "wasmparser" version = "0.244.0" @@ -3225,57 +1691,57 @@ checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" dependencies = [ "bitflags", "hashbrown 0.15.5", - "indexmap 2.14.0", + "indexmap", "semver", ] [[package]] name = "wasmparser" -version = "0.246.2" +version = "0.248.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71cde4757396defafd25417cfb36aa3161027d06d865b0c24baaae229aac005d" +checksum = "aa4439c5eee9df71ee0c6efb37f63b1fcb1fec38f85f5142c54e7ed05d33091a" dependencies = [ "bitflags", - "hashbrown 0.16.1", - "indexmap 2.14.0", + "indexmap", "semver", - "serde", ] [[package]] name = "wasmparser" -version = "0.248.0" +version = "0.254.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa4439c5eee9df71ee0c6efb37f63b1fcb1fec38f85f5142c54e7ed05d33091a" +checksum = "d5769a29f799fbab136aaf65b4fe5384cd7d93fe6fc9ba0dcb6c8382a1f16e27" dependencies = [ "bitflags", - "indexmap 2.14.0", + "hashbrown 0.17.0", + "indexmap", "semver", + "serde", ] [[package]] name = "wasmprinter" -version = "0.246.2" +version = "0.254.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e41f7493ba994b8a779430a4c25ff550fd5a40d291693af43a6ef48688f00e3" +checksum = "64e3ba11e024f504698f87b629b2b66c22a1231758de7607754963f7d198be39" dependencies = [ "anyhow", "termcolor", - "wasmparser 0.246.2", + "wasmparser 0.254.0", ] [[package]] name = "wasmtime" -version = "44.0.1" +version = "48.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "372db8bbad8ec962038101f75ab2c3ffcd18797d7d3ae877a58ab9873cd0c4bd" +checksum = "699c2cb832d70d284b2839ebeb12721ca2ef80aa4a3389d2d2d13acf3e19c821" dependencies = [ "addr2line", "async-trait", "bitflags", "bumpalo", "cc", - "cfg-if", + "futures", "gimli", "libc", "log", @@ -3290,7 +1756,7 @@ dependencies = [ "serde_derive", "smallvec", "target-lexicon", - "wasmparser 0.246.2", + "wasmparser 0.254.0", "wasmtime-environ", "wasmtime-internal-component-macro", "wasmtime-internal-core", @@ -3300,14 +1766,14 @@ dependencies = [ "wasmtime-internal-jit-icache-coherence", "wasmtime-internal-unwinder", "wasmtime-internal-versioned-export-macros", - "windows-sys 0.61.2", + "windows-sys", ] [[package]] name = "wasmtime-environ" -version = "44.0.1" +version = "48.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e15aa0d1545e48d9b25ca604e9e27b4cd6d5886d30ac5787b57b3a2daf85b57" +checksum = "13f10ef166b4850535bc43c347797364cda5189870be163a617ebe5c0d72f139" dependencies = [ "anyhow", "cpp_demangle", @@ -3315,28 +1781,30 @@ dependencies = [ "cranelift-bitset", "cranelift-entity", "gimli", - "hashbrown 0.16.1", - "indexmap 2.14.0", + "hashbrown 0.17.0", + "indexmap", "log", "object", "postcard", "rustc-demangle", + "semver", "serde", "serde_derive", - "sha2", + "sha2 0.10.9", "smallvec", "target-lexicon", - "wasm-encoder 0.246.2", - "wasmparser 0.246.2", + "wasm-encoder 0.254.0", + "wasmparser 0.254.0", "wasmprinter", + "wasmtime-internal-component-util", "wasmtime-internal-core", ] [[package]] name = "wasmtime-internal-component-macro" -version = "44.0.1" +version = "48.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c136cb0d2d47850d6d04a58157130ac98b0df4c17626cd30b083d26b607b7027" +checksum = "5af067a5d17e17a424f99d31f08e08c5132dcc53a0f031a7be9279719a936b71" dependencies = [ "anyhow", "proc-macro2", @@ -3344,47 +1812,46 @@ dependencies = [ "syn", "wasmtime-internal-component-util", "wasmtime-internal-wit-bindgen", - "wit-parser 0.246.2", + "wit-parser 0.254.0", ] [[package]] name = "wasmtime-internal-component-util" -version = "44.0.1" +version = "48.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49df3d3b4fa2119c6fd161e475b4e21aaefb51d082353b922b433bea37facc65" +checksum = "540b7151342d75c3d596447c136d3862a280600bc6fd10b5dfc7c9e67c2f719f" [[package]] name = "wasmtime-internal-core" -version = "44.0.1" +version = "48.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f2c7fa6523647262bfb4095dbdf4087accefe525813e783f81a0c682f418ce4" +checksum = "8679b612a408274d91d1e0edf6baa36b63cc48d937f84dd1e7fb2fbe16a907ae" dependencies = [ - "hashbrown 0.16.1", + "hashbrown 0.17.0", "libm", "serde", ] [[package]] name = "wasmtime-internal-cranelift" -version = "44.0.1" +version = "48.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98c032f422e39061dfc43f32190c0a3526b04161ec4867f362958f3fe9d1fe29" +checksum = "fc46c460fb62d839ac441ccdf4b8b94bce30842ad4ccefd7744d9e28bf631232" dependencies = [ - "cfg-if", "cranelift-codegen", "cranelift-control", "cranelift-entity", "cranelift-frontend", "cranelift-native", "gimli", - "itertools 0.14.0", + "itertools", "log", "object", "pulley-interpreter", "smallvec", "target-lexicon", - "thiserror 2.0.18", - "wasmparser 0.246.2", + "thiserror", + "wasmparser 0.254.0", "wasmtime-environ", "wasmtime-internal-core", "wasmtime-internal-unwinder", @@ -3393,24 +1860,23 @@ dependencies = [ [[package]] name = "wasmtime-internal-fiber" -version = "44.0.1" +version = "48.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8dd76d80adf450cc260ba58f23c28030401930b19149695b1d121f7d621e791" +checksum = "31a94596377490aa56f5c58ebbcc39af49176dd9d4051a78e3fca3c024daf4f6" dependencies = [ "cc", - "cfg-if", "libc", "rustix", "wasmtime-environ", "wasmtime-internal-versioned-export-macros", - "windows-sys 0.61.2", + "windows-sys", ] [[package]] name = "wasmtime-internal-jit-debug" -version = "44.0.1" +version = "48.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab453cc600b28ee5d3f9495aa6d4cb2c81eda40903e9287296b548fba8b2391d" +checksum = "70026f001b46af7552ce89cfe748f0dea59097e7c8c3740998d8019b73d87ee5" dependencies = [ "cc", "wasmtime-internal-versioned-export-macros", @@ -3418,23 +1884,21 @@ dependencies = [ [[package]] name = "wasmtime-internal-jit-icache-coherence" -version = "44.0.1" +version = "48.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a1859e920871515d324fb9757c3e448d6ed1512ca6ccdff14b6e016505d6ada" +checksum = "bf7d8a6fc6d21eb177c38f096636661ca13ae924cccbb6f447c1498ca972ab10" dependencies = [ - "cfg-if", "libc", "wasmtime-internal-core", - "windows-sys 0.61.2", + "windows-sys", ] [[package]] name = "wasmtime-internal-unwinder" -version = "44.0.1" +version = "48.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1dfe405bd6adb1386d935a30f16a236bd4ef0d3c383e7cbbab98d063c9d9b73" +checksum = "4994aca882a65d2b06a7fc57a4a149de27b8851738d07ea2e778a21f74234a14" dependencies = [ - "cfg-if", "cranelift-codegen", "log", "object", @@ -3443,9 +1907,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-versioned-export-macros" -version = "44.0.1" +version = "48.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a9b9165fc45d42c81edfe3e9cb458e58720594ad5db6553c4079ea041a4a581" +checksum = "8f325614219c58c9f625b591ac79dbbefc76728720d72a721a6350c872d848e4" dependencies = [ "proc-macro2", "quote", @@ -3454,15 +1918,16 @@ dependencies = [ [[package]] name = "wasmtime-internal-wit-bindgen" -version = "44.0.1" +version = "48.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17c7ced16dc16d2027f9f8d3a503e191dcce0f53fe9218e7990135b31f8f6fdb" +checksum = "4ec33585b420181c41e26ac6153f0d18aa200e6aba9457dff6f8ebba3feedf98" dependencies = [ "anyhow", "bitflags", "heck", - "indexmap 2.14.0", - "wit-parser 0.246.2", + "indexmap", + "wit-component 0.254.0", + "wit-parser 0.254.0", ] [[package]] @@ -3487,23 +1952,13 @@ dependencies = [ "wast", ] -[[package]] -name = "web-time" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - [[package]] name = "winapi-util" version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys", ] [[package]] @@ -3512,15 +1967,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" -[[package]] -name = "windows-sys" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" -dependencies = [ - "windows-targets", -] - [[package]] name = "windows-sys" version = "0.61.2" @@ -3530,70 +1976,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "windows-targets" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" -dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" - -[[package]] -name = "windows_i686_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" - -[[package]] -name = "windows_i686_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" - -[[package]] -name = "windows_i686_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" - [[package]] name = "winnow" version = "0.7.15" @@ -3637,12 +2019,12 @@ checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" dependencies = [ "anyhow", "heck", - "indexmap 2.14.0", + "indexmap", "prettyplease", "syn", - "wasm-metadata", + "wasm-metadata 0.244.0", "wit-bindgen-core", - "wit-component", + "wit-component 0.244.0", ] [[package]] @@ -3668,17 +2050,36 @@ checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" dependencies = [ "anyhow", "bitflags", - "indexmap 2.14.0", + "indexmap", "log", "serde", "serde_derive", "serde_json", "wasm-encoder 0.244.0", - "wasm-metadata", + "wasm-metadata 0.244.0", "wasmparser 0.244.0", "wit-parser 0.244.0", ] +[[package]] +name = "wit-component" +version = "0.254.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0e65bb94c369b3c4741ce3d1d2704b1fec93db7c540df0e521a097e7ceeb5be" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder 0.254.0", + "wasm-metadata 0.254.0", + "wasmparser 0.254.0", + "wit-parser 0.254.0", +] + [[package]] name = "wit-parser" version = "0.244.0" @@ -3687,7 +2088,7 @@ checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" dependencies = [ "anyhow", "id-arena", - "indexmap 2.14.0", + "indexmap", "log", "semver", "serde", @@ -3699,139 +2100,34 @@ dependencies = [ [[package]] name = "wit-parser" -version = "0.246.2" +version = "0.254.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd979042b5ff288607ccf3b314145435453f20fc67173195f91062d2289b204d" +checksum = "1655131e4f7d3f0cb141f6eca71315ca40eff0f3d4de7cff0a82bacedd8c89b4" dependencies = [ "anyhow", - "hashbrown 0.16.1", + "hashbrown 0.17.0", "id-arena", - "indexmap 2.14.0", + "indexmap", "log", "semver", "serde", "serde_derive", "serde_json", - "unicode-xid", - "wasmparser 0.246.2", -] - -[[package]] -name = "wyz" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" -dependencies = [ - "tap", + "unicode-ident", + "wasmparser 0.254.0", ] [[package]] -name = "x25519-dalek" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7e468321c81fb07fa7f4c636c3972b9100f0346e5b6a9f2bd0603a52f7ed277" +name = "x25519" +version = "1.0.0" dependencies = [ - "curve25519-dalek", - "rand_core 0.6.4", - "serde", + "constant-time", "zeroize", ] -[[package]] -name = "yoke" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" -dependencies = [ - "stable_deref_trait", - "yoke-derive", - "zerofrom", -] - -[[package]] -name = "yoke-derive" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "synstructure", -] - -[[package]] -name = "zerocopy" -version = "0.8.48" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" -dependencies = [ - "zerocopy-derive", -] - -[[package]] -name = "zerocopy-derive" -version = "0.8.48" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "zerofrom" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69faa1f2a1ea75661980b013019ed6687ed0e83d069bc1114e2cc74c6c04c4df" -dependencies = [ - "zerofrom-derive", -] - -[[package]] -name = "zerofrom-derive" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "synstructure", -] - [[package]] name = "zeroize" -version = "1.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" -dependencies = [ - "zeroize_derive", -] - -[[package]] -name = "zeroize_derive" -version = "1.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "zip" -version = "7.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c42e33efc22a0650c311c2ef19115ce232583abbe80850bc8b66509ebef02de0" -dependencies = [ - "crc32fast", - "indexmap 2.14.0", - "memchr", - "typed-path", -] +version = "1.0.0" [[package]] name = "zmij" diff --git a/Cargo.toml b/Cargo.toml index 9bb301b..3d7724e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,12 +51,14 @@ toml = "0.8" postcard = { version = "1", default-features = false, features = ["use-std"] } hex = "0.4" -# Crypto -blake3 = "1" -ed25519-dalek = { version = "2", features = ["rand_core"] } -rand = "0.8" -rand_core = "0.6" -subtle = "2" +# Crypto +elib-blake = { package = "blake", path = "../elib-k0-nt/blake", version = "1.0.0" } +elib-ed25519 = { package = "ed25519", path = "../elib-k0-nt/ed25519", version = "1.0.0" } +elib-x25519 = { package = "x25519", path = "../elib-k0-nt/x25519", version = "1.0.0" } +elib-aes = { package = "aes", path = "../elib-k0-nt/aes", version = "1.0.0" } +elib-rng = { package = "rng", path = "../elib-k0-nt/rng", version = "1.0.0" } +elib-constant-time = { package = "constant-time", path = "../elib-k0-nt/constant-time", version = "1.0.0" } +elib-zeroize = { package = "zeroize", path = "../elib-k0-nt/zeroize", version = "1.0.0" } # Defense / matchers aho-corasick = "1" @@ -68,12 +70,13 @@ globset = "0.4" parking_lot = "0.12" # WASM (sandbox) -# v25 → 최신 line. RUSTSEC 다수 advisory (fd_renumber / wasi:http fields / +# v25 -> 최신 line. RUSTSEC 다수 advisory (fd_renumber / wasi:http fields / # 컴포넌트 model flags lift / Winch table.fill / UTF-16 transcoding panic) 패치 포함. -wasmtime = { version = "44", default-features = false, features = ["async", "cranelift", "runtime", "addr2line", "demangle"] } +wasmtime = { version = "48", default-features = false, features = ["async", "cranelift", "runtime", "addr2line", "demangle"] } wat = "1" -# Provenance — candle-onnx 가 0.7 을 끌어오므로 동일 메이저로 정렬해 dup 회피. +# Provenance — Hugging Face safetensors 헤더 sniff 용. 단독으로 동작하는 +# 작은 크레이트라 폐쇄망 빌드에 무리 없이 포함됩니다. safetensors = "0.7" # CLI @@ -83,28 +86,35 @@ clap = { version = "4", features = ["derive"] } tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["fmt", "env-filter"] } -# Optional crypto-channel feature -aes-gcm = "0.10" -x25519-dalek = { version = "2", features = ["static_secrets"] } +# Q. T. Felix NOTE: AES-GCM / x25519는 elib-k0-nt 모듈을 사용하므로 별도 외부 +# 크레이트가 필요하지 않음(위 elib-aes / elib-x25519 워크스페이스 정의 사용) # Dev / bench criterion = "0.5" -# halo2 (ZK 회로) - 매우 무거운 의존성. feature 뒤에서만 활성화. -halo2_proofs = "0.3" -ff = "0.13" -pasta_curves = "0.5" - -# candle (LLM 추론) - feature 뒤에서만 활성화. CPU 전용 빌드. -candle-core = { version = "0.10", default-features = false } -candle-onnx = { version = "0.10", default-features = false } -# candle-transformers: 실제 LLM 아키텍처 (LLaMA, Mistral, GPT-2 등). CPU 전용. -candle-transformers = { version = "0.10", default-features = false } -# HuggingFace 토크나이저 (tokenizer.json 형식). -# onig 는 C 소스에서 빌드되므로 시스템 라이브러리 불필요. -tokenizers = { version = "0.19" } +# 참고: 다음 의존성은 폐쇄형(Air-Gapped) 호환성 작업으로 제거되었습니다. +# - `halo2_proofs` / `ff` / `pasta_curves` : ZK 백엔드. mock prover 가 +# 이미 BLAKE3 기반이라 halo2 회로 없이 동작합니다. 실제 succinct ZK +# 백엔드 (SP1 / RISC Zero 등) 는 마일스톤 재검토 시 결정. +# - `candle-core` / `candle-onnx` / `candle-transformers` / `tokenizers` +# : GGUF·ONNX 추론 백엔드. 의존 트리가 매우 크고 (`gemm`, `pulp`, +# `paste`, `tokenizers`, `rand_core` 등) HF Hub 다운로드 코드를 +# 포함하므로 폐쇄망에 부적합. 도구 라우팅용 토크나이저는 자체 구현인 +# `lumen_inference::BpeTokenizer` 를 사용하고, 전체 LLM 생성은 +# `llama-cpp` 백엔드 또는 호스트 TEE 의 ChannelEngine forward 경로를 +# 사용합니다. +# - `rand` / `rand_core` : `lumen_core::rng` (HashDRBG 어댑터) 로 대체. [workspace.lints.rust] +# Q. T. Felix NOTE: 모든 경고를 deny 로 격상해 lumen 멤버 크레이트의 게이트로 +# 사용합니다. 과거에는 CI의 `RUSTFLAGS=-D warnings`로 같은 +# 효과를 냈으나, 그 방식은 워크스페이스 외부 path dep +# (`elib-k0-nt/*`) 까지 격상시켜 외부 변경에 lumen CI가 +# 끌려갔습니다. lints 테이블은 `[lints] workspace = true` +# 를 선언한 멤버에만 적용되므로 외부 path dep에 영향을 +# 주지 않습니다. priority -2로 가장 먼저 emit시켜 이후 +# 특정 lint(warn 레벨)가 자유롭게 덮어쓸 수 있도록 합니다. +warnings = { level = "deny", priority = -2 } unsafe_code = "deny" missing_docs = "warn" unreachable_pub = "warn" diff --git a/INTRODUCTION.md b/INTRODUCTION.md index 11feb10..0f4c369 100644 --- a/INTRODUCTION.md +++ b/INTRODUCTION.md @@ -34,7 +34,7 @@ Lumen의 모든 설계 결정을 관통하는 원칙은 다음과 같습니다. Lumen은 학술 발표나 폐쇄 솔루션이 아닌 **오픈소스 라이브러리**로 공개되어, 같은 문제를 마주한 사람들이 동일한 보안 기반을 공유할 수 있도록 합니다. [Apache-2.0](LICENSE-APACHE) 또는 [MIT](LICENSE-MIT) 듀얼 라이선스로 상업, 연구, 정부 사용 모두 가능합니다. -오픈소스인 이유는 단순합니다. 보안 인프라는 *감사 가능*해야 하고, 감사 가능하려면 코드가 공개되어 있어야 합니다. 폐쇄형 보안 솔루션의 신뢰성은 그것을 만든 회사에 대한 신뢰로 환원되며, 이는 제로 트러스트 철학과 정면 충돌합니다. 제로 트러스트으ㅢ 핵심 원칙은 "Never Trust, Always Verify"이며, 이는 내부 직원 뿐만 아니라 사용하는 도구와 공급망도 포함되기 때문입니다. +오픈소스인 이유는 단순합니다. 보안 인프라는 *감사 가능*해야 하고, 감사 가능하려면 코드가 공개되어 있어야 합니다. 폐쇄형 보안 솔루션의 신뢰성은 그것을 만든 회사에 대한 신뢰로 환원되며, 이는 제로 트러스트 철학과 정면 충돌합니다. 제로 트러스트의 핵심 원칙은 "Never Trust, Always Verify"이며, 이는 내부 직원 뿐만 아니라 사용하는 도구와 공급망도 포함되기 때문입니다. **Team Quant는 막강한 보안을 이상으로 생각하며, 언제든 오픈소스 생태계의 큰 발전을 위해 기여할겁니다.** diff --git a/KERNEL-COMPAT.md b/KERNEL-COMPAT.md new file mode 100644 index 0000000..91017ca --- /dev/null +++ b/KERNEL-COMPAT.md @@ -0,0 +1,187 @@ +# 폐쇄 커널 호환성 검증 보고서 (K0) + +> **목적**: Lumen 이 폐쇄형(Air-Gapped) `K0` 마이크로커널 환경에서 +> 정상 동작할 수 있는지 기술적으로 엄밀히 검증한 결과를 정리. + +--- + +## 1. 검증 방법론 + +세 단계 검증: + +| 단계 | 산출물 | 통과 기준 | +|------|--------|-----------| +| **(A) 정적 의존성 감사** | Cargo.lock + cargo build --message-format=json 추출 | 금지 크레이트 0건, 네트워크 스택 0건 | +| **(B) 와이어 호환성 회귀 테스트** | `crates/lumen-core/tests/elib_wire_compat.rs` (7 테스트) | lumen 래퍼 ≡ elib-k0-nt 직접 호출 (비트-동일) | +| **(C) 5-게이트 (offline 모드)** | `cargo build/test/clippy/fmt --check/deny check` 모두 `--offline --locked` | 모두 통과 | + +--- + +## 2. (A) 정적 의존성 감사 결과 + +### 2.1 lumen-cli (실제 운영 바이너리) 릴리스 빌드 + +| 항목 | 결과 | +|------|------| +| 워크스페이스 멤버 | 16 크레이트 | +| `lumen-cli` 릴리스 바이너리에 컴파일되는 외부 크레이트 | **102 개** | +| `wasmtime` / `cranelift` / `pulley` 포함 여부 | **❌ 0건** (lumen-sandbox 는 별도 라이브러리) | +| `getrandom` / `rand` / `rand_core` | **❌ 0건** (dev-only, 운영 바이너리에 없음) | +| `tokenizers` | **❌ 0건** | +| `reqwest` / `hyper` / `http` / `rustls` / `openssl` / `native-tls` | **❌ 0건** | +| `mio` / `socket2` (tokio 네트워크 백엔드) | **❌ 0건** (lumen 의 tokio feature 에 `net` 미포함) | +| `Cargo.lock` 의 `git+` source dep | **❌ 0건** (전부 crates.io 레지스트리) | +| `unsafe` 사용 | 13 개 lumen 크레이트가 `#![forbid(unsafe_code)]` 선언; 유일한 예외는 `lumen-sandbox` (wasmtime FFI) | + +### 2.2 lumen-sandbox (별도 옵션 컴포넌트) + +`lumen-sandbox` 는 **다른 어떤 lumen 크레이트도 의존하지 않는 독립 라이브러리**입니다. +즉, `lumen-cli` 기본 운영 바이너리에는 wasmtime 이 포함되지 않습니다. + +`grep -rn "lumen-sandbox" crates --include="Cargo.toml"` → 자기 자신 1건뿐. + +| 항목 | 결과 | +|------|------| +| Cranelift JIT 코드 생성 (W^X 위반 가능) | ✅ 포함 (현재 `cranelift` feature) | +| Pulley 인터프리터 (W^X 안전) | ✅ wasmtime 트리에 동시 존재 | +| 결정 | 운영 환경에 맞춰 `cranelift` 또는 `pulley` 중 선택 가능. 현재 기본은 Cranelift JIT (Ring 3 RWX 페이지 가정). 엄격한 W^X 정책 환경이라면 wasmtime feature 를 `["async","pulley","runtime"]` 으로 전환 필요 | + +### 2.3 런타임 시스템콜 표면 (lumen 운영 코드) + +`grep -rn "use std::fs|std::env|std::process|tokio::net|tokio::process"` 결과: + +| 사용처 | 시스템콜 | 폐쇄 커널 영향 | +|--------|----------|----------------| +| `lumen-core::hash::Blake3Hash::of_file` | `std::fs::File::open` | 모델 파일 검증; K0 가 Ring 3 에 fs 또는 IPC 전달을 제공하면 동작 | +| `lumen-provenance::*` | `std::path::Path` (경로 처리) + `std::fs` (파일 읽기) | 위와 동일 | +| `lumen-cli::cmd::run` 등 | `std::path::Path` | 경로 인자 처리 — 순수 메모리 연산 | +| `lumen-cli::cmd::verifier` | `std::env::var` (`privkey_env`, `fee_payer_env`) | **블록체인 배포용** (low-side 전용); 폐쇄망에서는 미사용 | +| `lumen-onchain::deploy` | `std::process::Command` (`forge` / `zk` 호출) | **블록체인 배포용**, `dry_run` 모드 제공 — 폐쇄망에서 텍스트 명령만 inspect 후 low-side 로 운반하도록 설계됨 | +| `tokio` features | `["rt-multi-thread", "macros", "sync", "time", "io-util", "fs"]` | `net` 미포함; `fs` / 멀티스레드는 Ring 3 ABI 가 제공해야 함 | +| 직접 `TcpStream`/`UdpSocket`/`reqwest`/`hyper` | **❌ 0건** | — | + +--- + +## 3. (B) 와이어 호환성 회귀 테스트 결과 + +`cargo test -p lumen-core --test elib_wire_compat` — **7/7 통과**. + +| 테스트 | 검증 대상 | 결과 | +|--------|-----------|------| +| `blake3_wrapper_byte_identical_to_direct_elib` | `Blake3Hash::of` ≡ `Blake3::new().update().finalize()` (4 입력) | ✅ | +| `blake3_wrapper_byte_identical_for_large_input` | 16 KiB 입력 (cv-스택 머지 경로) | ✅ | +| `blake3_keyed_derive_byte_identical_to_direct_elib` | `lumen_core::blake3_keyed_derive_32` ≡ `Blake3::new_keyed(...).update(...).finalize()` (lumen-channel KDF) | ✅ | +| `ed25519_sign_verify_via_wrapper_matches_direct_elib` | lumen 서명 → elib 검증 / elib 서명 → lumen 검증 양방향 | ✅ | +| `ed25519_signature_is_deterministic` | RFC 8032 결정성 (동일 msg+seed → 동일 64B 서명) | ✅ | +| `x25519_ecdh_produces_matching_shared_secret` | 양 피어 ECDH 32B 공유 비밀 일치 + 32B 와이어 라운드트립 | ✅ | +| `aes256_gcm_encrypts_with_tag_appended_format_used_by_lumen_channel` | `ciphertext \|\| tag` 와이어 포맷 + 결정성 + decrypt 라운드트립 | ✅ | + +**의미**: lumen 의 `lumen-core::SigningKey/VerifyingKey/Blake3Hash`, `lumen-channel::EncryptedChannel` 의 KDF/AEAD 출력은 `K0` 의 `elib-k0d-core` 가 elib-k0-nt 를 직접 호출했을 때와 비트 단위로 동일합니다. 즉, **두 프로젝트 사이의 와이어 호환성이 회귀-방지로 보장**됩니다. + +--- + +## 4. (C) 5-게이트 (Offline 모드) 결과 + +`cargo {build,test,clippy} --workspace --locked --offline ...`, `cargo fmt --all -- --check`, `cargo deny check`. + +| 게이트 | 결과 | +|--------|------| +| `cargo build --workspace --all-targets --locked --offline` | ✅ | +| `cargo build --workspace --no-default-features --locked --offline` | ✅ | +| `cargo test --workspace --locked --offline` | ✅ 38 테스트 그룹 (와이어 컴팟 7개 포함) | +| `cargo clippy --workspace --all-targets --offline -- -D warnings` | ✅ | +| `cargo fmt --all -- --check` | ✅ | +| `cargo deny check` | ✅ `advisories ok, bans ok, licenses ok, sources ok` | +| `cargo build --release --target wasm32-unknown-unknown --locked --offline` (echo-agent) | ✅ | + +`vendor-config.toml` 의 `[net] offline = true` 설정으로 외부 fetch 시도가 있으면 즉시 컴파일 실패. + +--- + +## 5. 폐쇄 커널 배포 모델 (검증 결과 요약) + +``` +┌────────────────────── High-Side (Air-Gapped) ───────────────────────┐ +│ │ +│ Ring 0 ┃ K0 마이크로커널 (no_std, x86_64-unknown-none) │ +│ ┃ - elib-k0-nt 암호 모듈 (PSK / TLS-PQ-Hybrid / ML-KEM) │ +│ ┃ - elib-k0d-core IPC 디스패처 (ed25519 sign/verify 등) │ +│ ─────────┃───────────────────────────────────────────────────────── │ +│ Ring 3 ┃ Lumen Ring 3 사용자공간 컴포넌트 (본 저장소) │ +│ ┃ │ +│ ┃ ┌──────────────────────────────────────────────┐ │ +│ ┃ │ lumen-cli + agent + orchestrator + channel + │ │ +│ ┃ │ inference (BPE) + zkml mock + provenance │ │ +│ ┃ │ - wasmtime 없음, 네트워크 스택 없음 │ │ +│ ┃ │ - 같은 elib-k0-nt 와 와이어 호환 (§3) │ │ +│ ┃ └──────────────────────────────────────────────┘ │ +│ ┃ │ +│ ┃ ┌──────────────────────────────────────────────┐ │ +│ ┃ │ lumen-sandbox (옵션, RWX 페이지 필요) │ │ +│ ┃ │ - Cranelift JIT — 강한 W^X 환경에서는 │ │ +│ ┃ │ wasmtime feature 를 pulley 로 전환 │ │ +│ ┃ └──────────────────────────────────────────────┘ │ +│ │ +└──────────────────────────────────────────────────────────────────────┘ +``` + +**K0 가 Ring 3 에 제공해야 하는 ABI 부분집합:** + +| 카테고리 | 필요 항목 | 현재 사용처 | +|----------|-----------|-------------| +| 메모리 | `mmap` (ANONYMOUS, R/W) + `munmap` | std/tokio 일반 할당 | +| 메모리 | `mprotect` 로 RWX (Cranelift JIT) | **lumen-sandbox 한정** — 사용 안 하면 불필요 | +| 동기화 | futex / `pthread_mutex_lock` | parking_lot, tokio sync | +| 스레드 | `clone(CLONE_THREAD)` 또는 동등 | tokio rt-multi-thread | +| 시간 | 단조 시계 (CLOCK_MONOTONIC 또는 동등) | tokio time | +| 파일시스템 | `open/read/close` 또는 IPC 매핑 | 모델 가중치 / SBOM 로드 | +| 표준 입출력 | `write(stderr)` | tracing fmt subscriber | +| 엔트로피 | OS 엔트로피 또는 `RDRAND` 인터페이스 | `lumen_core::rng::OsRng` (`elib_rng::os_entropy`) | + +**K0 가 *제공하지 않아도 되는* 것:** + +- 네트워크 (TCP/UDP/HTTP) — lumen 의 모든 통신은 `lumen-channel::SecureChannel` 을 통해 외부 transport 에 위임됨. 폐쇄망에서는 PSK 기반 IPC 가 transport 로 주입됨. +- 프로세스 spawn — `lumen-onchain` 만 사용하며 폐쇄망에서는 `dry_run=true` 로만 호출. +- 동적 링킹/로딩 — 정적 바이너리. + +--- + +## 6. 검증되지 않은(현재 환경에서 불가능한) 항목 + +이 항목들은 lumen 단독 자동 검증 범위를 벗어나므로 별도 통합 환경에서 확인 필요: + +1. **K0 의 실제 Ring 3 ABI 와의 결합**: 현재 `K0` 자체는 Ring 0 베어메탈 커널 단계까지만 구현되어 있어 Ring 3 사용자공간 프로세스를 띄울 수 없습니다. 결합 검증은 K0 의 `userspace_loader` 마일스톤이 도달한 뒤 가능합니다. +2. **TLS 1.3-PSK-PQ 핸드셰이크와 lumen-channel::EncryptedChannel 의 프로토콜 변환**: 두 프로토콜은 **기본 암호 프리미티브를 공유** (BLAKE3, AES-GCM, ed25519, x25519) 하지만 핸드셰이크 메시지 포맷은 다릅니다. lumen 은 이를 **외부 transport 가 처리** 하는 추상으로 격리해 두었으므로(§5 다이어그램), 실제 결합 시 thin adapter 한 단을 추가하면 됩니다. +3. **Cranelift JIT 의 페이지 보호 동작**: 현재 호스트 (Linux/macOS) 에서는 동작하지만, K0 가 Ring 3 에 W^X 강제 페이지를 부여할 경우 Cranelift 코드 생성이 실패합니다 — Pulley 인터프리터로 전환해야 합니다 (단순 feature 변경; 위 5절 참조). + +--- + +## 7. 결론 + +| 검증 영역 | 결과 | +|-----------|------| +| **암호 와이어 호환성** | ✅ lumen ≡ K0/elib-k0d-core 비트-동일 (회귀 테스트로 영구 보장) | +| **외부 의존성 정리** | ✅ 운영 바이너리에 네트워크 스택·tokenizers·rand_core·candle 없음 | +| **결정성** | ✅ Ed25519 서명, BLAKE3, AES-GCM, mock-prover 모두 결정론적 | +| **빌드 폐쇄성** | ✅ 5-게이트 모두 `--offline --locked` 통과, git source 0건 | +| **공급망 게이트** | ✅ `cargo deny check` 통과 (advisories / bans / licenses / sources 모두 clean) | +| **메모리 안전성** | ✅ 13/16 lumen 크레이트가 `forbid(unsafe_code)`; 예외는 wasmtime FFI 한 곳 | +| **JIT-free 운영 경로** | ✅ `lumen-cli` 기본 바이너리는 wasmtime 0건 | +| **샌드박스 환경 적응** | ⚠️ 강한 W^X 정책 시 wasmtime feature 를 `pulley` 로 전환 필요 (옵션) | + +**Lumen 은 폐쇄 커널 (K0) Ring 3 사용자공간 환경에서 정상 동작할 수 있도록 설계·검증되어 있습니다.** 위의 7개 와이어 호환성 회귀 테스트가 lumen ≡ elib-k0-nt 비트 동일성을 매 빌드마다 강제하며, 외부 네트워크 의존성과 JIT 컴포넌트는 운영 바이너리에서 분리되어 있습니다. + +--- + +## 부록 A. 회귀 테스트 실행 명령 + +```bash +cargo test -p lumen-core --test elib_wire_compat --offline +# 7 passed; 0 failed +``` + +## 부록 B. 참고 문서 + +- [`AIR-GAPPED.md`](AIR-GAPPED.md) — 폐쇄망 빌드 가이드 +- [`kernel-comp.md`](kernel-comp.md) — 의존성 점검 보고서 +- [`../K0/AIR-GAP.md`](../K0/AIR-GAP.md) — 커널 측 폐쇄망 프로토콜 명세 diff --git a/crates/lumen-agent/Cargo.toml b/crates/lumen-agent/Cargo.toml index 32904a3..c90848b 100644 --- a/crates/lumen-agent/Cargo.toml +++ b/crates/lumen-agent/Cargo.toml @@ -29,6 +29,5 @@ tracing.workspace = true parking_lot.workspace = true [dev-dependencies] -rand = { workspace = true } postcard = { workspace = true } tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/crates/lumen-agent/tests/determinism.rs b/crates/lumen-agent/tests/determinism.rs index 1d665ba..173c705 100644 --- a/crates/lumen-agent/tests/determinism.rs +++ b/crates/lumen-agent/tests/determinism.rs @@ -8,10 +8,10 @@ use lumen_capability::{ capability::{Capability, CapabilityBody}, PolicyEngine, Resource, }; +use lumen_core::rng::OsRng; use lumen_core::{AgentId, Blake3Hash, CapabilityId, SigningKey, Timestamp, ToolId}; use lumen_inference::DummyEngine; use lumen_zkml::mock::MockVk; -use rand::rngs::OsRng; fn build_runtime() -> AgentRuntime { let sk = SigningKey::generate(&mut OsRng); diff --git a/crates/lumen-agent/tests/determinism_stress.rs b/crates/lumen-agent/tests/determinism_stress.rs index 231e677..e653e00 100644 --- a/crates/lumen-agent/tests/determinism_stress.rs +++ b/crates/lumen-agent/tests/determinism_stress.rs @@ -88,9 +88,9 @@ fn make_runtime() -> AgentRuntime { /// `OsRng` here only feeds the *capability nonce* - the policy engine's replay /// table demands uniqueness; everything else in the proof binding is fixed. fn rand_nonce() -> [u8; 16] { - use rand::RngCore; + use lumen_core::rng::Rng; let mut n = [0u8; 16]; - rand::rngs::OsRng.fill_bytes(&mut n); + lumen_core::rng::OsRng.fill_bytes(&mut n); n } diff --git a/crates/lumen-capability/Cargo.toml b/crates/lumen-capability/Cargo.toml index 602d8d0..dc6cea2 100644 --- a/crates/lumen-capability/Cargo.toml +++ b/crates/lumen-capability/Cargo.toml @@ -23,6 +23,5 @@ parking_lot.workspace = true tracing.workspace = true [dev-dependencies] -rand = { workspace = true } serde_json = { workspace = true } toml.workspace = true diff --git a/crates/lumen-capability/src/capability.rs b/crates/lumen-capability/src/capability.rs index 6f7af28..98febb2 100644 --- a/crates/lumen-capability/src/capability.rs +++ b/crates/lumen-capability/src/capability.rs @@ -7,6 +7,14 @@ use serde::{Deserialize, Serialize}; use crate::resource::Resource; +/// Ed25519 도메인 분리 prefix. +/// +/// Lumen 의 다른 서명 페이로드 (보안 채널 핸드셰이크 / 데이터 프레임 / 모델 +/// 매니페스트) 와 동일한 신원 키를 공유하더라도, 이 prefix 가 서명 입력 첫 +/// 부분에 포함되므로 다른 프로토콜의 정상 서명을 capability 검증에 재사용 +/// 하는 cross-protocol replay 가 차단됩니다. +const CAPABILITY_SIGN_DOMAIN: &[u8] = b"lumen.capability.body.v1"; + /// Capability 의 본문 - 서명되는 모든 데이터. /// /// 본문과 서명을 분리해 두면 동일한 정규 표현을 서명과 검증 양쪽에서 @@ -39,9 +47,8 @@ pub struct Capability { impl Capability { /// 주어진 서명 키로 본문에 서명해 완성된 capability 를 만듭니다. pub fn sign(body: CapabilityBody, key: &SigningKey) -> Result { - let bytes = postcard::to_allocvec(&body) - .map_err(|e| Error::Decode(format!("capability body encode: {e}")))?; - let signature = key.sign(&bytes); + let payload = signing_payload(&body)?; + let signature = key.sign(&payload); Ok(Self { body, signature }) } @@ -50,17 +57,26 @@ impl Capability { /// 만료, audience, 리플레이는 **검사하지 않습니다** - 그것은 /// [`crate::PolicyEngine`] 의 책임입니다. pub fn verify_signature(&self, key: &VerifyingKey) -> Result<()> { - let bytes = postcard::to_allocvec(&self.body) - .map_err(|e| Error::Decode(format!("capability body encode: {e}")))?; - key.verify(&bytes, &self.signature) + let payload = signing_payload(&self.body)?; + key.verify(&payload, &self.signature) } } +/// `(CAPABILITY_SIGN_DOMAIN || postcard(body))` 를 정규 서명 페이로드로 반환. +fn signing_payload(body: &CapabilityBody) -> Result> { + let body_bytes = postcard::to_allocvec(body) + .map_err(|e| Error::Decode(format!("capability body encode: {e}")))?; + let mut out = Vec::with_capacity(CAPABILITY_SIGN_DOMAIN.len() + body_bytes.len()); + out.extend_from_slice(CAPABILITY_SIGN_DOMAIN); + out.extend_from_slice(&body_bytes); + Ok(out) +} + #[cfg(test)] mod tests { use super::*; + use lumen_core::rng::OsRng; use lumen_core::ToolId; - use rand::rngs::OsRng; use crate::Resource; @@ -86,4 +102,33 @@ mod tests { tampered.body.resource = Resource::ZkProofRequest; assert!(tampered.verify_signature(&vk).is_err()); } + + /// 도메인 분리 검증: domain prefix 가 없는 직접 postcard 서명은 + /// `verify_signature` 에서 거부되어야 합니다. cross-protocol replay + /// 보호의 회귀 테스트입니다. + #[test] + fn rejects_signature_without_domain_prefix() { + let sk = SigningKey::generate(&mut OsRng); + let vk = sk.verifying_key(); + let body = CapabilityBody { + id: CapabilityId::random(&mut OsRng), + audience: AgentId::random(&mut OsRng), + resource: Resource::Tool(ToolId::new("echo").unwrap()), + nonce: [3u8; 16], + expires_at: Timestamp::FOREVER, + issuer: AgentId::random(&mut OsRng), + }; + // 공격자가 도메인 prefix 없이 postcard(body) 만 서명 - 다른 프로토콜의 + // 정상 서명을 capability 서명으로 재사용하는 시나리오. + let raw = postcard::to_allocvec(&body).unwrap(); + let bad_sig = sk.sign(&raw); + let bad_cap = Capability { + body, + signature: bad_sig, + }; + assert!( + bad_cap.verify_signature(&vk).is_err(), + "domain prefix 없는 서명은 거부되어야 함" + ); + } } diff --git a/crates/lumen-capability/src/policy.rs b/crates/lumen-capability/src/policy.rs index 48a50e6..283bedb 100644 --- a/crates/lumen-capability/src/policy.rs +++ b/crates/lumen-capability/src/policy.rs @@ -137,8 +137,8 @@ fn _ensure_pattern_types_compile(_: &PathPattern, _: &HostPattern) {} #[cfg(test)] mod tests { use super::*; + use lumen_core::rng::OsRng; use lumen_core::{AgentId, CapabilityId, SigningKey}; - use rand::rngs::OsRng; use crate::capability::{Capability, CapabilityBody}; diff --git a/crates/lumen-channel/Cargo.toml b/crates/lumen-channel/Cargo.toml index 3a71ae9..3c7bc96 100644 --- a/crates/lumen-channel/Cargo.toml +++ b/crates/lumen-channel/Cargo.toml @@ -15,7 +15,7 @@ workspace = true [features] default = [] -crypto-channel = ["dep:aes-gcm", "dep:x25519-dalek", "dep:blake3"] +crypto-channel = ["dep:elib-aes", "dep:elib-x25519", "dep:elib-zeroize"] [dependencies] lumen-core.workspace = true @@ -25,10 +25,9 @@ tokio = { workspace = true } serde = { workspace = true } postcard.workspace = true thiserror.workspace = true -rand = { workspace = true } -aes-gcm = { workspace = true, optional = true } -x25519-dalek = { workspace = true, optional = true } -blake3 = { workspace = true, optional = true } +elib-aes = { workspace = true, optional = true } +elib-x25519 = { workspace = true, optional = true } +elib-zeroize = { workspace = true, optional = true } [dev-dependencies] tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/crates/lumen-channel/src/attested.rs b/crates/lumen-channel/src/attested.rs index 659f4dd..639a73e 100644 --- a/crates/lumen-channel/src/attested.rs +++ b/crates/lumen-channel/src/attested.rs @@ -44,6 +44,16 @@ pub const SOFTWARE_HELLO_MARKER: &str = "lumen.attested.v1"; /// 활성화됩니다. pub const TEE_HELLO_MARKER: &str = "lumen.attested-tee.v1"; +/// 핸드셰이크 Hello 서명 도메인 분리 prefix. +/// +/// 같은 Ed25519 신원 키가 capability / manifest / 데이터 프레임 서명에도 +/// 사용되므로, 다른 프로토콜의 서명을 Hello 서명으로 재사용하는 cross- +/// protocol replay 를 차단하기 위한 prefix 입니다. +const HELLO_SIGN_DOMAIN: &[u8] = b"lumen.attested.hello.v1"; + +/// 데이터 프레임 서명 도메인 분리 prefix. +const FRAME_SIGN_DOMAIN: &[u8] = b"lumen.attested.frame.v1"; + /// 서명되는 핸드셰이크 평문 페이로드. #[derive(Clone, Debug, Serialize, Deserialize)] struct Hello { @@ -158,9 +168,7 @@ impl AttestedChannel { nonce, attestation_doc, }; - let hello_bytes = - postcard::to_allocvec(&hello).map_err(|e| Error::Decode(format!("hello: {e}")))?; - let signature = sk.sign(&hello_bytes); + let signature = sk.sign(&hello_signing_payload(&hello)?); let signed = SignedHello { hello: hello.clone(), signature, @@ -184,9 +192,7 @@ impl AttestedChannel { "attested: peer hello 의 수신자가 우리 ID 가 아님".into(), )); } - let peer_bytes = postcard::to_allocvec(&peer.hello) - .map_err(|e| Error::Decode(format!("peer hello: {e}")))?; - peer_vk.verify(&peer_bytes, &peer.signature)?; + peer_vk.verify(&hello_signing_payload(&peer.hello)?, &peer.signature)?; // TEE 변종: peer attestation 문서 형식 + measurement 핀 검증. if marker == TEE_HELLO_MARKER { @@ -234,9 +240,7 @@ impl SecureChannel for AttestedChannel { seq: self.next_send_seq, payload: bytes, }; - let frame_bytes = postcard::to_allocvec(&frame) - .map_err(|e| Error::Decode(format!("frame encode: {e}")))?; - let signature = self.sk.sign(&frame_bytes); + let signature = self.sk.sign(&frame_signing_payload(&frame)?); let signed = SignedFrame { frame, signature }; self.inner.send(&signed).await?; self.next_send_seq = self @@ -248,9 +252,10 @@ impl SecureChannel for AttestedChannel { async fn recv_bytes(&mut self) -> Result> { let signed: SignedFrame = self.inner.recv().await?; - let frame_bytes = postcard::to_allocvec(&signed.frame) - .map_err(|e| Error::Decode(format!("frame decode: {e}")))?; - self.peer_vk.verify(&frame_bytes, &signed.signature)?; + // epoch / seq 를 먼저 빠르게 거부 - 적대적 가비지 위에서 ed25519 + // 검증을 낭비하지 않습니다. AEAD 가 없는 attested 변종에서도 위 + // 두 검사는 인증에 영향을 주지 않으며 실제 신원 검증은 아래 + // peer_vk.verify 가 담당합니다. if signed.frame.epoch != self.epoch { return Err(Error::Channel(format!( "attested: epoch mismatch (expected {}, got {})", @@ -263,6 +268,8 @@ impl SecureChannel for AttestedChannel { self.next_recv_seq, signed.frame.seq ))); } + self.peer_vk + .verify(&frame_signing_payload(&signed.frame)?, &signed.signature)?; self.next_recv_seq = self .next_recv_seq .checked_add(1) @@ -271,10 +278,28 @@ impl SecureChannel for AttestedChannel { } } +/// `(HELLO_SIGN_DOMAIN || postcard(hello))` 정규 서명 페이로드. +fn hello_signing_payload(hello: &Hello) -> Result> { + let body = postcard::to_allocvec(hello).map_err(|e| Error::Decode(format!("hello: {e}")))?; + let mut out = Vec::with_capacity(HELLO_SIGN_DOMAIN.len() + body.len()); + out.extend_from_slice(HELLO_SIGN_DOMAIN); + out.extend_from_slice(&body); + Ok(out) +} + +/// `(FRAME_SIGN_DOMAIN || postcard(frame))` 정규 서명 페이로드. +fn frame_signing_payload(frame: &DataFrame) -> Result> { + let body = postcard::to_allocvec(frame).map_err(|e| Error::Decode(format!("frame: {e}")))?; + let mut out = Vec::with_capacity(FRAME_SIGN_DOMAIN.len() + body.len()); + out.extend_from_slice(FRAME_SIGN_DOMAIN); + out.extend_from_slice(&body); + Ok(out) +} + fn random_nonce() -> [u8; 16] { - use rand::RngCore; + use lumen_core::rng::{OsRng, Rng}; let mut n = [0u8; 16]; - rand::rngs::OsRng.fill_bytes(&mut n); + OsRng.fill_bytes(&mut n); n } @@ -290,7 +315,7 @@ mod tests { AttestedChannel, AttestedChannel, ) { - let mut rng = rand::rngs::OsRng; + let mut rng = lumen_core::rng::OsRng; let sk_a = SigningKey::generate(&mut rng); let sk_b = SigningKey::generate(&mut rng); let vk_a = sk_a.verifying_key(); @@ -327,7 +352,7 @@ mod tests { #[tokio::test] async fn handshake_with_wrong_peer_fails() { - let mut rng = rand::rngs::OsRng; + let mut rng = lumen_core::rng::OsRng; let sk_a = SigningKey::generate(&mut rng); let sk_b = SigningKey::generate(&mut rng); let sk_c = SigningKey::generate(&mut rng); // attacker @@ -373,7 +398,7 @@ mod tests { async fn tampered_payload_rejected() { // We can't easily tamper without breaking the abstraction; instead, // wire two pairs and inject a different signer's frame. - let mut rng = rand::rngs::OsRng; + let mut rng = lumen_core::rng::OsRng; let sk_a = SigningKey::generate(&mut rng); let sk_b = SigningKey::generate(&mut rng); let sk_evil = SigningKey::generate(&mut rng); @@ -407,7 +432,7 @@ mod tests { doc_b[144..192].fill(0x22); let pin_b = vec![0x22u8; 48]; - let mut rng = rand::rngs::OsRng; + let mut rng = lumen_core::rng::OsRng; let sk_a = SigningKey::generate(&mut rng); let sk_b = SigningKey::generate(&mut rng); let vk_a = sk_a.verifying_key(); @@ -443,7 +468,7 @@ mod tests { // A 가 기대하는 pin 은 0xAA 인데 실제 B 는 0x22 -> 불일치. let bad_pin = vec![0xAAu8; 48]; - let mut rng = rand::rngs::OsRng; + let mut rng = lumen_core::rng::OsRng; let sk_a = SigningKey::generate(&mut rng); let sk_b = SigningKey::generate(&mut rng); let vk_a = sk_a.verifying_key(); @@ -465,7 +490,7 @@ mod tests { let mut doc = vec![0u8; 1184]; doc[0..4].copy_from_slice(&2u32.to_le_bytes()); - let mut rng = rand::rngs::OsRng; + let mut rng = lumen_core::rng::OsRng; let sk_a = SigningKey::generate(&mut rng); let sk_b = SigningKey::generate(&mut rng); let vk_a = sk_a.verifying_key(); diff --git a/crates/lumen-channel/src/encrypted.rs b/crates/lumen-channel/src/encrypted.rs index f22e94c..1f5e6e9 100644 --- a/crates/lumen-channel/src/encrypted.rs +++ b/crates/lumen-channel/src/encrypted.rs @@ -24,15 +24,20 @@ //! 아닙니다. attested 채널은 무결성 + 신원 + 리플레이 저항을 제공하지만 //! **기밀성** 은 없습니다. 이 모듈은 동일한 ed25519 핀을 재사용해 KEX 를 //! 인증하면서 기밀성을 추가합니다. +//! +//! 폐쇄형(Air-Gapped) K0 마이크로커널 환경 호환을 위해 모든 +//! 암호 프리미티브는 `elib-k0-nt` 모듈을 사용합니다 - AES-GCM 은 +//! `elib-k0-nt/aes::AES256GCM`, ECDH 는 `elib-k0-nt/x25519`, KDF 는 +//! `lumen_core::hash::blake3_keyed_derive_32` (BLAKE3 keyed 모드). -use aes_gcm::aead::{Aead, KeyInit, Payload}; -use aes_gcm::{Aes256Gcm, Key, Nonce}; use async_trait::async_trait; +use elib_aes::{AES256GCM, GCM_NONCE_SIZE, GCM_TAG_SIZE}; +use elib_x25519::{PublicKey as X25519Public, SecretKey as X25519Secret}; +use elib_zeroize::Zeroize; +use lumen_core::hash::blake3_keyed_derive_32; +use lumen_core::rng::{OsRng, Rng}; use lumen_core::{Error, Result, Signature, SigningKey, VerifyingKey}; -use rand::rngs::OsRng; -use rand::RngCore; use serde::{Deserialize, Serialize}; -use x25519_dalek::{EphemeralSecret, PublicKey as X25519Public}; use crate::SecureChannel; @@ -74,7 +79,8 @@ struct SignedEncryptedHello { struct EncryptedFrame { /// 송신 측 카운터; 수신 측은 정확히 다음 기대값과 일치해야 합니다. seq: u64, - /// AEAD 출력 (ciphertext || tag). + /// AEAD 출력 (ciphertext || tag) - 외부 와이어 호환을 위해 elib-k0-nt + /// 의 (ciphertext, tag) 쌍을 단일 바이트열로 직렬화한 것. ciphertext: Vec, } @@ -85,9 +91,9 @@ struct EncryptedFrame { pub struct EncryptedChannel { inner: C, /// 송신용 키 (자기 → 피어). - tx_cipher: Aes256Gcm, + tx_cipher: AES256GCM, /// 수신용 키 (피어 → 자기). - rx_cipher: Aes256Gcm, + rx_cipher: AES256GCM, /// 자신의 역할 (initiator=0, responder=1) - nonce 첫 바이트로 사용. my_role: u8, /// 피어의 역할 - 수신 시 nonce 검증. @@ -134,8 +140,15 @@ impl EncryptedChannel { my_role: u8, ) -> Result { let my_vk = sk.verifying_key(); - let my_eph = EphemeralSecret::random_from_rng(OsRng); - let my_eph_pub = X25519Public::from(&my_eph); + + // 임시 x25519 키 + 16 바이트 핸드셰이크 nonce 를 OS 엔트로피에서 생성. + // eph_seed 는 X25519Secret 으로 흡수된 직후 zeroize 하여 forensic + // 메모리 덤프에서 ephemeral 비밀이 회수되지 않도록 합니다. + let mut eph_seed = [0u8; 32]; + OsRng.fill_bytes(&mut eph_seed); + let my_eph = X25519Secret::from_bytes(eph_seed); + eph_seed.zeroize(); + let my_eph_pub = my_eph.public_key(); let mut nonce16 = [0u8; 16]; OsRng.fill_bytes(&mut nonce16); @@ -183,20 +196,34 @@ impl EncryptedChannel { peer_vk.verify(&peer_payload, &peer.signature)?; // x25519 ECDH. - let peer_x25519 = X25519Public::from(peer.hello.ephemeral_x25519); + let peer_x25519 = X25519Public::from_bytes(peer.hello.ephemeral_x25519); let shared = my_eph.diffie_hellman(&peer_x25519); + // RFC 7748 권고: 모든 비트가 0 인 공유 비밀은 비여직성(non-contributory) + // 키 합의를 의미하므로 거부합니다. + if shared.is_zero() { + return Err(Error::Crypto( + "encrypted: x25519 produced all-zero shared secret".into(), + )); + } + // 디렉션별 키 도출. initiator → responder 와 responder → initiator // 두 개를 만들고, 자신의 송신/수신을 그에 매핑합니다. - let key_i2r = derive_key(shared.as_bytes(), KDF_LABEL_INITIATOR, epoch); - let key_r2i = derive_key(shared.as_bytes(), KDF_LABEL_RESPONDER, epoch); - let (tx_key, rx_key, peer_role) = if my_role == 0 { + // AES256GCM 이 키를 라운드키로 확장한 직후 원본 32 바이트 사본을 + // zeroize 하여 스택 잔류 비밀을 최소화합니다. + let mut key_i2r = derive_key(shared.as_bytes(), KDF_LABEL_INITIATOR, epoch); + let mut key_r2i = derive_key(shared.as_bytes(), KDF_LABEL_RESPONDER, epoch); + let (mut tx_key, mut rx_key, peer_role) = if my_role == 0 { (key_i2r, key_r2i, 1u8) } else { (key_r2i, key_i2r, 0u8) }; - let tx_cipher = Aes256Gcm::new(Key::::from_slice(&tx_key)); - let rx_cipher = Aes256Gcm::new(Key::::from_slice(&rx_key)); + let tx_cipher = AES256GCM::new(&tx_key); + let rx_cipher = AES256GCM::new(&rx_key); + tx_key.zeroize(); + rx_key.zeroize(); + key_i2r.zeroize(); + key_r2i.zeroize(); Ok(Self { inner, @@ -226,16 +253,13 @@ impl SecureChannel for EncryptedChannel { async fn send_bytes(&mut self, bytes: Vec) -> Result<()> { let nonce = build_nonce(self.my_role, self.next_send_seq); let aad = aad_bytes(self.epoch, self.my_role); - let ciphertext = self - .tx_cipher - .encrypt( - Nonce::from_slice(&nonce), - Payload { - msg: &bytes, - aad: &aad, - }, - ) - .map_err(|e| Error::Crypto(format!("aes-gcm seal: {e}")))?; + let mut ciphertext = vec![0u8; bytes.len()]; + let mut tag = [0u8; GCM_TAG_SIZE]; + self.tx_cipher + .encrypt(&nonce, &aad, &bytes, &mut ciphertext, &mut tag); + // 와이어 포맷: ciphertext || tag (구버전 aes-gcm 0.10 호환). + ciphertext.extend_from_slice(&tag); + let frame = EncryptedFrame { seq: self.next_send_seq, ciphertext, @@ -256,18 +280,26 @@ impl SecureChannel for EncryptedChannel { self.next_recv_seq, frame.seq ))); } + if frame.ciphertext.len() < GCM_TAG_SIZE { + return Err(Error::Crypto( + "encrypted: frame shorter than GCM tag".into(), + )); + } let nonce = build_nonce(self.peer_role, frame.seq); let aad = aad_bytes(self.epoch, self.peer_role); - let plaintext = self + let split = frame.ciphertext.len() - GCM_TAG_SIZE; + let (ct, tag_slice) = frame.ciphertext.split_at(split); + let mut tag = [0u8; GCM_TAG_SIZE]; + tag.copy_from_slice(tag_slice); + let mut plaintext = vec![0u8; ct.len()]; + let ok = self .rx_cipher - .decrypt( - Nonce::from_slice(&nonce), - Payload { - msg: &frame.ciphertext, - aad: &aad, - }, - ) - .map_err(|_| Error::Crypto("encrypted: AEAD authentication failed".into()))?; + .decrypt(&nonce, &aad, ct, &tag, &mut plaintext); + if !ok { + return Err(Error::Crypto( + "encrypted: AEAD authentication failed".into(), + )); + } self.next_recv_seq = self .next_recv_seq .checked_add(1) @@ -287,16 +319,11 @@ fn signing_payload(hello: &EncryptedHello) -> Result> { fn derive_key(shared: &[u8; 32], label: &[u8], epoch: u64) -> [u8; 32] { // BLAKE3 keyed-hash 로 (shared || label || epoch) → 32-byte key. - let mut hasher = blake3::Hasher::new_keyed(shared); - hasher.update(label); - hasher.update(&epoch.to_le_bytes()); - let mut out = [0u8; 32]; - out.copy_from_slice(hasher.finalize().as_bytes()); - out + blake3_keyed_derive_32(shared, label, &epoch.to_le_bytes()) } -fn build_nonce(role: u8, seq: u64) -> [u8; 12] { - let mut n = [0u8; 12]; +fn build_nonce(role: u8, seq: u64) -> [u8; GCM_NONCE_SIZE] { + let mut n = [0u8; GCM_NONCE_SIZE]; n[0] = role; // n[1..4] 은 0 으로 둡니다 - 카운터 오버플로 시 명시적 거부. n[4..12].copy_from_slice(&seq.to_le_bytes()); @@ -321,7 +348,7 @@ mod tests { EncryptedChannel, EncryptedChannel, )> { - let mut rng = rand::rngs::OsRng; + let mut rng = OsRng; let sk_a = SigningKey::generate(&mut rng); let sk_b = SigningKey::generate(&mut rng); let vk_a = sk_a.verifying_key(); @@ -373,7 +400,7 @@ mod tests { #[tokio::test] async fn handshake_with_wrong_peer_pin_fails() { - let mut rng = rand::rngs::OsRng; + let mut rng = OsRng; let sk_a = SigningKey::generate(&mut rng); let sk_b = SigningKey::generate(&mut rng); let sk_c = SigningKey::generate(&mut rng); // 공격자 @@ -394,7 +421,7 @@ mod tests { #[tokio::test] async fn epoch_mismatch_rejected() { - let mut rng = rand::rngs::OsRng; + let mut rng = OsRng; let sk_a = SigningKey::generate(&mut rng); let sk_b = SigningKey::generate(&mut rng); let vk_a = sk_a.verifying_key(); diff --git a/crates/lumen-cli/Cargo.toml b/crates/lumen-cli/Cargo.toml index 87f4cd8..b507e92 100644 --- a/crates/lumen-cli/Cargo.toml +++ b/crates/lumen-cli/Cargo.toml @@ -36,4 +36,3 @@ tracing.workspace = true tracing-subscriber.workspace = true anyhow.workspace = true hex.workspace = true -rand = { workspace = true } diff --git a/crates/lumen-cli/examples/hello_agent.rs b/crates/lumen-cli/examples/hello_agent.rs index bc354db..7e4e6f7 100644 --- a/crates/lumen-cli/examples/hello_agent.rs +++ b/crates/lumen-cli/examples/hello_agent.rs @@ -17,11 +17,11 @@ use lumen_capability::{ capability::{Capability, CapabilityBody}, PolicyEngine, Resource, }; +use lumen_core::rng::OsRng; use lumen_core::{AgentId, Blake3Hash, CapabilityId, SigningKey, Timestamp, ToolId}; use lumen_inference::DummyEngine; use lumen_orchestrator::{AgentSpec, Orchestrator}; use lumen_zkml::mock::MockVk; -use rand::rngs::OsRng; fn issue_tool_cap( sk: &SigningKey, diff --git a/crates/lumen-core/Cargo.toml b/crates/lumen-core/Cargo.toml index 85c93b6..bb08e55 100644 --- a/crates/lumen-core/Cargo.toml +++ b/crates/lumen-core/Cargo.toml @@ -16,13 +16,15 @@ workspace = true [dependencies] thiserror.workspace = true serde = { workspace = true } -blake3 = { workspace = true } -ed25519-dalek = { workspace = true } -rand_core.workspace = true -subtle.workspace = true +elib-blake.workspace = true +elib-ed25519.workspace = true +elib-rng.workspace = true +elib-constant-time.workspace = true +elib-zeroize.workspace = true hex.workspace = true postcard.workspace = true [dev-dependencies] -rand = { workspace = true } serde_json = { workspace = true } +elib-aes.workspace = true +elib-x25519.workspace = true diff --git a/crates/lumen-core/src/crypto.rs b/crates/lumen-core/src/crypto.rs index 5664829..2feb53b 100644 --- a/crates/lumen-core/src/crypto.rs +++ b/crates/lumen-core/src/crypto.rs @@ -1,51 +1,60 @@ //! Ed25519 래퍼 + serde 글루. //! -//! `lumen-core` 가 `ed25519-dalek` 에 직접 의존하는 유일한 크레이트가 되어야 -//! 합니다. Capability 토큰과 provenance 서명 모두 이 타입들을 거칩니다. +//! `lumen-core` 가 Ed25519 구현 (`elib-k0-nt/ed25519`) 에 직접 의존하는 +//! 유일한 크레이트가 되어야 합니다. Capability 토큰과 provenance 서명 +//! 모두 이 타입들을 거칩니다. +//! +//! 폐쇄형(Air-Gapped) K0 환경 호환을 위해 외부 `ed25519-dalek` +//! 크레이트는 모두 elib-k0-nt 모듈로 교체되었습니다. use std::fmt; -use ed25519_dalek::{ - Signer, SigningKey as DalekSigningKey, Verifier, VerifyingKey as DalekVerifyingKey, - SECRET_KEY_LENGTH, SIGNATURE_LENGTH, -}; -use rand_core::{CryptoRng, RngCore}; +use elib_ed25519::{self as ed25519, Ed25519Error}; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use crate::error::{Error, Result}; +use crate::rng::Rng; + +/// Ed25519 비밀키 시드 길이 (32 바이트). +pub const SECRET_KEY_LENGTH: usize = 32; +/// Ed25519 서명 길이 (64 바이트). +pub const SIGNATURE_LENGTH: usize = 64; /// Ed25519 서명 (개인) 키. /// -/// `ed25519_dalek::SigningKey` 를 감쌉니다. **절대 직렬화되지 않습니다** - +/// `elib-k0-nt/ed25519::SecretKey` 를 감쌉니다. **절대 직렬화되지 않습니다** - /// 개인 키 export 는 명시적이고 감사된 작업이어야 합니다. 그 이유로 래퍼는 /// 의도적으로 `Serialize` / `Display` impl 을 두지 않습니다. #[derive(Clone)] -pub struct SigningKey(DalekSigningKey); +pub struct SigningKey { + secret: ed25519::SecretKey, + /// 공개 검증 키. 시드에서 매번 재유도하지 않도록 캐시. + public: ed25519::PublicKey, +} impl SigningKey { /// CSPRNG 으로부터 새 키를 생성합니다. - pub fn generate(rng: &mut R) -> Self { - Self(DalekSigningKey::generate(rng)) + pub fn generate(rng: &mut R) -> Self { + let mut seed = [0u8; SECRET_KEY_LENGTH]; + rng.fill_bytes(&mut seed); + Self::from_seed(&seed) } /// 32 바이트 원시 시드로부터 생성. pub fn from_seed(seed: &[u8; SECRET_KEY_LENGTH]) -> Self { - Self(DalekSigningKey::from_bytes(seed)) - } - - /// 시드 바이트를 반환 (민감 - 로그에 남기지 마세요). - pub fn to_seed(&self) -> [u8; SECRET_KEY_LENGTH] { - self.0.to_bytes() + let secret = ed25519::SecretKey::from_bytes(seed); + let public = ed25519::PublicKey::from(&secret); + Self { secret, public } } /// 공개 검증 키를 도출합니다. pub fn verifying_key(&self) -> VerifyingKey { - VerifyingKey(self.0.verifying_key()) + VerifyingKey(self.public) } /// 메시지에 서명합니다. pub fn sign(&self, msg: &[u8]) -> Signature { - Signature(self.0.sign(msg)) + Signature(ed25519::sign(msg, &self.secret)) } } @@ -58,32 +67,38 @@ impl fmt::Debug for SigningKey { } /// Ed25519 검증 (공개) 키. -#[derive(Clone, Copy, PartialEq, Eq, Hash)] -pub struct VerifyingKey(DalekVerifyingKey); +#[derive(Clone, Copy, PartialEq, Eq)] +pub struct VerifyingKey(ed25519::PublicKey); + +impl std::hash::Hash for VerifyingKey { + fn hash(&self, state: &mut H) { + self.0.as_bytes().hash(state); + } +} impl VerifyingKey { /// 32 바이트 원시 데이터로부터 파싱. + /// + /// 현재 elib-k0-nt 의 `PublicKey::from_bytes` 는 형태 검증 없이 32 바이트를 + /// 그대로 받아들이고, 실제 곡선 점 디코딩은 `verify` 시점에 수행됩니다. + /// API 계약 유지를 위해 길이 검증만 수행합니다. pub fn from_bytes(bytes: &[u8; 32]) -> Result { - DalekVerifyingKey::from_bytes(bytes) - .map(Self) - .map_err(|e| Error::Crypto(format!("verifying key: {e}"))) + Ok(Self(ed25519::PublicKey::from_bytes(bytes))) } /// 32 바이트 원시 데이터로 인코딩. pub fn to_bytes(&self) -> [u8; 32] { - self.0.to_bytes() + *self.0.as_bytes() } /// 서명 검증. pub fn verify(&self, msg: &[u8], sig: &Signature) -> Result<()> { - self.0 - .verify(msg, &sig.0) - .map_err(|e| Error::Crypto(format!("signature: {e}"))) + ed25519::verify(msg, &sig.0, &self.0).map_err(map_ed25519_err) } /// 64자 소문자 hex 로 렌더. pub fn to_hex(&self) -> String { - hex::encode(self.0.to_bytes()) + hex::encode(self.0.as_bytes()) } } @@ -104,7 +119,7 @@ impl Serialize for VerifyingKey { if ser.is_human_readable() { ser.serialize_str(&self.to_hex()) } else { - ser.serialize_bytes(&self.0.to_bytes()) + ser.serialize_bytes(self.0.as_bytes()) } } } @@ -129,22 +144,22 @@ impl<'de> Deserialize<'de> for VerifyingKey { /// Ed25519 detached 서명. #[derive(Clone, Copy, PartialEq, Eq)] -pub struct Signature(ed25519_dalek::Signature); +pub struct Signature(ed25519::Signature); impl Signature { /// 64 바이트 원시 데이터로부터 파싱. pub fn from_bytes(bytes: &[u8; SIGNATURE_LENGTH]) -> Self { - Self(ed25519_dalek::Signature::from_bytes(bytes)) + Self(ed25519::Signature::from_bytes(bytes)) } /// 64 바이트 원시 데이터로 렌더. pub fn to_bytes(&self) -> [u8; SIGNATURE_LENGTH] { - self.0.to_bytes() + *self.0.as_bytes() } /// 소문자 hex (128자). pub fn to_hex(&self) -> String { - hex::encode(self.0.to_bytes()) + hex::encode(self.0.as_bytes()) } } @@ -165,7 +180,7 @@ impl Serialize for Signature { if ser.is_human_readable() { ser.serialize_str(&self.to_hex()) } else { - ser.serialize_bytes(&self.0.to_bytes()) + ser.serialize_bytes(self.0.as_bytes()) } } } @@ -188,10 +203,14 @@ impl<'de> Deserialize<'de> for Signature { } } +fn map_ed25519_err(e: Ed25519Error) -> Error { + Error::Crypto(format!("signature: {e:?}")) +} + #[cfg(test)] mod tests { use super::*; - use rand::rngs::OsRng; + use crate::rng::OsRng; #[test] fn sign_verify_roundtrip() { diff --git a/crates/lumen-core/src/hash.rs b/crates/lumen-core/src/hash.rs index ed465b9..f19c37d 100644 --- a/crates/lumen-core/src/hash.rs +++ b/crates/lumen-core/src/hash.rs @@ -1,11 +1,15 @@ //! Lumen 전체에서 사용되는 BLAKE3 기반 해시 프리미티브. +//! +//! 폐쇄형(Air-Gapped) K0 환경 호환을 위해 구현체는 +//! `elib-k0-nt/blake` 의 [`Blake3`] 를 사용합니다. 외부 `blake3` 크레이트 +//! 의존을 제거합니다. use std::fmt; use std::path::Path; use std::str::FromStr; +use elib_blake::Blake3; use serde::{Deserialize, Deserializer, Serialize, Serializer}; -use subtle::ConstantTimeEq; use crate::error::{Error, Result}; @@ -19,7 +23,9 @@ pub struct Blake3Hash(pub [u8; 32]); impl Blake3Hash { /// 주어진 byte 슬라이스에 대해 해시를 계산합니다. pub fn of(bytes: &[u8]) -> Self { - Self(*blake3::hash(bytes).as_bytes()) + let mut hasher = Blake3::new(); + hasher.update(bytes); + Self(finalize_32(hasher)) } /// 디스크의 파일을 스트리밍으로 점진적으로 해시합니다. @@ -29,7 +35,7 @@ impl Blake3Hash { pub fn of_file(path: &Path) -> std::io::Result { use std::io::Read; let mut file = std::fs::File::open(path)?; - let mut hasher = blake3::Hasher::new(); + let mut hasher = Blake3::new(); let mut buf = [0u8; 64 * 1024]; loop { let n = file.read(&mut buf)?; @@ -38,7 +44,7 @@ impl Blake3Hash { } hasher.update(&buf[..n]); } - Ok(Self(*hasher.finalize().as_bytes())) + Ok(Self(finalize_32(hasher))) } /// 원시 바이트 차용. @@ -54,7 +60,8 @@ impl Blake3Hash { impl PartialEq for Blake3Hash { fn eq(&self, other: &Self) -> bool { - self.0.ct_eq(&other.0).into() + // elib-k0-nt/constant-time 의 상수시간 슬라이스 비교. + elib_blake::ct_eq_slice(&self.0, &other.0).unwrap_u8() == 1 } } @@ -134,6 +141,29 @@ pub fn hash_postcard(value: &T) -> Result { Ok(Blake3Hash::of(&bytes)) } +/// `Blake3` 해시를 32 바이트 배열로 종결합니다. +/// +/// elib-k0-nt 의 `finalize` 는 메모리 보안을 위해 `SecureBuffer` 를 반환합니다. +/// Lumen 의 식별자는 short-lived 한 32 바이트 다이제스트이므로 즉시 일반 배열로 +/// 복사해 사용합니다 (`SecureBuffer` 는 함수 종료와 동시에 zeroize 됨). +fn finalize_32(hasher: Blake3) -> [u8; 32] { + let buf = hasher.finalize().expect("blake3 finalize failed"); + let mut out = [0u8; 32]; + out.copy_from_slice(buf.as_slice()); + out +} + +/// `(key, label, epoch_le_bytes)` 트리플로 32 바이트 키 자료를 도출합니다. +/// +/// `lumen-channel` 의 키 도출에서 사용됩니다. BLAKE3 keyed 모드는 +/// `key` 자체를 IV 로 사용하므로 키 바이트가 누설되지 않습니다. +pub fn blake3_keyed_derive_32(key: &[u8; 32], label: &[u8], extra: &[u8]) -> [u8; 32] { + let mut hasher = Blake3::new_keyed(key); + hasher.update(label); + hasher.update(extra); + finalize_32(hasher) +} + #[cfg(test)] mod tests { use super::*; @@ -157,4 +187,13 @@ mod tests { fn distinct_inputs_distinct_hashes() { assert_ne!(Blake3Hash::of(b"a"), Blake3Hash::of(b"b")); } + + #[test] + fn keyed_derive_distinct_for_different_keys() { + let k1 = [1u8; 32]; + let k2 = [2u8; 32]; + let a = blake3_keyed_derive_32(&k1, b"label", b""); + let b = blake3_keyed_derive_32(&k2, b"label", b""); + assert_ne!(a, b); + } } diff --git a/crates/lumen-core/src/ids.rs b/crates/lumen-core/src/ids.rs index dbdb49b..f0e7ef8 100644 --- a/crates/lumen-core/src/ids.rs +++ b/crates/lumen-core/src/ids.rs @@ -8,10 +8,10 @@ use std::fmt; use std::str::FromStr; -use rand_core::RngCore; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use crate::error::{Error, Result}; +use crate::rng::Rng; macro_rules! binary_id { ($name:ident, $doc:literal) => { @@ -21,7 +21,7 @@ macro_rules! binary_id { impl $name { #[doc = "암호학적으로 안전한 RNG 로 임의의 ID 를 생성합니다."] - pub fn random(rng: &mut R) -> Self { + pub fn random(rng: &mut R) -> Self { let mut bytes = [0u8; 16]; rng.fill_bytes(&mut bytes); Self(bytes) @@ -169,7 +169,7 @@ impl<'de> Deserialize<'de> for ToolId { #[cfg(test)] mod tests { use super::*; - use rand::rngs::OsRng; + use crate::rng::OsRng; #[test] fn agent_id_hex_roundtrip() { diff --git a/crates/lumen-core/src/lib.rs b/crates/lumen-core/src/lib.rs index ddc5f89..e118227 100644 --- a/crates/lumen-core/src/lib.rs +++ b/crates/lumen-core/src/lib.rs @@ -11,10 +11,12 @@ pub mod crypto; pub mod error; pub mod hash; pub mod ids; +pub mod rng; pub mod time; pub use crypto::{Signature, SigningKey, VerifyingKey}; pub use error::{Error, Result}; pub use hash::Blake3Hash; pub use ids::{AgentId, CapabilityId, RequestId, ToolId}; +pub use rng::{HashDrbg, OsRng, Rng}; pub use time::Timestamp; diff --git a/crates/lumen-core/src/rng.rs b/crates/lumen-core/src/rng.rs new file mode 100644 index 0000000..0d4814d --- /dev/null +++ b/crates/lumen-core/src/rng.rs @@ -0,0 +1,135 @@ +//! Lumen의 RNG 추상화 모듈입니다. +//! +//! 폐쇄형(Air-Gapped) K0 마이크로커널 환경 호환을 위해 외부 +//! `rand` / `rand_core` 크레이트 대신 `elib-k0-nt/rng` 의 OS 엔트로피 +//! 수집과 NIST SP 800-90A Hash_DRBG 를 그대로 노출하는 얇은 래퍼입니다. +//! +//! # 트레이트와 구체 타입 +//! +//! - [`Rng`] : 단일 메서드 `fill_bytes` 만 갖는 최소 트레이트. 임의의 +//! 고정 길이 식별자 / 키 시드 / nonce 채움에 사용됩니다. +//! - [`OsRng`] : OS 엔트로피 (Linux `getrandom`, macOS / *BSD `arc4random`, +//! Windows BCrypt 등) 를 직접 사용하는 무상태 RNG. 프로덕션 기본값. +//! - [`HashDrbg`] : OS 엔트로피로 한번 시드된 SHA-256 기반 Hash_DRBG. +//! 동일 페어가 다량의 키 자료를 생성할 때 외부 entropy 호출 횟수를 +//! 줄이고 reseed 인터벌을 명시적으로 제어할 수 있습니다. + +use elib_rng::{DrbgError, HashDRBGSHA256}; +use elib_zeroize::Zeroize; + +use crate::error::{Error, Result}; + +/// 단일 책임의 RNG 트레이트입니다. +/// +/// `rand_core::RngCore` 를 대체합니다. 호출자는 이 트레이트만 의존하면 +/// 테스트에서는 결정론적 RNG (예: 직접 시딩한 [`HashDrbg`]) 를, 프로덕션 +/// 에서는 [`OsRng`] 를 주입할 수 있습니다. +pub trait Rng { + /// 주어진 슬라이스를 암호학적으로 안전한 난수로 채웁니다. + /// + /// # Panics + /// OS 엔트로피 소스가 사용 불가 등 회복 불가능한 실패 시 panic 합니다. + /// 회복 가능한 처리가 필요한 경우 [`Rng::try_fill_bytes`] 를 사용하세요. + fn fill_bytes(&mut self, dst: &mut [u8]) { + self.try_fill_bytes(dst) + .expect("CSPRNG fill_bytes failed irrecoverably") + } + + /// 주어진 슬라이스를 암호학적으로 안전한 난수로 채우고, 실패를 + /// `Result` 로 보고합니다. + fn try_fill_bytes(&mut self, dst: &mut [u8]) -> Result<()>; +} + +/// OS 엔트로피 직접 사용 RNG (무상태). +/// +/// 매 호출마다 `elib-k0-nt/rng::os_entropy::fill_bytes` 를 호출해 OS 의 +/// 보안 엔트로피 소스를 직접 사용합니다. 단일 호출당 32~64 바이트 수준의 +/// 사용에는 충분히 빠르고, 이 정도 RNG 호출 빈도는 핸드셰이크 / ID 생성 +/// 등 매 단위 동작당 1~2 회뿐입니다. +#[derive(Clone, Copy, Debug, Default)] +pub struct OsRng; + +impl Rng for OsRng { + fn try_fill_bytes(&mut self, dst: &mut [u8]) -> Result<()> { + elib_rng::os_entropy::fill_bytes(dst).map_err(map_drbg_err) + } +} + +/// SHA-256 기반 Hash_DRBG 어댑터. +/// +/// OS 엔트로피로 한번 시드한 뒤 [`Rng`] 인터페이스로 노출합니다. NIST +/// SP 800-90A 의 reseed 인터벌 (`2^48` 호출) 도달 시 자동으로 OS 엔트 +/// 로피를 추가 수집해 reseed 합니다. 호출자는 이 동작을 신경 쓸 필요가 +/// 없습니다. +pub struct HashDrbg(HashDRBGSHA256); + +impl HashDrbg { + /// OS 엔트로피로 새 DRBG 인스턴스를 생성합니다. + /// + /// `personalization` 은 "이 DRBG 인스턴스가 어떤 컨텍스트에서 사용 + /// 되는지" 를 나타내는 도메인 분리 라벨입니다 (예: `b"lumen.agent.v1"`). + /// 동일 컨텍스트의 두 호출자가 다른 DRBG 출력을 받도록 보장합니다. + pub fn from_os(personalization: Option<&[u8]>) -> Result { + HashDRBGSHA256::new_from_os(personalization) + .map(Self) + .map_err(map_drbg_err) + } +} + +impl Rng for HashDrbg { + fn try_fill_bytes(&mut self, dst: &mut [u8]) -> Result<()> { + // 단일 generate 호출당 최대 65536 바이트. lumen 호출자는 보통 + // 16~64 바이트만 채우므로 일반적으로 단일 호출로 끝납니다. + let mut remaining = dst; + while !remaining.is_empty() { + let chunk = remaining.len().min(65_536); + let (head, tail) = remaining.split_at_mut(chunk); + match self.0.generate(head, None) { + Ok(()) => {} + Err(DrbgError::ReseedRequired) => { + let mut entropy = [0u8; 64]; + elib_rng::os_entropy::fill_bytes(&mut entropy).map_err(map_drbg_err)?; + let reseed_res = self.0.reseed(&entropy, None); + // 엔트로피가 DRBG 내부 상태에 흡수되었으므로 스택 사본은 + // 즉시 zeroize 합니다 - 실패 경로에서도 누설되지 않도록 + // map_err 보다 먼저 수행합니다. + entropy.zeroize(); + reseed_res.map_err(map_drbg_err)?; + self.0.generate(head, None).map_err(map_drbg_err)?; + } + Err(e) => return Err(map_drbg_err(e)), + } + remaining = tail; + } + Ok(()) + } +} + +fn map_drbg_err(e: DrbgError) -> Error { + Error::Crypto(format!("rng: {e:?}")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn os_rng_fills_bytes() { + let mut a = [0u8; 32]; + let mut b = [0u8; 32]; + OsRng.fill_bytes(&mut a); + OsRng.fill_bytes(&mut b); + assert_ne!(a, b, "OsRng must produce distinct outputs across calls"); + assert_ne!(a, [0u8; 32], "OsRng must not return all-zero buffer"); + } + + #[test] + fn hash_drbg_fills_bytes() { + let mut drbg = HashDrbg::from_os(Some(b"lumen.test.v1")).unwrap(); + let mut a = [0u8; 32]; + let mut b = [0u8; 32]; + drbg.fill_bytes(&mut a); + drbg.fill_bytes(&mut b); + assert_ne!(a, b); + } +} diff --git a/crates/lumen-core/tests/elib_wire_compat.rs b/crates/lumen-core/tests/elib_wire_compat.rs new file mode 100644 index 0000000..dc4923d --- /dev/null +++ b/crates/lumen-core/tests/elib_wire_compat.rs @@ -0,0 +1,187 @@ +//! 폐쇄 커널(K0) 환경 호환성 회귀 테스트. +//! +//! Lumen 의 `lumen-core` 래퍼가 `elib-k0-nt` 의 모듈을 *직접* 호출했을 때와 +//! **비트-동일한 출력** 을 산출함을 보증합니다. K0 의 elib-k0d-core +//! 디스패처는 elib-k0-nt 를 직접 호출하므로, 본 테스트가 통과하면 lumen 이 +//! 폐쇄망 피어와 와이어 호환됩니다. +//! +//! # 검증 범위 +//! - BLAKE3: `Blake3Hash::of` ≡ 직접 호출 +//! - BLAKE3 keyed KDF: `blake3_keyed_derive_32` ≡ 직접 호출 +//! - Ed25519: `SigningKey::sign` ≡ `elib_ed25519::sign` +//! - Ed25519: `VerifyingKey::verify` ≡ `elib_ed25519::verify` +//! - x25519 ECDH: 양방향 공유 비밀 일치 +//! - 결정성: 동일 입력 → 동일 출력 (RNG 가 끼지 않은 경로) + +use elib_blake::Blake3; +use elib_ed25519::{ + self as ed25519, PublicKey as RawPublicKey, SecretKey as RawSecretKey, + Signature as RawSignature, +}; +use lumen_core::hash::{blake3_keyed_derive_32, Blake3Hash}; +use lumen_core::{SigningKey, VerifyingKey}; + +fn collect_finalize(input: &[u8]) -> [u8; 32] { + let mut h = Blake3::new(); + h.update(input); + let buf = h.finalize().expect("finalize"); + let mut out = [0u8; 32]; + out.copy_from_slice(buf.as_slice()); + out +} + +#[test] +fn blake3_wrapper_byte_identical_to_direct_elib() { + let inputs: &[&[u8]] = &[b"", b"a", b"hello world", b"lumen.air-gap.kernel.compat.v1"]; + for input in inputs { + let lumen = Blake3Hash::of(input); + let direct = collect_finalize(input); + assert_eq!( + lumen.as_bytes(), + &direct, + "BLAKE3 wrapper must match direct elib-blake call (input len={})", + input.len() + ); + } +} + +#[test] +fn blake3_wrapper_byte_identical_for_large_input() { + // 청크 경계(>1 KiB) 와 cv-스택 머지가 모두 일어나는 크기. + let big = vec![0xA5u8; 16 * 1024]; + let lumen = Blake3Hash::of(&big); + let direct = collect_finalize(&big); + assert_eq!(lumen.as_bytes(), &direct); +} + +#[test] +fn blake3_keyed_derive_byte_identical_to_direct_elib() { + // lumen-channel 의 KDF 가 폐쇄망 피어의 KDF 와 비트-동일함을 회귀 보장. + let key = [0x11u8; 32]; + let label = b"lumen.encrypted.k-i2r.v1"; + let epoch_le = 42u64.to_le_bytes(); + + let via_wrapper = blake3_keyed_derive_32(&key, label, &epoch_le); + + let mut h = Blake3::new_keyed(&key); + h.update(label); + h.update(&epoch_le); + let buf = h.finalize().expect("finalize"); + let mut direct = [0u8; 32]; + direct.copy_from_slice(buf.as_slice()); + + assert_eq!(via_wrapper, direct); +} + +#[test] +fn ed25519_sign_verify_via_wrapper_matches_direct_elib() { + // 와이어로 운반되는 객체는 (msg, signature, public_key) 3-튜플이므로 + // - lumen 으로 서명하고 elib 직접 검증 + // - elib 로 서명하고 lumen 직접 검증 + // 두 방향 모두에서 동일하게 통과해야 합니다. + let seed = [0x42u8; 32]; + let msg = b"lumen.kernel.compat.ed25519.v1"; + + // lumen 측 서명 → elib 측 검증 + let sk_lumen = SigningKey::from_seed(&seed); + let vk_lumen = sk_lumen.verifying_key(); + let sig_lumen = sk_lumen.sign(msg); + + let raw_sk = RawSecretKey::from_bytes(&seed); + let raw_pk = RawPublicKey::from(&raw_sk); + assert_eq!(*raw_pk.as_bytes(), vk_lumen.to_bytes()); + + let raw_sig = RawSignature::from_bytes(&sig_lumen.to_bytes()); + ed25519::verify(msg, &raw_sig, &raw_pk).expect("elib-direct verify of lumen signature"); + + // elib 측 서명 → lumen 측 검증 + let direct_sig = ed25519::sign(msg, &raw_sk); + assert_eq!( + direct_sig.as_bytes(), + &sig_lumen.to_bytes(), + "Ed25519 결정론적 서명: lumen 과 elib 결과가 비트-동일해야 함" + ); + + let vk_lumen_via_raw = VerifyingKey::from_bytes(raw_pk.as_bytes()).unwrap(); + let lumen_sig_obj = lumen_core::Signature::from_bytes(direct_sig.as_bytes()); + vk_lumen_via_raw + .verify(msg, &lumen_sig_obj) + .expect("lumen-wrapper verify of elib direct signature"); +} + +#[test] +fn ed25519_signature_is_deterministic() { + // RFC 8032 Ed25519 는 결정론적 서명입니다 - 동일 (msg, sk) 는 항상 동일 + // 서명을 산출해야 하며, 와이어 호환성 회귀의 사전 조건입니다. + let seed = [0x77u8; 32]; + let sk = SigningKey::from_seed(&seed); + let s1 = sk.sign(b"msg-1"); + let s2 = sk.sign(b"msg-1"); + assert_eq!(s1.to_bytes(), s2.to_bytes()); +} + +#[test] +fn x25519_ecdh_produces_matching_shared_secret() { + // 양 피어가 elib_x25519 만으로 공유 비밀을 도출할 때 비트-동일한 32 바이트 + // 결과를 얻는다는 회귀 보장. lumen-channel::EncryptedChannel 의 핸드셰이크가 + // 이 동작 위에 의존합니다. + use elib_x25519::{PublicKey, SecretKey}; + + let sk_a = SecretKey::from_bytes([0x01u8; 32]); + let sk_b = SecretKey::from_bytes([0x02u8; 32]); + let pk_a = sk_a.public_key(); + let pk_b = sk_b.public_key(); + + // 한 쪽이 만든 공유 비밀은 다른 쪽이 자신의 secret 으로 만든 결과와 동일. + let shared_a = sk_a.diffie_hellman(&pk_b); + let shared_b = sk_b.diffie_hellman(&pk_a); + assert_eq!(shared_a.as_bytes(), shared_b.as_bytes()); + + // 와이어로 운반된 32 바이트 공개키만으로 라운드트립 가능해야 함. + let pk_a_wire: [u8; 32] = *pk_a.as_bytes(); + let pk_b_wire: [u8; 32] = *pk_b.as_bytes(); + let shared_a_again = sk_a.diffie_hellman(&PublicKey::from_bytes(pk_b_wire)); + let shared_b_again = sk_b.diffie_hellman(&PublicKey::from_bytes(pk_a_wire)); + assert_eq!(shared_a_again.as_bytes(), shared_b_again.as_bytes()); + assert_eq!(shared_a_again.as_bytes(), shared_a.as_bytes()); +} + +#[test] +fn aes256_gcm_encrypts_with_tag_appended_format_used_by_lumen_channel() { + // lumen-channel 은 와이어 포맷을 `ciphertext || tag` 로 직렬화합니다. + // elib-k0-nt 의 raw API 는 (ct, tag) 분리 출력이므로 두 형태가 정확히 + // 동일한 17~N 바이트 와이어를 만드는지 회귀 보장. + use elib_aes::{AES256GCM, GCM_NONCE_SIZE, GCM_TAG_SIZE}; + + let key = [0x33u8; 32]; + let nonce = [0x44u8; GCM_NONCE_SIZE]; + let aad = b"lumen.aad.v1"; + let plaintext = b"lumen.kernel.compat.aesgcm.v1"; + + let cipher = AES256GCM::new(&key); + let mut ct_a = vec![0u8; plaintext.len()]; + let mut tag_a = [0u8; GCM_TAG_SIZE]; + cipher.encrypt(&nonce, aad, plaintext, &mut ct_a, &mut tag_a); + + let mut ct_b = vec![0u8; plaintext.len()]; + let mut tag_b = [0u8; GCM_TAG_SIZE]; + cipher.encrypt(&nonce, aad, plaintext, &mut ct_b, &mut tag_b); + + // 결정성: 동일 (key, nonce, aad, pt) → 동일 (ct, tag) + assert_eq!(ct_a, ct_b); + assert_eq!(tag_a, tag_b); + + // 와이어 형식: ciphertext || tag + let mut wire = ct_a.clone(); + wire.extend_from_slice(&tag_a); + + // 디코드 측 검증 (lumen-channel 이 수행하는 절차와 동일) + let split = wire.len() - GCM_TAG_SIZE; + let (ct_back, tag_back) = wire.split_at(split); + let mut tag_arr = [0u8; GCM_TAG_SIZE]; + tag_arr.copy_from_slice(tag_back); + let mut pt_back = vec![0u8; ct_back.len()]; + let ok = cipher.decrypt(&nonce, aad, ct_back, &tag_arr, &mut pt_back); + assert!(ok, "AEAD 검증 실패"); + assert_eq!(pt_back, plaintext); +} diff --git a/crates/lumen-inference/Cargo.toml b/crates/lumen-inference/Cargo.toml index 8cc8064..68c7fa4 100644 --- a/crates/lumen-inference/Cargo.toml +++ b/crates/lumen-inference/Cargo.toml @@ -6,7 +6,7 @@ rust-version.workspace = true license.workspace = true authors.workspace = true repository.workspace = true -description = "Lumen - InferenceEngine trait + streaming + verified loader + quantization config; candle/llama.cpp behind feature flags" +description = "Lumen - InferenceEngine trait + streaming + verified loader + quantization + minimal BPE tokenizer; llama.cpp behind feature flag" keywords.workspace = true categories.workspace = true @@ -15,11 +15,8 @@ workspace = true [features] default = [] -# ONNX 라우팅 엔진 (candle-onnx 기반, 기존 CandleEngine). -candle = ["dep:candle-core", "dep:candle-onnx"] -# 전체 LLM 생성 엔진 (candle-transformers + GGUF 양자화 모델 + HF 토크나이저). -candle-llm = ["dep:candle-core", "dep:candle-transformers", "dep:tokenizers"] # llama.cpp 백엔드 (cmake 빌드 필요, 현재 인터페이스 스텁). +# 전체 LLM 텍스트 생성이 필요한 경우 이 백엔드를 사용하세요. llama-cpp = [] [dependencies] @@ -32,12 +29,7 @@ futures.workspace = true serde = { workspace = true } serde_json.workspace = true tokio = { workspace = true, features = ["sync"] } -blake3 = { workspace = true } tracing.workspace = true -candle-core = { workspace = true, optional = true } -candle-onnx = { workspace = true, optional = true } -candle-transformers = { workspace = true, optional = true } -tokenizers = { workspace = true, optional = true } [dev-dependencies] tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/crates/lumen-inference/src/backend.rs b/crates/lumen-inference/src/backend.rs index c0f7042..152a6bb 100644 --- a/crates/lumen-inference/src/backend.rs +++ b/crates/lumen-inference/src/backend.rs @@ -8,14 +8,16 @@ //! | 백엔드 | feature | 설명 | //! |--------------------|--------------|--------------------------------------| //! | `Dummy` | *(없음)* | 결정론적 패턴 매칭, 테스트 전용. | -//! | `CandleOnnx` | `candle` | ONNX 라우팅 (기존 `CandleEngine`). | -//! | `CandleLlm` | `candle-llm` | GGUF LLM (candle-transformers). | -//! | `LlamaCpp` | `llama-cpp` | llama.cpp 스텁 (v0.5 완성 예정). | +//! | `LlamaCpp` | `llama-cpp` | llama.cpp 연동 (v0.5 완성 예정). | +//! +//! `candle` (candle-core / candle-onnx) 백엔드는 폐쇄형 환경 호환을 위해 +//! v0.4 에서 제거되었습니다. ONNX 도구 라우팅이 필요한 경우 자체 BPE +//! 토크나이저 + 호스트 TEE 의 추론 서비스로 [`crate::ChannelEngine`] 을 +//! 통해 forward 하는 경로를 사용하세요. 전체 LLM 텍스트 생성은 `llama-cpp` +//! 백엔드를 사용하세요. -#[cfg(any(feature = "candle", feature = "candle-llm", feature = "llama-cpp"))] +#[cfg(feature = "llama-cpp")] use crate::loader::VerifiedModelHandle; -#[cfg(feature = "candle-llm")] -use std::path::PathBuf; use std::sync::Arc; use lumen_core::Result; @@ -28,28 +30,6 @@ pub enum BackendConfig { /// 테스트 및 데모용 결정론적 패턴-매칭 엔진. Dummy, - /// ONNX 모델 기반 도구 라우팅 엔진. - /// - /// `candle` feature 가 활성화되어야 합니다. - #[cfg(feature = "candle")] - CandleOnnx { - /// 검증된 `.onnx` 모델 핸들. - handle: VerifiedModelHandle, - /// argmax 인덱스 → ToolId 매핑 테이블. - tool_table: Vec, - }, - - /// GGUF 양자화 LLM (candle-transformers 기반). - /// - /// `candle-llm` feature 가 활성화되어야 합니다. - #[cfg(feature = "candle-llm")] - CandleLlm { - /// 검증된 `.gguf` 모델 핸들. - handle: VerifiedModelHandle, - /// HuggingFace `tokenizer.json` 경로. - tokenizer_path: PathBuf, - }, - /// llama.cpp 기반 엔진 (v0.5 완성 예정). /// /// `llama-cpp` feature 가 활성화되어야 합니다. @@ -72,24 +52,6 @@ pub fn create_engine(config: BackendConfig) -> Result> match config { BackendConfig::Dummy => Ok(Arc::new(DummyEngine::new())), - #[cfg(feature = "candle")] - BackendConfig::CandleOnnx { handle, tool_table } => { - use crate::candle::CandleEngine; - Ok(Arc::new(CandleEngine::load(handle.path(), tool_table)?)) - } - - #[cfg(feature = "candle-llm")] - BackendConfig::CandleLlm { - handle, - tokenizer_path, - } => { - use crate::candle_llm::CandleLlmEngine; - Ok(Arc::new(CandleLlmEngine::from_gguf( - &handle, - tokenizer_path, - )?)) - } - #[cfg(feature = "llama-cpp")] BackendConfig::LlamaCpp { handle, diff --git a/crates/lumen-inference/src/candle.rs b/crates/lumen-inference/src/candle.rs deleted file mode 100644 index 316d73c..0000000 --- a/crates/lumen-inference/src/candle.rs +++ /dev/null @@ -1,165 +0,0 @@ -//! candle-onnx 기반 추론 엔진 - `candle` feature 가 활성화된 경우만 컴파일. -//! -//! ## 범위 (v0.3) -//! -//! - 작은 ONNX 분류 모델을 [`std::path::Path`] 로 로드. -//! - 프롬프트를 BLAKE3 해시 -> 고정 길이 `f32` 벡터로 결정론적 인코딩. -//! - 모델 forward 후 출력 벡터의 argmax 를 도구 ID 매핑으로 변환. -//! - 모든 추론 경로는 CPU 만 (CUDA/Metal feature 미활성). -//! -//! ## 결정성 -//! -//! candle 의 `f32` 연산은 미세하게 비결정적일 수 있으므로 **이 백엔드의 -//! 출력은 ZK witness 에 직접 묶어서는 안 됩니다.** 진정한 결정성 추론은 -//! `lumen-fixed` 위에 작성된 자체 quantize 그래프 (v0.5 예정) 에서 다룹니다. -//! 현재는 *Tool routing decision* 에 ONNX 출력을 *입력으로* 사용하되, ZK -//! 증명은 *호스트가 결정한 정수 라우팅 인덱스* 만을 바인드합니다 - float 의 -//! 전파를 차단하기 위해서. - -use std::path::PathBuf; - -use async_trait::async_trait; -use candle_core::{DType, Device, Tensor}; -use lumen_core::{Error, Result, ToolId}; - -use crate::{Completion, InferenceEngine, SamplingParams, ToolCall}; - -/// candle-onnx 백엔드. ONNX 모델 한 개를 로드해 inference 에 사용합니다. -pub struct CandleEngine { - /// ONNX 모델 객체 (candle 의 `ModelProto`). - model: candle_onnx::onnx::ModelProto, - /// ONNX 입력 텐서 이름 (모델의 첫 입력). - input_name: String, - /// ONNX 입력 길이 (1차원 f32 벡터로 가정). - input_len: usize, - /// argmax 인덱스를 ToolId 로 매핑하는 테이블. - tool_table: Vec, - /// 진단용 모델 경로 (audit 출력). - model_path: PathBuf, -} - -impl CandleEngine { - /// 모델 파일 (`.onnx`) 을 로드하고 `tool_table` 을 핀 합니다. - pub fn load(path: impl Into, tool_table: Vec) -> Result { - let model_path = path.into(); - let model = candle_onnx::read_file(&model_path) - .map_err(|e| Error::Inference(format!("candle-onnx read: {e}")))?; - let graph = model - .graph - .as_ref() - .ok_or_else(|| Error::Inference("candle: ONNX graph 없음".into()))?; - let input_info = graph - .input - .first() - .ok_or_else(|| Error::Inference("candle: ONNX 입력 없음".into()))?; - let input_name = input_info.name.clone(); - let input_len = input_tensor_length(input_info)?; - Ok(Self { - model, - input_name, - input_len, - tool_table, - model_path, - }) - } - - /// 프롬프트를 결정론적 `f32` 벡터로 인코딩. - /// - /// BLAKE3 다이제스트의 32 바이트를 8 개 `f32` 로 분할한 다음, 모델 입력 - /// 길이에 맞게 반복/잘라냅니다. 진짜 토크나이저는 v0.4 에서. - fn encode_prompt(&self, prompt: &str) -> Vec { - let digest = blake3::hash(prompt.as_bytes()); - let bytes = digest.as_bytes(); - let mut out = Vec::with_capacity(self.input_len); - for i in 0..self.input_len { - let byte = bytes[i % bytes.len()]; - // 0..255 -> -1.0..1.0 (대략). - let v = (byte as f32 / 127.5) - 1.0; - out.push(v); - } - out - } - - fn run_inference(&self, prompt: &str) -> Result> { - let input_data = self.encode_prompt(prompt); - let input = Tensor::from_vec(input_data, (1, self.input_len), &Device::Cpu) - .map_err(|e| Error::Inference(format!("candle tensor: {e}")))? - .to_dtype(DType::F32) - .map_err(|e| Error::Inference(format!("candle dtype: {e}")))?; - let mut inputs = std::collections::HashMap::new(); - inputs.insert(self.input_name.clone(), input); - let outputs = candle_onnx::simple_eval(&self.model, inputs) - .map_err(|e| Error::Inference(format!("candle eval: {e}")))?; - let (_name, tensor) = outputs - .into_iter() - .next() - .ok_or_else(|| Error::Inference("candle: ONNX 출력 없음".into()))?; - tensor - .flatten_all() - .and_then(|t| t.to_vec1::()) - .map_err(|e| Error::Inference(format!("candle output: {e}"))) - } - - /// 출력 벡터의 argmax 인덱스. - fn argmax(scores: &[f32]) -> Option { - let (idx, _) = scores - .iter() - .enumerate() - .max_by(|(_, a), (_, b)| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal))?; - Some(idx) - } - - /// 모델 경로 (디버그/audit 용). - pub fn model_path(&self) -> &std::path::Path { - &self.model_path - } -} - -#[async_trait] -impl InferenceEngine for CandleEngine { - async fn complete(&self, prompt: &str, _params: &SamplingParams) -> Result { - let scores = self.run_inference(prompt)?; - let chosen = Self::argmax(&scores).and_then(|i| self.tool_table.get(i).cloned()); - - let tool_call = chosen.map(|tool_id| ToolCall { - id: tool_id, - args_json: serde_json::json!({ "prompt": prompt }).to_string(), - }); - - Ok(Completion { - text: format!("[candle] scores={scores:?}"), - tool_call, - }) - } -} - -fn input_tensor_length(info: &candle_onnx::onnx::ValueInfoProto) -> Result { - let ty = info - .r#type - .as_ref() - .ok_or_else(|| Error::Inference("candle: input type 없음".into()))?; - let value = ty - .value - .as_ref() - .ok_or_else(|| Error::Inference("candle: input value 없음".into()))?; - use candle_onnx::onnx::type_proto::Value as TPV; - let TPV::TensorType(tensor_type) = value else { - return Err(Error::Inference("candle: 비텐서 입력 미지원".into())); - }; - let shape = tensor_type - .shape - .as_ref() - .ok_or_else(|| Error::Inference("candle: shape 없음".into()))?; - // 마지막 차원을 입력 길이로 채택. 첫 차원이 batch size 라고 가정. - let dims = &shape.dim; - let last = dims - .last() - .ok_or_else(|| Error::Inference("candle: 빈 shape".into()))?; - use candle_onnx::onnx::tensor_shape_proto::dimension::Value as DV; - match &last.value { - Some(DV::DimValue(v)) => Ok(*v as usize), - _ => Err(Error::Inference( - "candle: 동적 shape 미지원 (마지막 dim 이 정적이어야 함)".into(), - )), - } -} diff --git a/crates/lumen-inference/src/candle_llm.rs b/crates/lumen-inference/src/candle_llm.rs deleted file mode 100644 index bb4c3f5..0000000 --- a/crates/lumen-inference/src/candle_llm.rs +++ /dev/null @@ -1,301 +0,0 @@ -//! candle-transformers 기반 LLM 추론 엔진 — `candle-llm` feature 가 활성화된 경우만 컴파일. -//! -//! ## 지원 모델 -//! -//! GGUF 양자화 가중치 (LLaMA / Mistral 계열) 를 candle-transformers 의 -//! `quantized_llama::ModelWeights` 로 로드합니다. Safetensors 포맷은 -//! 추후 지원 예정입니다. -//! -//! ## 보안 불변식 -//! -//! 생성자 [`CandleLlmEngine::from_gguf`] 는 raw 경로 대신 -//! [`VerifiedModelHandle`] 을 받습니다. 검증을 건너뛰면 컴파일 에러가 -//! 발생합니다. -//! -//! ## 스트리밍 -//! -//! [`StreamingEngine::stream_complete`] 는 별도 `tokio::task` 에서 생성 루프를 -//! 실행하고 [`tokio::sync::mpsc`] 채널을 통해 토큰을 전달합니다. -//! 스트림을 drop 하면 채널이 닫히고 task 가 자동 종료됩니다. -//! -//! ## 결정성 주의 -//! -//! candle 의 f32 연산은 CPU 아키텍처 간 미세하게 비결정적일 수 있습니다. -//! 텍스트 생성 출력은 ZK witness 에 직접 바인딩하지 마세요. 도구 라우팅 -//! *결정* (정수 인덱스) 만 ZK 에 바인딩하며, `CandleEngine` (ONNX) 과 -//! 동일한 접근 방식을 따릅니다. - -use std::path::Path; -use std::sync::Arc; - -use async_trait::async_trait; -use candle_core::{DType, Device, Tensor}; -use candle_transformers::generation::LogitsProcessor; -use candle_transformers::models::quantized_llama::ModelWeights; -use futures::stream; -use lumen_core::{Error, Result}; -use tokenizers::Tokenizer; -use tokio::sync::Mutex; - -use crate::loader::VerifiedModelHandle; -use crate::streaming::{FinishReason, StreamingEngine, Token, TokenStream}; -use crate::{Completion, InferenceEngine, SamplingParams}; - -/// candle-transformers 기반 LLM 엔진. -/// -/// 생성은 항상 CPU 에서 실행됩니다 (`Device::Cpu`). GPU/Metal 지원은 -/// 추후 feature flag 로 추가 예정입니다. -pub struct CandleLlmEngine { - model: Arc>, - tokenizer: Arc, - eos_token_id: Option, -} - -impl CandleLlmEngine { - /// GGUF 양자화 모델을 검증된 핸들로부터 로드합니다. - /// - /// `tokenizer_path` 는 HuggingFace `tokenizer.json` 형식이어야 합니다. - /// - /// # 에러 - /// - /// - 파일 열기 실패 → [`Error::Inference`] - /// - GGUF 파싱 실패 → [`Error::Inference`] - /// - 토크나이저 로드 실패 → [`Error::Inference`] - pub fn from_gguf( - handle: &VerifiedModelHandle, - tokenizer_path: impl AsRef, - ) -> Result { - let device = Device::Cpu; - - tracing::info!( - target: "lumen.inference.candle_llm", - path = ?handle.path(), - model = %handle.model_info.name, - "GGUF 모델 로딩 중" - ); - - // GGUF 파싱 - let mut file = std::fs::File::open(handle.path()) - .map_err(|e| Error::Inference(format!("GGUF 파일 열기 실패: {e}")))?; - let gguf_content = candle_core::quantized::gguf_file::Content::read(&mut file) - .map_err(|e| Error::Inference(format!("GGUF 파싱 실패: {e}")))?; - let model = ModelWeights::from_gguf(gguf_content, &mut file, &device) - .map_err(|e| Error::Inference(format!("모델 가중치 로드 실패: {e}")))?; - - // 토크나이저 로드 - let tokenizer = Tokenizer::from_file(tokenizer_path) - .map_err(|e| Error::Inference(format!("토크나이저 로드 실패: {e}")))?; - - // EOS 토큰 탐색 (LLaMA: , GPT-계열: <|endoftext|>) - let eos_token_id = tokenizer - .token_to_id("") - .or_else(|| tokenizer.token_to_id("<|endoftext|>")) - .or_else(|| tokenizer.token_to_id("")); - - tracing::info!( - target: "lumen.inference.candle_llm", - eos_token_id, - "모델 로드 완료" - ); - - Ok(Self { - model: Arc::new(Mutex::new(model)), - tokenizer: Arc::new(tokenizer), - eos_token_id, - }) - } - - /// 프롬프트를 토큰 ID 벡터로 인코딩합니다. - fn encode_prompt(&self, prompt: &str) -> Result> { - let encoding = self - .tokenizer - .encode(prompt, true) - .map_err(|e| Error::Inference(format!("인코딩 실패: {e}")))?; - Ok(encoding.get_ids().to_vec()) - } -} - -#[async_trait] -impl InferenceEngine for CandleLlmEngine { - /// 스트림을 내부적으로 소진해 전체 completion 텍스트로 조립합니다. - async fn complete(&self, prompt: &str, params: &SamplingParams) -> Result { - use futures::StreamExt as _; - let mut token_stream = self.stream_complete(prompt, params).await?; - let mut text = String::new(); - while let Some(tok) = token_stream.next().await { - text.push_str(&tok?.text); - } - Ok(Completion { - text, - tool_call: None, - }) - } -} - -#[async_trait] -impl StreamingEngine for CandleLlmEngine { - async fn stream_complete(&self, prompt: &str, params: &SamplingParams) -> Result { - let prompt_tokens = self.encode_prompt(prompt)?; - if prompt_tokens.is_empty() { - return Err(Error::Inference( - "프롬프트 토큰화 결과가 비어있습니다".into(), - )); - } - - let max_tokens = params.max_tokens as usize; - let seed = params.seed.unwrap_or(0); - let temperature = params.temperature; - let top_p = params.top_p; - let eos_token_id = self.eos_token_id; - let model = Arc::clone(&self.model); - let tokenizer = Arc::clone(&self.tokenizer); - - let (tx, rx) = tokio::sync::mpsc::channel::>(128); - - tokio::spawn(async move { - if let Err(e) = run_generation( - model, - tokenizer, - prompt_tokens, - max_tokens, - temperature, - top_p, - eos_token_id, - seed, - tx.clone(), - ) - .await - { - // 채널이 이미 닫혔으면(호출자가 drop) 에러 전송은 무시합니다. - let _ = tx.send(Err(e)).await; - } - }); - - // mpsc Receiver → futures::Stream 변환 - let s = stream::unfold(rx, |mut rx| async move { - rx.recv().await.map(|item| (item, rx)) - }); - - Ok(Box::pin(s)) - } -} - -/// 생성 루프 (별도 tokio task 에서 실행). -/// -/// 모델 잠금을 보유한 채 토큰을 순차적으로 생성하고 `tx` 로 전달합니다. -/// 채널 수신자가 drop 되면 (`tx.send` 실패) 즉시 종료합니다. -async fn run_generation( - model: Arc>, - tokenizer: Arc, - prompt_tokens: Vec, - max_tokens: usize, - temperature: f32, - top_p: f32, - eos_token_id: Option, - seed: u64, - tx: tokio::sync::mpsc::Sender>, -) -> Result<()> { - let device = Device::Cpu; - - // LogitsProcessor: temperature == 0 이면 greedy decoding. - let temp_opt = if temperature == 0.0 { - None - } else { - Some(temperature as f64) - }; - let top_p_opt = if top_p > 0.0 && top_p < 1.0 { - Some(top_p as f64) - } else { - None - }; - let mut logits_processor = LogitsProcessor::new(seed, temp_opt, top_p_opt); - - let prompt_len = prompt_tokens.len(); - - // 모델 잠금을 보유한 채 프롬프트 prefill + 첫 토큰 샘플링 - let mut model_guard = model.lock().await; - - let input = Tensor::new(prompt_tokens.as_slice(), &device) - .and_then(|t| t.unsqueeze(0)) - .map_err(|e| Error::Inference(format!("프롬프트 텐서 생성 실패: {e}")))?; - - // prefill — KV 캐시 위치 0부터 시작 - let logits = model_guard - .forward(&input, 0) - .map_err(|e| Error::Inference(format!("prefill forward 실패: {e}")))?; - - // 마지막 위치의 로짓 추출: [1, seq_len, vocab] → [vocab] - let logits = extract_last_logit(logits, prompt_len)?; - - let mut next_token = logits_processor - .sample(&logits) - .map_err(|e| Error::Inference(format!("샘플링 실패: {e}")))?; - - let mut pos = prompt_len; - - for step in 0..max_tokens { - let decoded = tokenizer - .decode(&[next_token], true) - .map_err(|e| Error::Inference(format!("토큰 디코딩 실패: {e}")))?; - - let is_eos = eos_token_id.map_or(false, |eos| next_token == eos); - let finish_reason = if is_eos { - Some(FinishReason::Eos) - } else if step + 1 >= max_tokens { - Some(FinishReason::MaxTokens) - } else { - None - }; - - let token = Token { - id: next_token, - text: decoded, - logprob: None, - finish_reason, - }; - - // 채널 수신자가 drop 되면 생성을 중단합니다. - if tx.send(Ok(token)).await.is_err() { - break; - } - - if is_eos || finish_reason.is_some() { - break; - } - - // 다음 토큰 생성 - let input = Tensor::new(&[next_token], &device) - .and_then(|t| t.unsqueeze(0)) - .map_err(|e| Error::Inference(format!("단일 토큰 텐서 실패: {e}")))?; - - let logits = model_guard - .forward(&input, pos) - .map_err(|e| Error::Inference(format!("autoregressive forward 실패: {e}")))?; - - let logits = extract_last_logit(logits, 1)?; - - next_token = logits_processor - .sample(&logits) - .map_err(|e| Error::Inference(format!("샘플링 실패: {e}")))?; - - pos += 1; - } - - Ok(()) -} - -/// `[1, n, vocab]` 텐서에서 위치 `n - 1` 의 로짓 벡터 `[vocab]` 를 추출합니다. -fn extract_last_logit(logits: Tensor, n: usize) -> Result { - // [1, n, vocab] → [n, vocab] - let logits = logits - .squeeze(0) - .map_err(|e| Error::Inference(format!("batch squeeze 실패: {e}")))?; - // [n, vocab] → [vocab] (마지막 시퀀스 위치) - let logits = logits - .get(n - 1) - .map_err(|e| Error::Inference(format!("마지막 로짓 추출 실패: {e}")))?; - // 양자화 텐서를 f32 로 변환 (LogitsProcessor 요구사항) - logits - .to_dtype(DType::F32) - .map_err(|e| Error::Inference(format!("f32 변환 실패: {e}"))) -} diff --git a/crates/lumen-inference/src/lib.rs b/crates/lumen-inference/src/lib.rs index 11bdbd0..4fcc7ed 100644 --- a/crates/lumen-inference/src/lib.rs +++ b/crates/lumen-inference/src/lib.rs @@ -13,21 +13,26 @@ //! | 백엔드 | feature | 용도 | //! |------------------|--------------|----------------------------------| //! | `DummyEngine` | *(없음)* | 결정론적 테스트 / 데모 | -//! | `CandleEngine` | `candle` | ONNX 도구 라우팅 | -//! | `CandleLlmEngine`| `candle-llm` | GGUF LLM 텍스트 생성 + 스트리밍 | -//! | `LlamaCppEngine` | `llama-cpp` | llama.cpp (v0.5 예정) | +//! | `LlamaCppEngine` | `llama-cpp` | llama.cpp 연동 (v0.5 예정) | //! | `ChannelEngine` | *(없음)* | TEE 채널 forward | //! //! [`BackendConfig`] + [`backend::create_engine`] 으로 팩토리 패턴을 사용할 //! 수 있습니다. +//! +//! ## 폐쇄형(Air-Gapped) 환경 메모 +//! +//! HuggingFace 의 `tokenizers` 크레이트와 `candle-*` (candle-core / +//! candle-onnx / candle-transformers) 백엔드는 외부 다운로드 코드를 +//! 포함하고 의존 트리가 매우 커서 폐쇄망 빌드 부담이 큽니다. 따라서 +//! v0.4 부터 모든 candle feature 는 제거되었습니다. 도구 라우팅용 BPE +//! 토크나이저가 필요한 경우 자체 구현인 [`tokenizer::BpeTokenizer`] 를 +//! 사용하세요. 전체 LLM 텍스트 생성이 필요한 경우 `llama-cpp` (llama.cpp +//! 백엔드, 내장 BPE / SentencePiece 토크나이저) 또는 호스트 TEE 의 +//! 추론 서비스로 [`ChannelEngine`] 을 통해 forward 하는 경로를 사용하세요. #![forbid(unsafe_code)] #![warn(missing_docs)] -#[cfg(feature = "candle")] -pub mod candle; -#[cfg(feature = "candle-llm")] -pub mod candle_llm; #[cfg(feature = "llama-cpp")] pub mod llama_cpp; @@ -37,6 +42,7 @@ pub mod loader; pub mod quantize; pub mod streaming; pub mod tee_channel; +pub mod tokenizer; use async_trait::async_trait; use lumen_core::{Result, ToolId}; @@ -117,8 +123,7 @@ pub use loader::{VerifiedModelHandle, VerifiedModelLoader}; pub use quantize::{GgufLevel, QuantizationConfig, QuantizationKind}; pub use streaming::{FinishReason, StreamingEngine, Token, TokenStream}; pub use tee_channel::ChannelEngine; +pub use tokenizer::{BpeTokenizer, TokenId}; -#[cfg(feature = "candle-llm")] -pub use candle_llm::CandleLlmEngine; #[cfg(feature = "llama-cpp")] pub use llama_cpp::LlamaCppEngine; diff --git a/crates/lumen-inference/src/llama_cpp.rs b/crates/lumen-inference/src/llama_cpp.rs index bca3dcd..bc06e49 100644 --- a/crates/lumen-inference/src/llama_cpp.rs +++ b/crates/lumen-inference/src/llama_cpp.rs @@ -52,7 +52,7 @@ impl LlamaCppEngine { impl InferenceEngine for LlamaCppEngine { async fn complete(&self, _prompt: &str, _params: &SamplingParams) -> Result { Err(Error::NotImplemented( - "LlamaCppEngine: llama-cpp-2 연동이 v0.5 에서 구현됩니다".into(), + "LlamaCppEngine: llama-cpp-2 연동이 v0.5 에서 구현됩니다", )) } } @@ -65,7 +65,7 @@ impl StreamingEngine for LlamaCppEngine { _params: &SamplingParams, ) -> Result { Err(Error::NotImplemented( - "LlamaCppEngine: 스트리밍은 v0.5 에서 구현됩니다".into(), + "LlamaCppEngine: 스트리밍은 v0.5 에서 구현됩니다", )) } } diff --git a/crates/lumen-inference/src/tokenizer.rs b/crates/lumen-inference/src/tokenizer.rs new file mode 100644 index 0000000..9b29248 --- /dev/null +++ b/crates/lumen-inference/src/tokenizer.rs @@ -0,0 +1,321 @@ +//! 폐쇄형(Air-Gapped) 환경 친화적 최소 BPE 토크나이저. +//! +//! HuggingFace `tokenizers` 크레이트 의존을 끊기 위한 자체 구현입니다. +//! GGUF 토크나이저 메타데이터(`tokenizer.ggml.tokens` + `tokenizer.ggml.merges`) +//! 또는 GPT-2 호환 `vocab.json` + `merges.txt` 를 그대로 받아 사용할 수 있는 +//! 바이트 수준(byte-level) BPE 인코더/디코더입니다. +//! +//! # 결정성 +//! +//! 본 토크나이저의 출력은 동일 vocab/merges 입력에 대해 비트-동일한 결정론적 +//! 결과를 보장합니다. 부동소수 / 해시 테이블 비결정성을 피하기 위해 모든 내부 +//! 자료구조는 [`BTreeMap`] 으로 통일했습니다. +//! +//! # 알고리즘 복잡도 +//! +//! 단순 $O(n^2)$ 머지 루프(가장 낮은 rank 의 인접 페어를 매 라운드 검색) +//! 입니다. 일반적으로 토크나이저 입력은 ≤ 8 KiB 이고 호출 빈도가 적어 +//! 실제 latency 에는 무시할 만한 수준입니다. 더 큰 입력에 대해서는 우선순위 +//! 큐 기반 최적화(`O(n log n)`) 가 가능하나 본 마이크로커널 빌드에서는 +//! 필요성이 낮아 의도적으로 채택하지 않았습니다. +//! +//! # GGUF 호환 +//! +//! GGUF 의 토크나이저 메타데이터를 그대로 매핑할 수 있도록 다음 규약을 +//! 따릅니다: +//! +//! - vocab 은 `Vec>` 형태로 받으며, 인덱스가 곧 토큰 ID 입니다. +//! - merges 는 `Vec<(Vec, Vec)>` 형태로 받으며, 등록 순서가 곧 +//! 머지 우선순위(낮을수록 먼저 적용) 입니다. +//! - 토큰 ID 는 [`TokenId`] (`u32`) 로 표현됩니다. +//! +//! 실제 GGUF 파일에서 메타데이터를 읽어내는 파서는 추후 `gguf` 모듈로 +//! 분리됩니다 - 본 모듈은 순수 BPE 로직만 다룹니다. + +use std::collections::BTreeMap; + +use lumen_core::{Error, Result}; + +/// BPE 토큰 식별자. +pub type TokenId = u32; + +/// 바이트 수준 BPE 토크나이저. +#[derive(Clone, Debug)] +pub struct BpeTokenizer { + /// 토큰 ID -> 바이트 시퀀스. + vocab_by_id: Vec>, + /// 바이트 시퀀스 -> 토큰 ID (역색인). + id_by_token: BTreeMap, TokenId>, + /// 머지 규칙 `(left, right) -> rank` (낮을수록 우선). + merge_ranks: BTreeMap<(Vec, Vec), u32>, + /// 알 수 없는 토큰 fallback ID. `None` 이면 인코드 실패시 에러. + unk_id: Option, +} + +impl BpeTokenizer { + /// vocab + merges 로부터 새 인스턴스를 만듭니다. + /// + /// # Errors + /// vocab 크기가 `u32` 범위를 초과하거나, 중복 토큰이 발견되거나, merges + /// 길이가 `u32` 범위를 초과하면 [`Error::Invalid`] 를 반환합니다. + pub fn new(vocab: Vec>, merges: Vec<(Vec, Vec)>) -> Result { + if vocab.len() > u32::MAX as usize { + return Err(Error::Invalid( + "tokenizer: vocab too large for u32 ids".into(), + )); + } + let mut id_by_token = BTreeMap::new(); + for (idx, tok) in vocab.iter().enumerate() { + if id_by_token.insert(tok.clone(), idx as TokenId).is_some() { + return Err(Error::Invalid(format!( + "tokenizer: duplicate vocab token at id {idx}" + ))); + } + } + let mut merge_ranks = BTreeMap::new(); + for (rank, pair) in merges.into_iter().enumerate() { + if rank > u32::MAX as usize { + return Err(Error::Invalid( + "tokenizer: too many merges for u32 rank".into(), + )); + } + // 동일 페어가 중복 등록되어도 첫 등록이 가장 낮은 rank 이므로 + // 그것을 보존합니다. + merge_ranks.entry(pair).or_insert(rank as u32); + } + Ok(Self { + vocab_by_id: vocab, + id_by_token, + merge_ranks, + unk_id: None, + }) + } + + /// 알 수 없는 토큰을 만났을 때 사용할 fallback ID 를 설정합니다. + pub fn with_unk_id(mut self, unk_id: TokenId) -> Self { + self.unk_id = Some(unk_id); + self + } + + /// 단일 토큰 바이트를 ID 로 조회합니다. + pub fn token_to_id(&self, token: &[u8]) -> Option { + self.id_by_token.get(token).copied() + } + + /// ID 를 토큰 바이트로 조회합니다. + pub fn id_to_token(&self, id: TokenId) -> Option<&[u8]> { + self.vocab_by_id.get(id as usize).map(Vec::as_slice) + } + + /// vocab 크기. + pub fn vocab_size(&self) -> usize { + self.vocab_by_id.len() + } + + /// 등록된 머지 규칙 수. + pub fn merge_count(&self) -> usize { + self.merge_ranks.len() + } + + /// UTF-8 텍스트를 토큰 ID 시퀀스로 인코드합니다. + pub fn encode(&self, text: &str) -> Result> { + self.encode_bytes(text.as_bytes()) + } + + /// 임의의 바이트 시퀀스를 토큰 ID 시퀀스로 인코드합니다. + /// + /// # 알고리즘 + /// 1. 입력 바이트마다 한 글자 토큰으로 분해. + /// 2. 인접 페어 중 가장 낮은 rank 의 머지를 1 회 적용. + /// 3. 더 이상 적용 가능한 머지가 없을 때까지 반복. + /// 4. 각 토큰 바이트를 vocab ID 로 변환. + pub fn encode_bytes(&self, bytes: &[u8]) -> Result> { + if bytes.is_empty() { + return Ok(Vec::new()); + } + let mut pieces: Vec> = bytes.iter().map(|b| vec![*b]).collect(); + + loop { + let mut best_rank: u32 = u32::MAX; + let mut best_idx: Option = None; + for i in 0..pieces.len().saturating_sub(1) { + // 머지 규칙 검색은 페어 자체를 키로 사용합니다. BTreeMap + // 키는 빌릴 수 없으므로 검색용 페어를 만듭니다 - 짧은 + // 토큰이라 재할당 비용은 무시 가능합니다. + let pair = (pieces[i].clone(), pieces[i + 1].clone()); + if let Some(&rank) = self.merge_ranks.get(&pair) { + if rank < best_rank { + best_rank = rank; + best_idx = Some(i); + } + } + } + let Some(i) = best_idx else { break }; + let mut merged = Vec::with_capacity(pieces[i].len() + pieces[i + 1].len()); + merged.extend_from_slice(&pieces[i]); + merged.extend_from_slice(&pieces[i + 1]); + pieces[i] = merged; + pieces.remove(i + 1); + } + + let mut ids = Vec::with_capacity(pieces.len()); + for piece in pieces { + if let Some(id) = self.id_by_token.get(&piece) { + ids.push(*id); + } else if let Some(unk) = self.unk_id { + ids.push(unk); + } else { + return Err(Error::Inference(format!( + "tokenizer: unknown token (len={}) and no unk_id set", + piece.len() + ))); + } + } + Ok(ids) + } + + /// 토큰 ID 시퀀스를 바이트로 디코드합니다. + pub fn decode_bytes(&self, ids: &[TokenId]) -> Result> { + let mut out = Vec::new(); + for &id in ids { + let token = self.id_to_token(id).ok_or_else(|| { + Error::Inference(format!("tokenizer: unknown id {id} for decode")) + })?; + out.extend_from_slice(token); + } + Ok(out) + } + + /// 토큰 ID 시퀀스를 UTF-8 문자열로 디코드합니다. 비-UTF-8 시퀀스는 + /// lossy 변환됩니다. + pub fn decode(&self, ids: &[TokenId]) -> Result { + let bytes = self.decode_bytes(ids)?; + Ok(String::from_utf8_lossy(&bytes).into_owned()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn b(s: &str) -> Vec { + s.as_bytes().to_vec() + } + + /// 기본: 모든 단일 바이트가 vocab 에 있고 머지가 없으면, 인코드는 + /// 입력 바이트마다 1 토큰을 생성합니다. + #[test] + fn no_merges_byte_per_token() { + let vocab = (0u8..=255u8).map(|b| vec![b]).collect(); + let tk = BpeTokenizer::new(vocab, vec![]).unwrap(); + let ids = tk.encode("abc").unwrap(); + assert_eq!(ids, vec![b'a' as u32, b'b' as u32, b'c' as u32]); + assert_eq!(tk.decode(&ids).unwrap(), "abc"); + } + + /// 우선순위 검증: rank 0 페어가 rank 1 페어보다 먼저 머지됩니다. + #[test] + fn merge_priority_lowest_rank_first() { + // vocab: a b c ab abc + let vocab = vec![b("a"), b("b"), b("c"), b("ab"), b("abc")]; + // merges: (a,b) -> rank 0, (ab,c) -> rank 1 + let merges = vec![(b("a"), b("b")), (b("ab"), b("c"))]; + let tk = BpeTokenizer::new(vocab, merges).unwrap(); + let ids = tk.encode("abc").unwrap(); + // (a,b) -> ab, (ab,c) -> abc → 단일 토큰 "abc" (id=4). + assert_eq!(ids, vec![4]); + assert_eq!(tk.decode(&ids).unwrap(), "abc"); + } + + /// 동일 입력 / 동일 vocab 에서 결정론적 출력. + #[test] + fn deterministic() { + let vocab = vec![b("a"), b("b"), b("ab")]; + let merges = vec![(b("a"), b("b"))]; + let tk = BpeTokenizer::new(vocab, merges).unwrap(); + let a = tk.encode("ab").unwrap(); + let b = tk.encode("ab").unwrap(); + assert_eq!(a, b); + } + + /// 알 수 없는 토큰: unk_id 미설정 시 에러. + #[test] + fn unknown_without_unk_errors() { + let vocab = vec![b("a")]; // 'b' 미포함 + let tk = BpeTokenizer::new(vocab, vec![]).unwrap(); + assert!(tk.encode("b").is_err()); + } + + /// 알 수 없는 토큰: unk_id 설정 시 fallback. + #[test] + fn unknown_with_unk_uses_fallback() { + let vocab = vec![b("a"), b("")]; + let tk = BpeTokenizer::new(vocab, vec![]).unwrap().with_unk_id(1); + let ids = tk.encode("b").unwrap(); + assert_eq!(ids, vec![1]); + } + + /// 빈 입력은 빈 출력. + #[test] + fn empty_input() { + let tk = BpeTokenizer::new(vec![b("a")], vec![]).unwrap(); + assert!(tk.encode("").unwrap().is_empty()); + assert!(tk.decode(&[]).unwrap().is_empty()); + } + + /// 중복 vocab 토큰은 거부. + #[test] + fn duplicate_vocab_rejected() { + let vocab = vec![b("a"), b("a")]; + assert!(BpeTokenizer::new(vocab, vec![]).is_err()); + } + + /// 디코드 시 알 수 없는 ID 는 에러. + #[test] + fn decode_unknown_id_errors() { + let tk = BpeTokenizer::new(vec![b("a")], vec![]).unwrap(); + assert!(tk.decode(&[42]).is_err()); + } + + /// 바이트 수준: UTF-8 다바이트 문자도 동작. + #[test] + fn byte_level_handles_multibyte_utf8() { + let mut vocab: Vec> = (0u8..=255u8).map(|b| vec![b]).collect(); + // "한" 의 UTF-8 = 0xED 0x95 0x9C (3 바이트). + let han: Vec = vec![0xED, 0x95, 0x9C]; + vocab.push(han.clone()); + let merges = vec![(vec![0xED], vec![0x95]), (vec![0xED, 0x95], vec![0x9C])]; + let tk = BpeTokenizer::new(vocab, merges).unwrap(); + let ids = tk.encode("한").unwrap(); + // 머지 후 단일 토큰 (마지막 ID = 256). + assert_eq!(ids, vec![256]); + assert_eq!(tk.decode(&ids).unwrap(), "한"); + } + + /// 인코드/디코드 round-trip. + #[test] + fn roundtrip_arbitrary_bytes() { + // 256 개 단일 바이트 vocab + 일부 머지 규칙. + let mut vocab: Vec> = (0u8..=255u8).map(|b| vec![b]).collect(); + vocab.push(b("ab")); + vocab.push(b("cd")); + let merges = vec![(b("a"), b("b")), (b("c"), b("d"))]; + let tk = BpeTokenizer::new(vocab, merges).unwrap(); + let original = "abcd-abcd"; + let ids = tk.encode(original).unwrap(); + assert_eq!(tk.decode(&ids).unwrap(), original); + } + + /// vocab/merge 메타데이터 조회. + #[test] + fn metadata_accessors() { + let vocab = vec![b("a"), b("b"), b("ab")]; + let merges = vec![(b("a"), b("b"))]; + let tk = BpeTokenizer::new(vocab, merges).unwrap(); + assert_eq!(tk.vocab_size(), 3); + assert_eq!(tk.merge_count(), 1); + assert_eq!(tk.token_to_id(b"ab"), Some(2)); + assert_eq!(tk.id_to_token(2), Some(&b"ab"[..])); + assert_eq!(tk.id_to_token(99), None); + } +} diff --git a/crates/lumen-orchestrator/Cargo.toml b/crates/lumen-orchestrator/Cargo.toml index e760d44..222cddd 100644 --- a/crates/lumen-orchestrator/Cargo.toml +++ b/crates/lumen-orchestrator/Cargo.toml @@ -29,5 +29,4 @@ lumen-defense.workspace = true lumen-inference.workspace = true lumen-zkml.workspace = true serde_json = { workspace = true } -rand = { workspace = true } tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/crates/lumen-orchestrator/src/supervisor.rs b/crates/lumen-orchestrator/src/supervisor.rs index 8311a31..734f4d7 100644 --- a/crates/lumen-orchestrator/src/supervisor.rs +++ b/crates/lumen-orchestrator/src/supervisor.rs @@ -426,10 +426,10 @@ mod tests { capability::{Capability, CapabilityBody}, PolicyEngine, Resource, }; + use lumen_core::rng::OsRng; use lumen_core::{Blake3Hash, CapabilityId, SigningKey, Timestamp, ToolId}; use lumen_inference::DummyEngine; use lumen_zkml::mock::MockVk; - use rand::rngs::OsRng; fn build_runtime(nonce: [u8; 16]) -> (AgentId, Arc) { let sk = SigningKey::generate(&mut OsRng); diff --git a/crates/lumen-provenance/Cargo.toml b/crates/lumen-provenance/Cargo.toml index 00294c1..d5c6971 100644 --- a/crates/lumen-provenance/Cargo.toml +++ b/crates/lumen-provenance/Cargo.toml @@ -25,4 +25,3 @@ tracing.workspace = true [dev-dependencies] tempfile = "3" -rand = { workspace = true } diff --git a/crates/lumen-provenance/src/lib.rs b/crates/lumen-provenance/src/lib.rs index 5f2fa10..7ee3adb 100644 --- a/crates/lumen-provenance/src/lib.rs +++ b/crates/lumen-provenance/src/lib.rs @@ -63,20 +63,31 @@ pub fn verify_model( ))); } - if let (Some(sig), Some(signer)) = (&manifest.signature, &manifest.signer) { - let body = manifest.signing_payload()?; - let mut accepted = false; - for trusted in trusted_signers { - if trusted == signer && trusted.verify(&body, sig).is_ok() { - accepted = true; - break; + // trusted_signers 가 비어있지 않다면 매니페스트는 반드시 서명되어 + // 있어야 합니다 - 이전 구현은 None 매니페스트를 silently skip 하여 + // 미서명 모델이 hash 매칭만으로 통과될 수 있는 결함이 있었습니다. + match (&manifest.signature, &manifest.signer) { + (Some(sig), Some(signer)) => { + let body = manifest.signing_payload()?; + let mut accepted = false; + for trusted in trusted_signers { + if trusted == signer && trusted.verify(&body, sig).is_ok() { + accepted = true; + break; + } + } + if !accepted { + return Err(Error::Provenance( + "signature did not verify against any trusted signer".into(), + )); } } - if !accepted { + _ if !trusted_signers.is_empty() => { return Err(Error::Provenance( - "signature did not verify against any trusted signer".into(), + "manifest is unsigned but trusted signers were configured".into(), )); } + _ => {} } match manifest.format { diff --git a/crates/lumen-provenance/src/manifest.rs b/crates/lumen-provenance/src/manifest.rs index d47c358..f6a72d5 100644 --- a/crates/lumen-provenance/src/manifest.rs +++ b/crates/lumen-provenance/src/manifest.rs @@ -5,6 +5,13 @@ use std::path::PathBuf; use lumen_core::{Blake3Hash, Error, Result, Signature, SigningKey, VerifyingKey}; use serde::{Deserialize, Serialize}; +/// Ed25519 도메인 분리 prefix. +/// +/// Capability / 보안 채널 핸드셰이크 / 프레임 서명과 동일한 신원 키를 공유 +/// 하는 환경에서, 매니페스트 서명을 다른 프로토콜의 서명에 재사용하는 +/// cross-protocol replay 를 차단합니다. +const MANIFEST_SIGN_DOMAIN: &[u8] = b"lumen.provenance.manifest.v1"; + /// 매니페스트가 기술하는 파일 형식. #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] pub enum Format { @@ -43,8 +50,8 @@ pub struct ModelManifest { impl ModelManifest { /// 서명이 덮는 바이트. /// - /// `name | version | format | hash | license` 를 연접합니다. 경로를 - /// 제외하면 배포 위치에 무관하게 서명이 안정적입니다. + /// `MANIFEST_SIGN_DOMAIN || postcard({name | version | format | hash | license})`. + /// 경로 (`path`) 를 제외하므로 배포 위치에 무관하게 서명이 안정적입니다. pub fn signing_payload(&self) -> Result> { #[derive(Serialize)] struct Body<'a> { @@ -61,7 +68,12 @@ impl ModelManifest { hash: self.hash, license: self.license.as_deref(), }; - postcard::to_allocvec(&body).map_err(|e| Error::Decode(format!("manifest payload: {e}"))) + let body_bytes = postcard::to_allocvec(&body) + .map_err(|e| Error::Decode(format!("manifest payload: {e}")))?; + let mut out = Vec::with_capacity(MANIFEST_SIGN_DOMAIN.len() + body_bytes.len()); + out.extend_from_slice(MANIFEST_SIGN_DOMAIN); + out.extend_from_slice(&body_bytes); + Ok(out) } /// `key` 로 매니페스트에 서명하고 `signature` 와 `signer` 를 채웁니다. diff --git a/crates/lumen-provenance/src/pinset.rs b/crates/lumen-provenance/src/pinset.rs index 2f38a59..5dc76df 100644 --- a/crates/lumen-provenance/src/pinset.rs +++ b/crates/lumen-provenance/src/pinset.rs @@ -154,20 +154,35 @@ pub fn verify_model_against_pinset( }; debug_assert!(accepted); - if let (Some(sig), Some(signer)) = (&manifest.signature, &manifest.signer) { - let body = manifest.signing_payload()?; - let mut ok = false; - for trusted in trusted_signers { - if trusted == signer && trusted.verify(&body, sig).is_ok() { - ok = true; - break; + // trusted_signers 가 비어있지 않다면 매니페스트는 반드시 서명되어 + // 있어야 합니다. 이전 구현은 signature/signer 가 None 인 매니페스트에 + // 대해 검증을 silently skip 하여, pinset 의 grace 윈도가 느슨한 경우 + // 미서명 매니페스트가 통과될 수 있는 결함이 있었습니다. + match (&manifest.signature, &manifest.signer) { + (Some(sig), Some(signer)) => { + let body = manifest.signing_payload()?; + let mut ok = false; + for trusted in trusted_signers { + if trusted == signer && trusted.verify(&body, sig).is_ok() { + ok = true; + break; + } + } + if !ok { + return Err(Error::Provenance( + "signature did not verify against any trusted signer".into(), + )); } } - if !ok { + _ if !trusted_signers.is_empty() => { return Err(Error::Provenance( - "signature did not verify against any trusted signer".into(), + "manifest is unsigned but trusted signers were configured".into(), )); } + _ => { + // trusted_signers 가 비어 있고 매니페스트도 미서명 - pinset + // 검증만으로 진행 (호출자가 명시적으로 unsigned 모드 선택). + } } match manifest.format { diff --git a/crates/lumen-provenance/tests/verify_safetensors.rs b/crates/lumen-provenance/tests/verify_safetensors.rs index de094e5..98efcdf 100644 --- a/crates/lumen-provenance/tests/verify_safetensors.rs +++ b/crates/lumen-provenance/tests/verify_safetensors.rs @@ -68,8 +68,8 @@ fn verify_tampered_file_fails() { #[test] fn verify_with_signature() { + use lumen_core::rng::OsRng; use lumen_core::SigningKey; - use rand::rngs::OsRng; let dir = tempfile::tempdir().unwrap(); let path = write_tiny_safetensors(&dir); @@ -95,3 +95,34 @@ fn verify_with_signature() { let err = verify_model(&path, &manifest, &[other.verifying_key()]).unwrap_err(); assert!(err.to_string().contains("signature")); } + +/// 회귀: trusted_signers 가 비어있지 않은데 매니페스트가 미서명이면 거부. +/// 이전 구현은 (signature, signer) 가 None 이면 silently skip 했음. +#[test] +fn unsigned_manifest_rejected_when_signers_required() { + use lumen_core::rng::OsRng; + use lumen_core::SigningKey; + + let dir = tempfile::tempdir().unwrap(); + let path = write_tiny_safetensors(&dir); + let hash = Blake3Hash::of_file(&path).unwrap(); + let trusted = SigningKey::generate(&mut OsRng); + + // signature/signer 가 None 이지만 trusted_signers 는 비어있지 않음. + let manifest = ModelManifest { + name: "tiny".into(), + version: "0.0.1".into(), + path: path.clone(), + format: Format::Safetensors, + hash, + license: None, + signature: None, + signer: None, + }; + let err = verify_model(&path, &manifest, &[trusted.verifying_key()]).unwrap_err(); + let msg = err.to_string(); + assert!( + msg.contains("unsigned"), + "expected unsigned-rejection, got: {msg}" + ); +} diff --git a/crates/lumen-sandbox/Cargo.toml b/crates/lumen-sandbox/Cargo.toml index 0f8083e..1b04d18 100644 --- a/crates/lumen-sandbox/Cargo.toml +++ b/crates/lumen-sandbox/Cargo.toml @@ -13,6 +13,7 @@ categories.workspace = true # wasmtime FFI requires `unsafe`. Override the workspace deny rule locally and # enforce documented `// SAFETY:` blocks via clippy. [lints.rust] +warnings = { level = "deny", priority = -2 } unsafe_code = "allow" missing_docs = "warn" unreachable_pub = "warn" @@ -37,5 +38,4 @@ thiserror.workspace = true [dev-dependencies] wat.workspace = true -rand = { workspace = true } tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/crates/lumen-sandbox/src/imports.rs b/crates/lumen-sandbox/src/imports.rs index 60f8b7e..67869fa 100644 --- a/crates/lumen-sandbox/src/imports.rs +++ b/crates/lumen-sandbox/src/imports.rs @@ -18,6 +18,26 @@ use wasmtime::{Caller, Linker, Memory}; use crate::host::{HostState, ToolCallRecord}; +/// 단일 호스트 임포트 호출이 게스트 메모리에서 호스트 Vec 으로 복사할 수 +/// 있는 최대 바이트 수 (64 KiB). +/// +/// 이 한도가 없으면 게스트가 16 MiB 선형 메모리 페이지를 모두 채워 +/// `lumen_log` / `lumen_call_tool` 인자로 반복 전달함으로써 호스트 측 +/// 메모리를 과다 소비시키는 `DoS` 가 가능합니다. 도구 인자나 로그 메시지가 +/// 정상 운영에서 64 KiB 를 넘는 경우는 거의 없으므로 보수적인 상한으로 +/// 충분합니다. +const MAX_GUEST_BUFFER: usize = 64 * 1024; + +/// 단일 실행에서 audit 버퍼가 누적할 수 있는 최대 항목 수. +/// +/// 게스트가 짧은 메시지로 반복 `lumen_log` 를 호출해 호스트 audit Vec 을 +/// 무한 증식시키지 못하도록 보호합니다. 한도 도달 후 모든 후속 `lumen_log` +/// 호출은 무시됩니다 (실행은 계속 — fail-safe). +const MAX_AUDIT_ENTRIES: usize = 4096; + +/// 단일 실행에서 기록할 수 있는 최대 도구 호출 시도 수. +const MAX_TOOL_CALLS: usize = 1024; + /// 모듈 이름 `lumen` 아래 Lumen 호스트 임포트를 등록합니다. /// /// `caps` 콜백은 도구 호출 시도에 적용 가능한 capability 토큰을 반환합니다 - @@ -43,7 +63,12 @@ where let msg = String::from_utf8_lossy(&b).into_owned(); let line = format!("[guest level={level}] {msg}"); tracing::info!(target: "lumen.sandbox", "{line}"); - caller.data().audit.lock().push(line); + let mut audit = caller.data().audit.lock(); + if audit.len() < MAX_AUDIT_ENTRIES { + audit.push(line); + } + // 한도 도달 시 silently 드롭 - tracing 은 이미 발행되었고, + // panic 또는 trap 으로 호스트를 흔들지 않습니다. }, ) .map_err(|e| Error::Sandbox(format!("register lumen_log: {e}")))?; @@ -113,6 +138,11 @@ fn read_guest_bytes( ) -> Result> { let start = usize::try_from(ptr).map_err(|_| Error::Sandbox("negative ptr".into()))?; let n = usize::try_from(len).map_err(|_| Error::Sandbox("negative len".into()))?; + if n > MAX_GUEST_BUFFER { + return Err(Error::Sandbox(format!( + "guest buffer too large: {n} > {MAX_GUEST_BUFFER}" + ))); + } let data = memory.data(&*caller); let end = start .checked_add(n) @@ -124,8 +154,12 @@ fn read_guest_bytes( } fn record(caller: &Caller<'_, HostState>, tool: &str, authorised: bool) { - caller.data().tool_calls.lock().push(ToolCallRecord { - tool: tool.to_string(), - authorised, - }); + let mut tool_calls = caller.data().tool_calls.lock(); + if tool_calls.len() < MAX_TOOL_CALLS { + tool_calls.push(ToolCallRecord { + tool: tool.to_string(), + authorised, + }); + } + // 한도 도달 시 드롭 - 이미 audit::deny / allow 가 tracing 으로 기록. } diff --git a/crates/lumen-sandbox/src/runner.rs b/crates/lumen-sandbox/src/runner.rs index 46e57bb..4a3dbf1 100644 --- a/crates/lumen-sandbox/src/runner.rs +++ b/crates/lumen-sandbox/src/runner.rs @@ -72,8 +72,8 @@ impl Sandbox { mod tests { use super::*; use lumen_capability::PolicyEngine; + use lumen_core::rng::OsRng; use lumen_core::{AgentId, SigningKey, Timestamp}; - use rand::rngs::OsRng; const LOG_ONLY_WAT: &str = r#" (module diff --git a/crates/lumen-sandbox/tests/wasm_echo_agent.rs b/crates/lumen-sandbox/tests/wasm_echo_agent.rs index c4e7264..8a37d39 100644 --- a/crates/lumen-sandbox/tests/wasm_echo_agent.rs +++ b/crates/lumen-sandbox/tests/wasm_echo_agent.rs @@ -19,9 +19,9 @@ use lumen_capability::{ capability::{Capability, CapabilityBody}, PolicyEngine, Resource, }; +use lumen_core::rng::OsRng; use lumen_core::{AgentId, CapabilityId, SigningKey, Timestamp, ToolId}; use lumen_sandbox::{HostState, Sandbox, SandboxConfig}; -use rand::rngs::OsRng; /// 워크스페이스 루트 기준 wasm 산출물 경로. fn echo_agent_wasm_path() -> PathBuf { diff --git a/crates/lumen-sdk-macros/Cargo.toml b/crates/lumen-sdk-macros/Cargo.toml index 0d1014a..3cfbe8c 100644 --- a/crates/lumen-sdk-macros/Cargo.toml +++ b/crates/lumen-sdk-macros/Cargo.toml @@ -13,6 +13,7 @@ categories.workspace = true # proc-macro 크레이트는 호스트에서만 실행되며 자체적인 unsafe 사용은 없습니다. # (syn/quote 가 내부에서 unsafe 를 쓸 수 있으나 그 부분은 외부 검증.) [lints.rust] +warnings = { level = "deny", priority = -2 } unsafe_code = "deny" missing_docs = "warn" unreachable_pub = "warn" diff --git a/crates/lumen-sdk/Cargo.toml b/crates/lumen-sdk/Cargo.toml index 1c38452..f16e4cb 100644 --- a/crates/lumen-sdk/Cargo.toml +++ b/crates/lumen-sdk/Cargo.toml @@ -13,6 +13,7 @@ categories.workspace = true # `extern "C"` 호스트 임포트 선언과 메모리 포인터 변환에는 unsafe 가 불가피합니다. # 워크스페이스 lint 를 로컬에서 완화하고 모든 unsafe 블록에 SAFETY 주석을 강제합니다. [lints.rust] +warnings = { level = "deny", priority = -2 } unsafe_code = "allow" missing_docs = "warn" unreachable_pub = "warn" diff --git a/crates/lumen-zkml/Cargo.toml b/crates/lumen-zkml/Cargo.toml index 542e724..fd9935e 100644 --- a/crates/lumen-zkml/Cargo.toml +++ b/crates/lumen-zkml/Cargo.toml @@ -6,7 +6,7 @@ rust-version.workspace = true license.workspace = true authors.workspace = true repository.workspace = true -description = "Lumen - proving-system trait + mock commitment prover; ezkl/halo2 stubs behind feature flags" +description = "Lumen - proving-system trait + mock BLAKE3 commitment prover; ezkl stub behind feature flag" keywords.workspace = true categories.workspace = true @@ -16,18 +16,14 @@ workspace = true [features] default = [] ezkl = [] -halo2 = ["dep:halo2_proofs", "dep:ff", "dep:pasta_curves"] [dependencies] lumen-core.workspace = true serde = { workspace = true } postcard.workspace = true -blake3 = { workspace = true } +elib-blake.workspace = true thiserror.workspace = true tracing.workspace = true -halo2_proofs = { workspace = true, optional = true } -ff = { workspace = true, optional = true } -pasta_curves = { workspace = true, optional = true } [dev-dependencies] serde_json = { workspace = true } diff --git a/crates/lumen-zkml/src/halo2.rs b/crates/lumen-zkml/src/halo2.rs deleted file mode 100644 index 3d89ed3..0000000 --- a/crates/lumen-zkml/src/halo2.rs +++ /dev/null @@ -1,379 +0,0 @@ -//! halo2 기반 라우팅 증명 - `halo2` feature 가 활성화된 경우에만 컴파일됩니다. -//! -//! v0.3 범위: 도구 라우팅 결정의 핵심 제약을 halo2 `Circuit` 으로 표현하고 -//! `MockProver::verify` 로 *제약 만족* 을 검증합니다. 진정한 succinct ZK -//! proof (KZG 셋업 + `create_proof`/`verify_proof`) 는 v0.4 작업입니다. -//! -//! ## 회로 -//! -//! `RoutingProofCircuit` 는 다음을 증명합니다. -//! -//! - 공개 입력 `selected_value` 가 -//! - 비공개 witness `(score_a, score_b, choice_bit)` 와 함께, -//! - 제약 `selected_value = choice_bit * score_b + (1 - choice_bit) * score_a` 를 -//! 만족함. -//! - 추가 제약 `choice_bit * (1 - choice_bit) = 0` 으로 `choice_bit ∈ {0, 1}` -//! 을 강제. -//! -//! 이는 두 후보 라우팅 점수 중 하나를 선택하는 *binary argmax* 의 최소 -//! 형태입니다. 더 큰 N 에 대한 일반적 argmax 는 비교 게이트 (range check) -//! 가 필요하므로 v0.4 까지 미룹니다. -//! -//! ## 진정한 ZK 가 아닌 이유 -//! -//! `MockProver` 는 회로의 모든 advice 셀을 평문으로 보유한 채 제약식을 -//! 평가하므로 **소중한 (소위 "real") ZK 보장은 제공하지 않습니다.** 다만: -//! -//! - 회로 자체는 진짜 halo2 PLONKish 제약식이며, -//! - 위트니스가 제약을 만족하는지 검증한다는 점에서 정직하고, -//! - v0.4 에서 동일한 회로를 KZG 백엔드로 옮기면 그대로 succinct ZK 증명이 -//! 됩니다. -//! -//! 따라서 본 백엔드의 [`ProvingSystem::verify`] 는 [`Verification::ZkVerified`] -//! 를 반환합니다 - 회로 + 제약식이 진짜이기 때문이며, 다만 succinct 형태의 -//! "off-line verifiability" 가 빠져 있다는 점은 docstring 으로 명시합니다. - -use ff::{Field, PrimeField}; -use halo2_proofs::{ - circuit::{AssignedCell, Layouter, SimpleFloorPlanner, Value}, - dev::MockProver, - plonk::{Advice, Circuit, Column, ConstraintSystem, Error as Halo2Error, Instance, Selector}, - poly::Rotation, -}; -use lumen_core::{Error, Result}; -use pasta_curves::pallas::Base as Fp; -use serde::{Deserialize, Serialize}; - -use crate::{ProvingSystem, Verification}; - -/// halo2 회로 회로의 행 크기 = 2^K. -pub const CIRCUIT_K: u32 = 4; - -/// `RoutingProofCircuit` 의 PLONK 게이트 설정. -#[derive(Clone, Debug)] -pub struct RoutingConfig { - advice: Column, - instance: Column, - selector: Selector, - bit_selector: Selector, -} - -/// Witness - 두 후보 점수와 선택 비트. -#[derive(Clone, Debug, Default)] -pub struct RoutingProofCircuit { - /// 후보 A 의 점수. - pub score_a: Value, - /// 후보 B 의 점수. - pub score_b: Value, - /// 선택 비트 (0 = A, 1 = B). - pub choice_bit: Value, -} - -impl Circuit for RoutingProofCircuit { - type Config = RoutingConfig; - type FloorPlanner = SimpleFloorPlanner; - - fn without_witnesses(&self) -> Self { - Self::default() - } - - fn configure(meta: &mut ConstraintSystem) -> Self::Config { - let advice = meta.advice_column(); - let instance = meta.instance_column(); - let selector = meta.selector(); - let bit_selector = meta.selector(); - meta.enable_equality(advice); - meta.enable_equality(instance); - - // 게이트 1: selected_value = choice_bit * score_b + (1 - choice_bit) * score_a - // 행 레이아웃 (advice 컬럼): - // row+0: score_a - // row+1: score_b - // row+2: choice_bit - // row+3: selected_value - meta.create_gate("선택값 일치", |meta| { - let s = meta.query_selector(selector); - let a = meta.query_advice(advice, Rotation::cur()); - let b = meta.query_advice(advice, Rotation::next()); - let bit = meta.query_advice(advice, Rotation(2)); - let sel = meta.query_advice(advice, Rotation(3)); - // sel = bit * b + (1 - bit) * a - // = bit * (b - a) + a - let computed = bit * (b - a.clone()) + a; - vec![s * (sel - computed)] - }); - - // 게이트 2: choice_bit * (1 - choice_bit) = 0 ⇒ bit ∈ {0, 1} - meta.create_gate("선택 비트는 0 또는 1", |meta| { - let s = meta.query_selector(bit_selector); - let bit = meta.query_advice(advice, Rotation::cur()); - let one = halo2_proofs::plonk::Expression::Constant(Fp::ONE); - vec![s * bit.clone() * (one - bit)] - }); - - RoutingConfig { - advice, - instance, - selector, - bit_selector, - } - } - - fn synthesize( - &self, - config: Self::Config, - mut layouter: impl Layouter, - ) -> std::result::Result<(), Halo2Error> { - let selected_cell: AssignedCell = layouter.assign_region( - || "라우팅 영역", - |mut region| { - config.selector.enable(&mut region, 0)?; - config.bit_selector.enable(&mut region, 2)?; - region.assign_advice(|| "score_a", config.advice, 0, || self.score_a)?; - region.assign_advice(|| "score_b", config.advice, 1, || self.score_b)?; - region.assign_advice(|| "choice_bit", config.advice, 2, || self.choice_bit)?; - let zero = Fp::from(0u64); - let selected: Value = self - .choice_bit - .zip(self.score_a.zip(self.score_b)) - .map(|(bit, (a, b))| if bit == zero { a } else { b }); - region.assign_advice(|| "selected", config.advice, 3, || selected) - }, - )?; - layouter.constrain_instance(selected_cell.cell(), config.instance, 0)?; - Ok(()) - } -} - -/// halo2 백엔드의 검증 키 - 회로 식별자만 보유. -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct Halo2Vk { - /// 회로 식별자 (예: `"lumen.routing.binary.v1"`). - pub circuit_id: String, -} - -/// halo2 백엔드의 증명 - witness 평문 + 공개 입력의 직렬화. -/// -/// `MockProver` 가 succinct 한 증명 객체를 노출하지 않으므로, v0.3 에서는 -/// 회로를 재현하기 위한 평문 witness 를 운반합니다. v0.4 에서 KZG 기반 -/// `create_proof` 의 byte vector 로 대체됩니다. -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct Halo2Proof { - /// 직렬화된 witness 트리플 `(score_a, score_b, choice_bit)`. - pub witness_repr: Vec<[u8; 32]>, -} - -/// halo2 회로의 공개 입력. -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct Halo2Public { - /// 32 바이트 LE 직렬화된 선택 점수. - pub selected_value: [u8; 32], -} - -/// halo2 회로의 비공개 witness. -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct Halo2Witness { - /// 32 바이트 LE 직렬화된 score_a. - pub score_a: [u8; 32], - /// 32 바이트 LE 직렬화된 score_b. - pub score_b: [u8; 32], - /// 32 바이트 LE 직렬화된 choice_bit (0 또는 1). - pub choice_bit: [u8; 32], -} - -/// halo2 백엔드 - 무상태. -#[derive(Clone, Debug, Default)] -pub struct Halo2Prover; - -impl Halo2Prover { - /// 새 인스턴스. - pub fn new() -> Self { - Self - } -} - -fn fp_from_u64(v: u64) -> Fp { - Fp::from(v) -} - -fn fp_from_bytes(b: &[u8; 32]) -> Result { - let opt: Option = Fp::from_repr(*b).into(); - opt.ok_or_else(|| Error::Zkml("halo2: invalid Fp bytes".into())) -} - -fn fp_to_bytes(v: Fp) -> [u8; 32] { - v.to_repr() -} - -impl ProvingSystem for Halo2Prover { - type Witness = Halo2Witness; - type PublicInputs = Halo2Public; - type Proof = Halo2Proof; - type Vk = Halo2Vk; - - fn setup(&self, circuit_id: &str) -> Result { - Ok(Halo2Vk { - circuit_id: circuit_id.to_string(), - }) - } - - fn prove( - &self, - _vk: &Self::Vk, - public: &Self::PublicInputs, - witness: &Self::Witness, - ) -> Result { - // 회로 + MockProver 로 제약 만족 검증을 시도하고, 통과 시 witness 를 - // 그대로 운반하는 proof 를 반환합니다. - let score_a = fp_from_bytes(&witness.score_a)?; - let score_b = fp_from_bytes(&witness.score_b)?; - let choice_bit = fp_from_bytes(&witness.choice_bit)?; - let selected = fp_from_bytes(&public.selected_value)?; - - let circuit = RoutingProofCircuit { - score_a: Value::known(score_a), - score_b: Value::known(score_b), - choice_bit: Value::known(choice_bit), - }; - let prover = MockProver::run(CIRCUIT_K, &circuit, vec![vec![selected]]) - .map_err(|e| Error::Zkml(format!("halo2 mockprover run: {e:?}")))?; - prover - .verify() - .map_err(|e| Error::Zkml(format!("halo2 제약 미만족: {e:?}")))?; - Ok(Halo2Proof { - witness_repr: vec![witness.score_a, witness.score_b, witness.choice_bit], - }) - } - - fn verify( - &self, - _vk: &Self::Vk, - public: &Self::PublicInputs, - proof: &Self::Proof, - ) -> Result { - // 회로를 proof 의 witness 로 재구성하고 MockProver 를 다시 돌립니다. - if proof.witness_repr.len() != 3 { - return Ok(Verification::Invalid); - } - let score_a = match fp_from_bytes(&proof.witness_repr[0]) { - Ok(v) => v, - Err(_) => return Ok(Verification::Invalid), - }; - let score_b = match fp_from_bytes(&proof.witness_repr[1]) { - Ok(v) => v, - Err(_) => return Ok(Verification::Invalid), - }; - let choice_bit = match fp_from_bytes(&proof.witness_repr[2]) { - Ok(v) => v, - Err(_) => return Ok(Verification::Invalid), - }; - let selected = match fp_from_bytes(&public.selected_value) { - Ok(v) => v, - Err(_) => return Ok(Verification::Invalid), - }; - let circuit = RoutingProofCircuit { - score_a: Value::known(score_a), - score_b: Value::known(score_b), - choice_bit: Value::known(choice_bit), - }; - let prover = match MockProver::run(CIRCUIT_K, &circuit, vec![vec![selected]]) { - Ok(p) => p, - Err(_) => return Ok(Verification::Invalid), - }; - match prover.verify() { - Ok(()) => Ok(Verification::ZkVerified), - Err(_) => Ok(Verification::Invalid), - } - } -} - -/// 편의 헬퍼 - 작은 정수 점수로 witness 와 공개 입력을 만듭니다. -pub fn build_inputs(score_a: u64, score_b: u64, choice_bit: u64) -> (Halo2Witness, Halo2Public) { - debug_assert!(choice_bit == 0 || choice_bit == 1); - let a = fp_from_u64(score_a); - let b = fp_from_u64(score_b); - let bit = fp_from_u64(choice_bit); - let sel = if choice_bit == 0 { a } else { b }; - ( - Halo2Witness { - score_a: fp_to_bytes(a), - score_b: fp_to_bytes(b), - choice_bit: fp_to_bytes(bit), - }, - Halo2Public { - selected_value: fp_to_bytes(sel), - }, - ) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn prove_verify_roundtrip_choice_zero() { - let p = Halo2Prover::new(); - let vk = p.setup("lumen.routing.binary.v1").unwrap(); - let (w, pub_) = build_inputs(7, 13, 0); // 후보 A 선택 - let proof = p.prove(&vk, &pub_, &w).unwrap(); - let v = p.verify(&vk, &pub_, &proof).unwrap(); - assert_eq!(v, Verification::ZkVerified); - } - - #[test] - fn prove_verify_roundtrip_choice_one() { - let p = Halo2Prover::new(); - let vk = p.setup("lumen.routing.binary.v1").unwrap(); - let (w, pub_) = build_inputs(7, 13, 1); // 후보 B 선택 - let proof = p.prove(&vk, &pub_, &w).unwrap(); - let v = p.verify(&vk, &pub_, &proof).unwrap(); - assert_eq!(v, Verification::ZkVerified); - } - - #[test] - fn tampered_witness_invalid() { - let p = Halo2Prover::new(); - let vk = p.setup("lumen.routing.binary.v1").unwrap(); - let (w, pub_) = build_inputs(7, 13, 0); - let mut proof = p.prove(&vk, &pub_, &w).unwrap(); - // witness[0] (score_a) 를 변조 -> 제약 미만족 - proof.witness_repr[0][0] ^= 0xFF; - let v = p.verify(&vk, &pub_, &proof).unwrap(); - assert_eq!(v, Verification::Invalid); - } - - #[test] - fn wrong_public_input_invalid() { - let p = Halo2Prover::new(); - let vk = p.setup("lumen.routing.binary.v1").unwrap(); - let (w, _pub) = build_inputs(7, 13, 0); - // 잘못된 공개 입력 - 13 (score_b) 을 선택했다고 주장하지만 witness 는 0 - let bad_public = Halo2Public { - selected_value: fp_to_bytes(fp_from_u64(13)), - }; - let proof = p.prove(&vk, &bad_public, &w); - // prove 가 거부해야 함 (제약 미만족) - assert!(proof.is_err()); - } - - #[test] - fn invalid_choice_bit_rejected() { - let p = Halo2Prover::new(); - let vk = p.setup("lumen.routing.binary.v1").unwrap(); - // bit = 2 - bit constraint (bit ∈ {0,1}) 위반 - let a = fp_from_u64(7); - let b = fp_from_u64(13); - let bit = fp_from_u64(2); - let sel = b * bit + a * (Fp::ONE - bit); // 알파 산술상 일치하지만 bit constraint 가 위반 - let w = Halo2Witness { - score_a: fp_to_bytes(a), - score_b: fp_to_bytes(b), - choice_bit: fp_to_bytes(bit), - }; - let pub_ = Halo2Public { - selected_value: fp_to_bytes(sel), - }; - let res = p.prove(&vk, &pub_, &w); - assert!(res.is_err(), "bit ∈ {{0, 1}} 제약을 위반해야 함"); - } -} diff --git a/crates/lumen-zkml/src/lib.rs b/crates/lumen-zkml/src/lib.rs index cbe7509..c622046 100644 --- a/crates/lumen-zkml/src/lib.rs +++ b/crates/lumen-zkml/src/lib.rs @@ -6,7 +6,7 @@ //! 실행 가능합니다. //! //! [`ProvingSystem`] trait 가 백엔드 선택을 에이전트 런타임으로부터 숨깁니다. -//! 세 가지 백엔드가 scaffold 되어 있습니다: +//! 두 가지 백엔드가 scaffold 되어 있습니다: //! //! - **`mock`** - 항상 사용 가능. `(circuit_id, public_inputs, witness)` 를 //! 바인드하는 BLAKE3 commitment 를 생성합니다. 검증은 binding 을 다시 @@ -14,7 +14,15 @@ //! 아닙니다.** Verdict variant 는 의도적으로 [`Verification::ZkVerified`] //! 와 분리되어 있어 호출 사이트가 두 가지를 절대 혼동할 수 없습니다. //! - **`ezkl`** - feature-gated stub. -//! - **`halo2`** - feature 뒤 실제 PLONK 회로 (binary argmax). v0.3. +//! +//! ## 폐쇄형(Air-Gapped) 환경 메모 +//! +//! v0.3 까지 존재하던 `halo2` feature 와 그 PLONK 라우팅 회로는 v0.4 에서 +//! 제거되었습니다. `halo2_proofs` / `pasta_curves` 등 타원곡선 의존성이 +//! 매우 무거우면서도 `MockProver` 검증 단계에서는 witness 가 평문으로 +//! 노출되어 ZK 보장 자체가 없었기 때문입니다. 향후 succinct ZK 가 필요해 +//! 지면 SP1 / RISC Zero 등 별도 프레임워크 채택 여부를 마일스톤 재검토 +//! 시점에 결정합니다. #![forbid(unsafe_code)] #![warn(missing_docs)] @@ -24,8 +32,6 @@ pub mod verification; #[cfg(feature = "ezkl")] pub mod ezkl; -#[cfg(feature = "halo2")] -pub mod halo2; use lumen_core::Result; use serde::{de::DeserializeOwned, Serialize}; diff --git a/crates/lumen-zkml/src/mock.rs b/crates/lumen-zkml/src/mock.rs index c265116..676a75c 100644 --- a/crates/lumen-zkml/src/mock.rs +++ b/crates/lumen-zkml/src/mock.rs @@ -10,6 +10,7 @@ //! 제공하지 않으며**, 약한 soundness 만 - 유일한 보장은 `(public, witness)` //! 양쪽을 모두 가진 누군가가 증명을 생성했다는 사실 - 만 제공합니다. +use elib_blake::Blake3; use lumen_core::{Blake3Hash, Error, Result}; use serde::{Deserialize, Serialize}; @@ -48,7 +49,7 @@ impl MockCommitmentProver { postcard::to_allocvec(public).map_err(|e| Error::Decode(format!("public: {e}")))?; let witness_bytes = postcard::to_allocvec(witness).map_err(|e| Error::Decode(format!("witness: {e}")))?; - let mut hasher = blake3::Hasher::new(); + let mut hasher = Blake3::new(); hasher.update(b"lumen.mock.commit.v1\x00"); hasher.update(&u64::to_le_bytes(circuit_id.len() as u64)); hasher.update(circuit_id.as_bytes()); @@ -56,7 +57,12 @@ impl MockCommitmentProver { hasher.update(&public_bytes); hasher.update(&u64::to_le_bytes(witness_bytes.len() as u64)); hasher.update(&witness_bytes); - Ok(Blake3Hash(*hasher.finalize().as_bytes())) + let buf = hasher + .finalize() + .map_err(|e| Error::Zkml(format!("blake3 finalize: {e:?}")))?; + let mut out = [0u8; 32]; + out.copy_from_slice(buf.as_slice()); + Ok(Blake3Hash(out)) } } diff --git a/deny.toml b/deny.toml index 337850f..2a3bc65 100644 --- a/deny.toml +++ b/deny.toml @@ -9,18 +9,7 @@ no-default-features = false db-path = "~/.cargo/advisory-db" db-urls = ["https://github.com/rustsec/advisory-db"] yanked = "deny" -ignore = [ - # `paste` 는 archived/unmaintained 이지만 eager macro expansion 만 수행하는 - # 작은 proc-macro 크레이트로 알려진 취약점이 없습니다. 우리 직접 의존이 - # 아니라 `candle-core` → `gemm` / `tokenizers` / `pulp` 트리(즉 `candle` - # feature 활성화 시에만)에서만 옵니다. Drop-in 대체(`pastey`)가 있지만 - # 상위 크레이트들이 채택하기 전까지는 회피 불가. - { id = "RUSTSEC-2024-0436", reason = "transitive via candle-core (feature-gated), no upstream alternative yet" }, - # `number_prefix` 는 unmaintained 이지만 알려진 보안 취약점이 없습니다. - # `tokenizers v0.19` → `indicatif` 를 통한 transitive 의존으로 직접 제거 - # 불가. 안전한 업그레이드 경로가 없으며 상위 크레이트가 채택 전까지 회피 불가. - { id = "RUSTSEC-2025-0119", reason = "transitive via tokenizers→indicatif, no safe upgrade available upstream" }, -] +ignore = [] [licenses] confidence-threshold = 0.93 diff --git a/kernel-comp.md b/kernel-comp.md new file mode 100644 index 0000000..1639f78 --- /dev/null +++ b/kernel-comp.md @@ -0,0 +1,366 @@ +# 폐쇄형(Air-Gapped) 환경 의존성 점검 보고서 + +> **목적**: Lumen을 K0 마이크로커널 환경에서 동작하도록 개조하기 위해, +> 현재 사용 중인 외부 의존성을 전수 점검하고 교체/제거/유지 방침 정리. + +--- + +## 요약 + +| 분류 | 의존성 | 조치 | +|---------------------------|----------------------------------------|--------------------------------------------| +| **암호학 (교체 대상)** | `blake3` | elib-k0-nt/blake 로 교체 | +| | `ed25519-dalek` | elib-k0-nt/ed25519 로 교체 | +| | `subtle` | elib-k0-nt/constant-time 로 교체 | +| | `rand` + `rand_core` | elib-k0-nt/rng 로 교체 + 어댑터 작성 | +| | `aes-gcm` (crypto-channel) | elib-k0-nt/aes 로 교체 | +| | `x25519-dalek` (crypto-channel) | elib-k0-nt/x25519 로 교체 | +| **대형 외부 (평가 필요)** | `wasmtime` | 벤더링 유지 (대체재 없음) | +| | `halo2_proofs` + `ff` + `pasta_curves` | 현재 mock-only → 별도 결정 필요 | +| | `candle-*` | 벤더링 또는 llama.cpp 백엔드 전환 | +| | `tokenizers` | GGUF 내장 토크나이저로 제거 가능 | +| **유지** | 나머지 (serde, tokio 등) | 순수 Rust, 네트워크 무관 → 벤더링으로 유지 | + +--- + +## 1. 교체 대상: 암호학 의존성 + +### 1-1. `blake3` + +**현재 사용처** + +| 위치 | 사용 내용 | +|--------------------------------------|-------------------------------------------------------------------------------------| +| `lumen-core/src/hash.rs` | `Blake3Hash::of()`, `Blake3Hash::of_file()` — 모델 파일 및 데이터 해시 | +| `lumen-channel/src/encrypted.rs:290` | `blake3::Hasher::new_keyed(shared)` — x25519 공유 비밀에서 방향별 AES 키 도출 (KDF) | +| `lumen-zkml` | 증명 witness commitment 해시 | +| `lumen-inference` | 프롬프트/모델 디제스트 tracing용 해시 | + +**elib-k0-nt 대체** + +`elib-k0-nt/blake` 에 `Blake3` 구현 존재. API 매핑: + +```rust +// 기존: blake3::hash(data) +// 신규: Blake3::new().update(data).finalize() + +// 기존: blake3::Hasher::new_keyed(&key).update(data).finalize() +// → elib-k0-nt/blake 에 keyed 모드 없음 → 추가 구현 필요 (아래 참고) +``` + +**추가 구현 필요 사항** + +`elib-k0-nt/blake` 에 `Blake3::new_keyed(key: &[u8; 32])` 지원 추가 필요. +BLAKE3 스펙의 keyed hash 모드(domain separation context `"hash")는 입력 첫 블록에 키를 혼합하는 방식으로 구현되어 있음. 현재 `lumen-channel` 의 KDF 사용처 1곳이 이에 해당. + +--- + +### 1-2. `ed25519-dalek` + +**현재 사용처** + +| 위치 | 사용 내용 | +|----------------------------------|-----------------------------------------------------------| +| `lumen-core/src/crypto.rs` | `SigningKey`, `VerifyingKey` 래퍼 — 에이전트 ID 서명 키쌍 | +| `lumen-channel/src/encrypted.rs` | 핸드셰이크 서명/검증 | +| `lumen-capability` | Capability 토큰 서명/검증 | +| `lumen-provenance` | 모델 매니페스트 Ed25519 서명 검증 | + +**elib-k0-nt 대체** + +`elib-k0-nt/ed25519` 로 직접 교체. API 차이: + +```rust +// 기존 키 생성: SigningKey::generate(&mut OsRng) +// 신규: +let mut seed = [0u8; 32]; +rng.fill_bytes(&mut seed); // elib-k0-nt/rng 사용 +let sk = SecretKey::from_bytes(&seed)?; +let pk = PublicKey::from(&sk); + +// 기존 서명: sk.sign(msg) +// 신규: sign(msg, &sk) → Signature + +// 기존 검증: pk.verify(msg, &sig) +// 신규: verify(msg, &sig, &pk) → Result +``` + +`lumen-core` 의 `SigningKey`/`VerifyingKey` 공개 타입 래퍼 유지 가능 (내부 구현만 교체). + +--- + +### 1-3. `subtle` + +**현재 사용처** + +`lumen-core/src/hash.rs:56–58` 한 곳: `Blake3Hash` 의 타이밍 공격 방어용 상수시간 동등 비교. + +```rust +// 기존: subtle::ConstantTimeEq::ct_eq(&self.0, &other.0).into() +// 신규: CtEqOps::eq(&a, &b).unwrap_u8() == 1 (elib-k0-nt/constant-time) +``` + +단일 사용처이므로 교체 난이도 낮음. + +--- + +### 1-4. `rand` + `rand_core` + +**현재 사용처** + +| 위치 | 사용 내용 | +|----------------------------------|----------------------------------------------------------------------------------------------------| +| `lumen-core/src/ids.rs` | `AgentId::random(&mut rng)`, `CapabilityId::random(&mut rng)` — `rand_core::RngCore` 트레이트 수용 | +| `lumen-core/src/crypto.rs` | `SigningKey::generate(&mut OsRng)` | +| `lumen-channel/src/encrypted.rs` | `OsRng` 기반 x25519 임시 키 생성, 논스 생성 | +| 다수 테스트/예제 | `OsRng` | + +**elib-k0-nt 대체** + +`elib-k0-nt/rng` 제공 API: `HashDRBGSHA256::new(entropy, nonce, personalization)` + `generate_fixed::()`. + +```rust +// OS 엔트로피 수집 (elib-k0-nt/rng) +let entropy = rng::os_entropy::get_entropy(64)?; +let mut drbg = HashDRBGSHA256::new(&entropy, &nonce, &personalization)?; +let key_bytes: [u8; 32] = drbg.generate_fixed()?; +``` + +**마이그레이션 시 주의 사항** + +- `rand_core::RngCore` 트레이트를 수용하는 제네릭 함수들(`AgentId::random`)이 있어 트레이트 바운드를 변경하거나, 얇은 `RngCore` 어댑터를 구현해야 함. +- `rand` 크레이트 자체는 dev-only (`[dev-dependencies]`)로 쓰이는 곳이 많으므로, 프로덕션 코드에서는 `rand_core` 만 정리하면 됨. +- `rand_core::OsRng` 는 `elib-k0-nt/rng::os_entropy` 로 대체. + +--- + +### 1-5. `aes-gcm` (feature: `crypto-channel`) + +**현재 사용처** + +`lumen-channel/src/encrypted.rs` 전용: +- `Aes256Gcm::new(key)` — tx/rx 방향별 사이퍼 초기화 +- `cipher.encrypt(nonce, payload)` / `cipher.decrypt(nonce, payload)` — 프레임 암복호화 +- `aes-gcm` 은 태그를 암호문 뒤에 붙여 반환 + +**elib-k0-nt 대체** + +`elib-k0-nt/aes` 의 `AES256GCM` 사용. API 차이 (태그 분리): + +```rust +// 기존: let ciphertext_with_tag = cipher.encrypt(nonce, Payload { msg, aad })?; +// 신규: +let mut ciphertext = vec![0u8; plaintext.len()]; +let mut tag = [0u8; 16]; +AES256GCM::new(key).encrypt(nonce, aad, plaintext, &mut ciphertext, &mut tag)?; +// 프레임 직렬화 시 ciphertext || tag 로 직접 이어붙이면 동일 포맷 유지 가능 +``` + +--- + +### 1-6. `x25519-dalek` (feature: `crypto-channel`) + +**현재 사용처** + +`lumen-channel/src/encrypted.rs:137`: +```rust +let my_eph = EphemeralSecret::random_from_rng(OsRng); +let my_pub = X25519Public::from(&my_eph); +let shared = my_eph.diffie_hellman(&their_pub); +``` + +**elib-k0-nt 대체** + +`elib-k0-nt/x25519` 직접 교체: + +```rust +// 임시 키 생성: rng로 32바이트 생성 후 SecretKey 래핑 +let eph_bytes: [u8; 32] = drbg.generate_fixed()?; +let my_eph = x25519::SecretKey::from_bytes(eph_bytes); +let my_pub = my_eph.public_key(); +let shared: SharedSecret = my_eph.diffie_hellman(&their_pub); +// shared.expose() 로 [u8;32] 접근 가능 +``` + +`EphemeralSecret` 타입 소멸 후 키 파기는 elib-k0-nt `SharedSecret` 의 자동 zeroize로 동일하게 보장됨. + +--- + +## 2. 평가 대상: 대형 외부 의존성 + +### 2-1. `wasmtime` + +**현재 사용처** + +`lumen-sandbox` 전용. 에이전트 코드를 WASM으로 격리 실행하는 핵심 보안 경계. + +- `Engine::new()` — 결정론적 설정 (fuel, epoch, Cranelift JIT) +- `Module::new()` — WASM 바이트코드 컴파일 +- `Store::new()` + `set_fuel()` / `set_epoch_deadline()` — 실행 한계 강제 +- `Linker::instantiate_async()` — 격리 인스턴스 생성 +- `instance.get_func().call_async()` — 진입점 호출 + +**폐쇄형 환경 문제점** + +- 네트워크 비의존적(런타임 무관), 하지만 Cranelift JIT 컴파일러 포함으로 소스 트리가 방대함 (~100개 이상 크레이트). +- 커널 Ring 0/1에서는 JIT 실행 불가 — Ring 3 또는 별도 프로세스에서만 사용 가능. + +**대안 검토** + +| 대안 | 장점 | 단점 | +|-----------------|--------------------------------------|----------------------------------| +| `wasmi` v0.31+ | 순수 Rust 인터프리터, 소스 트리 소형 | 성능 10–50배 저하, fuel API 상이 | +| `wasm3-sys` | 경량 C 인터프리터 | C 바인딩 → 커널 no_std 불가 | +| wasmtime 벤더링 | 현재 보안 속성 그대로 유지 | 벤더 소스 관리 부담 (~15 MB) | + +**권고**: WASM 샌드박스는 보안의 핵심이므로 wasmtime을 벤더링하여 유지. `wasmi` 로의 전환은 성능 허용 시 별도 검토. 커널 통합 시 Ring 3 사용자공간 프로세스에서 실행하는 구조 필요. + +--- + +### 2-2. `halo2_proofs` + `ff` + `pasta_curves` + +**현재 사용처** + +`lumen-zkml` 의 `halo2` feature 뒤에서만 활성화. + +- `MockProver::verify()` — ZK 회로 제약 충족 여부 검증 (증명 비공개 X, witness 노출) +- `RoutingProofCircuit` — 에이전트 툴 라우팅 결정의 PLONKish 회로 정의 +- 현재 v0.3 기준으로 간결한 ZK(succinct proof)는 구현되지 않음 + +**폐쇄형 환경 문제점** + +- `pasta_curves` 등 타원곡선 연산 의존성이 크고 빌드 시간이 매우 긺. +- MockProver는 실제 ZK가 아니어서 witness가 노출됨 → 보안 기여 미미. +- 네트워크 비의존적이지만 벤더링 크기 상당. + +**직접 구현 방향 (mock 대체)** + +현재 MockProver 사용처는 "회로 제약 만족 여부 검증"에 불과. halo2 없이 대체 가능: + +```rust +// RoutingProofCircuit의 제약 조건들을 순수 단언(assertion) 기반으로 직접 검증 +// → lumen-zkml의 mock 프루버를 halo2 불필요 버전으로 재작성 +// BLAKE3 기반 commitment만 유지 (elib-k0-nt/blake) +``` + +실제 succinct ZK가 필요한 v0.4 시점에는 별도 프레임워크(SP1, RISC Zero 등) 선택 필요. + +**권고**: `halo2` feature를 기본에서 제거하고, mock prover를 halo2 불필요 버전으로 재구현. 실제 ZK는 마일스톤 재검토 시 결정. + +--- + +### 2-3. `candle-core` + `candle-onnx` + `candle-transformers` + +**현재 사용처** + +`lumen-inference` 의 `candle` / `candle-llm` feature 뒤에서만 활성화. + +- `candle-core`: `Device::Cpu`, `Tensor` 연산 (CPU 전용) +- `candle-transformers`: `ModelWeights` (GGUF 양자화 LLaMA), `LogitsProcessor` 샘플링 +- `candle-onnx`: ONNX 모델 라우팅 엔진 + +**폐쇄형 환경 문제점** + +- 런타임 네트워크 비의존적이나, candle은 HuggingFace Hub 다운로드 예제 코드 포함 → **소스 벤더링 시 해당 부분 제거 필요**. +- `rayon` (병렬 연산), `blas`/`metal` 선택적 의존 → 커널 환경에서는 단일 스레드 강제 필요. +- 빌드 의존 크레이트 수 매우 많음. + +**대안** + +| 대안 | 비고 | +|--------------------------|-------------------------------------------------------------------------------| +| `llama-cpp-2` (C 바인딩) | 이미 `lumen-inference/src/llama_cpp.rs` 스텁 존재. GGUF 내장 토크나이저 지원. | +| candle 소스 벤더링 | CPU-only feature 강제, HF Hub 코드 제거 필요 | +| 자체 GGUF 파서 + 추론 | 구현 비용 매우 높음 | + +**권고**: llama.cpp 백엔드(`llama-cpp-2`)를 우선 완성. air-gapped에서는 C 코드가 허용된다면 llama.cpp가 더 성숙하고 가벼움. candle은 소스 벤더링 후 CPU-only로 유지하는 것도 가능. + +--- + +### 2-4. `tokenizers` (HuggingFace) + +**현재 사용처** + +`lumen-inference/src/candle_llm.rs:86`: +```rust +let tokenizer = Tokenizer::from_file(tokenizer_path) // HF tokenizer.json 로컬 파일 +``` + +- BPE/WordPiece 토크나이저를 `tokenizer.json` 형식으로 로드 +- `encode()` / `decode()` — 텍스트 ↔ 토큰 ID 변환 +- EOS 토큰 감지 (``, `<|endoftext|>`) + +**폐쇄형 환경 문제점** + +- `onig` feature: C 소스(Oniguruma 정규식)를 소스에서 빌드 → 커널 no_std 환경 불가. +- 대형 dep tree (rayon, serde_json, unicode 등). +- **GGUF 모델 파일에는 토크나이저 어휘/머지 규칙이 내장되어 있음** — `tokenizer.json` 불필요. + +**직접 구현 방향** + +GGUF 형식(`gguf-rs` 또는 직접 파서)에서 어휘를 추출 후 최소 BPE 토크나이저 구현: + +``` +GGUF 헤더 → tokenizer.ggml.* 메타데이터 추출 + → vocab (id → str), merges (str pair → id) 로딩 + → encode(): pre-tokenize → BPE 머지 반복 + → decode(): id → str 매핑 +``` + +LLaMA 3 / Mistral 계열은 SentencePiece BPE, GPT-2 계열은 Byte-BPE. +약 300–500 LoC 순수 Rust 구현 가능 (no_std 호환). + +**권고**: `tokenizers` 제거하고 GGUF 내장 어휘 기반 최소 BPE 토크나이저를 `lumen-inference` 에 직접 구현. llama.cpp 백엔드 사용 시 해당 백엔드가 토크나이저를 내장 처리하므로 불필요. + +--- + +## 3. 유지 대상 (문제없는 의존성) + +아래 의존성들은 순수 Rust, 네트워크 무관하며 직접 구현 비용이 유지 비용을 초과. **소스 벤더링**으로 유지. + +| 의존성 | 이유 | +|-------------------------------------------|-------------------------------------------------------------------| +| `tokio` | 비동기 런타임. OS API만 사용. 커널 Ring 3 프로세스에서 동작 가능. | +| `async-trait`, `futures` | 순수 Rust 비동기 추상화. | +| `serde`, `serde_json`, `toml`, `postcard` | 직렬화. 네트워크 무관. | +| `hex` | 16진수 인코딩. 60 LoC 대체 가능하나 유지가 합리적. | +| `thiserror`, `anyhow` | 에러 추상화. 순수 Rust. | +| `aho-corasick`, `regex` | 프롬프트 인젝션 방어 패턴 매칭. 순수 Rust, 네트워크 무관. | +| `globset` | 파일 경로 Capability 매칭. 순수 Rust. | +| `once_cell` | 정적 초기화. 순수 Rust. | +| `parking_lot` | 동시성 기본기. 순수 Rust. | +| `safetensors` | 모델 헤더 읽기 전용. 네트워크 무관. 순수 Rust. | +| `clap` | CLI 전용. 커널 내부 로직 불관여. | +| `tracing`, `tracing-subscriber` | 감사 로그. 순수 Rust. | +| `criterion` | dev-only 벤치마크. | +| `wat` | dev-only WASM 텍스트 변환. | +| `safetensors` | 테스트 픽스처 생성 전용 (프로덕션 불사용). | + +--- + +## 4. 마이그레이션 우선순위 + +### Phase 1 — 암호학 교체 (즉시 착수) + +교체 자체가 API 레벨 수정에 그침. elib-k0-nt 모듈로 1:1 대체. + +1. `subtle` → `elib-k0-nt/constant-time` (`lumen-core/src/hash.rs` 1곳) +2. `blake3` → `elib-k0-nt/blake` (keyed 모드 추가 필요) +3. `ed25519-dalek` → `elib-k0-nt/ed25519` +4. `rand` + `rand_core` → `elib-k0-nt/rng` + `RngCore` 어댑터 작성 +5. `aes-gcm` → `elib-k0-nt/aes` (crypto-channel feature) +6. `x25519-dalek` → `elib-k0-nt/x25519` (crypto-channel feature) + +### Phase 2 — 추론 백엔드 정비 + +1. `tokenizers` 제거 → GGUF 내장 BPE 토크나이저 직접 구현 또는 llama.cpp 백엔드 전환 +2. candle 의존성을 llama.cpp 백엔드로 대체 (또는 소스 벤더링 후 CPU-only 강제) + +### Phase 3 — ZK 정리 및 WASM 벤더링 + +1. `halo2` feature 기본 비활성화 + mock prover 독립 재구현 (halo2_proofs 제거) +2. `wasmtime` 전체 소스 벤더링 + HF Hub 참조 제거 + +### Phase 4 — 나머지 의존성 벤더링 + +나머지 "유지 대상" 의존성들을 오프라인 소스 벤더링(`cargo vendor`)으로 전환. `K0` 빌드 시스템에 통합. \ No newline at end of file diff --git a/lumen-demo-screenshot.png b/lumen-demo-screenshot.png new file mode 100644 index 0000000..3c467ee Binary files /dev/null and b/lumen-demo-screenshot.png differ diff --git a/lumen-demo.gif b/lumen-demo.gif new file mode 100644 index 0000000..d6fa5d0 Binary files /dev/null and b/lumen-demo.gif differ diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 0459460..9381a24 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,7 +1,7 @@ [toolchain] # Lumen CI 와 동일한 toolchain 으로 로컬 개발 환경을 강제 동기화합니다. # 변경 시 .github/docker/Dockerfile.ci 의 `FROM rust:-bookworm` 도 함께 갱신. -channel = "1.93.0" +channel = "1.95.0" components = ["clippy", "rustfmt"] targets = ["wasm32-unknown-unknown"] profile = "minimal" diff --git a/scripts/vendor.sh b/scripts/vendor.sh new file mode 100755 index 0000000..be78933 --- /dev/null +++ b/scripts/vendor.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash +# +# Lumen 폐쇄형(Air-Gapped) 환경 의존성 벤더링 스크립트. +# +# 동작: +# 1. cargo vendor 로 모든 워크스페이스 의존성 소스를 vendor/ 에 복제 +# 2. .cargo/vendor-config.toml 을 .cargo/config.toml 로 활성화 +# 3. 폐쇄망에서 필요한 시스템 바이너리 (cmake 등) 위치 안내 +# +# 사용: +# ./scripts/vendor.sh # 기본 (default features) +# ./scripts/vendor.sh --all # crypto-channel + llama-cpp 까지 포함 +# ./scripts/vendor.sh --check # vendor/ 무결성 재검증 (재실행 안전) +# +# 산출물: +# vendor/ ~500MB–1GB, .gitignore 됨 +# .cargo/config.toml 소스 교체 + net.offline=true +# +# 폐쇄망 이전 절차: +# 1. (온라인) ./scripts/vendor.sh --all +# 2. tar czf lumen-airgap-bundle.tar.gz . --exclude=target --exclude=.git +# 3. (오프라인) tar xzf lumen-airgap-bundle.tar.gz && cargo build --offline ... +# + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$REPO_ROOT" + +MODE="default" +if [[ "${1:-}" == "--all" ]]; then + MODE="all" +elif [[ "${1:-}" == "--check" ]]; then + MODE="check" +elif [[ -n "${1:-}" ]]; then + echo "사용법: $0 [--all | --check]" >&2 + exit 1 +fi + +echo "==> Lumen 의존성 벤더링 (mode=${MODE})" + +# 1. 사전 점검: cargo, git +command -v cargo >/dev/null 2>&1 || { echo "cargo 가 PATH 에 없습니다." >&2; exit 1; } + +# 2. cargo vendor 실행 +VENDOR_ARGS=() +if [[ "$MODE" == "all" ]]; then + # 모든 optional feature 의 의존성도 함께 가져옵니다. + # llama-cpp 는 cmake 사전 설치 필요. 이 스크립트는 vendor 만 수행하고 + # 빌드는 별도이므로 cmake 부재여도 vendor 는 성공합니다. + VENDOR_ARGS+=( + --sync crates/lumen-channel/Cargo.toml + --sync crates/lumen-inference/Cargo.toml + ) +fi + +if [[ "$MODE" == "check" ]]; then + if [[ ! -d vendor ]]; then + echo "vendor/ 디렉토리가 없습니다. 먼저 $0 또는 $0 --all 을 실행하세요." >&2 + exit 1 + fi + echo "==> cargo vendor 무결성 재검증 (--no-delete)" + cargo vendor --no-delete vendor >/dev/null +else + echo "==> cargo vendor 실행 — 약 500MB~1GB 의 소스를 vendor/ 로 가져옵니다..." + cargo vendor "${VENDOR_ARGS[@]}" vendor >/dev/null +fi + +# 3. .cargo/config.toml 활성화 (이미 존재하면 백업) +if [[ -f .cargo/config.toml && ! -L .cargo/config.toml ]]; then + cp .cargo/config.toml ".cargo/config.toml.backup-$(date +%Y%m%d-%H%M%S)" + echo "==> 기존 .cargo/config.toml 을 백업했습니다." +fi + +cp .cargo/vendor-config.toml .cargo/config.toml +echo "==> .cargo/config.toml 활성화 (vendor-config.toml 복사)" + +# 4. 결과 요약 +VENDOR_SIZE=$(du -sh vendor 2>/dev/null | cut -f1) +VENDOR_COUNT=$(find vendor -mindepth 1 -maxdepth 1 -type d | wc -l | tr -d ' ') +echo "" +echo "==> 완료" +echo " vendor/ 크기 : ${VENDOR_SIZE}" +echo " 벤더된 크레이트 : ${VENDOR_COUNT} 개" +echo "" +echo "다음 단계 (4-게이트):" +echo " cargo build --offline --workspace --all-targets" +echo " cargo test --offline --workspace" +echo " cargo clippy --offline --workspace --all-targets -- -D warnings" +echo " cargo fmt --all -- --check" +echo "" +echo "옵셔널 feature 빌드:" +echo " cargo build --offline --workspace --features lumen-channel/crypto-channel,lumen-inference/llama-cpp" +echo "" +echo "폐쇄망 시스템 바이너리 (워크스페이스 외부):" +echo " - cmake (lumen-inference/llama-cpp feature 활성화 시 필수, v0.5)" +echo "" +echo "AIR-GAPPED.md 참고."