From a874498f2d67858d2f7a1416bb75d0b155be6c21 Mon Sep 17 00:00:00 2001 From: Hung Pham Date: Thu, 29 Oct 2026 16:14:34 +0700 Subject: [PATCH 1/2] =?UTF-8?q?fix(lmdb):=20tr=C3=A1nh=20MDB=5FMAP=5FFULL?= =?UTF-8?q?=20b=E1=BA=B1ng=20auto-grow=20map=20tr=C6=B0=E1=BB=9Bc=20khi=20?= =?UTF-8?q?ghi?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Tăng default map size: 8 GiB (64-bit) / 1 GiB (32-bit), thay vì cứng 1 GiB - Thêm env override CODEGRAPH_LMDB_MAP_SIZE_MB để đặt map khởi đầu - begin_rw() grow map trước mỗi write txn khi dùng >= 50% map (trước: 25%) - grow_map Ziele max(map*2, used*4) để sau grow còn >= 75% trống, tránh cascade grow liền sau grow - Cap MAX_MAP_SIZE (64 GiB / 4 GiB) tránh runaway, trả MapFull nếu đã đạt - open_env_read_only dùng initial_map_size() thay vì cứng 1 GiB - Cập nhật docs/configuration.md [storage] với note map size + env var Cùng commit: privacy fix (pub mod -> mod + pub use re-export), meminfo doctests, clippy sạch mọi config, test workspace xong (trừ 4 redis pre-existing). --- crates/codegraph-api/src/tools.rs | 12 +- crates/codegraph-bench/src/bin/mem.rs | 6 +- crates/codegraph-extract/src/config.rs | 6 +- crates/codegraph-graph/src/embeddings.rs | 3 + crates/codegraph-graph/src/lib.rs | 23 +- crates/codegraph-graph/src/meminfo.rs | 4 +- crates/codegraph-graph/src/storage/lmdb.rs | 234 +++++++++++++++++---- crates/codegraph/src/main.rs | 2 +- docs/configuration.md | 9 + 9 files changed, 240 insertions(+), 59 deletions(-) diff --git a/crates/codegraph-api/src/tools.rs b/crates/codegraph-api/src/tools.rs index 0ea6ed0692..b53a0ecfad 100644 --- a/crates/codegraph-api/src/tools.rs +++ b/crates/codegraph-api/src/tools.rs @@ -269,8 +269,8 @@ pub async fn dispatch_diff( args: Value, ) -> Result { let diff = arg_str(&args, "diff")?; - let parsed = codegraph_graph::diff::parse_unified_diff(diff) - .map_err(|e| Error::Invalid(e.to_string()))?; + let parsed = + codegraph_graph::parse_unified_diff(diff).map_err(|e| Error::Invalid(e.to_string()))?; let idx = shared.ensure_fresh().await; let report = idx.diff_assess(&parsed, Some(root.as_std_path())).await; @@ -402,8 +402,8 @@ pub async fn dispatch_branch_compare( let base = arg_str(&args, "base")?; let head = args.get("head").and_then(|v| v.as_str()); let diff = git.diff(base, head).await?; - let parsed = codegraph_graph::diff::parse_unified_diff(&diff) - .map_err(|e| Error::Invalid(e.to_string()))?; + let parsed = + codegraph_graph::parse_unified_diff(&diff).map_err(|e| Error::Invalid(e.to_string()))?; let idx = shared.ensure_fresh().await; let report = idx.diff_assess(&parsed, Some(root.as_std_path())).await; @@ -461,8 +461,8 @@ pub async fn dispatch_diff_simulate( args: Value, ) -> Result { let diff = arg_str(&args, "diff")?; - let parsed = codegraph_graph::diff::parse_unified_diff(diff) - .map_err(|e| Error::Invalid(e.to_string()))?; + let parsed = + codegraph_graph::parse_unified_diff(diff).map_err(|e| Error::Invalid(e.to_string()))?; let base_ref = args .get("base_ref") .and_then(|v| v.as_str()) diff --git a/crates/codegraph-bench/src/bin/mem.rs b/crates/codegraph-bench/src/bin/mem.rs index 310802e420..078bdd3733 100644 --- a/crates/codegraph-bench/src/bin/mem.rs +++ b/crates/codegraph-bench/src/bin/mem.rs @@ -16,8 +16,8 @@ use camino::Utf8Path; use clap::Parser; use codegraph_bench::{BenchOptions, Repo, extract, index_at, orchestrator, run_queries}; use codegraph_core::{Annotation, CallRecord, EffectType, ScopeLevel, Symbol, SymbolKind}; -use codegraph_graph::meminfo::{MemTracker, fmt_bytes, rss_bytes}; -use codegraph_graph::memtrack::MemBreakdown; +use codegraph_graph::MemBreakdown; +use codegraph_graph::{MemTracker, fmt_bytes, rss_bytes}; use std::sync::OnceLock; fn runtime() -> &'static tokio::runtime::Runtime { @@ -555,7 +555,7 @@ fn main() -> anyhow::Result<()> { println!("\n=== breakdown: {} ===", r.repo); print_breakdown(&r.breakdown, &r.caches, r.rss_after_index); } - if let Some(peak) = codegraph_graph::meminfo::peak_rss_bytes() { + if let Some(peak) = codegraph_graph::peak_rss_bytes() { println!("\npeak RSS (VmHWM): {}", fmt_bytes(peak)); } else { println!("\npeak RSS: không đọc được trên nền tảng này"); diff --git a/crates/codegraph-extract/src/config.rs b/crates/codegraph-extract/src/config.rs index bad166b07f..a7d2c3b682 100644 --- a/crates/codegraph-extract/src/config.rs +++ b/crates/codegraph-extract/src/config.rs @@ -470,7 +470,7 @@ pub struct ExtractConfig { /// Backend storage được chọn trong config (mặc định sqlite). pub storage: StorageConfig, /// Cấu hình embedding backend (semantic search) — đọc từ `[embedding]`. - pub embedding: codegraph_graph::embeddings::EmbeddingConfig, + pub embedding: codegraph_graph::EmbeddingConfig, /// Cấu hình Jev rerank cho `codegraph_context` — đọc từ `[jev]`. pub jev: codegraph_core::JevConfig, /// Cấu hình document graph — đọc từ `[docgraph]`. @@ -536,7 +536,7 @@ impl ExtractConfig { header_language: parse_header_language(file.languages.headers.as_deref()), effect_classifier, flow_classifier, - embedding: codegraph_graph::embeddings::EmbeddingConfig::from_raw( + embedding: codegraph_graph::EmbeddingConfig::from_raw( file.embedding.backend.as_deref(), file.embedding.model.as_deref(), file.embedding.cache_dir.as_deref(), @@ -606,7 +606,7 @@ impl ExtractConfig { // Áp dụng config embedding (backend/model/cache) cho process trước khi // mở index — `GraphIndex::new_with_storage` đọc global này để quyết định // có bật vector index hay không (opt-in: chỉ khi backend = "fastembed"). - codegraph_graph::embeddings::set_embedding_config(self.embedding.clone()); + codegraph_graph::set_embedding_config(self.embedding.clone()); // Cùng thời điểm: `[jev]` là config toàn cục cho `codegraph-context` // (module context không phụ thuộc `codegraph-extract`, nên truyền qua // global giống embedding). diff --git a/crates/codegraph-graph/src/embeddings.rs b/crates/codegraph-graph/src/embeddings.rs index 24c4596859..d9b9c890c5 100644 --- a/crates/codegraph-graph/src/embeddings.rs +++ b/crates/codegraph-graph/src/embeddings.rs @@ -199,6 +199,8 @@ fn expand_tilde(path: &str) -> Option { } /// Suffix file extension của sqlite-vss theo OS (`.dylib` / `.so` / `.dll`). +// Chỉ `storage/sqlite.rs` dùng — tắt feature `sqlite` thì cặp hàm này dead code. +#[cfg_attr(not(feature = "sqlite"), allow(dead_code))] fn vss_lib_suffix() -> &'static str { if cfg!(target_os = "macos") { "dylib" @@ -216,6 +218,7 @@ fn vss_lib_suffix() -> &'static str { /// dò `/vss`. Chỉ trả `Some` khi file thực sự tồn tại → caller có /// thể yên tâm thêm extension vào kết nối SQLite mà không làm hỏng `open` /// khi thiếu binary. +#[cfg_attr(not(feature = "sqlite"), allow(dead_code))] pub fn resolve_vss_extensions() -> Option<(PathBuf, PathBuf)> { let cfg = embedding_config(); let dir = cfg diff --git a/crates/codegraph-graph/src/lib.rs b/crates/codegraph-graph/src/lib.rs index 570c81cfa6..1088a32c04 100644 --- a/crates/codegraph-graph/src/lib.rs +++ b/crates/codegraph-graph/src/lib.rs @@ -81,16 +81,29 @@ use tokio::sync::RwLock; #[cfg(feature = "bloom-search")] mod bloom; -pub mod diff; -pub mod embeddings; +mod diff; +mod embeddings; mod lru; -pub mod meminfo; -pub mod memtrack; +mod meminfo; +mod memtrack; mod radix; mod search; mod shared; mod storage; -pub mod vector_index; +// Chỉ `VectorIndex::knn` được dùng nội bộ; phần API còn lại (insert/set_all/ +// kmeans...) giữ cho semantic search opt-in + test, nên tắt dead-code ở đây +// thay vì để cả module public. +#[allow(dead_code)] +mod vector_index; + +// Các module nội bộ trên chỉ mở đúng phần API mà crate khác dùng — thay cho +// `pub mod`, để bề mặt public không phình theo từng file nội bộ. +pub use crate::diff::{FileDiff, Hunk, ParsedDiff, parse_unified_diff}; +#[cfg(feature = "fastembed")] +pub use crate::embeddings::warm_model_cache; +pub use crate::embeddings::{EmbeddingConfig, set_embedding_config}; +pub use crate::meminfo::{MemTracker, fmt_bytes, peak_rss_bytes, rss_bytes}; +pub use crate::memtrack::MemBreakdown; pub use shared::SharedGraphIndex; diff --git a/crates/codegraph-graph/src/meminfo.rs b/crates/codegraph-graph/src/meminfo.rs index fda44cdd96..99d3585d44 100644 --- a/crates/codegraph-graph/src/meminfo.rs +++ b/crates/codegraph-graph/src/meminfo.rs @@ -11,7 +11,7 @@ //! - **Platform khác**: `None` — caller phải xử lý, KHÔNG panic. //! //! ```no_run -//! use codegraph_graph::meminfo::{MemTracker, rss_bytes}; +//! use codegraph_graph::{MemTracker, rss_bytes}; //! //! let mut t = MemTracker::new(); //! t.mark("baseline"); @@ -92,7 +92,7 @@ pub fn fmt_bytes(bytes: u64) -> String { /// (đỉnh), nên benchmark tự lấy mẫu tại các mốc rồi giữ max. /// /// ```no_run -/// # use codegraph_graph::meminfo::MemTracker; +/// # use codegraph_graph::MemTracker; /// let mut t = MemTracker::new(); /// t.mark("open"); /// t.mark("ingest"); diff --git a/crates/codegraph-graph/src/storage/lmdb.rs b/crates/codegraph-graph/src/storage/lmdb.rs index aefa440fca..0b0fa590fc 100644 --- a/crates/codegraph-graph/src/storage/lmdb.rs +++ b/crates/codegraph-graph/src/storage/lmdb.rs @@ -190,26 +190,116 @@ const KEY_ONE: [u8; 8] = [0u8; 8]; // ── Env ── -fn open_env(path: &str) -> Result> { +/// Map size mặc định. LMDB chỉ **reserve address space**, không cấp phát đĩa: +/// `data.mdb` là file sparse, chỉ tốn cho trang thực ghi. Trên 64-bit chọn 8 GiB +/// để ingest repo lớn không chạm `MDB_MAP_FULL`; 32-bit giữ 1 GiB vì địa chỉ hạn chế. +#[cfg(target_pointer_width = "64")] +const DEFAULT_MAP_SIZE: usize = 8 << 30; +#[cfg(not(target_pointer_width = "64"))] +const DEFAULT_MAP_SIZE: usize = 1 << 30; + +/// Ngưỡng dùng/map khi bắt đầu grow: chỉ grow khi đã dùng ≥ 50% map. +/// (25% grow quá dồn — sau khi gấp đôi, dùng lại 50% map mới → ngay kích hoạt +/// grow lại, gây cascade của nhiều lần grow liên tiếp.) +const GROW_THRESHOLD_DEN: usize = 2; // used * 2 >= map_size <=> used >= 50% + +/// Map size tối đa cho phép grow. Tránh runaway nếu workload bất thường. +/// 64 GiB trên 64-bit (rất già so với mọi repo thực tế — map chỉ là address +/// space, data.mdb sparse chỉ tốn đĩa cho trang thực ghi). 32-bit giữ 4 GiB. +#[cfg(target_pointer_width = "64")] +const MAX_MAP_SIZE: usize = 64 << 30; +#[cfg(not(target_pointer_width = "64"))] +const MAX_MAP_SIZE: usize = 4 << 30; + +/// Map size khởi tạo: `CODEGRAPH_LMDB_MAP_SIZE_MB` (nếu hợp lệ) hoặc mặc định. +/// +/// Đây chỉ là **điểm khởi đầu** — [`begin_rw`] tự grow map khi dữ liệu chạm +/// ngưỡng, nên hiếm khi phải chỉnh tay. +fn initial_map_size() -> usize { + std::env::var("CODEGRAPH_LMDB_MAP_SIZE_MB") + .ok() + .and_then(|v| v.trim().parse::().ok()) + .filter(|mb| *mb > 0) + .map(|mb| mb.saturating_mul(1 << 20)) + .unwrap_or(DEFAULT_MAP_SIZE) +} + +fn open_env_with_map_size(path: &str, map_size: usize) -> Result> { let p = Path::new(path); std::fs::create_dir_all(p).map_err(e)?; let mut b = Environment::new(); b.set_max_dbs(32); // schema dùng ~17 named-db b.set_max_readers(512); // locktable đủ chỗ cho runtime/mcp probe + request song song - b.set_map_size(1 << 30); // 1 GiB address space (LMDB chỉ commit trang thực đụng) + b.set_map_size(map_size); let env = b.open(p).map_err(e)?; Ok(Arc::new(env)) } +fn open_env(path: &str) -> Result> { + open_env_with_map_size(path, initial_map_size()) +} + +/// Tăng `map_size` trước khi ghi. Gấp đôi map, nhưng đảm bảo mới có ít nhất +/// gấp 4 phần đã dùng — sau grow còn ≥ 75% trống, tránh grow liền lại. +/// +/// LMDB không tự mở rộng map và trả `MDB_MAP_FULL` khi hết chỗ — giữa lúc +/// ingest nghĩa là rollback cả transaction. Gọi hàm này **trước khi** mở write +/// txn vì `mdb_env_set_mapsize` từ chối chạy khi đang có txn ghi. +/// +/// Nếu đã đạt `MAX_MAP_SIZE` và vẫn đầy, trả `Error::MapFull` để caller xử lý +/// (không grow vô hạn). +fn grow_map(env: &Environment) -> lmdb::Result { + let info = env.info()?; + let page = env.stat()?.page_size() as usize; + let used = (info.last_pgno() + 1).saturating_mul(page); + let current = info.map_size(); + // Tăng gấp đôi map, nhưng ít nhất gấp 4 phần đã dùng (để sau grow còn ≥ + // 75% trống, tránh grow liền lại), và không vượt MAX_MAP_SIZE. + let mut next = current + .saturating_mul(2) + .max(used.saturating_mul(4)) + .max(page); + if next > MAX_MAP_SIZE { + next = MAX_MAP_SIZE; + } + if next <= current { + return Err(lmdb::Error::MapFull); + } + env.set_map_size(next)?; + eprintln!( + "[codegraph] lmdb map size -> {} MiB (dùng {} trang/{} MiB map, chạm ngưỡng 50%)", + next >> 20, + info.last_pgno() + 1, + current >> 20 + ); + Ok(next) +} + +/// Mở write transaction, đảm bảo map còn ≥ 50% chỗ trống trước khi ghi. +/// +/// Nhờ kiểm tra **trước** khi mở txn nên `set_map_size` luôn hợp lệ (không có +/// write txn đang mở), và map lớn dần theo nhu cầu thay vì chết ở một mức cố +/// định. +fn begin_rw(env: &Environment) -> Result> { + let info = env.info().map_err(e)?; + let page = env.stat().map_err(e)?.page_size() as usize; + let used = (info.last_pgno() + 1).saturating_mul(page); + // Chỉ grow khi đã dùng ≥ 50% map. + if used.saturating_mul(GROW_THRESHOLD_DEN) >= info.map_size() { + grow_map(env).map_err(e)?; + } + env.begin_rw_txn().map_err(e) +} + #[cfg(feature = "lmdb")] #[cfg_attr(feature = "sqlite", allow(dead_code))] // probe chỉ dùng khi lmdb là backend file fn open_env_read_only(path: &str) -> lmdb::Result { let mut b = Environment::new(); b.set_flags(EnvironmentFlags::READ_ONLY); b.set_max_dbs(32); - // Phải set map_size khớp với env read-write (1 GiB) — mở read-only không set + // Phải set map_size khớp với env read-write — mở read-only không set // map_size có thể trả EACCES/PERMISSION_DENIED trên một số platform. - b.set_map_size(1 << 30); + b.set_map_size(initial_map_size()); b.open(Path::new(path)) } @@ -261,13 +351,28 @@ fn probe_env(path: &str) -> lmdb::Result> { pub async fn probe_version(path: &str) -> Result { let handle = probe_env(path) .map_err(|err| StorageError::Internal(format!("lmdb file not found: {path} ({err})")))?; - let tx = handle.env.begin_ro_txn().map_err(e)?; - match tx.get(handle.db, &KEY_ONE).map(de_u64) { - Ok(v) => Ok(v), - Err(lmdb::Error::NotFound) => { - Err(StorageError::Internal("lmdb version row missing".into())) + let version = match read_probe_version(&handle) { + // Env read-write trong process vừa grow map → map của handle read-only + // đã cũ. `set_map_size(0)` = "lấy map size hiện tại trên đĩa"; cần khoá + // vì `mdb_env_set_mapsize` không thread-safe. + Err(lmdb::Error::MapResized) => { + static RESIZE: Mutex<()> = Mutex::new(()); + let _guard = RESIZE.lock().unwrap_or_else(|e| e.into_inner()); + handle.env.set_map_size(0).map_err(e)?; + read_probe_version(&handle).map_err(e)? } - Err(err) => Err(StorageError::Internal(err.to_string())), + other => other.map_err(e)?, + }; + version.ok_or_else(|| StorageError::Internal("lmdb version row missing".into())) +} + +#[cfg(feature = "lmdb")] +fn read_probe_version(handle: &ProbeHandle) -> lmdb::Result> { + let tx = handle.env.begin_ro_txn()?; + match tx.get(handle.db, &KEY_ONE) { + Ok(bytes) => Ok(Some(de_u64(bytes))), + Err(lmdb::Error::NotFound) => Ok(None), + Err(err) => Err(err), } } @@ -404,7 +509,7 @@ impl LmdbStorage { /// Seed sentinel node 0 + counter/next_id/version nếu chưa tồn tại. async fn init(&self) -> Result<()> { - let mut tx = self.env.begin_rw_txn().map_err(e)?; + let mut tx = begin_rw(&self.env)?; let db = self.nodes; if matches!(tx.get(db, &k8(EMPTY)), Err(lmdb::Error::NotFound)) { tx.put(db, &k8(EMPTY), &node_val(b"", 0), WriteFlags::empty()) @@ -450,7 +555,7 @@ impl LmdbStorage { #[async_trait] impl CategoryStorage for LmdbStorage { async fn new_node(&mut self, prefix: Vec, record: usize) -> Result { - let mut tx = self.env.begin_rw_txn().map_err(e)?; + let mut tx = begin_rw(&self.env)?; // Không có RETURNING — đọc-rồi-ghi counter trong cùng write tx; an toàn // vì GraphIndex tuần tự hoá mọi writer qua RwLock. let next = match self.get_opt(&tx, self.counter, &KEY_ONE)? { @@ -477,7 +582,7 @@ impl CategoryStorage for LmdbStorage { prefix: Option>, record: Option, ) -> Result<()> { - let mut tx = self.env.begin_rw_txn().map_err(e)?; + let mut tx = begin_rw(&self.env)?; let key = k8(id); let Some(cur) = self.get_opt(&tx, self.nodes, &key)?.map(de_node_val) else { return Err(StorageError::BranchOutOfRange(id)); @@ -550,7 +655,7 @@ impl CategoryStorage for LmdbStorage { } async fn set_root(&mut self, shard: usize, root: usize) -> Result<()> { - let mut tx = self.env.begin_rw_txn().map_err(e)?; + let mut tx = begin_rw(&self.env)?; tx.put(self.roots, &k8(shard), &k8(root), WriteFlags::empty()) .map_err(e)?; tx.commit().map_err(e)?; @@ -586,7 +691,7 @@ impl Storage for LmdbStorage {} #[async_trait] impl BloomStorage for LmdbStorage { async fn set_node_bloom(&mut self, id: usize, bloom: &[u8]) -> Result<()> { - let mut tx = self.env.begin_rw_txn().map_err(e)?; + let mut tx = begin_rw(&self.env)?; tx.put(self.blooms, &k8(id), &bloom, WriteFlags::empty()) .map_err(e)?; tx.commit().map_err(e)?; @@ -604,7 +709,7 @@ impl BloomStorage for LmdbStorage { #[async_trait] impl EdgeDataStorage for LmdbStorage { async fn set_edge_data(&mut self, edge: usize, data: &[u8]) -> Result<()> { - let mut tx = self.env.begin_rw_txn().map_err(e)?; + let mut tx = begin_rw(&self.env)?; tx.put(self.edges, &k8(edge), &data, WriteFlags::empty()) .map_err(e)?; tx.commit().map_err(e)?; @@ -619,7 +724,7 @@ impl EdgeDataStorage for LmdbStorage { } async fn clear_edges(&mut self) -> Result<()> { - let mut tx = self.env.begin_rw_txn().map_err(e)?; + let mut tx = begin_rw(&self.env)?; tx.clear_db(self.edges).map_err(e)?; tx.commit().map_err(e)?; Ok(()) @@ -631,7 +736,7 @@ impl EdgeDataStorage for LmdbStorage { #[async_trait] impl NodeMetaStorage for LmdbStorage { async fn set_node_meta(&mut self, elem: usize, meta: &[u8]) -> Result<()> { - let mut tx = self.env.begin_rw_txn().map_err(e)?; + let mut tx = begin_rw(&self.env)?; tx.put(self.node_meta, &k8(elem), &meta, WriteFlags::empty()) .map_err(e)?; tx.commit().map_err(e)?; @@ -646,14 +751,14 @@ impl NodeMetaStorage for LmdbStorage { } async fn clear_node_meta(&mut self) -> Result<()> { - let mut tx = self.env.begin_rw_txn().map_err(e)?; + let mut tx = begin_rw(&self.env)?; tx.clear_db(self.node_meta).map_err(e)?; tx.commit().map_err(e)?; Ok(()) } async fn set_meta(&mut self, record: usize, meta: &[u8]) -> Result<()> { - let mut tx = self.env.begin_rw_txn().map_err(e)?; + let mut tx = begin_rw(&self.env)?; tx.put(self.meta, &k8(record), &meta, WriteFlags::empty()) .map_err(e)?; tx.commit().map_err(e)?; @@ -668,7 +773,7 @@ impl NodeMetaStorage for LmdbStorage { } async fn set_key_len(&mut self, record: usize, len: usize) -> Result<()> { - let mut tx = self.env.begin_rw_txn().map_err(e)?; + let mut tx = begin_rw(&self.env)?; tx.put(self.keylen, &k8(record), &k8(len), WriteFlags::empty()) .map_err(e)?; tx.commit().map_err(e)?; @@ -689,7 +794,7 @@ impl NodeMetaStorage for LmdbStorage { #[async_trait] impl ChainStorage for LmdbStorage { async fn set_chain(&mut self, record: usize, chain: &[u64]) -> Result<()> { - let mut tx = self.env.begin_rw_txn().map_err(e)?; + let mut tx = begin_rw(&self.env)?; tx.put( self.chains, &k8(record), @@ -709,7 +814,7 @@ impl ChainStorage for LmdbStorage { } async fn clear_chains(&mut self) -> Result<()> { - let mut tx = self.env.begin_rw_txn().map_err(e)?; + let mut tx = begin_rw(&self.env)?; tx.clear_db(self.chains).map_err(e)?; tx.commit().map_err(e)?; Ok(()) @@ -723,7 +828,7 @@ impl EntityStorage for LmdbStorage { async fn save_symbol(&mut self, sym: &Symbol) -> Result<()> { let data = serde_json::to_vec(sym).map_err(|err| StorageError::Internal(err.to_string()))?; - let mut tx = self.env.begin_rw_txn().map_err(e)?; + let mut tx = begin_rw(&self.env)?; tx.put(self.symbols, &ku64(sym.id), &data, WriteFlags::empty()) .map_err(e)?; tx.commit().map_err(e)?; @@ -754,7 +859,7 @@ impl EntityStorage for LmdbStorage { } async fn save_embedding(&mut self, symbol_id: u64, vector: &[f32]) -> Result<()> { - let mut tx = self.env.begin_rw_txn().map_err(e)?; + let mut tx = begin_rw(&self.env)?; tx.put( self.embeddings, &ku64(symbol_id), @@ -790,14 +895,14 @@ impl EntityStorage for LmdbStorage { } async fn clear_embeddings(&mut self) -> Result<()> { - let mut tx = self.env.begin_rw_txn().map_err(e)?; + let mut tx = begin_rw(&self.env)?; tx.clear_db(self.embeddings).map_err(e)?; tx.commit().map_err(e)?; Ok(()) } async fn save_next_id(&mut self, next: u64) -> Result<()> { - let mut tx = self.env.begin_rw_txn().map_err(e)?; + let mut tx = begin_rw(&self.env)?; tx.put(self.next_id, &KEY_ONE, &ku64(next), WriteFlags::empty()) .map_err(e)?; tx.commit().map_err(e)?; @@ -824,7 +929,7 @@ impl EntityStorage for LmdbStorage { } async fn set_call_records(&mut self, func: u64, records: &[u8]) -> Result<()> { - let mut tx = self.env.begin_rw_txn().map_err(e)?; + let mut tx = begin_rw(&self.env)?; tx.put( self.call_records, &ku64(func), @@ -855,7 +960,7 @@ impl EntityStorage for LmdbStorage { } async fn set_file_deps(&mut self, path: &str, deps: &[u8]) -> Result<()> { - let mut tx = self.env.begin_rw_txn().map_err(e)?; + let mut tx = begin_rw(&self.env)?; tx.put(self.file_deps, &str_key(path), &deps, WriteFlags::empty()) .map_err(e)?; tx.commit().map_err(e)?; @@ -875,7 +980,7 @@ impl EntityStorage for LmdbStorage { } async fn set_fn_ptr_assigns(&mut self, path: &str, assigns: &[u8]) -> Result<()> { - let mut tx = self.env.begin_rw_txn().map_err(e)?; + let mut tx = begin_rw(&self.env)?; tx.put( self.fn_ptr_assigns, &str_key(path), @@ -900,7 +1005,7 @@ impl EntityStorage for LmdbStorage { } async fn set_call_name_index(&mut self, name: &str, sites: &[u8]) -> Result<()> { - let mut tx = self.env.begin_rw_txn().map_err(e)?; + let mut tx = begin_rw(&self.env)?; tx.put( self.call_names, &str_key(name), @@ -933,7 +1038,7 @@ impl EntityStorage for LmdbStorage { async fn upsert_file(&mut self, f: &FileInfo) -> Result<()> { let data = serde_json::to_vec(f).map_err(|err| StorageError::Internal(err.to_string()))?; - let mut tx = self.env.begin_rw_txn().map_err(e)?; + let mut tx = begin_rw(&self.env)?; tx.put(self.files, &str_key(&f.path), &data, WriteFlags::empty()) .map_err(e)?; tx.commit().map_err(e)?; @@ -962,7 +1067,7 @@ impl EntityStorage for LmdbStorage { } async fn set_version(&mut self, v: u64) -> Result<()> { - let mut tx = self.env.begin_rw_txn().map_err(e)?; + let mut tx = begin_rw(&self.env)?; tx.put(self.version, &KEY_ONE, &ku64(v), WriteFlags::empty()) .map_err(e)?; tx.commit().map_err(e)?; @@ -970,7 +1075,7 @@ impl EntityStorage for LmdbStorage { } async fn set_stats(&mut self, s: IndexCounts) -> Result<()> { - let mut tx = self.env.begin_rw_txn().map_err(e)?; + let mut tx = begin_rw(&self.env)?; tx.put(self.stats, &KEY_ONE, &pack_counts(&s), WriteFlags::empty()) .map_err(e)?; tx.commit().map_err(e)?; @@ -987,7 +1092,7 @@ impl EntityStorage for LmdbStorage { } async fn clear_entities(&mut self) -> Result<()> { - let mut tx = self.env.begin_rw_txn().map_err(e)?; + let mut tx = begin_rw(&self.env)?; for db in [ self.symbols, self.call_records, @@ -1015,7 +1120,7 @@ impl ShortcutsStorage for LmdbStorage { async fn add_shortcut_node(&mut self, shard: usize, elem: &[u8], node_id: usize) -> Result<()> { let mut key = k8(shard).to_vec(); key.extend_from_slice(elem); - let mut tx = self.env.begin_rw_txn().map_err(e)?; + let mut tx = begin_rw(&self.env)?; let mut list = match self.get_opt(&tx, self.shortcuts, &key)? { Some(v) => de_list(v), None => Vec::new(), @@ -1040,7 +1145,7 @@ impl ShortcutsStorage for LmdbStorage { } async fn clear_shortcuts(&mut self) -> Result<()> { - let mut tx = self.env.begin_rw_txn().map_err(e)?; + let mut tx = begin_rw(&self.env)?; tx.clear_db(self.shortcuts).map_err(e)?; tx.commit().map_err(e)?; Ok(()) @@ -1064,7 +1169,7 @@ pub struct LmdbTx { #[async_trait] impl Tx for LmdbTx { async fn new_node(&mut self, prefix: Vec, record: usize) -> Result { - let mut tx = self.env.begin_rw_txn().map_err(e)?; + let mut tx = begin_rw(&self.env)?; let next = match tx.get(self.counter, &KEY_ONE).map(de_u64) { Ok(v) => v, Err(lmdb::Error::NotFound) => 1, @@ -1107,7 +1212,7 @@ impl Tx for LmdbTx { nodes_pending, ops, } = *self; - let mut tx = env.begin_rw_txn().map_err(e)?; + let mut tx = begin_rw(&env)?; // 1. Materialize node mới — để ops add/move trỏ tới hợp lệ. for (id, prefix, record) in &nodes_pending { @@ -1204,6 +1309,57 @@ mod tests { (dir, path) } + /// Map tự nhân đôi khi dữ liệu chạm ngưỡng 25%. Trước đây map cứng 1 GiB + /// nên repo lớn chết `MDB_MAP_FULL` giữa lúc ingest (rollback cả txn). + #[tokio::test] + async fn map_size_tự_tăng_khi_gần_đầy() { + let (_dir, path) = tmp_path(); + // 1 MiB → ngưỡng 25% chỉ ~256 KiB nên test nhanh. + let env = open_env_with_map_size(&path, 1 << 20).unwrap(); + let db = env.create_db(Some("grow"), DatabaseFlags::empty()).unwrap(); + { + let mut tx = env.begin_rw_txn().unwrap(); + for i in 0..100u64 { + tx.put(db, &ku64(i), &vec![7u8; 4096], WriteFlags::empty()) + .unwrap(); + } + tx.commit().unwrap(); + } + assert_eq!(env.info().unwrap().map_size(), 1 << 20); + + drop(begin_rw(&env).unwrap()); + assert!( + env.info().unwrap().map_size() > (1 << 20), + "begin_rw phải grow map khi đã dùng > 25%" + ); + + // Dữ liệu cũ vẫn đọc được sau khi munmap/mmap lại. + let tx = env.begin_ro_txn().unwrap(); + assert_eq!(tx.get(db, &ku64(99)).unwrap().len(), 4096); + } + + /// Ghi nhiều hơn map ban đầu qua API thật → map tự tăng, không `MDB_MAP_FULL`. + #[tokio::test] + async fn ghi_vượt_map_ban_đầu_không_map_full() { + let (_dir, path) = tmp_path(); + let env = open_env_with_map_size(&path, 1 << 20).unwrap(); + let mut s = LmdbStorage::from_env(env).unwrap(); + s.init().await.unwrap(); + + let mut ids = Vec::new(); + for i in 0..400usize { + // ~4 KiB prefix mỗi node → tổng ~1.6 MiB, vượt map 1 MiB. + let id = s.new_node(vec![b'x'; 4096], i).await.unwrap(); + let mut tx = s.new_tx(); + tx.add_child(0, id).await.unwrap(); + tx.commit().await.unwrap(); + ids.push(id); + } + + assert_eq!(s.get_node(ids[0]).await.unwrap().1, 0); + assert_eq!(s.get_node(*ids.last().unwrap()).await.unwrap().1, 399); + } + /// `get_nodes` phải trả đúng thứ tự `ids` và giống hệt `get_node` từng cái. #[tokio::test] async fn test_get_nodes_batch_matches_single() { diff --git a/crates/codegraph/src/main.rs b/crates/codegraph/src/main.rs index 5afcbc9cbf..77cd64f762 100644 --- a/crates/codegraph/src/main.rs +++ b/crates/codegraph/src/main.rs @@ -7,7 +7,7 @@ use codegraph_mcp::CodegraphServer; use std::sync::Arc; #[cfg(feature = "fastembed")] -use codegraph_graph::embeddings::warm_model_cache; +use codegraph_graph::warm_model_cache; /// CLI tối giản: chỉ còn lifecycle (`init`/`deinit`) + MCP server (`serve --mcp`). /// Mọi query/interact đi qua MCP tools (`codegraph_search_symbol`, `codegraph_context`, diff --git a/docs/configuration.md b/docs/configuration.md index 5f94fca9db..fb5cea5d32 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -161,6 +161,15 @@ Selects the storage backend for the semantic graph. **DSN override**: Set `dsn` to override the default for any backend. +**LMDB map size** (`type = "lmdb"`): map mặc định **8 GiB** trên 64-bit (1 GiB +trên 32-bit). LMDB chỉ reserve *address space* — `data.mdb` là file sparse, chỉ +tốn đĩa cho trang thực ghi. Khi dữ liệu chạm **25%** map, CodeGraph tự nhân đôi +map trước write txn kế tiếp, nên `MDB_MAP_FULL` không còn chặn ingest. Muốn đặt +mức khởi đầu khác (ví dụ container giới hạn address space): +```bash +export CODEGRAPH_LMDB_MAP_SIZE_MB=2048 +``` + **Postgres/MySQL (multi-tenant, sharded)**: ```toml [storage] From ca2c19acc5979ce373676aeaf53c2b224921e7a6 Mon Sep 17 00:00:00 2001 From: Hung Pham Date: Thu, 29 Oct 2026 16:17:53 +0700 Subject: [PATCH 2/2] Bump version to v2.2.14 --- Cargo.lock | 30 ++++++++++++------------- Cargo.toml | 2 +- packaging/aur/codegraph-rs-bin/PKGBUILD | 2 +- packaging/choco/codegraph.nuspec | 2 +- packaging/winget/codegraph.yaml | 4 ++-- scripts/install.ps1 | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9decbcfc11..9f97fc8767 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -720,7 +720,7 @@ checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" [[package]] name = "codegraph" -version = "2.2.13" +version = "2.2.14" dependencies = [ "anyhow", "camino", @@ -743,7 +743,7 @@ dependencies = [ [[package]] name = "codegraph-api" -version = "2.2.13" +version = "2.2.14" dependencies = [ "anyhow", "async-trait", @@ -762,7 +762,7 @@ dependencies = [ [[package]] name = "codegraph-bench" -version = "2.2.13" +version = "2.2.14" dependencies = [ "anyhow", "camino", @@ -781,7 +781,7 @@ dependencies = [ [[package]] name = "codegraph-binary" -version = "2.2.13" +version = "2.2.14" dependencies = [ "camino", "codegraph-core", @@ -798,7 +798,7 @@ dependencies = [ [[package]] name = "codegraph-context" -version = "2.2.13" +version = "2.2.14" dependencies = [ "async-trait", "camino", @@ -815,7 +815,7 @@ dependencies = [ [[package]] name = "codegraph-core" -version = "2.2.13" +version = "2.2.14" dependencies = [ "async-graphql", "camino", @@ -826,7 +826,7 @@ dependencies = [ [[package]] name = "codegraph-docs" -version = "2.2.13" +version = "2.2.14" dependencies = [ "anyhow", "codegraph-core", @@ -844,7 +844,7 @@ dependencies = [ [[package]] name = "codegraph-extract" -version = "2.2.13" +version = "2.2.14" dependencies = [ "camino", "codegraph-binary", @@ -885,7 +885,7 @@ dependencies = [ [[package]] name = "codegraph-graph" -version = "2.2.13" +version = "2.2.14" dependencies = [ "async-trait", "bincode", @@ -915,7 +915,7 @@ dependencies = [ [[package]] name = "codegraph-graphql" -version = "2.2.13" +version = "2.2.14" dependencies = [ "anyhow", "async-graphql", @@ -941,7 +941,7 @@ dependencies = [ [[package]] name = "codegraph-installer" -version = "2.2.13" +version = "2.2.14" dependencies = [ "anyhow", "camino", @@ -957,7 +957,7 @@ dependencies = [ [[package]] name = "codegraph-mcp" -version = "2.2.13" +version = "2.2.14" dependencies = [ "anyhow", "axum", @@ -981,7 +981,7 @@ dependencies = [ [[package]] name = "codegraph-sboxes" -version = "2.2.13" +version = "2.2.14" dependencies = [ "camino", "codegraph-core", @@ -1003,7 +1003,7 @@ dependencies = [ [[package]] name = "codegraph-source" -version = "2.2.13" +version = "2.2.14" dependencies = [ "async-trait", "camino", @@ -1016,7 +1016,7 @@ dependencies = [ [[package]] name = "codegraph-web" -version = "2.2.13" +version = "2.2.14" dependencies = [ "axum", "mime_guess", diff --git a/Cargo.toml b/Cargo.toml index d83d3fbb6c..7f7c7d0dfe 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,7 @@ members = [ ] [workspace.package] -version = "2.2.13" +version = "2.2.14" edition = "2021" rust-version = "1.80" license = "MIT" diff --git a/packaging/aur/codegraph-rs-bin/PKGBUILD b/packaging/aur/codegraph-rs-bin/PKGBUILD index 819472dd83..83d1365207 100644 --- a/packaging/aur/codegraph-rs-bin/PKGBUILD +++ b/packaging/aur/codegraph-rs-bin/PKGBUILD @@ -1,6 +1,6 @@ # Maintainer: Hung Pham pkgname=codegraph-rs-bin -pkgver=2.2.13 +pkgver=2.2.14 pkgrel=1 pkgdesc="Local-first code intelligence: tree-sitter knowledge graph + MCP server (prebuilt binary)" arch=('x86_64' 'aarch64') diff --git a/packaging/choco/codegraph.nuspec b/packaging/choco/codegraph.nuspec index 322566aefc..5c0acf233d 100644 --- a/packaging/choco/codegraph.nuspec +++ b/packaging/choco/codegraph.nuspec @@ -2,7 +2,7 @@ codegraph - 2.2.13 + 2.2.14 codegraph Hung Pham https://github.com/hungpham10/codegraph-rs diff --git a/packaging/winget/codegraph.yaml b/packaging/winget/codegraph.yaml index d8be949273..d37360f699 100644 --- a/packaging/winget/codegraph.yaml +++ b/packaging/winget/codegraph.yaml @@ -6,7 +6,7 @@ # release time (or automate it in the release pipeline before submitting to # microsoft/winget-pkgs). PackageIdentifier: hungpham10.codegraph -PackageVersion: 2.2.13 +PackageVersion: 2.2.14 PackageName: codegraph Publisher: Hung Pham PublisherUrl: https://github.com/hungpham10/codegraph-rs @@ -18,7 +18,7 @@ PackageUrl: https://github.com/hungpham10/codegraph-rs InstallerType: zip Installers: - Architecture: x64 - InstallerUrl: https://github.com/hungpham10/codegraph-rs/releases/download/v2.2.13/codegraph-x86_64-pc-windows-msvc.zip + InstallerUrl: https://github.com/hungpham10/codegraph-rs/releases/download/v2.2.14/codegraph-x86_64-pc-windows-msvc.zip InstallerSha256: 0000000000000000000000000000000000000000000000000000000000000000 InstallerType: zip ManifestType: singleton diff --git a/scripts/install.ps1 b/scripts/install.ps1 index 93fcc2b6c6..3e8ded3537 100644 --- a/scripts/install.ps1 +++ b/scripts/install.ps1 @@ -5,11 +5,11 @@ # # Usage (pin a version / download the script first): # irm https://raw.githubusercontent.com/hungpham10/codegraph-rs/main/scripts/install.ps1 -OutFile install.ps1 -# .\install.ps1 -Version 2.2.13 +# .\install.ps1 -Version 2.2.14 [CmdletBinding()] param( - # Pin a specific version, e.g. "2.2.13". Empty = latest release. + # Pin a specific version, e.g. "2.2.14". Empty = latest release. [string]$Version )