Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 11 additions & 1 deletion REQUIREMENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

> **Note:** This document is automatically generated and verified against the live test suite by `scripts/generate_requirements.py` and `tests/backend/test_requirements_sync.py`.

**Test Verification Baseline:** **1344 Automated Tests** (980 Pytest Backend + 314 Vitest Frontend + 50 Playwright E2E).
**Test Verification Baseline:** **1352 Automated Tests** (988 Pytest Backend + 314 Vitest Frontend + 50 Playwright E2E).

---

Expand Down Expand Up @@ -1133,6 +1133,16 @@ code results are enriched with signature/kind/ast_symbol_id from SQLite._
- `test_restart_clears_relational_metadata_on_collection_recreate` - _Verifies that on app restart, if ensure_collection detects a dimension mismatch and recreates the collection, relational metadata is cleared._
- `test_rollback_and_complete_rebuild` - _Tests model transition, rollback, and complete rebuild across local paths._

#### `tests/test_embedding_parallel_batch.py` (8 tests)
- `test_parallel_slices_execution_and_deterministic_ordering` - _Verifies get_dense_embeddings_batch dispatches slices in parallel across EMBEDDING_API_CONCURRENCY threads and reassembles results in exact input order._
- `test_token_limit_slicing_respects_max_tokens_per_request` - _Verifies partition_texts_into_slices splits batches when estimated tokens exceed EMBEDDING_MAX_TOKENS_PER_REQUEST even if item count is below batch_size._
- `test_http_429_retry_after_aimd_and_telemetry` - _Verifies HTTP 429 rate-limit responses parse Retry-After headers, log a warning, multiplicatively reduce concurrency, and record rate_limit_wait_seconds in telemetry and progress_tracker._
- `test_cross_file_batch_embedding_in_git_and_local_syncer` - _Verifies git_syncer and local_syncer collect chunks across multiple files and call get_hybrid_embeddings_batch once per multi-file batch instead of once per file._
- `test_atomic_openai_client_swap_during_concurrent_embeddings` - _Verifies init_embeddings replaces _openai_client atomically without ever setting it to None while concurrent embedding calls are active._
- `test_env_embedding_batch_size_takes_effect_across_restarts` - _Verifies init_db does not freeze embedding_batch_size or embedding_num_threads in system_metadata, allowing .env changes to take effect on restart._
- `test_bounded_channel_pipeline_producer_consumer_backpressure` - _Verifies BoundedChannel and ChannelIngestionPipeline stream parsed files through coalescing, parallel API embedding slices, and persistence._
- `test_class_and_methods_do_not_double_chunk` - _Verifies extract_symbols_and_chunks does not duplicate method bodies inside parent class chunks and keeps method chunk hashes stable when sibling methods change._

#### `tests/test_engineering_style_guardrails.py` (4 tests)
- `test_repository_root_structure_hygiene` - _Verify repository root contains required living docs, no stray test_*.py files, and no committed UI build._
- `test_all_python_files_and_classes_under_500_loc` - _Verify every Python file and class across main.py, app/, scripts/, and tests/ stays <= 500 LOC._
Expand Down
125 changes: 88 additions & 37 deletions app/services/chunking/symbol_extractor.py
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,55 @@ def extract_symbols_and_chunks(
is_csharp = language in ("c_sharp", "csharp")
receiver_type_cache: Dict[Any, Dict[str, str]] = {}

def _resolve_container_info(n) -> Optional[Tuple[str, str]]:
if n.type not in target_types:
return None
if n.type in ("variable_declarator", "pair"):
return extract_callable_variable_or_pair(n, source_bytes)
n_name = extract_node_name(n, source_bytes)
if not n_name:
return None
n_kind = n.type
if language in ("c_sharp", "csharp") and n.type == "record_declaration":
if any(c.type == "struct" for c in n.children):
n_kind = "record_struct_declaration"
return n_name, n_kind

def _collect_direct_child_containers(parent_node) -> List[Any]:
found: List[Any] = []

def _walk(cur):
for ch in cur.children:
if _resolve_container_info(ch) is not None:
found.append(ch)
else:
_walk(ch)

_walk(parent_node)
return found

def _emit_chunk_span(sym_full: str, sym_kind: str, text_span: str, span_start: int, span_end: int) -> None:
if not text_span.strip():
return
heading_str = f"{sym_full} ({sym_kind})"
if len(text_span) <= max_chunk_chars:
chunks.append({
"symbol": sym_full,
"kind": sym_kind,
"heading": heading_str,
"content": text_span,
"start_line": span_start,
"end_line": span_end,
})
else:
sub = split_by_length(text_span, heading=heading_str, max_chars=max_chunk_chars)
for s in sub:
s["symbol"] = sym_full
s["kind"] = sym_kind
s["start_line"] = span_start + s.get("start_line", 1) - 1
s["end_line"] = span_start + s.get("end_line", span_end - span_start + 1) - 1
chunks.append(s)

def traverse(node, parent_symbol: Optional[str] = None):
line_num = node.start_point[0] + 1

Expand All @@ -102,27 +151,19 @@ def traverse(node, parent_symbol: Optional[str] = None):
kind = node.type

if node.type in target_types:
if node.type in ("variable_declarator", "pair"):
res_var = extract_callable_variable_or_pair(node, source_bytes)
if res_var:
name, kind = res_var
is_container = True
elif node.type == "variable_declarator":
name_node = node.child_by_field_name("name")
val_node = node.child_by_field_name("value")
if name_node and val_node and name_node.type == "identifier" and val_node.type == "object":
obj_name = source_bytes[name_node.start_byte:name_node.end_byte].decode("utf-8", errors="ignore").strip()
next_parent = f"{parent_symbol}.{obj_name}" if parent_symbol else obj_name
for child in node.children:
traverse(child, parent_symbol=next_parent)
return
else:
name = extract_node_name(node, source_bytes)
if name:
is_container = True
if language in ("c_sharp", "csharp") and node.type == "record_declaration":
if any(c.type == "struct" for c in node.children):
kind = "record_struct_declaration"
res_info = _resolve_container_info(node)
if res_info is not None:
name, kind = res_info
is_container = True
elif node.type == "variable_declarator":
name_node = node.child_by_field_name("name")
val_node = node.child_by_field_name("value")
if name_node and val_node and name_node.type == "identifier" and val_node.type == "object":
obj_name = source_bytes[name_node.start_byte:name_node.end_byte].decode("utf-8", errors="ignore").strip()
next_parent = f"{parent_symbol}.{obj_name}" if parent_symbol else obj_name
for child in node.children:
traverse(child, parent_symbol=next_parent)
return

if is_container and name:
full_symbol = f"{parent_symbol}.{name}" if parent_symbol else name
Expand Down Expand Up @@ -191,23 +232,33 @@ def traverse(node, parent_symbol: Optional[str] = None):
"line_number": irel["line"]
})

if len(node_text) <= max_chunk_chars:
chunks.append({
"symbol": full_symbol,
"kind": kind,
"heading": f"{full_symbol} ({kind})",
"content": node_text,
"start_line": start_line,
"end_line": end_line
})
child_containers = _collect_direct_child_containers(node)
if not child_containers:
_emit_chunk_span(full_symbol, kind, node_text, start_line, end_line)
else:
sub = split_by_length(node_text, heading=f"{full_symbol} ({kind})", max_chars=max_chunk_chars)
for s in sub:
s["symbol"] = full_symbol
s["kind"] = kind
s["start_line"] = start_line + s.get("start_line", 1) - 1
s["end_line"] = start_line + s.get("end_line", end_line - start_line + 1) - 1
chunks.append(s)
covered_lines = set()
for ch_node in child_containers:
ch_start = ch_node.start_point[0] + 1
ch_end = ch_node.end_point[0] + 1
covered_lines.update(range(ch_start, ch_end + 1))

spans: List[Tuple[int, int]] = []
seg_start: Optional[int] = None
for ln in range(start_line, end_line + 1):
if ln not in covered_lines:
if seg_start is None:
seg_start = ln
else:
if seg_start is not None:
spans.append((seg_start, ln - 1))
seg_start = None
if seg_start is not None:
spans.append((seg_start, end_line))

for idx_span, (s_ln, e_ln) in enumerate(spans):
seg_text = "\n".join(lines[s_ln - 1 : e_ln])
if idx_span == 0 or re.search(r"[A-Za-z0-9_]", seg_text):
_emit_chunk_span(full_symbol, kind, seg_text, s_ln, e_ln)

for child in node.children:
traverse(child, parent_symbol=full_symbol)
Expand Down
98 changes: 63 additions & 35 deletions app/services/database/config_store.py
Original file line number Diff line number Diff line change
Expand Up @@ -206,10 +206,20 @@ def _resolve_default_embedding_config(conn: Optional[Any] = None) -> Dict[str, A
env_threads = os.getenv("EMBEDDING_NUM_THREADS") or os.getenv("EMBEDDING_THREADS")
threads = int(env_threads) if env_threads and env_threads.isdigit() else default_threads

provider = os.getenv("EMBEDDING_PROVIDER", "local").lower().strip()
env_batch = os.getenv("EMBEDDING_BATCH_SIZE")
batch_size = int(env_batch) if env_batch and env_batch.isdigit() else 32
default_bs = 128 if provider == "api" else 32
batch_size = int(env_batch) if env_batch and env_batch.isdigit() else default_bs

env_concurrency = os.getenv("EMBEDDING_API_CONCURRENCY")
api_concurrency = int(env_concurrency) if env_concurrency and env_concurrency.isdigit() else 4

env_max_tokens = os.getenv("EMBEDDING_MAX_TOKENS_PER_REQUEST")
max_tokens_per_request = int(env_max_tokens) if env_max_tokens and env_max_tokens.isdigit() else 100000

env_tpm = os.getenv("EMBEDDING_TPM_LIMIT")
tpm_limit = int(env_tpm) if env_tpm and env_tpm.isdigit() else 0

provider = os.getenv("EMBEDDING_PROVIDER", "local").lower().strip()
dense_model = os.getenv("EMBEDDING_MODEL", "BAAI/bge-small-en-v1.5").strip()
sparse_model = os.getenv("SPARSE_MODEL", "Qdrant/bm25").strip()
litellm_url = (
Expand All @@ -234,6 +244,9 @@ def _resolve_default_embedding_config(conn: Optional[Any] = None) -> Dict[str, A
"sparse_model": sparse_model,
"threads": max(1, threads),
"batch_size": max(1, batch_size),
"api_concurrency": max(1, api_concurrency),
"max_tokens_per_request": max(1, max_tokens_per_request),
"tpm_limit": max(0, tpm_limit),
"openai_api_url": litellm_url,
"litellm_url": litellm_url,
"litellm_api_key": litellm_api_key,
Expand All @@ -252,36 +265,47 @@ def get_embedding_db_config() -> Dict[str, Any]:
sparse_model = _get_meta("embedding_sparse_model")
threads_str = _get_meta("embedding_num_threads")
batch_size_str = _get_meta("embedding_batch_size")
concurrency_str = _get_meta("embedding_api_concurrency")
max_tokens_str = _get_meta("embedding_max_tokens_per_request")
tpm_limit_str = _get_meta("embedding_tpm_limit")
litellm_url = _get_meta("openai_api_url") or _get_meta("embedding_litellm_url")
litellm_api_key = _get_meta("openai_api_key") or _get_meta("embedding_litellm_api_key")
vision_ocr_model = (
_get_meta("openai_vision_model")
or _get_meta("vision_ocr_model")
or _get_meta("embedding_vision_ocr_model")
)
chat_model = (
_get_meta("openai_chat_model")
or _get_meta("chat_model")
or _get_meta("embedding_chat_model")
_get_meta("openai_vision_model") or _get_meta("vision_ocr_model") or _get_meta("embedding_vision_ocr_model")
)
chat_model = _get_meta("openai_chat_model") or _get_meta("chat_model") or _get_meta("embedding_chat_model")
allow_fallback_str = _get_meta("embedding_allow_fallback")
fallback_dense_model = _get_meta("embedding_fallback_dense_model")

default_cfg = _resolve_default_embedding_config()
eff_provider = (provider or default_cfg["provider"]).lower().strip()

threads = int(threads_str) if threads_str and threads_str.isdigit() else default_cfg["threads"]
batch_size = int(batch_size_str) if batch_size_str and batch_size_str.isdigit() else default_cfg["batch_size"]
env_bs = os.getenv("EMBEDDING_BATCH_SIZE", "")
if batch_size_str and batch_size_str.isdigit():
batch_size = int(batch_size_str)
elif env_bs.isdigit():
batch_size = int(env_bs)
else:
batch_size = 128 if eff_provider == "api" else 32

api_concurrency = int(concurrency_str) if concurrency_str and concurrency_str.isdigit() else default_cfg["api_concurrency"]
max_tokens = int(max_tokens_str) if max_tokens_str and max_tokens_str.isdigit() else default_cfg["max_tokens_per_request"]
tpm_limit = int(tpm_limit_str) if tpm_limit_str and tpm_limit_str.isdigit() else default_cfg["tpm_limit"]
allow_fallback = (allow_fallback_str == "1") if allow_fallback_str is not None else default_cfg["allow_fallback"]

resolved_url = (litellm_url or default_cfg["litellm_url"]).strip()
raw_db_key = (litellm_api_key or "").strip()
effective_key = ("" if _is_placeholder_api_key(raw_db_key) else raw_db_key) or default_cfg["litellm_api_key"]
return {
"provider": (provider or default_cfg["provider"]).lower().strip(),
"provider": eff_provider,
"dense_model": (dense_model or default_cfg["dense_model"]).strip(),
"sparse_model": (sparse_model or default_cfg["sparse_model"]).strip(),
"threads": max(1, threads),
"batch_size": max(1, batch_size),
"api_concurrency": max(1, api_concurrency),
"max_tokens_per_request": max(1, max_tokens),
"tpm_limit": max(0, tpm_limit),
"openai_api_url": resolved_url,
"litellm_url": resolved_url,
"litellm_api_key": effective_key,
Expand All @@ -302,6 +326,9 @@ def set_embedding_db_config(
sparse_model: Optional[str] = None,
threads: Optional[int] = None,
batch_size: Optional[int] = None,
api_concurrency: Optional[int] = None,
max_tokens_per_request: Optional[int] = None,
tpm_limit: Optional[int] = None,
litellm_url: Optional[str] = None,
litellm_api_key: Optional[str] = None,
vision_ocr_model: Optional[str] = None,
Expand All @@ -312,28 +339,32 @@ def set_embedding_db_config(
if provider is not None:
_set_meta("embedding_provider", provider.lower().strip())
if dense_model is not None:
_set_meta("embedding_dense_model", dense_model.strip())
_set_meta("openai_embedding_model", dense_model.strip())
for k in ("embedding_dense_model", "openai_embedding_model"):
_set_meta(k, dense_model.strip())
if sparse_model is not None:
_set_meta("embedding_sparse_model", sparse_model.strip())
if threads is not None:
_set_meta("embedding_num_threads", str(max(1, threads)))
if batch_size is not None:
_set_meta("embedding_batch_size", str(max(1, batch_size)))
if api_concurrency is not None:
_set_meta("embedding_api_concurrency", str(max(1, api_concurrency)))
if max_tokens_per_request is not None:
_set_meta("embedding_max_tokens_per_request", str(max(1, max_tokens_per_request)))
if tpm_limit is not None:
_set_meta("embedding_tpm_limit", str(max(0, tpm_limit)))
if litellm_url is not None:
_set_meta("openai_api_url", litellm_url.strip())
_set_meta("embedding_litellm_url", litellm_url.strip())
for k in ("openai_api_url", "embedding_litellm_url"):
_set_meta(k, litellm_url.strip())
if litellm_api_key is not None:
_set_meta("openai_api_key", litellm_api_key.strip())
_set_meta("embedding_litellm_api_key", litellm_api_key.strip())
for k in ("openai_api_key", "embedding_litellm_api_key"):
_set_meta(k, litellm_api_key.strip())
if vision_ocr_model is not None:
_set_meta("openai_vision_model", vision_ocr_model.strip())
_set_meta("vision_ocr_model", vision_ocr_model.strip())
_set_meta("embedding_vision_ocr_model", vision_ocr_model.strip())
for k in ("openai_vision_model", "vision_ocr_model", "embedding_vision_ocr_model"):
_set_meta(k, vision_ocr_model.strip())
if chat_model is not None:
_set_meta("openai_chat_model", chat_model.strip())
_set_meta("chat_model", chat_model.strip())
_set_meta("embedding_chat_model", chat_model.strip())
for k in ("openai_chat_model", "chat_model", "embedding_chat_model"):
_set_meta(k, chat_model.strip())
if allow_fallback is not None:
_set_meta("embedding_allow_fallback", "1" if allow_fallback else "0")
if fallback_dense_model is not None:
Expand Down Expand Up @@ -406,20 +437,17 @@ def set_ai_gateway_config(
) -> Dict[str, Any]:
"""Persists OpenAI-compatible AI Gateway configuration in system_metadata."""
if url is not None:
_set_meta("openai_api_url", url.strip())
_set_meta("embedding_litellm_url", url.strip())
for k in ("openai_api_url", "embedding_litellm_url"):
_set_meta(k, url.strip())
if api_key is not None:
cleaned_key = api_key.strip()
_set_meta("openai_api_key", cleaned_key)
_set_meta("embedding_litellm_api_key", cleaned_key)
for k in ("openai_api_key", "embedding_litellm_api_key"):
_set_meta(k, api_key.strip())
if chat_model is not None:
_set_meta("openai_chat_model", chat_model.strip())
_set_meta("chat_model", chat_model.strip())
_set_meta("embedding_chat_model", chat_model.strip())
for k in ("openai_chat_model", "chat_model", "embedding_chat_model"):
_set_meta(k, chat_model.strip())
if vision_ocr_model is not None:
_set_meta("openai_vision_model", vision_ocr_model.strip())
_set_meta("vision_ocr_model", vision_ocr_model.strip())
_set_meta("embedding_vision_ocr_model", vision_ocr_model.strip())
for k in ("openai_vision_model", "vision_ocr_model", "embedding_vision_ocr_model"):
_set_meta(k, vision_ocr_model.strip())
if embedding_model is not None:
cleaned_emb = embedding_model.strip()
_set_meta("openai_embedding_model", cleaned_emb)
Expand Down
2 changes: 0 additions & 2 deletions app/services/database/engine.py
Original file line number Diff line number Diff line change
Expand Up @@ -289,8 +289,6 @@ def _seed_defaults(engine: Engine, vault_path: str = "/docs") -> None:
("embedding_provider", emb_cfg["provider"]),
("embedding_dense_model", emb_cfg["dense_model"]),
("embedding_sparse_model", emb_cfg["sparse_model"]),
("embedding_num_threads", str(emb_cfg["threads"])),
("embedding_batch_size", str(emb_cfg["batch_size"])),
("openai_api_url", emb_cfg["litellm_url"]),
("embedding_litellm_url", emb_cfg["litellm_url"]),
]:
Expand Down
Loading
Loading