diff --git a/CHANGELOG.md b/CHANGELOG.md index 6f08cdde..8ba1bf19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ All notable changes to this project are documented in this file. +## Unreleased + +### Removed +- **The `--threads` / `-t` CLI options (on `build-kg` and `build-fullmap`), the `threads` parameter on the Python API (`fullmap.resolve`, etc.), and the agent command's `--gepa-threads` option (`dspy.GEPA` evaluation pool) are gone.** All parallelism is now automatic: every parallel stage uses all available CPU threads, with the fullmap build still capped on Linux by available memory (~2 GB per thread, read from `/proc/meminfo`) to avoid OOMs, and entity-resolution lookups fanning out across the fullmap shards automatically only for batches of 1024+ terms (smaller batches stay serial). GEPA now uses its library default parallelism. Results are unchanged. + ## 16.7.0 - 2026-09-04 ### Changed diff --git a/docs/agent.md b/docs/agent.md index b50542b1..78df8efe 100644 --- a/docs/agent.md +++ b/docs/agent.md @@ -519,15 +519,15 @@ Following GEPA best practice, the optimizer splits the models: a **strong reflec proposes the few instruction edits, and an optional **fast task LM** (`--task-model`) runs the many candidate program evaluations. Pointing `--task-model` at a cheap model (e.g. a flash model) keeps the run fast while the strong model does the thinking; without `--task-model` the reflection LM is used for -both. `--gepa-threads` parallelizes GEPA's candidate **LM forward passes** only: the coverage-scoring +both. GEPA runs its candidate evaluations with its library default parallelism; the coverage-scoring builds stay serialized on the process-wide `_GEPA_BUILD_LOCK` (`agent.py`, since `os.chdir` is -process-global), so a higher thread count does not speed up the expensive build/coverage step. +process-global), so extra parallelism does not speed up the expensive build/coverage step. ```bash # optimize the agent prompt over a dataset of examples, writing the result to a file tablassert agent PMC11708054 --configuration-file ./graph.yaml --optimize \ --dataset examples/gepa-dataset.yaml --task-model qwen-flash \ - --max-metric-calls 30 --gepa-threads 4 \ + --max-metric-calls 30 \ --instructions-out .tablassert/agent/optimized_instructions.yaml # later, run the supervisor with the optimized prompt diff --git a/docs/api/fullmap.md b/docs/api/fullmap.md index 1350581f..02178fdf 100644 --- a/docs/api/fullmap.md +++ b/docs/api/fullmap.md @@ -21,7 +21,6 @@ def resolve( config_file: Optional[str] = None, column_context: bool = True, tag: str = "_two", - threads: Optional[int] = None, ) -> pl.LazyFrame ``` @@ -79,10 +78,6 @@ Suffix appended to `col` to locate the `level_two` output column. The default `"_two"` matches `level_two`'s default tag. -**`threads: Optional[int]` (default: `None`)** - -Optional worker-thread count passed through to the Rust lookup for parallel term batching. - ### Return Value Returns a Polars LazyFrame with these columns added: diff --git a/docs/cli.md b/docs/cli.md index 4e9ca6f9..50672fe9 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -75,7 +75,6 @@ page lists the flags; see | `--max-metric-calls` | int | No | `8` | GEPA metric-call budget for `--optimize` | | `--dataset` | Path | No | `None` | YAML/JSON list of `{table_summary, coverage_feedback}` examples for `--optimize` (an example may also carry `fullmap`, `workdir`, and `head` to score each proposed config with real coverage) | | `--task-model` | str | No | `None` | Fast model id for GEPA's many program evaluations (cheap task LM + strong reflection LM); `--model-id` is the reflection LM. Defaults to the reflection LM | -| `--gepa-threads` | int | No | `None` | Thread count for GEPA's evaluation pool (`--optimize`): parallelizes candidate LM forward passes only; coverage-scoring builds stay serialized on `_GEPA_BUILD_LOCK` | ```bash tablassert agent PMC11708054 --configuration-file ./graph.yaml @@ -124,7 +123,6 @@ tablassert build-fullmap [ARGS] | `--output`, `-o` | Path | No | `./fullmap/data/fullmap.redb` | Path to write the redb file (prebuilt extraction or build output) | | `--cache`, `-c` | Path | No | `./fullmap/downloads` | Directory for downloaded BABEL files when building from scratch (`classes/`, `synonyms/`) | | `--version`, `-v` | str | No | `2026jul22` | BABEL snapshot date to fetch (a RENCI stamp, **not** Tablassert's version) | -| `--threads`, `-t` | int | No | `None` (auto) | Worker threads for a from-scratch build; auto-capped by memory on Linux (`/proc/meminfo`), else ~90% of CPUs | | `--aria2c`, `-a` | Flag | No | `False` | Opt into the bundled `aria2c` binary from the `[aria2]` extra for resumable segmented downloads (the prebuilt archive **or** BABEL files); fails loud (exit 2, before any download starts) if the extra is missing or unsupported on the current platform, and on a non-zero aria2c exit | | `--force`, `-f` | Flag | No | `False` | Skip the prebuilt download and always rebuild from BABEL outputs | @@ -148,6 +146,10 @@ prebuilt exists for this version (or the download or extraction fails), it falls from-scratch BABEL build and logs a warning. A database already present at `--output` is reused as-is; pass `--force` to rebuild. +A from-scratch build parallelizes automatically across all available CPU threads — on Linux the +worker count is capped by available memory (~2 GB per thread, read from `/proc/meminfo`) to avoid +OOMs. There is no flag to tune. + See [Fullmap](fullmap.md) for the data pipeline, output schema, and graph-config usage. --- @@ -171,12 +173,16 @@ The positional `GRAPH-CONFIGURATION-FILE` (also `--configuration-file`, `-f`) is | `--log`, `-l` | Flag | No | `False` | Enable verbose per-section logging | | `--head`, `-hd` | Flag | No | `False` | Fast output-shape preview: ≤5 random rows/section, cached to `.head.parquet`, never clobbers a full build | | `--no-original`, `-no` | Flag | No | `False` | Omit the verbatim source-cell copies (`original_subject`, `original_object`, and any other `original_*` fields) from the final edge NDJSON | -| `--threads`, `-t` | int | No | `None` (auto) | Worker threads for the parallel fullmap reads behind entity resolution. Readers fan out across the 16 record-shard files, and values above the (non-empty) shard count further split the busiest shards' term buckets across more concurrent readers of the same shard; redb readers share-lock, so they never contend with each other. Unset keeps the auto behavior: large batches (≥ 1024 terms) fan out, small ones stay serial. Results are identical at any worker count | ```bash tablassert build-kg graph.yaml --qc --log ``` +The parallel fullmap reads behind entity resolution are automatic: large lookup batches (≥ 1024 +terms) fan out across the record-shard files on all available CPU threads (redb readers share-lock, +so they never contend), while smaller batches stay serial. There is no flag to tune, and results are +identical at any worker count. + Output is written to `rig.artifact_base_path` (created when missing) as `{name}_{version}.nodes.ndjson`, `{name}_{version}.edges.ndjson`, and `{name}_{version}.RIG.yaml`; intermediate parquet lands in `.tablassert/store/`. The RIG document is audited in memory before it is written: an invalid diff --git a/docs/fullmap.md b/docs/fullmap.md index 167ee5ca..d668b2ab 100644 --- a/docs/fullmap.md +++ b/docs/fullmap.md @@ -26,7 +26,7 @@ tablassert build-fullmap --aria2c ``` See the [CLI Reference → build-fullmap](cli.md#build-fullmap) for the complete flag table (output path, -cache directory, BABEL snapshot version, worker threads, the optional `--aria2c` / `-a` downloader, +cache directory, BABEL snapshot version, the optional `--aria2c` / `-a` downloader, and the `--force` / `-f` rebuild flag), their defaults, and more examples. By default, `build-fullmap` first downloads a **prebuilt** database published for this Tablassert @@ -49,9 +49,10 @@ Two facts matter most when planning a build: - The BABEL **version** flag selects a RENCI BABEL snapshot date (default `2026jul22`), *not* Tablassert's package version. Bumping it fetches a different snapshot and requires rebuilding; the value used is recorded in the primary's `meta` table (`source_version`). -- With **threads** left unset, the Rust build caps workers at `min(available_CPUs, MemAvailable_GB / 2)` - on Linux (reading `MemAvailable:` from `/proc/meminfo`, each worker budgeting ~2 GB of local buffers) - and falls back to ~90% of CPUs elsewhere, so a large build stays within a fixed memory budget. +- The build **parallelizes automatically** across all available CPU threads: the Rust build caps + workers at `min(available_CPUs, MemAvailable_GB / 2)` on Linux (reading `MemAvailable:` from + `/proc/meminfo`, each worker budgeting ~2 GB of local buffers) and falls back to ~90% of CPUs + elsewhere, so a large build stays within a fixed memory budget. There is no flag to tune. ### Data Pipeline diff --git a/examples/agent/README.md b/examples/agent/README.md index 3117e012..459ad65d 100644 --- a/examples/agent/README.md +++ b/examples/agent/README.md @@ -44,7 +44,7 @@ export TABLASSERT_AGENT_API_KEY="sk-***" tablassert agent PMC11947420 --configuration-file /path/to/graph.yaml --optimize \ --dataset examples/agent/gepa-dataset.yaml \ --task-model qwen3.6-flash \ - --max-metric-calls 30 --gepa-threads 4 \ + --max-metric-calls 30 \ --instructions-out examples/agent/optimized_instructions.yaml ``` @@ -52,8 +52,8 @@ tablassert agent PMC11947420 --configuration-file /path/to/graph.yaml --optimize GEPA best practice (and what the flags above do): a **strong reflection LM** (`--model-id`) proposes the few instruction edits, while a **fast task LM** (`--task-model`) runs the many candidate evaluations. -`--max-metric-calls` bounds the budget; `--gepa-threads` parallelizes the candidate LM forward passes -(the coverage-scoring builds stay serialized on the process-wide `_GEPA_BUILD_LOCK`). +`--max-metric-calls` bounds the budget; GEPA parallelizes the candidate LM forward passes with its +library default (the coverage-scoring builds stay serialized on the process-wide `_GEPA_BUILD_LOCK`). ## QC state-directory requirement diff --git a/rust/src/fullmap.rs b/rust/src/fullmap.rs index 9f52efc6..9d8e9403 100644 --- a/rust/src/fullmap.rs +++ b/rust/src/fullmap.rs @@ -2297,13 +2297,12 @@ fn build_fullmap_inner( } #[pyfunction] -#[pyo3(signature = (output, classes, synonyms, threads=None, progress=None))] +#[pyo3(signature = (output, classes, synonyms, progress=None))] pub fn build_fullmap_db( py: Python<'_>, output: PathBuf, classes: Vec, synonyms: Vec, - threads: Option, progress: Option>, ) -> PyResult<()> { if synonyms.is_empty() { @@ -2319,35 +2318,34 @@ pub fn build_fullmap_db( // the build if this doesn't succeed. let _ = rlimit::increase_nofile_limit(u64::MAX); - let worker_count = threads - .unwrap_or_else(|| { - let cpus = std::thread::available_parallelism() - .map(std::num::NonZero::get) - .unwrap_or(1); - // Cap at available_memory_gb / 2 to prevent swap on memory-constrained - // machines. Each thread uses ~400 MB of local buffers; the cap is - // generous (2 GB/thread) to avoid limiting CPU-bound throughput. - let avail_kb = std::fs::read_to_string("/proc/meminfo") - .ok() - .and_then(|s| { - s.lines() - .find(|l| l.starts_with("MemAvailable:")) - .and_then(|l| { - l.split_whitespace() - .nth(1) - .and_then(|v| v.parse::().ok()) - }) - }) - .unwrap_or(0); - if avail_kb > 0 { - let avail_gb = avail_kb / (1024 * 1024); - let mem_cap = (avail_gb / 2).max(1); - cpus.min(mem_cap) - } else { - cpus * 9 / 10 - } - }) - .max(1); + let worker_count = { + let cpus = std::thread::available_parallelism() + .map(std::num::NonZero::get) + .unwrap_or(1); + // Cap at available_memory_gb / 2 to prevent swap on memory-constrained + // machines. Each thread uses ~400 MB of local buffers; the cap is + // generous (2 GB/thread) to avoid limiting CPU-bound throughput. + let avail_kb = std::fs::read_to_string("/proc/meminfo") + .ok() + .and_then(|s| { + s.lines() + .find(|l| l.starts_with("MemAvailable:")) + .and_then(|l| { + l.split_whitespace() + .nth(1) + .and_then(|v| v.parse::().ok()) + }) + }) + .unwrap_or(0); + if avail_kb > 0 { + let avail_gb = avail_kb / (1024 * 1024); + let mem_cap = (avail_gb / 2).max(1); + cpus.min(mem_cap) + } else { + cpus * 9 / 10 + } + } + .max(1); if let Some(parent) = output.parent() { std::fs::create_dir_all(parent).map_err(py_err)?; @@ -3382,8 +3380,8 @@ fn lookup_pair_terms_db( Ok(tagged.into_iter().map(|(_, pairs)| pairs).collect()) } -/// Smallest batch that defaults to parallel shard fan-out when the caller passes -/// no `threads`. Below this, lookups stay single-threaded: a point/small lookup +/// Smallest batch that fans out across the shards in parallel. Below this, +/// lookups stay single-threaded: a point/small lookup /// (and any cache-warm path) finishes faster serially than the cost of spawning /// shard-reader threads. At/above it, the per-shard fan-out in /// `lookup_pair_terms_db` wins. 1024 terms ~= a few ms of serial redb point @@ -3392,9 +3390,9 @@ fn lookup_pair_terms_db( /// while never penalizing small lookups. const LOOKUP_PARALLEL_MIN: usize = 1024; -/// Default worker count for lookups when the caller passes no `threads`. +/// Worker count for lookups, selected automatically from the batch size. /// The production build-kg resolve sends ONE batch of all distinct node-column -/// terms (often huge) with `threads=None`; parallelizing that across the RECORDS +/// terms (often huge); parallelizing that across the RECORDS /// shards is the win, so large batches default to `available_parallelism`. Small /// batches (< `LOOKUP_PARALLEL_MIN`) stay single-threaded to avoid spawn overhead. /// The fan-out is already capped by the non-empty shard count inside @@ -3409,13 +3407,8 @@ fn default_lookup_workers(terms_len: usize) -> usize { .unwrap_or(1) } -fn lookup_pair_terms( - db: PathBuf, - terms: Vec, - threads: Option, -) -> PyResult { - let workers = threads - .unwrap_or_else(|| default_lookup_workers(terms.len())) +fn lookup_pair_terms(db: PathBuf, terms: Vec) -> PyResult { + let workers = default_lookup_workers(terms.len()) .max(1) .min(terms.len().max(1)); // Open (and schema-validate) the primary, then route pair lookups to shards. @@ -3463,11 +3456,7 @@ fn hydrate_curie_rows(database: &ReadOnlyDatabase, curie_ids: &[u32]) -> PyResul Ok(out) } -fn lookup_terms( - db: PathBuf, - terms: Vec, - threads: Option, -) -> PyResult)>> { +fn lookup_terms(db: PathBuf, terms: Vec) -> PyResult)>> { // Open the primary ONCE (the cached shared-lock handle) for dims/CURIES // hydration; pair lookups route to the shard files. One handle per file is // a cache choice, not a lock constraint — read-only opens coexist. @@ -3476,8 +3465,7 @@ fn lookup_terms( let category_map = load_string_table(&database, CATEGORIES)?; let source_map = load_sources(&database)?; let shards = open_cached_shards(&db)?; - let workers = threads - .unwrap_or_else(|| default_lookup_workers(terms.len())) + let workers = default_lookup_workers(terms.len()) .max(1) .min(terms.len().max(1)); let pair_rows = lookup_pair_terms_db(&shards, &terms, workers)?; @@ -3519,26 +3507,23 @@ fn lookup_terms( Ok(out) } -/// Look up fullmap records for `terms`. `threads=None` (the production -/// build-kg default) auto-selects the worker count via `default_lookup_workers`: -/// batches >= `LOOKUP_PARALLEL_MIN` fan out across the RECORDS shards in -/// parallel (up to 16 by default), smaller batches stay single-threaded. An -/// explicit `threads=Some(1)` always forces the serial path. The GIL is released -/// for the whole lookup. +/// Look up fullmap records for `terms`. The worker count is selected +/// automatically via `default_lookup_workers`: batches >= `LOOKUP_PARALLEL_MIN` +/// fan out across the RECORDS shards in parallel (up to 16 by default), smaller +/// batches stay single-threaded. The GIL is released for the whole lookup. #[pyfunction] -#[pyo3(signature = (db, terms, threads=None, return_format="rows"))] +#[pyo3(signature = (db, terms, return_format="rows"))] pub fn lookup_fullmap_terms<'py>( py: Python<'py>, db: PathBuf, terms: Vec, - threads: Option, return_format: &str, ) -> PyResult> { if return_format == "pairs" { // Release the GIL for the whole lookup (pure-Rust shard reads); the // PyList is built only after re-acquiring it so rich's Live display // thread can repaint and Ctrl-C works mid-lookup. - let pair_rows = py.detach(move || lookup_pair_terms(db, terms, threads))?; + let pair_rows = py.detach(move || lookup_pair_terms(db, terms))?; let list = PyList::empty(py); for (term, pairs) in pair_rows { let row = PyDict::new(py); @@ -3556,7 +3541,7 @@ pub fn lookup_fullmap_terms<'py>( // Release the GIL for the whole lookup (pure-Rust shard reads + CURIE/dim // hydration against the primary); the PyList is built only after // re-acquiring the GIL. - let rows = py.detach(move || lookup_terms(db, terms, threads))?; + let rows = py.detach(move || lookup_terms(db, terms))?; let list = PyList::empty(py); for (term, records) in rows { for record in records { @@ -3768,12 +3753,8 @@ mod tests { .unwrap(); build_test(output.clone(), vec![classes], vec![synonyms], 1, 4_000_000).unwrap(); - let rows = lookup_terms( - output, - vec!["brca1".to_string(), "ncbigene672".to_string()], - Some(1), - ) - .unwrap(); + let rows = + lookup_terms(output, vec!["brca1".to_string(), "ncbigene672".to_string()]).unwrap(); assert_eq!(rows.len(), 2); assert_eq!(rows[0].1[0].curie, "HGNC:1100"); @@ -3836,7 +3817,7 @@ mod tests { } // The single indexed term still resolves (routed through its shard). - let rows = lookup_terms(output, vec!["brca1".to_string()], Some(1)).unwrap(); + let rows = lookup_terms(output, vec!["brca1".to_string()]).unwrap(); assert_eq!(rows.len(), 1); assert_eq!(rows[0].1[0].curie, "HGNC:1100"); } @@ -3979,7 +3960,7 @@ mod tests { .flat_map(|i| [format!("gene{i}"), format!("alias{i}")]) .collect(); let norm = |db: PathBuf| -> Vec<(String, Vec)> { - let rows = lookup_terms(db, probes.clone(), Some(4)).unwrap(); + let rows = lookup_terms(db, probes.clone()).unwrap(); let mut out: Vec<(String, Vec)> = rows .into_iter() .map(|(t, recs)| { @@ -4133,7 +4114,7 @@ mod tests { .flat_map(|i| [format!("gene{i}"), format!("alias{i}")]) .collect(); let norm = |db: PathBuf| -> Vec<(String, Vec)> { - let rows = lookup_terms(db, probes.clone(), Some(4)).unwrap(); + let rows = lookup_terms(db, probes.clone()).unwrap(); let mut out: Vec<(String, Vec)> = rows .into_iter() .map(|(t, recs)| { @@ -4215,12 +4196,19 @@ mod tests { let got_order: Vec = parallel.iter().map(|(t, _)| t.clone()).collect(); assert_eq!(got_order, expected_order, "merge broke input order"); - // End-to-end hydration (through the primary) also agrees across thread - // counts and yields one row group per hit term. - let rows_par = lookup_terms(output.clone(), probes.clone(), Some(4)).unwrap(); - let rows_ser = lookup_terms(output, probes, Some(1)).unwrap(); - assert_eq!(rows_par.len(), expected_order.len()); - assert_eq!(rows_par, rows_ser); + // End-to-end hydration (through the primary) agrees with the shard-level + // pairs and yields one row group per hit term. + let rows = lookup_terms(output, probes.clone()).unwrap(); + assert_eq!(rows.len(), expected_order.len()); + let pair_counts: Vec<(String, usize)> = parallel + .iter() + .map(|(t, pairs)| (t.clone(), pairs.len())) + .collect(); + let row_counts: Vec<(String, usize)> = rows + .iter() + .map(|(t, recs)| (t.clone(), recs.len())) + .collect(); + assert_eq!(pair_counts, row_counts); } /// `split_counts` hands surplus workers to the busiest bucket (most @@ -4324,29 +4312,37 @@ mod tests { "bucket splitting broke input order" ); - // End-to-end (through `lookup_pair_terms`, which clamps workers to the - // term count) and through full hydration alike. - let pairs_above = lookup_pair_terms(output.clone(), probes.clone(), Some(above)).unwrap(); - let pairs_serial = lookup_pair_terms(output.clone(), probes.clone(), Some(1)).unwrap(); - assert_eq!(pairs_above, pairs_serial); - let rows_above = lookup_terms(output.clone(), probes.clone(), Some(above)).unwrap(); - let rows_serial = lookup_terms(output, probes, Some(1)).unwrap(); - assert_eq!(rows_above, rows_serial); + // End-to-end (through `lookup_pair_terms` / `lookup_terms`, which clamp + // the auto-selected workers to the term count) matches the shard-level + // serial result, through full hydration alike. + let pairs_auto = lookup_pair_terms(output.clone(), probes.clone()).unwrap(); + assert_eq!(pairs_auto, via_serial); + let rows_auto = lookup_terms(output, probes.clone()).unwrap(); + let row_counts: Vec<(String, usize)> = rows_auto + .iter() + .map(|(t, recs)| (t.clone(), recs.len())) + .collect(); + let pair_counts: Vec<(String, usize)> = via_serial + .iter() + .map(|(t, pairs)| (t.clone(), pairs.len())) + .collect(); + assert_eq!(row_counts, pair_counts); } - /// The production build-kg resolve calls `lookup_fullmap_terms` with - /// `threads=None`, so the parallel shard fan-out must kick in from the Rust - /// DEFAULT alone — not just when a test passes `threads>=2`. This builds a + /// The production build-kg resolve calls `lookup_fullmap_terms` with no + /// thread tuning, so the parallel shard fan-out must kick in from the Rust + /// DEFAULT alone. This builds a /// large fixture and probes it with a batch that crosses `LOOKUP_PARALLEL_MIN` /// and spans every default shard, then asserts: (a) the default worker count /// is >1 on any multi-core host, and reaches the 16-shard fan-out cap on a /// host with at least 16 CPUs (so `lookup_pair_terms_db` can spawn one reader - /// per non-empty shard), and (b) `threads=None` returns results IDENTICAL - /// (content + order) to the forced-serial `threads=Some(1)`, with misses + /// per non-empty shard), and (b) the auto-selected parallel lookup returns + /// results IDENTICAL (content + order) to a forced-serial + /// `lookup_pair_terms_db(..., 1)`, with misses /// dropped. On smaller hosts the maximum-fanout assertion is skipped but /// equivalence still holds. #[test] - fn threads_none_defaults_to_parallel_for_large_batch() { + fn large_batch_lookup_defaults_to_parallel() { pyo3::Python::initialize(); let dir = tempfile::tempdir().unwrap(); let synonyms = dir.path().join("large.ndjson"); @@ -4413,7 +4409,7 @@ mod tests { "large batch should be able to fan out across all default shards" ); } - // Explicit threads=Some(1) still forces serial regardless of batch size. + // Sub-threshold batches stay serial. assert_eq!(default_lookup_workers(0), 1, "empty batch stays serial"); assert_eq!( default_lookup_workers(LOOKUP_PARALLEL_MIN - 1), @@ -4421,13 +4417,14 @@ mod tests { "sub-threshold batch stays serial" ); - // threads=None (production default) == forced-serial Some(1): identical - // content AND order, misses dropped. - let via_default = lookup_pair_terms(output.clone(), probes.clone(), None).unwrap(); - let via_serial = lookup_pair_terms(output, probes.clone(), Some(1)).unwrap(); + // The auto-selected lookup (production path) == forced-serial shard read: + // identical content AND order, misses dropped. + let via_default = lookup_pair_terms(output.clone(), probes.clone()).unwrap(); + let shards = open_cached_shards(&output).unwrap(); + let via_serial = lookup_pair_terms_db(&shards, &probes, 1).unwrap(); assert_eq!( via_default, via_serial, - "threads=None diverged from threads=Some(1)" + "auto-selected workers diverged from forced-serial" ); let expected_order: Vec = probes .iter() @@ -4461,9 +4458,9 @@ mod tests { let body = &tail[..end]; // Rows fetch: exactly one call, and it is the detached one. - let rows_calls = body.matches("lookup_terms(db, terms, threads)").count(); + let rows_calls = body.matches("lookup_terms(db, terms)").count(); let rows_detached = body - .matches("py.detach(move || lookup_terms(db, terms, threads)") + .matches("py.detach(move || lookup_terms(db, terms)") .count(); assert_eq!(rows_calls, 1, "rows fetch must be called exactly once"); assert_eq!( @@ -4472,11 +4469,9 @@ mod tests { ); // Pairs fetch: exactly one call, and it is the detached one. - let pair_calls = body - .matches("lookup_pair_terms(db, terms, threads)") - .count(); + let pair_calls = body.matches("lookup_pair_terms(db, terms)").count(); let pair_detached = body - .matches("py.detach(move || lookup_pair_terms(db, terms, threads)") + .matches("py.detach(move || lookup_pair_terms(db, terms)") .count(); assert_eq!(pair_calls, 1, "pairs fetch must be called exactly once"); assert_eq!( @@ -4562,7 +4557,7 @@ mod tests { let shards = open_cached_shards(&output).unwrap(); assert_eq!(shards.len(), 2); let terms: Vec = (0..50).map(|i| format!("gene{i}")).collect(); - let rows = lookup_terms(output, terms, Some(4)).unwrap(); + let rows = lookup_terms(output, terms).unwrap(); assert_eq!(rows.len(), 50); } @@ -4593,14 +4588,7 @@ mod tests { std::env::set_var("TABLASSERT_FULLMAP_SHARDS", "2"); let built = Python::attach(|py| { - build_fullmap_db( - py, - output.clone(), - Vec::new(), - vec![synonyms], - Some(2), - None, - ) + build_fullmap_db(py, output.clone(), Vec::new(), vec![synonyms], None) }); std::env::remove_var("TABLASSERT_FULLMAP_SHARDS"); built.unwrap(); @@ -4623,7 +4611,7 @@ mod tests { // Lookups still resolve across all 16 shards. let terms: Vec = (0..50).map(|i| format!("gene{i}")).collect(); - let rows = lookup_terms(output, terms, Some(4)).unwrap(); + let rows = lookup_terms(output, terms).unwrap(); assert_eq!(rows.len(), 50); } @@ -4692,7 +4680,7 @@ mod tests { } let terms: Vec = (0..80).map(|i| format!("gene{i}")).collect(); - let rows = lookup_terms(output, terms, Some(SHARD_COUNT_SHARDS)).unwrap(); + let rows = lookup_terms(output, terms).unwrap(); assert_eq!(rows.len(), 80); } @@ -4710,7 +4698,7 @@ mod tests { write.commit().unwrap(); drop(database); - let err = lookup_terms(output, vec!["brca1".to_string()], Some(1)).unwrap_err(); + let err = lookup_terms(output, vec!["brca1".to_string()]).unwrap_err(); assert!(err .to_string() .contains("fullmap DB is outdated; rebuild with 'tablassert build-fullmap'")); @@ -4733,7 +4721,7 @@ mod tests { write.commit().unwrap(); drop(database); - let err = lookup_terms(output, vec!["brca1".to_string()], Some(1)).unwrap_err(); + let err = lookup_terms(output, vec!["brca1".to_string()]).unwrap_err(); assert!(err .to_string() .contains("fullmap DB is outdated; rebuild with 'tablassert build-fullmap'")); @@ -5087,7 +5075,7 @@ mod tests { ) .unwrap(); build_test(output.clone(), Vec::new(), vec![synonyms1], 1, 4_000_000).unwrap(); - let rows = lookup_terms(output.clone(), vec!["brca1".to_string()], Some(1)).unwrap(); + let rows = lookup_terms(output.clone(), vec!["brca1".to_string()]).unwrap(); assert_eq!(rows.len(), 1); assert_eq!(rows[0].1[0].curie, "HGNC:1100"); @@ -5105,14 +5093,14 @@ mod tests { swap_build_over(&built, &output); // Old-generation term must be gone (stale shards would resurrect it). - let rows = lookup_terms(output.clone(), vec!["brca1".to_string()], Some(1)).unwrap(); + let rows = lookup_terms(output.clone(), vec!["brca1".to_string()]).unwrap(); assert!( rows.is_empty(), "stale shard generation resurrected an old term: {rows:?}" ); // New term resolves against the NEW primary's dims/CURIES (a stale // primary would hydrate the wrong curie/preferred_name). - let rows = lookup_terms(output, vec!["tp53".to_string()], Some(1)).unwrap(); + let rows = lookup_terms(output, vec!["tp53".to_string()]).unwrap(); assert_eq!(rows.len(), 1); assert_eq!(rows[0].1[0].curie, "NCBIGene:7157"); assert_eq!(rows[0].1[0].preferred_name, "TP53"); @@ -5385,12 +5373,7 @@ mod tests { drop(database); // Every gene resolves through the streamed CURIES table. - let rows = lookup_terms( - output, - (0..5).map(|i| format!("gene{i}")).collect(), - Some(1), - ) - .unwrap(); + let rows = lookup_terms(output, (0..5).map(|i| format!("gene{i}")).collect()).unwrap(); let mut got: Vec = rows .iter() .flat_map(|(_, recs)| recs.iter().map(|r| r.curie.clone())) @@ -5419,11 +5402,11 @@ mod tests { build_test(output.clone(), Vec::new(), vec![synonyms], 1, 4_000_000).unwrap(); - let alive = lookup_terms(output.clone(), vec!["realname".to_string()], Some(1)).unwrap(); + let alive = lookup_terms(output.clone(), vec!["realname".to_string()]).unwrap(); assert_eq!(alive.len(), 1); assert_eq!(alive[0].1[0].curie, "HGNC:1100"); - let dead = lookup_terms(output, vec!["12345".to_string()], Some(1)).unwrap(); + let dead = lookup_terms(output, vec!["12345".to_string()]).unwrap(); assert!(dead.is_empty() || dead[0].1.is_empty()); } @@ -5480,11 +5463,11 @@ mod tests { drop(read); drop(database); - let kept = lookup_terms(output.clone(), vec!["water".to_string()], Some(1)).unwrap(); + let kept = lookup_terms(output.clone(), vec!["water".to_string()]).unwrap(); assert_eq!(kept.len(), 1); assert_eq!(kept[0].1[0].curie, "CHEBI:2"); - let dropped = lookup_terms(output, vec!["genea".to_string()], Some(1)).unwrap(); + let dropped = lookup_terms(output, vec!["genea".to_string()]).unwrap(); assert!(dropped.is_empty() || dropped[0].1.is_empty()); } @@ -5543,7 +5526,7 @@ mod tests { .into_iter() .map(|i| format!("gene{i}")) .collect(); - let rows = lookup_terms(output, terms, Some(4)).unwrap(); + let rows = lookup_terms(output, terms).unwrap(); let mut got: Vec = rows .iter() .flat_map(|(_, recs)| recs.iter().map(|r| r.curie.clone())) @@ -5562,7 +5545,6 @@ mod tests { PathBuf::from("/tmp/should-not-exist.redb"), Vec::new(), Vec::new(), - Some(1), None, ) .expect_err("empty synonyms should fail"); @@ -5587,7 +5569,7 @@ mod tests { .unwrap(); build_test(output.clone(), Vec::new(), vec![synonyms], 1, 4_000_000).unwrap(); - let rows = lookup_terms(output, vec!["alias disease".to_string()], Some(1)).unwrap(); + let rows = lookup_terms(output, vec!["alias disease".to_string()]).unwrap(); assert_eq!(rows.len(), 1); assert_eq!(rows[0].1[0].curie, "MONDO:1"); @@ -5613,7 +5595,6 @@ mod tests { let rows = lookup_terms( output, vec!["hypothetical protein".to_string(), "gene1".to_string()], - Some(1), ) .unwrap(); @@ -5636,7 +5617,7 @@ mod tests { .unwrap(); build_test(output.clone(), Vec::new(), vec![synonyms], 1, 4_000_000).unwrap(); - let rows = lookup_terms(output, vec!["quoted gene".to_string()], Some(1)).unwrap(); + let rows = lookup_terms(output, vec!["quoted gene".to_string()]).unwrap(); assert_eq!(rows.len(), 1); assert_eq!(rows[0].1[0].preferred_name, "Quoted Gene"); @@ -5677,8 +5658,8 @@ mod tests { "beta".to_string(), "delta".to_string(), ]; - let big = lookup_terms(out_big, probes.clone(), Some(1)).unwrap(); - let tiny = lookup_terms(out_tiny, probes, Some(1)).unwrap(); + let big = lookup_terms(out_big, probes.clone()).unwrap(); + let tiny = lookup_terms(out_tiny, probes).unwrap(); // Same terms resolved, same hydrated records (order-independent compare). let norm = |v: Vec<(String, Vec)>| -> Vec<(String, Vec)> { @@ -5727,7 +5708,7 @@ mod tests { // Several consecutive lookups (rows path) must all succeed. for _ in 0..3 { - let rows = lookup_terms(output.clone(), vec!["brca1".to_string()], Some(1)).unwrap(); + let rows = lookup_terms(output.clone(), vec!["brca1".to_string()]).unwrap(); assert_eq!(rows.len(), 1); } // open_cached returns the same handle across calls. @@ -6033,7 +6014,7 @@ mod tests { } write.commit().unwrap(); drop(database); - let err = lookup_terms(v2, vec!["brca1".to_string()], Some(1)).unwrap_err(); + let err = lookup_terms(v2, vec!["brca1".to_string()]).unwrap_err(); assert!(err .to_string() .contains("fullmap DB is outdated; rebuild with 'tablassert build-fullmap'")); @@ -6049,7 +6030,7 @@ mod tests { } write.commit().unwrap(); drop(database); - let err = lookup_terms(missing, vec!["brca1".to_string()], Some(1)).unwrap_err(); + let err = lookup_terms(missing, vec!["brca1".to_string()]).unwrap_err(); assert!(err.to_string().contains("unsupported fullmap redb schema")); // Garbage schema value => unsupported. @@ -6062,7 +6043,7 @@ mod tests { } write.commit().unwrap(); drop(database); - let err = lookup_terms(garbage, vec!["brca1".to_string()], Some(1)).unwrap_err(); + let err = lookup_terms(garbage, vec!["brca1".to_string()]).unwrap_err(); assert!(err.to_string().contains("unsupported fullmap redb schema")); } diff --git a/rust/tests/build_golden.rs b/rust/tests/build_golden.rs index 6e5dc3bd..159fac98 100644 --- a/rust/tests/build_golden.rs +++ b/rust/tests/build_golden.rs @@ -100,7 +100,7 @@ t|HGNC:3 #[test] fn golden_output_is_pinned() { let dir = tempfile::tempdir().unwrap(); - let output = build_fixture(dir.path(), 1); + let output = build_fixture(dir.path()); let map = term_curie_map(&output); let actual = canonical_dump(&map); assert!( @@ -119,7 +119,7 @@ fn golden_output_is_pinned() { #[ignore] fn regenerate_golden() { let dir = tempfile::tempdir().unwrap(); - let output = build_fixture(dir.path(), 1); + let output = build_fixture(dir.path()); let map = term_curie_map(&output); println!("===GOLDEN-START==="); print!("{}", canonical_dump(&map)); @@ -134,8 +134,8 @@ fn regenerate_golden() { fn deterministic_across_rebuilds() { let dir_a = tempfile::tempdir().unwrap(); let dir_b = tempfile::tempdir().unwrap(); - let out_a = build_fixture(dir_a.path(), 1); - let out_b = build_fixture(dir_b.path(), 1); + let out_a = build_fixture(dir_a.path()); + let out_b = build_fixture(dir_b.path()); let dump_a = canonical_dump(&term_curie_map(&out_a)); let dump_b = canonical_dump(&term_curie_map(&out_b)); assert!(!dump_a.is_empty()); @@ -146,20 +146,24 @@ fn deterministic_across_rebuilds() { } // --------------------------------------------------------------------------- -// (c) THREAD INVARIANCE — threads=1 and threads=4 agree on term -> CURIE strings -// (curie_ids are scheduling-dependent and deliberately NOT compared). +// (c) THREAD INVARIANCE — the public build API now selects parallelism +// automatically, so an explicit threads=1 vs threads=4 comparison can no longer +// be driven from an integration test. The explicit worker-count determinism +// coverage (worker_count 1 vs 4 through `build_test` / `build_fullmap_inner`, +// comparing term -> CURIE results and per-shard record counts) lives in the +// in-crate unit test `parallel_writers_match_single_writer_build` in +// src/fullmap.rs. Here we pin the integration-level half: a build whose +// worker count was auto-selected (parallel on any multi-core host) reproduces +// the pinned GOLDEN — which was generated single-threaded — exactly. // --------------------------------------------------------------------------- #[test] -fn thread_count_does_not_change_results() { - let dir_a = tempfile::tempdir().unwrap(); - let dir_b = tempfile::tempdir().unwrap(); - let serial = term_curie_map(&build_fixture(dir_a.path(), 1)); - let parallel = term_curie_map(&build_fixture(dir_b.path(), 4)); - assert!(!serial.is_empty()); +fn automatic_parallelism_matches_golden() { + let dir = tempfile::tempdir().unwrap(); + let actual = canonical_dump(&term_curie_map(&build_fixture(dir.path()))); assert_eq!( - serial, parallel, - "thread count must not change term -> CURIE results" + GOLDEN, actual, + "auto-parallel build diverged from the pinned (serial-origin) golden" ); } @@ -170,7 +174,7 @@ fn thread_count_does_not_change_results() { #[test] fn dimension_tables_are_complete_and_consistent() { let dir = tempfile::tempdir().unwrap(); - let output = build_fixture(dir.path(), 1); + let output = build_fixture(dir.path()); let db = open_primary_copy(&output); let read = db.begin_read().unwrap(); @@ -250,7 +254,7 @@ fn dimension_tables_are_complete_and_consistent() { #[test] fn schema_and_shard_count_are_pinned() { let dir = tempfile::tempdir().unwrap(); - let output = build_fixture(dir.path(), 1); + let output = build_fixture(dir.path()); let db = open_primary_copy(&output); let read = db.begin_read().unwrap(); let meta = read.open_table(META).unwrap(); @@ -289,7 +293,7 @@ fn schema_and_shard_count_are_pinned() { fn gz_input_matches_plain_input() { // Plain build. let dir_plain = tempfile::tempdir().unwrap(); - let plain = build_fixture(dir_plain.path(), 1); + let plain = build_fixture(dir_plain.path()); let plain_map = term_curie_map(&plain); // Gz build: same class + synonym content, synonym file gzipped. The source @@ -319,7 +323,6 @@ fn gz_input_matches_plain_input() { output_gz.clone(), vec![classes], vec![synonyms_gz], - Some(1), None, ) .unwrap(); @@ -348,15 +351,8 @@ fn empty_synonym_file_builds_empty_db() { write_jsonl(&synonyms, &[]); // 0 rows let output = dir.path().join("fullmap.redb"); pyo3::Python::attach(|py| { - tablassert_rs::build_fullmap_db( - py, - output.clone(), - vec![classes], - vec![synonyms], - Some(1), - None, - ) - .unwrap(); + tablassert_rs::build_fullmap_db(py, output.clone(), vec![classes], vec![synonyms], None) + .unwrap(); }); assert!( @@ -387,8 +383,7 @@ fn synonym_row_with_no_names_indexes_only_curie() { ); let output = dir.path().join("fullmap.redb"); pyo3::Python::attach(|py| { - tablassert_rs::build_fullmap_db(py, output.clone(), vec![], vec![synonyms], Some(1), None) - .unwrap(); + tablassert_rs::build_fullmap_db(py, output.clone(), vec![], vec![synonyms], None).unwrap(); }); let map = term_curie_map(&output); @@ -414,8 +409,7 @@ fn null_preferred_name_falls_back_to_curie() { ); let output = dir.path().join("fullmap.redb"); pyo3::Python::attach(|py| { - tablassert_rs::build_fullmap_db(py, output.clone(), vec![], vec![synonyms], Some(1), None) - .unwrap(); + tablassert_rs::build_fullmap_db(py, output.clone(), vec![], vec![synonyms], None).unwrap(); }); let db = open_primary_copy(&output); @@ -450,15 +444,8 @@ fn class_row_without_equivalents_builds() { ); let output = dir.path().join("fullmap.redb"); pyo3::Python::attach(|py| { - tablassert_rs::build_fullmap_db( - py, - output.clone(), - vec![classes], - vec![synonyms], - Some(1), - None, - ) - .unwrap(); + tablassert_rs::build_fullmap_db(py, output.clone(), vec![classes], vec![synonyms], None) + .unwrap(); }); let map = term_curie_map(&output); diff --git a/rust/tests/common/mod.rs b/rust/tests/common/mod.rs index 5b0785f4..0a2a4842 100644 --- a/rust/tests/common/mod.rs +++ b/rust/tests/common/mod.rs @@ -106,10 +106,10 @@ pub fn shard_path(primary: &Path, index: usize) -> PathBuf { primary.with_file_name(format!("{stem}.s{index}.{ext}")) } -/// Build the fixed fixture at `/fullmap.redb` with `threads` workers and -/// return the primary path. Uses the public `build_fullmap_db` (the production -/// entry point), exactly as Python callers do. -pub fn build_fixture(dir: &Path, threads: usize) -> PathBuf { +/// Build the fixed fixture at `/fullmap.redb` (parallelism is selected +/// automatically by the build) and return the primary path. Uses the public +/// `build_fullmap_db` (the production entry point), exactly as Python callers do. +pub fn build_fixture(dir: &Path) -> PathBuf { pyo3::Python::initialize(); let classes = dir.join("classes.ndjson"); let synonyms = dir.join("SRC.ndjson"); @@ -117,15 +117,8 @@ pub fn build_fixture(dir: &Path, threads: usize) -> PathBuf { write_jsonl(&synonyms, SYNONYM_LINES); let output = dir.join("fullmap.redb"); pyo3::Python::attach(|py| { - tablassert_rs::build_fullmap_db( - py, - output.clone(), - vec![classes], - vec![synonyms], - Some(threads), - None, - ) - .unwrap(); + tablassert_rs::build_fullmap_db(py, output.clone(), vec![classes], vec![synonyms], None) + .unwrap(); }); output } diff --git a/rust/tests/extract_prebuilt.rs b/rust/tests/extract_prebuilt.rs index 840cb024..bb58bcfa 100644 --- a/rust/tests/extract_prebuilt.rs +++ b/rust/tests/extract_prebuilt.rs @@ -150,7 +150,7 @@ fn extract_prebuilt_matches_force_build() { // 1. Build the fixture with the production entry point (single-threaded // for a deterministic, fast build). let fixture_dir = tempfile::tempdir().unwrap(); - let fixture_primary = common::build_fixture(fixture_dir.path(), 1); + let fixture_primary = common::build_fixture(fixture_dir.path()); // 2. Package the primary + all 16 shards into `fullmap.tar.zst`. Members // are nested under a `bundle/` subdirectory to exercise the recursive @@ -272,7 +272,7 @@ fn corrupt_archive_is_rejected_and_leaves_nothing() { #[test] fn archive_without_primary_is_rejected() { let fixture_dir = tempfile::tempdir().unwrap(); - let fixture_primary = common::build_fixture(fixture_dir.path(), 1); + let fixture_primary = common::build_fixture(fixture_dir.path()); let archive = fixture_dir.path().join("fullmap.tar.zst"); package_tar_zst( &archive, @@ -290,7 +290,7 @@ fn archive_without_primary_is_rejected() { #[test] fn missing_shard_is_rejected_and_named() { let fixture_dir = tempfile::tempdir().unwrap(); - let fixture_primary = common::build_fixture(fixture_dir.path(), 1); + let fixture_primary = common::build_fixture(fixture_dir.path()); let mut members = vec![("fullmap.redb".to_string(), fixture_primary.clone())]; members.extend(shard_members(&fixture_primary, common::SHARD_COUNT - 1)); // s0..s14 let archive = fixture_dir.path().join("fullmap.tar.zst"); @@ -311,7 +311,7 @@ fn missing_shard_is_rejected_and_named() { #[test] fn extra_shard_is_rejected_and_named() { let fixture_dir = tempfile::tempdir().unwrap(); - let fixture_primary = common::build_fixture(fixture_dir.path(), 1); + let fixture_primary = common::build_fixture(fixture_dir.path()); // No s16 exists in a real build; misuse a copy of s0's bytes as the stray. let stray = fixture_dir.path().join("fullmap.s16.redb"); std::fs::copy(common::shard_path(&fixture_primary, 0), &stray).unwrap(); @@ -373,7 +373,7 @@ fn outdated_primary_copy(fixture_primary: &Path) -> PathBuf { #[test] fn outdated_schema_is_rejected_and_demands_rebuild() { let fixture_dir = tempfile::tempdir().unwrap(); - let fixture_primary = common::build_fixture(fixture_dir.path(), 1); + let fixture_primary = common::build_fixture(fixture_dir.path()); let outdated = outdated_primary_copy(&fixture_primary); let mut members = vec![("fullmap.redb".to_string(), outdated)]; members.extend(shard_members(&fixture_primary, common::SHARD_COUNT)); @@ -395,7 +395,7 @@ fn outdated_schema_is_rejected_and_demands_rebuild() { #[test] fn non_redb_primary_is_rejected() { let fixture_dir = tempfile::tempdir().unwrap(); - let fixture_primary = common::build_fixture(fixture_dir.path(), 1); + let fixture_primary = common::build_fixture(fixture_dir.path()); let garbage = fixture_dir.path().join("garbage.redb"); std::fs::write(&garbage, b"this is definitely not a redb database").unwrap(); let mut members = vec![("fullmap.redb".to_string(), garbage)]; @@ -524,7 +524,7 @@ fn pax_sparse_archive_is_rejected() { #[test] fn multiple_unnamed_primaries_are_rejected_and_listed() { let fixture_dir = tempfile::tempdir().unwrap(); - let fixture_primary = common::build_fixture(fixture_dir.path(), 1); + let fixture_primary = common::build_fixture(fixture_dir.path()); let stray_a = fixture_dir.path().join("primary_a.redb"); let stray_b = fixture_dir.path().join("primary_b.redb"); std::fs::copy(&fixture_primary, &stray_a).unwrap(); @@ -558,7 +558,7 @@ fn multiple_unnamed_primaries_are_rejected_and_listed() { #[test] fn named_fullmap_primary_is_preferred_over_strays() { let fixture_dir = tempfile::tempdir().unwrap(); - let fixture_primary = common::build_fixture(fixture_dir.path(), 1); + let fixture_primary = common::build_fixture(fixture_dir.path()); let stray = fixture_dir.path().join("stray.redb"); std::fs::copy(&fixture_primary, &stray).unwrap(); let mut members = vec![ @@ -598,7 +598,7 @@ fn named_fullmap_primary_is_preferred_over_strays() { #[test] fn progress_callback_details_are_pinned() { let fixture_dir = tempfile::tempdir().unwrap(); - let fixture_primary = common::build_fixture(fixture_dir.path(), 1); + let fixture_primary = common::build_fixture(fixture_dir.path()); let archive = fixture_dir.path().join("fullmap.tar.zst"); let mut members = vec![("fullmap.redb".to_string(), fixture_primary.clone())]; members.extend(shard_members(&fixture_primary, common::SHARD_COUNT)); diff --git a/src/tablassert/agent.py b/src/tablassert/agent.py index d6de5d76..343f3540 100644 --- a/src/tablassert/agent.py +++ b/src/tablassert/agent.py @@ -4979,7 +4979,6 @@ def run_gepa( task_lm: object | None = None, gepa_cls: object | None = None, max_metric_calls: int | None = 8, - num_threads: int | None = None, dataset: list[dict[str, Any]] | None = None, ) -> dict[str, Any]: """Optimize the agent's instructions as a BLACK BOX with dspy.GEPA (Pareto-native, textual feedback). @@ -4994,8 +4993,7 @@ def run_gepa( LM split (GEPA best practice): GEPA evaluates candidate programs MANY times but reflects only a few times. ``task_lm`` (when given) is the FAST model configured for those many program evaluations (``dspy.configure``), while ``reflection_lm`` is the STRONG model GEPA uses for the few - instruction-proposal steps. When ``task_lm`` is None, ``reflection_lm`` is used for both. ``num_threads`` - parallelizes GEPA's evaluation pool when set. + instruction-proposal steps. When ``task_lm`` is None, ``reflection_lm`` is used for both. """ _require("dspy") import dspy as _dspy # pyright: ignore[reportMissingImports] @@ -5007,8 +5005,6 @@ def run_gepa( "reflection_lm": reflection_lm, "max_metric_calls": max_metric_calls, } - if num_threads is not None: - gepa_kwargs["num_threads"] = num_threads try: optimizer: Any = cls(**gepa_kwargs) except TypeError: diff --git a/src/tablassert/cli.py b/src/tablassert/cli.py index 4be74cc0..6292f880 100644 --- a/src/tablassert/cli.py +++ b/src/tablassert/cli.py @@ -146,11 +146,10 @@ def build_pipeline( log: bool = False, head: bool = False, no_original: bool = False, - threads: int | None = None, ) -> None: """Load a graph YAML and build it through the shared in-process core.""" graph: Graph = _load_graph(configuration_file) - build_graph_pipeline(graph, configuration_file, progress, release=release, qc=qc, log=log, head=head, no_original=no_original, threads=threads) + build_graph_pipeline(graph, configuration_file, progress, release=release, qc=qc, log=log, head=head, no_original=no_original) def build_graph_pipeline( @@ -162,7 +161,6 @@ def build_graph_pipeline( log: bool = False, head: bool = False, no_original: bool = False, - threads: int | None = None, audit_sources: bool = True, ) -> None: """Build a validated :class:`Graph` without loading another graph YAML. @@ -182,8 +180,6 @@ def build_graph_pipeline( head: When ``True``, build a random sample of up to five rows per section. no_original: When ``True``, omit the verbatim ``original_*`` source-cell copies from the final edge NDJSON. - threads: Optional worker thread count for the parallel fullmap reads behind - entity resolution (auto when unset). """ from tablassert.fullmap import fullmap_db_path from tablassert.lib import Tcode, compile_graph, compile_subgraph @@ -243,7 +239,6 @@ def build_graph_pipeline( "qc": qc, "release": release, "head": head, - "threads": threads, "name": g.name, "infores": g.rig.source_info.infores_id, } @@ -697,7 +692,6 @@ def build_kg( log: Annotated[bool, cyclopts.Parameter(name=["--log", "-l"], negative="")] = False, head: Annotated[bool, cyclopts.Parameter(name=["--head", "-hd"], negative="")] = False, no_original: Annotated[bool, cyclopts.Parameter(name=["--no-original", "-no"], negative="")] = False, - threads: Annotated[int | None, cyclopts.Parameter(name=["--threads", "-t"])] = None, ) -> None: """Build a knowledge graph from a YAML configuration file. @@ -706,8 +700,7 @@ def build_kg( ``--no-original`` omits the verbatim source-cell copies (``original_subject``, ``original_object``, and any other ``original_*`` fields) from the final edge - NDJSON. ``--threads`` sets the worker count for the parallel fullmap reads behind entity - resolution (auto when unset). ``--qc`` requires the ``[qc]`` extra (``pip install + NDJSON. ``--qc`` requires the ``[qc]`` extra (``pip install "tablassert[qc]"``); it is checked before the build starts, because the audit stage runs LAST and a missing extra would otherwise surface only after entity resolution has finished. It also runs a final study stage that asserts over the emitted NDJSON @@ -716,16 +709,9 @@ def build_kg( malformed lines, no null or empty values in any field, and no stray whitespace -- and fails the build (non-zero exit) when any assertion is violated. """ - # A non-positive thread count would only fail deep inside the Rust lookup; fail loud - # up front, matching the --gepa-threads pattern. - if threads is not None and threads < 1: - print("tablassert build-kg: --threads must be a positive integer.", file=sys.stderr) - raise SystemExit(2) if qc: extras.require("qc", required_by="--qc") - run( - 7 if qc else 6, build_pipeline, graph_configuration_file, release=release, qc=qc, log=log, head=head, no_original=no_original, threads=threads - ) + run(7 if qc else 6, build_pipeline, graph_configuration_file, release=release, qc=qc, log=log, head=head, no_original=no_original) @APP.command(name="validate") @@ -809,7 +795,6 @@ def agent( max_metric_calls: Annotated[int, cyclopts.Parameter(name=["--max-metric-calls"])] = 8, dataset: Annotated[Path | None, cyclopts.Parameter(name=["--dataset"])] = None, task_model: Annotated[str | None, cyclopts.Parameter(name=["--task-model"])] = None, - gepa_threads: Annotated[int | None, cyclopts.Parameter(name=["--gepa-threads"])] = None, ) -> None: """Autonomously derive, build, audit, and improve KG configs from PMC articles. @@ -866,9 +851,6 @@ def agent( task_model: Optional FAST model id for GEPA's many program evaluations (GEPA best practice: a cheap task LM + a strong reflection LM); ``--model-id`` is the strong reflection LM. Defaults to the reflection LM when unset. - gepa_threads: Optional thread count for GEPA's evaluation pool. Parallelizes the candidate LM - forward passes only; the coverage-scoring builds stay serialized on the process-wide - ``_GEPA_BUILD_LOCK`` (``os.chdir`` is process-global), so more threads do not speed up builds. """ from tablassert import agent as agent_mod from tablassert.graph_target import prepare_graph @@ -902,12 +884,6 @@ def agent( print("tablassert agent: --biolink-threshold must be a finite number between 0 and 1.", file=sys.stderr) raise SystemExit(2) - # A non-positive thread count would only fail deep inside dspy/ThreadPoolExecutor AFTER the models are - # built; fail loud up front, matching the --judge-threshold pattern. - if gepa_threads is not None and gepa_threads < 1: - print("tablassert agent: --gepa-threads must be a positive integer.", file=sys.stderr) - raise SystemExit(2) - # --distill records the SUPERVISOR's model calls; the --optimize path returns early below and # GEPA's dspy LM bypasses the recording seam, so the combination would silently record nothing. if distill and optimize: @@ -1009,7 +985,6 @@ def parse_local(specs: list[str] | None) -> dict[str, Path] | Path | None: task_lm=task_lm, dataset=gepa_dataset, max_metric_calls=max_metric_calls, - num_threads=gepa_threads, ) # A failed GEPA compile falls back to the SEED instructions with stats["error"]; do NOT persist that # unoptimized prompt or report success -- fail loud with a non-zero status. @@ -1256,12 +1231,7 @@ def report_progress(downloaded: int, total: int) -> None: def build_fullmap_pipeline( - output: Path, - progress: PipelineProgress, - cache: Path = Path("./fullmap/downloads"), - version: str = BABEL_VERSION, - threads: int | None = None, - aria2c: bool = False, + output: Path, progress: PipelineProgress, cache: Path = Path("./fullmap/downloads"), version: str = BABEL_VERSION, aria2c: bool = False ) -> None: """Build an embedded fullmap redb database from BABEL outputs. @@ -1273,7 +1243,6 @@ def build_fullmap_pipeline( progress: Pipeline progress reporter. cache: Directory for downloaded BABEL files. version: BABEL version label. - threads: Optional thread count forwarded to Rust. aria2c: Use the bundled aria2c binary from the optional ``[aria2]`` extra for downloads when true. """ from tablassert import rs @@ -1317,7 +1286,7 @@ def download_one(filename: str, url: str, destination: Path) -> Path: # Rust drives per-phase progress (equivalents -> synonyms -> writing) via the # callback; the GIL is released during the build so the bar repaints live. on_progress = progress.dynamic_loop("Build") - rs.build_fullmap_db(output, class_files, synonym_files, threads=threads, progress=on_progress) + rs.build_fullmap_db(output, class_files, synonym_files, progress=on_progress) progress.end_section_task() logger.info( @@ -1334,7 +1303,6 @@ def build_fullmap( output: Annotated[Path, cyclopts.Parameter(name=["--output", "-o"])] = Path("./fullmap/data/fullmap.redb"), cache: Annotated[Path, cyclopts.Parameter(name=["--cache", "-c"])] = Path("./fullmap/downloads"), version: Annotated[str, cyclopts.Parameter(name=["--version", "-v"])] = BABEL_VERSION, - threads: Annotated[int | None, cyclopts.Parameter(name=["--threads", "-t"])] = None, aria2c: Annotated[bool, cyclopts.Parameter(name=["--aria2c", "-a"], negative="")] = False, force: Annotated[bool, cyclopts.Parameter(name=["--force", "-f"], negative="")] = False, ) -> None: @@ -1353,7 +1321,6 @@ def build_fullmap( output: Path to write the redb file (prebuilt extraction or build output). cache: Directory for downloaded BABEL files when building from scratch. version: BABEL snapshot date to fetch (a RENCI stamp, NOT Tablassert's version). - threads: Worker threads for a from-scratch build (auto when unset). aria2c: Use the bundled aria2c binary from the ``[aria2]`` extra for downloads (prebuilt or BABEL). force: Skip the prebuilt download and always rebuild from BABEL outputs. @@ -1374,4 +1341,4 @@ def build_fullmap( return except PrebuiltFullmapUnavailable as exc: logger.warning("Prebuilt fullmap unavailable ({reason}); building from BABEL outputs.", reason=exc) - run(3, build_fullmap_pipeline, output, cache=cache, version=version, threads=threads, aria2c=aria2c) + run(3, build_fullmap_pipeline, output, cache=cache, version=version, aria2c=aria2c) diff --git a/src/tablassert/fullmap.py b/src/tablassert/fullmap.py index dc23540a..1ab791d7 100644 --- a/src/tablassert/fullmap.py +++ b/src/tablassert/fullmap.py @@ -178,13 +178,12 @@ def _dimension_maps(db: Path, cache_key: tuple[Path, float]) -> tuple[list[str], return value -def lookup_rows(db: Path, terms: list[str], threads: int | None = None) -> list[dict[str, object]]: +def lookup_rows(db: Path, terms: list[str]) -> list[dict[str, object]]: """Lookup terms using the v2 raw-pair path and hydrate rows once per batch. Args: db: Path to the fullmap redb file. terms: Terms to query. - threads: Optional thread count forwarded to Rust. Returns: Hydrated rows matching the legacy ``lookup_fullmap_terms`` shape. @@ -203,7 +202,7 @@ def lookup_rows(db: Path, terms: list[str], threads: int | None = None) -> list[ if misses: try: - pair_rows: list[dict[str, object]] = _call_with_lock_retry(rs.lookup_fullmap_terms, db, misses, threads=threads, return_format="pairs") + pair_rows: list[dict[str, object]] = _call_with_lock_retry(rs.lookup_fullmap_terms, db, misses, return_format="pairs") except TypeError as exc: # Only swallow the signature-mismatch TypeError from an old extension that # lacks return_format; any other TypeError is a real bug and must propagate. @@ -212,12 +211,12 @@ def lookup_rows(db: Path, terms: list[str], threads: int | None = None) -> list[ # Legacy extension without return_format: re-query the FULL term set so # already-cached terms are not dropped from the returned rows. _warn_legacy_compat("no return_format support") - return _call_with_lock_retry(rs.lookup_fullmap_terms, db, terms, threads=threads) + return _call_with_lock_retry(rs.lookup_fullmap_terms, db, terms) if pair_rows and "records" not in pair_rows[0]: # Legacy row shape covers only `misses`; re-query the FULL term set so # already-cached terms are not dropped when _TERM_CACHE is partially warm. _warn_legacy_compat("legacy row shape") - return _call_with_lock_retry(rs.lookup_fullmap_terms, db, terms, threads=threads) + return _call_with_lock_retry(rs.lookup_fullmap_terms, db, terms) seen: set[str] = set() for row in pair_rows: term = str(row["term"]) @@ -568,7 +567,6 @@ def resolve_batch( config_file: str | None = None, column_context: bool = True, tag: str = "_two", - threads: int | None = None, on_phase: Callable[[str], None] | None = None, ) -> pl.LazyFrame: """Resolve multiple node columns against one shared redb fetch. @@ -587,7 +585,6 @@ def resolve_batch( config_file: Originating config file (for log context). column_context: Whether to compute/use category frequency as a tiebreaker. tag: Suffix used to derive level-two column names. - threads: Optional thread count forwarded to the Rust lookup. on_phase: Optional callback fired with ``"resolve:"`` before each column is processed, used to drive fine-grained progress UX. @@ -643,7 +640,7 @@ def resolve_batch( union_terms: list[str] = pl.concat([t.select("term") for t in terms_by_col.values()]).unique().get_column("term").to_list() - rows: list[dict[str, object]] = lookup_rows(db, union_terms, threads=threads) if union_terms else [] + rows: list[dict[str, object]] = lookup_rows(db, union_terms) if union_terms else [] raw: pl.DataFrame = pl.DataFrame(rows) result: pl.DataFrame = df @@ -675,7 +672,6 @@ def resolve( config_file: str | None = None, column_context: bool = True, tag: str = "_two", - threads: int | None = None, ) -> pl.LazyFrame: """Case-dependent, provenance-rich named-entity recognition (single-column wrapper). @@ -696,7 +692,6 @@ def resolve( config_file: Originating config file (for log context). column_context: Whether to compute/use category frequency as a tiebreaker. tag: Suffix used to derive the level-two column name. - threads: Optional thread count forwarded to the Rust lookup. Returns: LazyFrame with resolved columns added. @@ -710,5 +705,4 @@ def resolve( config_file=config_file, column_context=column_context, tag=tag, - threads=threads, ) diff --git a/src/tablassert/lib.py b/src/tablassert/lib.py index c0892c7d..135efc38 100644 --- a/src/tablassert/lib.py +++ b/src/tablassert/lib.py @@ -1148,7 +1148,6 @@ class Tcode(Section): qc: bool = Field(False) release: bool = Field(False) head: bool = Field(False) - threads: int | None = Field(None) name: str | None = Field(None) infores: str | None = Field(None) @@ -1299,10 +1298,7 @@ def _node_ops(self: Self, db: Path) -> list[Any]: # copied them into named slots; trim before resolution so the frame that # resolve_batch materializes and joins stays narrow. (trim, ()), - # ``"_two"`` is spelled explicitly (it is ``resolve_batch``'s own default tag) - # only so ``threads`` can follow positionally: ``compile_subgraph`` applies op - # args positionally (``on_phase`` arrives separately as a keyword). - (resolve_batch, (specs, db, self.log, self.store.stem, self.config.name, True, "_two", self.threads)), + (resolve_batch, (specs, db, self.log, self.store.stem, self.config.name, True, "_two")), # QC audits only the strict columns: a nullable qualifier's nulls are expected # (blank cell / no match), not resolution errors for the audit to delete. [ diff --git a/src/tablassert/rs.pyi b/src/tablassert/rs.pyi index c55b22fe..87c6571e 100644 --- a/src/tablassert/rs.pyi +++ b/src/tablassert/rs.pyi @@ -5,7 +5,7 @@ from pathlib import Path from typing import Any def build_fullmap_db( - output: Path, classes: list[Path], synonyms: list[Path], threads: int | None = None, progress: Callable[[int, int, int, str], None] | None = None + output: Path, classes: list[Path], synonyms: list[Path], progress: Callable[[int, int, int, str], None] | None = None ) -> None: ... def dedup_ndjson( input: Path, output: Path, is_edges: bool, domain: str | None = None, uuid_fields: list[str] | None = None, on_collision: str | None = None @@ -16,6 +16,6 @@ def hydrate_categories(db: Path) -> list[str]: ... def hydrate_curies(db: Path, curie_ids: list[int]) -> list[dict[str, Any]]: ... def hydrate_prefixes(db: Path) -> list[str]: ... def hydrate_sources(db: Path) -> list[str]: ... -def lookup_fullmap_terms(db: Path, terms: list[str], threads: int | None = None, return_format: str = "rows") -> list[dict[str, Any]]: ... +def lookup_fullmap_terms(db: Path, terms: list[str], return_format: str = "rows") -> list[dict[str, Any]]: ... def namespace_uuid(domain: str, values: list[str]) -> str: ... def xxh64(data: str) -> str: ... diff --git a/tests/test_agent_branches.py b/tests/test_agent_branches.py index 388c375f..81fab0f0 100644 --- a/tests/test_agent_branches.py +++ b/tests/test_agent_branches.py @@ -114,7 +114,7 @@ def jl(p: Path, rows: list[dict[str, Any]]) -> Path: ], ) output = root / "data" / "fullmap.redb" - rs.build_fullmap_db(output, [classes], [synonyms], threads=2) + rs.build_fullmap_db(output, [classes], [synonyms]) return output diff --git a/tests/test_agent_build.py b/tests/test_agent_build.py index 831ebabe..2efad332 100644 --- a/tests/test_agent_build.py +++ b/tests/test_agent_build.py @@ -45,7 +45,7 @@ def _build_real_redb(root: Path) -> Path: [_synonym_row("HGNC:1100", "BRCA1", ["BRCA1", "brca1"], "Gene"), _synonym_row("HGNC:6871", "MAPK1", ["MAPK1", "mapk1"], "Gene")], ) output: Path = root / "data" / "fullmap.redb" - rs.build_fullmap_db(output, [classes], [synonyms], threads=2) + rs.build_fullmap_db(output, [classes], [synonyms]) return output @@ -290,7 +290,7 @@ def _gene_disease_redb(root: Path) -> Path: [_synonym_row("HGNC:1100", "BRCA1", ["BRCA1", "brca1"], "Gene"), _synonym_row("MONDO:0008903", "lung cancer", ["lung cancer"], "Disease")], ) output: Path = root / "data" / "fullmap.redb" - rs.build_fullmap_db(output, [classes], [synonyms], threads=2) + rs.build_fullmap_db(output, [classes], [synonyms]) return output diff --git a/tests/test_agent_cli.py b/tests/test_agent_cli.py index 444ffec5..c819b873 100644 --- a/tests/test_agent_cli.py +++ b/tests/test_agent_cli.py @@ -415,29 +415,6 @@ def fake_run_supervisor(pmc_ids: list[str], **kwargs: object) -> dict[str, objec assert captured["judge_threshold"] == 0.7 -@pytest.mark.parametrize("bad_threads", [0, -1, -8]) -def test_agent_gepa_threads_non_positive_exits_2(bad_threads: int, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: - """CodeRabbit: a non-positive --gepa-threads fails loud (exit 2) before any model is built.""" - monkeypatch.setenv(ENV_MODEL_ID, "m") - monkeypatch.setenv(ENV_API_BASE, "b") - monkeypatch.setenv(ENV_API_KEY, "k") - - def fail_supervisor(*a: object, **k: object) -> object: - raise AssertionError("run_supervisor must NOT run with an invalid --gepa-threads") - - def fail_model_init(*a: object, **k: object) -> object: - raise AssertionError("make_dspy_lm must NOT run with an invalid --gepa-threads") - - monkeypatch.setattr("tablassert.agent.run_supervisor", fail_supervisor) - # also prove NO model construction happens before validation (not just no supervisor run) - monkeypatch.setattr("tablassert.agent.make_dspy_lm", fail_model_init) - - with pytest.raises(SystemExit) as exc_info: - agent(["PMC1"], graph_configuration_file=_graph_path(), gepa_threads=bad_threads) - assert exc_info.value.code == 2 - assert "gepa-threads" in capsys.readouterr().err - - @pytest.mark.parametrize("bad_spec", ["PMC1=", "=DIR", "PMC1= "]) def test_agent_local_rejects_empty_mapping_components(bad_spec: str, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: """CodeRabbit: --local PMCid=DIR with a blank PMC id or blank DIR fails loud (exit 2), not Path('.').""" diff --git a/tests/test_agent_coverage.py b/tests/test_agent_coverage.py index d9eb235f..0417148f 100644 --- a/tests/test_agent_coverage.py +++ b/tests/test_agent_coverage.py @@ -43,7 +43,7 @@ def _build_real_redb(root: Path) -> Path: [_synonym_row("HGNC:1100", "BRCA1", ["BRCA1", "brca1"], "Gene"), _synonym_row("HGNC:6871", "MAPK1", ["MAPK1", "mapk1"], "Gene")], ) output: Path = root / "data" / "fullmap.redb" - rs.build_fullmap_db(output, [classes], [synonyms], threads=2) + rs.build_fullmap_db(output, [classes], [synonyms]) return output diff --git a/tests/test_agent_edgecount.py b/tests/test_agent_edgecount.py index 251b7584..a25556d9 100644 --- a/tests/test_agent_edgecount.py +++ b/tests/test_agent_edgecount.py @@ -113,7 +113,7 @@ def _build_real_redb(root: Path) -> Path: classes_path: Path = _write_jsonl(root / "classes.ndjson", classes) synonyms_path: Path = _write_jsonl(root / "synonyms.ndjson", synonyms) output: Path = root / "data" / "fullmap.redb" - rs.build_fullmap_db(output, [classes_path], [synonyms_path], threads=2) + rs.build_fullmap_db(output, [classes_path], [synonyms_path]) return output diff --git a/tests/test_agent_eval.py b/tests/test_agent_eval.py index 1c830c6b..da9e1613 100644 --- a/tests/test_agent_eval.py +++ b/tests/test_agent_eval.py @@ -457,28 +457,6 @@ def compile(self, program: object, *, trainset: object = None, **kwargs: object) assert configured[-1] is refl -def test_run_gepa_forwards_num_threads() -> None: - """num_threads is forwarded to GEPA only when set.""" - pytest.importorskip("dspy") - created: dict[str, Any] = {} - - class StubGEPA: - def __init__(self, metric: object = None, **kwargs: Any) -> None: - created["kwargs"] = kwargs - self.gepa_stats: dict[str, object] = {} - - def compile(self, program: object, *, trainset: object = None, **kwargs: object) -> object: - predictor = SimpleNamespace(signature=SimpleNamespace(instructions="OPT")) - return SimpleNamespace(named_predictors=lambda: [("propose", predictor)]) - - run_gepa(seed_instructions="SEED", gepa_cls=StubGEPA, reflection_lm=SimpleNamespace(), trainset=[], num_threads=4) - assert created["kwargs"]["num_threads"] == 4 - - created.clear() - run_gepa(seed_instructions="SEED", gepa_cls=StubGEPA, reflection_lm=SimpleNamespace(), trainset=[]) - assert "num_threads" not in created["kwargs"] - - # --------------------------------------------------------------------------- # # Offline integration: build the reference KGX from the fixture + score F1 # --------------------------------------------------------------------------- # @@ -520,7 +498,7 @@ def class_row(curie: str) -> dict[str, Any]: classes_path.write_text("\n".join(json.dumps(r) for r in classes) + "\n") synonyms_path.write_text("\n".join(json.dumps(r) for r in synonyms) + "\n") output = root / "data" / "fullmap.redb" - rs.build_fullmap_db(output, [classes_path], [synonyms_path], threads=2) + rs.build_fullmap_db(output, [classes_path], [synonyms_path]) return output diff --git a/tests/test_agent_multisection.py b/tests/test_agent_multisection.py index e23d5cc0..2d6aa74c 100644 --- a/tests/test_agent_multisection.py +++ b/tests/test_agent_multisection.py @@ -55,7 +55,7 @@ def redb(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: [_synonym_row("HGNC:1100", "BRCA1", ["BRCA1", "brca1"], "Gene"), _synonym_row("HGNC:6871", "MAPK1", ["MAPK1", "mapk1"], "Gene")], ) output: Path = root / "data" / "fullmap.redb" - rs.build_fullmap_db(output, [classes], [synonyms], threads=2) + rs.build_fullmap_db(output, [classes], [synonyms]) return output diff --git a/tests/test_agent_speed.py b/tests/test_agent_speed.py index 1d2d7976..36e4b457 100644 --- a/tests/test_agent_speed.py +++ b/tests/test_agent_speed.py @@ -502,7 +502,7 @@ def test_build_and_audit_tool_memoizes_identical_config(tmp_path: Path, monkeypa + "\n" ) output: Path = root / "data" / "fullmap.redb" - rs.build_fullmap_db(output, [classes], [synonyms], threads=2) + rs.build_fullmap_db(output, [classes], [synonyms]) calls: list[str] = [] real_build_and_audit = build_and_audit diff --git a/tests/test_agent_storage.py b/tests/test_agent_storage.py index d7a60d4b..ea9e07df 100644 --- a/tests/test_agent_storage.py +++ b/tests/test_agent_storage.py @@ -155,7 +155,7 @@ def fullmap_db(tmp_path: Path) -> Path: ], ) output: Path = root / "data" / "fullmap.redb" - rs.build_fullmap_db(output, [classes], [synonyms], threads=2) + rs.build_fullmap_db(output, [classes], [synonyms]) return output diff --git a/tests/test_agent_supervisor.py b/tests/test_agent_supervisor.py index 3a3b3706..5f621e0c 100644 --- a/tests/test_agent_supervisor.py +++ b/tests/test_agent_supervisor.py @@ -79,7 +79,7 @@ def fullmap_db(tmp_path: Path) -> Path: [_synonym_row("HGNC:1100", "BRCA1", ["BRCA1", "brca1"], "Gene"), _synonym_row("HGNC:6871", "MAPK1", ["MAPK1", "mapk1"], "Gene")], ) output: Path = root / "data" / "fullmap.redb" - rs.build_fullmap_db(output, [classes], [synonyms], threads=2) + rs.build_fullmap_db(output, [classes], [synonyms]) return output diff --git a/tests/test_agent_workflow_metrics.py b/tests/test_agent_workflow_metrics.py index c88402e2..3bcd31b3 100644 --- a/tests/test_agent_workflow_metrics.py +++ b/tests/test_agent_workflow_metrics.py @@ -125,7 +125,7 @@ def fullmap_db(tmp_path: Path) -> Path: [_synonym_row("HGNC:1100", "BRCA1", ["BRCA1", "brca1"], "Gene"), _synonym_row("HGNC:6871", "MAPK1", ["MAPK1", "mapk1"], "Gene")], ) output: Path = root / "data" / "fullmap.redb" - rs.build_fullmap_db(output, [classes], [synonyms], threads=2) + rs.build_fullmap_db(output, [classes], [synonyms]) return output @@ -565,7 +565,7 @@ def _build_golden_redb(root: Path) -> Path: classes_path: Path = _write_jsonl(root / "classes.ndjson", classes) synonyms_path: Path = _write_jsonl(root / "synonyms.ndjson", synonyms) output: Path = root / "data" / "fullmap.redb" - rs.build_fullmap_db(output, [classes_path], [synonyms_path], threads=2) + rs.build_fullmap_db(output, [classes_path], [synonyms_path]) return output diff --git a/tests/test_cover_agent_core.py b/tests/test_cover_agent_core.py index d3eb77a3..635abeec 100644 --- a/tests/test_cover_agent_core.py +++ b/tests/test_cover_agent_core.py @@ -196,7 +196,7 @@ def _build_real_redb(root: Path) -> Path: [_synonym_row("HGNC:1100", "BRCA1", ["BRCA1", "brca1"], "Gene"), _synonym_row("HGNC:6871", "MAPK1", ["MAPK1", "mapk1"], "Gene")], ) output: Path = root / "data" / "fullmap.redb" - rs.build_fullmap_db(output, [classes], [synonyms], threads=2) + rs.build_fullmap_db(output, [classes], [synonyms]) return output diff --git a/tests/test_cover_agent_propose.py b/tests/test_cover_agent_propose.py index 59d9ff5d..872029d0 100644 --- a/tests/test_cover_agent_propose.py +++ b/tests/test_cover_agent_propose.py @@ -81,7 +81,7 @@ def fullmap_db(tmp_path: Path) -> Path: [_synonym_row("HGNC:1100", "BRCA1", ["BRCA1", "brca1"], "Gene"), _synonym_row("HGNC:6871", "MAPK1", ["MAPK1", "mapk1"], "Gene")], ) output: Path = root / "data" / "fullmap.redb" - rs.build_fullmap_db(output, [classes], [synonyms], threads=2) + rs.build_fullmap_db(output, [classes], [synonyms]) return output diff --git a/tests/test_cover_cli.py b/tests/test_cover_cli.py index 6b18676d..dd3e65ef 100644 --- a/tests/test_cover_cli.py +++ b/tests/test_cover_cli.py @@ -22,7 +22,7 @@ from urllib.error import HTTPError, URLError import pytest -from cyclopts.exceptions import CoercionError, UnknownOptionError # pyright: ignore[reportMissingImports] +from cyclopts.exceptions import UnknownOptionError # pyright: ignore[reportMissingImports] from tablassert import cli, extras, rs from tablassert.cli import build_fullmap_pipeline, build_kg, download_babel_file, download_babel_file_aria2c, validate_graph_pipeline @@ -358,50 +358,6 @@ def _fake_run(command: list[str], **kwargs: Any) -> subprocess.CompletedProcess[ assert control.read_bytes() == b"resume-state" # failure path preserves aria2 resume metadata -def test_build_kg_command_delegates_to_run(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - """Cover cli.py:501 — the ``build-kg`` cyclopts command forwards to ``run(7, build_pipeline, ...)``. - - ``cli.run`` is stubbed to a recorder so the command body executes (line 501) without a real - multi-hour build. Asserts the stage count (7 with ``--qc``: the study stage over the final - NDJSON is appended), pipeline function, config path, and every flag — including the fullmap - lookup ``threads`` — are threaded through unchanged. - """ - config: Path = tmp_path / "graph.yaml" - calls: list[tuple[Any, ...]] = [] - - def _fake_run(stages: int, fn: Any, arg: Path, **kwargs: Any) -> None: - calls.append((stages, fn, arg, kwargs)) - - monkeypatch.setattr(cli, "run", _fake_run) - monkeypatch.setattr(extras, "missing", lambda extra: ()) - build_kg(config, release=True, qc=True, log=True, head=True, no_original=True, threads=8) - assert calls == [(7, cli.build_pipeline, config, {"release": True, "qc": True, "log": True, "head": True, "no_original": True, "threads": 8})] - calls.clear() - build_kg(config) - assert calls == [ - (6, cli.build_pipeline, config, {"release": False, "qc": False, "log": False, "head": False, "no_original": False, "threads": None}) - ] - - -@pytest.mark.parametrize("bad_threads", [0, -1, -8]) -def test_build_kg_non_positive_threads_exits_2( - bad_threads: int, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] -) -> None: - """A non-positive ``--threads`` fails loud (exit 2) before any build work starts. - - Mirrors the ``--gepa-threads`` gate: the invalid count would otherwise surface only deep - inside the Rust lookup, after table loading had already begun. - """ - config: Path = tmp_path / "graph.yaml" - monkeypatch.setattr(cli, "run", lambda *args, **kwargs: pytest.fail("the build started with an invalid --threads")) - - with pytest.raises(SystemExit) as exc_info: - build_kg(config, threads=bad_threads) - - assert exc_info.value.code == 2 - assert "--threads" in capsys.readouterr().err - - def test_build_kg_qc_without_the_extra_stops_before_the_build(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: """``--qc`` without the ``[qc]`` extra fails immediately, naming the install command. @@ -452,8 +408,8 @@ def _fake_run(stages: int, fn: Any, arg: Path, **kwargs: Any) -> None: monkeypatch.setattr(cli, "run", _fake_run) monkeypatch.setattr(extras, "missing", lambda extra: ()) - cli.build_fullmap(output=output, cache=cache, version="v", threads=2, aria2c=True, force=True) - assert calls == [(3, cli.build_fullmap_pipeline, output, {"cache": cache, "version": "v", "threads": 2, "aria2c": True})] + cli.build_fullmap(output=output, cache=cache, version="v", aria2c=True, force=True) + assert calls == [(3, cli.build_fullmap_pipeline, output, {"cache": cache, "version": "v", "aria2c": True})] def test_build_fullmap_aria2c_without_the_extra_stops_before_downloading( @@ -554,9 +510,9 @@ def parse(argv: list[str]) -> dict[str, Any]: for removed in (["--table-config"], ["--fullmap", str(config)]): with pytest.raises(UnknownOptionError): parse(["build-kg", str(config), *removed]) - # The removed ``-tc`` is likewise unusable: now that ``-t`` is the threads alias, cyclopts - # reads the cluster as ``-t c`` and rejects the non-integer value instead of the option. - with pytest.raises(CoercionError): + # The removed ``-tc`` is likewise unusable: with the threads option (and its ``-t`` alias) + # gone, cyclopts rejects the cluster's leading ``-t`` as an unknown option. + with pytest.raises(UnknownOptionError): parse(["build-kg", str(config), "-tc"]) # --no-original / -no bind the no_original flag. assert parse(["build-kg", str(config), "--no-original"])["no_original"] is True @@ -595,22 +551,22 @@ def _recording_detail(downloaded: int, total: int) -> str: built: list[tuple[Any, ...]] = [] - def _fake_build(output: Path, class_files: list[Path], synonym_files: list[Path], threads: int | None = None, progress: Any = None) -> None: - built.append((output, class_files, synonym_files, threads)) + def _fake_build(output: Path, class_files: list[Path], synonym_files: list[Path], progress: Any = None) -> None: + built.append((output, class_files, synonym_files)) monkeypatch.setattr(rs, "build_fullmap_db", _fake_build) output: Path = tmp_path / "fullmap.redb" cache: Path = tmp_path / "downloads" - build_fullmap_pipeline(output, PipelineProgress(total_stages=3), cache=cache, version="v", threads=1) + build_fullmap_pipeline(output, PipelineProgress(total_stages=3), cache=cache, version="v") # Line 653 fired for each downloaded file (one chunk each), ending at the full payload size. assert detail_calls == [(len(payload), len(payload)), (len(payload), len(payload))] # The real downloader spooled both files to their class/synonym cache dirs. assert (cache / "classes" / "c.gz").read_bytes() == payload assert (cache / "synonyms" / "s.gz").read_bytes() == payload - # Stage 3 received the downloaded paths and the thread count. - assert built == [(output, [cache / "classes" / "c.gz"], [cache / "synonyms" / "s.gz"], 1)] + # Stage 3 received the downloaded paths. + assert built == [(output, [cache / "classes" / "c.gz"], [cache / "synonyms" / "s.gz"])] def test_build_fullmap_pipeline_uses_aria2c_when_opted_in(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: @@ -639,8 +595,8 @@ def _fake_aria2c(filename: str, url: str, destination: Path, retries: int = 5) - monkeypatch.setattr(cli, "download_babel_file_aria2c", _fake_aria2c) built: list[tuple[Any, ...]] = [] - def _fake_build(output: Path, class_files: list[Path], synonym_files: list[Path], threads: int | None = None, progress: Any = None) -> None: - built.append((output, class_files, synonym_files, threads)) + def _fake_build(output: Path, class_files: list[Path], synonym_files: list[Path], progress: Any = None) -> None: + built.append((output, class_files, synonym_files)) monkeypatch.setattr(rs, "build_fullmap_db", _fake_build) @@ -673,12 +629,12 @@ def end_section_task(self) -> None: progress = _RecordingProgress() output: Path = tmp_path / "fullmap.redb" cache: Path = tmp_path / "downloads" - build_fullmap_pipeline(output, progress, cache=cache, version="v", threads=1, aria2c=True) # type: ignore[arg-type] + build_fullmap_pipeline(output, progress, cache=cache, version="v", aria2c=True) # type: ignore[arg-type] assert aria_calls == [("c.gz", "https://example.com/c.gz", cache / "classes"), ("s.gz", "https://example.com/s.gz", cache / "synonyms")] assert progress.sub_steps.count("aria2c downloading") == 2 assert progress.advances == 4 # two discovery entries + two downloaded files - assert built == [(output, [cache / "classes" / "c.gz"], [cache / "synonyms" / "s.gz"], 1)] + assert built == [(output, [cache / "classes" / "c.gz"], [cache / "synonyms" / "s.gz"])] # --- prebuilt fullmap downloader (download-first default; --force rebuilds from BABEL) --- @@ -1008,11 +964,11 @@ def _fake_run(stages: int, fn: Any, arg: Path, **kwargs: Any) -> None: monkeypatch.setattr(extras, "missing", lambda extra: ()) # --aria2c preflight: report [aria2] as installed output: Path = tmp_path / "fullmap.redb" # absent cache: Path = tmp_path / "c" - cli.build_fullmap(output=output, cache=cache, version="v", threads=4, aria2c=True) + cli.build_fullmap(output=output, cache=cache, version="v", aria2c=True) assert len(calls) == 2 assert calls[0][0] == 2 assert calls[0][1] is cli.fetch_prebuilt_fullmap - assert calls[1] == (3, cli.build_fullmap_pipeline, output, {"cache": cache, "version": "v", "threads": 4, "aria2c": True}) + assert calls[1] == (3, cli.build_fullmap_pipeline, output, {"cache": cache, "version": "v", "aria2c": True}) def test_build_fullmap_force_flag_parses() -> None: @@ -1076,7 +1032,7 @@ def _build_real_force_fullmap(directory: Path) -> Path: _write_gzip_ndjson(directory / "synonyms" / "MONDO.ndjson.gz", _REAL_SYNONYM_LINES_MONDO), ] output: Path = directory / "force" / "fullmap.redb" - rs.build_fullmap_db(output, classes, synonyms, threads=2) + rs.build_fullmap_db(output, classes, synonyms) return output diff --git a/tests/test_cover_fullmap.py b/tests/test_cover_fullmap.py index 72b6b50c..ad6669f2 100644 --- a/tests/test_cover_fullmap.py +++ b/tests/test_cover_fullmap.py @@ -58,7 +58,7 @@ def fullmap_db(tmp_path: Path) -> Path: classes: Path = write_jsonl(tmp_path / "classes.ndjson", [class_row("HGNC:1100", ["NCBIGene:672"])]) synonyms: Path = write_jsonl(tmp_path / "HGNC.ndjson", [synonym_row("HGNC:1100", "BRCA1", ["BRCA1", "breast cancer 1"], "Gene")]) output: Path = tmp_path / "data" / "fullmap.redb" - rs.build_fullmap_db(output, [classes], [synonyms], threads=1) + rs.build_fullmap_db(output, [classes], [synonyms]) return output diff --git a/tests/test_e2e_smoke.py b/tests/test_e2e_smoke.py index 01fc9eac..3c1df737 100644 --- a/tests/test_e2e_smoke.py +++ b/tests/test_e2e_smoke.py @@ -48,7 +48,7 @@ def _build_real_redb(root: Path) -> Path: [_synonym_row("HGNC:1100", "BRCA1", ["BRCA1", "brca1"], "Gene"), _synonym_row("HGNC:6871", "MAPK1", ["MAPK1", "mapk1"], "Gene")], ) output: Path = root / "data" / "fullmap.redb" - rs.build_fullmap_db(output, [classes], [synonyms], threads=2) + rs.build_fullmap_db(output, [classes], [synonyms]) return output @@ -298,7 +298,7 @@ def _build_context_redb(root: Path) -> Path: ], ) output: Path = root / "data" / "fullmap.redb" - rs.build_fullmap_db(output, [classes], [synonyms], threads=2) + rs.build_fullmap_db(output, [classes], [synonyms]) return output diff --git a/tests/test_fullmap.py b/tests/test_fullmap.py index 338445ac..c09b06c5 100644 --- a/tests/test_fullmap.py +++ b/tests/test_fullmap.py @@ -69,7 +69,7 @@ def fullmap_db(tmp_path: Path) -> Path: ], ) output: Path = tmp_path / "data" / "fullmap.redb" - rs.build_fullmap_db(output, [classes], [synonyms], threads=2) + rs.build_fullmap_db(output, [classes], [synonyms]) return output @@ -580,9 +580,9 @@ def test_resolve_batch_makes_one_redb_call_regardless_of_spec_count(fullmap_db: calls: list[list[str]] = [] original = rs.lookup_fullmap_terms - def counting_lookup(db: Path, terms: list[str], threads: Any = None) -> list[dict[str, Any]]: + def counting_lookup(db: Path, terms: list[str]) -> list[dict[str, Any]]: calls.append(list(terms)) - return original(db, terms, threads=threads) + return original(db, terms) monkeypatch.setattr(rs, "lookup_fullmap_terms", counting_lookup) @@ -659,13 +659,6 @@ def test_resolve_batch_three_node_columns_on_sharded_db(fullmap_db: Path) -> Non assert result["disease_context_qualifier_taxon"] == "NCBITaxon:9606" -def test_lookup_threads_match(fullmap_db: Path) -> None: - """rust lookup is deterministic with one or more threads.""" - single: list[dict[str, Any]] = rs.lookup_fullmap_terms(fullmap_db, ["brca1", "mapk1"], threads=1) - multi: list[dict[str, Any]] = rs.lookup_fullmap_terms(fullmap_db, ["brca1", "mapk1"], threads=2) - assert single == multi - - def test_fullmap_db_path_variants(tmp_path: Path, fullmap_db: Path) -> None: """fullmap base path helper supports file, direct base, and data/fullmap.redb.""" direct: Path = tmp_path / "fullmap.redb" @@ -694,7 +687,7 @@ def test_term_cache_invalidates_across_rebuild(tmp_path: Path) -> None: # v1: "brca1" resolves to HGNC:1100 and warms the term cache. synonyms_v1: Path = write_jsonl(tmp_path / "v1.ndjson", [synonym_row("HGNC:1100", "BRCA1", ["brca1"], "Gene")]) - rs.build_fullmap_db(output, [classes], [synonyms_v1], threads=2) + rs.build_fullmap_db(output, [classes], [synonyms_v1]) first: list[dict[str, object]] = lookup_rows(output, ["brca1"]) assert first[0]["CURIE"] == "HGNC:1100" assert any(term == "brca1" for _path, _mtime, term in _TERM_CACHE) @@ -702,7 +695,7 @@ def test_term_cache_invalidates_across_rebuild(tmp_path: Path) -> None: # v2: rebuild at the SAME path with different content ("brca1" -> HGNC:2222), # then guarantee a distinct mtime so the cache key changes deterministically. synonyms_v2: Path = write_jsonl(tmp_path / "v2.ndjson", [synonym_row("HGNC:2222", "BRCA1", ["brca1"], "Gene")]) - rs.build_fullmap_db(output, [classes], [synonyms_v2], threads=2) + rs.build_fullmap_db(output, [classes], [synonyms_v2]) bumped: float = output.stat().st_mtime + 10.0 os.utime(output, (bumped, bumped)) @@ -721,7 +714,7 @@ def test_lookup_rows_propagates_unrelated_typeerror(fullmap_db: Path, monkeypatc """ _TERM_CACHE.clear() - def boom(db: Path, terms: list[str], threads: Any = None, return_format: str = "rows") -> list[dict[str, Any]]: + def boom(db: Path, terms: list[str], return_format: str = "rows") -> list[dict[str, Any]]: raise TypeError("internal rust panic: null pointer") monkeypatch.setattr(rs, "lookup_fullmap_terms", boom) @@ -745,11 +738,11 @@ def test_lookup_rows_legacy_signature_typeerror_requeries_full_term_set(fullmap_ calls: list[tuple[list[str], str]] = [] original = rs.lookup_fullmap_terms - def fake(db: Path, terms: list[str], threads: Any = None, return_format: str = "rows") -> list[dict[str, Any]]: + def fake(db: Path, terms: list[str], return_format: str = "rows") -> list[dict[str, Any]]: calls.append((list(terms), return_format)) if return_format == "pairs": raise TypeError("lookup_fullmap_terms() got an unexpected keyword argument 'return_format'") - return original(db, terms, threads=threads) + return original(db, terms) monkeypatch.setattr(rs, "lookup_fullmap_terms", fake) rows: list[dict[str, object]] = lookup_rows(fullmap_db, ["brca1", "mapk1"]) @@ -776,12 +769,12 @@ def test_lookup_rows_legacy_shape_requeries_full_term_set(fullmap_db: Path, monk calls: list[tuple[list[str], str]] = [] original = rs.lookup_fullmap_terms - def fake(db: Path, terms: list[str], threads: Any = None, return_format: str = "rows") -> list[dict[str, Any]]: + def fake(db: Path, terms: list[str], return_format: str = "rows") -> list[dict[str, Any]]: calls.append((list(terms), return_format)) if return_format == "pairs": # Legacy shape: rows carry no "records" key and cover only the queried misses. return [{"term": term, "CURIE": "X:1"} for term in terms] - return original(db, terms, threads=threads) + return original(db, terms) monkeypatch.setattr(rs, "lookup_fullmap_terms", fake) rows: list[dict[str, object]] = lookup_rows(fullmap_db, ["brca1", "mapk1"]) @@ -812,10 +805,10 @@ def test_lookup_rows_legacy_fallback_warns_once(fullmap_db: Path, monkeypatch: p original = rs.lookup_fullmap_terms - def fake(db: Path, terms: list[str], threads: Any = None, return_format: str = "rows") -> list[dict[str, Any]]: + def fake(db: Path, terms: list[str], return_format: str = "rows") -> list[dict[str, Any]]: if return_format == "pairs": return [{"term": term, "CURIE": "X:1"} for term in terms] # legacy shape: no "records" - return original(db, terms, threads=threads) + return original(db, terms) monkeypatch.setattr(rs, "lookup_fullmap_terms", fake) @@ -855,10 +848,10 @@ def fake_download_babel_file( monkeypatch.setattr(cli, "download_babel_file", fake_download_babel_file) monkeypatch.chdir(tmp_path) - build_fullmap(output=output, version="test-version", threads=1, force=True) + build_fullmap(output=output, version="test-version", force=True) assert downloaded_paths == [Path("fullmap/downloads/classes/classes.ndjson"), Path("fullmap/downloads/synonyms/HGNC.ndjson")] - rows: list[dict[str, Any]] = rs.lookup_fullmap_terms(output, ["brca1"], threads=1) + rows: list[dict[str, Any]] = rs.lookup_fullmap_terms(output, ["brca1"]) assert rows[0]["CURIE"] == "HGNC:1100" diff --git a/tests/test_fullmap_golden.py b/tests/test_fullmap_golden.py index 040c3c3c..a5d400b6 100644 --- a/tests/test_fullmap_golden.py +++ b/tests/test_fullmap_golden.py @@ -229,13 +229,13 @@ def golden_db(tmp_path: Path) -> Path: classes: Path = write_jsonl(tmp_path / "classes.ndjson", CLASSES) synonyms: Path = write_jsonl(tmp_path / "SRC.ndjson", SYNONYMS) output: Path = tmp_path / "fullmap.redb" - rs.build_fullmap_db(output, [classes], [synonyms], threads=1) + rs.build_fullmap_db(output, [classes], [synonyms]) return output def test_golden_lookup_is_pinned(golden_db: Path) -> None: """Looking up every indexed term yields exactly the pinned hydrated rows.""" - rows: list[dict[str, Any]] = rs.lookup_fullmap_terms(golden_db, PROBES, threads=1, return_format="rows") + rows: list[dict[str, Any]] = rs.lookup_fullmap_terms(golden_db, PROBES, return_format="rows") # SOURCE_VERSION is a compile-time constant on every row (asserted once here, # not repeated inline in the golden). @@ -252,7 +252,7 @@ def test_pairs_format_matches_golden_curies(golden_db: Path) -> None: """return_format='pairs' yields (curie_id, source_id) records whose hydrated CURIE strings match the golden term -> CURIE mapping (raw curie_ids are scheduling-dependent and deliberately not pinned).""" - pair_rows: list[dict[str, Any]] = rs.lookup_fullmap_terms(golden_db, PROBES, threads=1, return_format="pairs") + pair_rows: list[dict[str, Any]] = rs.lookup_fullmap_terms(golden_db, PROBES, return_format="pairs") prefixes: list[str] = list(rs.hydrate_prefixes(golden_db)) expected: dict[str, list[str]] = _expected_term_curies() @@ -271,7 +271,7 @@ def test_pairs_format_matches_golden_curies(golden_db: Path) -> None: def test_hydration_round_trip_is_consistent(golden_db: Path) -> None: """Every curie_id from the pairs hydrates to a complete, consistent CURIE row.""" - pair_rows: list[dict[str, Any]] = rs.lookup_fullmap_terms(golden_db, PROBES, threads=1, return_format="pairs") + pair_rows: list[dict[str, Any]] = rs.lookup_fullmap_terms(golden_db, PROBES, return_format="pairs") curie_ids: list[int] = sorted({int(a) for row in pair_rows for a, _b in row["records"]}) prefixes: list[str] = list(rs.hydrate_prefixes(golden_db)) @@ -328,16 +328,16 @@ def test_rebuild_stability(tmp_path: Path) -> None: synonyms: Path = write_jsonl(tmp_path / "SRC.ndjson", SYNONYMS) output: Path = tmp_path / "fullmap.redb" - rs.build_fullmap_db(output, [classes], [synonyms], threads=1) - first: list[dict[str, object]] = _canonical(rs.lookup_fullmap_terms(output, PROBES, threads=1, return_format="rows")) + rs.build_fullmap_db(output, [classes], [synonyms]) + first: list[dict[str, object]] = _canonical(rs.lookup_fullmap_terms(output, PROBES, return_format="rows")) assert first == GOLDEN_ROWS # Rebuild identical content at the SAME path; bump mtime so the Python-side # term/dimension caches (keyed on path+mtime) invalidate deterministically. - rs.build_fullmap_db(output, [classes], [synonyms], threads=1) + rs.build_fullmap_db(output, [classes], [synonyms]) bumped: float = output.stat().st_mtime + 10.0 os.utime(output, (bumped, bumped)) - second: list[dict[str, object]] = _canonical(rs.lookup_fullmap_terms(output, PROBES, threads=1, return_format="rows")) + second: list[dict[str, object]] = _canonical(rs.lookup_fullmap_terms(output, PROBES, return_format="rows")) assert second == GOLDEN_ROWS assert first == second diff --git a/tests/test_lib.py b/tests/test_lib.py index 081037bd..1547918e 100644 --- a/tests/test_lib.py +++ b/tests/test_lib.py @@ -77,8 +77,8 @@ def install_fake_fullmap(monkeypatch: Any, rows: dict[str, list[dict[str, object """Monkeypatch fullmap lookup and return captured term batches.""" calls: list[list[str]] = [] - def fake_lookup(db: Path, terms: list[str], threads: int | None = None, return_format: str = "rows") -> list[dict[str, object]]: - del db, threads, return_format + def fake_lookup(db: Path, terms: list[str], return_format: str = "rows") -> list[dict[str, object]]: + del db, return_format calls.append(terms) return [row for term in terms for row in rows.get(term, [])] @@ -437,59 +437,6 @@ def test_tcode_collect_audits_follow_single_resolve_batch_with_qualifiers(fixtur assert all(i > batch_idx for i, _ in audit_ops) -def test_tcode_collect_threads_nullable_into_resolve_specs(fixtures_path: Path) -> None: - """A nullable qualifier's ResolveSpec carries nullable=True; subject/object stay strict.""" - data: Any = from_yaml(fixtures_path / "minimal_section.yaml") - store: Path = Path("/tmp/sectionhash_nullable_spec.parquet") - data["statement"]["qualifiers"] = [ - {"qualifier": "disease_context_qualifier", "method": "column", "encoding": "C", "nullable": True}, - {"qualifier": "anatomical_context_qualifier", "method": "column", "encoding": "D"}, - ] - - tcode_model: Tcode = Tcode.model_validate( # pyright: ignore - {**data, "config": fixtures_path / "minimal_section.yaml", "store": store} - ) - - collected: list[tuple[Any, tuple[Any]]] = tcode_model.collect(Path("/tmp/fullmap.redb")) # pyright: ignore - batch_ops: list[tuple[Any, tuple[Any]]] = [op for op in collected if op[0].__name__ == "resolve_batch"] - specs: list[ResolveSpec] = batch_ops[0][1][0] - by_col: dict[str, ResolveSpec] = {spec.col: spec for spec in specs} - - assert by_col["subject"].nullable is False - assert by_col["object"].nullable is False - assert by_col["disease_context_qualifier"].nullable is True - assert by_col["anatomical_context_qualifier"].nullable is False - - -def test_tcode_collect_threads_reach_resolve_batch(fixtures_path: Path) -> None: - """``Tcode.threads`` rides the resolve_batch op args; the tag defaults to ``"_two"``. - - ``compile_subgraph`` applies op args positionally, so the resolve_batch op spells the - tag explicitly to reach ``threads`` (positionals: specs, db, log, section_hash, - config_file, column_context, tag, threads). Unset threads keeps the Rust auto behavior. - """ - data: Any = from_yaml(fixtures_path / "minimal_section.yaml") - store: Path = Path("/tmp/sectionhash_threads.parquet") - - tcode_model: Tcode = Tcode.model_validate( # pyright: ignore - {**data, "config": fixtures_path / "minimal_section.yaml", "store": store, "threads": 8} - ) - collected: list[tuple[Any, tuple[Any]]] = tcode_model.collect(Path("/tmp/fullmap.redb")) # pyright: ignore - batch_ops: list[tuple[Any, tuple[Any]]] = [op for op in collected if op[0].__name__ == "resolve_batch"] - assert len(batch_ops) == 1 - args: tuple[Any, ...] = tuple(batch_ops[0][1]) - assert args[6] == "_two" - assert args[7] == 8 - - default_model: Tcode = Tcode.model_validate( # pyright: ignore - {**data, "config": fixtures_path / "minimal_section.yaml", "store": Path("/tmp/sectionhash_threads_default.parquet")} - ) - default_ops: list[tuple[Any, tuple[Any]]] = [op for op in default_model.collect(Path("/tmp/fullmap.redb")) if op[0].__name__ == "resolve_batch"] # pyright: ignore - default_args: tuple[Any, ...] = tuple(default_ops[0][1]) - assert default_args[6] == "_two" - assert default_args[7] is None - - def test_tcode_collect_excludes_nullable_qualifier_from_audit(fixtures_path: Path) -> None: """QC audit skips a nullable qualifier column (its nulls are expected, not resolution errors).""" data: Any = from_yaml(fixtures_path / "minimal_section.yaml") diff --git a/tests/test_production_patterns.py b/tests/test_production_patterns.py index 5bc96f2d..c62ffeb6 100644 --- a/tests/test_production_patterns.py +++ b/tests/test_production_patterns.py @@ -105,7 +105,7 @@ def _build_rich_redb(root: Path) -> Path: ], ) output: Path = root / "data" / "fullmap.redb" - rs.build_fullmap_db(output, [classes], [synonyms], threads=2) + rs.build_fullmap_db(output, [classes], [synonyms]) return output