perf(rust): make the uuid_on_collision merge pass near-linear

CHANGELOG.md

@@ -7,6 +7,9 @@ All notable changes to this project are documented in this file.
 ### Fixed
 - **`number_of_cases` is now emitted as a JSON integer instead of source text.** The slot is typed `int` on biolink-model's `EntityToDiseaseAssociation` / `EntityToPhenotypicFeatureAssociation`, but it was absent from `lib.numeric_columns`' exact set — since 15.1's `STUDY_SIZE_EXEMPT_PATTERN` (#119) stopped it being *renamed* onto `study_size`, nothing cast its *values*, so the raw TSV cell (`"1"`) shipped on the edge NDJSON. Pydantic's lax validation coerced the string back to int inside `validate_kgx`, so the record still validated, and the Rust `uuid_on_collision: merge` recompute already wrote a real int — leaving the shipped graph type-inconsistent edge to edge. `number_of_cases` now rides the same `clean_numeric` / `format_numeric` machinery as `study_size`: `numeric_slot_kind` reads the `int` typing off the installed model, fractional / negative / non-numeric counts become null, and the release-mode `drop_low_number_of_cases` filter is untouched (it already cast inline).
 
+### Performance
+- **The `uuid_on_collision: merge` edge pass is now near-linear in divergent rows per edge instead of quadratic, and its canonical serializer no longer clones per record.** The merge fold re-canonicalized and re-sorted every list item already stored on an edge on each incoming row — O(n²) in rows × `supporting_case_ids` — and `canonical_json_bytes` deep-cloned into a sorted-key `Value` tree before writing, a fixed cost paid once per record on every pass. Array fields now carry their items' canonical bytes, computed once when an item arrives, so each fold is O(1) per incoming item (hash-set membership, one deferred stable sort at write-out, and only for lists a real union touched), while the sorted-key form is written directly into the output buffer with scalars delegated to `serde_json`'s own writers. Measured on release builds (`uv run maturin develop --release`) over the seeded corpora (seed 42) of the committed harness — `TABLASSERT_BENCH=1 uv run pytest tests/bench_merge_bench.py -s -n 0 -q` — on an AMD Ryzen AI 9 HX 370 with Python 3.13: **37.45 s → 1.311 s (28.6x)** for 100,000 lines of 1k triples × 100 divergent rows at 50 case ids per row, **20.69 s → 2.155 s (9.6x)** for 100,000 lines of 5k triples × 20 rows at 100 case ids per row, **6.71 s → 5.184 s (1.29x)** for 600,000 lines of 200k triples × 3 rows at 10–20 case ids per row, and **0.59 s → 0.495 s (1.19x)** for 100,000 byte-identical rows, so the no-fold path does not pay for the new bookkeeping either. Merged-record and conflict counters are unchanged on every scenario (400,000 / 1,199,899; 95,000 / 284,979; 99,000 / 296,969), and the emitted bytes are identical to the previous fold — pinned by a seeded fuzz against a frozen verbatim copy of the old implementation (`cargo test merge_fold_matches_reference_on_fuzz`), an end-to-end pure-Python reference (`tests/test_rs.py::test_dedup_edges_merge_matches_python_reference`), and an in-process guard that fails below a 5x bound against that frozen quadratic reference (`cargo test fold_speedup`). The buffered merge state does carry per-list-item bookkeeping memory — one canonical-bytes copy per distinct list item, a single allocation shared between the membership set and the ordered write-out list — which is the price of the O(1) membership check and one more reason the mode stays opt-in: sharing that allocation rather than storing two copies drops peak RSS on the harness's 1k triples × 100 rows corpus (~5M stored list items) from 1.93 GB to 1.26 GB. ([#143](https://github.com/SkyeAv/Tablassert/pull/143))
+
 ## 16.6.1 - 2026-09-04
 
 ### Performance

rust/src/json.rs

@@ -79,30 +79,61 @@ pub fn emitted_json_bytes(value: &Value) -> serde_json::Result<Vec<u8>> {
 /// different bytes.  The edge deduper compares records for equality, so it needs a form
 /// where "same content" means "same bytes"; array order is preserved because it is
 /// semantic.
+///
+/// US-003: the sorted-key form is written DIRECTLY into the output buffer instead of
+/// first deep-cloning into a canonical `Value` tree -- this runs once per record on
+/// every pass, so the clone was the fixed per-record cost.  Scalars are delegated to
+/// serde_json's own writers, keeping escaping and number formatting byte-identical to
+/// the old clone-then-`to_vec` form (pinned by `canonical_json_bytes_matches_reference_on_fuzz`).
 pub fn canonical_json_bytes(value: &Value) -> serde_json::Result<Vec<u8>> {
-    serde_json::to_vec(&canonical_value(value))
+    let mut buf: Vec<u8> = Vec::with_capacity(128);
+    write_canonical(value, &mut buf)?;
+    Ok(buf)
 }
 
-fn canonical_value(value: &Value) -> Value {
+fn write_canonical(value: &Value, buf: &mut Vec<u8>) -> serde_json::Result<()> {
     match value {
         Value::Object(entries) => {
             let mut keys: Vec<&String> = entries.keys().collect();
             keys.sort_unstable();
-            let mut sorted: Map<String, Value> = Map::with_capacity(entries.len());
+            buf.push(b'{');
+            let mut first: bool = true;
             for key in keys {
-                sorted.insert(key.clone(), canonical_value(&entries[key]));
+                if !first {
+                    buf.push(b',');
+                }
+                first = false;
+                serde_json::to_writer(&mut *buf, key)?;
+                buf.push(b':');
+                write_canonical(entries.get(key).expect("key came from the same map"), buf)?;
             }
-            Value::Object(sorted)
+            buf.push(b'}');
+        }
+        Value::Array(items) => {
+            buf.push(b'[');
+            let mut first: bool = true;
+            for item in items {
+                if !first {
+                    buf.push(b',');
+                }
+                first = false;
+                write_canonical(item, buf)?;
+            }
+            buf.push(b']');
+        }
+        // null / bool / number / string: serde_json's own writer emits exactly the bytes
+        // `to_vec` would for the same scalar (ryu/itoa numbers, full string escaping).
+        Value::Null | Value::Bool(_) | Value::Number(_) | Value::String(_) => {
+            serde_json::to_writer(&mut *buf, value)?
         }
-        Value::Array(items) => Value::Array(items.iter().map(canonical_value).collect()),
-        _ => value.clone(),
     }
+    Ok(())
 }
 
 #[cfg(test)]
 mod tests {
-    use super::strip_nulls;
-    use serde_json::json;
+    use super::{canonical_json_bytes, strip_nulls};
+    use serde_json::{json, Map, Value};
 
     #[test]
     fn strip_nulls_removes_absent_and_null_like_values() {
@@ -153,4 +184,271 @@ mod tests {
             })
         );
     }
+
+    /// Byte-verbatim copy of the pre-US-003 canonical serializer: deep-clone into a
+    /// key-sorted `Value` at every depth, then `to_vec` the clone.  Kept ONLY as the
+    /// equivalence oracle for `canonical_json_bytes_matches_reference_on_fuzz` --
+    /// production code must never call it (the deep clone was the fixed per-record cost
+    /// US-003 removed).
+    fn canonical_json_bytes_reference(value: &Value) -> serde_json::Result<Vec<u8>> {
+        serde_json::to_vec(&canonical_value_reference(value))
+    }
+
+    fn canonical_value_reference(value: &Value) -> Value {
+        match value {
+            Value::Object(entries) => {
+                let mut keys: Vec<&String> = entries.keys().collect();
+                keys.sort_unstable();
+                let mut sorted: Map<String, Value> = Map::with_capacity(entries.len());
+                for key in keys {
+                    sorted.insert(key.clone(), canonical_value_reference(&entries[key]));
+                }
+                Value::Object(sorted)
+            }
+            Value::Array(items) => {
+                Value::Array(items.iter().map(canonical_value_reference).collect())
+            }
+            _ => value.clone(),
+        }
+    }
+
+    /// Tiny deterministic PRNG (splitmix64): the fuzz stream must be reproducible across
+    /// machines without pulling in a `rand` dependency.
+    struct Rng(u64);
+
+    impl Rng {
+        fn next_u64(&mut self) -> u64 {
+            self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15);
+            let mut z: u64 = self.0;
+            z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
+            z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
+            z ^ (z >> 31)
+        }
+
+        fn below(&mut self, bound: usize) -> usize {
+            (self.next_u64() % u64::try_from(bound).expect("bound fits in u64")) as usize
+        }
+
+        fn one_in(&mut self, odds: usize) -> bool {
+            self.below(odds) == 0
+        }
+    }
+
+    fn shuffle<T>(rng: &mut Rng, items: &mut [T]) {
+        for high in (1..items.len()).rev() {
+            let low: usize = rng.below(high + 1);
+            items.swap(high, low);
+        }
+    }
+
+    /// Strings loaded with escape-sensitive content: quotes, backslashes, control chars,
+    /// DEL, NEL/line separators, BMP and non-BMP unicode.
+    const FUZZ_STRINGS: [&str; 14] = [
+        "",
+        "plain",
+        "two words",
+        "quote \" inside",
+        "back\\slash",
+        "\"both\" and \\ together",
+        "tab\tnewline\nreturn\r",
+        "\u{0}\u{1}\u{1f}",
+        "\u{7f}",
+        "\u{85}\u{2028}\u{2029}",
+        "unicode: \u{e9} \u{fc} \u{df} \u{6f22}\u{5b57}",
+        "non-bmp: \u{1d11e} \u{1f600} \u{1f980}",
+        "\u{10ffff}",
+        "trailing backslash\\",
+    ];
+
+    /// Keys chosen to stress the byte-wise key sort: digits-vs-letters, case, punctuation
+    /// above and below letters, spaces, and multi-byte UTF-8.
+    const FUZZ_KEYS: [&str; 12] = [
+        "a", "b", "A", "B", "_", "~", "10", "2", "z", "\u{e9}", "zz", "a b",
+    ];
+
+    /// Shapes the fuzz stream actually covered, so a passing run cannot be vacuous.
+    #[derive(Default)]
+    struct FuzzStats {
+        multi_key_objects: usize,
+        arrays: usize,
+        nested_containers: usize,
+        escaped_strings: usize,
+    }
+
+    fn fuzz_number(rng: &mut Rng) -> Value {
+        match rng.below(4) {
+            0 => Value::Number(rng.next_u64().into()), // u64, incl. above i64::MAX
+            1 => Value::Number((rng.next_u64() as i64).into()), // i64, incl. negatives
+            2 => {
+                let pool: [f64; 10] = [
+                    0.0,
+                    -0.0,
+                    0.1,
+                    -0.1,
+                    1e30,
+                    1.5e-7,
+                    0.3333333333333333,
+                    5e-324,
+                    1.7976931348623157e308,
+                    987654321.5,
+                ];
+                Value::Number(
+                    serde_json::Number::from_f64(pool[rng.below(pool.len())])
+                        .expect("pool values are finite"),
+                )
+            }
+            _ => {
+                // Random finite fraction with exponent in [-8, 8]: exercises ryu's
+                // decimal-vs-exponent formatting boundary and negative signs.
+                let mantissa: f64 = (rng.next_u64() % 1_000_000) as f64;
+                let exponent: i32 = rng.below(17) as i32 - 8;
+                let mut magnitude: f64 = mantissa * 10f64.powi(exponent);
+                if rng.one_in(2) {
+                    magnitude = -magnitude;
+                }
+                Value::Number(serde_json::Number::from_f64(magnitude).expect("finite"))
+            }
+        }
+    }
+
+    fn fuzz_value(rng: &mut Rng, depth: usize, stats: &mut FuzzStats) -> Value {
+        // Beyond depth 4 only scalars: recursion stays bounded while still nesting deep
+        // enough to exercise the recursive sorted-key write.
+        match rng.below(if depth >= 4 { 4 } else { 8 }) {
+            0 => Value::Null,
+            1 => Value::Bool(rng.one_in(2)),
+            2 => fuzz_number(rng),
+            3 => {
+                let text: &str = FUZZ_STRINGS[rng.below(FUZZ_STRINGS.len())];
+                if text.chars().any(|c| c == '"' || c == '\\' || c < ' ') {
+                    stats.escaped_strings += 1;
+                }
+                Value::String(text.to_string())
+            }
+            4 => {
+                stats.arrays += 1;
+                if depth > 0 {
+                    stats.nested_containers += 1;
+                }
+                let mut items: Vec<Value> = Vec::new();
+                for _ in 0..rng.below(5) {
+                    items.push(fuzz_value(rng, depth + 1, stats));
+                }
+                Value::Array(items)
+            }
+            _ => {
+                let mut entries: Map<String, Value> = Map::new();
+                for _ in 0..rng.below(5) {
+                    let key: String = FUZZ_KEYS[rng.below(FUZZ_KEYS.len())].to_string();
+                    entries.insert(key, fuzz_value(rng, depth + 1, stats));
+                }
+                if entries.len() >= 2 {
+                    stats.multi_key_objects += 1;
+                }
+                if depth > 0 {
+                    stats.nested_containers += 1;
+                }
+                Value::Object(entries)
+            }
+        }
+    }
+
+    #[test]
+    fn canonical_json_bytes_matches_reference_on_fuzz() {
+        // WHY: US-003 rewrote `canonical_json_bytes` from "deep-clone into a canonical
+        // Value, then to_vec" to "write the sorted-key JSON directly into the output
+        // buffer".  Node dedup (`record_if_new`) keys on these exact bytes and the
+        // default-mode edge content hash hashes them, so even one drifted byte (string
+        // escaping, number formatting, key sort, separators) silently shifts node ids,
+        // dedup decisions, and edge hashes.  This seeded fuzz drives the new writer and
+        // a byte-verbatim copy of the old serializer over nested objects at depth,
+        // arrays, every number shape (u64/i64/negative/fractional/exponent), bools,
+        // nulls, escape-heavy unicode strings, and empty containers, and demands
+        // byte-identical output.
+        let mut rng = Rng(0x5EED_2024_0000_0003);
+        let mut stats = FuzzStats::default();
+
+        for _ in 0..4096 {
+            let value: Value = fuzz_value(&mut rng, 0, &mut stats);
+            let direct: Vec<u8> = canonical_json_bytes(&value).expect("canonical");
+            let reference: Vec<u8> = canonical_json_bytes_reference(&value).expect("reference");
+            assert_eq!(direct, reference, "divergence on: {value}");
+        }
+
+        // Key-order permutations of the same object: canonical bytes must not depend on
+        // insertion order (serde_json's preserve_order leaks it to plain to_vec).
+        let mut order: Vec<usize> = (0..FUZZ_KEYS.len()).collect();
+        for _ in 0..128 {
+            shuffle(&mut rng, &mut order);
+            let count: usize = 1 + rng.below(FUZZ_KEYS.len());
+            let entries: Vec<(String, Value)> = order[..count]
+                .iter()
+                .map(|index| {
+                    (
+                        FUZZ_KEYS[*index].to_string(),
+                        fuzz_value(&mut rng, 1, &mut stats),
+                    )
+                })
+                .collect();
+            let mut forward: Map<String, Value> = Map::new();
+            for (key, entry) in &entries {
+                forward.insert(key.clone(), entry.clone());
+            }
+            let mut reversed: Map<String, Value> = Map::new();
+            for (key, entry) in entries.iter().rev() {
+                reversed.insert(key.clone(), entry.clone());
+            }
+            let forward_value = Value::Object(forward);
+            let reversed_value = Value::Object(reversed);
+            let direct_forward: Vec<u8> = canonical_json_bytes(&forward_value).expect("canonical");
+            let direct_reversed: Vec<u8> =
+                canonical_json_bytes(&reversed_value).expect("canonical");
+            let reference_forward: Vec<u8> =
+                canonical_json_bytes_reference(&forward_value).expect("reference");
+            assert_eq!(direct_forward, direct_reversed, "key order leaked");
+            assert_eq!(
+                direct_forward, reference_forward,
+                "divergence on permutation"
+            );
+        }
+
+        // Duplicate keys in raw JSON: serde_json's parser collapses them last-wins
+        // before either serializer sees the Value; pin that both implementations agree
+        // on the collapsed form (plus the degenerate scalars/empties).
+        let raw_cases: [&str; 8] = [
+            r#"{"a":1,"a":2}"#,
+            r#"{"b":{"x":[1,2],"x":null},"b":{}}"#,
+            "{}",
+            "[]",
+            "null",
+            "true",
+            "\"\"",
+            "-0.0",
+        ];
+        for raw in raw_cases {
+            let value: Value = serde_json::from_str(raw).expect("parse raw case");
+            let direct: Vec<u8> = canonical_json_bytes(&value).expect("canonical");
+            let reference: Vec<u8> = canonical_json_bytes_reference(&value).expect("reference");
+            assert_eq!(direct, reference, "raw case: {raw}");
+        }
+
+        // Non-vacuity: the seeded stream must actually exercise the risky shapes or the
+        // equivalence check could pass on a trivially degenerate input.
+        assert!(
+            stats.multi_key_objects >= 100,
+            "expected multi-key objects, got {}",
+            stats.multi_key_objects
+        );
+        assert!(stats.arrays >= 100, "expected arrays, got {}", stats.arrays);
+        assert!(
+            stats.nested_containers >= 50,
+            "expected nested containers, got {}",
+            stats.nested_containers
+        );
+        assert!(
+            stats.escaped_strings >= 50,
+            "expected escape-heavy strings, got {}",
+            stats.escaped_strings
+        );
+    }
 }