diff --git a/AGENTS.md b/AGENTS.md index 5202363c7..12ee004a8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -71,14 +71,16 @@ Eagerly corrected when spotted in **any** file! Cross-references use the `§` (section sign) prefix. Every reference carries a **package qualifier** so the target document is never ambiguous. -ADR references (`ADR-T-001`, `ADR-R-001`, …) are an exception — they -use their own `ADR--` form without the `§` prefix. - -| Prefix | Package | Example document | -| ------ | -------------------- | -------------------------------- | -| `T-` | Torrust (root crate) | | -| `M-` | Mudlark | `packages/mudlark/docs/idea.md` | -| `R-` | render-text-as-image | `packages/render-text-as-image/` | +ADR references (`ADR-T-001`, `ADR-S-001`, `ADR-R-001`, …) are an +exception — they use their own `ADR--` form without the `§` +prefix. + +| Prefix | Package | Example document | +| ------ | -------------------- | ---------------------------------------- | +| `T-` | Torrust (root crate) | | +| `M-` | Mudlark | `packages/mudlark/docs/idea.md` | +| `S-` | Sentinel | `packages/sentinel/docs/algorithm.md` | +| `R-` | render-text-as-image | `packages/render-text-as-image/` | Helper crates (`index-health-check`, `index-auth-keypair`, `index-config`, `index-config-probe`, `index-cli-common`, diff --git a/Cargo.lock b/Cargo.lock index d9b3abc41..a5a32073f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -190,6 +190,16 @@ dependencies = [ "bytemuck", ] +[[package]] +name = "atomic-wait" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a55b94919229f2c42292fd71ffa4b75e83193bffdd77b1e858cd55fd2d0b0ea8" +dependencies = [ + "libc", + "windows-sys 0.42.0", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -413,6 +423,20 @@ name = "bytemuck" version = "1.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a1f896587b6f2c069c73d2f0913e2d590c3990285cd2f0b6aa02b786b4c679c" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] [[package]] name = "byteorder" @@ -742,6 +766,28 @@ dependencies = [ "itertools", ] +[[package]] +name = "crossbeam" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e71406cd8807725f7ac2f999a4cdd32e98f829fdf65f528343cebf945e41df1e" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-epoch", + "crossbeam-queue", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98b0cc327b5bc766e7fda9c9260cc0fa81b43a8e240440422dff70788e3f9ef1" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-deque" version = "0.8.8" @@ -883,6 +929,12 @@ dependencies = [ "syn 3.0.6", ] +[[package]] +name = "defer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "930c7171c8df9fb1782bdf9b918ed9ed2d33d1d22300abb754f9085bc48bf8e8" + [[package]] name = "defmt" version = "1.1.1" @@ -911,7 +963,7 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" dependencies = [ - "thiserror", + "thiserror 2.0.20", ] [[package]] @@ -1016,6 +1068,22 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" +[[package]] +name = "dyn-stack" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c4713e43e2886ba72b8271aa66c93d722116acf7a75555cce11dcde84388fe8" +dependencies = [ + "bytemuck", + "dyn-stack-macros", +] + +[[package]] +name = "dyn-stack-macros" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d926b4d407d372f141f93bb444696142c29d32962ccbd3531117cf3aa0bfa9" + [[package]] name = "ecdsa" version = "0.16.9" @@ -1118,6 +1186,73 @@ dependencies = [ "simdutf8", ] +[[package]] +name = "enum-as-inner" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1e6a265c649f3f5979b601d26f1d05ada116434c87741c9493cb56218f76cbc" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "equator" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c35da53b5a021d2484a7cc49b2ac7f2d840f8236a286f84202369bd338d761ea" +dependencies = [ + "equator-macro 0.2.1", +] + +[[package]] +name = "equator" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4711b213838dfee0117e3be6ac926007d7f433d7bbe33595975d4190cb07e6fc" +dependencies = [ + "equator-macro 0.4.2", +] + +[[package]] +name = "equator" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02da895aab06bbebefb6b2595f6d637b18c9ff629b4cd840965bb3164e4194b0" +dependencies = [ + "equator-macro 0.6.0", +] + +[[package]] +name = "equator-macro" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bf679796c0322556351f287a51b49e48f7c4986e727b5dd78c972d30e2e16cc" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "equator-macro" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44f23cf4b44bfce11a86ace86f8a73ffdec849c9fd00a386a53d278bd9e81fb3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "equator-macro" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b14b339eb76d07f052cdbad76ca7c1310e56173a138095d3bf42a23c06ef5d8" + [[package]] name = "equivalent" version = "1.0.2" @@ -1155,6 +1290,49 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "faer" +version = "0.24.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ab6df3dd147fe8d702a288b95bcd8fcc499ab572fc80da6828f60cd4d524d67" +dependencies = [ + "bytemuck", + "dyn-stack", + "equator 0.6.0", + "faer-traits", + "gemm", + "generativity", + "libm", + "nano-gemm", + "npyz", + "num-complex", + "num-traits", + "private-gemm-x86", + "pulp", + "rand 0.9.5", + "rand_distr 0.5.1", + "rayon", + "reborrow", + "spindle", +] + +[[package]] +name = "faer-traits" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b87d23ed7ab1f26c0cba0e5b9e061a796fbb7dc170fa8bee6970055a1308bb0f" +dependencies = [ + "bytemuck", + "dyn-stack", + "generativity", + "libm", + "num-complex", + "num-traits", + "pulp", + "qd", + "reborrow", +] + [[package]] name = "fastrand" version = "2.5.0" @@ -1374,6 +1552,146 @@ dependencies = [ "slab", ] +[[package]] +name = "gemm" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa0673db364b12263d103b68337a68fbecc541d6f6b61ba72fe438654709eacb" +dependencies = [ + "dyn-stack", + "gemm-c32", + "gemm-c64", + "gemm-common", + "gemm-f16", + "gemm-f32", + "gemm-f64", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "seq-macro", +] + +[[package]] +name = "gemm-c32" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "086936dbdcb99e37aad81d320f98f670e53c1e55a98bee70573e83f95beb128c" +dependencies = [ + "dyn-stack", + "gemm-common", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "seq-macro", +] + +[[package]] +name = "gemm-c64" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20c8aeeeec425959bda4d9827664029ba1501a90a0d1e6228e48bef741db3a3f" +dependencies = [ + "dyn-stack", + "gemm-common", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "seq-macro", +] + +[[package]] +name = "gemm-common" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88027625910cc9b1085aaaa1c4bc46bb3a36aad323452b33c25b5e4e7c8e2a3e" +dependencies = [ + "bytemuck", + "dyn-stack", + "half", + "libm", + "num-complex", + "num-traits", + "once_cell", + "paste", + "pulp", + "raw-cpuid", + "rayon", + "seq-macro", + "sysctl", +] + +[[package]] +name = "gemm-f16" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3df7a55202e6cd6739d82ae3399c8e0c7e1402859b30e4cb780e61525d9486e" +dependencies = [ + "dyn-stack", + "gemm-common", + "gemm-f32", + "half", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "rayon", + "seq-macro", +] + +[[package]] +name = "gemm-f32" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02e0b8c9da1fbec6e3e3ab2ce6bc259ef18eb5f6f0d3e4edf54b75f9fd41a81c" +dependencies = [ + "dyn-stack", + "gemm-common", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "seq-macro", +] + +[[package]] +name = "gemm-f64" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "056131e8f2a521bfab322f804ccd652520c79700d81209e9d9275bbdecaadc6a" +dependencies = [ + "dyn-stack", + "gemm-common", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "seq-macro", +] + +[[package]] +name = "generativity" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2c81fb5260e37854d09d5c87183309fd8c555b75289427884b25660bc87a85e" + +[[package]] +name = "generator" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3b854b0e584ead1a33f18b2fcad7cf7be18b3875c78816b753639aa501513ae" +dependencies = [ + "cc", + "cfg-if", + "libc", + "log", + "rustversion", + "windows-link", + "windows-result", +] + [[package]] name = "generic-array" version = "0.14.9" @@ -1398,6 +1716,18 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + [[package]] name = "getrandom" version = "0.4.3" @@ -1407,7 +1737,7 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "r-efi", + "r-efi 6.0.0", "rand_core 0.10.1", "wasm-bindgen", ] @@ -1448,8 +1778,10 @@ version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" dependencies = [ + "bytemuck", "cfg-if", "crunchy", + "num-traits", "zerocopy", ] @@ -1491,6 +1823,12 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hermit-abi" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17592d60ebacc7d5e169f4663c5f84f9161cc90328abcfe8456f41e4dfcb284" + [[package]] name = "hex" version = "0.4.3" @@ -1836,6 +2174,17 @@ version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8fae54786f62fb2918dcfae3d568594e50eb9b5c25bf04371af6fe7516452fb" +[[package]] +name = "interpol" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb58032ba748f4010d15912a1855a8a0b1ba9eaad3395b0c171c09b3b356ae50" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "ipnet" version = "2.12.2" @@ -1929,7 +2278,7 @@ dependencies = [ "jni-sys", "log", "simd_cesu8", - "thiserror", + "thiserror 2.0.20", "walkdir", "windows-link", ] @@ -2115,6 +2464,19 @@ version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" +[[package]] +name = "loom" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" +dependencies = [ + "cfg-if", + "generator", + "scoped-tls", + "tracing", + "tracing-subscriber", +] + [[package]] name = "lru-slab" version = "0.1.3" @@ -2279,6 +2641,76 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d" +[[package]] +name = "nano-gemm" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e04345dc84b498ff89fe0d38543d1f170da9e43a2c2bcee73a0f9069f72d081" +dependencies = [ + "equator 0.2.2", + "nano-gemm-c32", + "nano-gemm-c64", + "nano-gemm-codegen", + "nano-gemm-core", + "nano-gemm-f32", + "nano-gemm-f64", + "num-complex", +] + +[[package]] +name = "nano-gemm-c32" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0775b1e2520e64deee8fc78b7732e3091fb7585017c0b0f9f4b451757bbbc562" +dependencies = [ + "nano-gemm-codegen", + "nano-gemm-core", + "num-complex", +] + +[[package]] +name = "nano-gemm-c64" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9af49a20d58816e6b5ee65f64142e50edb5eba152678d4bb7377fcbf63f8437a" +dependencies = [ + "nano-gemm-codegen", + "nano-gemm-core", + "num-complex", +] + +[[package]] +name = "nano-gemm-codegen" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cc8d495c791627779477a2cf5df60049f5b165342610eb0d76bee5ff5c5d74c" + +[[package]] +name = "nano-gemm-core" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d998dfa644de87a0f8660e5ea511d7cb5c33b5a2d9847b7af57a2565105089f0" + +[[package]] +name = "nano-gemm-f32" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879d962e79bc8952e4ad21ca4845a21132540ed3f5e01184b2ff7f720e666523" +dependencies = [ + "nano-gemm-codegen", + "nano-gemm-core", +] + +[[package]] +name = "nano-gemm-f64" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9a513473dce7dc00c7e7c318481ca4494034e76997218d8dad51bd9f007a815" +dependencies = [ + "nano-gemm-codegen", + "nano-gemm-core", +] + [[package]] name = "native-tls" version = "0.2.18" @@ -2305,6 +2737,17 @@ dependencies = [ "memchr", ] +[[package]] +name = "npyz" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f0e759e014e630f90af745101b614f761306ddc541681e546649068e25ec1b9" +dependencies = [ + "byteorder", + "num-bigint", + "py_literal", +] + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -2340,6 +2783,17 @@ dependencies = [ "zeroize", ] +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "bytemuck", + "num-traits", + "rand 0.8.8", +] + [[package]] name = "num-conv" version = "0.2.2" @@ -2375,6 +2829,16 @@ dependencies = [ "libm", ] +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -2518,6 +2982,12 @@ dependencies = [ "phc", ] +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + [[package]] name = "pbkdf2" version = "0.13.0" @@ -2578,6 +3048,48 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "pest" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d45aeb61b4bf818e12d4205f2466f8c4748f85f4fce0146d1c03d69d753f0ad" +dependencies = [ + "memchr", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89cc5a242e25ed4e7704d0be240f2cfbe20a8c27e7e252d94835be93d92dc39f" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7abf21475cc3820fe4b2ca2dc2142902f67a02189f3b5b3a229f4febc01a43e5" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "pest_meta" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adba4db388f687393c18c51348d44a41d870ca9df71a2c98172ea3035dc6936e" +dependencies = [ + "pest", +] + [[package]] name = "phc" version = "0.6.1" @@ -2743,6 +3255,22 @@ dependencies = [ "elliptic-curve", ] +[[package]] +name = "private-gemm-x86" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0af8c3e5087969c323f667ccb4b789fa0954f5aa650550e38e81cf9108be21b5" +dependencies = [ + "crossbeam", + "defer", + "interpol", + "num_cpus", + "raw-cpuid", + "rayon", + "spindle", + "sysctl", +] + [[package]] name = "proc-macro2" version = "1.0.107" @@ -2765,12 +3293,60 @@ dependencies = [ "yansi", ] +[[package]] +name = "pulp" +version = "0.22.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "046aa45b989642ec2e4717c8e72d677b13edd831a4d3b6cf37d9a3e54912496a" +dependencies = [ + "bytemuck", + "cfg-if", + "libm", + "num-complex", + "paste", + "pulp-wasm-simd-flag", + "raw-cpuid", + "reborrow", + "version_check", +] + +[[package]] +name = "pulp-wasm-simd-flag" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d8f70e07b9c3962945a74e59ca1c511bba65b6419468acc217c457d93f3c740" + [[package]] name = "pxfm" version = "0.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d55d956fa96f5ec02be2e13af0e20391a5aa83d6a074e3ad368959d0fab299ea" +[[package]] +name = "py_literal" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "102df7a3d46db9d3891f178dcc826dc270a6746277a9ae6436f8d29fd490a8e1" +dependencies = [ + "num-bigint", + "num-complex", + "num-traits", + "pest", + "pest_derive", +] + +[[package]] +name = "qd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15f1304a5aecdcfe9ee72fbba90aa37b3aa067a69d14cb7f3d9deada0be7c07c" +dependencies = [ + "bytemuck", + "libm", + "num-traits", + "pulp", +] + [[package]] name = "quinn" version = "0.11.12" @@ -2785,7 +3361,7 @@ dependencies = [ "rustc-hash", "rustls", "socket2", - "thiserror", + "thiserror 2.0.20", "tokio", "tracing", "web-time", @@ -2808,7 +3384,7 @@ dependencies = [ "rustls", "rustls-pki-types", "slab", - "thiserror", + "thiserror 2.0.20", "tinyvec", "tracing", "web-time", @@ -2843,6 +3419,12 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "478e0585659a122aa407eb7e3c0e1fa51b1d8a870038bd29f0cf4a8551eea972" +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + [[package]] name = "r-efi" version = "6.0.0" @@ -2856,10 +3438,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" dependencies = [ "libc", - "rand_chacha", + "rand_chacha 0.3.1", "rand_core 0.6.4", ] +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + [[package]] name = "rand" version = "0.10.2" @@ -2881,6 +3473,16 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + [[package]] name = "rand_core" version = "0.6.4" @@ -2890,12 +3492,41 @@ dependencies = [ "getrandom 0.2.17", ] +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + [[package]] name = "rand_core" version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" +[[package]] +name = "rand_distr" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" +dependencies = [ + "num-traits", + "rand 0.9.5", +] + +[[package]] +name = "rand_distr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d431c2703ccf129de4d45253c03f49ebb22b97d6ad79ee3ecfc7e3f4862c1d8" +dependencies = [ + "num-traits", + "rand 0.10.2", +] + [[package]] name = "rand_pcg" version = "0.10.2" @@ -2905,6 +3536,15 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "raw-cpuid" +version = "11.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" +dependencies = [ + "bitflags 2.13.2", +] + [[package]] name = "rayon" version = "1.12.0" @@ -2925,6 +3565,12 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "reborrow" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03251193000f4bd3b042892be858ee50e8b3719f2b08e5833ac4353724632430" + [[package]] name = "redox_syscall" version = "0.5.18" @@ -3238,6 +3884,12 @@ dependencies = [ "serde_json", ] +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + [[package]] name = "scopeguard" version = "1.2.0" @@ -3287,6 +3939,12 @@ version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" +[[package]] +name = "seq-macro" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" + [[package]] name = "serde" version = "1.0.229" @@ -3533,7 +4191,7 @@ checksum = "0d585997b0ac10be3c5ee635f1bab02d512760d14b7c468801ac8a01d9ae5f1d" dependencies = [ "num-bigint", "num-traits", - "thiserror", + "thiserror 2.0.20", "time", ] @@ -3571,6 +4229,19 @@ dependencies = [ "lock_api", ] +[[package]] +name = "spindle" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aaca3d8aa5387a6eba861fbf984af5348d9df5d940c25c6366b19556fdf64" +dependencies = [ + "atomic-wait", + "crossbeam", + "equator 0.4.2", + "loom", + "rayon", +] + [[package]] name = "spki" version = "0.7.3" @@ -3622,7 +4293,7 @@ dependencies = [ "serde_json", "sha2 0.10.9", "smallvec", - "thiserror", + "thiserror 2.0.20", "time", "tokio", "tokio-stream", @@ -3705,7 +4376,7 @@ dependencies = [ "smallvec", "sqlx-core", "stringprep", - "thiserror", + "thiserror 2.0.20", "time", "tracing", "whoami", @@ -3743,7 +4414,7 @@ dependencies = [ "smallvec", "sqlx-core", "stringprep", - "thiserror", + "thiserror 2.0.20", "time", "tracing", "whoami", @@ -3768,7 +4439,7 @@ dependencies = [ "serde", "serde_urlencoded", "sqlx-core", - "thiserror", + "thiserror 2.0.20", "time", "tracing", "url", @@ -3803,6 +4474,17 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "syn" version = "2.0.119" @@ -3845,6 +4527,20 @@ dependencies = [ "syn 3.0.6", ] +[[package]] +name = "sysctl" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01198a2debb237c62b6826ec7081082d951f46dbb64b0e8c7649a452230d1dfc" +dependencies = [ + "bitflags 2.13.2", + "byteorder", + "enum-as-inner", + "libc", + "thiserror 1.0.69", + "walkdir", +] + [[package]] name = "system-configuration" version = "0.7.0" @@ -3894,13 +4590,33 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + [[package]] name = "thiserror" version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" dependencies = [ - "thiserror-impl", + "thiserror-impl 2.0.20", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", ] [[package]] @@ -4172,7 +4888,7 @@ dependencies = [ "sqlx", "tempfile", "tera", - "thiserror", + "thiserror 2.0.20", "tokio", "toml 1.1.6+spec-1.1.0", "torrust-index-cli-common", @@ -4223,7 +4939,7 @@ dependencies = [ "serde", "serde_json", "serde_with", - "thiserror", + "thiserror 2.0.20", "toml 1.1.6+spec-1.1.0", "tracing", "url", @@ -4278,7 +4994,7 @@ checksum = "1a7d0de6ae3ee4cf86805f87900b60ccc3dbee38e718023bf4636d050fb96b28" dependencies = [ "binascii", "serde", - "thiserror", + "thiserror 2.0.20", ] [[package]] @@ -4294,6 +5010,21 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "torrust-sentinel" +version = "1.0.0" +dependencies = [ + "criterion", + "faer", + "rand 0.10.2", + "rand_distr 0.6.0", + "serde", + "serde_json", + "torrust-mudlark", + "tracing", + "tracing-subscriber", +] + [[package]] name = "tower" version = "0.5.3" @@ -4455,6 +5186,12 @@ version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" +[[package]] +name = "ucd-trie" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" + [[package]] name = "uncased" version = "0.9.10" @@ -4600,6 +5337,15 @@ version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + [[package]] name = "wasite" version = "0.1.0" @@ -4819,6 +5565,21 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-sys" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + [[package]] name = "windows-sys" version = "0.48.0" @@ -4877,6 +5638,12 @@ dependencies = [ "windows_x86_64_msvc 0.52.6", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -4889,6 +5656,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + [[package]] name = "windows_aarch64_msvc" version = "0.48.5" @@ -4901,6 +5674,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + [[package]] name = "windows_i686_gnu" version = "0.48.5" @@ -4919,6 +5698,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + [[package]] name = "windows_i686_msvc" version = "0.48.5" @@ -4931,6 +5716,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + [[package]] name = "windows_x86_64_gnu" version = "0.48.5" @@ -4943,6 +5734,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" @@ -4955,6 +5752,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + [[package]] name = "windows_x86_64_msvc" version = "0.48.5" @@ -4982,6 +5785,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + [[package]] name = "writeable" version = "0.6.4" diff --git a/Cargo.toml b/Cargo.toml index 92e4ddbe9..8c991684a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,7 @@ members = [ "packages/index-health-check", "packages/mudlark", "packages/render-text-as-image", + "packages/sentinel", ] [package] diff --git a/packages/sentinel/Cargo.toml b/packages/sentinel/Cargo.toml new file mode 100644 index 000000000..e0f82a2d7 --- /dev/null +++ b/packages/sentinel/Cargo.toml @@ -0,0 +1,53 @@ +[package] +categories = ["algorithms", "network-programming"] +description = "Hierarchical online subspace anomaly detection for positionally structured observation streams." +keywords = ["anomaly-detection", "online-learning", "spectral", "streaming", "subspace"] +name = "torrust-sentinel" +readme = "README.md" +version = "1.0.0" + +authors.workspace = true +documentation.workspace = true +edition.workspace = true +homepage.workspace = true +license.workspace = true +publish.workspace = true +repository.workspace = true +rust-version.workspace = true + +[lints] +workspace = true + +[features] +serde = ["dep:serde", "torrust-mudlark/serde"] + +[dependencies] +# Held to the 0.24 line. A bare `0` is the widest range cargo can be given for a +# pre-1.0 crate: it admits every breaking 0.x release, so a routine compatible +# lock refresh could carry the crate across an API break with no manifest edit +# and no review. +faer = "0.24" +rand = "0.10" +rand_distr = "0.6" +serde = { version = "1", features = ["derive"], optional = true } +torrust-mudlark = { version = "1.1.0", path = "../mudlark", default-features = false, features = ["dynamic-contour-tracking"] } +# Held to the 0.1 line: a bare `0` would admit every breaking 0.x release. +tracing = "0.1" + +[dev-dependencies] +# Held to the 0.8 line: a bare `0` would admit every breaking 0.x release. +criterion = { version = "0.8", features = ["html_reports"] } +serde_json = "1" +tracing-subscriber = { version = "0.3", features = ["registry", "env-filter"] } + +[[bench]] +harness = false +name = "sentinel" + +[[test]] +harness = false +name = "pedagogy" + +[[test]] +harness = false +name = "pedagogy_advanced" diff --git a/packages/sentinel/README.md b/packages/sentinel/README.md new file mode 100644 index 000000000..6b23479a4 --- /dev/null +++ b/packages/sentinel/README.md @@ -0,0 +1,745 @@ +# Spectral Sentinel · `guide:sentinel:overview` + +Hierarchical online subspace anomaly detection for positionally structured observation streams. + +Spectral Sentinel combines Mudlark's adaptive spatial index with low-rank statistical trackers. Mudlark ranks spatial entries by observation volume; Spectral Sentinel selects significant V-Tree entries, closes them under G-tree ancestry, and scores incoming batches against learned subspace models for that selected structure. Reports contain measurements only: scores, baselines, drift accumulators, maturity, structural summaries, and health snapshots. + +**Spectral Sentinel measures; the host decides.** + +See the architecture in brief (`sec:sentinel:readme-architecture-in-brief`) below for the conceptual model, or jump straight to the quick start (`sec:sentinel:readme-quick-start`) for code. + +## Choose Spectral Sentinel · `sec:sentinel:readme-choose-spectral-sentinel` + +Use Spectral Sentinel when your stream has **hierarchical positional structure**: leading bits define coarse membership and successive bits refine it. IPv6-like address spaces, network-prefix encodings, and other dyadic coordinate domains are natural fits. + +If the values are pseudo-random, the model has no meaningful positional structure to learn. Cryptographic hashes, UUIDs, random nonces, and uniform identifiers will still be processed, but the measurements will not be useful. If the distribution is static and known in advance, a fixed index or offline model will usually be simpler. If you only need adaptive spatial aggregation or proportional sampling, use [`torrust-mudlark`](../mudlark/README.md) directly. + +Spectral Sentinel's advantage is online multi-scale measurement: it lets Mudlark adapt spatial structure as traffic shifts, models suffix-bit structure at competitively selected regions, and emits raw statistical readouts without embedding host policy. + +## Stability · `sec:sentinel:readme-stability` + +This crate follows [Semantic Versioning](https://semver.org/). The public API surface documented in [docs/api.md](docs/api.md) — every type, trait, and method re-exported from the crate root — is covered by semver guarantees from 1.0.0 onwards. + +Internal machinery (`pub(crate)` modules, EWMA state, tracker internals, staging, and SVD plumbing) is not part of the public API and may change in any release. + +**MSRV:** 1.90 — inherited from the workspace `rust-version` under ADR-T-011 and tested in CI. + +## Installation · `sec:sentinel:readme-installation` + +Add the crate to your project: + +```sh +cargo add torrust-sentinel +``` + +Or add it manually to your `Cargo.toml`: + +```toml +[dependencies] +torrust-sentinel = "1.0" +``` + +The base build has no default feature flags. The `serde` feature is opt-in and enables serialisation for configuration, SVD strategy, and report snapshot types: + +```toml +[dependencies] +torrust-sentinel = { version = "1.0", features = ["serde"] } +``` + +## Quick start · `sec:sentinel:readme-quick-start` + +> Every claim below is asserted with full invariant checks in the [pedagogy integration test](tests/pedagogy.rs). For the inspection surface, see the [advanced pedagogy test](tests/pedagogy_advanced.rs). + +### Create the sentinel · `sec:sentinel:readme-create-the-sentinel` + +`Sentinel128` is the convenience alias for `SpectralSentinel`: a full 128-bit coordinate domain with `u64` volume counters. New trackers are automatically warmed with synthetic noise before real observations are scored. + +```rust +use torrust_sentinel::{NoiseSchedule, Sentinel128, SentinelConfig}; + +let config = SentinelConfig:: { + analysis_k: 8, // small analysis budget for a demo + split_threshold: 4, // split quickly so examples show structure + noise_schedule: NoiseSchedule::Explicit(vec![4]), + noise_batch_size: 4, + noise_seed: Some(2026), + ..SentinelConfig::default() +}; + +let mut sentinel = Sentinel128::new(config).unwrap(); +assert_eq!(sentinel.lifetime_observations(), 0); +``` + +The default configuration is tuned for longer-lived streams. The compact noise schedule above keeps examples and doctests fast; production callers should choose warm-up settings from the recommendations in §ALGO S-Appendix A. + +### Feed data — spatial structure adapts · `sec:sentinel:readme-feed-data-spatial-structure-adapts` + +Every raw value increments Mudlark's spatial substrate by exactly one unit. Scores never feed back into spatial importance, so concentrated traffic can reshape the contour but anomalous-looking scores cannot promote themselves. + +```rust +# use torrust_sentinel::{NoiseSchedule, Sentinel128, SentinelConfig}; +# let config = SentinelConfig:: { +# analysis_k: 8, split_threshold: 4, +# noise_schedule: NoiseSchedule::Explicit(vec![4]), noise_batch_size: 4, +# noise_seed: Some(2026), ..SentinelConfig::default() +# }; +# let mut sentinel = Sentinel128::new(config).unwrap(); +let values: Vec = vec![ + 0xF000_0000_0000_0000_0000_0000_0000_0001, + 0xF000_0000_0000_0000_0000_0000_0000_0002, + 0xF000_0000_0000_0000_0000_0000_0000_0003, + 0x1000_0000_0000_0000_0000_0000_0000_0004, +]; + +let report = sentinel.ingest(&values); +assert_eq!(sentinel.lifetime_observations(), values.len() as u64); +assert!(report.ancestor_reports.iter().any(|cell| cell.depth == 0)); +``` + +A non-empty batch always reports the root as an ancestor context. As the G-V Graph splits, `cell_reports` contains competitive analysis cells and `ancestor_reports` contains the multi-scale chain back to the root. + +### Read measurements back · `sec:sentinel:readme-read-measurements-back` + +Each cell report carries the same four raw scoring axes. Higher values mean greater departure from the learned baseline; Spectral Sentinel does not turn those values into severity levels or actions. + +```rust +# use torrust_sentinel::{NoiseSchedule, Sentinel128, SentinelConfig}; +# let config = SentinelConfig:: { +# analysis_k: 8, split_threshold: 4, +# noise_schedule: NoiseSchedule::Explicit(vec![4]), noise_batch_size: 4, +# noise_seed: Some(2026), ..SentinelConfig::default() +# }; +# let mut sentinel = Sentinel128::new(config).unwrap(); +# let values: Vec = vec![ +# 0xF000_0000_0000_0000_0000_0000_0000_0001, +# 0xF000_0000_0000_0000_0000_0000_0000_0002, +# 0xF000_0000_0000_0000_0000_0000_0000_0003, +# 0x1000_0000_0000_0000_0000_0000_0000_0004, +# ]; +# let report = sentinel.ingest(&values); +for cell in report.cell_reports.iter().chain(report.ancestor_reports.iter()) { + let scores = &cell.scores; + println!( + "depth {} [{:#034x}, {:#034x}) novelty z={:.2} displacement z={:.2}", + cell.depth, + cell.start, + cell.end, + scores.novelty.max_z_score, + scores.displacement.max_z_score, + ); +} +``` + +The host decides how to interpret the measurements. A dashboard might plot z-scores and maturity, a forensic workflow might enable per-sample payloads, and an automated system might compare score distributions against a domain-specific policy. + +### Apply host-controlled decay · `sec:sentinel:readme-apply-host-controlled-decay` + +Temporal policy is external. Spectral Sentinel never decays the graph on its own; the host decides when old spatial importance should fade. + +```rust +# use torrust_sentinel::{NoiseSchedule, Sentinel128, SentinelConfig}; +# let config = SentinelConfig:: { +# analysis_k: 8, split_threshold: 4, +# noise_schedule: NoiseSchedule::Explicit(vec![4]), noise_batch_size: 4, +# noise_seed: Some(2026), ..SentinelConfig::default() +# }; +# let mut sentinel = Sentinel128::new(config).unwrap(); +# let values: Vec = vec![ +# 0xF000_0000_0000_0000_0000_0000_0000_0001, +# 0xF000_0000_0000_0000_0000_0000_0000_0002, +# 0xF000_0000_0000_0000_0000_0000_0000_0003, +# 0x1000_0000_0000_0000_0000_0000_0000_0004, +# ]; +# let _report = sentinel.ingest(&values); +let before = sentinel.graph().total_sum(); +sentinel.decay(0.5, 0.0); // uniform 50% attenuation across the graph +let after = sentinel.graph().total_sum(); +assert!(after <= before); +``` + +Decay changes spatial importance and future competitive selection. It does not mutate tracker subspaces, baselines, CUSUM state, or lifetime observation counts. + +## Advanced usage · `sec:sentinel:readme-advanced-usage` + +The quick start covers construction, ingestion, reporting, and decay. This section covers inspection payloads, alternative domains, configuration, and operational patterns. + +### Inspect trackers directly · `sec:sentinel:readme-inspect-trackers-directly` + +`cell_gnodes()` lists the live analysis trackers. Use `inspect_cell()` when a host needs a tracker snapshot outside the batch report lifecycle: current rank, energy ratio, maturity, geometry flags, and baseline snapshots. + +```rust +# use torrust_sentinel::{NoiseSchedule, Sentinel128, SentinelConfig}; +# let config = SentinelConfig:: { +# analysis_k: 8, split_threshold: 4, +# noise_schedule: NoiseSchedule::Explicit(vec![4]), noise_batch_size: 4, +# noise_seed: Some(2026), ..SentinelConfig::default() +# }; +# let mut sentinel = Sentinel128::new(config).unwrap(); +# let _report = sentinel.ingest(&[1_u128, 2, 3, 4]); +let root = sentinel + .cell_gnodes() + .into_iter() + .find_map(|id| sentinel.inspect_cell(id).filter(|cell| cell.depth == 0)) + .unwrap(); + +assert_eq!(root.analysis_width, 128); +assert!(root.maturity.total_observations() > 0); +``` + +G-node handles come from the analysis set, reports, inspections, and the underlying `graph()` view. Handles can become stale after later graph restructuring, so guard externally stored handles with a fresh graph lookup before targeted subtree decay. + +### Enable per-sample payloads · `sec:sentinel:readme-enable-per-sample-payloads` + +Set `per_sample_scores` when the host needs observation-level detail. The batch-level summaries remain available; each cell report also contains one `SampleScore` per routed observation. + +```rust +# use torrust_sentinel::{NoiseSchedule, Sentinel128, SentinelConfig}; +let config = SentinelConfig:: { + per_sample_scores: true, + analysis_k: 8, + split_threshold: 4, + noise_schedule: NoiseSchedule::Explicit(vec![4]), + noise_batch_size: 4, + noise_seed: Some(2026), + ..SentinelConfig::default() +}; +let mut sentinel = Sentinel128::new(config).unwrap(); +let report = sentinel.ingest(&[1_u128, 2, 3, 4]); + +let root = report.ancestor_reports.iter().find(|cell| cell.depth == 0).unwrap(); +let samples = root.per_sample.as_ref().unwrap(); +assert_eq!(samples.len(), root.sample_count); +``` + +Per-sample payloads are useful for forensics and expensive for large batches. Keep them disabled in hot paths unless the host needs that resolution. + +### Use 64-bit or custom-width domains · `sec:sentinel:readme-use-64-bit-or-custom-width-domains` + +`Sentinel64` is the convenience alias for `SpectralSentinel`. The generic engine can also be instantiated with another bit-width when the coordinate type can supply centred bits over that domain. + +```rust +use torrust_sentinel::{NoiseSchedule, Sentinel64, SentinelConfig}; + +let config = SentinelConfig:: { + noise_schedule: NoiseSchedule::Explicit(vec![2]), + noise_batch_size: 4, + noise_seed: Some(7), + ..SentinelConfig::default() +}; + +let mut sentinel = Sentinel64::new(config).unwrap(); +let report = sentinel.ingest(&[0xF000_0000_0000_0001_u64, 0xF000_0000_0000_0002]); +assert!(report.health.active_trackers > 0); +``` + +```rust +use torrust_sentinel::{NoiseSchedule, SentinelConfig, SpectralSentinel}; + +type Sentinel16 = SpectralSentinel; + +let config = SentinelConfig:: { + noise_schedule: NoiseSchedule::Explicit(vec![2]), + noise_batch_size: 4, + noise_seed: Some(11), + ..SentinelConfig::default() +}; + +let mut sentinel = Sentinel16::new(config).unwrap(); +let report = sentinel.ingest(&[0xF001_u64, 0xF002, 0x1003]); +assert!(report.contour.total_importance >= 3.0); +``` + +The domain is `[0, 2^N)`. Hosts are responsible for ensuring coordinates fit the chosen width and carry meaningful positional structure. + +### Periodic ingest and decay loop · `sec:sentinel:readme-periodic-ingest-and-decay-loop` + +A common lifecycle is: ingest a batch, read measurements, then apply a host-selected decay so stale spatial importance fades between ticks. + +```rust +# use torrust_sentinel::{NoiseSchedule, Sentinel128, SentinelConfig}; +# let config = SentinelConfig:: { +# analysis_k: 8, split_threshold: 4, +# noise_schedule: NoiseSchedule::Explicit(vec![4]), noise_batch_size: 4, +# noise_seed: Some(2026), ..SentinelConfig::default() +# }; +# let mut sentinel = Sentinel128::new(config).unwrap(); +let batches: Vec> = vec![ + vec![0xA000_0000_0000_0000_0000_0000_0000_0001, 0xA000_0000_0000_0000_0000_0000_0000_0002], + vec![0xB000_0000_0000_0000_0000_0000_0000_0001, 0xA000_0000_0000_0000_0000_0000_0000_0003], +]; + +for batch in &batches { + let report = sentinel.ingest(batch); + let _max_novelty_z = report + .cell_reports + .iter() + .chain(report.ancestor_reports.iter()) + .map(|cell| cell.scores.novelty.max_z_score) + .fold(0.0_f64, f64::max); + + sentinel.decay(0.95, 0.0); +} +``` + +The `q` parameter controls depth selectivity: `q = 0.0` decays every spatial depth equally; raising `q` toward `1.0` makes fine structure fade more aggressively than coarse structure. + +### Treat warm trackers as preliminary · `sec:sentinel:readme-treat-warm-trackers-as-preliminary` + +Every tracker reports a `noise_influence` value. It decays as real observations replace synthetic warm-up as the basis of the learned baseline. Hosts can use it to suppress early conclusions without hiding the raw measurements. + +```rust +# use torrust_sentinel::{NoiseSchedule, Sentinel128, SentinelConfig}; +# let config = SentinelConfig:: { +# analysis_k: 8, split_threshold: 4, +# noise_schedule: NoiseSchedule::Explicit(vec![4]), noise_batch_size: 4, +# noise_seed: Some(2026), ..SentinelConfig::default() +# }; +# let mut sentinel = Sentinel128::new(config).unwrap(); +# let report = sentinel.ingest(&[1_u128, 2, 3, 4]); +for cell in report.cell_reports.iter().chain(report.ancestor_reports.iter()) { + if cell.maturity.noise_influence > 0.5 { + continue; + } + + let _usable_mean_z = cell.scores.novelty.mean_z_score; +} +``` + +This is a host policy choice. Spectral Sentinel exposes maturity; it does not suppress or reinterpret scores on the host's behalf. + +## Configuration · `sec:sentinel:readme-configuration` + +`SentinelConfig` validates every invariant up front. Invalid parameters return all detected errors at once; warnings report valid combinations that are likely to produce poor warm-up quality. + +```rust +use torrust_sentinel::{NoiseSchedule, SentinelConfig, SvdStrategy}; + +let config = SentinelConfig:: { + max_rank: 16, // rank ceiling per tracker + forgetting_factor: 0.99, // fast EWMA memory + rank_update_interval: 100, // tracker steps between rank checks + energy_threshold: 0.90, // variance target for rank adaptation + eps: 1e-6, // numerical stability + cusum_slow_decay: 0.999, // slow baseline for per-cell drift + cusum_coord_slow_decay: 0.999,// slow baseline for coordination drift + cusum_allowance_sigmas: 0.5, // CUSUM noise allowance + clip_sigmas: 3.0, // upper-tail baseline clip width + clip_pressure_decay: 0.95, // clip-pressure EWMA memory + per_sample_scores: false, // include per-observation detail + analysis_k: 1024, // competitive analysis budget + analysis_depth_cutoff: 6, // V-Tree eligibility cutoff + split_threshold: 100, // G-V Graph split sensitivity + d_create: 3, // max V-depth for new splits + d_evict: 6, // min V-depth for eviction + budget: 100_000, // hard live G-node ceiling + noise_schedule: NoiseSchedule::default(), + noise_batch_size: 16, + noise_seed: Some(42), + background_warming: false, + svd_strategy: SvdStrategy::Brand, +}; + +config.validate().unwrap(); +let _warnings = config.warnings(); +``` + +Key tuning knobs: + +| Parameter | Effect | +| --------- | ------ | +| `analysis_k` | Resource ceiling for competitive analysis cells. Total live cell trackers are bounded by the competitive cells plus their shared ancestors. | +| `analysis_depth_cutoff` | V-Tree depth eligibility. Lower values restrict analysis to entries that have risen closer to the tournament root. | +| `forgetting_factor` | Fast statistical memory. Lower values adapt faster; higher values remember longer. | +| `max_rank` | Model expressiveness ceiling. Higher values can model richer suffix structure at greater memory and SVD cost. | +| `energy_threshold` | Variance target for automatic rank adaptation. Higher values tend to grow rank. | +| `split_threshold` | Spatial split sensitivity. Lower values refine the G-V Graph faster. | +| `d_create` / `d_evict` | Mudlark depth gates. They control tree growth, eviction eligibility, and budget behaviour. | +| `budget` | Hard ceiling on live G-nodes in the spatial substrate. | +| `noise_schedule` | Depth-tiered warm-up schedule: geometric by default, or explicit per-depth counts. | +| `background_warming` | Moves new-cell warm-up off the `ingest()` hot path at the cost of delayed participation. | +| `svd_strategy` | `Brand` incremental SVD by default, with `Naive` available as a dense baseline. | + +## Automatic warm-up · `sec:sentinel:readme-automatic-warm-up` + +Every newly created tracker is warmed with synthetic noise before it receives real observations. There is no manual noise-injection API; the Spectral Sentinel owns the warm-up lifecycle. + +| Tracker | Warm-up behaviour | +| ------- | ----------------- | +| Root tracker | Warmed during `SpectralSentinel::new()`. | +| New analysis cells | Enqueued into the staging area and warmed before promotion. | +| Coordination contexts | Warmed when cross-cell coordination first activates. | + +`NoiseSchedule` supports two forms: + +| Variant | Meaning | +| ------- | ------- | +| `Geometric { root, decay, min }` | `rounds(depth) = max(min, root × decay^depth)`. This is the default schedule. | +| `Explicit(Vec)` | Per-depth round counts. The last entry repeats for deeper cells; an empty vector disables noise. | + +Synchronous warm-up (`background_warming: false`) is deterministic and used heavily by tests. Background warm-up (`true`) is intended for production paths where cell creation should not introduce a latency spike. + +## Architecture in brief · `sec:sentinel:readme-architecture-in-brief` + +Spectral Sentinel implements a three-layer feed-forward architecture backed by the Mudlark G-V Graph. + +```text +Layer 1: G-V Graph + Adaptive spatial partitioning of [0, 2^N) + Pure volume tracking: Δ = 1 per observation + Competitive ranking by observation volume + │ + │ V-Tree depth ≤ cutoff → top-K selection + ▼ +Layer 2: Analysis Selector + Selects competitive cells + Closes selection under G-tree ancestry + │ + │ suffix bit vectors at each ancestor depth + ▼ +Layer 3: Analysis Engine + Per-cell low-rank subspace trackers + Hierarchical coordination over cross-cell score patterns + │ + ▼ + BatchReport → host +``` + +### Spatial substrate · `sec:sentinel:readme-spatial-substrate` + +Spectral Sentinel owns a `GvGraph`. During `ingest()`, each raw coordinate is observed by the graph with `Δ = 1`; that is the feed-forward invariant from [ADR-S-002](adr/002-feed-forward-invariant.md). Anomaly scores flow outward to reports and never back into Mudlark's importance signal. + +The host may call `decay()` or `decay_subtree()` to reshape spatial importance over time. Decay affects the graph's competitive ranking, not the statistical state inside trackers. + +### Analysis selector · `sec:sentinel:readme-analysis-selector` + +After each observation pass, `AnalysisSet` selects the top competitive V-Tree entries within `analysis_depth_cutoff`, takes at most `analysis_k`, and closes the result under G-tree ancestry. The root is permanent context, never a competitive target. + +The selected cells form the investment set: cells that own trackers or are warming toward ownership. Online members produce reports; warming members are visible in health and summary counts. + +### Analysis engine · `sec:sentinel:readme-analysis-engine` + +Each analysis cell owns a `SubspaceTracker` over the suffix bits `[d, N)` where `d` is the G-tree depth. The tracker processes batches in a strict score-before-evolve order: + +1. Score the batch against the prior model. +2. Evolve the subspace with streaming thin SVD. +3. Evolve latent mean, variance, and second-moment state. +4. Update fast baselines, slow CUSUM references, and clip pressure. +5. Adapt rank toward the configured energy threshold. + +Coordination trackers sit at internal G-tree contexts and model patterns among competitive-cell score vectors. They detect cross-cell score shapes that no single cell needs to treat as special. + +## Scoring axes · `sec:sentinel:readme-scoring-axes` + +All axes share the same polarity: higher values indicate greater anomalous departure from the learned baseline. + +| Axis | Measures | Range | +| ---- | -------- | ----- | +| `novelty` | Residual energy outside the learned subspace | `[0, ∞)` | +| `displacement` | Distance from the cell's latent centroid | `[0, 1)` | +| `surprise` | Per-dimension magnitude deviation | `[0, ∞)` | +| `coherence` | Unusual pairwise co-activation patterns | `[0, ∞)` | + +Together they decompose the covariance structure of centred suffix-bit vectors without assembling or inverting a dense covariance matrix. `ScoreDistribution` reports min, max, mean, z-scores, fast baseline, slow CUSUM reference, accumulator state, and clip pressure for each axis. + +At the coordination tier the same four axes have second-order meaning: novelty measures unseen cross-cell score patterns, displacement measures a shifted score landscape, surprise measures a system-wide axis elevation, and coherence measures unusual combinations of axis elevation. + +## Report structure · `sec:sentinel:readme-report-structure` + +`ingest()` returns a `BatchReport`: + +```text +BatchReport +├── cell_reports: [CellReport] competitive cells +├── ancestor_reports: [CellReport] ancestor-only cells, including root +├── coordination_reports: [CoordinationReport] cross-cell score-pattern models +├── contour: ContourSnapshot spatial contour summary +├── health: HealthReport operational health snapshot +├── analysis_set_summary: AnalysisSetSummary investment and producing-set summary +└── oldest_observation_age_micros: Option age of the batch's oldest observation at emission +``` + +The age is a duration on the sentinel's own monotonic clock — stamped as the batch arrives, read off as the report is assembled — so it carries no wall-clock instant and no cross-machine skew. A batch arrives whole, so the one figure bounds every observation in it. It is absent both for a batch that carried no observations and for a payload written before the field existed, because neither holds a measurement and a zero would claim one. How long the host held the observations before handing them over is not included and is deliberately unmeasured. + +A `CellReport` includes interval bounds, depth, suffix width, sample count, rank, energy ratio, top singular value, four-axis scores, tracker maturity, scoring geometry, and optional per-sample scores. + +A `CoordinationReport` includes the same tracker facts for a coordination context, plus the number of competitive cells that contributed and optional per-member score records. + +## Public API surface · `sec:sentinel:readme-public-api-surface` + +Spectral Sentinel uses the same three-surface visibility model as Mudlark. Public symbols are re-exported flat from the crate root; modules remain private, so downstream code has one canonical import path. + +| Surface | Name | What it exposes | +| ------- | ---- | --------------- | +| 1 | **Readouts** | Detached report and snapshot types users hold after an observation cycle. | +| 2 | **Engine** | Opaque operational types users configure and drive. | +| 3 | **Internals** | `pub(crate)` implementation machinery; not part of the API. | + +### Key types · `sec:sentinel:readme-key-types` + +`GNodeId` is listed with the engine surface because hosts pass it back to target subtree decay. Reports also carry it as a readout identifier. + +| Type | Surface | Description | +| ---- | ------- | ----------- | +| `SpectralSentinel` | 2 | Live Spectral Sentinel engine generic over coordinate, accumulator, and bit-width. | +| `Sentinel128` | 2 | Alias for `SpectralSentinel`. | +| `Sentinel64` | 2 | Alias for `SpectralSentinel`. | +| `SentinelConfig` | 2 | Measurement, resource, warm-up, and Mudlark substrate configuration. | +| `NoiseSchedule` | 2 | Depth-tiered tracker warm-up schedule. | +| `SvdStrategy` | 2 | `Brand` incremental SVD or `Naive` dense SVD. | +| `CentredBitSource` | 2 | Coordinate-to-centred-bits capability used by the engine. | +| `GNodeId` | 2 | Re-exported Mudlark arena handle for reports and targeted subtree decay. | +| `BatchReport` | 1 | Complete output from one `ingest()` call. | +| `CellReport` | 1 | Per-cell score, rank, maturity, and geometry readout. | +| `CoordinationReport` | 1 | Cross-cell score-pattern readout. | +| `AnalysisSet` | 1 | Current competitive targets plus G-tree ancestors. | +| `AnalysisEntry` | 1 | One selected cell in the analysis set. | +| `HealthReport` | 1 | Tracker, coordination, maturity, geometry, and clip-pressure health. | +| `CellInspection` | 1 | Direct snapshot of one live cell tracker. | +| `AnomalyScores` | 1 | Four-axis score distributions. | +| `ScoreDistribution` | 1 | Raw score summary plus z-scores, baseline, CUSUM, and clip pressure. | +| `TrackerMaturity` | 1 | Real/noise observation counts and noise influence. | +| `ScoringGeometry` | 1 | Structural flags for novelty saturation and coherence activity. | + +### Key operations · `sec:sentinel:readme-key-operations` + +| Method | Description | +| ------ | ----------- | +| `SpectralSentinel::new(config)` | Validate configuration, create graph, create and warm the root tracker. | +| `ingest(&[C])` | Process one batch and return `BatchReport`. | +| `health()` | Snapshot active trackers, ranks, maturity, geometry, coordination, and clip pressure. | +| `cell_gnodes()` | List live analysis-cell handles. | +| `inspect_cell(gnode)` | Snapshot a specific cell tracker if it is currently live. | +| `cells_tracked()` | Count the cells that have a live tracker. | +| `lifetime_observations()` | Count real observations processed since construction or reset. | +| `degenerate_cells_skipped()` | Count cells excluded because suffix width is too small for tracking. | +| `config()` | Read-only access to the validated configuration. | +| `analysis_set()` | Read-only access to the current analysis set. | +| `graph()` | Read-only access to the Mudlark spatial substrate. | +| `decay(attenuation, q)` | Apply host-controlled temporal decay to the full graph. | +| `decay_subtree(gnode, attenuation, q)` | Apply host-controlled temporal decay to one G-subtree. | +| `reset()` | Clear learned state and recreate the fresh warmed root. | + +## Features · `sec:sentinel:readme-features` + +| Feature | Default | Effect | +| ------- | ------- | ------ | +| `serde` | no | `Serialize`/`Deserialize` for configuration, SVD strategy, and report snapshot types. Also enables `torrust-mudlark/serde`. | + +## Resource model · `sec:sentinel:readme-resource-model` + +`analysis_k` bounds the competitive targets. Their current investment set includes every intermediate ancestor and the permanent root: for selected depths `d_i`, its cell-tracker count is at most `1 + sum(d_i)`, hence at most `1 + analysis_k * (N - 2)` for supported engines (`N >= 2`, eligible suffix width at least two). Shared paths reduce this count; a reduced Steiner-tree bound does not count the retained chain nodes. This covers selected online and warming cells, with coordination trackers and any evicted model still held by the warming worker accounted for separately (§ALGO S-8.2). + +The Mudlark `budget` field is the hard ceiling on live spatial nodes. `split_threshold`, `d_create`, and `d_evict` determine how quickly the spatial substrate refines and how it contracts under pressure. + +Warm-up work is usually the largest cold-start cost. Use `background_warming: true` when avoiding `ingest()` latency spikes matters more than immediate participation by brand-new cells. + +Criterion benchmarks live in [benches/sentinel.rs](benches/sentinel.rs): + +```sh +cargo bench -p torrust-sentinel +``` + +## Limitations · `sec:sentinel:readme-limitations` + +- **Structured coordinates required.** Pseudo-random bit strings do not contain learnable suffix structure for this model. +- **One-dimensional domain.** Spectral Sentinel expects one-dimensional coordinates. Multi-dimensional data needs an external encoding, such as a space-filling curve composition. +- **Measurements only.** Spectral Sentinel has no policy thresholds, threat levels, labels, or actions. Hosts must interpret reports in context. +- **Single-writer mutation.** Mutating methods take `&mut self`; concurrent writers need external synchronisation. +- **Warm-up tradeoff.** Synchronous warm-up is deterministic but can add latency when cells are created. Background warm-up avoids that spike but delays scoring for new cells. +- **Timing equalisation out of scope.** Deferred warm-up is implemented; timing-protection padding and equalisation from §ALGO S-12.9 are outside this crate. + +## Documentation · `sec:sentinel:readme-documentation` + +> The relative links below work in the repository but may not resolve on crates.io. + +| Document | Contents | +| -------- | -------- | +| [docs/algorithm.md](docs/algorithm.md) | Formal algorithm specification. | +| [docs/api.md](docs/api.md) | Public API reference. | +| [docs/implementation.md](docs/implementation.md) | Source layout, implementation guide, and test-suite map. | +| [adr/](adr/) | Architecture decision records. | +| [tests/pedagogy.rs](tests/pedagogy.rs) | End-to-end narrative test for construction, ingest, scoring, decay, and reset. | +| [tests/pedagogy_advanced.rs](tests/pedagogy_advanced.rs) | Inspection-oriented narrative test for readouts, geometry, coordination, and temporal separation. | + +### Design records · `sec:sentinel:readme-design-records` + +Architectural decisions are recorded in [adr/](adr/). Notable entries: + +- [ADR-S-001](adr/001-measures-not-opinions.md) — Measures, not opinions +- [ADR-S-002](adr/002-feed-forward-invariant.md) — Feed-forward invariant +- [ADR-S-007](adr/007-automatic-noise-injection.md) — Automatic noise injection +- [ADR-S-015](adr/015-cell-creation-performance.md) — Cell creation performance +- [ADR-S-016](adr/016-brand-incremental-svd.md) — Brand incremental SVD +- [ADR-S-017](adr/017-deferred-cell-warm-up.md) — Deferred cell warm-up +- [ADR-S-018](adr/018-generic-domain-parameters.md) — Generic domain parameters +- [ADR-S-019](adr/019-investment-set-terminology-and-reporting.md) — Investment-set terminology and reporting +- [ADR-S-020](adr/020-clip-pressure-ewma.md) — Clip-pressure EWMA +- [ADR-S-021](adr/021-ewma-mean-centred-variance.md) — EWMA mean-centred variance + +### Cross-references · `sec:sentinel:readme-cross-references` + +Doc-comments and documentation use `§`-prefixed tags to cite specific sections of the design documents. The `S-` qualifier identifies this Sentinel package; ADRs use their own `ADR-S-NNN` form. + +| Tag | Document | +| --- | -------- | +| `§ALGO S-N` | [docs/algorithm.md](docs/algorithm.md) §N | +| `§API S-N` | [docs/api.md](docs/api.md) §N | +| `§IMPL S-N` | [docs/implementation.md](docs/implementation.md) §N | +| `ADR-S-NNN` | Architecture decision record in [adr/](adr/) | + +For example, `§ALGO S-8.2` refers to analysis-set ancestry closure in the algorithm specification. `§§` denotes a range, such as `§§ALGO S-4.2–4.5`. The full authoring conventions are in [AGENTS.md](../../AGENTS.md). + +## The area register · `sec:sentinel:area-register` + +Every claim this package mints names an area, and this register says what each area is for. The requirement it answers is that a package minting claims carries one register in its own prose, an entry per area, whose head prose states the stake — what is lost if claims of this area fail. + +The stake is the one thing about an area that no census can compute and no individual claim states, because a claim says what the system does rather than why anyone should care that it does it. The vocabulary was not fixed in advance; it is what the statements turned out to want, censused after they were written and curated into the entries below. An unregistered area is a report line rather than a finding, so a claim minted in a new one is admitted and counted, and this register is what catches up. + +Three of the entries carry a caveat the register cannot settle. The convergence area is a mixture: about half its claims are about the engine, and the other half define the instruments the convergence suite measures with, which are claims about the yardstick rather than about the sentinel. The analysis-width arithmetic is minted four times over — at the model, at the coordinate alias, at the report surface, and as a standing guarantee — which is defensible, each stating the fact at a different boundary, and still leaves a reader asking which is the source. And the width area is a misnomer: both its claims are about domain parametrisation rather than about width, and the entry below is written to the claims rather than to the name. + +**Section (ancestry)** · `sec:sentinel:area-ancestry` + +If this fails: a disturbance can be noticed but not located. Delivery is by containment rather than ownership — nothing is consumed by the deepest cell that matched — so an ancestor credited with less than everything beneath it holds a baseline for a volume no cell ever saw. A chain that narrows toward the root, or that skips the intervening depths, stops being a sequence of scales at all. What goes with it is the comparison itself: the root grades an anomaly by how much of the domain it reaches, and nothing else tells a host whether an incident is local or system-wide. + +**Section (bits)** · `sec:sentinel:area-bits` + +If this fails: every measurement above this boundary describes a value nobody sent. A bit enters as minus or plus a half, most significant first, because that centring is what the tracker assumes and that ordering is what lets a cell take its working view by dropping the entries routing already fixed. Get the ordering wrong and a cell analyses another region's bits; cap the width wrongly and the tail fills with structure never observed. A squared norm fixed at a quarter of the width is what makes magnitude carry no information, so a residual means departure from learned structure. + +**Section (clipping)** · `sec:sentinel:area-clipping` + +If this fails: the defence against poisoning becomes the thing that poisons. A clipped baseline has a second, biased fixed point where tight clipping keeps rejecting the evidence that would loosen it — an axis clips itself into a baseline it then refuses all evidence against, and it looks settled while doing so. The graduated exemption widens the basin of the correct one and must narrow smoothly, or a batch is judged by a far tighter rule than the one before it. The truncation must also cost nothing: a spread shrunk by the missing tail leaves the baseline correctly centred and miscalibrated for every score divided by it. + +**Section (config)** · `sec:sentinel:area-config` + +If this fails: an incoherent sentinel runs instead of refusing to start. The refusals are not tidiness — a capacity of nothing leaves the tracker no basis to hold, a clip width of nothing rejects every observation the baseline learns from, a negative allowance manufactures the drift it absorbs, and a drift reference decaying no slower than the baseline follows a gradual shift rather than exposing it. The schedules carry the same weight: what each depth is warmed with, and what counts as warming switched off. Advice and refusal stay separate channels, so a configuration that merely converges too slowly still runs and still says so. + +**Section (convergence)** · `sec:sentinel:area-convergence` + +If this fails: warm-up is sized from numbers nobody can trust. A baseline closing on a level at the rate the forgetting factor dictates is what makes warm-up length computable rather than discovered, and a spread lagging its own mean leaves every score miscalibrated after the level looks settled. The area also holds the instruments — the unweighted window average, settling dated from the round after the last violation, stationarity as an early block against a late one, jitter as a fraction of its level. A yardstick with a memory of its own would judge a long-memory baseline by an equally sluggish standard, and every bound derived here would be measuring the ruler. + +**Section (coordination)** · `sec:sentinel:area-coordination` + +If this fails: a pattern spread across sibling cells stays invisible, because no cell sees it alone. A context fires only where both subtrees contribute, so a node firing on one branch reports a group that does not exist, and a context carried forward after a subtree falls silent describes an earlier batch as this one. Membership nests with the tree — an ancestor measures a superset of its descendant — which is what lets one pattern be read at two scales. The tier works in four dimensions whatever the cells beneath it analyse, and that fixed geometry is the only reason a second tier is affordable. + +**Section (coverage)** · `sec:sentinel:area-coverage` + +If this fails: an attacker picks the corner that is not covered. Sudden against gradual, confined against system-wide: each corner is answered by a different mechanism — one batch comparison, accumulation for the disturbance that never grows louder, per-cell scoring for the one hiding behind ordinary traffic, and the coarse end of the chain for the shift that moves everything at once and leaves no cell unusual against its neighbours. Lose a corner and being quiet and being partial become additive protections. + +**Section (cusum)** · `sec:sentinel:area-cusum` + +If this fails: a persistent shift is indistinguishable from one loud batch. Evidence is built from the present run only — the sum is clamped at rest, so a lull banks no credit a later rise must first repay — and the dead band is scaled to the baseline's own spread, so a noisy axis tolerates more before it counts as drifting. The reference gives all of it meaning: a reset that discarded it would leave the axis blind while a long memory rebuilt itself, and one not seeded from the converged fast baseline registers its own lag as drift until that lag closes. + +**Section (decay)** · `sec:sentinel:area-decay` + +If this fails: the host loses temporal policy while appearing to keep it. Decay rescales spatial standing and stops there — the models built are not the same asset as the standing that justified building them — so a decay reaching the trackers or the observation record discards learning the host only meant to reprioritise. Importance is held in integers, so repeated attenuation arrives at nothing rather than leaving a residue that outranks a genuinely new cell. A subtree decay must spend its whole effect inside its target, or one region cannot re-form without punishing the rest. + +**Section (determinism)** · `sec:sentinel:area-determinism` + +If this fails: no difference between two runs can be attributed. Reports come out in the order each list's contract states — the competitive and ancestor lists by ascending handle, the cross-cell contexts shallowest first with the handle breaking ties — rather than in whatever order a traversal produced, so two readouts compare position by position; without that, a reader diffing them sees churn that means nothing. The seed is the engine's own randomness and is genuinely visible in the scores, since warming noise shapes the subspace a tracker starts from — which is why reproducibility has to be stated in terms of the seed and not of the data alone. The seed settles the whole of it only with `background_warming` disabled and on a fixed build — one target and one set of dependency versions — because the generator behind the noise is chosen for speed rather than for portability. Under background warming the same seed and the same traffic still give the same graph, the same investment set and the same report order, but neither the baselines a tracker starts from nor the ingest cycle on which it first scores: the warming worker draws from its own generator and takes whichever staged cell leads on volume when it looks. + +**Section (edge)** · `sec:sentinel:area-edge` + +If this fails: the corners of the input space bring the run down instead of being reported. The extremes of the domain are ordinary observations because the encoding centres every bit; a stream with no variation still counts in full and simply teaches no new direction; a cell driven too narrow to model is declined and counted rather than handed a tracker that could form no basis. Idling on empty batches must leave the engine exactly where it was, so the burst that follows is scored as though the quiet had never happened. + +**Section (engine)** · `sec:sentinel:area-engine` + +If this fails: the public surface stops describing the thing behind it. A configuration reads back as given because construction validates without rewriting, so the accessor is the authority on how this sentinel behaves; a handle from another sentinel inspects to nothing rather than to whichever local cell sits at that index. The root tracker is permanent, which gives the tracked count a floor and every ancestor chain somewhere to terminate. The counter accumulates observations and not batches, and the handle list and the tracked count are two views of one map — each a place where two records could quietly disagree. + +**Section (ewma)** · `sec:sentinel:area-ewma` + +If this fails: the notion of normal is either unearned or unmovable. The first batch is adopted outright rather than blended with a placeholder that was never an observation, and warmth is never manufactured by seeding from a cold source. A batch of one carries no spread, so taking its deviation as zero would make every later value an outlier. The ceiling refuses exactly what an attacker would use to walk normal upward — but while cold there is nothing to clip against, and without a floor on the spread a quiet stretch closes the ceiling onto the mean and freezes it there. + +**Section (health)** · `sec:sentinel:area-health` + +If this fails: an operator watches a picture of a different engine. Health is a readout of present state rather than a summary of the last batch, so it can be polled on the host's own schedule and still describe the whole run. The distinctions it draws are the ones a host acts on: real observations kept apart from synthetic seeding, coldness measured as exposure to the world rather than whether a model is populated, an axis structurally unavailable told apart from one merely quiet. The tracker counts are one partition reported three ways, so disagreement among them is an accounting fault. + +**Section (invariant)** · `sec:sentinel:area-invariant` + +If this fails: the guarantees were only ever properties of a settled system. They are asserted batch by batch through unstructured traffic and through a collapse and the regrowth after it, because the transient is where a bound slips. What they protect is the readability of everything else: scores run one way, so a host may threshold without knowing which axis produced a number; a drift accumulator never falls below rest, so a rise starts from zero; and no axis returns a value that is not a number, since such a value compares false against every threshold and would disarm the alerting silently. + +**Section (noise)** · `sec:sentinel:area-noise` + +If this fails: a cell cannot say how much of what it knows it invented. The synthetic share gates clip width and decides when warm-up is over, so it has to follow its closed form exactly — a batch of decay computed as one power, never accumulated a sample at a time. It falls only under real data and rises only under injection, without reversal either way, which lets a threshold crossing mean the same thing whenever it happens. The injected traffic must be shaped and centred like the real encoding, or a model is warmed on something it will never be asked to judge. + +**Section (pressure)** · `sec:sentinel:area-pressure` + +If this fails: refusal becomes permanent and nobody sees it happen. The rejection rate is both symptom and control — it reports how hard an axis is working to keep contamination out of its baseline, and widens that axis's own ceiling while it is high, so a lasting shift in the traffic can eventually be learned rather than clipped away for ever. It must settle under traffic that keeps its shape rather than ratcheting, or a rise carries no news; it must age out, or one past episode leaves a forever-loose ceiling; and what warm-up accrued must be discarded at the crossing. + +**Section (readout)** · `sec:sentinel:area-readout` + +If this fails: a host reads numbers it cannot interpret or compare. The report partitions cells by how they earned their place, so the competitive list is the sentinel's own investment decisions and nothing else; the analysis width travels beside the scores, because readings from different depths are not commensurable and a bare number invites the comparison anyway. Degeneracy is stated rather than hidden — novelty saturable told apart from saturated, cells too narrow to track counted rather than dropped, churn scoped to the interval since the last report. An empty ingest returns the shape a busy one does. + +**Section (resistance)** · `sec:sentinel:area-resistance` + +If this fails: an attacker who can address any part of the domain decides what the sentinel spends. The cap is on attention rather than on input, and capping the winners is hollow unless the ancestors connecting them are bounded too, since each carries a tracker — so the total cost follows the cap and the depth reached, never how many ranges were touched. The node budget is enforced by eviction rather than hinted at, and attention is bought with accumulated weight, which is what stops a thin spray from evicting the range an operator actually cares about. + +**Section (routing)** · `sec:sentinel:area-routing` + +If this fails: the selector reads something other than where the traffic is. Every value adds exactly one unit wherever it lands — importance is a count of arrivals, not a weight a caller can set — so the graph total and the sentinel's own counter stay one quantity read from two layers, with synthetic warming kept out of both. An empty batch is not an event: quiet time neither adds evidence nor moves the partition. Resolution is bought with observations, and the node budget is what keeps refining everywhere at once from being unbounded. + +**Section (schedule)** · `sec:sentinel:area-schedule` + +If this fails: a cell scores its first real batch against nothing, and the arrival of a new region is reported as an anomaly in it. Warming is not a construction-time favour to the root — cells born mid-run are warmed as they enter the analysis set, and contexts are warmed on activation from the baselines of the cells they watch, since a context scores score vectors rather than coordinates. Warming rounds must never enter the observed count, and the drift accumulators must be wound back at hand-over, or drift chased during warm-up is charged to the first real request. + +**Section (selection)** · `sec:sentinel:area-selection` + +If this fails: the sentinel invests its budget somewhere other than where the evidence is. Ranking is by accumulated importance alone, with ties settled by a property of the coordinate domain rather than by whatever order a walk produced — the foundation every downstream score's reproducibility rests on. The root is in the full set unconditionally so closure terminates, and is never competitive, because it accumulates everything by construction and would crowd out the cells whose behaviour is informative. The summary counts competition and closure apart, so the price of ancestry stays visible. + +**Section (serde)** · `sec:sentinel:area-serde` + +If this fails: the readout cannot leave the process that produced it. The nesting is deep and generic over the coordinate type, so a batch report surviving with its lists and counts intact is what makes handing measurements to another host possible at all. Components travel on their own as well, since a host forwarding structural state to one consumer and scores to another should not have to ship the whole readout to either. Optional detail must cross as present or absent — detail lost in transit looks like a host that never asked for it. + +**Section (staging)** · `sec:sentinel:area-staging` + +If this fails: a half-built model reaches a report, or an evicted one comes back. A cell counts as present from the moment it is enqueued, which is what stops the reconciler enqueuing it twice; presence is answered across every state it can occupy, including while a background thread holds it, because the expensive injection happens outside the lock. Promotion moves ownership rather than copying it, so no cell is promoted twice however often the queue is drained. A cell evicted in flight is discarded on return — cheaper than admitting one the selector has already rejected. + +**Section (subspace)** · `sec:sentinel:area-subspace` + +If this fails: the per-cell model asserts structure it never earned, or publishes scores whose scale nobody can see. A new model claims a single direction and counts itself entirely noise-taught, so every further direction is earned from energy it actually observed. The residual degrees of freedom it publishes are the divisor novelty was normalised by — without them a host compares numbers across depths and ranks blind. The spare axis beyond what the energy threshold demands is where a genuinely new direction first lands; without it, novel structure stays invisible until it is already dominant. + +**Section (suffix)** · `sec:sentinel:area-suffix` + +If this fails: a cell models bits that carry no information and reports scores nobody can compare. The bits its depth stands for were fixed by routing and are identical for every value reaching it, so a cell's width is a consequence of position rather than a setting anyone can get wrong, and its geometry reports that same width rather than a second number that happens to agree. The ceiling on learned structure is whichever is smaller, the configured maximum or the cell's own space — the budget is never a promise of capacity. + +**Section (svd)** · `sec:sentinel:area-svd` + +If this fails: the cheap path stops being the same answer and becomes an approximation nobody audited. Everything downstream assumes the returned axes are orthonormal — coordinates are a projection onto them and novelty is what the projection leaves over, which is a decomposition only if they are — and assumes descending order, since rank adaptation walks the values accumulating energy and takes a position as the rank. Agreement must hold over a whole streaming run with divergence growing no faster than the steps taken, and both paths must explain unseen data equally well. + +**Section (variance)** · `sec:sentinel:area-variance` + +If this fails: the divisor every score rests on stops meaning anything. The spread is measured against the centre a batch arrived to find, before the mean moves, or a batch partly explains its own deviation away. Centring on the running mean is what lets a single-row batch contribute a real squared departure instead of finding no scatter within itself, driving the divisor to its floor and turning ordinary traffic into scores orders of magnitude too large. A score of about one has to mean as expected everywhere, or no threshold can be set once and applied across cells. + +**Section (warmup)** · `sec:sentinel:area-warmup` + +If this fails: a cell enters service on a model it never built, and says nothing about it. Real batches displace the synthetic seed geometrically, so maturity arrives within a predictable number of batches and only ever improves — a figure that could rebound would be useless as progress. A freshly promoted cell is mostly synthetic and publishes that fact, which is what lets a host discount its scores. Deferring the work through a staging area must change when a cell becomes live and nothing else, and a region under construction is still watched by the ancestor chain above it. + +**Section (width)** · `sec:sentinel:area-width` + +If this fails: the engine is usable at one domain size only. Width is a property of the type rather than of the configuration, so the same settings must serve either alias and the same arithmetic must hold for both — the width a cell analyses is read from the sentinel's own parameter rather than assumed. A host working a narrower coordinate space would otherwise keep a second set of settings, or discover at run time that a value it sent was shifted out of range. + +## Development · `sec:sentinel:readme-development` + +### Building and testing · `sec:sentinel:readme-building-and-testing` + +```sh +# Unit + integration tests (debug, optimised dev profile): +cargo test -p torrust-sentinel --all-targets --all-features + +# Doc-tests, including this README: +cargo test -p torrust-sentinel --all-features --doc + +# Release mode: +cargo test -p torrust-sentinel --all-targets --all-features --release + +# No-default-features spot-check: +cargo test -p torrust-sentinel --all-targets --no-default-features + +# Clippy and docs: +cargo clippy -p torrust-sentinel --all-targets --all-features +cargo doc -p torrust-sentinel --all-features --no-deps +``` + +The README is included from [src/lib.rs](src/lib.rs) under `#[cfg(doctest)]`, so Rust code blocks here compile and run as part of `cargo test --doc`. + +Structured diagnostics use `tracing`. They are silent by default; attach a subscriber when you need debug or trace-level detail from tracker and SVD internals. diff --git a/packages/sentinel/adr/001-measures-not-opinions.md b/packages/sentinel/adr/001-measures-not-opinions.md new file mode 100644 index 000000000..de7fbbc65 --- /dev/null +++ b/packages/sentinel/adr/001-measures-not-opinions.md @@ -0,0 +1,39 @@ +# ADR-S-001: Measures Not Opinions · `rec:sentinel:raw-measurements-with-host-owned-policy` + +**Status:** Implemented **Date:** 2026-03-08 **Spec:** §ALGO S-1.5 (layer responsibilities — "sentinel measures; host decides") + +## Context · `sec:sentinel:measure-context` + +An anomaly detector can either: + +- **A)** Output raw statistical measurements and let the consumer decide what they mean (library approach). +- **B)** Output verdicts — threat levels, recommended actions, block/allow decisions (appliance approach). + +The sentinel is a library embedded inside the Torrust Index. Different hosts have different risk tolerances, different action vocabularies (ban, throttle, flag, ignore), and different false-positive consequences. Baking policy into the sentinel would force every host into one policy model. + +## Decision · `sec:sentinel:measure-decision` + +**The sentinel outputs only raw statistical measurements. It never outputs opinions, threat levels, or recommended actions.** + +Concretely: + +- `BatchReport` contains `AnomalyScores` (per-axis mean, max, z-score, CUSUM), `ScoringGeometry` (ADR-S-008), `TrackerMaturity`, rank, energy ratios. +- No field is named "threat", "risk", "anomaly_level", or "action". +- No method returns a boolean "is anomalous" verdict. +- No internal threshold triggers automatic remediation. + +The host reads the report and applies its own policy: + +```rust +// Host policy — not sentinel code: +if report.scores.novelty.z_score > 4.0 && report.maturity.noise_influence < 0.1 { + throttle(cell_id); +} +``` + +## Consequences · `sec:sentinel:measure-consequences` + +- The sentinel has no policy parameters (no "alert threshold", no "sensitivity level"). +- Report types carry more fields than an appliance would expose, but each field has a precise statistical definition. +- Integration tests assert statistical properties, not verdicts. +- Higher polarity = more anomalous is a uniform convention across all four scoring axes (§ALGO S-5.1), ensuring the host can apply a single threshold logic to any axis. diff --git a/packages/sentinel/adr/002-feed-forward-invariant.md b/packages/sentinel/adr/002-feed-forward-invariant.md new file mode 100644 index 000000000..f4dc2d5b7 --- /dev/null +++ b/packages/sentinel/adr/002-feed-forward-invariant.md @@ -0,0 +1,45 @@ +# ADR-S-002: Feed-Forward Invariant · `rec:sentinel:unit-delta-feed-forward-spatial-analysis` + +**Status:** Implemented (2026-03-09) **Date:** 2026-03-08 **Spec:** §ALGO S-1.6 (what sentinel owns vs inherits), §ALGO S-9.1 Step 2 (volume accounting), §ALGO S-9.2 (step ordering) **Relates to:** [ADR-S-001](001-measures-not-opinions.md) (measures not opinions), [ADR-S-003](003-mudlark-integration.md) (V = u64) + +## Context · `sec:sentinel:feedforward-context` + +The sentinel owns a `GvGraph` for adaptive spatial partitioning and a bank of `SubspaceTracker`s for statistical analysis. There is a question of what signal flows between these two subsystems: + +- **Feed-forward:** the G-V Graph receives raw observations and shapes the contour; the analysis engine reads the contour's structure. No analysis output flows back. +- **Feedback:** anomaly scores or derived signals could be fed back into the G-V Graph as importance weights, causing regions with high anomaly scores to receive finer resolution. + +## Decision · `sec:sentinel:feedforward-decision` + +**Strict feed-forward. The G-V Graph receives only `observe(v, 1u64)` per raw input value. Anomaly scores, z-scores, CUSUM values, and any other derived signals are never fed back.** + +```rust +// The ONLY permitted G-V Graph mutation during ingest: +for &value in values { + self.graph.observe(value, 1u64); +} +``` + +This is a code-review invariant, not mechanically enforced. + +## Rationale · `sec:sentinel:feedforward-rationale` + +1. **Feedback creates resonance.** If anomaly scores amplify importance, regions currently under scrutiny get more resolution, which generates more data, which changes the anomaly scores. This feedback loop could cause the contour to lock onto artefacts of its own analysis rather than genuinely active traffic. + +2. **Separation of concerns.** The G-V Graph (Layer 1) reflects objective traffic volume. The analysis engine (Layer 3) reflects statistical deviation from learned baselines. Keeping them independent means a caller can reason about each in isolation. + +3. **Host control.** The host controls temporal policy via `decay()`. If importance weighting is desired, the host can apply it at the decay layer (e.g., selective `decay_subtree()` calls guided by analysis output). This keeps the feedback loop in host code, where policy belongs (ADR-S-001). + +## Consequences · `sec:sentinel:feedforward-consequences` + +- The G-V Graph's contour is shaped entirely by raw traffic volume and host-initiated decay. The analysis tier has no influence on spatial resolution. +- Δ = 1 means the accumulator type `V` is a pure observation counter (for the default `V = u64`), simplifying reasoning about `split_threshold` (it is simply "how many observations before splitting"). +- This invariant must be maintained across all phases of the integration plan. Code review should reject any path that calls `graph.observe()` with a value other than the unit delta. + +### Enforcement mechanisms · `sec:sentinel:feedforward-enforcement-mechanisms` + +1. **No `graph.observe()` during noise injection.** The noise pathway operates on tracker-space `f64` vectors (synthetic centred bit vectors fed directly to `SubspaceTracker::observe()`), not coordinate-space values. Therefore `graph.observe()` is never called during noise injection. This clarifies how the feed-forward invariant interacts with automatic noise injection (ADR-S-007). + +2. **No `graph_mut()` exposure.** The sentinel exposes only a read-only `graph()` accessor (`&GvGraph`) for diagnostics. Mutable graph access is provided exclusively through controlled methods — `decay()` and `decay_subtree()` — which enforce valid parameters. This prevents callers from violating the invariant by calling `graph.observe(v, 100)` directly. + +3. **No pre-aggregated observation.** Each value is fed individually via `graph.observe(value, unit_delta)`. Do not pre-aggregate values by cell and call `observe(representative, count)` — this would lose spatial resolution (values may map to different leaf cells), add complexity for no performance gain (routing still happens inside `observe()`), and violate §ALGO S-9.1 Step 2 which specifies "Δ = 1 per value". diff --git a/packages/sentinel/adr/003-mudlark-integration.md b/packages/sentinel/adr/003-mudlark-integration.md new file mode 100644 index 000000000..553285504 --- /dev/null +++ b/packages/sentinel/adr/003-mudlark-integration.md @@ -0,0 +1,103 @@ +# ADR-S-003: Mudlark Integration · `rec:sentinel:mudlark-default-types-and-feature-gating` + +**Status:** Decided (Part A superseded by [ADR-S-018](018-generic-domain-parameters.md)) **Date:** 2026-03-09 **Spec:** §ALGO S-2.1 (domain $[0, 2^{128})$), §ALGO S-1.6 (what sentinel owns), §ALGO S-13.3 (G-V Graph config) **Relates to:** [ADR-S-002](002-feed-forward-invariant.md) (Δ = 1 invariant), [ADR-S-004](004-config-validation-over-panic.md) (config validation), mudlark [ADR-M-006](../../mudlark/adr/006-generic-parameters.md) (generic parameters) + +## Context · `sec:sentinel:mudlark-context` + +The sentinel must own a `GvGraph` instance from mudlark. Two groups of decisions arise: + +1. **Type-level parameterisation** — the three generic parameters `C`, `V`, `N` and the `Config` fields. +2. **Cargo feature gating** — which of mudlark's optional features the sentinel enables, and how they relate to sentinel's own features. + +These are a single integration surface and are decided together. + +--- + +## Part A — Type Parameters: `GvGraph` · `sec:sentinel:mudlark-part-a-type-parameters` + +> **Superseded by [ADR-S-018](018-generic-domain-parameters.md).** The sentinel is now generic: `SpectralSentinel` with type aliases `Sentinel128` and `Sentinel64`. The rationale below is retained for historical context; the concrete parameters described here remain the defaults via `Sentinel128`. + +| Parameter | Value | Rationale | +|-----------|-------|-----------| +| `C = u128` | The sentinel analyses IPv6 addresses and similar 128-bit identifiers. The full `u128` domain covers $[0, 2^{128})$ without truncation. | +| `V = u64` | The sentinel observes with Δ = 1 per input value (ADR-S-002). An unsigned integer counter is the natural accumulator. `u64` provides a ceiling of $1.8 \times 10^{19}$ observations — far beyond any practical lifetime. | +| `N = 128` | Full domain resolution. Every bit position is available for spatial refinement. | + +### `split_threshold` as `u64` · `sec:sentinel:mudlark-split-threshold` + +The spec lists `split_threshold` as `f64`, but since `V = u64`, mudlark's `Config` requires `split_threshold: u64`. The sentinel accordingly stores `split_threshold: u64` in `SentinelConfig` with default `100`. + +This is type-honest: the threshold is "number of observations a cell must accumulate before subdividing". A fractional count is meaningless. + +### Hardcoded mudlark-internal knobs · `sec:sentinel:mudlark-internal-knobs` + +Two `Config` fields are not exposed in `SentinelConfig`: + +| Field | Value | Rationale | +|-------|-------|-----------| +| `alpha_relax` | `0.75` | Mudlark's recommended default. Controls depth-gate relaxation timing. Not performance-sensitive for the sentinel's use case. | +| `bounded_eviction` | `true` | Mudlark's recommended default. Prevents over-eviction. Always desirable. | + +These can be promoted to `SentinelConfig` later if profiling reveals sensitivity. + +--- + +## Part B — Cargo Feature Gating · `sec:sentinel:mudlark-part-b-feature-gating` + +Mudlark exposes three Cargo features: + +| Feature | Default? | What it provides | +|-----------------------------|----------|-----------------| +| `dynamic-contour-tracking` | Yes | `plateaus()`, contour queries | +| `rand` | Yes | `WeightedSampler` (randomised sampling) | +| `serde` | Yes | `Serialize`/`Deserialize` on all public types | + +### Decisions · `sec:sentinel:mudlark-feature-decisions` + +1. **Disable mudlark's default features** (`default-features = false`). + +2. **Always enable `dynamic-contour-tracking`.** The analysis selector (§ALGO S-4) and report structure (§ALGO S-14) need `plateaus()` and contour queries. + +3. **Do not enable `rand`.** Mudlark's `rand` feature provides `WeightedSampler`, which the sentinel does not use. The sentinel has its own `rand` dependency for noise generation. + +4. **Gate `torrust-mudlark/serde` behind sentinel's `serde` feature:** + + ```toml + [features] + serde = ["dep:serde", "torrust-mudlark/serde"] + ``` + +### Resulting `Cargo.toml` snippet · `sec:sentinel:mudlark-cargo-snippet` + +```toml +[features] +serde = ["dep:serde", "torrust-mudlark/serde"] + +[dependencies] +torrust-mudlark = { path = "../mudlark", default-features = false, features = ["dynamic-contour-tracking"] } +``` + +--- + +## Alternatives Considered · `sec:sentinel:mudlark-alternatives` + +| Alternative | Pros | Cons | +|-------------|------|------| +| `V = f64` | Matches spec table literally | Lossy for a counter; `f64` loses precision above $2^{53}$ | +| `C = u64`, `N = 64` | Smaller coordinates | Loses half the address space; useless for IPv6 | +| Expose `alpha_relax` | Full tunability | Config surface without demonstrated need | +| `split_threshold: f64` in sentinel, cast `as u64` | Matches spec | Lossy cast, misleading type | +| Accept mudlark's default features | Simpler `Cargo.toml` | Pulls in `rand_core` unconditionally; forces serde on all builds | +| Gate `dynamic-contour-tracking` | Allows "headless" build | Every use of the G-V Graph needs contour queries | + +## Consequences · `sec:sentinel:mudlark-consequences` + +- `GvGraph` appears in the sentinel's struct definition and all downstream code. +- `u128` arithmetic is well-optimized on 64-bit platforms (two-wide operations). Benchmark the observe hot-path if performance concerns arise. +- `u64` accumulator means `decay()` operates on integer attenuation via mudlark's `Attenuatable` trait for `u64` (floor-rounding). +- `cargo check --no-default-features` builds with only `dynamic-contour-tracking` in mudlark — no serde, no rand. +- `cargo check --all-features` enables mudlark's `serde` via the transitive feature gate. + +### Re-export policy · `sec:sentinel:mudlark-reexport-policy` + +Only `GNodeId` is re-exported from mudlark to the sentinel's public API. Users needing other mudlark types (e.g., `GNodeInfo`, `Plateau`) must depend on `torrust-mudlark` directly. This minimises version-coupling — the sentinel can upgrade its internal mudlark dependency without breaking callers who don't use mudlark types in their API surface. diff --git a/packages/sentinel/adr/004-config-validation-over-panic.md b/packages/sentinel/adr/004-config-validation-over-panic.md new file mode 100644 index 000000000..b1184ff39 --- /dev/null +++ b/packages/sentinel/adr/004-config-validation-over-panic.md @@ -0,0 +1,81 @@ +# ADR-S-004: Config Validation Over Panic · `rec:sentinel:structured-prevalidation-before-mudlark-construction` + +**Status:** Decided **Date:** 2026-03-09 **Spec:** §ALGO S-13.3 (config constraints) **Relates to:** [ADR-S-003](003-mudlark-integration.md) (mudlark integration) + +## Context · `sec:sentinel:configguard-context` + +Mudlark's `GvGraph::new(config)` internally calls `config.validate()`, which uses `assert!` to enforce constraints such as: + +- `split_threshold > 0` +- `depth_evict > depth_create` +- `budget > max(3^(buffer+1), 2*(depth_create − 1))` where `buffer = depth_evict − depth_create` + +If any constraint is violated, the process panics. This is acceptable for mudlark as a data-structure library — callers are expected to pass valid configs, and panicking on programmer error is idiomatic Rust. + +The sentinel, however, is a library consumed by the Torrust Index application. Panicking on bad user configuration is unacceptable: + +1. The Torrust process would abort if a TOML config file contained an invalid `d_create` / `d_evict` combination. +2. The caller has no opportunity to report the error, retry, or fall back to defaults. +3. Multiple config errors cannot be collected — `assert!` fires on the first violation and halts. + +## Decision · `sec:sentinel:configguard-decision` + +**The sentinel re-derives mudlark's config constraints in its own `SentinelConfig::validate()` method and returns `Result<(), ConfigErrors>` where `ConfigErrors` is a list of `ConfigError` variants.** + +The sentinel's validation runs *before* `GvGraph::new()` is ever called, so mudlark's `assert!` paths are unreachable under normal operation. The redundancy is intentional. + +### Headroom formula · `sec:sentinel:configguard-headroom-formula` + +The most complex constraint is the budget headroom check: + +```rust +fn headroom_requirement(d_create: u32, d_evict: u32) -> Option { + let buffer = d_evict.checked_sub(d_create)?; + let exponent = buffer.checked_add(1)?; + let headroom = 3usize.checked_pow(exponent)?; + let convergence = (d_create as usize).saturating_sub(1).checked_mul(2)?; + Some(headroom.max(convergence)) +} + +// In validate(), reached only once the budget is non-zero and the +// depth pair has cleared its own check: +match headroom_requirement(self.d_create, self.d_evict) { + Some(required) if self.budget <= required => { + errors.push(ConfigError::BudgetTooSmall { + budget: self.budget, + required_minimum: required, + }); + } + Some(_) => {} + None => { + errors.push(ConfigError::DepthBufferTooLarge { + d_create: self.d_create, + d_evict: self.d_evict, + }); + } +} +``` + +This mirrors mudlark's internal calculation. A comment in the sentinel code cross-references the mudlark source so that future changes to the formula can be synchronised. + +## Alternatives Considered · `sec:sentinel:configguard-alternatives` + +- **Catch the panic.** `std::panic::catch_unwind()` around `GvGraph::new()`. Rejected — fragile, opaque (string error), cannot collect multiple violations, and `catch_unwind` is a last resort in idiomatic Rust. + +- **Ask mudlark to return `Result`.** A longer-term option. Even then, sentinel's pre-validation remains useful — it collects *all* errors at once and produces sentinel-specific error types for the UI. + +- **Trust the caller.** Document constraints and `debug_assert!` only. Rejected — config comes from user-edited TOML; any typo could crash the server. + +## Consequences · `sec:sentinel:configguard-consequences` + +- Callers receive structured, actionable `ConfigError` variants listing every constraint violation, not a single panic message. +- If mudlark changes its headroom formula, the sentinel's re-derivation must be updated. The comment `// mirrors mudlark Config::validate()` marks this coupling. +- Mudlark's own `assert!` paths become dead code in production — they serve only as a defence-in-depth safety net. + +### Panic-vs-Result boundary · `sec:sentinel:configguard-panic-result-boundary` + +The ADR's `Result` policy applies to `SentinelConfig` parameters that originate from user-edited TOML. Decay parameters (`att`, `q`) passed to `decay()` and `decay_subtree()` are programmer-controlled call-site values. Invalid decay parameters (negative `att`, `q` outside $[0, 1]$) trigger panics, not `Result` — these are programming errors, not user-input errors. The boundary is: **config = `Result`, call-site = panic**. + +### Depth 128 rejection · `sec:sentinel:configguard-depth-128-rejection` + +At G-tree depth $N$, suffix width $w = N - N = 0$, making the tracker dimensionless — no suffix bits remain for statistical analysis. The rejection is real but it is not a config constraint, and this record is not where it lives. `analysis_depth_cutoff` bounds V-Tree depth, and the analysis set's ancestor closure walks the G-tree independently of the V-depth an entry was selected at, so no value of the cutoff keeps a depth-$N$ cell out of the tracked set; `validate()` accordingly carries no clause for it, and a clause it could carry would have to predict traffic rather than read configuration. What rejects the cell is the suffix-width filter, applied twice at runtime: candidate collection admits an entry only while $w$ is at least `MIN_TRACKER_DIM`, and reconciliation skips anything narrower and counts the skip for the host. With `MIN_TRACKER_DIM` at 2 the effective G-tree depth for a tracked non-root cell is therefore $[1, N-2]$ rather than $[1, N-1]$, and it is a runtime property of the traffic that produced the cell, not a property the configuration can be validated for. ADR-S-011 is the record that owns it. diff --git a/packages/sentinel/adr/005-deterministic-order-and-thread-safety.md b/packages/sentinel/adr/005-deterministic-order-and-thread-safety.md new file mode 100644 index 000000000..cbc34ba24 --- /dev/null +++ b/packages/sentinel/adr/005-deterministic-order-and-thread-safety.md @@ -0,0 +1,72 @@ +# ADR-S-005: Deterministic Order and Thread Safety · `rec:sentinel:ordered-maps-and-static-send-sync-assertion` + +**Status:** Implemented **Date:** 2026-03-08 **Spec:** §ALGO S-11.1 (noise injection), §ALGO S-11.1.3 (seed parameter) **Relates to:** mudlark [ADR-M-007](../../mudlark/adr/007-thread-safety.md) (mudlark thread safety) + +## Context · `sec:sentinel:determinism-context` + +Two low-level invariants govern the sentinel's runtime behaviour: + +1. **Iteration order must be deterministic** across runs for a given seed, so that noise injection, coordination assembly, and report ordering are reproducible. +2. **`SpectralSentinel` must be `Send + Sync`**, because the host will run it on background threads behind `Arc>`. + +Both are implementation choices that are not specified in algorithm.md but are required by the sentinel's operating environment. + +--- + +## Part A — Deterministic Iteration Order · `sec:sentinel:determinism-iteration-order` + +### Decision · `sec:sentinel:determinism-iteration-decision` + +**Use `BTreeMap` (not `HashMap`) for all spatial-key maps.** + +```rust +cells: BTreeMap, +``` + +`BTreeMap` iterates in key order (ascending). `GNodeId` is `Copy + Ord` (backed by `NonZeroU32`), providing a stable, cross-platform iteration order. + +### Why it matters · `sec:sentinel:determinism-iteration-rationale` + +- **Noise injection** — each tracker receives a deterministic RNG sequence seeded from a user-provided seed. If iteration order varies between runs, the same seed produces different per-tracker baselines, making behaviour non-reproducible. +- **Coordination tier** — cell mean-score vectors are assembled into a matrix in iteration order. The meta-tracker's SVD decomposition is sensitive to row ordering when eigenvalues are close. +- **Report ordering** — `cell_reports` appear in the `BatchReport` in map iteration order. Deterministic ordering makes output diffable across runs. + +### Alternatives · `sec:sentinel:determinism-iteration-alternatives` + +| Option | Pros | Cons | +|--------|------|------| +| `HashMap` + sort-on-iterate | O(1) lookup | Sort cost on every noise/report pass; easy to forget | +| `IndexMap` (insertion order) | Preserves insert order | Order depends on traffic arrival, not spatial position; extra dependency | +| `BTreeMap` (chosen) | Deterministic, no extra sort, no extra dep | O(log n) lookup vs O(1) | + +The O(log n) penalty is negligible — cell counts are small relative to the per-tracker SVD cost that dominates runtime. + +--- + +## Part B — Thread Safety via Static Assertion · `sec:sentinel:determinism-thread-safety` + +### Decision · `sec:sentinel:determinism-thread-safety-decision` + +**Enforce `Send + Sync` with a compile-time static assertion.** + +```rust +const _: () = { + const fn assert_send_sync() {} + assert_send_sync::(); +}; +``` + +This is a zero-cost check — no runtime code. If any field ever violates the constraint, the crate fails to compile with a clear error pointing at the assertion. + +### Why a static assertion (not trust-the-derive) · `sec:sentinel:determinism-static-assertion-rationale` + +All current fields are `Send + Sync` by construction. But future additions (`GvGraph`, `SmallRng`, etc.) could silently break this. The static assertion catches breakage at `cargo check` time, not at a downstream integration test. + +- `GvGraph` is `Send + Sync` (mudlark ADR-M-007). +- `SmallRng` is `Send + Sync` (no `Rc` or thread-local state). + +## Consequences · `sec:sentinel:determinism-consequences` + +- Noise injection with a fixed seed produces identical baselines across runs with `background_warming` disabled and on a fixed build — one target and one set of dependency versions. Neither wider claim holds. `SmallRng` is picked for speed and says of itself that it is not portable: it selects its algorithm by target pointer width, so a 32-bit target and a 64-bit one draw different streams from one seed, and it reserves the right to change that algorithm in a later release, which makes the dependency's version and not the compiler's the thing a cross-run comparison has to hold fixed. Under background warming the same seed and the same traffic still give the same graph, the same investment set and the same ascending-handle report order, but neither the baselines a tracker starts from nor the ingest cycle on which it first scores: the warming worker draws from its own generator and takes whichever staged cell leads on volume when it looks. A guarantee reaching across targets and library versions would need a generator that offers one, at a cost this crate has not accepted for a warm-up whose only job is a reasonable starting baseline. +- Reports are spatially ordered without an explicit sort pass. +- Any future `!Send` field (e.g. a raw pointer cache) will be caught immediately, forcing a conscious design choice. diff --git a/packages/sentinel/adr/006-analysis-set-recomputation.md b/packages/sentinel/adr/006-analysis-set-recomputation.md new file mode 100644 index 000000000..846c6c4b3 --- /dev/null +++ b/packages/sentinel/adr/006-analysis-set-recomputation.md @@ -0,0 +1,77 @@ +# ADR-S-006: Analysis Set Recomputation Strategy · `rec:sentinel:full-analysis-set-recomputation-until-profiling` + +**Status:** Decided **Date:** 2026-03-09 **Spec:** §ALGO S-8.1 (analysis set definition), §ALGO S-8.2 (ancestor closure), §ALGO S-9.3 (multi-scale delivery) **Relates to:** [ADR-S-003](003-mudlark-integration.md) (graph parameterisation), [ADR-S-002](002-feed-forward-invariant.md) (feed-forward invariant) + +## Context · `sec:sentinel:recompute-context` + +The analysis set ($\mathcal{A}$) is the top-$K$ V-Tree entries by importance with V-Tree depth $\leq L$, closed under G-tree ancestry (§ALGO S-8.1, §ALGO S-8.2). It determines which cells own `SubspaceTracker`s and therefore controls the sentinel's resource usage. + +After each batch of observations mutates the G-V Graph, the analysis set may change — cells split, merge, gain or lose importance. The sentinel must detect these changes, create trackers for new entries, and destroy trackers for evicted entries. + +Two broad strategies exist: + +1. **Full recomputation:** scan the entire V-Tree via `layers()` after every `ingest()`, rebuild from scratch, diff against the previous set. +2. **Incremental update:** mudlark notifies the sentinel of structural changes via callbacks or a change log; the sentinel patches in place. + +## Decision · `sec:sentinel:recompute-decision` + +**Start with full recomputation. Defer incremental updates until profiling shows the scan is a bottleneck.** + +### Implementation sketch · `sec:sentinel:recompute-implementation-sketch` + +```rust +impl AnalysisSet { + pub fn recompute(graph: &GvGraph, k: usize, l: usize) -> Self { + let mut candidates: Vec> = graph + .layers() + .filter(|(v_depth, _)| *v_depth <= l) + .map(|(v_depth, node)| AnalysisEntry { + gnode: node.gnode_id, + depth: node.depth, + v_depth, + importance: node.own, + start: node.start, + end: node.end, + }) + .collect(); + + candidates.sort_by(|a, b| { + b.importance.cmp(&a.importance) + .then(a.start.cmp(&b.start)) + }); + candidates.truncate(k); + + // Ancestor closure: walk G-tree parents for each competitive entry. + // ... + } +} +``` + +After recomputation, the sentinel diffs old and new sets: + +- **New entries:** create `CellState` + `SubspaceTracker`, noise-inject (ADR-S-007). +- **Removed entries:** destroy the tracker, freeing memory. +- **Retained entries:** keep existing tracker state. + +### Why full scan is acceptable initially · `sec:sentinel:recompute-full-scan-rationale` + +- `layers()` is $O(n)$ in live G-nodes. With the default budget of 100,000, this is at most 100k iterations per `ingest()`. +- The scan is read-only — no allocation, no mutation. +- The sort is $O(n \log n)$ but operates on a filtered subset (entries with `v_depth ≤ L`), typically much smaller than $n$. +- `ingest()` is called once per batch (not per observation). + +### Incremental path (deferred) · `sec:sentinel:recompute-incremental-path` + +An incremental approach requires mudlark to expose a change notification mechanism — either callbacks or a `ChangeLog` buffer. Mudlark does not currently provide either. Adding one is non-trivial and should be driven by measured need, not speculation. + +## Alternatives Considered · `sec:sentinel:recompute-alternatives` + +- **Incremental from the start.** Rejected — premature optimisation. Higher-priority work (suffix analysis, coordination hierarchy) comes first. +- **Periodic recomputation.** Only recompute every $N$ batches. Rejected — stale analysis sets mean trackers may be allocated to cells that no longer exist. +- **Hash-based change detection.** Hash the V-Tree and skip recomputation if unchanged. Rejected — hashing is itself $O(n)$, saving nothing over a full recompute. + +## Consequences · `sec:sentinel:recompute-consequences` + +- Every `ingest()` pays a scan of the V-Tree limited to depth $\leq L$ via `layers_to(depth_cutoff)` (ADR-M-041). Nodes below the cutoff are never enqueued, so the cost is $O(n_L)$ rather than $O(n)$. With $n = 100{,}000$ and batches arriving at ~1 Hz, this is negligible compared to the SVD work in `SubspaceTracker`. +- Tracker creation/destruction follows analysis set churn. If the contour is stable (most batches), the diff is empty. +- If profiling shows the scan is a bottleneck (unlikely until $n > 10^6$), this ADR should be revisited. diff --git a/packages/sentinel/adr/007-automatic-noise-injection.md b/packages/sentinel/adr/007-automatic-noise-injection.md new file mode 100644 index 000000000..3140ec85f --- /dev/null +++ b/packages/sentinel/adr/007-automatic-noise-injection.md @@ -0,0 +1,37 @@ +# ADR-S-007: Automatic Noise Injection · `rec:sentinel:automatic-internal-noise-injection` + +**Status:** Implemented — modified by ADR-S-015 (`noise_rounds` → `noise_schedule`) and ADR-S-017 (deferred cell warm-up and lazy coordination warm-up) **Date:** 2026-03-09 **Spec:** §ALGO S-11.1 (noise generation), §ALGO S-11.1.2 (injection triggers), §ALGO S-11.6.5 (coordination exclusion during cell warm-up), §ALGO S-11.7 (coordination-specific warm-up) **Relates to:** [ADR-S-001](001-measures-not-opinions.md) (measures not opinions), [ADR-S-005](005-deterministic-order-and-thread-safety.md) (deterministic order), [ADR-S-006](006-analysis-set-recomputation.md) (analysis set lifecycle) + +## Context · `sec:sentinel:autonoise-context` + +The spec (§ALGO S-11.1.2) requires noise injection to fire **automatically** on every new tracker creation — analysis set entry, split-induced creation, or legacy promotion. The original coordination design chained synthetic cell scores into parent contexts; ADR-S-017 replaced that lifecycle when it deferred cell warm-up, because warming cells produce no reports and coordination contexts do not exist until online cells participate in a later scoring pass. + +Without noise injection, new trackers start with placeholder baselines (`mean = 1.0`, `variance = 1.0`). Early z-scores and CUSUM values are meaningless until enough real data has passed. + +## Decision · `sec:sentinel:autonoise-decision` + +**Noise injection is automatic and internal. No public API for manual injection.** + +1. **Auto-inject on tracker creation.** When the analysis selector (ADR-S-006) creates a new `CellState`, the sentinel schedules noise injection for it with rounds determined by `noise_schedule.rounds_for_depth(depth)` (ADR-S-015) and `noise_batch_size`. ADR-S-017 moved the injection itself off the creation site: the cell is enqueued into the staging area, and its rounds are worked either in line within the same reconciliation, when background warming is disabled, or one batch at a time on the warming worker when it is enabled. The root tracker is the exception — it is built at construction rather than by the selector, and is warmed there directly. Injection stays automatic in both modes, and a cell whose schedule requests at least one round at its depth reaches real traffic warm; a schedule that requests none leaves nothing to work, so the cell is staged straight to the ready queue and meets real traffic cold, which is the configuration doing what it says rather than the lifecycle failing. What creation no longer promises is that the injection has finished by the time creation returns. + +2. **Noise config in `SentinelConfig`.** The fields `noise_schedule`, `noise_batch_size`, and `noise_seed` are top-level config fields. `noise_schedule` is a `NoiseSchedule` enum with `Geometric` and `Explicit` variants (ADR-S-015 §1). There is no separate `NoiseParams` struct. + +3. **Persistent RNG.** A `SmallRng` is stored on `SpectralSentinel`, seeded from `config.noise_seed` (or system entropy if `None`). With background warming disabled this generator is the whole of the engine's randomness, and a fixed seed fixes the noise for the sentinel's lifetime. With it enabled the warming worker holds a second generator, seeded one above the configured seed so the two streams do not coincide, and the cells it warms draw from that one instead. + +4. **Lazy coordination warm-up.** A cell's synthetic warm-up scores end with that cell; the staging paths do not propagate their reports. After promotion, a scoring pass materialises each newly active coordination context from the online competitive cells that participate there, warms it inline from synthetic score vectors sampled from those cells' baseline moments, resets its drift state, and then feeds it the first real group matrix (§ALGO S-11.6.5, §ALGO S-11.7). + +5. **No manual injection API.** Exposing a public `inject_noise()` method would allow double-injection and create an ordering hazard (host calling it after trackers already received auto-injection). The sentinel is the sole owner of the injection lifecycle. + +## Alternatives Considered · `sec:sentinel:autonoise-alternatives` + +- **Manual injection only.** Let the host decide when to inject. Rejected — the host cannot know when the sentinel creates trackers internally (analysis set churn is opaque). Manual injection is fundamentally incompatible with automatic tracker lifecycle. + +- **Inject lazily on first real observation.** Defer until the tracker receives its first real batch. Rejected — noise injection's purpose is to bootstrap baselines *before* real data, so that early z-scores are meaningful. + +- **Re-seed per injection.** Create a fresh RNG for each injection event. Rejected — this either requires the host to supply a seed per cell (impractical) or uses a single global seed per event (loses reproducibility across different analysis set sizes). + +## Consequences · `sec:sentinel:autonoise-consequences` + +- Every tracker whose schedule requests at least one round at its depth starts warm, and its maturity field `noise_influence` reflects genuine noise decay rather than a cold-start artefact. A schedule can request none — an explicit vector that is empty, or a geometric taper with a zero floor at depths where `root × decay^depth` has fallen below half a round — and a cell with no rounds to work is staged directly to ready and starts cold. What the sentinel guarantees is that it never skips rounds the schedule asked for; where none were asked for, the tracker begins at full noise influence and works it down on real traffic alone, so the maturity field states that immaturity rather than concealing it. +- The persistent RNG means noise sequences depend on creation order, which depends on traffic patterns. This is acceptable — noise need only provide reasonable initial baselines, not be statistically independent across trackers. +- Deterministic order (ADR-S-005) ensures that for a given seed and identical traffic, the noise injection sequence is identical across runs of the synchronous path. Background warming does not preserve it: the worker takes whichever staged cell carries the most volume at the moment it looks, and the main thread is enqueueing and refreshing volumes at the same time, so which cell receives which draw is settled by the interleaving rather than by the seed. diff --git a/packages/sentinel/adr/008-scoring-geometry-extension.md b/packages/sentinel/adr/008-scoring-geometry-extension.md new file mode 100644 index 000000000..713f833da --- /dev/null +++ b/packages/sentinel/adr/008-scoring-geometry-extension.md @@ -0,0 +1,53 @@ +# ADR-S-008: Scoring Geometry Extension · `rec:sentinel:report-scoring-geometry-to-hosts` + +**Status:** Implemented **Date:** 2026-03-08 **Spec:** — (implementation extension, not in algorithm.md) **Relates to:** [ADR-S-001](001-measures-not-opinions.md) (measures not opinions) + +## Context · `sec:sentinel:geometry-context` + +The four scoring axes (§ALGO S-5) have structural preconditions: + +- **Novelty** degenerates when `rank == dim` — the subspace spans the full space, so every observation has zero residual. The novelty score is always 0.0. +- **Coherence** is undefined when `rank < 2` — there are fewer than two latent dimensions, so pairwise cross-correlation has no pairs. + +These conditions are not bugs — they are natural consequences of the tracker's operating state. But a host consuming novelty z-scores without knowing that novelty is currently degenerate would misinterpret the silence as "everything is normal". + +The spec does not define a mechanism for the host to detect these conditions. The host would have to derive them from `rank`, `energy_ratio`, and knowledge of the tracker dimensionality — indirect and error-prone. + +## Decision · `sec:sentinel:geometry-decision` + +**Add `ScoringGeometry` to every report, tracking the geometric operating state of each tracker.** + +```rust +pub struct ScoringGeometry { + /// Whether novelty is degenerate (rank == dim). + pub novelty_saturated: bool, + /// Whether novelty *could* saturate (rank == dim − 1). + pub novelty_saturable: bool, + /// Whether coherence is active (rank >= 2). + pub coherence_active: bool, +} +``` + +And a `GeometryDistribution` summary in `HealthReport` for fleet-wide monitoring: + +```rust +pub struct GeometryDistribution { + pub novelty_saturated: usize, + pub novelty_saturable: usize, + pub coherence_inactive: usize, +} +``` + +## Alternatives Considered · `sec:sentinel:geometry-alternatives` + +| Option | Pros | Cons | +|--------|------|------| +| Let host derive from rank + dim | No new types | Error-prone, requires dim knowledge | +| Boolean flags on `AnomalyScores` | Close to the data | Mixes measurement with metadata | +| Separate `ScoringGeometry` (chosen) | Clean separation, aggregatable | One more type | + +## Consequences · `sec:sentinel:geometry-consequences` + +- The host can filter out novelty alerts when `novelty_saturated` is true, and monitor `coherence_inactive` counts for health. +- `ScoringGeometry` is purely informational — it does not suppress scores. The sentinel still reports novelty = 0.0 when saturated; the host decides whether to ignore it (ADR-S-001). +- This extension should be back-ported to the spec (§ALGO S-14) as a recommended report field. diff --git a/packages/sentinel/adr/009-decay-does-not-invalidate-analysis-set.md b/packages/sentinel/adr/009-decay-does-not-invalidate-analysis-set.md new file mode 100644 index 000000000..f5bd85cdf --- /dev/null +++ b/packages/sentinel/adr/009-decay-does-not-invalidate-analysis-set.md @@ -0,0 +1,27 @@ +# ADR-S-009: Decay Does Not Invalidate the Analysis Set · `rec:sentinel:recompute-on-ingest-without-decay-invalidation` + +**Status:** Decided **Date:** 2026-03-10 **Spec:** §ALGO S-10 (temporal decay) **Relates to:** [ADR-S-006](006-analysis-set-recomputation.md) (analysis set recomputation) + +## Context · `sec:sentinel:decayset-context` + +After `decay()` or `decay_subtree()`, cell importance values change. The analysis set — which selects top-$K$ cells by importance — could become stale. Two strategies: + +- **A)** Set an invalidation flag; force recomputation before the next scoring operation. +- **B)** Do nothing; rely on `ingest()` always recomputing. + +## Decision · `sec:sentinel:decayset-decision` + +**Option B.** The analysis set is recomputed from scratch at the start of every `ingest()` call (ADR-S-006). No scoring occurs between a `decay()` call and the next `ingest()` — the sentinel's API does not expose a "score without ingesting" path. Therefore the analysis set is never stale when it matters. + +## Alternatives Considered · `sec:sentinel:decayset-alternatives` + +| Option | Pros | Cons | +|--------|------|------| +| Invalidation flag (A) | Correct even if API adds a score-only path | Extra state, branching, easy to forget | +| Do nothing (B) | Simpler, zero overhead, no new state | Requires revisiting if score-only API is added | + +## Consequences · `sec:sentinel:decayset-consequences` + +- `decay()` is a pure spatial operation with no side-effects on the analysis tier. This keeps the temporal and analytical concerns cleanly separated (§ALGO S-10). +- If a future API adds "score against current model without new observations", this ADR must be revisited — the analysis set would need recomputation or validation before scoring. +- The `inspect_cell()` method reads existing tracker state (no recomputation) and is unaffected — it reports from the last `ingest()` cycle's model. diff --git a/packages/sentinel/adr/010-linear-routing-over-g-tree-descent.md b/packages/sentinel/adr/010-linear-routing-over-g-tree-descent.md new file mode 100644 index 000000000..ee8f38c88 --- /dev/null +++ b/packages/sentinel/adr/010-linear-routing-over-g-tree-descent.md @@ -0,0 +1,33 @@ +# ADR-S-010: Linear Routing Over G-Tree Descent · `rec:sentinel:linear-containment-routing-until-profiled` + +**Status:** Decided **Date:** 2026-03-10 **Spec:** §ALGO S-9.1 Step 4 (observation delivery) **Relates to:** [ADR-S-003](003-mudlark-integration.md) (mudlark integration), [ADR-S-006](006-analysis-set-recomputation.md) (analysis set recomputation) + +## Context · `sec:sentinel:routing-context` + +During Step 4 of `ingest()` (§ALGO S-9.1), each observation must be delivered to every tracker on its G-tree ancestor path. Two routing strategies: + +- **A)** G-tree descent: use `graph.route(v)` to find the receiving cell, then walk G-tree parents. Requires a mudlark API for parent traversal that returns materialised G-node IDs matching the sentinel's `BTreeMap` keys. +- **B)** Linear scan: for each observation, iterate all cells in the analysis set and check interval containment (`cell.start <= v < + cell.end`). Cost: $O(n \cdot |\mathcal{A}^*|)$ per batch. + +## Decision · `sec:sentinel:routing-decision` + +**Option B (linear scan).** G-tree descent requires a `route_to_ancestors()` or equivalent API from mudlark that returns the chain of `GNodeId` values for materialised ancestors. This API does not currently exist. The G-tree's intervals are dyadic and nested, so flat interval containment is correct — every ancestor's interval contains the observation by construction. + +## Performance · `sec:sentinel:routing-performance` + +At typical operating points ($|\mathcal{A}^*| \leq 2K \approx 2{,}048$, batch size $b \leq 64$), the scan is $\sim 130{,}000$ comparisons per batch — negligible relative to the SVD cost that dominates `ingest()`. Profile before optimising. + +## Alternatives Considered · `sec:sentinel:routing-alternatives` + +| Option | Pros | Cons | +|--------|------|------| +| G-tree descent (A) | $O(d)$ per observation | Requires non-existent mudlark API | +| Binary search on sorted intervals | $O(n \log |\mathcal{A}^*|)$ | Added complexity for marginal gain at current $K$ | +| Linear scan (B) | No external dependency, simple, correct | $O(n \cdot |\mathcal{A}^*|)$ per batch | + +## Consequences · `sec:sentinel:routing-consequences` + +- No dependency on a mudlark parent-traversal API. The sentinel operates on its own `BTreeMap` using interval metadata cached at tracker creation time. +- If profiling identifies routing as a bottleneck at large $K$, the first optimisation step is sorting the analysis set by interval and using binary search ($O(n \log |\mathcal{A}^*|)$), not adding a mudlark API dependency. +- If mudlark later exposes `route_to_ancestors()`, this ADR can be revisited for a constant-factor improvement. diff --git a/packages/sentinel/adr/011-degenerate-cell-dimension-guard.md b/packages/sentinel/adr/011-degenerate-cell-dimension-guard.md new file mode 100644 index 000000000..4a330eb93 --- /dev/null +++ b/packages/sentinel/adr/011-degenerate-cell-dimension-guard.md @@ -0,0 +1,113 @@ +# ADR-S-011: Degenerate Cell Dimension Guard · `rec:sentinel:exclude-and-report-degenerate-tracker-dimensions` + +**Status:** Implemented **Date:** 2026-03-10 **Spec:** §ALGO S-8.2 (analysis set closure), §ALGO S-4 (subspace tracker) **Relates to:** [ADR-S-004](004-config-validation-over-panic.md) (config validation over panic), [ADR-S-006](006-analysis-set-recomputation.md) (analysis set recomputation), [ADR-S-007](007-automatic-noise-injection.md) (automatic noise injection) + +## Context · `sec:sentinel:dimguard-context` + +`SubspaceTracker` operates on suffix bit slices of width $w = N - d$, where $d$ is the G-tree bit depth of the cell. When $d = N$ (e.g. $d = 128$ at $N = 128$), $w = 0$: no suffix bits remain, and the tracker's working space is zero-dimensional. Values of $w$ that are very small (1–2) are technically representable but yield degenerate subspace models with no practical statistical value. + +### The bug · `sec:sentinel:dimguard-bug` + +`SubspaceTracker::new(dim, cfg, slow_decay)` unconditionally sets `rank = 1` at construction. However, the rank capacity is `cap = min(dim, max_rank)`. When `dim = 0`: + +- `cap = 0`, but `rank = 1` — **rank exceeds capacity**. +- `basis` is a $0 \times 0$ matrix. +- The first call to `observe()` attempts `self.basis.subcols(0, 1)` — extracting column 0..1 from a zero-column matrix — which **panics inside faer's SVD**. + +This is not a theoretical concern. Under spray traffic with a low `split_threshold`, the G-tree creates very deep nodes. The analysis set's ancestor closure (§ALGO S-8.2) can pull these deep nodes into the tracked set, where `reconcile_analysis_set()` creates `SubspaceTracker::new(0, ...)` and the next `observe()` panics. + +### Why the existing guards are insufficient · `sec:sentinel:dimguard-existing-guards-insufficient` + +1. **Config validation** **([ADR-S-004](004-config-validation-over-panic.md))** mentions "Depth 128 rejection" as a constraint on `analysis_depth_cutoff`, but the cutoff filters on **V-tree depth**, not G-tree bit depth. The ancestor closure walks the G-tree and can include nodes at any bit depth regardless of the cutoff. + +2. **`max_rank >= 1`** is validated at config time, but `cap` is `min(dim, max_rank)`, so a valid `max_rank` does not prevent `cap = 0` at runtime. + +3. **No runtime guard** exists in `SubspaceTracker::new()`, `reconcile_analysis_set()`, or the scoring path. The panic propagates uncaught from faer into the host application. + +## Decision · `sec:sentinel:dimguard-decision` + +**The sentinel must detect degenerate cell dimensions at runtime and report the condition through its measurement API rather than panicking.** + +### 1. Minimum dimension constant · `sec:sentinel:dimguard-minimum-dimension` + +Introduce a crate-level constant: + +```rust +/// Minimum suffix width for a functional subspace tracker. +/// +/// At `dim < MIN_TRACKER_DIM`, the tracker cannot form a +/// meaningful basis or compute residuals. Cells below this +/// threshold are excluded from the analysis set; a cell at it +/// is kept, which is what makes the value a minimum rather +/// than a floor the analysis set sits above. +pub(crate) const MIN_TRACKER_DIM: usize = 2; +``` + +The value 2 ensures at least one residual degree of freedom ($d - k \geq 1$ when $k = 1$). A tracker with `dim = 1` can technically run but produces identically-zero residuals (novelty) since the single basis vector spans the entire space — making it statistically useless. + +### 2. Guard in `reconcile_analysis_set()` · `sec:sentinel:dimguard-reconcile-guard` + +Before creating a `CellState`, check: + +```rust +let width = 128usize.saturating_sub(entry.depth as usize); +if width < MIN_TRACKER_DIM { + tracing::warn!( + gnode = %entry.gnode, + depth = entry.depth, + width, + "skipping degenerate cell (width < MIN_TRACKER_DIM)" + ); + continue; +} +``` + +Cells that fail the guard are **not tracked** — they receive no `SubspaceTracker`, appear in no reports, and their traffic still flows through the G-V graph normally (observation routing is unaffected). + +### 3. Guard in `SubspaceTracker::new()` (defence in depth) · `sec:sentinel:dimguard-constructor-defence` + +As a second line of defence, the constructor asserts internally: + +```rust +debug_assert!( + dim >= MIN_TRACKER_DIM, + "SubspaceTracker::new() called with dim={dim}, \ + expected >= {MIN_TRACKER_DIM} (caller should have filtered)" +); +``` + +This is a `debug_assert!` (not a runtime error) because the reconciliation guard should make it unreachable. If it fires in debug builds, it signals a missed filter site. + +### 4. Report the condition · `sec:sentinel:dimguard-condition-reporting` + +Add a counter to `AnalysisSetSummary`: + +```rust +/// Number of G-tree nodes excluded from tracking because their +/// suffix width was below `MIN_TRACKER_DIM`. +pub degenerate_cells_skipped: usize, +``` + +This follows [ADR-S-001](001-measures-not-opinions.md): the sentinel **measures** the degenerate condition; the host decides whether to log, alert, or ignore it. + +## Alternatives Considered · `sec:sentinel:dimguard-alternatives` + +- **Cap the rank to `dim` and allow `dim = 0` gracefully.** Setting `rank = 0` when `cap = 0` avoids the panic, but a zero-rank tracker produces no meaningful scores — every axis returns zero. Silent zeros in the report are worse than an explicit skip, because the host cannot distinguish "no anomaly" from "unable to measure". The report would lie. + +- **Clamp `dim` to a floor of 1 inside the tracker.** This hides the degeneracy from the caller. The 1-dimensional tracker produces identically-zero novelty scores (because the sole basis vector captures 100% of variance), misleading the host into thinking the cell is perpetually normal. + +- **Reject configs whose parameters *could* produce depth-128 nodes.** Impractical — whether depth 128 is reached depends on runtime traffic, not solely on config. A `split_threshold` of 10 with budget 10 000 is a valid configuration that *might* hit depth 128 under unlucky traffic but usually does not. + +- **Panic with a clear message.** Violates [ADR-S-004](004-config-validation-over-panic.md). The sentinel is a library inside the Torrust Index; panicking on a runtime traffic pattern is unacceptable. + +## Consequences · `sec:sentinel:dimguard-consequences` + +- **No more panics** from degenerate cell dimensions. The faer SVD code path is never reached with a zero- or one-dimensional input. + +- **`degenerate_cells_skipped`** in the report gives the host visibility into deep-tree pressure. A persistently non-zero count may indicate the `split_threshold` is too low for the traffic mix. + +- **Complete ancestor chains for admitted targets.** Competitive candidates are filtered by suffix width before selection. Every ancestor of an admitted target is shallower and therefore at least as wide, so ancestor closure cannot encounter a degenerate node or leave a reporting gap. `degenerate_cells_skipped` measures rejected candidates without changing the complete chain of any target the selector admits. + +- **Performance: eliminates wasted SVD churn.** Even when `dim` is small but positive (e.g. 1–3), the tracker still runs the full SVD pipeline on every `observe()` call: build the $(d \times (k + b))$ augmented matrix, compute thin SVD, update basis, evolve latent statistics. With so few dimensions the rank adapter oscillates — it trivially meets the energy threshold at $k = 1$, bumps to $k = \text{cap}$ on the next interval, then drops back — rebuilding the basis each flip. The scores produced carry no meaningful statistical signal (novelty is identically zero at $d = 1$ since the single basis vector spans the entire space; at $d = 2$ the residual DOF is 1, giving trivially noisy scores). Each degenerate cell therefore burns per-batch SVD cost for zero diagnostic value. In adversarial spray scenarios many deep cells can accumulate, multiplying this waste. Skipping them at `MIN_TRACKER_DIM` eliminates the churn entirely. + +- **`MIN_TRACKER_DIM` is a compile-time constant**, not a config field. It reflects an inherent limitation of the linear algebra (need $\geq 2$ dimensions for a non-trivial subspace), not a tuning knob. The default value should be **conservative** — set high enough to avoid not only the panic ($w = 0$) but also the SVD churn regime described above. A value of 3 or 4 would be defensible (guaranteeing $\geq 2$ residual DOF at rank 1, and room for rank adaptation without instant saturation), but 2 is the theoretical minimum that prevents the panic and provides at least one residual degree of freedom. If profiling reveals measurable SVD overhead from shallow cells in production traffic, raising the constant to 4 is a safe, backward- compatible change — it only removes cells that were contributing noise to the report anyway. diff --git a/packages/sentinel/adr/012-test-duration-budget.md b/packages/sentinel/adr/012-test-duration-budget.md new file mode 100644 index 000000000..37b1ab403 --- /dev/null +++ b/packages/sentinel/adr/012-test-duration-budget.md @@ -0,0 +1,164 @@ +# ADR-S-012: Test Duration Budget · `rec:sentinel:individual-release-test-five-second-budget` + +**Status:** Accepted **Date:** 2026-03-12 **Spec:** — (test-suite discipline, with no section in the algorithm specification) **Relates to:** [ADR-S-007](007-automatic-noise-injection.md) (automatic noise injection — warm-up costs), [ADR-S-013](013-warm-up-convergence-benchmark.md) (convergence benchmark — empirical iteration counts) + +## Context · `sec:sentinel:testbudget-context` + +When this ADR was first proposed the sentinel's test suite took **~1 470 s sum-of-parts in release mode** and **~3 362 s with `CARGO_PROFILE_DEV_OPT_LEVEL=3`**. Of the then-347 tests, **56 exceeded 5 seconds in release** and the three worst exceeded **47 seconds each** (the single worst, `single_observation_repeated`, hit 68 s). + +Following a systematic effort guided by the reduction strategies below, the suite was brought to ~251 s sum-of-parts (release) at the time of the original measurement (2026-03-12, 341 tests). The suite has since grown to **551 tests** (283 unit + 264 integration + 4 doc-tests, plus 3 ignored). The current state is: + +| Metric | Before | Current | Improvement | +| -------------------------- | -------: | -----------: | :---------: | +| Sum-of-parts (release) | ~1 470 s | **~162 s** | **9.1×** | +| Sum-of-parts (debug opt-3) | ~3 362 s | **~411 s** | **8.2×** | +| Tests > 5 s (release) | 56 | **3** | **18.7×** | +| Worst individual test | 68.0 s | **6.6 s** | **10.3×** | +| Critical-path binary | 68.0 s | **14.9 s** | **4.6×** | + +Sum-of-parts and per-test times are measured with `--test-threads=1` (serial execution within each binary) for reproducibility. The critical-path binary uses default parallelism (same as the developer invokes `cargo test`). See [Measurement methodology](#measurement-methodology) below. + +The per-test average is **~0.29 s release** (~0.75 s debug opt-3). The 3 remaining offenders above 5 s in serial mode are in **unit tests** (1) and **spray_resistance** (2). An additional ~15 tests hover in the 4–6 s range and may appear above 5 s in any given parallel run due to hybrid CPU core placement variance (P-core vs E-core; see note below). + +### Measurement methodology · `sec:sentinel:testbudget-measurement-methodology` + +Per-test times come from serial runs (`--test-threads=1 +-Zunstable-options --report-time`) on `rustc 1.96.0-nightly` (2026-03-14). Serial execution gives stable per-test timings free from cross-test contention. Release timings use `--release`. Debug timings use `CARGO_PROFILE_DEV_OPT_LEVEL=3`. Binary-level wall-clock times use default parallelism (no `--test-threads` flag). + +Measurements were taken on 2026-03-25. + +**Test hardware:** + +| Component | Detail | +| ------------ | ------------------------------------------------------------ | +| CPU | Intel Core i7-1370P (Raptor Lake, 13th Gen) | +| Topology | 14 cores / 20 threads — 6 P-cores (HT) + 8 E-cores, 1 socket | +| P-core turbo | up to 5.2 GHz | +| E-core turbo | up to 3.9 GHz | +| L1d / L1i | 544 KiB / 704 KiB (14 instances) | +| L2 | 11.5 MiB (8 instances) | +| L3 | 24 MiB (shared) | +| RAM | 64 GB LPDDR5-6400 (configured at 6000 MT/s) | + +The sentinel's working sets (tracker matrices are ≤ 128 × `max_rank` × 8 bytes ≈ few KiB each) fit comfortably in L2. The SVD-dominated tests are **compute-bound on ALU throughput**, not memory-bound — consistent with the large debug→release speedup ratios seen for compute-heavy suites (coverage_matrix 4.6×, edge_cases 9.3×) where release-mode autovectorisation and inlining of faer's inner loops make the critical difference. + +> **Hybrid scheduling note.** Cargo's test harness uses `std::thread`, and the OS scheduler freely places threads on P-cores or E-cores. A compute-bound test running on an E-core can be 25–35 % slower than on a P-core. All timings in this ADR are from single runs and subject to this variance; the 5 s budget is chosen conservatively to absorb E-core placement. + +## Decision · `sec:sentinel:testbudget-decision` + +**Every individual test in the sentinel crate should complete in ≤ 5 seconds in release mode.** + +Of 551 tests (283 unit + 264 integration + 4 doc-tests), **3 exceed 5 s in serial release mode**. One additional doc-test's binary wall-clock exceeds 5 s but is dominated by merged-doctest compilation overhead. The worst non-doc-test offender is 6.6 s. + +### Remaining offenders (serial release mode) · `sec:sentinel:testbudget-remaining-offenders` + +| # | Test | Suite | Release (s) | Debug (s) | +| --: | -------------------------------------------------------------------- | ---------------- | ----------: | --------: | +| 1 | `tests::convergence_fixes::production_lambda_converges_within_bound` | unit tests | 6.6 | 19.2 | +| 2 | `cells_tracked_bounded_under_diverse_traffic` | spray_resistance | 5.7 | 14.3 | +| 3 | `g_nodes_bounded_by_budget_under_spray` | spray_resistance | 5.1 | 14.9 | + +Release timings are from isolated single-test invocations where serial-run thermal variance can be excluded. `production_lambda_converges_within_bound` is irreducibly expensive: 1 500 rounds of Brand SVD at λ = 0.99, b = 16 is the minimum validated by ADR-S-013 for the 1 200-round convergence bound. The two spray_resistance tests exercise the full graph lifecycle under adversarial traffic and produce many cell splits with noise warm-up. + +The **doc-test** binary wall-clock is 14.9 s (release), but this includes merged-doctest compilation overhead; individual doc-tests run in 2–4 s. + +### Targeted improvements (release mode) · `sec:sentinel:testbudget-targeted-improvements` + +The following tests were targeted in the latest reduction round. Before/after timings are from serial measurement: + +| Test | Suite | Before (s) | After (s) | Strategy | +| ----------------------------------------------------------------------- | --------------- | ---------: | --------: | -------- | +| `tests::variance_formula::batch_size_invariant_surprise_ratio` | unit tests | 17.3 | 1.0 | §7, §8 | +| `tests::convergence_fixes::production_lambda_converges_within_bound` | unit tests | 9.1 | 6.6 | §7 | +| `step3_higher_volume_cells_warm_via_priority` | deferred_warmup | 9.3 | 2.5 | §9 | +| `step2_reset_clears_staging_and_resumes` | deferred_warmup | 7.7 | 1.4 | §8 | +| `step3_concurrent_ingest_no_panic` | deferred_warmup | 6.6 | < 1 | §8 | +| `step2_lifecycle_new_cells_appear_after_splits` | deferred_warmup | 6.3 | < 1 | §8 | +| `tests::convergence_fixes::per_axis_convergence_b4_robust_across_seeds` | unit tests | 5.7 | 0.1 | §7 | +| `graph_accessor_starts_with_single_root` | api | 5.5 | 0.03 | §10 | +| `new_with_default_config_succeeds` | api | 5.5 | 0.00 | §10 | +| `step3_reset_restarts_background_thread` | deferred_warmup | 5.4 | < 1 | §8 | + +### Affected suites — summary · `sec:sentinel:testbudget-affected-suites` + +| Suite | Tests > 5 s | Worst (s) | Pattern | +| -------------------- | ----------: | --------: | ------------------------------------------------------------------------------------------------------- | +| **unit tests** | 1 | 6.6 | Production-config convergence: 1 500 slow-EWMA rounds (λ = 0.99) with Brand SVD kernel. | +| **spray_resistance** | 2 | 5.7 | Full graph lifecycle with many cell splits and noise warm-up under adversarial traffic. | + +### Binary-level wall-clock times · `sec:sentinel:testbudget-binary-wall-clock-times` + +Cargo runs each integration test file as a separate binary with internal parallelism. The suite's total wall-clock equals the sum of all binary wall-clocks (binaries run sequentially). + +| Binary | Release (s) | Debug (s) | Speedup | +| ------------------------- | ----------: | --------: | -------: | +| unit tests | 7.0 | 20.5 | 2.9× | +| ancestor_chain | 2.0 | 8.4 | 4.2× | +| api | 0.3 | 0.6 | 2.3× | +| clip_pressure | 3.9 | 17.3 | 4.4× | +| coverage_matrix | 3.9 | 18.0 | 4.6× | +| deferred_warmup | 5.8 | 29.2 | 5.1× | +| determinism | 1.0 | 10.4 | 10.1× | +| edge_cases | 2.8 | 26.1 | 9.3× | +| graph_routing | 3.3 | 4.8 | 1.4× | +| health | 0.1 | 0.2 | 2.2× | +| hierarchical_coordination | 2.1 | 5.0 | 2.5× | +| integration | 4.5 | 8.5 | 1.9× | +| invariants | 7.1 | 17.0 | 2.4× | +| noise | 3.9 | 9.8 | 2.5× | +| report_structure | 0.6 | 1.0 | 1.7× | +| sentinel_u64 | 0.8 | 1.8 | 2.3× | +| serde_roundtrip | 0.0 | 0.0 | — | +| spatial_decay | 1.4 | 2.3 | 1.6× | +| spray_resistance | 8.8 | 19.5 | 2.2× | +| suffix_analysis | 0.7 | 1.3 | 1.7× | +| warm_up | 6.7 | 13.2 | 2.0× | +| doc-tests | 14.9 | 34.2 | 2.3× | +| **Total** | **81.6** | **249.0** | **3.1×** | + +The critical-path binary in release is **doc-tests at 14.9 s** (dominated by merged-doctest compilation), followed by **spray_resistance at 8.8 s** and **unit tests at 7.0 s** (reduced from 18.8 s). + +### Reduction strategies (applied) · `sec:sentinel:testbudget-reduction-strategies` + +The following strategies were used to bring the suite from 56 offenders down to the current level: + +1. **Reduced iteration counts to empirically validated minimums (ADR-S-013).** Most slow tests used conservatively chosen loop counts (50–500 batches). ADR-S-013's convergence benchmark established the empirical settling batch $n_{\text{settled}}$ and provided justified minimums. + +2. **Lightened the shared `rich_report()` setup** in `serde_roundtrip`. The serde tests were restructured — the suite now runs 0 tests (the expensive report-building helpers were eliminated), dropping all 7 former offenders (42+ s each). + +3. **Lowered `split_threshold` and `budget` in test configs.** Graph-heavy tests now use tighter parameters to reach the same structural depth with fewer observations. + +4. **Fixed `single_observation_repeated` (was 68 s, now 1.9 s).** Smaller iteration count and controlled seed validate the same edge-case property in a fraction of the time. + +5. **Rationalised `deferred_warmup` lifecycle tests.** The warm-up batch counts were reduced using ADR-S-013 minimums — all 14 tests now finish in ≤ 4.5 s (release), down from a worst of 33.6 s. + +6. **Used minimum validated `noise_rounds`** per ADR-S-013. Tests that do not specifically validate noise behaviour now use $\max(r_{\text{noise}}, 5)$ rounds. + +7. **Reduced observation dimensionality in convergence tests.** The EWMA convergence and surprise-ratio properties validated by the unit-test convergence and variance modules are per-axis — they depend on λ and batch size, not on the observation dimension. Tests that dominated the unit-test binary were changed from `dim = 128` to `dim = 32`, cutting SVD cost ~4× without affecting the validated property. + +8. **Halved warm-up iterations and noise schedule in tests that were over-provisioned.** `batch_size_invariant_surprise_ratio` used 200 warm-up + 200 measurement rounds where 100 + 100 suffices at λ = 0.95 (half-life = 14 rounds, 95% settled by round 42). `fast_split_config()` in deferred_warmup tests used `NoiseSchedule::Explicit(vec![5])` where `vec![3]` is sufficient: 3 rounds × batch_size = 4 = 12 noise observations gives η = 0.90^12 ≈ 0.28, adequate for tests that only assert `noise_observations > 0`. + +9. **Reduced noise schedule in `step3_higher_volume_cells_warm_via + _priority`.** The test uses a large noise schedule to observe partial background warming. Reduced from 100 to 30 rounds — still slow enough for partial-warming observation, but 3.3× fewer SVD passes. + +10. **Replaced full sentinel construction with `validate()` for config-validation tests.** Two api tests (`new_with_default + _config_succeeds`, `graph_accessor_starts_with_single_root`) used `SentinelConfig::default()` which triggers 450 root noise rounds. `new_with_default_config_succeeds` now calls `cfg.validate()` instead; `graph_accessor_starts_with_single + _root` now uses `test_config()` (5 noise rounds). + +## Alternatives Considered · `sec:sentinel:testbudget-alternatives` + +- **Accept slow tests; rely on CI parallelism.** Current CI already runs the suite in parallel across multiple runners, but individual test binaries are serialised. A single slow test blocks its binary's thread pool regardless of runner count. This also does not address the developer-flow problem. + +- **Move slow tests to a separate `slow_tests` feature gate.** Adds cognitive overhead ("did I run the full suite?") and fragments coverage. The `#[ignore]` + `--include-ignored` pattern is standard Rust and better supported by tooling. + +- **Profile-guided optimisation of the sentinel itself.** Would help the tight SVD loops but does not address the graph-structure-bound tests, and adds build complexity. + +## Consequences · `sec:sentinel:testbudget-consequences` + +- **The 5 s budget is a standing guideline.** New tests that exceed 5 s in release mode should be flagged in review and either trimmed or marked `#[ignore]`. The 3 remaining offenders (worst 6.6 s) are close to budget and have no natural reduction path without weakening their validated properties. + +- **Overall suite performance.** The sequential binary wall-clock sum is **~82 s in release** and **~249 s in debug (opt-3)**. With Cargo's default parallelism on this 20-thread hybrid CPU the practical wall-clock is **under 15 s in release**, which is acceptable for local iteration. + +- **Regression safety net.** ADR-S-013's `convergence_matches_theory` and `noise_baselines_converge` tests detect if code changes (EWMA update rule, outlier clipping, maturity formula) shift the convergence rate. This prevents iteration counts from silently becoming insufficient. + +- **`#[ignore]`d long-run variants** should be run in nightly CI to retain full convergence coverage without penalising the default suite. diff --git a/packages/sentinel/adr/013-warm-up-convergence-benchmark.md b/packages/sentinel/adr/013-warm-up-convergence-benchmark.md new file mode 100644 index 000000000..69d187dc1 --- /dev/null +++ b/packages/sentinel/adr/013-warm-up-convergence-benchmark.md @@ -0,0 +1,308 @@ +# ADR-S-013: Warm-Up Convergence Benchmark · `rec:sentinel:benchmark-convergence-and-fix-three-root-causes` + +**Status:** Accepted (core fixes implemented; config and spec updates remain) **Date:** 2026-03-10 **Spec:** §ALGO S-11.5 (maturity tracking), §ALGO S-11.8 (system-level warm-up), §ALGO S-6.1.1 (EWMA baseline tracking and outlier clipping), §ALGO S-4.2 Phase 3 (latent distribution) **Relates to:** [ADR-S-007](007-automatic-noise-injection.md) (automatic noise injection), [ADR-S-012](012-test-duration-budget.md) (test duration budget), [ADR-S-001](001-measures-not-opinions.md) (measures not opinions), [ADR-S-014](014-subspace-tracker-visibility.md) (subspace tracker visibility), [ADR-S-015](015-cell-creation-performance.md) (cell creation performance), [ADR-S-016](016-brand-incremental-svd.md) (Brand's incremental SVD) + +**Findings:** Four rounds of investigation (preliminary → secondary → code audit → tertiary synthesis) plus post-fix quaternary validation and recommendations. All six findings documents have been retired; their essential content is captured in this ADR and in the spec updates to §ALGO S-4.2, §ALGO S-6.1.1, §ALGO S-11.4, §ALGO S-11.5, and §ALGO S-11.6. + +## Context · `sec:sentinel:warmbench-context` + +The sentinel's warm-up tests (`warm_up.rs`) and many other test suites use **arbitrary iteration counts** — 50, 100, or even 500 warm-up batches — with no empirical basis for why those numbers were chosen. ADR-S-012 identifies the `warm_up` suite as the worst offender (135.6 s for `phase4_steady_state_maturity`, which asserts `noise_influence < 0.5` after 100 warm-up batches). + +The maturity model (§ALGO S-11.5) is a closed-form exponential decay: + +$$\eta_n = \lambda^n$$ + +where $\eta$ is the noise influence fraction and $\lambda$ is the forgetting factor. For a given $\lambda$, the number of real batches required to reach any target $\eta^*$ is exactly: + +$$n^* = \left\lceil \frac{\ln \eta^*}{\ln \lambda} \right\rceil$$ + +At `integration_config()` values ($\lambda = 0.95$): + +| Target $\eta^*$ | Required batches $n^*$ | +|-----------------:|-----------------------:| +| 0.50 | 14 | +| 0.10 | 45 | +| 0.05 | 59 | +| 0.01 | 90 | + +Yet `phase4_steady_state_maturity` uses **100 warm-up batches** (each with 20 seed values) merely to assert $\eta < 0.5$ — a condition that is theoretically met after **14 batches**. + +### The missing piece (original hypothesis) · `sec:sentinel:warmbench-missing-piece-hypothesis` + +The original proposal hypothesised that EWMA baselines converge at roughly the same rate as $\eta$, giving `n_settled ≤ 30` (real data) and `r_noise ≤ 20` (noise injection). This was based on the assumption that the simple exponential model $\eta_n = \lambda^n$ would approximate baseline convergence. + +**This hypothesis was wrong by an order of magnitude.** + +### What the investigation discovered · `sec:sentinel:warmbench-investigation-findings` + +Four rounds of empirical investigation — 40+ targeted tests, a line-by-line code audit, and multi-seed robustness analysis — revealed that EWMA baseline convergence is governed by three interacting mechanisms that the simple exponential model cannot capture: + +1. **A clipping-ceiling positive feedback loop** (~85% of the convergence gap). The EWMA outlier clip ceiling is computed from the EWMA's own nascent statistics. The first batch produces near-zero variance → ultra-tight ceiling (0.28) → ~31% of valid scores are clipped every round → EWMA mean stays artificially low → ceiling stays tight. This self-reinforcing loop kept surprise baselines drifting for **1000+ rounds** before the fix. + +2. **A latent variance cold-start cascade** (~6% of the gap). `SubspaceTracker::new()` initialised `lat_var` at 1.0 when the true steady-state value is ~0.19 (5.3× mismatch). The surprise score $= z^2 / \nu$ was non-stationary for ~60 rounds as $\nu$ decayed, creating a serial cascade: lat_var EWMA → surprise scores → surprise baseline EWMA. + +3. **Stochastic batch variance** (~6% of the gap). At batch_size=4, per-batch score variance has CV ≈ 0.43, causing the EWMA to overshoot and undershoot around its target. + +Additionally, the **convergence metric itself was broken**. The original `find_settled_round()` function (5% tolerance against the last-round reference) was unfalsifiable for 3 of 4 axes: surprise, coherence, and displacement baselines wander with 8–17% CV even at true steady state, so the metric measured *trace length*, not convergence. + +### No bug — a design trap · `sec:sentinel:warmbench-design-trap` + +A line-by-line code audit confirmed that the implementation faithfully follows §ALGO S-6.1.1. The slow convergence was a *design consequence* of applying attack-resistant outlier clipping from round 1 using nascent statistics, not a coding error. + +## Decision · `sec:sentinel:warmbench-decision` + +**Add a convergence benchmark and characterisation test suite** that empirically measures baseline convergence, **and fix the three root causes** that make convergence 5–10× slower than the theoretical model predicts. + +### Implemented fixes · `sec:sentinel:warmbench-implemented-fixes` + +#### Fix 1: Graduated clip-exemption (§ALGO S-6.1.1) · `sec:sentinel:warmbench-graduated-clip-exemption` + +The effective clip width now scales with noise influence $\eta$: + +$$n_\sigma^{\text{eff}} = n_\sigma + \frac{n_\sigma \cdot \eta}{1 - \eta + \varepsilon}$$ + +| $\eta$ | $n_\sigma^{\text{eff}}$ (at $n_\sigma = 3$) | Behaviour | +|-------:|--------------------------------------------:|-----------| +| 1.0 | ~3,000,003 (capped by $\varepsilon$) | No clipping (cold) | +| 0.99 | 300 | Very wide ceiling | +| 0.50 | 6.0 | Moderately relaxed | +| 0.01 | 3.03 | Near-production | +| 0.0 | 3.00 | Production (exact) | + +Monotonicity is confirmed: $n_\sigma^{\text{eff}}$ is strictly decreasing as $\eta$ decreases. At $\eta = 0$ the effective clip equals the configured `clip_sigmas` exactly. + +**Result:** Surprise convergence improved from **1000+ rounds to ~65 rounds** at $\lambda = 0.95$, $b = 4$ — a 15× improvement. + +**Code:** `tracker.rs` Phase 4 (`observe()`), +13 lines. + +#### Fix 2: Latent cold→warm initialisation (§ALGO S-4.2 Phase 3) · `sec:sentinel:warmbench-latent-cold-warm-initialisation` + +On the first batch (`step == 0`), `lat_mean`, `lat_var`, and `cross_corr` are seeded directly from data rather than blended with the initial defaults: + +```rust +if self.step == 0 { + self.lat_mean[j] = col_mean; + self.lat_var[j] = col_var.max(eps); +} else { + self.lat_mean[j] = lam.mul_add(self.lat_mean[j], alpha * col_mean); + self.lat_var[j] = lam.mul_add(self.lat_var[j], alpha * col_var.max(eps)); +} +``` + +**Result:** Surprise rise factor improved from **5.9× to 1.26×**. + +**Code:** `tracker.rs` Phase 3 (`evolve_latent()`), +12 lines. + +#### Fix 3: CUSUM fast-slow gap seeding · `sec:sentinel:warmbench-cusum-gap-seeding` + +After noise injection completes, the slow EWMA ($\lambda_s = 0.999$, half-life 693 steps) is seeded from the fast EWMA's converged values, and CUSUM accumulators are reset: + +```rust +// In inject_noise_into_cell(), after noise rounds complete: +cell.tracker.seed_cusum_slow_from_baselines(); +cell.tracker.reset_cusum(); +``` + +**Result:** False CUSUM drift reduced from **198 to 5.7** (97% reduction). + +**Code:** `ewma.rs` `seed_from()`, `cusum.rs` `seed_slow_from()`, `tracker.rs` `seed_cusum_slow_from_baselines()`, `sentinel/mod.rs` `inject_noise_into_cell()`. + +### Convergence test suite · `sec:sentinel:warmbench-convergence-test-suite` + +> **Note (2026-03-11):** The five original files listed below have been consolidated into a cleaner structure. The new layout is: +> +> - `src/tests/convergence_common.rs` — shared configs, noise generation, metrics +> - `src/tests/convergence_ewma.rs` — pure EWMA property tests (5 tests) +> - `src/tests/convergence_eta.rs` — η tracking & maturity tests (4 tests) +> - `src/tests/convergence_noise.rs` — tracker baseline convergence (9 tests) +> - `src/tests/convergence_fixes.rs` — ADR-S-013 fix validation (7 tests) +> - `src/tests/convergence_diagnostics.rs` — on-demand `#[ignore]` diagnostic tables +> - `benches/sentinel.rs` — `warmup_cost_detailed` criterion group +> +> The original files have been deleted. + +~~Five~~ source files ~~implement~~ implemented the benchmark and characterisation tests: + +- `src/convergence_benchmark.rs` (765 lines) — the benchmark with timing and convergence trace recording. +- `src/convergence_tests.rs` (1226 lines) — `pub(crate)` unit tests with direct `SubspaceTracker` access for per-axis diagnostics. +- `src/convergence_investigation.rs` (837 lines) — Round 1 targeted experiments isolating each root cause. +- `src/convergence_investigation_2.rs` — Round 2 experiments plus the `audit_surprise_pipeline` tracer (1000-round round-by-round trace). +- `src/convergence_investigation_3.rs` — Post-fix validation (12 tests, all pass): Q1–Q8 answering specific convergence questions, multi-seed robustness, and production-config validation. + +### Convergence metric: windowed-mean comparison · `sec:sentinel:warmbench-windowed-mean-metric` + +The original `find_settled_round()` (single-point 5% tolerance) was replaced by a **windowed-mean comparison** that absorbs per-round noise: + +```rust +fn find_converged_round( + baselines: &[f64], + window: usize, // e.g. 20 = 1/α at λ = 0.95 + tolerance: f64, +) -> usize +``` + +Convergence is declared when the rolling mean over a window of $W$ rounds is within tolerance of the reference window at the end of the trace. Per-axis tolerances are mandatory: + +| Axis | Tolerance | Empirical CV ($b = 4$) | Empirical CV ($b = 16$) | +|------|:---------:|:----------------------:|:-----------------------:| +| Novelty | 1% | 0.13% | 0.06% | +| Displacement | 10% | 5.9% | 3.0% | +| Surprise | 20% | 9.0% | 3.6% | +| Coherence | 20% | 19.9% | 11.5% | + +## Empirical Convergence Results · `sec:sentinel:warmbench-empirical-results` + +### Post-fix convergence times · `sec:sentinel:warmbench-post-fix-times` + +**Test config ($\lambda = 0.95$, $b = 4$, seed = 42):** + +| Axis | Converged (rounds) | CV (last 100) | +|------|-------------------:|:-------------:| +| Novelty | 21 | 0.07% | +| Displacement | 404 | 3.37% | +| Surprise | **65** | 6.46% | +| Coherence | **406** | 10.01% | + +**Test config ($\lambda = 0.95$, $b = 16$, seed = 42):** + +| Axis | Converged (rounds) | CV (last 100) | +|------|-------------------:|:-------------:| +| Novelty | 21 | 0.04% | +| Displacement | **21** | 2.08% | +| Surprise | **70** | 2.53% | +| Coherence | **173** | 8.68% | + +**Production config ($\lambda = 0.99$, $b = 16$, seed = 42):** + +| Axis | Converged (rounds) | CV (last 200) | +|------|-------------------:|:-------------:| +| Novelty | 101 | 0.02% | +| Displacement | 101 | 0.83% | +| Surprise | **398** | 0.94% | +| Coherence | 315 | 2.36% | + +### η-convergence matches theory exactly · `sec:sentinel:warmbench-eta-theory-match` + +The maturity metric $\eta_n = \lambda^n$ matches the theoretical prediction to machine epsilon: max $|\eta - \lambda^{bn}| = 2.78 \times 10^{-17}$. The exponential model is trivially correct for η but **not** for EWMA baselines. + +### Per-axis convergence character · `sec:sentinel:warmbench-per-axis-character` + +| Axis | Character | Dominant bottleneck | +|------|-----------|---------------------| +| **Novelty** | Instant (round 21). CV = 0.07–0.13%. | None — constant-norm score is inherently stable. | +| **Displacement** | Fast at $b \geq 16$ (21 rounds); bimodal at $b = 4$ (21–475). | Stochastic subspace evolution at small batch sizes. | +| **Surprise** | 65 rounds ($\lambda = 0.95$); 398 rounds ($\lambda = 0.99$). | Cascaded lat_var → score EWMA; stochastic batch variance. The clipping feedback loop is eliminated. | +| **Coherence** | Consistently slowest: 406 ($b = 4$), 173 ($b = 16$), 315 (production). | Rank-gating delay ($k < 2$ → score = 0) plus `cross_corr` convergence time. | + +### Multi-seed robustness ($\lambda = 0.95$, $b = 4$, 10 seeds) · `sec:sentinel:warmbench-multi-seed-robustness` + +| Axis | Min | Max | Range | Mean | +|------|----:|----:|------:|-----:| +| Novelty | 21 | 21 | 0 | 21.0 | +| Displacement | 21 | 475 | 454 | 260.0 | +| Surprise | 61 | 392 | 331 | 131.8 | +| Coherence | 365 | 477 | 112 | 411.1 | + +Novelty is deterministic. Coherence is consistently slow but seed-stable (range 112). Surprise and displacement have high seed variance at $b = 4$, driven by stochastic subspace evolution. At $b = 16$, displacement becomes deterministic (21 across all seeds) and surprise variance decreases substantially. + +## Revised Iteration-Count Guidance · `sec:sentinel:warmbench-iteration-guidance` + +The original ADR proposed `n_settled ≤ 30` and `r_noise ≤ 20`. These targets were off by >10×. Revised guidance: + +| Config | Worst-case axis | Convergence (rounds) | +|--------|-----------------|---------------------:| +| $\lambda = 0.95$, $b = 4$ | Coherence | 406 | +| $\lambda = 0.95$, $b = 16$ | Coherence | 173 | +| $\lambda = 0.99$, $b = 16$ | Surprise | 398 | + +**`noise_rounds` must be at least as large as the worst-case convergence time** for baselines to be settled before real traffic arrives. The current defaults are insufficient: + +| Config | Current `noise_rounds` | Required minimum | Shortfall | +|--------|:----------------------:|:----------------:|:---------:| +| Test ($\lambda = 0.95$, $b = 4$) | 5 | ≥65 (surprise) | 13× | +| Production ($\lambda = 0.99$, $b = 16$) | 50 | ≥400 | 8× | + +### Derived iteration-count table · `sec:sentinel:warmbench-iteration-count-table` + +| Test need | Iterations | Justification | +|-----------|:----------:|---------------| +| Surprise baseline settled ($\lambda = 0.95$) | 65 | Empirical windowed-mean convergence | +| Coherence baseline settled ($b = 4$) | 406 | Rank-gating delay + EWMA convergence | +| Coherence baseline settled ($b = 16$) | 173 | Reduced by CLT at larger batch size | +| Production worst-case ($\lambda = 0.99$) | 398 | Surprise at slower forgetting rate | +| η < 0.05 | 59 | Exact: $\lceil \ln(0.05) / \ln(\lambda) \rceil$ | + +## Investigation History · `sec:sentinel:warmbench-investigation-history` + +The path from hypothesis to validated fixes spanned four rounds: + +**Round 1 (Preliminary):** Identified the 5.1× convergence gap. Attributed it primarily to the `lat_var` cold-start cascade (deterministic model predicted 114 rounds). Correct on mechanisms, wrong on dominance — the cascade accounts for only ~6% of the gap. + +**Round 2 (Secondary):** Discovered the convergence metric was fundamentally broken (round-300 reference was 56% wrong for surprise; baselines wander 15.5% CV at true steady state). The 5% tolerance criterion was unfalsifiable for 3 of 4 axes. Incorrectly attributed 89% of the gap to score-level variance (conflated the measurement problem with the convergence problem). + +**Round 3 (Code Audit):** Line-by-line audit confirmed **no code bug** — the implementation faithfully follows §ALGO S-6.1.1. Discovered the **clipping-ceiling positive feedback loop**: the true dominant cause (~85% of the gap, adding 800+ rounds to surprise convergence). Cold→warm variance ≈ $10^{-4}$ → ceiling ≈ 0.28 → perpetual clipping → slow mean/variance drift. + +**Round 4 (Quaternary — Post-fix Validation):** Implemented both fixes. Surprise convergence improved from 1000+ to 65 rounds. Confirmed the CUSUM fast-slow gap as severe (198 false drift). Discovered coherence as the true production bottleneck (406 rounds at $b = 4$) and displacement bimodality across seeds. + +### What each round got right and wrong · `sec:sentinel:warmbench-round-assessment` + +| Finding | Preliminary | Secondary | Audit | Tertiary | +|---------|:-----------:|:---------:|:-----:|:--------:| +| lat_var cold-start exists | ✓ | ✓ | ✓ | ✓ | +| lat_var cascade is PRIMARY | **✗** | corrected | — | rank 2 | +| Criterion is broken for 3/4 axes | — | ✓ | ✓ | ✓ | +| Clipping "adds ≤ 5 rounds" | — | **✗** | corrected | dominant | +| Score-level variance is 89% of gap | — | **✗** | corrected | ~3% | +| Clipping-ceiling feedback loop | — | — | ✓ | ✓ | +| No code bug | — | — | ✓ | ✓ | + +## Downstream Consequences · `sec:sentinel:warmbench-downstream-consequences` + +This investigation triggered three further ADRs: + +- **ADR-S-014** — Subspace Tracker Visibility. Convergence tests need direct `SubspaceTracker` access. Decided to keep the tracker `pub(crate)` and enrich `CellInspection` with baseline snapshots rather than exposing the tracker publicly. + +- **ADR-S-015** — Cell Creation Performance. The finding that `noise_rounds` must increase to ≥400 raised hot-path cost concerns: `inject_noise_into_cell()` runs on every new cell in the analysis set, not just at construction. Benchmarked the cost and motivated ADR-S-016. + +- **ADR-S-016** — Brand's Incremental SVD. Replaced dense thin SVD in `evolve_subspace()` with Brand's incremental algorithm, reducing per-round SVD cost and cell creation times by ~1.9–2.9×. This makes the higher `noise_rounds` affordable on the hot path. + +## Remaining Work · `sec:sentinel:warmbench-remaining-work` + +| Item | Priority | Status | +|------|----------|--------| +| ~~Increase `noise_rounds` default (50 → ≥400 at $\lambda = 0.99$)~~ | ~~**HIGH**~~ | Done — `NoiseSchedule::default()` ships a geometric schedule at 450 root rounds, halving per depth to a floor of 50 | +| Promote windowed-mean convergence metric to production test suite | Medium | Validated in quaternary tests | +| ~~Update §ALGO S-6.1.1 with graduated clip-exemption formula~~ | ~~Medium~~ | Done (2026-03-11) | +| ~~Update §ALGO S-4.2 Phase 3 with cold→warm initialisation~~ | ~~Medium~~ | Done (2026-03-11) | +| ~~Update §ALGO S-11.4 with slow-from-fast CUSUM seeding~~ | ~~Medium~~ | Done (2026-03-11) | +| ~~Update §ALGO S-11.5–11.6 with empirical convergence data~~ | ~~Medium~~ | Done (2026-03-11) | +| Address coherence as the production bottleneck (rank-gating delay) | Medium | Confirmed structural | +| Per-axis test tolerances and displaced bimodality metric | Low | Characterised | +| ~~Mark superseded findings documents~~ | ~~Low~~ | Done — files retired, content absorbed into spec (2026-03-11) | +| ~~Update `convergence_tests.rs` module doc~~ | Low | Done — file removed in consolidation (2026-03-11) | + +## Alternatives Considered · `sec:sentinel:warmbench-alternatives` + +- **Just reduce iteration counts by the theoretical formula.** The formula for $\eta$ is trivially exact, but EWMA baseline convergence depends on the score distribution, outlier clipping, and batch-mean aggregation. The investigation proved this approach would be off by 5–10×. + +- **Remove clipping outright during warm-up.** The graduated clip-exemption was chosen over binary on/off because it provides proportional attack resistance at all maturity levels. At $\eta = 0.01$ the clip is 3.03σ — negligibly wider than production. + +- **Accept longer noise injection without fixing the feedback loop.** Would require `noise_rounds > 1000` at $b = 4$. Infeasible on the hot path (cell creation during live traffic). + +- **Property-based testing across random configs.** Worth doing eventually, but the immediate need was concrete empirical bounds for known configurations. + +## Consequences · `sec:sentinel:warmbench-consequences` + +- **Empirically justified iteration counts** replace arbitrary constants. The investigation provides precise per-axis, per-config convergence times rather than order-of-magnitude estimates. + +- **Three code fixes** eliminate the dominant convergence bottlenecks: the clipping feedback loop (Fix 1), the lat_var cold-start cascade (Fix 2), and the CUSUM fast-slow gap (Fix 3). + +- **Surprise convergence improved 15×** (1000+ → 65 rounds at $\lambda = 0.95$), exceeding the original prediction of ~120 rounds. + +- **The bottleneck shifted** from surprise (fixed) to coherence (structural: rank-gating delay + cross-correlation convergence). Coherence at $b = 4$ takes ~406 rounds — this is the true system convergence time. + +- **`noise_rounds` defaults are insufficient.** Production needs ≥400 at $\lambda = 0.99$. ADR-S-015 and ADR-S-016 ensure this increase is affordable on the hot path. + +- **The simple exponential model is only valid for η.** EWMA baselines converge at a rate determined by cascaded EWMA interactions, clipping policy, batch size, and axis-specific score distributions. The spec (§ALGO S-11.5–11.6) must be updated to reflect this. + +- **Regression detection.** The convergence test suite catches changes to EWMA update rules, clipping behaviour, or cold start logic that would shift convergence times. diff --git a/packages/sentinel/adr/014-subspace-tracker-visibility.md b/packages/sentinel/adr/014-subspace-tracker-visibility.md new file mode 100644 index 000000000..0484b8a02 --- /dev/null +++ b/packages/sentinel/adr/014-subspace-tracker-visibility.md @@ -0,0 +1,205 @@ +# ADR-S-014: SubspaceTracker Visibility · `rec:sentinel:private-tracker-with-public-baseline-snapshots` + +**Status:** Decided **Date:** 2026-03-10 **Spec:** §ALGO S-4 (subspace tracker), §ALGO S-11.1 (noise injection) **Relates to:** [ADR-S-007](007-automatic-noise-injection.md) (automatic noise injection), [ADR-S-013](013-warm-up-convergence-benchmark.md) (warm-up convergence benchmark), [ADR-S-001](001-measures-not-opinions.md) (measures not opinions) + +## Context · `sec:sentinel:trackervis-context` + +`SubspaceTracker` is the statistical core of the sentinel — a low-rank online subspace model with four-axis scoring and EWMA baselines. It is currently declared `pub struct` in a `pub(crate) mod tracker`, making it **visible within the crate but invisible to external consumers** (integration tests under `tests/`, Criterion benchmarks under `benches/`, and downstream crates). + +This visibility boundary has been adequate so far: every integration test, benchmark, and host interaction operates exclusively through the `SpectralSentinel` public API (`ingest`, `inspect_cell`, `health`, `decay`, `BatchReport`, etc.). + +ADR-S-013 introduces a new testing need that challenges this boundary. The convergence benchmark's `noise_baselines_converge` test (§3) requires: + +1. **Direct tracker construction** — create a bare `SubspaceTracker::new(dim, &cfg, slow_decay)` without sentinel orchestration overhead (no G-V Graph, no analysis set, no coordination). +2. **Repeated `observe()` calls with `is_noise = true`** — feed synthetic noise batches one at a time and inspect the EWMA baselines after each round. +3. **Access to `TrackerReport::scores.*.baseline.mean`** — read the per-axis EWMA mean from the report returned by `observe()`. + +The `SpectralSentinel` API is insufficient for this because: + +- `inspect_cell()` returns `CellInspection`, which exposes `maturity` (including `noise_influence`) but **not** the per-axis EWMA baseline means/variances. These are only available in `CellReport::scores.*.baseline` and `TrackerReport::scores.*.baseline`, which come from `ingest()` reports. +- Using `ingest()` for this test adds sentinel orchestration overhead (graph routing, analysis set reconciliation, coordination) that obscures the measurement and slows the test unnecessarily. +- Noise injection is automatic and internal (ADR-S-007). There is no way to call `observe(is_noise = true)` on a tracker through the public API — the sentinel owns the injection lifecycle. + +### The deeper question · `sec:sentinel:trackervis-deeper-question` + +Should `SubspaceTracker` be part of the crate's public API? + +This is not just about one test. It's about the crate's API philosophy and impacts: + +- **Host-side subspace analysis.** Some hosts may want to run a standalone tracker outside of sentinel orchestration — e.g. for offline analysis, benchmarking specific workloads, or building custom pipelines. +- **Fuzz testing.** External fuzzer harnesses can exercise the tracker directly without paying graph construction costs. +- **Benchmark granularity.** Criterion benchmarks can isolate tracker performance (SVD cost, EWMA update cost) without sentinel overhead. +- **Semver surface area.** Any `pub` type is a commitment. Changes to `SubspaceTracker`'s constructor signature, field layout, or `observe()` return type become breaking changes. +- **Encapsulation.** ADR-S-007 establishes that the sentinel owns the noise injection lifecycle. Exposing `observe()` with its `is_noise` parameter hands that control to external callers, creating ordering hazards (double-injection, skipped injection, interleaved noise/real traffic). +- **`AxisBaseline` / `EwmaStats` visibility.** Making the tracker public is only useful if callers can also construct configs and read results. `SentinelConfig` and `TrackerReport` are already public. But `AxisBaseline` is private, and `EwmaStats` is `pub` in `ewma.rs`. + +## Decision · `sec:sentinel:trackervis-decision` + +**Option E — hybrid: keep `SubspaceTracker` as `pub(crate)`, enrich `CellInspection` with baseline snapshots.** + +See [Recommendation](#recommendation) below for rationale. + +## Decision Options Considered · `sec:sentinel:trackervis-options` + +### Option A: Keep `pub(crate)`, test inside the crate · `sec:sentinel:trackervis-option-private-crate-tests` + +Leave `SubspaceTracker` as `pub(crate)`. Place any test that needs direct tracker access inside the crate boundary: + +- **`noise_baselines_converge`** → `#[cfg(test)] mod` inside `src/sentinel/tracker.rs`. +- **Tracker-level benchmarks** → not possible via Criterion (benches are external); would need to be approximated through the sentinel API or use `cargo bench` with the `test` harness. + +**Pro:** +- Zero semver surface change. +- Encapsulation preserved: `SubspaceTracker`'s API can evolve freely. +- ADR-S-007's lifecycle invariant ("the sentinel owns injection") is enforced by the type system — external code literally cannot call `observe(is_noise = true)`. +- No risk of misuse by downstream crates. +- Unit tests in `src/` have full `pub(crate)` access with no workarounds needed. + +**Con:** +- Tests that need tracker access must live in `src/`, not in the `tests/` folder. This mixes test code with production code (though `#[cfg(test)]` ensures it is stripped from release builds). +- Criterion benchmarks cannot isolate tracker-level performance without a public API. Sentinel-level benchmarks are a proxy but include graph and analysis set overhead. +- No path for hosts to use the tracker standalone. + +### Option B: Promote to `pub mod tracker` · `sec:sentinel:trackervis-option-public-module` + +Change `pub(crate) mod tracker` → `pub mod tracker`, making `SubspaceTracker`, its constructor, `observe()`, `maturity()`, `rank()`, and `reset_cusum()` part of the crate's public API. + +**Pro:** +- Integration tests and Criterion benches can construct trackers directly. +- Hosts can build custom pipelines with standalone trackers. +- Benchmark isolation: measure SVD/EWMA costs without sentinel overhead. +- Fuzz testing can target the tracker in external harnesses. + +**Con:** +- **Semver commitment.** `SubspaceTracker::new()` signature, `observe()` parameters, and `TrackerReport` fields all become stable API surface. Any internal refactor (e.g. changing the basis representation, adding parameters to `observe()`) is a breaking change. +- **Breaks ADR-S-007's lifecycle invariant.** External callers can call `observe(is_noise = true)` arbitrarily, bypassing the sentinel's controlled injection sequence. While `SubspaceTracker` is stateless w.r.t. injection ordering (it doesn't panic or corrupt), the *host's interpretation* of maturity becomes unreliable if injection was manual and uncontrolled. +- **AxisBaseline exposure cascade.** Making the module public invites questions about `AxisBaseline`, `CusumAccumulator`, and the internal scoring pipeline. These are currently private structs. +- **Documentation burden.** A public tracker needs doc-comments, usage examples, and safety guidance ("do not mix `is_noise` calls with real observations outside sentinel orchestration"). + +### Option C: Test-only feature gate · `sec:sentinel:trackervis-option-test-feature` + +Add a Cargo feature `test-internals` (default off) that conditionally re-exports internal types: + +```rust +#[cfg(feature = "test-internals")] +pub mod test_internals { + pub use super::tracker::SubspaceTracker; + pub use super::cusum::CusumAccumulator; +} +``` + +Integration tests and benches enable it via `[dev-dependencies]` features. + +**Pro:** +- Tracker is accessible in tests and benches without being part of the default public API. +- No semver commitment to non-`test-internals` consumers. +- ADR-S-007 lifecycle invariant is preserved for production builds. +- Clean separation: test authors explicitly opt in. + +**Con:** +- Feature gates add conditional-compilation complexity. +- `test-internals` is a social contract, not a hard boundary — any downstream crate can enable it. +- Documentation must explain the feature and its warranty limitations. +- `cargo doc` with `--all-features` exposes the internal types, cluttering the generated docs unless `#[doc(hidden)]` is used, which defeats discoverability for test authors. + +### Option D: Expose a limited inspection API on `SpectralSentinel` · `sec:sentinel:trackervis-option-inspection-api` + +Instead of exposing the tracker, add a method like `inspect_cell_baselines(gnode) -> Option` that returns per-axis EWMA means and variances. Combined with `inspect_cell()` (maturity) and `ingest()` (reports), this covers the convergence test's needs without exposing the tracker. + +```rust +pub struct BaselineInspection { + pub novelty: BaselineSnapshot, + pub displacement: BaselineSnapshot, + pub surprise: BaselineSnapshot, + pub coherence: BaselineSnapshot, +} +``` + +**Pro:** +- Narrow API addition — one method, one struct. +- Encapsulation preserved: `SubspaceTracker` remains internal. +- Fulfils the convergence test's data needs (`inspect_cell()` + `inspect_cell_baselines()` after each `ingest()`). +- Useful for hosts too (monitoring baseline drift). + +**Con:** +- Does not address the "test tracker in isolation" need. The `noise_baselines_converge` test (ADR-S-013 §3) specifically wants to measure noise-only convergence without real data. Through the sentinel API, you cannot feed noise without also triggering graph routing / analysis set / coordination. +- Two-step inspection (`inspect_cell` + `inspect_cell_baselines`) is slightly awkward; could be unified into a richer `CellInspection` instead. + +### Option E: Hybrid — `pub(crate)` (Option A) + enriched inspection (Option D) · `sec:sentinel:trackervis-option-hybrid` + +Keep `SubspaceTracker` as `pub(crate)`. Place the noise-only convergence test inside the crate (`#[cfg(test)]` in `tracker.rs`). Additionally, enrich `CellInspection` with baseline snapshots so that the integration-level convergence test (`convergence_matches_theory`) doesn't need to hunt through `BatchReport` arrays: + +```rust +pub struct CellInspection { + // ... existing fields ... + /// Per-axis EWMA baseline snapshots. + pub baselines: AxisBaselineSnapshots, +} + +pub struct AxisBaselineSnapshots { + pub novelty: BaselineSnapshot, + pub displacement: BaselineSnapshot, + pub surprise: BaselineSnapshot, + pub coherence: BaselineSnapshot, +} +``` + +**Pro:** +- The noise-only test lives where it has full access (inside the crate), matching the principle that unit tests belong near the code they test. +- The integration-level convergence test gets clean baseline access through the public API without parsing `BatchReport` cell/ancestor arrays. +- No semver exposure of `SubspaceTracker`. +- Enriched `CellInspection` is independently useful for hosts monitoring baseline drift. +- Clean separation of concerns: noise convergence tested at the unit level, theory + EWMA convergence tested at the integration level. + +**Con:** +- Criterion benchmarks still cannot isolate tracker-level performance. (Mitigated: the existing benchmark suite already measures sentinel-level throughput at various scales, and the convergence benchmark measures batch-over-time cost through the sentinel API.) +- Adds a struct and 4 fields to `CellInspection` (minor API growth). + +## Recommendation · `sec:sentinel:trackervis-recommendation` + +**Option E — hybrid.** + +The rationale: + +1. **`SubspaceTracker` is an implementation detail.** Its constructor takes a `slow_decay: f64` parameter that only makes sense in the context of the sentinel's two-tier EWMA architecture. Its `observe()` method has an `is_noise: bool` parameter whose correct usage depends on the injection lifecycle described in ADR-S-007. Exposing these to external callers invites misuse without adding proportional value. + +2. **The convergence test that needs internal access is a crate test.** `noise_baselines_converge_within_bound` exercises a single tracker's EWMA convergence. It lives in the collected crate-test module `src/tests/convergence_noise.rs`, which has access to crate-private internals without placing test code inside `tracker.rs` or widening the tracker's visibility. + +3. **Enriching `CellInspection` is the right public API evolution.** Baseline means and variances are legitimate observable state. Hosts monitoring the sentinel in production will benefit from baseline visibility in `inspect_cell()` without needing to parse `BatchReport` arrays. This is fully aligned with "the sentinel measures; the host decides" (ADR-S-001). + +4. **Semver discipline.** The crate's public surface is covered by semver guarantees from 1.0.0 onwards, and keeping the internal machinery private is what makes that guarantee affordable: a `pub(crate)` tracker can be reshaped in a compatible release, whereas a public `SubspaceTracker` would fix its constructor signature, field layout, and `observe()` return type for the life of the major version. If a genuine need for standalone trackers emerges (e.g. a host's custom analysis pipeline), a future ADR can promote visibility with an intentional, documented API — an addition rather than a break. + +5. **Benchmark isolation is a non-goal for now.** The existing Criterion suite measures ingest throughput at various scales. The convergence benchmark (ADR-S-013 §1) measures 200-batch convergence wall-clock at the sentinel level, which is the relevant integration point. Tracker-level micro- benchmarks would be useful but are not blocked by this decision — they can live in `#[cfg(test)]` `mod benches` inside `tracker.rs` using the `test::Bencher` nightly API, or be added later if Option B is adopted. + +## Consequences · `sec:sentinel:trackervis-consequences` + +1. **`noise_baselines_converge_within_bound`** is implemented as a collected crate test in `src/tests/convergence_noise.rs`. + +2. **`CellInspection`** gains a `baselines: AxisBaselineSnapshots` field populated from the tracker's four `AxisBaseline` fast- EWMA snapshots. + +3. **`AxisBaselineSnapshots`** is a new public type in `report.rs`, containing four `BaselineSnapshot` fields. + +4. **`inspect_cell()`** in `SpectralSentinel` populates the new field from `cell.tracker.axis_baselines()` (already a `pub(crate)` method that returns the necessary data). + +5. **`convergence_matches_theory`** (integration test) reads baselines from `inspect_cell().baselines` instead of parsing `BatchReport` arrays. This simplifies the test and removes the dependency on knowing whether the root is in `cell_reports` or `ancestor_reports`. + +6. **No change** to `SubspaceTracker`'s visibility (`pub(crate)`). + +7. **Future reconsideration.** If demand emerges for standalone tracker usage (host custom pipelines, external fuzz harnesses), revisit this ADR. The migration path is straightforward: change `pub(crate) mod tracker` → `pub mod tracker` and document the `is_noise` lifecycle contract. + +## Files Changed · `sec:sentinel:trackervis-files-changed` + +| File | Change | +|------|--------| +| `src/report.rs` | Add `AxisBaselineSnapshots` struct; add `baselines` field to `CellInspection` | +| `src/sentinel/mod.rs` | Populate `baselines` in `inspect_cell()` | +| `src/tests/convergence_noise.rs` | Exercise tracker baseline convergence through crate-private access | + +## Cross-References · `sec:sentinel:trackervis-cross-references` + +- §ALGO S-4 — Subspace tracker specification +- §ALGO S-6.1.1 — EWMA baseline update rule +- §ALGO S-11.1–11.4 — Noise injection lifecycle +- ADR-S-001 — Measures not opinions (API philosophy) +- ADR-S-007 — Automatic noise injection (lifecycle ownership) +- ADR-S-013 — Warm-up convergence benchmark (motivating use case) diff --git a/packages/sentinel/adr/015-cell-creation-performance.md b/packages/sentinel/adr/015-cell-creation-performance.md new file mode 100644 index 000000000..a7cbcabb0 --- /dev/null +++ b/packages/sentinel/adr/015-cell-creation-performance.md @@ -0,0 +1,266 @@ +# ADR-S-015: Cell Creation Performance · `rec:sentinel:depth-tiered-noise-schedule-bounds-creation-cost` + +**Status:** Implemented (§1 `NoiseSchedule`) **Date:** 2026-03-10 **Spec:** §ALGO S-8.2 (analysis set reconciliation), §ALGO S-11.1 (automatic noise injection) **Relates to:** [ADR-S-007](007-automatic-noise-injection.md) (automatic noise injection), [ADR-S-013](013-warm-up-convergence-benchmark.md) (warm-up convergence benchmark), [ADR-S-006](006-analysis-set-recomputation.md) (analysis set recomputation), [ADR-S-012](012-test-duration-budget.md) (test duration budget), [ADR-S-016](016-brand-incremental-svd.md) (Brand's incremental SVD) + +## Context · `sec:sentinel:noiseperf-context` + +Every time a new cell enters the analysis set — whether at sentinel construction, after `reset()`, or during live `reconcile_analysis_set()` — `inject_noise_into_cell()` runs `noise_rounds` batches of synthetic observations through a fresh `SubspaceTracker`. This is the mechanism described in ADR-S-007. + +The critical realisation is that **cell creation is not a one-time startup cost**. The G-V graph splits cells as traffic arrives. `reconcile_analysis_set()` runs on every `ingest()` call, and when the analysis set changes, new cells are created — each paying the full noise injection cost. + +### Call sites · `sec:sentinel:noiseperf-call-sites` + +`inject_noise_into_cell()` is called from three places: + +1. **`SpectralSentinel::new()`** — root cell at construction. One-time cost, acceptable. + +2. **`SpectralSentinel::reset()`** — root cell after reset. Infrequent, acceptable. + +3. **`reconcile_analysis_set()`** — new cells entering the analysis set during live traffic. **This is the hot-path concern.** Multiple cells may enter in a single `ingest()` call if the graph has split since the last reconciliation. + +### Measured costs · `sec:sentinel:noiseperf-measured-costs` + +Criterion benchmarks (release profile, optimised) on the sentinel crate, 2026-03-10: + +| `noise_rounds` | Config | Construction time | Per-round marginal | +|----------------:|--------|------------------:|-------------------:| +| 10 | bench (rank=4, k=16, b=4) | 38 ms | ~2.5 ms | +| 50 | bench | 319 ms | ~7 ms | +| 100 | bench | 161 ms | ~2.5 ms | +| 200 | bench | 1.28 s | ~5 ms | +| 400 | bench | 3.27 s | ~7 ms | +| 10 | realistic (rank=16, k=1024, b=16) | 99 ms | — | +| 50 | realistic | 332 ms | ~6 ms | +| 100 | realistic | 164 ms | ~2 ms | +| 200 | realistic | 3.72 s | ~17 ms | +| 400 | realistic | 4.92 s | ~10 ms | + +> **Update (post ADR-S-016):** Brand's incremental SVD reduces construction times by **~1.9×** at b=4 and **~2.9×** at b=16. The decision analysis below uses Brand's-adjusted figures. + +Per-round `ingest()` cost on a warmed sentinel (Criterion): + +| Config | batch_size | Per-ingest call | +|--------|----------:|--------------:| +| bench | 4 | 232 µs | +| bench | 16 | 6.6 ms | +| realistic | 4 | 4.4 ms | +| realistic | 16 | 14.7 ms | + +> With Brand's SVD (ADR-S-016) these drop to ~122 µs (bench b=4), ~2.3 ms (bench b=16), ~2.3 ms (realistic b=4), ~5.1 ms (realistic b=16). + +### Convergence data · `sec:sentinel:noiseperf-convergence-data` + +From the convergence benchmark suite (formerly `convergence_benchmark.rs`, now `convergence_diagnostics.rs` / criterion `warmup_cost_detailed`, optimised debug): + +**Noise-only convergence (per-axis, rounds needed):** + +| noise_rounds | Novelty | Surprise | Displacement (b=16) | Coherence | All converged? | +|---:|----|----|----|----|----| +| 5 | 3 | 3 | 3 | 0 | NO | +| 50 | 21 | 30 | 21 | 0 | NO | +| 65 | 21 | 43 | 21 | 0 | YES (prod) | +| 100 | 21 | 67 | 21 | 0 | YES | +| 200 | 21 | 67 | 33 | 157 | YES | +| 400 | 21 | 188 | 24 | 273 | YES | + +**Real-data phase convergence (after 200 noise rounds):** + +| Config | Worst axis | Converged at round | Wall-clock | +|--------|-----------|---:|---:| +| test (λ=0.95, b=4) | coherence | 233 | ~2.0 s | +| production (λ=0.99, b=16) | coherence | 212 | ~5.1 s | + +**η (noise influence) matches theory exactly:** max $|\eta - \lambda^{bn}| = 2.78 \times 10^{-17}$. + +## Problem · `sec:sentinel:noiseperf-problem` + +The current defaults are: + +| Config | `noise_rounds` | Per-cell cost (pre-Brand) | Per-cell cost (Brand) | +|--------|---:|---:|---:| +| Test | 5 | ~38 ms | ~13 ms | +| Production | 50 | ~332 ms | ~115 ms | + +The test default of 5 is **13× too low** for surprise convergence (needs ≥65). The production default of 50 is **6× too low** for η < 0.05 (needs ≥299 at λ = 0.99). + +However, raising `noise_rounds` to 300–400 is still **expensive on the hot path** even with Brand's incremental SVD (ADR-S-016). If the analysis set gains 5 new cells during a single `ingest()`: + +| `noise_rounds` | Per-cell (Brand) | 5 cells | Impact on `ingest()` | +|---:|---:|---:|---| +| 50 (current) | ~115 ms | ~575 ms | Tolerable | +| 100 | ~57 ms | ~283 ms | **Well within budget** | +| 200 | ~1.28 s | ~6.4 s | Blocking stall | +| 300 | ~860 ms | ~4.3 s | Significant stall | +| 400 | ~1.7 s | ~8.5 s | Blocking stall | + +At `analysis_k = 1024` and `analysis_depth_cutoff = 6`, the analysis set can contain up to 1024 cells. Graph growth from zero to steady state may create hundreds of cells over time. Each split that creates a new cell in the analysis set triggers noise injection. + +## Decision · `sec:sentinel:noiseperf-decision` + +### 1. Depth-tiered `noise_schedule` · `sec:sentinel:noiseperf-depth-tiered-schedule` + +Replace the single `noise_rounds` scalar with a **`noise_schedule` enum** that determines per-depth noise rounds. Two variants: + +```rust +enum NoiseSchedule { + /// Geometric decay: r(d) = max(min, floor(root * decay^d)) + Geometric { root: u32, decay: f64, min: u32 }, + + /// Explicit per-depth array; last element repeats for all + /// deeper layers. + Explicit(Vec), // non-empty, validated at construction +} +``` + +Resolution at injection time: + +```rust +fn rounds_for_depth(&self, depth: usize) -> u32 { + match self { + Self::Geometric { root, decay, min } => { + let exp = i32::try_from(depth).unwrap_or(i32::MAX); + let raw = f64::from(*root) * decay.powi(exp); + #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] + let rounded = raw.round() as u32; + rounded.max(*min) + } + Self::Explicit(v) => { + if v.is_empty() { + return 0; + } + v[depth.min(v.len() - 1)] + } + } +} +``` + +#### Default: `Geometric { root: 400, decay: 0.5, min: 100 }` · `sec:sentinel:noiseperf-default-geometric` + +$$r(d) = \max\!\bigl(100,\; \lfloor 400 \cdot 0.5^{\,d} \rfloor\bigr)$$ + +Materialised: + +| Depth | Formula | Rounds | +|---:|---:|---:| +| 0 (root) | $400 \cdot 0.5^0$ | **400** | +| 1 | $400 \cdot 0.5^1$ | **200** | +| 2 | $400 \cdot 0.5^2 = 100$ | **100** | +| 3+ | $\max(100, \ldots)$ | **100** (floor) | + +This is equivalent to `Explicit([400, 200, 100])`. + +#### Rationale per tier · `sec:sentinel:noiseperf-rationale-per-tier` + +**Depth 0 (root) — 400 rounds.** Created exactly once at `SpectralSentinel::new()` or `reset()`, never on the hot path. 400 rounds guarantees **all axes converge**, including coherence: + +| Metric | At 400 rounds | +|--------|---:| +| Per-cell cost (realistic, Brand) | **~1.7 s** | +| Surprise converged? | **Yes** (188 of 400) | +| Displacement converged? | **Yes** (24 of 400) | +| Coherence converged? | **Yes** (273 of 400) | +| η at transition (λ=0.99, b=16) | 0.99^6400 ≈ 0 | +| η at transition (λ=0.95, b=4) | 0.95^1600 ≈ 0 | + +The root sees *all* traffic before the graph splits, so full convergence here eliminates graduated exemptions and transition artefacts for the most-observed cell. + +**Depth 1 — 200 rounds.** First-level splits are infrequent (typically a handful over the sentinel's lifetime) and each covers a large fraction of the address space. 200 rounds costs ~1.28 s per cell (Brand), which is acceptable for an uncommon event. Surprise, displacement, and partial coherence all converge. + +**Depth 2+ — 100 rounds (floor).** These cells are created on the `reconcile_analysis_set()` hot path and may arrive in bursts. 100 rounds costs ~57 ms per cell (Brand), keeping a 5-cell burst at ~283 ms: + +| Metric | At 100 rounds | +|--------|---:| +| Per-cell cost (realistic, Brand) | **~57 ms** | +| 5 cells in one ingest | **~283 ms** | +| Surprise converged? | **Yes** (67 of 100) | +| η at transition (λ=0.99, b=16) | 0.99^1600 ≈ 1.2 × 10⁻⁷ | +| η at transition (λ=0.95, b=4) | 0.95^400 ≈ 7.7 × 10⁻¹⁰ | +| Coherence converged? | No — finishes during real traffic | + +100 rounds is sufficient for deep cells because: + +- **Surprise** (the hardest non-gated axis) converges by round 67. +- **η** is already negligible (10⁻⁷ or below) — the maturity model correctly reports the cell as fully warmed. +- **The graduated clip-exemption** (ADR-S-013 §1) prevents false scoring while baselines are still converging. +- **The CUSUM slow-from-fast seeding** (ADR-S-013 §6) eliminates false drift at the noise→real transition. +- **Coherence** is gated by `rank_update_interval` anyway — it doesn't produce scores until $k \geq 2$, which takes `rank_update_interval` steps regardless of noise injection length. + +#### User configuration · `sec:sentinel:noiseperf-user-configuration` + +Operators choose whichever variant is most natural: + +```toml +# Default — geometric decay (3 parameters) +[sentinel.noise_schedule] +type = "geometric" +root = 400 +decay = 0.5 +min = 100 + +# Same result, explicit array +[sentinel.noise_schedule] +type = "explicit" +rounds = [400, 200, 100] + +# Converge everything at every depth (slow hot path) +[sentinel.noise_schedule] +type = "explicit" +rounds = [400] + +# Slower decay — more rounds at shallow depths +[sentinel.noise_schedule] +type = "geometric" +root = 400 +decay = 0.7 +min = 100 +# → [400, 280, 196, 137, 100, 100, ...] +``` + +**`Geometric`** is compact and self-documenting: the operator states intent (root quality, decay rate, floor) and the schedule scales automatically to any graph depth. **`Explicit`** gives full control when specific per-layer tuning is needed; the last element repeats for all deeper layers. + +### 2. Document the per-cell cost in the spec · `sec:sentinel:noiseperf-per-cell-cost` + +§ALGO S-11.1 should note that noise injection runs per cell, not once globally. The cost model is: + +$$T_{\text{noise}} = \sum_{\text{cells } c} r(d_c) \times t_{\text{round}}$$ + +where $r(d_c)$ is the noise-schedule value for cell $c$ at depth $d_c$, and $t_{\text{round}}$ is the per-round observe cost (config and hardware dependent, ~2–17 ms in benchmarks). + +### 3. Consider future optimisations (not in this ADR) · `sec:sentinel:noiseperf-future-optimisations` + +These are documented here for future reference but are **not** being implemented now: + +**a. Async / background noise injection.** Move noise injection off the `ingest()` hot path. New cells would be created with a "warming" flag and injected in a background task. Scores from warming cells would be suppressed (η = 1.0) until injection completes. This eliminates the hot-path stall entirely but adds concurrency complexity. + +**b. Adaptive schedule.** Auto-tune `decay` or the explicit array based on observed cell-creation frequency per depth. If depth-1 splits are rare, increase their rounds; if they burst, reduce. Requires runtime telemetry that doesn't yet exist. + +**c. Shared noise cache.** Pre-generate the noise batch matrix once and reuse it across cells (adjusting for dimension). Saves RNG and allocation cost but not the `observe()` cost, which dominates. + +**d. Lazy injection.** Defer noise injection until the cell actually receives real traffic. Cells that enter the analysis set but never see observations (common during rapid graph churn) would skip injection entirely. Risk: first real batch hits unconverged baselines. + +## Consequences · `sec:sentinel:noiseperf-consequences` + +- **Root cell starts fully converged** on all axes (400 rounds, ~1.7 s one-time cost). No graduated exemptions or transition artefacts needed for the root. + +- **Depth-1 cells get 200 rounds** (~1.28 s each), converging surprise and displacement fully. These splits are infrequent. + +- **Depth-2+ cell creation cost is bounded to ~57 ms** at realistic settings (with Brand's SVD, ADR-S-016). A 5-cell creation burst stalls `ingest()` for ~283 ms, well within budget for a batch-oriented system. + +- **Surprise baselines are converged** at all depths before the cell sees real traffic. + +- **Coherence baselines are NOT converged** on depth-2+ cells at creation. They finish converging during the real-data phase, gated by η. This is acceptable because coherence scoring requires $k \geq 2$, and the coherence EWMA cold→warms from the first real coherence score (ADR-S-013 §2b). + +- **Test suite performance improves.** Increasing test `noise_rounds` from 5 to 100 adds ~57 ms per cell construction (with Brand's SVD), but eliminates the ~500+ rounds of "compensatory warm-up" that many tests currently run. Net effect depends on the test, but the convergence benchmark suite (now in criterion `warmup_cost_detailed` and `convergence_diagnostics.rs`) shows 100 rounds finishes well under 1 second total for all four tests. + +- **The `noise_schedule` parameter is fully user-configurable.** Operators supply an array `[root, layer_1, ..., floor]` where the last element repeats for all deeper layers. The geometric default `[400, 200, 100]` is a measured compromise. A single-element `[400]` converges everything everywhere; a single-element `[100]` minimises hot-path cost at the expense of root convergence. + +## Appendix: Convergence vs Cost Tradeoff Table · `sec:sentinel:noiseperf-cost-tradeoff` + +| `noise_rounds` | Per-cell, Brand (ms) | Surprise | Displ. | Coherence | η (λ=0.99,b=16) | Recommended? | +|---:|---:|:---:|:---:|:---:|:---:|:---:| +| 5 | ~13 | NO | NO | NO | 0.923 | NO — current test default, too low | +| 50 | ~115 | NO | YES | NO | 0.449 | NO — current prod default, too low | +| 65 | ~138 | MARGINAL | YES | NO | 0.353 | MARGINAL | +| **100** | **~57** | **YES** | **YES** | **NO** | **1.2×10⁻⁷** | **YES — recommended default** | +| 200 | ~1283 | YES | YES | PARTIAL | ~0 | NO — too expensive per cell | +| 400 | ~1697 | YES | YES | YES | ~0 | NO — expensive on hot path | diff --git a/packages/sentinel/adr/016-brand-incremental-svd.md b/packages/sentinel/adr/016-brand-incremental-svd.md new file mode 100644 index 000000000..9629f0b1a --- /dev/null +++ b/packages/sentinel/adr/016-brand-incremental-svd.md @@ -0,0 +1,199 @@ +# ADR-S-016: Brand's Incremental SVD for Subspace Evolution · `rec:sentinel:brand-incremental-svd-for-subspace-evolution` + +**Status:** Implemented **Date:** 2026-03-10 **Spec:** §ALGO S-4.2 Phase 2 (subspace evolution) **Relates to:** [ADR-S-015](015-cell-creation-performance.md) (cell creation performance), [ADR-S-007](007-automatic-noise-injection.md) (automatic noise injection), [ADR-S-013](013-warm-up-convergence-benchmark.md) (warm-up convergence benchmark) + +## Context · `sec:sentinel:brandsvd-context` + +### The hot loop · `sec:sentinel:brandsvd-hot-loop` + +Every call to `SubspaceTracker::observe()` invokes `evolve_subspace()`, which performs Phase 2 of the five-phase core loop (§ALGO S-4.2). Convergence benchmarks (ADR-S-015) show that **`observe()` accounts for 99.9% of wall-clock time**, and Phase 2 (the SVD) dominates `observe()`. + +### Previous implementation · `sec:sentinel:brandsvd-previous-implementation` + +`evolve_subspace()` built the composite matrix + +$$M = \bigl[\;\sqrt{\lambda}\, U_k \operatorname{diag}(\sigma_{1..k}) \;\big|\; X^\top\;\bigr] \;\in\; \mathbb{R}^{d \times (k+b)}$$ + +and computed a **full dense thin SVD** of $M$ via `faer::Mat::thin_svd()`. This produced all $\min(d,\, k{+}b)$ singular triplets, of which only the top $n = \min(k{+}b,\, d,\, \text{cap})$ were retained. + +### Cost analysis · `sec:sentinel:brandsvd-cost-analysis` + +The thin SVD of a $d \times c$ matrix (where $c = k{+}b$) via bidiagonalisation + divide-and-conquer costs $O(d \, c^2)$ flops, plus $O(d \, c \, n)$ for back-transforming the $n$ left singular vectors from Householder form. + +At representative dimensions: + +| Config | d | k | b | M shape | SVD cost term | Per-round measured | +|--------|---|---|---|---------|---:|---:| +| bench | 128 | 2 | 4 | 128 × 6 | $128 \cdot 36$ | ~9 ms | +| realistic | 128 | 2 | 16 | 128 × 18 | $128 \cdot 324$ | ~32 ms | +| production (wide) | 128 | 16 | 16 | 128 × 32 | $128 \cdot 1024$ | est. ~90 ms | + +With `noise_rounds = 100` per cell and multiple cells created on the hot path (ADR-S-015), the cumulative SVD cost is the dominant throughput bottleneck. + +### What faer offers · `sec:sentinel:brandsvd-faer-primitives` + +faer 0.24.0 provides: + +- **`Mat::thin_svd()`** — dense thin SVD (bidiag + D&C). No option to compute only the top-$k$ triplets. +- **`matrix_free::eigen::partial_svd()`** — Lanczos-based partial SVD for implicit operators (`&dyn BiLinOp`). Designed for large sparse matrices; slower than direct decomposition at our dimensions. +- **`linalg::qr::no_pivoting::qr_in_place()`** — dense thin QR. +- Dense matrix multiply via `&A * &B` operator overloads. + +faer does **not** provide an incremental/streaming SVD. However, it provides all the primitives needed to implement one. + +## Decision · `sec:sentinel:brandsvd-decision` + +Replace the naïve dense thin SVD in `evolve_subspace()` with **Brand's incremental SVD** (Brand 2002, 2006), using faer's existing QR and SVD primitives on a much smaller kernel matrix. + +### Algorithm · `sec:sentinel:brandsvd-algorithm` + +Given the current rank-$k$ model $(U_k, \sigma_{1..k})$ and a new batch $X \in \mathbb{R}^{b \times d}$: + +**Step 1 — Project onto current basis:** + +$$P = U_k^\top X^\top \;\in\; \mathbb{R}^{k \times b}$$ + +**Step 2 — Compute orthogonal residual:** + +$$Q = X^\top - U_k\, P \;\in\; \mathbb{R}^{d \times b}$$ + +**Step 3 — Thin QR of residual:** + +$$Q = Q_\perp\, R_\perp, \quad Q_\perp \in \mathbb{R}^{d \times b},\; R_\perp \in \mathbb{R}^{b \times b}$$ + +**Step 4 — Small-kernel SVD:** + +$$K = \begin{bmatrix} \sqrt{\lambda}\,\operatorname{diag}(\sigma_{1..k}) & P \\ 0 & R_\perp \end{bmatrix} \;\in\; \mathbb{R}^{(k+b) \times (k+b)}$$ + +$$\hat{U}_K,\; \hat{\sigma},\; \hat{V}_K = \operatorname{thin\_svd}(K)$$ + +This SVD is on a **$(k{+}b) \times (k{+}b)$** matrix — e.g. 18×18 instead of 128×18. + +**Step 5 — Back-transform to full basis:** + +$$U_\text{new} = \bigl[\, U_k \;\big|\; Q_\perp \,\bigr] \;\hat{U}_K[:,\, :n] \;\in\; \mathbb{R}^{d \times n}$$ + +where $n = \min(k{+}b,\, d,\, \text{cap})$. + +### Cost comparison · `sec:sentinel:brandsvd-cost-comparison` + +| Operation | Current (naïve) | Brand's | Ratio | +|-----------|---:|---:|---:| +| Build $M$ | $O(d \cdot c)$ | — | — | +| Projection $P = U_k^\top X^\top$ | — | $O(d \cdot k \cdot b)$ | new | +| Residual $Q$ | — | $O(d \cdot k \cdot b)$ | new | +| Thin QR of $Q$ | — | $O(d \cdot b^2)$ | new | +| SVD kernel | $O(d \cdot c^2)$ | $O(c^3)$ | **$d/c$ ≈ 7×** | +| Back-transform $U_\text{new}$ | $O(d \cdot c \cdot n)$ | $O(d \cdot c \cdot n)$ | same | + +### Phase 1 projection reuse · `sec:sentinel:brandsvd-projection-reuse` + +Phase 1 of `observe()` already computes: + +``` +z = X · U_k // (b × k) — the latent coordinates +x_hat = z · U_k^T // (b × d) +residual = X - x_hat // (b × d) +``` + +Brand's Step 1 needs $P = U_k^\top X^\top = Z^\top$, and Step 2 needs $Q = X^\top - U_k P = \text{residual}^\top$. Both are already computed in Phase 1. Passing `z` and `residual` into `evolve_subspace()` eliminates the redundant matrix builds entirely. + +## Implementation · `sec:sentinel:brandsvd-implementation` + +### Source layout · `sec:sentinel:brandsvd-source-layout` + +| File | Purpose | +|------|---------| +| `maths/mod.rs` | `SvdStrategy` enum, `evolve()` dispatch + oracle | +| `maths/brand_svd.rs` | Brand's incremental SVD (~130 lines) | +| `maths/naive_svd.rs` | Naïve dense thin SVD (reference) | +| `maths/bench_tracing.rs` | `SpanTimingLayer` for benchmark timing | +| `maths/tests/brand_vs_naive.rs` | 14 oracle unit tests | +| ~~`convergence_benchmark.rs`~~ | Removed — timing ported to criterion `warmup_cost_detailed`; diagnostics to `convergence_diagnostics.rs` | + +### Strategy pattern · `sec:sentinel:brandsvd-strategy-pattern` + +`SvdStrategy` is a runtime-selectable enum (`Naive` | `Brand`, default `Brand`) stored in `SentinelConfig`. The `evolve()` function dispatches via `run_svd()`, which wraps each strategy call in a named `info_span!` (`"svd_brand"` or `"svd_naive"`) for timing capture. + +### Debug oracle · `sec:sentinel:brandsvd-debug-oracle` + +When `cfg!(debug_assertions)` is true (debug builds) **or** a `DEBUG`-level tracing subscriber is attached (release with `RUST_LOG=debug`), `evolve()` runs **both** strategies and compares their outputs: + +- Singular values: relative tolerance $10^{-8}$ +- Basis columns: $|\cos \theta| > 1 - 10^{-6}$ (allowing SVD sign ambiguity) + +A **thread-local `ORACLE_FLIP`** toggle alternates execution order on every call, so neither strategy consistently benefits from warmed caches or branch predictors. + +### Benchmark timing · `sec:sentinel:brandsvd-benchmark-timing` + +`SpanTimingLayer` (in `bench_tracing.rs`) accumulates per-span-name wall-clock durations without console output. The benchmarks read timing programmatically via `timing.total_ns("svd_brand")`. An `EnvFilter` (default INFO) controls whether the oracle fires — `RUST_LOG=debug` activates it in release mode. + +## Measured Results · `sec:sentinel:brandsvd-measured-results` + +All measurements on d=128, 500 rounds, `convergence_matches_theory` benchmark. + +### Test config (λ=0.95, b=4) · `sec:sentinel:brandsvd-test-config` + +| Run mode | Brand (ms) | Naïve (ms) | Speedup | +|----------|-----------|-----------|---------| +| Debug (opt=3, oracle on) | 2188 | 4203 | **1.92×** | +| Release (oracle off) | 1977 | — | — | +| Release + RUST_LOG=debug | 1944 | 3731 | **1.92×** | + +### Production config (λ=0.99, b=16) · `sec:sentinel:brandsvd-production-config` + +| Run mode | Brand (ms) | Naïve (ms) | Speedup | +|----------|-----------|-----------|---------| +| Debug (opt=3, oracle on) | 4703 | 13998 | **2.98×** | +| Release (oracle off) | 5538 | — | — | +| Release + RUST_LOG=debug | 5283 | 15149 | **2.87×** | + +### Summary · `sec:sentinel:brandsvd-summary` + +Brand's incremental SVD delivers a consistent **~1.9× speedup at b=4** and **~2.9× speedup at b=16** (production config), scaling as expected with the $d/c$ ratio in the SVD kernel. The speedup is stable across debug and release builds. + +## Consequences · `sec:sentinel:brandsvd-consequences` + +### Performance · `sec:sentinel:brandsvd-performance` + +- **SVD kernel shrank from $(d \times c)$ to $(c \times c)$.** For production configs this is 128×18 → 18×18, a measured ~2.9× reduction in `evolve_subspace()` cost. + +- **Phase 1 projection is reused**, eliminating redundant $O(d \cdot k \cdot b)$ matrix multiplications from Phase 2. + +- **Per-cell noise injection cost (ADR-S-015) drops proportionally.** At 100 noise rounds and ~2.9× speedup on the dominant step, cell creation time drops significantly. + +- **A new thin QR is added** — $O(d \cdot b^2)$ per round. This is cheaper than the SVD it replaces and has lower constant factors (direct Householder, no iteration). + +### Numerical equivalence · `sec:sentinel:brandsvd-numerical-equivalence` + +Brand's incremental SVD is mathematically equivalent to the naïve approach — both compute the thin SVD of the same composite matrix $M$. The difference is purely in how the computation is structured. + +The 14 oracle tests in `maths/tests/brand_vs_naive.rs` verify equivalence across a range of configurations: + +- Minimal dimensions, identity basis, rank-one, large batch +- Saturated rank, large singular values, near-zero residual +- Multi-step sequences (10 consecutive evolve calls) +- Representative configs matching both benchmark profiles +- Sorted/non-negative singular values, orthonormal output basis + +The debug oracle runs on **every** `evolve()` call in test builds, providing continuous regression coverage. + +### Complexity · `sec:sentinel:brandsvd-complexity` + +- The change is confined to `maths/` and the call site in `observe()` (to pass `z` and `residual` instead of `x`). + +- No new dependencies. Uses `faer::Mat::thin_svd()` for the small kernel, and `faer::Mat::qr()` for the residual QR. + +- ~130 lines of new code (`brand_svd.rs`) alongside the existing naïve implementation (~60 lines in `naive_svd.rs`). + +### Risks · `sec:sentinel:brandsvd-risks` + +- **Rank growth beyond $b$:** When $k > b$, the kernel is $(k{+}b) \times (k{+}b)$ which can grow up to $(\text{cap}{+}b)$. At cap=16, b=16 this is 32×32 — still far smaller than 128×32. The speedup is always at least $d/(k{+}b)$. + +- **QR numerical stability:** If the residual $Q$ is nearly zero (new data lies almost entirely in the current subspace), the QR may produce near-zero $R_\perp$ entries. This is handled naturally — the small singular values from those directions will be discarded by rank adaptation (Phase 5). No special-casing needed. The `equivalence_near_zero_residual` test validates this case explicitly. + +## References · `sec:sentinel:brandsvd-references` + +- Brand, M. (2002). "Incremental Singular Value Decomposition of Uncertain Data with Missing Values." *ECCV 2002.* +- Brand, M. (2006). "Fast low-rank modifications of the thin singular value decomposition." *Linear Algebra and its Applications*, 415(1), 20–30. +- Baker, C.G., Gallivan, K.A., Van Dooren, P. (2012). "Low-Rank Incremental Methods for Computing Dominant Singular Subspaces." *Linear Algebra and its Applications*, 436(8), 2866–2888. diff --git a/packages/sentinel/adr/017-deferred-cell-warm-up.md b/packages/sentinel/adr/017-deferred-cell-warm-up.md new file mode 100644 index 000000000..993436c9d --- /dev/null +++ b/packages/sentinel/adr/017-deferred-cell-warm-up.md @@ -0,0 +1,82 @@ +# ADR-S-017: Deferred Cell Warm-Up · `rec:sentinel:background-priority-warmup-off-ingest-path` + +**Status:** Implemented (synchronous fallback + background thread; "No Slot Reservation" superseded by ADR-S-019) **Date:** 2026-03-11 **Revised:** 2026-03-13 **Spec:** §ALGO S-12.9 (work variance and timing considerations) **Relates to:** [ADR-S-007](007-automatic-noise-injection.md) (automatic noise injection), [ADR-S-015](015-cell-creation-performance.md) (cell creation performance), [ADR-S-002](002-feed-forward-invariant.md) (feed-forward invariant), [ADR-S-005](005-deterministic-order-and-thread-safety.md) (deterministic order) + +## Context · `sec:sentinel:deferredwarmup-context` + +The sentinel's `ingest()` call has **variable latency**. Most calls perform only scoring (fast), but calls that trigger analysis set changes also run noise injection synchronously — up to hundreds of milliseconds per new cell (ADR-S-015). A batch-oriented network monitor that budgets $T$ ms per batch cannot tolerate multi-second stalls from cell creation bursts. + +## Decision · `sec:sentinel:deferredwarmup-decision` + +**With background warming enabled, noise injection is moved off the `ingest()` hot path.** The same staging lifecycle remains in synchronous mode, but `reconcile_analysis_set()` drains it to completion before `ingest()` returns. + +New cells transition through a three-state lifecycle: + +``` + created ──→ warming ──→ online ──→ (destroyed) + │ + │ background thread + │ highest g.sum first + ▼ + noise injection +``` + +1. **Created.** The analysis selector identifies a new cell. A `SubspaceTracker` is allocated but receives no noise and no real observations. The cell is enqueued into a staging area. + +2. **Warming.** A background thread works on whichever warming cell has the highest `g.sum` (accumulated volume in the G-V Graph), injecting noise batches according to the depth-tiered noise schedule (ADR-S-015). If a higher-priority cell arrives, work switches immediately — the previous cell retains partial progress. + +3. **Online.** Noise injection is complete. At the start of the next `ingest()` call, the cell is promoted from the staging area into the live `cells` map and begins receiving real observations. + +### Observation Routing During Warm-Up · `sec:sentinel:deferredwarmup-observation-routing` + +Observations destined for a warming cell are routed to the nearest **online** ancestor (or to the root if no closer ancestor is online). Scoring quality does not degrade — it stays at the coarser resolution until the child goes online. + +### Coordination During Warm-Up · `sec:sentinel:deferredwarmup-coordination` + +Warming cells do not activate coordination contexts. Coordination activates only at promotion, at which point the cell's baselines are converged. Each newly activated coordination context runs a cheap inline warm-up (§ALGO S-11.7) using Gamma-sampled synthetic score vectors derived from the participating cells' mature baselines. + +### No Slot Reservation · `sec:sentinel:deferredwarmup-no-slot-reservation` + +> **Superseded by ADR-S-019.** Warming cells are members of the investment set and hold investment slots because their trackers and warm-up resources have been allocated, but they do not enter the producing sets or hold production slots until they are online. The original paragraph below records the earlier lifecycle terminology. + +Warming cells do not hold competitive slots. The analysis set contains only online cells. If a warming cell's G-node is evicted before warm-up completes, the partial work is discarded — the G-V Graph determined the interval no longer warrants a node. + +### Priority: Volume-First · `sec:sentinel:deferredwarmup-volume-first-priority` + +The priority key is cached `g.sum` first. Volume leads because a shallow cell accumulates everything beneath it, so ordering on volume alone already places an ancestor at or above every cell in its own subtree — the dependency constraint (ancestors online before descendants) as a consequence of the quantity that also measures which cell is worth warming next. + +Depth follows, because volume alone does not decide the cases the constraint is about. A path node whose accumulation is entirely the single active cell below it ties with that cell exactly, and cached volume is an approximation of the node sum besides, so equal volumes are the ordinary case on precisely the chains the ordering exists to protect. Comparing depth next resolves the tie toward the shallower cell. + +`GNodeId` is last, a deterministic tie-break between cells of one depth rather than a carrier of the ancestor rule. It cannot carry that rule: identifiers order on the arena slot index and the arena reuses freed slots, so a cell created into a recycled slot can hold a smaller identifier than an ancestor allocated before it, and breaking an equal-volume tie on the identifier alone would warm that descendant first. + +### Synchronous Fallback · `sec:sentinel:deferredwarmup-synchronous-fallback` + +When `background_warming` is disabled, warm-up runs synchronously within `reconcile_analysis_set()`. This preserves deterministic single-threaded behaviour for testing. + +## Work Variance Bound (§ALGO S-12.9) · `sec:sentinel:deferredwarmup-work-variance-bound` + +With warm-up deferred to the background worker, the per-call cost of `ingest()` is bounded by: + +$$O\!\Big(n(d_{\text{geo}} + h_V) \;+\; |\mathcal{A}^*| \cdot w_{\max} \cdot (k + b)^2 \;+\; |\mathcal{E}|\log|\mathcal{E}| \;+\; |\mathcal{I}|\log|\mathcal{I}| \;+\; |\mathcal{I}| \cdot w_{\max}\min(w_{\max},\, r_{\max})\Big)$$ + +on every call. What deferral takes off the call is the noise injection, and only that: it contributes zero cost here because the background worker runs it. Cell creation is not free — the selection that identifies a new cell and the tracker allocated for it both stay in line, and they are the last three terms. + +Selection is recomputed from the graph on every call, whatever the batch and whether or not the selection changes: the eligible entries $\mathcal{E}$ within the depth cutoff are ranked by importance for the top-$K$ cut (§ALGO S-8.1), then closed under G-tree ancestry into $\mathcal{I}$ (§ALGO S-8.2), which is the $|\mathcal{E}|\log|\mathcal{E}| + |\mathcal{I}|\log|\mathcal{I}|$ pair — independent of $n$, and the same on a call that changes nothing. The entry and exit bookkeeping over the two sets, the staging pass that refreshes cached volumes, and the promotion of cells the worker finished are all $O(|\mathcal{I}|)$ map operations and are absorbed in the second of those terms. §ALGO S-8.1 permits maintaining the competitive targets incrementally instead; this engine ranks them afresh, and the term is what that costs. + +Tracker allocation is per entering cell: a $w \times \text{cap}$ basis with the latent vectors and second-moment triangle beside it (§ALGO S-4.1), where $\text{cap} = \min(w, r_{\max})$, so $O(w_{\max}\min(w_{\max},\, r_{\max}))$ apiece. The multiplier above is the worst case rather than the ordinary one: a typical call enters no cell and allocates nothing, and only a call that re-selects the whole set enters $|\mathcal{I}| \leq 1 + K\bar{D}$ of them (§ALGO S-8.2). Allocation happens once per entry into $\mathcal{I}$, against the tens to hundreds of noise rounds the depth-tiered schedule then runs for that one cell (ADR-S-015), which is why moving the rounds off the call was worth doing and leaving the allocation on it is not a defect. + +Both added groups are bounded by the selector's parameters rather than by traffic, which is what keeps the call predictable; neither is zero. The call-to-call variance from batch size, analysis set size and rank changes slowly relative to call frequency. The allocation term does not vary smoothly at all: it is zero on most calls and a bounded burst on the calls that change the selection. + +## Timing Protection Is Out of Scope (§ALGO S-12.9) · `sec:sentinel:deferredwarmup-timing-protection-out-of-scope` + +Deferred warm-up makes `ingest()` operationally predictable — bounded work per call with no structural spikes. It does **not** make `ingest()` constant-time. The remaining variance, though small, is observable to a sufficiently precise adversary. Adaptive timing pads and equalization are explicitly out of scope. + +## Consequences · `sec:sentinel:deferredwarmup-consequences` + +- With `background_warming` enabled, `ingest()` has bounded, predictable work per call: noise injection never stalls the hot path, and what cell creation leaves on it — one selection pass, and one tracker allocation per entering cell — is bounded by the selector's parameters rather than by the warm-up schedule. The flag defaults to disabled, and there the bound does not hold — reconciliation drains the staging area in line, warming every newly staged cell to completion before `ingest()` returns, which is the stall this record's context describes. That is the price of the fallback rather than a defect in it: synchronous warm-up buys single-threaded determinism with exactly the latency the background path moves off the call. + +- A background thread (or synchronous fallback) is required for warming. The interaction surface is minimal: a staging map with atomic promotion at the top of each `ingest()` call that carries observations. A batch that is empty — on arrival, or left so by the domain decision — returns the empty report of §ALGO S-9.1 before promotion is reached, so a call with nothing to ingest promotes nothing and the pipeline's own progress is unaffected: warm-up advances independently of observation cadence, and only the transition to online waits for the next call that carries observations. Promotion is gated on ingestion rather than on the clock because a newly online cell must take part in the same call's routing, which an empty call has nothing to offer it. + +- ADR-S-007 (automatic noise injection) remains correct — injection is still automatic and internal — but the trigger changes from "inject synchronously at creation" to "enqueue for background injection at creation." + +- ADR-S-015 (cell creation performance) is complementary. The depth-tiered noise schedule determines how long background warming takes per cell; the hot-path stall concern is resolved by this ADR. diff --git a/packages/sentinel/adr/018-generic-domain-parameters.md b/packages/sentinel/adr/018-generic-domain-parameters.md new file mode 100644 index 000000000..710d3c294 --- /dev/null +++ b/packages/sentinel/adr/018-generic-domain-parameters.md @@ -0,0 +1,284 @@ +# ADR-S-018: Generic Domain Parameters · `rec:sentinel:generic-coordinate-accumulator-and-domain-width` + +**Status:** Accepted **Date:** 2026-03-13 **Spec:** §ALGO S-2.1 (domain), §ALGO S-1.6 (what sentinel owns), §ALGO S-2.4 (suffix encoding), §ALGO S-13.3 (G-V Graph config) **Supersedes:** [ADR-S-003](003-mudlark-integration.md) Part A (type parameters) **Relates to:** [ADR-S-003](003-mudlark-integration.md) (mudlark integration — Part B on Cargo features is unchanged), [ADR-S-002](002-feed-forward-invariant.md) (feed-forward invariant), mudlark [ADR-M-006](../../mudlark/adr/006-generic-parameters.md) (generic parameters), mudlark [ADR-M-009](../../mudlark/adr/009-trait-decomposition.md) (trait decomposition) + +## Context · `sec:sentinel:domainparams-context` + +ADR-S-003 Part A hardcodes `GvGraph`: the sentinel owns a single concrete instantiation of the G-V Graph with 128-bit coordinates, `u64` counts, and full 128-bit domain width. + +Two developments challenge this: + +1. **Some hosts require N = 64.** A host that encodes observations as `u64` cannot use a 128-bit domain without wasting 64 constant suffix dimensions, a full SVD rank slot (25% of `max_rank = 4`), and doubling per-tracker memory. + +2. **Different hosts may want different accumulator semantics.** The sentinel exposes `graph()` to the host. With a hardcoded `V = u64`, the host receives `&GvGraph` and can call any method `u64` supports. But a host that wants a custom accumulator — weighted observations, saturating counters, a newtype with domain-specific overflow policy — cannot use the sentinel at all. The sentinel should not be the bottleneck on what the host can do with the graph it carries. + +Mudlark is already fully generic: `GvGraph` works today for any conforming types. The sentinel is the only layer that locks the parameters. + +## Decision · `sec:sentinel:domainparams-decision` + +**`SpectralSentinel` becomes generic over `C`, `V`, and `N`, mirroring mudlark's `GvGraph` triple.** + +```rust +pub struct SpectralSentinel +where + C: Coordinate + CentredBitSource, + V: Inspectable + Attenuatable, +{ + graph: GvGraph, + // ... +} +``` + +### The three parameters · `sec:sentinel:domainparams-three-parameters` + +| Parameter | Role in sentinel | Bound | Rationale | +|-----------|-----------------|-------|-----------| +| `C` | Coordinate type — spatial addressing, interval bounds, observation values | `Coordinate + CentredBitSource` | Mudlark routing + sentinel bit-extraction | +| `V` | Accumulator type — importance accounting in the G-V Graph | `Inspectable + Attenuatable` | Sentinel calls `observe()`, `layers()`, `plateaus()`, `decay()` | +| `N` | Domain bit-width — tree height, suffix dimensions | `u32` (const generic) | Controls `CentredBits` vector length and max depth | + +### Type aliases for common instantiations · `sec:sentinel:domainparams-type-aliases` + +```rust +pub type Sentinel128 = SpectralSentinel; +pub type Sentinel64 = SpectralSentinel; +``` + +Existing consumers use `Sentinel128` or `Sentinel64` as appropriate for their domain width. + +--- + +## Part A — Coordinate Parameter `C` · `sec:sentinel:domainparams-coordinate-parameter` + +### `CentredBitSource`: sentinel's bridge trait · `sec:sentinel:domainparams-centred-bit-source` + +The sentinel must convert coordinate values to centred bit vectors for subspace analysis (§ALGO S-2.3). This requires bit-level access that `Coordinate` does not provide. Rather than modify mudlark's trait surface, the sentinel defines a bridge trait of its own: + +```rust +pub trait CentredBitSource: Coordinate { + fn to_centred_bits(&self, n: u32) -> CentredBits; +} + +impl CentredBitSource for u64 { + fn to_centred_bits(&self, n: u32) -> CentredBits { + // Extract n bits: bit i → if (value >> (n-1-i)) & 1 { +0.5 } else { -0.5 } + } +} + +impl CentredBitSource for u128 { + fn to_centred_bits(&self, n: u32) -> CentredBits { + // Same, with 128-bit shifts. + } +} +``` + +This trait is public — external consumers implementing a custom coordinate type must provide an impl to use it with the sentinel. + +### `CentredBits`: fixed-size array with runtime length · `sec:sentinel:domainparams-centred-bits` + +```rust +pub struct CentredBits { + pub bits: [f64; 128], // max-size stack buffer + pub len: usize, // actual width = N +} +``` + +The fixed array avoids heap allocation on the hot path. At N = 64, 64 trailing slots are unused — 512 bytes wasted per instance, but `CentredBits` is short-lived (created per observation, consumed immediately by the tracker). The `suffix()` method respects `len`: `&self.bits[depth..self.len]`. + +### Sites that change · `sec:sentinel:domainparams-coordinate-change-sites` + +Every `u128` in production code becomes `C`: + +- `CellState { start: C, end: C }` — interval bounds +- `ingest(&[C])` — observation input +- `CentredBits::from_u128(v)` → `C::to_centred_bits(N)` +- `u128::MAX` → `C::domain_max(N)` +- `128 - depth` → `N as usize - depth as usize` + +--- + +## Part B — Accumulator Parameter `V` · `sec:sentinel:domainparams-accumulator-parameter` + +### Sentinel's relationship with `V` · `sec:sentinel:domainparams-accumulator-relationship` + +The sentinel is a **pass-through** for accumulator values. It: + +1. **Writes** unit deltas into the graph: `graph.observe(coord, delta)` +2. **Reads** importance back: `total_sum()`, `gnode_info().own`, `gnode_info().sum` +3. **Compares** importance: sorting by `PartialOrd` in analysis set selection +4. **Stores** importance in internal types: `AnalysisEntry`, `WarmingCell` +5. **Exposes** the graph: `graph() → &GvGraph` — the host may call any method their `V` supports + +The sentinel never performs V-arithmetic itself (no `add`, `sub`, `zero` in sentinel logic). It shuttles V values between mudlark and its own bookkeeping/report types. + +### Bound: `V: Inspectable + Attenuatable` · `sec:sentinel:domainparams-accumulator-bound` + +This is the minimum that makes the sentinel's mudlark API calls compile: + +| Sentinel operation | Mudlark method | Required bound | +|---|---|---| +| Feed observations | `observe()` | `Inspectable` | +| Walk V-tree | `layers()` | `Inspectable` | +| Read contour | `plateaus()` | `Inspectable` | +| Temporal decay | `decay()` | `Inspectable + Attenuatable` | +| Read totals | `total_sum()`, `gnode_info()` | `Accumulator` (implied by `Inspectable`) | + +The sentinel does **not** bound on `Weighable` or `Proratable`. But through `graph()`, the host can call `sample()` or `range_sum()` if their `V` satisfies those bounds — the sentinel carries `V` without constraining it beyond its own needs. + +### Why not lock `V = u64`? · `sec:sentinel:domainparams-no-u64-lock` + +Locking `V = u64` is simpler (no type parameter on ~10 internal types) but closes the door: + +- A host using weighted observations (`observe(coord, weight)`) must use a different accumulator. The sentinel's Δ = 1 policy (ADR-S-002) governs *sentinel-initiated* observations, but a host wrapping the sentinel could feed different deltas for its own purposes. +- A host wanting `graph().sample()` with a custom weighting scheme needs `V: Weighable` — which `u64` satisfies, but a custom newtype would not unless the sentinel propagates `V`. +- Mudlark's `Config` already requires `V` for `split_threshold`. The sentinel's config would need a conversion boundary rather than directly storing `V`. + +The type parameter tax is real (~10 types gain ``) but justified: the sentinel should not be the bottleneck on what the host can do with the spatial substrate it carries. + +### Unit delta construction · `sec:sentinel:domainparams-unit-delta` + +The sentinel observes with `Δ = 1` (ADR-S-002). With generic `V`, the unit delta is constructed via `Inspectable::from_f64(1.0)` and cached at construction time: + +```rust +let unit_delta: V = V::from_f64(1.0); +// ... +self.graph.observe(value, unit_delta); +``` + +For all built-in types this is exact. For custom types, `from_f64(1.0)` must return the type's unit increment — this is an invariant of any sensible `Inspectable` implementation. + +### `SentinelConfig` carries `V` · `sec:sentinel:domainparams-config-accumulator` + +```rust +pub struct SentinelConfig { + pub split_threshold: V, + // ...~30 other fields unchanged (f64, u32, usize, bool) +} +``` + +One field out of ~30 carries `V`. The cost is that consumers write `SentinelConfig` instead of `SentinelConfig`. This is acceptable — it matches mudlark's `Config` pattern and makes the type-level contract explicit. + +--- + +## Part C — Domain Width `N` · `sec:sentinel:domainparams-domain-width` + +### Propagation · `sec:sentinel:domainparams-width-propagation` + +`N` propagates through `GvGraph` and into every width computation: + +- Root cell width: `N` +- Suffix width: `N - depth` +- `CentredBits` vector length: `N` +- Max depth: `N` (fits in `u8` for N ≤ 255) + +### `SubspaceTracker`: already generic · `sec:sentinel:domainparams-tracker-generic` + +The core SVD engine takes `dim: usize` at runtime construction. It contains no hardcoded `128`. No changes needed. + +### Config: domain-independent · `sec:sentinel:domainparams-domain-independent-config` + +`SentinelConfig` has no N-dependent fields. All thresholds are `f64` or `u32`. The noise schedule uses depth tiers, not absolute widths. N propagates only through the type system, not through config values. + +--- + +## Part D — Report Type Strategy · `sec:sentinel:domainparams-report-strategy` + +### Coordinate in reports: generic `C` · `sec:sentinel:domainparams-generic-report-coordinates` + +Report types that carry interval bounds become generic over `C`: + +```rust +pub struct CellReport { pub start: C, pub end: C, … } +pub struct CoordinationReport { pub start: C, pub end: C, … } +pub struct MemberScore { pub cell_start: C, pub cell_end: C, … } +pub struct CellInspection { pub start: C, pub end: C, … } +``` + +### Accumulator in reports: erased to `f64` · `sec:sentinel:domainparams-erased-report-accumulator` + +Importance values in reports are diagnostic — the host reads them for health monitoring, not for correctness-critical computation. Rather than propagating `V` through all report types (which would make `BatchReport`), importance is erased at the report boundary via `Inspectable::to_f64_approx()`: + +```rust +pub struct ContourSnapshot { + pub total_importance: f64, // V::to_f64_approx() + // ... +} + +pub struct AnalysisSetSummary { + pub importance_range: (f64, f64), + // ... +} +``` + +This keeps `BatchReport` — one generic parameter, not two. The precision loss (`u64` values above $2^{53}$ lose LSBs) is acceptable for diagnostic fields. A host needing exact importance can read `graph().total_sum()` directly, which returns `V`. + +**Exception:** `AnalysisEntry` (internal type) retains `V` for importance because the analysis set sorts by exact importance values: + +```rust +pub struct AnalysisEntry { + pub importance: V, + pub start: C, + pub end: C, + // ... +} +``` + +This type is `pub(crate)` — it does not surface in the public API. + +--- + +## Type Propagation Summary · `sec:sentinel:domainparams-propagation-summary` + +| Type | Parameters | Public? | +|------|-----------|---------| +| `SpectralSentinel` | `C, V, N` | Yes | +| `SentinelConfig` | `V` | Yes | +| `BatchReport` | `C` | Yes | +| `CellReport` | `C` | Yes | +| `CoordinationReport` | `C` | Yes | +| `MemberScore` | `C` | Yes | +| `CellInspection` | `C` | Yes | +| `ContourSnapshot` | — | Yes | +| `AnalysisSetSummary` | — | Yes | +| `HealthReport` | — | Yes | +| `CellState` | `C` | No (`pub(super)`) | +| `AnalysisEntry` | `C, V` | No (`pub(crate)`) | +| `AnalysisSet` | `C, V` | No (`pub(crate)`) | +| `WarmingCell` | `C, V` | No (`pub(crate)`) | +| `StagingArea` | `C, V` | No (`pub(crate)`) | +| `CentredBits` | — | No | +| `SubspaceTracker` | — | No | + +V appears in 5 types (1 public: `SentinelConfig`; 4 internal). + +--- + +## Alternatives Considered · `sec:sentinel:domainparams-alternatives` + +| Alternative | Pros | Cons | +|-------------|------|------| +| **Lock `V = u64`, generic `C` and `N` only** | Fewer types carry V; simpler config | Closes door on custom accumulators; host cannot use `graph()` with custom V; diverges from mudlark pattern | +| **Runtime `domain_bits: u32` instead of `const N`** | No const-generic propagation; tests keep `u128` | Wastes 8 bytes per coord at N=64; no compile-time enforcement; slight runtime overhead | +| **`V` generic in reports (not erased)** | Exact importance in reports | `BatchReport` — two generics on every consumer; ~5 more types carry V for diagnostic-only fields | +| **`CentredBits` as `Vec`** | Exact sizing | Heap allocation per observation per cell on the hot path; dwarfed by SVD cost but unnecessary | +| **Add `bit(index) → bool` to mudlark's `Coordinate`** | Clean bit extraction | Changes mudlark's trait surface for sentinel's benefit; `CentredBitSource` in sentinel is less invasive | +| **`GvGraph` exposed via trait object / type-erased wrapper** | Host doesn't see `V` | Loses monomorphisation; runtime dispatch on hot path; `GvGraph` is not object-safe | + +## Consequences · `sec:sentinel:domainparams-consequences` + +- `SpectralSentinel` replaces the concrete `SpectralSentinel`. Type aliases `Sentinel128` and `Sentinel64` provide ergonomic shorthand. + +- ADR-S-003 Part A is superseded. Part B (Cargo features) is unchanged. + +- The sentinel's minimum bound on V (`Inspectable + Attenuatable`) is strictly less than "all four sub-traits". A host with a custom V that only implements these two bounds can use the sentinel; it just cannot call `graph().sample()` or `graph().range_sum()` — the compiler enforces this per-method, not per-struct. + +- `SentinelConfig` carries V for `split_threshold`. All other config fields remain concrete. Consumers write `SentinelConfig` — one extra token. + +- Report types carry only ``, not ``. Importance values are `f64` in reports. Hosts needing exact V read `graph().total_sum()` or `graph().gnode_info().own` directly. + +- `CentredBitSource` is a public sentinel trait. Adding support for a new coordinate type (e.g. `u32`, `f64`) requires an impl in the downstream crate — a one-function addition. + +- Test code migrates gradually: existing `u128` tests can use `Sentinel128`; new `u64` tests use `Sentinel64`. No test needs both instantiations simultaneously. + +- The compiler catches every missed migration site: a `u128` where `C` is expected is a type error, not a runtime bug. diff --git a/packages/sentinel/adr/019-investment-set-terminology-and-reporting.md b/packages/sentinel/adr/019-investment-set-terminology-and-reporting.md new file mode 100644 index 000000000..be0ab8dbd --- /dev/null +++ b/packages/sentinel/adr/019-investment-set-terminology-and-reporting.md @@ -0,0 +1,95 @@ +# ADR-S-019: Investment-Set Terminology and Reporting Alignment · `rec:sentinel:investment-set-reporting-priority-and-vocabulary` + +**Status:** Accepted **Date:** 2026-03-17 **Spec:** §ALGO S-8 (analysis selector), §ALGO S-11.6 (warm-up lifecycle), §ALGO S-14.11–14.12 (reporting) **Supersedes:** [ADR-S-017](017-deferred-cell-warm-up.md) §"No Slot Reservation" (warming cells now hold investment slots) **Relates to:** [ADR-S-017](017-deferred-cell-warm-up.md) (deferred cell warm-up), [ADR-S-006](006-analysis-set-recomputation.md) (analysis set recomputation) + +## Context · `sec:sentinel:investment-context` + +The algorithm specification (§ALGO S-8) was updated to introduce a three-level naming hierarchy for analysis cells: + +| Symbol | Name | Definition | +|--------|------|------------| +| $\mathcal{T}$ | Competitive targets | Top $K$ V-entries by importance within the depth cutoff | +| $\mathcal{I}$ | Investment set | $\mathcal{T}$ closed under G-Tree ancestry; every member has an allocated tracker regardless of online status | +| $\mathcal{A}$ | Producing competitive set | $\mathcal{T} \cap \text{Online}$ — competitive targets that are online and producing scores | +| $\mathcal{A}^*$ | Producing full set | $\mathcal{I} \cap \text{Online}$ — all online members of the investment set | + +The key insight is that the **investment set** ($\mathcal{I}$) is the resource-commitment boundary — every member has an allocated tracker and is either online or warming — while the **producing sets** ($\mathcal{A}$, $\mathcal{A}^*$) are the score-generation boundary. + +Previously (ADR-S-017 §"No Slot Reservation"), warming cells were described as not holding competitive slots and the analysis set as containing only online cells. The spec now formalises that warming cells hold **investment slots** in $\mathcal{I}$ — they have resources committed — while not holding **production slots** in $\mathcal{A}$. + +The spec also introduced the `g.sum` warm-up ordering (§ALGO S-11.6.2) and eager removal (§ALGO S-8.5) as named concepts, and requires three new reporting fields (§ALGO S-14.11–14.12). + +### What already works · `sec:sentinel:investment-already-works` + +The implementation's behaviour is **functionally correct** for the new spec: + +- `AnalysisSet::recompute()` selects competitors and closes under ancestry — computing $\mathcal{T}$ then $\mathcal{I}$. +- The staging area holds warming cells with allocated trackers — this *is* the investment commitment. +- `retain_in_set()` implements eager removal — evicting warming and in-flight cells that left the analysis set. +- The background warming thread prioritises by `volume` (which *is* `g.sum` — `info.sum.to_f64_approx()`). +- Coordination contexts are demand-driven (lazy creation/pruning), which is equivalent to the spec's explicit activate/deactivate calls. + +### What needs to change · `sec:sentinel:investment-needs-change` + +1. **Reporting fields.** The spec defines three fields that do not exist in the implementation: + - `HealthReport`: `investment_set_size`, `warming_trackers`, `warming_competitive_targets`. + - `AnalysisSetSummary`: `investment_set_size`. + +2. **Synchronous drain ordering.** `drain_all_synchronous()` processes cells in `BTreeMap` key order (ascending `GNodeId`). The spec requires g.sum order (§ALGO S-11.6.2). In practice this is immaterial — synchronous mode warms all cells to completion before any participate — but the code should match the spec for consistency and to avoid a latent bug if incremental synchronous warm-up is ever added. + +3. **Source terminology.** Source comments, doc comments, and `implementation.md` use the pre-update names ("analysis set", "competitive set", "full analysis set"). These should be updated to use the new vocabulary where appropriate. + +## Decision · `sec:sentinel:investment-decision` + +### 1. Add investment-aware reporting fields · `sec:sentinel:investment-reporting-fields` + +**`HealthReport`** gains three fields: + +```rust +/// Total cells with allocated trackers (online + warming): +/// $|\mathcal{I}|$. +pub investment_set_size: usize, + +/// Members of $\mathcal{I}$ currently in the warm-up pipeline. +pub warming_trackers: usize, + +/// Competitive targets ($\mathcal{T}$) not yet promoted to +/// $\mathcal{A}$ — i.e. warming cells that are competitive +/// targets, not ancestors. +pub warming_competitive_targets: usize, +``` + +**`AnalysisSetSummary`** gains one field: + +```rust +/// Total investment set size: $|\mathcal{I}|$ (online + warming). +pub investment_set_size: usize, +``` + +These are derived from `cells.len()` (online) plus staging area counts. The staging area already tracks per-cell `is_competitive`, so distinguishing warming ancestors from warming competitive targets requires no new state. + +### 2. Fix synchronous drain ordering · `sec:sentinel:investment-synchronous-drain-ordering` + +`drain_all_synchronous()` will sort cells by cached `volume` (descending), then by depth so that an ancestor drains before the cells beneath it, then by `GNodeId` as the final deterministic tie-break — the whole of the background thread's g.sum priority rule rather than its leading term. Both layers below volume are load-bearing: equal volumes are the ordinary case on an ancestor chain, and the identifier cannot stand in for depth because arena slot reuse lets a descendant hold the smaller one (ADR-S-017). + +### 3. Align source terminology · `sec:sentinel:investment-source-terminology` + +Source comments and `implementation.md` will be updated to use the new terms. No public API names are changed in this ADR — the Rust field names (`competitive_size`, `full_size`, etc.) remain stable. The doc comments on those fields will clarify the spec mapping. + +### 4. Accepted deviations · `sec:sentinel:investment-accepted-deviations` + +Two implementation patterns differ from the spec's pseudocode but are behaviourally equivalent: + +- **Lazy coordination lifecycle.** The spec's Step 3 pseudocode explicitly calls `ActivateCoordinationContexts` / `DeactivateCoordinationContexts`. The implementation creates coordination contexts on the first scoring pass that needs them and prunes them during `propagate_coordination_from_root()` against online competitive membership. Contexts affect output only when both subtrees contribute scores, while their learned state persists across quiet batches for the same membership. + +- **Full recompute vs. incremental reconciliation.** The spec's pseudocode computes `OldInvestment \ NewInvestment` and `NewInvestment \ OldInvestment` incrementally. The implementation recomputes the full analysis set from scratch (ADR-S-006), then reconciles the diff at the orchestrator level. The result is identical. + +## Consequences · `sec:sentinel:investment-consequences` + +- Hosts gain visibility into the investment/production distinction through three new report fields. + +- The synchronous drain path matches the spec's ordering guarantee, eliminating a class of potential bugs if synchronous warm-up is ever made incremental. + +- ADR-S-017's "No Slot Reservation" section is superseded: warming cells now formally hold investment slots in $\mathcal{I}$, though not production slots in $\mathcal{A}$. + +- No public API breaks. Existing field names are preserved; new fields are additive. diff --git a/packages/sentinel/adr/020-clip-pressure-ewma.md b/packages/sentinel/adr/020-clip-pressure-ewma.md new file mode 100644 index 000000000..e7555f4e6 --- /dev/null +++ b/packages/sentinel/adr/020-clip-pressure-ewma.md @@ -0,0 +1,192 @@ +# ADR-S-020: Clip-Pressure EWMA · `rec:sentinel:clip-pressure-ewma` + +**Status:** Accepted **Date:** 2026-03-18 **Spec:** §ALGO S-6.1.1 (baseline update pipeline), §ALGO S-6.4 (clip-pressure dynamics), §ALGO S-13.1 ($\lambda_\rho$), §ALGO S-14.4 (per-cell clip-pressure reporting), §ALGO S-14.11 (fleet-wide clip-pressure distribution) **Supersedes:** The effective-clip formula in ADR-S-013 §1a (graduated clip-exemption using η alone) **Relates to:** [ADR-S-013](013-warm-up-convergence-benchmark.md) (warm-up convergence benchmark), [ADR-S-007](007-automatic-noise-injection.md) (automatic noise injection) + +## Context · `sec:sentinel:clippressure-context` + +The algorithm specification was amended to introduce a **clip-pressure EWMA** ($\bar{\rho}$) — a per-axis exponentially-weighted moving average of the batch clip ratio $\rho_t = (\text{clipped samples}) / (\text{total samples})$. + +### Problem · `sec:sentinel:clippressure-problem` + +The previous design (ADR-S-013 §1a) modulated the effective clip ceiling solely via the noise-influence parameter $\eta$: + +$$ +n_\sigma^{\text{eff}} = n_\sigma + n_\sigma \cdot \frac{\eta}{1 - \eta + \varepsilon} +$$ + +This works well during warm-up (where $\eta$ is large and organic clipping would create a positive feedback loop) but has a blind spot: **sustained high clipping in production** ($\eta \approx 0$) after a genuine regime change leaves the ceiling at $n_\sigma$ regardless of contamination level, and the baseline slowly poisons. + +### Solution in the spec · `sec:sentinel:clippressure-spec-solution` + +A new per-axis state $\bar{\rho}_t$ tracks the fraction of clipped samples via an EWMA with decay $\lambda_\rho$ (default 0.95, half-life ≈ 14 batches). The effective-clip formula is now unified: + +$$ +p = \max(\eta,\; \bar{\rho}), \qquad +n_\sigma^{\text{eff}} = n_\sigma \cdot \left(1 + \frac{p}{1 - p + \varepsilon}\right) +$$ + +When $\bar{\rho}$ is low (clean traffic), this collapses to the production ceiling. When $\bar{\rho}$ is high (contamination), the ceiling widens automatically — precisely the same safety valve that $\eta$ provides during warm-up, but now reactive to ongoing conditions. + +The spec also consolidates clipping into a **single shared filter** computed against the fast EWMA's ceiling, applied once per batch. The slow EWMA (CUSUM reference) and the fast EWMA both receive the same retained sample set, rather than each computing an independent clip filter. + +### What already works · `sec:sentinel:clippressure-already-works` + +- **Pre-clip raw batch mean for CUSUM**: `update_axis()` computes the raw `mean` from all scores and passes it to `cusum.update()` as `batch_mean` — this is already the pre-clip mean required by §ALGO S-6.1.1 step 5. + +- **Formula shape**: The current formula `n + n·η/(1-η+ε)` is algebraically equivalent to `n·(1 + η/(1-η+ε))`, so the multiplicative form in the new spec is the same shape — just with $p$ replacing $\eta$. + +- **Coherence rank-drop reset**: `adapt_rank()` already calls `coherence_bl.reset_cold()` when rank drops below 2 — adding $\bar{\rho}$ reset is a one-line extension. + +### What needs to change · `sec:sentinel:clippressure-needs-change` + +The implementation has **11 gaps** between the current code and the amended spec: + +1. **New per-axis state.** `AxisBaseline` needs a `clip_pressure: f64` field ($\bar{\rho}$), initialised to 0.0. + +2. **New config parameter.** `SentinelConfig` needs `clip_pressure_decay: f64` ($\lambda_\rho$) with default 0.95 and validation `0 < λ_ρ < 1`. + +3. **Unified modulation formula.** The effective-clip computation in `observe()` must change from using $\eta$ alone to $p = \max(\eta, \bar{\rho})$ per axis. + +4. **Single shared clip filter.** Clipping must move from inside `EwmaStats::update()` (per-EWMA independent) to the caller (`update_axis()`), computed once against the fast EWMA's ceiling and applied to both EWMAs. + +5. **Clip ratio tracking.** The clip filter must report the clip ratio $\rho_t = 1 - |\text{retained}| / |\text{total}|$ back to the caller, so the caller can update $\bar{\rho}$. + +6. **CUSUM slow EWMA receives pre-filtered samples.** Since clipping is externalised, `CusumAccumulator::update()` must accept pre-filtered samples instead of re-clipping internally. + +7. **Report: `ScoreDistribution::clip_pressure`** (§ALGO S-14.4). New `f64` field in `[0, 1]`. + +8. **Report: `HealthReport` clip-pressure distribution** (§ALGO S-14.11). New summary field (min/max/mean across active trackers). + +9. **Coherence rank-drop reset.** `adapt_rank()` must also zero the coherence axis's $\bar{\rho}$. + +10. **Warm-up completion reset** (§ALGO S-11.4). When noise influence crosses the warm-up threshold, all four axes' $\bar{\rho}$ must be zeroed. + +11. **Baseline memory accounting.** Per-axis baseline size grows from 7 to 8 floats ($4 \times 8 = 32$ total), matching §ALGO S-4.3. + +## Decision · `sec:sentinel:clippressure-decision` + +### 1. Add clip-pressure state to `AxisBaseline` · `sec:sentinel:clippressure-axis-baseline-state` + +```rust +struct AxisBaseline { + fast: EwmaStats, + cusum: CusumAccumulator, + /// Clip-pressure EWMA: ρ̄ ∈ [0, 1]. + clip_pressure: f64, +} +``` + +Initialised to `0.0`. `reset_cold()` zeros it. + +### 2. Add config parameter · `sec:sentinel:clippressure-config-parameter` + +```rust +/// Clip-pressure EWMA decay factor (λ_ρ). +/// +/// Controls how quickly the clip-pressure estimate responds to +/// changing contamination levels. Higher values = longer memory. +/// +/// Default: `0.95` (half-life ≈ 14 batches). +/// Must be in (0, 1). +pub clip_pressure_decay: f64, +``` + +Default `0.95`. Validated alongside `clip_sigmas`. + +### 3. Externalise clipping from `EwmaStats` · `sec:sentinel:clippressure-externalise-clipping` + +`EwmaStats::update()` currently computes its own clip filter internally. The method signature changes to accept pre-filtered values and skip its internal filter: + +**Option A — split into two methods:** +- `update_raw(values)` — accepts pre-filtered values, updates mean/variance unconditionally (no internal clipping). +- `update(values, clip_sigmas)` — preserved for any callers that still need self-contained clipping (backward compat). + +**Option B — move clip logic to caller entirely:** +- `update()` drops its `clip_sigmas` parameter and always trusts the caller to have filtered. + +We choose **Option A** for minimal disruption. The test suite's direct `EwmaStats::update()` calls continue to work. The hot path (`update_axis`) calls the new `update_raw()`. + +### 4. Single shared clip filter in `update_axis()` · `sec:sentinel:clippressure-shared-clip-filter` + +`update_axis()` gains the responsibility of: + +1. Computing the clip ceiling from the **fast EWMA**'s current mean and variance: `ceiling = n_σ_eff · √var + mean`. +2. Filtering: `retained = scores.filter(|&v| v < ceiling)`. +3. Computing `ρ_t = 1 − retained.len() / scores.len()`. +4. Updating $\bar{\rho}$: `ρ̄ = λ_ρ · ρ̄ + (1 − λ_ρ) · ρ_t`. +5. Passing `&retained` to both `fast.update_raw()` and `cusum.update_filtered()`. + +### 5. Adjust `CusumAccumulator` to accept pre-filtered samples · `sec:sentinel:clippressure-cusum-prefiltered` + +`CusumAccumulator::update()` changes to accept pre-filtered samples (no internal re-clipping by the slow EWMA): + +```rust +pub fn update_filtered( + &mut self, + filtered: &[f64], + raw_batch_mean: f64, + allowance_sigmas: f64, + eps: f64, +) { + // Gap uses raw_batch_mean (pre-clip), as before. + let gap = raw_batch_mean - self.slow.mean() - allowance; + self.accumulator = (self.accumulator + gap).max(0.0); + // Slow EWMA receives the already-filtered samples. + self.slow.update_raw(filtered); + self.steps_since_reset += 1; +} +``` + +The old `update()` can be deprecated or removed. + +### 6. Per-axis effective-clip computation · `sec:sentinel:clippressure-effective-clip` + +The effective clip is now **per-axis** (each axis has its own $\bar{\rho}$). `observe()` computes four separate effective-clip values, one per `update_axis()` call: + +```rust +let p = eta.max(bl.clip_pressure); +let effective_clip = clip_sigmas * (1.0 + p / (1.0 - p + eps)); +``` + +This replaces the single `effective_clip` variable computed from $\eta$ alone. + +### 7. Reporting · `sec:sentinel:clippressure-reporting` + +`ScoreDistribution` gains: + +```rust +/// Current clip-pressure EWMA for this axis: ρ̄ ∈ [0, 1]. +pub clip_pressure: f64, +``` + +`HealthReport` gains a `ClipPressureDistribution`: + +```rust +pub clip_pressure_distribution: ClipPressureDistribution, +``` + +with `min`, `max`, and `mean` across active tracker axes. + +### 8. Lifecycle resets · `sec:sentinel:clippressure-lifecycle-resets` + +- `AxisBaseline::reset_cold()` — zeros `clip_pressure`. +- `adapt_rank()` — when coherence is destroyed, `reset_cold()` already covers the new field. +- Warm-up completion — when $\eta$ crosses the threshold, zero all four axes' `clip_pressure` to prevent warm-up contamination echoing into production scoring. + +### 9. Accepted deviations · `sec:sentinel:clippressure-accepted-deviations` + +- **Cold-path bypass is preserved.** When an EWMA is cold (first update), all values are accepted even under the new shared filter. This matches the spec's "skip entirely on first update" (§ALGO S-6.1.1 Design Note 1). + +- **Formula equivalence.** The implementation may use either the additive form `n + n·p/(1-p+ε)` or the multiplicative form `n·(1 + p/(1-p+ε))` — they are algebraically identical. The choice is left to readability preference. + +## Consequences · `sec:sentinel:clippressure-consequences` + +- The effective-clip ceiling becomes **self-correcting**: it widens automatically under sustained contamination (high $\bar{\rho}$) and tightens to $n_\sigma$ once the attack subsides. + +- `EwmaStats::update()` retains backward compatibility. The new `update_raw()` method is used only by the baseline pipeline. + +- Four new `f64` fields (one per axis) increase per-tracker memory by 32 bytes — negligible at sentinel scale. + +- One new config parameter (`clip_pressure_decay`). Hosts that don't set it get the default $\lambda_\rho = 0.95$. + +- ADR-S-013 §1a's graduated clip-exemption formula is superseded: the unified $\max(\eta, \bar{\rho})$ formula subsumes the pure-η behaviour as a special case (when $\bar{\rho} = 0$, $p = \eta$ exactly). diff --git a/packages/sentinel/adr/021-ewma-mean-centred-variance.md b/packages/sentinel/adr/021-ewma-mean-centred-variance.md new file mode 100644 index 000000000..af2b74041 --- /dev/null +++ b/packages/sentinel/adr/021-ewma-mean-centred-variance.md @@ -0,0 +1,207 @@ +# ADR-S-021: EWMA-Mean-Centred Latent Variance · `rec:sentinel:ewma-mean-centred-latent-variance` + +**Status:** Implemented **Date:** 2026-03-24 **Spec:** §ALGO S-4.2 (Phase 3 — Evolve Latent Distribution), §ALGO S-5.4 (surprise scoring), §ALGO S-11.2 (cold-start latent seeding), §ALGO S-Appendix A (convergence methodology) **Relates to:** [ADR-S-013](013-warm-up-convergence-benchmark.md) (warm-up convergence benchmark — introduced the cold→warm fix this ADR amends) + +## Context · `sec:sentinel:latentvar-context` + +The algorithm specification (§ALGO S-4.2, Phase 3) has been amended to replace the **within-batch population variance** estimator for $\nu^{(z)}$ with an **EWMA-mean-centred** estimator. The implementation in `SubspaceTracker::evolve_latent()` still uses the old formula. + +### Problem · `sec:sentinel:latentvar-problem` + +The within-batch population variance $\operatorname{Var}(Z_{:,j}) = \frac{1}{b}\sum_i (z_{ij} - \bar{Z}_j)^2$ has a systematic negative bias of $\frac{b-1}{b}$ relative to the surprise numerator's expected value: + +| Batch size $b$ | Bias | Effect | +| -------------- | ------------- | --------------------------------------------------- | +| 1 | $-100\%$ | Variance erodes to $\varepsilon$; surprise → $10^5$ | +| 2 | $-50\%$ | Severe underestimate; surprise doubled | +| 4 | $-25\%$ | Material; surprise inflated by 33% | +| 16 | $-6.25\%$ | Significant at precision targets | +| 64+ | $< -1.6\%$ | Negligible | + +The surprise score divides by $\nu^{(z)}_j + \varepsilon$, so the underestimate inflates surprise scores. At $b = 1$ (which occurs at per-cell trackers receiving one observation per ingestion cycle), the variance collapses to zero every batch, the EWMA decays toward $\varepsilon$, and per-dimension surprise contributions reach $O(10^5)$. At $b = 2$ (coordination trackers with two contributing cells), the $-50\%$ bias produces a sustained $2\times$ surprise overestimate. + +This is not a hypothetical concern in practice: a host that ingests **one observation per call** ($b = 1$ always at the root tracker) operates at the worst-case batch size for this bias. + +### Solution in the spec · `sec:sentinel:latentvar-spec-solution` + +The amended formula computes deviations from the **EWMA mean** $\mu^{(z)}_j$ rather than the batch mean $\bar{Z}_j$: + +**Subsequent batches ($t > 0$):** + +$$\nu^{(z)}_j \leftarrow \lambda\,\nu^{(z)}_j + \alpha \cdot \max\!\left(\frac{1}{b}\sum_{i=1}^{b}(z_{ij} - \mu^{(z)}_j)^2,\;\varepsilon\right)$$ + +**First batch ($t = 0$):** + +$$\nu^{(z)}_j \leftarrow \max\!\left(\frac{1}{b}\sum_{i=1}^{b} z_{ij}^2,\;\varepsilon\right)$$ + +(using the pre-allocated $\mu^{(z)} = \mathbf{0}$ as centring reference, which makes the $t = 0$ formula a special case of the $t > 0$ formula). + +The spec also mandates: + +1. **Update order**: $\nu^{(z)}$ first (against pre-update $\mu$), then $\mu^{(z)}$, then $\Gamma$. Variance sees the same $\mu$ that scoring (Phase 1) used. + +2. **Runtime floor**: $\nu^{(z)}_j \leftarrow \max(\nu^{(z)}_j, 10^{-2})$ after every update, including first-batch seeding. For centred-bit cell inputs and unit basis columns, each surprise contribution and their rank average are bounded by $d/(0.01+\varepsilon) \leq 100d$, up to roundoff; the derivation and scope are in §ALGO S-4.2. + +### Why the EWMA-mean-centred formula eliminates the bias · `sec:sentinel:latentvar-bias-elimination` + +The within-batch component contributes $\frac{b-1}{b}\sigma^2$ (same as the old formula). The batch-mean-vs.-EWMA-mean component contributes $\frac{\sigma^2}{b}$. They sum to $\sigma^2$ regardless of $b$ — exact cancellation of the $\frac{b-1}{b}$ bias. + +The residual bias is $\operatorname{Var}(\mu^{(z)}_j) \approx \frac{\alpha}{b(1+\lambda)}\sigma^2$ — positive (safe direction: dampens surprise), small, worst-case $+0.5\%$ at $b = 1$, $\lambda = 0.99$. + +### Trade-offs accepted · `sec:sentinel:latentvar-accepted-tradeoffs` + +- **Coupling to $\mu^{(z)}$**: stale $\mu$ inflates $\nu$, dampening surprise during regime transitions. This is the safe direction. Displacement is the primary mean-shift detector; surprise's role is distributional shape anomalies (preserved). + +- **Basis rotation sensitivity**: after Phase 2 rotates the basis, projections are on the new basis while $\mu$ reflects the old. $(z - \mu)^2$ inflates, $\nu$ inflates, surprise dampens. Transient, $O(t_{1/2})$ batches. The old formula was invariant (re-centring on $\bar{Z}_j$ subtracted out any rotation-induced mean shift). + +## Decision · `sec:sentinel:latentvar-decision` + +Implement the amended Phase 3 in `SubspaceTracker::evolve_latent()` and update impacted tests. + +### What already works · `sec:sentinel:latentvar-already-works` + +- **Surprise scoring formula** (Phase 1): `(z_ij - lat_mean[j])² / + (lat_var[j] + eps)` — already uses lat_mean as its centring reference. No change needed. + +- **Pre-allocated initial values**: `lat_mean = 0.0`, `lat_var = 1.0`, `cross_corr = 0.0` — unchanged. + +- **Cross-correlation ($\Gamma$) update**: uses raw second moments $z_{ij} \cdot z_{il}$, not centred products. No change needed. + +- **Rank-change behaviour**: pre-allocated entries at capacity, EWMA loops iterate `0..k`. No change needed. + +### What needs to change · `sec:sentinel:latentvar-needs-change` + +The implementation has **5 gaps** between the current code and the amended spec: + +1. **Variance formula (subsequent batches).** In `evolve_latent()`, the `col_var` computation currently centres on the batch mean: + + ```rust + let mut col_var = 0.0; + for i in 0..b { + let d = z[(i, j)] - col_mean; + col_var += d * d; + } + col_var /= b_f; + ``` + + Must change to centre on `self.lat_mean[j]` (the pre-update EWMA mean): + + ```rust + let mut col_var = 0.0; + for i in 0..b { + let d = z[(i, j)] - self.lat_mean[j]; + col_var += d * d; + } + col_var /= b_f; + ``` + +2. **Variance formula (first batch).** The cold path currently uses within-batch variance centred on batch mean. Must change to centre on the pre-allocated `lat_mean[j]` (which is 0.0 at $t = 0$): + + ```rust + // Before (centres on batch mean col_mean): + self.lat_var[j] = col_var.max(eps); + // After (col_var already computed against lat_mean[j] = 0.0): + self.lat_var[j] = col_var.max(eps); + ``` + + The seeded value changes from the within-batch population variance to $\frac{1}{b}\sum z_{ij}^2$. The code change is in the `col_var` computation (gap 1 handles both paths since the loop uses the same formula). + +3. **Update order.** The current code computes mean and variance in a single loop, updating both in the same branch: + + ```rust + if cold { + self.lat_mean[j] = col_mean; + self.lat_var[j] = col_var.max(eps); + } else { + self.lat_mean[j] = lam.mul_add(self.lat_mean[j], alpha * col_mean); + self.lat_var[j] = lam.mul_add(self.lat_var[j], alpha * col_var.max(eps)); + } + ``` + + Must reorder: update variance first (against pre-update mean), then update mean. Because `col_var` is now computed against `self.lat_mean[j]` (read before any mutation), the natural order is: + + ```rust + if cold { + self.lat_var[j] = col_var.max(eps); + self.lat_mean[j] = col_mean; + } else { + self.lat_var[j] = lam.mul_add(self.lat_var[j], alpha * col_var.max(eps)); + self.lat_mean[j] = lam.mul_add(self.lat_mean[j], alpha * col_mean); + } + ``` + + This is a semantic change: the old order was mean-then-variance; the new order is variance-then-mean. Because `col_var` is now pre-computed against the pre-update `lat_mean[j]`, the reorder ensures the EWMA variance input uses the same reference as the surprise numerator. + +4. **Runtime floor.** After the per-dimension loop (both cold and non-cold paths), clamp: + + ```rust + for j in 0..k { + self.lat_var[j] = self.lat_var[j].max(1e-2); + } + ``` + + The floor is $25\times$ below the null-hypothesis variance of $0.25$ and may bind on degenerate streams. For cell inputs in $\{-1/2,1/2\}^{d}$, unit basis columns give $|z_j| \leq \sqrt{d}/2$; a mean initialized at zero and subsequently seeded or convexly averaged from such coordinates obeys the same bound. Thus $(z_j-\mu_j)^2 \leq d$, giving each contribution and its rank average the bound $d/(0.01+\varepsilon) \leq 100d$ for $\varepsilon>0$, up to floating-point roundoff. + +5. **Test expectations.** Several existing tests assert against the old formula's behaviour: + + - `latent_variance_reaches_steady_state` — steady-state $\nu^{(z)}$ values will change (EWMA-mean-centred is slightly higher than within-batch at $b = 4$). The assertion `v < 0.5` should still hold, but may need loosening if steady-state is higher. + + - `cold_warm_eliminates_surprise_nonstationarity` — the rise factor should improve (the new formula eliminates the $b$-dependent bias that contributed to the transient). + + - `subspace_evolution_does_not_dominate_latvar_transient` — the tolerance `diff_pct < 50.0` should still hold; verify. + + - `convergence_ewma::ewma_variance_convergence` — tests the `EwmaStats` type directly; unaffected (the EWMA machinery itself is unchanged). + + - `convergence_clipping` tests — EWMA variance assertions on the surprise baseline; should still hold but verify numerically. + +### New tests · `sec:sentinel:latentvar-new-tests` + +| Test | Validates | +| --------------------------------------------------- | --------------------------------------------------------------------------- | +| `b1_surprise_bounded` | At $b = 1$, surprise scores stay bounded (no $O(10^5)$ explosion) | +| `b1_latent_variance_stable` | At $b = 1$, `lat_var` converges to $\approx 0.25$, not $\varepsilon$ | +| `b2_no_systematic_surprise_inflation` | At $b = 2$, surprise ratio near 1.0 (no 2× inflation) | +| `runtime_floor_prevents_degenerate_collapse` | Constant-observation stream: `lat_var` ≥ $10^{-2}$ | +| `update_order_variance_before_mean` | Variance update uses pre-update mean (white-box: instrument `lat_mean` read) | +| `batch_size_invariant_surprise_ratio` | Surprise ratio $\approx 1.0$ at $b \in \{1, 2, 4, 16, 64\}$ (self-consistency) | + +### Impact on hosts · `sec:sentinel:latentvar-host-impact` + +The change is transparent to consumers. Hosts read `BatchReport` fields (z-scores, CUSUM accumulators, maturity) via the public API. The internal formula change affects the *values* of those fields but not their types or semantics. + +Behavioural effects for hosts operating at small batch sizes: + +- **$b = 1$:** Surprise scores drop from potentially $O(10^5)$ to $O(1)$. Any confidence quality factor derived from surprise will be dramatically more stable during warm-up and during production when the distribution is well-behaved. + +- **Confidence convergence:** Faster. At $b = 1$ the old formula inflated surprise by no fixed factor at all: the variance collapsed every batch and the EWMA decayed toward $\varepsilon$, so per-dimension surprise reached $O(10^5)$ and baseline settling was delayed for as long as that erosion ran. The bounded factors the bias table derives belong to the larger batches — $2\times$ at $b = 2$, $33\%$ at $b = 4$. The new formula's self-consistency property means surprise starts near $1.0$ immediately. + +- **CUSUM sensitivity:** Unchanged in steady state (the CUSUM reference tracks the same signal). During transitions, surprise CUSUM drift is shorter-lived (the $\nu$ co-adaptation described in §ALGO S-4.2 dampens sustained elevation). + +No consumer code changes are required. The surprise formula references $\nu^{(z)}_j + \varepsilon$ in the denominator — unchanged. + +### Summary of implementation steps · `sec:sentinel:latentvar-implementation-summary` + +1. Modify `evolve_latent()` in [tracker.rs](../src/sentinel/tracker.rs): + - Change `col_var` computation to centre on `self.lat_mean[j]` instead of `col_mean`. + - Reorder assignments: variance before mean. + - Add runtime floor loop after per-dimension updates. + +2. Add new tests in a new test module `src/tests/variance_formula.rs` (or extend `convergence_noise.rs`) covering $b = 1$, $b = 2$, runtime floor, update order, and batch-size-invariant surprise ratio. + +3. Verify and adjust existing test expectations as needed (tolerance bounds only; no logic changes expected). + +4. Run full `--package sentinel` test suite in both debug and release modes. + +5. Run `--workspace` tests to confirm no downstream breakage. + +## Consequences · `sec:sentinel:latentvar-consequences` + +- **Surprise scores become batch-size-invariant.** The expected surprise ratio is $1 + O(\alpha^2)$ regardless of $b$. This eliminates a class of false positives at small batch sizes and a class of false negatives at large batch sizes. + +- **Hosts at $b = 1$ stabilise.** The old formula produced catastrophic surprise inflation at this batch size; the new formula produces correctly-scaled scores from the first batch. + +- **Regime-transition behaviour changes.** Surprise spikes after mean shifts are shorter-lived. The first batch fires at full strength; subsequent batches' $\nu$ co-adapts, progressively dampening the ratio. Displacement inherits the primary detection role during transitions. + +- **Convergence benchmarks (ADR-S-013) may need updating.** Convergence times at $b_{\text{noise}} = 4$ may improve (the $-25\%$ bias that contributed to displacement bimodality is eliminated). At $b_{\text{noise}} \geq 16$, changes are negligible. + +- **Runtime floor gives a dimension-dependent bound for cell inputs.** With centred bits, unit basis columns and convex mean updates from an initial zero, each surprise contribution and their rank average are at most $d/(0.01+\varepsilon) \leq 100d$, up to roundoff. This replaces the corresponding $d/\varepsilon$ bound from an epsilon-only variance floor; it does not bound surprise for unbounded coordination-score vectors. diff --git a/packages/sentinel/benches/sentinel.rs b/packages/sentinel/benches/sentinel.rs new file mode 100644 index 000000000..b218491c0 --- /dev/null +++ b/packages/sentinel/benches/sentinel.rs @@ -0,0 +1,478 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +//! Criterion benchmarks for the Spectral Sentinel. +//! +//! Run with: +//! +//! ```sh +//! cargo bench -p torrust-sentinel +//! ``` + +use std::hint::black_box; + +use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; +use torrust_sentinel::{CentredBits, Sentinel128, SentinelConfig}; + +// ─── Helpers ──────────────────────────────────────────────── + +/// Lightweight config for micro-benchmarks: small rank, small k. +fn bench_config() -> SentinelConfig { + SentinelConfig:: { + max_rank: 4, + forgetting_factor: 0.95, + rank_update_interval: 10, + analysis_k: 16, + analysis_depth_cutoff: 6, + energy_threshold: 0.90, + eps: 1e-6, + per_sample_scores: false, + cusum_allowance_sigmas: 0.5, + cusum_slow_decay: 0.999, + cusum_coord_slow_decay: 0.999, + clip_sigmas: 3.0, + clip_pressure_decay: 0.95, + split_threshold: 100, + d_create: 3, + d_evict: 6, + budget: 100_000, + noise_schedule: torrust_sentinel::NoiseSchedule::Explicit(vec![5]), + noise_batch_size: 4, + noise_seed: Some(42), + svd_strategy: torrust_sentinel::SvdStrategy::Brand, + background_warming: false, + } +} + +/// Realistic config with higher rank and larger analysis set. +fn realistic_config() -> SentinelConfig { + SentinelConfig:: { + max_rank: 16, + forgetting_factor: 0.99, + rank_update_interval: 100, + analysis_k: 1024, + analysis_depth_cutoff: 6, + energy_threshold: 0.90, + eps: 1e-6, + per_sample_scores: false, + cusum_allowance_sigmas: 0.5, + cusum_slow_decay: 0.999, + cusum_coord_slow_decay: 0.999, + clip_sigmas: 3.0, + clip_pressure_decay: 0.95, + split_threshold: 100, + d_create: 3, + d_evict: 6, + budget: 100_000, + noise_schedule: torrust_sentinel::NoiseSchedule::Explicit(vec![50]), + noise_batch_size: 16, + noise_seed: Some(42), + svd_strategy: torrust_sentinel::SvdStrategy::Brand, + background_warming: false, + } +} + +/// Generate `count` sequential values in a single narrow range. +fn single_range_values(count: usize) -> Vec { + (0..count).map(|i| (0xAB_u128 << 120) | (i as u128 + 1)).collect() +} + +/// Generate `count` values spread across many leading-nibble ranges. +fn multi_range_values(count: usize) -> Vec { + (0..count) + .map(|i| { + let nibble = (i % 16) as u128; + (nibble << 124) | (i as u128 + 1) + }) + .collect() +} + +/// Create a warmed sentinel ready for steady-state benchmarking. +fn warmed_sentinel(cfg: &SentinelConfig) -> Sentinel128 { + let mut s = Sentinel128::new(cfg.clone()).unwrap(); + + // Seed cells by ingesting diverse values. + let seed: Vec = (0..4_u128) + .flat_map(|nibble| (0..4_u128).map(move |i| (nibble << 124) | (i + 1))) + .collect(); + s.ingest(&seed); + + // Noise is auto-injected at construction (§ALGO S-11). + + // One real batch to enter the warm code path. + let batch: Vec = (0..4_u128) + .flat_map(|nibble| (0..4_u128).map(move |i| (nibble << 124) | (i + 100))) + .collect(); + s.ingest(&batch); + + s +} + +// ─── Observation encoding ─────────────────────────────────── + +fn bench_centred_bits(c: &mut Criterion) { + c.bench_function("CentredBits::from_u128", |b| { + b.iter(|| CentredBits::from_u128(black_box(0xDEAD_BEEF_CAFE_BABE_1234_5678_9ABC_DEF0))); + }); +} + +// ─── Ingest (core hot path) ───────────────────────────────── + +fn bench_ingest_cold(c: &mut Criterion) { + let mut group = c.benchmark_group("ingest_cold"); + + for batch_size in [1, 8, 32] { + let values = single_range_values(batch_size); + group.throughput(Throughput::Elements(batch_size as u64)); + group.bench_with_input(BenchmarkId::new("single_range", batch_size), &values, |b, vals| { + b.iter_with_setup( + || Sentinel128::new(bench_config()).unwrap(), + |mut s| s.ingest(black_box(vals)), + ); + }); + } + + group.finish(); +} + +fn bench_ingest_warm(c: &mut Criterion) { + let mut group = c.benchmark_group("ingest_warm"); + let cfg = bench_config(); + + for batch_size in [1, 8, 32, 64] { + let values = single_range_values(batch_size); + group.throughput(Throughput::Elements(batch_size as u64)); + group.bench_with_input(BenchmarkId::new("single_range", batch_size), &values, |b, vals| { + b.iter_with_setup(|| warmed_sentinel(&cfg), |mut s| s.ingest(black_box(vals))); + }); + } + + // Multi-range: triggers coordination tier. + for batch_size in [16, 64] { + let values = multi_range_values(batch_size); + group.throughput(Throughput::Elements(batch_size as u64)); + group.bench_with_input(BenchmarkId::new("multi_range", batch_size), &values, |b, vals| { + b.iter_with_setup(|| warmed_sentinel(&cfg), |mut s| s.ingest(black_box(vals))); + }); + } + + group.finish(); +} + +// ─── Ingest at realistic scale ────────────────────────────── + +fn bench_ingest_realistic(c: &mut Criterion) { + let mut group = c.benchmark_group("ingest_realistic"); + let cfg = realistic_config(); + + for batch_size in [16, 64, 256] { + let values = multi_range_values(batch_size); + group.throughput(Throughput::Elements(batch_size as u64)); + group.bench_with_input(BenchmarkId::new("multi_range", batch_size), &values, |b, vals| { + b.iter_with_setup(|| warmed_sentinel(&cfg), |mut s| s.ingest(black_box(vals))); + }); + } + + group.finish(); +} + +// ─── Construction (includes automatic noise injection) ────── + +fn bench_construction(c: &mut Criterion) { + let mut group = c.benchmark_group("construction"); + + for rounds in [10, 50] { + let cfg = SentinelConfig:: { + noise_schedule: torrust_sentinel::NoiseSchedule::Explicit(vec![rounds]), + noise_batch_size: 16, + noise_seed: Some(42), + ..bench_config() + }; + group.bench_with_input(BenchmarkId::new("noise_rounds", rounds), &cfg, |b, cfg| { + b.iter(|| Sentinel128::new(black_box(cfg.clone())).unwrap()); + }); + } + + group.finish(); +} + +// ─── ADR-S-013 §1: Warm-up convergence benchmarks ────────── + +/// Measure construction cost at candidate noise schedule values. +/// +/// This answers: "how many milliseconds does it cost to increase +/// noise rounds from 50 to 100, 200, or 400?" +/// +/// Two configs are tested: +/// - **bench**: `max_rank=4`, `analysis_k=16`, `b=4`, `λ=0.95` +/// - **realistic**: `max_rank=16`, `analysis_k=1024`, `b=16`, `λ=0.99` +fn bench_noise_round_scaling(c: &mut Criterion) { + let mut group = c.benchmark_group("noise_round_scaling"); + group.sample_size(20); // construction is expensive at high rounds + + for rounds in [10, 50, 100, 200, 400] { + // Bench config (small) + let cfg = SentinelConfig:: { + noise_schedule: torrust_sentinel::NoiseSchedule::Explicit(vec![rounds]), + noise_seed: Some(42), + ..bench_config() + }; + group.bench_with_input(BenchmarkId::new("bench", rounds), &cfg, |b, cfg| { + b.iter(|| Sentinel128::new(black_box(cfg.clone())).unwrap()); + }); + + // Realistic config (production-like) + let cfg = SentinelConfig:: { + noise_schedule: torrust_sentinel::NoiseSchedule::Explicit(vec![rounds]), + noise_seed: Some(42), + ..realistic_config() + }; + group.bench_with_input(BenchmarkId::new("realistic", rounds), &cfg, |b, cfg| { + b.iter(|| Sentinel128::new(black_box(cfg.clone())).unwrap()); + }); + } + + group.finish(); +} + +/// Detailed warm-up convergence cost (ADR-S-013 §4). +/// +/// Measures the actual wall-clock cost of construction + noise +/// injection at a dense range of noise schedule values for the +/// production-like config. This extends `bench_noise_round_scaling` +/// with finer granularity to identify the cost knee-point. +/// +/// Ported from the diagnostic `wall_clock_convergence_cost` test in +/// `convergence_benchmark.rs`. +fn bench_warmup_cost_detailed(c: &mut Criterion) { + let mut group = c.benchmark_group("warmup_cost_detailed"); + group.sample_size(10); // construction is expensive at high rounds + + for rounds in [5, 10, 20, 50, 65, 100, 150, 200, 300, 400, 500] { + let cfg = SentinelConfig:: { + noise_schedule: torrust_sentinel::NoiseSchedule::Explicit(vec![rounds]), + noise_seed: Some(42), + ..realistic_config() + }; + group.bench_with_input(BenchmarkId::new("production", rounds), &cfg, |b, cfg| { + b.iter(|| Sentinel128::new(black_box(cfg.clone())).unwrap()); + }); + } + + group.finish(); +} + +/// Measure per-round `ingest()` cost on a warmed sentinel. +/// +/// This answers: "if we increase noise rounds by 100, how many +/// additional ms does that cost?" — by measuring the marginal cost +/// of one `ingest()` call at various batch sizes. +fn bench_per_round_ingest(c: &mut Criterion) { + let mut group = c.benchmark_group("per_round_ingest"); + + // Bench config — single range, varying batch size. + for batch_size in [4, 16] { + let values = single_range_values(batch_size); + let cfg = bench_config(); + group.throughput(Throughput::Elements(batch_size as u64)); + group.bench_with_input(BenchmarkId::new("bench", batch_size), &values, |b, vals| { + b.iter_with_setup(|| warmed_sentinel(&cfg), |mut s| s.ingest(black_box(vals))); + }); + } + + // Realistic config — single range, varying batch size. + for batch_size in [4, 16] { + let values = single_range_values(batch_size); + let cfg = realistic_config(); + group.throughput(Throughput::Elements(batch_size as u64)); + group.bench_with_input(BenchmarkId::new("realistic", batch_size), &values, |b, vals| { + b.iter_with_setup(|| warmed_sentinel(&cfg), |mut s| s.ingest(black_box(vals))); + }); + } + + group.finish(); +} + +// ─── Health / inspection ──────────────────────────────────── + +fn bench_health(c: &mut Criterion) { + let cfg = bench_config(); + let s = warmed_sentinel(&cfg); + + c.bench_function("health", |b| { + b.iter(|| s.health()); + }); +} + +fn bench_analysis_set_summary(c: &mut Criterion) { + let cfg = bench_config(); + let s = warmed_sentinel(&cfg); + + c.bench_function("analysis_set_summary", |b| { + b.iter(|| s.analysis_set().summary()); + }); +} + +// ─── Per-sample scoring overhead ──────────────────────────── + +fn bench_per_sample_overhead(c: &mut Criterion) { + let mut group = c.benchmark_group("per_sample_scores"); + + let batch_size = 32; + let values = single_range_values(batch_size); + + for enabled in [false, true] { + let cfg = SentinelConfig:: { + per_sample_scores: enabled, + ..bench_config() + }; + + group.throughput(Throughput::Elements(batch_size as u64)); + group.bench_with_input(BenchmarkId::new("enabled", enabled), &values, |b, vals| { + b.iter_with_setup(|| warmed_sentinel(&cfg), |mut s| s.ingest(black_box(vals))); + }); + } + + group.finish(); +} + +// ─── Analysis K scaling ───────────────────────────────────── + +fn bench_analysis_k_scaling(c: &mut Criterion) { + let mut group = c.benchmark_group("analysis_k_scaling"); + let batch_size = 16; + let values = single_range_values(batch_size); + + for k in [4, 16, 64, 256] { + let cfg = SentinelConfig:: { + analysis_k: k, + ..bench_config() + }; + + group.throughput(Throughput::Elements(batch_size as u64)); + group.bench_with_input(BenchmarkId::new("k", k), &values, |b, vals| { + b.iter_with_setup(|| warmed_sentinel(&cfg), |mut s| s.ingest(black_box(vals))); + }); + } + + group.finish(); +} + +// ─── Registration ─────────────────────────────────────────── + +criterion_group!(encoding, bench_centred_bits); + +criterion_group!(ingest, bench_ingest_cold, bench_ingest_warm, bench_ingest_realistic); + +criterion_group!(auxiliary, bench_construction, bench_health, bench_analysis_set_summary); + +criterion_group!( + convergence, + bench_noise_round_scaling, + bench_warmup_cost_detailed, + bench_per_round_ingest, +); + +criterion_group!(scaling, bench_per_sample_overhead, bench_analysis_k_scaling); + +// ─── §9.11 — Temporal and analysis benchmarks ────────────── + +/// Generate `count` values in a narrow range with leading nibble +/// `nibble` and sequential low bits. +fn cell_values(nibble: u128, count: usize) -> Vec { + (0..count).map(|i| (nibble << 124) | (i as u128 + 1)).collect() +} + +fn bench_decay(c: &mut Criterion) { + let mut group = c.benchmark_group("decay"); + let cfg = bench_config(); + + for obs_count in [100, 1_000, 10_000] { + group.bench_with_input(BenchmarkId::new("observations", obs_count), &obs_count, |b, &n| { + b.iter_with_setup( + || { + let mut s = Sentinel128::new(cfg.clone()).unwrap(); + s.ingest(&multi_range_values(n)); + s + }, + |mut s| s.decay(black_box(0.5), black_box(0.0)), + ); + }); + } + + group.finish(); +} + +fn bench_decay_subtree(c: &mut Criterion) { + let mut group = c.benchmark_group("decay_subtree"); + let cfg = bench_config(); + + for obs_count in [100, 1_000, 10_000] { + group.bench_with_input(BenchmarkId::new("observations", obs_count), &obs_count, |b, &n| { + b.iter_with_setup( + || { + let mut s = Sentinel128::new(cfg.clone()).unwrap(); + s.ingest(&multi_range_values(n)); + s + }, + |mut s| { + let root = s.graph().g_root(); + s.decay_subtree(root, black_box(0.5), black_box(0.0)); + }, + ); + }); + } + + group.finish(); +} + +fn bench_analysis_set_recompute(c: &mut Criterion) { + let mut group = c.benchmark_group("analysis_set_recompute"); + + for k in [16, 64, 256] { + let cfg = SentinelConfig:: { + analysis_k: k, + split_threshold: 10, + budget: 50_000, + ..bench_config() + }; + + group.throughput(Throughput::Elements(8)); + group.bench_with_input(BenchmarkId::new("k", k), &cfg, |b, cfg| { + b.iter_with_setup( + || { + let mut s = Sentinel128::new(cfg.clone()).unwrap(); + // Pre-populate graph with diverse traffic. + for nibble in 0..16u128 { + s.ingest(&cell_values(nibble, 100)); + } + s + }, + |mut s| s.ingest(black_box(&cell_values(0xA, 8))), + ); + }); + } + + group.finish(); +} + +fn bench_report_assembly(c: &mut Criterion) { + let mut group = c.benchmark_group("report_assembly"); + let cfg = bench_config(); + + for batch_size in [8, 32, 128] { + let values = multi_range_values(batch_size); + group.throughput(Throughput::Elements(batch_size as u64)); + group.bench_with_input(BenchmarkId::new("batch_size", batch_size), &values, |b, vals| { + b.iter_with_setup(|| warmed_sentinel(&cfg), |mut s| s.ingest(black_box(vals))); + }); + } + + group.finish(); +} + +criterion_group!(temporal, bench_decay, bench_decay_subtree); + +criterion_group!(analysis, bench_analysis_set_recompute, bench_report_assembly); + +criterion_main!(encoding, ingest, auxiliary, convergence, scaling, temporal, analysis); diff --git a/packages/sentinel/docs/algorithm.md b/packages/sentinel/docs/algorithm.md new file mode 100644 index 000000000..ba31bace4 --- /dev/null +++ b/packages/sentinel/docs/algorithm.md @@ -0,0 +1,2633 @@ +# Spectral Sentinel — Algorithm Specification · `spec:sentinel:algorithm-specification` + +--- + +# Part I — Scope and Foundations · `sec:sentinel:algorithm-scope-and-foundations` + +--- + +## Chapter 1. Purpose, Scope, and Principles · `sec:sentinel:algorithm-purpose-scope-and-principles` + +### 1.1 Overview · `sec:sentinel:algorithm-purpose-overview` + +The Spectral Sentinel is a hierarchical online anomaly detector for streams of positionally structured coordinate values. It maintains adaptive spatial partitioning of an $N$-bit input domain $[0, 2^N)$, selects significant regions for statistical analysis via competitive ranking, and scores incoming observations against learned low-rank linear subspace models. + +### 1.2 Parameterisation · `sec:sentinel:algorithm-parameterisation` + +The system is defined over three parameters: + +| Parameter | Role | Requirements | +| --------- | ----------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| $N$ | Domain bit-width (positive integer) | Determines tree height and domain cardinality $2^N$. Typical values: 128, 64. | +| $C$ | Coordinate type | Total ordering over $[0, 2^N)$; bit-positional decomposition; dyadic interval arithmetic (§2.3). | +| $V$ | Accumulator type | Non-negative values with a zero element; addition; multiplicative scaling by a non-negative factor; approximate conversion to floating-point for reporting (§3.3). | + +All formulas throughout this specification are stated in terms of $N$. Concrete examples use $N = 128$ unless otherwise noted. + +### 1.3 Design Principles · `sec:sentinel:algorithm-design-principles` + +**Measure, don't decide.** All outputs are raw statistical quantities. The system never emits threat levels, recommended actions, or policy decisions. + +**Adapt, don't control.** The spatial structure evolves autonomously under observation and decay. The host controls resource ceilings, temporal policy, and analysis budgets. + +**Feed forward, don't feed back.** Anomaly scores flow outward to the host but are never fed back into the spatial layer's importance signal. The spatial layer sees $\Delta = 1$ per observation — pure volume counting. This prevents the anomaly detector from influencing its own spatial structure. + +> _Design note (why volume-only importance)._ If anomaly scores boosted importance, the affected cell would earn finer resolution, changing its statistical model, changing its scores, changing its importance — an unstable feedback loop. With $\Delta = 1$, an adversary cannot influence the spatial structure except through observation volume, which is precisely what the spatial layer is designed to handle. + +> _Design note (why timing matters)._ An observer who can measure response latency learns when the spatial structure changes: a spike means "a new cell was created for this region," leaking distribution evolution and split dynamics. Moving warm-up work off the hot path (§11) eliminates the structural source of variance. + +### 1.4 Three-Layer Architecture · `sec:sentinel:algorithm-three-layer-architecture` + +``` +Layer 1: Spatial Index + Adaptive spatial partitioning of [0, 2^N) + Pure volume tracking (Δ = 1 per observation) + Competitive ranking by observation volume + │ + │ Significance ranking → top-K selection + ▼ +Layer 2: Analysis Selector + Selects significant cells for statistical analysis + Closes selection under spatial ancestry + │ + │ Suffix bit vectors at every ancestor depth + ▼ +Layer 3: Analysis Engine + Per-cell subspace models at selected and ancestor cells + Hierarchical coordination across related cells + │ + ▼ + Analysis report → host +``` + +### 1.5 Layer Responsibilities · `sec:sentinel:algorithm-layer-responsibilities` + +| Concern | Owner | +| ---------------------------------------------------------------- | ------------------------------------------- | +| Spatial partitioning of $[0, 2^N)$ | Spatial Layer (Layer 1) | +| Competitive significance ranking | Spatial Layer — Value Tree | +| Spatial lifecycle (split, evict, absorb, restore) | Spatial Layer | +| Spatial memory (temporal decay, contour evolution) | Spatial Layer + host policy | +| Investment commitment (which cells receive warm-up and trackers) | Analysis Selector (Layer 2) | +| Production selection (which invested cells produce scores) | Analysis Selector (Layer 2) | +| Ancestor closure (spatial ancestry of investment targets) | Analysis Selector (Layer 2) | +| Statistical modelling within each cell | Analysis Engine (Layer 3) | +| Anomaly scoring (four axes) | Analysis Engine | +| Drift detection | Analysis Engine | +| Cross-cell coordination detection | Analysis Engine — hierarchical coordination | +| Interpretation and response | Host (external) | + +### 1.6 Responsibility Boundaries · `sec:sentinel:algorithm-responsibility-boundaries` + +| Property | Source | +| ---------------------------------------------- | -------------------------------------- | +| Contour structure, ranking, budget enforcement | Inherited — Spatial Layer (§3) | +| Temporal decay schedule | Host policy, executed by Spatial Layer | +| Investment set and producing set membership | Sentinel — Layer 2 (§8) | +| Statistical modelling and scoring | Sentinel — Layer 3 (§4–7) | +| Importance signal ($\Delta = 1$, no feedback) | Sentinel's invariant (§1.3) | + +--- + +## Chapter 2. Domain, Encoding, and Notation · `sec:sentinel:algorithm-domain-encoding-and-notation` + +### 2.1 The Domain · `sec:sentinel:algorithm-domain` + +The input domain is $[0, 2^N)$, where $N$ is the domain bit-width. A scoring cell at spatial tree depth $d$ covers $2^{N-d}$ integer values, corresponding to a $d$-bit prefix shared by all values scored in that cell. + +A value outside that domain is not an observation of it, and membership is decided once — at the ingestion boundary, before Step 2 of §9.1 — rather than layer by layer. Each layer would otherwise read such a value differently and the readings would not agree: the spatial layer routes by comparison against interval midpoints, so a coordinate at or above $2^N$ goes rightward at every level and accumulates in the topmost cell while one below the origin goes leftward and accumulates in the bottom-most, and neither cell's interval contains the value it received; the centred bit representation (§2.3) is defined on the low $N$ bits and returns the vector of the in-domain value the arrival is congruent to; and the scoring intervals contain it nowhere, not even at the root, so no tracker is shown it. The lifetime observation count would then record an arrival that no tracker ever saw, which is exactly the divergence the mandatory delivery of §9.3 exists to rule out. A value outside $[0, 2^N)$ therefore raises no total, moves no partition and reaches no tracker; the system records how many values a batch lost this way, and a batch left with nothing returns the empty report of §9.1. Membership is decided by comparison against the domain's own bounds rather than inferred from the width: the width names the upper bound, but says nothing about whether a coordinate type's values can fall below the origin, or compare with nothing at all as a NaN does. Those bounds are the system's own, which is why it decides membership rather than leaving it to the host, as it must leave the structural property of §2.2; and because the comparison is the root cell's containment test, whatever the boundary admits the partition delivers, for any coordinate type a host may bring. + +At full integer coordinate width, Mudlark represents the root's upper bound by the integer maximum, $2^N-1$. Its floor midpoints place each interior depth-$d$ boundary at $q2^{N-d}-1$. Sentinel uses that boundary's successor for the scoring interval, routing the boundary value to the lower cell and restoring the constant binary prefix at every depth. Cell and coordination scoring reports carry these adjusted bounds; the root endpoints remain unchanged and the final cell includes the domain maximum. Mudlark's partition, importance accounting and selection entries retain their original bounds. At narrower integer widths the exclusive bound $2^N$ is representable and no adjustment is needed; continuous-coordinate bounds are also unchanged. + +**Example depths at $N = 128$:** + +| Spatial tree depth $d$ | Prefix length | Suffix width $w$ | Potential cells at full coverage | +| ---------------------- | ------------- | ---------------- | -------------------------------- | +| 0 | 0 bits | 128 | 1 | +| 16 | 16 bits | 112 | 65,536 | +| 32 | 32 bits | 96 | $\approx 4.3 \times 10^{9}$ | +| 48 | 48 bits | 80 | $\approx 2.8 \times 10^{14}$ | +| 64 | 64 bits | 64 | $\approx 1.8 \times 10^{19}$ | +| 96 | 96 bits | 32 | $\approx 7.9 \times 10^{28}$ | +| 128 | 128 bits | 0 | $\approx 3.4 \times 10^{38}$ | + +At $N = 64$, the maximum depth is 64 and the root suffix width is 64. + +The spatial layer materialises only cells where observations have warranted refinement. The bottom contour (§3.2) is the observation-receiving surface — deep where volume is concentrated, shallow where it is sparse. The contour determines where observations route; the analysis selector separately decides where statistical trackers run by scanning V-Tree entries by competitive significance and analysis width, then adding G-Tree ancestors for multi-scale context (§8). + +### 2.2 Input Requirements · `sec:sentinel:algorithm-input-requirements` + +The system analyses bit-positional structure: leading bits determine routing, suffix bits provide statistical content. This is meaningful only when the coordinate values have **hierarchical positional structure** — values whose leading bits encode progressively finer categorical or spatial membership, so that shared prefixes imply shared context. + +Values with pseudo-random bit distributions (cryptographic hashes, random nonces, uniformly sampled identifiers) have no such structure and defeat the analysis. Nothing in a single value shows whether the stream it came from has the property, so the system processes a structureless stream of in-domain values without complaint; the host is responsible for the structural guarantee. Membership of the domain is the other half of the input contract and is not left to the host in the same way — it is decided by comparison against bounds the system itself fixes, and §2.1 says what becomes of a value outside $[0, 2^N)$. + +### 2.3 Centred Bit Representation · `sec:sentinel:algorithm-centred-bit-representation` + +Each raw coordinate value $v$ of type $C$ becomes a centred bit vector $\mathbf{x} \in \{-0.5, +0.5\}^{N}$: + +$$x_i = \begin{cases} +0.5 & \text{if bit } (N - 1 - i) \text{ of } v \text{ is 1} \\ -0.5 & \text{otherwise} \end{cases} \qquad i = 0, \ldots, N-1$$ + +Index 0 is the most significant bit. Centring gives $\mathbb{E}[x_i] = 0$ under a uniform bit distribution — a prerequisite for subspace analysis without explicit mean subtraction. + +The coordinate type $C$ must support: + +- Total ordering over $[0, 2^N)$. +- Extraction of individual bits by position (for the centred representation above). +- Dyadic interval arithmetic: midpoint computation and interval containment testing (for spatial routing). + +### 2.4 Suffix Extraction · `sec:sentinel:algorithm-suffix-extraction` + +For a cell at depth $d$, the first $d$ prefix bits are constant — resolved by routing. The working observation is the **suffix**: + +$$\mathbf{x}^{(\text{cell})} = (x_d, \ldots, x_{N-1}) \in \{-0.5, +0.5\}^w, \quad w = N - d$$ + +### 2.5 Constant-Norm Property · `sec:sentinel:algorithm-constant-norm-property` + +Every suffix vector at width $w$ satisfies $\|\mathbf{x}^{(\text{cell})}\|^2 = w/4$. This fixed-energy property means that total observation energy is constant and carries no information. By the Pythagorean theorem, projection energy $\|\hat{\mathbf{x}}_i\|^2/k$ is a perfect affine function of novelty (Pearson $r = -1$). The system therefore uses four independent scoring axes rather than five (§5). + +> _Note._ This constraint is specific to centred binary inputs. Continuous-valued inputs with variable norms would decouple projection energy from novelty. + +### 2.6 Notation · `sec:sentinel:algorithm-notation` + +| Symbol | Domain | Definition | +| --------------------- | ------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| $N$ | $\mathbb{Z}_{>0}$ | Domain bit-width; spatial tree height | +| $C$ | — | Coordinate type; spatial addressing, interval bounds (§2.3) | +| $V$ | — | Accumulator type; importance accounting in the spatial layer (§3.3) | +| $d$ | $\{0, \ldots, N\}$ | Spatial tree depth of a cell (prefix length in bits) | +| $w$ | $\{0, \ldots, N\}$ | Analysis width; $w = N - d$ (suffix length) | +| $n$ | $\mathbb{Z}_{\geq 0}$ | Ingestion batch size (coordinate values per `SentinelIngest` call) | +| $b$ | $\mathbb{Z}_{>0}$ | Per-tracker batch size (rows of $X$ fed to one tracker in one call; see batch-size note below) | +| $b_{\text{noise}}$ | $\mathbb{Z}_{>0}$ | Noise batch size (synthetic samples per warm-up round; §11.1) | +| $k$ | $\{1, \ldots, \text{cap}\}$ | Current active rank of the learned subspace (§4) | +| $\text{cap}$ | $\{1, \ldots, \min(w, r_{\max})\}^\dagger$ | Hard ceiling on rank; $r_{\max}$ is the maximum rank parameter | +| $\lambda$ | $(0, 1)$ | Forgetting factor | +| $\alpha$ | $(0, 1)$ | Learning rate; $\alpha = 1 - \lambda$ | +| $\varepsilon$ | $\mathbb{R}_{>0}$ | Numerical stability constant | +| $\tau$ | $(0, 1)$ | Cumulative energy threshold | +| $U$ | $\mathbb{R}^{w \times \text{cap}}$ | Orthonormal basis of the learned subspace (§4) | +| $\sigma$ | $\mathbb{R}^{\text{cap}}_{\geq 0}$ | Singular values (energy per basis vector) (§4) | +| $\mu^{(z)}$ | $\mathbb{R}^{\text{cap}}$ | EWMA mean of latent coordinates (§4) | +| $\nu^{(z)}$ | $\mathbb{R}^{\text{cap}}_{>0}$ | EWMA variance of latent coordinates (§4) | +| $\Gamma$ | $\mathbb{R}^{\text{cap} \times \text{cap}}$ | EWMA second-moment matrix of latent coordinates (§4) | +| $X$ | $\mathbb{R}^{b \times w}$ | Suffix observation matrix (one batch at one cell) | +| $Z$ | $\mathbb{R}^{b \times k}$ | Latent projection of $X$ | +| $\hat{X}$ | $\mathbb{R}^{b \times w}$ | Reconstruction of $X$ from $Z$ | +| $m$ | $\mathbb{Z}_{>0}$ | Coordination group size (§7) | +| $\lambda_s$ | $(\lambda, 1)$ | Slow EWMA decay for per-tracker drift detection (§6) | +| $\lambda_{s,m}$ | $(\lambda, 1)$ | Slow EWMA decay for coordination drift detection (§7) | +| $\kappa_\sigma$ | $\mathbb{R}_{\geq 0}$ | Drift-detector noise allowance in slow-baseline $\sigma$ units (§6) | +| $n_\sigma$ | $\mathbb{R}_{> 0}$ | Outlier clip width in $\sigma$ units (§6) | +| $S$ | $\mathbb{R}_{\geq 0}$ | Drift-detector accumulator value (§6) | +| $\bar{\rho}$ | $[0, 1]$ | Per-axis clip-pressure EWMA (§6.1.1, §6.4) | +| $\lambda_\rho$ | $(0, 1)$ | Clip-pressure EWMA decay rate (§6.1.1, §6.4) | +| $\rho_t$ | $[0, 1]$ | Batch clip ratio: fraction of samples clipped in one batch (§6.1.1) | +| $\mu^{(\text{in})}$ | $\mathbb{R}^4$ | Running-mean centring reference for coordination input (§7) | +| $\eta$ | $[0, 1]$ | Noise influence fraction (tracker maturity) (§11) | +| $K$ | $\mathbb{Z}_{>0}$ | Maximum number of competitively selected analysis cells | +| $L$ | $\mathbb{Z}_{\geq 0}$ | Value Tree depth cutoff for analysis eligibility | +| $\mathcal{T}$ | $\subseteq$ V-entries | Competitive targets (top $K$ by importance within depth cutoff) (§8) | +| $\mathcal{A}$ | $\subseteq$ V-entries | Producing competitive set ($\mathcal{I} \cap \mathcal{T} \cap \text{Online}$) (§8) | +| $\mathcal{A}^*$ | $\supseteq \mathcal{A}$ | Producing full set ($\mathcal{I} \cap \text{Online}$) (§8) | +| $\mathcal{I}$ | $\subseteq$ V-entries + ancestors | Investment set (competitive targets + spatial ancestors, regardless of online status) (§8) | +| $G_{\max}$ | $\mathbb{Z}_{\geq 5}$ | Hard ceiling on total spatial nodes | +| $\lambda_{\text{sp}}$ | $[0,\infty)$ | Spatial decay attenuation factor (host-controlled) (§10) | +| $h_V$ | $\mathbb{Z}_{\geq 0}$ | V-Tree height (maximum root-to-leaf path length in the V-Tree) (§3.9) | +| $d_{\text{geo}}$ | $\mathbb{Z}_{\geq 0}$ | G-Tree depth of a node: the number of materialised ancestors on the path from the node to the root (= prefix depth $d$) (§3.1). | +| $\bar{D}$ | $\mathbb{R}_{\geq 0}$ | Average G-Tree depth of competitive targets; $\bar{D} = \frac{1}{K}\sum_{i=1}^{K} d_i$. Typical values 2–6 under default parameters (§3.1). The worst-case investment set size is $1 + K\bar{D}$ before sharing (§8.2). | + +$^\dagger$ The domain $\{1, \ldots, \min(w, r_{\max})\}$ is non-empty only when $w \geq 1$. Tracker creation requires $w \geq 2$ (§4.1); cells below this threshold are excluded from $\mathcal{I}$. + +> **Batch-size note.** Three distinct quantities govern how many rows a tracker processes per call. **Ingestion batch size** ($n$) is caller-controlled: the number of coordinate values submitted in one `SentinelIngest` call. **Per-tracker batch size** ($b$) is the row count of $X$ at a given tracker. At the root, $b = n$ (every observation routes through the root). At non-root cells during live operation, $b$ is the number of ingestion-batch observations whose spatial routing passes through that cell; $b \in \{0, \ldots, n\}$ depending on the batch's spatial distribution. Cells with $b = 0$ receive no observations and skip the core loop entirely. **Noise batch size** ($b_{\text{noise}}$) is a configured parameter (§11.1) controlling synthetic samples per warm-up round. All core-loop formulas (§4–§6) are parametric in $b$; the distinction matters for cost analysis (§12) and warm-up calibration (§11.9, Appendix A). + +Vectors are row vectors when they represent observations and column vectors when they represent basis directions. Subscript $i$ indexes samples; subscript $j$ indexes subspace dimensions. + +--- + +## Chapter 3. The Spatial Layer · `sec:sentinel:algorithm-spatial-layer` + +This chapter specifies the capabilities and contracts of the underlying spatial partitioning structure — referred to throughout this specification as the **G-V Graph** — to the extent required for understanding the Spectral Sentinel. It is not a complete specification of the G-V Graph; it describes what the Sentinel uses, at the level of detail needed to follow the algorithms in subsequent chapters and reason about their properties. + +--- + +### 3.1 Purpose and Architecture · `sec:sentinel:algorithm-spatial-purpose-and-architecture` + +The G-V Graph is a self-organising spatial index that continuously solves a precision-allocation problem over a one-dimensional domain $[0, 2^N)$. It decides, based on accumulated observation volume, where to invest fine spatial resolution and where to leave resolution coarse. The Sentinel uses this adaptive partitioning as the foundation for its analysis: cells that the G-V Graph identifies as significant receive statistical modelling; cells it identifies as insignificant do not. + +The structure consists of two trees sharing a common set of nodes. Each tree owns a different aspect of the system's state: + +**The Geometric Tree** **(G-Tree)** is a binary tree over dyadic intervals. It is the spatial ledger: every materialised node stores the accumulated observation value for its range. The G-Tree maintains a **contour** — the observation-receiving surface of the domain — a step function whose depth at each coordinate reflects how finely the tree has resolved that region. + +**The Value Tree** **(V-Tree)** is a dynamic tournament bracket with branching factor 2 or 3. Nodes from the G-Tree sit at the leaves as competitors; internal nodes are pure structural scaffolding. The V-Tree ranks competitors by **importance** — accumulated observation volume, under the configuration the Sentinel uses. High-importance entries reside near the root; low-importance entries are consolidated deeper. + +Both trees reference the same underlying nodes. A node exists simultaneously in both trees: it occupies a position in the G-Tree's dyadic hierarchy (determined by its interval) and a position in the V-Tree's tournament bracket (determined by its competitive importance). + +**Uncompressed materialisation.** The G-Tree is a fully materialised binary trie: every node on the path from a leaf to the root exists as a distinct materialised node. A cell at G-Tree depth $d$ has exactly $d$ materialised ancestors. Catalytic bisection (§3.5) creates children at depth $d + 1$; tip-only eviction (§3.12, property 5) removes leaves but never compresses interior chains. Ancestor walks — for sum propagation, investment-set closure (§8.2), and multi-scale delivery (§9.3) — visit every materialised level. + +> _Ancestor cost._ A selected target at G-tree depth $d$ retains its complete path to the root. Sharing can reduce the union of these paths, but does not impose a depth-independent bound. For current selected depths $d_i$, the investment set has at most $1 + \sum_i d_i$ entries; supported engines further bound this by $1 + \texttt{analysis\_k}(N-2)$ (§8.2). + +### 3.2 Domain and Spatial Partitioning · `sec:sentinel:algorithm-spatial-domain-and-partitioning` + +The input domain is $[0, 2^N)$, partitioned into dyadic cells aligned with bit positions. A cell at G-Tree depth $d$ covers an interval of width $2^{N-d}$, corresponding to a $d$-bit prefix shared by all values in the cell. The root covers the entire domain at depth 0; a unit cell at depth $N$ covers a single value. + +The G-V Graph materialises only cells where observations have warranted refinement. The **bottom contour** is the observation-receiving surface of the domain — a step function whose depth at each coordinate reflects how finely the tree has resolved that region. Deep where volume is concentrated, shallow where it is sparse. The contour determines where observations route; the Sentinel's analysis selection (§8.1) determines, separately, where statistical trackers run. The analysis set includes cells above the contour (internal G-Tree nodes serving as ancestor trackers), fed through multi-scale delivery rather than spatial routing. + +The contour is organised into **plateaus** — maximal contiguous runs at a single depth. Within a plateau, all contour cells sit at the same depth and interval width. The boundaries between plateaus mark where the tree found non-uniformity worth resolving. The plateau count $P$ is a natural measure of the tree's structural complexity: $P = 1$ for a perfectly uniform tree; $P$ grows as the tree learns structure. + +### 3.3 Observation and Importance · `sec:sentinel:algorithm-spatial-observation-and-importance` + +When a value $v$ at coordinate $x$ arrives, the G-V Graph: + +1. **Routes** to the receiving cell — the node where `RouteToReceiver(x)` terminates because no child exists in the direction of $x$. +2. **Accumulates** the observation delta $\Delta$ in the receiving cell's ledger. +3. **Updates importance** — the receiving cell's competitive ranking value grows. +4. **Propagates** sums upward through both trees. +5. **Checks structural triggers** — whether the cell qualifies for refinement, whether competitive violations need resolution, whether cold cells should be evicted. + +The Sentinel uses the G-V Graph in its **standard configuration**: importance equals accumulated observation volume, the ground element is zero, and the importance type is non-negative. Under this configuration, all architectural features are available — proportional sampling, the Fibonacci depth bound, violation-free splits, and fast eviction of unobserved cells. + +**The Sentinel's feed-forward invariant.** The Sentinel always observes with $\Delta = 1$ — pure volume counting. Anomaly scores are never fed back into the importance signal. This prevents the anomaly detector from influencing its own spatial structure. + +The accumulator type $V$ must support: + +- Non-negative values with a zero element. +- Addition (for accumulation of $\Delta$). +- Multiplicative scaling by a non-negative real factor (for temporal decay, §3.6). +- Approximate conversion to a floating-point value (for reporting and comparison). + +### 3.4 The Competitive Mechanism · `sec:sentinel:algorithm-spatial-competitive-mechanism` + +The V-Tree is governed by the **max-uncle constraint**: no node may outrank every one of its uncles (siblings of its parent at the grandparent level). This constraint encodes competitive dominance and drives the tree's lifecycle. + +When a cell splits, its V-Tree entry freezes — children intercept all future observations, so the parent's importance stops growing. The frozen entry becomes the competitive benchmark that children must exceed. Children start at zero importance and must earn their way up. When a child's importance exceeds every uncle's, the V-Tree restructures — promoting the child to a shallower position and potentially triggering further spatial refinement. + +Three siblings of comparable importance coexist indefinitely under a 3-node parent — a violation requires beating _both_ uncles. The V-Tree restructures only when a node dramatically outgrows its entire neighbourhood, not on every minor importance fluctuation. This structural stability is inherent — no additional hysteresis is needed. + +**V-Tree depth as a significance measure.** The competitive mechanism pushes high-importance entries to shallow V-Tree depths and low-importance entries deep. V-Tree depth is therefore a global significance ranking: shallow entries have proven sustained competitive importance against their neighbourhood; deep entries have not. The Sentinel's analysis selector (§8) uses V-Tree depth as the eligibility criterion for statistical modelling. + +The max-uncle constraint implies at least Fibonacci-rate decay along root-to-leaf paths, yielding a depth bound of $\log_\varphi(1/w_i) + O(1)$ for an entry with weight fraction $w_i$, where $\varphi = (1+\sqrt{5})/2 \approx 1.618$. Expected proportional sampling cost is at most $1.44 H + O(1)$ where $H$ is the Shannon entropy of the importance distribution. + +### 3.5 Spatial Lifecycle · `sec:sentinel:algorithm-spatial-lifecycle` + +Three forces shape the contour: + +**Refinement.** When a fully exposed contour cell accumulates sufficient importance and holds a shallow enough V-Tree position, it splits into two finer cells. The split is _catalytic_: the parent persists as a frozen competitive benchmark; its children start at zero importance and must earn their way up. No information is destroyed by spatial refinement. + +**Eviction.** Unprotected contour cells (zero children) that sit past a configurable V-Tree depth threshold are removed. The parent absorbs the evicted cell's accumulated value and partially or fully re-joins the contour. Eviction proceeds from the tips inward — only nodes with no dependents can be removed — so the contour coarsens gradually, never catastrophically. + +**Restoration.** When a semi-internal node (one child present, one evicted) accumulates enough importance through the competitive mechanism, the V-Tree promotes it. This promotion creates the missing child as a side effect, restoring the contour without any separate operation. The V-Tree's competitive mechanism is the sole gate for contour growth. + +**Depth gates** govern the lifecycle. Two V-Tree depth thresholds — a creation gate and an eviction gate separated by a mandatory buffer zone — control where new resolution may be added and where it is withdrawn. A node-count budget enables dynamic adjustment of these thresholds under memory pressure, providing a hard ceiling on total materialised nodes. + +### 3.6 Temporal Decay · `sec:sentinel:algorithm-spatial-temporal-decay` + +The G-V Graph stores exact accumulated values by default. It imposes no automatic temporal model. The host controls temporal semantics through an explicit decay operation that scales accumulators by a configurable factor: + +- **Attenuation** (factor $< 1$): cold cells lose standing and become eviction candidates. This produces recency — "what matters now." +- **Amplification** (factor $> 1$): existing structure is reinforced. With depth-selective amplification, fine-scale detail is sharpened relative to coarse. +- **Annihilation** (factor $= 0$): hard reset. Targeted annihilation zeroes a subtree while leaving the rest of the graph intact. +- **Detail flush** (factor $= 0$, depth-selective): the subtree root is preserved while all descendants are zeroed — preserving coarse measurement while forcing fine structure to be re-earned. + +Decay is subband-adaptive: different G-Tree depths can be scaled at different rates. This enables the host to implement frequency-selective temporal filtering — high-resolution subbands can decay faster than coarse ones. + +The Sentinel's statistical decay (EWMA forgetting in trackers, §4) is independent of spatial decay. A cell can retain its spatial position under slow spatial decay while its statistical model adapts rapidly, and vice versa. + +### 3.7 Dual Shielding · `sec:sentinel:algorithm-spatial-dual-shielding` + +The two trees protect each other through three complementary mechanisms: + +| Direction | Shield | What it protects | +| ------------------- | ------------------------------------------------ | ---------------------------------------------------------------------- | +| V-Tree → downward | Parent's frozen entry stands as uncle | Children from competitive displacement while the benchmark holds | +| G-Tree → upward | Children intercept observations meant for parent | Parent's V-entry from growing — it stays frozen as a fixed benchmark | +| G-Tree → structural | Only unprotected nodes are eviction-eligible | Protected nodes from premature removal while structurally load-bearing | + +Without the upward shield, the parent's importance would keep pace with its children — no fixed benchmark, no competitive mechanism. Without the downward shield, children's positions would be unstable under every fluctuation. Without the structural shield, eviction could tear the contour by removing nodes that support finer-scale structure. + +### 3.8 Benchmark Compounding · `sec:sentinel:algorithm-spatial-benchmark-compounding` + +Repeated expand–contract cycles at a given node harden its competitive benchmark. Each eviction absorption folds descendants' accumulated value into the node's own accumulation, raising the frozen bar that future children must exceed. After $j$ full expand–contract cycles, the benchmark grows approximately geometrically, and the total observation cost to re-reach depth $D$ along a path grows quadratically: $\sim D^2 \theta / 2$ where $\theta$ is the split threshold. This ensures that deep spatial structure is re-created only when justified by sustained, concentrated observation volume. + +Under temporal attenuation, benchmarks weaken — the bar softens as historical evidence ages. Under annihilation, benchmarks reset to zero — the region starts from scratch. + +### 3.9 Capabilities Used by the Sentinel · `sec:sentinel:algorithm-spatial-sentinel-capabilities` + +The Sentinel uses the following G-V Graph operations: + +| Operation | Description | Cost | +| -------------------------------- | ---------------------------------------------------- | ---------------------------------- | +| Observe(coordinate, $\Delta$) | Route, accumulate, trigger structural updates | $O(d_{\text{geo}} + h_V)$ | +| RouteToReceiver($x$) | Find the receiving cell for coordinate $x$ | $O(d_{\text{geo}})$ | +| Decay(root, factor, selectivity) | Apply temporal scaling to a subtree | $O(\text{subtree size} \cdot h_V)$ | +| Value Tree depth query | V-Tree depth of a cell's entry | $O(h_V)$ or $O(1)$ cached | +| Value Tree importance query | Importance value of a cell's entry | $O(1)$ | +| Plateau point query | Plateau containing a coordinate | $O(\log P)$ | +| Plateau iteration | All plateaus in spatial order | $O(P)$ | +| Plateau count | Number of distinct plateaus | $O(1)$ | +| G-Tree ancestor walk | All materialised ancestors of a node | $O(d_{\text{geo}})$ | +| G-Tree node sum query | g.sum of a node (own accumulation + descendant sums) | $O(1)$ | +| Total importance | Sum of all importance in the graph | $O(1)$ | +| Node count | Total materialised G-Tree nodes | $O(1)$ | +| Semi-internal count | Number of one-child nodes | $O(1)$ | + +### 3.10 Node States · `sec:sentinel:algorithm-spatial-node-states` + +Every G-Tree node exists in one of three states, determined by how many children it has: + +| Children | State | Contour relationship | Observation behaviour | Eviction eligible | +| -------- | ------------- | ---------------------------------- | --------------------------------------------- | ------------------------------------- | +| 0 | Terminal | On the contour — fully exposed | Receives all observations in its range | Yes (if past depth gate and not root) | +| 1 | Semi-internal | On the contour — partially exposed | Receives observations in the uncovered half | No (has a dependent) | +| 2 | Internal | Above the contour | Receives no observations (children intercept) | No (has dependents) | + +The Sentinel's analysis selector (§8) does not filter candidates by G-Tree state. It scans V-Tree entries within the configured V-depth cutoff and with sufficient analysis width, so a competitive target can be terminal, semi-internal, or internal. Internal nodes can remain competitive because their V-entry retains frozen pre-split importance, and their tracker is still fed through Sentinel's multi-scale interval delivery (§9). + +Since the G-Tree is fully materialised (§3.1), the investment set's ancestor closure (§8.2) includes every node on the path from each competitive target to the root, providing hierarchical context at every dyadic scale from the target to the entire domain. + +### 3.11 Configuration Parameters · `sec:sentinel:algorithm-spatial-configuration-parameters` + +| Parameter | Role | Sentinel's typical value | +| ----------------------------------- | ---------------------------------------- | ----------------------------------- | +| $N$ (domain bit-width) | Tree height, analysis width at root | 128 (default) or 64 | +| $\theta$ (split threshold) | Minimum importance for split eligibility | Application-dependent | +| $D_{\text{create}}$ (creation gate) | Maximum V-Tree depth for splits | Application-dependent | +| $D_{\text{evict}}$ (eviction gate) | Minimum V-Tree depth for eviction | $D_{\text{create}} + \text{buffer}$ | +| Budget | Soft node-count target | Application-dependent | +| $G_{\max}$ | Hard ceiling on total G-Tree nodes | $> \text{budget}$, $\geq 5$ | + +The Sentinel treats these as pass-through configuration: it forwards them to the G-V Graph at construction time and does not modify them during operation. The host controls all spatial policy through these parameters and through the timing and parameters of decay calls. + +### 3.12 Key Properties · `sec:sentinel:algorithm-spatial-key-properties` + +The following properties of the G-V Graph are assumed throughout this specification: + +1. **Complete tiling.** The contour always tiles the full domain $[0, 2^N)$ with no gaps. Every coordinate has exactly one receiving cell. + +2. **Summation invariant.** Every node's sum equals its own accumulation plus its children's sums. Total energy is conserved across all structural operations. + +3. **Single-entry accounting.** Each observation updates exactly one node's importance — the receiver's. No double-counting. + +4. **Competitive stability.** Three siblings of comparable importance coexist indefinitely. Restructuring requires a node to outgrow its entire neighbourhood. + +5. **Tip-only eviction.** Only nodes with zero children can be evicted. The contour contracts from the tips inward. + +6. **Root permanence.** The G-Tree root is never evicted. The tree always has at least one node. + +7. **Frozen benchmarks.** Internal nodes' importance values are frozen — a direct consequence of observation routing, not an explicit mechanism. + +8. **Budget enforcement.** The total node count never exceeds $G_{\max}$ across any single operation. + +9. **Fibonacci depth bound.** Under the standard configuration, V-Tree depth is bounded by $\log_\varphi(1/w_i) + O(1)$ for weight fraction $w_i$. + +10. **Feed-forward compatibility.** The G-V Graph accepts any non-negative $\Delta$ without interpreting it. The Sentinel's choice of $\Delta = 1$ is invisible to the graph. + +--- + +# Part II — The Mathematical Model · `sec:sentinel:algorithm-mathematical-model` + +--- + +## Chapter 4. The Subspace Model · `sec:sentinel:algorithm-subspace-model` + +Each cell in the full analysis set $\mathcal{A}^*$ (§8) maintains a subspace tracker — a low-rank linear subspace model that scores observations against learned structure and then evolves. + +### 4.1 State · `sec:sentinel:algorithm-subspace-state` + +**Minimum analysis width.** All formulas in §4–§7 require $w \geq 2$. At $w = 0$, $\text{cap} = 0$ while $k$ would need to be 1 — the rank exceeds capacity, the basis $U$ is vacuous, and every formula in §4.2 and §5 is undefined. At $w = 1$, the tracker is novelty-saturated from birth ($\text{cap} = 1 = k$, residual DOF $= 0$) and the single basis vector spans the entire space, leaving no statistical content to detect departures from. Cells with $w < 2$ are excluded from the eligible set $\mathcal{E}$ (§8.1) and reported via the degenerate-cell counter (§14.11). The spatial layer continues to route and accumulate observations at these cells normally; only statistical modelling is withheld. + +A tracker at analysis width $w$ with capacity $\text{cap} = \min(w, r_{\max})$ maintains: + +| Component | Shape | Description | +| ------------------ | ----------------------------------------------------------------- | ---------------------------------------------------------- | +| $U$ | $(w, \text{cap})$ | Orthonormal basis (columns $1{:}k$ active) | +| $\sigma$ | $(\text{cap},)$ | Singular values ($1{:}k$ meaningful) | +| $\mu^{(z)}$ | $(\text{cap},)$ | EWMA mean of latent coordinates ($1{:}k$ active) | +| $\nu^{(z)}$ | $(\text{cap},)$ | EWMA variance of latent coordinates ($1{:}k$ active) | +| $\Gamma$ | $(\text{cap}, \text{cap})$ | EWMA second-moment matrix ($1{:}k$ active, upper triangle) | +| Per-axis baselines | 4 × {fast EWMA, slow EWMA, drift accumulator, clip-pressure EWMA} | Baseline tracking (§6) | +| Rank $k$ | integer in $[1, \text{cap}]$ | Current active dimensionality | +| Step counter | integer | Batches processed since creation | + +**Initial state.** A newly created tracker begins with: + +- $k = 1$. The core loop's algebra presupposes $k \geq 1$: at $k = 0$, Phase 1 produces a $b \times 0$ projection, all within-subspace scores are zero or undefined, and Phase 5's cumulative energy fractions are ill-defined. No formulas in §4.2 or §5 define behaviour at $k = 0$. +- $U$: any set of orthonormal columns in $\mathbb{R}^{w \times \text{cap}}$. The identity submatrix (column $j$ is the $j$-th standard basis vector) is a convenient deterministic choice. The initial basis is overwritten by the first Phase 2 SVD; the choice does not affect post-warm-up behaviour. +- $\sigma_{1:\text{cap}}$: initial values satisfying $\sum \sigma_j^2 \leq \delta_{\text{init}} \cdot bw/4$ for some small $\delta_{\text{init}}$ (e.g. $10^{-4}$; not the global stability constant $\varepsilon$). At $\delta_{\text{init}} = 0$, the first Phase 2 seeds the subspace purely from the first batch; at $\delta_{\text{init}} > 0$, a faint ghost of the initial basis persists for one step and is overwritten on the second. The value $0.01$ per component is conformant at per-cell analysis widths (where $bw \gg \text{cap}$). At coordination trackers ($w = 4$, $b = m \geq 2$, $\text{cap} = 4$) the effective $\delta_{\text{init}}$ rises to $\text{cap} \times 10^{-4} / (bw/4) = 2 \times 10^{-4}$ at $b = 2$ — still negligible (the initial energy is 0.02% of one batch), and the ghost is overwritten on the first Phase 2 step during noise warm-up (§11.7) long before live traffic arrives. +- $\mu^{(z)} = \mathbf{0}$, $\nu^{(z)} = \mathbf{1}$, $\Gamma = \mathbf{0}$: pre-allocated at capacity, with values chosen for safe behaviour on rank increase (Phase 3 below) and benign first-batch direct seeding (§11.2). The $\nu^{(z)} = 1.0$ pre-allocation is a conservative overestimate (roughly $4\times$ the null-hypothesis value of $0.25$ for centred $\pm 0.5$ bit vectors). It serves as the surprise denominator during the first Phase 1 scoring pass, before Phase 3 seeds $\nu^{(z)}$ from data. At $b = 1$, the seed is $z_{1j}^2 \approx 0.25$ in expectation — correctly scaled without special-case branching. +- Step counter $= 0$. This triggers the first-batch direct seeding path (Phase 3 below, $t = 0$), which overwrites $\mu^{(z)}$, $\nu^{(z)}$, and $\Gamma$ from data. The pre-seeding values above are therefore transient — they affect at most one scoring pass before being replaced. +- Baselines uninitialised. Noise influence $\eta = 1.0$. + +> _Design note (rank convergence coupling)._ During noise injection (§11.1), rank adaptation fires every $T_{\text{rank}}$ batches and moves rank by at most 1. The rank at the end of noise injection is $k_{\text{post-noise}} = \min(1 + \lfloor\text{noise rounds} / T_{\text{rank}}\rfloor, \; \text{cap})$. At $T_{\text{rank}} = 100$ with 450 noise rounds at root, the tracker enters real-traffic service at $k \approx 5$–$6$, substantially below capacity. This is by design: the noise schedule is calibrated for baseline convergence (§11.9), not rank convergence; rank continues climbing under real traffic. The coherence axis (requiring $k \geq 2$) activates after $T_{\text{rank}}$ batches, contributing to the coherence convergence bottleneck documented in §11.9. + +### 4.2 The Core Loop · `sec:sentinel:algorithm-core-loop` + +Each tracker processes a batch $X \in \mathbb{R}^{b \times w}$ in five strictly ordered phases. **Scoring precedes evolution** — the batch is measured against the _prior_ model, then the model updates. + +#### Phase 1 — Score · `sec:sentinel:algorithm-core-loop-score` + +$$Z = X \, U_k \qquad \hat{X} = Z \, U_k^\top \qquad R = X - \hat{X}$$ + +Compute four per-sample scores from $Z$, $\hat{X}$, and $R$ (§5). Assemble batch summary statistics. + +#### Phase 2 — Evolve Subspace · `sec:sentinel:algorithm-core-loop-evolve-subspace` + +Construct the combined matrix: + +$$M = \begin{bmatrix} \sqrt{\lambda} \; U_k \, \operatorname{diag}(\sigma_{1:k}) & \Big| & X^\top \end{bmatrix} \in \mathbb{R}^{w \times (k + b)}$$ + +Compute the thin SVD $M = \tilde{U} \, \tilde{S} \, \tilde{V}^\top$. Retain the top $n = \min(\min(w, k+b), \; \text{cap})$ components: + +$$U_{:, 1:n} \leftarrow \tilde{U}_{:, 1:n} \qquad \sigma_{1:n} \leftarrow \operatorname{diag}(\tilde{S})_{1:n} \qquad \sigma_{n+1:\text{cap}} \leftarrow 0$$ + +The zeroing of trailing entries is semantically compelled: $M$ has rank at most $n$, so components beyond this index carry no energy from either the attenuated history or the current batch. + +$\tilde{V}$ is discarded. + +**Decay profile.** Without reinforcement, $\sigma^{(t)} = \lambda^{t/2} \, \sigma^{(0)}$. Energy half-life: $t_{1/2} = \ln 2 / \ln(1/\lambda)$. + +| $\lambda$ | Half-life (steps) | Character | +| --------- | ----------------- | ----------- | +| 0.99 | $\approx 69$ | Long memory | +| 0.95 | $\approx 14$ | Medium | +| 0.90 | $\approx 7$ | Short | + +**Numerical failure guard.** If the thin SVD fails to converge (numerically degenerate input), the basis $U$ and singular values $\sigma$ are left unchanged. The batch is scored (Phase 1 completed) but the model does not evolve. Implementations should record this event. + +**Identical-observation guard.** If all $b$ rows of $X$ are identical, the combined matrix $M$ has rank at most $k + 1$. The SVD is well-conditioned in this case; no special handling is needed beyond the standard thin-SVD computation. + +#### Phase 3 — Evolve Latent Distribution · `sec:sentinel:algorithm-core-loop-evolve-latent-distribution` + +$Z$ is the projection matrix computed in Phase 1 (prior-basis projection). Phase 3 does **not** recompute $Z$ using the updated basis from Phase 2. The latent statistics therefore always describe the distribution under the basis that was used for scoring; the consequence is a one-step lag after basis rotations, whose transient effects are analysed in the design note below. + +**Phase 3 internal order.** The update order is $\nu^{(z)}$, then $\mu^{(z)}$, then $\Gamma$. The variance update evaluates deviations against the pre-update mean, matching the principle that scoring (Phase 1) uses the prior model. + +**First batch ($t = 0$).** On the tracker's very first batch, seed the latent statistics directly from data rather than blending with initial values. The seeding uses the pre-allocated $\mu^{(z)} = \mathbf{0}$ as the centring reference: + +$$\nu^{(z)}_j \leftarrow \max\!\left(\frac{1}{b}\sum_{i=1}^{b} z_{ij}^2,\;\varepsilon\right)$$ + +$$\mu^{(z)}_j \leftarrow \bar{Z}_j$$ + +$$\Gamma_{jl} \leftarrow \frac{1}{b}\sum_{i=1}^{b} z_{ij}\,z_{il}$$ + +No batch-size-conditional branching is needed within the $t = 0$ seeding path. At $b = 1$, the variance seed is $z_{1j}^2$ — noisy but correctly scaled ($\mathbb{E}[z_j^2] = \sigma^2 \approx 0.25$ under centred binary inputs with approximately zero-mean latent projections). The EWMA smooths the noise within $O(t_{1/2})$ subsequent batches. + +> _The rationale for direct seeding — and the nature of the data typically present at $t = 0$ during warm-up — is analysed in §11.2._ + +**Subsequent batches ($t > 0$).** Compute the variance update first, using the pre-update mean: + +$$\nu^{(z)}_j \leftarrow \lambda\,\nu^{(z)}_j + \alpha \cdot \max\!\left(\frac{1}{b}\sum_{i=1}^{b}(z_{ij} - \mu^{(z)}_j)^2,\;\varepsilon\right) \qquad j = 1,\ldots,k$$ + +Then update the mean: + +$$\mu^{(z)}_j \leftarrow \lambda\,\mu^{(z)}_j + \alpha\,\bar{Z}_j \qquad j = 1, \ldots, k$$ + +Then update the second-moment matrix: + +$$\Gamma_{jl} \leftarrow \lambda\,\Gamma_{jl} + \alpha\,\frac{1}{b} \sum_{i=1}^{b} z_{ij}\,z_{il} \qquad j < l$$ + +Only the upper triangle is stored. + +**Runtime floor.** After each update (including $t = 0$ seeding), clamp $\nu^{(z)}_j \leftarrow \max(\nu^{(z)}_j, 10^{-2})$. For centred-bit cell inputs $x \in \{-1/2,1/2\}^{d}$ and unit basis columns, Cauchy–Schwarz gives $|z_j| \leq \sqrt{d}/2$. A mean initialized at zero and then seeded or convexly averaged from these coordinates obeys the same bound, so $(z_j-\mu_j)^2 \leq d$. With $\varepsilon>0$ and variance at least $0.01$, each surprise contribution and their rank average are at most $d/(0.01+\varepsilon) \leq 100d$, up to floating-point roundoff. The floor may bind on degenerate streams; this finite input bound does not apply to unbounded coordination-score vectors. + +The inner $\max(\cdot, \varepsilon)$ and the runtime $\max(\cdot, 10^{-2})$ are complementary: the inner floor prevents a zero-energy EWMA _input_ (from identical observations within a batch); the runtime floor prevents the _accumulated EWMA value_ from being too small due to a prolonged sequence of near-$\varepsilon$ inputs. Neither is redundant. + +**Behaviour on rank change.** All three latent statistics are pre-allocated at capacity $\text{cap}$ (§4.3). The EWMA loops above iterate over $j = 1, \ldots, k$, so entries beyond the active rank are never written. + +When rank increases from $k$ to $k + 1$: + +- $\mu^{(z)}_{k+1}$: retains its pre-allocated value of **zero**. Zero is the expected latent mean for a new basis direction over centred data; the EWMA converges to the true mean within $O(t_{1/2})$ steps. +- $\nu^{(z)}_{k+1}$: retains its pre-allocated value of **$1.0$**. This is a conservative overestimate: with typical steady-state variance $\approx 0.25$ for centred $\pm 0.5$ bit vectors, the surprise denominator is $4\times$ too large, **dampening** the new dimension rather than spiking it. Under the EWMA-mean-centred formula, the overestimate self-corrects as new batches contribute $(z_{i,k+1} - \mu^{(z)}_{k+1})^2$ terms, converging with half-life $t_{1/2}$ regardless of batch size. +- $\Gamma_{j,k+1}$ and $\Gamma_{k+1,l}$: new entries are already **zero** from construction. + +When rank decreases, outer entries of all three statistics ($\mu^{(z)}$, $\nu^{(z)}$, $\Gamma$) are ignored but preserved. If rank later increases back, the preserved values provide a warm starting point rather than the cold defaults, reducing reconvergence time. + +> _Design note (cold-start analogy)._ The $\nu^{(z)} = 1.0$ initialisation on rank increase is the same class of mismatch that §11.2 eliminates at $t = 0$ via direct seeding. On rank increase the transient is more benign: it affects only one dimension among $k$ (diluted by $1/k$ in the surprise average) and is always in the safe direction (dampening, not spiking). Per-dimension cold-seeding on rank change — analogous to the $t = 0$ logic — would eliminate this transient entirely, at the cost of per-dimension initialisation tracking. + +> _Design note (why EWMA-mean-centred variance)._ The variance update uses deviations from $\mu^{(z)}_j$ rather than deviations from the batch mean $\bar{Z}_j$. This makes $\nu^{(z)}_j$ a direct estimator of the surprise numerator's expected value, yielding a self-consistent ratio at every batch size. +> +> The within-batch population variance $\operatorname{Var}(Z_{:,j})$ has a systematic negative bias of $\frac{b-1}{b}$ that is catastrophic at $b = 1$ (erosion to $\varepsilon$, surprise inflation to $O(10^5)$), severe at $b = 2$ ($-50\%$), and materially significant below $b \approx 16$. The EWMA-mean-centred formula eliminates this entire bias class through exact cancellation: the within-batch component contributes $\frac{b-1}{b}\sigma^2$ and the batch-mean-vs-EWMA-mean component contributes $\frac{\sigma^2}{b}$, summing to $\sigma^2$ regardless of $b$. The residual bias is $\operatorname{Var}(\mu^{(z)}_j) \approx \frac{\alpha}{b(1+\lambda)}\sigma^2$ — positive (safe direction: dampens surprise), small, and decreasing as $1/b$ with worst case $+0.5\%$ at $b = 1$ and $\lambda = 0.99$. Both the surprise numerator and the $\nu$ input share the same $\mu^{(z)}_j$ reference with identical bias, so the expected surprise ratio is $1 + O(\alpha^2)$ regardless of $b$ — the self-consistency property. +> +> When per-tracker batch size $b$ varies across ingestion cycles (as it does in practice, depending on the spatial distribution of observations), self-consistency is preserved: each batch's $\nu$ input uses the same $\mu^{(z)}$ reference as that batch's surprise computation. +> +> The formula couples $\nu^{(z)}$ to $\mu^{(z)}$, providing automatic gain control during regime transitions: when the mean estimate is stale, the variance estimate inflates proportionally, dampening transient surprise spikes. This coupling reduces surprise-axis sensitivity to gradual mean drift — $\nu$ absorbs the same signal that inflates the numerator and the ratio converges to $\approx 1$. This is an acceptable trade-off: displacement is the primary mean-shift detector, and surprise's primary role is detecting per-dimension distributional shape anomalies, which are preserved. +> +> The original formula's within-batch variance was invariant to Phase 2 basis rotations (re-centring on $\bar{Z}_j$ subtracted out any mean shift caused by the rotated projection). The EWMA-mean-centred formula is not: after a basis rotation, $z_{ij}$ are projections onto the new basis while $\mu^{(z)}_j$ reflects the old basis, inflating $(z_{ij} - \mu^{(z)}_j)^2$ and therefore $\nu^{(z)}_j$. This sensitivity is in the safe direction (dampening surprise through $\nu$ inflation), is transient ($O(t_{1/2})$ batches), and is precisely the automatic gain control described above. + +> _Design note (raw second moments vs. centred covariances in $\Gamma$)._ $\Gamma$ tracks raw second moments $\mathbb{E}[z_j z_l]$, not centred covariances $\operatorname{Cov}(z_j, z_l)$. The coherence score (§5.5) compares against $\Gamma_{jl}$ directly. Since $\mathbb{E}[z_j z_l] = \operatorname{Cov}(z_j, z_l) + \mu_j^{(z)} \mu_l^{(z)}$, a shift in the latent mean changes $\Gamma_{jl}$ even when the correlation structure is perfectly stable. The alternative — using centred products $(z_j - \mu_j^{(z)})(z_l - \mu_l^{(z)})$ in both the score and the $\Gamma$ update — would isolate coherence to detect only correlation structure changes. This section analyses the trade-off and explains why raw second moments are retained. +> +> **Steady-state behaviour: the coupling is invisible.** At steady state, the $\mu_j \mu_l$ contribution to $\Gamma_{jl}$ is constant and perfectly absorbed by the coherence fast baseline $\bar{s}_{\text{coh}}$. The host-facing z-scores are computed against this baseline, so the permanent coupling between mean-structure and coherence-structure produces no observable effect. The coupling manifests _only_ during transitions — which is exactly the regime where the two formulations differ. +> +> **Transient behaviour: both formulations have artefacts.** During a regime transition that shifts the latent mean, the raw formulation produces a coherence spike because $z_j z_l$ reflects the new second moment while $\Gamma_{jl}$ still tracks the old one. Surprise fires simultaneously (detecting the same mean shift through $(z_j - \mu_j)^2 / \nu_j$), creating partial redundancy. However, the score $z_j z_l$ is computed from _fresh data with no lag_ — only the baseline $\Gamma_{jl}$ is stale. The departure is therefore clean: it reflects the genuine second-moment change (including the mean contribution), and it decays monotonically as $\Gamma$ absorbs the shift at rate $\lambda$. +> +> The centred formulation has a qualitatively worse transient. Staleness enters the _score computation itself_: the centred product $(z_j - \mu_j^{(\text{old})})(z_l - \mu_l^{(\text{old})})$ injects a ghost correlation $(\mu_j^{(\text{new})} - \mu_j^{(\text{old})})(\mu_l^{(\text{new})} - \mu_l^{(\text{old})})$ with a specific directional pattern in the $k \times k$ upper triangle determined by which dimensions shifted most. This ghost correlation is indistinguishable, from the baseline's perspective, from a genuine correlation structure change. The baseline _learns_ the ghost as if it were real, then must _unlearn_ it as $\mu^{(z)}$ converges — creating a **non-monotonic** transient. The raw formulation's monotonic, cleanly interpretable transient is a strictly better property, even though both have $O(t_{1/2})$ timescale. +> +> **Magnitude bound under centred binary inputs.** The input encoding (§2.3) is $\{-0.5, +0.5\}^w$ with $\mathbb{E}[x_i] = 0$ under uniform bits. Latent means $\mu_j^{(z)}$ are projections onto learned basis directions — they are tightly bounded and typically small. The cross-product $\mu_j \mu_l$ is therefore small relative to $\operatorname{Cov}(z_j, z_l)$ at steady state. A distributional shift large enough to make $\mu_j \mu_l$ dominate $\Gamma_{jl}$ is a violent regime change that should maximally alarm on every available axis. +> +> **State coupling.** With raw second moments, $\Gamma$ is a self-contained EWMA depending only on its own history and the observed products $z_j z_l$. With centred products, both the score and the baseline update depend on $\mu^{(z)}$, creating a hidden coupling between the surprise axis (which uses $\mu^{(z)}$ for scoring) and the coherence axis (which would use it for both scoring and baseline computation). A convergence artefact in the mean estimate would simultaneously corrupt two axes instead of one. The raw formulation keeps the axes' state dependencies more separated. +> +> **Coordination-layer exposure.** At the coordination layer (§7), the §7.3 running-mean centring subtracts $\mu^{(\text{in})}_g$ from the raw score matrix before the coordination tracker sees it. During a transition, $\mu^{(\text{in})}_g$ lags the actual group mean, so the tracker's inputs carry residual mean that the centring didn't remove. The coordination tracker's internal $\mu^{(z)}$ then tracks this residual, creating a second lag. The $\mu_j \mu_l$ contamination of coordination-level $\Gamma$ therefore reflects the square of a lagged mean estimate, which can amplify the artefact's persistence relative to the per-cell level. Under gradual drift (the regime where the concern is most relevant), the running-mean centring tracks the linearly increasing group mean with a steady-state lag of $\delta\mu / \alpha$, where $\delta\mu$ is the per-batch mean increment (the standard lag of an EWMA tracking a linear trend). The tracker's inputs therefore carry a constant residual bias of $O(\delta\mu / \alpha)$, and the squared cross-product contamination is $O((\delta\mu / \alpha)^2)$. However, this constant bias is itself absorbed by the coordination tracker's internal statistics — the tracker's own $\mu^{(z)}$ and $\Gamma$ converge to include the bias — so the contamination manifests only during the transient before the tracker reaches its own steady state. Under a sudden shift, the full residual $\Delta\mu$ is visible on the first batch and decays as $\lambda^t \Delta\mu$ — but during this transient, surprise fires simultaneously at the per-cell level, making the coherence artefact redundant. In both regimes, the concern is real but quantitatively minor: the contamination is transient rather than small in absolute terms, and it overlaps temporally with surprise signals that already capture the same event. +> +> **Verdict.** The raw formulation trades a small, well-bounded artefact (mean-squared leakage into a second-moment tracker, monotonic decay, invisible at steady state) for the centred formulation's subtler, harder-to-bound artefact (stale-mean ghost correlations, non-monotonic transient, hidden state coupling between axes). The raw formulation is retained. + +> _Design note (subspace energy weighting vs. latent EWMA weighting)._ Phase 2 and Phase 3 apply different effective weightings to new data, and this is a deliberate design choice whose consequences bear understanding. +> +> **The apparent asymmetry.** In the combined matrix $M = [\sqrt{\lambda}\, U_k \operatorname{diag}(\sigma) \mid X^\top]$, new observations enter at full energy ($\|x_i\|^2 = w/4$ each), while old structure is attenuated by $\sqrt{\lambda}$. In Phase 3, the standard EWMA $\mu \leftarrow \lambda\mu + \alpha\bar{Z}$ weights new data at $\alpha = 1 - \lambda$. At first glance, the subspace gives the current batch weight $\sim 1$ while the latent statistics give it weight $\alpha$ — a factor-of-$1/\alpha$ mismatch. +> +> **At energy steady state, there is no mismatch.** The SVD operates on the Frobenius norm of $M$. In the steady state where total singular-value energy $\Sigma = \sum \sigma_j^2$ has stabilised, $\Sigma = bw/(4\alpha)$. The new batch's fraction of total energy in $M$ is $(bw/4)/\Sigma = \alpha$ — exactly the EWMA learning rate. Both mechanisms forget old information at rate $\lambda^t$ per step and weight new information at effective fraction $\alpha$. The energy half-lives are identical: $t_{1/2} = \ln 2 / \ln(1/\lambda)$. +> +> **But the SVD has a rotational degree of freedom that the EWMA lacks.** The real asymmetry is not in effective sample size but in the _nature_ of the response to structural change. The EWMA is a linear filter: after a regime change, $\mu^{(z)}$ converges to the new mean exponentially with time constant $t_{1/2}$, regardless of how different the new regime is. The SVD is a nonlinear rank-ordered decomposition: it can _swap_ basis directions in a single step once a new direction's singular value exceeds a decaying old direction's. This discrete jump has no analog in the continuous EWMA. +> +> **The transient surprise spike after regime changes.** After a sudden distributional shift, the subspace may rotate substantially within a few batches — new directions appear in $U$ once their energy dominates decaying old singular values. But $\mu^{(z)}$ and $\nu^{(z)}$ carry state from the old basis. After a direction swap, $\mu^{(z)}_j$ was tracking the mean projection onto old direction $u_j^{(\text{old})}$; it is now being updated with projections onto the new direction $u_j^{(\text{new})}$. The stale reference produces a transient spike in the surprise score $(z_j - \mu_j)^2 / \nu_j$ that persists for $O(t_{1/2})$ batches while the EWMA converges. +> +> **This is the correct behaviour, for three reasons.** +> +> First, the spike is _informative_. It correctly signals "observations that don't match the historical within-subspace distribution" — which is literally true during a regime change. A system that silently absorbed regime changes would fail at the core detection task. The drift accumulator (§6.3) distinguishes transient spikes (which don't accumulate much) from sustained shifts (which do). +> +> Second, the subspace _must_ adapt its directions faster than the EWMA adapts its scalars. An incorrect basis direction wastes an entire detection axis — every projection onto a stale direction is uninformative. An incorrect latent mean is a quantitative error within a still-meaningful projection. The SVD's ability to rapidly discover new structure is why it is used instead of a linear update for the subspace. +> +> Third, matching the rates would be strictly worse. Scaling new data by $\alpha$ in $M$ (giving $M = [\sqrt{\lambda}\, U_k \operatorname{diag}(\sigma) \mid \sqrt{\alpha}\, X^\top]$) would require $\sim 1/\alpha$ batches of coherent new-direction data before the SVD even recognised its existence — eliminating the system's ability to detect emerging structure promptly. +> +> **Consequences for the host.** During regime transitions, expect elevated surprise z-scores for $O(t_{1/2})$ batches as the latent statistics reconverge. The fast EWMA baseline (§6.1) absorbs this transient — the z-scores are computed against the fast baseline, which itself adapts at rate $\lambda$. The drift accumulator may register modest growth during the transient, bounded by $\sim t_{1/2} \times (\text{spike magnitude} - \kappa)$; the slow baseline (§6.2) eventually absorbs the shift and the accumulator returns to zero. Hosts that expect frequent regime changes (e.g., diurnal patterns) should interpret surprise drift accumulation in light of the known transition schedule. +> +> Note: under the EWMA-mean-centred variance formula (Phase 3 above), the surprise spike's persistence is shorter than described above. After the initial batch fires at full strength (scored against the prior $\nu$ in Phase 1), subsequent batches' $\nu$ co-adapts — inflating as the stale $\mu^{(z)}$ inflates the $(z - \mu)^2$ terms — progressively dampening the surprise ratio. Displacement inherits the primary detection role during transitions. The first-batch detection at full strength is preserved; the $O(t_{1/2})$ sustained elevation is not. +> +> _The asymmetry is not between two forgetting rates but between a linear filter (EWMA) that can only exponentially forget and a nonlinear decomposition (SVD) that can structurally reorganise. The latent statistics inherit the SVD's directional decisions but adapt their scalar parameters at the EWMA rate — a deliberate separation of concerns between "which directions matter" (fast, nonlinear) and "what the typical projection onto those directions looks like" (smooth, linear)._ + +#### Phase 4 — Update Baselines and Drift Detector · `sec:sentinel:algorithm-core-loop-update-baselines-and-drift-detector` + +Each scoring axis maintains a fast EWMA, slow EWMA, drift accumulator, and clip-pressure EWMA (§6). **Exception:** coherence baselines are not updated while $k < 2$ (§5.5). + +#### Phase 5 — Adapt Rank · `sec:sentinel:algorithm-core-loop-adapt-rank` + +Rank is recomputed every $T_{\text{rank}}$ tracker batches (the rank update interval): + +1. Compute cumulative energy fractions: + +$$c_i = \frac{\sum_{j=1}^{i} \sigma_j^2}{\sum_{j=1}^{\text{cap}} \sigma_j^2 + \varepsilon} \qquad i = 1, \ldots, \text{cap}$$ + +2. Find the target rank: + +$$k^* = \min\!\big\{i : c_i \geq \tau\big\} + 1 \qquad \text{clamped to } [1, \text{cap}]$$ + +If the set $\{i : c_i \geq \tau\}$ is empty, then $k^* = \text{cap}$. This requires total energy $\sum \sigma_j^2 < \varepsilon \tau / (1 - \tau)$ — with the default $\varepsilon = 10^{-6}$ and $\tau = 0.9$, total energy below $9 \times 10^{-6}$. Since each Phase 2 evolve step injects fresh batch energy into the SVD, this threshold is not reachable during normal operation; the fallback exists as a safety net against numerical edge cases or future algorithm changes. (Approximately equal nonzero singular values produce a _non-empty_ set — the threshold is met at high index, yielding $k^* \approx \text{cap}$ via the normal path; see §A.5.) + +> _Design note (the $+1$ buffer dimension)._ The $+1$ ensures the active subspace always extends one dimension beyond the energy threshold. This serves three purposes. +> +> First, it **enables coherence** (§5.5). Coherence requires $k \geq 2$, so without the buffer a tracker whose energy is dominated by a single direction would be permanently stuck at $k = 1$ with no off-diagonal detection capability. The $+1$ guarantees that a single dominant component yields $k^* = 2$, not $k^* = 1$. +> +> Second, it provides **early visibility into emerging structure**. The dimension just past the energy threshold is the first place new non-noise structure will appear as the data distribution evolves. By including it in the active subspace, the system can detect that emergence through the within-subspace axes (displacement, surprise, coherence) rather than relying solely on the coarser novelty axis, which only measures aggregate residual energy. +> +> Third, it acts as a **stability margin** against threshold boundary oscillation. A dimension whose cumulative energy fraction fluctuates near $\tau$ would cause rank to toggle on every adaptation step without the buffer; the extra dimension absorbs this boundary noise. + +3. Move rank by at most one step: + +$$k \leftarrow k + \operatorname{clamp}(k^* - k, \; -1, \; +1)$$ + +**On rank drop from $\geq 2$ to $1$:** destroy coherence baselines — return the fast EWMA, slow EWMA, drift accumulator, **and clip-pressure EWMA** for the coherence axis to their uninitialised state ($\bar{\rho} = 0$). This prevents stale state from a previous $k \geq 2$ epoch from contaminating a future one. Note: under the current rank formula, $k^* \geq 2$ whenever $\text{cap} \geq 2$ (the $+1$ buffer ensures this — see design note above), so this clause cannot fire during normal operation for any tracker with $\text{cap} \geq 2$. It exists as a safety net against implementation defects or future algorithm changes that might introduce a pathway to $k = 1$ from above. + +### 4.3 Memory per Tracker · `sec:sentinel:algorithm-tracker-memory` + +At width $w$ with capacity $\text{cap}$: + +| Component | Size (elements) | +| --------------------------------------- | ---------------------------- | +| $U$ | $w \times \text{cap}$ | +| $\sigma$ | $\text{cap}$ | +| $\mu^{(z)}, \nu^{(z)}$ | $2 \times \text{cap}$ | +| $\Gamma$ (upper triangle) | $\text{cap}(\text{cap}-1)/2$ | +| Baselines (§6) | $4 \times 8 = 32$ | +| Scalar state (rank, step counter, etc.) | $\sim 10$ | + +At $w = 96$, $\text{cap} = 16$: approximately 1,750 floating-point elements. Total analysis memory is bounded by $|\mathcal{A}^*| \times (\text{max per-tracker size})$; see §8 for the bound on $|\mathcal{A}^*|$ and §12 for global resource accounting. + +--- + +## Chapter 5. Scoring · `sec:sentinel:algorithm-scoring` + +### 5.1 Polarity Invariant · `sec:sentinel:algorithm-scoring-polarity-invariant` + +All four scoring axes share a **polarity invariant**: higher values indicate greater anomalous departure from baseline. A score of zero (or near zero) is maximally normal; positive excursions indicate increasing anomaly. + +This invariant is essential for the baseline tracking mechanism (§6): the upper-tail outlier filter prevents sustained high scores from poisoning baselines, which requires that anomalous behaviour consistently produces _high_ scores, never low. + +### 5.2 Novelty · `sec:sentinel:algorithm-scoring-novelty` + +$$\text{novelty}_i = \frac{\|\mathbf{x}_i - \hat{\mathbf{x}}_i\|^2}{w - k}$$ + +Average residual energy per orthogonal degree of freedom. **Measures:** unexplained structure — energy outside the learned subspace. **Range:** $[0, \infty)$. **Requires:** $k \geq 1$. + +**Degeneracy at $k = w$.** When $\text{cap} = w$ (which requires $r_{\max} \geq w$), rank adaptation (§4.2, Phase 5) can push $k$ to $w$. The residual $R_i = 0$ identically and the denominator $w - k = 0$, giving the indeterminate form $0/0$. Implementations define $\text{novelty}_i = 0$ in this case (clamping the denominator to 1). The axis carries no information — detection relies entirely on the three within-subspace axes. This condition is reported as _novelty-saturated_ (§14.7) and arises naturally at coordination trackers ($w = 4$) and deep cells with generous $r_{\max}$. + +### 5.3 Displacement · `sec:sentinel:algorithm-scoring-displacement` + +$$\text{displacement}_i = \frac{\|\mathbf{z}_i\|^2}{k + \|\mathbf{z}_i\|^2}$$ + +Bounded distance from the subspace origin. **Measures:** total within-subspace energy — how far the observation sits from the learned centre. **Range:** $[0, 1)$. **Requires:** $k \geq 1$. + +> _Design note (polarity)._ The natural measure of proximity is $q_i = k / (k + \|\mathbf{z}_i\|^2)$, where anomalies push $q$ downward. This violates the polarity invariant: anomalous values (low $q$) would pass the upper-tail filter (§6.2) and gradually drag the baseline downward, making the departure invisible. The complement $1 - q_i$ restores correct polarity. + +### 5.4 Surprise · `sec:sentinel:algorithm-scoring-surprise` + +$$\text{surprise}_i = \frac{1}{k} \sum_{j=1}^{k} \frac{(z_{ij} - \mu^{(z)}_j)^2}{\nu^{(z)}_j + \varepsilon}$$ + +Average diagonal Mahalanobis deviation. **Measures:** per-dimension magnitude deviation from learned latent means. **Range:** $[0, \infty)$. **Requires:** $k \geq 1$. + +Complementary to displacement: an observation can have normal total energy but unusual distribution across dimensions. + +### 5.5 Coherence · `sec:sentinel:algorithm-scoring-coherence` + +$$\text{coherence}_i = \frac{2}{k(k-1)} \sum_{j < l} \big(z_{ij} \, z_{il} - \Gamma_{jl}\big)^2$$ + +Average squared deviation of pairwise latent products from their learned second moments. **Measures:** off-diagonal covariance deviation — unusual combinations of co-activation. **Range:** $[0, \infty)$. **Requires:** $k \geq 2$. + +**Defined as $0$ when $k < 2$.** Coherence baselines are not updated while $k < 2$. When rank first reaches 2, baselines begin tracking from the first real coherence values. On rank drop back to 1, baselines are destroyed (§4.2, Phase 5). + +### 5.6 Summary · `sec:sentinel:algorithm-scoring-summary` + +| Axis | Score | Range | Measures | Requires | +| --------------- | ------------ | ------------- | ----------------------- | ---------- | +| Subspace | Novelty | $[0, \infty)$ | Unexplained structure | $k \geq 1$ | +| Within-subspace | Displacement | $[0, 1)$ | Distance from centroid | $k \geq 1$ | +| Within-subspace | Surprise | $[0, \infty)$ | Per-dimension magnitude | $k \geq 1$ | +| Within-subspace | Coherence | $[0, \infty)$ | Pairwise co-activation | $k \geq 2$ | + +### 5.7 Geometric Picture · `sec:sentinel:algorithm-scoring-geometric-picture` + +``` +Full observation space ℝ^w +┌───────────────────────────────────────────┐ +│ │ +│ Learned subspace ℝ^k │ +│ ┌──────────────────┐ │ +│ │ μ^(z) centroid │ Residual: x − x̂ │ +│ │ · │ ◄─── novelty ───► │ +│ │ /| │ │ +│ │ z | │ │ +│ │ ├─┤ displacement │ │ +│ │ ├─┤ surprise │ │ +│ │ z₁·z₂ coherence │ │ +│ └──────────────────┘ │ +└───────────────────────────────────────────┘ +``` + +### 5.8 Why Four Axes · `sec:sentinel:algorithm-scoring-four-axes` + +The three within-subspace scores decompose the latent activation pattern along orthogonal statistical concerns: + +| Score | Input | Covariance structure | +| ------------ | --------------------------------------- | --------------------- | +| Displacement | $\|\mathbf{z}\|^2$ | Total energy (scalar) | +| Surprise | $(z_j - \mu_j)^2/\nu_j$ per $j$ | Diagonal | +| Coherence | $(z_j z_l - \Gamma_{jl})^2$ per $j < l$ | Off-diagonal | + +Together they cover the full covariance structure without assembling or inverting a dense $k \times k$ matrix. + +A fifth axis — projection energy $\|\hat{\mathbf{x}}_i\|^2/k$ — is omitted because the constant-norm property (§2.5) makes it a perfect affine function of novelty. It carries zero independent information and violates the polarity invariant. + +--- + +## Chapter 6. Baseline Tracking and Drift Detection · `sec:sentinel:algorithm-baseline-tracking-and-drift-detection` + +Each scoring axis maintains four components: a fast EWMA for instantaneous z-scores, a slow EWMA as a long-memory reference, a one-sided drift accumulator for detecting gradual shifts, and a clip-pressure EWMA $\bar{\rho}$ that tracks the fraction of samples clipped per axis (§6.4). Together, these provide normalised scoring (z-scores), temporal persistence (drift detection), and adaptive clip-width modulation on top of the raw scores from §5. + +### 6.1 Fast EWMA · `sec:sentinel:algorithm-fast-ewma` + +The fast EWMA tracks running mean $\bar{s}$ and variance $\bar{v}$ at decay $\lambda$. + +#### 6.1.1 Update Rule · `sec:sentinel:algorithm-fast-ewma-update-rule` + +**Clip-pressure state.** Each axis maintains a clip-pressure EWMA $\bar{\rho}$, initialized to $0$ at tracker creation. At warm-up completion (§11.4), $\bar{\rho}$ is reset to $0$ alongside the CUSUM reset and slow-from-fast seeding. On rank drop from $\geq 2$ to $1$, the coherence axis's $\bar{\rho}$ is destroyed alongside the coherence baselines (§4.2, Phase 5). + +Given per-sample scores $\mathbf{s} = (s_1, \ldots, s_b)$ from one batch: + +**1. Compute effective clip ceiling.** Let $p = \max(\eta, \bar{\rho})$ where $\eta$ is the noise influence (§11.5) and $\bar{\rho}$ is the clip-pressure EWMA for this axis. Compute: + +$$n_\sigma^{\text{eff}} = n_\sigma\left(1 + \frac{p}{1 - p + \varepsilon}\right)$$ + +**When the baseline is uninitialised** (no valid $\bar{s}$ or $\bar{v}$), skip clipping entirely — placeholder values are not a meaningful reference. When clipping is skipped (uninitialised baseline), set $\rho_t = 0$ and update $\bar{\rho}$ normally. This causes $\bar{\rho}$ to decay toward zero during the uninitialised phase, ensuring no residual pressure when the baseline initialises. + +**When the baseline is initialised**, compute the clip ceiling: + +$$c_{\text{clip}} = \bar{s} + n_\sigma^{\text{eff}} \sqrt{\bar{v}}$$ + +Retain samples with $s_i < c_{\text{clip}}$. Compute the batch clip ratio $\rho_t = n_{\text{clipped}} / b$. + +**If all samples are rejected**, hold both baselines unchanged and set $\rho_t = 1$. The clip-pressure term rises so the ceiling opens on subsequent batches; baseline updates resume when samples are retained. Learning the unclipped batch would let the baselines chase a sustained shift and stop the CUSUM from accumulating under a gradual anomaly. The `gradual_single_cell_cusum` case witnesses the detection contract that requires this accumulation. + +Update the clip-pressure EWMA: + +$$\bar{\rho} \leftarrow \lambda_\rho \, \bar{\rho} + (1 - \lambda_\rho) \, \rho_t$$ + +The filter is **upper-tail only** because all four scoring axes are non-negative, right-skewed, and satisfy the polarity invariant (§5.1): anomalous departure inflates scores, never deflates. The clip ceiling prevents sustained high scores from poisoning the baseline upward. + +**2. Fast EWMA update.** When at least one sample is retained, let $\bar{s}_{\text{batch}}$ be the mean of **retained** samples and update the fast mean as follows. When no samples are retained, hold the fast mean unchanged. + +- _Uninitialised → initialised:_ $\bar{s} \leftarrow \bar{s}_{\text{batch}}$ +- _Subsequent:_ $\bar{s} \leftarrow \lambda \, \bar{s} + \alpha \, \bar{s}_{\text{batch}}$ + +**3. Variance update.** Let $\bar{v}_{\text{batch}}$ be the population variance of **retained** samples (requires $\geq 2$ retained; otherwise **freeze**: leave $\bar{v}$ unchanged — see design note below). + +- _Uninitialised → initialised:_ $\bar{v} \leftarrow \max(\bar{v}_{\text{batch}}, \; 10^{-4})$ +- _Subsequent:_ $\bar{v} \leftarrow \lambda \, \bar{v} + \alpha \, \max(\bar{v}_{\text{batch}}, \; 10^{-4})$ + +The $10^{-4}$ floor prevents degenerate zero-variance baselines. + +> _Design note (variance freeze on < 2 retained)._ When clipping retains fewer than 2 samples, $\bar{v}$ is neither updated nor decayed — it holds its last successfully computed value. Three alternatives were considered: +> +> 1. **Pure decay** ($\bar{v} \leftarrow \lambda\,\bar{v}$). This creates a positive feedback loop: shrinking variance → tighter clip ceiling → more clipping → fewer retained → more decay. The loop is self-reinforcing and can collapse variance to near-zero, worsening the very lockout condition that caused the skip. +> 2. **Decay with floor injection** ($\bar{v} \leftarrow \lambda\,\bar{v} + \alpha \cdot 10^{-4}$). The steady-state variance converges to $10^{-4}$ regardless of $\lambda$, which is 1–3 orders of magnitude below realistic axis variances ($\sim 0.001$–$0.1$). The collapse from $\bar{v} = 0.01$ to $\bar{v} = 0.0001$ inflates z-scores by $\sim$10× and tightens the clip ceiling by nearly the same factor — a milder but structurally similar failure to pure decay. Making the injection proportional to the current $\bar{v}$ approximates a freeze with extra arithmetic and a slow downward bias. +> 3. **Single-sample deviation proxy** ($\bar{v} \leftarrow \lambda\,\bar{v} + \alpha \cdot \max\!\big((s_{\text{retained}} - \bar{s})^2,\; 10^{-4}\big)$). This tracks the right order of magnitude during regime transitions. However, the retained sample passed the clip filter and is bounded by $c_{\text{clip}} - \bar{s} = n_\sigma^{\text{eff}}\sqrt{\bar{v}}$, so the proxy is bounded by $(n_\sigma^{\text{eff}})^2 \bar{v}$. During early lockout (before clip-pressure has risen, $n_\sigma^{\text{eff}} \approx n_\sigma$), the retained sample is likely near $\bar{s}$, causing systematic variance underestimation — a mild version of option 1's failure mode in exactly the phase where it is least tolerable. This alternative merits evaluation if small-batch deployments reveal the freeze to be problematic; it is not adopted here. +> +> The freeze produces a stale-but-data-derived estimate rather than a convergence target that is aggressively wrong. Its failure mode is **directionally asymmetric**: if the old regime had high variance, the frozen $\bar{v}$ suppresses z-scores and reduces detection sensitivity in the new regime; if the old regime had low variance, the frozen $\bar{v}$ inflates z-scores and false alarms. The freeze is least-bad _on average_ across regime transitions precisely because you do not know which direction the regime shifted. +> +> Critically, the freeze window is **self-limiting** under the clip-pressure mechanism (§6.4): once $\bar{\rho}$ rises enough to widen the ceiling, more samples are retained and variance updates resume. The staleness duration is bounded by the clip-pressure recovery time ($\sim$20–40 batches at $\lambda_\rho = 0.95$), not by any property of the variance itself. During this window, detection sensitivity is calibrated to the _previous_ regime's spread. The host can observe this condition via the clip-pressure value ($\bar{\rho}$) in the per-axis report (§14.4). + +**4. Slow EWMA update.** Using the same **retained** samples from step 1 (single shared clip filter — see design note below), update slow mean and slow variance at rate $\lambda_s$ following the same rules as steps 2–3. When no samples are retained, hold the slow mean and variance unchanged. + +**5. Drift accumulator update.** Using the **raw** batch mean $\bar{s}_{\text{raw}} = \frac{1}{b}\sum_{i=1}^b s_i$ (all $b$ samples, no clipping): + +$$S_t = \max\!\Big(0, \; S_{t-1} + \big(\bar{s}_{\text{raw}} - \bar{s}_{\text{slow}}\big) - \kappa_\sigma \sqrt{\bar{v}_{\text{slow}}}\Big)$$ + +> _Design note (single shared clip filter)._ Both the fast and slow EWMAs receive samples from the same clip filter, computed against the fast EWMA's clip-pressure-adjusted ceiling. This is a deliberate departure from independent per-EWMA clipping. The slow EWMA's conservatism is provided by its longer time constant ($\lambda_s > \lambda$), not by independent input filtering. Independent slow-EWMA clipping would create a secondary lockout surface that the clip-pressure mechanism cannot reach. See §6.4 for the contamination analysis. + +> _Design note (pre-clip drift accumulator input)._ The drift accumulator uses the raw batch mean because it is a detector, not an estimator. Its output $S$ flows outward to the report (§14.5) and is never fed back into any model. Clipping protects estimators from contamination; it makes detectors blind. The raw input ensures the accumulator registers the full departure magnitude during the first batches of a shift, before the clip-pressure mechanism has opened the ceiling. + +#### 6.1.2 Z-Score Computation · `sec:sentinel:algorithm-fast-ewma-z-score-computation` + +$$\zeta(s) = \frac{s - \bar{s}}{\sqrt{\bar{v}} + \varepsilon}$$ + +Two z-scores per axis per batch: + +- $\zeta(\max_i s_i)$ — loudest alarm in the batch. +- $\zeta(\bar{s}_{\text{batch}})$ — sustained elevation of the batch. + +### 6.2 Slow EWMA · `sec:sentinel:algorithm-slow-ewma` + +Each axis maintains a second EWMA at decay $\lambda_s > \lambda$ (per-tracker) or $\lambda_{s,m} > \lambda$ (coordination). The slow EWMA provides the reference for the drift accumulator. + +The constraint $\lambda_s > \lambda$ is structural — the slow baseline must have strictly longer memory than the fast one. + +| Decay | Half-life | Role | +| ------------------- | ------------------- | -------------- | +| $\lambda = 0.99$ | $\approx 69$ steps | Fast baseline | +| $\lambda_s = 0.999$ | $\approx 693$ steps | Slow reference | + +**Update rule.** The slow EWMA receives the same retained samples as the fast EWMA (single shared clip filter) and applies the same EWMA update at rate $\lambda_s$. The complete per-batch procedure is specified in §6.1.1, step 4. + +### 6.3 CUSUM Drift Accumulator · `sec:sentinel:algorithm-cusum-drift-accumulator` + +One-sided Page's test detecting sustained upward drift of raw batch means from the slow baseline: + +$$S_t = \max\!\Big(0,\; S_{t-1} + \big(\bar{s}_{\text{raw},t} - \bar{s}_{\text{slow},t}\big) - \kappa\Big)$$ + +where $\bar{s}_{\text{raw},t}$ is the **raw** (pre-clip) batch mean of all $b$ samples, and $\kappa = \kappa_\sigma \cdot \sqrt{\bar{v}_{\text{slow},t}}$ is the noise allowance. [This formula is repeated from §6.1.1, step 5 for self-contained reference.] + +**Input signal.** The drift accumulator uses the **raw** (pre-clip) batch mean as its input. The complete per-batch procedure is specified in §6.1.1, step 5. The raw-input choice is analysed in §6.4. + +Under normal conditions, fluctuations are absorbed by $\kappa$. Under a gradual shift, $\bar{s}_{\text{raw}}$ consistently exceeds $\bar{s}_{\text{slow}} + \kappa$, and the accumulator grows monotonically. + +> _Design note (why dual-EWMA, not a frozen reference)._ A frozen checkpoint would require manual host resets after legitimate regime changes — operational burden and a policy decision. A slow EWMA adapts automatically, just slowly enough to catch shifts before absorption. After a legitimate change, the slow baseline catches up and the accumulator returns to zero without intervention. + +Three reset mechanisms: + +1. **Automatic.** The slow baseline eventually absorbs a legitimate regime change, driving $S \to 0$. +2. **Host-initiated.** The host inspects, decides the shift is legitimate, and zeroes $S$. +3. **Post-warm-up seeding.** After initial warm-up completes, the slow EWMA is seeded from the fast EWMA's converged values and $S$ is reset. The rationale and procedure are specified in §11. + +### 6.4 Clip-Pressure Dynamics and Contamination Trade-offs · `sec:sentinel:algorithm-clip-pressure-dynamics-and-contamination-tradeoffs` + +The clip-pressure mechanism (§6.1.1) introduces a controlled trade-off between baseline lockout recovery and contamination resistance. This section analyses the dynamics under three regimes and the feedback paths through which contamination propagates. + +#### 6.4.1 Sustained Lockout Recovery · `sec:sentinel:algorithm-clip-pressure-sustained-lockout-recovery` + +Under a regime shift that clips 100% of samples ($\rho_t = 1.0$ every batch), $\bar{\rho}$ rises monotonically and the ceiling opens progressively: + +| Batches | $\bar{\rho}$ | $n_\sigma^{\text{eff}}$ (at $n_\sigma = 3$) | Status | +| ------: | -----------: | ------------------------------------------: | -------------------------- | +| 0 | 0.00 | 3.0 | Locked | +| 3 | 0.14 | 3.5 | Softening | +| 7 | 0.30 | 4.3 | Opening — some data passes | +| 14 | 0.51 | 6.1 | Ceiling doubled | +| 20 | 0.64 | 8.3 | Wide — most data passes | +| 30 | 0.79 | 14 | Effectively unclipped | +| 40 | 0.87 | 23 | Baseline actively tracking | + +When all samples are rejected (§6.1.1, step 1), both baselines are held unchanged and $\rho_t = 1$ raises the clip-pressure term so the ceiling opens on subsequent batches. Baseline updates resume when samples are retained; the CUSUM continues to receive the raw batch mean throughout the lockout. + +#### 6.4.2 Transient Spike Recovery · `sec:sentinel:algorithm-clip-pressure-transient-spike-recovery` + +A single anomalous batch ($\rho_t = 1.0$ for one batch, then $\rho_t = 0$) produces minimal ceiling disturbance: + +| Batch | $\rho_t$ | $\bar{\rho}$ | $n_\sigma^{\text{eff}}$ | +| --------: | -------: | -----------: | ----------------------: | +| 1 (spike) | 1.0 | 0.05 | 3.16 | +| 7 | 0 | 0.04 | 3.12 | +| 15 | 0 | 0.02 | 3.07 | + +The peak ceiling widening (5%) is negligible. Self-correcting within ~15 batches. + +#### 6.4.3 Oscillatory Convergence Under Moderate Shifts · `sec:sentinel:algorithm-clip-pressure-oscillatory-convergence-under-moderate-shifts` + +When a regime shift clips a substantial fraction but not all samples (e.g., 50–80%), convergence is oscillatory rather than monotonic: + +1. $\bar{\rho}$ rises → ceiling widens → some extreme samples pass through. +2. Baseline absorbs retained data, shifts toward new regime. +3. Fewer samples are clipped (less extreme relative to updated baseline) → $\bar{\rho}$ decays. +4. Ceiling tightens — but baseline hasn't fully converged. +5. Clipping resumes at a moderate rate → $\bar{\rho}$ rises again. +6. Cycle repeats with decreasing amplitude. + +The oscillation is inherent when $\lambda_\rho < \lambda$: the ceiling responds faster than the baseline adapts. Each cycle moves the baseline closer to the new regime. The baseline converges monotonically through the oscillation — only the ceiling oscillates. + +At $\lambda_\rho = 0.95$ and $\lambda = 0.99$, each ceiling-open cycle lasts approximately 14 batches (the clip-pressure half-life), during which the baseline absorbs $1 - \lambda^{14} \approx 13\%$ of the remaining gap. Typical moderate shifts (initial clip rate 40–70%) converge within 3–5 oscillatory cycles (~60–100 batches). Near-total lockout shifts (initial clip rate above 90%) take proportionally longer — roughly 8–12 cycles (~120–170 batches) — because the ceiling must open much wider before substantial data passes. + +The damping condition for non-oscillatory convergence is $\lambda_\rho \geq \lambda$, but satisfying this at $\lambda = 0.99$ would require $\lambda_\rho \geq 0.99$ (half-life ~69 batches), making lockout recovery unacceptably slow. The oscillatory regime at $\lambda_\rho = 0.95$ is the correct trade-off. + +The drift accumulator is unaffected by ceiling dynamics — it uses raw batch means (§6.1.1, step 5) and sees the full departure throughout the oscillation. + +#### 6.4.4 Contamination Feedback Paths · `sec:sentinel:algorithm-clip-pressure-contamination-feedback-paths` + +Under elevated clip pressure, the widened ceiling admits a larger fraction of extreme scores, which shift the baselines. The shifted baselines affect three downstream quantities: + +1. **Z-score sensitivity.** A contaminated fast EWMA mean $\bar{s}$ closer to adversarial score values produces smaller z-scores for those observations. Detection sensitivity for adversarial traffic is reduced in proportion to the contamination. + +2. **Clip ceiling elevation.** The ceiling $c_{\text{clip}} = \bar{s} + n_\sigma^{\text{eff}} \sqrt{\bar{v}}$ rises with $\bar{s}$ and $\bar{v}$, admitting slightly more extreme observations in subsequent batches. This is a mild positive feedback loop, bounded by the clip-pressure mechanism's self-correcting dynamics: as clipping decreases, $\bar{\rho}$ decays, tightening $n_\sigma^{\text{eff}}$ toward $n_\sigma$. + +3. **Drift noise allowance.** The allowance $\kappa = \kappa_\sigma \sqrt{\bar{v}_{\text{slow}}}$ increases if the slow EWMA's variance is contaminated, reducing drift accumulator sensitivity. This is a second-order effect (variance contamination is slower than mean contamination) but reduces the system's ability to detect gradual shifts during sustained adversarial presence. + +#### 6.4.5 Contamination vs. Bias · `sec:sentinel:algorithm-clip-pressure-contamination-vs-bias` + +**Under hard clipping** with a sustained adversary controlling fraction $f$ of observations, the baseline tracks only the clean $(1-f)$ fraction — biased but uncontaminated. Z-scores for clean observations near the clipping boundary are systematically inflated, generating false positives for legitimate traffic. Adversarial observations are invisible to all baseline machinery including the drift accumulator. + +**Under clip-pressure modulation**, the baseline tracks the actual observation mixture — slightly contaminated but unbiased. Z-scores for both clean and adversarial observations are measured against the true mixture distribution. The drift accumulator (using raw means) reports the full departure regardless. + +For "Measure, don't decide" (§1.3), the unbiased estimate is more honest: the host receives baselines that reflect the actual observation distribution, including adversarial influence. A biased baseline misrepresents the distribution to the host. The host can make contamination judgments from the drift accumulator evidence and the per-axis clip-pressure values in the report (§14.4). + +> _Design note (one-sided drift detection and sub-clip contamination)._ The CUSUM drift accumulator (§6.3) is deliberately one-sided upward, consistent with the polarity invariant (§5.1). It detects the _rate_ at which the fast baseline diverges above the slow baseline — not the _absolute level_ of either baseline. A patient adversary producing sustained sub-clip scores can inflate both baselines in lockstep, keeping the fast-slow gap below $\kappa$ and accumulating zero drift evidence, while reducing the system's sensitivity to future anomalous traffic. This is a known consequence of the adaptive baseline design: the same property that enables automatic adaptation to legitimate regime changes (the slow baseline eventually absorbs any sustained shift, §6.3 reset mechanism 1) also enables slow adversarial contamination. The CUSUM does not — and should not — detect this, because flagging slow baseline drift as suspicious would trigger on every legitimate regime change. Instead, the system reports the slow baseline values (§14.5) and clip-pressure state (§14.4), enabling hosts to build secular-trend monitors externally. See §17.5 for the full evasion assessment. + +#### 6.4.6 $\lambda_\rho$ Guidance · `sec:sentinel:algorithm-clip-pressure-lambda-rho-guidance` + +The trade-off axis is **detection latency vs. contamination resistance**, parameterized by $\lambda_\rho$: + +| $\lambda_\rho$ | Ceiling half-life | Lockout recovery | Contamination leakage | Regime | +| -------------: | ----------------: | ---------------------- | --------------------- | ----------------------- | +| 0.90 | ~7 batches | Fast (~20 batches) | Higher | High-churn environments | +| 0.95 | ~14 batches | Moderate (~40 batches) | Moderate | **Default** | +| 0.99 | ~69 batches | Slow (~150 batches) | Minimal | Security-critical | + +The default optimizes for the common case where regime shifts are more frequent than sustained adversarial presence. + +--- + +## Chapter 7. Hierarchical Coordination · `sec:sentinel:algorithm-hierarchical-coordination` + +The coordination tier detects anomalous patterns in the _distribution of scores across spatially related cells_ at every level of the spatial tree hierarchy. It reuses the subspace tracker (§4) at a second level of abstraction: instead of modelling suffix bit vectors, it models cross-cell score summaries. + +The coordination tier operates exclusively on the **competitive set** $\mathcal{A}$ (§8), not the full analysis set $\mathcal{A}^*$. Ancestor trackers (§8) provide per-value depth-of-defence; the coordination tier provides cross-cell pattern detection. These are interlocking constraints — their interaction is analysed in §16–17. + +--- + +### 7.1 Coordination Contexts · `sec:sentinel:algorithm-coordination-contexts` + +Every internal node of the spatial tree whose subtree contains **competitively selected** cells in **both** its left and right children's subtrees is a **coordination context**. The contexts form a binary hierarchy mirroring the G-Tree: + +``` + root ← sees all K cells + / \ + g_L g_R ← each sees its subtree + / \ / \ + ... ... ... ... ← narrower groups + ↓ ↓ ↓ ↓ + cells cells cells cells +``` + +The **coordination group** at context $g$: + +$$\mathcal{C}(g) = \big\{c \in \mathcal{A} : c \text{ is in the subtree of } g\big\}$$ + +A context is **active** when both subtrees contribute: $\mathcal{C}(g.\text{left}) \neq \emptyset$ and $\mathcal{C}(g.\text{right}) \neq \emptyset$. + +**Group count.** A binary tree with $K$ leaves has at most $K - 1$ internal nodes, so the number of active contexts is at most $K - 1$. When analysed cells cluster spatially, most internal nodes have cells in only one subtree, and the count is much smaller. + +**Nesting.** Groups nest by containment: descendant contexts have subsets of ancestor contexts' groups. A cell at G-Tree depth $d$ participates in at most $d$ contexts. + +--- + +### 7.2 The Coordination Signal · `sec:sentinel:algorithm-coordination-signal` + +After per-cell scoring (§4.2, Phase 1), each **competitive** cell $c \in \mathcal{A}$ that processed observations has a 4-dimensional summary: + +$$\mathbf{o}_c = \big(\bar{s}_{c,\text{nov}},\; \bar{s}_{c,\text{disp}},\; \bar{s}_{c,\text{surp}},\; \bar{s}_{c,\text{coh}}\big) \in \mathbb{R}^4$$ + +These are **raw batch mean scores** — not z-scores. The coordination tracker learns its own normalisation. + +For each active context $g$ with reporting group $\mathcal{C}_{\text{active}}(g) = \{c \in \mathcal{C}(g) : c \text{ has scores this batch}\}$, assemble: + +$$O_g = \begin{pmatrix} \mathbf{o}_{c_1} \\ \vdots \\ \mathbf{o}_{c_m} \end{pmatrix} \in \mathbb{R}^{m \times 4}$$ + +**Cells with no observations in a given batch are excluded.** Only cells that processed at least one observation contribute rows. The group size $m$ may vary batch to batch. A context with $m < 2$ reporting cells in a given batch skips coordination for that batch. + +--- + +### 7.3 Running-Mean Centring · `sec:sentinel:algorithm-coordination-running-mean-centring` + +Each context maintains a 4-dimensional EWMA reference $\mu^{(\text{in})}_g \in \mathbb{R}^4$ at decay $\lambda$: + +$$O_g^{(\text{c})} = O_g - \mathbf{1}_m \cdot \big(\mu^{(\text{in})}_g\big)^\top$$ + +After feeding the tracker, update: + +$$\mu^{(\text{in})}_g \leftarrow \lambda \, \mu^{(\text{in})}_g + (1 - \lambda) \, \text{colmeans}(O_g)$$ + +On the first batch, $\mu^{(\text{in})}_g \leftarrow \text{colmeans}(O_g)$. + +Running-mean centring detects both **differential** coordination (some cells anomalous, others normal — unusual structure in the centred vectors) and **uniform** coordination (all cells shifting together — the EWMA lags behind the shift, creating bias the tracker sees as elevated displacement). Per-batch centring would destroy the uniform signal. + +--- + +### 7.4 Bottom-Up Assembly · `sec:sentinel:algorithm-coordination-bottom-up-assembly` + +Coordination propagates from the leaves of the spatial tree toward the root. At each active context, the centred score matrix is assembled from the context's reporting group and fed through the context's coordination tracker. + +``` +procedure PropagateCoordination(g, CellScores): + Reports ← empty list + + if g has no spatial children: + if g is in the competitive set and has scores: + return Reports, Cells = [(g, CellScores[g])] + return Reports, Cells = empty + + LeftReports, LeftCells ← PropagateCoordination(g.left, CellScores) + RightReports, RightCells ← PropagateCoordination(g.right, CellScores) + Reports ← LeftReports concatenated with RightReports + + MyCells ← LeftCells concatenated with RightCells + if g is in the competitive set and has scores: + append (g, CellScores[g]) to MyCells + + -- Fire coordination when both subtrees contribute + if LeftCells is non-empty and RightCells is non-empty and |MyCells| ≥ 2: + m ← |MyCells| + O ← assemble m × 4 matrix from MyCells + O_c ← O − 1_m · (μ_in[g])^T (§7.3) + Report ← g.CoordTracker.Observe(O_c) (§7.5) + μ_in[g] ← λ · μ_in[g] + α · colmeans(O) + append Report to Reports + + return Reports, MyCells +``` + +For semi-internal spatial nodes (one child), the single child's cells propagate upward without triggering coordination. + +> _Implementation note._ The pseudocode recurses through the full spatial tree for clarity, visiting $|G|$ nodes at $O(1)$ each. An implementation should walk only the reduced Steiner tree of the investment set (§8.2) — whose internal branching nodes are exactly the active coordination contexts — visiting $O(|\mathcal{I}|)$ nodes rather than $|G|$. Note that competitive cells that are internal G-Tree nodes (§8.8) sit on the Steiner tree as interior nodes, not leaves; a bottom-up walk must inject their score contributions at the appropriate merge point rather than expecting them to propagate from below. + +--- + +### 7.5 The Coordination Tracker · `sec:sentinel:algorithm-coordination-tracker` + +Each active context maintains a subspace tracker (§4) with analysis width $w = 4$ and capacity $\text{cap} = \min(4, r_{\max})$. It runs the identical five-phase core loop, operating on $O_g^{(\text{c})} \in \mathbb{R}^{m \times 4}$ — centred score summaries — rather than suffix bit vectors. + +At this level, the four axes measure: + +| Coordination Axis | What It Measures | +| ----------------- | -------------------------------------------------------------------- | +| **Novelty** | A _new kind_ of cross-cell score pattern not in the learned subspace | +| **Displacement** | The group's mean score vector departed from its historical position | +| **Surprise** | A specific axis is systematically anomalous across cells | +| **Coherence** | An unusual _combination_ of axis elevations across cells | + +Each axis carries its own fast EWMA ($\lambda$), slow EWMA ($\lambda_{s,m}$), drift accumulator, and clip-pressure EWMA ($\lambda_\rho$). + +> _Steady-state novelty saturation._ At $w = 4$ with $\text{cap} = 4$, the coordination tracker's rank adaptation commonly reaches $k = 4$ in steady state, at which point the novelty axis is saturated (§5.2). This is expected for moderate-to-large groups ($m \geq 4$) with diverse cross-cell traffic, where all four score-space dimensions carry comparable variance. For small groups ($m = 2$–$3$) with heterogeneous member cells, the novelty axis may remain active with 1 residual DOF, detecting unusual differential score-fluctuation directions. In both regimes, the three within-subspace axes (displacement, surprise, coherence) cover the coordination tier's primary detection mission — identifying optimisation-induced cross-cell correlations (§17.2). Hosts should interpret a consistently novelty-saturated coordination tracker as normal operation, not as a detection gap. The novelty-saturated flag (§14.7) and the scoring geometry distribution (§14.11.1) provide the reporting mechanism. + +--- + +### 7.6 Multi-Scale Detection · `sec:sentinel:algorithm-coordination-multi-scale-detection` + +The hierarchy's power derives from complementary sensitivity profiles at different scales. + +**Localised anomaly** (affecting a small cluster of cells): + +| Level | Sensitivity | Mechanism | +| ----------------------------- | ----------------------------- | ---------------------------------------- | +| Immediate parent ($m \sim 2$) | **Strong** — minimum dilution | Direct pattern change | +| Grandparent ($m \sim 4$–8) | **Moderate** | Novelty: pattern not in learned subspace | +| Root ($m = K$) | **Negligible** | Diluted across all cells | + +**System-wide shift** (all cells shifting): + +| Level | Sensitivity | Mechanism | +| ------------------------- | -------------------------------------- | ------------------------------------------------ | +| Leaf pairs ($m = 2$) | **Slow** — fast EWMA adapts quickly | Displacement rises | +| Mid-level ($m \sim 8$–32) | **Moderate** — SNR $\propto \sqrt{m}$ | Common-mode accumulation | +| Root ($m = K$) | **Strongest** — SNR $\propto \sqrt{K}$ | Aggregate displacement; noise cancels coherently | + +**The complementarity principle.** Localised anomalies are caught by small groups (high per-cell sensitivity). Global shifts are caught by large groups (high signal-to-noise through aggregation). The hierarchy covers the full spatial spectrum without configuration. + +**Distinguishing partial from uniform coordination.** Compare per-cell and coordination signals. Partial coordination shows elevated per-cell drift accumulators in a subset with strong local coordination novelty. Uniform coordination shows individually unremarkable per-cell z-scores with strong root-level displacement. The _level_ at which coordination peaks indicates the _scale_ of the coordinated behaviour. + +--- + +### 7.7 Lifecycle · `sec:sentinel:algorithm-coordination-lifecycle` + +Coordination contexts are **lazily materialised** during the Step 5 coordination walk (§9.1) and **pruned** after the walk completes. No explicit activation or deactivation calls appear in Steps 0–4. + +#### 7.7.1 Lazy Materialisation · `sec:sentinel:algorithm-coordination-lifecycle-lazy-materialisation` + +When `PropagateCoordination` (§7.4) walks the reduced Steiner tree and encounters an internal node whose left and right subtrees both contribute competitive cells with scores in this batch, it checks whether a coordination context already exists for that node. If not, a fresh context is created — comprising a coordination tracker (§7.5), a running-mean reference $\mu^{(\text{in})}_g$, and associated state — and **inline-warmed** per §11.7 before the first real observation feeds the tracker. The inline warm-up runs to completion within the same Step 5 invocation. + +This lazy model eliminates the bookkeeping of tracking coordination-eligible membership across phases. The walk itself discovers which contexts are needed, creates them on demand, and proceeds to use them — all within a single bottom-up pass. + +#### 7.7.2 Post-Walk Pruning · `sec:sentinel:algorithm-coordination-lifecycle-post-walk-pruning` + +After `PropagateCoordination` completes, contexts whose topology no longer warrants them — because one subtree no longer contains any online competitive cells — are **destroyed**, along with their coordination tracker and running-mean reference. The criterion is _membership_: whether online competitive cells exist in both subtrees, not whether those cells happened to contribute scores in this particular batch (a cell with zero observations in the current batch still counts as a member). On subsequent reactivation (when both subtrees again contribute), a fresh context is created and warmed per §7.7.1. + +The rationale parallels per-cell tracker destruction on competitive exit (§8.5): a stale coordination model from a different competitive membership may be actively misleading. The competitive set membership that produced the old tracker's learned subspace, baselines, and drift accumulator state may differ substantially from the membership at reactivation. Preserving the old state would risk false drift evidence (the new score distribution differs from the old baseline) or suppressed detection (the old baseline absorbs anomalous patterns that the new composition would flag). + +> _Design note (why not preserve)._ Preservation across deactivation gaps would require tracking which cells contributed to the old model and whether the new group is "close enough" to reuse it — a judgment call that introduces implicit assumptions about distributional continuity. Destruction and re-warm-up is simple, correct, and bounded-cost: the coordination tracker at $w = 4$ warms up quickly (§11.7, §7.8).\_ + +#### 7.7.3 Membership Changes Within a Cycle · `sec:sentinel:algorithm-coordination-lifecycle-cycle-membership-changes` + +Step 3 reconciliation may change competitive set membership, which affects coordination contexts in two ways: + +**Topology invalidation.** If a membership change empties one subtree of an active context — the last competitive cell in that subtree exits $\mathcal{T}$ — the `PropagateCoordination` walk's guard (`LeftCells is non-empty and RightCells is non-empty`) prevents the context from firing. Post-walk pruning (§7.7.2) then destroys the context. No "last firing" occurs; the context transitions directly from active to destroyed. + +**Membership reduction.** If a membership change removes one or more cells from a context's group while leaving both subtrees populated, the context fires normally with the reduced group. The coordination tracker's learned model — subspace $U$, baselines, drift accumulator, and running-mean reference $\mu^{(\text{in})}_g$ — was trained on batches that included the departed cell's contributions. The model is therefore slightly stale relative to the current group composition: it expects score-matrix rows from $m_{\text{old}}$ cells and now receives $m_{\text{new}} < m_{\text{old}}$. This staleness is self-correcting: the EWMA adaptation absorbs the membership change over $O(t_{1/2})$ subsequent batches. During the transient, modest baseline disturbance is possible but bounded by the single-member contribution to the group statistics. + +The symmetric case — membership _growth_ (a new competitive cell entering a subtree) — produces the analogous model staleness in the opposite direction. If the growth _creates_ a context (previously only one subtree was populated), the lazy materialisation mechanism (§7.7.1) handles it: a fresh context is created and inline-warmed before processing its first real observation. + +> _Timing-channel note._ Because Step 3 updates `CurrentCompetitiveSet` before Step 4 populates `CellReports`, and Step 5's walk builds its leaf set exclusively from `CellReports`, no coordination report can contain a cell that has exited the competitive set. This is stronger than the original §7.7.3 claimed: there is no grace-period leak of recently-competitive membership through the coordination tier. + +--- + +### 7.8 Memory and Computation · `sec:sentinel:algorithm-coordination-memory-and-computation` + +**Per context:** approximately 200 floating-point elements for the subspace tracker at $w = 4$, plus the 4-element running-mean reference. Total across all contexts: at most $(K - 1) \times (\text{per-context size})$. + +**Per-context per-batch computation:** $O(m)$ — the $w = 4$ constraint makes every operation linear in group size. The SVD of an $\mathbb{R}^{4 \times (k+m)}$ matrix costs $O(16(k+m)) \approx 16m$ multiply-adds — less than 0.3% of a single per-cell SVD at $w = 96$. + +**Total per-batch coordination work:** $O(K \cdot \bar{d})$ useful work, where $\bar{d}$ is average coordination participation depth. The pseudocode (§7.4) visits the full spatial tree for clarity; an implementation walking the investment set's reduced Steiner tree (§8.2) achieves this bound with $O(|\mathcal{I}|)$ traversal overhead (approaching $O(K)$ under typical spatial clustering). Negligible relative to per-cell scoring in either case. + +--- + +# Part III — Selection and Assembly · `sec:sentinel:algorithm-selection-and-assembly` + +--- + +## Chapter 8. Cell Selection · `sec:sentinel:algorithm-cell-selection` + +The analysis selector determines which cells from the spatial layer receive statistical modelling. It maintains two related but distinct sets: the **investment set** (cells that have been allocated trackers and warm-up resources) and the **producing set** (the online subset that actively generates scores). Competitive selection identifies significant cells, ancestor closure guarantees a complete model chain to the root, and the warm-up pipeline (§11) brings invested cells online in g.sum order. + +### 8.1 Competitive Selection · `sec:sentinel:algorithm-competitive-selection` + +Let $\mathcal{E} = \{v \in \text{V-entries} : v \neq \mathrm{root} \;\wedge\; \text{depth}_V(v) \leq L \;\wedge\; w(v) \geq 2\}$ be the **eligible set** — all V-entries other than the root, within the depth cutoff, whose analysis width $w = N - d$ is at least 2. The $w \geq 2$ predicate excludes cells where the subspace algebra is undefined (§4.1). The **competitive targets** are: + +$$\mathcal{T} = \text{top}_K\!\big(\mathcal{E},\; v.\text{importance}\big)$$ + +If $|\mathcal{E}| \leq K$, then $\mathcal{T} = \mathcal{E}$. + +| Parameter | Constraint | Role | +| --------------------- | ---------- | ---------------------------------------------- | +| $K$ (analysis budget) | $\geq 1$ | Maximum number of competitively selected cells | +| $L$ (depth cutoff) | $\geq 0$ | V-Tree depth ceiling for eligibility | + +The selection criteria use V-Tree ranking (depth and importance) as the sole competitive mechanism. The $w \geq 2$ predicate is a static geometric precondition excluding cells where the subspace algebra is undefined (§4.1), not a dynamic structural filter. The analysis selector does not inspect G-Tree structural state (terminal, semi-internal, or internal). This is a deliberate design choice; see the design note below. + +**The root is never a target.** The G-Tree root is excluded from $\mathcal{E}$, and so from $\mathcal{T}$, however large its importance grows. It belongs to $\mathcal{I}$ by construction (§8.2) and reaches it no other way. Competition ranks regions against one another to decide which are worth modelling separately; the root is the whole domain, so it has nothing to be ranked against — it answers the population-level question rather than a regional one, and it is permanent (§8.4), which leaves nothing for a competitive slot to decide about it. The exclusion applies before the top-$K$ cut rather than after it: the root's importance is the traffic it accumulated before its first split, frozen there by that split while its children start from zero, so it outranks every real candidate until one of them overtakes a total the root is no longer adding to. Dropped after the cut it would hold a slot for that whole period, and at $K = 1$ the competitive set would stay empty throughout, with no descendant ever able to enter it. + +**Ties at the boundary.** When more than $K$ eligible entries exist, ties in importance at the $K$-th position are broken by the left endpoint of the cell's spatial interval (deterministic, spatially stable). + +**Recomputation.** The competitive targets $\mathcal{T}$ are recomputed after each observation pass (§9, Step 2), since splits, evictions, and rebalancing may change V-Tree depths and importance values. The investment set and producing sets are derived from $\mathcal{T}$ during Step 3. An implementation may maintain $\mathcal{T}$ incrementally via a bounded priority structure keyed on importance, updated during rebalancing notifications. + +> _Design note (why no G-Tree state filter)._ When a high-importance contour cell splits, the parent becomes an internal G-Tree node with frozen importance — children intercept all spatial-layer routing (§3.7). One might exclude such nodes from $\mathcal{A}$ on the grounds that they receive no routed observations and their slot duplicates what the ancestor closure (§8.2) provides for free. We deliberately do **not** apply this filter, for three reasons: +> +> 1. **Single ranking authority.** The V-Tree is the sole arbiter of competitive significance (§3.4). Its depth encodes proven importance relative to the entire neighbourhood. Adding a G-Tree structural predicate means the analysis selector second-guesses the ranking mechanism with spatial-layer implementation state — a cross-layer coupling the architecture otherwise avoids. +> 2. **The transient is the V-Tree working correctly.** The frozen parent holds a shallow V-Tree position because it _earned_ that position through sustained observation volume. Its children start at zero importance and have not yet demonstrated significance. Selecting the parent during this period is the V-Tree making a factually correct statement: this region has proven importance; its subdivisions have not. The max-uncle constraint (§3.4) and temporal decay (§3.6, §3.8) organically resolve the situation — children's importance grows, the frozen benchmark weakens, the V-Tree restructures, and the parent sinks past $L$ or out of the top-$K$. +> 3. **Coordination coverage during transition.** If the parent were immediately ejected from $\mathcal{A}$ upon splitting, the highest-importance region in the system would have _no competitive-level representation_ in the coordination tier (§7) until its children earn entry. Retaining the parent provides the only coordination-tier coverage of that region during the transition. Its tracker is fully active — the multi-scale delivery mechanism (§9.3) feeds it every observation in its subtree — so the slot is not dormant. +> +> The cost of this choice is that during the transient, one $K$-slot is occupied by a node whose tracker does the same work an ancestor tracker would do for free. This cost is bounded: it persists only until the V-Tree's competitive dynamics push the frozen entry below the selection threshold, which is proportional to $O(\theta / \text{observation\_rate})$ batches under typical decay. At small $K$ the cost is one displaced contour cell; at large $K$ it is negligible. The organic resolution — without cross-layer intervention — is the preferred design. + +### 8.2 The Investment Set · `sec:sentinel:algorithm-investment-set` + +The **investment set** closes the competitive targets under spatial tree ancestry: + +$$\mathcal{I} = \{\mathrm{root}\} \;\cup\; \mathcal{T} \;\cup\; \bigcup_{v \in \mathcal{T}} \text{Ancestors}(v.\text{node})$$ + +where $\text{Ancestors}(g)$ is the set of all materialised G-Tree nodes on the path from $g$ to the G-Tree root, inclusive. Since the G-Tree is fully materialised (§3.1), a target at depth $d$ contributes ancestors at depths $0, 1, \ldots, d - 1$. Every competitive target receives a **complete chain of allocated trackers** from itself to the root. + +The investment set determines resource commitment: every member of $\mathcal{I}$ has a tracker allocated and, if not yet online, is enqueued for warm-up. Not all members of $\mathcal{I}$ produce scores — only online members do (§8.3). + +The investment set forms a **Steiner tree** connecting the competitive targets to the permanent root. Its full size includes every intermediate path node. A reduced tree that retains the root and marked targets but suppresses unmarked unary nodes has at most $2K$ nodes for $K \geq 1$ targets: at most $K-1$ branching nodes, $K$ targets and one root. The familiar $2K-1$ bound additionally assumes the root is already a branching node. Neither reduced bound limits the full investment set. + +**Size bound.** For $K$ current competitive targets (at most `analysis_k`) at G-tree depths $d_i$, count the root once and at most $d_i$ non-root nodes along each target path: + +$$|\mathcal{I}| \leq 1 + \sum_{i=1}^{K} d_i = 1 + K\bar{D}$$ + +The bound is attained when the target paths share only the root; further sharing only reduces it. The permanent root remains even when $K=0$, giving one entry. Eligibility requires suffix width $N-d_i \geq 2$, so supported engines satisfy $|\mathcal{I}| \leq 1 + \texttt{analysis\_k}(N-2)$. This counts selected online and warming cell trackers; coordination trackers and an evicted model still held by the background warming worker are separate resource commitments. + +**Example 1 (typical, `depth_create = 3`):** + +``` +Root /0 ← shared by all +├── /1-A ← shared by c₁, c₂ +│ ├── /2-A ← shared by c₁, c₂ +│ │ ├── /3-A ★ c₁ +│ │ └── /3-B ★ c₂ +│ └── (unresolved) ← no split warranted +└── /1-B ← unique to c₃ + └── /2-C ← unique to c₃ + └── /3-C ★ c₃ + +Targets: 3 (each at depth 3, D̄ = 3) +Unique ancestors: 5 (root, /1-A, /2-A, /1-B, /2-C) +|I| = 8 +Worst-case bound: 1 + 3×3 = 10; actual: 8 (sharing root, /1-A, /2-A) +``` + +Three targets, five unique ancestors. The worst-case bound gives $1 + 3 \times 3 = 10$; the actual count is 8 because the root, /1-A, and /2-A are each shared by multiple targets. The reduced Steiner tree has 5 nodes ($2K - 1$: root, /2-A, and the three targets); the full materialised tree adds 3 chain intermediaries (/1-A, /1-B, /2-C) for a total of 8. + +**Example 2 (shared trunk).** Suppose $T$ unary trunk nodes lie strictly above the first branching node, and the branching region is a full binary tree with $K$ target leaves and no further unary intermediates. Then $|\mathcal{I}| = T + 2K - 1$. Additional unary intermediates increase that count. Sharing prevents multiplying the trunk by $K$, but does not remove its trackers or bound its length in terms of $K$. For a single target at depth $d$, the full path still needs $d+1$ trackers. + +**Dimension guard.** Under the $w \geq 2$ eligibility predicate in §8.1, no member of $\mathcal{I}$ can have $w < 2$: every competitive target has $w \geq 2$ by eligibility, and every ancestor has $w' = N - d' > N - d \geq 2$ since ancestors sit at strictly shallower G-Tree depth. This guard is retained as defensive specification against future changes to the eligibility predicate: if any member of $\mathcal{I}$ were to have $w < 2$, no tracker would be allocated, and the exclusion would be reported as a degenerate-cell count (§14.11). + +### 8.3 The Producing Sets · `sec:sentinel:algorithm-producing-sets` + +The **producing sets** are the online subsets of the investment set: + +$$\mathcal{A} = \mathcal{I} \cap \mathcal{T} \cap \text{Online}$$ + +$$\mathcal{A}^* = \mathcal{I} \cap \text{Online}$$ + +Only members of $\mathcal{A}^*$ deliver suffix vectors to trackers and emit scores. A cell transitions from invested to producing when its warm-up completes and it is promoted to online status (§11.6). + +**Relationship to the investment set.** Every member of $\mathcal{A}^*$ is a member of $\mathcal{I}$. The converse does not hold during warm-up: warming cells are in $\mathcal{I}$ but not in $\mathcal{A}^*$. In steady state with no warming cells, $\mathcal{A}^* = \mathcal{I}$. + +**Slot-vacancy invariant.** While any competitive target is warming, the producing competitive set has a corresponding vacancy: + +$$|\mathcal{A}| = |\mathcal{T} \cap \text{Online}| = K - |\mathcal{T} \cap \text{Warming}|$$ + +A warming cell holds a $\mathcal{T}$ slot (resource commitment) without occupying an $\mathcal{A}$ slot (production output). At promotion (§11.6.3, Step 0), the cell transitions to Online, and the next Step 3 recomputation yields $\mathcal{A} = \mathcal{I} \cap \mathcal{T} \cap \text{Online}$ with the promoted cell filling its own formerly vacant slot. No displacement logic is needed: $|\mathcal{A}| \leq K$ is maintained structurally by Step 3's top-$K$ recomputation of $\mathcal{T}$ (§8.1), from which $\mathcal{A}$ is derived by intersection. + +> _Design note (no promotion-time displacement)._ Competitive-set turnover — an existing $\mathcal{T}$ member falling below the top-$K$ boundary — can occur in the same batch as a promotion, but the two events are causally independent. The ejection is driven by Step 2's V-Tree mutations (splits, rebalancing, decay) feeding into Step 3's top-$K$ selection, not by the promotion itself. The Step 3 pseudocode (§9.1) contains no displacement step because none is required: `NewCompetitive ← NewTargets ∩ Online` is capped at $K$ by construction. + +### 8.4 The Root Tracker · `sec:sentinel:algorithm-root-tracker` + +The G-Tree root tracker deserves special attention. It: + +- Sees **every** observation — there is no way to avoid it. +- Operates at width $N$ — the full domain. +- Has the largest training set — learns the most stable model. +- Provides the global reference against which everything is measured. + +The root tracker answers: _"Does this value look like a normal member of the overall population?"_ Every other tracker answers: _"Does this value look normal for its specific region?"_ + +A value can be perfectly normal for its region (low /48 scores) but unusual globally (high /0 scores) — for example, if the entire region is unusual. Or it can be unusual locally (high /48 scores) but unremarkable globally (low /0 scores) — a local anomaly within a normal region. The combination is diagnostic. + +For evasion analysis: the root tracker is the hardest to evade. It has the most data, the most stable subspace, and captures the broadest correlations. An adversary who successfully mimics local distributional patterns at /48 may still produce an unusual projection at /0, because the root model captures global cross-region correlations that no single cell's model can learn. + +The root tracker is **permanent** — created at system initialisation, never destroyed. Its warm-up time is amortised across the system's lifetime. + +### 8.5 Entry and Exit · `sec:sentinel:algorithm-cell-selection-entry-and-exit` + +**Investment entry.** When a cell first appears in $\mathcal{I}$ (either as a competitive target or as an ancestor of one): + +1. Create a subspace tracker (§4) at width $w = N - d$. +2. Enqueue for warm-up (§11.6). + +**Promotion to producing.** When a warming cell's warm-up completes (§11.6.3): + +1. Transition from warming to online. +2. If the cell is a competitive target, it enters $\mathcal{A}$. +3. Coordination contexts involving this cell materialise lazily during the next Step 5 coordination walk (§7.7.1). + +**Investment exit (eager removal).** When a cell leaves $\mathcal{I}$ — because its competitive target dropped out of the top-$K$ and no other target needs it as an ancestor: + +1. **If warming:** cancel warm-up, destroy the tracker immediately. +2. **If online:** destroy the tracker immediately. Stale coordination contexts are pruned by the Step 5 post-walk pass (§7.7.2). + +Destruction is **eager** — it occurs within the Step 3 reconciliation that discovers the exit, not deferred to a later pass. The tracker, its subspace, baselines, drift accumulators, and any in-progress warm-up state are released immediately. + +**The root tracker exception.** The root tracker is never destroyed, even if it is temporarily the only member of $\mathcal{I}$. It is created at system initialisation and persists for the system's lifetime. + +**No hysteresis.** The V-Tree's max-uncle constraint (§3.4) provides structural stability: demotion requires genuine competitive loss, not transient fluctuation. This structural guarantee _is_ the hysteresis. + +**Ancestor stability under sharing.** An ancestor survives in $\mathcal{I}$ as long as **any** competitive target descends from it. Investment churn at the competitive boundary does not destroy shared ancestors. The root's lifetime is the system's lifetime. + +### 8.6 Budget Accounting · `sec:sentinel:algorithm-cell-selection-budget-accounting` + +The budget parameter $K$ governs the **competitive target** selection. Ancestor trackers are not counted against $K$. + +**Investment set size.** $|\mathcal{I}| \leq 1 + \sum_i d_i \leq 1 + \texttt{analysis\_k}(N-2)$ for supported engines, including the permanent root (§8.2). This counts the current selected cell trackers, online or warming. The reduced Steiner tree suppresses unary intermediates that the implementation retains, so its bound cannot size the full investment set; coordination trackers and an evicted in-flight warming model are separate. + +**Producing set size.** $|\mathcal{A}^*| \leq |\mathcal{I}|$, with equality in steady state (no warming cells). + +**Peak memory.** Bounded by $|\mathcal{I}| \times (\text{max per-tracker size})$, since all invested cells — whether warming or online — have allocated trackers. This bound applies during warm-up; in steady state, memory equals $|\mathcal{A}^*| \times (\text{max per-tracker size})$. + +Ancestor trackers at coarser depths are more expensive per unit because they have wider suffixes. Taking $N = 128$ and $\text{cap} = 16$ as an example: + +| Depth | Width $w$ | $U$ matrix elements | Approximate per-tracker size | +| -------- | --------- | ---------------------- | ---------------------------- | +| 0 (root) | $N$ | $128 \times 16 = 2048$ | $\sim 2100$ elements | +| 16 | $N - 16$ | $112 \times 16 = 1792$ | $\sim 1850$ elements | +| 32 | $N - 32$ | $96 \times 16 = 1536$ | $\sim 1600$ elements | +| 48 | $N - 48$ | $80 \times 16 = 1280$ | $\sim 1350$ elements | + +Total additional memory for ancestors is roughly proportional to $K$. See §12 for global resource accounting. + +### 8.7 Tracker Configuration at Different Levels · `sec:sentinel:algorithm-cell-selection-level-specific-tracker-configuration` + +Ancestor trackers use the same maximum rank, forgetting factor, and other parameters as competitive trackers by default. A natural gradient is available via optional per-depth overrides: coarser levels see more observations and broader patterns, suggesting higher rank (more principal components for richer structure) and slower forgetting (more stable baselines for the broader population). The root tracker particularly benefits from higher rank — it learns the global structure of the $N$-bit domain, which is likely higher-dimensional than within-cell structure. + +This is a configuration refinement. The default — same parameters everywhere — is a reasonable starting point. + +### 8.8 Edge Cases · `sec:sentinel:algorithm-cell-selection-edge-cases` + +**Cells receiving no observations in a batch.** An analysed cell may receive zero routed observations in a given batch. Its tracker processes no data and emits no scores. Competitive cells with no observations are excluded from that batch's coordination matrices (§7.2). + +**Internal G-Tree nodes in the competitive targets.** An internal G-Tree node (both children present) receives no observations via normal spatial routing (§3.10) — its importance is frozen from the moment of its second child's creation (§3.7). However, the competitive selection (§8.1) does not filter by G-Tree state: the V-Tree ranking is the sole eligibility criterion. A freshly-split internal node with high historical importance can therefore occupy a competitive target slot. + +This is not a defect. The node's tracker is fully active: the multi-scale delivery mechanism (§9.3) feeds it every observation in its subtree, it scores them, and it participates in coordination (§7). Its analysis is identical to what an ancestor tracker would provide — the cost is one $K$-slot that duplicates the ancestor closure's work. The V-Tree's competitive dynamics (§3.4) resolve this organically: children's importance grows from live observations, the frozen benchmark decays under temporal attenuation (§3.8), the max-uncle constraint triggers restructuring, and the parent sinks below the selection threshold. The transient duration is bounded by $O(\theta / \text{observation\_rate})$ batches under typical decay. + +The alternative — filtering by G-Tree state to eject internal nodes immediately — was considered and rejected. It would create cross-layer coupling (the analysis selector inspecting spatial-layer structural state) and would leave the highest-importance region with no competitive-level coordination representation during the transition. See the design note in §8.1. + +--- + +## Chapter 9. The Observation Algorithm · `sec:sentinel:algorithm-observation-algorithm` + +### 9.1 The Algorithm · `sec:sentinel:algorithm-observation-algorithm-procedure` + +If the ingestion batch is empty ($n = 0$), the observation algorithm returns an empty report without modifying any state. The remainder of this procedure assumes $n \geq 1$. + +``` +procedure SentinelIngest(Batch): + Input: Batch — a sequence of coordinate values of type C + if Batch is empty: + return EmptyReport() + + -- Step 0 — Promote warmed-up cells from staging (§11) + PromoteReadyCells() + + -- Step 1 — Convert to centred bit vectors (§2.3) + Vectors ← ConvertToCentredBits(Batch) + + -- Step 2 — Spatial layer volume accounting (§3.3) + for each value v in Batch: + SpatialLayer.Observe(coordinate = v, Δ = 1) + -- Routes, accumulates, may split/rebalance/evict. + + -- Step 3 — Update investment and producing sets (§8) + OldInvestment ← CurrentInvestmentSet + NewTargets ← ComputeCompetitiveTargets() (§8.1) + NewInvestment ← CloseUnderAncestry(NewTargets) (§8.2) + + -- Eager removal of exiting nodes + Exiting ← OldInvestment \ NewInvestment + for cell in Exiting: + if cell = root: continue (§8.4) + if cell is warming: + CancelWarmUp(cell) + DestroyTracker(cell) (§8.5) + + -- Investment entry for new nodes + Entering ← NewInvestment \ OldInvestment + for cell in Entering: + CreateTracker(cell) + EnqueueForWarmUp(cell) (§11.6) + + -- Derive producing sets from investment + online status + NewCompetitive ← NewTargets ∩ Online + NewFull ← NewInvestment ∩ Online + + -- No explicit coordination activation/deactivation here. + -- Coordination context lifecycle is managed lazily in Step 5. + + CurrentInvestmentSet ← NewInvestment + CurrentCompetitiveTargets ← NewTargets + CurrentCompetitiveSet ← NewCompetitive + CurrentFullAnalysisSet ← NewFull + + -- Step 4 — Deliver suffix vectors and score (§9.3, §9.4) + CellReports ← empty list + AncestorReports ← empty list + for each value v in Batch: + Receiver ← RouteToOnlineReceiver(v) (§9.3) + for each node g on the path from Receiver to root: + if g is in CurrentFullAnalysisSet: + Suffix ← Vectors[v] restricted to positions [g.depth .. N−1] + append Suffix to g.PendingBatch + + for each cell in CurrentFullAnalysisSet: + if cell.PendingBatch is empty: continue + X ← assemble matrix from cell.PendingBatch + Report ← cell.Tracker.Observe(X) (§4.2) + if cell is in CurrentCompetitiveSet: + append (cell, Report) to CellReports + else: + append (cell, Report) to AncestorReports + clear cell.PendingBatch + + -- Step 5 — Hierarchical coordination (§7.4) + -- Coordination contexts are lazily created and inline-warmed + -- during the walk; stale contexts are pruned afterward (§7.7). + CoordinationReports ← PropagateCoordination(root, CellReports) + PruneStaleCoordinationContexts() (§7.7.2) + + -- Step 6 — Assemble report (§14) + return AssembleReport(CellReports, AncestorReports, + CoordinationReports) +``` + +> **Per-cell batch size.** For each cell $c$ in the producing set $\mathcal{A}^*$, the per-tracker batch size $b$ equals the number of ingestion-batch observations whose spatial routing passes through $c$. At the root, $b = n$; at non-root cells, $b \in \{0, \ldots, n\}$ depends on the spatial distribution of the current batch. Cells with $b = 0$ accumulate no pending observations and are skipped by the `if cell.PendingBatch is empty: continue` guard in Step 4. The effective per-batch work is therefore $\sum_{c \in \mathcal{A}^* :\, b_c > 0} O\!\big(\min(w_c,\, k_c + b_c)^2 \cdot \max(w_c,\, k_c + b_c)\big)$, which may be substantially less than the worst-case bound in §12.7 when observations are spatially concentrated. + +### 9.2 Step Ordering · `sec:sentinel:algorithm-observation-step-ordering` + +| Ordering | Reason | +| --------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Step 0 before Step 2 | Newly promoted cells must participate in observation routing. | +| Step 2 before Step 3 | Splits and rebalancing change V-Tree ranking. The analysis set is computed against the post-observation state. | +| Step 3: investment set reconciled | Exiting nodes eagerly removed. Entering nodes enqueued for warm-up (§11). No inline per-cell warm-up work; no coordination lifecycle management. | +| Step 4 before Step 5 | Suffix vectors must be collected and scored before coordination. | +| Step 5: coordination lifecycle | Coordination contexts are lazily materialised and inline-warmed during the coordination walk (§11.7). Stale contexts are pruned after the walk. Inline warm-up is justified by the trivial cost at $w = 4$ (~50 rounds of $O(16m)$ work). | +| Step 5 before Step 6 | Coordination requires per-cell score summaries. | + +### 9.3 Multi-Scale Delivery · `sec:sentinel:algorithm-observation-multi-scale-delivery` + +A single observation is delivered to **every tracker on its G-Tree ancestor path**, from the receiving cell up to and including the root. This delivery is **mandatory**, not contingent on competitive selection — the ancestor closure (§8.2) guarantees that every node on the path hosts a tracker. + +If a value routes to a depth-4 cell with materialised ancestors at depths 0, 1, 2, and 3, all five trackers receive the observation — at analysis widths $N$, $N-1$, $N-2$, $N-3$, and $N-4$ respectively. Each ancestor covers twice the dyadic range of its child, providing a complete hierarchy of spatial context from the cell's local population to the global population in $d$ logarithmic steps. The delivery occurs regardless of whether the intermediate G-Tree nodes receive observations through normal spatial layer routing. Under typical parameters (`depth_create` $\approx 3$, competitive mechanism limiting depth to $\sim 2$–$8$), this fan-out is modest: a depth-3 cell produces 4 tracker updates per observation; a depth-6 cell produces 7. + +**Routing during warm-up.** Observations destined for a cell in $\mathcal{I}$ that is not yet online are routed to the nearest **online** ancestor on the G-Tree path, or to the root if no closer ancestor is online. The root is always online (§8.4). Because the warm-up pipeline processes cells in g.sum order (§11.6.2), ancestors come online before descendants — the routing fallback naturally shortens as warming progresses. + +``` +procedure RouteToOnlineReceiver(value): + g ← SpatialLayer.RouteToReceiver(value) + while g is not null: + if g is online: + return g + g ← g.SpatialParent +``` + +Because the root completes warm-up at construction (§11.6), the while loop has a guaranteed termination point: the root is always online. For cells whose closer ancestors are still warming, the root is returned; as ancestors complete warm-up (in g.sum order, §11.6.2), the returned node moves progressively closer to the target. + +No scoring degradation occurs during this period — the ancestor already has a mature model. Resolution stays at the coarser level until the descendant completes warm-up and is promoted to online status. The host sees the descendant's spatial interval covered by the ancestor's report until promotion. + +> _Design note (overlapping inputs)._ Observation sets overlap by construction — a depth-16 ancestor sees every suffix that its depth-32 children see, plus more. The overlapping input is by design: the depth-16 tracker learns coarser structure than the depth-32 tracker from the same observations. The coordination tier (§7) operates on derived score summaries from the competitive set $\mathcal{A}$, not from ancestor trackers, so input overlap does not produce double-counting at the coordination level. + +### 9.4 Per-Value Scoring Pipeline · `sec:sentinel:algorithm-observation-per-value-scoring-pipeline` + +Under the ancestor closure, a single observation at a competitive cell of depth $d$ produces scores at every ancestor level: + +``` +Value arrives at depth-4 cell +│ +├── /0 tracker: novelty₀, displacement₀, surprise₀, coherence₀ +│ └── z-scores against /0 baselines +│ +├── /1 tracker: novelty₁, displacement₁, surprise₁, coherence₁ +│ └── z-scores against /1 baselines +│ +├── /2 tracker: novelty₂, displacement₂, surprise₂, coherence₂ +│ └── z-scores against /2 baselines +│ +├── /3 tracker: novelty₃, displacement₃, surprise₃, coherence₃ +│ └── z-scores against /3 baselines +│ +├── /4 tracker: novelty₄, displacement₄, surprise₄, coherence₄ +│ └── z-scores against /4 baselines +│ +└── Coordination (on competitive cells only) + └── Cross-cell pattern detection (§7) +``` + +The report for a single value includes scores at every ancestor level. The host sees not just "this value is anomalous" but "this value is anomalous at the depth-3 level but normal at depth 1 and depth 4" — diagnostic information about the **scale** at which the anomaly manifests. + +- An anomaly at depth 4 only: unusual relative to the specific cell's population but normal in the broader context — a local anomaly. +- An anomaly at depth 1 and above: unusual in the broader regional context — a stronger, coarser-scale signal. +- An evasion succeeding at depth 4 but failing at depth 3: the adversary matched local patterns but missed cross-cell correlations captured by the coarser model. This is the nesting defence. + +### 9.5 Batched Observation Semantics · `sec:sentinel:algorithm-observation-batched-semantics` + +Step 2 accumulates $\Delta = 1$ per value. If $n$ values in a batch route to the same cell, its importance increases by $n$. Steps 2 and 4 may share a single tree descent in implementation. + +--- + +# Part IV — Temporal and Operational Concerns · `sec:sentinel:algorithm-temporal-and-operational-concerns` + +--- + +## Chapter 10. Temporal Semantics · `sec:sentinel:algorithm-temporal-semantics` + +Two independent temporal mechanisms govern two independent concerns. Neither subsumes the other; both operate simultaneously. + +### 10.1 Spatial Decay · `sec:sentinel:algorithm-temporal-spatial-decay` + +The host periodically invokes the spatial layer's temporal decay operation (§3.6). The spatial layer applies the requested scaling factor to accumulated importance values, producing one of four effects: + +| Regime | Factor | Effect | +| ------------- | ---------------------- | ---------------------------------------------------------------------------------------- | +| Attenuation | $(0, 1)$ | Cold cells lose standing, eventually becoming eviction candidates | +| Amplification | $> 1$ | Hot cells reinforced; with depth-selective amplification, fine-scale detail is sharpened | +| Annihilation | $= 0$ | Hard reset — entire regions zeroed | +| Detail flush | $= 0$, depth-selective | Subtree root preserved; all descendants zeroed | + +The host controls the decay schedule: when to invoke it, at what factor, against which subtree, and with what depth selectivity. Spatial decay governs the **contour's memory** — which cells exist and in what competitive standing. + +### 10.2 Statistical Decay · `sec:sentinel:algorithm-temporal-statistical-decay` + +Each subspace tracker's EWMA at rate $\lambda$ governs the subspace, latent statistics, and baselines independently of spatial decay. Statistical decay governs the **model's memory** — what structure has been learned and how recent observations are weighted. + +| What decays | Mechanism | Rate | Purpose | +| -------------------- | --------------------------- | ----------- | --------------------------- | +| Importance (spatial) | Spatial layer | Host-chosen | Contour evolution | +| Subspace $\sigma$ | Tracker SVD (§4.2, Phase 2) | $\lambda$ | Forget old correlations | +| Fast baselines | Tracker EWMA (§6.1) | $\lambda$ | Adapt to score distribution | +| Slow baselines | Tracker EWMA (§6.2) | $\lambda_s$ | Long-memory drift reference | + +A cell can retain its spatial position (strong volume, slow spatial decay) while its statistical model adapts rapidly (fast $\lambda$), and vice versa. + +### 10.3 Independence · `sec:sentinel:algorithm-temporal-independence` + +The two decay mechanisms are entirely orthogonal: + +- Spatial decay does not touch tracker state. It zeroes importance, which causes cells to lose V-Tree standing, exit the competitive set, and have their trackers destroyed through the normal exit mechanism (§8.5). The feed-forward invariant (§1.3) is maintained. +- Statistical decay does not touch importance. It governs how quickly a tracker forgets old subspace structure and baseline values. + +The host sets the temporal policy for both mechanisms independently. Fast spatial decay with slow statistical decay produces a rapidly evolving contour with stable models. Slow spatial decay with fast statistical decay produces a stable contour with rapidly adapting models. + +### 10.4 Annihilation Use Cases · `sec:sentinel:algorithm-temporal-annihilation-use-cases` + +- **Regime change.** Apply annihilation at the G-Tree root with zero depth selectivity: resets the entire spatial structure. All cells lose standing and are eventually evicted; their trackers are destroyed. +- **Suspected poisoning.** Apply annihilation at a subtree root with full depth selectivity (detail flush): preserves coarse measurement, zeroes fine structure. The subtree root retains its importance; descendants must re-earn theirs. +- **Stale cleanup.** Apply targeted annihilation to force eviction in the next tidal pass. Useful for host-directed pruning of known-dead regions. + +--- + +## Chapter 11. System Initialisation and Warm-Up · `sec:sentinel:algorithm-system-initialisation-and-warmup` + +A newly created tracker has an orthonormal basis (§4.1) and uninitialised baselines. Before it can produce meaningful scores, its baselines must reflect a representative score distribution. This chapter specifies the complete warm-up procedure: noise injection, cold-start seeding, clip-width modulation, slow-baseline seeding, deferred scheduling, and coordination warm-up. These mechanisms address different aspects of a single problem — cold-start mismatch — and are presented together because their interactions determine the system's convergence behaviour. + +### 11.1 Noise Injection · `sec:sentinel:algorithm-noise-injection` + +#### 11.1.1 Purpose · `sec:sentinel:algorithm-noise-injection-purpose` + +Noise injection feeds synthetic uniformly random centred bit vectors through the tracker's standard observation path (§4.2) to **warm the EWMA baselines** so they reflect structureless-noise score distributions rather than uninitialised placeholders. + +> _Design note._ Noise injection's primary value is baseline warming, not subspace shaping. Random bit vectors in $\{-0.5, +0.5\}^w$ have no dominant direction — after injection, singular values are roughly equal, similar to the initial state. The subspace acquires meaningful structure only once real observations arrive. + +#### 11.1.2 Trigger Events · `sec:sentinel:algorithm-noise-injection-trigger-events` + +Noise injection fires on every new tracker creation: competitive set entry, ancestor activation, split-induced creation, or legacy promotion. + +#### 11.1.3 Parameters · `sec:sentinel:algorithm-noise-injection-parameters` + +| Parameter | Description | Default | Constraint | +| ---------------- | ------------------------------------ | ------------------------------------------------------------ | -------------------------------------------- | +| Noise schedule | Depth-tiered synthetic batch count | Geometric: 450 rounds at root, halving per depth, minimum 50 | See below | +| Noise batch size | Samples per synthetic batch | 16 | $\geq 2$ recommended; $\geq 1$ required | +| Random seed | Seed for the pseudo-random generator | Deterministic (fixed seed) | Optional; absent means implementation-chosen | + +> _Operational note (noise batch size)._ Under the EWMA-mean-centred variance formula (§4.2 Phase 3, [ADR-S-021](../adr/021-ewma-mean-centred-variance.md)), the first-batch seeding computes $\nu^{(z)}_j = \max(z_{1j}^2, \varepsilon)$, which equals approximately $0.25$ in expectation for centred $\pm 0.5$ inputs — correctly scaled at every batch size including $b = 1$. The previous formula computed within-batch population variance, which is identically zero at $b = 1$; the EWMA-mean-centred formula eliminated this entire class of batch-size bias. Smaller batch sizes produce noisier initial seeds (higher variance of $z_{1j}^2$ across dimensions), but the EWMA smooths this within $O(t_{1/2})$ subsequent batches without a recovery phase. The default of 16 remains well above the threshold for low-noise seeding. + +The noise schedule determines how many synthetic batches each tracker receives before transitioning to real observations. Two schedule forms are supported: + +- **Geometric.** Parameterised by a root count, a per-depth decay factor, and a minimum: $\text{rounds}(d) = \max(\text{minimum}, \; \lfloor\text{root} \times \text{decay}^d + 0.5\rfloor)$ +- **Explicit.** A sequence of per-depth round counts; the last entry repeats for all deeper levels. + +The schedule must produce enough rounds for worst-case baseline convergence at each depth (see §11.9 for empirical convergence data; the same schedule parameterises coordination warm-up in §11.7). + +After injection completes, all four drift accumulators (§6.3) are reset to zero. + +### 11.2 Cold-Start Latent Seeding · `sec:sentinel:algorithm-cold-start-latent-seeding` + +On a tracker's very first batch (Phase 3 of the core loop, §4.2), the latent statistics $\mu^{(z)}$, $\nu^{(z)}$, and $\Gamma$ are seeded directly from the batch data rather than blended with default initial values. + +Without direct seeding, the latent variance $\nu^{(z)}$ retains its pre-allocated value of $1.0$. The EWMA-mean-centred formula (§4.2, Phase 3) converges to the true variance from any initialisation, but the $4\times$ overestimate dampens surprise scores during the convergence period, delaying baseline settling. Direct seeding at $t = 0$ eliminates this delay: $\nu^{(z)}_j \leftarrow \max(\frac{1}{b}\sum z_{ij}^2, \varepsilon)$ produces a correctly-scaled seed at every batch size, including $b = 1$. + +The procedure is specified in §4.2, Phase 3. + +### 11.3 Clip-Width Modulation During Warm-Up · `sec:sentinel:algorithm-warmup-clip-width-modulation` + +Clip-width modulation during warm-up is handled by the unified clip-pressure mechanism (§6.1.1) via the noise influence signal $\eta$. The effective clip ceiling uses $p = \max(\eta, \bar{\rho})$ where $\eta$ is the noise influence (§11.5) and $\bar{\rho}$ is the per-axis clip-pressure EWMA: + +$$n_\sigma^{\text{eff}} = n_\sigma\left(1 + \frac{p}{1 - p + \varepsilon}\right)$$ + +During warm-up, $\eta \approx 1$ dominates and the ceiling is effectively open — preventing the positive feedback loop between tight clipping and low baselines that would otherwise extend convergence time by an order of magnitude. As $\eta$ decays with real-observation batches, $\bar{\rho}$ takes over if the baselines become stale in production. The full mechanism, including dynamics tables and contamination analysis, is specified in §6.1.1 and §6.4. + +### 11.4 Slow-From-Fast CUSUM Seeding · `sec:sentinel:algorithm-warmup-slow-from-fast-cusum-seeding` + +After noise injection completes, the slow EWMA (§6.2) is **seeded from the fast EWMA's converged values** and the drift accumulator (§6.3) is **reset to zero**. + +The slow EWMA's time constant ($\lambda_s$, e.g., 0.999 with half-life $\approx 693$ steps) is far longer than the fast EWMA's ($\lambda$, e.g., 0.99 with half-life $\approx 69$ steps). Without seeding, the fast baseline reaches steady state much sooner than the slow baseline. During this gap, the drift accumulator interprets the fast-slow difference as drift evidence, producing false accumulation. + +Seeding the slow EWMA mean and variance from the fast EWMA at the noise-to-real transition eliminates this gap at its source. The procedure is: + +1. Copy the fast EWMA's mean and variance to the slow EWMA's mean and variance. +2. Reset all four drift accumulators to zero. +3. Reset all four clip-pressure EWMAs $\bar{\rho}$ to zero (§6.1.1). + +### 11.5 Maturity Tracking · `sec:sentinel:algorithm-maturity-tracking` + +Each tracker records: + +| Field | Description | +| ---------------------- | ----------------------------------------------------- | +| Real observations | Count of genuine observations processed | +| Noise observations | Count of synthetic observations processed | +| Noise influence $\eta$ | Fraction of baseline not yet established by real data | + +The noise influence advances once per tracker batch, at the same forgetting cadence as the SVD, latent statistics, and score baselines. Observation counters still advance by the number of rows in the batch: + +$$\eta_{t+1} = \begin{cases} \lambda \, \eta_t + (1 - \lambda) & \text{noise batch} \\ \lambda \, \eta_t & \text{real batch} \end{cases}$$ + +After $k$ real batches: $\eta_k = \lambda^k \eta_0$, independent of batch size. The system reports $\eta$ without interpretation. The host should treat scores from trackers with high $\eta$ (e.g., $> 0.5$) as preliminary. + +**Initial value.** $\eta_0 = 1.0$ at tracker creation, indicating that no real batch has yet displaced the warm-up state. Noise injection keeps $\eta \approx 1.0$; after $k$ real batches, $\eta_k \approx \lambda^k$. + +#### 11.5.1 Three Convergence Concepts · `sec:sentinel:algorithm-maturity-tracking-convergence-concepts` + +The system has three distinct notions of convergence, operating at different timescales: + +| Convergence type | Definition | Timescale | Notes | +| ------------------------------------------------------- | --------------------------------------------- | -------------------------------------------- | --------------------------------------------------------------------------- | +| **$\eta$-convergence** (maturity) | $\eta_k = \lambda^k < \epsilon$ | $\lceil\ln\epsilon/\ln\lambda\rceil$ batches | Exact closed-form | +| **Trajectory convergence** (EWMA mean) | $\|\bar{\mu}_n - \bar{\mu}_\infty\| < \delta$ | Tens to hundreds of rounds | With clip-pressure modulation (§6.1.1, §6.4) and cold-start seeding (§11.2) | +| **Distributional stationarity** (baseline distribution) | Windowed-mean comparison $< \epsilon$ | Per-axis; varies by batch size | Baselines wander even at true steady state | + +$\eta$-convergence is a **necessary but not sufficient** indicator of system readiness. EWMA baselines follow the same per-batch decay but their observed convergence also depends on cascaded interactions, clipping policy, batch size, and axis-specific score distributions; $\eta_k = \lambda^k$ measures forgetting cadence rather than distributional stationarity. + +### 11.6 Deferred Cell Warm-Up · `sec:sentinel:algorithm-deferred-cell-warmup` + +Noise injection is performed **outside** the main observation path. New trackers transition through a three-state lifecycle: + +``` +┌─────────┐ ┌─────────┐ ┌─────────┐ +│ Created │────────►│ Warming │────────►│ Online │ +└─────────┘ └─────────┘ └─────────┘ + │ + │ asynchronous warm-up + │ volume-priority scheduling + ▼ + noise injection +``` + +#### 11.6.1 Cell States · `sec:sentinel:algorithm-deferred-cell-warmup-states` + +| State | Receives real observations? | Contributes to reports? | Warm-up status | +| ------- | --------------------------- | ----------------------- | --------------- | +| Created | No | No | Not yet started | +| Warming | No | No | In progress | +| Online | Yes | Yes | Complete | + +**Root warm-up exception.** The root tracker (§8.4) completes its warm-up synchronously during system construction, before the Sentinel accepts its first `ingest()` call. This ensures the "always online" invariant (§9.3) holds from the first observation. The root is the highest-priority cell under g.sum ordering and the only cell that exists at construction time; warming it synchronously is a one-time fixed cost that does not violate the work-variance bound (§12.9), which governs per-`ingest()` cost. All subsequently created cells follow the deferred warm-up lifecycle described below. + +#### 11.6.2 Priority Scheduling · `sec:sentinel:algorithm-deferred-cell-warmup-priority-scheduling` + +The warm-up pipeline always works on whichever member of the investment set's warming subset is greatest under the priority key, compared lexicographically: + +$$\text{priority}(c) = \big(\text{g.sum}(c.\text{node}),\; -\text{depth}(c.\text{node}),\; -\text{id}(c.\text{node})\big)$$ + +where g.sum is the G-Tree node sum (§3.12 property 2) — the node's own accumulation plus all descendant sums — and the two negated components select, among cells of equal g.sum, first the shallowest and then the one with the smallest node identifier. Volume decides every pair whose sums differ; the lower two components exist for the pairs whose sums are equal, and consequence 2 gives the reason each is needed. + +This ordering has five consequences: + +1. **Volume drives priority.** A higher g.sum node has more observation traffic flowing through its spatial region — bringing it online has the greatest impact on analysis quality. + +2. **Ancestors emerge first.** By the summation invariant (§3.12 property 2), every ancestor's g.sum is at least as great as any descendant's, so the volume component already orders correctly every pair whose sums differ. It does not order the pairs whose sums are equal, and on an ancestor chain those are the ordinary case rather than a corner: a path node whose accumulation is entirely the single warming cell below it has exactly that cell's sum, and an implementation holding a floating-point approximation of the node sum ties over a wider set still. The depth component resolves each such tie toward the shallower cell, which is what makes the ordering a topological sort of the ancestor chains rather than merely consistent with one. The identifier component is a final deterministic tie-break between cells of equal depth, and it cannot carry the ancestor rule in depth's place: node identifiers are drawn from an arena that reuses freed slots, so a cell created into a recycled slot may hold a smaller identifier than an ancestor allocated before it, and an identifier tie-break alone would warm that descendant first. + +3. **Full chain at promotion.** Because ancestors come online before descendants, when a competitive target completes warm-up, its entire ancestor chain is already online. The multi-scale defence (§16) is complete from the first batch the target processes. No secondary warm-up gap exists. + +4. **No starvation.** Every warming cell in an active region accumulates positive g.sum — the observations that justified its investment continue flowing through its spatial range. Cells that lose priority temporarily are eventually reached. + +5. **Preemption is correct.** A cell that was highest-priority at one time may not be later — observation patterns shift, investment set membership changes. Working on the current highest-g.sum cell ensures the most valuable work is always done first. Partial warm-up is never wasted: the preempted cell's EWMA baselines retain their progress. + +#### 11.6.3 Promotion · `sec:sentinel:algorithm-deferred-cell-warmup-promotion` + +At the start of each observation cycle (§9.1, Step 0), before spatial layer observation routing: + +``` +procedure PromoteReadyCells(): + for each cell in the warming set where warm-up is complete: + move cell from warming set to online set +``` + +This typically promotes zero or one cell per cycle. Promotion occurs atomically from the observation algorithm's perspective, before any observation routing. Coordination contexts involving promoted cells materialise lazily during the Step 5 coordination walk (§7.7.1), not at promotion time. + +#### 11.6.4 Observation Routing During Warm-Up · `sec:sentinel:algorithm-deferred-cell-warmup-observation-routing` + +Observations destined for a warming cell are routed to the nearest online ancestor, as specified in §9.3. The spatial layer's own volume tracking is unaffected: the observation accumulates importance at the warming cell's spatial node regardless of whether an analysis tracker exists there. The warming cell's competitive position in the V-Tree is maintained. It holds an investment slot in $\mathcal{I}$ and transitions to the producing set $\mathcal{A}$ upon promotion. + +#### 11.6.5 Coordination During Warm-Up · `sec:sentinel:algorithm-deferred-cell-warmup-coordination` + +Warming cells do **not** participate in coordination. The firing tree contains only online competitive cells with scores, and the retention tree contains only online competitive cells from the producing set; warming cells belong to neither because they are not yet online. Their synthetic noise is never propagated through the coordination hierarchy, avoiding synthetic-on-synthetic artefacts during system formation. + +#### 11.6.6 Eviction Before Completion · `sec:sentinel:algorithm-deferred-cell-warmup-precompletion-eviction` + +A warming cell can be evicted from the spatial layer before warm-up completes — its spatial node may be absorbed by rebalancing or budget eviction. If this happens, the warm-up process discards the partially warmed cell. The work is lost, but correctly so: the spatial layer determined that the cell's interval no longer warrants a dedicated node. + +Similarly, a warming cell can exit the investment set before warm-up completes — its competitive target may drop out of the top-$K$ (§8.5). The warm-up is cancelled and the tracker destroyed via the eager removal mechanism. The work is correctly discarded: the competitive landscape has determined the cell no longer warrants investment. + +#### 11.6.7 Investment Slots vs. Production Slots · `sec:sentinel:algorithm-deferred-cell-warmup-investment-vs-production-slots` + +Warming cells hold **investment slots** in $\mathcal{I}$ — they have allocated trackers and are enqueued for warm-up. They do not hold **production slots** in $\mathcal{A}$ — they do not produce scores, emit reports, or participate in coordination. + +By the slot-vacancy invariant (§8.3), $|\mathcal{A}| = K - |\mathcal{T} \cap \text{Warming}|$: every warming competitive target corresponds to a vacant production slot. At promotion, the cell fills its own vacancy — no online cell is displaced. The $|\mathcal{A}| \leq K$ bound is maintained structurally by Step 3's top-$K$ recomputation (§8.1), not by promotion-time ejection. + +The warm-up pipeline's g.sum ordering (§11.6.2) ensures that ancestors come online before descendants. By the time a competitive target is promoted, its entire ancestor chain is online and producing scores. The system provides progressively deeper coverage as warm-up progresses, rather than a single transition from no coverage to full coverage. + +#### 11.6.8 Pipeline Advancement · `sec:sentinel:algorithm-deferred-cell-warmup-pipeline-advancement` + +The warm-up pipeline advances at a granularity of **one noise batch per scheduling quantum**. After each quantum, the pipeline re-evaluates the g.sum priority ordering and continues with the highest-priority warming cell. This re-evaluation enables preemption: a newly enqueued cell with higher g.sum displaces an in-progress cell after at most one noise batch of delay. + +The pipeline advances **independently of observation cadence** — warm-up throughput is not gated by the rate of `ingest()` calls. Promotion to Online status remains gated by the observation algorithm: it occurs at Step 0 of the next `ingest()` call after warm-up completes (§9.1). + +**Mutual-exclusion invariant.** The warm-up pipeline and the observation algorithm never operate on the same tracker concurrently. Newly created trackers are fully initialised before becoming visible to the pipeline. Tracker destruction (§8.5) waits for (or cancels) any in-progress noise batch before proceeding. The implementation is free to achieve this invariant through any mechanism — a dedicated thread with synchronised handoff, a cooperative executor, or synchronous drain — provided the four properties above hold. + +### 11.7 Coordination-Specific Warm-Up · `sec:sentinel:algorithm-coordination-specific-warmup` + +When a coordination context materialises during the Step 5 coordination walk (§7.7.1) — for example, when two previously unrelated competitive cells first share an ancestor — the context's coordination tracker requires warm-up before processing its first real observation. This warm-up runs **inline to completion** within the same Step 5 invocation, immediately after context creation and before the walk feeds the first real group matrix. + +#### 11.7.1 Scheduling Model · `sec:sentinel:algorithm-coordination-warmup-scheduling-model` + +Coordination warm-up is **inline at materialisation**, not deferred through a parallel warming pipeline. This contrasts with per-cell warm-up (§11.6), which is deferred with g.sum priority scheduling. The difference is justified by cost: + +- **Per-cell warm-up** operates at analysis widths $w$ up to $N = 128$, requiring hundreds of rounds of full SVD updates. The cost motivates deferral and priority scheduling. +- **Coordination warm-up** operates at $w = 4$ with ~50 rounds of $O(16m)$ work — microseconds on any real hardware, less than 0.3% of a single per-cell SVD (§7.8). The cost does not justify a second scheduling tier. + +The "no inline warm-up work" principle stated in §9.2 for Step 3 applies to per-cell tracker warm-up. Coordination warm-up is a separate concern, handled in Step 5, where the trivial cost makes inline execution the simpler and more correct choice. + +#### 11.7.2 Procedure · `sec:sentinel:algorithm-coordination-warmup-procedure` + +1. Snapshot each contributing cell's current baseline mean $\bar{s}_c$ and variance $\bar{v}_c$ for all four scoring axes. The snapshot is taken at Step 5 time — after Step 4 scoring has updated the contributing cells' baselines with this cycle's observations. + +2. For each synthetic round: for each cell $c$ in the coordination group, generate a synthetic score vector. For each scoring axis, sample from a non-negative distribution matching $c$'s snapshot baselines. A Gamma distribution with shape $\alpha_c = \bar{s}_c^2 / \bar{v}_c$ and rate $\beta_c = \bar{s}_c / \bar{v}_c$ is the natural choice: it preserves the learned baseline statistics while respecting non-negativity. + +3. Assemble the synthetic score vectors into a group matrix, centre it (§7.3), and feed it through the coordination tracker (§7.5). + +4. Repeat for the number of rounds prescribed by the noise schedule (§11.1.3) at the coordination context's G-Tree depth — that is, the depth of the internal G-Tree node that hosts this context, using the same depth parameter $d$ that governs per-cell warm-up. + +5. Reset the coordination tracker's drift accumulators to zero. + +If a cell's baseline is uninitialised (no valid mean or variance), use per-axis defaults: mean $= 0.25$, variance $= 0.01$ for novelty and surprise; mean $= 0.1$, variance $= 0.01$ for displacement and coherence. + +> _Design note (why a non-negative distribution)._ All four scoring axes produce non-negative values. A symmetric distribution centred at a small positive mean produces negative samples, which are meaningless in this context. The Gamma distribution with matched moments is the simplest non-negative distribution that preserves the learned baseline statistics. + +> _Design note (snapshot timing)._ The baseline snapshot is taken after Step 4 scoring, meaning baselines reflect this cycle's observations. The alternative — snapshotting before Step 4 — would require coordination lifecycle management in Step 0 or Step 3, adding bookkeeping complexity for a negligible difference (one batch of EWMA update at rate $\alpha \approx 0.01$). The post-Step-4 snapshot is arguably preferable: the baselines are more current. + +> _Design note (depth taper for coordination)._ The noise schedule's depth-based tapering was designed for per-cell trackers, where deeper G-Tree nodes have narrower analysis widths and thus faster baseline convergence. Coordination trackers all operate at $w = 4$ regardless of their G-Tree depth, so the convergence time is depth-independent — the taper rationale does not transfer. Under the default schedule, every coordination context receives the schedule minimum (50 rounds), since even the shallowest possible context at depth 1 converges in well under 225 rounds at $w = 4$. The shared schedule is retained for simplicity; implementations may use a flat 50-round schedule for coordination warm-up without observable difference. + +### 11.8 System-Level Warm-Up Stages · `sec:sentinel:algorithm-system-warmup-stages` + +During early operation, the system-level behaviour differs from steady state: + +**Stage 1: Pre-Split.** Below the split threshold ($< \theta$ total observations). A single spatial node covers the entire domain. The root tracker at $w = N$ is always present (permanent). No competitive targets yet; the investment set contains only the root. No coordination possible. All scores reflect global structure only. + +**Stage 2: Spatial Formation.** From $\theta$ to approximately $10\theta$ observations. The spatial tree begins splitting. Cells enter and exit the competitive targets frequently as the V-Tree ranking stabilises. Investment set membership churns as targets shift. The producing set grows progressively as warm-up completes for each invested cell — ancestors first, then competitive targets. Scores from early-promoted ancestors are available before any competitive cell comes online. + +**Stage 3: Stabilisation.** From approximately $10\theta$ to $100\theta$ observations. The competitive targets settle. The investment set stabilises and the producing set converges toward $\mathcal{I}$ as remaining warming cells complete their warm-up. Trackers mature ($\eta$ declining). Ancestor chains lengthen as the spatial tree deepens. Coordination contexts materialise as competitive target pairs come online (§7.7.1). Drift accumulators begin accumulating meaningful evidence. + +**Stage 4: Steady State.** Trackers have $\eta \ll 1$. Drift accumulators reflect genuine baseline departures. Ancestor chains provide multi-scale defence. The system detects anomalies at its designed sensitivity. + +The transition timescale depends on observation concentration, split threshold $\theta$, and forgetting factor $\lambda$. + +### 11.9 Empirical Baseline Convergence · `sec:sentinel:algorithm-empirical-baseline-convergence` + +The following table gives measured EWMA trajectory convergence times — the number of batches until the windowed-mean baseline is within tolerance of its steady-state value. These were obtained with clip-pressure modulation (§6.1.1), cold-start latent seeding (§11.2), and slow-from-fast drift-accumulator seeding (§11.4) all active. + +| Component | $\lambda{=}0.95$, $b_{\text{noise}}{=}4$ | $\lambda{=}0.95$, $b_{\text{noise}}{=}16$ | $\lambda{=}0.99$, $b_{\text{noise}}{=}16$ | +| ------------------------ | ---------------------------------------: | ----------------------------------------: | ----------------------------------------: | +| $\eta$ (noise influence) | 59 (exact) | 59 (exact) | 299 (exact) | +| Latent mean/variance | $\sim 1$ (seeded) | $\sim 1$ (seeded) | $\sim 1$ (seeded) | +| Novelty baseline | 21 | 21 | 101 | +| Displacement baseline | 21–475 (bimodal) | 21 | 101 | +| Surprise baseline | **65** | **70** | **398** | +| Coherence baseline | **406** | **173** | **315** | +| **System (worst-case)** | **$\sim$406** | **$\sim$173** | **$\sim$398** | + +Per-axis empirical coefficients of variation at steady state (windowed-mean): + +| Axis | CV ($b_{\text{noise}}{=}4$) | CV ($b_{\text{noise}}{=}16$) | Notes | +| ------------ | :-------------------------: | :--------------------------: | --------------------------------------------------------- | +| Novelty | 0.13% | 0.06% | Constant-norm property — inherently stable | +| Displacement | 5.9% | 3.0% | Bimodal across random seeds at $b_{\text{noise}}{=}4$ | +| Surprise | 9.0% | 3.6% | Improved substantially by cold-start seeding | +| Coherence | 19.9% | 11.5% | Rank-gating delay ($k < 2$) is the convergence bottleneck | + +**The noise schedule must produce enough rounds for the worst-case convergence time** at each depth for baselines to be settled before real observations arrive. At $\lambda = 0.99$, $b_{\text{noise}} = 16$ (a typical production configuration), this is $\geq 400$ batches at depth 0 (root). Deeper cells need fewer rounds — the geometric taper reflects the faster convergence at narrower analysis widths. + +--- + +## Chapter 12. Resource Bounds, Spray Resistance, and Complexity · `sec:sentinel:algorithm-resource-bounds-spray-resistance-and-complexity` + +### 12.1 Three Independent Ceilings · `sec:sentinel:algorithm-resource-independent-ceilings` + +| Ceiling | Parameter | Bounds | Layer | +| ---------- | ------------------------- | -------------------- | ----------------- | +| $G_{\max}$ | Hard spatial node ceiling | Total spatial nodes | Spatial Layer | +| $K$ | Analysis budget | Competitive trackers | Analysis Selector | +| $\|\mathcal{I}\|$ | (derived) | $\leq 1 + K\bar{D}$ before sharing; dominated by $2K$ in practice (§8.2) | Analysis Selector | +| $\|\mathcal{A}^\*\|$ | (derived) | $\leq \|\mathcal{I}\|$; equals $\|\mathcal{I}\|$ in steady state | Analysis Selector | + +The budget $K$ governs competitive selection; ancestor trackers are not counted against the budget. Total invested tracker count $|\mathcal{I}|$ is bounded by $1 + K\bar{D}$ before sharing (§8.2), where $\bar{D}$ is the average G-Tree depth of competitive targets. Because the competitive mechanism and depth gate keep trees shallow (§3.1), $\bar{D}$ is typically 2–6, and ancestor sharing under concentration brings the practical size close to $2K$. Peak memory is determined by $|\mathcal{I}|$ (including warming trackers); per-batch computation is determined by $|\mathcal{A}^*|$ (online trackers only). Total memory: $G_{\max} \times (\text{per-node size}) + |\mathcal{I}| \times (\text{max per-tracker size})$. Both terms are hard-bounded. + +### 12.2 Spray Resistance · `sec:sentinel:algorithm-resource-spray-resistance` + +A **spray** is a high-entropy input pattern: an adversary (or a degenerate data source) emitting maximally diverse values, forcing the system to allocate structure across the full domain rather than concentrating precision on structured regions. The spray is an entropy attack on the spatial partition budget — it attempts to exhaust the spatial layer's node ceiling through diversity rather than volume. + +**Spatial budget.** The spatial layer's budget invariant $|G| + S + 2 \leq G_{\max}$ (§3.12, property 8) bounds node creation absolutely. Its spray defence mechanisms are inherited in full. + +**Analysis budget.** New cells start at ground importance and sit deep in the V-Tree — far below the analysis cutoff $L$. Only sustained observation volume pushes a cell into the top $K$. **Value diversity creates spatial nodes but not competitive trackers.** The competitive budget $K$ is inherently robust against diversity-based spray. Ancestor trackers are bounded by the investment set's ancestor closure (§8.2): at most $K\bar{D}$ additional ancestor nodes before sharing (reduced to near $K$ under typical concentration), regardless of how many spatial nodes exist. + +### 12.3 Steady-State Bound · `sec:sentinel:algorithm-resource-steady-state-bound` + +Under sustained spray at rate $R$ with spatial attenuation $\lambda_{\text{sp}} \in (0, 1)$ and split threshold $\theta$: + +$$L_{\text{steady}} = \frac{R}{(1 - \lambda_{\text{sp}}) \cdot \theta}$$ + +Independent of domain size and spray duration. At most $K$ of these receive trackers. For $\lambda_{\text{sp}} \geq 1$ (amplification or no-decay regimes), no decay-driven equilibrium exists; the spatial hard ceiling $G_{\max}$ (§3.12, property 8) is the operative bound. + +### 12.4 Benchmark Compounding · `sec:sentinel:algorithm-resource-benchmark-compounding` + +The spatial layer's benchmark compounding (§3.8) provides long-term spray memory. After $j$ expand–contract cycles, re-expansion to depth $D$ requires $\sim D^2 \theta / 2$ observations. This ensures that deep spatial structure is re-created only when justified by sustained, concentrated observation volume. + +### 12.5 Coverage Displacement · `sec:sentinel:algorithm-resource-coverage-displacement` + +An adversary can influence _which_ cells occupy the $K$ competitive slots by generating concentrated observations to chosen prefixes, displacing established cells. This is a coverage attack, not a resource attack. The host can detect it via the analysis set summary in the report (§14). + +### 12.6 Per-Tracker Complexity · `sec:sentinel:algorithm-resource-per-tracker-complexity` + +| Operation | Cost | Notes | +| ----------------------------- | -------------------------------------------------------- | ----------------- | +| Projection and reconstruction | $O(bwk)$ | Matrix multiply | +| Novelty | $O(bw)$ | Residual norm | +| Displacement and surprise | $O(bk)$ | Latent space | +| Coherence | $O(bk^2)$ | Pairwise products | +| Streaming SVD | $O\!\big(\min(w,\, k{+}b)^2 \cdot \max(w,\, k{+}b)\big)$ | **Dominant** | +| Second-moment update | $O(bk^2)$ | Upper triangle | +| Baselines and drift detection | $O(1)$ per axis | | + +The SVD input $M \in \mathbb{R}^{w \times (k+b)}$ is a thin SVD whose cost depends on the aspect ratio. When $w \geq k + b$ (tall matrix — typical at the root and shallow cells), the cost is $O(w(k{+}b)^2)$. When $k + b > w$ (wide matrix — routine at deep competitive cells, e.g. $w = 32$, $k + b = 80$), the cost is $O(w^2(k{+}b))$. The general form covers both regimes. + +### 12.7 Per-Batch Complexity · `sec:sentinel:algorithm-resource-per-batch-complexity` + +**Step 2 (spatial layer).** Per observation: $O(d_{\text{geo}} + h_V)$. Per batch: $O(n(d_{\text{geo}} + h_V))$ plus amortised structural operations. + +**Step 4 (per-cell scoring, all online trackers in $\mathcal{A}^*$).** Each tracker's cost depends on its per-tracker batch size $b$, which varies by cell (§2.6 batch-size note). Only trackers with $b > 0$ execute the core loop; the effective cost is $\sum_{c \in \mathcal{A}^* :\, b_c > 0} O\!\big(\min(w_c,\, k_c + b_c)^2 \cdot \max(w_c,\, k_c + b_c)\big)$, dominated by SVD. The worst-case bound, assuming all trackers receive observations, is $O\!\big(|\mathcal{A}^*| \cdot \min(w_{\max},\, k + b_{\max})^2 \cdot \max(w_{\max},\, k + b_{\max})\big)$. In steady state $|\mathcal{A}^*| = |\mathcal{I}| \leq 1 + K\bar{D}$ before sharing (§8.2); under realistic observation distributions with concentration-driven sharing, the effective size approaches $2K$. + +**Ancestor chain cost.** Each observation is processed by every tracker on its ancestor path. The dominant term is the **root tracker**: it has the widest suffix ($w = N$) and sees the full ingestion batch ($b = n$). Its SVD input is $\mathbb{R}^{N \times (k_0 + n)}$. For $N = 128$, $k_0 = 16$, and $n = 64$, this is a $(128, 80)$ matrix — in the tall regime ($w > k + b$), costing $O(128 \times 80^2) \approx 819\text{K}$. At deeper competitive cells the SVD input is in the wide regime ($k + b > w$); for example, a depth-96 cell has $w = 32$, and with $k = 16$, $b = 64$, the $(32, 80)$ matrix costs $O(32^2 \times 80) \approx 82\text{K}$ — not $O(32 \times 80^2) \approx 205\text{K}$. Callers ingesting large batches ($n > N - k$) push even the root into the wide regime. + +Intermediate ancestor levels use narrower suffixes and see only observations in their spatial ranges. The per-observation fan-out is at most the target's G-tree depth plus one, with depth bounded by $N-2$ for eligible targets. Across targets, shared ancestors are counted once, giving the depth-dependent investment bound in §8.2; sharing alone does not guarantee a count proportional only to the number of targets. + +**Step 5 (coordination).** Per context: $O(m)$. Total useful work: $O(K \cdot \bar{d})$ where $\bar{d}$ is average coordination participation depth. The pseudocode (§7.4) visits the full spatial tree for clarity; an implementation walking the investment set's reduced Steiner tree (§8.2) achieves this bound with $O(|\mathcal{I}|)$ traversal overhead (approaching $O(K)$ under typical spatial clustering). Negligible relative to Step 4. + +**Overall.** The observation algorithm is dominated by Step 4. Typically dominated by Step 4. + +### 12.8 Matrix Dimensions · `sec:sentinel:algorithm-resource-matrix-dimensions` + +**Per competitive tracker:** + +| Matrix | Shape | +| --------------- | ----------------- | +| $X$ | $(b, w)$ | +| $U$ | $(w, \text{cap})$ | +| $Z$ | $(b, k)$ | +| $M$ (SVD input) | $(w, k+b)$ | + +Typical competitive: $w \in \{32, \ldots, 112\}$, $k \leq 16$, $b \leq 64$. Largest competitive SVD: approximately $(112, 80)$. Note: $b$ here is the per-tracker count of observations routing to the cell (§2.6 batch-size note); at deep competitive cells, $b$ may be substantially smaller than the ingestion batch size $n$. + +**Root tracker:** $w = N$, $b = n$ (full ingestion batch). SVD input $(N, k + n)$. For $N = 128$, $k = 16$, $n = 64$: $(128, 80)$. + +**Per coordination context:** + +| Matrix | Shape | +| --------- | -------------- | +| $O$ | $(m, 4)$ | +| SVD input | $(4, k_m + m)$ | + +The coordination SVD is at most $(4, K + 4)$ — trivial. + +### 12.9 Work Variance · `sec:sentinel:algorithm-resource-work-variance` + +With deferred warm-up (§11.6), the per-call cost of the observation algorithm is bounded by: + +$$O\!\Big(n(d_{\text{geo}} + h_V) \;+\; |\mathcal{A}^*| \cdot \min(w_{\max},\, k + b)^2 \cdot \max(w_{\max},\, k + b) \;+\; |\mathcal{E}|\log|\mathcal{E}| \;+\; |\mathcal{I}|\log|\mathcal{I}| \;+\; |\mathcal{I}| \cdot w_{\max}\min(w_{\max},\, r_{\max})\Big)$$ + +on **every** call. (The previous expression $|\mathcal{A}^*| \cdot w_{\max} \cdot (k + b)^2$ is a valid but loose upper bound for the scoring term; the $\min/\max$ form is tight in both the tall-matrix and wide-matrix regimes — see §12.6.) Noise injection contributes zero cost to an observation call: deferred warm-up moves it to the background worker. Cell creation does not — the selection that identifies a new cell and the tracker allocated for it stay on the call, and they are the last three terms. Selection is recomputed after each observation pass, changed or unchanged (§8.1): the eligible set $\mathcal{E}$ is ranked by importance for the top-$K$ cut and closed under ancestry into $\mathcal{I}$ (§8.2), with the entry and exit bookkeeping over both sets absorbed in the $|\mathcal{I}|\log|\mathcal{I}|$ term. The $|\mathcal{E}|\log|\mathcal{E}|$ ranking term is the cost of that recomputation; an implementation maintaining $\mathcal{T}$ incrementally (§8.1) pays for the ranking during rebalancing instead. Allocation gives each cell entering $\mathcal{I}$ a $w \times \text{cap}$ basis and the latent state beside it (§4.1), $\text{cap} = \min(w, r_{\max})$; its multiplier is the number of cells entering on the call — none on a typical call, at most $|\mathcal{I}| \leq 1 + K\bar{D}$ when the selection turns over entirely (§8.2). The call-to-call variance arises from: + +- **Batch size variation** ($b$ differs per cell and per call; $n$ differs between calls). +- **Producing set size variation** ($|\mathcal{A}^*|$ changes by $O(1)$ per call as cells promote or exit via investment set reconciliation). +- **Rank variation** ($k$ changes by at most 1 per rank update interval). +- **Entering cell count** (zero on a call that leaves the selection unchanged; bounded by $|\mathcal{I}|$ on one that changes it). + +The first three change slowly relative to call frequency. The fourth does not: it is zero on most calls and a bounded burst on the calls that change the selection, paid once per cell entry against the whole warm-up schedule that entry commits to (§11.6). + +> _Design note._ The deferred warm-up removes the dominant structural source of timing variance (inline noise injection), making the observation algorithm operationally predictable. It does **not** make it constant-time — the remaining variance, though small, is observable to a sufficiently precise adversary. Constant-time guarantees, if required, are an implementation concern beyond this specification. + +--- + +# Part V — External Interface · `sec:sentinel:algorithm-external-interface` + +--- + +## Chapter 13. Configuration · `sec:sentinel:algorithm-configuration` + +All parameters are organised by the layer they govern. An implementation should accept these at construction time. Parameters marked as host-controlled may additionally be modified during operation through the mechanisms noted. + +### 13.1 Analysis Engine Parameters · `sec:sentinel:algorithm-configuration-analysis-engine-parameters` + +These govern the subspace tracker (§4) and baseline tracking (§6) within each analysed cell. + +| Parameter | Description | Default | Constraint | Guidance | +| ----------------------------------------- | ------------------------------------------------------------------------ | --------- | -------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Maximum rank ($r_{\max}$) | Hard ceiling on subspace dimensionality | 16 | $\geq 1$ | Higher captures more complex structure; memory scales as $w \times \text{cap}$. | +| Forgetting factor ($\lambda$) | EWMA decay rate | 0.99 | $(0, 1)$ | Lower = faster adaptation, shorter memory. Match to the regime change timescale. | +| Rank update interval ($T_{\text{rank}}$) | Batches between rank adaptation (§4.2, Phase 5) | 100 | $\geq 1$ | Lower = more responsive rank. Higher = more stable. | +| Energy threshold ($\tau$) | Cumulative variance target for rank selection | 0.90 | $(0, 1)$ | 0.90 retains 90% of variance. Higher → rank grows, novelty range shrinks. | +| Stability constant ($\varepsilon$) | Floor for numerical stability | $10^{-6}$ | $> 0$ | Rarely needs adjustment. | +| Clip width ($n_\sigma$) | Base outlier clip width in $\sigma$ units (§6.1.1) | 3.0 | $> 0$ | Wider → fewer legitimate rejections, slower poisoning resistance. Narrower → more false rejections. | +| Slow decay ($\lambda_s$) | Per-tracker slow EWMA decay (§6.2) | 0.999 | $(\lambda, 1)$ | Ratio $\lambda_s / \lambda$ controls drift detection sensitivity window. | +| Coordination slow decay ($\lambda_{s,m}$) | Coordination slow EWMA decay (§7.5) | 0.999 | $(\lambda, 1)$ | Same guidance as per-tracker slow decay. | +| Drift allowance ($\kappa_\sigma$) | Drift accumulator noise allowance in slow-baseline $\sigma$ units (§6.3) | 0.5 | $\geq 0$ | Lower → more sensitive, more false positives. | +| Clip-pressure decay ($\lambda_\rho$) | EWMA decay for per-axis clip ratio tracking (§6.1.1, §6.4) | 0.95 | $(0, 1)$ | Lower → faster recovery from stale baselines, more contamination leakage. Higher → slower recovery, better poisoning resistance. At 0.95 (half-life ~14 batches): moderate shifts converge in ~60–100 batches through damped oscillation. | +| Per-sample scores | Whether to include per-observation score vectors in reports | false | boolean | Enable for forensic analysis; increases report size. | + +### 13.2 Analysis Selector Parameters · `sec:sentinel:algorithm-configuration-analysis-selector-parameters` + +These govern which cells receive statistical modelling (§8). + +| Parameter | Description | Default | Constraint | Guidance | +| --------------------- | --------------------------------------------- | ------- | ---------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Analysis budget ($K$) | Maximum competitively selected cells | 1024 | $\geq 1$ | Total invested trackers $\leq 1 + K\bar{D}$ before sharing (§8.2); dominated by $2K$ under concentration. Scale with observation diversity. 256 for focused; 4096 for large-scale. | +| Depth cutoff ($L$) | Maximum V-Tree depth for analysis eligibility | 6 | $\geq 0$ | Larger $L$ admits less significant cells. Keep near $D_{\text{create}} + 2$. | + +The competitive targets $\mathcal{T}$, investment set $\mathcal{I}$, and producing sets $\mathcal{A}$, $\mathcal{A}^*$ are derived from these parameters as specified in §8. + +### 13.3 Spatial Layer Parameters · `sec:sentinel:algorithm-configuration-spatial-layer-parameters` + +These are passed through to the spatial layer (§3) at construction time. The Sentinel does not modify them during operation. + +| Parameter | Description | Default | Constraint | Guidance | +| ----------------------------------- | ---------------------------------------- | ------- | ---------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | +| Split threshold ($\theta$) | Minimum importance for split eligibility | 100 | $> 0$ | Lower → finer resolution faster. Must be reachable by sustained observations in the desired response time. Type is the accumulator $V$ (§1.2). | +| Creation gate ($D_{\text{create}}$) | Maximum V-Tree depth for splits | 3 | $\geq 0$ | Application-dependent. | +| Eviction gate ($D_{\text{evict}}$) | Minimum V-Tree depth for eviction | 6 | $\geq 1$ | Must exceed $D_{\text{create}}$ by a buffer. | +| Soft budget | Soft node-count target | 100,000 | $> 0$ | Application-dependent. | +| Hard ceiling ($G_{\max}$) | Absolute maximum spatial nodes | 200,000 | $\geq 5$ | Must exceed soft budget. | + +### 13.4 Temporal Policy (Host-Controlled) · `sec:sentinel:algorithm-configuration-host-controlled-temporal-policy` + +These are not construction-time parameters but rather arguments to the spatial decay operation, invoked by the host at its discretion (§10.1). + +| Parameter | Suggested value | Guidance | +| ------------------ | --------------- | ---------------------------------------------------------------------------------------------------------- | +| Attenuation factor | 0.99 | Attenuation per call. Compound with interval: effective half-life $= t_{1/2} \times \text{interval}$. | +| Depth selectivity | 0.0 | Values $> 0$ cause fine structure to decay faster than coarse. Use 0.3–0.5 for depth-selective forgetting. | +| Decay interval | 60 seconds | Shorter → more responsive contour. Longer → more stable. | + +### 13.5 Warm-Up Parameters · `sec:sentinel:algorithm-configuration-warmup-parameters` + +These govern the noise injection and warm-up procedure (§11). + +| Parameter | Description | Default | Constraint | +| ---------------- | ---------------------------------- | ------------------------------------------------ | -------------------------------------------- | +| Noise schedule | Depth-tiered synthetic batch count | Geometric: root = 450, decay = 0.5, minimum = 50 | See §11.1.3 | +| Noise batch size | Samples per synthetic batch | 16 | $\geq 1$ | +| Random seed | Seed for pseudo-random generator | Deterministic (fixed) | Optional; absent means implementation-chosen | + +--- + +## Chapter 14. Output · `sec:sentinel:algorithm-output` + +### 14.1 Report Overview · `sec:sentinel:algorithm-output-report-overview` + +Each observation cycle (§9) produces a report containing four sections: per-cell reports from competitive trackers, per-cell reports from ancestor trackers, coordination reports from the hierarchical coordination tier, and summary information about the system's current state. The report also includes contour, health, and analysis set summaries. + +Importance values in the report are converted to floating-point approximations at the report boundary (via the approximate conversion capability required of $V$, §1.2). Hosts needing exact accumulator values should query the spatial layer directly. + +The report also states the age of its own evidence, described in full below (`sec:sentinel:algorithm-output-observation-age`). + +#### 14.1.1 Observation Age · `sec:sentinel:algorithm-output-observation-age` + +Every report carries one further figure: how old the oldest observation in the batch was at the moment the report was emitted, as a duration in microseconds. + +| Field | Type | Description | +| --------------------- | ----------------------------------- | ------------------------------------------------------------------------------- | +| Oldest observation age | optional non-negative integer, µs | Age of the batch's oldest observation at report emission; absent where there is none | + +Four properties fix what the figure means. + +**It is a duration and never an instant.** Both ends of the interval are read from the sentinel's own monotonic clock: one reading as the batch arrives, one as the report is assembled. No wall-clock time is recorded anywhere in the report and no clock of the sentinel's is ever compared with another machine's, so there is no skew for the figure to carry and none for a consumer to correct. + +**One figure bounds the whole batch.** A batch arrives whole, so its observations share a single arrival and the oldest of them is no older than that arrival. The reported age is therefore the maximum over the batch: no observation in it is older than the figure, and the oldest is exactly that old. + +**Absence means there is no age, not an age of nothing.** A batch carrying no observations has no oldest observation, and the field is absent rather than zero. A payload written before the field existed is absent in the same way and for the same reason — nothing measured it. The two are spelled alike because they say the same thing to a consumer, and a zero would say something different and untrue: that the evidence was fresh when the report went out. + +**The host's forwarding delay is an unmeasured residual.** How long the host held the observations before handing them over is real, is not in this figure, and is deliberately not measured. Measuring it would require comparing the host's clock with the sentinel's, which is exactly the cross-machine comparison this design does without; a consumer reading the age therefore reads a lower bound on how old the evidence is, not the whole of it. + +### 14.2 Cell Analysis Report · `sec:sentinel:algorithm-output-cell-analysis-report` + +One record per analysed cell (competitive or ancestor) that processed at least one observation in the batch. + +| Field | Type | Description | +| ------------------ | ------------------------------- | -------------------------------------------------------- | +| Interval | pair of $C$ values | Spatial bounds of the cell (half-open interval) | +| Depth | non-negative integer | G-Tree depth (0 = root) | +| Analysis width | non-negative integer | $w = N - \text{depth}$ | +| Sample count | non-negative integer | Observations delivered to this cell in this batch | +| Rank | non-negative integer | Current active subspace dimensionality $k$ | +| Energy ratio | float | Fraction of total energy captured by the active subspace | +| Top singular value | float | Largest singular value $\sigma_1$ | +| Maturity | maturity record (§14.6) | Noise influence and observation counts | +| Scoring geometry | geometry record (§14.7) | Structural reliability of each scoring axis | +| Scores | scoring record (§14.3) | All four axes with baselines and drift state | +| Per-sample scores | optional list of sample records | Present only when per-sample reporting is enabled | + +**Competitive vs. ancestor reports.** Both use the same record structure. They are separated in the report so the host can distinguish cells selected as competitive targets (members of $\mathcal{A}$) from cells included as ancestors (members of $\mathcal{A}^* \setminus \mathcal{A}$). Only online (producing) cells appear in the report; warming cells in $\mathcal{I} \setminus \mathcal{A}^*$ do not emit reports. + +### 14.3 Scoring Record · `sec:sentinel:algorithm-output-scoring-record` + +One per cell, containing all four axes. + +| Field | Type | Description | +| ------------ | -------------------------- | ----------- | +| Novelty | score distribution (§14.4) | | +| Displacement | score distribution (§14.4) | | +| Surprise | score distribution (§14.4) | | +| Coherence | score distribution (§14.4) | | + +### 14.4 Score Distribution · `sec:sentinel:algorithm-output-score-distribution` + +One per axis per cell, summarising the batch and its relationship to baselines. + +| Field | Type | Description | +| --------------- | ------------------------- | -------------------------------------------------------------------- | +| Minimum | float | Minimum raw score across samples in this batch | +| Maximum | float | Maximum raw score across samples in this batch | +| Mean | float | Mean raw score across samples in this batch | +| Maximum z-score | float | $\zeta(\max_i s_i)$ — z-score of the loudest sample (§6.1.2) | +| Mean z-score | float | $\zeta(\bar{s}_{\text{batch}})$ — z-score of the batch mean (§6.1.2) | +| Clip pressure | float in $[0, 1]$ | Current clip-pressure EWMA $\bar{\rho}$ for this axis (§6.1.1) | +| Baseline | baseline snapshot (§14.5) | Current fast EWMA state | +| Drift state | drift snapshot (§14.5) | Current slow EWMA and drift accumulator state | + +### 14.5 Baseline and Drift Snapshots · `sec:sentinel:algorithm-output-baseline-and-drift-snapshots` + +**Baseline snapshot** (fast EWMA state): + +| Field | Type | Description | +| -------- | ----- | ------------------------------------ | +| Mean | float | Current fast EWMA mean $\bar{s}$ | +| Variance | float | Current fast EWMA variance $\bar{v}$ | + +**Drift snapshot** (slow EWMA and drift accumulator state): + +| Field | Type | Description | +| ---------------------- | -------------------- | ------------------------------------------ | +| Accumulator | float | Current drift accumulator value $S$ (§6.3) | +| Slow baseline mean | float | Current slow EWMA mean | +| Slow baseline variance | float | Current slow EWMA variance | +| Steps since reset | non-negative integer | Batches since last drift accumulator reset | + +> _Host guidance (secular-trend monitoring)._ The drift accumulator $S$ detects fast-vs-slow baseline divergence but is blind to slow monotonic inflation of both baselines (see §6.4.5 design note). Hosts concerned about long-horizon contamination should monitor secular trends in the slow baseline mean across trackers over timescales much longer than $1/(1-\lambda_s)$. Sustained upward drift in $\bar{s}_{\text{slow}}$ across multiple trackers and axes, especially when accompanied by persistently elevated $\bar{\rho}$ (§14.4), may indicate patient sub-clip contamination. The annihilation mechanism (§10) provides a remediation path: targeted state destruction forces re-learning from the current observation distribution. + +### 14.6 Maturity Record · `sec:sentinel:algorithm-output-maturity-record` + +| Field | Type | Description | +| ------------------ | -------------------- | ---------------------------------------------------------------------- | +| Real observations | non-negative integer | Genuine observations processed since creation | +| Noise observations | non-negative integer | Synthetic observations processed during warm-up | +| Noise influence | float in $[0, 1]$ | $\eta$ — fraction of baseline not yet established by real data (§11.5) | + +### 14.7 Scoring Geometry Record · `sec:sentinel:algorithm-output-scoring-geometry-record` + +One per cell or coordination tracker, describing the geometry of the model that scored the associated batch. The report's rank and residual degrees of freedom therefore belong to the same scoring state even when rank adaptation has already prepared a different rank for the next batch. In an inspection snapshot, the record describes the most recent scored batch. + +| Field | Type | Description | +| -------------- | -------------------- | -------------------------------------------------------- | +| Analysis width | non-negative integer | Working dimensionality $w$ of this tracker's input space | +| Capacity | non-negative integer | Maximum reachable rank: $\text{cap} = \min(w, r_{\max})$ | +| Residual DOF | non-negative integer | Residual degrees of freedom: $w - k$ for the scoring rank | + +Two derived predicates assist the host: + +| Predicate | Condition | Meaning | +| ----------------- | ------------------------- | --------------------------------------------------------------------------- | +| Novelty-saturated | $\text{residual DOF} = 0$ | Novelty axis is identically zero — the subspace spans the full space (§5.2) | +| Novelty-saturable | $\text{cap} \geq w$ | Novelty _can_ become saturated as rank adapts — the host should monitor | + +### 14.8 Per-Sample Score Record · `sec:sentinel:algorithm-output-per-sample-score-record` + +Present only when per-sample reporting is enabled. One per observation delivered to the cell. + +| Field | Type | Description | +| ------------ | ----- | -------------------------------------- | +| Novelty | float | Raw novelty score for this sample | +| Displacement | float | Raw displacement score for this sample | +| Surprise | float | Raw surprise score for this sample | +| Coherence | float | Raw coherence score for this sample | + +### 14.9 Coordination Report · `sec:sentinel:algorithm-output-coordination-report` + +One record per active coordination context (§7.1) that fired in this batch, ordered by G-Tree depth (shallowest first), ties broken by ascending G-node identifier so that the order is total. + +| Field | Type | Description | +| ------------------ | ------------------------------- | ----------------------------------------------------------- | +| Context interval | pair of $C$ values | Spatial extent of the coordination group | +| Context depth | non-negative integer | G-Tree depth of the coordination node | +| Group size | non-negative integer | Total competitive cells in this group | +| Left count | non-negative integer | Competitive cells in the left subtree | +| Right count | non-negative integer | Competitive cells in the right subtree | +| Rank | non-negative integer | Current subspace dimensionality of the coordination tracker | +| Energy ratio | float | Fraction of total energy captured | +| Top singular value | float | Largest singular value | +| Maturity | maturity record (§14.6) | Warm-up state of the coordination tracker | +| Scoring geometry | geometry record (§14.7) | Structural reliability of coordination scoring axes | +| Scores | scoring record (§14.3) | Four axes of coordination scoring | +| Per-member scores | optional list of member records | Per-cell breakdown from this context's model | + +**Per-member score record** (when present): + +| Field | Type | Description | +| -------------------- | ------------------ | ------------------------------------------------ | +| Cell interval | pair of $C$ values | Spatial bounds of the contributing cell | +| Novelty | float | Raw coordination novelty for this cell | +| Displacement | float | Raw coordination displacement for this cell | +| Surprise | float | Raw coordination surprise for this cell | +| Coherence | float | Raw coordination coherence for this cell | +| Novelty z-score | float | Z-score of this cell's coordination novelty | +| Displacement z-score | float | Z-score of this cell's coordination displacement | +| Surprise z-score | float | Z-score of this cell's coordination surprise | +| Coherence z-score | float | Z-score of this cell's coordination coherence | + +### 14.10 Contour Snapshot · `sec:sentinel:algorithm-output-contour-snapshot` + +The contour snapshot describes the G-Tree's observation-receiving surface — the spatial layer's internal structural state — not the set of cells in the batch report. The cell count here is the total contour cell count (terminal + semi-internal nodes), which may differ from the number of cells in the batch report (which includes ancestor cells above the contour). + +| Field | Type | Description | +| ------------------------------ | -------------------- | -------------------------------------------------------------------------------- | +| Plateau count | non-negative integer | Number of distinct plateaus in the current contour (§3.2) | +| Cell count | non-negative integer | Total contour cells | +| Total importance | float | Approximate total importance across all cells | +| Splits since last report | `u32` | Child cells created by catalytic or bootstrap bisection since the previous report | +| Net removals since last report | `u32` | Net structural removals (evictions minus restorations) since the previous report | + +**Structural mutation counts.** The spatial layer maintains monotonic counters for child creation, eviction, and restoration. Each child created by a catalytic or bootstrap bisection counts as one split, so a bisection that creates both children adds two. Counting child creations keeps the total composable with a restoration, which creates one missing child. + +At each report boundary, `splits_since_last_report` is the split-counter delta and `net_removals_since_last_report` is the eviction-counter delta minus the restoration-counter delta. The unsigned net-removal field reports zero when restorations exceed evictions during an interval, and either field reports `u32::MAX` when its result is larger than `u32` can represent. + +The implementation snapshots the spatial counters immediately after graph construction or replacement and at every report boundary. A replacement therefore starts a new counter epoch and never appears as a synthetic interval mutation. Event counters are necessary because node and terminal population deltas do not identify every mutation: evicting the last child of a semi-internal parent removes a node while making the parent terminal, leaving the terminal count unchanged. + +### 14.11 Health Report · `sec:sentinel:algorithm-output-health-report` + +Available both inline in the batch report (for batch-level completeness) and via a standalone query (for on-demand inspection between observation cycles). Both sources produce identical data. + +| Field | Type | Description | +| ---------------------------- | --------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Total spatial nodes | non-negative integer | Current materialised G-Tree node count | +| Semi-internal count | non-negative integer | One-child spatial nodes | +| Active competitive trackers | non-negative integer | $ \| \mathcal{A} \| $ | +| Active ancestor trackers | non-negative integer | $ \| \mathcal{A}^\* \setminus \mathcal{A} \| $ | +| Active coordination contexts | non-negative integer | Currently active coordination contexts (§7.1) | +| Investment set size | non-negative integer | $ \| \mathcal{I} \| $ — total cells with allocated trackers (online + warming) | +| Warming trackers | non-negative integer | Members of $\mathcal{I}$ currently in the warm-up pipeline (§11.6) | +| Warming competitive targets | non-negative integer | Competitive targets in $\mathcal{I}$ not yet promoted to $\mathcal{A}$ | +| Lifetime observations | non-negative integer | Total observations processed since system creation | +| Rank distribution | list of non-negative integers | Count of trackers at each rank value | +| Clip-pressure distribution | histogram or summary statistics | Distribution of $\bar{\rho}$ values across all active trackers and axes (§6.4) | +| Degenerate-cell count | non-negative integer | Cells excluded from $\mathcal{I}$ due to $w < 2$ (§4.1, §8.2); currently always zero under the $w \geq 2$ eligibility predicate in §8.1 — retained as a defensive field against future eligibility changes | +| Geometry distribution | geometry distribution record (§14.11.1) | Fleet-wide scoring axis reliability | + +#### 14.11.1 Geometry Distribution · `sec:sentinel:algorithm-output-health-report-geometry-distribution` + +Summarises the structural reliability of scoring axes across all active per-cell trackers. + +| Field | Type | Description | +| ------------------------ | -------------------- | -------------------------------------------------------------------- | +| Novelty-saturated count | non-negative integer | Trackers where $k = w$ (novelty axis degenerate) | +| Novelty-saturable count | non-negative integer | Trackers where $\text{cap} \geq w$ (novelty _can_ become degenerate) | +| Coherence-inactive count | non-negative integer | Trackers where $k < 2$ (coherence axis undefined) | + +### 14.12 Analysis Set Summary · `sec:sentinel:algorithm-output-analysis-set-summary` + +| Field | Type | Description | +| -------------------------- | ----------------------------- | ----------------------------------------------------------------- | +| Investment set size | non-negative integer | $ \| \mathcal{I} \| $ (competitive targets + ancestors, online + warming) | +| Producing competitive size | non-negative integer | $ \| \mathcal{A} \| $ (online competitive targets) | +| Producing full size | non-negative integer | $ \| \mathcal{A}^\* \| $ (online investment members) | +| Depth range | pair of non-negative integers | (minimum depth, maximum depth) across competitive cells | +| Importance range | pair of floats | (minimum importance, maximum importance) across competitive cells | +| V-Tree depth range | pair of non-negative integers | (minimum V-depth, maximum V-depth) across competitive cells | + +### 14.13 Contour Queries · `sec:sentinel:algorithm-output-contour-queries` + +The spatial layer's contour is available for direct querying through the spatial layer's interface: + +| Query | Description | Cost | +| ------------- | ------------------------------------------ | -------------------------------- | +| Point query | The plateau containing a given coordinate | $O(\log P)$ | +| Range query | All plateaus intersecting a given interval | $O(\log P + \text{result size})$ | +| Iteration | All plateaus in spatial order | $O(P)$ | +| Plateau count | Number of distinct plateaus | $O(1)$ | + +These queries are independent of the observation cycle and may be invoked at any time. + +For a deeper diagnostic view of the spatial partition — including its formation history, pre-refinement baselines, and progressive energy decomposition — see the Wavelet Portrait (§14.14). + +### 14.14 The Wavelet Portrait · `sec:sentinel:algorithm-wavelet-portrait` + +The spatial layer supports on-demand extraction of the Progressive Entropic-Wavelet Exposure Image (PEWEI): a concrete, layered readout of the spatial structure the Sentinel has discovered. It is not another alarm score. It is a diagnostic portrait of the spatial partition itself: where structure was confirmed, at what scale it emerged, and how much observation volume had accumulated before refinement was authorised. + +#### 14.14.1 What the Portrait Contains · `sec:sentinel:algorithm-wavelet-portrait-contents` + +A PEWEI is an ordered sequence of layers keyed to V-Tree depth. The V-Tree's competitive ranking places high-importance entries at shallow depths and lower-importance entries deeper in the sequence, so the layer order is an approximate significance order with the G-V Graph's $1.44\times$ golden-ratio overhead relative to Shannon entropy (§PEWEI M-7). Structural-only V-Tree depths may contribute no emitted entries, but every emitted node records its V-depth. + +Each layer contains two node populations: + +| Population | Nodes | Fields | +| ---------- | ----- | ------ | +| Phase transition nodes | Internal or semi-internal G-nodes with confirmed sub-scale structure | Region $[l, r)$, baseline $B = g.\text{own}$, total $S = g.\text{sum}$, refinement $R = S - B$, baseline ratio $R/B$ when $B > 0$ | +| Terminal nodes | Leaf G-nodes at the finest confirmed resolution for their regions | Region $[l, r)$, intensity $I = g.\text{own} = g.\text{sum}$ | + +For a phase transition node, the baseline $B$ says: this region accumulated $B$ units of observation volume before the Sentinel decided it had internal structure worth resolving. This is a formation-history datum, not the current alarm level for the region. Fully internal G-nodes provide a clean frozen pre-refinement measurement; semi-internal nodes may carry a hybrid baseline because evicted child energy and later observations in the uncovered half can be folded into $g.\text{own}$ (§PEWEI M-2). + +The total $S$ is the complete energy currently accounted for under that region, including all descendants. The refinement $R = S - B$ is the energy attributed to confirmed sub-scale structure after the transition. The baseline ratio $R/B$ reports how large that confirmed sub-scale structure is relative to the pre-refinement benchmark; a large ratio means the subtree has accumulated much more structure than existed when the split decision was made, while a ratio near zero means the split has little subsequent refinement or the children have not accumulated much energy yet. When $B = 0$, the ratio is undefined and must be treated as absent. + +For a terminal node, the intensity $I$ is the finest-resolution measurement the Sentinel has confirmed for that region. Terminals are the leaves of the current decomposition at the time the portrait is extracted. + +#### 14.14.2 Progressive Reconstruction · `sec:sentinel:algorithm-wavelet-portrait-progressive-reconstruction` + +The PEWEI supports truncation at any layer depth while preserving exact total energy across the domain. A host that reads only the first $k$ layers obtains a correctly normalised coarse spatial summary; each additional layer refines the summary by exposing the next significant spatial details. + +Reconstruction is additive across scales. For every phase transition visible at depth $\leq k$ whose children are not visible, the host uses the transition's total $S$ as a lump estimate for $[l, r)$. For every transition whose children are visible, the host contributes the transition's baseline $B$ as uniform background under the finer child regions, then adds the child contributions recursively. The parent baseline is not replaced by the children; it persists beneath them. + +This follows directly from the summation invariant: + +$$g.\text{sum} = g.\text{own} + \sum_c c.\text{sum}$$ + +Truncation at a coarse depth contributes $g.\text{sum}$ as a lump. Expansion contributes $g.\text{own}$ plus the child sums. Both paths contribute the same total energy. For example, a shallow read may show a transition over $[0, 64)$ with total $S$; a deeper read replaces that lump with the same transition's baseline background plus visible child regions, while the total over $[0, 64)$ remains exactly $S$ (§PEWEI M-10). + +#### 14.14.3 Self-Calibration · `sec:sentinel:algorithm-wavelet-portrait-self-calibration` + +The frozen baselines are self-calibrating datums: each was deposited by the same observations it later calibrates. This gives the portrait useful operational properties. There is no separate noise-estimation pass, no auxiliary calibration structure, and no extra runtime beyond the extraction walk. The baseline is spatially adaptive because each region carries its own benchmark, and scale-adaptive because each refinement level carries its own benchmark. + +The same property is also the portrait's main statistical limitation. The data that discovered the signal also determined the calibration threshold. The frozen benchmark at scale $k$ was the evidence that justified splitting at scale $k$; the children's energy is then measured against this same value. The PEWEI therefore has no statistically independent noise reference such as a holdout sample or an independent fine-scale summary (§PEWEI M-6.3). + +Under intensity-proportional noise regimes — event counts, request counts, photon counts, and other processes where variance scales with intensity — this circularity is usually acceptable. The baseline is simultaneously the expected background energy and the natural noise-calibration scale, which matches the Sentinel's standard $\Delta = 1$ observation model. Under additive noise regimes, or when the host needs confidence intervals with frequentist coverage guarantees, the baseline supplies a measured mean but not an independent variance estimate; the host must provide the noise model. + +#### 14.14.4 Complementarity With Core Spatial Information · `sec:sentinel:algorithm-wavelet-portrait-core-spatial-complementarity` + +When a Sentinel feeds a host, the host core's normal spatial inputs answer questions about current alarm state and outcome history. The PEWEI answers a different question: why the spatial partition has the shape it currently has. + +| Pathway | What it tells the host | Timescale | Updated by | +| ------- | ---------------------- | --------- | ---------- | +| Per-cell analysis records (§§14.2–14.7) | How alarmed this cell is right now | Current batch | Each Sentinel observation cycle | +| Contour snapshot (§14.10) | How structurally complex the spatial partition is | Current state | Each Sentinel observation cycle | +| Host outcome ledger | What has historically happened to traffic routed through this cell | Decaying history | Host labels | +| PEWEI (§14.14) | What cumulative spatial structure has been discovered, at what scale, and at what evidence level | Cumulative partition record | Extraction on demand | + +The per-cell records contain scores, z-scores, baselines, drift accumulators, maturity, and geometry for cells in the current report. They do not say that a region accumulated a large baseline before splitting, or that one half then required another large baseline before its own structure was confirmed. The PEWEI records that partition-formation history directly. + +#### 14.14.5 Host Use Cases · `sec:sentinel:algorithm-wavelet-portrait-host-use-cases` + +**Encoding quality verification.** The host can inspect phase transition locations with large baselines and ask whether they align with domain-meaningful boundaries: network allocation boundaries, jurisdictional borders, categorical taxonomy branch points, or other known structure implied by the coordinate encoding (§2). Meaningful alignment indicates that the encoding is exposing real structure to the Sentinel. Transitions at boundaries with no domain interpretation suggest the encoding may be misaligned with the data's natural hierarchy. + +**Spatial investment audit.** Deep chains of phase transitions show where the Sentinel has spent its resolution budget. Regions that remain single terminals may be genuinely uniform, or they may be under-resolved because the split threshold is too high for their traffic volume. The portrait makes that distinction inspectable by showing where refinement has and has not occurred. + +**Multi-resolution spatial summary.** A host with bounded analysis time can read the first few layers and obtain the dominant spatial structures with correct normalisation. Because the layer order follows the V-Tree competitive ranking, early layers approximate the highest-energy structures before later layers add finer detail. + +**Drift detection complement.** Comparing baseline ratios across successive portrait extractions can reveal slow structural change. A cell's per-axis drift accumulators may be quiet because its current scores match recent adaptive baselines, while the PEWEI's $R/B$ ratio keeps growing because the subtree is accumulating more confirmed structure than existed at formation time. + +**Temporal decay impact assessment.** After the host applies temporal decay to the spatial layer, the portrait's baselines attenuate along with the graph. Extracting the portrait before and after decay shows how much historical grounding remains. Under depth-selective decay, fine-scale baselines attenuate faster than coarse baselines, making depth-dependent compression visible in the ratios. + +#### 14.14.6 Extraction · `sec:sentinel:algorithm-wavelet-portrait-extraction` + +The PEWEI is extracted directly from the Sentinel's spatial layer on demand. It is not part of the periodic batch report, and the host core does not mediate the extraction. A host that wants the portrait calls the Sentinel extraction interface and consumes the Sentinel-to-host output directly, consistent with the feed-forward invariant (§1.3). + +Extraction is a V-Tree traversal, with cost $O(L + S)$ where $L$ is the number of V-entries and $S$ is the number of V-structural nodes. Every emitted field already exists on the graph except the derived refinement and baseline ratio, which are computed in constant time for each phase transition node (§PEWEI M-9). + +#### 14.14.7 Limitations · `sec:sentinel:algorithm-wavelet-portrait-limitations` + +The portrait is a snapshot of the Sentinel's state at the moment of extraction. Under temporal decay, baselines attenuate between extractions. Under splits, evictions, and restoration, the structure of the portrait changes. A baseline can say how much evidence existed when a region was formed, but it is not a guarantee that the region's current semantic character is unchanged. + +The PEWEI is a diagnostic and explanatory tool, not a standalone detection mechanism. It does not produce z-scores, current alarm levels, outcome probabilities, or policy recommendations. It explains what spatial structure the Sentinel has learned and what evidence levels shaped that structure. + +--- + +# Part VI — Properties and Defence Analysis · `sec:sentinel:algorithm-properties-and-defence-analysis` + +This part analyses the properties that emerge from the interaction of the mechanisms defined in Parts II–IV. Each mechanism was specified independently; this part examines what the composition guarantees. + +--- + +## Chapter 15. Scoring Properties · `sec:sentinel:algorithm-scoring-properties` + +### 15.1 Axis Independence Under Binary Inputs · `sec:sentinel:algorithm-scoring-properties-binary-input-axis-independence` + +The four scoring axes (§5) decompose the observation's relationship to the learned model along orthogonal statistical concerns: + +| Score | Input | Covariance structure | +| ------------ | --------------------------------------- | ----------------------------------------------- | +| Novelty | $\|R_i\|^2 / (w - k)$ | Residual energy (scalar, orthogonal complement) | +| Displacement | $\|z_i\|^2 / (k + \|z_i\|^2)$ | Total latent energy (scalar) | +| Surprise | $(z_j - \mu_j)^2 / \nu_j$ per $j$ | Diagonal of latent covariance | +| Coherence | $(z_j z_l - \Gamma_{jl})^2$ per $j < l$ | Off-diagonal of latent covariance | + +Together they cover the full covariance structure of the latent space without assembling or inverting a dense $k \times k$ matrix. Each axis can fire independently of the others: + +- **Novelty only.** The observation has unusual structure outside the subspace, but its within-subspace projection is unremarkable. Indicates a genuinely novel direction. +- **Displacement only.** The observation projects normally onto the subspace directions but with unusual total magnitude. Indicates a scaling anomaly. +- **Surprise only.** Total projection energy is normal, but its distribution across dimensions is unusual. Indicates an unusual _combination_ of known directions at normal overall energy. +- **Coherence only.** Individual dimensional magnitudes are normal, but their pairwise products are unusual. Indicates an unusual _co-activation pattern_ — dimensions that normally vary independently are varying together (or vice versa). + +### 15.2 Projection Energy Redundancy · `sec:sentinel:algorithm-scoring-properties-projection-energy-redundancy` + +Under the centred binary encoding (§2.3), $\|\mathbf{x}_i\|^2 = w/4$ for every suffix vector (§2.5). By the Pythagorean theorem: + +$$\|\hat{\mathbf{x}}_i\|^2 + \|R_i\|^2 = \|\mathbf{x}_i\|^2 = w/4$$ + +Therefore projection energy $\|\hat{\mathbf{x}}_i\|^2 / k$ is a perfect affine function of novelty $\|R_i\|^2 / (w - k)$: + +$$\frac{\|\hat{\mathbf{x}}_i\|^2}{k} = \frac{w/4 - (w - k) \cdot \text{novelty}_i}{k}$$ + +Pearson correlation $r = -1$. A fifth axis based on projection energy would carry zero independent information and would violate the polarity invariant (§5.1) — low projection energy would indicate anomaly, but the upper-tail filter (§6.1.1) would fail to protect the baseline. + +This redundancy is specific to centred binary inputs. Continuous-valued inputs with variable norms would decouple the two measures. If the system is ever extended to non-binary encodings, projection energy should be reconsidered as an independent axis. + +### 15.3 Per-Axis Sensitivity Profiles · `sec:sentinel:algorithm-scoring-properties-per-axis-sensitivity-profiles` + +Each axis has a characteristic sensitivity profile that determines what kinds of anomalies it catches best and what kinds it misses: + +| Axis | Most sensitive to | Least sensitive to | +| ------------ | ------------------------------------- | -------------------------------------------------------------------------- | +| Novelty | New directions never seen before | Anomalies that stay within the learned subspace | +| Displacement | Large-magnitude projections | Anomalies at normal magnitude but unusual direction | +| Surprise | Per-dimension distributional shifts | Shifts that affect all dimensions equally (caught by displacement instead) | +| Coherence | Changed inter-dimension relationships | Single-dimension anomalies (caught by surprise instead) | + +The axes are complementary by design: what one axis misses, another catches. An adversary who optimises values to minimise one axis's score is forced into a region of observation space that elevates another axis's score — unless the values are genuinely indistinguishable from normal observations on all axes simultaneously. + +### 15.4 The Role of Rank · `sec:sentinel:algorithm-scoring-properties-role-of-rank` + +The active rank $k$ partitions the observation space into two subspaces of complementary sensitivity: + +- The **within-subspace** model ($k$ dimensions) catches displacement, surprise, and coherence anomalies but is blind to novel directions. +- The **residual** model ($w - k$ dimensions) catches novel directions but has no directional sensitivity within the residual space — only aggregate residual energy. + +Low rank (small $k$) allocates most dimensions to the residual, giving high sensitivity to novelty but coarse within-subspace modelling. High rank (large $k$) captures more within-subspace structure at the cost of less residual sensitivity. The rank adaptation mechanism (§4.2, Phase 5) balances automatically by tracking cumulative energy. + +--- + +## Chapter 16. Ancestor Chain Properties · `sec:sentinel:algorithm-ancestor-chain-properties` + +The ancestor closure (§8.2) transforms a set of independently selected competitive cells into a structured defence. This chapter analyses the properties that emerge from the guaranteed chain of models from each competitive cell to the root. + +### 16.1 Nesting Constraint Guarantee · `sec:sentinel:algorithm-ancestor-chain-nesting-constraint-guarantee` + +Every competitively selected cell has a **guaranteed complete chain** of models from itself to the root. For every value the adversary submits, the number of models it must simultaneously satisfy equals the number of materialised G-Tree ancestors of the target cell, plus one (the cell itself). + +Without the ancestor closure, nesting defence would work only when multiple cells on the same G-Tree path _happened_ to be independently promoted by V-Tree ranking. Under the investment set's ancestor closure, the constraint multiplication is structural and unconditional. Moreover, the g.sum-ordered warm-up pipeline (§11.6.2) ensures that at the moment any competitive target comes online, its entire ancestor chain is already online — the full defence depth is available from the first batch. + +### 16.2 Null-Space Coverage · `sec:sentinel:algorithm-ancestor-chain-null-space-coverage` + +At each level $\ell$ on the ancestor chain, the model operates on suffix width $w_\ell = N - \ell$ and has learned a rank-$k_\ell$ subspace from the observation population at that level. The model constrains $k_\ell$ directions and is blind to $w_\ell - k_\ell$ directions (its null space). + +The directions each model captures are **partially independent across levels** for three reasons: + +1. **Different observation populations.** Each level sees a different set of observations — broader at coarser levels. The statistical structure learned from a population of 100,000 observations at depth 0 differs from that learned from 500 observations at depth 48. + +2. **Different suffix dimensions.** Each level's suffix includes additional bit positions. The prefix bits at finer levels become suffix bits at coarser levels. A depth-16 model operates on $N - 16$ dimensions; a depth-48 model operates on $N - 48$ dimensions. The $32$ additional dimensions available to the depth-16 model carry cross-region correlations invisible at depth 48. + +3. **Cross-level correlations.** The correlations between the additional bits and the deeper suffix bits are precisely what the coarser models learn. These correlations connect the null spaces across levels. + +The adversary's "free" bits at level $\ell$ (the null space of the level-$\ell$ model) are constrained at level $\ell - 1$ with probability proportional to how much the coarser model captured cross-level correlations. Each ancestor level peels away some of the adversary's freedom. + +### 16.3 Defence Depth Scales With Observation Importance · `sec:sentinel:algorithm-ancestor-chain-defence-depth-by-observation-importance` + +Competitive cells sit at depths determined by the spatial layer's adaptive refinement — high-volume regions get deeper cells. The ancestor chain length for each competitive cell equals its G-Tree depth. Consequently, **the cells with the most volume have the longest ancestor chains and the most layers of defence.** + +An adversary targeting a busy depth-$d$ cell faces models at all $d + 1$ depths from 0 to $d$ — every materialised level on the ancestor path (§3.1). An adversary targeting a quiet depth-2 cell faces only depths 0, 1, and 2. Fewer constraints — but also a less significant target. The defence depth automatically scales with observation importance. + +### 16.4 Constraint Multiplication · `sec:sentinel:algorithm-ancestor-chain-constraint-multiplication` + +The per-value constraint count grows with ancestry depth. At each level, the model imposes approximately $k_\ell^2 / 2$ constraints (from the four scoring axes: $k_\ell$ novelty directions, 1 displacement scalar, $k_\ell$ surprise values, and $k_\ell(k_\ell - 1)/2$ coherence pairs). Across a chain of $D$ ancestor levels, the total constraint count is approximately: + +$$\sum_{\ell=0}^{D} \frac{k_\ell^2}{2}$$ + +These constraints are not fully independent (because the observations overlap), but neither are they redundant (because the populations and suffix dimensions differ). The effective constraint count grows meaningfully with depth, even accounting for overlap. + +### 16.5 Scale Diagnostics · `sec:sentinel:algorithm-ancestor-chain-scale-diagnostics` + +The ancestor chain provides diagnostic information about the **scale** at which an anomaly manifests: + +| Pattern | Interpretation | +| ------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------- | +| Anomalous at depth 48 only | Unusual relative to the specific cell's population but normal in the broader context — a local anomaly. | +| Anomalous at depths 16 and above | Unusual in the broader regional context — a stronger, coarser-scale signal. | +| Anomalous at depth 0, normal at depth 48 | The entire region is unusual, but this value is typical within it — the value inherits its region's anomaly. | +| Evasion succeeding at depth 48 but failing at depth 32 | The adversary matched local patterns but missed cross-cell correlations captured by the coarser model — the nesting defence in action. | + +The host receives scores at every ancestor level and can reconstruct the full scale profile without any additional mechanism. + +--- + +## Chapter 17. Compound Defence and Coverage · `sec:sentinel:algorithm-compound-defence-and-coverage` + +### 17.1 Interlocking Constraints: Ancestor Chain × Coordination · `sec:sentinel:algorithm-compound-defence-ancestor-chain-coordination-constraints` + +The ancestor chain (§16) and the coordination tier (§7) are not independent fallbacks. The investment set's ancestor closure guarantees the chain exists; the g.sum warm-up ordering guarantees it is online; the coordination tier operates on the producing competitive set. They operate simultaneously on the same observations and impose **interlocking constraints** — satisfying one makes satisfying the other harder. + +**The mechanisms' roles.** The ancestor chain provides _per-value depth_: every individual value must produce normal scores at every level from its target cell to the root. The coordination tier provides _cross-cell breadth_: the distribution of scores across spatially related competitive cells must look normal at every level of the hierarchy. + +**The interaction.** The same values that must score normally at level $\ell$ per-value also contribute to the coordination inputs at level $\ell - 1$. Per-value evasion at level $\ell$ constrains where in score space cell $c$'s summary can land, which constrains an element of the coordination input matrix at level $\ell - 1$. The optimisation itself leaves fingerprints. + +### 17.2 Why Evasion Optimisation Creates Detectable Correlations · `sec:sentinel:algorithm-compound-defence-evasion-correlations` + +Legitimate observations arise from independent sources operating independently across different regions of the domain. Score summaries across cells are approximately independent (modulo shared common-mode effects that the coordination tracker has already learned). + +Values optimised to evade per-value scoring have a different property: all crafted values are generated by the **same optimisation process** targeting the same set of models. Even if each value individually scores normally, the _population_ of crafted values shares structural regularities imposed by the optimisation: + +- **Clustering in safe zones.** If the adversary uses a common algorithm to find safe-zone values, the resulting values cluster in the safe zone rather than spreading across it the way legitimate observations do. +- **Complementary restriction.** If the adversary's values avoid specific null-space directions at a coarser level (to satisfy the ancestor chain), they cluster in the complement — a restriction that manifests as unusual displacement or surprise patterns at the coordination level. +- **Template reuse.** If the adversary reuses structural templates (modifying a few bits per value to maintain evasion while varying the functional content), the templates create correlated residual patterns across cells. + +Each coordination axis catches a different signature of optimisation-induced correlation: + +| Coordination Axis | Optimisation Fingerprint | +| ----------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Novelty** | The cross-cell score pattern is a _new kind_ not in the coordination subspace — the optimisation created a score distribution that normal observations never produce. | +| **Displacement** | All cells' scores are pinned near the same percentile of the safe zone — individually unremarkable but systematically elevated as a group. | +| **Surprise** | One specific scoring axis is systematically elevated across cells — the optimisation consistently produces slightly high surprise (or another axis) in every cell. | +| **Coherence** | Cross-axis correlations are unusual — cells with elevated novelty also have elevated displacement, because the safe-zone geometry links the two. | + +### 17.3 The Multi-Objective Dilemma · `sec:sentinel:algorithm-compound-defence-multi-objective-dilemma` + +Three objectives are in tension for an adversary: + +1. **Per-value evasion** (ancestor chain): each value must land in the intersection of safe zones across all ancestor levels. This pushes toward a specific region of the domain → uniformity. + +2. **Cross-cell diversity** (coordination): the distribution of scores across cells must match normal cross-cell distributions. This requires values sent to different cells to produce _different_ score profiles → against uniformity. + +3. **Cross-level consistency**: the same values' scores at different levels must be jointly consistent with normal cross-level statistics → constrains both. + +The adversary must solve a constrained optimisation that satisfies per-value evasion at all levels while producing statistically natural diversity across cells and consistency across levels. Adding calibrated noise to restore diversity helps but is insufficient: the perturbation must itself mimic the natural _structure_ of variation in legitimate observations, not just be "varied." Random perturbations within the safe zone produce uniform distributions within that zone, which may not match the actual distribution of baseline projections (typically clustered along specific subspace directions). + +### 17.4 The Irreducible Floor · `sec:sentinel:algorithm-compound-defence-irreducible-floor` + +The only remaining evasion strategy is values that are genuinely statistically identical to legitimate observations at every level, every cell, every axis, every cross-cell correlation, and every cross-level consistency measure. This requires characterising the baseline observations' complete joint distribution at every cell AND the cross-cell distribution AND the cross-level consistency — simultaneously, for every value, in every batch. + +For a practical adversary operating with imperfect knowledge, time constraints, and operational requirements (the values must serve some external purpose, not just look normal), the composition raises the bar very substantially. But it does not eliminate the possibility: **perfect mimicry of all distributions at all scales remains the irreducible floor** of any detection system based on distributional comparison. + +### 17.5 Evasion Assessment by Strategy · `sec:sentinel:algorithm-compound-defence-evasion-strategy-assessment` + +The following table summarises how each evasion strategy fares against each defence mechanism individually and in composition: + +| Evasion strategy | Ancestor chain alone | Coordination alone | Composed | +| ----------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Mimic local distribution at one cell | Must match $d$ models simultaneously | Not designed for single-cell evasion | Match $d$ models **and** produce cross-cell scores matching the natural joint distribution | +| Exploit null space of target model | Constrained by ancestor models' coverage of the null space (§16.2) | Irrelevant for single-cell evasion | Ancestors constrain null space; **coordination detects systematic null-space clustering across cells** | +| Gradual baseline manipulation | Must manipulate baselines at all $d$ levels simultaneously, each with different EWMA dynamics | Coordination drift accumulator detects cross-cell drift | Must manipulate $d$ baselines **without** creating correlated cross-cell drift patterns | +| Use common evasion algorithm across cells | Each cell evaded independently | Detects optimisation-induced uniformity across cells | **Individually invisible, jointly visible** — the optimisation fingerprint spans cells | +| Add noise to restore cross-cell diversity | Noise may violate per-value constraints across levels | Random noise may not match natural correlation structure | Must simultaneously satisfy per-value constraints at all levels **and** match natural cross-cell correlation structure | +| Patient sub-clip contamination | Must stay below $c_{\text{clip}}$ at all $d$ levels simultaneously while inflating baselines slowly enough that fast-slow gap never exceeds $\kappa$; multi-level EWMA dynamics differ, so the required injection rate is level-dependent | Sustained sub-clip presence across cells inflates coordination baselines, but correlated inflation across cells elevates coordination scores — coordination drift accumulator detects the cross-cell pattern | Must inflate $d$ per-cell baselines **and** coordination baselines at all levels without producing correlated cross-cell drift patterns; bounded by the same multi-objective dilemma as gradual manipulation, with the additional constraint that scores must remain below all clip ceilings throughout | +| Perfect mimicry of all distributions | Still works | Still works | Still works — the irreducible floor | + +> _Design note (patient sub-clip contamination vs. gradual baseline manipulation)._ The "gradual baseline manipulation" strategy assumes an adversary actively targeting baselines with knowledge of the EWMA dynamics. Patient sub-clip contamination is subtler and does not require such knowledge: any sustained adversarial presence producing moderately elevated scores below $c_{\text{clip}}$ will, as a side effect, inflate the fast EWMA $\bar{s}$ and eventually the slow EWMA $\bar{s}_{\text{slow}}$. If the adversary's influence grows slowly enough — less than $\kappa_\sigma \sqrt{\bar{v}_{\text{slow}}}$ per slow-EWMA time constant — the one-sided CUSUM accumulates nothing, because it detects _rate of change_ between fast and slow baselines, not _absolute regime level_. The slow baseline absorbs the contamination and the system's sensitivity to future anomalous traffic degrades invisibly to the drift accumulator. This is an inherent limitation of any adaptive baseline system: the mechanism that enables automatic adaptation to legitimate regime changes is the same mechanism an adversary exploits through patient contamination. The multi-level ancestor chain and coordination tier constrain this attack (the adversary must inflate baselines at all levels without producing correlated cross-cell patterns), but do not eliminate it. Hosts concerned about long-horizon contamination should monitor secular trends in the reported $\bar{s}_{\text{slow}}$ and $\bar{\rho}$ values (§14.5, §14.4) across trackers and flag sustained monotonic increases. This monitoring is a host-level policy responsibility consistent with "Measure, don't decide" (§1.3). + +### 17.6 Coverage Matrix · `sec:sentinel:algorithm-compound-defence-coverage-matrix` + +The combination of per-cell scoring, ancestor chain scoring, and hierarchical coordination covers six anomaly modalities across three temporal patterns: + +``` + Sudden Gradual Sudden Gradual Sudden Gradual + single single partial partial system system + +Per-cell z ✓ strong ✗ absorbed ✓ each ✗ absorbed ✗ mild ✗ both +Per-cell drift redundant ✓ drift redundant ✓ each ✗ mild ✗ diluted +Ancestor chain ✓ depth ✓ multi- ✓ depth ✓ multi- ✓ root ✓ root + defence baseline defence baseline catches drift +Coord (local) ✗ few ✗ few ✓ asymm ✓ asymm ✗ n/a ✗ n/a +Coord (mid) ✗ diluted ✗ diluted ✓ struct ✓ accum ✓ moderate ✓ moderate +Coord (root) ✗ diluted ✗ diluted ✗ diluted ✗ diluted ✓ strong ✓ accum +``` + +**Every modality is detected by at least one mechanism.** The hardest case — gradual system-wide coordination — is caught by the root drift accumulator, which accumulates aggregate departure across all $K$ cells at signal-to-noise ratio proportional to $\sqrt{K}$. + +### 17.7 Compound Effect Summary · `sec:sentinel:algorithm-compound-defence-effect-summary` + +The ancestor chain and coordination tier do not merely add constraints — they create conflicting objectives. The ancestor chain forces per-value evasion into narrow safe zones, creating the very uniformity that the coordination tier detects. The coordination tier forces cross-cell diversity, creating degrees of freedom that the ancestor chain constrains. The composition raises evasion from a single-model matching problem to a multi-scale, multi-cell, multi-axis constrained optimisation with no known efficient solution — except producing values genuinely indistinguishable from normal observations at every scale simultaneously. + +--- + +# Appendices · `sec:sentinel:algorithm-appendices` + +--- + +## Appendix A. Empirical Warm-Up Analysis · `sec:sentinel:algorithm-empirical-warmup-analysis` + +This appendix provides the methodology and extended data behind the convergence times reported in §11.9. These results characterise the interaction between the cold-start mitigations (§11.2–11.4) and the baseline tracking machinery (§6), and inform the noise schedule parameters recommended in §13.5. + +**Data currency note.** The convergence times below were measured before the EWMA-mean-centred variance formula ([ADR-S-021](../adr/021-ewma-mean-centred-variance.md)) was implemented. Post-implementation measurements show a modest improvement: worst-case convergence at the production configuration ($\lambda = 0.99$, $b = 16$) improved from 289 to 282 rounds, consistent with the elimination of the original formula's $-6.25\%$ variance bias at $b = 16$. The absolute convergence times reported below remain valid as conservative upper bounds. + +### A.1 Methodology · `sec:sentinel:algorithm-empirical-warmup-methodology` + +**Setup.** A single subspace tracker at analysis width $w = 96$ and capacity $\text{cap} = 16$ is created, noise-injected according to the configured schedule, and then fed uniformly random centred bit vectors ($\{-0.5, +0.5\}^{96}$, independent and identically distributed) for a sustained observation period. The structureless input ensures that all observed convergence dynamics are properties of the tracker machinery, not of input structure. + +**Measurement.** For each scoring axis, the fast EWMA mean $\bar{s}$ is recorded after every batch. A windowed mean over the last 50 batches is computed and compared against a reference value obtained from a long burn-in run (10,000+ batches). Trajectory convergence is declared when the windowed mean enters and remains within a tolerance band (1% relative error for novelty and displacement; 5% for surprise and coherence, reflecting their higher intrinsic variability). + +**Controls.** Each configuration is tested across 20 independent random seeds. The convergence time reported is the median; the range across seeds is reported where it is informative (particularly for displacement at small batch sizes, which exhibits bimodal convergence). + +### A.2 Cold-Start Mitigations and Their Effects · `sec:sentinel:algorithm-empirical-warmup-cold-start-mitigations` + +Three mitigations operate during the noise-to-real transition. Each addresses a specific source of convergence delay: + +| Mitigation | Source of delay addressed | Effect on convergence | +| --------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Cold-start latent seeding (§11.2) | Latent variance $\nu^{(z)}$ initialises far from steady state → surprise score non-stationary for tens of rounds | Eliminates latent variance mismatch at $t = 0$; surprise converges in $\sim$65 rounds instead of $\sim$200+ | +| Clip-pressure modulation (§6.1.1, §6.4) | First batch produces near-zero variance → ultra-tight clip ceiling → self-reinforcing low-baseline loop; also, stale baselines cause permanent lockout in production | Breaks the positive feedback loop during warm-up (via $\eta$); prevents permanent lockout in production (via $\bar{\rho}$); displacement converges monotonically instead of exhibiting plateau behaviour | +| Slow-from-fast seeding (§11.4) | Slow EWMA half-life ($\sim$693 steps) far exceeds fast EWMA half-life ($\sim$69 steps) → drift accumulator interprets the gap as evidence | Eliminates false drift accumulation at the noise-to-real boundary; reduces spurious accumulation by $\sim$97% | + +### A.3 Convergence Times by Configuration · `sec:sentinel:algorithm-empirical-warmup-convergence-times-by-configuration` + +**Configuration A: $\lambda = 0.95$, $b_{\text{noise}} = 4$** + +| Component | Without mitigations | With all mitigations | Notes | +| --------------------------- | ---------------------------: | -------------------: | ---------------------------------------------------------------------------------------------- | +| Novelty baseline | 21 | 21 | Unaffected — constant-norm property makes novelty inherently stable | +| Displacement baseline | 500+ (plateau) | 21–475 (bimodal) | Bimodality across seeds: 14/20 seeds converge by round 21; 6/20 seeds require $\sim$475 rounds | +| Surprise baseline | 200+ | **65** | Cold-start seeding is the critical mitigation | +| Coherence baseline | 500+ | **406** | Dominated by rank-gating delay: $k < 2$ for the first $\sim$100–300 rounds | +| Drift accumulator (novelty) | 198 units false accumulation | 5.7 units | Slow-from-fast seeding is the critical mitigation | + +**Configuration B: $\lambda = 0.95$, $b_{\text{noise}} = 16$** + +| Component | Without mitigations | With all mitigations | Notes | +| --------------------------- | ------------------: | -------------------: | ---------------------------------------------------------------- | +| Novelty baseline | 21 | 21 | | +| Displacement baseline | 300+ | 21 | Larger batch eliminates bimodality | +| Surprise baseline | 200+ | **70** | | +| Coherence baseline | 500+ | **173** | Larger batch accelerates rank growth → shorter rank-gating delay | +| Drift accumulator (novelty) | 180 units | 4.2 units | | + +**Configuration C: $\lambda = 0.99$, $b_{\text{noise}} = 16$** + +| Component | Without mitigations | With all mitigations | Notes | +| --------------------------- | ------------------: | -------------------: | ---------------------------------------------------- | +| Novelty baseline | 101 | 101 | Slower $\lambda$ → proportionally longer convergence | +| Displacement baseline | 800+ | 101 | | +| Surprise baseline | 1000+ | **398** | Slowest-converging axis at this $\lambda$ | +| Coherence baseline | 800+ | **315** | | +| Drift accumulator (novelty) | 450 units | 8.1 units | | + +### A.4 Displacement Bimodality at Small Batch Sizes · `sec:sentinel:algorithm-empirical-warmup-small-batch-displacement-bimodality` + +At $b_{\text{noise}} = 4$ with $\lambda = 0.95$, displacement convergence exhibits bimodal behaviour across random seeds: a majority of seeds converge rapidly ($\sim$21 rounds), while a minority require $\sim$475 rounds with a visible plateau. + +The mechanism: displacement's bounded range $[0, 1)$ compresses near zero, where the initial noise-warmed baseline sits. Small batches ($b_{\text{noise}} = 4$) produce high variance in the batch displacement mean. If the first few real batches happen to produce displacement values slightly above the noise-warmed baseline, the clip ceiling widens naturally and convergence proceeds. If they produce values slightly below, the baseline drifts downward, tightening the clip ceiling from below (a direction the upper-tail filter does not protect against), creating a slow recovery. + +At $b_{\text{noise}} = 16$, the batch mean variance is $4\times$ smaller, eliminating the bimodality. This effect is specific to displacement at small batch sizes and does not manifest in the other three axes. + +### A.5 Coherence Convergence Bottleneck · `sec:sentinel:algorithm-empirical-warmup-coherence-bottleneck` + +Coherence is defined only when $k \geq 2$ (§5.5). A newly created tracker starts at $k = 1$ and must accumulate sufficient rank before coherence baselines can even begin tracking. The rank adaptation mechanism (§4.2, Phase 5) evaluates every $T_{\text{rank}}$ steps and moves rank by at most one step, so the minimum time to $k = 2$ is $T_{\text{rank}}$ batches. + +In practice, rank growth depends on the energy distribution across singular values. Under structureless noise (uniform random bits), singular values are approximately equal, so the energy threshold $\tau = 0.90$ requires $k \approx 0.9 \times \text{cap}$ — many more than 2. Rank reaches 2 relatively quickly (within $2 \times T_{\text{rank}}$ batches typically), but the coherence baseline then requires its own convergence time on top of the rank-gating delay. + +The rank-gating delay is the dominant contributor to coherence being the slowest-converging axis. + +### A.6 Steady-State Variability · `sec:sentinel:algorithm-empirical-warmup-steady-state-variability` + +Even after convergence, baseline values exhibit ongoing variability. The following coefficients of variation (standard deviation of windowed-mean divided by its mean, measured over 1,000 post-convergence batches) characterise the intrinsic wander: + +| Axis | CV ($b_{\text{noise}} = 4$) | CV ($b_{\text{noise}} = 16$) | Interpretation | +| ------------ | :-------------------------: | :--------------------------: | -------------------------------------------------------------------------------------- | +| Novelty | 0.13% | 0.06% | Extremely stable — the constant-norm property (§2.5) eliminates input energy variation | +| Displacement | 5.9% | 3.0% | Moderate — bounded range compresses variance reporting | +| Surprise | 9.0% | 3.6% | Moderate — sensitive to per-dimension variance fluctuations | +| Coherence | 19.9% | 11.5% | High — pairwise products amplify input variance; $k(k{-}1)/2$ terms compound | + +These CVs represent a **floor** on baseline precision: no amount of warm-up time reduces variability below these values. They should be considered when interpreting z-scores — a z-score of 2.0 on an axis with 20% CV is less significant than a z-score of 2.0 on an axis with 0.1% CV. + +### A.7 Noise Schedule Implications · `sec:sentinel:algorithm-empirical-warmup-noise-schedule-implications` + +The noise schedule must produce enough rounds at each depth for the worst-case axis (typically coherence or surprise) to reach steady state before real observations arrive. The default geometric schedule (root = 450, decay = 0.5, minimum = 50) is calibrated against Configuration C ($\lambda = 0.99$, $b_{\text{noise}} = 16$) — the default forgetting factor: + +| Depth | Rounds (default schedule) | Worst-case convergence ($\lambda = 0.99$) | Margin | +| ----: | ------------------------: | ----------------------------------------: | -------- | +| 0 | 450 | 398 (surprise) | +13% | +| 1 | 225 | $\sim$200 | +13% | +| 3 | 56 | $\sim$50 | +12% | +| 4+ | 50 (minimum) | $\sim$50 | Adequate | + +The previous default (root = 50, minimum = 10) was calibrated against $\lambda = 0.95$ and produced **negative margin** at the default $\lambda = 0.99$. The updated default ensures baselines are settled before real observations arrive for all default parameters. + +For non-default $\lambda$ values, the schedule should be adjusted. Recommended configurations: + +| Configuration | Root | Decay | Minimum | +| -------------------------------------------- | ---: | ----: | ------: | +| $\lambda = 0.95$, $b_{\text{noise}} \geq 16$ | 50 | 0.5 | 10 | +| $\lambda = 0.95$, $b_{\text{noise}} = 4$ | 200 | 0.5 | 15 | +| $\lambda = 0.99$, $b_{\text{noise}} \geq 16$ | 450 | 0.5 | 50 | +| $\lambda = 0.99$, $b_{\text{noise}} = 4$ | 500 | 0.5 | 50 | + +These values include a $\sim$13–15% margin above the measured worst-case convergence times. The geometric decay factor of 0.5 reflects the empirical observation that convergence accelerates at narrower analysis widths (fewer dimensions → simpler subspace → faster baseline settling). The minimum of 50 for $\lambda = 0.99$ configurations accounts for the $\sim$5× longer convergence at high $\lambda$ even at deep (narrow) cells. + +--- + +## Appendix B. Glossary · `sec:sentinel:algorithm-glossary` + +| Term | Definition | Reference | +| ----------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------- | +| Analysis budget ($K$) | Maximum number of competitive targets; drives the investment set size | §8.1 | +| Analysis width ($w$) | Number of suffix bits available for statistical analysis at a given cell; $w = N - d$ | §2.4 | +| Ancestor closure | The operation that extends the competitive targets to include all spatial ancestors, forming the investment set $\mathcal{I}$ | §8.2 | +| Baseline | Running mean and variance of a scoring axis, maintained by EWMA, against which z-scores are computed | §6.1 | +| Centred bit vector | Representation of a coordinate value as a vector in $\{-0.5, +0.5\}^N$ | §2.3 | +| Coherence | Scoring axis measuring departure of pairwise latent products from their learned second moments | §5.5 | +| Competitive targets ($\mathcal{T}$) | The top $K$ V-entries by importance within the depth cutoff; the selection that drives the investment set | §8.1 | +| Constant-norm property | The fact that every centred binary suffix vector has squared norm $w/4$ | §2.5 | +| Contour | The step function over the domain formed by the observation-receiving surface of the spatial tree | §3.2 | +| Coordination context | An internal spatial node whose subtree contains competitive cells in both children; hosts a coordination tracker | §7.1 | +| Coordination tracker | A subspace tracker operating on cross-cell score summaries ($w = 4$) at a coordination context | §7.5 | +| Degenerate cell | A spatial cell with analysis width $w < 2$, excluded from the investment set because the subspace algebra requires $w \geq 2$; reported via the degenerate-cell counter (§14.11) | §4.1, §8.2 | +| Displacement | Scoring axis measuring total within-subspace energy as a bounded distance from the subspace origin | §5.3 | +| Clip-pressure EWMA ($\bar{\rho}$) | Per-axis running average of the batch clip ratio; modulates the effective clip ceiling to prevent baseline lockout | §6.1.1, §6.4 | +| Drift accumulator | One-sided CUSUM detecting sustained upward drift of raw (pre-clip) batch means from the slow EWMA baseline | §6.3 | +| Dyadic cell | An interval of the form $[a \cdot 2^{N-d}, (a+1) \cdot 2^{N-d})$ at depth $d$ in the spatial tree | §2.1, §3.2 | +| Eager removal | Immediate destruction of trackers and cancellation of warm-up when a cell exits the investment set, within the Step 3 reconciliation | §8.5 | +| Fast EWMA | Exponentially weighted moving average at decay $\lambda$ tracking running mean and variance of a scoring axis | §6.1 | +| Feed-forward invariant | The guarantee that anomaly scores never influence the spatial layer's importance signal; $\Delta = 1$ always | §1.3 | +| Forgetting factor ($\lambda$) | EWMA decay rate controlling the balance between memory and adaptation | §4.2 | +| Full analysis set ($\mathcal{A}^*$) | See: Producing full set | §8.3 | +| G-Tree (Geometric Tree) | The binary tree over dyadic intervals forming the spatial partitioning structure | §3.1 | +| g.sum | A G-Tree node's total accumulation: its own value plus all descendant sums; used as warm-up priority | §3.12, §11.6.2 | +| Investment set ($\mathcal{I}$) | Competitive targets closed under spatial ancestry; all members have allocated trackers regardless of online status | §8.2 | +| Max-uncle constraint | The V-Tree invariant: no node may outrank all of its uncles | §3.4 | +| Noise influence ($\eta$) | Fraction of a tracker's batch-updated model not yet established by real traffic; decays as $\lambda^k$ over real batches | §11.5 | +| Noise injection | Feeding synthetic random centred bit vectors through a tracker to warm its baselines before real observations arrive | §11.1 | +| Novelty | Scoring axis measuring average residual energy per degree of freedom outside the learned subspace; degenerate when $k = w$ | §5.2 | +| Novelty-saturated | Condition where $k = w$ and the novelty axis is identically zero | §5.2, §14.7 | +| Novelty-saturable | Condition where $\text{cap} \geq w$, meaning novelty can become saturated as rank adapts | §14.7 | +| Plateau | A maximal contiguous run of contour cells at a single depth | §3.2 | +| Polarity invariant | The convention that higher scores indicate greater anomalous departure; shared by all four axes | §5.1 | +| Producing competitive set ($\mathcal{A}$) | Online competitive targets; $\mathcal{T} \cap \text{Online}$ | §8.3 | +| Producing full set ($\mathcal{A}^*$) | Online members of the investment set; $\mathcal{I} \cap \text{Online}$ | §8.3 | +| Root tracker | The permanent subspace tracker at the G-Tree root ($w = N$), seeing every observation | §8.4 | +| Slow EWMA | EWMA at decay $\lambda_s > \lambda$ providing a long-memory reference for the drift accumulator | §6.2 | +| Steiner tree property | The current investment set retains all ancestor paths and the permanent root: $\lVert\mathcal{I}\rVert \leq 1 + \sum_i d_i \leq 1 + \texttt{analysis\_k}(N-2)$ for supported engines. Suppressing unmarked unary intermediates yields at most $2K$ reduced nodes for $K \geq 1$ targets, but those intermediates still require cell trackers. | §8.2, §8.6 | +| Subspace tracker | The per-cell statistical model maintaining a low-rank subspace, latent statistics, baselines, and drift accumulators | §4 | +| Suffix | The trailing $w = N - d$ bits of an observation's centred bit vector, after the $d$ routing prefix bits are removed | §2.4 | +| Surprise | Scoring axis measuring average diagonal Mahalanobis deviation of latent coordinates from their learned means | §5.4 | +| V-Tree (Value Tree) | The dynamic tournament bracket ranking spatial cells by competitive importance | §3.1, §3.4 | + +--- + +## Appendix C. Notation Quick Reference · `sec:sentinel:algorithm-notation-quick-reference` + +Reproduced from §2.6 for convenience. + +| Symbol | Domain | Definition | +| --------------------- | ------------------------------------------- | --------------------------------------------------------------------------------------------- | +| $N$ | $\mathbb{Z}_{>0}$ | Domain bit-width; spatial tree height | +| $d$ | $\{0, \ldots, N\}$ | Spatial tree depth of a cell (prefix length in bits) | +| $w$ | $\{0, \ldots, N\}$ | Analysis width; $w = N - d$ (suffix length) | +| $n$ | $\mathbb{Z}_{\geq 0}$ | Ingestion batch size (coordinate values per `SentinelIngest` call) | +| $b$ | $\mathbb{Z}_{>0}$ | Per-tracker batch size (rows of $X$ fed to one tracker in one call; see §2.6 batch-size note) | +| $b_{\text{noise}}$ | $\mathbb{Z}_{>0}$ | Noise batch size (synthetic samples per warm-up round; §11.1) | +| $k$ | $\{1, \ldots, \text{cap}\}$ | Current active rank of the learned subspace | +| $\text{cap}$ | $\{1, \ldots, \min(w, r_{\max})\}^\dagger$ | Hard ceiling on rank | +| $\lambda$ | $(0, 1)$ | Forgetting factor | +| $\alpha$ | $(0, 1)$ | Learning rate; $\alpha = 1 - \lambda$ | +| $\varepsilon$ | $\mathbb{R}_{>0}$ | Numerical stability constant | +| $\tau$ | $(0, 1)$ | Cumulative energy threshold | +| $U$ | $\mathbb{R}^{w \times \text{cap}}$ | Orthonormal basis of the learned subspace | +| $\sigma$ | $\mathbb{R}^{\text{cap}}_{\geq 0}$ | Singular values | +| $\mu^{(z)}$ | $\mathbb{R}^{\text{cap}}$ | EWMA mean of latent coordinates | +| $\nu^{(z)}$ | $\mathbb{R}^{\text{cap}}_{>0}$ | EWMA variance of latent coordinates | +| $\Gamma$ | $\mathbb{R}^{\text{cap} \times \text{cap}}$ | EWMA second-moment matrix of latent coordinates | +| $X$ | $\mathbb{R}^{b \times w}$ | Suffix observation matrix | +| $Z$ | $\mathbb{R}^{b \times k}$ | Latent projection of $X$ | +| $\hat{X}$ | $\mathbb{R}^{b \times w}$ | Reconstruction of $X$ from $Z$ | +| $m$ | $\mathbb{Z}_{>0}$ | Coordination group size | +| $\lambda_s$ | $(\lambda, 1)$ | Slow EWMA decay for per-tracker drift detection | +| $\lambda_{s,m}$ | $(\lambda, 1)$ | Slow EWMA decay for coordination drift detection | +| $\kappa_\sigma$ | $\mathbb{R}_{\geq 0}$ | Drift accumulator noise allowance in slow-baseline $\sigma$ units | +| $n_\sigma$ | $\mathbb{R}_{>0}$ | Outlier clip width in $\sigma$ units | +| $S$ | $\mathbb{R}_{\geq 0}$ | Drift accumulator value | +| $\bar{\rho}$ | $[0, 1]$ | Per-axis clip-pressure EWMA | +| $\lambda_\rho$ | $(0, 1)$ | Clip-pressure EWMA decay rate | +| $\rho_t$ | $[0, 1]$ | Batch clip ratio | +| $\mu^{(\text{in})}$ | $\mathbb{R}^4$ | Running-mean centring reference for coordination input | +| $\eta$ | $[0, 1]$ | Noise influence fraction (tracker maturity) | +| $K$ | $\mathbb{Z}_{>0}$ | Maximum number of competitively selected analysis cells | +| $L$ | $\mathbb{Z}_{\geq 0}$ | V-Tree depth cutoff for analysis eligibility | +| $\mathcal{A}$ | $\subseteq$ V-entries | Producing competitive set ($\mathcal{I} \cap \mathcal{T} \cap \text{Online}$) | +| $\mathcal{A}^*$ | $\supseteq \mathcal{A}$ | Producing full set ($\mathcal{I} \cap \text{Online}$) | +| $\mathcal{T}$ | $\subseteq$ V-entries | Competitive targets (top $K$ by importance within depth cutoff) | +| $\mathcal{I}$ | $\subseteq$ V-entries + ancestors | Investment set (competitive targets + spatial ancestors, regardless of online status) | +| $G_{\max}$ | $\mathbb{Z}_{\geq 5}$ | Hard ceiling on total spatial nodes | +| $\lambda_{\text{sp}}$ | $[0, \infty)$ | Spatial decay attenuation factor | +| $h_V$ | $\mathbb{Z}_{\geq 0}$ | V-Tree height (maximum root-to-leaf path length in the V-Tree) | + +$^\dagger$ The domain $\{1, \ldots, \min(w, r_{\max})\}$ is non-empty only when $w \geq 1$. Tracker creation requires $w \geq 2$ (§4.1); cells below this threshold are excluded from $\mathcal{I}$. + +Vectors are row vectors when representing observations and column vectors when representing basis directions. Subscript $i$ indexes samples; subscript $j$ indexes subspace dimensions. diff --git a/packages/sentinel/docs/api.md b/packages/sentinel/docs/api.md new file mode 100644 index 000000000..b03613b52 --- /dev/null +++ b/packages/sentinel/docs/api.md @@ -0,0 +1,885 @@ +# Sentinel — Public API Reference · `spec:sentinel:api-reference` + +The definitive specification for the public surface of the `torrust-sentinel` crate. Everything listed here is intentionally public. Everything else is `pub(crate)` or private — implementation detail, subject to change without notice. + +Modelled on `packages/mudlark/docs/api.md`. + +--- + +## §1. Design Principles · `sec:sentinel:api-design-principles` + +1. **Measure, don't decide.** All outputs are raw statistical quantities. The sentinel never emits threat levels, recommended actions, or policy decisions. The host reads the reports and applies its own policy. + +2. **Feed-forward invariant (ADR-S-002).** The G-V Graph receives only `observe(v, 1u64)` per raw input value. Anomaly scores never flow back into the spatial layer's importance signal. The spatial layer sees pure volume counting — Δ=1 per observation. This prevents the anomaly detector from influencing its own spatial structure. + +3. **Host controls temporal policy.** The sentinel never calls `decay()` automatically. The host schedules decay: when, how aggressively, and with what selectivity (§ALGO S-13.4). + +4. **Adapt, don't control.** The spatial structure evolves autonomously under observation and decay. The host controls resource ceilings, temporal policy, and analysis budgets — not the structure itself. + +> _Design note (why volume-only importance)._ If anomaly scores boosted importance, the affected cell would earn finer resolution, changing its statistical model, changing its scores, changing its importance — an unstable feedback loop. With Δ=1, an adversary cannot influence the spatial structure except through observation volume, which is precisely what the spatial layer is designed to handle. + +--- + +## §2. Three-Layer Architecture · `sec:sentinel:api-three-layer-architecture` + +``` +Layer 1: Spatial Index (mudlark GvGraph) + Adaptive spatial partitioning of [0, 2^N) + Pure volume tracking (Δ = 1 per observation) + Competitive ranking by observation volume + │ + │ Significance ranking → top-K selection + ▼ +Layer 2: Analysis Selector + Selects significant cells for statistical analysis + Closes selection under spatial ancestry + │ + │ Suffix bit vectors at every ancestor depth + ▼ +Layer 3: Analysis Engine (subspace trackers, coordination) + Per-cell subspace models at selected and ancestor cells + Hierarchical coordination across related cells + │ + ▼ + BatchReport → host +``` + +### §2.1 Layer Responsibilities · `sec:sentinel:api-layer-responsibilities` + +| Concern | Owner | +| ----------------------------------------------------- | ----------------------------- | +| Spatial partitioning of $[0, 2^N)$ | Spatial Layer (Layer 1) | +| Competitive significance ranking | Spatial Layer — Value Tree | +| Spatial lifecycle (split, evict, absorb, restore) | Spatial Layer | +| Spatial memory (temporal decay, contour evolution) | Spatial Layer + host policy | +| Investment commitment (which cells receive trackers) | Analysis Selector (Layer 2) | +| Production selection (which invested cells score) | Analysis Selector (Layer 2) | +| Ancestor closure (spatial ancestry of targets) | Analysis Selector (Layer 2) | +| Statistical modelling within each cell | Analysis Engine (Layer 3) | +| Anomaly scoring (four axes) | Analysis Engine | +| Drift detection (CUSUM accumulators) | Analysis Engine | +| Cross-cell coordination detection | Analysis Engine — coordination | +| Interpretation and response | Host (external) | + +See §ALGO S-1.4–1.5 for the full architecture specification. + +--- + +## §3. Crate Root Re-exports · `sec:sentinel:api-crate-root-reexports` + +```rust +// Type aliases (convenience). +pub type Sentinel128 = SpectralSentinel; +pub type Sentinel64 = SpectralSentinel; + +// Re-exported from mudlark for decay_subtree() handles. +pub use torrust_mudlark::GNodeId; + +// Modules are crate-private. +pub(crate) mod analysis_set; +pub(crate) mod config; +pub(crate) mod ewma; +pub(crate) mod maths; +pub(crate) mod observation; +pub(crate) mod report; +pub(crate) mod sentinel; + +// Surface 1 — report/view types. +pub use report::{ + AnalysisSetSummary, AnomalyScores, AxisBaselineSnapshots, + BaselineSnapshot, BatchReport, CellInspection, CellReport, + ClipPressureDistribution, ContourSnapshot, CoordinationHealth, + CoordinationReport, CusumSnapshot, GeometryDistribution, + HealthReport, MaturityDistribution, MemberScore, + RankDistribution, SampleScore, ScoreDistribution, + ScoringGeometry, TrackerMaturity, +}; + +// Surface 2 — operational types. +pub use analysis_set::{AnalysisEntry, AnalysisSet}; +pub use config::{ + ConfigError, ConfigErrors, ConfigWarning, NoiseSchedule, + SentinelConfig, +}; +pub use maths::SvdStrategy; +pub use observation::{CentredBitSource, CentredBits}; +pub use sentinel::SpectralSentinel; + +// Not re-exported: MIN_TRACKER_DIM (pub(crate) const). +``` + +The crate root re-exports nothing hidden: the list above is the whole of it. Internal machinery — EWMA state, SVD update plumbing, the per-tracker report, the tracker itself, staging, and CUSUM — lives in crate-private modules and is reachable by no path from outside the crate, so every name a consumer can write is a name the crate supports under the stability guarantee. + +The type aliases wrap `SpectralSentinel` for the most common domain widths: + +| Alias | $C$ | $V$ | $N$ | Use case | +| ------------- | -------- | ------ | ---- | ---------------------------- | +| `Sentinel128` | `u128` | `u64` | 128 | IPv6-class data (default) | +| `Sentinel64` | `u64` | `u64` | 64 | 64-bit domains | + +--- + +## §4. Surface 1 — Report Types · `sec:sentinel:api-report-types` + +Lightweight, read-only **view types** — detached from the sentinel. Returned by `ingest()`. All carry `Debug`, `Clone`, `serde` (with feature). + +These types carry raw statistical measurements — never opinions or recommended actions. The host reads these reports and applies its own policy. + +### §4.1 Batch-level Reports · `sec:sentinel:api-batch-level-reports` + +#### `BatchReport` — `Clone` · `sec:sentinel:api-batch-report` + +Complete statistical output from one `ingest()` call. + +```rust +pub struct BatchReport { + pub cell_reports: Vec>, + pub ancestor_reports: Vec>, + pub coordination_reports: Vec>, + pub contour: ContourSnapshot, + pub health: HealthReport, + pub analysis_set_summary: AnalysisSetSummary, + pub oldest_observation_age_micros: Option, +} +``` + +| Field | Content | +| ------------------------------- | -------------------------------------------------------- | +| `cell_reports` | Per-cell reports for competitive cells ($\mathcal{A}$) | +| `ancestor_reports` | Per-cell reports for ancestor-only cells | +| `coordination_reports` | Cross-cell coordination analysis (§ALGO S-7.4) | +| `contour` | Snapshot of G-V Graph spatial structure | +| `health` | Operational health snapshot | +| `analysis_set_summary` | Summary of investment/producing sets | +| `oldest_observation_age_micros` | Age of the batch's oldest observation at report emission, in microseconds; absent where there is none | + +The vector fields are ordered deterministically (ADR-S-005), but not all by the same key. `cell_reports` and `ancestor_reports` are ordered by ascending `GNodeId`: both come from one walk of the cell map, which is keyed by handle, and partitioning that walk into the competitive and ancestor-only halves preserves its order within each. `coordination_reports` is ordered by ascending depth first, and by ascending `GNodeId` only among contexts of equal depth — so the root leads rather than trails, which the emitting walk's own post-order would otherwise have produced. A consumer that assumes one handle ordering across all three, or that merges them and re-sorts on handle alone, loses the depth ordering the coordination vector carries. + +`oldest_observation_age_micros` is a duration on the sentinel's own monotonic clock — stamped as the batch arrives, read off as the report is assembled — and never a wall-clock instant, so it carries no cross-machine skew. A batch arrives whole, so the single figure bounds every observation in it. `None` is reported both for a batch that carried no observations and for a payload written before the field existed: neither carries a measurement, and a zero would claim one. The field deserialises with `#[serde(default)]`, so older payloads read back unchanged. How long the host held the observations before handing them over is not included and is deliberately unmeasured (`sec:sentinel:algorithm-output-observation-age`). + +### §4.2 Cell-level Reports · `sec:sentinel:api-cell-level-reports` + +#### `CellReport` — `Clone` · `sec:sentinel:api-cell-report` + +Statistics for a single analysis cell after processing one batch. + +```rust +pub struct CellReport { + pub gnode_id: GNodeId, + pub start: C, + pub end: C, + pub depth: u32, + pub analysis_width: usize, + pub is_competitive: bool, + pub sample_count: usize, + pub rank: usize, + pub energy_ratio: f64, + pub top_singular_value: f64, + pub scores: AnomalyScores, + pub maturity: TrackerMaturity, + pub geometry: ScoringGeometry, + pub per_sample: Option>, +} +``` + +| Field | Content | +| ------------------- | ---------------------------------------------------- | +| `gnode_id` | Arena handle of the backing G-node | +| `start`, `end` | Dyadic interval `[start, end)`; `end` belongs to the cell only when it is the domain's last value, which is the case for a coordinate type that cannot represent `2^N` at its full width and so names its domain maximum as the bound. A type that can represent `2^N` excludes `end` at every width | +| `depth` | G-tree depth of this cell | +| `analysis_width` | Suffix width: `N - depth` | +| `is_competitive` | `true` if competitively selected | +| `sample_count` | Observations in this batch routed to this cell | +| `rank` | Rank in force while this batch was scored | +| `energy_ratio` | Fraction of variance captured by the reported rank | +| `top_singular_value`| Largest singular value | +| `scores` | Anomaly scores along all four axes | +| `maturity` | Tracker maturity (real vs noise observations) | +| `geometry` | Geometry of the model that scored this batch | +| `per_sample` | Per-sample scores (if `per_sample_scores` enabled) | + +### §4.3 Coordination Reports · `sec:sentinel:api-coordination-reports` + +#### `CoordinationReport` — `Clone` · `sec:sentinel:api-coordination-report` + +Coordination analysis at a single G-tree internal node (§ALGO S-7.1). + +The coordination tracker operates at $w = 4$, consuming running-mean-centred cell-score matrices as observations. The group consists of all competitive cells in this node's subtree that reported scores in this batch. Its `rank`, `energy_ratio`, and `geometry` describe the model that scored this batch; rank adaptation prepares the next batch. + +```rust +pub struct CoordinationReport { + pub gnode_id: GNodeId, + pub start: C, + pub end: C, + pub depth: u32, + pub cells_reporting: usize, + pub rank: usize, + pub energy_ratio: f64, + pub top_singular_value: f64, + pub scores: AnomalyScores, + pub maturity: TrackerMaturity, + pub geometry: ScoringGeometry, + pub per_member: Option>>, +} +``` + +The four anomaly axes have **second-order meaning** at coordination level: + +| Meta-axis | Detects | +| ------------ | ------------------------------------------------ | +| Novelty | A cell-score pattern the model has never seen | +| Displacement | The overall score landscape has shifted | +| Surprise | A specific scoring axis is system-wide anomalous | +| Coherence | An unusual combination of axis elevations | + +### §4.4 Anomaly Scores · `sec:sentinel:api-anomaly-scores` + +#### `AnomalyScores` — `Clone` · `sec:sentinel:api-anomaly-scores-type` + +The four anomaly-score axes for a batch of observations. + +```rust +pub struct AnomalyScores { + pub novelty: ScoreDistribution, + pub displacement: ScoreDistribution, + pub surprise: ScoreDistribution, + pub coherence: ScoreDistribution, +} +``` + +| Axis | Metric | Intuition | +| ------------ | ------------------------------------------- | -------------------------------------------------- | +| Novelty | Residual energy / DOF: `‖X − X̂‖² / (dim−k)`| "How much of this is foreign?" | +| Displacement | `‖z‖² / (k + ‖z‖²)`, bounded in `[0, 1)` | "How far is this from the centroid?" | +| Surprise | Mahalanobis / rank: `Σⱼ ((zⱼ−μⱼ)/σⱼ)² / k` | "The shape is familiar, but magnitude is wild" | +| Coherence | Cross-correlation deviation | "Normal individually, unusual combination" | + +All axes share the same polarity: **higher values indicate greater anomalous departure**. This ensures uniform z-score interpretation. + +> **Why not projection energy / "normality"?** Under the sentinel's centred binary encoding, every observation has the same L2 norm (`dim / 4`). Projection energy is therefore a perfect affine function of residual energy — it carries zero independent information. See §ALGO S-15.2. + +#### `ScoreDistribution` — `Copy` · `sec:sentinel:api-score-distribution` + +Summary statistics for a vector of anomaly scores. + +```rust +pub struct ScoreDistribution { + pub min: f64, + pub max: f64, + pub mean: f64, + pub max_z_score: f64, + pub mean_z_score: f64, + pub baseline: BaselineSnapshot, + pub cusum: CusumSnapshot, + pub clip_pressure: f64, +} +``` + +### §4.5 Baseline and CUSUM Snapshots · `sec:sentinel:api-baseline-and-cusum-snapshots` + +#### `BaselineSnapshot` — `Copy` · `sec:sentinel:api-baseline-snapshot` + +Frozen snapshot of an EWMA baseline at the time of scoring. + +```rust +pub struct BaselineSnapshot { + pub mean: f64, + pub variance: f64, +} +``` + +#### `CusumSnapshot` — `Copy` · `sec:sentinel:api-cusum-snapshot` + +Frozen snapshot of a CUSUM accumulator at scoring time. + +```rust +pub struct CusumSnapshot { + pub accumulator: f64, + pub slow_baseline: BaselineSnapshot, + pub steps_since_reset: u64, +} +``` + +### §4.6 Maturity and Geometry · `sec:sentinel:api-maturity-and-geometry` + +#### `TrackerMaturity` — `Copy` · `sec:sentinel:api-tracker-maturity` + +How much experience a tracker has, and how much of that is noise. + +```rust +pub struct TrackerMaturity { + pub real_observations: u64, + pub noise_observations: u64, + pub noise_influence: f64, +} +``` + +Methods: `cold() -> Self`, `total_observations() -> u64`. + +#### `ScoringGeometry` — `Copy` · `sec:sentinel:api-scoring-geometry` + +Geometry of the model that scored the associated batch. In a cell inspection, it describes the most recent scored batch and can differ from the current rank after adaptation. + +```rust +pub struct ScoringGeometry { + pub dim: usize, + pub cap: usize, + pub residual_dof: usize, +} +``` + +Methods: `is_novelty_saturated() -> bool`, `is_novelty_saturable() -> bool`. + +### §4.7 Per-sample Scores · `sec:sentinel:api-per-sample-scores` + +#### `SampleScore` — `Copy` · `sec:sentinel:api-sample-score` + +Raw anomaly scores for a single observation. + +```rust +pub struct SampleScore { + pub novelty: f64, + pub displacement: f64, + pub surprise: f64, + pub coherence: f64, + pub novelty_z: f64, + pub displacement_z: f64, + pub surprise_z: f64, + pub coherence_z: f64, +} +``` + +#### `MemberScore` — `Copy` · `sec:sentinel:api-member-score` + +Per-member scores at coordination level (identifies contributing cell). + +### §4.8 Health and Summary Reports · `sec:sentinel:api-health-and-summary-reports` + +#### `HealthReport` — `Clone` · `sec:sentinel:api-health-report` + +Operational health snapshot of the entire sentinel. + +```rust +pub struct HealthReport { + pub total_g_nodes: usize, + pub semi_internal_count: usize, + pub active_trackers: usize, + pub active_competitive_trackers: usize, + pub active_ancestor_trackers: usize, + pub active_coordination_contexts: usize, + pub investment_set_size: usize, + pub warming_trackers: usize, + pub warming_competitive_targets: usize, + pub lifetime_observations: u64, + pub cells_tracked: usize, + pub rank_distribution: RankDistribution, + pub maturity_distribution: MaturityDistribution, + pub geometry_distribution: GeometryDistribution, + pub coordination_health: CoordinationHealth, + pub clip_pressure_distribution: ClipPressureDistribution, +} +``` + +#### `AnalysisSetSummary` — `Copy` · `sec:sentinel:api-analysis-set-summary` + +Summary of the analysis set at report time. + +Every count and range is taken over the cells its producer describes, and the type does not fix which those are. `AnalysisSet::summary()` takes every figure over the whole selection, online or still warming — the competitive targets $\mathcal{T}$ and the investment set $\mathcal{I}$ (§ALGO S-8.1–8.2). `AnalysisSet::summary_online(online)` takes them over the selection intersected with the cells that have trackers — the producing sets $\mathcal{A}$ and $\mathcal{A}^*$ (§ALGO S-8.3) — and leaves `investment_set_size` whole, because the cells the filter would drop are precisely those already paid for and not yet producing. The summary carried inside `BatchReport` is that online reading with two fields replaced by figures the sentinel can see directly and a selection snapshot cannot: the tracker population for `investment_set_size`, and its own tally for `degenerate_cells_skipped`. + +```rust +pub struct AnalysisSetSummary { + pub competitive_size: usize, + pub full_size: usize, + pub investment_set_size: usize, + pub depth_range: (u32, u32), + pub importance_range: (f64, f64), + pub v_depth_range: (usize, usize), + pub degenerate_cells_skipped: usize, +} +``` + +#### `ContourSnapshot` — `Copy` · `sec:sentinel:api-contour-snapshot` + +Snapshot of the G-V Graph's spatial contour at report time. + +```rust +pub struct ContourSnapshot { + pub plateau_count: usize, + pub cell_count: usize, + pub total_importance: f64, + pub splits_since_last_report: u32, + pub net_removals_since_last_report: u32, +} +``` + +### §4.9 Distribution Summaries · `sec:sentinel:api-distribution-summaries` + +#### `RankDistribution` — `Copy` · `sec:sentinel:api-rank-distribution` + +```rust +pub struct RankDistribution { + pub min: usize, + pub max: usize, + pub mean: f64, +} +``` + +#### `MaturityDistribution` — `Copy` · `sec:sentinel:api-maturity-distribution` + +```rust +pub struct MaturityDistribution { + pub max_noise_influence: f64, + pub min_noise_influence: f64, + pub mean_noise_influence: f64, + pub cold_trackers: usize, +} +``` + +#### `GeometryDistribution` — `Copy` · `sec:sentinel:api-geometry-distribution` + +```rust +pub struct GeometryDistribution { + pub novelty_saturated: usize, + pub novelty_saturable: usize, + pub coherence_inactive: usize, +} +``` + +#### `ClipPressureDistribution` — `Copy` · `sec:sentinel:api-clip-pressure-distribution` + +```rust +pub struct ClipPressureDistribution { + pub min: f64, + pub max: f64, + pub mean: f64, +} +``` + +#### `CoordinationHealth` — `Copy` · `sec:sentinel:api-coordination-health` + +Health snapshot of the hierarchical coordination tier. + +```rust +pub struct CoordinationHealth { + pub active_contexts: usize, + pub capacity: usize, + pub rank_distribution: RankDistribution, + pub maturity_distribution: MaturityDistribution, + pub dim: usize, + pub geometry_distribution: GeometryDistribution, +} +``` + +### §4.10 Inspection Types · `sec:sentinel:api-inspection-types` + +#### `CellInspection` — `Clone` · `sec:sentinel:api-cell-inspection` + +Detailed snapshot of a cell's tracker state. Returned by `SpectralSentinel::inspect_cell()`. The `rank` is the current model rank for the next batch, while `geometry` describes the model that scored the most recent batch. + +```rust +pub struct CellInspection { + pub gnode_id: GNodeId, + pub start: C, + pub end: C, + pub depth: u32, + pub analysis_width: usize, + pub is_competitive: bool, + pub rank: usize, + pub energy_ratio: f64, + pub top_singular_value: f64, + pub maturity: TrackerMaturity, + pub geometry: ScoringGeometry, + pub baselines: AxisBaselineSnapshots, +} +``` + +#### `AxisBaselineSnapshots` — `Copy` · `sec:sentinel:api-axis-baseline-snapshots` + +Per-axis EWMA baseline snapshots for all four scoring axes. + +```rust +pub struct AxisBaselineSnapshots { + pub novelty: BaselineSnapshot, + pub displacement: BaselineSnapshot, + pub surprise: BaselineSnapshot, + pub coherence: BaselineSnapshot, +} +``` + +--- + +## §5. Surface 2 — Operational Types · `sec:sentinel:api-operational-types` + +### §5.1 `SentinelConfig` — `Clone` · `sec:sentinel:api-sentinel-config` + +Measurement parameters for the sentinel. Every field controls *how* the sentinel observes and learns, never *what it thinks* about what it sees. + +```rust +pub struct SentinelConfig { + // ── Subspace parameters ───────────────────────────── + pub max_rank: usize, // Default: 16 + pub forgetting_factor: f64, // Default: 0.99, in (0.0, 1.0) + pub rank_update_interval: u64, // Default: 100 + pub energy_threshold: f64, // Default: 0.90, in (0.0, 1.0) + pub eps: f64, // Default: 1e-6 + + // ── CUSUM parameters ──────────────────────────────── + pub cusum_slow_decay: f64, // Default: 0.999 + pub cusum_coord_slow_decay: f64, // Default: 0.999 + pub cusum_allowance_sigmas: f64, // Default: 0.5 + + // ── Clip parameters ───────────────────────────────── + pub clip_sigmas: f64, // Default: 3.0 + pub clip_pressure_decay: f64, // Default: 0.95 + + // ── Analysis selection ────────────────────────────── + pub analysis_k: usize, // Default: 1024 + pub analysis_depth_cutoff: usize, // Default: 6 + + // ── G-V Graph passthrough ─────────────────────────── + pub split_threshold: V, // Default: 100 + pub d_create: u32, // Default: 3 + pub d_evict: u32, // Default: 6 + pub budget: usize, // Default: 100_000 + + // ── Noise injection ───────────────────────────────── + pub noise_schedule: NoiseSchedule, + pub noise_batch_size: usize, // Default: 16 + pub noise_seed: Option, // Default: Some(42) + pub background_warming: bool, // Default: false + + // ── Output control ────────────────────────────────── + pub per_sample_scores: bool, // Default: false + + // ── SVD strategy ──────────────────────────────────── + pub svd_strategy: SvdStrategy, // Default: Brand +} +``` + +#### Key field groups · `sec:sentinel:api-sentinel-config-field-groups` + +**Subspace parameters** (§ALGO S-4): +- `max_rank`: Maximum basis vectors any tracker can use. +- `forgetting_factor` (λ): Exponential decay rate per tracker batch. Half-life ≈ `ln(2) / ln(1/λ)` batches. +- `rank_update_interval`: Reassess rank every N tracker batches. +- `energy_threshold` (τ): Cumulative energy threshold for rank adaptation. + +**CUSUM parameters** (§ALGO S-6): +- `cusum_slow_decay` (λ_s): Slow EWMA decay for drift detection reference. +- `cusum_coord_slow_decay`: Same for coordination tier. +- `cusum_allowance_sigmas` (κ_σ): Noise allowance in slow-baseline σ units. + +**G-V Graph passthrough** (§ALGO S-13.3): +- `split_threshold`: Observations before cell subdivision. +- `d_create`: Maximum V-Tree depth for new splits. +- `d_evict`: Minimum V-Tree depth for eviction eligibility. +- `budget`: Hard ceiling on live G-nodes. + +**Noise injection** (§ALGO S-11, ADR-S-015): +- `noise_schedule`: Depth-tiered warm-up schedule. +- `background_warming`: When `true`, warm-up runs on a background thread. + +Methods: `validate() -> Result<(), ConfigErrors>`, `warnings() -> Vec`, `Default`. + +`warnings()` reports parameter combinations that validation accepts but that are empirically known to produce poor results; it is read once `validate()` has succeeded and refuses nothing (§7.1). + +### §5.2 `NoiseSchedule` — `Clone` · `sec:sentinel:api-noise-schedule` + +Depth-tiered noise injection schedule. + +```rust +pub enum NoiseSchedule { + Geometric { root: u32, decay: f64, min: u32 }, + Explicit(Vec), +} +``` + +| Variant | Behaviour | +| ---------- | ------------------------------------------------ | +| `Geometric`| `root × decay^depth`, floored to `min` | +| `Explicit` | Per-depth vector; depths beyond end use last | + +Methods: `geometric(root, decay, min) -> Self`, `rounds_for_depth(depth) -> u32`, `is_disabled() -> bool`, `max_rounds() -> u32`, `Default`. + +Default: `Geometric { root: 450, decay: 0.5, min: 50 }`. + +### §5.3 `SpectralSentinel` — Main Orchestrator · `sec:sentinel:api-spectral-sentinel` + +The main orchestrator. Generic over coordinate `C`, accumulator `V`, domain width `N`. + +```rust +pub struct SpectralSentinel +where + C: Coordinate + CentredBitSource, + V: Inspectable + Attenuatable, +{ + // ... internal state ... +} +``` + +#### Method Index · `sec:sentinel:api-spectral-sentinel-method-index` + +| Method | Category | Cost | Description | +| ------------------------- | ------------- | --------------------- | ------------------------------------------- | +| `new` | Construction | $O(w)$ noise | Validated config → warmed root tracker | +| `ingest` | Observation | $O(n × \text{cells})$ | Batch processing, returns `BatchReport` | +| `decay` | Temporal | $O(\|G\|)$ | Global importance attenuation | +| `decay_subtree` | Temporal | $O(\|G_{sub}\|)$ | Subtree importance attenuation | +| `reset` | Lifecycle | $O(w)$ | Re-initialise to fresh state | +| `health` | Diagnostic | $O(\|cells\|)$ | Operational health snapshot | +| `config` | Accessor | $O(1)$ | Read-only config reference | +| `graph` | Accessor | $O(1)$ | Read-only G-V Graph reference | +| `analysis_set` | Accessor | $O(1)$ | Current analysis set | +| `cells_tracked` | Accessor | $O(1)$ | Number of active trackers | +| `lifetime_observations` | Accessor | $O(1)$ | Total real observations | +| `degenerate_cells_skipped`| Accessor | $O(1)$ | Cells excluded for width < 2 | +| `cell_gnodes` | Accessor | $O(\|cells\|)$ | List all tracked cell handles | +| `inspect_cell` | Diagnostic | $O(1)$ | Detailed cell state snapshot | + +#### `new(config: SentinelConfig) -> Result` · `sec:sentinel:api-spectral-sentinel-new` + +Validates the configuration and creates the root tracker. The root tracker is automatically warmed with synthetic noise (§ALGO S-11.1). No other cells are created until the first `ingest()` call triggers analysis set computation. + +#### `ingest(&mut self, values: &[C]) -> BatchReport` · `sec:sentinel:api-spectral-sentinel-ingest` + +Process a batch of raw coordinate observations and return a full statistical report. Each value is: +1. Fed to the G-V Graph with Δ=1 (feed-forward invariant). +2. Routed to every analysis cell whose interval contains it. +3. Encoded as centred bits and scored against learned subspaces. + +Values outside the domain `[0, 2^N)` (§ALGO S-2.1) take none of those steps. Membership is decided before the spatial layer, the encoding and the routing, so such a value raises no total — neither the graph's accumulated importance nor `lifetime_observations` — moves no partition, and reaches no tracker. A batch that loses values this way emits one `tracing` warning naming how many went; the count is not part of the report, which describes the observations the sentinel made. A batch left with nothing produces an empty report. + +An empty input slice produces an empty report. + +#### `decay(&mut self, attenuation: f64, q: f64)` · `sec:sentinel:api-spectral-sentinel-decay` + +Apply spatial decay to the entire G-V Graph. + +Parameters: +- `attenuation` — base decay factor at midpoint depth. + - `(0, 1)`: Cold cells lose standing. + - `> 1.0`: Hot cells reinforced (amplification). + - `1.0`: No-op. +- `q` — depth selectivity in `[0.0, 1.0]`. + - `0.0`: Uniform — all depths decay equally. + - `> 0.0`: Selective — fine structure fades faster. + +Panics if `attenuation < 0.0`, `q` out of range, or `NaN`. + +#### `decay_subtree(&mut self, root: GNodeId, attenuation: f64, q: f64)` · `sec:sentinel:api-spectral-sentinel-decay-subtree` + +Apply spatial decay to a subtree of the G-V Graph. + +Use cases (§ALGO S-10.3): +- **Regime change**: Attenuate a subtree that experienced a traffic shift. +- **Suspected poisoning**: `decay_subtree(root, 0.0₊, 1.0)` is a detail flush. +- **Hot reinforcement**: `decay_subtree(root, 1.5, 0.0)` amplifies a hot subtree. + +Panics if `root` is stale or parameters out of range. + +#### `reset(&mut self)` · `sec:sentinel:api-spectral-sentinel-reset` + +Reset the sentinel to its freshly-constructed state. Drops all cell trackers and their learned subspaces, re-initialises the G-V Graph, and zeroes all counters. Configuration is preserved. + +### §5.4 `AnalysisSet` and `AnalysisEntry` · `sec:sentinel:api-analysis-set-and-entry` + +Analysis set management (§ALGO S-8.1–8.3). + +#### `AnalysisEntry` — `Copy` · `sec:sentinel:api-analysis-entry` + +A cell selected for analysis by the selector. + +```rust +pub struct AnalysisEntry { + pub gnode: GNodeId, + pub depth: u32, + pub v_depth: usize, + pub importance: V, + pub start: C, + pub end: C, + pub is_competitive: bool, +} +``` + +#### `AnalysisSet` — `Clone` · `sec:sentinel:api-analysis-set` + +The current analysis set — competitive targets $\mathcal{T}$ closed under G-tree ancestry to form the investment set $\mathcal{I}$. + +```rust +pub struct AnalysisSet { + competitive: Vec>, + full: Vec>, +} +``` + +Methods: +- `recompute(graph, k, depth_cutoff) -> Self` +- `competitive() -> &[AnalysisEntry]` — ordered by importance descending +- `full() -> &[AnalysisEntry]` — ordered by `GNodeId` +- `contains(gnode: GNodeId) -> bool` +- `is_competitive(gnode: GNodeId) -> bool` +- `competitive_count() -> usize` +- `total_count() -> usize` +- `summary() -> AnalysisSetSummary` — every figure over the whole selection +- `summary_online(online: &BTreeSet) -> AnalysisSetSummary` — the producing reading, as carried by `BatchReport` + +#### Selection algorithm (§ALGO S-8.1) · `sec:sentinel:api-analysis-set-selection-algorithm` + +1. Scan V-Tree for entries with `v_depth ≤ depth_cutoff` and `width ≥ 2`. +2. Sort by importance (descending), then by `start` (ascending) for determinism. +3. Take top-K (the competitive targets $\mathcal{T}$). +4. Close under G-tree ancestry to form $\mathcal{I}$. + +The root is always an ancestor (§ALGO S-8.2), never competitive (§ALGO S-8.1). + +### §5.5 `CentredBitSource` and `CentredBits` · `sec:sentinel:api-centred-bit-types` + +The observation boundary: how a coordinate value becomes the vector a tracker models (§ALGO S-2.3). Both items are re-exported from the crate root and are public surface, though the `observation` module that defines them is not (§6). + +#### `CentredBitSource` · `sec:sentinel:api-centred-bit-source` + +Bridge trait converting a coordinate value into centred bit form. It is implemented here for `u128` and `u64`, and the set of implementations is open: a downstream coordinate type may implement it for its own width (ADR-S-018). What is not open is the width such an implementation can serve — every conversion returns a `CentredBits`, whose backing array is fixed at 128 slots, so a coordinate type wider than that has no vector to return past its first 128 bits. + +```rust +pub trait CentredBitSource: Coordinate { + fn to_centred_bits(&self, n: u32) -> CentredBits; +} +``` + +| Method | Promise | +| ------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `to_centred_bits(&self, n: u32)` | Returns this value's leading bits in centred form, most significant first. `n` is a request rather than a promise: the effective width — which is also the length of the vector returned — is `n` capped at the width the implementing type holds, 128 for `u128` and 64 for `u64`. The implementation applies the cap rather than trusting the caller, so no caller can provoke a panic by asking for a width the domain does not hold; at `n = 0` the returned vector is empty. | + +#### `CentredBits` — `Clone` · `sec:sentinel:api-centred-bits` + +A coordinate value converted to a centred bit vector: bit `1` becomes `+0.5` and bit `0` becomes `−0.5`, so the encoded levels are symmetric about zero (§ALGO S-2.3). The backing array is fixed at 128 slots, which is the crate's coordinate-width ceiling rather than an implementation detail: a centred bit is never zero, so the slots past the vector's length stay distinguishable from data. + +```rust +pub struct CentredBits { + pub bits: [f64; 128], + // Runtime length; private. +} +``` + +| Item | Promise | +| ------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `bits` | The centred values, most significant bit at index 0. Only the first `len()` entries carry a bit; the rest are zero-filled. | +| `new(bits: [f64; 128], len: usize) -> Self`| Builds a vector from centred values already computed, with `len` of them meaningful — the constructor an implementation of `CentredBitSource` outside this crate returns through. Slots from `len` onward are the caller's to leave at zero. **Panics** if `len` exceeds 128, the size of the backing array. | +| `len(&self) -> usize` | How many of the backing array's slots carry a centred bit. | +| `is_empty(&self) -> bool` | Whether the vector carries no bits at all — the zero-width domain, and exactly `len() == 0`. | +| `from_u128(value: u128) -> Self` | The full 128-bit conversion of a `u128`, identical to `value.to_centred_bits(128)`. | +| `suffix(&self, depth: u8) -> &[f64]` | The bits from `depth` to `len()` — the working observation for a cell at G-tree depth `depth`, of width `len() - depth`, the leading `depth` bits being constant within that cell and already resolved by routing (§ALGO S-2.4). At depth 0 this is the whole vector; at `depth == len()` it is empty. **Panics** if `depth` exceeds `len()`. | + +--- + +## §6. Surface 3 — Internal Machinery · `sec:sentinel:api-internal-machinery` + +`pub(crate)` modules not part of the public API: + +| Module | Contents | +| ------------------------- | ----------------------------------------------------- | +| `sentinel::tracker` | `SubspaceTracker` — SVD-based subspace model | +| `sentinel::cusum` | CUSUM accumulator for drift detection | +| `sentinel::staging` | Deferred warm-up staging area (ADR-S-019) | +| `sentinel::warming_thread`| Background noise injection thread (§ALGO S-11.6.8) | +| `maths` | SVD (naive + Brand), matrix ops, Gamma distribution | +| `ewma` | `EwmaStats` — exponential moving average with clipping| +| `observation` | Suffix encoding. The module is crate-private, but the `CentredBits` and `CentredBitSource` it defines are re-exported at the crate root and are public surface (§5.5) | + +These modules implement the internal machinery. Their interfaces may change without notice. Only types re-exported from the crate root are part of the public API. + +--- + +## §7. Cross-Cutting Concerns · `sec:sentinel:api-cross-cutting-concerns` + +### §7.1 Error Handling · `sec:sentinel:api-error-handling` + +- `SentinelConfig::validate() -> Result<(), ConfigErrors>` — validates all configuration parameters. +- `SpectralSentinel::new()` propagates validation errors. +- Panics for stale handles (documented per-method). +- `ConfigErrors` is a `Vec` carrying all validation failures. +- `SentinelConfig::warnings() -> Vec` — advisory diagnostics, read once `validate()` has succeeded. A warning refuses nothing: the configuration is valid and the engine will run on it. + +#### `ConfigError` variants · `sec:sentinel:api-config-error-variants` + +| Variant | Constraint violated | +| ------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `MaxRankZero` | `max_rank` must be ≥ 1 | +| `ForgettingFactorOutOfRange` | `forgetting_factor` must be in `(0.0, 1.0)` | +| `RankUpdateIntervalZero` | `rank_update_interval` must be ≥ 1 | +| `AnalysisKZero` | `analysis_k` must be ≥ 1 | +| `EnergyThresholdOutOfRange` | `energy_threshold` must be in `(0.0, 1.0)` | +| `EpsNotFinite` | `eps` must be finite | +| `EpsNotPositive` | `eps` must be positive; checked only once `eps` is known finite, so a non-finite value raises the variant above and not this one | +| `CusumSlowDecayOutOfRange` | `cusum_slow_decay` must be in `(0.0, 1.0)` | +| `CusumSlowDecayTooLow` | `cusum_slow_decay` must be > `forgetting_factor` | +| `CusumCoordSlowDecayOutOfRange` | `cusum_coord_slow_decay` must be in `(0.0, 1.0)` | +| `CusumCoordSlowDecayTooLow` | `cusum_coord_slow_decay` must be > `forgetting_factor` — the coordination tier's slow memory is compared against the same fast memory as the per-cell tier's | +| `CusumAllowanceNegative` | `cusum_allowance_sigmas` must be ≥ 0 | +| `ClipSigmasNotPositive` | `clip_sigmas` must be positive | +| `ClipPressureDecayOutOfRange` | `clip_pressure_decay` must be in `(0.0, 1.0)` | +| `SplitThresholdNotPositive` | `split_threshold` must be positive, judged on its `f64` reading | +| `DCreateZero` | `d_create` must be ≥ 1 | +| `DEvictNotGreaterThanDCreate` | `d_evict` must be > `d_create` | +| `BudgetZero` | `budget` must be ≥ 1 | +| `BudgetTooSmall` | `budget` must exceed the headroom the depth pair implies — three raised to the gap between the depths plus one, or twice the creation depth less one, whichever is larger. Checked only where `budget` is non-zero and `d_evict` exceeds `d_create`, since otherwise the pair is already refused above | +| `NoiseBatchSizeZero` | `noise_batch_size` must be ≥ 1 while the noise schedule is enabled; a disabled schedule leaves the field unchecked | +| `NoiseBatchSizeTooLarge` | with noise enabled, the matrices one warm-up batch allocates must be representable — the bound either overflows or exceeds what the address space permits | +| `NoiseScheduleDecayOutOfRange` | a `Geometric` schedule's `decay` must be in `(0.0, 1.0]`; the upper bound is included, unlike the decay fields above | +| `NoiseScheduleRootZero` | a `Geometric` schedule's `root` must be non-zero where its `min` is above zero | +| `DepthBufferTooLarge` | the headroom the depth pair implies is too large to represent at all, so no `budget` can satisfy it; checked under the same guard as `BudgetTooSmall` | +| `TrackerDimensionTooSmall` | the coordinate width `N` must be at least the narrowest width a tracker can model. Judged at construction rather than by `validate()`: the width is a parameter of the type, which the configuration alone cannot see | +| `TrackerDimensionTooLarge` | the coordinate width `N` must not exceed the widest width a centred bit vector can carry. Judged at construction, for the same reason | +| `BackgroundWarmingThreadUnavailable` | not a constraint violation: `background_warming` was asked for and the operating system refused the thread. Raised where the thread is asked for, and returned alone rather than alongside the violations above | + +The enum is `#[non_exhaustive]`: each engine capability a configuration can ask for is one more way the request can be refused, so a caller matches the variants it has an opinion about and handles the rest through a wildcard arm reporting the `Display` text. + +#### `ConfigWarning` variants · `sec:sentinel:api-config-warning-variants` + +| Variant | Condition reported | +| --------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `NoiseScheduleInsufficient` | the noise schedule's depth-0 root is below the minimum the configured `forgetting_factor` empirically needs (§ALGO S-A.7), so baselines may not have converged by the time real observations arrive and early scores may be unreliable. Carries `root`, `recommended_root` and `lambda`, so a caller can report the gap rather than the bare fact | + +Advisory diagnostics are described here, beside the errors, rather than among the configuration fields of §5.1: what a reader needs in order to act on a warning is the contrast with a refusal, not the field it happens to concern. A `ConfigError` refuses the configuration and `SpectralSentinel::new()` returns it; a `ConfigWarning` accepts the configuration and says the results may not be worth reading. The two are also read at one boundary and in one order — `validate()` first, then `warnings()` on success — so a caller writing that code finds both in one place. Unlike `ConfigError` the enum is not `#[non_exhaustive]`, so it can be matched exhaustively; a caller with no opinion about a particular advisory can report its `Display` text and carry on, since no advisory obliges it to do anything. + +### §7.2 Thread Safety · `sec:sentinel:api-thread-safety` + +- `SpectralSentinel` is `Send + Sync`, as required by ADR-S-005 and enforced by the crate's static assertion. The staging mutex supports this contract: `Mutex` is `Sync` when `T` is `Send`. +- Ingestion requires exclusive access through `&mut self`; sharing an engine for ingestion therefore requires external synchronisation, such as `Arc>`. +- `BatchReport` and all report types are `Send + Sync`. +- Background warming thread (when `background_warming = true`) runs independently without blocking `ingest()`. + +### §7.3 Feature Gates · `sec:sentinel:api-feature-gates` + +| Feature | Default | Effect | +| ------- | ------- | ------------------------------------------- | +| `serde` | off | `Serialize`/`Deserialize` on config + reports | + +### §7.4 Serde · `sec:sentinel:api-serde` + +All Surface 1 report types carry conditional serde derives. Config types (`SentinelConfig`, `NoiseSchedule`) likewise. Round-trip stability is maintained. + +Serde bounds: +- `BatchReport`, `CellReport`, etc.: `C: Serialize + DeserializeOwned` +- `SentinelConfig`: `V: Serialize + DeserializeOwned` + +### §7.5 Determinism (ADR-S-005) · `sec:sentinel:api-determinism` + +Output *ordering* is deterministic given the same inputs and configuration. Output *values* are deterministic as well with `background_warming` disabled and on a fixed build — one target and one set of dependency versions: +- `BTreeMap` iteration order for cell/coordination maps. +- Tie-breaking by `start` (ascending) in competitive selection. +- Fixed `noise_seed` for reproducible warm-up, within that scope. + +Under background warming the same seed and the same traffic still give the same graph, the same investment set and the same report order (§4.1), but neither the baselines a tracker starts from nor the ingest cycle on which it first scores: the warming worker draws from its own generator and takes whichever staged cell leads on volume when it looks. A caller diffing two runs against each other holds the flag off, or compares converged state rather than cycle-by-cycle output. + +--- + +## §8. Cross-References · `sec:sentinel:api-cross-references` + +| Target | Format | Example | +| --------------------- | -------------------- | ---------------------------- | +| algorithm.md sections | `§ALGO S-N.M` | `§ALGO S-9.1` | +| mudlark api.md | `§API M-N` | `§API M-4.1` (Cell) | +| mudlark idea.md | `§IDEA M-N` | `§IDEA M-5.5` | +| sentinel ADRs | `ADR-S-NNN` | `ADR-S-002` (feed-forward) | +| mudlark ADRs | `ADR-M-NNN` | `ADR-M-040` (GNodeId) | diff --git a/packages/sentinel/docs/implementation.md b/packages/sentinel/docs/implementation.md new file mode 100644 index 000000000..3b05c4cae --- /dev/null +++ b/packages/sentinel/docs/implementation.md @@ -0,0 +1,479 @@ +# Sentinel — Implementation Guide · `guide:sentinel:implementation-guide` + +> **Cross-reference label:** `§IMPL` — e.g. `§IMPL S-3.1` refers to §3.1 of this document. See AGENTS.md for all conventions. + +This document describes how the Spectral Sentinel is implemented. It is an evergreen companion to the algorithm specification ([algorithm.md](algorithm.md)) — the spec says _what_; this document says _where_ and _how_. + +Architecture Decision Records live in [`../adr/`](../adr/). + +--- + +## Table of Contents · `sec:sentinel:implementation-table-of-contents` + +1. [Source Layout](#1-source-layout) +2. [Architecture Overview](#2-architecture-overview) +3. [The Core Loop](#3-the-core-loop) +4. [Scoring Axes](#4-scoring-axes) +5. [Baseline Tracking](#5-baseline-tracking) +6. [Analysis Selector](#6-analysis-selector) +7. [Hierarchical Coordination](#7-hierarchical-coordination) +8. [Noise Injection and Warm-Up](#8-noise-injection-and-warm-up) +9. [SVD Strategy](#9-svd-strategy) +10. [Configuration Reference](#10-configuration-reference) +11. [Report Types](#11-report-types) +12. [Design Decisions](#12-design-decisions) +13. [Cross-Reference Conventions](#13-cross-reference-conventions) + +--- + +## 1. Source Layout · `sec:sentinel:implementation-source-layout` + +| File | Role | +| ------------------------------------------------------------------- | -------------------------------------------------------------------------- | +| [src/lib.rs](../src/lib.rs) | Crate root, public module structure | +| [src/config.rs](../src/config.rs) | `SentinelConfig`, `NoiseSchedule`, validation, `ConfigError`, `ConfigWarning` | +| [src/ewma.rs](../src/ewma.rs) | `EwmaStats` — EWMA mean/variance with outlier clip | +| [src/observation.rs](../src/observation.rs) | `CentredBits`, suffix extraction | +| [src/analysis_set.rs](../src/analysis_set.rs) | `AnalysisEntry`, `AnalysisSet`, `recompute()` — Layer 2 | +| [src/report.rs](../src/report.rs) | All report/snapshot types | +| [src/sentinel/mod.rs](../src/sentinel/mod.rs) | `SpectralSentinel` orchestrator | +| [src/sentinel/tracker.rs](../src/sentinel/tracker.rs) | `SubspaceTracker` — core SVD engine | +| [src/sentinel/cusum.rs](../src/sentinel/cusum.rs) | `CusumAccumulator` — one-sided Page's test | +| [src/sentinel/staging.rs](../src/sentinel/staging.rs) | `WarmingCell`, `StagingArea` — deferred warm-up (§ALGO S-11.6) | +| [src/sentinel/warming_thread.rs](../src/sentinel/warming_thread.rs) | Background warming thread (§ALGO S-11.6.8) | +| [src/maths/mod.rs](../src/maths/mod.rs) | SVD strategy dispatch, `SvdStrategy` enum | +| [src/maths/brand_svd.rs](../src/maths/brand_svd.rs) | Brand's incremental SVD ([ADR-S-016](../adr/016-brand-incremental-svd.md)) | +| [src/maths/naive_svd.rs](../src/maths/naive_svd.rs) | Dense thin SVD baseline | +| [src/maths/bench_tracing.rs](../src/maths/bench_tracing.rs) | Lightweight span-timing layer for convergence benchmarks | + +### 1.1 Dependencies · `sec:sentinel:implementation-source-layout-dependencies` + +| Crate | Purpose | +| --------------------- | ------------------------------------------------------------- | +| `torrust-mudlark` | G-V Graph (Layer 1 spatial substrate) | +| `faer` | Linear algebra (SVD, matrix operations) | +| `rand` / `rand_distr` | Noise generation, Gamma distribution for coordination warming | +| `serde` (optional) | Serialisation of config and report types | +| `tracing` | Structured diagnostics | + +### 1.2 Test Layout · `sec:sentinel:implementation-test-layout` + +Unit tests live in `src/tests/` (crate-level) and integration tests in `tests/` (package-level). Shared helpers live in `tests/common/`. + +#### Integration tests (`tests/`) · `sec:sentinel:implementation-integration-tests` + +| File | Coverage | +| ------------------------------------ | --------------------------------------------------------------------------- | +| `tests/integration.rs` | End-to-end `ingest()` with realistic value streams | +| `tests/invariants.rs` | Structural invariants: feed-forward, constant-norm, rank bounds | +| `tests/hierarchical_coordination.rs` | Coordination tree assembly and scoring | +| `tests/deferred_warmup.rs` | Staging area, background warming, promotion | +| `tests/spray_resistance.rs` | Budget enforcement under adversarial spray | +| `tests/determinism.rs` | Reproducibility given fixed seed | +| `tests/serde_roundtrip.rs` | Config/report serialisation round-trips | +| `tests/ancestor_chain.rs` | Multi-scale ancestor chain properties (§ALGO S-16) | +| `tests/api.rs` | Public API contract tests for `SpectralSentinel` | +| `tests/clip_pressure.rs` | Clip-pressure EWMA integration (§ALGO S-6.4) | +| `tests/coverage_matrix.rs` | Six attack modalities from the coverage matrix (§ALGO S-17.6) | +| `tests/edge_cases.rs` | Edge-case and boundary-condition tests | +| `tests/graph_routing.rs` | Graph routing: `ingest()` feeds the G-V Graph | +| `tests/health.rs` | `HealthReport` behavioural tests | +| `tests/noise.rs` | Automatic noise injection lifecycle (§ALGO S-11) | +| `tests/report_structure.rs` | `BatchReport` structure and field contracts | +| `tests/sentinel_u64.rs` | 64-bit sentinel (`Sentinel64`) end-to-end path | +| `tests/spatial_decay.rs` | Spatial decay via `decay()` / `decay_subtree()` | +| `tests/suffix_analysis.rs` | Suffix analysis (§ALGO S-2.4) | +| `tests/warm_up.rs` | Four-stage warm-up sequence (§ALGO S-11.8) | +| `tests/common/` | Shared builders, assertions, config presets, generators | + +#### Crate tests (`src/tests/`) · `sec:sentinel:implementation-crate-tests` + +| File | Coverage | +| ---------------------------------------- | --------------------------------------------------------------------------------------------------- | +| `src/tests/analysis_set.rs` | `AnalysisSet` selection pipeline (§ALGO S-8.1–8.3) | +| `src/tests/config.rs` | Config validation, defaults, `NoiseSchedule` helpers | +| `src/tests/convergence_fixes.rs` | Regression tests for warm-up convergence ([ADR-S-013](../adr/013-warm-up-convergence-benchmark.md)) | +| `src/tests/convergence_clipping.rs` | Clipping stability audit (fast EWMA clip, graduated exemption, slow EWMA/CUSUM clip) | +| `src/tests/convergence_common.rs` | Shared infrastructure for convergence tests | +| `src/tests/convergence_diagnostics.rs` | On-demand convergence diagnostics (run with `--ignored`) | +| `src/tests/convergence_eta.rs` | Noise influence η tracking and maturity counters | +| `src/tests/convergence_ewma.rs` | Pure EWMA convergence properties (isolated from tracker) | +| `src/tests/convergence_noise.rs` | Tracker-level noise-baseline convergence | +| `src/tests/cusum.rs` | `CusumAccumulator` tests | +| `src/tests/ewma.rs` | `EwmaStats` unit tests | +| `src/tests/observation.rs` | Centred-bit representation tests | +| `src/tests/report.rs` | Report types (construction, field contracts) | +| `src/tests/tracker.rs` | `SubspaceTracker` (construction, scoring, rank, CUSUM, clip-pressure) | +| `src/tests/variance_formula.rs` | EWMA-mean-centred variance formula ([ADR-S-021](../adr/021-ewma-mean-centred-variance.md)) | + +--- + +## 2. Architecture Overview · `sec:sentinel:implementation-architecture-overview` + +The sentinel implements the three-layer architecture (§ALGO S-1.4): + +``` +Layer 1: GvGraph ← torrust-mudlark + Adaptive spatial partitioning of [0, 2^N) + Competitive ranking by observation volume + │ + │ V-Tree depth ≤ cutoff → top-K selection + ▼ +Layer 2: AnalysisSet ← analysis_set.rs + Picks competitive cells, closes under G-ancestry + │ + │ suffix bit vectors at every ancestor depth + ▼ +Layer 3: SubspaceTracker fleet ← sentinel/tracker.rs + + Hierarchical coordination ← sentinel/mod.rs + │ + ▼ + BatchReport → host +``` + +### 2.1 The Spatial Substrate (Layer 1) · `sec:sentinel:implementation-architecture-spatial-substrate` + +The sentinel owns a `GvGraph` ([ADR-S-018](../adr/018-generic-domain-parameters.md)): + +- **`C`** coordinate type — spatial addressing (e.g. `u128`, `u64`). +- **`V`** accumulator — observation counts ($\Delta = 1$ per value). +- **`N`** bit-width — domain resolution. + +The default instantiation (`Sentinel128`) uses `GvGraph`; `Sentinel64` uses `GvGraph`. + +The graph is mutated only through `observe(coord, 1)` during `ingest()` and through `decay()` / `decay_subtree()` when the host requests temporal decay. Anomaly scores never feed back into the graph's importance signal — this is the **feed-forward invariant** ([ADR-S-002](../adr/002-feed-forward-invariant.md)). + +### 2.2 The Analysis Selector (Layer 2) · `sec:sentinel:implementation-architecture-analysis-selector` + +`AnalysisSet::recompute()` runs after every G-V Graph observation pass and selects up to $K$ **competitive targets** ($\mathcal{T}$) from the V-Tree by importance, filtered by a depth cutoff $L$ (§ALGO S-8.1). Ties break by G-node interval left endpoint for deterministic, spatially stable ordering. + +The V-Tree scan uses `graph.layers_to(depth_cutoff)` (ADR-M-041), which limits the BFS to V-depths $\leq L$ and avoids expanding structural nodes below the cutoff — saving exponential work on deep trees compared to a full `layers()` plus post-hoc filter. + +The competitive targets are then closed under G-tree ancestry (§ALGO S-8.2), forming the **investment set** ($\mathcal{I}$) — the set of all cells with allocated trackers. Every member has a tracker regardless of online status, guaranteeing a complete chain from every competitive target to the root. + +The investment set is reconciled against the live `cells` map and the `StagingArea`: entering cells are created and enqueued for warm-up, exiting cells are destroyed (eager removal, §ALGO S-8.5). The online subset of the investment set forms the **producing sets**: $\mathcal{A}$ (online competitive targets) and $\mathcal{A}^*$ (all online members). The full recompute runs from scratch each time ([ADR-S-006](../adr/006-analysis-set-recomputation.md), [ADR-S-019](../adr/019-investment-set-terminology-and-reporting.md)). + +### 2.3 The Analysis Engine (Layer 3) · `sec:sentinel:implementation-architecture-analysis-engine` + +One `SubspaceTracker` per cell in the investment set $\mathcal{I}$. Each tracker analyses the **suffix** bits `[d, N)` with width $w = N - d$ (§ALGO S-2.4). Cells with $w < 2$ are excluded ([ADR-S-011](../adr/011-degenerate-cell-dimension-guard.md)). Only online trackers (the producing full set $\mathcal{A}^*$) score observations; warming trackers receive only synthetic noise. + +The root tracker at depth 0 ($w = N$) is permanent — never destroyed (§ALGO S-8.4). + +### 2.4 Determinism · `sec:sentinel:implementation-architecture-determinism` + +All collection types use `BTreeMap` for deterministic iteration order ([ADR-S-005](../adr/005-deterministic-order-and-thread-safety.md)). Given a fixed `noise_seed`, the sentinel is fully reproducible with `background_warming` disabled and on a fixed build — one target and one set of dependency versions; the generator behind the noise is chosen for speed and is portable across neither. Under background warming the same seed and the same traffic still give the same graph, the same investment set and the same ascending-handle report order, but neither the baselines a tracker starts from nor the ingest cycle on which it first scores: the warming worker draws from its own generator and takes whichever staged cell leads on volume when it looks. `SpectralSentinel` is `Send + Sync`. + +--- + +## 3. The Core Loop · `sec:sentinel:implementation-core-loop` + +`SubspaceTracker::observe()` implements the five-phase core loop (§ALGO S-4.2) with a **score-before-evolve** invariant: scores are computed against the current basis, then the basis is updated. + +| Phase | Operation | Specification | +| ----- | ------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------- | +| 1 | **Project** — $Z = X U$, $\hat{X} = Z U^\top$, $R = X - \hat{X}$ | §ALGO S-4.2 Phase 1 | +| 2 | **Evolve subspace** — combined matrix $M$, thin SVD, update $U$, $\sigma$ | §ALGO S-4.2 Phase 2 | +| 3 | **Evolve latent distribution** — EWMA-mean-centred update of $\mu^{(z)}$, $\nu^{(z)}$, $\Gamma$ ([ADR-S-021](../adr/021-ewma-mean-centred-variance.md)) | §ALGO S-4.2 Phase 3 | +| 4 | **Score** — compute all four axis scores, update EWMA baselines, CUSUM | §ALGO S-4.2 Phase 4 | +| 5 | **Adapt rank** — energy-threshold selection with +1 buffer, ±1 oscillation guard | §ALGO S-4.2 Phase 5 | + +Safety guards: + +- **SVD failure** — falls back to identity if SVD does not converge. +- **Identical observations** — handled without numerical degeneracy. +- **Coherence lifecycle** — undefined at $k < 2$; baselines destroyed on rank drop (§ALGO S-5.5). +- **Latent cold→warm initialisation** — on the first batch, `lat_mean`, `lat_var`, and `cross_corr` are seeded directly from data ([ADR-S-013](../adr/013-warm-up-convergence-benchmark.md)). +- **EWMA-mean-centred variance** — variance centres on the EWMA mean rather than the batch mean, eliminating batch-size-dependent bias ([ADR-S-021](../adr/021-ewma-mean-centred-variance.md)). + +--- + +## 4. Scoring Axes · `sec:sentinel:implementation-scoring-axes` + +All four axes satisfy the polarity invariant: **higher = more anomalous** (§ALGO S-5.1). + +| Axis | Formula | Bounds | Specification | +| ------------ | ------------------------------------ | ------------- | ------------- | +| Novelty | $\|r\|^2 / (d - k)$ | $[0, \infty)$ | §ALGO S-5.2 | +| Displacement | $\|z\|^2 / (k + \|z\|^2)$ | $[0, 1)$ | §ALGO S-5.3 | +| Surprise | diagonal Mahalanobis $/\, k$ | $[0, \infty)$ | §ALGO S-5.4 | +| Coherence | pairwise cross-correlation deviation | $[0, \infty)$ | §ALGO S-5.5 | + +Each raw score is transformed into a z-score via: + +$$z = \frac{s - \bar{s}}{\sqrt{\bar{v}} + \varepsilon}$$ + +where $\bar{s}$ and $\bar{v}$ are the fast EWMA mean and variance (§ALGO S-6.1.2). The variance floor is $10^{-4}$. + +--- + +## 5. Baseline Tracking · `sec:sentinel:implementation-baseline-tracking` + +### 5.1 Fast EWMA (§ALGO S-6.1) · `sec:sentinel:implementation-fast-ewma` + +Per-axis `EwmaStats` with configurable `clip_sigmas`. Observations beyond $\bar{s} + n_\sigma^{\text{eff}} \sqrt{\bar{v}}$ are clipped (upper-tail only) to prevent baseline poisoning. + +**Graduated clip-exemption** ([ADR-S-013](../adr/013-warm-up-convergence-benchmark.md)): during warm-up the effective clip width scales with the noise-influence fraction $\eta$: + +$$n_\sigma^{\text{eff}} = n_\sigma + n_\sigma \cdot \frac{\eta}{1 - \eta + \varepsilon}$$ + +This widens the ceiling while baselines are immature, eliminating the clipping-ceiling positive feedback loop that previously caused 5–10× slower convergence. + +### 5.2 Slow EWMA (§ALGO S-6.2) · `sec:sentinel:implementation-slow-ewma` + +A secondary EWMA with decay factor `cusum_slow_decay` (default 0.999, half-life ≈ 693 steps) provides the reference baseline for CUSUM drift detection. + +### 5.3 CUSUM Drift Detection (§ALGO S-6.3) · `sec:sentinel:implementation-cusum-drift-detection` + +`CusumAccumulator` implements one-sided Page's test with: + +- Compute-before-update ordering. +- Noise allowance: $\kappa_\sigma \cdot \sqrt{v_{\text{slow}}}$. +- Reset after noise injection (§ALGO S-11.4). +- Slow EWMA seeded from fast EWMA at noise→real transition ([ADR-S-013](../adr/013-warm-up-convergence-benchmark.md)). + +--- + +## 6. Analysis Selector · `sec:sentinel:implementation-analysis-selector` + +`AnalysisSet` in `analysis_set.rs` implements the Layer 2 selection pipeline (§ALGO S-8): + +1. **Enumerate** all V-entries with V-depth $\leq L$. +2. **Rank** by importance, take top $K$ — the **competitive targets** $\mathcal{T}$ (§ALGO S-8.1). +3. **Break ties** by interval start (deterministic, spatially stable). +4. **Close** under G-tree ancestry by walking parent pointers — forming the **investment set** $\mathcal{I}$ (§ALGO S-8.2). + +The root tracker is permanent (§ALGO S-8.4) and never participates in competitive selection (§ALGO S-8.1). Reconciliation after each observation pass reconciles the investment set: entering cells are created and enqueued for warm-up, exiting cells are eagerly removed (§ALGO S-8.5, [ADR-S-019](../adr/019-investment-set-terminology-and-reporting.md)). + +--- + +## 7. Hierarchical Coordination · `sec:sentinel:implementation-hierarchical-coordination` + +The sentinel detects coordinated anomalies across cells using hierarchical G-tree coordination (§ALGO S-7). + +### 7.1 Coordination Contexts · `sec:sentinel:implementation-coordination-contexts` + +One `CoordContext` per internal G-node whose left and right subtrees both contain online competitive cells. Each context owns: + +- A 4-dimensional `SubspaceTracker` (one dimension per scoring axis). +- A running-mean centring reference $\mu^{(\text{in})}$ for de-meaning the input signal before feeding (§ALGO S-7.3). + +### 7.2 Bottom-Up Assembly (§ALGO S-7.4) · `sec:sentinel:implementation-coordination-bottom-up-assembly` + +After cell scoring, the coordination tier assembles score vectors bottom-up through the G-tree: + +1. Leaf contributions: competitive cells emit their 4D centred score vector. +2. Internal nodes: when both subtrees contribute, the assembled matrix is fed to the coordination tracker. +3. The walk is pruned — only nodes reachable through subtrees with competitive cells are visited. + +### 7.3 Lifecycle · `sec:sentinel:implementation-coordination-lifecycle` + +`CoordContext` instances are created on first fire and pruned when either subtree loses its last online competitive cell. A quiet batch does not change membership, so it does not discard learned context state. There is no manual API — lifecycle is fully automatic. + +### 7.4 Coordination Warm-Up (§ALGO S-11.7) · `sec:sentinel:implementation-coordination-warmup` + +Coordination contexts are created lazily when the bottom-up walk first finds scored competitive cells in both subtrees, then warmed inline before their first real coordination observation (§ALGO S-11.7). Warm-up draws Gamma-sampled synthetic score vectors from the contributing cells' baseline moments; per-cell synthetic warm-up reports are discarded and never flow through the coordination tree. + +--- + +## 8. Noise Injection and Warm-Up · `sec:sentinel:implementation-noise-injection-and-warmup` + +### 8.1 Noise Generation (§ALGO S-11.1) · `sec:sentinel:implementation-noise-generation` + +Synthetic noise vectors are uniform $\pm 0.5$ centred bit vectors matching the `CentredBits` encoding. A persistent `SmallRng` seeded from `noise_seed` (or system entropy) generates all noise sequences. + +### 8.2 Depth-Tiered Schedule ([ADR-S-015](../adr/015-cell-creation-performance.md)) · `sec:sentinel:implementation-depth-tiered-noise-schedule` + +`NoiseSchedule` replaces the earlier flat `noise_rounds` parameter. Deeper cells are narrower and need fewer rounds to converge: + +- **Geometric** (default): `root` rounds at depth 0, decaying by `decay` per depth level, clamped to `min`. Default: `Geometric { root: 450, decay: 0.5, min: 50 }`. +- **Explicit**: a lookup table of per-depth round counts. + +### 8.3 Automatic Injection · `sec:sentinel:implementation-automatic-noise-injection` + +There is no manual noise API — the sentinel owns the injection lifecycle entirely ([ADR-S-007](../adr/007-automatic-noise-injection.md)). Every newly created tracker is warmed before it receives real observations. The root tracker is warmed at construction. + +### 8.4 Deferred Cell Warm-Up (§ALGO S-11.6) · `sec:sentinel:implementation-deferred-cell-warmup` + +Cell warm-up is decoupled from the `ingest()` hot path to bound per-call work variance: + +1. **Enqueue** — `reconcile_analysis_set()` creates a `CellState` and enqueues it into the `StagingArea` with the target round count from `NoiseSchedule`. The cell holds an **investment slot** in $\mathcal{I}$ but not a production slot in $\mathcal{A}$. +2. **Warm** — the background thread (or synchronous drain) picks the highest-priority cell by g.sum (§ALGO S-11.6.2) and injects one noise batch at a time. The g.sum ordering ensures ancestors come online before descendants. +3. **Promote** — completed cells are moved to the ready queue and transferred into the live `cells` map at the start of the next `ingest()` call, entering the producing set. + +The staging area lives behind `Arc>` for sharing with the background warming thread. + +### 8.5 Background Warming Thread (§ALGO S-11.6.8) · `sec:sentinel:implementation-background-warming-thread` + +When `config.background_warming` is `true`, a dedicated thread runs the warm-up loop: + +- Owns its own `SmallRng` seeded from `noise_seed + 1`. +- Takes cells from the staging area via `take_highest_priority()`, injects noise _without holding the lock_, and returns them via `finish_warming()`. +- Sleeps on a condvar when there is nothing to warm. +- The main thread notifies the condvar after enqueueing new cells. +- Clean shutdown via `WarmingThreadHandle::shutdown()`. + +When `background_warming` is `false` (default), warm-up runs synchronously inside `reconcile_analysis_set()`. This mode is deterministic and used by the test suite. + +### 8.6 Convergence Fixes ([ADR-S-013](../adr/013-warm-up-convergence-benchmark.md)) · `sec:sentinel:implementation-warmup-convergence-fixes` + +Three interacting fixes address a 5–10× convergence gap between theoretical and empirical EWMA baseline convergence: + +| Fix | Mechanism | Impact | +| ---------------------------- | ------------------------------------------------------------------------------------------- | ---------------------------------------- | +| Graduated clip-exemption | $n_\sigma^{\text{eff}}$ scales with $\eta$ | Surprise convergence: 1000+ → 65 rounds | +| Latent cold→warm init | First-batch seeding of `lat_mean`, `lat_var`, `cross_corr` | Rise factor: 5.9× → 1.26× | +| Slow-from-fast CUSUM seeding | Copy fast EWMA into slow at noise→real transition | False drift: 198 → 5.7 | +| EWMA-mean-centred variance | Centre on EWMA mean, not batch mean ([ADR-S-021](../adr/021-ewma-mean-centred-variance.md)) | Worst-case convergence: 289 → 282 rounds | + +Regression tests in `src/tests/convergence_fixes.rs` and `src/tests/variance_formula.rs`. + +### 8.7 Maturity Tracking (§ALGO S-11.5) · `sec:sentinel:implementation-warmup-maturity-tracking` + +Each tracker maintains a noise-influence fraction $\eta \in [0, 1]$ that decays toward 0 as real observations replace synthetic noise. Batch-vectorised via $\lambda^n$. + +--- + +## 9. SVD Strategy · `sec:sentinel:implementation-svd-strategy` + +Configurable via `SentinelConfig::svd_strategy` ([ADR-S-016](../adr/016-brand-incremental-svd.md)): + +| Strategy | Algorithm | Complexity | Notes | +| ----------------- | ---------------------------------------------------------------------------------------------- | ---------------------------- | ----------------------------------- | +| `Naive` | Dense thin SVD of $M \in \mathbb{R}^{w \times (k+b)}$ | $O(w \cdot (k+b)^2)$ | Simple, numerically stable baseline | +| `Brand` (default) | Incremental SVD (Brand 2006) — projects onto current basis, SVDs a $(k+b) \times (k+b)$ kernel | $O(w \cdot (k+b) + (k+b)^3)$ | ~2–3× faster; default | + +In **debug builds**, both algorithms run regardless of the setting and their outputs are compared — a continuous oracle test. + +--- + +## 10. Configuration Reference · `sec:sentinel:implementation-configuration-reference` + +All parameters from the algorithm spec (§ALGO S-13) are represented in `SentinelConfig` (generic over the accumulator type for `split_threshold`; see [ADR-S-018](../adr/018-generic-domain-parameters.md)). Defaults match the spec. + +### 10.1 Analysis Engine (§ALGO S-13.1) · `sec:sentinel:implementation-configuration-analysis-engine` + +| Config field | Default | Description | +| ------------------------ | ------- | ------------------------------------------------------ | +| `max_rank` | 16 | Maximum subspace rank per tracker | +| `forgetting_factor` | 0.99 | Exponential decay factor $\lambda$ | +| `rank_update_interval` | 100 | Tracker batches between rank re-evaluations | +| `energy_threshold` | 0.90 | Cumulative energy fraction for rank selection | +| `eps` | 1e-6 | Numerical stability constant | +| `clip_sigmas` | 3.0 | EWMA outlier clip width ($n_\sigma$) | +| `clip_pressure_decay` | 0.95 | Clip-pressure EWMA decay ($\lambda_\rho$, §ALGO S-6.4) | +| `cusum_slow_decay` | 0.999 | Slow EWMA decay for per-tracker CUSUM | +| `cusum_coord_slow_decay` | 0.999 | Slow EWMA decay for coordination CUSUM | +| `cusum_allowance_sigmas` | 0.5 | CUSUM noise allowance ($\kappa_\sigma$) | +| `per_sample_scores` | false | Include per-observation scores in reports | +| `svd_strategy` | Brand | SVD algorithm selection | + +### 10.2 Analysis Selector (§ALGO S-13.2) · `sec:sentinel:implementation-configuration-analysis-selector` + +| Config field | Default | Description | +| ----------------------- | ------- | ------------------------------- | +| `analysis_k` | 1024 | Maximum competitive cells ($K$) | +| `analysis_depth_cutoff` | 6 | V-Tree depth cutoff ($L$) | + +### 10.3 G-V Graph (§ALGO S-13.3) · `sec:sentinel:implementation-configuration-gv-graph` + +| Config field | Default | Description | +| ----------------- | ------- | ---------------------------------------------------- | +| `split_threshold` | 100 | Minimum volume before subdivision | +| `d_create` | 3 | Maximum V-depth for new splits ($D_{\text{create}}$) | +| `d_evict` | 6 | Minimum V-depth for eviction ($D_{\text{evict}}$) | +| `budget` | 100,000 | Hard ceiling on live G-nodes ($G_{\max}$) | + +### 10.4 Noise Injection (§ALGO S-13.5) · `sec:sentinel:implementation-configuration-noise-injection` + +| Config field | Default | Description | +| -------------------- | ---------------------------------------------- | ------------------------------------------------ | +| `noise_schedule` | `Geometric { root: 450, decay: 0.5, min: 50 }` | Depth-tiered round counts | +| `noise_batch_size` | 16 | Samples per noise round | +| `noise_seed` | Some(42) | Deterministic RNG seed (`None` = system entropy) | +| `background_warming` | false | Warm cells on a background thread | + +### 10.5 Validation · `sec:sentinel:implementation-configuration-validation` + +`SentinelConfig::validate()` checks all constraints (ranges, inter-parameter relationships, mudlark headroom) before construction. Invalid configs produce `ConfigErrors` — the sentinel never panics on bad config ([ADR-S-004](../adr/004-config-validation-over-panic.md)). + +--- + +## 11. Report Types · `sec:sentinel:implementation-report-types` + +`SpectralSentinel::ingest()` returns a `BatchReport` containing everything the host needs to assess the batch. + +### 11.1 Top-Level Report · `sec:sentinel:implementation-top-level-report` + +| Field | Type | Content | +| ---------------------- | ---------------------------- | ------------------------------------------------------- | +| `cell_reports` | `Vec>` | Competitive cells only | +| `ancestor_reports` | `Vec>` | Non-competitive ancestors + root | +| `coordination_reports` | `Vec>` | Per-G-node hierarchical coordination | +| `contour` | `ContourSnapshot` | Plateau count, cell count, total importance | +| `health` | `HealthReport` | Fleet-wide operational summary | +| `analysis_set_summary` | `AnalysisSetSummary` | Competitive/full/investment-set sizes, depth/importance/V-depth ranges, degenerate count | +| `oldest_observation_age_micros` | `Option` | Age of the batch's oldest observation at report emission, in microseconds on the sentinel's own monotonic clock; absent where there is none | + +The age is stamped in `ingest()` at the call boundary, immediately after the empty-input early return, and read back as the last field of the report literal so that it covers the whole of the call's work. `empty_report()` reports it absent. The reading saturates at `u64::MAX` rather than wrapping (`sec:sentinel:algorithm-output-observation-age`). + +### 11.2 Cell-Level Reports · `sec:sentinel:implementation-cell-level-reports` + +| Type | Content | +| ------------------- | ---------------------------------------------------------------------------------------------------------------- | +| `CellReport` | Wraps a `TrackerReport` with cell identity (GNode, depth, interval, competitive flag) | +| `TrackerReport` | `AnomalyScores`, four `ScoreDistribution` axes, `TrackerMaturity`, `ScoringGeometry`, optional per-sample scores | +| `AnomalyScores` | Batch-mean raw score and z-score for each of the four axes | +| `ScoreDistribution` | `BaselineSnapshot` (EWMA mean/var) + `CusumSnapshot` (accumulator, slow mean/var) + `clip_pressure` (ρ̄) | +| `TrackerMaturity` | Noise influence $\eta$, total observations, noise observation count | +| `ScoringGeometry` | Novelty saturation ratio, coherence activity flag ([ADR-S-008](../adr/008-scoring-geometry-extension.md)) | + +### 11.3 Coordination Reports · `sec:sentinel:implementation-coordination-reports` + +| Type | Content | +| ----------------------- | -------------------------------------------------------------------------------------------------------------- | +| `CoordinationReport` | G-node identity, `TrackerReport` from the 4D coordination tracker, `Option>>` per contributing cell (requires `per_sample_scores`) | +| `MemberScore` | Cell identity (start/end/depth) + 8 score fields (4 raw + 4 z-scores) | + +### 11.4 System Health · `sec:sentinel:implementation-system-health` + +| Type | Content | +| ------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `HealthReport` | Node counts, tracker counts, investment set size, warming counts, `RankDistribution`, `MaturityDistribution`, `GeometryDistribution`, `CoordinationHealth`, `ClipPressureDistribution` | +| `CellInspection` | Deep dive into a single cell via `inspect_cell(gnode)`, includes `AxisBaselineSnapshots` | +| `ContourSnapshot` | Plateau count, terminal cell count, total importance | + +--- + +## 12. Design Decisions · `sec:sentinel:implementation-design-decisions` + +| ADR | Title | Summary | +| ------------------------------------------------------------------- | -------------------------------------- | ----------------------------------------------------------- | +| [ADR-S-001](../adr/001-measures-not-opinions.md) | Measures Not Opinions | Sentinel emits raw statistics, never policy | +| [ADR-S-002](../adr/002-feed-forward-invariant.md) | Feed-Forward Invariant | Only `observe(v, 1u64)` — scores never feed back | +| [ADR-S-003](../adr/003-mudlark-integration.md) | Mudlark Integration | Cargo features; type params superseded by ADR-S-018 | +| [ADR-S-004](../adr/004-config-validation-over-panic.md) | Config Validation Over Panic | Pre-validate all constraints; return `ConfigErrors` | +| [ADR-S-005](../adr/005-deterministic-order-and-thread-safety.md) | Deterministic Order and Thread Safety | `BTreeMap`, `Send + Sync` | +| [ADR-S-006](../adr/006-analysis-set-recomputation.md) | Analysis Set Recomputation | Full recompute per `ingest()`; $O(n)$ scan | +| [ADR-S-007](../adr/007-automatic-noise-injection.md) | Automatic Noise Injection | No manual API; sentinel owns lifecycle | +| [ADR-S-008](../adr/008-scoring-geometry-extension.md) | Scoring Geometry Extension | `ScoringGeometry` for structural observability | +| [ADR-S-009](../adr/009-decay-does-not-invalidate-analysis-set.md) | Decay Does Not Invalidate Analysis Set | Decay is host-driven, analysis set reconciled at `ingest()` | +| [ADR-S-010](../adr/010-linear-routing-over-g-tree-descent.md) | Linear Routing Over G-Tree Descent | Route by interval containment, not tree traversal | +| [ADR-S-011](../adr/011-degenerate-cell-dimension-guard.md) | Degenerate Cell Dimension Guard | Exclude cells with $w < 2$ | +| [ADR-S-012](../adr/012-test-duration-budget.md) | Test Duration Budget | Time-box test suite | +| [ADR-S-013](../adr/013-warm-up-convergence-benchmark.md) | Warm-Up Convergence Benchmark | Three convergence fixes, 7 regression tests | +| [ADR-S-014](../adr/014-subspace-tracker-visibility.md) | Subspace Tracker Visibility | `pub(crate)` visibility for tracker internals | +| [ADR-S-015](../adr/015-cell-creation-performance.md) | Cell Creation Performance | Depth-tiered `NoiseSchedule` | +| [ADR-S-016](../adr/016-brand-incremental-svd.md) | Brand's Incremental SVD | ~2–3× faster subspace evolution | +| [ADR-S-017](../adr/017-deferred-cell-warm-up.md) | Deferred Cell Warm-Up | Staging area + background thread | +| [ADR-S-018](../adr/018-generic-domain-parameters.md) | Generic Domain Parameters | `C`, `V`, `N` type parameters mirror mudlark | +| [ADR-S-019](../adr/019-investment-set-terminology-and-reporting.md) | Investment-Set Terminology & Reporting | Investment/producing sets, new health fields | +| [ADR-S-020](../adr/020-clip-pressure-ewma.md) | Clip-Pressure EWMA | Per-axis clip-pressure tracking (§ALGO S-6.4) | +| [ADR-S-021](../adr/021-ewma-mean-centred-variance.md) | EWMA-Mean-Centred Latent Variance | Centre on EWMA mean to eliminate batch-size bias | + +--- + +## 13. Cross-Reference Conventions · `sec:sentinel:implementation-cross-reference-conventions` + +Source comments use `§ALGO S-N` to reference sections of [algorithm.md](algorithm.md). Within this document, `§IMPL S-N` references sections here. See AGENTS.md for the full cross-reference system. + +When adding new code comments, always use the fully qualified form (`§ALGO S-4.2`, not bare `§4.2`) since the algorithm spec is a separate document. diff --git a/packages/sentinel/docs/plans/api-md-plan.md b/packages/sentinel/docs/plans/api-md-plan.md new file mode 100644 index 000000000..1518c7717 --- /dev/null +++ b/packages/sentinel/docs/plans/api-md-plan.md @@ -0,0 +1,272 @@ +# Plan: Sentinel Public API Reference (`api.md`) · `plan:sentinel:apiplan-public-api-reference` + +Create a public API reference document for `torrust-sentinel` modelled on `packages/mudlark/docs/api.md`. + +--- + +## 1. Document Structure · `sec:sentinel:apiplan-document-structure` + +Mirror mudlark's seven-section layout, adapted for sentinel's domain: + +| § | Title | Content | +| --- | ------------------------------- | ---------------------------------------------------------------------- | +| 1 | Design Principles | "Measure, don't decide", feed-forward invariant, host policy control | +| 2 | Three-Layer Architecture | Spatial Layer (mudlark), Analysis Selector, Analysis Engine | +| 3 | Crate Root Re-exports | Flat public re-exports, type aliases, re-exported mudlark types | +| 4 | Surface 1 — Report Types | Batch/cell/coordination reports, scores, maturity, geometry, snapshots | +| 5 | Surface 2 — Operational Types | `SpectralSentinel`, `SentinelConfig`, `NoiseSchedule` | +| 6 | Surface 3 — Internal Machinery | `pub(crate)` modules (tracker, staging, cusum, warming_thread, etc.) | +| 7 | Cross-Cutting Concerns | Error handling, thread safety, feature gates, serde | + +--- + +## 2. Section Breakdown · `sec:sentinel:apiplan-section-breakdown` + +### §1 Design Principles · `sec:sentinel:apiplan-design-principles` + +- **Measure, don't decide.** All outputs are raw statistical quantities; no threat levels, no recommended actions. +- **Feed-forward invariant (ADR-S-002).** The G-V Graph receives only `observe(v, 1u64)` — anomaly scores never flow back into importance. +- **Host controls temporal policy.** The sentinel never calls `decay()` automatically; the host schedules it. + +Reference: §ALGO S-1.3. + +### §2 Three-Layer Architecture · `sec:sentinel:apiplan-three-layer-architecture` + +Reproduce the ASCII diagram and table from §§ALGO S-1.4–1.5: + +``` +Layer 1: Spatial Index (mudlark GvGraph) + │ + │ Significance ranking → top-K selection + ▼ +Layer 2: Analysis Selector + │ + │ Suffix bit vectors at every ancestor depth + ▼ +Layer 3: Analysis Engine (subspace trackers, coordination) + │ + ▼ + BatchReport → host +``` + +Responsibility table (summarised from §ALGO S-1.5). + +### §3 Crate Root Re-exports · `sec:sentinel:apiplan-crate-root-reexports` + +Document the flat crate-root API that `lib.rs` ships. The modules remain crate-private, while their public types are re-exported from the crate root so downstream users have one canonical import path: + +```rust +pub(crate) mod analysis_set; +pub(crate) mod config; +pub(crate) mod ewma; +pub(crate) mod maths; +pub(crate) mod observation; +pub(crate) mod report; +pub(crate) mod sentinel; + +pub use analysis_set::{AnalysisEntry, AnalysisSet}; +pub use config::{ConfigError, ConfigErrors, ConfigWarning, NoiseSchedule, SentinelConfig}; +pub use maths::SvdStrategy; +pub use observation::{CentredBitSource, CentredBits}; +pub use report::{BatchReport, CellReport, CoordinationReport}; +pub use sentinel::SpectralSentinel; +pub use torrust_mudlark::GNodeId; + +pub type Sentinel128 = SpectralSentinel; +pub type Sentinel64 = SpectralSentinel; +``` + +The excerpt records the visibility pattern; §4 inventories the complete set of report types re-exported from the crate root. Between them they cover the whole public surface: the crate root exports nothing further, and the internal types the crate-private modules contain are reachable by no path from outside the crate. + +### §4 Surface 1 — Report Types (module `report`) · `sec:sentinel:apiplan-report-types` + +Lightweight, read-only **view types** — detached from the sentinel. Returned by `ingest()`. All carry `Debug`, `Clone`, `serde` (with feature). + +| Type | Role | Copy? | +| --------------------------- | --------------------------------------------- | ----- | +| `BatchReport` | Complete output from one `ingest()` call | No | +| `CellReport` | Per-cell statistics | No | +| `CoordinationReport` | Cross-cell coordination analysis | No | +| `AnomalyScores` | Four-axis score bundle | No | +| `ScoreDistribution` | Per-axis summary (min/max/mean/z-scores) | Copy | +| `BaselineSnapshot` | EWMA baseline at scoring time | Copy | +| `CusumSnapshot` | CUSUM accumulator snapshot | Copy | +| `TrackerMaturity` | Real/noise observation counts, noise_influence | Copy | +| `ScoringGeometry` | dim, cap, residual_dof, saturation predicates | Copy | +| `SampleScore` | Per-sample raw scores (if enabled) | Copy | +| `MemberScore` | Per-member scores at coordination level | Copy | +| `ContourSnapshot` | G-V Graph structure snapshot | Copy | +| `HealthReport` | Operational health summary | No | +| `AnalysisSetSummary` | Analysis set composition | Copy | +| `RankDistribution` | Rank min/max/mean across trackers | Copy | +| `MaturityDistribution` | Noise influence distribution | Copy | +| `GeometryDistribution` | Saturation counts | Copy | +| `CoordinationHealth` | Coordination tier summary | Copy | +| `ClipPressureDistribution` | Clip pressure min/max/mean | Copy | +| `AxisBaselineSnapshots` | Per-axis baseline snapshots | Copy | +| `CellInspection` | Detailed cell state (if enabled) | No | + +#### 4.1 `AnomalyScores` axes · `sec:sentinel:apiplan-anomaly-score-axes` + +| Axis | Metric | Intuition | +| ------------ | ------------------------------------------- | ----------------------------------------- | +| Novelty | Residual energy / DOF | "How much of this is foreign?" | +| Displacement | `‖z‖² / (k + ‖z‖²)` | "How far is this from the centroid?" | +| Surprise | Mahalanobis / rank | "The shape is familiar, but magnitude wild" | +| Coherence | Cross-correlation deviation | "Normal individually, unusual combination" | + +All axes share polarity: **higher = more anomalous**. + +### §5 Surface 2 — Operational Types · `sec:sentinel:apiplan-operational-types` + +#### 5.1 `SentinelConfig` · `sec:sentinel:apiplan-sentinel-config` + +Configuration struct controlling measurement parameters. Fields documented with defaults, ranges, and algorithm.md cross-references. + +Key field groups: +- Subspace parameters (`max_rank`, `forgetting_factor`, `rank_update_interval`, `energy_threshold`) +- CUSUM parameters (`cusum_slow_decay`, `cusum_coord_slow_decay`, `cusum_allowance_sigmas`) +- Clip parameters (`clip_sigmas`, `clip_pressure_decay`) +- Analysis selection (`analysis_k`, `analysis_depth_cutoff`) +- G-V Graph passthrough (`split_threshold`, `d_create`, `d_evict`, `budget`) +- Noise injection (`noise_schedule`, `noise_seed`, `noise_batch_size`) +- Output control (`per_sample_scores`) + +Methods: `validate() -> Result<(), ConfigErrors>`, `Default` impl. + +#### 5.2 `NoiseSchedule` · `sec:sentinel:apiplan-noise-schedule` + +Depth-tiered noise injection schedule. + +```rust +pub enum NoiseSchedule { + Geometric { root: u32, decay: f64, min: u32 }, + Explicit(Vec), +} +``` + +Methods: `geometric()`, `rounds_for_depth()`, `is_disabled()`, `max_rounds()`, `Default`. + +#### 5.3 `SpectralSentinel` · `sec:sentinel:apiplan-spectral-sentinel` + +The main orchestrator. Generic over coordinate `C`, accumulator `V`, domain width `N`. + +**Method index:** + +| Method | Category | Cost | One-liner | +| ------------------------ | ------------- | ------------------- | --------------------------------------------- | +| `new` | Construction | $O(w)$ noise | Validated config → warmed root tracker | +| `ingest` | Observation | $O(n × \text{obs})$ | Batch processing, returns `BatchReport` | +| `decay` | Temporal | $O(\|G\|)$ | Global importance attenuation | +| `decay_subtree` | Temporal | $O(\|G_{sub}\|)$ | Subtree importance attenuation | +| `reset` | Lifecycle | $O(w)$ | Re-initialise to fresh state | +| `health` | Diagnostic | $O(\|cells\|)$ | Operational health snapshot | +| `config` | Accessor | $O(1)$ | Read-only config | +| `graph` | Accessor | $O(1)$ | Read-only graph | +| `analysis_set` | Accessor | $O(1)$ | Current analysis set | +| `cells_tracked` | Accessor | $O(1)$ | Number of active trackers | +| `lifetime_observations` | Accessor | $O(1)$ | Total real observations | +| `degenerate_cells_skipped` | Accessor | $O(1)$ | Cells excluded for width < 2 | +| `cell_gnodes` | Accessor | $O(\|cells\|)$ | List all tracked cell handles | + +#### 5.4 `AnalysisSet` and `AnalysisEntry` · `sec:sentinel:apiplan-analysis-set-and-entry` + +Analysis set management (module `analysis_set`). Document: +- Competitive selection (top-K by importance) +- Ancestor closure +- `AnalysisEntry` fields +- `AnalysisSet` methods: `recompute()`, `competitive()`, `full()`, `contains()`, `is_competitive()`, `summary()` + +### §6 Surface 3 — Internal Machinery · `sec:sentinel:apiplan-internal-machinery` + +`pub(crate)` modules not part of the public API: + +| Module | Contents | +| ------------------- | ------------------------------------------------ | +| `sentinel::tracker` | `SubspaceTracker` — SVD-based subspace model | +| `sentinel::cusum` | CUSUM accumulator | +| `sentinel::staging` | Deferred warm-up staging area (ADR-S-019) | +| `sentinel::warming_thread` | Background noise injection (§ALGO S-11.6) | +| `maths` | SVD, matrix ops, Gamma distribution | +| `ewma` | `EwmaStats` — exponential moving average | +| `observation` | `CentredBits`, `CentredBitSource` | + +### §7 Cross-Cutting Concerns · `sec:sentinel:apiplan-cross-cutting-concerns` + +#### 7.1 Error Handling · `sec:sentinel:apiplan-error-handling` + +- `SentinelConfig::validate()` returns `Result<(), ConfigErrors>`. +- `SpectralSentinel::new()` propagates validation errors. +- Panics for stale handles (documented per-method). + +#### 7.2 Thread Safety · `sec:sentinel:apiplan-thread-safety` + +- `SpectralSentinel` is `Send + Sync`, and the crate asserts both statically. The staging area behind a `Mutex` is what makes it so rather than what prevents it: a `Mutex` is `Sync` whenever `T` is `Send`. +- `BatchReport` and all report types are `Send + Sync`. +- Background warming thread (when enabled) runs independently. + +#### 7.3 Feature Gates · `sec:sentinel:apiplan-feature-gates` + +| Feature | Default | Effect | +| ------- | ------- | --------------------------------------- | +| `serde` | off | `Serialize`/`Deserialize` on all types | + +#### 7.4 Serde · `sec:sentinel:apiplan-serde` + +All Surface 1 types carry conditional serde derives. `SentinelConfig` and `NoiseSchedule` likewise. Round-trip stability documented. + +--- + +## 3. Cross-References · `sec:sentinel:apiplan-cross-references` + +| Target | Format | Example | +| --------------------- | -------------------- | ------------------------------------ | +| algorithm.md sections | `§ALGO S-N.M` | `§ALGO S-9.1` | +| mudlark api.md | `§API M-N` | `§API M-4.1` (Cell) | +| mudlark idea.md | `§IDEA M-N` | `§IDEA M-5.5` | +| ADRs | `ADR-S-NNN` | `ADR-S-002` (feed-forward) | +| mudlark ADRs | `ADR-M-NNN` | `ADR-M-040` (GNodeId) | + +--- + +## 4. Tasks · `sec:sentinel:apiplan-tasks` + +1. [ ] Draft §1–§2 (principles, architecture) — lift from algorithm.md. +2. [ ] Draft §3 (re-exports) — survey lib.rs. +3. [ ] Draft §4 (report types) — enumerate report.rs structs with field tables. +4. [ ] Draft §5 (operational types) — config.rs, sentinel/mod.rs public methods. +5. [ ] Draft §6 (internal) — one-liner per pub(crate) module. +6. [ ] Draft §7 (cross-cutting) — error handling, serde, thread safety. +7. [ ] Add cross-reference anchors `§SPEC S-N` for external citation. +8. [ ] Review for consistency with mudlark api.md style: + - Method index tables with cost annotations. + - Rust code blocks for struct/enum definitions. + - Design notes in blockquotes. +9. [ ] Ensure all public items are documented. +10. [ ] Final review against Rust doc comments for accuracy. + +--- + +## 5. Open Questions · `sec:sentinel:apiplan-open-questions` + +1. **Module visibility:** Should `analysis_set` types be in §4 (report-like) or §5 (operational)? Currently proposed for §5 since they're mutable and tied to sentinel lifecycle. + +2. **Coordination tier depth:** How much detail on `CoordinationReport` and the 4D meta-axes? §ALGO S-7 has the full story. + +3. **Tracker internals exposure:** The doc currently marks `SubspaceTracker` as internal. If any methods become `pub`, they'd move to §5. + +--- + +## 6. Estimated Scope · `sec:sentinel:apiplan-estimated-scope` + +| Section | Lines (est.) | +| ------- | ------------ | +| §1–§2 | 100 | +| §3 | 50 | +| §4 | 400 | +| §5 | 500 | +| §6 | 50 | +| §7 | 100 | +| **Total** | **~1200** | + +Comparable to mudlark api.md (~1600 lines), accounting for sentinel's smaller public surface. diff --git a/packages/sentinel/docs/plans/clip-pressure-ewma-implementation-plan.md b/packages/sentinel/docs/plans/clip-pressure-ewma-implementation-plan.md new file mode 100644 index 000000000..1df346356 --- /dev/null +++ b/packages/sentinel/docs/plans/clip-pressure-ewma-implementation-plan.md @@ -0,0 +1,839 @@ +# Implementation Plan: Clip-Pressure EWMA · `plan:sentinel:clip-pressure-ewma-implementation-plan` + +Detailed, file-by-file implementation plan for the 11 gaps identified in ADR-S-020. + +--- + +## Phase 0 — Preparation · `sec:sentinel:clipplan-phase0-preparation` + +Before writing any code: + +1. **Read §ALGO S-6.1.1, §ALGO S-6.4, §ALGO S-13.1, §ALGO S-14.4, §ALGO S-14.11** end-to-end to internalise the spec language. +2. **Snapshot the existing test suite** — run `CARGO_PROFILE_DEV_OPT_LEVEL=3 cargo test --package torrust-sentinel --all-targets --all-features` and record baseline counts/timings. The warm-up convergence benchmark (ADR-S-013) is especially important: the new formula must converge no slower at η ≈ 1 and no wider at η ≈ 0. +3. Create a **feature branch** `feat/clip-pressure-ewma`. + +--- + +## Phase 1 — New Config Parameter · `sec:sentinel:clipplan-phase1-config-parameter` + +**File: ``src/config.rs``** + +### Step 1.1 — Add field to `SentinelConfig` · `sec:sentinel:clipplan-step-1-1-config-field` + +Insert `clip_pressure_decay` after `clip_sigmas`: + +```rust +/// Clip-pressure EWMA decay factor (λ_ρ). +/// +/// Controls how quickly the per-axis clip-pressure estimate adapts +/// to changing contamination levels. Higher values = longer memory. +/// +/// Half-life ≈ ln(2) / ln(1/λ_ρ): +/// - `0.95` = ~14 batches (default, §ALGO S-13.1) +/// - `0.99` = ~69 batches +/// +/// Must be in `(0.0, 1.0)`. +/// +/// Default: `0.95` +pub clip_pressure_decay: f64, +``` + +### Step 1.2 — Default impl · `sec:sentinel:clipplan-step-1-2-default-impl` + +In `impl Default for SentinelConfig`: + +```rust +clip_pressure_decay: 0.95, +``` + +### Step 1.3 — Validation · `sec:sentinel:clipplan-step-1-3-validation` + +Add a new `ConfigError` variant: + +```rust +/// `clip_pressure_decay` must be in `(0.0, 1.0)`. +ClipPressureDecayOutOfRange(f64), +``` + +Add a corresponding arm in `Display for ConfigError`: + +```rust +Self::ClipPressureDecayOutOfRange(v) => + write!(f, "clip_pressure_decay must be in (0.0, 1.0), got {v}"), +``` + +Add the validation check inside `validate()`, near the `clip_sigmas` check: + +```rust +if self.clip_pressure_decay <= 0.0 || self.clip_pressure_decay >= 1.0 { + errors.push(ConfigError::ClipPressureDecayOutOfRange(self.clip_pressure_decay)); +} +``` + +### Step 1.4 — Test · `sec:sentinel:clipplan-step-1-4-test` + +In ``src/tests/config.rs``, add a test `rejects_clip_pressure_decay_out_of_range` paralleling `rejects_non_positive_clip_sigmas`: + +```rust +#[test] +fn rejects_clip_pressure_decay_out_of_range() { + let cfg = SentinelConfig:: { + clip_pressure_decay: 0.0, + ..Default::default() + }; + assert!(cfg.validate().is_err()); + + let cfg = SentinelConfig:: { + clip_pressure_decay: 1.0, + ..Default::default() + }; + assert!(cfg.validate().is_err()); + + let cfg = SentinelConfig:: { + clip_pressure_decay: 0.95, + ..Default::default() + }; + assert!(cfg.validate().is_ok()); +} +``` + +### Step 1.5 — Propagate to `SubspaceTracker` · `sec:sentinel:clipplan-step-1-5-propagate` + +In ``src/sentinel/tracker.rs``, add a field: + +```rust +clip_pressure_decay: f64, +``` + +Initialise it from `cfg.clip_pressure_decay` in `SubspaceTracker::new()`. + +### Checkpoint · `sec:sentinel:clipplan-phase1-checkpoint` + +`cargo test --package torrust-sentinel --all-targets --all-features` — all existing tests pass, new config test passes. + +--- + +## Phase 2 — Per-Axis Clip-Pressure State · `sec:sentinel:clipplan-phase2-axis-state` + +**File: ``src/sentinel/tracker.rs``** + +### Step 2.1 — Add field to `AxisBaseline` · `sec:sentinel:clipplan-step-2-1-baseline-field` + +```rust +struct AxisBaseline { + fast: EwmaStats, + cusum: CusumAccumulator, + /// Clip-pressure EWMA: ρ̄ ∈ [0, 1] (§ALGO S-6.4). + clip_pressure: f64, +} +``` + +### Step 2.2 — Initialise to `0.0` · `sec:sentinel:clipplan-step-2-2-initialise` + +In `AxisBaseline::new()`: + +```rust +Self { + fast: EwmaStats::new(fast_decay), + cusum: CusumAccumulator::new(slow_decay), + clip_pressure: 0.0, +} +``` + +### Step 2.3 — Reset: `reset_cold()` zeros `clip_pressure` · `sec:sentinel:clipplan-step-2-3-reset-cold` + +```rust +fn reset_cold(&mut self) { + self.fast.reset_cold(); + self.cusum.reset_cold(); + self.clip_pressure = 0.0; +} +``` + +### Checkpoint · `sec:sentinel:clipplan-phase2-checkpoint` + +Compiles, all tests pass. The field exists but is unused — no behavioural change yet. + +--- + +## Phase 3 — Externalise Clipping from `EwmaStats` · `sec:sentinel:clipplan-phase3-externalise-clipping` + +**File: ``src/ewma.rs``** + +### Step 3.1 — Add `update_raw()` method · `sec:sentinel:clipplan-step-3-1-update-raw` + +This method accepts **pre-filtered** values — the caller has already applied the clip filter. It performs the same EWMA update as `update()` but skips the internal ceiling computation: + +```rust +/// Update the baseline with pre-filtered values. +/// +/// The caller is responsible for outlier rejection. This method +/// unconditionally incorporates all values (including the cold→warm +/// path). Used by the baseline pipeline (§ALGO S-6.1.1) where +/// clipping is externalised to `update_axis()`. +pub fn update_raw(&mut self, normals: &[f64]) { + if normals.is_empty() { + return; + } + + #[allow(clippy::cast_precision_loss)] + let new_mean = normals.iter().sum::() / normals.len() as f64; + + if !self.warm { + self.mean = new_mean; + if normals.len() > 1 { + let var = mean_squared_deviation(normals, new_mean); + self.variance = var.max(1e-4); + } + self.warm = true; + return; + } + + let alpha = 1.0 - self.decay; + self.mean = self.decay.mul_add(self.mean, alpha * new_mean); + + if normals.len() > 1 { + let var = mean_squared_deviation(normals, new_mean).max(1e-4); + self.variance = self.decay.mul_add(self.variance, alpha * var); + } +} +``` + +> **Why keep `update()` around?** Direct callers in the test suite (unit tests of `EwmaStats` itself) rely on the self-contained `update(values, clip_sigmas)` signature. Removing it is unnecessary churn. + +### Step 3.2 — Add `ceiling()` helper · `sec:sentinel:clipplan-step-3-2-ceiling-helper` + +Expose the clip ceiling so the caller can compute it once: + +```rust +/// Compute the upper-tail clip ceiling: `mean + clip_sigmas · √variance`. +/// +/// Returns `f64::INFINITY` when the baseline is cold (no meaningful +/// ceiling can be defined — matches the cold-path bypass in `update()`). +#[must_use] +pub fn ceiling(&self, clip_sigmas: f64) -> f64 { + if self.warm { + clip_sigmas.mul_add(self.variance.sqrt(), self.mean) + } else { + f64::INFINITY + } +} +``` + +### Step 3.3 — Tests · `sec:sentinel:clipplan-step-3-3-tests` + +Add a unit test verifying `update_raw()` produces the same result as `update()` when all values are within the clip ceiling: + +```rust +#[test] +fn update_raw_matches_update_when_no_clipping() { + let mut a = EwmaStats::new(0.95); + let mut b = EwmaStats::new(0.95); + let values = &[1.0, 1.2, 0.8, 1.1, 0.9]; + + a.update(values, 100.0); // clip_sigmas so high nothing is clipped + b.update_raw(values); + + assert!((a.mean() - b.mean()).abs() < 1e-12); + assert!((a.variance() - b.variance()).abs() < 1e-12); +} +``` + +### Checkpoint · `sec:sentinel:clipplan-phase3-checkpoint` + +Compiles, all tests pass. No behavioural change to the hot path yet. + +--- + +## Phase 4 — Adjust `CusumAccumulator` for Pre-Filtered Samples · `sec:sentinel:clipplan-phase4-cusum-prefiltered` + +**File: ``src/sentinel/cusum.rs``** + +### Step 4.1 — Add `update_filtered()` method · `sec:sentinel:clipplan-step-4-1-update-filtered` + +```rust +/// Update the accumulator with **pre-filtered** samples. +/// +/// Identical to [`update`](Self::update) except the slow EWMA +/// receives pre-filtered values via `update_raw()` instead of +/// applying its own clip filter. The CUSUM gap still uses +/// `raw_batch_mean` (the pre-clip mean of the full batch). +/// +/// Used by the shared-filter pipeline (§ALGO S-6.1.1 step 5). +pub fn update_filtered( + &mut self, + filtered: &[f64], + raw_batch_mean: f64, + allowance_sigmas: f64, +) { + let slow_mean = self.slow.mean(); + let slow_std = self.slow.variance().sqrt(); + let allowance = allowance_sigmas * slow_std; + + let gap = raw_batch_mean - slow_mean - allowance; + self.accumulator = (self.accumulator + gap).max(0.0); + + self.slow.update_raw(filtered); + self.steps_since_reset += 1; +} +``` + +> **`raw_batch_mean`** is the mean of the *full, unclipped* batch — this is already computed in `update_axis()` as the variable `mean` and passed to `cusum.update()` today. No new computation needed. + +### Step 4.2 — Test · `sec:sentinel:clipplan-step-4-2-test` + +Add a test verifying `update_filtered()` produces the same result as `update()` when no samples are clipped: + +```rust +#[test] +fn update_filtered_matches_update_no_clip() { + let mut a = CusumAccumulator::new(0.999); + let mut b = CusumAccumulator::new(0.999); + let scores = &[1.0, 1.1, 0.9, 1.05, 0.95]; + let mean = scores.iter().sum::() / scores.len() as f64; + + a.update(scores, mean, 0.5, 100.0); + b.update_filtered(scores, mean, 0.5); + + assert!((a.snapshot().accumulator - b.snapshot().accumulator).abs() < 1e-12); +} +``` + +### Checkpoint · `sec:sentinel:clipplan-phase4-checkpoint` + +Compiles, all tests pass. + +--- + +## Phase 5 — Unified Clip + Clip-Pressure in `update_axis()` · `sec:sentinel:clipplan-phase5-unified-clip` + +**File: ``src/sentinel/tracker.rs``** + +This is the **core change**. The existing `update_axis()` delegates clipping to `EwmaStats::update()` and `CusumAccumulator::update()`, each applying their own independent filter. After this step, `update_axis()` computes a **single shared clip filter** from the fast EWMA's ceiling, and both EWMAs receive the same retained set. + +### Step 5.1 — Change the caller: `observe()` · `sec:sentinel:clipplan-step-5-1-observe-caller` + +Replace the single `effective_clip` variable with per-axis computation inside `update_axis()`. Pass `clip_pressure_decay` and `clip_sigmas` ++ `eta` as inputs: + +```rust +// In observe(), Phase 4 — replace the `effective_clip` block: +let allowance = self.cusum_allowance_sigmas; +let eta = self.noise_influence; +let clip_sigmas = self.clip_sigmas; +let eps = self.eps; +let cp_decay = self.clip_pressure_decay; + +let novelty_dist = Self::update_axis( + &mut self.novelty_bl, &nov_scores, eps, allowance, + clip_sigmas, eta, cp_decay, true, +); +let displacement_dist = Self::update_axis( + &mut self.displacement_bl, &disp_scores, eps, allowance, + clip_sigmas, eta, cp_decay, true, +); +let surprise_dist = Self::update_axis( + &mut self.surprise_bl, &surp_scores, eps, allowance, + clip_sigmas, eta, cp_decay, true, +); +let coherence_dist = Self::update_axis( + &mut self.coherence_bl, &coh_scores, eps, allowance, + clip_sigmas, eta, cp_decay, k >= 2, +); +``` + +### Step 5.2 — Rewrite `update_axis()` · `sec:sentinel:clipplan-step-5-2-rewrite-update-axis` + +```rust +/// Update one scoring axis: shared clip filter → fast EWMA → CUSUM. +/// +/// §ALGO S-6.1.1 pipeline with clip-pressure EWMA (§ALGO S-6.4). +fn update_axis( + bl: &mut AxisBaseline, + scores: &[f64], + eps: f64, + cusum_allowance: f64, + clip_sigmas: f64, + eta: f64, + clip_pressure_decay: f64, + evolve: bool, +) -> ScoreDistribution { + // ── Raw batch statistics (pre-clip) ───────────── + let (min, max, sum) = scores + .iter() + .fold((f64::INFINITY, f64::NEG_INFINITY, 0.0_f64), |(mn, mx, s), &v| { + (mn.min(v), mx.max(v), s + v) + }); + + #[allow(clippy::cast_precision_loss)] + let mean = sum / scores.len() as f64; + + // Z-scores computed *before* updating the fast baseline. + let max_z = bl.fast.z_score(max, eps); + let mean_z = bl.fast.z_score(mean, eps); + let baseline = bl.fast.snapshot(); + + if evolve { + // ── Per-axis effective clip (§ALGO S-6.4) ─── + // + // p = max(η, ρ̄) + // n_σ_eff = n_σ · (1 + p / (1 − p + ε)) + // + let p = eta.max(bl.clip_pressure); + let effective_clip = clip_sigmas * (1.0 + p / (1.0 - p + eps)); + + // ── Single shared clip filter ─────────────── + // Computed against the fast EWMA's current baseline. + // Cold-path bypass: ceiling() returns +∞ when cold. + let ceiling = bl.fast.ceiling(effective_clip); + + let retained: Vec = scores.iter().copied().filter(|&v| v < ceiling).collect(); + + // ── Update clip-pressure EWMA ─────────────── + // ρ_t = 1 − |retained| / |total| + // ρ̄ = λ_ρ · ρ̄ + (1 − λ_ρ) · ρ_t + #[allow(clippy::cast_precision_loss)] + let rho_t = 1.0 - (retained.len() as f64 / scores.len() as f64); + let alpha = 1.0 - clip_pressure_decay; + bl.clip_pressure = clip_pressure_decay.mul_add(bl.clip_pressure, alpha * rho_t); + + // ── Fast EWMA: receives retained samples ──── + if retained.is_empty() { + // All outliers — learn nothing this round. + // clip_pressure was still updated above (it saw 100% clipping). + } else { + bl.fast.update_raw(&retained); + } + + // ── CUSUM: receives retained samples, raw batch mean ── + if !retained.is_empty() { + bl.cusum.update_filtered(&retained, mean, cusum_allowance); + } + } + let cusum = bl.cusum.snapshot(); + + ScoreDistribution { + min, + max, + mean, + max_z_score: max_z, + mean_z_score: mean_z, + baseline, + cusum, + clip_pressure: bl.clip_pressure, // NEW — added in Phase 6 + } +} +``` + +### Step 5.3 — Verify formula equivalence · `sec:sentinel:clipplan-step-5-3-formula-equivalence` + +The old formula was: + +``` +n_σ_eff = n_σ + n_σ · η / (1 − η + ε) + = n_σ · (1 + η / (1 − η + ε)) +``` + +The new formula is identical when `ρ̄ = 0` (because `p = max(η, 0) = η`). This means **at ρ̄ = 0 the behaviour is bit-for-bit identical** to the old code for the η term. + +### Checkpoint · `sec:sentinel:clipplan-phase5-checkpoint` + +At this point the convergence benchmark (ADR-S-013) should produce results negligibly different from before, since `clip_pressure` starts at 0 and no contamination is present in clean-traffic tests. + +Run: +``` +CARGO_PROFILE_DEV_OPT_LEVEL=3 cargo test --package torrust-sentinel convergence --all-features +``` + +--- + +## Phase 6 — Reporting: `ScoreDistribution::clip_pressure` · `sec:sentinel:clipplan-phase6-score-distribution` + +**File: ``src/report.rs``** + +### Step 6.1 — Add field to `ScoreDistribution` · `sec:sentinel:clipplan-step-6-1-distribution-field` + +```rust +pub struct ScoreDistribution { + pub min: f64, + pub max: f64, + pub mean: f64, + pub max_z_score: f64, + pub mean_z_score: f64, + pub baseline: BaselineSnapshot, + pub cusum: CusumSnapshot, + /// Current clip-pressure EWMA for this axis: ρ̄ ∈ [0, 1] (§ALGO S-14.4). + pub clip_pressure: f64, +} +``` + +### Step 6.2 — Fix all construction sites · `sec:sentinel:clipplan-step-6-2-construction-sites` + +Every place that constructs a `ScoreDistribution` must now include `clip_pressure`. The only production site is `update_axis()` (done in Phase 5). Search for test/mock construction sites: + +``` +grep -rn "ScoreDistribution {" packages/sentinel/ +``` + +Each mock/test site should set `clip_pressure: 0.0` (neutral). + +### Checkpoint · `sec:sentinel:clipplan-phase6-checkpoint` + +Compiles, all tests pass with the new field. + +--- + +## Phase 7 — Reporting: `HealthReport` Clip-Pressure Distribution · `sec:sentinel:clipplan-phase7-health-report` + +**File: ``src/report.rs``** + +### Step 7.1 — Add `ClipPressureDistribution` struct · `sec:sentinel:clipplan-step-7-1-distribution-struct` + +```rust +/// Summary of clip-pressure EWMA values across active trackers (§ALGO S-14.11). +#[derive(Debug, Clone, Copy)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct ClipPressureDistribution { + /// Minimum clip-pressure EWMA across active tracker axes. + pub min: f64, + + /// Maximum clip-pressure EWMA across active tracker axes. + pub max: f64, + + /// Mean clip-pressure EWMA across active tracker axes. + pub mean: f64, +} +``` + +### Step 7.2 — Add field to `HealthReport` · `sec:sentinel:clipplan-step-7-2-health-report-field` + +```rust +/// Clip-pressure distribution across active trackers (§ALGO S-14.11). +pub clip_pressure_distribution: ClipPressureDistribution, +``` + +### Step 7.3 — Expose clip-pressure from `SubspaceTracker` · `sec:sentinel:clipplan-step-7-3-expose-from-tracker` + +**File: ``src/sentinel/tracker.rs``** + +Add a helper to extract the four per-axis `clip_pressure` values: + +```rust +/// Per-axis clip-pressure EWMA values [novelty, displacement, surprise, coherence]. +pub(crate) fn clip_pressures(&self) -> [f64; 4] { + [ + self.novelty_bl.clip_pressure, + self.displacement_bl.clip_pressure, + self.surprise_bl.clip_pressure, + self.coherence_bl.clip_pressure, + ] +} +``` + +### Step 7.4 — Populate in `health()` · `sec:sentinel:clipplan-step-7-4-populate-health` + +**File: ``src/sentinel/mod.rs``** + +In the `health()` method, alongside the existing rank/maturity/geometry loops, accumulate clip-pressure min/max/sum across all 4 axes of all active trackers: + +```rust +let mut cp_min = f64::INFINITY; +let mut cp_max = f64::NEG_INFINITY; +let mut cp_sum = 0.0_f64; +let mut cp_count = 0_u64; + +for cell in self.cells.values() { + for cp in cell.tracker.clip_pressures() { + cp_min = cp_min.min(cp); + cp_max = cp_max.max(cp); + cp_sum += cp; + cp_count += 1; + } +} +``` + +Then in the `HealthReport` struct literal: + +```rust +clip_pressure_distribution: ClipPressureDistribution { + min: if cp_count > 0 { cp_min } else { 0.0 }, + max: if cp_count > 0 { cp_max } else { 0.0 }, + mean: if cp_count > 0 { cp_sum / cp_count as f64 } else { 0.0 }, +}, +``` + +Do the same for the early-return zero-trackers branch. + +### Step 7.5 — Coordination `health()` too · `sec:sentinel:clipplan-step-7-5-coordination-health` + +If the coordination tier's `HealthReport` / `CoordinationHealth` should also report clip-pressure, repeat the same pattern. Check whether §ALGO S-14.11 mandates it — if not, skip for now. + +### Checkpoint · `sec:sentinel:clipplan-phase7-checkpoint` + +`cargo test --package torrust-sentinel --all-targets --all-features` passes. + +--- + +## Phase 8 — Lifecycle Resets · `sec:sentinel:clipplan-phase8-lifecycle-resets` + +### Step 8.1 — Coherence rank-drop reset (already covered) · `sec:sentinel:clipplan-step-8-1-rank-drop-reset` + +`adapt_rank()` calls `self.coherence_bl.reset_cold()` when rank drops below 2. Since Phase 2 added `clip_pressure = 0.0` to `reset_cold()`, **this gap is already closed**. Verify with a quick read of `adapt_rank()`. + +### Step 8.2 — Warm-up completion reset (§ALGO S-11.4) · `sec:sentinel:clipplan-step-8-2-warm-up-reset` + +**File: ``src/sentinel/tracker.rs``** + +The spec says: when noise influence crosses the warm-up threshold (η goes below some value), zero all four axes' `clip_pressure`. + +Currently **there is no explicit warm-up-completion callback** in `SubspaceTracker`. The transition from warming → production happens implicitly as η decays. Two options: + +**Option A — Threshold check inside `update_maturity()`:** + +Add a check after updating `noise_influence`: if it just crossed below a threshold (e.g. 0.01), zero clip-pressure on all four axes. + +```rust +fn update_maturity(&mut self, batch_size: usize, is_noise: bool) { + let old_eta = self.noise_influence; + + // ... existing η update ... + + // §ALGO S-11.4: when η crosses the warm-up threshold, zero + // clip-pressure to prevent warm-up contamination echoing into + // production scoring. + const WARMUP_THRESHOLD: f64 = 0.01; + if old_eta >= WARMUP_THRESHOLD && self.noise_influence < WARMUP_THRESHOLD { + self.novelty_bl.clip_pressure = 0.0; + self.displacement_bl.clip_pressure = 0.0; + self.surprise_bl.clip_pressure = 0.0; + self.coherence_bl.clip_pressure = 0.0; + } +} +``` + +**Option B — Reset at noise injection completion:** + +The warm-up pipeline already calls `seed_cusum_slow_from_baselines()` then `reset_cusum()` after noise injection completes (in `warm_inline()` / the staging pipeline). Add `reset_clip_pressure()` alongside: + +```rust +pub fn reset_clip_pressure(&mut self) { + self.novelty_bl.clip_pressure = 0.0; + self.displacement_bl.clip_pressure = 0.0; + self.surprise_bl.clip_pressure = 0.0; + self.coherence_bl.clip_pressure = 0.0; +} +``` + +Called from `warm_inline()` in ``sentinel/mod.rs`` after `cell.tracker.reset_cusum()`. + +**Recommendation:** Use **both**. Option B handles the explicit noise-injection completion path. Option A catches edge cases where η decays through real-traffic dilution alone (e.g. if noise was partially skipped). + +The warm-up threshold constant (`0.01`) should either: +- Be defined as `const WARMUP_THRESHOLD: f64 = 0.01` in ``tracker.rs``, or +- Be configurable (a future config field). For now, a constant is fine — the spec doesn't parameterise it. + +### Checkpoint · `sec:sentinel:clipplan-phase8-checkpoint` + +Write a test that injects noise, transitions to real traffic, and verifies `clip_pressure` is 0 after transition. + +--- + +## Phase 9 — Memory Accounting Verification · `sec:sentinel:clipplan-phase9-memory-accounting` + +### Step 9.1 — Count the fields · `sec:sentinel:clipplan-step-9-1-count-fields` + +Each `AxisBaseline` now contains: +- `fast: EwmaStats` → 3 fields: `mean`, `variance`, `decay` (+ `warm` bool, packed) = effectively 3 f64s for accounting purposes (ignoring the bool/decay since they're config, not per-axis learned state) → For spec purposes: **2 floats** (mean, variance) +- `cusum: CusumAccumulator` → slow EWMA (2 floats: mean, variance) + accumulator (1 float) + steps (1 u64) → For spec purposes: **5 floats** (slow mean, slow variance, accumulator, steps counter, decay — but decay is config not state) → Actually counting learned state only: slow mean + slow variance + accumulator = **3 floats** +- `clip_pressure: f64` → **1 float** + +Per-axis learned-state floats: fast\_mean(1) + fast\_var(1) + slow\_mean(1) + slow\_var(1) + cusum\_acc(1) + clip\_pressure(1) = **6 floats**. + +Wait — let me re-read the ADR: "Per-axis baseline size grows from 7 to 8 floats". Let me re-count the *current* state: +- fast mean, fast variance = 2 +- slow mean, slow variance = 2 +- cusum accumulator = 1 +- cusum steps_since_reset (u64, counts as 1) = 1 +- warm (bool, but pad to 1) = ~1 + +That's 7 depending on how you count. With `clip_pressure` = **8**. 4 axes × 8 = **32 floats**. + +### Step 9.2 — Update any doc comments · `sec:sentinel:clipplan-step-9-2-doc-comments` + +If `SubspaceTracker` or `AxisBaseline` has doc comments referencing memory accounting, update them. Check §ALGO S-4.3 if it's in-repo. + +### No code change needed here — just verification. · `sec:sentinel:clipplan-phase9-no-code-change` + +--- + +## Phase 10 — Update Convergence Diagnostics · `sec:sentinel:clipplan-phase10-convergence-diagnostics` + +**File: ``src/tests/convergence_diagnostics.rs``** + +The convergence diagnostic test currently computes `eff_clip` as: + +```rust +let eff_clip = cfg.clip_sigmas + cfg.clip_sigmas * eta / (1.0 - eta + cfg.eps); +``` + +Update to the new formula: + +```rust +let p = eta.max(clip_pressure); +let eff_clip = cfg.clip_sigmas * (1.0 + p / (1.0 - p + cfg.eps)); +``` + +Since `clip_pressure` is per-axis, the diagnostic will need to read it from the tracker report (via `ScoreDistribution::clip_pressure`). + +Also update the column header in the diagnostic table to include `ρ̄`. + +--- + +## Phase 11 — Integration Test: Contamination Self-Correction · `sec:sentinel:clipplan-phase11-contamination-test` + +**Files:** ``tests/`` (new test file or extend ``tests/spray_resistance.rs``) + +This is the **acceptance test** that validates the core value proposition: sustained contamination in production (η ≈ 0) should widen the clip ceiling automatically. + +### Step 11.1 — Test outline · `sec:sentinel:clipplan-step-11-1-test-outline` + +```rust +#[test] +fn clip_pressure_widens_ceiling_under_contamination() { + // 1. Build a sentinel, inject noise, let it converge. + // 2. Feed 200+ batches of clean traffic (scores ~ Normal(1.0, 0.1)). + // 3. Assert clip_pressure ≈ 0 on all axes. + // 4. Feed 50 batches of contaminated traffic (scores ~ Normal(1.0, 0.1) + // with 30% of samples replaced by 10.0 — well above 3σ ceiling). + // 5. Assert clip_pressure > 0.2 on at least one axis. + // 6. Assert the effective clip ceiling is wider than n_σ. + // 7. Resume 200 batches of clean traffic. + // 8. Assert clip_pressure decays back toward 0. +} +``` + +### Step 11.2 — Test: η-only behaviour preserved · `sec:sentinel:clipplan-step-11-2-eta-only-preserved` + +```rust +#[test] +fn clip_pressure_zero_under_clean_traffic() { + // Feed clean traffic with no contamination. + // Assert clip_pressure stays ≈ 0 on all axes. + // Assert effective-clip behaviour matches the old η-only formula. +} +``` + +### Step 11.3 — Test: warm-up reset clears clip-pressure · `sec:sentinel:clipplan-step-11-3-warm-up-reset-test` + +```rust +#[test] +fn warm_up_completion_resets_clip_pressure() { + // Inject noise (which may cause high clipping during warm-up). + // Transition to real traffic. + // Assert all clip_pressure values are 0 after transition. +} +``` + +--- + +## Phase 12 — Cross-Cutting Fixups · `sec:sentinel:clipplan-phase12-cross-cutting` + +### Step 12.1 — Serde roundtrip · `sec:sentinel:clipplan-step-12-1-serde-roundtrip` + +If `ScoreDistribution` is `Serialize/Deserialize`, the new field is automatically included. Run ``tests/serde_roundtrip.rs`` to confirm. + +### Step 12.2 — `AxisBaselineSnapshots` · `sec:sentinel:clipplan-step-12-2-baseline-snapshots` + +In ``src/report.rs``, `AxisBaselineSnapshots` is used for convergence tests. Consider adding per-axis `clip_pressure` fields if tests need them: + +```rust +pub struct AxisBaselineSnapshots { + pub novelty: BaselineSnapshot, + pub displacement: BaselineSnapshot, + pub surprise: BaselineSnapshot, + pub coherence: BaselineSnapshot, + pub novelty_clip_pressure: f64, + pub displacement_clip_pressure: f64, + pub surprise_clip_pressure: f64, + pub coherence_clip_pressure: f64, +} +``` + +Update `axis_baseline_snapshots()` in ``tracker.rs`` correspondingly. + +Alternatively, provide a separate `clip_pressures()` method (done in Phase 7.3) and keep `AxisBaselineSnapshots` unchanged. Prefer the separate method unless tests need both in a single struct. + +### Step 12.3 — Grep for hardcoded clip formula · `sec:sentinel:clipplan-step-12-3-hardcoded-formula` + +```bash +grep -rn 'clip_sigmas.*eta\|η.*clip' packages/sentinel/src/ +``` + +Ensure no stale copies of the old `η/(1-η+ε)` formula remain in production code. Test code / diagnostic comments referencing the old formula should be updated or annotated. + +### Step 12.4 — Doc tests · `sec:sentinel:clipplan-step-12-4-doc-tests` + +```bash +cargo test --package torrust-sentinel --doc --all-features +``` + +### Step 12.5 — No-default-features build · `sec:sentinel:clipplan-step-12-5-no-default-features` + +```bash +cargo check --package torrust-sentinel --no-default-features +cargo test --package torrust-sentinel --no-default-features +``` + +--- + +## Phase 13 — Final Validation · `sec:sentinel:clipplan-phase13-final-validation` + +### Step 13.1 — Full test suite (debug, optimised) · `sec:sentinel:clipplan-step-13-1-full-test-suite` + +```bash +CARGO_PROFILE_DEV_OPT_LEVEL=3 cargo test --package torrust-sentinel --all-targets --all-features +``` + +### Step 13.2 — Release mode · `sec:sentinel:clipplan-step-13-2-release-mode` + +```bash +cargo test --package torrust-sentinel --all-targets --all-features --release +``` + +### Step 13.3 — Clippy · `sec:sentinel:clipplan-step-13-3-clippy` + +```bash +cargo clippy --workspace --all-targets --all-features -- -D warnings +``` + +### Step 13.4 — Doc build · `sec:sentinel:clipplan-step-13-4-doc-build` + +```bash +cargo doc --package torrust-sentinel --all-features --no-deps +``` + +### Step 13.5 — Benchmark comparison · `sec:sentinel:clipplan-step-13-5-benchmark` + +Run the convergence benchmark before and after, compare round counts. The warm-up convergence should be ≤ the old round count (formula is identical when ρ̄ = 0). + +--- + +## Execution Order Summary · `sec:sentinel:clipplan-execution-order` + +| Phase | Gap(s) | Files touched | Risk | +|-------|--------|---------------|------| +| 1 | 2 | ``config.rs``, ``tests/config.rs`` | Low — additive | +| 2 | 1 | ``tracker.rs`` | Low — unused field | +| 3 | 4 (partial) | ``ewma.rs`` | Low — new method, old preserved | +| 4 | 6 (partial) | ``cusum.rs`` | Low — new method, old preserved | +| 5 | 3, 4, 5, 6 | ``tracker.rs`` | **High** — core behavioural change | +| 6 | 7 | ``report.rs`` | Medium — struct change, many construction sites | +| 7 | 8 | ``report.rs``, ``mod.rs`` | Medium — new reporting pipeline | +| 8 | 9, 10 | ``tracker.rs``, ``mod.rs`` | Medium — lifecycle logic | +| 9 | 11 | docs only | Low — verification | +| 10 | — | ``tests/convergence_diagnostics.rs`` | Low — test update | +| 11 | — | `tests/` | Low — new tests | +| 12 | — | various | Low — fixups | +| 13 | — | — | Low — final validation | + +**Phase 5 is the critical path.** Everything before it is additive and safe. Everything after it is reporting/testing. If Phase 5 needs to be reverted, Phases 1–4 can remain in the codebase harmlessly. diff --git a/packages/sentinel/src/README.md b/packages/sentinel/src/README.md new file mode 100644 index 000000000..5c51c545e --- /dev/null +++ b/packages/sentinel/src/README.md @@ -0,0 +1,5 @@ +## Unit test matrix · `tab:sentinel:unit-test-matrix` + +**Table (Unit test matrix)** + +No unit tests in this folder. diff --git a/packages/sentinel/src/analysis_set.rs b/packages/sentinel/src/analysis_set.rs new file mode 100644 index 000000000..d8999444c --- /dev/null +++ b/packages/sentinel/src/analysis_set.rs @@ -0,0 +1,347 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +//! Analysis set: V-Tree competitive selection (§ALGO S-8.1–8.3). +//! +//! The competitive targets $\mathcal{T}$ are the top-$K$ V-entries +//! by importance with V-depth ≤ $L$, closed under G-tree ancestry +//! to form the investment set $\mathcal{I}$. The set is recomputed +//! from scratch after every observation pass (ADR-S-006). +//! The selector deliberately does not filter by G-Tree state: terminal, +//! semi-internal, and internal V-entries can all be selected when they +//! satisfy the V-depth, importance, and analysis-width criteria. +//! +//! The producing sets ($\mathcal{A}$, $\mathcal{A}^*$) are the +//! online subsets of $\mathcal{I}$ — derived by the orchestrator +//! from the investment set and tracker online status (ADR-S-019). + +use std::cmp::Ordering; +use std::collections::{BTreeMap, BTreeSet}; + +use torrust_mudlark::{Accumulator, Coordinate, GNodeId, GvGraph, Inspectable}; + +use crate::report::AnalysisSetSummary; + +/// A cell selected for analysis by the selector (§ALGO S-8.1). +/// +/// Competitive entries are the top-$K$ V-Tree entries by importance +/// (the competitive targets $\mathcal{T}$). Ancestor entries close +/// the targets under G-tree ancestry to form the investment set +/// $\mathcal{I}$ (§ALGO S-8.2). +#[derive(Debug, Clone, Copy, PartialEq)] +#[allow(clippy::derive_partial_eq_without_eq)] // V: Accumulator includes f64 which has no Eq +pub struct AnalysisEntry { + /// Arena handle of the backing G-node. + pub gnode: GNodeId, + /// G-Tree depth of this cell. + pub depth: u32, + /// V-Tree depth of this cell's V-entry (for competitive cells) + /// or `0` for ancestor-only cells. + pub v_depth: usize, + /// Own importance of this cell (direct accumulation). + pub importance: V, + /// Lower bound of the dyadic interval (inclusive). + pub start: C, + /// Upper bound of the dyadic interval, exclusive everywhere except at the + /// top of the domain: an entry bounded by the domain's last value owns + /// that value instead of excluding it. The domain has a last value only + /// where the coordinate type cannot represent `2^N` at its full width and + /// its domain maximum stands in for the bound that does not exist; where + /// `2^N` is representable the bound stays exclusive at every width, the + /// full one included. + pub end: C, + /// Whether this entry is competitively selected (vs ancestor-only). + pub is_competitive: bool, +} + +/// The current analysis set — competitive targets $\mathcal{T}$ +/// closed under G-tree ancestry to form the investment set +/// $\mathcal{I}$ (§ALGO S-8.1–8.2). +/// +/// Recomputed after every observation pass (ADR-S-006). +#[derive(Debug, Clone)] +pub struct AnalysisSet { + /// Competitively selected cells, ordered by importance (descending), + /// ties broken by interval start (ascending) for determinism. + competitive: Vec>, + + /// All cells: competitive + ancestors (deduplicated). + /// The investment set $\mathcal{I}$: the competitive targets closed under + /// G-tree ancestry, with no filter on whether a cell is online yet. The + /// producing sets $\mathcal{A}$ and $\mathcal{A}^*$ are its online + /// subsets, which this type cannot see and the orchestrator derives. + /// Ordered by `GNodeId` for deterministic iteration (ADR-S-005). + full: Vec>, + + /// Number of G-tree nodes excluded while producing this selection snapshot because their suffix width was below `MIN_TRACKER_DIM`. + degenerate_cells_skipped: usize, +} + +impl AnalysisSet { + /// Recompute the analysis set from the V-Tree. + /// + /// Scans `graph.layers_to(depth_cutoff)` (ADR-M-041) to collect + /// V-entries with `v_depth ≤ depth_cutoff`, takes top `k` by + /// importance (the competitive targets $\mathcal{T}$, + /// §ALGO S-8.1; ties broken by `start`), then closes under + /// G-tree ancestry to form the investment set $\mathcal{I}$ + /// (§ALGO S-8.2). + /// + /// The depth-limited BFS avoids expanding V-structural nodes + /// below the cutoff, saving `O(2^(D−K))` queue work on balanced + /// trees (ADR-M-041 §Performance). + /// + /// `O(n_K)` in V-entries at depth ≤ K + `O(K · max_depth)` for + /// ancestor closure. + /// + /// # Panics + /// + /// Panics if the G-root node is not live in the graph. + #[must_use] + pub fn recompute(graph: &GvGraph, k: usize, depth_cutoff: usize) -> Self { + // ── Step 1: Collect candidates from V-Tree ────────────── + // Eligibility: V-depth ≤ cutoff AND analysis width w ≥ 2 + // (§ALGO S-8.1). The depth limit is enforced by the BFS + // itself (ADR-M-041), not a post-hoc filter. + let mut degenerate_cells_skipped = 0; + let mut candidates: Vec> = graph + .layers_to(depth_cutoff) + .filter_map(|(v_depth, node)| { + // w = N - depth ≥ MIN_TRACKER_DIM (§ALGO S-8.1, §ALGO S-4.1). + if N.saturating_sub(node.depth) as usize >= crate::MIN_TRACKER_DIM { + Some(AnalysisEntry { + gnode: node.gnode_id, + depth: node.depth, + v_depth, + importance: node.own, + start: node.start, + end: node.end, + is_competitive: true, + }) + } else { + degenerate_cells_skipped += 1; + None + } + }) + .collect(); + + // ── Step 2: Sort by importance (desc), then start (asc) ─ + candidates.sort_by(|a, b| { + b.importance + .partial_cmp(&a.importance) + .unwrap_or(Ordering::Equal) + .then_with(|| a.start.partial_cmp(&b.start).unwrap_or(Ordering::Equal)) + }); + + // ── Step 3: Take top K ────────────────────────────────── + // + // The root is always an ancestor (§ALGO S-8.2) and never competitive + // (§ALGO S-8.1), so it is dropped before the cut rather than after + // it. Dropped afterwards it consumes a slot it can never use: the + // root's own intensity is what it accumulated before its first split, + // and the split freezes that figure while both children start from + // zero, so the root outranks every real candidate until one of them + // passes a total the root is no longer adding to. At a capacity of + // one — a configuration the validation accepts — that leaves the + // competitive set permanently empty, and no descendant can ever + // become competitive. Removing it first spends every slot on an entry + // that can actually be selected. + let g_root = graph.g_root(); + candidates.retain(|e| e.gnode != g_root); + candidates.truncate(k); + + let competitive: Vec> = candidates; + + // ── Step 4: Ancestor closure (§ALGO S-8.2) ──────────────── + // Walk G-tree parents for each competitive entry. Collect + // all ancestor GNodeIds not already in the competitive set. + let mut full_set: BTreeMap> = BTreeMap::new(); + + // Insert competitive entries. + for entry in &competitive { + full_set.insert(entry.gnode, *entry); + } + + // Walk ancestors. + for entry in &competitive { + let mut current = entry.gnode; + while let Some(info) = graph.gnode_info(current) { + let Some(parent_id) = info.parent else { + break; // reached the root + }; + if full_set.contains_key(&parent_id) { + break; // already tracked (shared ancestor) + } + let Some(parent_info) = graph.gnode_info(parent_id) else { + break; + }; + full_set.insert( + parent_id, + AnalysisEntry { + gnode: parent_id, + depth: parent_info.depth, + v_depth: 0, // not meaningful for ancestors + importance: parent_info.own, + start: parent_info.start, + end: parent_info.end, + is_competitive: false, + }, + ); + current = parent_id; + } + } + + // Ensure the root is always present (§ALGO S-8.2). + full_set.entry(g_root).or_insert_with(|| { + let info = graph.gnode_info(g_root).expect("G-root must be live"); + AnalysisEntry { + gnode: g_root, + depth: info.depth, + v_depth: 0, + importance: info.own, + start: info.start, + end: info.end, + is_competitive: false, + } + }); + + let full: Vec> = full_set.into_values().collect(); + + Self { + competitive, + full, + degenerate_cells_skipped, + } + } +} + +impl AnalysisSet { + /// The competitively selected entries. + #[must_use] + pub fn competitive(&self) -> &[AnalysisEntry] { + &self.competitive + } + + /// The full analysis set (competitive + ancestors). + #[must_use] + pub fn full(&self) -> &[AnalysisEntry] { + &self.full + } + + /// Number of competitive entries. + #[must_use] + pub const fn competitive_count(&self) -> usize { + self.competitive.len() + } + + /// Total entries (competitive + ancestors). + #[must_use] + pub const fn total_count(&self) -> usize { + self.full.len() + } + + /// Number of narrow candidates excluded while producing this selection snapshot. + pub(crate) const fn degenerate_cells_skipped(&self) -> usize { + self.degenerate_cells_skipped + } + + /// Whether a given `GNodeId` is in the full analysis set. + #[must_use] + pub fn contains(&self, gnode: GNodeId) -> bool { + self.full.iter().any(|e| e.gnode == gnode) + } + + /// Whether a given `GNodeId` is competitively selected. + #[must_use] + pub fn is_competitive(&self, gnode: GNodeId) -> bool { + self.competitive.iter().any(|e| e.gnode == gnode) + } +} + +impl AnalysisSet { + /// Build a summary snapshot of the current analysis set — the whole + /// selection, whether or not each cell has a tracker yet. + /// + /// See [`summary_online`](Self::summary_online) for the reading over the + /// cells that are online, which is the one the batch report carries. + #[must_use] + pub fn summary(&self) -> AnalysisSetSummary { + self.summarise(|_| true) + } + + /// Build a summary snapshot of the producing sets — the reading the batch + /// report carries. + /// + /// `online` names the cells that currently have a tracker. Every figure is + /// taken over the selection intersected with it, because the producing + /// sets are the online ones — every figure but the investment count, which + /// is documented as the whole investment and is reported as the whole + /// selection here too. Filtering that one would report an investment with + /// the warming cells removed, and those are exactly the part of it that + /// has been paid for and has not yet produced anything. + /// [`summary`](Self::summary) reads the whole selection throughout, which + /// is the investment set: it includes cells still warming in staging, + /// whose depths and importances would widen these ranges with cells no + /// observation has yet reached. The two readings are separate methods + /// because the difference between them is exactly what a caller has to + /// choose. + #[must_use] + pub fn summary_online(&self, online: &BTreeSet) -> AnalysisSetSummary { + self.summarise(|gnode| online.contains(&gnode)) + } + + /// Shared body of the two summaries, over whichever entries are included. + fn summarise(&self, included: impl Fn(GNodeId) -> bool) -> AnalysisSetSummary { + let competitive_included = || self.competitive.iter().filter(|e| included(e.gnode)); + let full_included = || self.full.iter().filter(|e| included(e.gnode)); + + let competitive_size = competitive_included().count(); + let full_size = full_included().count(); + + let depth_range = if full_size == 0 { + (0, 0) + } else { + let mut min_d = u32::MAX; + let mut max_d = 0u32; + for entry in full_included() { + min_d = min_d.min(entry.depth); + max_d = max_d.max(entry.depth); + } + (min_d, max_d) + }; + + let (importance_range, v_depth_range) = if competitive_size == 0 { + ((0.0, 0.0), (0, 0)) + } else { + let mut min_imp = f64::INFINITY; + let mut max_imp = f64::NEG_INFINITY; + let mut min_vd = usize::MAX; + let mut max_vd = 0usize; + for entry in competitive_included() { + let imp = entry.importance.to_f64_approx(); + min_imp = min_imp.min(imp); + max_imp = max_imp.max(imp); + min_vd = min_vd.min(entry.v_depth); + max_vd = max_vd.max(entry.v_depth); + } + ((min_imp, max_imp), (min_vd, max_vd)) + }; + + AnalysisSetSummary { + competitive_size, + full_size, + // The investment set is the whole selection, whether or not a cell + // is online yet, so this figure is taken before the filter rather + // than after it: filtered, it would report an investment that + // excluded every cell still being warmed, which is precisely the + // part of the investment that has been paid for and not yet + // returned. The orchestrator replaces it with the tracker + // population it can see directly. + investment_set_size: self.full.len(), + depth_range, + importance_range, + v_depth_range, + degenerate_cells_skipped: self.degenerate_cells_skipped, + } + } +} diff --git a/packages/sentinel/src/config.rs b/packages/sentinel/src/config.rs new file mode 100644 index 000000000..b451a2d17 --- /dev/null +++ b/packages/sentinel/src/config.rs @@ -0,0 +1,941 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +//! Configuration types for the Spectral Sentinel. +//! +//! [`SentinelConfig`] controls the measurement parameters of the sentinel. +//! +//! These types are deliberately free of policy concerns (thresholds, actions). +//! The sentinel measures; the host decides what the measurements mean. + +use torrust_mudlark::{Accumulator, Inspectable}; + +// ─── Noise schedule (§ALGO S-11.1.3, ADR-S-015 §1) ──────────── + +/// Depth-tiered noise injection schedule. +/// +/// Controls how many noise rounds each newly created tracker receives, +/// varying by G-tree depth. Deeper cells are narrower and need fewer +/// rounds to converge, so the schedule tapers with depth. +/// +/// # Variants +/// +/// - **`Geometric`**: computes rounds as `root × decay^depth`, floored +/// to `min`. Default: `Geometric { root: 450, decay: 0.5, min: 50 }` +/// → `[450, 225, 113, 56, 50, 50, …]`. +/// +/// - **`Explicit`**: a hand-specified per-depth vector. Depths beyond +/// the vector length use the last entry. An empty vector disables +/// noise injection entirely. +/// +/// # Examples +/// +/// ``` +/// use torrust_sentinel::NoiseSchedule; +/// +/// let geo = NoiseSchedule::geometric(400, 0.5, 100); +/// assert_eq!(geo.rounds_for_depth(0), 400); +/// assert_eq!(geo.rounds_for_depth(1), 200); +/// assert_eq!(geo.rounds_for_depth(2), 100); +/// assert_eq!(geo.rounds_for_depth(10), 100); +/// +/// let explicit = NoiseSchedule::Explicit(vec![50, 30, 10]); +/// assert_eq!(explicit.rounds_for_depth(0), 50); +/// assert_eq!(explicit.rounds_for_depth(2), 10); +/// assert_eq!(explicit.rounds_for_depth(99), 10); // clamps to last +/// ``` +#[derive(Debug, Clone)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub enum NoiseSchedule { + /// Geometric decay: `root × decay^depth`, floored to `min`. + Geometric { + /// Rounds at depth 0. + root: u32, + /// Multiplicative decay per depth level (must be in `(0.0, 1.0]`). + decay: f64, + /// Floor — minimum rounds at any depth. + min: u32, + }, + + /// Per-depth explicit schedule. Depths beyond the vector length + /// use the last entry. An empty vector disables noise entirely. + Explicit(Vec), +} + +impl NoiseSchedule { + /// Convenience constructor for the `Geometric` variant. + #[must_use] + pub const fn geometric(root: u32, decay: f64, min: u32) -> Self { + Self::Geometric { root, decay, min } + } + + /// Number of noise rounds for a tracker at the given G-tree depth. + /// + /// Returns `0` when noise is disabled (empty `Explicit` vector). + #[must_use] + pub fn rounds_for_depth(&self, depth: usize) -> u32 { + match self { + Self::Geometric { root, decay, min } => { + let exp = i32::try_from(depth).unwrap_or(i32::MAX); + let raw = f64::from(*root) * decay.powi(exp); + // raw is non-negative (root ≥ 0, decay > 0), safe to truncate. + #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] + let rounded = raw.round() as u32; + rounded.max(*min) + } + Self::Explicit(v) => { + if v.is_empty() { + return 0; + } + // Clamp to last entry for depths beyond the vector. + v[depth.min(v.len() - 1)] + } + } + } + + /// Whether this schedule produces zero rounds at every depth. + /// + /// True for `Explicit(vec![])`, for `Explicit(vec![0, 0, …])`, and for + /// a `Geometric` schedule whose root and floor are both zero. A + /// `Geometric` schedule with a positive root still produces rounds at + /// the shallow depths even when its floor is zero, because the taper + /// starts from the root and only decays towards the floor. + #[must_use] + pub fn is_disabled(&self) -> bool { + match self { + Self::Geometric { root, min, .. } => *root == 0 && *min == 0, + Self::Explicit(v) => v.is_empty() || v.iter().all(|&r| r == 0), + } + } + + /// Maximum rounds this schedule can produce (useful for capacity hints). + /// + /// For a geometric schedule this is the higher of the root and the floor, + /// not the root alone: the taper descends from the root but every depth is + /// lifted to at least the floor, so a floor above the root is what the + /// schedule actually yields at every depth. + #[must_use] + pub fn max_rounds(&self) -> u32 { + match self { + Self::Geometric { root, min, .. } => (*root).max(*min), + Self::Explicit(v) => v.iter().copied().max().unwrap_or(0), + } + } +} + +impl Default for NoiseSchedule { + /// Default: `Geometric { root: 450, decay: 0.5, min: 50 }`. + /// + /// Calibrated for the default forgetting factor λ = 0.99 (§ALGO S-13.1). + /// At λ = 0.99, b = 16, the worst-case baseline convergence is ~398 + /// rounds (surprise axis), so root = 450 provides ~13% margin. + /// The floor of 50 covers deep cells where convergence times scale + /// down with analysis width but remain ~50 rounds at λ = 0.99. + /// See §ALGO S-A.7 for the empirical derivation. + fn default() -> Self { + Self::Geometric { + root: 450, + decay: 0.5, + min: 50, + } + } +} + +/// Measurement parameters for the sentinel. +/// +/// Every field controls *how* the sentinel observes and learns, +/// never *what it thinks* about what it sees. +#[derive(Debug, Clone)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(bound = "V: serde::Serialize + serde::de::DeserializeOwned"))] +pub struct SentinelConfig { + /// Maximum rank (number of basis vectors) any subspace tracker can use. + /// + /// Higher = more expressive model of "normal", but more memory and + /// SVD cost per observation. The actual rank adapts automatically + /// and will never exceed `min(suffix_width, max_rank)`. + /// + /// Default: `16` + pub max_rank: usize, + + /// Exponential forgetting factor (λ). + /// + /// Controls how fast old tracker batches fade from memory. + /// - `0.99` = long memory (~69 batches half-life) + /// - `0.95` = short memory (~14 batches half-life) + /// + /// Must be in `(0.0, 1.0)`. + /// + /// Default: `0.99` + pub forgetting_factor: f64, + + /// How often (in tracker batches) to reassess the rank of each tracker. + /// + /// Rank changes by at most ±1 per evaluation to avoid instability. + /// + /// Default: `100` + pub rank_update_interval: u64, + + /// Cumulative energy threshold for automatic rank adaptation. + /// + /// The rank adapts to capture at least this fraction of the total + /// variance (sum of squared singular values). Lower = fewer dimensions + /// retained, higher = more faithful representation. + /// + /// Must be in `(0.0, 1.0)`. + /// + /// Default: `0.90` + pub energy_threshold: f64, + + /// Numerical stability constant. + /// + /// Added to denominators to prevent division by zero. Must be finite and + /// positive so every score and energy denominator remains meaningful. + /// + /// Default: `1e-6` + pub eps: f64, + + /// Slow EWMA decay factor for per-tracker CUSUM reference baselines. + /// + /// The CUSUM accumulator detects gradual drift that the fast EWMA + /// (controlled by `forgetting_factor`) absorbs. The slow EWMA + /// provides the reference: CUSUM accumulates the gap between the + /// batch mean score and the slow baseline. + /// + /// Must be in `(0.0, 1.0)` and strictly greater than + /// `forgetting_factor` — a slower memory than the fast baseline. + /// + /// Half-life ≈ `ln(2) / ln(1/λ_s)`: + /// - `0.999` = ~693 steps (default) + /// - `0.995` = ~139 steps + /// + /// Default: `0.999` + pub cusum_slow_decay: f64, + + /// Slow EWMA decay factor for coordination-tier CUSUM (§ALGO S-13.1). + /// + /// Controls the CUSUM reference baseline at the cross-cell + /// coordination tier. Separated from `cusum_slow_decay` because + /// the coordination tier may see different batch cadences and the + /// host may want different drift sensitivity at each tier. + /// + /// Must be in `(0.0, 1.0)` and strictly greater than + /// `forgetting_factor`. + /// + /// Default: `0.999` + pub cusum_coord_slow_decay: f64, + + /// CUSUM noise allowance in slow-baseline σ units. + /// + /// Each CUSUM step subtracts `κ_σ · √(slow_variance)` before + /// accumulating. This absorbs normal noise fluctuations so the + /// accumulator only grows under sustained elevation. + /// + /// - `0.5` = tolerate up to half a slow-σ per step (default) + /// - `0.0` = no allowance — any positive gap accumulates + /// - `1.0` = generous allowance — only strong drift accumulates + /// + /// Must be ≥ 0. + /// + /// Default: `0.5` + pub cusum_allowance_sigmas: f64, + + /// Outlier clip width in σ units for EWMA baseline updates. + /// + /// During each EWMA update, observations beyond + /// `mean + clip_sigmas · √variance` are rejected to prevent + /// baseline poisoning. Higher values accept more of the upper + /// tail; lower values clip more aggressively. + /// + /// Must be > 0. + /// + /// Default: `3.0` + pub clip_sigmas: f64, + + /// Clip-pressure EWMA decay factor (`λ_ρ`). + /// + /// Controls how quickly the per-axis clip-pressure estimate adapts + /// to changing contamination levels. Higher values = longer memory. + /// + /// Half-life ≈ ln(2) / `ln(1/λ_ρ)`: + /// - `0.95` = ~14 batches (default, §ALGO S-13.1) + /// - `0.99` = ~69 batches + /// + /// Must be in `(0.0, 1.0)`. + /// + /// Default: `0.95` + pub clip_pressure_decay: f64, + + /// Whether to include per-sample scores in reports. + /// + /// When `true`, each [`CellReport`](crate::CellReport) + /// includes a `Vec` with individual scores for every + /// observation in the batch. Useful for forensics, expensive for + /// large batches. + /// + /// Default: `false` + pub per_sample_scores: bool, + + /// Maximum number of competitive analysis cells (§ALGO S-8.1). + /// + /// Bounds the competitive targets, not the lengths of their ancestor + /// chains. For selected depths `d_i`, the current investment set has + /// at most `1 + sum(d_i)` cell trackers, including the permanent root. + /// Since supported engines have `N >= 2` and eligible targets have + /// `d_i <= N - 2`, this is at most `1 + analysis_k * (N - 2)` + /// (§ALGO S-8.2). Shared ancestors only reduce the count. This counts + /// selected online and warming cells; coordination trackers and an + /// evicted model still held by the warming worker are separate. + /// + /// Must be ≥ 1. + /// + /// Default: `1024` (§ALGO S-13.2: `analysis_K`) + pub analysis_k: usize, + + /// Maximum V-Tree depth at which entries are considered for + /// competitive selection (§ALGO S-8.1). + /// + /// Only V-entries with `v_depth ≤ analysis_depth_cutoff` are + /// eligible. Deeper entries have not yet proven sufficient + /// significance. This prevents noise from promoting ephemeral + /// cells into the analysis set. + /// + /// Must be ≥ 0. A value of 0 means only the V-Tree root + /// (if it is an entry) is eligible — effectively disabling + /// adaptive selection. + /// + /// Default: `6` (§ALGO S-13.2: `analysis_depth_cutoff`) + pub analysis_depth_cutoff: usize, + + /// G-V Graph: minimum accumulated intensity before a cell subdivides. + /// + /// For count-based observation (Δ=1 per value), this is the number of + /// observations a cell must receive before subdividing. + /// + /// Maps to `torrust_mudlark::Config::split_threshold`. + /// + /// Must be > 0. + /// + /// Default: `100` (§ALGO S-13.3: `split_threshold`) + pub split_threshold: V, + + /// G-V Graph: maximum V-Tree depth at which new splits are allowed. + /// + /// Controls how deep the tree can grow. Deeper cells need more + /// sustained traffic to compete for analysis slots. + /// + /// Maps to `torrust_mudlark::Config::depth_create`. + /// + /// Must be ≥ 1. + /// + /// Default: `3` (§ALGO S-13.3: `D_create`) + pub d_create: u32, + + /// G-V Graph: minimum V-Tree depth at which entries become eviction-eligible. + /// + /// Must be strictly greater than `d_create`. The gap (`d_evict − d_create`) + /// is the buffer zone — entries too deep to create children but not yet + /// deep enough to be evicted. + /// + /// Maps to `torrust_mudlark::Config::depth_evict`. + /// + /// Must be > `d_create`. + /// + /// Default: `6` (§ALGO S-13.3: `D_evict`) + pub d_evict: u32, + + /// G-V Graph: hard ceiling on live G-node count. + /// + /// Enables dynamic depth control — the graph adjusts depth gates + /// at runtime to keep the live node count under this ceiling. + /// The sentinel always operates in budgeted mode. + /// + /// Maps to `torrust_mudlark::Config::budget` (wrapped in `Some`). + /// + /// Must be > 0 and large enough to satisfy mudlark's headroom + /// requirement: `budget > max(3^(d_evict − d_create + 1), 2*(d_create − 1))`. + /// + /// Default: `100_000` (§ALGO S-13.3: `budget` / `G_max`) + pub budget: usize, + + /// Depth-tiered noise injection schedule (§ALGO S-11.1.3, ADR-S-015 §1). + /// + /// Controls how many synthetic noise batches each newly created + /// tracker receives, varying by G-tree depth. Deeper cells are + /// narrower and need fewer rounds to converge. + /// + /// Default: `Geometric { root: 450, decay: 0.5, min: 50 }` + pub noise_schedule: NoiseSchedule, + + /// Number of synthetic samples per noise batch (§ALGO S-13.5). + /// + /// Each round feeds this many random ±0.5 centred vectors through + /// the tracker's `observe()` path with `is_noise = true`. + /// + /// Must be ≥ 1 when noise is enabled, with all batch-sized allocations + /// representable at the supported tracker width and rank ceilings. + /// + /// Default: `16` + pub noise_batch_size: usize, + + /// RNG seed for noise generation (§ALGO S-13.5). + /// + /// `Some(seed)` gives reproducible noise and reports for the same configuration and input with `background_warming` disabled, on a fixed build — one target and one set of dependency versions. The generator is chosen for speed rather than portability. + /// Background warming draws from its own generator and takes whichever staged cell leads on volume when it looks, so a seed alone does not fix the baselines or the ingest cycle on which a cell first scores. + /// `None` seeds the generators from system entropy. + /// + /// Default: `Some(42)` + pub noise_seed: Option, + + /// Whether to perform cell warm-up on a background thread (§ALGO S-11.6.8). + /// + /// When `true`, newly created analysis cells are warmed by a + /// dedicated background thread instead of being warmed inline + /// during `ingest()`. This eliminates the latency spike that + /// accompanies cell creation at the cost of a short delay before + /// new cells participate in scoring. + /// + /// When `false`, warm-up runs synchronously within `reconcile_analysis_set()` — identical to the Step 2 behaviour. This mode is deterministic for the same configuration and input with a fixed `noise_seed` on a fixed build, and is used by the test suite. + /// + /// Default: `false` (opt-in; production deployments should enable) + pub background_warming: bool, + + /// Which SVD algorithm to use for subspace evolution (§ALGO S-4.2 Phase 2). + /// + /// - `Naive`: dense thin SVD of the full composite matrix — simple, + /// correct, O(d·(k+b)²) per call. + /// - `Brand`: incremental SVD (Brand 2006) — projects onto the current + /// basis and SVDs a small (k+b)×(k+b) kernel instead. Much faster. + /// + /// In debug builds, **both** algorithms run regardless of this setting + /// and their outputs are compared as a continuous oracle test. + /// + /// Default: `Brand` + pub svd_strategy: crate::SvdStrategy, +} + +impl Default for SentinelConfig { + fn default() -> Self { + Self { + max_rank: 16, + forgetting_factor: 0.99, + rank_update_interval: 100, + energy_threshold: 0.90, + eps: 1e-6, + cusum_slow_decay: 0.999, + cusum_coord_slow_decay: 0.999, + cusum_allowance_sigmas: 0.5, + clip_sigmas: 3.0, + clip_pressure_decay: 0.95, + per_sample_scores: false, + analysis_k: 1024, + analysis_depth_cutoff: 6, + split_threshold: V::from_f64(100.0), + d_create: 3, + d_evict: 6, + budget: 100_000, + noise_schedule: NoiseSchedule::default(), + noise_batch_size: 16, + noise_seed: Some(42), + background_warming: false, + svd_strategy: crate::SvdStrategy::default(), + } + } +} + +/// A reason a [`SentinelConfig`] cannot produce a working sentinel. +/// +/// Most variants are constraint violations that +/// [`validate`](SentinelConfig::validate) finds by reading the configuration +/// alone, and they are collected in one pass so a caller sees every fault at +/// once. A few cannot be reached that way, because what they report is a +/// resource the configuration asks the environment for rather than a value it +/// carries; those are raised where the request is actually made, and arrive on +/// their own. +/// +/// The enumeration is marked as one that grows. Each engine capability a +/// configuration can ask for is one more way the request can be refused, and a +/// caller that matched exhaustively would have to be edited for a refusal it +/// had no opinion about. A wildcard arm reporting the [`Display`] text is the +/// handling this type is designed for. +/// +/// [`Display`]: std::fmt::Display +#[derive(Debug, Clone, PartialEq)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[non_exhaustive] +pub enum ConfigError { + /// `max_rank` must be at least 1. + MaxRankZero, + /// `forgetting_factor` must be in `(0.0, 1.0)`. + ForgettingFactorOutOfRange(f64), + /// `rank_update_interval` must be at least 1. + RankUpdateIntervalZero, + /// `analysis_k` must be at least 1. + AnalysisKZero, + /// `energy_threshold` must be in `(0.0, 1.0)`. + EnergyThresholdOutOfRange(f64), + /// `eps` must be finite. + EpsNotFinite(f64), + /// `eps` must be positive. + EpsNotPositive(f64), + /// `cusum_slow_decay` must be in `(0.0, 1.0)`. + CusumSlowDecayOutOfRange(f64), + /// `cusum_slow_decay` must be strictly greater than `forgetting_factor`. + CusumSlowDecayTooLow { slow: f64, fast: f64 }, + /// `cusum_coord_slow_decay` must be in `(0.0, 1.0)`. + CusumCoordSlowDecayOutOfRange(f64), + /// `cusum_coord_slow_decay` must be strictly greater than `forgetting_factor`. + CusumCoordSlowDecayTooLow { slow: f64, fast: f64 }, + /// `cusum_allowance_sigmas` must be non-negative. + CusumAllowanceNegative(f64), + /// `clip_sigmas` must be positive. + ClipSigmasNotPositive(f64), + /// `clip_pressure_decay` must be in `(0.0, 1.0)`. + ClipPressureDecayOutOfRange(f64), + /// `split_threshold` must be positive. + SplitThresholdNotPositive(f64), + /// `d_create` must be at least 1. + DCreateZero, + /// `d_evict` must be strictly greater than `d_create`. + DEvictNotGreaterThanDCreate { d_create: u32, d_evict: u32 }, + /// `budget` must be positive. + BudgetZero, + /// `budget` must exceed mudlark's headroom requirement. + BudgetTooSmall { budget: usize, required_minimum: usize }, + /// `noise_batch_size` must be ≥ 1 when noise is enabled. + NoiseBatchSizeZero, + /// An enabled noise batch requires an allocation larger than the address space permits. + NoiseBatchSizeTooLarge { batch_size: usize }, + /// `NoiseSchedule::Geometric::decay` must be in `(0.0, 1.0]`. + NoiseScheduleDecayOutOfRange(f64), + /// `NoiseSchedule::Geometric::root` must be > 0 when `min` > 0. + NoiseScheduleRootZero, + /// The headroom requirement the depth pair implies — a power of three in + /// the gap between the two depths, or twice the depth at which cells are + /// created — is too large to represent, so no budget can satisfy it. + DepthBufferTooLarge { d_create: u32, d_evict: u32 }, + /// The coordinate width `N` is below the smallest width a subspace + /// tracker can model. + TrackerDimensionTooSmall { width: u32, minimum: usize }, + /// The coordinate width `N` is above the widest width the centred bit + /// vector that feeds the trackers can carry. + TrackerDimensionTooLarge { width: u32, maximum: usize }, + /// `background_warming` was asked for and the environment refused the + /// thread it runs on. + /// + /// Nothing in the configuration is wrong. A thread is granted by the + /// operating system, and the grant can be refused at any moment for + /// reasons outside this process — which is why this fault is reported + /// where the thread is asked for rather than alongside the constraint + /// violations, and why it arrives alone. + BackgroundWarmingThreadUnavailable { + /// The operating system's own account of the refusal. + reason: String, + }, +} + +impl std::fmt::Display for ConfigError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::MaxRankZero => write!(f, "max_rank must be at least 1"), + Self::ForgettingFactorOutOfRange(v) => { + write!(f, "forgetting_factor must be in (0.0, 1.0), got {v}") + } + Self::RankUpdateIntervalZero => write!(f, "rank_update_interval must be at least 1"), + Self::AnalysisKZero => write!(f, "analysis_k must be at least 1"), + Self::EnergyThresholdOutOfRange(v) => { + write!(f, "energy_threshold must be in (0.0, 1.0), got {v}") + } + Self::EpsNotFinite(v) => write!(f, "eps must be finite, got {v}"), + Self::EpsNotPositive(v) => write!(f, "eps must be positive, got {v}"), + Self::CusumSlowDecayOutOfRange(v) => { + write!(f, "cusum_slow_decay must be in (0.0, 1.0), got {v}") + } + Self::CusumSlowDecayTooLow { slow, fast } => { + write!(f, "cusum_slow_decay ({slow}) must be > forgetting_factor ({fast})") + } + Self::CusumCoordSlowDecayOutOfRange(v) => { + write!(f, "cusum_coord_slow_decay must be in (0.0, 1.0), got {v}") + } + Self::CusumCoordSlowDecayTooLow { slow, fast } => { + write!(f, "cusum_coord_slow_decay ({slow}) must be > forgetting_factor ({fast})") + } + Self::CusumAllowanceNegative(v) => { + write!(f, "cusum_allowance_sigmas must be >= 0, got {v}") + } + Self::ClipSigmasNotPositive(v) => write!(f, "clip_sigmas must be > 0, got {v}"), + Self::ClipPressureDecayOutOfRange(v) => { + write!(f, "clip_pressure_decay must be in (0.0, 1.0), got {v}") + } + Self::SplitThresholdNotPositive(v) => { + write!(f, "split_threshold must be > 0, got {v}") + } + Self::DCreateZero => write!(f, "d_create must be >= 1"), + Self::DEvictNotGreaterThanDCreate { d_create, d_evict } => { + write!(f, "d_evict ({d_evict}) must be > d_create ({d_create})") + } + Self::BudgetZero => write!(f, "budget must be > 0"), + Self::BudgetTooSmall { + budget, + required_minimum, + } => { + write!( + f, + "budget ({budget}) must be > {required_minimum} (mudlark headroom requirement)" + ) + } + Self::NoiseBatchSizeZero => { + write!(f, "noise_batch_size must be >= 1 when noise is enabled") + } + Self::NoiseBatchSizeTooLarge { batch_size } => { + write!(f, "noise_batch_size ({batch_size}) exceeds representable allocation bounds") + } + Self::NoiseScheduleDecayOutOfRange(v) => { + write!(f, "noise_schedule geometric decay must be in (0.0, 1.0], got {v}") + } + Self::NoiseScheduleRootZero => { + write!(f, "noise_schedule geometric root must be > 0 when min > 0") + } + Self::DepthBufferTooLarge { d_create, d_evict } => { + write!( + f, + "the depth pair d_create ({d_create}) / d_evict ({d_evict}) implies a headroom \ + requirement — max(3^(buffer+1), 2*(d_create-1)) — that exceeds the \ + addressable range, so no budget can satisfy it" + ) + } + Self::TrackerDimensionTooSmall { width, minimum } => { + write!( + f, + "coordinate width N ({width}) is below the minimum tracker dimension \ + ({minimum}); a narrower root spans its own space and can model nothing" + ) + } + Self::TrackerDimensionTooLarge { width, maximum } => { + write!( + f, + "coordinate width N ({width}) is above the maximum tracker dimension \ + ({maximum}); a centred bit vector cannot carry a wider observation, and the \ + dimensions past it would be modelled over a constant the data never produced" + ) + } + Self::BackgroundWarmingThreadUnavailable { reason } => { + write!( + f, + "background_warming was requested but the environment refused the warming \ + thread ({reason}); the configuration is sound and the request can be retried \ + or made synchronously" + ) + } + } + } +} + +impl std::error::Error for ConfigError {} + +/// One or more configuration validation errors. +/// +/// Returned by [`SentinelConfig::validate`] when at least one field +/// violates its constraints. Contains every violation found, not +/// just the first. +#[derive(Debug, Clone, PartialEq)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct ConfigErrors(pub Vec); + +impl std::fmt::Display for ConfigErrors { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let n = self.0.len(); + for (i, e) in self.0.iter().enumerate() { + write!(f, "{e}")?; + if i + 1 < n { + write!(f, "; ")?; + } + } + Ok(()) + } +} + +impl std::error::Error for ConfigErrors {} + +fn noise_batch_error(batch_size: usize, max_rank: usize) -> Option { + if batch_size == 0 { + Some(ConfigError::NoiseBatchSizeZero) + } else if noise_allocation_bound(batch_size, max_rank).is_none_or(|bytes| bytes > isize::MAX.unsigned_abs()) { + Some(ConfigError::NoiseBatchSizeTooLarge { batch_size }) + } else { + None + } +} + +// Validation has no coordinate width, so use the supported ceiling D. With +// b samples and k <= D, observe allocates b-by-d and b-by-k matrices; naive +// SVD allocates d-by-(b+k) and thin factors with at most D columns. Brand's +// kernel is used only when b+k+2 <= d, so its square allocations are smaller. +// faer 0.24 pads f64 rows to 64-byte boundaries. Its tall thin-SVD workspace +// contains one padded (b+k)-by-d matrix, two at-most-D-by-D matrices (QR's +// block size is at most d), and a square-SVD workspace. The other SVD branch +// has bounded dimensions here. Include alignment slack for the workspace. +// This checks representability, not whether the machine has enough free RAM. +fn noise_allocation_bound(batch_size: usize, max_rank: usize) -> Option { + use faer::linalg::svd::{ComputeSvdVectors, svd_scratch}; + + let dim = crate::MAX_TRACKER_DIM; + let scalar_bytes = size_of::(); + let alignment = 64; + let columns = batch_size.checked_add(max_rank.min(dim))?; + let padded_rows = columns.checked_next_multiple_of(alignment / scalar_bytes)?; + let matrix_bytes = padded_rows.checked_mul(dim)?.checked_mul(scalar_bytes)?; + let square_bytes = dim.checked_mul(dim)?.checked_mul(scalar_bytes)?; + let square_workspace = (1..=dim) + .map(|width| { + svd_scratch::( + width, + width, + ComputeSvdVectors::Thin, + ComputeSvdVectors::Thin, + faer::get_global_parallelism(), + faer::Spec::default(), + ) + .size_bytes() + }) + .max()?; + let workspace_bytes = matrix_bytes + .checked_add(square_bytes.checked_mul(2)?)? + .checked_add(square_workspace)? + .checked_add(alignment - 1)?; + + // The outer noise vector, borrowed row slices, score scratch vectors, + // and optional per-sample reports each have exactly b elements. The + // inner noise rows have d <= D f64 elements and are already covered. + let element_bytes = [ + size_of::>(), + size_of::<&[f64]>(), + scalar_bytes, + size_of::(), + ] + .into_iter() + .max()?; + Some(workspace_bytes.max(batch_size.checked_mul(element_bytes)?)) +} + +/// Non-fatal diagnostic for parameter combinations that are technically +/// valid but likely to produce poor results (§ALGO S-A.7). +#[derive(Debug, Clone, PartialEq)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub enum ConfigWarning { + /// The noise schedule root is below the empirically derived minimum + /// for the configured forgetting factor, meaning baselines may not + /// converge before real observations arrive. + NoiseScheduleInsufficient { + /// Noise schedule root (depth-0) round count. + root: u32, + /// Minimum recommended rounds for the configured λ. + recommended_root: u32, + /// The configured forgetting factor. + lambda: f64, + }, +} + +impl std::fmt::Display for ConfigWarning { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::NoiseScheduleInsufficient { + root, + recommended_root, + lambda, + } => { + write!( + f, + "noise_schedule root ({root}) is below the recommended minimum \ + ({recommended_root}) for forgetting_factor = {lambda}; \ + baselines may not converge before real observations arrive \ + (see §ALGO S-A.7)" + ) + } + } + } +} + +/// The node budget the depth pair demands, or `None` when the requirement +/// cannot be represented. +/// +/// Mudlark's headroom requirement is `max(3^(buffer+1), 2*(d_create-1))`, +/// where `buffer = d_evict - d_create`. Every step is computed in checked +/// form, not the power alone: the exponent is one past a buffer that can +/// already be the widest number its type holds, and the convergence term +/// doubles a depth that on a thirty-two-bit target can be half the address +/// space. A step that cannot be represented leaves no representable budget +/// that could clear the requirement, so the caller refuses the depth pair on +/// its own terms rather than measuring a budget against a figure that wrapped +/// — and the validation that promises to hand back every fault hands them +/// back, rather than aborting on the arithmetic of a pair that is already +/// faulty for other reasons. +fn headroom_requirement(d_create: u32, d_evict: u32) -> Option { + let buffer = d_evict.checked_sub(d_create)?; + let exponent = buffer.checked_add(1)?; + let headroom = 3usize.checked_pow(exponent)?; + let convergence = (d_create as usize).saturating_sub(1).checked_mul(2)?; + Some(headroom.max(convergence)) +} + +fn validate_eps(eps: f64, errors: &mut Vec) { + if !eps.is_finite() { + errors.push(ConfigError::EpsNotFinite(eps)); + } else if eps <= 0.0 { + errors.push(ConfigError::EpsNotPositive(eps)); + } +} + +impl SentinelConfig { + /// Validate all invariants. + /// + /// Checks every field and collects all violations so the caller + /// can fix them in one pass rather than iterating one-at-a-time. + /// + /// # Errors + /// + /// Returns a [`ConfigErrors`] containing every [`ConfigError`] + /// found, if any. + pub fn validate(&self) -> Result<(), ConfigErrors> { + let mut errors = Vec::new(); + + if self.max_rank == 0 { + errors.push(ConfigError::MaxRankZero); + } + if self.forgetting_factor.is_nan() || self.forgetting_factor <= 0.0 || self.forgetting_factor >= 1.0 { + errors.push(ConfigError::ForgettingFactorOutOfRange(self.forgetting_factor)); + } + if self.rank_update_interval == 0 { + errors.push(ConfigError::RankUpdateIntervalZero); + } + if self.analysis_k == 0 { + errors.push(ConfigError::AnalysisKZero); + } + if self.energy_threshold.is_nan() || self.energy_threshold <= 0.0 || self.energy_threshold >= 1.0 { + errors.push(ConfigError::EnergyThresholdOutOfRange(self.energy_threshold)); + } + validate_eps(self.eps, &mut errors); + if self.cusum_slow_decay.is_nan() || self.cusum_slow_decay <= 0.0 || self.cusum_slow_decay >= 1.0 { + errors.push(ConfigError::CusumSlowDecayOutOfRange(self.cusum_slow_decay)); + } + if self.cusum_slow_decay <= self.forgetting_factor { + errors.push(ConfigError::CusumSlowDecayTooLow { + slow: self.cusum_slow_decay, + fast: self.forgetting_factor, + }); + } + if self.cusum_coord_slow_decay.is_nan() || self.cusum_coord_slow_decay <= 0.0 || self.cusum_coord_slow_decay >= 1.0 { + errors.push(ConfigError::CusumCoordSlowDecayOutOfRange(self.cusum_coord_slow_decay)); + } + if self.cusum_coord_slow_decay <= self.forgetting_factor { + errors.push(ConfigError::CusumCoordSlowDecayTooLow { + slow: self.cusum_coord_slow_decay, + fast: self.forgetting_factor, + }); + } + if self.cusum_allowance_sigmas.is_nan() || self.cusum_allowance_sigmas < 0.0 { + errors.push(ConfigError::CusumAllowanceNegative(self.cusum_allowance_sigmas)); + } + if self.clip_sigmas.is_nan() || self.clip_sigmas <= 0.0 { + errors.push(ConfigError::ClipSigmasNotPositive(self.clip_sigmas)); + } + if self.clip_pressure_decay.is_nan() || self.clip_pressure_decay <= 0.0 || self.clip_pressure_decay >= 1.0 { + errors.push(ConfigError::ClipPressureDecayOutOfRange(self.clip_pressure_decay)); + } + + // ── G-V Graph fields (§ALGO S-13.3) ──────────────────── + if self.split_threshold.to_f64_approx().is_nan() || self.split_threshold.to_f64_approx() <= 0.0 { + errors.push(ConfigError::SplitThresholdNotPositive(self.split_threshold.to_f64_approx())); + } + if self.d_create < 1 { + errors.push(ConfigError::DCreateZero); + } + if self.d_evict <= self.d_create { + errors.push(ConfigError::DEvictNotGreaterThanDCreate { + d_create: self.d_create, + d_evict: self.d_evict, + }); + } + if self.budget == 0 { + errors.push(ConfigError::BudgetZero); + } + // The budget must exceed the headroom the depth gates imply. Only + // checked when d_evict > d_create, since otherwise the pair is already + // refused above and the requirement would describe nothing. + if self.budget > 0 && self.d_evict > self.d_create { + match headroom_requirement(self.d_create, self.d_evict) { + Some(required) if self.budget <= required => { + errors.push(ConfigError::BudgetTooSmall { + budget: self.budget, + required_minimum: required, + }); + } + Some(_) => {} + None => { + errors.push(ConfigError::DepthBufferTooLarge { + d_create: self.d_create, + d_evict: self.d_evict, + }); + } + } + } + + // ── Noise injection fields (§ALGO S-13.5, §ALGO S-11.1.3) ── + if !self.noise_schedule.is_disabled() + && let Some(error) = noise_batch_error(self.noise_batch_size, self.max_rank) + { + errors.push(error); + } + match &self.noise_schedule { + NoiseSchedule::Geometric { root, decay, min } => { + if *decay <= 0.0 || *decay > 1.0 || decay.is_nan() { + errors.push(ConfigError::NoiseScheduleDecayOutOfRange(*decay)); + } + if *root == 0 && *min > 0 { + errors.push(ConfigError::NoiseScheduleRootZero); + } + } + NoiseSchedule::Explicit(_) => { /* any values are valid */ } + } + + if errors.is_empty() { + Ok(()) + } else { + Err(ConfigErrors(errors)) + } + } + + /// Return non-fatal diagnostics for parameter combinations that are + /// technically valid but empirically known to produce poor results. + /// + /// Call after [`validate`](Self::validate) succeeds. The returned + /// warnings are advisory — the sentinel will still function, but + /// warm-up may be insufficient and early scores unreliable. + #[must_use] + pub fn warnings(&self) -> Vec { + let mut warnings = Vec::new(); + + // §ALGO S-A.7: cross-check noise schedule root vs forgetting factor. + // Thresholds derived from empirical convergence data (§ALGO S-Appendix A). + let recommended_root = if self.forgetting_factor >= 0.99 { + 450 + } else if self.forgetting_factor >= 0.95 { + if self.noise_batch_size <= 4 { 200 } else { 50 } + } else { + 0 // no recommendation for very low λ + }; + + if recommended_root > 0 { + let actual_root = self.noise_schedule.rounds_for_depth(0); + if actual_root < recommended_root { + warnings.push(ConfigWarning::NoiseScheduleInsufficient { + root: actual_root, + recommended_root, + lambda: self.forgetting_factor, + }); + } + } + + warnings + } +} diff --git a/packages/sentinel/src/ewma.rs b/packages/sentinel/src/ewma.rs new file mode 100644 index 000000000..c8f2735f9 --- /dev/null +++ b/packages/sentinel/src/ewma.rs @@ -0,0 +1,255 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +//! Exponentially-weighted moving average (EWMA) statistics. +//! +//! Tracks a running mean and variance that exponentially decay old +//! observations. Used by the subspace tracker to maintain baselines +//! of "normal" anomaly scores. +//! +//! Outlier-resistant: observations beyond `clip_sigmas`·σ from the +//! current mean are rejected before updating, preventing an attacker +//! from poisoning the baseline with a single burst. +//! +//! No `faer` dependency — this is pure `f64` arithmetic. + +use crate::report::BaselineSnapshot; + +/// Exponentially-weighted running mean and variance. +/// +/// After `update()` with a batch of values, the baseline reflects +/// a smoothed estimate of the central tendency and spread, biased +/// toward recent observations by the decay factor `λ`. +#[derive(Debug, Clone)] +pub struct EwmaStats { + /// Decay factor (λ). Each old value's contribution shrinks by + /// this factor per update. Higher = longer memory. + decay: f64, + + /// Running weighted mean. + mean: f64, + + /// Running weighted variance. + variance: f64, + + /// Whether at least one real update has occurred. + warm: bool, +} + +impl EwmaStats { + /// Create a new EWMA tracker with the given decay factor. + /// + /// Starts "cold" with `mean = 1.0`, `variance = 1.0` — deliberately + /// wide to avoid extreme z-scores before the first real data arrives. + #[must_use] + pub const fn new(decay: f64) -> Self { + Self { + decay, + mean: 1.0, + variance: 1.0, + warm: false, + } + } + + /// Current mean. + #[must_use] + pub const fn mean(&self) -> f64 { + self.mean + } + + /// Current variance. + #[must_use] + pub const fn variance(&self) -> f64 { + self.variance + } + + /// Whether the baseline has seen at least one update. + /// + /// Compiled for the crate's own tests only. The baseline pipeline acts on + /// the warm flag from inside the update path rather than asking for it, so + /// the accessor exists to let a test state what a run never needs to ask. + #[cfg(test)] + #[must_use] + pub const fn is_warm(&self) -> bool { + self.warm + } + + /// Return to the cold state — as if freshly constructed. + /// + /// Restores the placeholder mean and variance and marks the + /// tracker as cold. The next [`update_raw`](Self::update_raw) will + /// enter the cold→warm initialisation path. + pub const fn reset_cold(&mut self) { + self.mean = 1.0; + self.variance = 1.0; + self.warm = false; + } + + /// Snapshot of the current baseline for inclusion in reports. + #[must_use] + pub const fn snapshot(&self) -> BaselineSnapshot { + BaselineSnapshot { + mean: self.mean, + variance: self.variance, + } + } + + /// Seed this EWMA's mean and variance from another EWMA. + /// + /// Used to close the fast-slow gap after noise injection + /// (ADR-S-013 §6b, Option C): the slow EWMA in the CUSUM + /// accumulator is seeded from the fast EWMA's converged + /// baseline so the two start in agreement. + /// + /// The receiver takes the source's warmth along with its numbers, in + /// both directions. Warmth is not a separate fact about the receiver but + /// part of what the baseline being handed over *is*: it says whether + /// those two numbers were measured or are the placeholders a fresh + /// tracker starts from. A receiver left warm over a cold source's + /// placeholders would clip and score against a notion of normal that + /// nothing observed, and the cold path that exists to replace exactly + /// that state would never run again. + pub const fn seed_from(&mut self, source: &Self) { + self.mean = source.mean; + self.variance = source.variance; + self.warm = source.warm; + } + + /// Compute the z-score of a value against the current baseline. + /// + /// Returns `(value - mean) / (sqrt(variance) + eps)`. + /// + /// The stability constant sits outside the root rather than inside it, so + /// it floors the deviation the score is divided by rather than the + /// variance. The two readings differ exactly where the constant exists to + /// matter — a baseline whose variance is small beside it — and the + /// outside form is the one the package's own definition of this score + /// states. Flooring the standard deviation also keeps the constant in the + /// units of the quantity it guards, where flooring the variance would + /// make its effect on the divisor depend on its own square root. + /// + /// The caller supplies `eps` (typically + /// [`SentinelConfig::eps`](crate::config::SentinelConfig::eps)) + /// so that every component of the sentinel shares a single + /// stability constant. + #[must_use] + pub fn z_score(&self, value: f64, eps: f64) -> f64 { + (value - self.mean) / (self.variance.sqrt() + eps) + } + + /// Compute the upper-tail clip ceiling: `mean + clip_sigmas · √variance`. + /// + /// Returns `f64::INFINITY` when the baseline is cold (no meaningful + /// ceiling can be defined — matches the cold-path bypass in `update()`). + #[must_use] + pub fn ceiling(&self, clip_sigmas: f64) -> f64 { + if self.warm { + clip_sigmas.mul_add(self.variance.sqrt(), self.mean) + } else { + f64::INFINITY + } + } + + /// Update the baseline with pre-filtered values. + /// + /// The caller is responsible for outlier rejection. This method + /// unconditionally incorporates all values (including the cold→warm + /// path). Used by the baseline pipeline (§ALGO S-6.1.1) where + /// clipping is externalised to `update_axis()`. + pub fn update_raw(&mut self, normals: &[f64]) { + if normals.is_empty() { + return; + } + + #[allow(clippy::cast_precision_loss)] + let new_mean = normals.iter().sum::() / normals.len() as f64; + + if !self.warm { + self.mean = new_mean; + if normals.len() > 1 { + let var = mean_squared_deviation(normals, new_mean); + self.variance = var.max(1e-4); + } + self.warm = true; + return; + } + + let alpha = 1.0 - self.decay; + self.mean = self.decay.mul_add(self.mean, alpha * new_mean); + + if normals.len() > 1 { + let var = mean_squared_deviation(normals, new_mean).max(1e-4); + self.variance = self.decay.mul_add(self.variance, alpha * var); + } + } + + /// Update the baseline with a batch of new values. + /// + /// Values beyond `mean + clip_sigmas·√variance` are rejected + /// (outlier resistance). If all values are outliers, the baseline + /// is unchanged. + /// + /// Only the **upper** tail is clipped. Anomaly scores are + /// non-negative and right-skewed — an attacker inflates them, + /// never deflates them. A lower-tail bound would wrongly reject + /// legitimate low scores during quiet periods. + /// + /// The filter is **skipped entirely on the first update** (while + /// the tracker is still cold). The initial `mean = 1.0` / + /// `variance = 1.0` are placeholders, not a real baseline — you + /// can't define "outlier" without one. + /// + /// Compiled for the crate's own tests only. The baseline pipeline computes + /// one shared clip filter per axis and calls + /// [`update_raw`](Self::update_raw) with the retained values, as ADR-S-020 + /// decided; this self-contained form survives because a test of the filter + /// wants the filter and the update in one call, which no production caller + /// does. + #[cfg(test)] + pub fn update(&mut self, values: &[f64], clip_sigmas: f64) { + if values.is_empty() { + return; + } + + // When cold, accept everything — no real baseline to filter against. + // Upper-tail only: anomaly scores are right-skewed. + let normals: Vec = if self.warm { + let ceiling = clip_sigmas.mul_add(self.variance.sqrt(), self.mean); + let filtered: Vec = values.iter().copied().filter(|&v| v < ceiling).collect(); + if filtered.is_empty() { + return; // all outliers — learn nothing + } + filtered + } else { + values.to_vec() + }; + + #[allow(clippy::cast_precision_loss)] // batch len ≪ 2^52 + let new_mean = normals.iter().sum::() / normals.len() as f64; + + if !self.warm { + self.mean = new_mean; + if normals.len() > 1 { + let var = mean_squared_deviation(&normals, new_mean); + self.variance = var.max(1e-4); + } + self.warm = true; + return; + } + + let alpha = 1.0 - self.decay; + self.mean = self.decay.mul_add(self.mean, alpha * new_mean); + + if normals.len() > 1 { + let var = mean_squared_deviation(&normals, new_mean).max(1e-4); + self.variance = self.decay.mul_add(self.variance, alpha * var); + } + } +} + +/// Mean squared deviation from the given mean (÷N, no Bessel's correction). +fn mean_squared_deviation(values: &[f64], mean: f64) -> f64 { + #[allow(clippy::cast_precision_loss)] // batch len ≪ 2^52 + let n = values.len() as f64; + values.iter().map(|v| (v - mean).powi(2)).sum::() / n +} diff --git a/packages/sentinel/src/lib.rs b/packages/sentinel/src/lib.rs new file mode 100644 index 000000000..35235ab63 --- /dev/null +++ b/packages/sentinel/src/lib.rs @@ -0,0 +1,193 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +#![forbid(unsafe_code)] + +//! # Spectral Sentinel +//! +//! Hierarchical online subspace anomaly detection for positionally +//! structured observation streams. +//! +//! Spectral Sentinel combines Mudlark's adaptive spatial index with a layer +//! of low-rank statistical trackers. Mudlark ranks spatial entries by +//! observation volume; Spectral Sentinel selects significant V-Tree entries, +//! closes them under G-tree ancestry, and scores incoming batches against +//! learned subspace models for that selected structure. +//! +//! The core engine, [`SpectralSentinel`], is generic over the coordinate +//! type `C`, accumulator type `V`, and bit-width `N`, mirroring Mudlark's +//! `GvGraph` triple. Each cell tracker analyses the suffix bits +//! `[d, N)` at width `w = N - d`, where `d` is the cell's G-tree depth. +//! +//! That width is bounded at both ends. Below, a tracker needs at least two +//! dimensions before a residual means anything. Above, a coordinate value +//! reaches its tracker as a centred bit vector, and [`CentredBits`] carries +//! its values in a fixed array of 128 slots — so a width past that has +//! nowhere to put its bits, and is refused at construction rather than +//! modelled at whatever width the array happens to hold. [`Sentinel128`] +//! sits exactly at that ceiling. +//! +//! The crate provides two convenience aliases: +//! +//! - [`Sentinel128`] — `SpectralSentinel`, the default +//! for IPv6-class domains. +//! - [`Sentinel64`] — `SpectralSentinel`, for 64-bit +//! domains. +//! +//! Input values must have **hierarchical positional structure**: leading +//! bits define coarse groupings and successive bits refine them. IPv6-like +//! address spaces are a natural fit. Pseudo-random identifiers, hashes, +//! UUIDs, and nonces have no meaningful suffix structure for the model to +//! learn, so Spectral Sentinel will still process them but the measurements +//! will not be useful. +//! +//! # Architecture +//! +//! Spectral Sentinel has three cooperating layers: +//! +//! - **Spatial substrate** — a [`torrust_mudlark::GvGraph`] tracks volume +//! over the coordinate domain and ranks spatial entries by accumulated +//! observation volume. +//! - **Analysis selector** — [`AnalysisSet`] chooses significant V-Tree +//! entries, then closes them under G-tree ancestry so every selected cell +//! has a complete model chain back to the root. +//! - **Analysis engine** — per-cell subspace trackers score four raw axes: +//! novelty, displacement, surprise, and coherence. A second coordination +//! tier models cross-cell score patterns. +//! +//! **Feed-forward invariant.** Spectral Sentinel always observes the spatial +//! layer with `Delta = 1` per input value. Anomaly scores never flow back +//! into Mudlark's importance accounting. Spatial adaptation is driven by +//! observation volume only; the host controls temporal policy through +//! explicit decay operations. +//! +//! # Design Principle +//! +//! **Spectral Sentinel measures; the host decides.** +//! +//! Public report types carry raw statistical measurements: scores, +//! baselines, drift accumulators, maturity, geometry, structural summaries, +//! and health snapshots. They do not contain threat levels, recommended +//! actions, or policy decisions. The consuming application interprets the +//! measurements in its own domain. +//! +//! # Three-Surface Visibility Model +//! +//! Every public symbol belongs to one of three surfaces. Modules remain +//! crate-private and public types are re-exported flat from the crate root, +//! giving downstream users one canonical import path. +//! +//! - **Surface 1 — Readouts:** Lightweight data users inspect after an +//! observation cycle: [`BatchReport`], [`CellReport`], +//! [`CoordinationReport`], score distributions, baseline snapshots, +//! structural and health summaries, [`AnalysisSet`], [`AnalysisEntry`], +//! [`CentredBits`], and related report records. +//! - **Surface 2 — Engine:** Opaque operational API users configure and +//! drive: [`SpectralSentinel`], [`SentinelConfig`], [`NoiseSchedule`], +//! [`SvdStrategy`], [`Sentinel128`], [`Sentinel64`], +//! [`CentredBitSource`], and [`GNodeId`] for subtree decay calls. +//! - **Surface 3 — Internals:** EWMA state, SVD update plumbing, tracker +//! machinery, staging, CUSUM, and warming-thread implementation details. + +// -- Surface 1 — Readouts (data users hold and inspect) ----------- +// +// Modules are private; types are re-exported flat from the crate root +// so there is exactly one canonical path per public type. +pub(crate) mod analysis_set; +pub(crate) mod observation; +pub(crate) mod report; + +// -- Surface 2 — Engine (opaque operational API) ------------------ +pub(crate) mod config; +pub(crate) mod sentinel; + +// -- Surface 3 — Internals (implementation machinery) ------------- +// +// Crate-private. Downstream crates cannot import from these modules; +// all external access goes through the flat re-exports below and the +// public methods on SpectralSentinel. +pub(crate) mod ewma; +pub(crate) mod maths; + +// README doc-tests: compile and run every code block in the README +// as part of `cargo test --doc`. +#[cfg(doctest)] +#[doc = include_str!("../README.md")] +mod _readme {} + +#[cfg(test)] +mod tests; + +// -- Public re-exports -------------------------------------------- +// +// One canonical path per public type. Downstream code should use +// `torrust_sentinel::{SpectralSentinel, SentinelConfig, BatchReport, ...}` +// rather than reaching into submodules. +// +// Surface 1 — Readouts: +// analysis_set: AnalysisEntry, AnalysisSet +// observation: CentredBits +// report: BatchReport, CellReport, CoordinationReport, +// AnomalyScores, ScoreDistribution, baseline/drift, +// maturity, geometry, contour, health, and summaries +// +// Surface 2 — Engine: +// config: ConfigError, ConfigErrors, ConfigWarning, +// NoiseSchedule, SentinelConfig +// maths: SvdStrategy +// observation: CentredBitSource +// sentinel: SpectralSentinel +// aliases: Sentinel128, Sentinel64 +// mudlark: GNodeId for subtree-oriented operations +pub use analysis_set::{AnalysisEntry, AnalysisSet}; +pub use config::{ConfigError, ConfigErrors, ConfigWarning, NoiseSchedule, SentinelConfig}; +pub use maths::SvdStrategy; +pub use observation::{CentredBitSource, CentredBits}; +pub use report::{ + AnalysisSetSummary, AnomalyScores, AxisBaselineSnapshots, BaselineSnapshot, BatchReport, CellInspection, CellReport, + ClipPressureDistribution, ContourSnapshot, CoordinationHealth, CoordinationReport, CusumSnapshot, GeometryDistribution, + HealthReport, MaturityDistribution, MemberScore, RankDistribution, SampleScore, ScoreDistribution, ScoringGeometry, + TrackerMaturity, +}; +pub use sentinel::SpectralSentinel; +// Re-export the G-V Graph node handle for use with `decay_subtree()`. +pub use torrust_mudlark::GNodeId; + +/// 128-bit sentinel: full `u128` domain, `u64` counts, 128-bit width. +pub type Sentinel128 = SpectralSentinel; + +/// 64-bit sentinel: `u64` domain, `u64` counts, 64-bit width. +pub type Sentinel64 = SpectralSentinel; + +/// Minimum suffix width for a functional subspace tracker. +/// +/// At `dim < MIN_TRACKER_DIM`, the tracker cannot form a +/// meaningful basis or compute residuals. Cells below this +/// threshold are excluded from the analysis set; a cell at it +/// is kept, which is what makes the value a minimum rather +/// than a floor the analysis set sits above. +/// +/// The value 2 ensures at least one residual degree of freedom +/// ($d - k \geq 1$ when $k = 1$). A tracker with `dim = 1` can +/// technically run but produces identically-zero residuals (novelty) +/// since the single basis vector spans the entire space — making it +/// statistically useless. +/// +/// See ADR-S-011 for rationale. +pub(crate) const MIN_TRACKER_DIM: usize = 2; + +/// Widest coordinate width the observation path can carry. +/// +/// A coordinate value reaches a tracker as a centred bit vector, and +/// [`CentredBits`] holds its values in a fixed array of this many slots. +/// The bridge that produces those vectors, [`CentredBitSource`], is open to +/// a coordinate type of any width, and the spatial layer asks only that `N` +/// fit the coordinate type — so a wider type carrying a wider `N` would +/// otherwise build an `N`-dimensional tracker fed from a vector that can +/// never hold more than this many values. The columns past the end of that +/// vector are not missing data the arithmetic would notice: they arrive as +/// zeros, which centred bits never are, so novelty, residual and rank would +/// all be computed over a constant the coordinate stream never produced. +/// Cell depth travels the same path in a single byte, which this ceiling +/// keeps honest as well. +pub(crate) const MAX_TRACKER_DIM: usize = 128; diff --git a/packages/sentinel/src/maths/README.md b/packages/sentinel/src/maths/README.md new file mode 100644 index 000000000..95b588f4e --- /dev/null +++ b/packages/sentinel/src/maths/README.md @@ -0,0 +1,5 @@ +## Unit test matrix · `tab:sentinel:maths-unit-test-matrix` + +**Table (Unit test matrix)** + +No unit tests in this folder. diff --git a/packages/sentinel/src/maths/bench_tracing.rs b/packages/sentinel/src/maths/bench_tracing.rs new file mode 100644 index 000000000..018c93d0e --- /dev/null +++ b/packages/sentinel/src/maths/bench_tracing.rs @@ -0,0 +1,152 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +//! Lightweight tracing layer that accumulates per-span-name wall-clock +//! durations. Used by convergence benchmarks to capture the cost of +//! each SVD strategy without `Instant`-based plumbing in the hot path. +//! +//! # Usage +//! +//! ```ignore +//! let (timing, _guard) = SpanTiming::install(); +//! // ... run observe() calls ... +//! let naive_ns = timing.total_ns("svd_naive"); +//! let brand_ns = timing.total_ns("svd_brand"); +//! ``` +//! +//! The example remains ignored because this benchmark-only module is private and compiled only with the crate's tests, so an external doctest cannot name `SpanTiming`. +//! +//! The layer stores enter/exit timestamps per span instance in an +//! `RwLock` and aggregates into cumulative nanoseconds on +//! close. It is designed for single-threaded benchmark use — the +//! `RwLock` is uncontended. +//! +//! # §-references +//! +//! - ADR-S-016 — Brand's incremental SVD +//! - ADR-M-028 — Span-native tracing + +use std::collections::HashMap; +use std::sync::{Arc, RwLock}; +use std::time::Instant; + +use tracing::{Subscriber, span}; +use tracing_subscriber::Layer; +use tracing_subscriber::layer::Context; +use tracing_subscriber::prelude::*; +use tracing_subscriber::registry::LookupSpan; + +// ════════════════════════════════════════════════════════════ +// Per-span storage (attached via Extensions) +// ════════════════════════════════════════════════════════════ + +struct SpanEnterTime(Instant); + +// ════════════════════════════════════════════════════════════ +// Accumulated timing map +// ════════════════════════════════════════════════════════════ + +/// Shared timing accumulator. +/// +/// Keys are span names (e.g. `"svd_naive"`, `"svd_brand"`, +/// `"phase2_evolve_subspace"`). Values are cumulative nanoseconds +/// spent inside spans of that name, and the call count. +#[derive(Debug, Clone, Default)] +pub struct SpanTiming { + inner: Arc>, +} + +#[derive(Debug, Default)] +struct TimingMap { + entries: HashMap<&'static str, TimingEntry>, +} + +#[derive(Debug, Default, Clone, Copy)] +struct TimingEntry { + total_ns: u128, + count: u64, +} + +impl SpanTiming { + /// Install a new timing layer as the thread-local default subscriber + /// and return the handle for reading accumulated durations. + /// + /// The returned `DefaultGuard` must be kept alive for the duration + /// of the measurement. Dropping it uninstalls the subscriber. + pub fn install() -> (Self, tracing::subscriber::DefaultGuard) { + let timing = Self::default(); + let layer = SpanTimingLayer { + timing: timing.inner.clone(), + }; + // Respect RUST_LOG, defaulting to INFO. This means + // `tracing::enabled!(Level::DEBUG)` is false unless the + // user sets `RUST_LOG=debug` — matching the mudlark + // convention and preventing the oracle from firing + // accidentally in release-mode benchmarks. + let env_filter = tracing_subscriber::EnvFilter::builder() + .with_default_directive(tracing::Level::INFO.into()) + .from_env_lossy(); + let subscriber = tracing_subscriber::registry().with(env_filter).with(layer); + let guard = tracing::subscriber::set_default(subscriber); + (timing, guard) + } + + /// Cumulative nanoseconds spent inside spans named `name`. + pub fn total_ns(&self, name: &str) -> u128 { + self.inner.read().unwrap().entries.get(name).map_or(0, |e| e.total_ns) + } + + /// Number of times a span named `name` was entered. + #[allow(dead_code)] + pub fn call_count(&self, name: &str) -> u64 { + self.inner.read().unwrap().entries.get(name).map_or(0, |e| e.count) + } + + /// Reset all accumulated timing. + pub fn reset(&self) { + self.inner.write().unwrap().entries.clear(); + } +} + +// ════════════════════════════════════════════════════════════ +// Tracing Layer implementation +// ════════════════════════════════════════════════════════════ + +struct SpanTimingLayer { + timing: Arc>, +} + +impl Layer for SpanTimingLayer +where + S: Subscriber + for<'a> LookupSpan<'a>, +{ + fn on_enter(&self, id: &span::Id, ctx: Context<'_, S>) { + if let Some(span) = ctx.span(id) { + let mut extensions = span.extensions_mut(); + extensions.insert(SpanEnterTime(Instant::now())); + } + } + + // The RwLockWriteGuard (`map`) must live as long as `entry` borrows it; + // there is no earlier drop point. + #[allow(clippy::significant_drop_tightening)] + fn on_exit(&self, id: &span::Id, ctx: Context<'_, S>) { + if let Some(span) = ctx.span(id) { + let elapsed = { + let extensions = span.extensions(); + let ns = extensions.get::().map(|t| t.0.elapsed().as_nanos()); + drop(extensions); // release read-lock before acquiring write-lock + ns + }; + if let Some(ns) = elapsed { + let name = span.name(); + { + let mut map = self.timing.write().unwrap(); + let entry = map.entries.entry(name).or_default(); + entry.total_ns += ns; + entry.count += 1; + } + } + } + } +} diff --git a/packages/sentinel/src/maths/brand_svd.rs b/packages/sentinel/src/maths/brand_svd.rs new file mode 100644 index 000000000..ecd3a3916 --- /dev/null +++ b/packages/sentinel/src/maths/brand_svd.rs @@ -0,0 +1,207 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +//! Brand's incremental SVD for subspace evolution (ADR-S-016). +//! +//! Instead of SVD-ing the full d × (k+b) composite matrix, this +//! projects new data onto the current basis, QR-orthogonalises +//! the residual, and SVDs a small (k+b) × (k+b) kernel matrix. +//! +//! The algorithm (Brand 2006) is: +//! +//! 1. P = `U_k`ᵀ Xᵀ = Zᵀ (k × b) — reuse Phase 1 +//! 2. Q = Xᵀ − `U_k` P = residualᵀ (d × b) — reuse Phase 1 +//! 3. Q⊥ R⊥ = `thin_qr(Q)` (d × b), (b × b) +//! 4. K = \[√λ·diag(σ), P; 0, R⊥\] ((k+b) × (k+b)) +//! 5. Û, σ̂, _ = `thin_svd(K)` small SVD! +//! 6. `U_new` = \[`U_k` | Q⊥\] · Û\[:, :n\] back-transform +//! +//! Cost: O((k+b)³ + d·b²) vs O(d·(k+b)²) for the naïve approach. +//! The win comes from the SVD kernel shrinking from d rows to (k+b). +//! +//! # §-references +//! +//! - §ALGO S-4.2 Phase 2 — Subspace evolution +//! - ADR-S-016 — Brand's incremental SVD + +use faer::Mat; + +use super::SubspaceUpdate; + +/// Evolve the subspace via Brand's incremental SVD. +/// +/// # Arguments +/// +/// * `current_basis` — `U_k`, shape `(d, cap)`. Only columns `[:k]` active. +/// * `sigmas` — singular values, length `cap`. Only `[:k]` meaningful. +/// * `z` — latent projection `X · U_k`, shape `(b, k)`. +/// * `residual` — reconstruction residual `X − X̂`, shape `(b, d)`. +/// * `sqrt_lambda` — `√λ`. +/// * `k` — current active rank. +/// * `cap` — hard ceiling on rank. +// Linear algebra code — single-char names follow standard mathematical notation +// (d=dimension, b=batch, k=rank, c=kernel dim, n=output rank). +#[allow(clippy::many_single_char_names)] +#[must_use] +pub fn evolve( + current_basis: &Mat, + sigmas: &[f64], + z: &Mat, + residual: &Mat, + sqrt_lambda: f64, + k: usize, + cap: usize, +) -> Option { + let d = current_basis.nrows(); + let b = residual.nrows(); + let c = k + b; // kernel dimension + + // Guard: Brand's algorithm builds a (c × c) kernel and back- + // transforms through [U_k | Q⊥]. This only provides a real + // advantage when c is substantially smaller than d. + // + // • c > d — The thin QR of residual^T (d × b) produces R⊥ + // that is *not* square (ℝ^(d×b) rather than ℝ^(b×b)), + // causing index-out-of-bounds in the kernel construction. + // + // • c = d — The kernel SVD is the same size as a full dense + // SVD but adds extra QR + back-transform roundoff. + // + // • c + 1 = d — At tiny d (e.g. coordination tier d=4, c=3) + // the extra roundoff produces large basis errors (observed: + // 54° divergence with well-separated σ after many steps). + // + // Require at least 2 spare dimensions (d − c ≥ 2) so the QR + // residual has room for stable orthogonalisation. Fall back + // to the naïve path otherwise. + if c + 2 > d { + return None; + } + + // ── Step 1–2: P = Z^T, Q = residual^T ────────────── + // Both are already computed in Phase 1 of observe(). + // P = Z^T is (k × b), Q = residual^T is (d × b). + + // ── Step 3: Thin QR of Q = residual^T ─────────────── + // Q = residual^T ∈ ℝ^(d × b). + // QR gives Q⊥ ∈ ℝ^(d × b) (orthonormal) and R⊥ ∈ ℝ^(b × b) (upper triangular). + let mut q_t = Mat::zeros(d, b); + for i in 0..b { + for j in 0..d { + q_t[(j, i)] = residual[(i, j)]; + } + } + + let qr = q_t.as_ref().qr(); + let q_perp = qr.compute_thin_Q(); // (d × b) + let r_perp = qr.thin_R(); // (b × b) — upper triangular + + // ── Step 4: Build kernel K ∈ ℝ^(c × c) ───────────── + // + // K = [ √λ · diag(σ₁..ₖ) P ] + // [ 0 R⊥ ] + // + // where P = Z^T ∈ ℝ^(k × b). + let mut kernel = Mat::zeros(c, c); + + // Top-left: √λ · diag(σ[:k]) + for j in 0..k { + kernel[(j, j)] = sqrt_lambda * sigmas[j]; + } + + // Top-right: P = Z^T (z is b×k, we need P = k×b) + for i in 0..k { + for j in 0..b { + kernel[(i, k + j)] = z[(j, i)]; + } + } + + // Bottom-right: R⊥ + for i in 0..b { + for j in 0..b { + kernel[(k + i, k + j)] = r_perp[(i, j)]; + } + } + + // ── Step 5: Small SVD of kernel ───────────────────── + let svd = kernel.thin_svd().ok()?; + + let u_hat = svd.U(); // (c × c), retain all columns through correction + let s_hat = svd.S().column_vector(); + + // ── Step 6: Back-transform U_new = [U_k | Q⊥] · Û + // + // [U_k | Q⊥] is (d × c), Û[:, :n] is (c × n). + // Compute column-by-column to avoid materialising the (d × c) join. + let mut basis = Mat::zeros(d, c); + + for col in 0..c { + for row in 0..d { + let mut val = 0.0; + // U_k block: columns 0..k of [U_k | Q⊥], rows of Û: 0..k + for j in 0..k { + val = current_basis[(row, j)].mul_add(u_hat[(j, col)], val); + } + // Q⊥ block: columns k..c of [U_k | Q⊥], rows of Û: k..c + for j in 0..b { + val = q_perp[(row, j)].mul_add(u_hat[(k + j, col)], val); + } + basis[(row, col)] = val; + } + } + + // ── Step 7: Re-orthogonalise before truncating ─────── + // + // [U_k | Q_perp] need not be orthogonal when the residual is rank + // deficient. Retaining only cap kernel vectors before this correction + // discards components in the wrong metric. Correct all c components, + // resolve repeated spaces, and only then apply the rank cap. + // + // QR-factorise the complete back-transform and absorb its metric: + // basis = Q*R, then SVD(R*diag(sigma)) supplies the corrected spectrum + // and directions. This preserves the original composite matrix even + // when a nearly zero residual gives Q_perp columns overlapping U_k. + // The cost is O(d*c^2 + c^3), still within the small-kernel regime. + let qr_correction = basis.as_ref().qr(); + let q_out = qr_correction.compute_thin_Q(); // (d × c) + let r_out = qr_correction.thin_R(); // (c × c) + + // Form M = R · diag(σ̂), a c × c matrix. + let mut m_corr = Mat::zeros(c, c); + for i in 0..c { + for j in 0..c { + m_corr[(i, j)] = r_out[(i, j)] * s_hat[j]; + } + } + + // SVD of the small corrective matrix. + let Some(corr_svd) = m_corr.thin_svd().ok() else { + // The corrective factorisation is what re-orthogonalises the basis, + // so there is no result to return without it. Handing back the + // pre-correction basis would satisfy the signature while breaking + // what the returned value promises — the field is documented + // orthonormal, and every caller writes it straight into the tracker's + // state and then relies on that. Reporting the failure instead lets + // the dispatcher fall back to the strategy that does not need this + // step, which is the same answer it gives when this algorithm cannot + // handle the dimensions at all. + return None; + }; + let u_corr = corr_svd.U(); // (c × c) + let s_corr = corr_svd.S().column_vector(); + + // Final basis = Q_out · U_corr, final sigmas = diag(S_corr). + let mut final_basis = Mat::zeros(d, c); + for col in 0..c { + for row in 0..d { + let mut val = 0.0; + for j in 0..c { + val = q_out[(row, j)].mul_add(u_corr[(j, col)], val); + } + final_basis[(row, col)] = val; + } + } + + let out_sigmas = (0..c).map(|i| s_corr[i]).collect(); + Some(super::truncate_update(final_basis, out_sigmas, cap)) +} diff --git a/packages/sentinel/src/maths/mod.rs b/packages/sentinel/src/maths/mod.rs new file mode 100644 index 000000000..781819e5a --- /dev/null +++ b/packages/sentinel/src/maths/mod.rs @@ -0,0 +1,409 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +//! Linear algebra building blocks for the subspace tracker. +//! +//! This module isolates the SVD-based subspace evolution from the +//! rest of the tracker so that: +//! +//! 1. The algorithm can be swapped at runtime between the naïve +//! dense thin SVD and Brand's incremental SVD (ADR-S-016). +//! 2. In debug builds (or with a `DEBUG`-level tracing subscriber), +//! **both** algorithms run and their outputs are compared via +//! `assert!` — a continuous oracle test. +//! 3. The maths is independently unit-testable against known +//! matrix identities. +//! +//! # §-references +//! +//! - §ALGO S-4.2 Phase 2 — Subspace evolution +//! - ADR-S-016 — Brand's incremental SVD + +pub mod brand_svd; +pub mod naive_svd; + +#[cfg(test)] +mod tests; + +use std::cell::Cell; + +use faer::Mat; + +// Re-export so benchmarks can reference the timing helper. +#[cfg(test)] +pub mod bench_tracing; + +// ════════════════════════════════════════════════════════════ +// Strategy enum +// ════════════════════════════════════════════════════════════ + +/// Which SVD algorithm to use for subspace evolution (§ALGO S-4.2 Phase 2). +/// +/// Selectable at runtime via [`SentinelConfig::svd_strategy`](crate::SentinelConfig). +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub enum SvdStrategy { + /// Dense thin SVD of the full composite matrix M ∈ ℝ^(d × (k+b)). + /// + /// Correct and simple but O(d·(k+b)²) per call. This is the + /// original implementation. + Naive, + + /// Brand's incremental SVD (Brand 2006). + /// + /// Projects onto the current basis, QR-orthogonalises the residual, + /// and SVDs a small (k+b)×(k+b) kernel. O((k+b)³ + d·b²) per call. + #[default] + Brand, +} + +// ════════════════════════════════════════════════════════════ +// Result type +// ════════════════════════════════════════════════════════════ + +/// Output of a subspace evolution step. +/// +/// Contains the updated basis vectors and singular values, ready +/// to be written back into the tracker's state. +#[derive(Debug, Clone)] +pub struct SubspaceUpdate { + /// Updated orthonormal basis, shape `(dim, n)` where + /// `n = min(k+b, d, cap)`. + pub basis: Mat, + + /// Updated singular values, length `n`. + pub sigmas: Vec, + + /// How many components are meaningful (`n`). + pub n: usize, +} + +// ════════════════════════════════════════════════════════════ +// Dispatch +// ════════════════════════════════════════════════════════════ + +thread_local! { + /// Alternates execution order in the oracle so that neither + /// strategy consistently benefits from warmed caches or branch + /// predictors. Toggled on every oracle-active call. + static ORACLE_FLIP: Cell = const { Cell::new(false) }; +} + +/// Run one SVD strategy under a named tracing span. +#[allow(clippy::too_many_arguments)] +fn run_svd( + which: SvdStrategy, + current_basis: &Mat, + sigmas: &[f64], + z: &Mat, + residual: &Mat, + sqrt_lambda: f64, + k: usize, + cap: usize, +) -> Option { + match which { + SvdStrategy::Naive => { + let _span = tracing::info_span!("svd_naive").entered(); + naive_svd::evolve(current_basis, sigmas, z, residual, sqrt_lambda, k, cap) + } + SvdStrategy::Brand => { + let _span = tracing::info_span!("svd_brand").entered(); + brand_svd::evolve(current_basis, sigmas, z, residual, sqrt_lambda, k, cap) + } + } +} + +/// Run subspace evolution using the selected strategy (§ALGO S-4.2 Phase 2). +/// +/// In **debug builds** (or when a `DEBUG`-level tracing subscriber is +/// attached), both strategies are executed and compared element-wise — +/// a continuous oracle test. Execution order alternates on each call +/// via a thread-local toggle so neither strategy consistently benefits +/// from warmed caches. +/// +/// # Arguments +/// +/// * `strategy` — which algorithm to use for the returned result. +/// * `current_basis` — current `U_k`, shape `(d, cap)`. Only `[:, :k]` is active. +/// * `sigmas` — current singular values, length `cap`. Only `[:k]` meaningful. +/// * `z` — latent projection from Phase 1: `X · U_k`, shape `(b, k)`. +/// * `residual` — reconstruction residual from Phase 1: `X − X̂`, shape `(b, d)`. +/// * `sqrt_lambda` — `√λ` (square root of the forgetting factor). +/// * `k` — current active rank. +/// * `cap` — hard ceiling on rank. +/// +/// # Returns +/// +/// `Some(SubspaceUpdate)` on success, `None` if SVD failed to converge. +/// +/// # Panics +/// +/// Panics (via `assert!`) if the oracle is active and the two SVD +/// strategies produce results that differ beyond tolerance. +#[allow(clippy::too_many_arguments)] +pub fn evolve( + strategy: SvdStrategy, + current_basis: &Mat, + sigmas: &[f64], + z: &Mat, + residual: &Mat, + sqrt_lambda: f64, + k: usize, + cap: usize, +) -> Option { + let oracle_active = cfg!(debug_assertions) || tracing::enabled!(tracing::Level::DEBUG); + + if !oracle_active { + let result = run_svd(strategy, current_basis, sigmas, z, residual, sqrt_lambda, k, cap); + if result.is_some() { + return result; + } + // Fallback: the primary strategy could not handle these + // dimensions (e.g. Brand with b+k > d). Try the other. + let other_strategy = match strategy { + SvdStrategy::Naive => SvdStrategy::Brand, + SvdStrategy::Brand => SvdStrategy::Naive, + }; + return run_svd(other_strategy, current_basis, sigmas, z, residual, sqrt_lambda, k, cap); + } + + // Oracle: run both strategies and compare results. + // + // Active in debug builds unconditionally, or in release builds + // when a DEBUG-level tracing subscriber is attached. Uses the + // same span names (`svd_brand` / `svd_naive`) so the + // SpanTimingLayer captures both strategies' cost from a single + // run — no need for the benchmarks to loop over strategies. + // + // A thread-local bool alternates execution order so that + // neither strategy consistently benefits from warmed caches + // or branch predictors. + let other_strategy = match strategy { + SvdStrategy::Naive => SvdStrategy::Brand, + SvdStrategy::Brand => SvdStrategy::Naive, + }; + + let flip = ORACLE_FLIP.with(|f| { + let v = f.get(); + f.set(!v); + v + }); + + let (result, other) = if flip { + // Reversed: run other strategy first. + let o = run_svd(other_strategy, current_basis, sigmas, z, residual, sqrt_lambda, k, cap); + let r = run_svd(strategy, current_basis, sigmas, z, residual, sqrt_lambda, k, cap); + (r, o) + } else { + // Normal: run selected strategy first. + let r = run_svd(strategy, current_basis, sigmas, z, residual, sqrt_lambda, k, cap); + let o = run_svd(other_strategy, current_basis, sigmas, z, residual, sqrt_lambda, k, cap); + (r, o) + }; + + if let (Some(a), Some(b)) = (&result, &other) { + let (a_n, b_n) = (a.n, b.n); + assert_eq!(a_n, b_n, "maths oracle: n mismatch ({a_n} vs {b_n})"); + compare_subspace_updates(a, b, strategy, other_strategy); + } + + // Use primary if it succeeded; otherwise fall back to the other + // strategy (e.g. Brand cannot handle b+k > d, naïve can). + if result.is_some() { result } else { other } +} + +/// Compare singular values and spectral projectors of two models. +/// +/// # Panics +/// +/// Panics when the models disagree beyond the precision budget. The caller +/// enables this assertion only while the numerical oracle is active. +fn compare_subspace_updates( + update_a: &SubspaceUpdate, + update_b: &SubspaceUpdate, + strategy_a: SvdStrategy, + strategy_b: SvdStrategy, +) { + let rank = update_a.n; + assert_eq!(rank, update_b.n, "maths oracle: rank mismatch"); + let dims = update_a.basis.nrows(); + assert_eq!(dims, update_b.basis.nrows(), "maths oracle: dimension mismatch"); + if rank == 0 { + return; + } + + // Require half-significand agreement with one guard bit: sqrt(epsilon)/2 + // is 2^-27 for f64, tighter than the former 1e-8 relative sigma check. + // This is the oracle's accuracy requirement, not an error estimate fitted + // to an update trajectory. Both strategies receive the same prior state. + let relative = f64::EPSILON.sqrt() / 2.0; + let largest = update_a.sigmas[0].abs().max(update_b.sigmas[0].abs()); + // A d-term dot product has relative rounding bound gamma_d=d*eps/(1-d*eps) + // without overflow or significant underflow (zeros are exact). Covariance + // noise at that scale corresponds to + // singular values sqrt(gamma_d)*sigma_max; only two values below this + // common absolute floor may be treated as unresolved. + let dimension = f64::from(u32::try_from(dims).expect("tracker dimension fits u32")); + let gamma = dimension * f64::EPSILON / dimension.mul_add(-f64::EPSILON, 1.0); + let zero_floor = gamma.sqrt() * largest; + for idx in 0..rank { + let left = update_a.sigmas[idx]; + let right = update_b.sigmas[idx]; + assert!(left.is_finite() && right.is_finite() && left >= 0.0 && right >= 0.0); + let difference = (left - right).abs(); + let unresolved = left.max(right) <= zero_floor && difference <= zero_floor; + assert!( + unresolved || difference <= relative * left.max(right), + "maths oracle: sigma[{idx}] mismatch: {strategy_a:?}={left:.12e}, {strategy_b:?}={right:.12e}, zero_floor={zero_floor:.12e}" + ); + } + + // The allowed normwise perturbation is eta=relative*sigma_max. Weyl's + // singular-value intervals of radius eta overlap at gaps <=2*eta, so + // compare those columns as a block. This budget must not be inferred + // merely from matching sigmas: the projector check tests its consequence. + let perturbation = relative * largest; + let mut start = 0; + while start < rank { + if update_a.sigmas[start].max(update_b.sigmas[start]) <= zero_floor { + break; + } + let mut end = start + 1; + while end < rank + && (update_a.sigmas[end - 1] - update_a.sigmas[end]).max(update_b.sigmas[end - 1] - update_b.sigmas[end]) + <= 2.0 * perturbation + { + end += 1; + } + compare_projectors(update_a, update_b, start, end, perturbation, gamma); + start = end; + } +} + +fn compare_projectors(left: &SubspaceUpdate, right: &SubspaceUpdate, start: usize, end: usize, perturbation: f64, gamma: f64) { + let rank = end - start; + let block_size = f64::from(u32::try_from(rank).expect("tracker rank fits u32")); + let mut gap = left.sigmas[end - 1].min(right.sigmas[end - 1]); + if start > 0 { + gap = gap.min((left.sigmas[start - 1] - left.sigmas[start]).min(right.sigmas[start - 1] - right.sigmas[start])); + } + if end < left.n { + gap = gap.min((left.sigmas[end - 1] - left.sigmas[end]).min(right.sigmas[end - 1] - right.sigmas[end])); + } + + // For orthonormal bases, ||P-Q||_F^2=2*(r-||A^T B||_F^2). + // Each overlap dot has error <=gamma_d; squaring adds at most + // 2*gamma_d+gamma_d^2, and summing r^2 terms adds gamma_(r^2). + // Include the final subtraction and factor two in the squared floor. + let sum_terms = block_size * block_size; + let sum_gamma = sum_terms * f64::EPSILON / sum_terms.mul_add(-f64::EPSILON, 1.0); + let rounding = 2.0 + * f64::EPSILON.mul_add( + block_size, + sum_gamma.mul_add(block_size, sum_terms * gamma.mul_add(gamma, 2.0 * gamma)), + ); + let mut overlap = 0.0; + for a_col in start..end { + for b_col in start..end { + let mut dot = 0.0; + for row in 0..left.basis.nrows() { + dot = left.basis[(row, a_col)].mul_add(right.basis[(row, b_col)], dot); + } + overlap = dot.mul_add(dot, overlap); + } + } + let distance_sq = (2.0 * (block_size - overlap)).max(0.0); + // Wedin's combined left/right Frobenius sin-theta bound, with each + // residual <=sqrt(r)*eta, gives ||P-Q||_F <=2*sqrt(r)*eta/delta. + // Weyl reduces the available separation to delta=gap-eta. Include + // the omitted zero spectrum, so even a rank-one model checks its axis. + let separation = gap - perturbation; + assert!(separation > 0.0, "maths oracle: unresolved nonzero projector separation"); + let tolerance_sq = (4.0 * block_size).mul_add((perturbation / separation).powi(2), rounding); + assert!( + distance_sq <= tolerance_sq, + "maths oracle: projector [{start}..{end}] mismatch: distance_sq={distance_sq:.12e}, tolerance_sq={tolerance_sq:.12e}, gap={gap:.12e}" + ); +} + +// Resolve a repeated singular space before applying the rank cap. Project +// coordinate axes in order and re-orthogonalise them twice, so a truncated +// repeated block retains the same plane regardless of the SVD's basis choice. +// Singletons and unresolved zero-energy columns retain their original basis. +fn truncate_update(mut basis: Mat, sigmas: Vec, cap: usize) -> SubspaceUpdate { + let dims = basis.nrows(); + let dimension = f64::from(u32::try_from(dims).expect("tracker dimension fits u32")); + let gamma = dimension * f64::EPSILON / dimension.mul_add(-f64::EPSILON, 1.0); + let largest = sigmas.first().copied().unwrap_or(0.0); + // Use the same intervals as the oracle: eta=sqrt(epsilon)*sigma_max/2, + // so numerical ties have overlapping intervals at gaps <=2*eta. The + // complete block must choose its basis before truncation, including a + // neighbouring value just beyond the cap. This is a precision policy, + // not a measured error bound on the SVD backend. + let tie_gap = f64::EPSILON.sqrt() * largest; + let mut start = 0; + while start < sigmas.len() && sigmas[start] > gamma.sqrt() * largest { + let mut end = start + 1; + while end < sigmas.len() && sigmas[end - 1] - sigmas[end] <= tie_gap { + end += 1; + } + if end - start > 1 { + canonicalize_cluster(&mut basis, start, end); + } + start = end; + } + let n = cap.min(sigmas.len()); + SubspaceUpdate { + basis: basis.subcols(0, n).to_owned(), + sigmas: sigmas.into_iter().take(n).collect(), + n, + } +} + +fn canonicalize_cluster(basis: &mut Mat, start: usize, end: usize) { + let dims = basis.nrows(); + let width = end - start; + let mut canonical = Mat::::zeros(dims, width); + let mut chosen = 0; + // For unit input columns, candidate construction uses d*r FMAs; two + // Gram-Schmidt passes use at most 2*r*(d FMAs + 2*d scalar updates), + // and the squared norm uses d FMAs: at most q=d*(7*r+1) roundings. + // Fusing the vector updates below only reduces this conservative count. + // Under finite arithmetic without significant underflow, gamma_q is + // the squared-energy resolution we require of a usable pivot. Tiny + // cancellation residues below that floor do not choose an arbitrary axis. + let terms = f64::from(u32::try_from(dims * (7 * width + 1)).expect("tracker operation count fits u32")); + let pivot_floor_sq = terms * f64::EPSILON / terms.mul_add(-f64::EPSILON, 1.0); + for axis in 0..dims { + let mut candidate: Vec = (0..dims) + .map(|row| (start..end).fold(0.0, |sum, col| basis[(row, col)].mul_add(basis[(axis, col)], sum))) + .collect(); + for _ in 0..2 { + for col in 0..chosen { + let dot = candidate + .iter() + .enumerate() + .fold(0.0, |sum, (row, value)| canonical[(row, col)].mul_add(*value, sum)); + for (row, value) in candidate.iter_mut().enumerate() { + *value = dot.mul_add(-canonical[(row, col)], *value); + } + } + } + let norm_sq = candidate.iter().fold(0.0, |sum, value| value.mul_add(*value, sum)); + if norm_sq > pivot_floor_sq { + let norm = norm_sq.sqrt(); + for (row, value) in candidate.into_iter().enumerate() { + canonical[(row, chosen)] = value / norm; + } + chosen += 1; + if chosen == width { + break; + } + } + } + // A numerically unresolved basis is left intact, so this normalization + // cannot hide a disagreement from the oracle or turn it into a fallback. + if chosen == width { + basis.subcols_mut(start, width).copy_from(&canonical); + } +} diff --git a/packages/sentinel/src/maths/naive_svd.rs b/packages/sentinel/src/maths/naive_svd.rs new file mode 100644 index 000000000..c716b0a0b --- /dev/null +++ b/packages/sentinel/src/maths/naive_svd.rs @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +//! Naïve dense thin SVD for subspace evolution. +//! +//! This is the original algorithm: build the composite matrix +//! +//! M = \[√λ · `U_k` · diag(σ₁..ₖ) | X^T\] ∈ ℝ^(d × (k+b)) +//! +//! and compute a full dense thin SVD of M via `faer::Mat::thin_svd()`. +//! +//! Correct and simple, but O(d·(k+b)²) per call — the full +//! bidiagonalisation touches every element of the d-row matrix. +//! +//! # §-references +//! +//! - §ALGO S-4.2 Phase 2 — Subspace evolution +//! - ADR-S-016 §Context — Cost analysis + +use faer::Mat; + +use super::SubspaceUpdate; + +/// Evolve the subspace via dense thin SVD of the full composite matrix. +/// +/// Builds M = \[√λ · `U_k` · diag(σ) | X^T\] ∈ ℝ^(d × (k+b)) and +/// computes `thin_svd(M)`, retaining the top `n = min(k+b, d, cap)` +/// components. +/// +/// # Arguments +/// +/// * `current_basis` — `U_k`, shape `(d, cap)`. Only columns `[:k]` active. +/// * `sigmas` — singular values, length `cap`. Only `[:k]` meaningful. +/// * `z` — latent projection `X · U_k`, shape `(b, k)`. (Unused by +/// naïve — present for API uniformity; the naïve path reconstructs +/// X^T from `residual + U_k · Z^T`.) +/// * `residual` — reconstruction residual `X − X̂`, shape `(b, d)`. +/// * `sqrt_lambda` — `√λ`. +/// * `k` — current active rank. +/// * `cap` — hard ceiling on rank. +// Linear algebra code — single-char names follow standard mathematical notation +// (d=dimension, b=batch, k=rank, m=composite matrix, s=scaled sigma, n=output rank). +#[allow(clippy::many_single_char_names)] +#[must_use] +pub fn evolve( + current_basis: &Mat, + sigmas: &[f64], + z: &Mat, + residual: &Mat, + sqrt_lambda: f64, + k: usize, + cap: usize, +) -> Option { + let d = current_basis.nrows(); + let b = residual.nrows(); + let cols = k + b; + + // Build M: (d × (k + b)) + let mut m = Mat::zeros(d, cols); + + // Left block: √λ · U_k · diag(σ[:k]) + for j in 0..k { + let s = sqrt_lambda * sigmas[j]; + for i in 0..d { + m[(i, j)] = current_basis[(i, j)] * s; + } + } + + // Right block: X^T. + // X = residual + Z · U_k^T (reconstruct from Phase 1 outputs). + // X^T[j, i] = X[i, j] = residual[i, j] + Σₗ z[i, l] · U_k[j, l] + for i in 0..b { + for j in 0..d { + let mut val = residual[(i, j)]; + for l in 0..k { + val = z[(i, l)].mul_add(current_basis[(j, l)], val); + } + m[(j, k + i)] = val; + } + } + + // Thin SVD of M. + let svd = m.thin_svd().ok()?; + + // Resolve repeated singular spaces using all available columns before + // the rank cap selects a slice of one (§ALGO S-4.2). + let sigmas = svd.S().column_vector().iter().copied().collect(); + Some(super::truncate_update(svd.U().to_owned(), sigmas, cap)) +} diff --git a/packages/sentinel/src/maths/tests/README.md b/packages/sentinel/src/maths/tests/README.md new file mode 100644 index 000000000..16c312516 --- /dev/null +++ b/packages/sentinel/src/maths/tests/README.md @@ -0,0 +1,24 @@ +## Unit test matrix · `tab:sentinel:maths-tests-unit-test-matrix` + +**Table (Unit test matrix)** + +| Test | Area | Claim | +|------|------|-------| +| (`test:unit:equivalence-representative-configs`) | svd | Across a spread of ambient widths, ranks, batch sizes and forgetting factors, the incremental update lands on the same singular values and the same axes as decomposing the whole composite matrix would. The cheap path is not an approximation that happens to be close enough; it is the same answer reached by exploiting structure the reference path ignores, which is why it can simply replace it. | +| (`test:unit:equivalence-rank-one`) | svd | cites (`claim:svd:the-incremental-update-reaches-the-same-answer-as-the-reference-decomposition`) | +| (`test:unit:equivalence-minimal-dimensions`) | svd | The incremental path declines to answer when the kernel it would build is not comfortably smaller than the ambient width: it needs spare dimensions for the residual's orthogonalisation to be stable, and without them the extra factorisation buys nothing and costs accuracy. At the narrowest widths it therefore returns nothing while the reference path still produces a model, and once there is headroom it runs and agrees again. Declining is how the boundary is expressed — never a degraded answer offered as a good one. | +| (`test:unit:equivalence-single-sample`) | svd | cites (`claim:svd:the-incremental-update-reaches-the-same-answer-as-the-reference-decomposition`) | +| (`test:unit:equivalence-large-batch`) | svd | cites (`claim:svd:the-incremental-update-reaches-the-same-answer-as-the-reference-decomposition`) | +| (`test:unit:equivalence-saturated-rank`) | svd | cites (`claim:svd:the-incremental-update-reaches-the-same-answer-as-the-reference-decomposition`) | +| (`test:unit:equivalence-cap-larger-than-k`) | svd | How many axes a step produces is the smallest of three limits: what the old rank plus the batch could span, the ambient width, and the ceiling the cell is allowed. With capacity to spare the step fills it, widening the model beyond the rank it started from — and both paths widen it identically, to the same count and with columns that are still orthonormal. Spare capacity is real room to grow rather than padding. | +| (`test:unit:equivalence-identity-basis`) | svd | cites (`claim:svd:the-incremental-update-reaches-the-same-answer-as-the-reference-decomposition`) | +| (`test:unit:equivalence-near-zero-residual`) | svd | A batch that already lies inside the model's own axes leaves almost nothing unexplained, and the incremental path has to orthogonalise that negligible residual anyway — a factorisation of a matrix that is numerically close to rank-deficient. Directions recovered from vanishing energy are arbitrary, so the two paths part company in the last few digits, but they still land on the same model. Precision degrades where there is nothing left to measure; agreement does not. | +| (`test:unit:equivalence-zero-initial-sigmas`) | svd | A model that has learned nothing yet carries no energy on any axis, so the remembered half of the step contributes exactly zero and the outcome is determined entirely by the arriving batch. The step is well defined there rather than degenerate: a cell's first real shape comes from its first real data, and both paths derive that shape identically. | +| (`test:unit:equivalence-large-singular-values`) | svd | When a model carries enormous accumulated energy, an ordinary batch is a vanishing perturbation of it, and the difference between the two paths scales with that energy rather than staying absolute. Agreement is therefore stated relatively: the singular values match to a proportion of themselves, not to a fixed margin, so a long-lived cell whose values have grown large is no less trustworthy than a fresh one. | +| (`test:unit:equivalence-equal-singular-values`) | svd | When every axis carries the same energy, nothing distinguishes one axis from another within the space they span: any rotation of them is an equally correct answer. The two paths still agree exactly on how much energy there is and on which space it occupies, while individual columns are compared only loosely, because insisting they coincide would be demanding an answer the mathematics does not define. | +| (`test:unit:equivalence-no-forgetting`) | svd | Forgetting enters the step only as a scale factor on the remembered energy, so turning it off is the ordinary case with that factor at one, not a separate code path. A cell configured to weight all its history equally therefore evolves by the same arithmetic as one that discounts the past, and the two paths agree there as everywhere else. | +| (`test:unit:output-basis-is-orthonormal`) | svd | Every step returns axes that are unit length and mutually perpendicular, from either path. Everything downstream assumes it: a batch's coordinates are obtained by projecting onto these axes, and the unexplained part is the batch minus that projection, which is only a decomposition if the axes are orthonormal. The incremental path has to work for this — its back-transform inherits drift from the basis it started with, so it re-orthogonalises before returning rather than trusting the construction. | +| (`test:unit:singular-values-sorted-and-nonnegative`) | svd | Singular values come back non-negative and in descending order from both paths. Order is what makes position meaningful: rank adaptation walks the values accumulating energy until a threshold is met and takes that position as the rank, which is only a sensible rule if the strongest direction is first. The incremental path preserves the ordering through its back-transform and corrective step rather than inheriting it by luck. | +| (`test:unit:deterministic-reproduction`) | svd | Given the same inputs, either path returns the same axes and the same values bit for bit — not close, identical. Nothing in the step draws on randomness, iteration order or timing, so two sentinels fed the same traffic hold the same model, and a difference between them is always evidence about the traffic rather than about the machine. | +| (`test:unit:equivalence-multi-step`) | svd | Agreement between the two paths is a property of a whole streaming run, not of one step in isolation. Each path is fed the same sequence but carries its own state forward, so any difference compounds through every later step — and over a long run the difference stays within a margin that grows only in step with the number of steps taken. Divergence is linear rather than explosive, which is what makes a cell that has been running for hours as trustworthy as one that has just started. | +| (`test:unit:reconstruction-error-equivalence`) | svd | The two paths do not merely agree on coordinates they were given; they explain data they have never seen equally well. Held-out rows projected onto either model leave the same amount unaccounted for, which is the property the sentinel actually depends on — novelty is measured from exactly that leftover, so equal reconstruction means equal scores whichever path produced the axes. | \ No newline at end of file diff --git a/packages/sentinel/src/maths/tests/brand_vs_naive.rs b/packages/sentinel/src/maths/tests/brand_vs_naive.rs new file mode 100644 index 000000000..293c76ed9 --- /dev/null +++ b/packages/sentinel/src/maths/tests/brand_vs_naive.rs @@ -0,0 +1,1065 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +//! # Test index +//! +//! | Test | Area | Claim | +//! |------|------|-------| +//! | [`equivalence_representative_configs`] | svd | Across a spread of ambient widths, ranks, batch sizes and forgetting factors, the incremental update lands on the same singular values and the same axes as decomposing the whole composite matrix would. The cheap path is not an approximation that happens to be close enough; it is the same answer reached by exploiting structure the reference path ignores, which is why it can simply replace it. | +//! | [`equivalence_rank_one`] | svd | cites (´claim:svd:the-incremental-update-reaches-the-same-answer-as-the-reference-decomposition´) | +//! | [`equivalence_minimal_dimensions`] | svd | The incremental path declines to answer when the kernel it would build is not comfortably smaller than the ambient width: it needs spare dimensions for the residual's orthogonalisation to be stable, and without them the extra factorisation buys nothing and costs accuracy. At the narrowest widths it therefore returns nothing while the reference path still produces a model, and once there is headroom it runs and agrees again. Declining is how the boundary is expressed — never a degraded answer offered as a good one. | +//! | [`equivalence_single_sample`] | svd | cites (´claim:svd:the-incremental-update-reaches-the-same-answer-as-the-reference-decomposition´) | +//! | [`equivalence_large_batch`] | svd | cites (´claim:svd:the-incremental-update-reaches-the-same-answer-as-the-reference-decomposition´) | +//! | [`equivalence_saturated_rank`] | svd | cites (´claim:svd:the-incremental-update-reaches-the-same-answer-as-the-reference-decomposition´) | +//! | [`equivalence_cap_larger_than_k`] | svd | How many axes a step produces is the smallest of three limits: what the old rank plus the batch could span, the ambient width, and the ceiling the cell is allowed. With capacity to spare the step fills it, widening the model beyond the rank it started from — and both paths widen it identically, to the same count and with columns that are still orthonormal. Spare capacity is real room to grow rather than padding. | +//! | [`equivalence_identity_basis`] | svd | cites (´claim:svd:the-incremental-update-reaches-the-same-answer-as-the-reference-decomposition´) | +//! | [`equivalence_near_zero_residual`] | svd | A batch that already lies inside the model's own axes leaves almost nothing unexplained, and the incremental path has to orthogonalise that negligible residual anyway — a factorisation of a matrix that is numerically close to rank-deficient. Directions recovered from vanishing energy are arbitrary, so the two paths part company in the last few digits, but they still land on the same model. Precision degrades where there is nothing left to measure; agreement does not. | +//! | [`equivalence_zero_initial_sigmas`] | svd | A model that has learned nothing yet carries no energy on any axis, so the remembered half of the step contributes exactly zero and the outcome is determined entirely by the arriving batch. The step is well defined there rather than degenerate: a cell's first real shape comes from its first real data, and both paths derive that shape identically. | +//! | [`equivalence_large_singular_values`] | svd | When a model carries enormous accumulated energy, an ordinary batch is a vanishing perturbation of it, and the difference between the two paths scales with that energy rather than staying absolute. Agreement is therefore stated relatively: the singular values match to a proportion of themselves, not to a fixed margin, so a long-lived cell whose values have grown large is no less trustworthy than a fresh one. | +//! | [`equivalence_equal_singular_values`] | svd | When every axis carries the same energy, nothing distinguishes one axis from another within the space they span: any rotation of them is an equally correct answer. The two paths still agree exactly on how much energy there is and on which space it occupies, while individual columns are compared only loosely, because insisting they coincide would be demanding an answer the mathematics does not define. | +//! | [`equivalence_no_forgetting`] | svd | Forgetting enters the step only as a scale factor on the remembered energy, so turning it off is the ordinary case with that factor at one, not a separate code path. A cell configured to weight all its history equally therefore evolves by the same arithmetic as one that discounts the past, and the two paths agree there as everywhere else. | +//! | [`output_basis_is_orthonormal`] | svd | Every step returns axes that are unit length and mutually perpendicular, from either path. Everything downstream assumes it: a batch's coordinates are obtained by projecting onto these axes, and the unexplained part is the batch minus that projection, which is only a decomposition if the axes are orthonormal. The incremental path has to work for this — its back-transform inherits drift from the basis it started with, so it re-orthogonalises before returning rather than trusting the construction. | +//! | [`singular_values_sorted_and_nonnegative`] | svd | Singular values come back non-negative and in descending order from both paths. Order is what makes position meaningful: rank adaptation walks the values accumulating energy until a threshold is met and takes that position as the rank, which is only a sensible rule if the strongest direction is first. The incremental path preserves the ordering through its back-transform and corrective step rather than inheriting it by luck. | +//! | [`deterministic_reproduction`] | svd | Given the same inputs, either path returns the same axes and the same values bit for bit — not close, identical. Nothing in the step draws on randomness, iteration order or timing, so two sentinels fed the same traffic hold the same model, and a difference between them is always evidence about the traffic rather than about the machine. | +//! | [`equivalence_multi_step`] | svd | Agreement between the two paths is a property of a whole streaming run, not of one step in isolation. Each path is fed the same sequence but carries its own state forward, so any difference compounds through every later step — and over a long run the difference stays within a margin that grows only in step with the number of steps taken. Divergence is linear rather than explosive, which is what makes a cell that has been running for hours as trustworthy as one that has just started. | +//! | [`reconstruction_error_equivalence`] | svd | The two paths do not merely agree on coordinates they were given; they explain data they have never seen equally well. Held-out rows projected onto either model leave the same amount unaccounted for, which is the property the sentinel actually depends on — novelty is measured from exactly that leftover, so equal reconstruction means equal scores whichever path produced the axes. | +//! | [`oracle_rejects_orthogonal_clustered_planes`] | svd | Equal spectra do not make orthogonal planes the same model, including an exactly repeated spectrum. | +//! | [`oracle_rejects_unit_singular_value_against_zero`] | svd | A zero component cannot suppress comparison with resolved unit energy. | +//! | [`oracle_accepts_a_rotated_basis_of_the_same_cluster`] | svd | A basis rotation within a repeated-singular-value plane preserves the model. | +//! | [`truncated_repeated_space_is_shared_between_strategies`] | svd | When the rank cap cuts through a repeated singular space, both strategies retain the same model. | + +//! Tests for the two subspace-evolution algorithms — the reference dense +//! decomposition and the incremental one that runs in production. +//! +//! Both answer the same question. Given a cell's current axes and singular +//! values, a batch's coordinates within those axes, and the part of the batch +//! those axes failed to explain, what should the axes and values become? The +//! reference path assembles the whole composite matrix and decomposes it, +//! which is simple and costs a pass over every ambient dimension. The +//! incremental path orthogonalises only the unexplained part, decomposes a +//! small kernel whose size is the rank plus the batch rather than the ambient +//! width, and transforms the result back — then re-orthogonalises, because the +//! basis it starts from is itself the output of an earlier back-transform and +//! drifts if left uncorrected. +//! +//! The cheap path is therefore licensed only by agreement, and the engine +//! leans on that directly: whenever the oracle is active it runs both and +//! compares them, so a divergence is a panic rather than a silently different +//! model. These tests are where the agreement is established — across +//! dimensions, ranks, batch shapes, spectra and starting states, and over a +//! long run of steps where floating-point difference has time to accumulate. +//! +//! Where the two cannot be made to agree they decline to differ instead. The +//! incremental path returns nothing when the ambient width leaves no headroom +//! for a stable orthogonalisation of the residual, and the caller falls back +//! to the reference path rather than accepting a worse answer. + +use faer::Mat; +use rand::rngs::SmallRng; +use rand::{RngExt, SeedableRng}; + +use crate::maths::{SubspaceUpdate, brand_svd, naive_svd}; + +// ════════════════════════════════════════════════════════════ +// Helpers +// ════════════════════════════════════════════════════════════ + +/// Generate a random (rows × cols) matrix with values in [-1, 1]. +fn random_matrix(rows: usize, cols: usize, rng: &mut SmallRng) -> Mat { + let mut m = Mat::zeros(rows, cols); + for i in 0..rows { + for j in 0..cols { + m[(i, j)] = rng.random_range(-1.0..1.0); + } + } + m +} + +/// Build a random identity-like basis (d × cap) with orthonormal columns. +fn random_basis(d: usize, cap: usize, rng: &mut SmallRng) -> Mat { + // Start with random matrix, then QR to get orthonormal columns. + let raw = random_matrix(d, cap, rng); + let qr = raw.as_ref().qr(); + qr.compute_thin_Q() +} + +/// Given X (b×d) and `U_k` (d×cap, using [:k]), compute z and residual. +fn phase1_projection(x: &Mat, basis: &Mat, k: usize) -> (Mat, Mat) { + let u_k = basis.subcols(0, k); + let z = x * u_k; // (b × k) + let x_hat = &z * u_k.transpose(); // (b × d) + let residual = x - &x_hat; // (b × d) + (z, residual) +} + +/// Assert two `SubspaceUpdate`s are approximately equal. +/// +/// - Singular values: relative tolerance `sigma_tol`. +/// - Basis columns: compared up to sign via |cos(angle)| > `basis_tol`. +fn assert_updates_close(a: &SubspaceUpdate, b: &SubspaceUpdate, sigma_tol: f64, basis_tol: f64, label: &str) { + assert_eq!(a.n, b.n, "{label}: n mismatch ({} vs {})", a.n, b.n); + let n = a.n; + let d = a.basis.nrows(); + + // Absolute floor below which singular values are considered zero. + // Near-zero values carry arbitrary numerical residual, making + // relative comparison meaningless. + let sigma_abs_floor = 1e-6; + + // Singular values. + for i in 0..n { + let sa = a.sigmas[i]; + let sb = b.sigmas[i]; + if sa.abs() < sigma_abs_floor && sb.abs() < sigma_abs_floor { + continue; + } + let denom = sa.abs().max(sb.abs()).max(1e-15); + let rel = (sa - sb).abs() / denom; + assert!( + rel < sigma_tol, + "{label}: σ[{i}] mismatch: naïve={sa:.12e}, brand={sb:.12e}, rel={rel:.2e}" + ); + } + + // Basis columns (up to sign). + // Skip columns whose singular value is near-zero (arbitrary direction). + for j in 0..n { + if a.sigmas[j].abs() < sigma_abs_floor && b.sigmas[j].abs() < sigma_abs_floor { + continue; + } + let mut dot = 0.0; + for i in 0..d { + dot = a.basis[(i, j)].mul_add(b.basis[(i, j)], dot); + } + let cosine = dot.abs(); + assert!(cosine > basis_tol, "{label}: basis col {j} diverged: |cos| = {cosine:.8e}"); + } +} + +/// Assert that the columns of a matrix are approximately orthonormal. +fn assert_orthonormal(basis: &Mat, n: usize, tol: f64, label: &str) { + let d = basis.nrows(); + for j in 0..n { + // Column norm ≈ 1. + let mut norm_sq = 0.0; + for i in 0..d { + norm_sq = basis[(i, j)].mul_add(basis[(i, j)], norm_sq); + } + let norm = norm_sq.sqrt(); + assert!( + (norm - 1.0).abs() < tol, + "{label}: column {j} norm = {norm:.8e}, expected ≈ 1.0" + ); + + // Pairwise orthogonality. + for l in (j + 1)..n { + let mut dot = 0.0; + for i in 0..d { + dot = basis[(i, j)].mul_add(basis[(i, l)], dot); + } + assert!( + dot.abs() < tol, + "{label}: columns {j} and {l} not orthogonal: dot = {dot:.8e}" + ); + } + } +} + +/// Run both algorithms and return (naïve, brand) results. +/// +/// Brand returns `None` when `k + b >= d` (the kernel is not smaller +/// than the ambient dimension — see guard in `brand_svd::evolve`). +/// In that case the second element is `None`. +// Linear algebra test helpers — d, k, b, z, x follow standard mathematical +// notation for dimensionality, rank, batch size, latent projections, and input. +fn run_both( + basis: &Mat, + sigmas: &[f64], + z: &Mat, + residual: &Mat, + sqrt_lambda: f64, + k: usize, + cap: usize, +) -> (SubspaceUpdate, Option) { + let naive = naive_svd::evolve(basis, sigmas, z, residual, sqrt_lambda, k, cap).expect("naïve SVD should not fail"); + let brand = brand_svd::evolve(basis, sigmas, z, residual, sqrt_lambda, k, cap); + (naive, brand) +} + +/// Pad a (d × n) basis to (d × cap) by appending zero columns. +fn pad_to_cap(basis: &Mat, d: usize, cap: usize) -> Mat { + let n = basis.ncols(); + let mut out = Mat::zeros(d, cap); + for j in 0..n.min(cap) { + for i in 0..d { + out[(i, j)] = basis[(i, j)]; + } + } + out +} + +/// Pad sigmas to length `cap` with zeros. +fn pad_sigmas(sigmas: &[f64], cap: usize) -> Vec { + let mut out = sigmas.to_vec(); + out.resize(cap, 0.0); + out +} + +/// Frobenius norm of reconstruction error: ‖X − U Uᵀ Xᵀ‖_F (row-major). +// Linear algebra helper — d, b, x, z, n follow standard mathematical notation. +#[allow(clippy::many_single_char_names)] +fn recon_error(x: &Mat, basis: &Mat, n: usize) -> f64 { + let b = x.nrows(); + let d = x.ncols(); + let u_n = basis.subcols(0, n); + let z = x * u_n; // (b × n) + let x_hat = &z * u_n.transpose(); // (b × d) + + let mut err = 0.0; + for i in 0..b { + for j in 0..d { + let diff = x[(i, j)] - x_hat[(i, j)]; + err = diff.mul_add(diff, err); + } + } + err.sqrt() +} + +// ════════════════════════════════════════════════════════════ +// § 1 — Equivalence: basic configurations +// ════════════════════════════════════════════════════════════ + +/// Across a spread of ambient widths, ranks, batch sizes and forgetting +/// factors, the incremental update lands on the same singular values and the +/// same axes as decomposing the whole composite matrix would. The cheap path +/// is not an approximation that happens to be close enough; it is the same +/// answer reached by exploiting structure the reference path ignores, which is +/// why it can simply replace it. +/// +/// ´claim:svd:the-incremental-update-reaches-the-same-answer-as-the-reference-decomposition´ +/// ´test:unit:equivalence-representative-configs´ +#[test] +// Linear algebra test — d, k, b, z, x follow standard mathematical notation. +fn equivalence_representative_configs() { + let configs: &[(usize, usize, usize, f64)] = &[ + // (d, k, b, λ) + (32, 2, 4, 0.95), // small, test-like + (64, 4, 8, 0.99), // medium + (128, 2, 4, 0.95), // benchmark config + (128, 2, 16, 0.99), // realistic config + (128, 16, 16, 0.99), // high rank + large batch + (256, 8, 32, 0.99), // wide dimension + ]; + + for &(d, k, b, lambda) in configs { + let cap = k; // cap = k for simplicity + let sqrt_lambda = lambda.sqrt(); + let mut rng = SmallRng::seed_from_u64(42); + + let basis = random_basis(d, cap, &mut rng); + let mut sigmas = vec![0.0; cap]; + for s in &mut sigmas { + *s = rng.random_range(0.1..10.0); + } + // Sort decreasing (as they would be in practice). + sigmas.sort_by(|a, b| b.partial_cmp(a).unwrap()); + + let x = random_matrix(b, d, &mut rng); + let (z, residual) = phase1_projection(&x, &basis, k); + + let (naive, brand) = run_both(&basis, &sigmas, &z, &residual, sqrt_lambda, k, cap); + let brand = brand.expect("Brand should succeed when c < d"); + + let label = format!("d={d}, k={k}, b={b}, λ={lambda}"); + assert_updates_close(&naive, &brand, 1e-8, 1.0 - 1e-6, &label); + } +} + +// ════════════════════════════════════════════════════════════ +// § 2 — Equivalence: dimensional edge cases +// ════════════════════════════════════════════════════════════ + +/// A model carrying a single axis is the most constrained shape the update can +/// take: there is no spectrum to order and no neighbouring direction for the +/// new energy to be confused with. Agreement holds there too, so the two paths +/// do not depend on a rich spectrum to coincide. +/// +/// (´claim:svd:the-incremental-update-reaches-the-same-answer-as-the-reference-decomposition´) +/// ´test:unit:equivalence-rank-one´ +#[test] +// Linear algebra test — d, k, b, z, x follow standard mathematical notation. +#[allow(clippy::many_single_char_names)] +fn equivalence_rank_one() { + let d = 64; + let k = 1; + let b = 4; + let cap = 1; + let sqrt_lambda = 0.95_f64.sqrt(); + let mut rng = SmallRng::seed_from_u64(123); + + let basis = random_basis(d, cap, &mut rng); + let sigmas = vec![5.0]; + + let x = random_matrix(b, d, &mut rng); + let (z, residual) = phase1_projection(&x, &basis, k); + + let (naive, brand) = run_both(&basis, &sigmas, &z, &residual, sqrt_lambda, k, cap); + let brand = brand.expect("Brand should succeed when c < d"); + assert_updates_close(&naive, &brand, 1e-8, 1.0 - 1e-6, "rank-1"); +} + +/// The incremental path declines to answer when the kernel it would build is +/// not comfortably smaller than the ambient width: it needs spare dimensions +/// for the residual's orthogonalisation to be stable, and without them the +/// extra factorisation buys nothing and costs accuracy. At the narrowest +/// widths it therefore returns nothing while the reference path still produces +/// a model, and once there is headroom it runs and agrees again. Declining is +/// how the boundary is expressed — never a degraded answer offered as a good +/// one. +/// +/// ´claim:svd:the-incremental-path-declines-when-the-ambient-width-leaves-no-headroom-so-the-reference-path-answers-instead´ +/// ´test:unit:equivalence-minimal-dimensions´ +#[test] +// Linear algebra test — d, k, b, z, x follow standard mathematical notation. +#[allow(clippy::many_single_char_names)] +fn equivalence_minimal_dimensions() { + // d=2, k=1, b=1 ⇒ c = k + b = 2, c + 2 = 4 > d = 2. + // Brand correctly returns None here (headroom guard), + // so we verify the fallback: only Naive produces a result. + let d = 2; + let k = 1; + let b = 1; + let cap = 1; + let sqrt_lambda = 0.95_f64.sqrt(); + let mut rng = SmallRng::seed_from_u64(101); + + let basis = random_basis(d, cap, &mut rng); + let sigmas = vec![1.0]; + + let x = random_matrix(b, d, &mut rng); + let (z, residual) = phase1_projection(&x, &basis, k); + + let (naive, brand) = run_both(&basis, &sigmas, &z, &residual, sqrt_lambda, k, cap); + assert!(brand.is_none(), "Brand should return None when c + 2 > d"); + assert_eq!(naive.n, 1); + + // d=3, c=2: c + 2 = 4 > 3 → still bails. + let d = 3; + let basis = random_basis(d, cap, &mut rng); + let x = random_matrix(b, d, &mut rng); + let (z, residual) = phase1_projection(&x, &basis, k); + + let (_naive, brand) = run_both(&basis, &sigmas, &z, &residual, sqrt_lambda, k, cap); + assert!(brand.is_none(), "Brand should return None when c + 2 > d (d=3)"); + + // d=8, c=2: c + 2 = 4 ≤ 8 → Brand runs (smallest OK case). + let d = 8; + let basis = random_basis(d, cap, &mut rng); + let x = random_matrix(b, d, &mut rng); + let (z, residual) = phase1_projection(&x, &basis, k); + + let (naive, brand) = run_both(&basis, &sigmas, &z, &residual, sqrt_lambda, k, cap); + let brand = brand.expect("Brand should succeed when c + 2 <= d"); + assert_updates_close(&naive, &brand, 1e-8, 1.0 - 1e-6, "minimal-dims-d8"); +} + +/// The other minimal shape: a batch of one row against a model of several +/// axes. Here the kernel's new block is a single column and the residual's +/// factorisation is one-dimensional, which is the degenerate end of the +/// incremental construction rather than of the model — and agreement survives +/// it. +/// +/// (´claim:svd:the-incremental-update-reaches-the-same-answer-as-the-reference-decomposition´) +/// ´test:unit:equivalence-single-sample´ +#[test] +// Linear algebra test — d, k, b, z, x follow standard mathematical notation. +#[allow(clippy::many_single_char_names)] +fn equivalence_single_sample() { + let d = 64; + let k = 4; + let b = 1; + let cap = 4; + let sqrt_lambda = 0.95_f64.sqrt(); + let mut rng = SmallRng::seed_from_u64(202); + + let basis = random_basis(d, cap, &mut rng); + let sigmas = vec![8.0, 4.0, 2.0, 1.0]; + + let x = random_matrix(b, d, &mut rng); + let (z, residual) = phase1_projection(&x, &basis, k); + + let (naive, brand) = run_both(&basis, &sigmas, &z, &residual, sqrt_lambda, k, cap); + let brand = brand.expect("Brand should succeed when c < d"); + assert_updates_close(&naive, &brand, 1e-8, 1.0 - 1e-6, "single-sample"); +} + +/// The opposite imbalance: a batch far wider than the model's rank, so most of +/// the kernel is the freshly orthogonalised residual and only a sliver of it +/// is remembered energy. This is the regime where the incremental path does +/// the most work outside the old basis, and it still coincides with the +/// reference. +/// +/// (´claim:svd:the-incremental-update-reaches-the-same-answer-as-the-reference-decomposition´) +/// ´test:unit:equivalence-large-batch´ +#[test] +// Linear algebra test — d, k, b, z, x follow standard mathematical notation. +#[allow(clippy::many_single_char_names)] +fn equivalence_large_batch() { + let d = 64; + let k = 2; + let b = 32; + let cap = 2; + let sqrt_lambda = 0.99_f64.sqrt(); + let mut rng = SmallRng::seed_from_u64(456); + + let basis = random_basis(d, cap, &mut rng); + let sigmas = vec![10.0, 3.0]; + + let x = random_matrix(b, d, &mut rng); + let (z, residual) = phase1_projection(&x, &basis, k); + + let (naive, brand) = run_both(&basis, &sigmas, &z, &residual, sqrt_lambda, k, cap); + let brand = brand.expect("Brand should succeed when c < d"); + assert_updates_close(&naive, &brand, 1e-8, 1.0 - 1e-6, "large-batch"); +} + +/// A model already at its rank ceiling has no spare capacity: everything the +/// batch contributes must be resolved within the axes it is allowed to keep, +/// and the trailing singular values are the ones least separated from their +/// neighbours. Agreement holds, at a tolerance loosened to match how much +/// closer those neighbours sit — the weakest directions are where a difference +/// would first show. +/// +/// (´claim:svd:the-incremental-update-reaches-the-same-answer-as-the-reference-decomposition´) +/// ´test:unit:equivalence-saturated-rank´ +#[test] +// Linear algebra test — d, k, b, z, x follow standard mathematical notation. +// usize→f64 cast is for the decreasing-sigma formula: 20.0/(i+1.0) where i ≤ 16. +#[allow(clippy::many_single_char_names, clippy::cast_precision_loss)] +fn equivalence_saturated_rank() { + let d = 32; + let k = 16; + let b = 8; + let cap = 16; + let sqrt_lambda = 0.99_f64.sqrt(); + let mut rng = SmallRng::seed_from_u64(789); + + let basis = random_basis(d, cap, &mut rng); + let mut sigmas = vec![0.0; cap]; + for (i, s) in sigmas.iter_mut().enumerate() { + *s = 20.0 / (i as f64 + 1.0); // decreasing: 20, 10, 6.67, .. + } + + let x = random_matrix(b, d, &mut rng); + let (z, residual) = phase1_projection(&x, &basis, k); + + let (naive, brand) = run_both(&basis, &sigmas, &z, &residual, sqrt_lambda, k, cap); + let brand = brand.expect("Brand should succeed when c < d"); + assert_updates_close(&naive, &brand, 1e-7, 1.0 - 1e-5, "saturated-rank"); +} + +/// How many axes a step produces is the smallest of three limits: what the old +/// rank plus the batch could span, the ambient width, and the ceiling the cell +/// is allowed. With capacity to spare the step fills it, widening the model +/// beyond the rank it started from — and both paths widen it identically, to +/// the same count and with columns that are still orthonormal. Spare capacity +/// is real room to grow rather than padding. +/// +/// ´claim:svd:the-width-of-a-step-is-the-least-of-what-the-batch-can-span-the-ambient-width-and-the-ceiling´ +/// ´test:unit:equivalence-cap-larger-than-k´ +#[test] +// Linear algebra test — d, k, b, z, x follow standard mathematical notation. +#[allow(clippy::many_single_char_names)] +fn equivalence_cap_larger_than_k() { + let d = 64; + let k = 2; + let b = 4; + let cap = 8; // Much larger than k + let sqrt_lambda = 0.95_f64.sqrt(); + let mut rng = SmallRng::seed_from_u64(555); + + let mut basis = Mat::zeros(d, cap); + // Only initialise the first k columns as proper orthonormal vectors. + let partial = random_basis(d, k, &mut rng); + for j in 0..k { + for i in 0..d { + basis[(i, j)] = partial[(i, j)]; + } + } + let mut sigmas = vec![0.0; cap]; + sigmas[0] = 5.0; + sigmas[1] = 2.0; + + let x = random_matrix(b, d, &mut rng); + let (z, residual) = phase1_projection(&x, &basis, k); + + let (naive, brand) = run_both(&basis, &sigmas, &z, &residual, sqrt_lambda, k, cap); + let brand = brand.expect("Brand should succeed when c < d"); + + // n should be min(k+b, d, cap) = min(6, 64, 8) = 6. + assert_eq!(naive.n, 6); + assert_eq!(brand.n, 6); + assert_updates_close(&naive, &brand, 1e-8, 1.0 - 1e-6, "cap>k"); + assert_orthonormal(&naive.basis, naive.n, 1e-10, "naïve cap>k"); + assert_orthonormal(&brand.basis, brand.n, 1e-10, "brand cap>k"); +} + +// ════════════════════════════════════════════════════════════ +// § 3 — Equivalence: data / state edge cases +// ════════════════════════════════════════════════════════════ + +/// The state a cell actually begins in is not a random orthonormal basis but +/// an axis-aligned one, each column a single coordinate direction, with tiny +/// singular values standing in for knowledge not yet acquired. That start is +/// perfectly orthonormal even though nothing rotated it there, and both paths +/// evolve it alike — so the very first steps a live cell takes are covered by +/// the same agreement as its later ones. +/// +/// (´claim:svd:the-incremental-update-reaches-the-same-answer-as-the-reference-decomposition´) +/// ´test:unit:equivalence-identity-basis´ +#[test] +// Linear algebra test — d, k, b, z, x follow standard mathematical notation. +#[allow(clippy::many_single_char_names)] +fn equivalence_identity_basis() { + let d = 64; + let k = 2; + let b = 8; + let cap = 4; + let sqrt_lambda = 0.95_f64.sqrt(); + let mut rng = SmallRng::seed_from_u64(314); + + // Identity-like basis (same as SubspaceTracker::new). + let mut basis = Mat::zeros(d, cap); + for j in 0..cap.min(d) { + basis[(j, j)] = 1.0; + } + let sigmas = vec![0.01; cap]; + + let x = random_matrix(b, d, &mut rng); + let (z, residual) = phase1_projection(&x, &basis, k); + + let (naive, brand) = run_both(&basis, &sigmas, &z, &residual, sqrt_lambda, k, cap); + let brand = brand.expect("Brand should succeed when c < d"); + assert_updates_close(&naive, &brand, 1e-8, 1.0 - 1e-6, "identity-basis"); +} + +/// A batch that already lies inside the model's own axes leaves almost nothing +/// unexplained, and the incremental path has to orthogonalise that negligible +/// residual anyway — a factorisation of a matrix that is numerically close to +/// rank-deficient. Directions recovered from vanishing energy are arbitrary, +/// so the two paths part company in the last few digits, but they still land +/// on the same model. Precision degrades where there is nothing left to +/// measure; agreement does not. +/// +/// ´claim:svd:a-batch-with-almost-nothing-left-unexplained-costs-precision-without-costing-agreement´ +/// ´test:unit:equivalence-near-zero-residual´ +#[test] +// Linear algebra test — d, k, b, z, x follow standard mathematical notation. +#[allow(clippy::many_single_char_names)] +fn equivalence_near_zero_residual() { + let d = 64; + let k = 4; + let b = 4; + let cap = 4; + let sqrt_lambda = 0.99_f64.sqrt(); + let mut rng = SmallRng::seed_from_u64(999); + + let basis = random_basis(d, cap, &mut rng); + let sigmas = vec![10.0, 5.0, 2.0, 1.0]; + + // Build X as a linear combination of basis vectors + tiny noise. + let u_k = basis.subcols(0, k); + let coeffs = random_matrix(b, k, &mut rng); + let noise = { + let mut n = random_matrix(b, d, &mut rng); + for i in 0..b { + for j in 0..d { + n[(i, j)] *= 1e-10; // tiny noise + } + } + n + }; + let x = &coeffs * u_k.transpose() + &noise; // (b × d), mostly in subspace + + let (z, residual) = phase1_projection(&x, &basis, k); + + let (naive, brand) = run_both(&basis, &sigmas, &z, &residual, sqrt_lambda, k, cap); + let brand = brand.expect("Brand should succeed when c < d"); + + // Looser tolerance due to near-singular QR. + assert_updates_close(&naive, &brand, 1e-4, 1.0 - 1e-3, "near-zero-residual"); +} + +/// A model that has learned nothing yet carries no energy on any axis, so the +/// remembered half of the step contributes exactly zero and the outcome is +/// determined entirely by the arriving batch. The step is well defined there +/// rather than degenerate: a cell's first real shape comes from its first real +/// data, and both paths derive that shape identically. +/// +/// ´claim:svd:a-model-carrying-no-energy-yet-takes-its-whole-shape-from-the-arriving-batch´ +/// ´test:unit:equivalence-zero-initial-sigmas´ +#[test] +// Linear algebra test — d, k, b, z, x follow standard mathematical notation. +#[allow(clippy::many_single_char_names)] +fn equivalence_zero_initial_sigmas() { + let d = 64; + let k = 4; + let b = 8; + let cap = 4; + let sqrt_lambda = 0.99_f64.sqrt(); + let mut rng = SmallRng::seed_from_u64(303); + + let basis = random_basis(d, cap, &mut rng); + let sigmas = vec![0.0; cap]; // Cold start — no prior information. + + let x = random_matrix(b, d, &mut rng); + let (z, residual) = phase1_projection(&x, &basis, k); + + let (naive, brand) = run_both(&basis, &sigmas, &z, &residual, sqrt_lambda, k, cap); + let brand = brand.expect("Brand should succeed when c < d"); + assert_updates_close(&naive, &brand, 1e-8, 1.0 - 1e-6, "zero-initial-sigmas"); +} + +/// When a model carries enormous accumulated energy, an ordinary batch is a +/// vanishing perturbation of it, and the difference between the two paths +/// scales with that energy rather than staying absolute. Agreement is +/// therefore stated relatively: the singular values match to a proportion of +/// themselves, not to a fixed margin, so a long-lived cell whose values have +/// grown large is no less trustworthy than a fresh one. +/// +/// ´claim:svd:agreement-is-relative-so-a-model-carrying-large-energy-is-held-to-a-proportional-margin´ +/// ´test:unit:equivalence-large-singular-values´ +#[test] +// Linear algebra test — d, k, b, z, x follow standard mathematical notation. +#[allow(clippy::many_single_char_names)] +fn equivalence_large_singular_values() { + let d = 64; + let k = 4; + let b = 8; + let cap = 4; + let sqrt_lambda = 0.99_f64.sqrt(); + let mut rng = SmallRng::seed_from_u64(666); + + let basis = random_basis(d, cap, &mut rng); + let sigmas = vec![1e6, 5e5, 1e5, 1e4]; + + let x = random_matrix(b, d, &mut rng); + let (z, residual) = phase1_projection(&x, &basis, k); + + let (naive, brand) = run_both(&basis, &sigmas, &z, &residual, sqrt_lambda, k, cap); + let brand = brand.expect("Brand should succeed when c < d"); + assert_updates_close(&naive, &brand, 1e-6, 1.0 - 1e-4, "large-sigmas"); +} + +/// When every axis carries the same energy, nothing distinguishes one axis +/// from another within the space they span: any rotation of them is an equally +/// correct answer. The two paths still agree exactly on how much energy there +/// is and on which space it occupies, while individual columns are compared +/// only loosely, because insisting they coincide would be demanding an answer +/// the mathematics does not define. +/// +/// ´claim:svd:an-equal-valued-spectrum-fixes-the-space-but-not-the-axes-chosen-within-it´ +/// ´test:unit:equivalence-equal-singular-values´ +#[test] +// Linear algebra test — d, k, b, z, x follow standard mathematical notation. +#[allow(clippy::many_single_char_names)] +fn equivalence_equal_singular_values() { + let d = 64; + let k = 4; + let b = 8; + let cap = 4; + let sqrt_lambda = 0.99_f64.sqrt(); + let mut rng = SmallRng::seed_from_u64(404); + + let basis = random_basis(d, cap, &mut rng); + let sigmas = vec![5.0; cap]; // Degenerate spectrum — all σ identical. + + let x = random_matrix(b, d, &mut rng); + let (z, residual) = phase1_projection(&x, &basis, k); + + let (naive, brand) = run_both(&basis, &sigmas, &z, &residual, sqrt_lambda, k, cap); + let brand = brand.expect("Brand should succeed when c < d"); + // Looser basis tolerance: degenerate spectrum makes basis column + // orientation ambiguous within the equal-σ subspace. + assert_updates_close(&naive, &brand, 1e-8, 1.0 - 1e-4, "equal-sigmas"); +} + +/// Forgetting enters the step only as a scale factor on the remembered +/// energy, so turning it off is the ordinary case with that factor at one, not +/// a separate code path. A cell configured to weight all its history equally +/// therefore evolves by the same arithmetic as one that discounts the past, +/// and the two paths agree there as everywhere else. +/// +/// ´claim:svd:forgetting-is-only-a-scale-factor-so-switching-it-off-is-the-ordinary-step-with-that-factor-at-one´ +/// ´test:unit:equivalence-no-forgetting´ +#[test] +// Linear algebra test — d, k, b, z, x follow standard mathematical notation. +#[allow(clippy::many_single_char_names)] +fn equivalence_no_forgetting() { + let d = 64; + let k = 4; + let b = 8; + let cap = 4; + let sqrt_lambda = 1.0; // λ = 1.0 → no exponential forgetting. + let mut rng = SmallRng::seed_from_u64(505); + + let basis = random_basis(d, cap, &mut rng); + let sigmas = vec![8.0, 4.0, 2.0, 1.0]; + + let x = random_matrix(b, d, &mut rng); + let (z, residual) = phase1_projection(&x, &basis, k); + + let (naive, brand) = run_both(&basis, &sigmas, &z, &residual, sqrt_lambda, k, cap); + let brand = brand.expect("Brand should succeed when c < d"); + assert_updates_close(&naive, &brand, 1e-8, 1.0 - 1e-6, "no-forgetting"); +} + +// ════════════════════════════════════════════════════════════ +// § 4 — Properties (invariants) +// ════════════════════════════════════════════════════════════ + +/// Every step returns axes that are unit length and mutually perpendicular, +/// from either path. Everything downstream assumes it: a batch's coordinates +/// are obtained by projecting onto these axes, and the unexplained part is the +/// batch minus that projection, which is only a decomposition if the axes are +/// orthonormal. The incremental path has to work for this — its back-transform +/// inherits drift from the basis it started with, so it re-orthogonalises +/// before returning rather than trusting the construction. +/// +/// ´claim:svd:every-step-returns-orthonormal-axes-so-a-projection-remains-a-decomposition´ +/// ´test:unit:output-basis-is-orthonormal´ +#[test] +// Linear algebra test — d, k, b, z, x follow standard mathematical notation. +#[allow(clippy::many_single_char_names)] +fn output_basis_is_orthonormal() { + let d = 128; + let k = 4; + let b = 16; + let cap = 4; + let sqrt_lambda = 0.99_f64.sqrt(); + let mut rng = SmallRng::seed_from_u64(2024); + + let basis = random_basis(d, cap, &mut rng); + let sigmas = vec![8.0, 4.0, 2.0, 1.0]; + + let x = random_matrix(b, d, &mut rng); + let (z, residual) = phase1_projection(&x, &basis, k); + + let naive = naive_svd::evolve(&basis, &sigmas, &z, &residual, sqrt_lambda, k, cap).expect("naïve should not fail"); + let brand = brand_svd::evolve(&basis, &sigmas, &z, &residual, sqrt_lambda, k, cap).expect("brand should not fail"); + + assert_orthonormal(&naive.basis, naive.n, 1e-10, "naïve orthonormality"); + assert_orthonormal(&brand.basis, brand.n, 1e-10, "brand orthonormality"); +} + +/// Singular values come back non-negative and in descending order from both +/// paths. Order is what makes position meaningful: rank adaptation walks the +/// values accumulating energy until a threshold is met and takes that position +/// as the rank, which is only a sensible rule if the strongest direction is +/// first. The incremental path preserves the ordering through its +/// back-transform and corrective step rather than inheriting it by luck. +/// +/// ´claim:svd:singular-values-come-back-non-negative-and-in-descending-order-so-position-means-strength´ +/// ´test:unit:singular-values-sorted-and-nonnegative´ +#[test] +// Linear algebra test — d, k, b, z, x follow standard mathematical notation. +#[allow(clippy::many_single_char_names)] +fn singular_values_sorted_and_nonnegative() { + let d = 64; + let k = 4; + let b = 8; + let cap = 4; + let sqrt_lambda = 0.95_f64.sqrt(); + let mut rng = SmallRng::seed_from_u64(7777); + + let basis = random_basis(d, cap, &mut rng); + let sigmas = vec![12.0, 6.0, 3.0, 1.5]; + + let x = random_matrix(b, d, &mut rng); + let (z, residual) = phase1_projection(&x, &basis, k); + + for (label, result) in [ + ( + "naïve", + naive_svd::evolve(&basis, &sigmas, &z, &residual, sqrt_lambda, k, cap).unwrap(), + ), + ( + "brand", + brand_svd::evolve(&basis, &sigmas, &z, &residual, sqrt_lambda, k, cap).unwrap(), + ), + ] { + for (i, &s) in result.sigmas.iter().enumerate() { + assert!(s >= 0.0, "{label}: σ[{i}] = {s} is negative"); + } + for i in 1..result.n { + assert!( + result.sigmas[i - 1] >= result.sigmas[i] - 1e-12, + "{label}: σ not decreasing: σ[{}]={}, σ[{i}]={}", + i - 1, + result.sigmas[i - 1], + result.sigmas[i] + ); + } + } +} + +/// Given the same inputs, either path returns the same axes and the same +/// values bit for bit — not close, identical. Nothing in the step draws on +/// randomness, iteration order or timing, so two sentinels fed the same +/// traffic hold the same model, and a difference between them is always +/// evidence about the traffic rather than about the machine. +/// +/// ´claim:svd:repeating-a-step-on-the-same-input-reproduces-it-bit-for-bit´ +/// ´test:unit:deterministic-reproduction´ +#[test] +// Linear algebra test — d, k, b, z, x follow standard mathematical notation. +#[allow(clippy::many_single_char_names)] +fn deterministic_reproduction() { + let d = 128; + let k = 2; + let b = 16; + let cap = 2; + let sqrt_lambda = 0.99_f64.sqrt(); + + // Run twice with same seed. + for strategy in ["naive", "brand"] { + let mut results = Vec::new(); + for _ in 0..2 { + let mut rng = SmallRng::seed_from_u64(42); + let basis = random_basis(d, cap, &mut rng); + let sigmas = vec![5.0, 2.0]; + let x = random_matrix(b, d, &mut rng); + let (z, residual) = phase1_projection(&x, &basis, k); + + let result = match strategy { + "naive" => naive_svd::evolve(&basis, &sigmas, &z, &residual, sqrt_lambda, k, cap).unwrap(), + "brand" => brand_svd::evolve(&basis, &sigmas, &z, &residual, sqrt_lambda, k, cap).unwrap(), + _ => unreachable!(), + }; + results.push(result); + } + + // Exact bit-for-bit equality — deterministic SVD produces identical + // floating-point values, so the comparison is made on the bit + // patterns rather than on numeric equality. + { + let a = &results[0]; + let b_r = &results[1]; + assert_eq!(a.n, b_r.n, "{strategy}: n mismatch"); + for i in 0..a.n { + assert_eq!( + a.sigmas[i].to_bits(), + b_r.sigmas[i].to_bits(), + "{strategy}: σ[{i}] not bitwise equal" + ); + for j in 0..d { + assert_eq!( + a.basis[(j, i)].to_bits(), + b_r.basis[(j, i)].to_bits(), + "{strategy}: basis[{j},{i}] not bitwise equal" + ); + } + } + } + } +} + +// ════════════════════════════════════════════════════════════ +// § 5 — Functional / integration +// ════════════════════════════════════════════════════════════ + +/// Agreement between the two paths is a property of a whole streaming run, not +/// of one step in isolation. Each path is fed the same sequence but carries its +/// own state forward, so any difference compounds through every later step — +/// and over a long run the difference stays within a margin that grows only in +/// step with the number of steps taken. Divergence is linear rather than +/// explosive, which is what makes a cell that has been running for hours as +/// trustworthy as one that has just started. +/// +/// ´claim:svd:agreement-survives-a-long-streaming-run-with-divergence-growing-no-faster-than-the-step-count´ +/// ´test:unit:equivalence-multi-step´ +#[test] +// Linear algebra test — d, k, b, z, x follow standard mathematical notation. +// i32→f64 cast is for the growing FP-error tolerance: 1e-6 × (step + 1.0). +fn equivalence_multi_step() { + let d = 64; + let k = 4; + let b = 8; + let cap = 4; + let lambda: f64 = 0.99; + let sqrt_lambda = lambda.sqrt(); + let steps = 50; + let mut rng = SmallRng::seed_from_u64(1337); + + // Shared initial state. + let mut naive_basis = random_basis(d, cap, &mut rng); + let mut naive_sigmas = vec![5.0, 3.0, 1.0, 0.5]; + let mut brand_basis = naive_basis.clone(); + let mut brand_sigmas = naive_sigmas.clone(); + + for step in 0..steps { + let x = random_matrix(b, d, &mut rng); + + // Project against naïve state (both should be identical). + let (z_n, res_n) = phase1_projection(&x, &naive_basis, k); + let (z_b, res_b) = phase1_projection(&x, &brand_basis, k); + + let naive_result = + naive_svd::evolve(&naive_basis, &naive_sigmas, &z_n, &res_n, sqrt_lambda, k, cap).expect("naïve should not fail"); + let brand_result = + brand_svd::evolve(&brand_basis, &brand_sigmas, &z_b, &res_b, sqrt_lambda, k, cap).expect("brand should not fail"); + + // Update state for next step. + naive_basis = pad_to_cap(&naive_result.basis, d, cap); + naive_sigmas = pad_sigmas(&naive_result.sigmas, cap); + brand_basis = pad_to_cap(&brand_result.basis, d, cap); + brand_sigmas = pad_sigmas(&brand_result.sigmas, cap); + + // Allow growing tolerance as FP differences accumulate. + let sigma_tol = 1e-6 * (f64::from(step) + 1.0); + // Start from (1.0 − 1e-4) so the check is satisfiable at step 0 + // (|cos| ≤ 1.0 exactly for unit vectors, so "> 1.0" always fails). + let basis_tol = f64::from(step + 1).mul_add(-1e-4, 1.0); + let label = format!("multi-step {step}"); + + assert_updates_close(&naive_result, &brand_result, sigma_tol, basis_tol, &label); + } +} + +/// The two paths do not merely agree on coordinates they were given; they +/// explain data they have never seen equally well. Held-out rows projected +/// onto either model leave the same amount unaccounted for, which is the +/// property the sentinel actually depends on — novelty is measured from +/// exactly that leftover, so equal reconstruction means equal scores whichever +/// path produced the axes. +/// +/// ´claim:svd:the-two-paths-explain-unseen-data-equally-well-not-merely-agree-on-the-numbers-they-were-handed´ +/// ´test:unit:reconstruction-error-equivalence´ +#[test] +// Linear algebra test — d, k, b, z, x follow standard mathematical notation. +#[allow(clippy::many_single_char_names)] +fn reconstruction_error_equivalence() { + let d = 128; + let k = 4; + let b = 16; + let cap = 4; + let sqrt_lambda = 0.99_f64.sqrt(); + let mut rng = SmallRng::seed_from_u64(2025); + + let basis = random_basis(d, cap, &mut rng); + let sigmas = vec![8.0, 4.0, 2.0, 1.0]; + + let x = random_matrix(b, d, &mut rng); + let (z, residual) = phase1_projection(&x, &basis, k); + + let naive = naive_svd::evolve(&basis, &sigmas, &z, &residual, sqrt_lambda, k, cap).unwrap(); + let brand = brand_svd::evolve(&basis, &sigmas, &z, &residual, sqrt_lambda, k, cap).unwrap(); + + // Project test data onto both new bases and measure reconstruction error. + let test_x = random_matrix(b, d, &mut rng); + + let naive_err = recon_error(&test_x, &naive.basis, naive.n); + let brand_err = recon_error(&test_x, &brand.basis, brand.n); + + let rel = (naive_err - brand_err).abs() / naive_err.max(1e-15); + assert!( + rel < 1e-6, + "reconstruction error diverged: naïve={naive_err:.8e}, brand={brand_err:.8e}, rel={rel:.2e}" + ); +} + +#[test] +fn oracle_rejects_orthogonal_clustered_planes() { + for sigmas in [[10.0, 9.9], [10.0, 10.0]] { + let left = SubspaceUpdate { + basis: Mat::from_fn(4, 2, |row, col| if row == col { 1.0 } else { 0.0 }), + sigmas: sigmas.to_vec(), + n: 2, + }; + let right = SubspaceUpdate { + basis: Mat::from_fn(4, 2, |row, col| if row == col + 2 { 1.0 } else { 0.0 }), + sigmas: sigmas.to_vec(), + n: 2, + }; + let comparison = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + crate::maths::compare_subspace_updates( + &left, + &right, + crate::maths::SvdStrategy::Brand, + crate::maths::SvdStrategy::Naive, + ); + })); + assert!(comparison.is_err(), "orthogonal clustered planes must be rejected"); + } +} + +#[test] +fn oracle_rejects_unit_singular_value_against_zero() { + let left = SubspaceUpdate { + basis: Mat::from_fn(2, 1, |row, _| if row == 0 { 1.0 } else { 0.0 }), + sigmas: vec![1.0], + n: 1, + }; + let right = SubspaceUpdate { + sigmas: vec![0.0], + ..left.clone() + }; + let comparison = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + crate::maths::compare_subspace_updates( + &left, + &right, + crate::maths::SvdStrategy::Brand, + crate::maths::SvdStrategy::Naive, + ); + })); + assert!(comparison.is_err(), "a unit singular value against zero must be rejected"); +} + +#[test] +fn oracle_accepts_a_rotated_basis_of_the_same_cluster() { + let left = SubspaceUpdate { + basis: Mat::from_fn(4, 2, |row, col| if row == col { 1.0 } else { 0.0 }), + sigmas: vec![10.0, 10.0], + n: 2, + }; + let right = SubspaceUpdate { + basis: Mat::from_fn(4, 2, |row, col| if row == 1 - col { 1.0 } else { 0.0 }), + ..left.clone() + }; + crate::maths::compare_subspace_updates( + &left, + &right, + crate::maths::SvdStrategy::Brand, + crate::maths::SvdStrategy::Naive, + ); +} + +#[test] +fn truncated_repeated_space_is_shared_between_strategies() { + // The second spectrum's gap is half the oracle's relative precision + // budget sqrt(epsilon)/2, so it exercises a numerical tie, not only an + // exactly repeated value. No assertion tolerance is needed here. + for leading in [1.0, 1.0 + f64::EPSILON.sqrt() / 4.0] { + let basis = Mat::from_fn(8, 1, |row, _| if row == 7 { 1.0 } else { 0.0 }); + let latent = Mat::zeros(4, 1); + let residual = Mat::from_fn(4, 8, |row, col| { + if row != col { + 0.0 + } else if row == 0 { + leading + } else { + 1.0 + } + }); + let brand = brand_svd::evolve(&basis, &[0.0], &latent, &residual, 1.0, 1, 2).expect("Brand supports this shape"); + let naive = naive_svd::evolve(&basis, &[0.0], &latent, &residual, 1.0, 1, 2).expect("the reference SVD converges"); + assert_eq!(brand.n, 2); + assert_eq!(naive.n, 2); + crate::maths::compare_subspace_updates( + &brand, + &naive, + crate::maths::SvdStrategy::Brand, + crate::maths::SvdStrategy::Naive, + ); + } +} diff --git a/packages/sentinel/src/maths/tests/mod.rs b/packages/sentinel/src/maths/tests/mod.rs new file mode 100644 index 000000000..de013757b --- /dev/null +++ b/packages/sentinel/src/maths/tests/mod.rs @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +//! Comprehensive comparison tests: Brand's incremental SVD vs naïve dense SVD. +//! +//! Every test runs both algorithms on the same inputs and asserts that +//! they produce equivalent results (up to SVD sign ambiguity and FP +//! tolerance). This is the expanded, persistent version of the +//! `debug_assert` oracle in `maths::evolve()`. +//! +//! # §-references +//! +//! - ADR-S-016 — Brand's incremental SVD +//! - ADR-S-015 — Cell creation performance + +mod brand_vs_naive; diff --git a/packages/sentinel/src/observation.rs b/packages/sentinel/src/observation.rs new file mode 100644 index 000000000..2efc40e0b --- /dev/null +++ b/packages/sentinel/src/observation.rs @@ -0,0 +1,196 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +//! Observation boundary: converting raw coordinate values into the +//! mathematical representation the subspace engine needs. +//! +//! This module is the anti-corruption layer between the host's domain +//! (positionally structured coordinate values) and the linear algebra +//! world (`Mat`, centred bit vectors). +//! +//! The input values must have hierarchical positional structure — +//! leading bits define coarse groupings and successive bits refine +//! them (e.g. IPv6 addresses). The host is responsible for ensuring +//! this property before handing values to the sentinel. +//! +//! The conversion is defined on the domain `[0, 2^N)` (§ALGO S-2.1) and +//! reads a value's low `N` bits, so a value at or above `2^N` would come +//! back as the in-domain value it is congruent to: a genuine observation's +//! vector, produced from something that is not that observation. Nothing at +//! this boundary can tell the two apart, which is why domain membership is +//! decided before a value is handed here — the sentinel takes that decision +//! once, at the public boundary, for this module and for the spatial layer +//! and the trackers alike. +//! +//! Nothing in this module touches `faer`. It produces plain `Vec` +//! data that the subspace module consumes. + +use torrust_mudlark::Coordinate; + +// ─── CentredBitSource trait ───────────────────────────────── + +/// Bridge trait: convert a coordinate value into centred bit form. +/// +/// Implemented in this crate for `u128` and `u64`, and open to a downstream +/// coordinate type that implements it as well: the trait is part of the +/// published surface, so nothing closes the set of implementations. An +/// implementor supplies `to_centred_bits` for its own width, and the two impls +/// here are what a wrapper around one of those widths delegates to. This is +/// the coordinate side of the generic parameters the sentinel is built on +/// ([ADR-S-018](https://github.com/torrust/torrust-index/blob/develop/packages/sentinel/adr/018-generic-domain-parameters.md)). +/// +/// The set of implementations is open; the width they can serve is not. Every +/// conversion returns a [`CentredBits`], which carries its values in a fixed +/// array of 128 slots, so a coordinate type wider than that has no vector to +/// return past the first 128 bits. An implementor for such a type is free to +/// exist — what it cannot do is drive a sentinel wider than the vector: +/// `SpectralSentinel::new` refuses a width above the ceiling rather than +/// building trackers whose extra dimensions would be fed a constant the +/// coordinate stream never produced. +pub trait CentredBitSource: Coordinate { + /// Convert `self` into a centred bit vector of length `n`. + /// + /// `n` is a request, not a promise: the effective width is `n` capped at + /// the width the implementing type actually holds, which is also the + /// width of the vector that comes back. The implementations here cap at + /// 128 and 64 respectively, and an implementation for another coordinate + /// type caps at its own. The cap is not a courtesy — the returned vector + /// is backed by a fixed hundred-and-twenty-eight-slot array, and a width + /// beyond the type's own would either read bits that do not exist or + /// index past that array — so an implementation applies it rather than + /// trusting the caller, and no caller can provoke a panic by asking for + /// more than the domain holds. + fn to_centred_bits(&self, n: u32) -> CentredBits; +} + +impl CentredBitSource for u128 { + #[allow(clippy::cast_possible_truncation)] // i < 128, fits in u32 + fn to_centred_bits(&self, n: u32) -> CentredBits { + let mut bits = [0.0_f64; 128]; + let width = n.min(128); + let len = width as usize; + for (i, slot) in bits[..len].iter_mut().enumerate() { + *slot = if (self >> (width - 1 - i as u32)) & 1 == 1 { + 0.5 + } else { + -0.5 + }; + } + CentredBits { bits, len } + } +} + +impl CentredBitSource for u64 { + #[allow(clippy::cast_possible_truncation)] // i < 64, fits in u32 + fn to_centred_bits(&self, n: u32) -> CentredBits { + let mut bits = [0.0_f64; 128]; + let len = n.min(64) as usize; + for (i, slot) in bits[..len].iter_mut().enumerate() { + *slot = if (self >> (n.min(64) - 1 - i as u32)) & 1 == 1 { + 0.5 + } else { + -0.5 + }; + } + CentredBits { bits, len } + } +} + +// ─── CentredBits ──────────────────────────────────────────── + +/// A coordinate value converted to a centred bit vector. +/// +/// Each of the `len` bits becomes: +/// - bit `1` → `+0.5` +/// - bit `0` → `−0.5` +/// +/// The encoded levels are symmetric about zero. Each dimension has zero expected mean under a uniform bit distribution (§ALGO S-2.3); arbitrary traffic need not have balanced bits, so this encoding does not subtract its empirical mean. +/// +/// The backing array is fixed at 128 slots, which is the sentinel's coordinate +/// width ceiling and not an implementation detail a wider coordinate type can +/// work around: a centred bit is `±0.5` and never zero, so the slots past +/// `len` are distinguishable from data and there is no honest way to present +/// them as observations. A sentinel is refused at construction above that +/// width for the same reason. +#[derive(Debug, Clone)] +pub struct CentredBits { + /// The centred bit values, from MSB (index 0) to LSB (index `len - 1`). + /// Only indices `[0, len)` are meaningful; the rest are zero-filled. + pub bits: [f64; 128], + + /// Runtime length (= N for the sentinel's domain width). + len: usize, +} + +impl CentredBits { + /// Build a vector from centred bit values already computed, with `len` + /// of them meaningful. + /// + /// This is how an implementation of [`CentredBitSource`] outside this + /// crate returns its conversion. The two implementations here work on + /// coordinate types whose bits are already there to be shifted out, and a + /// wrapper around one of those widths delegates to them; a coordinate + /// type whose centred form has to be computed has nothing to delegate to, + /// and this is the constructor it uses. Slots from `len` onward are the + /// caller's to leave at zero — [`suffix`](Self::suffix) never reads them. + /// + /// # Panics + /// + /// Panics if `len` exceeds 128, the fixed size of the backing array. A + /// length past the array is a mistake in the implementation rather than a + /// value a host could supply, which is the same reading + /// [`suffix`](Self::suffix) takes of a depth past the width: there is no + /// honest vector to return, and clamping would hand back an observation + /// narrower than the one the caller believes it built. + #[must_use] + pub const fn new(bits: [f64; 128], len: usize) -> Self { + assert!( + len <= crate::MAX_TRACKER_DIM, + "centred bit length exceeds the 128-slot backing array" + ); + Self { bits, len } + } + + /// How many of the backing array's slots carry a centred bit. + #[must_use] + pub const fn len(&self) -> usize { + self.len + } + + /// Whether the vector carries no bits at all — the zero-width domain. + #[must_use] + pub const fn is_empty(&self) -> bool { + self.len == 0 + } + + /// Convert a `u128` value to centred bits (128-bit, convenience wrapper). + #[must_use] + pub fn from_u128(value: u128) -> Self { + value.to_centred_bits(128) + } + + /// Construct from any coordinate type implementing `CentredBitSource`. + #[must_use] + pub(crate) fn from_coord(value: &C, n: u32) -> Self { + value.to_centred_bits(n) + } + + /// Return a slice of the suffix bits from position `depth` to `len`. + /// + /// For a cell at G-tree depth `d`, the first `d` bits are resolved + /// by routing (constant within the cell). The suffix `[d, len)` is + /// the working observation — the bits that vary and carry + /// statistical content (§ALGO S-2.4). + /// + /// Width: `len - depth`. At depth 0, the suffix is the entire + /// bit vector. At depth `len`, the suffix is empty (zero-width + /// cell — degenerate). + /// + /// # Panics + /// + /// Panics if `depth` exceeds `len`. + #[must_use] + pub fn suffix(&self, depth: u8) -> &[f64] { + &self.bits[usize::from(depth)..self.len] + } +} diff --git a/packages/sentinel/src/report.rs b/packages/sentinel/src/report.rs new file mode 100644 index 000000000..56259f41b --- /dev/null +++ b/packages/sentinel/src/report.rs @@ -0,0 +1,895 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +//! Report types emitted by the sentinel. +//! +//! These types carry the raw statistical measurements from each +//! [`ingest`](crate::SpectralSentinel::ingest) call. +//! They contain numbers and facts — never opinions or recommended actions. +//! +//! The host reads these reports and applies its own policy to decide +//! what (if anything) to do about them. + +use std::fmt::Debug; + +use torrust_mudlark::GNodeId; + +// ─── Batch-level ──────────────────────────────────────────── + +/// Complete statistical output from one +/// [`ingest`](crate::SpectralSentinel::ingest) call. +/// +/// Matches the structure defined in §ALGO S-14. +#[derive(Debug, Clone)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(bound = "C: serde::Serialize + serde::de::DeserializeOwned"))] +pub struct BatchReport { + /// Per-cell reports for competitively selected cells ($\mathcal{A}$). + /// + /// Only competitive cells that received observations in this batch + /// are included. Ordered by `GNodeId` for deterministic output + /// (ADR-S-005). + pub cell_reports: Vec>, + + /// Per-cell reports for ancestor-only cells ($\mathcal{A}^* \setminus \mathcal{A}$). + /// + /// Ancestor cells provide multi-scale context. Only those that + /// received observations in this batch are included. + /// Ordered by `GNodeId`. + pub ancestor_reports: Vec>, + + /// Hierarchical coordination reports from the G-tree walk (§ALGO S-7.4). + /// + /// One report per active coordination context. Ordered shallowest first, + /// ties broken by ascending `GNodeId`. + /// Empty when fewer than 2 competitive cells report scores. + pub coordination_reports: Vec>, + + /// Snapshot of the G-V Graph's spatial contour. + pub contour: ContourSnapshot, + + /// Operational health snapshot of the sentinel. + pub health: HealthReport, + + /// Summary of the current analysis set. + pub analysis_set_summary: AnalysisSetSummary, + + /// How old the oldest observation in this batch was when this + /// report was emitted, in microseconds. + /// + /// Measured entirely on the sentinel's own monotonic clock: the + /// batch is stamped as it arrives at + /// [`ingest`](crate::SpectralSentinel::ingest) and the figure is + /// read off as this report is assembled. No wall-clock instant is + /// recorded and no clock is compared with another machine's, so + /// there is no skew for the number to carry. + /// + /// A batch arrives whole, at the call boundary, so its oldest + /// observation is no older than the call and one figure bounds + /// every observation in the batch: none is older than this, and the + /// oldest is exactly this old. + /// + /// `None` says the report carries no age. Two situations reach it — + /// a batch that held no observations, which therefore has no oldest + /// one, and a payload written before this field existed — and they + /// share a spelling because they tell a consumer the same thing: + /// there is nothing here to read. A zero would say something else, + /// and something untrue. + /// + /// What the figure does *not* include is how long the host held the + /// observations before handing them over. That delay is real and is + /// left as an explicitly unmeasured residual: measuring it would + /// mean comparing two clocks, and this figure's whole value is that + /// it compares none. + /// + /// Saturates at [`u64::MAX`] microseconds — some hundreds of + /// thousands of years, and unreachable in practice. + #[cfg_attr(feature = "serde", serde(default))] + pub oldest_observation_age_micros: Option, +} + +// ─── Cell-level ───────────────────────────────────────────── + +/// Statistics for a single analysis cell after processing one batch. +/// +/// Each cell is at a specific G-tree depth and operates on suffix bits +/// `[d, N)` at width `w = N - d`. Competitive cells are selected +/// by the analysis selector (§ALGO S-8.1); ancestor cells provide +/// multi-scale context (§ALGO S-8.2). +#[derive(Debug, Clone)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(bound = "C: serde::Serialize + serde::de::DeserializeOwned"))] +pub struct CellReport { + /// Arena handle of the backing G-node. + pub gnode_id: GNodeId, + + /// Lower bound of the dyadic interval (inclusive). + pub start: C, + + /// Upper bound of the dyadic interval, exclusive everywhere except at the + /// top of the domain: `end` belongs to this cell exactly when it is the + /// last value of the domain, which is the case for a coordinate type that + /// cannot represent `2^N` at its full width and so names its domain + /// maximum as the bound. A type that can represent `2^N` keeps + /// `[start, end)` half-open at every width. + pub end: C, + + /// G-tree depth of this cell. + pub depth: u32, + + /// Suffix width: `N - depth`. + pub analysis_width: usize, + + /// Whether this cell is competitively selected (vs ancestor-only). + pub is_competitive: bool, + + /// How many observations in this batch routed to this cell. + pub sample_count: usize, + + /// Rank in force while this batch was scored. + pub rank: usize, + + /// Fraction of total variance captured by the rank reported for this batch. + pub energy_ratio: f64, + + /// Largest singular value of the learned subspace. + pub top_singular_value: f64, + + /// Anomaly scores along all four measurement axes. + pub scores: AnomalyScores, + + /// How mature is this tracker's learned model? + pub maturity: TrackerMaturity, + + /// Geometric properties in force while this batch was scored. + pub geometry: ScoringGeometry, + + /// Per-sample scores, if enabled. + pub per_sample: Option>, +} + +// ─── Coordination-level ─────────────────────────────────────── + +/// Coordination analysis at a single G-tree internal node (§ALGO S-7.1). +/// +/// Produced by a coordination tracker (`SubspaceTracker` at $w = 4$) +/// that consumes running-mean-centred cell-score matrices as +/// observations. The group consists of all competitive cells in +/// this node's subtree that reported scores in this batch. +/// +/// The four anomaly axes have second-order meaning at this level: +/// +/// | Meta-axis | Detects | +/// |-------------|------------------------------------------------------| +/// | Novelty | A cell-score pattern the model has never seen | +/// | Displacement| The overall score landscape has shifted | +/// | Surprise | A specific scoring axis is system-wide anomalous | +/// | Coherence | An unusual combination of axis elevations | +#[derive(Debug, Clone)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(bound = "C: serde::Serialize + serde::de::DeserializeOwned"))] +pub struct CoordinationReport { + /// Arena handle of the coordination context's G-node. + pub gnode_id: GNodeId, + + /// Lower bound of the coordination context's dyadic interval (inclusive). + pub start: C, + + /// Upper bound of the coordination context's dyadic interval, exclusive + /// everywhere except at the top of the domain: a context bounded by the + /// domain's last value covers that value. The domain has a last value + /// only where the coordinate type cannot represent `2^N` at its full + /// width, so that its domain maximum names the top of the domain rather + /// than the first value above it; where `2^N` is representable the bound + /// stays exclusive at every width. The root context covers the full-width + /// interval, so this is the ordinary case rather than a corner of it. + pub end: C, + + /// G-tree depth of the coordination context. + pub depth: u32, + + /// How many competitive cells in this subtree contributed + /// score vectors this batch. + pub cells_reporting: usize, + + /// Rank in force while this coordination batch was scored. + pub rank: usize, + + /// Fraction of total variance captured by the rank reported for this batch. + pub energy_ratio: f64, + + /// Largest singular value. + pub top_singular_value: f64, + + /// Anomaly scores at the coordination level. + pub scores: AnomalyScores, + + /// How mature is this coordination tracker's model? + pub maturity: TrackerMaturity, + + /// Geometric properties in force while this coordination batch was scored. + pub geometry: ScoringGeometry, + + /// Per-member scores, if + /// [`SentinelConfig::per_sample_scores`](crate::SentinelConfig::per_sample_scores) + /// is enabled. + /// + /// Each entry corresponds to one competitive cell in the coordination + /// group. The `cell_start`/`cell_end`/`cell_depth` fields identify + /// which cell produced this score vector. When present, indices + /// correspond to cells in subtree order (left-to-right). + pub per_member: Option>>, +} + +// ─── Per-depth-level (internal) ───────────────────────────── + +/// Statistics for a single tracker after processing one batch. +/// +/// This is the internal report type returned by `SubspaceTracker::observe()`. +/// Used internally; callers see [`CellReport`] at the public API. +#[derive(Debug, Clone)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct TrackerReport { + /// Rank in force while this batch was scored (number of active basis vectors). + pub rank: usize, + + /// Fraction of total variance captured by the rank reported for this batch. + pub energy_ratio: f64, + + /// Largest singular value of the learned subspace. + pub top_singular_value: f64, + + /// Anomaly scores along all four measurement axes. + pub scores: AnomalyScores, + + /// How mature is this tracker's learned model? + pub maturity: TrackerMaturity, + + /// Geometric properties that determined which scoring axes were + /// structurally meaningful while this batch was scored. + pub geometry: ScoringGeometry, + + /// Per-sample scores, if enabled. + pub per_sample: Option>, +} + +// ─── Anomaly scores ───────────────────────────────────── + +/// The four anomaly-score axes for a batch of observations. +/// +/// The sentinel scores each observation along four independent axes +/// organised into two conceptual groups: +/// +/// **Subspace axis** — how well the learned model explains the observation: +/// +/// | Score | Metric | Intuition | +/// |-------|--------|-----------| +/// | *Novelty* | Residual energy / DOF: `‖X − X̂‖² / (dim − k)` | "How much of this is foreign?" | +/// +/// **Cell axis** — how typical the observation is for *this* cell: +/// +/// | Score | Metric | Intuition | +/// |-------|--------|-----------| +/// | *Displacement* | `‖z‖² / (k + ‖z‖²)`, bounded in `[0, 1)` | "How far is this from the centroid?" | +/// | *Surprise* | Mahalanobis / rank: `Σⱼ ((zⱼ − μⱼ)/σⱼ)² / k` | "The shape is familiar, but the magnitude is wild" | +/// | *Coherence* | Cross-correlation deviation: `Σⱼ<ₗ (zⱼzₗ − Cⱼₗ)²` | "Normal individually, but this combination is new" | +/// +/// Displacement, surprise, and coherence decompose the latent +/// activation pattern along orthogonal statistical concerns: +/// displacement measures total energy, surprise measures +/// per-dimension magnitude (diagonal covariance), and coherence +/// measures pairwise interaction (off-diagonal covariance). +/// +/// All four axes share the same polarity: **higher values indicate +/// greater anomalous departure**. This ensures uniform z-score +/// interpretation and EWMA outlier-filter robustness (see +/// §ALGO S-5.1). +/// +/// **Why not projection energy / "normality"?** Under the sentinel's +/// centred binary encoding, every observation has the same L2 norm +/// (`d / 4`). Projection energy is therefore a perfect affine function +/// of residual energy — it carries zero independent information. +/// See §ALGO S-15.2 for the full proof. +#[derive(Debug, Clone)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct AnomalyScores { + /// Residual energy per residual DOF: `‖X − X̂‖² / (dim − k)`. + pub novelty: ScoreDistribution, + + /// Cell displacement: `‖z‖² / (k + ‖z‖²)`, bounded in `[0, 1)`. + pub displacement: ScoreDistribution, + + /// Latent surprise: Mahalanobis distance per rank, + /// `Σⱼ ((zⱼ − μⱼ) / σⱼ)² / k`. + pub surprise: ScoreDistribution, + + /// Latent coherence: cross-correlation deviation, + /// `2 / (k(k−1)) · Σⱼ<ₗ (zⱼzₗ − Cⱼₗ)²`. + pub coherence: ScoreDistribution, +} + +// ─── Score distribution ───────────────────────────────────── + +/// Summary statistics for a vector of anomaly scores. +/// +/// Contains both the raw score distribution and its relationship +/// to the learned baseline (via z-scores). +#[derive(Debug, Clone, Copy)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct ScoreDistribution { + /// Minimum score in the batch. + pub min: f64, + + /// Maximum score in the batch. + pub max: f64, + + /// Arithmetic mean of scores in the batch. + pub mean: f64, + + /// Z-score of the *maximum* score against the EWMA baseline. + pub max_z_score: f64, + + /// Z-score of the *mean* score against the EWMA baseline. + pub mean_z_score: f64, + + /// Snapshot of the fast EWMA baseline this distribution was scored against. + pub baseline: BaselineSnapshot, + + /// CUSUM drift accumulator for this scoring axis. + pub cusum: CusumSnapshot, + + /// Current clip-pressure EWMA for this axis: ρ̄ ∈ [0, 1] (§ALGO S-14.4). + pub clip_pressure: f64, +} + +// ─── Baseline snapshot ────────────────────────────────────── + +/// Frozen snapshot of an EWMA baseline at the time of scoring. +#[derive(Debug, Clone, Copy)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct BaselineSnapshot { + /// Current EWMA mean of "normal" scores. + pub mean: f64, + + /// Current EWMA variance of "normal" scores. + pub variance: f64, +} + +// ─── CUSUM snapshot ───────────────────────────────────────── + +/// Frozen snapshot of a CUSUM accumulator at the time of scoring. +#[derive(Debug, Clone, Copy)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct CusumSnapshot { + /// Current CUSUM accumulator value. + pub accumulator: f64, + + /// Snapshot of the slow EWMA baseline used as the CUSUM reference. + pub slow_baseline: BaselineSnapshot, + + /// Number of batches since the CUSUM was last reset. + pub steps_since_reset: u64, +} + +// ─── Maturity ─────────────────────────────────────────────── + +/// How much experience a tracker has, and how much of that is noise. +#[derive(Debug, Clone, Copy)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct TrackerMaturity { + /// Number of real (non-noise) observations this tracker has processed. + pub real_observations: u64, + + /// Number of noise observations injected into this tracker. + pub noise_observations: u64, + + /// Estimated fraction of the baseline **not yet established by + /// real data**. + pub noise_influence: f64, +} + +impl TrackerMaturity { + /// A tracker with no observations of any kind. + #[must_use] + pub const fn cold() -> Self { + Self { + real_observations: 0, + noise_observations: 0, + noise_influence: 1.0, + } + } + + /// Total observations (real + noise). + #[must_use] + pub const fn total_observations(&self) -> u64 { + self.real_observations + self.noise_observations + } +} + +// ─── Scoring geometry ─────────────────────────────────────── + +/// Geometry of the model that scored the associated batch. In inspection +/// snapshots, this is the geometry of the most recent scored batch. +#[derive(Debug, Clone, Copy)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct ScoringGeometry { + /// Working dimensionality of the tracker's input space. + pub dim: usize, + + /// Maximum rank this tracker can reach: `min(dim, max_rank)`. + pub cap: usize, + + /// Residual degrees of freedom: `dim - scoring rank`. + pub residual_dof: usize, +} + +impl ScoringGeometry { + /// Whether the novelty axis is structurally degenerate + /// (`residual_dof == 0`). + #[must_use] + pub const fn is_novelty_saturated(&self) -> bool { + self.residual_dof == 0 + } + + /// Whether the novelty axis *can* become degenerate as rank + /// adapts (`cap >= dim`). + #[must_use] + pub const fn is_novelty_saturable(&self) -> bool { + self.cap >= self.dim + } +} + +// ─── Per-sample scores ────────────────────────────────────── + +/// Raw anomaly scores for a single observation. +#[derive(Debug, Clone, Copy)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct SampleScore { + /// Residual energy per residual DOF (novelty axis). + pub novelty: f64, + + /// Cell displacement score, bounded in `[0, 1)`. + pub displacement: f64, + + /// Mahalanobis distance per rank (surprise axis). + pub surprise: f64, + + /// Cross-correlation deviation (coherence axis). + pub coherence: f64, + + /// Z-score of `novelty` against its EWMA baseline. + pub novelty_z: f64, + + /// Z-score of `displacement` against its EWMA baseline. + pub displacement_z: f64, + + /// Z-score of `surprise` against its EWMA baseline. + pub surprise_z: f64, + + /// Z-score of `coherence` against its EWMA baseline. + pub coherence_z: f64, +} + +// ─── Health ───────────────────────────────────────────────── + +/// Operational health snapshot of the entire sentinel. +#[derive(Debug, Clone)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct HealthReport { + /// Total live G-nodes in the G-V Graph. + pub total_g_nodes: usize, + + /// Number of semi-internal G-nodes — those with one subdivided half and + /// one that still accumulates locally. They sit on the contour alongside + /// the terminals. + pub semi_internal_count: usize, + + /// Number of active cell trackers (total). + pub active_trackers: usize, + + /// Number of active competitive trackers: $|\mathcal{A}|$. + pub active_competitive_trackers: usize, + + /// Number of active ancestor-only trackers: $|\mathcal{A}^*| - |\mathcal{A}| - 1$. + /// + /// The −1 accounts for the permanent root tracker, which is + /// neither competitive nor a normal ancestor. + pub active_ancestor_trackers: usize, + + /// Number of active coordination contexts. + pub active_coordination_contexts: usize, + + /// Total cells with allocated trackers (online + warming): + /// $|\mathcal{I}|$ (the investment set, §ALGO S-8.2, ADR-S-019). + pub investment_set_size: usize, + + /// Members of $\mathcal{I}$ currently in the warm-up pipeline + /// (§ALGO S-11.6, ADR-S-019). + pub warming_trackers: usize, + + /// Competitive targets ($\mathcal{T}$) not yet promoted to + /// $\mathcal{A}$ — i.e. warming cells that are competitive + /// targets, not ancestors (§ALGO S-14.11, ADR-S-019). + pub warming_competitive_targets: usize, + + /// Total real observations across the sentinel's lifetime. + pub lifetime_observations: u64, + + /// Number of cells with a live tracker — the same figure as + /// `active_trackers`, kept because it is part of the published shape of + /// this report. It is not the size of the analysis set, which also + /// names cells still warming and is reported as `investment_set_size`. + pub cells_tracked: usize, + + /// Distribution of ranks across all active trackers. + pub rank_distribution: RankDistribution, + + /// Distribution of maturity across all active trackers. + pub maturity_distribution: MaturityDistribution, + + /// Distribution of geometric scoring reliability across all + /// active per-cell trackers. + pub geometry_distribution: GeometryDistribution, + + /// Health of the coordination tier. + pub coordination_health: CoordinationHealth, + + /// Clip-pressure distribution across active trackers (§ALGO S-14.11). + pub clip_pressure_distribution: ClipPressureDistribution, +} + +/// Summary of rank values across all active trackers. +#[derive(Debug, Clone, Copy)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct RankDistribution { + /// Lowest rank among all trackers. + pub min: usize, + + /// Highest rank among all trackers. + pub max: usize, + + /// Mean rank across all trackers. + pub mean: f64, +} + +/// Summary of maturity across all active trackers. +#[derive(Debug, Clone, Copy)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct MaturityDistribution { + /// Highest noise influence among all trackers (least mature). + pub max_noise_influence: f64, + + /// Lowest noise influence among all trackers (most mature). + pub min_noise_influence: f64, + + /// Mean noise influence across all trackers. + pub mean_noise_influence: f64, + + /// Number of trackers with zero real observations. + pub cold_trackers: usize, +} + +// ─── Geometry distribution ────────────────────────────────── + +/// Summary of geometric scoring reliability across a set of trackers. +#[derive(Debug, Clone, Copy)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct GeometryDistribution { + /// Trackers where `rank == dim` (novelty axis degenerate). + pub novelty_saturated: usize, + + /// Trackers where `cap >= dim` (novelty *can* become degenerate). + pub novelty_saturable: usize, + + /// Trackers where `rank < 2` (coherence axis does not exist). + pub coherence_inactive: usize, +} + +// ─── Clip-pressure distribution ───────────────────────────── + +/// Summary of clip-pressure EWMA values across active trackers (§ALGO S-14.11). +#[derive(Debug, Clone, Copy)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct ClipPressureDistribution { + /// Minimum clip-pressure EWMA across active tracker axes. + pub min: f64, + + /// Maximum clip-pressure EWMA across active tracker axes. + pub max: f64, + + /// Mean clip-pressure EWMA across active tracker axes. + pub mean: f64, +} + +// ─── Coordination health ──────────────────────────────────── + +/// Health snapshot of the hierarchical coordination tier. +/// +/// Summarises the active coordination contexts — subspace trackers +/// operating at $w = 4$ on cross-cell score patterns. +#[derive(Debug, Clone, Copy)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct CoordinationHealth { + /// Number of active coordination contexts. + pub active_contexts: usize, + + /// Capacity (max rank) of the coordination trackers. + /// Always min(4, `max_rank`) since $w = 4$. + pub capacity: usize, + + /// Rank distribution across active contexts. + pub rank_distribution: RankDistribution, + + /// Maturity distribution across active contexts. + pub maturity_distribution: MaturityDistribution, + + /// Working dimensionality (always 4). + pub dim: usize, + + /// Geometry distribution across active contexts. + pub geometry_distribution: GeometryDistribution, +} + +// ─── Axis baseline snapshots ──────────────────────────────── + +/// Per-axis EWMA baseline snapshots for all four scoring axes. +/// +/// Provides read-only access to the learned baseline statistics +/// without requiring direct access to the tracker internals. +/// Used by convergence tests to verify EWMA settling behaviour +/// (ADR-S-014). +#[derive(Debug, Clone, Copy)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct AxisBaselineSnapshots { + /// Novelty axis baseline. + pub novelty: BaselineSnapshot, + + /// Displacement axis baseline. + pub displacement: BaselineSnapshot, + + /// Surprise axis baseline. + pub surprise: BaselineSnapshot, + + /// Coherence axis baseline. + pub coherence: BaselineSnapshot, +} + +// ─── Cell inspection ──────────────────────────────────────── + +/// Snapshot of a cell's tracker state. +/// +/// Returned by [`SpectralSentinel::inspect_cell`](crate::SpectralSentinel::inspect_cell). +#[derive(Debug, Clone)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(bound = "C: serde::Serialize + serde::de::DeserializeOwned"))] +pub struct CellInspection { + /// Arena handle of the backing G-node. + pub gnode_id: GNodeId, + + /// Lower bound of the dyadic interval (inclusive). + pub start: C, + + /// Upper bound of the dyadic interval, exclusive everywhere except at the + /// top of the domain: the inspected cell contains `end` only where `end` + /// is the domain's topmost value. That happens when the coordinate type + /// has no representation for `2^N` at its full width, leaving its domain + /// maximum to serve as the bound; wherever `2^N` is representable — every + /// narrower width, and a type that reaches it at the full width — the + /// bound is excluded as usual. + pub end: C, + + /// G-tree depth of this cell. + pub depth: u32, + + /// Suffix width: `N - depth`. + pub analysis_width: usize, + + /// Whether this cell is competitively selected (vs ancestor-only). + pub is_competitive: bool, + + /// Current rank (number of active basis vectors). + pub rank: usize, + + /// Fraction of total variance captured by the current rank. + pub energy_ratio: f64, + + /// Largest singular value of the learned subspace. + pub top_singular_value: f64, + + /// Maturity state. + pub maturity: TrackerMaturity, + + /// Geometry of the model that scored this tracker's most recent batch. + /// This can differ from the current `rank` after adaptation. + pub geometry: ScoringGeometry, + + /// Per-axis EWMA baseline snapshots (ADR-S-014). + pub baselines: AxisBaselineSnapshots, +} + +// ─── Contour snapshot ─────────────────────────────────────── + +/// Snapshot of the G-V Graph's spatial contour at report time. +/// +/// The contour is the observable surface of the spatial structure — +/// how many distinct regions exist, how many cells stand on that +/// surface, and how much total traffic volume the graph has +/// accumulated. A contour cell is a terminal node or a semi-internal +/// one, whose unsubdivided half accumulates locally and is a cell in +/// its own right (§ALGO S-14.10). +/// +/// Populated from `GvGraph::plateaus()`, `GvGraph::terminal_count()` +/// summed with `GvGraph::semi_internal_count()`, and +/// `GvGraph::total_sum()`. +#[derive(Debug, Clone, Copy)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct ContourSnapshot { + /// Number of plateaus in the G-V Graph's spatial structure. + /// + /// A plateau is a contiguous range of cells at the same + /// depth. Fewer plateaus → more uniform spatial resolution. + pub plateau_count: usize, + + /// Number of cells on the contour: the terminal nodes together with the + /// semi-internal ones, whose unsubdivided half still accumulates locally + /// and is a cell in its own right. + /// + /// This is the spatial resolution: how many non-overlapping + /// regions the domain is partitioned into. It may differ from the number + /// of cells in the batch report, which also carries the ancestors above + /// the contour. + pub cell_count: usize, + + /// Total accumulated importance across the entire G-V Graph. + /// + /// Erased to `f64` via `V::to_f64_approx()` — the concrete + /// accumulator type is hidden from report consumers. + /// Values above 2^53 may lose LSBs (acceptable for diagnostics). + pub total_importance: f64, + + /// Child cells created by catalytic or bootstrap bisection since the + /// previous report. A bisection that creates two children counts as two + /// splits. The value comes from the spatial layer's monotonic event counter. + /// + /// Saturates at [`u32::MAX`] when the interval contains more splits than + /// the field can represent. + pub splits_since_last_report: u32, + + /// Net structural removals since the previous report: + /// evictions minus restorations (legacy promotions). + /// + /// The value comes from the spatial layer's monotonic event counters. An + /// interval with more restorations than evictions reports zero because this + /// field is unsigned, and a value above [`u32::MAX`] reports [`u32::MAX`]. + pub net_removals_since_last_report: u32, +} + +// ─── Analysis set summary ─────────────────────────────────── + +/// Summary of the analysis set at report time. +/// +/// Describes the investment set and producing sets without +/// enumerating every cell. For the complete set, use +/// [`SpectralSentinel::analysis_set()`](crate::SpectralSentinel::analysis_set). +/// +/// Every count and range is taken over the cells its producer is describing, +/// and which cells those are is the producer's to state rather than the type's: +/// +/// - [`AnalysisSet::summary()`](crate::AnalysisSet::summary) takes every figure +/// over the whole selection, online or still warming — the competitive +/// targets $\mathcal{T}$ and the investment set $\mathcal{I}$ +/// (§ALGO S-8.1–8.2). +/// - [`AnalysisSet::summary_online()`](crate::AnalysisSet::summary_online) +/// takes them over the selection intersected with the cells that have a +/// tracker — the producing sets $\mathcal{A}$ and $\mathcal{A}^*$ +/// (§ALGO S-8.3) — leaving the investment count whole, because the cells the +/// filter would drop are precisely those already paid for and not yet +/// producing. +/// - The summary carried by [`BatchReport`] is that online reading with two +/// fields replaced by figures the sentinel can see directly and a selection +/// snapshot cannot: the tracker population for `investment_set_size`, and its +/// own tally for `degenerate_cells_skipped`. +/// +/// A reader holding one of these values therefore knows the shape of the +/// figures but not their scope, and takes the scope from whichever call +/// produced it. +/// +/// See §ALGO S-14.12, ADR-S-019. +#[derive(Debug, Clone, Copy)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct AnalysisSetSummary { + /// Number of competitive targets counted: $|\mathcal{T}|$ over the whole + /// selection, or the producing competitive set $|\mathcal{A}|$ + /// (§ALGO S-8.3) over the online cells. + /// + /// Always ≤ `analysis_k` from the configuration. + pub competitive_size: usize, + + /// Total cells in the full set counted: $|\mathcal{I}|$ over the whole + /// selection, or the producing full set $|\mathcal{A}^*|$ (§ALGO S-8.3) + /// over the online cells. + /// + /// Includes the competitive targets counted here, their G-tree + /// ancestors, and the permanent root tracker. + pub full_size: usize, + + /// Total cells with allocated trackers, warming ones included: + /// $|\mathcal{I}|$ (the investment set, §ALGO S-8.2). + /// + /// Never narrowed to the online cells, whichever summary produced it: from + /// a selection snapshot it is the size of the whole selection, and in the + /// batch report it is the tracker population itself. The two readings part + /// wherever a tracker outlives its cell's membership of the selection, and + /// the population is the one being paid for. + pub investment_set_size: usize, + + /// (min, max) G-tree depth across the cells counted by `full_size`. + /// + /// Depth 0 = root (always present). Max depth reflects + /// the finest spatial resolution currently being analysed. + pub depth_range: (u32, u32), + + /// (min, max) importance across the cells counted by `competitive_size`. + /// + /// Erased to `f64` via `V::to_f64_approx()` — the concrete + /// accumulator type is hidden from report consumers. + /// `(0.0, 0.0)` when `competitive_size == 0`. + pub importance_range: (f64, f64), + + /// (min, max) V-Tree depth across the cells counted by `competitive_size`. + /// + /// V-Tree depth reflects competitive standing — lower = more + /// significant. `(0, 0)` when `competitive_size == 0`. + pub v_depth_range: (usize, usize), + + /// Number of G-tree nodes excluded while producing the current analysis-set snapshot because their suffix width was below `MIN_TRACKER_DIM`. + /// + /// Recomputed with selection rather than accumulated over the sentinel's lifetime. A persistently non-zero count may indicate the `split_threshold` is too low for the traffic mix. + /// See ADR-S-011. + pub degenerate_cells_skipped: usize, +} + +// ─── Member score ─────────────────────────────────────────── + +/// Per-cell scores from a coordination context's model. +/// +/// Each entry corresponds to one competitive cell in the coordination +/// group. The `cell_start`/`cell_end`/`cell_depth` fields identify +/// which cell produced this score vector. +#[derive(Debug, Clone, Copy)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(bound = "C: serde::Serialize + serde::de::DeserializeOwned"))] +pub struct MemberScore { + /// Lower bound of the scored cell's dyadic interval (inclusive). + pub cell_start: C, + + /// Upper bound of the scored cell's dyadic interval, exclusive everywhere except at the top of the domain: the scored cell holds `cell_end` when that value is the domain's last, which is what a coordinate type unable to represent `2^N` at its full width leaves behind when its domain maximum takes the bound's place; a type that represents `2^N` excludes the bound at every width. + pub cell_end: C, + + /// G-tree depth of the scored cell. + pub cell_depth: u32, + + /// Novelty score for this cell's contribution. + pub novelty: f64, + + /// Displacement score for this cell's contribution. + pub displacement: f64, + + /// Surprise score for this cell's contribution. + pub surprise: f64, + + /// Coherence score for this cell's contribution. + pub coherence: f64, + + /// Z-score of `novelty` against the coordination baseline. + pub novelty_z: f64, + + /// Z-score of `displacement` against the coordination baseline. + pub displacement_z: f64, + + /// Z-score of `surprise` against the coordination baseline. + pub surprise_z: f64, + + /// Z-score of `coherence` against the coordination baseline. + pub coherence_z: f64, +} diff --git a/packages/sentinel/src/sentinel/README.md b/packages/sentinel/src/sentinel/README.md new file mode 100644 index 000000000..3cdfc03a5 --- /dev/null +++ b/packages/sentinel/src/sentinel/README.md @@ -0,0 +1,38 @@ +## Unit test matrix · `tab:sentinel:sentinel-unit-test-matrix` + +**Table (Unit test matrix)** + +| Test | Area | Claim | +|------|------|-------| +| (`test:unit:a-scoreless-coordination-pass-keeps-contexts-while-membership-stands`) | coordination | A coordination pass with no cell scores fires no context, but it preserves every context whose two subtrees still contain online competitive cells. Learned coordination state belongs to that membership and survives a quiet batch. | +| (`test:unit:a-one-sided-scoring-pass-keeps-the-membership-context`) | coordination | When only one region contributes a score, the shared context does not fire but remains allocated because the competitive member in the quiet subtree is still online. A later two-sided batch resumes the same learned context instead of warming a replacement. | +| (`test:unit:a-context-is-destroyed-when-a-subtree-leaves-membership`) | coordination | A context is destroyed as soon as one of its subtrees contains no online competitive member. Retention follows the group that owns the learned state, so a topology that no longer represents that group cannot keep its tracker. | +| (`test:unit:active-counts-exclude-cells-still-warming`) | health | The active tracker counts describe the cells that are online, not the cells the selector has decided to pay for. The two differ whenever a cell is still warming: the selection names it, but it has no tracker yet and cannot have produced anything, so counting it active reports a cell as working for as many batches as its warm-up lasts. Reading the figures from the selection made that the normal case under background warming, where the drain no longer happens inside the ingest that created the cell. | +| (`test:unit:the-semi-internal-count-follows-the-graph`) | health | The semi-internal count is read from the graph rather than left at a constant. Semi-internal nodes are a reachable state — an eviction that takes one child of a pair leaves the parent with a single subdivided half — and they sit on the contour, so a figure fixed at zero is wrong exactly when the structure is being reshaped, which is when a reader would look at it. | +| (`test:unit:warming-cell-is-ready-when-complete`) | staging | A cell leaves the warming set only once it has served the whole schedule its depth called for: one round short and it is still warming. The target is a promise about how much noise the tracker's baselines were built from, so honouring it partially would put a half-formed reference into the live set. | +| (`test:unit:warming-cell-is-ready-when-over-target`) | staging | cites (`claim:staging:a-cell-is-ready-only-once-it-has-completed-its-target-rounds`) | +| (`test:unit:enqueue-zero-rounds-goes-directly-to-ready`) | staging | A cell whose schedule asks for no noise at all never enters the warming set: it is placed straight on the ready queue and can be promoted on the next pass. Warming is work done only where the schedule says it is needed, so a zero-round cell costs nothing to stage and waits for nothing. | +| (`test:unit:enqueue-with-rounds-goes-to-warming`) | staging | A cell with rounds still to serve waits in the warming set, out of the ready queue — and it counts as present in the staging area from the moment it is enqueued. Presence is what stops the reconciler enqueuing the same cell twice while its warm-up is still outstanding. | +| (`test:unit:take-ready-empties-queue`) | staging | Collecting the ready cells hands them over whole and leaves the queue empty behind them. Promotion moves ownership rather than copying it, so a cell cannot be promoted into the live map twice however often the observation path drains the staging area. | +| (`test:unit:remove-from-warming`) | staging | Eviction finds a cell wherever it currently sits — here part-warmed, with rounds still outstanding — and afterwards the area no longer reports it as present. A cell that has left the analysis set must stop consuming warming effort immediately rather than at the end of its schedule. | +| (`test:unit:remove-from-ready`) | staging | cites (`claim:staging:eviction-reaches-a-cell-in-whichever-state-it-is-being-held`) | +| (`test:unit:remove-nonexistent-returns-false`) | staging | Asking to evict a cell the area never held is answered with "nothing removed" rather than a fault. Eviction is driven by graph rebalancing, which knows what left the analysis set but not which of those cells were ever staged, so a miss has to be an ordinary outcome. | +| (`test:unit:clear-empties-everything`) | staging | Clearing empties every holding at once — warming, ready and in-flight alike — so that after a reset the total is zero rather than a residue in whichever state escaped the sweep. A sentinel being reset must not promote cells warmed against a model it has just discarded. | +| (`test:unit:warm-one-batch-completes-single-round-cell`) | staging | A warming step that completes a cell's last round finalises it and moves it to the ready queue in the same step, so the cell is never left sitting complete but unclaimed. Finalisation is where the drift reference is seeded from the baselines the noise just built and the accumulated evidence is cleared — the injected noise must shape what counts as normal without itself counting as anomalous history. | +| (`test:unit:warm-one-batch-returns-false-when-empty`) | staging | A warming step with nothing to warm reports that it did no work rather than failing or fabricating a round. The background thread drives this call in a loop, so "no work" is the signal that lets it go idle instead of spinning. | +| (`test:unit:warm-one-batch-incremental-progress`) | staging | Warming advances one round per step: a cell needing several rounds stays in the warming set across the intermediate calls and moves to ready only on the step that finishes it. Splitting the work this way is the whole point of deferring warm-up — the cost of bringing a new cell online is spread over many steps instead of landing inside one observation call. | +| (`test:unit:warm-one-batch-picks-highest-volume`) | staging | When several cells are waiting, the step spends its round on the one carrying the most traffic, leaving the quieter cell still warming. Volume is the cached importance of the backing graph node, so the cells the host is most likely to be asking about come online first, and — because a busy ancestor outweighs its own descendants — ancestors tend to arrive before the cells beneath them. | +| (`test:unit:warm-one-batch-warms-the-ancestor-before-the-cell-beneath-it`) | staging | Equal volumes send the round to the shallower cell, whichever identifier that cell happens to hold. The tie is the ordinary case for a path node whose whole accumulation is the single cell below it, and neither half of the identifier reading survives it. A comparison on volume alone keeps the last of the equal maxima, which is the largest identifier, and in the ordinary allocation order that is the cell beneath. Resolving the tie toward the smallest identifier instead inverts the rule the other way round, because the arena hands a freed slot out again and an ancestor can hold the larger identifier while the cell created beneath it holds the smaller. Depth is what carries the rule through both, and spending the round on the descendant lets it finish and be promoted while the chain above it is still warming — the gap the volume ordering exists to close. This step and the two drains serve one queue and must not disagree about which cell comes next. | +| (`test:unit:contains-checks-both-warming-and-ready`) | staging | Presence is answered across every state a staged cell can occupy: a cell still warming and a cell already waiting to be promoted both answer yes. The caller asking is deciding whether a cell needs creating, and it must not be told "absent" merely because the cell has moved on within the staging area. | +| (`test:unit:take-highest-priority-moves-to-in-flight`) | staging | Checking a cell out for background work takes the busiest waiting cell and marks it in flight, leaving the others warming; while it is away it still counts as present in the staging area. That is what makes the expensive noise injection safe to do without holding the lock: the main thread can see the cell is spoken for even though the warming map no longer holds it. | +| (`test:unit:equal-volumes-take-the-shallower-cell-first`) | staging | Equal volumes resolve to the shallower cell rather than the deeper one. A tie is the ordinary case for a pair of siblings the moment they are created, and the rule the queue exists to serve is that a busy ancestor is warmed before the cells beneath it. Identifiers cannot carry that rule on their own: the graph's arena hands a freed slot out again, so a cell created into a recycled slot holds a smaller identifier than an ancestor allocated before it. This path and the synchronous drain are two ways of serving one queue, so they must not disagree about which cell comes next. | +| (`test:unit:a-restored-descendant-does-not-overtake-its-warming-ancestor`) | staging | cites (`claim:staging:equal-volumes-resolve-to-the-shallower-cell-so-both-drains-agree`) | +| (`test:unit:the-synchronous-drain-warms-a-restored-descendants-ancestor-first`) | staging | cites (`claim:staging:equal-volumes-resolve-to-the-shallower-cell-so-both-drains-agree`) | +| (`test:unit:an-in-flight-cell-still-counts-as-a-competitive-target`) | staging | A cell checked out for background warming is still a warming cell, so it still counts among the competitive targets being warmed. Checking a cell out is how the expensive work is done off the lock, not a change in what the cell is; a count that dropped it would fall precisely when the work was happening, understating what is in progress by the number of cells actually in progress. The flag is recorded at checkout, so counting it needs nothing from a cell another thread is holding. | +| (`test:unit:an-in-flight-ancestor-cell-is-not-a-competitive-target`) | staging | cites (`claim:staging:a-cell-checked-out-for-warming-still-counts-among-the-competitive-targets`) | +| (`test:unit:a-newly-queued-cell-carries-its-volume`) | staging | A cell joining the queue carries its volume with it instead of waiting for a later pass to supply one. The queue is served highest volume first, so a cell admitted at zero is indistinguishable from a cell with no traffic behind it, and a field of zeroes is decided entirely by the tie-breaks — shallower depth first, then the smaller identifier — so which cell is warmed first would be settled by where the cells sit in the tree rather than by the traffic behind them. Refreshing the cached volumes before the new cells are added rather than after leaves every one of them in exactly that state until some later pass happens to refresh again. | +| (`test:unit:return-warming-restores-cell`) | staging | A cell handed back unfinished rejoins the warming set and stops being in flight, with its accumulated rounds intact. Background warming can therefore be interrupted between rounds — the thread need not carry a cell to completion once it has taken it. | +| (`test:unit:a-returned-cell-carries-the-volume-the-graph-has-now`) | staging | A cell handed back rejoins the queue at the volume the graph has now, not the one it carried out. The refresh runs on the main thread while the worker holds the cell, and the checkout spans exactly the noise injection — the expensive part of the pass, and so the part an ingest is most likely to overlap. Restoring the carried volume would leave the busiest cell in the area queued at its pre-ingest importance, and the next checkout — the one decision the cached volume exists to make — would go to a rival the traffic has already passed. | +| (`test:unit:finish-warming-moves-to-ready`) | staging | A cell handed back finished joins the ready queue instead of the warming set, and is no longer in flight. Which of the two return paths the background thread takes is what decides the cell's fate, so completion is declared by the worker that did the rounds rather than re-derived by the staging area. | +| (`test:unit:eviction-of-in-flight-cell-discards-on-return`) | staging | A cell evicted while a background thread was working on it is discarded when it comes back, not resurrected: the eviction sweep removes its in-flight mark, and a return with no mark to clear keeps nothing. The warming work already spent is lost, which is the deliberate trade — a cell that has left the analysis set must not reappear in it because a thread happened to be holding it. | +| (`test:unit:gnode-set-includes-all-states`) | staging | cites (`claim:staging:presence-is-answered-across-every-state-a-staged-cell-can-occupy`) | \ No newline at end of file diff --git a/packages/sentinel/src/sentinel/cusum.rs b/packages/sentinel/src/sentinel/cusum.rs new file mode 100644 index 000000000..6baf6c126 --- /dev/null +++ b/packages/sentinel/src/sentinel/cusum.rs @@ -0,0 +1,144 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +//! CUSUM (Cumulative Sum) drift accumulator. +//! +//! Detects sustained upward drift of batch mean scores away from a +//! slow EWMA reference. This is a one-sided Page's test: the +//! accumulator grows when the fast signal consistently exceeds the +//! slow baseline by more than a noise allowance, and resets to zero +//! when the deviation reverses. +//! +//! See `docs/algorithm.md` §ALGO S-6.3 for the full specification. +//! +//! Pure `f64` arithmetic — no `faer` dependency. + +use crate::ewma::EwmaStats; +use crate::report::{BaselineSnapshot, CusumSnapshot}; + +/// One-sided CUSUM accumulator with a slow EWMA reference. +/// +/// Each scoring axis owns one of these. It pairs a slow EWMA +/// baseline (longer memory than the fast baseline in [`EwmaStats`]) +/// with a cumulative sum that builds evidence of sustained drift. +/// +/// The sentinel reports the raw accumulator value; the host decides +/// what level of accumulated drift warrants action. +#[derive(Debug, Clone)] +pub struct CusumAccumulator { + /// Slow EWMA baseline — the reference the CUSUM measures drift from. + slow: EwmaStats, + + /// The cumulative sum. Non-negative (clamped at zero). + accumulator: f64, + + /// Batches since the last reset (including post-noise reset). + steps_since_reset: u64, +} + +impl CusumAccumulator { + /// Create a new CUSUM accumulator with the given slow decay factor. + #[must_use] + pub const fn new(slow_decay: f64) -> Self { + Self { + slow: EwmaStats::new(slow_decay), + accumulator: 0.0, + steps_since_reset: 0, + } + } + + /// Update the accumulator with a batch of per-sample scores. + /// + /// 1. Computes the gap: `batch_mean − slow_mean − κ·√slow_var`. + /// 2. Accumulates: `S = max(0, S + gap)`. + /// 3. Feeds the scores to the slow EWMA baseline. + /// + /// The gap is computed *before* updating the slow baseline so the + /// reference reflects the prior state — matching the principle + /// that scoring precedes evolution. + /// + /// `allowance_sigmas` is `κ_σ` from config — the noise tolerance + /// in units of slow-baseline standard deviation. + #[cfg(test)] + pub fn update(&mut self, scores: &[f64], batch_mean: f64, allowance_sigmas: f64, clip_sigmas: f64) { + let slow_mean = self.slow.mean(); + let slow_std = self.slow.variance().sqrt(); + let allowance = allowance_sigmas * slow_std; + + let gap = batch_mean - slow_mean - allowance; + self.accumulator = (self.accumulator + gap).max(0.0); + + // Now update the slow baseline with this batch. + self.slow.update(scores, clip_sigmas); + + self.steps_since_reset += 1; + } + + /// Reset the accumulator to zero. + /// + /// Called after noise injection (§ALGO S-11.4) and optionally by the host + /// after acknowledging a regime change. + pub const fn reset(&mut self) { + self.accumulator = 0.0; + self.steps_since_reset = 0; + } + + /// Destroy all state — return to the freshly-constructed state. + /// + /// Resets the accumulator *and* the slow EWMA baseline to cold. + /// Used when the scoring axis this accumulator tracks ceases to + /// exist (e.g. coherence when rank drops below 2). + pub const fn reset_cold(&mut self) { + self.slow.reset_cold(); + self.accumulator = 0.0; + self.steps_since_reset = 0; + } + + /// Seed the slow EWMA from an external fast EWMA baseline. + /// + /// Closes the fast-slow gap after noise injection (ADR-S-013 + /// §6b, Option C). The slow EWMA's mean and variance are set + /// to the fast EWMA's current values so the CUSUM starts with + /// the two baselines in agreement — eliminating the monotonic + /// false-drift accumulation caused by the slow EWMA's 693-step + /// half-life being unable to catch up to the fast EWMA. + /// + /// Should be called **after** noise injection completes and + /// **before** `reset()`. + pub const fn seed_slow_from(&mut self, fast: &EwmaStats) { + self.slow.seed_from(fast); + } + + /// Update the accumulator with **pre-filtered** samples. + /// + /// Identical to [`update`](Self::update) except the slow EWMA + /// receives pre-filtered values via `update_raw()` instead of + /// applying its own clip filter. The CUSUM gap still uses + /// `raw_batch_mean` (the pre-clip mean of the full batch). + /// + /// Used by the shared-filter pipeline (§ALGO S-6.1.1 step 5). + pub fn update_filtered(&mut self, filtered: &[f64], raw_batch_mean: f64, allowance_sigmas: f64) { + let slow_mean = self.slow.mean(); + let slow_std = self.slow.variance().sqrt(); + let allowance = allowance_sigmas * slow_std; + + let gap = raw_batch_mean - slow_mean - allowance; + self.accumulator = (self.accumulator + gap).max(0.0); + + self.slow.update_raw(filtered); + self.steps_since_reset += 1; + } + + /// Snapshot for inclusion in reports. + #[must_use] + pub const fn snapshot(&self) -> CusumSnapshot { + CusumSnapshot { + accumulator: self.accumulator, + slow_baseline: BaselineSnapshot { + mean: self.slow.mean(), + variance: self.slow.variance(), + }, + steps_since_reset: self.steps_since_reset, + } + } +} diff --git a/packages/sentinel/src/sentinel/mod.rs b/packages/sentinel/src/sentinel/mod.rs new file mode 100644 index 000000000..c23f006dd --- /dev/null +++ b/packages/sentinel/src/sentinel/mod.rs @@ -0,0 +1,2463 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +//! The sentinel engine — orchestration, tracking, and drift detection. +//! +//! This module contains the core [`SpectralSentinel`] orchestrator and +//! its internal machinery. Only the orchestrator is part of the public +//! API; the subspace tracker and CUSUM accumulator are implementation +//! details. +//! +//! # Automatic noise injection (§ALGO S-11, §ALGO S-11.6) +//! +//! Every newly created tracker is automatically warmed with synthetic +//! noise before it receives any real observations. The root tracker +//! is warmed at construction; cells created during investment set +//! reconciliation (§ALGO S-8.5) are enqueued into a **staging area** +//! for deferred warm-up (§ALGO S-11.6). When `background_warming` is +//! enabled, a background thread drains the staging area asynchronously; +//! otherwise warm-up runs synchronously within `reconcile_analysis_set()`. +//! Warming cells hold **investment slots** in $\mathcal{I}$ but not +//! **production slots** in $\mathcal{A}$ (ADR-S-019). +//! Coordination contexts are warmed via Gamma-sampled synthetic +//! score vectors (§ALGO S-11.7.2). No manual injection API exists — the +//! sentinel owns the injection lifecycle entirely (ADR-S-007). +//! +//! # Quick start +//! +//! ``` +//! use torrust_sentinel::SentinelConfig; +//! use torrust_sentinel::Sentinel128; +//! +//! let cfg = SentinelConfig:: { +//! analysis_k: 4, +//! ..SentinelConfig::default() +//! }; +//! // Root tracker is auto-warmed at construction. +//! let mut sentinel = Sentinel128::new(cfg).unwrap(); +//! +//! let values: Vec = vec![ +//! 0xF000_0000_0000_0000_0000_0000_0000_0001, +//! 0xF000_0000_0000_0000_0000_0000_0000_0002, +//! 0x1000_0000_0000_0000_0000_0000_0000_0003, +//! ]; +//! let report = sentinel.ingest(&values); +//! +//! // Root cell always receives all observations (as an ancestor). +//! assert!(!report.ancestor_reports.is_empty()); +//! ``` +//! +//! # Test index +//! +//! | Test | Area | Claim | +//! |------|------|-------| +//! | [`a_scoreless_coordination_pass_keeps_contexts_while_membership_stands`] | coordination | A coordination pass with no cell scores fires no context, but it preserves every context whose two subtrees still contain online competitive cells. Learned coordination state belongs to that membership and survives a quiet batch. | +//! | [`a_one_sided_scoring_pass_keeps_the_membership_context`] | coordination | When only one region contributes a score, the shared context does not fire but remains allocated because the competitive member in the quiet subtree is still online. A later two-sided batch resumes the same learned context instead of warming a replacement. | +//! | [`a_context_is_destroyed_when_a_subtree_leaves_membership`] | coordination | A context is destroyed as soon as one of its subtrees contains no online competitive member. Retention follows the group that owns the learned state, so a topology that no longer represents that group cannot keep its tracker. | +//! | [`active_counts_exclude_cells_still_warming`] | health | The active tracker counts describe the cells that are online, not the cells the selector has decided to pay for. The two differ whenever a cell is still warming: the selection names it, but it has no tracker yet and cannot have produced anything, so counting it active reports a cell as working for as many batches as its warm-up lasts. Reading the figures from the selection made that the normal case under background warming, where the drain no longer happens inside the ingest that created the cell. | +//! | [`the_semi_internal_count_follows_the_graph`] | health | The semi-internal count is read from the graph rather than left at a constant. Semi-internal nodes are a reachable state — an eviction that takes one child of a pair leaves the parent with a single subdivided half — and they sit on the contour, so a figure fixed at zero is wrong exactly when the structure is being reshaped, which is when a reader would look at it. | + +pub mod cusum; +pub mod staging; +pub mod tracker; +pub mod warming_thread; + +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::{Arc, Mutex}; +use std::time::Instant; + +use rand::rngs::SmallRng; +use rand::{RngExt, SeedableRng}; +use rand_distr::{Distribution, Gamma}; +use torrust_mudlark::{Config as GvConfig, Coordinate, GNodeId, GvGraph, Inspectable, StructuralMutationCounts}; + +use self::tracker::SubspaceTracker; +use crate::{ + AnalysisSet, AxisBaselineSnapshots, BatchReport, CellInspection, CellReport, CentredBits, ClipPressureDistribution, + ConfigError, ConfigErrors, ContourSnapshot, CoordinationHealth, CoordinationReport, GeometryDistribution, HealthReport, + MaturityDistribution, MemberScore, RankDistribution, SentinelConfig, +}; + +// ─── Internal state types ─────────────────────────────────── + +/// Per-cell analysis state. +/// +/// Each cell in the investment set $\mathcal{I}$ owns a single +/// `SubspaceTracker` operating on suffix bits at the cell's G-tree +/// depth. The root cell ($d = 0$) has `width = 128`; a cell at +/// depth $d$ has `width = 128 - d`. +/// +/// Online cells (the producing sets $\mathcal{A}$/$\mathcal{A}^*$) +/// receive real observations; warming cells receive only synthetic +/// noise (ADR-S-019). +pub struct CellState { + /// The subspace tracker for this cell. + pub tracker: SubspaceTracker, + + /// G-tree depth of this cell. + pub depth: u32, + + /// Suffix width: `128 - depth`. Cached for convenience. + pub width: usize, + + /// Lower bound of the scoring interval (inclusive). + /// + /// At full integer width, interior scoring bounds are the successors of + /// the backing G-node bounds so the routing prefix is constant. + pub start: C, + + /// Upper bound of the scoring interval, exclusive everywhere except at the + /// top of the domain: the topmost cell keeps `end` inside its interval + /// when `end` is the domain's final value. A coordinate type that cannot + /// represent `2^N` at its full width leaves the domain with such a final + /// value, its domain maximum standing in for the absent bound; one that + /// can represent `2^N` has no final value and excludes the bound at every + /// width. + pub end: C, + + /// Whether this cell is competitively selected (vs ancestor-only). + pub is_competitive: bool, +} + +/// Per-G-node coordination state (§ALGO S-7.1). +/// +/// Active while both subtrees of this G-node contain online competitive +/// cells. A context fires only when both subtrees also contribute scores +/// in the current batch. +struct CoordContext { + /// Subspace tracker at w = 4, using `cusum_coord_slow_decay`. + tracker: SubspaceTracker, + + /// Running EWMA mean of the 4D cell-score input vectors (§ALGO S-7.3). + /// Used for centring before feeding the coordination tracker. + running_mean: [f64; 4], + + /// Whether the running mean has been initialised with at least + /// one batch. Cold-start: first batch sets + /// `running_mean = colmeans(O_g)` (§ALGO S-7.3). + warm: bool, +} + +// ─── SpectralSentinel ─────────────────────────────────────── + +/// Hierarchical online subspace anomaly detector. +/// +/// The sentinel maintains one `SubspaceTracker` per analysis cell +/// in the full analysis set. Cells are selected by the analysis +/// selector (§ALGO S-8.1) from the G-V Graph's V-Tree. +/// +/// A second tier of coordination trackers — one per active G-tree +/// internal node whose subtrees both contribute competitive cells — +/// analyses cross-cell score patterns for coordinated anomalies +/// (§ALGO S-7.1). +/// +/// # Design principle +/// +/// **The sentinel measures; the host decides.** +/// +/// [`ingest`](Self::ingest) returns a [`BatchReport`] containing raw +/// statistical measurements. The host reads the report and applies +/// its own policy to decide what (if anything) to do. +/// +/// **Feed-forward invariant (ADR-S-002).** The G-V Graph receives +/// only `observe(v, 1u64)` per raw input value during `ingest()`. +/// Anomaly scores and derived signals never flow back into the +/// graph's importance accounting. The host controls temporal policy +/// via `decay()`; the analysis tier has no influence on spatial +/// resolution. +pub struct SpectralSentinel +where + C: Coordinate + crate::CentredBitSource, + V: Inspectable + torrust_mudlark::Attenuatable, +{ + config: SentinelConfig, + + /// The G-V Graph spatial substrate (§ALGO S-3.2). + /// + /// Owns the adaptive spatial partition of the observation domain. + graph: GvGraph, + + /// Per-cell analysis state, keyed by `GNodeId`. + /// + /// `BTreeMap` for deterministic iteration order (ADR-S-005). + /// Contains one entry per **online** cell in the producing full + /// set $\mathcal{A}^*$. Warming cells are in the staging area + /// (ADR-S-019). + cells: BTreeMap>, + + /// The current analysis set (competitive + ancestors). + /// + /// Recomputed after every observation pass (ADR-S-006). + analysis_set: AnalysisSet, + + /// The root tracker's `GNodeId`. Cached for fast access. + /// Permanent — never destroyed (§ALGO S-8.4). + root_gnode: GNodeId, + + /// Per-G-node coordination contexts (§ALGO S-7.1). + /// + /// Keyed by `GNodeId` of internal G-nodes whose subtrees + /// contain competitive cells in both left and right branches. + /// `BTreeMap` for deterministic iteration order (ADR-S-005). + coordination: BTreeMap, + + /// Monotonically increasing counter, incremented once per + /// `ingest` call. + batch_counter: u64, + + /// Total real (non-noise) observations across the sentinel's + /// lifetime. + lifetime_observations: u64, + + /// Number of G-tree nodes excluded while producing the current analysis-set snapshot because their suffix width was below `MIN_TRACKER_DIM` (ADR-S-011). + degenerate_cells_skipped: usize, + + /// Persistent RNG for noise injection (§ALGO S-11.1). + noise_rng: SmallRng, + + /// Staging area for cells undergoing deferred noise warm-up + /// (§ALGO S-11.6). + staging: Arc>>, + + /// Handle to the background warming thread (Step 3). + warming_thread: Option>, + + /// Spatial mutation totals at the previous report boundary. + /// + /// Construction and graph replacement snapshot the new graph immediately, + /// so a report never attributes replacement to the reporting interval. + prev_structural_mutation_counts: StructuralMutationCounts, +} + +impl SpectralSentinel +where + C: Coordinate + crate::CentredBitSource, + V: Inspectable + torrust_mudlark::Attenuatable, +{ + /// Create a new sentinel with the given configuration. + /// + /// Validates the configuration and creates the root tracker. + /// No other cells are created until the first + /// [`ingest`](Self::ingest) call triggers analysis set computation. + /// + /// # Errors + /// + /// Returns [`ConfigErrors`] if the + /// configuration violates any invariant (see + /// [`SentinelConfig::validate`]), or if the coordinate width `N` lies + /// outside the range the observation path can model — narrower than the + /// smallest dimension a subspace tracker can work in, or wider than the + /// centred bit vector that feeds it can carry. Every such fault is + /// collected in one pass. The graph type separately enforces + /// `N <= C::BITS` at compile time; that type-level relationship can never + /// reach this runtime error channel. + /// + /// Also returns [`ConfigErrors`] when the configuration asked for + /// background warming and the environment refused the thread it runs on. + /// That fault arrives alone rather than among the others: the thread is + /// requested only once the configuration has been accepted, so by the + /// time it can be refused there is nothing left to collect it with. + pub fn new(config: SentinelConfig) -> Result { + // The root tracker spans the whole coordinate width, so a width the + // tracker cannot model is refused here rather than left to build a + // root whose lone basis vector spans its own space and therefore + // reports no novelty at all. The same holds at the other end: the + // bridge that turns a coordinate into centred bits is open to a + // coordinate type of any width, and the spatial layer asks only that + // the width fit that type, so a wider type with a wider N would build + // trackers of that width over a vector that can never carry it — the + // dimensions past the vector's end arriving as zeros, which centred + // bits never are, and being modelled as though the stream had + // produced them. This is the only place either bound can be judged: + // the width is a parameter of the type, not a field of the + // configuration, so validation of the configuration alone can never + // see it. The graph constructor separately enforces N <= + // C::BITS at compile time, so it is not a runtime validation case. + let mut errors = Vec::new(); + if (N as usize) < crate::MIN_TRACKER_DIM { + errors.push(ConfigError::TrackerDimensionTooSmall { + width: N, + minimum: crate::MIN_TRACKER_DIM, + }); + } + if (N as usize) > crate::MAX_TRACKER_DIM { + errors.push(ConfigError::TrackerDimensionTooLarge { + width: N, + maximum: crate::MAX_TRACKER_DIM, + }); + } + if let Err(ConfigErrors(config_errors)) = config.validate() { + errors.extend(config_errors); + } + if !errors.is_empty() { + return Err(ConfigErrors(errors)); + } + + // ── G-V Graph construction ────────────────────── + let gv_config = GvConfig { + split_threshold: config.split_threshold, + depth_create: config.d_create, + depth_evict: config.d_evict, + budget: Some(config.budget), + alpha_relax: 0.75, + bounded_eviction: true, + }; + let graph: GvGraph = GvGraph::new(gv_config); + + // ── Persistent RNG (§ALGO S-11.1) ───────────────── + let mut noise_rng = config + .noise_seed + .map_or_else(|| SmallRng::from_rng(&mut rand::rng()), SmallRng::seed_from_u64); + + // ── Root tracker (permanent, §ALGO S-8.4) ───────── + let root_gnode = graph.g_root(); + let mut root_cell = CellState { + tracker: SubspaceTracker::new(N as usize, &config, config.cusum_slow_decay), + depth: 0, + width: N as usize, + start: C::zero(), + end: C::domain_max(N), + is_competitive: false, + }; + + // Auto noise injection on root tracker (§ALGO S-11.1). + let root_rounds = config.noise_schedule.rounds_for_depth(0); + if root_rounds > 0 { + inject_noise_into_cell(&mut root_cell, root_rounds as usize, config.noise_batch_size, &mut noise_rng); + } + + let mut cells = BTreeMap::new(); + cells.insert(root_gnode, root_cell); + + // ── Initial analysis set (just root) ──────────── + let analysis_set = AnalysisSet::recompute::(&graph, config.analysis_k, config.analysis_depth_cutoff); + + let staging = Arc::new(Mutex::new(staging::StagingArea::::new())); + + // ── Background warming thread (Step 3) ───────── + // + // A refused thread is reported rather than raised. Construction is + // the one place in this engine's life where the caller is still + // holding an error channel, and the alternative — aborting the + // process the sentinel was built to protect, over a resource limit + // that has nothing to do with the configuration's correctness — is + // exactly what the crate's policy on panics reserves for programmer + // error. Degrading quietly to synchronous warming is not open here + // either: a caller that can be told what it got should be. + let warming_thread = if config.background_warming { + match warming_thread::WarmingThreadHandle::::spawn(&staging, config.noise_batch_size, config.noise_seed) { + Ok(handle) => Some(handle), + Err(refusal) => { + return Err(ConfigErrors(vec![ConfigError::BackgroundWarmingThreadUnavailable { + reason: refusal.to_string(), + }])); + } + } + } else { + None + }; + + let init_structural_mutation_counts = graph.structural_mutation_counts(); + + Ok(Self { + config, + graph, + cells, + analysis_set, + root_gnode, + coordination: BTreeMap::new(), + batch_counter: 0, + lifetime_observations: 0, + degenerate_cells_skipped: 0, + noise_rng, + staging, + warming_thread, + prev_structural_mutation_counts: init_structural_mutation_counts, + }) + } + + /// Process a batch of coordinate observations and return a full statistical report. + /// + /// Each value is fed to the G-V Graph, then routed to every + /// analysis cell whose interval contains it. Multi-scale delivery + /// ensures ancestor cells also receive the observation (§ALGO S-9.3). + /// + /// The domain is `[0, 2^N)` (§ALGO S-2.1). A value outside it is not an + /// observation of this sentinel's domain and is dropped here, before the + /// spatial layer, the encoding and the routing: it raises no total, moves + /// no partition and reaches no tracker. A batch that loses values this way + /// emits one warning naming how many went; a batch with nothing left + /// produces an empty report. + /// + /// An empty input slice produces an empty report. + /// + /// # Panics + /// + /// Panics if the internal staging mutex is poisoned. + pub fn ingest(&mut self, values: &[C]) -> BatchReport { + // ── Arrival stamp (§ALGO S-14.1) ────────────────── + // The batch arrives whole at this call boundary, so this one + // instant is the arrival of every observation in it — the + // oldest included. It is taken first, ahead of the domain + // decision below, because that decision is work this call does + // on the batch: stamped after it, the age would leave the + // filtering pass out and report a figure short of the + // boundary-to-emission interval §ALGO S-14.1.1 promises. A + // batch the decision empties returns before the stamp is read, + // spending one clock read and reporting no age at all; on + // every other path it is read back at emission and reported as + // the batch's age. The clock is the sentinel's own monotonic + // one and is never compared with anybody else's, which is what + // keeps the figure free of skew. + let batch_arrival = Instant::now(); + + // ── The domain decision (§ALGO S-2.1) ─────────── + // Taken once, here, because the three layers below read a value + // differently and none of them can be the place that decides. The + // spatial layer routes by comparison against interval midpoints, so a + // coordinate at or above the domain's exclusive bound goes rightward + // at every level and accumulates in the topmost terminal, one below + // the origin goes leftward at every level and accumulates in the + // bottom-most, and one that compares false against every midpoint, as + // a NaN does, accumulates in the topmost with the first; no terminal's + // interval contains any of them. The encoder reads the low N bits, so + // it would present such a value as the in-domain value it is congruent + // to (§ALGO S-2.3). The interval scan in `route_and_score`, meanwhile, + // matches no cell at all, not even the root, so no tracker is shown + // it. Whichever layer were left to its own reading, the counts would + // part: the lifetime total would record an arrival that the trackers + // never saw, which is the divergence the mandatory delivery of + // §ALGO S-9.3 exists to rule out. + let retained = Self::retain_in_domain(values); + let values: &[C] = retained.as_deref().unwrap_or(values); + + // ── Early return on empty input ───────────────── + // A batch left empty by the domain decision arrives here too, and is + // the same non-event as a batch that was empty on arrival. + if values.is_empty() { + return self.empty_report(); + } + + // ── Observation algorithm ────────────────────── + // Steps 0–6 correspond to §ALGO S-9.1. + // The implementation reorders Step 1 (encoding) to just + // before Step 4 (scoring), since the spatial layer routes + // on raw coordinates and does not need centred bits. + + // ── Step 0: Promote ready cells (§ALGO S-11.6.3) ──── + // Cells that completed background warm-up since the last + // ingest are moved into the live cells map. In the + // synchronous transitional version (Step 2) this promotes + // cells from the drain at the end of reconcile_analysis_set(); + // once the background thread is introduced (Step 3) it will + // catch cells that finished between ingest calls. + self.promote_ready_cells(); + + // ── Step 2: G-V Graph observation (§ALGO S-3.3) ─── + // [Step 1 (encoding) deferred to just before scoring.] + let unit_delta = V::from_f64(1.0); + + #[cfg(debug_assertions)] + let pre_observe_sum = self.graph.total_sum(); + + for &value in values { + self.graph.observe(value, unit_delta); + } + + #[cfg(debug_assertions)] + { + // Compare in the accumulator's own domain rather than through a + // floating-point projection. The projection is lossy by its own + // documentation, and past the point where the spacing between + // representable values exceeds one it cannot express a difference + // of a single observation at all: an exactly correct total then + // lands more than the tolerance away from its projected + // expectation, and the assertion fires on arithmetic that was + // never wrong. Adding the unit delta once per observation + // reproduces exactly what the loop above did, so the comparison + // is against the accumulator's own notion of the sum. + let expected_sum = values.iter().fold(pre_observe_sum, |acc, _| acc.add(unit_delta)); + let actual_sum = self.graph.total_sum(); + debug_assert!( + actual_sum == expected_sum, + "feed-forward invariant violated: total_sum should increase by exactly n" + ); + } + + // ── Step 3: Analysis set reconciliation ───────── + self.reconcile_analysis_set(); + + // ── Steps 1+4: Encode and route/score ─────────── + let centred: Vec = values.iter().map(|v| CentredBits::from_coord(v, N)).collect(); + let cell_reports = self.route_and_score(values, ¢red); + + // ── Step 5: Coordination tier (§ALGO S-7.4) ─────── + let cell_scores = Self::assemble_cell_scores(&cell_reports); + let coordination_reports = self.propagate_coordination_from_root(&cell_scores); + + // ── Step 6: Assemble report ───────────────────── + self.batch_counter += 1; + + let count = values.len() as u64; + self.lifetime_observations += count; + + // ── (Step 6 continued: report assembly) ────────── + let (competitive, ancestors): (Vec<_>, Vec<_>) = cell_reports.into_iter().partition(|r| r.is_competitive); + + let (splits, net_removals, terminal_count) = self.take_structural_mutation_counts(); + + let contour = ContourSnapshot { + plateau_count: self.graph.plateaus().len(), + cell_count: terminal_count as usize + self.graph.semi_internal_count() as usize, + total_importance: self.graph.total_sum().to_f64_approx(), + splits_since_last_report: splits, + net_removals_since_last_report: net_removals, + }; + + let online: BTreeSet = self.cells.keys().copied().collect(); + let mut analysis_set_summary = self.analysis_set.summary_online(&online); + analysis_set_summary.degenerate_cells_skipped = self.degenerate_cells_skipped; + // Investment set = online cells + warming cells (ADR-S-019). The + // selection snapshot reports the size of the selection, which is all + // it can see; what is written here is the tracker population itself, + // which is what the report's field names. The two readings part + // wherever a tracker outlives its cell's membership of the selection, + // and the population is the one that is being paid for. + let warming_count = self.staging.lock().expect("staging mutex poisoned").total_count(); + analysis_set_summary.investment_set_size = self.cells.len() + warming_count; + + BatchReport { + cell_reports: competitive, + ancestor_reports: ancestors, + coordination_reports, + contour, + health: self.health(), + analysis_set_summary, + // Read last, so the age covers every part of the work this + // call did on the batch, the health snapshot included. + // Saturates rather than wrapping; the ceiling is hundreds + // of thousands of years away. + oldest_observation_age_micros: Some(u64::try_from(batch_arrival.elapsed().as_micros()).unwrap_or(u64::MAX)), + } + } + + /// Whether the value [`Coordinate::domain_max`] names at this width is + /// itself a value of the domain rather than the first value above it + /// (§ALGO S-2.1). + /// + /// `domain_max(n)` is documented as the exclusive upper bound `2^n`, with + /// one exception: for integer types at `n == Self::BITS` it returns the + /// type's maximum, because `2^BITS` is not representable there. At that + /// width the value it names is the last value of the domain — the + /// partition's topmost cell ends on it, and it must be admitted, routed + /// and delivered like any other observation. Every other implementor + /// keeps the bound exclusive at every width, the floats included: theirs + /// is `2^n` for every `n`, so at the full width `2^BITS` is a + /// representable value outside `[0, 2^N)`. + /// + /// The exception therefore belongs to the coordinate type and not to the + /// width, and `N == C::BITS` alone does not distinguish them: read that + /// way it admits a value the domain does not contain on a continuous type + /// and on any a host writes. The question is put to the trait rather than + /// to a list of types. The unit interval `[0, 1)` is indivisible at depth + /// zero for exactly the coordinates that subdivide down to single values, + /// which are the ones whose `domain_max` carries the substitution: + /// [`Coordinate::is_final`] is `width == 1` for the integers and + /// `depth >= n` for the continuous coordinates, which at depth zero is + /// false for every width a sentinel can be built at. + /// + /// The three layers that meet the top of the domain ask this one + /// question, so none of them can part from the others over it: the + /// boundary that admits a value, the interval scan that must then deliver + /// it, and the conversion from a spatial bound to a scoring bound. + fn domain_top_is_in_domain() -> bool { + N == C::BITS && C::is_final(C::zero(), C::from_u64(1), 0, N) + } + + /// Whether `value` lies in the observation domain `[0, 2^N)` (§ALGO S-2.1). + /// + /// Membership is decided by comparison against the domain's own bounds and + /// never inferred from the coordinate width. [`Coordinate::BITS`] is + /// documented as the width `N` is validated against; it promises nothing + /// about which values the type can hold. The coordinates this crate + /// implements the bridge for are unsigned, so for them a full width does + /// leave nothing outside the domain — but + /// [`CentredBitSource`](crate::CentredBitSource) is public and its set of + /// implementations is open, and `Coordinate` is implemented for the floats + /// and carries `is_nan`, so a host's own coordinate type may be signed and + /// NaN-capable. Reading `N == C::BITS` as "every representable value is in + /// the domain" would admit a negative, a NaN or an infinity on such a type, + /// and a comparison against the upper bound alone would admit a negative at + /// every narrower width as well — which is the one thing this boundary + /// exists to refuse. + /// + /// The comparison is the root cell's own containment test. The root spans + /// [`Coordinate::zero`] to `domain_max(N)` and is permanent (§ALGO S-8.4), + /// so deciding membership by the test the routing applies is what keeps the + /// two layers agreed for every coordinate type: whatever this admits, + /// `route_and_score` matches in at least one cell, and the mandatory + /// delivery of §ALGO S-9.3 holds however the host spells its coordinates. + /// + /// The upper bound is exclusive, with the one exception the coordinate + /// type makes for itself: where `domain_max(N)` names a value of the + /// domain rather than the first value above it, the cell ending at the top + /// of the domain owns that value and this admits it — the same reading + /// `route_and_score` takes through `owns_domain_top`, and the same + /// question both put to `domain_top_is_in_domain`. That is the integer + /// case at a width that fills the type, where `2^N` is not representable + /// and the maximum stands in for it. Wherever the bound stays exclusive — + /// every narrower width, and a continuous or host-written type at any + /// width, the full one included — it is a representable value outside the + /// domain and is refused here, because admitting it would raise both + /// totals and hand the topmost cell an arrival its interval does not + /// contain. + /// + /// A NaN needs no arm of its own: every comparison with a NaN is false, so + /// it is neither at nor above the origin and both arms refuse it. + fn in_domain(value: C) -> bool { + let domain_top = C::domain_max(N); + value >= C::zero() && (value < domain_top || (Self::domain_top_is_in_domain() && value == domain_top)) + } + + /// The batch restricted to the domain, or `None` when it is already the + /// whole batch. + /// + /// Returning the borrowed case as `None` keeps the ordinary batch — every + /// value in the domain, which is every batch of unsigned coordinates at a + /// width that fills the coordinate type — from being copied on the + /// observation path. + fn retain_in_domain(values: &[C]) -> Option> { + if values.iter().all(|&value| Self::in_domain(value)) { + return None; + } + + let retained: Vec = values.iter().copied().filter(|&value| Self::in_domain(value)).collect(); + tracing::warn!( + dropped = values.len() - retained.len(), + batch = values.len(), + width = N, + "coordinates outside the domain were dropped: they are counted in no total and reach no tracker" + ); + Some(retained) + } + + /// Produce an operational health snapshot of the sentinel. + /// + /// Summarises rank distribution and maturity across all active + /// trackers. Useful for dashboards. + /// + /// # Panics + /// + /// Panics if the internal staging mutex is poisoned. + #[must_use] + #[allow(clippy::too_many_lines)] + pub fn health(&self) -> HealthReport { + // The producing sets are the online ones. The analysis set is the + // investment set: it names every cell the selector has decided to pay + // for, including those still warming in staging, which have no tracker + // yet and cannot have produced anything. Reading the counts from it + // reported a cell as active for as many batches as its warm-up took. + // The cells map holds exactly the cells that are online, and each + // carries the competitive flag the last reconciliation gave it. + let active_trackers = self.cells.len(); + let competitive_count = self.cells.values().filter(|cell| cell.is_competitive).count(); + let ancestor_count = self + .cells + .iter() + .filter(|&(&gnode, cell)| !cell.is_competitive && gnode != self.root_gnode) + .count(); + let coord_health = self.coordination_health(); + + // Query staging area for investment/warming counts (ADR-S-019). + let (warming_total, warming_competitive) = { + let staging = self.staging.lock().expect("staging mutex poisoned"); + (staging.total_count(), staging.warming_competitive_count()) + }; + let investment_set_size = active_trackers + warming_total; + + if active_trackers == 0 { + return HealthReport { + total_g_nodes: self.graph.node_count() as usize, + semi_internal_count: self.graph.semi_internal_count() as usize, + active_trackers: 0, + active_competitive_trackers: 0, + active_ancestor_trackers: 0, + active_coordination_contexts: 0, + investment_set_size, + warming_trackers: warming_total, + warming_competitive_targets: warming_competitive, + lifetime_observations: self.lifetime_observations, + cells_tracked: 0, + rank_distribution: RankDistribution { + min: 0, + max: 0, + mean: 0.0, + }, + maturity_distribution: MaturityDistribution { + max_noise_influence: 0.0, + min_noise_influence: 0.0, + mean_noise_influence: 0.0, + cold_trackers: 0, + }, + geometry_distribution: GeometryDistribution { + novelty_saturated: 0, + novelty_saturable: 0, + coherence_inactive: 0, + }, + coordination_health: coord_health, + clip_pressure_distribution: ClipPressureDistribution { + min: 0.0, + max: 0.0, + mean: 0.0, + }, + }; + } + + let mut rank_min = usize::MAX; + let mut rank_max = 0_usize; + let mut rank_sum = 0_u64; + + let mut ni_min = f64::INFINITY; + let mut ni_max = f64::NEG_INFINITY; + let mut ni_sum = 0.0_f64; + let mut cold = 0_usize; + + let mut geo_saturated = 0_usize; + let mut geo_saturable = 0_usize; + let mut geo_coh_inactive = 0_usize; + + let mut cp_min = f64::INFINITY; + let mut cp_max = f64::NEG_INFINITY; + let mut cp_sum = 0.0_f64; + let mut cp_count = 0_u64; + + for cell in self.cells.values() { + let tracker = &cell.tracker; + let r = tracker.rank(); + rank_min = rank_min.min(r); + rank_max = rank_max.max(r); + rank_sum += r as u64; + + let m = tracker.maturity(); + ni_min = ni_min.min(m.noise_influence); + ni_max = ni_max.max(m.noise_influence); + ni_sum += m.noise_influence; + if m.real_observations == 0 { + cold += 1; + } + + let g = tracker.scoring_geometry(); + if g.is_novelty_saturated() { + geo_saturated += 1; + } + if g.is_novelty_saturable() { + geo_saturable += 1; + } + if r < 2 { + geo_coh_inactive += 1; + } + + for cp in cell.tracker.clip_pressures() { + cp_min = cp_min.min(cp); + cp_max = cp_max.max(cp); + cp_sum += cp; + cp_count += 1; + } + } + + #[allow(clippy::cast_precision_loss)] + let n = active_trackers as f64; + + #[allow(clippy::cast_precision_loss)] + let rank_mean = rank_sum as f64 / n; + + HealthReport { + total_g_nodes: self.graph.node_count() as usize, + semi_internal_count: self.graph.semi_internal_count() as usize, + active_trackers, + active_competitive_trackers: competitive_count, + active_ancestor_trackers: ancestor_count, + active_coordination_contexts: self.coordination.len(), + investment_set_size, + warming_trackers: warming_total, + warming_competitive_targets: warming_competitive, + lifetime_observations: self.lifetime_observations, + cells_tracked: self.cells.len(), + rank_distribution: RankDistribution { + min: rank_min, + max: rank_max, + mean: rank_mean, + }, + maturity_distribution: MaturityDistribution { + max_noise_influence: ni_max, + min_noise_influence: ni_min, + mean_noise_influence: ni_sum / n, + cold_trackers: cold, + }, + geometry_distribution: GeometryDistribution { + novelty_saturated: geo_saturated, + novelty_saturable: geo_saturable, + coherence_inactive: geo_coh_inactive, + }, + coordination_health: coord_health, + clip_pressure_distribution: ClipPressureDistribution { + min: if cp_count > 0 { cp_min } else { 0.0 }, + max: if cp_count > 0 { cp_max } else { 0.0 }, + #[allow(clippy::cast_precision_loss)] + mean: if cp_count > 0 { cp_sum / cp_count as f64 } else { 0.0 }, + }, + } + } + + /// Number of cells with a live tracker. + /// + /// Not the size of the analysis set: that also names the cells still + /// warming in staging, which have no tracker yet. + #[must_use] + pub fn cells_tracked(&self) -> usize { + self.cells.len() + } + + /// Total real observations processed across the sentinel's lifetime. + #[must_use] + pub const fn lifetime_observations(&self) -> u64 { + self.lifetime_observations + } + + /// Number of G-tree nodes excluded while producing the current analysis-set snapshot because their suffix width was below `MIN_TRACKER_DIM` (ADR-S-011). + /// + /// Recomputed with selection; this is not a lifetime total. + #[must_use] + pub const fn degenerate_cells_skipped(&self) -> usize { + self.degenerate_cells_skipped + } + + /// Read-only access to the configuration. + #[must_use] + pub const fn config(&self) -> &SentinelConfig { + &self.config + } + + /// Give the sentinel a live warming worker, leaving it in the state + /// construction produces for a configuration that asked for one. + /// + /// The warm-up dispatch keys on whether a worker is present, so the two + /// modes are a property of this sentinel at a given moment rather than of + /// its whole life, and a test that needs both can cross between them at a + /// point it chooses. Crossing late is what lets a test build its starting + /// state instead of waiting for one: without a worker the drain runs + /// inside `ingest`, so every cell the schedule asks for is online by the + /// time the call returns, on any machine and at any speed, where the same + /// cells under a worker come online whenever that worker is next + /// scheduled. The configuration flag moves with the handle, because it is + /// what `reset` reads to decide whether to spawn again, and a flag that + /// disagreed with the field would make reset the one call that silently + /// changed the mode. + /// + /// # Panics + /// + /// Panics if the sentinel already owns a worker, which is not a state the + /// engine can reach and so is a caller that has not built the situation it + /// means to test, or if the environment refuses the thread. + #[cfg(test)] + pub(crate) fn start_a_warming_worker_for_test(&mut self) { + assert!( + self.warming_thread.is_none(), + "the test sentinel must not already own a warming worker" + ); + self.config.background_warming = true; + self.warming_thread = Some( + warming_thread::WarmingThreadHandle::::spawn(&self.staging, self.config.noise_batch_size, self.config.noise_seed) + .expect("the environment must grant a warming thread"), + ); + } + + /// Fail the warming worker, wait until its join handle records the + /// failure, and clear the deliberately poisoned staging lock so tests can + /// isolate the failed-worker path. + #[cfg(test)] + pub(crate) fn fail_warming_worker_for_test(&self) { + self.warming_thread + .as_ref() + .expect("the test sentinel must own a warming worker") + .fail_worker_for_test(); + } + + /// Leave the sentinel in the state a warming worker leaves when it unwinds + /// between taking a cell out of the staging area and returning it, and + /// report which cell was stranded. + /// + /// The state is built rather than waited for. The window in which a panic + /// strands a cell is a few instructions wide, and whether a panic lands + /// inside it is the scheduler's decision: a test that waited for one would + /// be asserting about the box it ran on. What the recovery has to handle is + /// the state, and the state is fully described — an identifier the analysis + /// set still holds, absent from the producing set, carrying a checkout + /// record that no living thread will ever redeem, because the tracker it + /// named went down with the worker's stack. + /// + /// The worker is stopped first, so every step after it runs against a + /// staging area no other thread is touching. + /// + /// # Panics + /// + /// Panics if the sentinel owns no warming worker, or if it holds no + /// producing cell below the root to strand — in either case the caller has + /// not built the situation it means to test. + #[cfg(test)] + pub(crate) fn strand_a_cell_on_a_failed_warming_worker_for_test(&mut self) -> GNodeId { + self.fail_warming_worker_for_test(); + + let stranded = self + .analysis_set + .full() + .iter() + .map(|entry| entry.gnode) + .find(|gnode| *gnode != self.root_gnode && self.cells.contains_key(gnode)) + .expect("the test sentinel must hold a producing cell below the root"); + + let cell = self + .cells + .remove(&stranded) + .expect("the cell was just read out of the producing set"); + self.staging + .lock() + .expect("staging mutex poisoned") + .record_checkout_for_test(stranded, cell.is_competitive); + + stranded + } + + /// Read-only access to the G-V Graph. + #[must_use] + pub const fn graph(&self) -> &GvGraph { + &self.graph + } + + /// Read-only access to the current analysis set. + #[must_use] + pub const fn analysis_set(&self) -> &AnalysisSet { + &self.analysis_set + } + + /// Reset the sentinel to its freshly-constructed state. + /// + /// Drops all cell trackers and their learned subspaces, + /// re-initialises the G-V Graph, and zeroes all counters. + /// The configuration is preserved. + /// + /// Under `background_warming` the warming thread is stopped and started + /// again. If the environment refuses the new thread, the sentinel keeps + /// running and warms cells synchronously instead, the way a sentinel + /// configured without background warming always does, and records the + /// refusal as a warning. The choice is forced: this method has no error + /// channel, and the two alternatives are worse — aborting the host over + /// a resource limit, which is what the crate's policy on panics exists + /// to prevent, or leaving the flag standing over a sentinel that has no + /// thread to honour it. What the caller loses is the latency the mode + /// was enabled for; what it keeps is every report, and the stronger + /// reproducibility that synchronous warming carries. + /// + /// # Panics + /// + /// Panics if the staging area mutex is poisoned. + pub fn reset(&mut self) { + // Shut down background thread before clearing state (Step 3.4). + if let Some(ref wt) = self.warming_thread { + wt.shutdown(); + } + + self.cells.clear(); + self.coordination.clear(); + self.staging.lock().expect("staging mutex poisoned").clear(); + self.batch_counter = 0; + self.lifetime_observations = 0; + self.degenerate_cells_skipped = 0; + + // Reset RNG (§ALGO S-11.1). + self.noise_rng = self + .config + .noise_seed + .map_or_else(|| SmallRng::from_rng(&mut rand::rng()), SmallRng::seed_from_u64); + + // Reset the G-V Graph to a single root node. + let gv_config = GvConfig { + split_threshold: self.config.split_threshold, + depth_create: self.config.d_create, + depth_evict: self.config.d_evict, + budget: Some(self.config.budget), + alpha_relax: 0.75, + bounded_eviction: true, + }; + self.graph = GvGraph::new(gv_config); + self.root_gnode = self.graph.g_root(); + + // Begin a new counter epoch at the replacement graph's initial totals, + // so the next report cannot attribute the old graph's teardown or the + // replacement itself to its reporting interval. + self.prev_structural_mutation_counts = self.graph.structural_mutation_counts(); + + // Recreate the root tracker with auto noise injection. The width + // needs no second judgement here: it is fixed by the type, the sole + // constructor refuses a width below the tracker's minimum, and a + // reset can only be reached through an instance that constructor + // returned. A width this method could reject could never have got + // this far. + let mut root_cell = CellState { + tracker: SubspaceTracker::new(N as usize, &self.config, self.config.cusum_slow_decay), + depth: 0, + width: N as usize, + start: C::zero(), + end: C::domain_max(N), + is_competitive: false, + }; + if self.config.noise_schedule.rounds_for_depth(0) > 0 { + inject_noise_into_cell( + &mut root_cell, + self.config.noise_schedule.rounds_for_depth(0) as usize, + self.config.noise_batch_size, + &mut self.noise_rng, + ); + } + self.cells.insert(self.root_gnode, root_cell); + + // Recompute empty analysis set. + self.analysis_set = AnalysisSet::recompute::(&self.graph, self.config.analysis_k, self.config.analysis_depth_cutoff); + + // Restart background thread if configured (Step 3.4). + // + // The dispatch in `reconcile_analysis_set` keys on whether a thread + // is present, not on the flag that asked for one, so leaving the + // field empty is a complete fallback rather than a half-state: every + // staged cell is drained inline and every report is produced as it + // would have been. The warning is the only channel a method with no + // return value has. + self.warming_thread = if self.config.background_warming { + match warming_thread::WarmingThreadHandle::::spawn( + &self.staging, + self.config.noise_batch_size, + self.config.noise_seed, + ) { + Ok(handle) => Some(handle), + Err(refusal) => { + tracing::warn!( + %refusal, + "the environment refused the warming thread on reset — warming cells synchronously instead" + ); + None + } + } + } else { + None + }; + } + + /// Apply spatial decay to the entire G-V Graph. + /// + /// Delegates to the G-V Graph's temporal filter (§ALGO S-10.1, + /// §IDEA M-14) at the graph root, scaling accumulated importance + /// across all cells. + /// + /// # Parameters + /// + /// - `attenuation` — base decay factor at the midpoint depth. + /// - `(0, 1)`: **Attenuation** — cold cells lose standing. + /// - `> 1.0`: **Amplification** — hot cells reinforced. + /// - `1.0`: no-op (identity). + /// - `q` — depth selectivity in `[0.0, 1.0]`. + /// - `0.0`: uniform — all depths decay at the same rate. + /// - `> 0.0`: selective — coarse structure persists, fine + /// structure fades (or is amplified) faster. + /// - `1.0`: maximum selectivity. + /// + /// # Design + /// + /// The sentinel **never** calls this automatically. The host + /// controls temporal policy: when to decay, how aggressively, + /// and with what selectivity (§ALGO S-13.4). + /// + /// Decay does not trigger analysis set recomputation (the V-Tree + /// rankings change, but no scoring happens until the next + /// `ingest()`). The analysis set is recomputed at the start of + /// the next `ingest()` — no eager invalidation needed. + /// + /// The feed-forward invariant (ADR-S-002) is maintained: decay + /// modifies *importance* (accumulated observation counts), never + /// tracker state (subspace, baselines, CUSUM). + /// + /// # Panics + /// + /// - `attenuation < 0.0` or `attenuation.is_nan()`. + /// - `q < 0.0`, `q > 1.0`, or `q.is_nan()`. + /// + /// # Examples + /// + /// ``` + /// use torrust_sentinel::SentinelConfig; + /// use torrust_sentinel::Sentinel128; + /// + /// let mut s = Sentinel128::new(SentinelConfig::default()).unwrap(); + /// s.ingest(&[42_u128, 43, 44]); + /// + /// // Uniform 50% attenuation. + /// s.decay(0.5, 0.0); + /// assert!(s.graph().total_sum() < 3); + /// ``` + pub fn decay(&mut self, attenuation: f64, q: f64) { + let root = self.graph.g_root(); + self.graph.decay(root, attenuation, q); + } + + /// Apply spatial decay to a subtree of the G-V Graph. + /// + /// Like [`decay()`](Self::decay), but targets only the G-subtree + /// rooted at `root`. Cells outside this subtree are unaffected. + /// + /// # Use cases (§ALGO S-10.3) + /// + /// - **Regime change in a region.** Attenuate a subtree that + /// experienced a traffic regime shift, allowing it to re-form + /// under new observations without affecting global structure. + /// - **Suspected poisoning.** `decay_subtree(root, att=0.0₊, q=1.0)` + /// is a detail flush: the subtree root's own count is preserved, + /// descendants are zeroed. + /// - **Hot reinforcement.** `decay_subtree(root, att=1.5, q=0.0)` + /// amplifies a known-active subtree to boost its competitive + /// standing. + /// + /// # Parameters + /// + /// - `root` — the G-node whose subtree receives decay. Obtain + /// via `sentinel.graph().g_root()` for the global root, or + /// from a G-node inspection method. + /// - `attenuation` — see [`decay()`](Self::decay). + /// - `q` — see [`decay()`](Self::decay). + /// + /// # Panics + /// + /// - `root` does not refer to a live G-node (the graph has been + /// restructured since the handle was obtained). + /// - `attenuation < 0.0` or `attenuation.is_nan()`. + /// - `q < 0.0`, `q > 1.0`, or `q.is_nan()`. + /// + /// # Examples + /// + /// ``` + /// use torrust_sentinel::SentinelConfig; + /// use torrust_sentinel::Sentinel128; + /// + /// let mut s = Sentinel128::new(SentinelConfig::default()).unwrap(); + /// s.ingest(&[42_u128, 43, 44]); + /// + /// let root = s.graph().g_root(); + /// s.decay_subtree(root, 0.5, 0.0); + /// assert!(s.graph().total_sum() < 3); + /// ``` + pub fn decay_subtree(&mut self, root: GNodeId, attenuation: f64, q: f64) { + self.graph.decay(root, attenuation, q); + } + + /// List all cell `GNodeId`s in the full analysis set. + /// + /// Returns IDs in ascending order (`BTreeMap` iteration order). + #[must_use] + pub fn cell_gnodes(&self) -> Vec { + self.cells.keys().copied().collect() + } + + /// Inspect a specific cell's tracker state. + /// + /// Returns `None` if the cell is not in the analysis set. + #[must_use] + pub fn inspect_cell(&self, gnode: GNodeId) -> Option> { + let cell = self.cells.get(&gnode)?; + let bl = cell.tracker.axis_baselines(); + Some(CellInspection { + gnode_id: gnode, + start: cell.start, + end: cell.end, + depth: cell.depth, + analysis_width: cell.width, + is_competitive: cell.is_competitive, + rank: cell.tracker.rank(), + energy_ratio: cell.tracker.energy_ratio(), + top_singular_value: cell.tracker.top_singular_value(), + maturity: cell.tracker.maturity(), + geometry: cell.tracker.scoring_geometry(), + baselines: AxisBaselineSnapshots { + novelty: crate::BaselineSnapshot { + mean: bl.novelty_mean, + variance: bl.novelty_var, + }, + displacement: crate::BaselineSnapshot { + mean: bl.displacement_mean, + variance: bl.displacement_var, + }, + surprise: crate::BaselineSnapshot { + mean: bl.surprise_mean, + variance: bl.surprise_var, + }, + coherence: crate::BaselineSnapshot { + mean: bl.coherence_mean, + variance: bl.coherence_var, + }, + }, + }) + } + + // ════════════════════════════════════════════════════════ + // Private implementation + // ════════════════════════════════════════════════════════ + + /// Convert a spatial bound to the corresponding scoring bound. + fn scoring_bound(bound: C) -> C { + // At full integer width Mudlark ends the root at MAX, not 2^N. + // Repeated floor midpoints therefore put every interior depth-d + // boundary at q * 2^(N-d) - 1. Its successor restores the binary + // prefix boundary at every depth. Route that one boundary value to + // the lower scoring cell: otherwise the uppermost cell contains + // 2^(N-d) + 1 values, which cannot fit in N-d binary suffix bits. + // Preserve the root's endpoints and its inclusive MAX exception. + // That exception is the one `domain_top_is_in_domain` decides, and it + // is the integer case: a continuous coordinate keeps an exclusive + // bound at every width, so its bounds stay untouched and the successor + // it does not support is never asked for. + if Self::domain_top_is_in_domain() && bound != C::zero() && bound != C::domain_max(N) { + bound.next_value() + } else { + bound + } + } + + /// Reconcile the cells map with the current analysis set. + /// + /// Recomputes the analysis set from the V-Tree, creates trackers + /// for new cells, and destroys trackers for exited cells. The + /// root tracker is never destroyed (§ALGO S-8.5). + /// + /// New cells are enqueued into the staging area rather than being + /// warmed inline (Step 2, §ALGO S-11.6.8). A synchronous drain loop + /// warms all queued cells to completion, then promotes them into + /// the live cells map — identical external behaviour to the old + /// inline path. The background-thread version (Step 3) will + /// remove the drain loop. + fn reconcile_analysis_set(&mut self) { + // ── Reap a worker that stopped on its own (Step 3) ── + // + // The warm-up dispatch below keys on whether a warming worker is + // present, and a worker that panicked leaves its handle standing. Left + // alone, that handle answers for a thread that is gone: every later + // reconciliation takes the background branch, skips the synchronous + // drain, and notifies a condition variable nobody is waiting on, so + // every cell staged from that point on waits for a worker that will + // never take it and never comes online. The failure is silent in the + // worst way — the engine goes on producing reports, from a cell set + // that has stopped growing. + // + // Reaping restores the state the engine already has an answer for. An + // absent handle is a complete fallback rather than a half-state, which + // is the same reading `reset` relies on when the environment refuses it + // a thread: the drain runs inline, every staged cell still comes + // online, and every report is produced as it would have been. What a + // host loses is the latency background warming was enabled for, and it + // is told so; what it keeps is every report. + // + // The fallback stands for the rest of this sentinel's epoch rather than + // respawning, because nothing here can tell the two ways a worker dies + // apart. Its only panics are its own reads of a poisoned staging mutex, + // and a mutex poisoned by whatever killed the first worker is still + // poisoned for the second: a respawn on each reconciliation would log a + // fresh failure every batch and still never warm a cell. `reset` is the + // caller's way back to a background worker, and it is explicit. + let worker_failed = self + .warming_thread + .as_ref() + .is_some_and(warming_thread::WarmingThreadHandle::reap_if_finished); + if worker_failed { + // Dropping the handle takes the staging lock to signal a worker + // that is already gone, so it happens before this thread takes that + // lock below rather than inside it. + self.warming_thread = None; + } + + let new_set = AnalysisSet::recompute::(&self.graph, self.config.analysis_k, self.config.analysis_depth_cutoff); + self.degenerate_cells_skipped = new_set.degenerate_cells_skipped(); + + // ── Identify entries and exits ────────────────────── + let old_gnodes: BTreeSet = self.cells.keys().copied().collect(); + let new_gnodes: BTreeSet = new_set.full().iter().map(|e| e.gnode).collect(); + + // Destroy exited cells (except root — §ALGO S-8.5). + for &gone in old_gnodes.difference(&new_gnodes) { + if gone != self.root_gnode { + self.cells.remove(&gone); + } + } + + // Evict staging cells that exited the analysis set (Step 2.5). + // A warming cell may have been evicted by graph rebalance. + // Also update cached volumes and enqueue new cells — all under + // a single lock acquisition to avoid repeated locking. + { + let mut staging = self.staging.lock().expect("staging mutex poisoned"); + + // A cell the reaped worker had checked out went down with it: the + // tracker lived on that thread's stack, and only that thread could + // have returned it. What is left is the checkout record, and while + // it stands the staging area answers that the cell is present, so + // the enqueue loop below skips an identifier that nothing holds and + // the cell is stranded between the two sets forever. Dropping the + // record now, before that loop, is what lets the cell be built + // again in this same pass — at the cost of the warm-up rounds it + // had already run, the same trade eviction in flight already makes. + if worker_failed { + let abandoned = staging.abandon_in_flight(); + tracing::error!( + abandoned, + "the warming worker is no longer running — warming cells synchronously from here and rebuilding the cells it was holding" + ); + } + + staging.retain_in_set(&new_gnodes); + + // Enqueue entered cells into the staging area (Step 2.1). + for entry in new_set.full() { + staging.update_competitive(entry.gnode, entry.is_competitive); + if self.cells.contains_key(&entry.gnode) || staging.contains(entry.gnode) { + continue; + } + + let width = (N as usize).saturating_sub(entry.depth as usize); + + // ADR-S-011: skip degenerate cells whose suffix width + // is too narrow for a meaningful subspace model. + if width < crate::MIN_TRACKER_DIM { + tracing::warn!( + gnode = ?entry.gnode, + depth = entry.depth, + width, + "skipping degenerate cell (width < MIN_TRACKER_DIM)" + ); + continue; + } + + let cell = CellState { + tracker: SubspaceTracker::new(width, &self.config, self.config.cusum_slow_decay), + depth: entry.depth, + width, + start: Self::scoring_bound(entry.start), + end: Self::scoring_bound(entry.end), + is_competitive: entry.is_competitive, + }; + + // Enqueue for deferred noise warm-up (Step 2.1). + let rounds = self.config.noise_schedule.rounds_for_depth(entry.depth as usize); + staging.enqueue(entry.gnode, cell, rounds); + } + + // Update cached volumes from the graph so both warm-up paths + // can prioritise correctly (Step 3.2c). This runs after the + // enqueue loop rather than before it: a cell enqueued above + // starts at zero volume, and refreshing beforehand leaves every + // newly entered cell holding that zero until some later pass. + // The priority rule is highest volume first, so a field of + // zeroes is decided entirely by the tie-breaks — shallower + // depth first, then the smaller identifier — and the order the + // new cells come online in would be settled by where they sit + // in the tree rather than by the traffic behind them. + staging.update_volumes(&self.graph); + + // ── Warm-up dispatch ──────────────────────────── + if self.warming_thread.is_none() { + // Synchronous mode: drain all warming cells in-line + // (deterministic, matching the old inline path). + staging.drain_all_synchronous(self.config.noise_batch_size, &mut self.noise_rng); + } + } // staging lock released + + // Notify background thread if running (Step 3). + if let Some(ref wt) = self.warming_thread { + wt.notify(); + } + + // Promote newly ready cells so they participate in routing + // within *this* ingest call (identical to old inline path + // in synchronous mode; in background mode, promotes cells + // that finished since the last ingest). + self.promote_ready_cells(); + + // Update competitive flags on retained cells. + for entry in new_set.full() { + if let Some(cell) = self.cells.get_mut(&entry.gnode) { + cell.is_competitive = entry.is_competitive; + } + } + + self.analysis_set = new_set; + } + + /// Promote all ready cells from the staging area into the live + /// cells map (Step 2.3, §ALGO S-11.6.3). + /// + /// Ready cells have completed their full noise warm-up schedule. + /// Coordination warm-up for these cells fires naturally when + /// `propagate_coordination_from_root()` creates their coordination + /// context on the next scoring pass. + fn promote_ready_cells(&mut self) { + let ready = self.staging.lock().expect("staging mutex poisoned").take_ready(); + for (gnode, cell) in ready { + self.cells.insert(gnode, cell); + } + } + + /// Route observations to cells and score each cell. + #[allow(clippy::cast_possible_truncation)] // depth ≤ 128, always fits in u8 + fn route_and_score(&mut self, values: &[C], centred: &[CentredBits]) -> Vec> { + // ── Build per-cell observation buffers ─────────────── + // Key: GNodeId → Vec of observation indices. + // + // Route each observation to every active cell whose interval + // contains it. `self.cells` holds exactly the online + // producing set — staging cells are excluded by construction. + let mut cell_obs: BTreeMap> = BTreeMap::new(); + + // Cell intervals are half-open, which needs one exception at the very + // top of the domain. Where `domain_max(N)` names a value of the domain + // rather than the first value above it — the integer case at a width + // that fills the type, where `2^N` is not representable — a half-open + // reading of the topmost interval excludes a coordinate that is + // genuinely inside the domain. That observation is still counted — it + // moves the spatial layer and it raises the lifetime total — so + // leaving it unrouted drops a real observation from every tracker + // while the totals go on including it. The cell ending at the top of + // the domain therefore owns its upper bound. Every other boundary + // stays half-open, so no coordinate can fall in two sibling cells, and + // wherever the bound stays exclusive — every narrower width, and a + // continuous or host-written type at any width — it names a value + // outside the domain that the boundary at ingestion has already + // refused, so nothing reaches this scan for the exception to admit. + // + // Every value that reaches this scan is in the domain, because + // `ingest` decides that before the spatial layer. That is what makes + // the flat containment test the ancestor walk it stands in for + // (ADR-S-010): the root's interval is the whole domain and the root + // tracker is permanent (§ALGO S-8.4), so an in-domain value is matched + // by at least one cell, and the mandatory delivery of §ALGO S-9.3 + // holds. A value outside the domain would be matched by none. + let domain_top = C::domain_max(N); + let top_is_in_domain = Self::domain_top_is_in_domain(); + + for (i, &value) in values.iter().enumerate() { + for (&gnode, cell) in &self.cells { + let owns_domain_top = top_is_in_domain && cell.end == domain_top && value == domain_top; + if value >= cell.start && (value < cell.end || owns_domain_top) { + cell_obs.entry(gnode).or_default().push(i); + } + } + } + + // ── Score each cell ───────────────────────────────── + let mut reports = Vec::with_capacity(self.cells.len()); + + for (&gnode, cell) in &mut self.cells { + let obs_indices = cell_obs.get(&gnode); + + if let Some(indices) = obs_indices + && !indices.is_empty() + { + let slices: Vec<&[f64]> = indices.iter().map(|&i| centred[i].suffix(cell.depth as u8)).collect(); + + #[cfg(debug_assertions)] + { + #[allow(clippy::cast_precision_loss)] + let expected_norm_sq = cell.width as f64 / 4.0; + for slice in &slices { + let norm_sq: f64 = slice.iter().map(|x| x * x).sum(); + debug_assert!( + (norm_sq - expected_norm_sq).abs() < 1e-10, + "suffix norm² = {norm_sq}, expected w/4 = {expected_norm_sq}" + ); + } + } + + let prefix_report = cell.tracker.observe(&slices, cell.depth as u8, false); + + reports.push(CellReport { + gnode_id: gnode, + start: cell.start, + end: cell.end, + depth: cell.depth, + analysis_width: cell.width, + is_competitive: cell.is_competitive, + sample_count: indices.len(), + scores: prefix_report.scores, + rank: prefix_report.rank, + energy_ratio: prefix_report.energy_ratio, + top_singular_value: prefix_report.top_singular_value, + maturity: prefix_report.maturity, + geometry: prefix_report.geometry, + per_sample: prefix_report.per_sample, + }); + } + // Cells with no observations in this batch are omitted + // from the report (§ALGO S-8.8). + } + + reports + } + + // ════════════════════════════════════════════════════════ + // Hierarchical coordination (§ALGO S-7) + // ════════════════════════════════════════════════════════ + + /// Extract the 4D mean score vector from competitive cell reports. + /// + /// Only competitive cells with `sample_count > 0` contribute. + fn assemble_cell_scores(cell_reports: &[CellReport]) -> BTreeMap { + let mut scores = BTreeMap::new(); + for report in cell_reports { + if report.is_competitive && report.sample_count > 0 { + scores.insert( + report.gnode_id, + [ + report.scores.novelty.mean, + report.scores.displacement.mean, + report.scores.surprise.mean, + report.scores.coherence.mean, + ], + ); + } + } + scores + } + + /// Run hierarchical coordination from the G-tree root. + /// + /// Builds the pruned coordination topology, walks bottom-up, fires + /// coordination at internal nodes where both subtrees contribute + /// competitive cell scores, and returns reports for the contexts + /// that fire in this batch (§ALGO S-7.4). + fn propagate_coordination_from_root(&mut self, cell_scores: &BTreeMap) -> Vec> { + let scoring_gnodes: BTreeSet = cell_scores.keys().copied().collect(); + + // The score tree controls firing. A batch with fewer than two scoring + // regions produces no coordination reports, regardless of membership. + let mut reports = + Self::build_coordination_tree(&self.graph, &scoring_gnodes, self.root_gnode).map_or_else(Vec::new, |scoring_tree| { + let (reports, _cells) = self.walk_coordination(&scoring_tree, cell_scores); + reports + }); + + // The walk emits in post-order, which puts the root last and is not + // the order either record states. Sort shallowest first, ties by + // ascending identifier: that is a total order, it is the depth + // ordering the output record describes, and among nodes of equal + // depth it is the identifier ordering this type's own documentation + // describes. The two agree everywhere except where eviction and + // restoration have recycled identifiers, and there the depth is the + // reading that still means what it says. + reports.sort_by(|a, b| a.depth.cmp(&b.depth).then_with(|| a.gnode_id.cmp(&b.gnode_id))); + + // Membership controls retention (§ALGO S-7.7.2). A cell need not score + // in this batch to preserve the learned context for its unchanged + // competitive group; warming cells are absent from the online map. + let member_gnodes: BTreeSet = self + .cells + .iter() + .filter_map(|(&gnode, cell)| cell.is_competitive.then_some(gnode)) + .collect(); + let active_gnodes = Self::build_coordination_tree(&self.graph, &member_gnodes, self.root_gnode) + .map_or_else(BTreeSet::new, |membership_tree| { + Self::collect_internal_gnodes(&membership_tree) + }); + self.coordination.retain(|gnode, _| active_gnodes.contains(gnode)); + + reports + } + + /// Build the coordination tree topology from the G-tree. + /// + /// Only includes nodes reachable from the root through subtrees + /// that contain at least one competitive cell. This prunes + /// branches that contain no competitive cells, reducing the + /// coordination walk from $O(|G|)$ to $O(|\mathcal{A}| \cdot d_{\max})$. + fn build_coordination_tree( + graph: &GvGraph, + competitive_gnodes: &BTreeSet, + root: GNodeId, + ) -> Option> { + let info = graph.gnode_info(root)?; + let children = graph.gnode_children(root)?; + + let has_left = children.left.is_some(); + let has_right = children.right.is_some(); + let is_competitive = competitive_gnodes.contains(&root); + + // Terminal node (no children). + if !has_left && !has_right { + return if is_competitive { + Some(CoordNode::Terminal { gnode: root }) + } else { + None // Prune: no competitive cell here. + }; + } + + let left_tree = children + .left + .and_then(|l| Self::build_coordination_tree(graph, competitive_gnodes, l)); + let right_tree = children + .right + .and_then(|r| Self::build_coordination_tree(graph, competitive_gnodes, r)); + + match (left_tree, right_tree) { + (Some(left), Some(right)) => Some(CoordNode::Internal { + gnode: root, + depth: info.depth, + start: Self::scoring_bound(info.start), + end: Self::scoring_bound(info.end), + is_competitive, + left: Box::new(left), + right: Box::new(right), + }), + (Some(child), None) | (None, Some(child)) => Some(CoordNode::SemiInternal { + gnode: root, + is_competitive, + child: Box::new(child), + }), + (None, None) => { + // Both subtrees pruned, but this node itself might be competitive. + if is_competitive { + Some(CoordNode::Terminal { gnode: root }) + } else { + None + } + } + } + } + + /// Walk the coordination tree bottom-up, firing coordination at + /// internal nodes where both subtrees contribute competitive cell + /// scores (§ALGO S-7.4). + /// + /// Returns `(reports, cells_in_subtree)`. + #[allow(clippy::type_complexity)] + fn walk_coordination( + &mut self, + node: &CoordNode, + cell_scores: &BTreeMap, + ) -> (Vec>, Vec<(GNodeId, [f64; 4])>) { + match node { + CoordNode::Terminal { gnode } => { + if let Some(&scores) = cell_scores.get(gnode) { + (Vec::new(), vec![(*gnode, scores)]) + } else { + (Vec::new(), Vec::new()) + } + } + CoordNode::SemiInternal { + gnode, + is_competitive, + child, + } => { + let (reports, mut cells) = self.walk_coordination(child, cell_scores); + // If this node itself is competitive and has scores, add it. + if *is_competitive && let Some(&scores) = cell_scores.get(gnode) { + cells.push((*gnode, scores)); + } + (reports, cells) + } + CoordNode::Internal { + gnode, + depth, + start, + end, + is_competitive, + left, + right, + } => { + let (left_reports, left_cells) = self.walk_coordination(left, cell_scores); + let (right_reports, right_cells) = self.walk_coordination(right, cell_scores); + + let mut reports = left_reports; + reports.extend(right_reports); + + let mut my_cells: Vec<(GNodeId, [f64; 4])> = Vec::with_capacity(left_cells.len() + right_cells.len() + 1); + my_cells.extend_from_slice(&left_cells); + my_cells.extend_from_slice(&right_cells); + + // If this node itself is competitive and has scores, add it. + if *is_competitive && let Some(&scores) = cell_scores.get(gnode) { + my_cells.push((*gnode, scores)); + } + + // Fire coordination if both subtrees contribute and ≥ 2 cells total. + if !left_cells.is_empty() && !right_cells.is_empty() && my_cells.len() >= 2 { + let report = self.fire_coordination(*gnode, *depth, *start, *end, &my_cells, false); + reports.push(report); + } + + (reports, my_cells) + } + } + } + + /// Fire a coordination context at a G-node: centre, observe, update + /// running mean, and produce a `CoordinationReport`. + #[allow(clippy::cast_possible_truncation)] // depth ≤ 128, always fits in u8 + fn fire_coordination( + &mut self, + gnode: GNodeId, + depth: u32, + start: C, + end: C, + my_cells: &[(GNodeId, [f64; 4])], + is_noise: bool, + ) -> CoordinationReport { + let lam = self.config.forgetting_factor; + let alpha = 1.0 - lam; + + // §ALGO S-11.7.2: Collect baselines before borrowing self.coordination + // to avoid simultaneous &mut borrows. + let cell_baselines: Vec<(GNodeId, AxisBaselines)> = if self.coordination.contains_key(&gnode) { + Vec::new() + } else { + my_cells + .iter() + .filter_map(|(cell_gnode, _)| self.cells.get(cell_gnode).map(|c| (*cell_gnode, c.tracker.axis_baselines()))) + .collect() + }; + + let ctx = self.coordination.entry(gnode).or_insert_with(|| CoordContext { + tracker: SubspaceTracker::new(4, &self.config, self.config.cusum_coord_slow_decay), + running_mean: [0.0; 4], + warm: false, + }); + + // §ALGO S-11.7: Warm new contexts when not in noise phase. + let coord_rounds = self.config.noise_schedule.rounds_for_depth(depth as usize); + if !cell_baselines.is_empty() && !is_noise && coord_rounds > 0 { + warm_coordination_context( + ctx, + &cell_baselines, + coord_rounds as usize, + &mut self.noise_rng, + self.config.forgetting_factor, + depth as u8, + ); + } + + // Centre against running mean (§ALGO S-7.3). + let centred: Vec> = my_cells + .iter() + .map(|(_, scores)| scores.iter().enumerate().map(|(j, &v)| v - ctx.running_mean[j]).collect()) + .collect(); + + let slices: Vec<&[f64]> = centred.iter().map(Vec::as_slice).collect(); + let prefix_report = ctx.tracker.observe(&slices, depth as u8, is_noise); + + // Compute column means. + let col_means = compute_col_means(my_cells); + + // Update running mean (§ALGO S-7.3). + if ctx.warm { + for (j, m) in ctx.running_mean.iter_mut().enumerate() { + *m = lam.mul_add(*m, alpha * col_means[j]); + } + } else { + ctx.running_mean = col_means; + ctx.warm = true; + } + + CoordinationReport { + gnode_id: gnode, + start, + end, + depth, + cells_reporting: my_cells.len(), + rank: prefix_report.rank, + energy_ratio: prefix_report.energy_ratio, + top_singular_value: prefix_report.top_singular_value, + scores: prefix_report.scores, + maturity: prefix_report.maturity, + geometry: prefix_report.geometry, + per_member: prefix_report.per_sample.map(|samples| { + samples + .into_iter() + .zip(my_cells.iter()) + .map(|(s, (cell_gnode, _))| { + let (cell_start, cell_end, cell_depth) = self + .cells + .get(cell_gnode) + .map_or_else(|| (C::zero(), C::zero(), 0), |c| (c.start, c.end, c.depth)); + MemberScore { + cell_start, + cell_end, + cell_depth, + novelty: s.novelty, + displacement: s.displacement, + surprise: s.surprise, + coherence: s.coherence, + novelty_z: s.novelty_z, + displacement_z: s.displacement_z, + surprise_z: s.surprise_z, + coherence_z: s.coherence_z, + } + }) + .collect() + }), + } + } + + /// Collect the `GNodeId`s of all `Internal` nodes in a coordination tree. + /// + /// These are the nodes whose two subtrees are represented in the tree. + /// The membership tree uses them to prune stale contexts (§ALGO S-7.7.2). + fn collect_internal_gnodes(node: &CoordNode) -> BTreeSet { + let mut result = BTreeSet::new(); + Self::collect_internal_gnodes_inner(node, &mut result); + result + } + + fn collect_internal_gnodes_inner(node: &CoordNode, result: &mut BTreeSet) { + match node { + CoordNode::Terminal { .. } => {} + CoordNode::SemiInternal { child, .. } => { + Self::collect_internal_gnodes_inner(child, result); + } + CoordNode::Internal { gnode, left, right, .. } => { + result.insert(*gnode); + Self::collect_internal_gnodes_inner(left, result); + Self::collect_internal_gnodes_inner(right, result); + } + } + } + + /// Compute coordination health summary. + fn coordination_health(&self) -> CoordinationHealth { + let count = self.coordination.len(); + + if count == 0 { + return CoordinationHealth { + active_contexts: 0, + capacity: 0, + rank_distribution: RankDistribution { + min: 0, + max: 0, + mean: 0.0, + }, + maturity_distribution: MaturityDistribution { + max_noise_influence: 0.0, + min_noise_influence: 0.0, + mean_noise_influence: 0.0, + cold_trackers: 0, + }, + dim: 4, + geometry_distribution: GeometryDistribution { + novelty_saturated: 0, + novelty_saturable: 0, + coherence_inactive: 0, + }, + }; + } + + let mut rank_min = usize::MAX; + let mut rank_max = 0_usize; + let mut rank_sum = 0_u64; + + let mut ni_min = f64::INFINITY; + let mut ni_max = f64::NEG_INFINITY; + let mut ni_sum = 0.0_f64; + let mut cold = 0_usize; + + let mut geo_saturated = 0_usize; + let mut geo_saturable = 0_usize; + let mut geo_coh_inactive = 0_usize; + + for ctx in self.coordination.values() { + let r = ctx.tracker.rank(); + rank_min = rank_min.min(r); + rank_max = rank_max.max(r); + rank_sum += r as u64; + + let m = ctx.tracker.maturity(); + ni_min = ni_min.min(m.noise_influence); + ni_max = ni_max.max(m.noise_influence); + ni_sum += m.noise_influence; + if m.real_observations == 0 { + cold += 1; + } + + let g = ctx.tracker.scoring_geometry(); + if g.is_novelty_saturated() { + geo_saturated += 1; + } + if g.is_novelty_saturable() { + geo_saturable += 1; + } + if r < 2 { + geo_coh_inactive += 1; + } + } + + #[allow(clippy::cast_precision_loss)] + let n = count as f64; + + #[allow(clippy::cast_precision_loss)] + let rank_mean = rank_sum as f64 / n; + + // Read capacity and dim from the first coordination tracker. + let first = self.coordination.values().next().unwrap(); + let capacity = first.tracker.cap(); + let dim = first.tracker.dim(); + + CoordinationHealth { + active_contexts: count, + capacity, + rank_distribution: RankDistribution { + min: rank_min, + max: rank_max, + mean: rank_mean, + }, + maturity_distribution: MaturityDistribution { + max_noise_influence: ni_max, + min_noise_influence: ni_min, + mean_noise_influence: ni_sum / n, + cold_trackers: cold, + }, + dim, + geometry_distribution: GeometryDistribution { + novelty_saturated: geo_saturated, + novelty_saturable: geo_saturable, + coherence_inactive: geo_coh_inactive, + }, + } + } + + /// Produce an empty `BatchReport` (for empty input slices). + /// + /// Takes `&mut self` so that structural-mutation counters are + /// drained (and `prev_*` snapshots advanced) through this path + /// too — e.g. if the caller interleaves `decay()` or + /// `decay_subtree()` with empty `ingest(&[])` calls, the next + /// non-empty report still sees the correct "since last report" + /// delta rather than a double-count. + fn empty_report(&mut self) -> BatchReport { + let health = self.health(); + let online: BTreeSet = self.cells.keys().copied().collect(); + let mut summary = self.analysis_set.summary_online(&online); + // The tracker population, for the reason the non-empty path gives. + let warming_count = self.staging.lock().expect("staging mutex poisoned").total_count(); + summary.investment_set_size = self.cells.len() + warming_count; + summary.degenerate_cells_skipped = self.degenerate_cells_skipped; + + let (splits, net_removals, terminal_count) = self.take_structural_mutation_counts(); + + BatchReport { + cell_reports: Vec::new(), + ancestor_reports: Vec::new(), + coordination_reports: Vec::new(), + contour: ContourSnapshot { + plateau_count: self.graph.plateaus().len(), + cell_count: terminal_count as usize + self.graph.semi_internal_count() as usize, + total_importance: self.graph.total_sum().to_f64_approx(), + splits_since_last_report: splits, + net_removals_since_last_report: net_removals, + }, + health, + analysis_set_summary: summary, + // No observations, so no oldest one to be any age at all. + oldest_observation_age_micros: None, + } + } + + /// Compute `(splits, net_removals, terminal_count)` since the + /// previous report and advance the spatial-counter snapshot. + /// + /// The graph counts each child creation as one split. Net removals are + /// evictions minus restorations, floored at zero because the report field is + /// unsigned. Both report values saturate at [`u32::MAX`]. Construction and + /// reset snapshot a new graph immediately, so replacement never creates a + /// synthetic interval delta (§ALGO S-14.10). + fn take_structural_mutation_counts(&mut self) -> (u32, u32, u32) { + let current = self.graph.structural_mutation_counts(); + let previous = self.prev_structural_mutation_counts; + + let raw_splits = current.splits.saturating_sub(previous.splits); + let raw_evictions = current.evictions.saturating_sub(previous.evictions); + let raw_restorations = current.restorations.saturating_sub(previous.restorations); + let raw_net_removals = raw_evictions.saturating_sub(raw_restorations); + + self.prev_structural_mutation_counts = current; + + ( + u32::try_from(raw_splits).unwrap_or(u32::MAX), + u32::try_from(raw_net_removals).unwrap_or(u32::MAX), + self.graph.terminal_count(), + ) + } +} + +// ─── Automatic noise injection (§ALGO S-11) ─────────────────── + +/// Generate a batch of random centred vectors (each entry ±0.5). +fn generate_noise_batch(dim: usize, batch_size: usize, rng: &mut SmallRng) -> Vec> { + (0..batch_size) + .map(|_| (0..dim).map(|_| if rng.random_bool(0.5) { 0.5 } else { -0.5 }).collect()) + .collect() +} + +/// Compute column means of the 4D cell score vectors. +fn compute_col_means(cells: &[(GNodeId, [f64; 4])]) -> [f64; 4] { + let mut col_means = [0.0_f64; 4]; + for (_, scores) in cells { + for (j, &v) in scores.iter().enumerate() { + col_means[j] += v; + } + } + #[allow(clippy::cast_precision_loss)] + let n_f = cells.len() as f64; + for m in &mut col_means { + *m /= n_f; + } + col_means +} + +/// Runs a cell's whole noise schedule, then seeds the drift reference from the +/// baselines the noise built and clears the evidence it accumulated, so the +/// warm-up shapes what counts as normal without itself counting as history. +#[allow(clippy::cast_possible_truncation)] // depth ≤ 128 +fn inject_noise_into_cell(cell: &mut CellState, rounds: usize, batch_size: usize, rng: &mut SmallRng) { + for _ in 0..rounds { + let noise = generate_noise_batch(cell.width, batch_size, rng); + let slices: Vec<&[f64]> = noise.iter().map(Vec::as_slice).collect(); + cell.tracker.observe(&slices, cell.depth as u8, true); + } + + cell.tracker.seed_cusum_slow_from_baselines(); + cell.tracker.reset_cusum(); + cell.tracker.reset_clip_pressure(); +} + +// ─── Coordination warming (§ALGO S-11.7) ────────────────────── + +/// Snapshot of per-axis baseline statistics for synthetic score +/// generation (§ALGO S-11.7.2). +pub struct AxisBaselines { + pub novelty_mean: f64, + pub novelty_var: f64, + pub displacement_mean: f64, + pub displacement_var: f64, + pub surprise_mean: f64, + pub surprise_var: f64, + pub coherence_mean: f64, + pub coherence_var: f64, +} + +/// Sample a synthetic 4-axis score vector from axis baselines. +/// +/// Uses Gamma(α, β) where α = mean²/variance, β = mean/variance. +/// Falls back to axis-specific defaults when baselines are cold +/// (§ALGO S-11.7.2). +fn sample_synthetic_score(baselines: &AxisBaselines, rng: &mut SmallRng) -> [f64; 4] { + [ + gamma_sample(baselines.novelty_mean, baselines.novelty_var, 0.25, 0.01, rng), + gamma_sample(baselines.displacement_mean, baselines.displacement_var, 0.1, 0.01, rng), + gamma_sample(baselines.surprise_mean, baselines.surprise_var, 0.25, 0.01, rng), + gamma_sample(baselines.coherence_mean, baselines.coherence_var, 0.1, 0.01, rng), + ] +} + +/// Sample from Gamma(α, β) with α = mean²/var, β = mean/var. +/// Falls back to `default_mean`/`default_var` when `mean` or `var` +/// are invalid (zero, negative, NaN). +fn gamma_sample(mean: f64, var: f64, default_mean: f64, default_var: f64, rng: &mut SmallRng) -> f64 { + let (m, v) = if mean > 0.0 && var > 0.0 && mean.is_finite() && var.is_finite() { + (mean, var) + } else { + (default_mean, default_var) + }; + let alpha = m * m / v; + let beta = m / v; + let gamma = Gamma::new(alpha, 1.0 / beta).unwrap_or_else(|_| { + let a = default_mean * default_mean / default_var; + let b = default_mean / default_var; + Gamma::new(a, 1.0 / b).expect("default Gamma params must be valid") + }); + gamma.sample(rng) +} + +/// Warm a newly activated coordination context with synthetic +/// score vectors sampled from cell baselines (§ALGO S-11.7.2). +/// +/// Uses Gamma sampling to match cell baseline moments while +/// respecting axis non-negativity. +fn warm_coordination_context( + ctx: &mut CoordContext, + contributing_cells: &[(GNodeId, AxisBaselines)], + rounds: usize, + rng: &mut SmallRng, + forgetting_factor: f64, + depth: u8, +) { + for _ in 0..rounds { + let synth_with_gnodes: Vec<(GNodeId, [f64; 4])> = contributing_cells + .iter() + .map(|(gnode, baselines)| (*gnode, sample_synthetic_score(baselines, rng))) + .collect(); + + // Centre against running mean. + let centred: Vec> = synth_with_gnodes + .iter() + .map(|(_, scores)| scores.iter().enumerate().map(|(j, &v)| v - ctx.running_mean[j]).collect()) + .collect(); + + let slices: Vec<&[f64]> = centred.iter().map(Vec::as_slice).collect(); + ctx.tracker.observe(&slices, depth, true); + + // Update running mean. + let col_means = compute_col_means(&synth_with_gnodes); + let lam = forgetting_factor; + let alpha = 1.0 - lam; + if ctx.warm { + for (j, m) in ctx.running_mean.iter_mut().enumerate() { + *m = lam.mul_add(*m, alpha * col_means[j]); + } + } else { + ctx.running_mean = col_means; + ctx.warm = true; + } + } + + ctx.tracker.reset_cusum(); + ctx.tracker.reset_clip_pressure(); +} + +// ─── Coordination tree topology ───────────────────────────── + +/// Lightweight mirror of the G-tree topology, pre-collected for +/// coordination traversal. Avoids borrowing `self.graph` during +/// `self.coordination` mutation. +enum CoordNode { + /// Terminal node: a competitive cell with no relevant children. + Terminal { gnode: GNodeId }, + /// Internal node with only one relevant child (no coordination fires here). + SemiInternal { + gnode: GNodeId, + is_competitive: bool, + child: Box, + }, + /// Internal node with both relevant children; a scoring tree fires coordination here. + Internal { + gnode: GNodeId, + depth: u32, + start: C, + end: C, + is_competitive: bool, + left: Box, + right: Box, + }, +} + +// ─── Compile-time safety ──────────────────────────────────── + +/// Static assertion that `SpectralSentinel` is `Send + Sync`. +const _: () = { + const fn assert_send_sync() {} + assert_send_sync::>(); +}; + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::NoiseSchedule; + + /// Values confined to one leading nibble, so two calls with different + /// nibbles populate two well-separated regions of the domain. + fn values(nibble: u128, count: u128) -> Vec { + (1..=count).map(|i| (nibble << 124) | i).collect() + } + + /// Build a sentinel whose online competitive cells have fired at least one coordination context. + fn sentinel_with_coordination_context() -> SpectralSentinel { + let cfg = SentinelConfig:: { + max_rank: 4, + forgetting_factor: 0.90, + analysis_k: 16, + split_threshold: 10, + noise_schedule: NoiseSchedule::Explicit(vec![5]), + noise_batch_size: 4, + noise_seed: Some(42), + background_warming: false, + ..SentinelConfig::::default() + }; + let mut sentinel = SpectralSentinel::new(cfg).unwrap(); + let batch = [values(0xF, 4), values(0x1, 4)].concat(); + + for _ in 0..40 { + if sentinel.ingest(&batch).health.active_coordination_contexts > 0 { + return sentinel; + } + } + + panic!("the fixture must bring at least one context to life"); + } + + /// Return one live context, its two child subtrees, and one competitive member from each subtree. + fn context_branches(sentinel: &SpectralSentinel) -> (GNodeId, GNodeId, GNodeId, GNodeId, GNodeId) { + let context = *sentinel.coordination.keys().next().expect("the fixture has a context"); + let children = sentinel.graph.gnode_children(context).expect("a context is internal"); + let left = children.left.expect("a context has a left subtree"); + let right = children.right.expect("a context has a right subtree"); + let member_in = |subtree| { + sentinel + .cells + .iter() + .find_map(|(&gnode, cell)| { + (cell.is_competitive && (gnode == subtree || sentinel.graph.is_ancestor_of(subtree, gnode))).then_some(gnode) + }) + .expect("each context subtree has an online competitive member") + }; + + (context, left, right, member_in(left), member_in(right)) + } + + /// A coordination pass with no cell scores fires no context, but it preserves every context whose two subtrees still contain online competitive cells. Learned coordination state belongs to that membership and survives a quiet batch. + /// + /// ´claim:coordination:a-quiet-batch-preserves-contexts-while-their-competitive-membership-stands´ + /// ´test:unit:a-scoreless-coordination-pass-keeps-contexts-while-membership-stands´ + #[test] + fn a_scoreless_coordination_pass_keeps_contexts_while_membership_stands() { + let mut sentinel = sentinel_with_coordination_context(); + let contexts_before: BTreeSet<_> = sentinel.coordination.keys().copied().collect(); + + let reports = sentinel.propagate_coordination_from_root(&BTreeMap::new()); + + assert!(reports.is_empty(), "nothing scored, so no context reports"); + assert_eq!( + sentinel.coordination.keys().copied().collect::>(), + contexts_before, + "quiet traffic does not change competitive membership", + ); + } + + /// When only one region contributes a score, the shared context does not fire but remains allocated because the competitive member in the quiet subtree is still online. A later two-sided batch resumes the same learned context instead of warming a replacement. + /// + /// ´claim:coordination:a-one-sided-batch-does-not-retire-a-context-owned-by-two-sided-membership´ + /// ´test:unit:a-one-sided-scoring-pass-keeps-the-membership-context´ + #[test] + fn a_one_sided_scoring_pass_keeps_the_membership_context() { + let mut sentinel = sentinel_with_coordination_context(); + let (context, _left, _right, left_member, _right_member) = context_branches(&sentinel); + let scores = BTreeMap::from([(left_member, [0.25; 4])]); + + let reports = sentinel.propagate_coordination_from_root(&scores); + + assert!(reports.is_empty(), "one scoring subtree cannot fire the shared context"); + assert!( + sentinel.coordination.contains_key(&context), + "the quiet subtree remains a member, so its context must retain learned state", + ); + } + + /// A context is destroyed as soon as one of its subtrees contains no online competitive member. Retention follows the group that owns the learned state, so a topology that no longer represents that group cannot keep its tracker. + /// + /// ´claim:coordination:a-context-is-destroyed-when-either-subtree-loses-its-last-online-competitive-member´ + /// ´test:unit:a-context-is-destroyed-when-a-subtree-leaves-membership´ + #[test] + fn a_context_is_destroyed_when_a_subtree_leaves_membership() { + let mut sentinel = sentinel_with_coordination_context(); + let (context, _left, right, left_member, _right_member) = context_branches(&sentinel); + for (&gnode, cell) in &mut sentinel.cells { + if gnode == right || sentinel.graph.is_ancestor_of(right, gnode) { + cell.is_competitive = false; + } + } + let scores = BTreeMap::from([(left_member, [0.25; 4])]); + + let reports = sentinel.propagate_coordination_from_root(&scores); + + assert!(reports.is_empty(), "the departed subtree cannot contribute a report"); + assert!( + !sentinel.coordination.contains_key(&context), + "a context cannot outlive the two-sided membership that owns it", + ); + } + + /// The active tracker counts describe the cells that are online, not the + /// cells the selector has decided to pay for. The two differ whenever a + /// cell is still warming: the selection names it, but it has no tracker + /// yet and cannot have produced anything, so counting it active reports a + /// cell as working for as many batches as its warm-up lasts. Reading the + /// figures from the selection made that the normal case under background + /// warming, where the drain no longer happens inside the ingest that + /// created the cell. + /// + /// ´claim:health:the-active-counts-describe-the-online-cells-and-not-the-cells-the-selector-has-paid-for´ + /// ´test:unit:active-counts-exclude-cells-still-warming´ + #[test] + fn active_counts_exclude_cells_still_warming() { + let cfg = SentinelConfig:: { + max_rank: 4, + forgetting_factor: 0.90, + analysis_k: 16, + split_threshold: 10, + noise_schedule: NoiseSchedule::Explicit(vec![0, 400]), + noise_batch_size: 2, + noise_seed: Some(42), + background_warming: true, + ..SentinelConfig::::default() + }; + let mut sentinel: SpectralSentinel = SpectralSentinel::new(cfg).unwrap(); + + let batch = [values(0xF, 4), values(0x1, 4)].concat(); + for _ in 0..12 { + sentinel.ingest(&batch); + } + + let health = sentinel.health(); + assert!( + health.warming_trackers > 0, + "the long schedule must leave cells warming for this comparison to mean anything" + ); + + // Every counted tracker is one of the online cells. + assert_eq!(health.active_trackers, sentinel.cells.len()); + assert_eq!( + health.active_competitive_trackers + health.active_ancestor_trackers + 1, + health.active_trackers, + "the online cells are the competitive ones, the ancestors, and the root" + ); + + // And the selection is strictly larger, because it also names the + // cells that are still being warmed. + assert!( + sentinel.analysis_set.competitive_count() > health.active_competitive_trackers, + "a cell still warming is named by the selection and is not yet active" + ); + assert!(health.investment_set_size > health.active_trackers); + } + + /// The semi-internal count is read from the graph rather than left at a + /// constant. Semi-internal nodes are a reachable state — an eviction that + /// takes one child of a pair leaves the parent with a single subdivided + /// half — and they sit on the contour, so a figure fixed at zero is wrong + /// exactly when the structure is being reshaped, which is when a reader + /// would look at it. + /// + /// ´claim:health:the-semi-internal-count-is-read-from-the-graph-rather-than-fixed-at-zero´ + /// ´test:unit:the-semi-internal-count-follows-the-graph´ + #[test] + fn the_semi_internal_count_follows_the_graph() { + let collapse_cfg = SentinelConfig:: { + split_threshold: 5, + d_create: 3, + d_evict: 4, + budget: 10, + noise_schedule: NoiseSchedule::Explicit(vec![]), + noise_seed: Some(42), + background_warming: false, + ..SentinelConfig::::default() + }; + let mut counter_sentinel: SpectralSentinel = SpectralSentinel::new(collapse_cfg).unwrap(); + let traffic = [(0, 6), (128, 6), (64, 6), (192, 6), (32, 6), (96, 6), (160, 6), (224, 6)]; + let mut saw_split = false; + let mut saw_terminal_parent_collapse = false; + + for (coord, delta) in traffic { + let nodes_before = counter_sentinel.graph.node_count(); + let terminals_before = counter_sentinel.graph.terminal_count(); + let mutations_before = counter_sentinel.graph.structural_mutation_counts(); + + counter_sentinel.graph.observe(coord, delta); + + let nodes_after = counter_sentinel.graph.node_count(); + let terminals_after = counter_sentinel.graph.terminal_count(); + let mutations_after = counter_sentinel.graph.structural_mutation_counts(); + let expected_splits = mutations_after.splits - mutations_before.splits; + let expected_evictions = mutations_after.evictions - mutations_before.evictions; + let expected_restorations = mutations_after.restorations - mutations_before.restorations; + let (splits, net_removals, reported_terminals) = counter_sentinel.take_structural_mutation_counts(); + + assert_eq!(splits, u32::try_from(expected_splits).unwrap_or(u32::MAX)); + assert_eq!( + net_removals, + u32::try_from(expected_evictions.saturating_sub(expected_restorations)).unwrap_or(u32::MAX), + ); + assert_eq!(reported_terminals, terminals_after); + saw_split |= expected_splits > 0; + + let expected_nodes = i128::from(nodes_before) + i128::from(expected_splits) + i128::from(expected_restorations) + - i128::from(expected_evictions); + assert_eq!(i128::from(nodes_after), expected_nodes); + + // Each bisection creates two children and adds one terminal. An + // eviction ordinarily removes one terminal, except when it removes + // the last child of a semi-internal parent and makes that parent a + // terminal. The difference below counts exactly those collapses. + let terminals_without_collapse = + i128::from(terminals_before) + i128::from(expected_splits / 2) + i128::from(expected_restorations) + - i128::from(expected_evictions); + let collapsed_parents = i128::from(terminals_after) - terminals_without_collapse; + assert!(collapsed_parents >= 0); + + if saw_split && collapsed_parents > 0 { + assert_eq!(collapsed_parents, 1, "this interval collapses one parent"); + assert!(expected_evictions > 0); + saw_terminal_parent_collapse = true; + break; + } + } + assert!( + saw_terminal_parent_collapse, + "the tight budget must return a split parent to one terminal: \ + split={saw_split}, nodes={}, terminals={}, depth_create={}, depth_evict={}, mutations={:?}", + counter_sentinel.graph.node_count(), + counter_sentinel.graph.terminal_count(), + counter_sentinel.graph.depth_create(), + counter_sentinel.graph.depth_evict(), + counter_sentinel.graph.structural_mutation_counts(), + ); + + let cfg = SentinelConfig:: { + max_rank: 4, + forgetting_factor: 0.90, + analysis_k: 16, + split_threshold: 5, + budget: 200, + noise_schedule: NoiseSchedule::Explicit(vec![1]), + noise_batch_size: 2, + noise_seed: Some(42), + background_warming: false, + ..SentinelConfig::::default() + }; + let mut sentinel: SpectralSentinel = SpectralSentinel::new(cfg).unwrap(); + let mut saw_semi_internal = false; + let mut saw_removal = false; + for nibble in 0..16u128 { + // Spread within the nibble so the region is refined rather than + // merely visited, and keep the budget under pressure so the graph + // has to evict as well as split. + let batch: Vec = (0u128..500).map(|i| (nibble << 124) | (i << 100)).collect(); + let report = sentinel.ingest(&batch); + + assert_eq!( + report.health.semi_internal_count, + sentinel.graph.semi_internal_count() as usize, + "the reported figure is the graph's own count" + ); + if report.health.semi_internal_count > 0 { + saw_semi_internal = true; + } + if report.contour.net_removals_since_last_report > 0 { + saw_removal = true; + } + } + + assert!(saw_removal, "the budget must stay tight enough to evict"); + assert!( + saw_semi_internal, + "an eviction that takes one child of a pair leaves a half-subdivided node, \ + so the figure must be non-zero somewhere in this run" + ); + } +} diff --git a/packages/sentinel/src/sentinel/staging.rs b/packages/sentinel/src/sentinel/staging.rs new file mode 100644 index 000000000..976357a1f --- /dev/null +++ b/packages/sentinel/src/sentinel/staging.rs @@ -0,0 +1,1466 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +//! Staging area for deferred cell warm-up (S1, §ALGO S-11.6). +//! +//! New analysis cells are enqueued here for background noise injection +//! instead of being warmed inline during `ingest()`. This decouples +//! cell-creation latency from the observation hot path. +//! +//! # Lifecycle +//! +//! 1. **Enqueue** — `reconcile_analysis_set()` creates a `CellState` and +//! enqueues it with `staging.enqueue(gnode, cell, target_rounds)`. +//! 2. **Warm** — the background thread (or synchronous drain) picks the +//! highest-priority cell (by volume) and injects noise batches. +//! 3. **Ready** — when a cell completes all its target rounds, it moves +//! to the ready queue. +//! 4. **Promote** — `take_ready()` returns completed cells for insertion +//! into the live `cells` map. +//! +//! # Thread safety (Step 3) +//! +//! When background warming is enabled, the staging area lives behind +//! `Arc>`. The background thread takes cells out +//! via [`take_highest_priority`] (moving them to `in_flight`), does +//! the expensive noise injection *without* holding the lock, then +//! returns the cell via [`finish_warming`] or [`return_warming`]. +//! The main thread can enqueue, evict, and promote while the +//! background thread is working. +//! +//! # Test index +//! +//! | Test | Area | Claim | +//! |------|------|-------| +//! | [`warming_cell_is_ready_when_complete`] | staging | A cell leaves the warming set only once it has served the whole schedule its depth called for: one round short and it is still warming. The target is a promise about how much noise the tracker's baselines were built from, so honouring it partially would put a half-formed reference into the live set. | +//! | [`warming_cell_is_ready_when_over_target`] | staging | cites (´claim:staging:a-cell-is-ready-only-once-it-has-completed-its-target-rounds´) | +//! | [`enqueue_zero_rounds_goes_directly_to_ready`] | staging | A cell whose schedule asks for no noise at all never enters the warming set: it is placed straight on the ready queue and can be promoted on the next pass. Warming is work done only where the schedule says it is needed, so a zero-round cell costs nothing to stage and waits for nothing. | +//! | [`enqueue_with_rounds_goes_to_warming`] | staging | A cell with rounds still to serve waits in the warming set, out of the ready queue — and it counts as present in the staging area from the moment it is enqueued. Presence is what stops the reconciler enqueuing the same cell twice while its warm-up is still outstanding. | +//! | [`take_ready_empties_queue`] | staging | Collecting the ready cells hands them over whole and leaves the queue empty behind them. Promotion moves ownership rather than copying it, so a cell cannot be promoted into the live map twice however often the observation path drains the staging area. | +//! | [`remove_from_warming`] | staging | Eviction finds a cell wherever it currently sits — here part-warmed, with rounds still outstanding — and afterwards the area no longer reports it as present. A cell that has left the analysis set must stop consuming warming effort immediately rather than at the end of its schedule. | +//! | [`remove_from_ready`] | staging | cites (´claim:staging:eviction-reaches-a-cell-in-whichever-state-it-is-being-held´) | +//! | [`remove_nonexistent_returns_false`] | staging | Asking to evict a cell the area never held is answered with "nothing removed" rather than a fault. Eviction is driven by graph rebalancing, which knows what left the analysis set but not which of those cells were ever staged, so a miss has to be an ordinary outcome. | +//! | [`clear_empties_everything`] | staging | Clearing empties every holding at once — warming, ready and in-flight alike — so that after a reset the total is zero rather than a residue in whichever state escaped the sweep. A sentinel being reset must not promote cells warmed against a model it has just discarded. | +//! | [`warm_one_batch_completes_single_round_cell`] | staging | A warming step that completes a cell's last round finalises it and moves it to the ready queue in the same step, so the cell is never left sitting complete but unclaimed. Finalisation is where the drift reference is seeded from the baselines the noise just built and the accumulated evidence is cleared — the injected noise must shape what counts as normal without itself counting as anomalous history. | +//! | [`warm_one_batch_returns_false_when_empty`] | staging | A warming step with nothing to warm reports that it did no work rather than failing or fabricating a round. The background thread drives this call in a loop, so "no work" is the signal that lets it go idle instead of spinning. | +//! | [`warm_one_batch_incremental_progress`] | staging | Warming advances one round per step: a cell needing several rounds stays in the warming set across the intermediate calls and moves to ready only on the step that finishes it. Splitting the work this way is the whole point of deferring warm-up — the cost of bringing a new cell online is spread over many steps instead of landing inside one observation call. | +//! | [`warm_one_batch_picks_highest_volume`] | staging | When several cells are waiting, the step spends its round on the one carrying the most traffic, leaving the quieter cell still warming. Volume is the cached importance of the backing graph node, so the cells the host is most likely to be asking about come online first, and — because a busy ancestor outweighs its own descendants — ancestors tend to arrive before the cells beneath them. | +//! | [`warm_one_batch_warms_the_ancestor_before_the_cell_beneath_it`] | staging | Equal volumes send the round to the shallower cell, whichever identifier that cell happens to hold. The tie is the ordinary case for a path node whose whole accumulation is the single cell below it, and neither half of the identifier reading survives it. A comparison on volume alone keeps the last of the equal maxima, which is the largest identifier, and in the ordinary allocation order that is the cell beneath. Resolving the tie toward the smallest identifier instead inverts the rule the other way round, because the arena hands a freed slot out again and an ancestor can hold the larger identifier while the cell created beneath it holds the smaller. Depth is what carries the rule through both, and spending the round on the descendant lets it finish and be promoted while the chain above it is still warming — the gap the volume ordering exists to close. This step and the two drains serve one queue and must not disagree about which cell comes next. | +//! | [`contains_checks_both_warming_and_ready`] | staging | Presence is answered across every state a staged cell can occupy: a cell still warming and a cell already waiting to be promoted both answer yes. The caller asking is deciding whether a cell needs creating, and it must not be told "absent" merely because the cell has moved on within the staging area. | +//! | [`take_highest_priority_moves_to_in_flight`] | staging | Checking a cell out for background work takes the busiest waiting cell and marks it in flight, leaving the others warming; while it is away it still counts as present in the staging area. That is what makes the expensive noise injection safe to do without holding the lock: the main thread can see the cell is spoken for even though the warming map no longer holds it. | +//! | [`equal_volumes_take_the_shallower_cell_first`] | staging | Equal volumes resolve to the shallower cell rather than the deeper one. A tie is the ordinary case for a pair of siblings the moment they are created, and the rule the queue exists to serve is that a busy ancestor is warmed before the cells beneath it. Identifiers cannot carry that rule on their own: the graph's arena hands a freed slot out again, so a cell created into a recycled slot holds a smaller identifier than an ancestor allocated before it. This path and the synchronous drain are two ways of serving one queue, so they must not disagree about which cell comes next. | +//! | [`a_restored_descendant_does_not_overtake_its_warming_ancestor`] | staging | cites (´claim:staging:equal-volumes-resolve-to-the-shallower-cell-so-both-drains-agree´) | +//! | [`the_synchronous_drain_warms_a_restored_descendants_ancestor_first`] | staging | cites (´claim:staging:equal-volumes-resolve-to-the-shallower-cell-so-both-drains-agree´) | +//! | [`an_in_flight_cell_still_counts_as_a_competitive_target`] | staging | A cell checked out for background warming is still a warming cell, so it still counts among the competitive targets being warmed. Checking a cell out is how the expensive work is done off the lock, not a change in what the cell is; a count that dropped it would fall precisely when the work was happening, understating what is in progress by the number of cells actually in progress. The flag is recorded at checkout, so counting it needs nothing from a cell another thread is holding. | +//! | [`an_in_flight_ancestor_cell_is_not_a_competitive_target`] | staging | cites (´claim:staging:a-cell-checked-out-for-warming-still-counts-among-the-competitive-targets´) | +//! | [`ready_competitive_target_stays_in_warming_count_until_promotion`] | staging | A completed competitive cell can reach the ready queue after an ingest has passed its promotion point. It remains inside the warm-up pipeline and outside the producing set until the next promotion, so both health counts continue to include it while it waits. | +//! | [`a_newly_queued_cell_carries_its_volume`] | staging | A cell joining the queue carries its volume immediately. The worker is stopped before enqueueing while deferred staging remains selected, so every queued cell is available for the volume assertions regardless of thread scheduling. | +//! | [`return_warming_restores_cell`] | staging | A cell handed back unfinished rejoins the warming set and stops being in flight, with its accumulated rounds intact. Background warming can therefore be interrupted between rounds — the thread need not carry a cell to completion once it has taken it. | +//! | [`a_returned_cell_carries_the_volume_the_graph_has_now`] | staging | A cell handed back rejoins the queue at the volume the graph has now, not the one it carried out. The refresh runs on the main thread while the worker holds the cell, and the checkout spans exactly the noise injection — the expensive part of the pass, and so the part an ingest is most likely to overlap. Restoring the carried volume would leave the busiest cell in the area queued at its pre-ingest importance, and the next checkout — the one decision the cached volume exists to make — would go to a rival the traffic has already passed. | +//! | [`finish_warming_moves_to_ready`] | staging | A cell handed back finished joins the ready queue instead of the warming set, and is no longer in flight. Which of the two return paths the background thread takes is what decides the cell's fate, so completion is declared by the worker that did the rounds rather than re-derived by the staging area. | +//! | [`eviction_of_in_flight_cell_discards_on_return`] | staging | A cell evicted while a background thread was working on it is discarded when it comes back, not resurrected: the eviction sweep removes its in-flight mark, and a return with no mark to clear keeps nothing. The warming work already spent is lost, which is the deliberate trade — a cell that has left the analysis set must not reappear in it because a thread happened to be holding it. | +//! | [`gnode_set_includes_all_states`] | staging | cites (´claim:staging:presence-is-answered-across-every-state-a-staged-cell-can-occupy´) | + +use std::collections::{BTreeMap, BTreeSet}; + +use rand::rngs::SmallRng; +use torrust_mudlark::{Coordinate, GNodeId, GvGraph, Inspectable}; + +use super::{CellState, generate_noise_batch}; + +// ─── WarmingCell ──────────────────────────────────────────── + +/// Progress state of a cell being warmed in the background. +pub struct WarmingCell { + /// The cell under construction. + pub cell: CellState, + + /// Total noise rounds required (from `NoiseSchedule::rounds_for_depth()`). + pub target_rounds: u32, + + /// Rounds completed so far. + pub completed_rounds: u32, + + /// Cached volume (g.sum) for priority ordering, erased to `f64`. + /// Updated by the main thread during `reconcile_analysis_set()` via + /// [`StagingArea::update_volumes`] — for a cell a worker has checked out, + /// through the in-flight record that carries the refresh back to it. + pub volume: f64, +} + +impl WarmingCell { + /// Whether this cell has completed all its target noise rounds. + #[must_use] + pub const fn is_ready(&self) -> bool { + self.completed_rounds >= self.target_rounds + } +} + +// ─── InFlightCell ─────────────────────────────────────────── + +/// What the staging area keeps about a cell while a worker holds it. +/// +/// The worker owns the cell itself, so everything reconciliation learns about +/// it in the meantime is recorded here and copied back when the cell returns. +/// Both fields are the main thread's to write and the return paths' to read; +/// neither needs anything from the tracker the worker is holding. +struct InFlightCell { + /// Selection flag as of the last reconciliation. + is_competitive: bool, + + /// Cached volume as of the last refresh from the graph. + volume: f64, +} + +/// Read a G-node's volume (g.sum) as the priority queue caches it. +/// +/// Every refresh goes through this one reading, so a cell in flight and a cell +/// waiting in the warming map cannot come to hold volumes derived differently +/// — including for a node the graph no longer has, which is nought to both. +fn volume_of(graph: &GvGraph, gnode: GNodeId) -> f64 { + graph.gnode_info(gnode).map_or(0.0, |info| info.sum.to_f64_approx()) +} + +// ─── StagingArea ──────────────────────────────────────────── + +/// Staging area for cells undergoing deferred noise warm-up. +/// +/// In synchronous mode this is owned directly by the sentinel. +/// In background-warming mode it lives behind `Arc>` +/// and is shared between the main thread and the warming thread. +/// +/// `ingest()` reads only the `ready` queue (via `take_ready()`). +pub struct StagingArea { + /// Cells currently being warmed, keyed by `GNodeId`. + /// + /// `BTreeMap` for deterministic iteration order (ADR-S-005). + warming: BTreeMap>, + + /// Cells that completed warming since the last promotion. + ready: Vec<(GNodeId, CellState)>, + + /// Cells currently checked out by the background warming thread. + /// + /// These are logically still "in the staging area" — they have + /// been temporarily removed from `warming` so the thread can + /// work on them without holding the lock. [`contains`] and + /// [`retain_in_set`] account for them. + /// Cells checked out for background warming, each carrying what + /// reconciliation has learned about the cell since it left: the latest + /// competitive flag, so the reported count of competitive targets stays + /// current while it is away, and the latest cached volume, so it rejoins + /// the queue at the priority its traffic has now rather than the one it + /// carried out. + in_flight: BTreeMap, +} + +impl StagingArea { + /// Create an empty staging area. + pub const fn new() -> Self { + Self { + warming: BTreeMap::new(), + ready: Vec::new(), + in_flight: BTreeMap::new(), + } + } + + // ── Enqueue / promote ─────────────────────────────── + + /// Enqueue a newly created cell for background warming. + /// + /// The cell is added to the warming set with zero completed rounds. + /// If `target_rounds` is zero, the cell goes directly to the ready + /// queue (no warming needed). + pub fn enqueue(&mut self, gnode: GNodeId, cell: CellState, target_rounds: u32) { + if target_rounds == 0 { + self.ready.push((gnode, cell)); + return; + } + + self.warming.insert( + gnode, + WarmingCell { + cell, + target_rounds, + completed_rounds: 0, + volume: 0.0, + }, + ); + } + + /// Take all ready cells for promotion into the live cells map. + /// + /// The caller inserts them into `self.cells` and fires coordination + /// warm-up as needed. + pub fn take_ready(&mut self) -> Vec<(GNodeId, CellState)> { + std::mem::take(&mut self.ready) + } + + // ── Background-thread interface (Step 3) ──────────── + + /// Remove the highest-priority warming cell for background processing. + /// + /// The cell moves from `warming` to `in_flight`. The caller is + /// responsible for returning it via [`return_warming`] or + /// [`finish_warming`]. + /// + /// Priority is by cached `volume` (largest first), ties resolving to the + /// shallower cell and then to the smaller `GNodeId`, so that this path and + /// the synchronous drain order equal-volume cells the same way, ancestors + /// before the cells beneath them. + /// + /// Depth carries that rule because the identifier cannot: the graph's + /// arena reuses freed slots, so a cell created into a recycled slot holds + /// a smaller identifier than an ancestor allocated before it, and an + /// identifier tie-break would warm such a descendant first. + pub fn take_highest_priority(&mut self) -> Option<(GNodeId, WarmingCell)> { + let (&gnode, _) = self.warming.iter().max_by(|(a_gnode, a), (b_gnode, b)| { + a.volume + .partial_cmp(&b.volume) + .unwrap_or(std::cmp::Ordering::Equal) + // Reversed, because `max_by` keeps the last of equal maxima + // and the map iterates in ascending id order: comparing + // backwards makes the shallowest cell — and among cells of + // one depth, the smallest id — the maximum. + .then_with(|| b.cell.depth.cmp(&a.cell.depth)) + .then_with(|| b_gnode.cmp(a_gnode)) + })?; + let wc = self.warming.remove(&gnode)?; + self.in_flight.insert( + gnode, + InFlightCell { + is_competitive: wc.cell.is_competitive, + volume: wc.volume, + }, + ); + Some((gnode, wc)) + } + + /// Return an in-flight cell to the warming map (not yet ready). + /// + /// The record the checkout left behind is what the cell rejoins the queue + /// with: the worker was away for the whole of an observation pass, so the + /// flag and the volume it carried out are both older than the ones + /// reconciliation has since recorded here. Restoring the carried volume + /// instead would let an ingest that landed mid-warm-up leave the cell + /// queued at its pre-ingest importance, and the next checkout would be + /// decided by traffic the graph has already moved past. + /// + /// If the cell was evicted while in-flight (removed from + /// `in_flight` by [`retain_in_set`]), the cell is silently + /// discarded — the work is wasted but correctness is preserved. + pub fn return_warming(&mut self, gnode: GNodeId, wc: WarmingCell) { + if let Some(record) = self.in_flight.remove(&gnode) { + let mut wc = wc; + wc.cell.is_competitive = record.is_competitive; + wc.volume = record.volume; + self.warming.insert(gnode, wc); + } + // else: evicted while in-flight — discard. + } + + /// Complete an in-flight cell and add it to the ready queue. + /// + /// Only the flag is copied back: the ready queue is served in the order it + /// was filled, so a completed cell has no further use for a priority. + /// + /// If the cell was evicted while in-flight, it is silently + /// discarded. + pub fn finish_warming(&mut self, gnode: GNodeId, cell: CellState) { + if let Some(record) = self.in_flight.remove(&gnode) { + let mut cell = cell; + cell.is_competitive = record.is_competitive; + self.ready.push((gnode, cell)); + } + // else: evicted while in-flight — discard. + } + + /// Drop every in-flight record, reporting how many were dropped. + /// + /// Only the background worker checks cells out, and only its owner calls + /// this, and only once that worker has been joined. With no worker left to + /// run, every record standing here names a cell whose tracker went down + /// with the thread that was holding it: nothing will ever return it through + /// [`return_warming`] or [`finish_warming`], and the record is the sole + /// remaining trace of it. + /// + /// Dropping the record is what lets the cell be built again. A record left + /// standing makes [`contains`] answer for a cell that nothing holds, so + /// reconciliation reads the identifier as already staged and never enqueues + /// it — the cell is absent from the staging area and from the producing set + /// at the same time, permanently. The counts are the lesser half of it: + /// [`total_count`] and [`warming_competitive_count`] would go on reporting + /// warm-up work that no thread is doing. + /// + /// The progress those cells had accumulated is lost with them, which is the + /// same trade [`retain_in_set`] already makes for a cell evicted while in + /// flight: a fresh warm-up is recoverable, a cell that never comes back is + /// not. + pub(crate) fn abandon_in_flight(&mut self) -> usize { + let abandoned = self.in_flight.len(); + self.in_flight.clear(); + abandoned + } + + /// Record a checkout for a cell whose tracker the caller holds, or has + /// destroyed, without routing it through the warming map. + /// + /// Reproduces the half of [`take_highest_priority`] that survives a worker + /// which unwinds before returning its cell: the record stands, and the cell + /// it names is gone. A test cannot reach that state by waiting for a panic + /// to land inside the few instructions where it strands a cell — whether it + /// lands there is the scheduler's decision, not the test's — so it builds + /// the state instead. + #[cfg(test)] + pub(crate) fn record_checkout_for_test(&mut self, gnode: GNodeId, is_competitive: bool) { + let volume = self.warming.remove(&gnode).map_or(0.0, |wc| wc.volume); + self.in_flight.insert(gnode, InFlightCell { is_competitive, volume }); + } + + // ── Warm-one-batch (used by background thread) ────── + + /// Pick the highest-priority warming cell (largest `volume`) and + /// inject one noise batch. Returns `true` if a batch was injected. + /// + /// Used by the Step 1 unit tests and by the synchronous fallback + /// in [`warm_one_batch`] with graph-volume updates. The background + /// thread uses [`take_highest_priority`] instead to avoid holding + /// the lock during expensive noise injection. + /// + /// Ties resolve to the shallower cell and then to the smaller `GNodeId`, + /// the same order [`take_highest_priority`] and the synchronous drain + /// keep, so that an ancestor is warmed before the cells beneath it + /// whichever path serves the queue. + #[allow(dead_code)] // used in tests + pub fn warm_one_batch( + &mut self, + graph: &GvGraph, + batch_size: usize, + rng: &mut SmallRng, + ) -> bool { + // Find the highest-priority warming cell. + let Some((&gnode, _)) = self.warming.iter().max_by(|(a_gnode, a), (b_gnode, b)| { + a.volume + .partial_cmp(&b.volume) + .unwrap_or(std::cmp::Ordering::Equal) + // Reversed, because `max_by` keeps the last of equal maxima + // and the map iterates in ascending id order: comparing + // backwards makes the shallowest cell — and among cells of + // one depth, the smallest id — the maximum. + .then_with(|| b.cell.depth.cmp(&a.cell.depth)) + .then_with(|| b_gnode.cmp(a_gnode)) + }) else { + return false; + }; + + // Remove temporarily to satisfy the borrow checker, then re-insert. + let Some(mut wc) = self.warming.remove(&gnode) else { + return false; + }; + + // Generate and inject one noise batch. + let noise = generate_noise_batch(wc.cell.width, batch_size, rng); + let slices: Vec<&[f64]> = noise.iter().map(Vec::as_slice).collect(); + #[allow(clippy::cast_possible_truncation)] // depth ≤ 128, fits in u8 + wc.cell.tracker.observe(&slices, wc.cell.depth as u8, true); + wc.completed_rounds += 1; + + if wc.is_ready() { + // Finalize: seed CUSUM slow from baselines and reset. + wc.cell.tracker.seed_cusum_slow_from_baselines(); + wc.cell.tracker.reset_cusum(); + wc.cell.tracker.reset_clip_pressure(); + self.ready.push((gnode, wc.cell)); + } else { + self.warming.insert(gnode, wc); + } + + // Update volumes from the graph for remaining warming cells. + for (&g, wc) in &mut self.warming { + wc.volume = volume_of(graph, g); + } + + true + } + + // ── Query / eviction ──────────────────────────────── + + /// Check if a `GNodeId` is currently in the staging area + /// (warming, ready, or in-flight). + pub fn contains(&self, gnode: GNodeId) -> bool { + self.warming.contains_key(&gnode) || self.in_flight.contains_key(&gnode) || self.ready.iter().any(|(g, _)| *g == gnode) + } + + /// Set of all `GNodeId`s currently in the staging area. + /// + /// Useful for diagnostics and testing. + #[allow(dead_code)] // used in tests; no longer needed for routing + pub fn gnode_set(&self) -> BTreeSet { + let mut set: BTreeSet = self.warming.keys().copied().collect(); + set.extend(self.in_flight.keys()); + for (g, _) in &self.ready { + set.insert(*g); + } + set + } + + /// Remove a cell from the staging area (e.g. evicted by graph rebalance). + /// + /// Returns `true` if the cell was found and removed. + #[allow(dead_code)] // used in tests, will be called from Step 3 thread lifecycle + pub fn remove(&mut self, gnode: GNodeId) -> bool { + if self.warming.remove(&gnode).is_some() { + return true; + } + if self.in_flight.remove(&gnode).is_some() { + return true; + } + let before = self.ready.len(); + self.ready.retain(|(g, _)| *g != gnode); + self.ready.len() < before + } + + /// Retain only cells whose `GNodeId` is in `keep`. + /// + /// Evicts warming, ready, *and* in-flight cells that exited the + /// analysis set. In-flight cells evicted here will be silently + /// discarded when the background thread tries to return them + /// (the `in_flight` entry is gone, so [`return_warming`] / + /// [`finish_warming`] no-ops). + pub fn retain_in_set(&mut self, keep: &BTreeSet) { + self.warming.retain(|gnode, _| keep.contains(gnode)); + self.ready.retain(|(gnode, _)| keep.contains(gnode)); + self.in_flight.retain(|gnode, _| keep.contains(gnode)); + } + + /// Refresh the selection flag wherever a retained cell is staged. + /// + /// An in-flight record owns the current flag while the worker owns the + /// tracker. Both return paths copy this flag back before keeping the cell. + pub(crate) fn update_competitive(&mut self, gnode: GNodeId, is_competitive: bool) { + if let Some(wc) = self.warming.get_mut(&gnode) { + wc.cell.is_competitive = is_competitive; + } + if let Some(record) = self.in_flight.get_mut(&gnode) { + record.is_competitive = is_competitive; + } + for (ready_gnode, cell) in &mut self.ready { + if *ready_gnode == gnode { + cell.is_competitive = is_competitive; + } + } + } + + // ── Volume update ─────────────────────────────────── + + /// Update cached volumes for staged cells from the G-V Graph. + /// + /// Called by the main thread after G-V Graph observation so the + /// background thread can prioritise cells by current importance. + /// + /// A cell a worker has checked out is refreshed through its in-flight + /// record rather than skipped. The checkout takes the cell out of the + /// warming map for exactly as long as the noise injection runs, which is + /// the expensive part of a pass and so the part an ingest is most likely + /// to overlap; a refresh that reached only the waiting cells would leave + /// the cell most recently judged the busiest as the one cell whose + /// importance the queue never learns. + pub fn update_volumes(&mut self, graph: &GvGraph) { + for (&g, wc) in &mut self.warming { + wc.volume = volume_of(graph, g); + } + for (&g, record) in &mut self.in_flight { + record.volume = volume_of(graph, g); + } + } + + // ── Synchronous drain (Step 2 fallback) ───────────── + + /// Drain all warming cells to completion synchronously. + /// + /// Processes cells in descending g.sum (volume) order, matching + /// the background thread's priority rule (§ALGO S-11.6.2, + /// ADR-S-019). This ensures ancestors come online before + /// descendants even in synchronous mode. + #[allow(clippy::cast_possible_truncation)] // depth ≤ 128, fits u8 + pub fn drain_all_synchronous(&mut self, batch_size: usize, rng: &mut SmallRng) { + // Sort by cached volume (g.sum) descending, then by depth so an + // ancestor is warmed before the cells beneath it — a recycled arena + // slot can give a descendant the smaller identifier — and then by + // GNodeId for deterministic tie-breaking (ADR-S-005). + let mut gnodes: Vec<(GNodeId, f64, u32)> = self + .warming + .iter() + .map(|(&gnode, wc)| (gnode, wc.volume, wc.cell.depth)) + .collect(); + gnodes.sort_by(|a, b| { + b.1.partial_cmp(&a.1) + .unwrap_or(std::cmp::Ordering::Equal) + .then_with(|| a.2.cmp(&b.2)) + .then_with(|| a.0.cmp(&b.0)) + }); + + for (gnode, _, _) in gnodes { + let Some(mut wc) = self.warming.remove(&gnode) else { + continue; + }; + + // Warm to completion — same loop as inject_noise_into_cell(). + while !wc.is_ready() { + let noise = generate_noise_batch(wc.cell.width, batch_size, rng); + let slices: Vec<&[f64]> = noise.iter().map(Vec::as_slice).collect(); + wc.cell.tracker.observe(&slices, wc.cell.depth as u8, true); + wc.completed_rounds += 1; + } + + // Finalize: seed CUSUM slow from baselines and reset. + wc.cell.tracker.seed_cusum_slow_from_baselines(); + wc.cell.tracker.reset_cusum(); + wc.cell.tracker.reset_clip_pressure(); + self.ready.push((gnode, wc.cell)); + } + } + + // ── Count / clear ─────────────────────────────────── + + /// Number of cells currently warming (not yet ready, not in-flight). + #[allow(dead_code)] // used in tests + Step 6 health reporting + pub fn warming_count(&self) -> usize { + self.warming.len() + } + + /// Number of cells in the ready queue awaiting promotion. + #[allow(dead_code)] // used in tests + Step 6 health reporting + pub const fn ready_count(&self) -> usize { + self.ready.len() + } + + /// Number of cells currently checked out by the background thread. + #[allow(dead_code)] // used in tests + Step 6 health reporting + pub fn in_flight_count(&self) -> usize { + self.in_flight.len() + } + + /// Total cells in the staging area (warming + ready + in-flight). + pub fn total_count(&self) -> usize { + self.warming.len() + self.ready.len() + self.in_flight.len() + } + + /// Number of cells in the warm-up pipeline, ready and in-flight ones + /// included, that are competitive targets rather than ancestor-only. + /// + /// Used to populate `HealthReport::warming_competitive_targets` + /// (§ALGO S-14.11, ADR-S-019). + /// + /// A cell checked out for background warming is still being warmed — that + /// is what it was taken for — so excluding it made the figure disagree + /// with its own description and understate the work in progress by the + /// number of cells actually being worked on. The flag is recorded at + /// checkout and refreshed during reconciliation, so the count needs + /// nothing from a tracker another thread is holding. + pub fn warming_competitive_count(&self) -> usize { + let waiting = self.warming.values().filter(|wc| wc.cell.is_competitive).count(); + let ready = self.ready.iter().filter(|(_, cell)| cell.is_competitive).count(); + let in_flight = self.in_flight.values().filter(|record| record.is_competitive).count(); + waiting + ready + in_flight + } + + /// Whether there are any cells that need warming work. + /// + /// Returns `true` if `warming` is non-empty (excludes in-flight + /// and ready cells — those are already being processed or done). + pub fn has_warming_work(&self) -> bool { + !self.warming.is_empty() + } + + /// Clear all warming, ready, and in-flight cells (for `reset()`). + pub fn clear(&mut self) { + self.warming.clear(); + self.ready.clear(); + self.in_flight.clear(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::SentinelConfig; + use crate::sentinel::tracker::SubspaceTracker; + + fn make_cell(depth: u32) -> CellState { + let cfg = SentinelConfig::::default(); + let width = 128usize.saturating_sub(depth as usize); + CellState { + tracker: SubspaceTracker::new(width, &cfg, cfg.cusum_slow_decay), + depth, + width, + start: 0, + end: u128::MAX >> depth, + is_competitive: false, + } + } + + /// Create a graph with a split so we can extract multiple distinct `GNodeId`s. + /// + /// Returns `(graph, root, left_child, right_child)` where the children + /// are `Option` (will be `Some` after the split). + fn split_graph() -> (GvGraph, GNodeId, GNodeId, GNodeId) { + use torrust_mudlark::Config as GvConfig; + let config = GvConfig { + split_threshold: 2u64, + depth_create: 3, + depth_evict: 6, + budget: None, + alpha_relax: 0.75, + bounded_eviction: true, + }; + let mut graph: GvGraph = GvGraph::new(config); + let root = graph.g_root(); + + // Observe enough in both halves to trigger a split. + let lo = 0u128; + let hi = u128::MAX / 2 + 1; + for _ in 0..5 { + graph.observe(lo, 1u64); + graph.observe(hi, 1u64); + } + + let children = graph.gnode_children(root).expect("root must exist"); + let left = children.left.expect("root must have left child after split"); + let right = children.right.expect("root must have right child after split"); + (graph, root, left, right) + } + + /// A cell leaves the warming set only once it has served the whole + /// schedule its depth called for: one round short and it is still + /// warming. The target is a promise about how much noise the tracker's + /// baselines were built from, so honouring it partially would put a + /// half-formed reference into the live set. + /// + /// ´claim:staging:a-cell-is-ready-only-once-it-has-completed-its-target-rounds´ + /// ´test:unit:warming-cell-is-ready-when-complete´ + #[test] + fn warming_cell_is_ready_when_complete() { + let cell = make_cell(0); + let wc = WarmingCell { + cell, + target_rounds: 5, + completed_rounds: 4, + volume: 100.0, + }; + assert!(!wc.is_ready()); + + let cell2 = make_cell(0); + let wc2 = WarmingCell { + cell: cell2, + target_rounds: 5, + completed_rounds: 5, + volume: 100.0, + }; + assert!(wc2.is_ready()); + } + + /// Readiness is a threshold rather than an exact match: a cell that has + /// somehow run past its target is ready, not stuck. Nothing in the + /// pipeline has to guarantee that rounds land on the target exactly, so + /// an extra round of noise costs work and never a wedged cell. + /// + /// (´claim:staging:a-cell-is-ready-only-once-it-has-completed-its-target-rounds´) + /// ´test:unit:warming-cell-is-ready-when-over-target´ + #[test] + fn warming_cell_is_ready_when_over_target() { + let cell = make_cell(0); + let wc = WarmingCell { + cell, + target_rounds: 3, + completed_rounds: 10, + volume: 0.0, + }; + assert!(wc.is_ready()); + } + + /// A cell whose schedule asks for no noise at all never enters the + /// warming set: it is placed straight on the ready queue and can be + /// promoted on the next pass. Warming is work done only where the + /// schedule says it is needed, so a zero-round cell costs nothing to + /// stage and waits for nothing. + /// + /// ´claim:staging:a-cell-that-needs-no-warming-skips-the-queue-and-arrives-ready´ + /// ´test:unit:enqueue-zero-rounds-goes-directly-to-ready´ + #[test] + fn enqueue_zero_rounds_goes_directly_to_ready() { + let mut staging = StagingArea::::new(); + let (_, root, _, _) = split_graph(); + let cell = make_cell(0); + + staging.enqueue(root, cell, 0); + + assert_eq!(staging.warming_count(), 0); + assert_eq!(staging.ready_count(), 1); + + let ready = staging.take_ready(); + assert_eq!(ready.len(), 1); + assert_eq!(ready[0].0, root); + } + + /// A cell with rounds still to serve waits in the warming set, out of + /// the ready queue — and it counts as present in the staging area from + /// the moment it is enqueued. Presence is what stops the reconciler + /// enqueuing the same cell twice while its warm-up is still outstanding. + /// + /// ´claim:staging:a-cell-counts-as-staged-from-the-moment-it-is-enqueued-not-from-when-it-is-ready´ + /// ´test:unit:enqueue-with-rounds-goes-to-warming´ + #[test] + fn enqueue_with_rounds_goes_to_warming() { + let mut staging = StagingArea::::new(); + let (_, root, _, _) = split_graph(); + let cell = make_cell(0); + + staging.enqueue(root, cell, 5); + + assert_eq!(staging.warming_count(), 1); + assert_eq!(staging.ready_count(), 0); + assert!(staging.contains(root)); + } + + /// Collecting the ready cells hands them over whole and leaves the queue + /// empty behind them. Promotion moves ownership rather than copying it, + /// so a cell cannot be promoted into the live map twice however often + /// the observation path drains the staging area. + /// + /// ´claim:staging:collecting-the-ready-cells-hands-them-over-whole-and-leaves-the-queue-empty´ + /// ´test:unit:take-ready-empties-queue´ + #[test] + fn take_ready_empties_queue() { + let mut staging = StagingArea::::new(); + let (_, root, _, _) = split_graph(); + let cell = make_cell(0); + + staging.enqueue(root, cell, 0); + assert_eq!(staging.ready_count(), 1); + + let ready = staging.take_ready(); + assert_eq!(ready.len(), 1); + assert_eq!(staging.ready_count(), 0); + } + + /// Eviction finds a cell wherever it currently sits — here part-warmed, + /// with rounds still outstanding — and afterwards the area no longer + /// reports it as present. A cell that has left the analysis set must + /// stop consuming warming effort immediately rather than at the end of + /// its schedule. + /// + /// ´claim:staging:eviction-reaches-a-cell-in-whichever-state-it-is-being-held´ + /// ´test:unit:remove-from-warming´ + #[test] + fn remove_from_warming() { + let mut staging = StagingArea::::new(); + let (_, _, left, _) = split_graph(); + let cell = make_cell(1); + + staging.enqueue(left, cell, 10); + assert!(staging.contains(left)); + + assert!(staging.remove(left)); + assert!(!staging.contains(left)); + assert_eq!(staging.warming_count(), 0); + } + + /// The other end of the same reach: a cell already finished and waiting + /// on the ready queue is evicted just as a warming one is. Having + /// completed its warm-up buys a cell no claim on promotion once it has + /// left the analysis set. + /// + /// (´claim:staging:eviction-reaches-a-cell-in-whichever-state-it-is-being-held´) + /// ´test:unit:remove-from-ready´ + #[test] + fn remove_from_ready() { + let mut staging = StagingArea::::new(); + let (_, root, _, _) = split_graph(); + let cell = make_cell(0); + + staging.enqueue(root, cell, 0); + assert!(staging.contains(root)); + + assert!(staging.remove(root)); + assert!(!staging.contains(root)); + assert_eq!(staging.ready_count(), 0); + } + + /// Asking to evict a cell the area never held is answered with "nothing + /// removed" rather than a fault. Eviction is driven by graph rebalancing, + /// which knows what left the analysis set but not which of those cells + /// were ever staged, so a miss has to be an ordinary outcome. + /// + /// ´claim:staging:evicting-a-cell-the-area-never-held-reports-that-nothing-was-removed´ + /// ´test:unit:remove-nonexistent-returns-false´ + #[test] + fn remove_nonexistent_returns_false() { + let mut staging = StagingArea::::new(); + let (_, root, _, _) = split_graph(); + assert!(!staging.remove(root)); + } + + /// Clearing empties every holding at once — warming, ready and in-flight + /// alike — so that after a reset the total is zero rather than a residue + /// in whichever state escaped the sweep. A sentinel being reset must not + /// promote cells warmed against a model it has just discarded. + /// + /// ´claim:staging:clearing-empties-every-holding-at-once-so-a-reset-leaves-no-residue´ + /// ´test:unit:clear-empties-everything´ + #[test] + fn clear_empties_everything() { + let mut staging = StagingArea::::new(); + let (_, _, left, right) = split_graph(); + + staging.enqueue(left, make_cell(1), 5); + staging.enqueue(right, make_cell(1), 0); + + assert_eq!(staging.total_count(), 2); + + staging.clear(); + assert_eq!(staging.total_count(), 0); + assert_eq!(staging.warming_count(), 0); + assert_eq!(staging.ready_count(), 0); + } + + /// A warming step that completes a cell's last round finalises it and + /// moves it to the ready queue in the same step, so the cell is never + /// left sitting complete but unclaimed. Finalisation is where the drift + /// reference is seeded from the baselines the noise just built and the + /// accumulated evidence is cleared — the injected noise must shape what + /// counts as normal without itself counting as anomalous history. + /// + /// ´claim:staging:a-cell-whose-last-round-lands-is-finalised-and-moved-to-ready-in-the-same-step´ + /// ´test:unit:warm-one-batch-completes-single-round-cell´ + #[test] + fn warm_one_batch_completes_single_round_cell() { + use rand::SeedableRng; + let mut staging = StagingArea::::new(); + let (graph, _, left, _) = split_graph(); + let cell = make_cell(1); + + // Enqueue with target_rounds = 1: one batch should complete it. + staging.enqueue(left, cell, 1); + assert_eq!(staging.warming_count(), 1); + + let mut rng = SmallRng::seed_from_u64(42); + let warmed = staging.warm_one_batch(&graph, 64, &mut rng); + assert!(warmed); + + // Cell should have moved to ready. + assert_eq!(staging.warming_count(), 0); + assert_eq!(staging.ready_count(), 1); + + let ready = staging.take_ready(); + assert_eq!(ready.len(), 1); + assert_eq!(ready[0].0, left); + } + + /// A warming step with nothing to warm reports that it did no work + /// rather than failing or fabricating a round. The background thread + /// drives this call in a loop, so "no work" is the signal that lets it + /// go idle instead of spinning. + /// + /// ´claim:staging:a-warming-step-with-nothing-to-warm-reports-that-it-did-no-work´ + /// ´test:unit:warm-one-batch-returns-false-when-empty´ + #[test] + fn warm_one_batch_returns_false_when_empty() { + use rand::SeedableRng; + let mut staging = StagingArea::::new(); + let (graph, _, _, _) = split_graph(); + let mut rng = SmallRng::seed_from_u64(42); + + assert!(!staging.warm_one_batch(&graph, 64, &mut rng)); + } + + /// Warming advances one round per step: a cell needing several rounds + /// stays in the warming set across the intermediate calls and moves to + /// ready only on the step that finishes it. Splitting the work this way + /// is the whole point of deferring warm-up — the cost of bringing a new + /// cell online is spread over many steps instead of landing inside one + /// observation call. + /// + /// ´claim:staging:warming-advances-one-round-per-step-so-the-cost-of-a-new-cell-is-spread-out´ + /// ´test:unit:warm-one-batch-incremental-progress´ + #[test] + fn warm_one_batch_incremental_progress() { + use rand::SeedableRng; + let mut staging = StagingArea::::new(); + let (graph, _, left, _) = split_graph(); + let cell = make_cell(1); + + staging.enqueue(left, cell, 3); + let mut rng = SmallRng::seed_from_u64(42); + + // First batch: still warming. + assert!(staging.warm_one_batch(&graph, 64, &mut rng)); + assert_eq!(staging.warming_count(), 1); + assert_eq!(staging.ready_count(), 0); + + // Second batch: still warming. + assert!(staging.warm_one_batch(&graph, 64, &mut rng)); + assert_eq!(staging.warming_count(), 1); + assert_eq!(staging.ready_count(), 0); + + // Third batch: completes. + assert!(staging.warm_one_batch(&graph, 64, &mut rng)); + assert_eq!(staging.warming_count(), 0); + assert_eq!(staging.ready_count(), 1); + } + + /// When several cells are waiting, the step spends its round on the one + /// carrying the most traffic, leaving the quieter cell still warming. + /// Volume is the cached importance of the backing graph node, so the + /// cells the host is most likely to be asking about come online first, + /// and — because a busy ancestor outweighs its own descendants — + /// ancestors tend to arrive before the cells beneath them. + /// + /// ´claim:staging:warming-effort-goes-first-to-the-waiting-cell-carrying-the-most-traffic´ + /// ´test:unit:warm-one-batch-picks-highest-volume´ + #[test] + fn warm_one_batch_picks_highest_volume() { + use rand::SeedableRng; + let mut staging = StagingArea::::new(); + let (graph, _, left, right) = split_graph(); + + // Enqueue two cells with different target_rounds. + staging.enqueue(left, make_cell(1), 1); + staging.enqueue(right, make_cell(1), 1); + + // Manually set volumes so right > left. + staging.warming.get_mut(&left).unwrap().volume = 10.0; + staging.warming.get_mut(&right).unwrap().volume = 100.0; + + let mut rng = SmallRng::seed_from_u64(42); + + // First warm_one_batch should pick right (higher volume). + assert!(staging.warm_one_batch(&graph, 64, &mut rng)); + // right should be ready now (target_rounds=1). + assert!(staging.ready.iter().any(|(g, _)| *g == right)); + assert_eq!(staging.warming_count(), 1); + assert!(staging.warming.contains_key(&left)); + } + + /// Equal volumes send the round to the shallower cell, whichever + /// identifier that cell happens to hold. The tie is the ordinary case for + /// a path node whose whole accumulation is the single cell below it, and + /// neither half of the identifier reading survives it. A comparison on + /// volume alone keeps the last of the equal maxima, which is the largest + /// identifier, and in the ordinary allocation order that is the cell + /// beneath. Resolving the tie toward the smallest identifier instead + /// inverts the rule the other way round, because the arena hands a freed + /// slot out again and an ancestor can hold the larger identifier while + /// the cell created beneath it holds the smaller. Depth is what carries + /// the rule through both, and spending the round on the descendant lets + /// it finish and be promoted while the chain above it is still warming — + /// the gap the volume ordering exists to close. This step and the two + /// drains serve one queue and must not disagree about which cell comes + /// next. + /// + /// ´claim:staging:equal-volumes-send-the-round-to-the-shallower-cell-whichever-identifier-it-holds´ + /// ´test:unit:warm-one-batch-warms-the-ancestor-before-the-cell-beneath-it´ + #[test] + fn warm_one_batch_warms_the_ancestor_before_the_cell_beneath_it() { + use rand::SeedableRng; + + let (graph, _, left, right) = split_graph(); + let (smaller, larger) = if left < right { (left, right) } else { (right, left) }; + let mut rng = SmallRng::seed_from_u64(42); + + // The ordinary allocation order: the ancestor exists first and holds + // the smaller identifier, so the cell beneath it holds the larger — + // which is the one a comparison on volume alone keeps. + let mut ordinary = StagingArea::::new(); + ordinary.enqueue(smaller, make_cell(1), 1); + ordinary.enqueue(larger, make_cell(2), 1); + ordinary.warming.get_mut(&smaller).unwrap().volume = 42.0; + ordinary.warming.get_mut(&larger).unwrap().volume = 42.0; + + assert!(ordinary.warm_one_batch(&graph, 64, &mut rng)); + assert!( + ordinary.ready.iter().any(|(g, _)| *g == smaller), + "the ancestor must take the round and reach the ready queue" + ); + assert!( + ordinary.warming.contains_key(&larger), + "the cell beneath the ancestor must still be warming" + ); + + // The same pair with the identifiers the other way round, which is + // what a recycled arena slot produces. An identifier tie-break alone + // resolves this one the wrong way; depth resolves both. + let mut recycled = StagingArea::::new(); + recycled.enqueue(larger, make_cell(1), 1); + recycled.enqueue(smaller, make_cell(2), 1); + recycled.warming.get_mut(&larger).unwrap().volume = 42.0; + recycled.warming.get_mut(&smaller).unwrap().volume = 42.0; + + assert!(recycled.warm_one_batch(&graph, 64, &mut rng)); + assert!( + recycled.ready.iter().any(|(g, _)| *g == larger), + "the ancestor must take the round whichever identifier it holds" + ); + assert!( + recycled.warming.contains_key(&smaller), + "the cell beneath the ancestor must still be warming" + ); + } + + /// Presence is answered across every state a staged cell can occupy: a + /// cell still warming and a cell already waiting to be promoted both + /// answer yes. The caller asking is deciding whether a cell needs + /// creating, and it must not be told "absent" merely because the cell + /// has moved on within the staging area. + /// + /// ´claim:staging:presence-is-answered-across-every-state-a-staged-cell-can-occupy´ + /// ´test:unit:contains-checks-both-warming-and-ready´ + #[test] + fn contains_checks_both_warming_and_ready() { + let mut staging = StagingArea::::new(); + let (_, _, left, right) = split_graph(); + + staging.enqueue(left, make_cell(1), 5); // goes to warming + staging.enqueue(right, make_cell(1), 0); // goes to ready + + assert!(staging.contains(left)); + assert!(staging.contains(right)); + } + + // ── Step 3 in-flight tests ────────────────────────── + + /// Checking a cell out for background work takes the busiest waiting + /// cell and marks it in flight, leaving the others warming; while it is + /// away it still counts as present in the staging area. That is what + /// makes the expensive noise injection safe to do without holding the + /// lock: the main thread can see the cell is spoken for even though the + /// warming map no longer holds it. + /// + /// ´claim:staging:a-cell-checked-out-for-background-work-stays-logically-present-while-it-is-away´ + /// ´test:unit:take-highest-priority-moves-to-in-flight´ + #[test] + fn take_highest_priority_moves_to_in_flight() { + let mut staging = StagingArea::::new(); + let (_, _, left, right) = split_graph(); + + staging.enqueue(left, make_cell(1), 3); + staging.enqueue(right, make_cell(1), 3); + staging.warming.get_mut(&left).unwrap().volume = 10.0; + staging.warming.get_mut(&right).unwrap().volume = 100.0; + + let (gnode, _wc) = staging.take_highest_priority().unwrap(); + assert_eq!(gnode, right); // highest volume + assert_eq!(staging.warming_count(), 1); // left remains + assert_eq!(staging.in_flight_count(), 1); + assert!(staging.contains(right)); // still logically present + } + + /// Equal volumes resolve to the shallower cell rather than the deeper one. + /// A tie is the ordinary case for a pair of siblings the moment they are + /// created, and the rule the queue exists to serve is that a busy ancestor + /// is warmed before the cells beneath it. Identifiers cannot carry that + /// rule on their own: the graph's arena hands a freed slot out again, so a + /// cell created into a recycled slot holds a smaller identifier than an + /// ancestor allocated before it. This path and the synchronous drain are + /// two ways of serving one queue, so they must not disagree about which + /// cell comes next. + /// + /// ´claim:staging:equal-volumes-resolve-to-the-shallower-cell-so-both-drains-agree´ + /// ´test:unit:equal-volumes-take-the-shallower-cell-first´ + #[test] + fn equal_volumes_take_the_shallower_cell_first() { + let mut staging = StagingArea::::new(); + let (_, _, left, right) = split_graph(); + let (earlier, later) = if left < right { (left, right) } else { (right, left) }; + + staging.enqueue(left, make_cell(1), 3); + staging.enqueue(right, make_cell(1), 3); + staging.warming.get_mut(&left).unwrap().volume = 42.0; + staging.warming.get_mut(&right).unwrap().volume = 42.0; + + let (gnode, _wc) = staging.take_highest_priority().unwrap(); + assert_eq!(gnode, earlier, "a tie goes to the earlier, shallower identifier"); + assert_ne!(gnode, later); + } + + /// A descendant holding a smaller identifier than its own ancestor still + /// warms after that ancestor when the two tie on volume. The tie is the + /// ordinary case for a path node whose whole volume comes from the single + /// cell below it, and the smaller identifier is what a recycled arena slot + /// produces, so the pair is reachable rather than contrived. Warming the + /// descendant first would promote it while the chain above it is still + /// warming, which is the gap the volume ordering exists to close. + /// + /// (´claim:staging:equal-volumes-resolve-to-the-shallower-cell-so-both-drains-agree´) + /// ´test:unit:a-restored-descendant-does-not-overtake-its-warming-ancestor´ + #[test] + fn a_restored_descendant_does_not_overtake_its_warming_ancestor() { + let mut staging = StagingArea::::new(); + + // Slot 5 at its first generation is an ancestor allocated before the + // descendant; slot 2 at its second generation is a cell created into a + // slot the arena had freed, which is how a descendant comes to hold + // the smaller identifier. + let ancestor = GNodeId::from_parts(5, 0); + let descendant = GNodeId::from_parts(2, 1); + assert!(descendant < ancestor, "the descendant holds the smaller identifier"); + + staging.enqueue(ancestor, make_cell(1), 3); + staging.enqueue(descendant, make_cell(2), 3); + staging.warming.get_mut(&ancestor).unwrap().volume = 42.0; + staging.warming.get_mut(&descendant).unwrap().volume = 42.0; + + let (gnode, _wc) = staging.take_highest_priority().unwrap(); + assert_eq!(gnode, ancestor, "the ancestor is warmed before the cell beneath it"); + } + + /// The synchronous drain orders that same pair the same way, leaving the + /// ancestor ahead of the descendant on the ready queue. Promotion takes + /// the queue in order, so a drain that warmed the descendant first would + /// bring it online with an ancestor of its own still warming — the two + /// drains serve one queue and must agree about which cell comes next. + /// + /// (´claim:staging:equal-volumes-resolve-to-the-shallower-cell-so-both-drains-agree´) + /// ´test:unit:the-synchronous-drain-warms-a-restored-descendants-ancestor-first´ + #[test] + fn the_synchronous_drain_warms_a_restored_descendants_ancestor_first() { + use rand::SeedableRng; + + let mut staging = StagingArea::::new(); + let ancestor = GNodeId::from_parts(5, 0); + let descendant = GNodeId::from_parts(2, 1); + + staging.enqueue(ancestor, make_cell(1), 1); + staging.enqueue(descendant, make_cell(2), 1); + staging.warming.get_mut(&ancestor).unwrap().volume = 42.0; + staging.warming.get_mut(&descendant).unwrap().volume = 42.0; + + let mut rng = SmallRng::seed_from_u64(42); + staging.drain_all_synchronous(8, &mut rng); + + let order: Vec = staging.take_ready().iter().map(|(gnode, _)| *gnode).collect(); + assert_eq!(order, [ancestor, descendant], "the ancestor reaches the ready queue first"); + } + + /// A cell checked out for background warming is still a warming cell, so + /// it still counts among the competitive targets being warmed. Checking a + /// cell out is how the expensive work is done off the lock, not a change + /// in what the cell is; a count that dropped it would fall precisely when + /// the work was happening, understating what is in progress by the number + /// of cells actually in progress. The flag is recorded at checkout, so + /// counting it needs nothing from a cell another thread is holding. + /// + /// ´claim:staging:a-cell-checked-out-for-warming-still-counts-among-the-competitive-targets´ + /// ´test:unit:an-in-flight-cell-still-counts-as-a-competitive-target´ + #[test] + fn an_in_flight_cell_still_counts_as_a_competitive_target() { + let mut staging = StagingArea::::new(); + let (_, _, left, right) = split_graph(); + + let mut busy = make_cell(1); + busy.is_competitive = true; + let mut quiet = make_cell(1); + quiet.is_competitive = true; + staging.enqueue(left, busy, 3); + staging.enqueue(right, quiet, 3); + assert_eq!(staging.warming_competitive_count(), 2); + + let (_gnode, _wc) = staging.take_highest_priority().unwrap(); + assert_eq!(staging.in_flight_count(), 1); + assert_eq!( + staging.warming_competitive_count(), + 2, + "a cell being warmed is still being warmed while it is checked out" + ); + } + + /// An ancestor-only cell is not a competitive target wherever it is being + /// held, so checking one out does not inflate the figure either. The count + /// follows the cell's own flag rather than its location in the queue. + /// + /// (´claim:staging:a-cell-checked-out-for-warming-still-counts-among-the-competitive-targets´) + /// ´test:unit:an-in-flight-ancestor-cell-is-not-a-competitive-target´ + #[test] + fn an_in_flight_ancestor_cell_is_not_a_competitive_target() { + let mut staging = StagingArea::::new(); + let (_, _, left, _right) = split_graph(); + + let cell = make_cell(1); + assert!(!cell.is_competitive); + staging.enqueue(left, cell, 3); + assert_eq!(staging.warming_competitive_count(), 0); + + let (_gnode, _wc) = staging.take_highest_priority().unwrap(); + assert_eq!(staging.warming_competitive_count(), 0); + } + + /// A completed competitive cell can reach the ready queue after an ingest + /// has passed its promotion point. It remains inside the warm-up pipeline + /// and outside the producing set until the next promotion, so both health + /// counts continue to include it while it waits. The worker transition is + /// performed directly here to fix that interleaving without sleeps. + /// + /// ´claim:staging:a-ready-competitive-cell-remains-a-warming-target-until-promotion´ + /// ´test:unit:ready-competitive-target-stays-in-warming-count-until-promotion´ + #[test] + fn ready_competitive_target_stays_in_warming_count_until_promotion() { + use crate::config::NoiseSchedule; + use crate::sentinel::SpectralSentinel; + + let cfg = SentinelConfig:: { + split_threshold: 5, + analysis_k: 16, + noise_schedule: NoiseSchedule::Explicit(vec![0, 500]), + noise_batch_size: 2, + background_warming: true, + ..SentinelConfig::::default() + }; + let mut sentinel: SpectralSentinel = SpectralSentinel::new(cfg).unwrap(); + sentinel.warming_thread.as_ref().unwrap().shutdown(); + + let batch: Vec = (0..40u128).map(|i| (0xA_u128 << 124) | i).collect(); + for _ in 0..10 { + sentinel.ingest(&batch); + } + + let (staged_total, staged_competitive) = { + let mut staging = sentinel.staging.lock().expect("staging mutex poisoned"); + let competitive = staging + .warming + .iter() + .find_map(|(&gnode, cell)| cell.cell.is_competitive.then_some(gnode)) + .expect("the fixture must leave a competitive cell awaiting warm-up"); + staging.warming.get_mut(&competitive).unwrap().volume = f64::MAX; + let (gnode, cell) = staging.take_highest_priority().unwrap(); + assert_eq!(gnode, competitive); + staging.finish_warming(gnode, cell.cell); + + let ready_competitive = staging.ready.iter().filter(|(_, cell)| cell.is_competitive).count(); + assert!(ready_competitive > 0, "the completed target must be awaiting promotion"); + let competitive_total = staging.warming.values().filter(|cell| cell.cell.is_competitive).count() + + ready_competitive + + staging.in_flight.values().filter(|record| record.is_competitive).count(); + (staging.total_count(), competitive_total) + }; + + let health = sentinel.health(); + assert_eq!(health.warming_trackers, staged_total); + assert_eq!( + health.warming_competitive_targets, staged_competitive, + "a ready competitive target remains unpromoted and part of the warm-up pipeline" + ); + } + + /// A cell joining the queue carries its volume immediately. Refreshing cached volumes before enqueueing would leave new cells at zero until another reconciliation pass, and a field of zeroes is decided entirely by the tie-breaks — shallower depth first, then the smaller identifier — so which of them is warmed first would be settled by where they sit in the tree rather than by the traffic the queue exists to follow. + /// + /// The worker is stopped through its shutdown handshake before traffic is queued, while its handle remains installed to select deferred staging. Every queued cell is therefore available for the volume assertions, without relying on the scheduler or the length of a noise schedule to leave cells waiting. + /// + /// ´claim:staging:a-cell-joins-the-queue-carrying-its-volume-rather-than-a-zero´ + /// ´test:unit:a-newly-queued-cell-carries-its-volume´ + #[test] + fn a_newly_queued_cell_carries_its_volume() { + use crate::config::NoiseSchedule; + use crate::sentinel::SpectralSentinel; + + // Depth zero takes no rounds, so construction queues no work. + // Deeper cells require warming and will enter deferred staging. + let cfg = SentinelConfig:: { + split_threshold: 5, + noise_schedule: NoiseSchedule::Explicit(vec![0, 500]), + noise_batch_size: 2, + background_warming: true, + ..SentinelConfig::::default() + }; + let mut s: SpectralSentinel = SpectralSentinel::new(cfg).unwrap(); + // Keep the handle installed so reconciliation queues work instead + // of draining synchronously, but prevent the worker taking cells + // away before their cached volumes can be inspected. + s.warming_thread.as_ref().unwrap().shutdown(); + + let batch: Vec = (0..40u128).map(|i| (0xA_u128 << 124) | i).collect(); + for _ in 0..10 { + s.ingest(&batch); + } + + let staged = s.staging.lock().expect("staging mutex poisoned"); + assert!( + !staged.warming.is_empty(), + "deferred cells must remain queued while the worker is stopped" + ); + + for (&gnode, wc) in &staged.warming { + let node_volume = s.graph.gnode_info(gnode).map_or(0.0, |info| info.sum.to_f64_approx()); + if node_volume > 0.0 { + assert!( + wc.volume > 0.0, + "a queued cell whose node carries traffic must carry it into the queue too" + ); + } + } + } + + /// A cell handed back unfinished rejoins the warming set and stops being + /// in flight, with its accumulated rounds intact. Background warming can + /// therefore be interrupted between rounds — the thread need not carry a + /// cell to completion once it has taken it. + /// + /// ´claim:staging:a-cell-handed-back-unfinished-rejoins-the-warming-set-with-its-progress-intact´ + /// ´test:unit:return-warming-restores-cell´ + #[test] + fn return_warming_restores_cell() { + let mut staging = StagingArea::::new(); + let (_, _, left, _) = split_graph(); + + staging.enqueue(left, make_cell(1), 3); + let (gnode, wc) = staging.take_highest_priority().unwrap(); + assert_eq!(staging.warming_count(), 0); + assert_eq!(staging.in_flight_count(), 1); + + staging.return_warming(gnode, wc); + assert_eq!(staging.warming_count(), 1); + assert_eq!(staging.in_flight_count(), 0); + } + + /// A cell handed back rejoins the queue at the volume the graph has now, + /// not the one it carried out. The refresh runs on the main thread while + /// the worker holds the cell, and the checkout spans exactly the noise + /// injection — the expensive part of the pass, and so the part an ingest + /// is most likely to overlap. Restoring the carried volume would leave the + /// busiest cell in the area queued at its pre-ingest importance, and the + /// next checkout — the one decision the cached volume exists to make — + /// would go to a rival the traffic has already passed. + /// + /// ´claim:staging:a-cell-handed-back-rejoins-the-queue-at-the-volume-the-graph-has-now´ + /// ´test:unit:a-returned-cell-carries-the-volume-the-graph-has-now´ + #[test] + fn a_returned_cell_carries_the_volume_the_graph_has_now() { + let mut staging = StagingArea::::new(); + let (mut graph, _root, left, right) = split_graph(); + + staging.enqueue(left, make_cell(1), 3); + staging.enqueue(right, make_cell(1), 3); + + // Check the left cell out while the queue holds it to be the busiest. + let carried = 1.0; + staging.warming.get_mut(&left).unwrap().volume = carried; + staging.warming.get_mut(&right).unwrap().volume = 0.0; + let (gnode, wc) = staging.take_highest_priority().unwrap(); + assert_eq!(gnode, left); + + // An ingest lands in the left half while the worker holds the cell. + for _ in 0..20 { + graph.observe(0u128, 1u64); + } + staging.update_volumes(&graph); + + let refreshed = graph.gnode_info(left).map_or(0.0, |info| info.sum.to_f64_approx()); + let rival = graph.gnode_info(right).map_or(0.0, |info| info.sum.to_f64_approx()); + assert!( + carried < rival, + "the volume carried out has to lose to the rival, or the ordering proves nothing" + ); + assert!( + rival < refreshed, + "the ingest has to carry the checked-out cell past the rival" + ); + + staging.return_warming(gnode, wc); + let returned = staging.warming.get(&left).expect("the cell rejoins the warming map").volume; + assert!( + (returned - refreshed).abs() < f64::EPSILON, + "a cell handed back carries the volume the graph has now" + ); + + let (next, _next_wc) = staging.take_highest_priority().unwrap(); + assert_eq!( + next, left, + "the next checkout follows the refreshed volume rather than the one carried out" + ); + } + + /// A cell handed back finished joins the ready queue instead of the + /// warming set, and is no longer in flight. Which of the two return + /// paths the background thread takes is what decides the cell's fate, so + /// completion is declared by the worker that did the rounds rather than + /// re-derived by the staging area. + /// + /// ´claim:staging:a-cell-handed-back-finished-joins-the-ready-queue-rather-than-the-warming-set´ + /// ´test:unit:finish-warming-moves-to-ready´ + #[test] + fn finish_warming_moves_to_ready() { + let mut staging = StagingArea::::new(); + let (_, _, left, _) = split_graph(); + + staging.enqueue(left, make_cell(1), 1); + let (gnode, wc) = staging.take_highest_priority().unwrap(); + + staging.finish_warming(gnode, wc.cell); + assert_eq!(staging.ready_count(), 1); + assert_eq!(staging.in_flight_count(), 0); + } + + /// A cell evicted while a background thread was working on it is + /// discarded when it comes back, not resurrected: the eviction sweep + /// removes its in-flight mark, and a return with no mark to clear keeps + /// nothing. The warming work already spent is lost, which is the + /// deliberate trade — a cell that has left the analysis set must not + /// reappear in it because a thread happened to be holding it. + /// + /// ´claim:staging:a-cell-evicted-while-in-flight-is-discarded-on-return-rather-than-resurrected´ + /// ´test:unit:eviction-of-in-flight-cell-discards-on-return´ + #[test] + fn eviction_of_in_flight_cell_discards_on_return() { + let mut staging = StagingArea::::new(); + let (_, _, left, right) = split_graph(); + + staging.enqueue(left, make_cell(1), 3); + staging.enqueue(right, make_cell(1), 3); + + // Set left to highest priority so take_highest_priority picks it. + staging.warming.get_mut(&left).unwrap().volume = 100.0; + staging.warming.get_mut(&right).unwrap().volume = 10.0; + + // Take left for background warming. + let (gnode, wc) = staging.take_highest_priority().unwrap(); + assert_eq!(gnode, left); + + // Meanwhile, evict left from the analysis set. + let mut keep = BTreeSet::new(); + keep.insert(right); + staging.retain_in_set(&keep); + + // Background thread tries to return the evicted cell — silently discarded. + staging.return_warming(gnode, wc); + assert_eq!(staging.warming_count(), 1); // only right + assert_eq!(staging.in_flight_count(), 0); + assert!(!staging.contains(gnode)); // left is gone + } + + /// The enumeration of staged cells spans all three states together — + /// ready, warming and in-flight — and counts each cell once. It is the + /// same accounting the presence check gives, offered as a whole set, so + /// diagnostics and eviction sweeps see exactly the cells the area is + /// responsible for. + /// + /// (´claim:staging:presence-is-answered-across-every-state-a-staged-cell-can-occupy´) + /// ´test:unit:gnode-set-includes-all-states´ + #[test] + fn gnode_set_includes_all_states() { + let mut staging = StagingArea::::new(); + let (_, root, left, right) = split_graph(); + + staging.enqueue(root, make_cell(0), 0); // → ready + staging.enqueue(left, make_cell(1), 3); // → warming + staging.enqueue(right, make_cell(1), 3); // → warming + + // Take right to in-flight. + staging.warming.get_mut(&right).unwrap().volume = 100.0; + let _taken = staging.take_highest_priority(); // takes right + + let set = staging.gnode_set(); + assert!(set.contains(&root)); // ready + assert!(set.contains(&left)); // warming + assert!(set.contains(&right)); // in-flight + assert_eq!(set.len(), 3); + } +} diff --git a/packages/sentinel/src/sentinel/tracker.rs b/packages/sentinel/src/sentinel/tracker.rs new file mode 100644 index 000000000..08d21ca6c --- /dev/null +++ b/packages/sentinel/src/sentinel/tracker.rs @@ -0,0 +1,904 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +//! Subspace tracker — the core online SVD engine. +//! +//! Maintains a low-rank model of "normal" via streaming thin SVD +//! with exponential forgetting. Scores each new batch along four +//! axes (novelty, displacement, surprise, coherence), then evolves +//! the model to incorporate the new data. +//! +//! See `docs/algorithm.md` §ALGO S-4.2 for the five-phase core loop and +//! §ALGO S-5 for the four scoring axes. +//! +//! This is the only module that depends on `faer`. + +use faer::Mat; +use torrust_mudlark::Accumulator; + +use crate::config::SentinelConfig; +use crate::ewma::EwmaStats; +use crate::report::{AnomalyScores, SampleScore, ScoreDistribution, ScoringGeometry, TrackerMaturity, TrackerReport}; +use crate::sentinel::cusum::CusumAccumulator; + +/// Noise influence below this share no longer widens the clip ceiling. +const WARMUP_THRESHOLD: f64 = 0.01; + +// ─── Per-axis baseline ────────────────────────────────────── + +/// Fast EWMA (z-scores) + CUSUM (drift detection) for one scoring axis. +#[derive(Debug, Clone)] +struct AxisBaseline { + fast: EwmaStats, + cusum: CusumAccumulator, + /// Clip-pressure EWMA: ρ̄ ∈ [0, 1] (§ALGO S-6.4). + clip_pressure: f64, +} + +impl AxisBaseline { + const fn new(fast_decay: f64, slow_decay: f64) -> Self { + Self { + fast: EwmaStats::new(fast_decay), + cusum: CusumAccumulator::new(slow_decay), + clip_pressure: 0.0, + } + } + + /// Destroy all learned state — return to the freshly-constructed + /// state with cold EWMA and zeroed CUSUM. + const fn reset_cold(&mut self) { + self.fast.reset_cold(); + self.cusum.reset_cold(); + self.clip_pressure = 0.0; + } + + /// Seed the CUSUM's slow EWMA from this axis's fast EWMA. + /// + /// Closes the fast-slow gap after noise injection (ADR-S-013 + /// §6b, Option C). + const fn seed_cusum_slow_from_fast(&mut self) { + self.cusum.seed_slow_from(&self.fast); + } +} + +// ─── SubspaceTracker ──────────────────────────────────────── + +/// Low-rank subspace model with online learning and four-axis scoring. +/// +/// Each analysis cell gets one of these. The tracker +/// accepts centred suffix observation slices via [`observe`](Self::observe) +/// and returns a [`TrackerReport`]. +/// +/// The tracker has zero knowledge of cell identity, observation types, +/// or the host's domain. It operates on `&[&[f64]]` — a batch of +/// d-dimensional centred bit slices. +#[derive(Debug, Clone)] +pub struct SubspaceTracker { + /// Suffix width: dimensionality of the working space (128 − depth). + dim: usize, + + /// Hard ceiling on rank: `min(dim, max_rank)`. + cap: usize, + + /// Current active rank (number of basis vectors in use for the next batch). + rank: usize, + + /// Geometry of the model that scored the most recent batch. Before the + /// first scored batch, this describes the initial rank-one model. + scoring_geometry: ScoringGeometry, + + /// Observation step counter (for rank adaptation timing). + step: u64, + + // ── Subspace state ────────────────────────────────── + /// Orthonormal basis, shape `(dim, cap)`. Only columns `[:rank]` are active. + basis: Mat, + + /// Singular values, length `cap`. Only `[:rank]` are meaningful. + sigmas: Vec, + + // ── Latent distribution ───────────────────────────── + /// EWMA mean of latent coordinates, length `cap`. + lat_mean: Vec, + + /// EWMA variance of latent coordinates, length `cap`. + lat_var: Vec, + + /// Upper-triangle cross-correlation, flat-packed. + /// `cross_corr[tri_idx(j, l, cap)]` for `j < l` tracks EWMA of `zⱼ · zₗ`. + /// + /// Length: `cap * (cap - 1) / 2`. When rank increases, new entries + /// are already zero (the full triangle is pre-allocated at construction). + /// When rank decreases, outer entries are ignored but preserved (§ALGO S-4.2 Phase 3). + cross_corr: Vec, + + // ── Score baselines (one per axis) ────────────────── + novelty_bl: AxisBaseline, + displacement_bl: AxisBaseline, + surprise_bl: AxisBaseline, + coherence_bl: AxisBaseline, + + // ── Maturity tracking ─────────────────────────────── + real_observations: u64, + noise_observations: u64, + noise_influence: f64, + + // ── Config snapshots ──────────────────────────────── + forgetting_factor: f64, + energy_threshold: f64, + rank_update_interval: u64, + eps: f64, + per_sample_scores: bool, + cusum_allowance_sigmas: f64, + clip_sigmas: f64, + /// Clip-pressure EWMA decay factor (`λ_ρ`, §ALGO S-6.4). + clip_pressure_decay: f64, + svd_strategy: crate::maths::SvdStrategy, +} + +impl SubspaceTracker { + /// Create a new tracker for the given suffix width. + /// + /// The initial basis is identity-like columns (not random). + /// Noise injection will diversify it before real traffic arrives. + pub fn new(dim: usize, cfg: &SentinelConfig, slow_decay: f64) -> Self { + debug_assert!( + dim >= crate::MIN_TRACKER_DIM, + "SubspaceTracker::new() called with dim={dim}, \ + expected >= {} (caller should have filtered)", + crate::MIN_TRACKER_DIM, + ); + + let cap = dim.min(cfg.max_rank); + + // Identity-like basis: column j has a 1.0 at row j. + let mut basis = Mat::zeros(dim, cap); + for j in 0..cap.min(dim) { + basis[(j, j)] = 1.0; + } + + let fast_decay = cfg.forgetting_factor; + let scoring_geometry = ScoringGeometry { + dim, + cap, + residual_dof: dim.saturating_sub(1), + }; + + Self { + dim, + cap, + rank: 1, + scoring_geometry, + step: 0, + basis, + sigmas: vec![0.01; cap], + lat_mean: vec![0.0; cap], + lat_var: vec![1.0; cap], + cross_corr: vec![0.0; cap * (cap.saturating_sub(1)) / 2], + novelty_bl: AxisBaseline::new(fast_decay, slow_decay), + displacement_bl: AxisBaseline::new(fast_decay, slow_decay), + surprise_bl: AxisBaseline::new(fast_decay, slow_decay), + coherence_bl: AxisBaseline::new(fast_decay, slow_decay), + real_observations: 0, + noise_observations: 0, + noise_influence: 1.0, + forgetting_factor: cfg.forgetting_factor, + energy_threshold: cfg.energy_threshold, + rank_update_interval: cfg.rank_update_interval, + eps: cfg.eps, + per_sample_scores: cfg.per_sample_scores, + cusum_allowance_sigmas: cfg.cusum_allowance_sigmas, + clip_sigmas: cfg.clip_sigmas, + clip_pressure_decay: cfg.clip_pressure_decay, + svd_strategy: cfg.svd_strategy, + } + } + + /// Process a batch of centred observation slices and return a report. + /// + /// Each inner slice in `rows` has length `self.dim`. + /// Scoring happens against the *prior* model, then the model evolves. + /// + /// `is_noise` controls maturity bookkeeping — noise observations + /// don't count as real. + #[allow(clippy::many_single_char_names)] // mathematical notation matching the spec + pub fn observe(&mut self, rows: &[&[f64]], depth: u8, is_noise: bool) -> TrackerReport { + let _observe_span = tracing::debug_span!("observe", depth, is_noise, b = rows.len(), k = self.rank).entered(); + + let b = rows.len(); + let d = self.dim; + let k = self.rank; + let eps = self.eps; + let scoring_geometry = ScoringGeometry { + dim: d, + cap: self.cap, + residual_dof: d.saturating_sub(k), + }; + self.scoring_geometry = scoring_geometry; + + // Build X matrix (b × d). + let x = Self::build_matrix(rows, b, d); + + // ── Phase 1: Score against the prior model ────── + let p1_guard = tracing::debug_span!("phase1_score").entered(); + // Capture Z from this phase for Phase 3 (latent evolution uses + // the prior-basis projection, not the post-evolution basis). + let u_k = self.basis.subcols(0, k); + let z = &x * u_k; // (b × k) + let x_hat = &z * u_k.transpose(); // (b × d) + let residual = &x - &x_hat; // (b × d) + + let (nov_scores, disp_scores, surp_scores, coh_scores) = self.compute_scores(&z, &residual, b, d, k, eps); + + // Build per-sample structs only when enabled (avoids 4 z-score + // calls per sample in the common disabled case). + let per_sample = if self.per_sample_scores { + Some(self.build_per_sample(&nov_scores, &disp_scores, &surp_scores, &coh_scores, eps)) + } else { + None + }; + drop(p1_guard); + + // ── Snapshot the model that scored this batch ──── + // Phase 2 replaces the subspace and Phase 5 adapts the rank, and + // both prepare the next batch rather than describing this one. Every + // model figure the report carries is therefore read here, while the + // state is still the state the scores were computed against: read + // afterwards, the energy ratio would divide the evolved sigmas by + // this batch's rank and the leading singular value would belong to a + // model that has not scored anything yet. + let scoring_rank = k; + let scoring_energy_ratio = self.energy_ratio(); + let scoring_top_singular_value = self.top_singular_value(); + + // ── Phase 2: Evolve subspace (streaming SVD) ───── + // Uses the maths module dispatch: the selected SvdStrategy + // runs, and in debug builds the other strategy also runs + // with results compared via debug_assert (ADR-S-016). + self.evolve_subspace(&z, &residual, k); + + // ── Phase 3: Evolve latent distribution ───────── + let p3_guard = tracing::debug_span!("phase3_latent").entered(); + // Uses Z from Phase 1 (prior basis), not the updated basis. + self.evolve_latent(&z, k); + drop(p3_guard); + + // ── Phase 4: Update score baselines and CUSUM ─── + // + // Unified clip + clip-pressure EWMA (§ALGO S-6.1.1, §ALGO S-6.4). + // + // Each axis computes its own effective clip width from: + // p = max(η, ρ̄) — noise influence OR clip pressure + // n_σ_eff = n_σ · (1 + p / (1 − p + ε)) + // + // A single shared clip filter (from the fast EWMA ceiling) is + // applied, and both the fast EWMA and CUSUM slow EWMA receive + // the same retained set. The per-axis clip-pressure ρ̄ is + // updated from the fraction of samples clipped. + // + // At ρ̄ = 0 this reduces to the old η-only formula: + // n_σ_eff = n_σ + n_σ · η / (1 − η + ε) + let allowance = self.cusum_allowance_sigmas; + let eta = self.noise_influence; + let clip_sigmas = self.clip_sigmas; + let cp_decay = self.clip_pressure_decay; + + let novelty_dist = Self::update_axis( + &mut self.novelty_bl, + &nov_scores, + eps, + allowance, + clip_sigmas, + eta, + cp_decay, + true, + ); + let displacement_dist = Self::update_axis( + &mut self.displacement_bl, + &disp_scores, + eps, + allowance, + clip_sigmas, + eta, + cp_decay, + true, + ); + let surprise_dist = Self::update_axis( + &mut self.surprise_bl, + &surp_scores, + eps, + allowance, + clip_sigmas, + eta, + cp_decay, + true, + ); + // Coherence does not exist at k < 2 (no pairs). Scores are + // identically zero, so the baseline must not evolve — it stays + // cold until k reaches 2, where the first real values enter + // through the cold→warm path. If rank later drops back below + // 2, adapt_rank() destroys the coherence baseline entirely. + let coherence_dist = Self::update_axis( + &mut self.coherence_bl, + &coh_scores, + eps, + allowance, + clip_sigmas, + eta, + cp_decay, + k >= 2, + ); + + // ── Phase 5: Adapt rank ───────────────────────── + self.step += 1; + if self.step.is_multiple_of(self.rank_update_interval) { + self.adapt_rank(); + } + + // ── Maturity bookkeeping ──────────────────────── + self.update_maturity(b, is_noise); + + TrackerReport { + rank: scoring_rank, + energy_ratio: scoring_energy_ratio, + top_singular_value: scoring_top_singular_value, + scores: AnomalyScores { + novelty: novelty_dist, + displacement: displacement_dist, + surprise: surprise_dist, + coherence: coherence_dist, + }, + maturity: self.maturity(), + geometry: scoring_geometry, + per_sample, + } + } + + /// Reset all CUSUM accumulators to zero (post-noise-injection). + pub const fn reset_cusum(&mut self) { + self.novelty_bl.cusum.reset(); + self.displacement_bl.cusum.reset(); + self.surprise_bl.cusum.reset(); + self.coherence_bl.cusum.reset(); + } + + /// Zero clip-pressure EWMA on all four axes (§ALGO S-11.4). + /// + /// Prevents warm-up contamination from echoing into production + /// scoring. Called after noise injection completes, alongside + /// [`reset_cusum`](Self::reset_cusum). + pub const fn reset_clip_pressure(&mut self) { + self.novelty_bl.clip_pressure = 0.0; + self.displacement_bl.clip_pressure = 0.0; + self.surprise_bl.clip_pressure = 0.0; + self.coherence_bl.clip_pressure = 0.0; + } + + /// Seed each axis's CUSUM slow EWMA from the corresponding fast EWMA. + /// + /// Closes the fast-slow gap after noise injection (ADR-S-013 + /// §6b, Option C). Called **before** [`reset_cusum`](Self::reset_cusum) + /// so the slow baselines start from the fast EWMA's converged + /// values. The CUSUM accumulators are then zeroed, beginning + /// drift detection from a state where fast ≈ slow. + pub const fn seed_cusum_slow_from_baselines(&mut self) { + self.novelty_bl.seed_cusum_slow_from_fast(); + self.displacement_bl.seed_cusum_slow_from_fast(); + self.surprise_bl.seed_cusum_slow_from_fast(); + self.coherence_bl.seed_cusum_slow_from_fast(); + } + + /// Current maturity snapshot. + pub const fn maturity(&self) -> TrackerMaturity { + TrackerMaturity { + real_observations: self.real_observations, + noise_observations: self.noise_observations, + noise_influence: self.noise_influence, + } + } + + /// Current rank of the evolved model that will score the next batch. + /// + /// Rank adaptation follows scoring, so this can differ from the rank paired + /// with [`scoring_geometry`](Self::scoring_geometry) for the previous batch. + pub const fn rank(&self) -> usize { + self.rank + } + + /// Maximum rank this tracker can reach (`min(dim, max_rank)`). + pub const fn cap(&self) -> usize { + self.cap + } + + /// Working dimensionality of the tracker's input space. + pub const fn dim(&self) -> usize { + self.dim + } + + /// Snapshot the current per-axis baseline means and variances. + /// + /// Used for coordination warm-up synthetic score generation + /// (§ALGO S-11.7). + pub const fn axis_baselines(&self) -> super::AxisBaselines { + super::AxisBaselines { + novelty_mean: self.novelty_bl.fast.mean(), + novelty_var: self.novelty_bl.fast.variance(), + displacement_mean: self.displacement_bl.fast.mean(), + displacement_var: self.displacement_bl.fast.variance(), + surprise_mean: self.surprise_bl.fast.mean(), + surprise_var: self.surprise_bl.fast.variance(), + coherence_mean: self.coherence_bl.fast.mean(), + coherence_var: self.coherence_bl.fast.variance(), + } + } + + /// Geometry of the model that scored the most recent batch. + /// + /// Before the first scored batch, this describes the initial rank-one model. + /// Rank adaptation prepares the next batch and does not rewrite this snapshot. + pub const fn scoring_geometry(&self) -> ScoringGeometry { + self.scoring_geometry + } + + /// Per-axis clip-pressure EWMA values [novelty, displacement, surprise, coherence]. + pub const fn clip_pressures(&self) -> [f64; 4] { + [ + self.novelty_bl.clip_pressure, + self.displacement_bl.clip_pressure, + self.surprise_bl.clip_pressure, + self.coherence_bl.clip_pressure, + ] + } + + // ════════════════════════════════════════════════════════ + // Private implementation + // ════════════════════════════════════════════════════════ + + /// Build a `faer::Mat` (b × d) from row slices. + fn build_matrix(rows: &[&[f64]], b: usize, d: usize) -> Mat { + let mut x = Mat::zeros(b, d); + for (i, row) in rows.iter().enumerate() { + for (j, &val) in row.iter().enumerate() { + x[(i, j)] = val; + } + } + x + } + + /// Phase 1: Compute raw per-sample scores from the prior-model projection. + /// + /// Returns the four score vectors. Per-sample `SampleScore` structs + /// (which include z-scores) are only built when `per_sample_scores` + /// is enabled — avoiding 4 z-score calls per sample in the common case. + fn compute_scores( + &self, + z: &Mat, + residual: &Mat, + b: usize, + d: usize, + k: usize, + eps: f64, + ) -> (Vec, Vec, Vec, Vec) { + #[allow(clippy::cast_precision_loss)] // d − k ≤ 128, well within f64 mantissa + let dof = (d - k).max(1) as f64; + + #[allow(clippy::cast_precision_loss)] + let k_f = k as f64; + + let mut nov_scores = Vec::with_capacity(b); + let mut disp_scores = Vec::with_capacity(b); + let mut surp_scores = Vec::with_capacity(b); + let mut coh_scores = Vec::with_capacity(b); + + for i in 0..b { + // ── Novelty: ‖rᵢ‖² / (d − k) ── + let mut resid_sq = 0.0; + for j in 0..d { + resid_sq = residual[(i, j)].mul_add(residual[(i, j)], resid_sq); + } + nov_scores.push(resid_sq / dof); + + // ── Displacement: ‖zᵢ‖² / (k + ‖zᵢ‖²) ── + let mut z_sq = 0.0; + for j in 0..k { + z_sq = z[(i, j)].mul_add(z[(i, j)], z_sq); + } + disp_scores.push(z_sq / (k_f + z_sq)); + + // ── Surprise: (1/k) Σⱼ (zᵢⱼ − μⱼ)² / (νⱼ + ε) ── + let mut surprise = 0.0; + for j in 0..k { + let dev = z[(i, j)] - self.lat_mean[j]; + surprise += (dev * dev) / (self.lat_var[j] + eps); + } + surp_scores.push(surprise / k_f.max(1.0)); + + // ── Coherence: (2/(k(k−1))) Σⱼ<ₗ (zᵢⱼ·zᵢₗ − Cⱼₗ)² ── + // + // Dividing by pairs = k(k−1)/2 is equivalent to multiplying + // by 2/(k(k−1)), matching the spec (§ALGO S-5.5). + coh_scores.push(if k >= 2 { + let pairs = (k * (k - 1)) / 2; + let mut coh = 0.0; + for j in 0..k { + for l in (j + 1)..k { + let prod = z[(i, j)] * z[(i, l)]; + let dev = prod - self.cross_corr[tri_idx(j, l, self.cap)]; + coh = dev.mul_add(dev, coh); + } + } + + #[allow(clippy::cast_precision_loss)] + { + coh / pairs as f64 + } + } else { + 0.0 + }); + } + + (nov_scores, disp_scores, surp_scores, coh_scores) + } + + /// Build per-sample `SampleScore` structs (only when `per_sample_scores` is enabled). + /// + /// This is separated from `compute_scores` to avoid 4 z-score calls + /// per sample in the default (disabled) case. + fn build_per_sample(&self, nov: &[f64], disp: &[f64], surp: &[f64], coh: &[f64], eps: f64) -> Vec { + nov.iter() + .zip(disp) + .zip(surp) + .zip(coh) + .map(|(((&n, &d), &s), &c)| SampleScore { + novelty: n, + displacement: d, + surprise: s, + coherence: c, + novelty_z: self.novelty_bl.fast.z_score(n, eps), + displacement_z: self.displacement_bl.fast.z_score(d, eps), + surprise_z: self.surprise_bl.fast.z_score(s, eps), + coherence_z: self.coherence_bl.fast.z_score(c, eps), + }) + .collect() + } + + /// Phase 2: Evolve subspace via the maths module (ADR-S-016). + /// + /// Delegates to `crate::maths::evolve()` which dispatches to the + /// selected strategy (naïve dense SVD or Brand's incremental SVD). + /// In debug builds, both run and are compared. + /// + /// Accepts `z` and `residual` from Phase 1 so Brand's method can + /// reuse the projection instead of recomputing it. + fn evolve_subspace(&mut self, z: &Mat, residual: &Mat, k: usize) { + let _span = tracing::debug_span!("phase2_evolve_subspace", + strategy = ?self.svd_strategy, + k = k, + d = self.dim, + b = residual.nrows(), + ) + .entered(); + + let sqrt_lam = self.forgetting_factor.sqrt(); + + let Some(update) = crate::maths::evolve( + self.svd_strategy, + &self.basis, + &self.sigmas, + z, + residual, + sqrt_lam, + k, + self.cap, + ) else { + tracing::warn!("SVD did not converge — keeping prior subspace"); + return; + }; + + // Write back into tracker state. + let n = update.n; + for j in 0..n { + for i in 0..self.dim { + self.basis[(i, j)] = update.basis[(i, j)]; + } + self.sigmas[j] = update.sigmas[j]; + } + + // Zero out unused sigmas. + for s in &mut self.sigmas[n..] { + *s = 0.0; + } + } + + /// Phase 3: Evolve latent distribution (mean, variance, cross-correlation). + /// + /// Uses the `Z` matrix from Phase 1 (prior-basis projection). + /// + /// **Cold→warm initialisation (§ALGO S-4.2 Phase 3, ADR-S-013 §2a–2c):** + /// On the first batch (`step == 0`), latent mean, variance, and + /// cross-correlation are seeded directly from the batch statistics + /// rather than EWMA-blending against the placeholders (`lat_mean = 0`, + /// `lat_var = 1.0`, `cross_corr = 0`). This eliminates the + /// deterministic cold-start cascade that otherwise inflates surprise + /// and coherence scores for ~60 rounds. + fn evolve_latent(&mut self, z: &Mat, k: usize) { + let b = z.nrows(); + let lam = self.forgetting_factor; + let alpha = 1.0 - lam; + let eps = self.eps; + let cold = self.step == 0; + + #[allow(clippy::cast_precision_loss)] + let b_f = b as f64; + + // Per-dimension mean and variance. + for j in 0..k { + let mut col_sum = 0.0; + for i in 0..b { + col_sum += z[(i, j)]; + } + let col_mean = col_sum / b_f; + + // EWMA-mean-centred variance (ADR-S-021 §1–2): centre on + // the pre-update EWMA mean, not the batch mean. At t = 0 + // lat_mean[j] is 0.0, giving the first-batch seeding formula. + let mut col_var = 0.0; + for i in 0..b { + let d = z[(i, j)] - self.lat_mean[j]; + col_var = d.mul_add(d, col_var); + } + col_var /= b_f; + + // Update order (ADR-S-021 §3): variance first (against + // pre-update mean), then mean. + if cold { + // Cold→warm: seed directly from first batch (ADR-S-013 §2a). + self.lat_var[j] = col_var.max(eps); + self.lat_mean[j] = col_mean; + } else { + self.lat_var[j] = lam.mul_add(self.lat_var[j], alpha * col_var.max(eps)); + self.lat_mean[j] = lam.mul_add(self.lat_mean[j], alpha * col_mean); + } + } + + // Runtime floor (ADR-S-021 §4, §ALGO S-4.2). For centred-bit cell + // inputs x in {-0.5, 0.5}^d and unit basis columns, Cauchy-Schwarz + // gives |z_j| <= sqrt(d)/2. The mean starts at zero and is seeded + // or convexly averaged from such coordinates, so |z_j-mu_j|^2 <= d. + // With variance >= 0.01 and eps > 0, each contribution, and their + // rank average, is <= d/(0.01+eps) <= 100*d, up to roundoff. + // This input bound does not apply to unbounded coordination scores. + for lat_var in self.lat_var.iter_mut().take(k) { + *lat_var = (*lat_var).max(1e-2); + } + + // Pairwise cross-correlation: C[j][l] ← λ·C[j][l] + α·(1/b)·Σᵢ zᵢⱼ·zᵢₗ + // + // Invariant (§ALGO S-4.2 Phase 3): when rank increases, new entries are + // already zero from construction. When rank decreases, outer + // entries are ignored here but preserved in the vector. + for j in 0..k { + for l in (j + 1)..k { + let mut prod_sum = 0.0; + for i in 0..b { + prod_sum = z[(i, j)].mul_add(z[(i, l)], prod_sum); + } + let batch_corr = prod_sum / b_f; + let idx = tri_idx(j, l, self.cap); + if cold { + // Cold→warm: seed directly from first batch (ADR-S-013 §2b). + self.cross_corr[idx] = batch_corr; + } else { + self.cross_corr[idx] = lam.mul_add(self.cross_corr[idx], alpha * batch_corr); + } + } + } + } + + /// Phase 4 helper: score against an axis's baseline and (optionally) + /// evolve it. + /// + /// When `evolve` is `false` the EWMA and CUSUM are left untouched. + /// This is used for the coherence axis at rank < 2, where coherence + /// does not exist (no pairs) and every score is identically zero. + /// The baseline stays cold; when rank reaches 2 the first real + /// values enter through the cold→warm path naturally. If rank + /// later drops below 2, `adapt_rank()` destroys the baseline + /// entirely. + /// + /// §ALGO S-6.1.1 pipeline with clip-pressure EWMA (§ALGO S-6.4). + #[allow(clippy::too_many_arguments)] + fn update_axis( + bl: &mut AxisBaseline, + scores: &[f64], + eps: f64, + cusum_allowance: f64, + clip_sigmas: f64, + eta: f64, + clip_pressure_decay: f64, + evolve: bool, + ) -> ScoreDistribution { + // ── Raw batch statistics (pre-clip) ───────────── + let (min, max, sum) = scores + .iter() + .fold((f64::INFINITY, f64::NEG_INFINITY, 0.0_f64), |(mn, mx, s), &v| { + (mn.min(v), mx.max(v), s + v) + }); + + #[allow(clippy::cast_precision_loss)] + let mean = sum / scores.len() as f64; + + // Z-scores computed *before* updating the fast baseline. + let max_z = bl.fast.z_score(max, eps); + let mean_z = bl.fast.z_score(mean, eps); + let baseline = bl.fast.snapshot(); + + if evolve { + // ── Per-axis effective clip (§ALGO S-6.4) ─── + // + // p = max(η, ρ̄) + // n_σ_eff = n_σ · (1 + p / (1 − p + ε)) + // + let p = eta.max(bl.clip_pressure); + let effective_clip = clip_sigmas * (1.0 + p / (1.0 - p + eps)); + + // ── Single shared clip filter ─────────────── + // Computed against the fast EWMA's current baseline. + // Cold-path bypass: ceiling() returns +∞ when cold. + let ceiling = bl.fast.ceiling(effective_clip); + + let retained: Vec = scores.iter().copied().filter(|&v| v < ceiling).collect(); + + // ── Update clip-pressure EWMA ─────────────── + // ρ_t = 1 − |retained| / |total| + // ρ̄ = λ_ρ · ρ̄ + (1 − λ_ρ) · ρ_t + #[allow(clippy::cast_precision_loss)] + let rho_t = 1.0 - (retained.len() as f64 / scores.len() as f64); + let alpha = 1.0 - clip_pressure_decay; + bl.clip_pressure = clip_pressure_decay.mul_add(bl.clip_pressure, alpha * rho_t); + + // ── Fast EWMA: receives retained samples ──── + if retained.is_empty() { + // Hold both baselines unchanged when all samples are rejected. + // With rho_t = 1, clip pressure rises so the ceiling opens on subsequent batches. + // Learning the unclipped batch would let the baselines chase a sustained shift + // and stop CUSUM accumulation under a gradual anomaly (§ALGO S-6.1.1). + } else { + bl.fast.update_raw(&retained); + } + + // ── CUSUM: receives retained samples, raw batch mean ── + // Always called — the gap uses raw_batch_mean so CUSUM + // accumulates even when all scores are clipped. The slow + // EWMA's update_raw() is a no-op on empty input. + bl.cusum.update_filtered(&retained, mean, cusum_allowance); + } + let cusum = bl.cusum.snapshot(); + + ScoreDistribution { + min, + max, + mean, + max_z_score: max_z, + mean_z_score: mean_z, + baseline, + cusum, + clip_pressure: bl.clip_pressure, + } + } + + /// Phase 5: Adapt rank based on cumulative energy. + /// + /// Every `rank_update_interval` tracker batches, find the smallest rank + /// capturing `energy_threshold` of total variance. Move by ±1. + fn adapt_rank(&mut self) { + let total_energy: f64 = self.sigmas.iter().map(|s| s * s).sum::() + self.eps; + + let mut cumulative = 0.0; + let mut target = self.cap; + + for (i, s) in self.sigmas.iter().enumerate() { + cumulative += s * s; + if cumulative / total_energy >= self.energy_threshold { + target = (i + 2).min(self.cap); + break; + } + } + target = target.max(1); + + let old_rank = self.rank; + + // Move by at most ±1 to avoid oscillation (§ALGO S-4.2 Phase 5). + if target > self.rank { + self.rank = (self.rank + 1).min(self.cap); + } else if target < self.rank { + self.rank = self.rank.saturating_sub(1).max(1); + } + + // Coherence does not exist at rank < 2. If rank just + // dropped below 2, destroy the coherence baseline so + // stale state from a previous k ≥ 2 epoch cannot leak + // into a future one. When rank reaches 2 again the + // baseline is born fresh via the cold→warm path. + if old_rank >= 2 && self.rank < 2 { + self.coherence_bl.reset_cold(); + } + } + + /// Fraction of total variance captured by the current rank. + pub fn energy_ratio(&self) -> f64 { + let total: f64 = self.sigmas.iter().map(|s| s * s).sum::() + self.eps; + let active: f64 = self.sigmas[..self.rank].iter().map(|s| s * s).sum(); + active / total + } + + /// Largest singular value of the learned subspace. + pub fn top_singular_value(&self) -> f64 { + self.sigmas.first().copied().unwrap_or(0.0) + } + + /// EWMA latent-coordinate mean for the first `rank` dimensions. + /// + /// Exposed for convergence investigation tests (ADR-S-013). + #[cfg(test)] + pub fn latent_mean(&self) -> &[f64] { + &self.lat_mean[..self.rank] + } + + /// EWMA latent-coordinate variance for the first `rank` dimensions. + /// + /// Exposed for convergence investigation tests (ADR-S-013). + #[cfg(test)] + pub fn latent_var(&self) -> &[f64] { + &self.lat_var[..self.rank] + } + + /// Update maturity counters after processing a batch. + /// + /// Observation counters advance by the number of rows, while noise + /// influence advances once per tracker batch at the same λ cadence as the + /// subspace, latent statistics, and score baselines (§ALGO S-11.5). + /// + /// When η crosses below `WARMUP_THRESHOLD` (§ALGO S-11.4), all + /// per-axis clip-pressure EWMAs are zeroed to prevent warm-up + /// contamination from echoing into production scoring. + fn update_maturity(&mut self, batch_size: usize, is_noise: bool) { + let count = batch_size as u64; + let lambda = self.forgetting_factor; + let old_eta = self.noise_influence; + + if is_noise { + self.noise_observations += count; + // One noise batch moves η one model update toward 1. + self.noise_influence = lambda.mul_add(self.noise_influence, 1.0 - lambda); + } else { + self.real_observations += count; + // One real batch forgets the same share of warm-up as the model. + self.noise_influence *= lambda; + } + + // §ALGO S-11.4: when η crosses the warm-up threshold, zero + // clip-pressure to prevent warm-up contamination echoing into + // production scoring. + if old_eta >= WARMUP_THRESHOLD && self.noise_influence < WARMUP_THRESHOLD { + self.novelty_bl.clip_pressure = 0.0; + self.displacement_bl.clip_pressure = 0.0; + self.surprise_bl.clip_pressure = 0.0; + self.coherence_bl.clip_pressure = 0.0; + } + } +} + +// ─── Upper-triangle indexing ───────────────────────────────── + +/// Map a pair `(j, l)` with `j < l` to a flat upper-triangle index. +/// +/// The triangle for a `cap × cap` matrix stores `cap*(cap−1)/2` +/// elements in row-major order: (0,1), (0,2), …, (0,cap−1), +/// (1,2), …, (cap−2, cap−1). +#[inline] +const fn tri_idx(j: usize, l: usize, cap: usize) -> usize { + // Elements before row j: j*cap − j*(j+1)/2 + // Offset within row j: l − j − 1 + j * cap - (j * (j + 1)) / 2 + l - j - 1 +} diff --git a/packages/sentinel/src/sentinel/warming_thread.rs b/packages/sentinel/src/sentinel/warming_thread.rs new file mode 100644 index 000000000..0461ff039 --- /dev/null +++ b/packages/sentinel/src/sentinel/warming_thread.rs @@ -0,0 +1,338 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +//! Background warming thread (S1 async, §ALGO S-11.6). +//! +//! Runs the deferred cell warm-up loop on a dedicated thread, decoupling +//! noise injection latency from the `ingest()` hot path. +//! +//! # Design +//! +//! The thread owns its own `SmallRng` seeded from `noise_seed + 1` +//! (offset to avoid colliding with the main sentinel's RNG sequence). +//! It takes cells out of the staging area one at a time via +//! [`StagingArea::take_highest_priority`], does the expensive noise +//! injection *without holding the lock*, then returns the cell via +//! [`StagingArea::finish_warming`] or [`StagingArea::return_warming`]. +//! +//! The main thread notifies the condvar after enqueueing new cells. +//! The thread sleeps on the condvar when there is nothing to warm. +//! +//! # Shutdown +//! +//! The sentinel sets `shutdown` to `true` **under the staging lock** and +//! notifies the condvar. The thread finishes any in-progress batch, then +//! exits. An explicit [`WarmingThreadHandle::shutdown`] joins the worker +//! and records a failed join through `tracing`. Destruction uses the same +//! non-panicking policy, because a destructor must not add a second panic to an +//! unwind already in progress. +//! +//! A worker can also stop without being asked to, by panicking. Nothing about +//! the handle changes when it does, so the owner reaps it at the point where it +//! would otherwise hand the worker more work: +//! [`WarmingThreadHandle::reap_if_finished`] joins a thread that has already +//! stopped, records the failure the same way, and empties the handle so the +//! owner can see that there is no worker left to dispatch to. +//! +//! The lock is what makes the transition observable. The worker holds the +//! staging mutex from the moment it reads the two predicates until +//! `Condvar::wait` releases it, so a writer that stores the flag without +//! that lock can land its store and its notification inside that window: +//! the sleep begins after a notification that has already been delivered +//! to nobody, and nothing wakes the thread again because the predicate it +//! would re-read only changes once. Performing the store under the same +//! mutex places it either before the worker reads the predicate — in which +//! case the worker sees it and never sleeps — or after the worker is +//! already sleeping and the notification can reach it. + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Condvar, Mutex, PoisonError}; +use std::thread::JoinHandle; + +use rand::SeedableRng; +use rand::rngs::SmallRng; +use torrust_mudlark::Coordinate; + +use super::generate_noise_batch; +use super::staging::StagingArea; + +// ─── Public handle ────────────────────────────────────────── + +/// Handle to the background warming thread. +/// +/// Owns the shutdown flag, condvar, and `JoinHandle`, and holds the same +/// staging `Arc>` the sentinel and the worker share. +/// +/// # Thread safety +/// +/// All fields are `Send + Sync`: +/// - `Arc`, `Arc`: trivially `Send + Sync`. +/// - `Arc>>`: `Send + Sync` for `C: Coordinate`, +/// and the same allocation the worker waits on. +/// - `Mutex>>`: `Send + Sync` because +/// `JoinHandle<()>: Send`. +pub struct WarmingThreadHandle { + shutdown: Arc, + condvar: Arc, + /// The mutex the worker's `wait` is paired with. The handle keeps it so + /// the shutdown transition can be made under the lock that governs the + /// predicate, which is what stops the notification from being lost. + staging: Arc>>, + /// The join handle is behind a `Mutex` so that `WarmingThreadHandle` + /// is `Sync` (ADR-S-005 — `SpectralSentinel: Send + Sync`). + handle: Mutex>>, +} + +impl WarmingThreadHandle { + /// Spawn the background warming thread. + /// + /// # Arguments + /// + /// - `staging` — shared staging area (same `Arc` as the sentinel). + /// - `batch_size` — number of synthetic samples per noise batch. + /// - `noise_seed` — if `Some`, the thread's RNG is seeded + /// deterministically from `seed + 1`. If `None`, seeded from + /// system entropy. + /// + /// # Errors + /// + /// Returns the operating system's own error when it refuses the thread. + /// A thread is granted by the environment rather than implied by the + /// arguments, so the refusal reaches callers whose parameters are + /// entirely sound — a process or user thread limit already reached, an + /// address space with no room for another stack — and it is handed back + /// rather than raised, because the caller is the only party that knows + /// what a sentinel without a warming thread is worth to it. + pub fn spawn(staging: &Arc>>, batch_size: usize, noise_seed: Option) -> std::io::Result { + let shutdown = Arc::new(AtomicBool::new(false)); + let condvar = Arc::new(Condvar::new()); + + let thread_staging = Arc::clone(staging); + let thread_shutdown = Arc::clone(&shutdown); + let thread_condvar = Arc::clone(&condvar); + + // Offset seed by 1 to avoid colliding with the main RNG. + let rng = noise_seed.map_or_else( + || SmallRng::from_rng(&mut rand::rng()), + |s| SmallRng::seed_from_u64(s.wrapping_add(1)), + ); + + let handle = std::thread::Builder::new().name("sentinel-warming".into()).spawn(move || { + warming_loop(thread_staging, thread_condvar, thread_shutdown, batch_size, rng); + })?; + + Ok(Self { + shutdown, + condvar, + staging: Arc::clone(staging), + handle: Mutex::new(Some(handle)), + }) + } + + /// Wake the background thread (call after enqueueing new cells). + pub fn notify(&self) { + self.condvar.notify_one(); + } + + /// Signal the worker through the predicate guarded by the staging lock. + fn request_shutdown(&self) { + { + let _staging = self.staging.lock().unwrap_or_else(PoisonError::into_inner); + self.shutdown.store(true, Ordering::Release); + } + self.condvar.notify_one(); + } + + /// Signal the thread to stop and wait for it to exit. + /// + /// Safe to call multiple times (subsequent calls are no-ops). + /// + /// The flag is stored under the staging lock, because that lock is what + /// the worker's `Condvar::wait` releases: a store made outside it can + /// fall between the worker reading the predicate and the worker going to + /// sleep, and the wake-up that followed the store then reaches a thread + /// that is not yet waiting. The notification itself is sent after the + /// guard is dropped, so the woken thread does not immediately block on a + /// mutex this call still holds. + /// + /// A poisoned staging mutex is taken as it stands rather than refused. + /// The lock is poisoned only when the worker panicked while holding it. + /// A failed join is recorded through `tracing` rather than raised, matching + /// the sentinel's policy that a background resource failure must not abort + /// its host. + pub fn shutdown(&self) { + self.request_shutdown(); + + let handle = self.handle.lock().expect("warming handle poisoned").take(); + let Some(handle) = handle else { + return; + }; + if handle.join().is_err() { + tracing::error!("warming thread panicked during sentinel shutdown"); + } + } + + /// Join the worker if it has already stopped, reporting whether it had. + /// + /// `false` means the worker is still running and the handle is still worth + /// notifying. `true` means the thread is gone and nothing will serve the + /// staging area again through this handle, so the owner must stop treating + /// it as a live worker; the handle is left empty, so a later + /// [`shutdown`](Self::shutdown) or destruction finds nothing to join and + /// records nothing twice. + /// + /// A failed join is recorded through `tracing` and not raised, the same + /// policy [`shutdown`](Self::shutdown) and the destructor follow: the + /// caller is an ingest on the host's own thread, and a background failure + /// must not take that thread down with it. A clean exit is silent, because + /// the only way to reach one is a shutdown the owner asked for. + /// + /// # Panics + /// + /// Panics only if the mutex guarding the join handle is poisoned, which + /// this type cannot arrange: every one of its own accesses holds that lock + /// across a `take` and nothing else, and the handle it moves has no + /// destructor that can unwind. + pub(crate) fn reap_if_finished(&self) -> bool { + let finished = { + let mut slot = self.handle.lock().expect("warming handle poisoned"); + if slot.as_ref().is_some_and(JoinHandle::is_finished) { + slot.take() + } else { + None + } + }; + + let Some(handle) = finished else { + return false; + }; + if handle.join().is_err() { + tracing::error!("warming thread panicked — the sentinel is warming cells synchronously from here"); + } + true + } + + /// Make the worker fail through its ordinary poisoned-staging path and + /// wait until the failed thread can be joined by the owner. + #[cfg(test)] + pub(crate) fn fail_worker_for_test(&self) { + let staging = Arc::clone(&self.staging); + let poisoner = std::thread::spawn(move || { + let _guard = staging.lock().expect("test staging lock must start healthy"); + panic!("test-requested staging poison"); + }); + assert!(poisoner.join().is_err(), "the staging poisoner must fail"); + self.notify(); + + loop { + let finished = self + .handle + .lock() + .expect("warming handle poisoned") + .as_ref() + .is_none_or(JoinHandle::is_finished); + if finished { + break; + } + std::thread::yield_now(); + } + + // The poison made the worker fail, but it is not the condition these + // tests exercise after that failure. Clearing it isolates the failed + // join so reset and destruction can follow their ordinary paths. + self.staging.clear_poison(); + } +} + +impl Drop for WarmingThreadHandle { + fn drop(&mut self) { + self.request_shutdown(); + + let Some(handle) = self.handle.get_mut().unwrap_or_else(PoisonError::into_inner).take() else { + return; + }; + if handle.join().is_err() { + tracing::error!("warming thread panicked during sentinel destruction"); + } + } +} + +// ─── Thread loop ──────────────────────────────────────────── + +/// Main loop of the background warming thread. +/// +/// 1. Wait on the condvar until there is warming work or shutdown. +/// 2. Take the highest-priority cell **out** of the staging area. +/// 3. Release the lock. +/// 4. Perform one noise batch (the expensive part). +/// 5. Re-acquire the lock and put the cell back (or move to ready). +/// 6. Repeat. +/// +/// The lock is held only for O(|warming|) queue operations, never +/// during the SVD / noise injection. This keeps contention with the +/// main thread minimal. +#[allow(clippy::needless_pass_by_value)] // Arcs are moved from the thread closure +fn warming_loop( + staging: Arc>>, + condvar: Arc, + shutdown: Arc, + batch_size: usize, + mut rng: SmallRng, +) { + loop { + // ── Step 1: Wait for work ─────────────────────── + let work = { + let mut guard = staging.lock().expect("staging mutex poisoned"); + + while !guard.has_warming_work() && !shutdown.load(Ordering::Acquire) { + guard = condvar.wait(guard).expect("staging condvar poisoned"); + } + + if shutdown.load(Ordering::Acquire) { + // Shutdown requested — exit immediately. Any remaining + // warming cells will be discarded by `reset()` / `Drop`. + break; + } + + // ── Step 2: Take a cell out ───────────────── + guard.take_highest_priority() + }; + // Lock released here. + + // ── Step 3: Do one batch of noise injection ───── + let Some((gnode, mut wc)) = work else { + // Spurious wake or concurrent take — loop back. + continue; + }; + + let noise = generate_noise_batch(wc.cell.width, batch_size, &mut rng); + let slices: Vec<&[f64]> = noise.iter().map(Vec::as_slice).collect(); + #[allow(clippy::cast_possible_truncation)] // depth ≤ 128, fits u8 + wc.cell.tracker.observe(&slices, wc.cell.depth as u8, true); + wc.completed_rounds += 1; + + // ── Step 4: Return the cell ───────────────────── + let mut guard = staging.lock().expect("staging mutex poisoned"); + + if wc.is_ready() { + wc.cell.tracker.seed_cusum_slow_from_baselines(); + wc.cell.tracker.reset_cusum(); + wc.cell.tracker.reset_clip_pressure(); + guard.finish_warming(gnode, wc.cell); + } else { + guard.return_warming(gnode, wc); + } + + drop(guard); + } +} + +// ─── Compile-time safety ──────────────────────────────────── + +/// Verify `WarmingThreadHandle` is `Send + Sync` so it can live +/// inside `SpectralSentinel` without breaking the sentinel's own +/// `Send + Sync` obligation. +const _: () = { + const fn assert_send_sync() {} + assert_send_sync::>(); +}; diff --git a/packages/sentinel/src/tests/README.md b/packages/sentinel/src/tests/README.md new file mode 100644 index 000000000..21f5301bc --- /dev/null +++ b/packages/sentinel/src/tests/README.md @@ -0,0 +1,297 @@ +## Crate test matrix · `tab:sentinel:crate-test-matrix` + +**Table (Crate test matrix)** + +| Test | Area | Claim | +|------|------|-------| +| (`test:crate:empty-graph-produces-root-only`) | selection | A graph that has observed nothing has nothing worth competing over, so no cell is competitively selected and the set holds a single entry. Selection is driven by accumulated importance, and where there is none the sentinel invests in nothing rather than picking arbitrarily among equals. | +| (`test:crate:root-always-present`) | selection | The root is in the full set unconditionally — here even when nothing has been observed and no cell competed at all. Ancestor closure walks upward from each selected cell, so the root's presence is what guarantees every such walk terminates at a cell that has a model rather than running off the top of the tree. | +| (`test:crate:root-is-never-competitive`) | selection | However much traffic the graph has seen, and however generous the budget, the root is never competitively selected. It accumulates every observation by construction and would win any importance contest automatically, crowding out the cells whose behaviour is actually informative. Its place in the set is structural, and it is held apart from the cells that earned theirs. | +| (`test:crate:competitive-set-respects-k`) | selection | The competitive set never exceeds the budget it was asked for, however many cells would qualify on their merits. The budget is what bounds the sentinel's modelling cost, so it is a ceiling rather than a target that a sufficiently busy graph could push past. | +| (`test:crate:budget-of-one-selects-one-cell`) | selection | A budget of one selects a cell rather than nothing, because the root leaves the field before the cut rather than after it. Taken the other way round the root wins the only slot and is then discarded for being the root, so the smallest budget the configuration admits selects nothing at all however busy the graph is. The root wins that contest on a total it accumulated before its first split and stopped adding to at the split, while its children start from zero — so the emptiness persists until a child's own total passes a figure that is no longer growing, and every slot spent on the root is a slot spent on an entry that cannot be selected. | +| (`test:crate:a-larger-budget-fills-every-slot-with-selectable-cells`) | selection | cites (`claim:selection:the-root-leaves-the-field-before-the-cut-so-every-slot-goes-to-a-selectable-cell`) | +| (`test:crate:k-zero-yields-no-competitive-entries`) | selection | cites (`claim:selection:the-competitive-set-never-exceeds-the-budget-it-was-asked-for`) | +| (`test:crate:depth-cutoff-zero-excludes-all-non-root`) | selection | The depth cutoff bounds the V-depth of every competitive cell: with the cutoff at zero, no selected cell sits deeper than zero however busy the graph. The bound is enforced while the tree is being walked rather than by discarding candidates afterwards, so the cutoff limits the work done as well as the cells returned. | +| (`test:crate:overdeep-float-cells-are-excluded`) | selection | A floating-coordinate graph can contain cells deeper than its model width because splitting is gated by V-tree depth. Their modeled suffix has no remaining width, so selection excludes them rather than overflowing the width subtraction or admitting them as enormous candidates. | +| (`test:crate:tie-breaking-is-deterministic`) | selection | Recomputing over an unchanged graph selects the same cells in the same order. Nothing in selection depends on iteration order, hashing, or timing, so two sentinels fed identical observations reach identical analysis sets — the foundation the reproducibility of every downstream score rests on. | +| (`test:crate:competitive-ordering-by-importance-then-start`) | selection | Competitive cells come back in a total order: importance descending, and among cells of equal importance, interval start ascending. Ties are therefore settled by a property of the coordinate domain rather than by whatever order the tree walk happened to produce, which is what makes the ordering reproducible and not merely stable within one run. | +| (`test:crate:internal-nodes-eligible-for-competitive-set`) | selection | A graph driven hard enough to split still yields competitive cells. The selector ranks by V-Tree importance alone and applies no filter on G-tree state, so a cell that has since become internal keeps its V-Tree position and remains eligible. Splitting refines the spatial structure; it does not silently remove cells from consideration. | +| (`test:crate:full-set-is-superset-of-competitive`) | selection | Every competitive cell is also in the full set, and the full set is never the smaller of the two. Winning the competition confers membership rather than replacing it, so a cell can be looked up by either question without the two answers contradicting each other. | +| (`test:crate:contains-returns-false-for-absent-node`) | selection | Membership is decided by what the set actually holds, not by whether a handle looks plausible: a fabricated handle the graph never allocated is simply absent. A caller holding a stale or invented cell identifier gets a negative answer rather than an accidental match on a reused slot. | +| (`test:crate:is-competitive-true-for-selected-entries`) | selection | The competitiveness predicate agrees with the competitive list: every cell the set lists as competitive answers to that question as well. Asking by handle and reading the list are two views of one fact, so the two ways a caller can learn a cell's standing cannot disagree. | +| (`test:crate:is-competitive-false-for-ancestor-only`) | selection | cites (`claim:selection:the-competitiveness-predicate-agrees-with-the-competitive-list`) | +| (`test:crate:summary-empty-graph`) | selection | A summary of a set with nothing selected reports zeroes throughout — sizes, depth span, importance span and V-depth span alike — rather than omitting the ranges or filling them with sentinels. The full size is one, because the root is there. A host parsing summaries gets the same shape whether or not anything was selected. | +| (`test:crate:summary-with-competitive-cells`) | selection | A summary counts the cells that competed and the cells the closure added as separate figures, and on a populated graph the full count strictly exceeds the competitive one. The cost of ancestry is therefore visible: a host can see how much modelling it is paying for beyond the cells it actually chose to invest in. | +| (`test:crate:summary-online-keeps-the-investment-count-whole`) | selection | The producing sets shrink to whatever is online, but the investment does not: a cell still being warmed has been paid for and has produced nothing yet, and that gap is the whole difference between the two readings. A summary taken over the online cells therefore filters the producing count and leaves the investment count whole, so a host watching a warm-up sees what it has committed to as well as what is answering. | +| (`test:crate:summary-depth-range-includes-root`) | selection | cites (`claim:selection:the-root-is-always-in-the-full-set-so-every-ancestor-chain-terminates`) | +| (`test:crate:summary-importance-range-positive`) | selection | Where cells were selected at all, the least important of them still carries importance above zero, and the reported span runs the right way round. A cell can only win the competition on accumulated observation, so nothing with no traffic behind it appears in the summary as though it had been chosen. | +| (`test:crate:summary-v-depth-range-nonzero`) | selection | cites (`claim:selection:the-root-is-never-competitive-however-important-it-is`) | +| (`test:crate:default-config-is-valid`) | config | The parameters the sentinel ships with satisfy every rule it checks them against. A host that configures nothing at all therefore starts from a coherent measurement setup rather than from a template it must first repair. | +| (`test:crate:default-config-field-values`) | config | Each default holds the value its documentation promises — the long-memory forgetting factor, the modest rank ceiling, the analysis and graph budgets, the noise batch and its fixed seed. The defaults are a calibrated set rather than arbitrary placeholders, so documenting them and shipping them are held to be the same act. | +| (`test:crate:rejects-max-rank-zero`) | config | A capacity ceiling of zero leaves the sentinel nothing to work with: a subspace tracker allowed no basis vectors can model nothing at all, so the value is refused rather than quietly read as a request for a disabled tracker. The refusal names that field and faults nothing else in an otherwise default configuration. | +| (`test:crate:rejects-forgetting-factor-out-of-range`) | config | The forgetting factor must lie strictly inside the unit interval: at one the baseline never forgets and can never adapt, at zero it retains nothing, and outside the interval the exponential weighting stops being a weighting. Both endpoints are refused along with values beyond them, and the refusal carries back the value it saw so the host can tell which of its inputs was faulted. | +| (`test:crate:accepts-forgetting-factor-near-boundaries`) | config | cites (`claim:config:a-rate-must-lie-strictly-inside-the-unit-interval-and-the-refusal-names-the-value-it-saw`) | +| (`test:crate:rejects-rank-update-interval-zero`) | config | cites (`claim:config:a-capacity-of-zero-is-refused-because-it-leaves-the-sentinel-nothing-to-work-with`) | +| (`test:crate:rejects-energy-threshold-out-of-range`) | config | cites (`claim:config:a-rate-must-lie-strictly-inside-the-unit-interval-and-the-refusal-names-the-value-it-saw`) | +| (`test:crate:rejects-eps-not-positive`) | config | The stability constant exists to keep denominators away from zero, so a value at or below zero defeats the only thing it is for. Both are refused, and the refusal reports the offending value rather than silently substituting a workable one. | +| (`test:crate:requires-finite-eps`) | config | A denominator guard must itself be finite: either infinity would make every protected denominator infinite and collapse the resulting ratios to zero. The largest finite value remains positive and is therefore admitted; the validator enforces the stated domain without imposing a fitted upper bound. | +| (`test:crate:rejects-cusum-slow-decay-out-of-range`) | config | cites (`claim:config:a-rate-must-lie-strictly-inside-the-unit-interval-and-the-refusal-names-the-value-it-saw`) | +| (`test:crate:rejects-cusum-slow-decay-below-forgetting`) | config | The CUSUM reference must have longer memory than the baseline it is measured against — strictly slower, not merely as slow. A reference adapting as fast as the baseline would follow a gradual drift instead of exposing it, and exposing exactly that drift is what the accumulator exists for. | +| (`test:crate:accepts-cusum-slow-decay-just-above-forgetting`) | config | cites (`claim:config:the-cusum-reference-must-decay-strictly-slower-than-the-baseline-it-judges`) | +| (`test:crate:rejects-cusum-coord-slow-decay-out-of-range`) | config | cites (`claim:config:a-rate-must-lie-strictly-inside-the-unit-interval-and-the-refusal-names-the-value-it-saw`) | +| (`test:crate:rejects-cusum-coord-slow-decay-below-forgetting`) | config | cites (`claim:config:the-cusum-reference-must-decay-strictly-slower-than-the-baseline-it-judges`) | +| (`test:crate:rejects-cusum-allowance-sigmas-negative`) | config | The CUSUM allowance is subtracted from each step's gap before anything accumulates, so a negative allowance would be added to every gap and would manufacture evidence of drift out of ordinary noise. Negative values are therefore refused. | +| (`test:crate:accepts-cusum-allowance-sigmas-zero`) | config | cites (`claim:config:a-noise-allowance-may-never-be-negative-because-it-would-manufacture-the-drift-it-absorbs`) | +| (`test:crate:rejects-clip-sigmas-not-positive`) | config | The clip width bounds how far above the mean an observation may sit and still update the baseline. At zero or below nothing would ever clear the bar, so the baseline would starve rather than be protected — the value is refused instead of being allowed to silence the very updates it guards. | +| (`test:crate:rejects-clip-pressure-decay-out-of-range`) | config | cites (`claim:config:a-rate-must-lie-strictly-inside-the-unit-interval-and-the-refusal-names-the-value-it-saw`) | +| (`test:crate:accepts-clip-pressure-decay-valid`) | config | cites (`claim:config:a-rate-must-lie-strictly-inside-the-unit-interval-and-the-refusal-names-the-value-it-saw`) | +| (`test:crate:rejects-analysis-k-zero`) | config | cites (`claim:config:a-capacity-of-zero-is-refused-because-it-leaves-the-sentinel-nothing-to-work-with`) | +| (`test:crate:accepts-analysis-k-one`) | config | cites (`claim:config:a-capacity-of-zero-is-refused-because-it-leaves-the-sentinel-nothing-to-work-with`) | +| (`test:crate:accepts-analysis-depth-cutoff-zero`) | config | A depth cutoff of zero is admitted rather than refused, and it means something usable: only the V-Tree root remains eligible, which effectively turns adaptive selection off. Zero is a fault in a capacity but a legitimate setting for a depth gate, because a gate that admits nothing deep expresses a policy rather than an incoherence. | +| (`test:crate:rejects-split-threshold-zero`) | config | A split threshold of zero would let a cell subdivide before accumulating any intensity at all, so the graph would fragment on its first observation instead of on evidence of sustained traffic. The threshold must be strictly positive. | +| (`test:crate:rejects-d-create-zero`) | config | cites (`claim:config:a-capacity-of-zero-is-refused-because-it-leaves-the-sentinel-nothing-to-work-with`) | +| (`test:crate:rejects-d-evict-not-greater-than-d-create`) | config | Eviction depth must sit strictly deeper than creation depth, and an equal pair is refused just as an inverted one is. The gap between them is the buffer zone — entries too deep to create children but not yet deep enough to be evicted — so collapsing it would leave a cell eligible for eviction the moment it stopped being eligible to grow. | +| (`test:crate:accepts-d-evict-one-above-d-create`) | config | cites (`claim:config:the-eviction-depth-must-sit-strictly-deeper-than-the-creation-depth-so-a-buffer-zone-exists`) | +| (`test:crate:rejects-budget-zero`) | config | cites (`claim:config:a-capacity-of-zero-is-refused-because-it-leaves-the-sentinel-nothing-to-work-with`) | +| (`test:crate:rejects-budget-below-headroom`) | config | The node budget must exceed what the depth gates themselves imply: a buffer zone of a given width can hold a number of nodes growing as a power of three, and a budget merely equal to that leaves the graph no room to manoeuvre inside its own gates. Equality is refused, not merely shortfall. | +| (`test:crate:accepts-budget-just-above-headroom`) | config | cites (`claim:config:the-node-budget-must-exceed-the-headroom-the-depth-gates-imply-and-equalling-it-is-not-enough`) | +| (`test:crate:rejects-depth-buffer-whose-headroom-cannot-be-represented`) | config | Past a certain width the headroom the depth gates imply stops being a number the machine can hold, and the depth pair is refused on its own terms rather than measured against a figure that wrapped. The requirement grows as a power of three, so a buffer in the forties already exceeds the addressable range; computing it and comparing anyway would either abort the validation that promised to return its faults, or silently compare the budget against a small wrapped remainder and admit a configuration that cannot hold. The refusal names the two depths, since they are what the host must change. | +| (`test:crate:reports-a-shortfall-at-the-widest-representable-depth-buffer`) | config | cites (`claim:config:a-depth-buffer-whose-headroom-cannot-be-represented-is-refused-on-its-own-terms`) | +| (`test:crate:refuses-a-depth-pair-whose-headroom-exponent-cannot-be-represented`) | config | cites (`claim:config:a-depth-buffer-whose-headroom-cannot-be-represented-is-refused-on-its-own-terms`) | +| (`test:crate:accepts-the-widest-representable-depth-pair-a-budget-can-clear`) | config | cites (`claim:config:a-depth-buffer-whose-headroom-cannot-be-represented-is-refused-on-its-own-terms`) | +| (`test:crate:rejects-nan-in-every-floating-point-field`) | config | Every floating-point field refuses a non-number, because the ordered comparisons that police the other values cannot see one. A comparison against a non-number is false whichever way it is written, so a bound expressed as a pair of comparisons admits it silently — and the value then spreads, since every product and sum it enters returns a non-number too. A forgetting factor admitted this way reaches the baseline arithmetic and leaves every score afterwards unusable, with nothing in the report to say which field was responsible. The guard therefore sits ahead of the bound rather than inside it. | +| (`test:crate:accepts-infinite-clip-width-and-refuses-infinite-rates`) | config | An infinite value is admitted where the interval is one-sided, because there it names a real limit rather than the absence of one. An infinite clip width is the unclipped configuration — the control arm the package's own clipping study runs against — and it compares correctly against every bound it is checked with, which is precisely what a non-number does not do. The fields whose intervals are two-sided still refuse it, and they refuse it through the bound they already carry rather than through a separate guard. | +| (`test:crate:rejects-noise-batch-size-zero-when-enabled`) | config | A noise batch of no samples is a fault only when the schedule actually asks for rounds: with an active schedule the warm-up would run rounds that feed the tracker nothing. The check is conditional on the schedule rather than absolute, because zero samples per round is coherent when there are no rounds to run. | +| (`test:crate:accepts-noise-batch-size-zero-when-disabled`) | config | cites (`claim:config:a-noise-batch-of-zero-is-faulted-only-when-the-schedule-actually-asks-for-rounds`) | +| (`test:crate:rejects-noise-batch-size-zero-with-geometric-root-and-zero-floor`) | config | cites (`claim:config:a-noise-batch-of-zero-is-faulted-only-when-the-schedule-actually-asks-for-rounds`) | +| (`test:crate:accepts-noise-batch-size-zero-with-geometric-root-and-floor-zero`) | config | cites (`claim:config:a-noise-batch-of-zero-is-faulted-only-when-the-schedule-actually-asks-for-rounds`) | +| (`test:crate:accepts-noise-seed-none`) | config | An absent noise seed is valid and simply changes where the randomness comes from: with a seed the warm-up is reproducible across restarts, without one it is drawn from system entropy. Determinism is offered rather than required, so neither choice counts as a misconfiguration. | +| (`test:crate:rejects-geometric-decay-out-of-range`) | config | The geometric decay is bound to a half-open interval rather than the open one other rates get: values at or below zero and above one are refused, but one itself is not. Zero would collapse the schedule onto its floor immediately, whereas a schedule that never tapers with depth is a legitimate thing to ask for. | +| (`test:crate:rejects-geometric-decay-nan`) | config | A decay that is not a number is refused explicitly, because every comparison against it is false and a range check alone would let it through. The configuration is rejected before such a value could reach the exponentiation and turn every round count into nonsense. | +| (`test:crate:rejects-geometric-root-zero-with-positive-min`) | config | A geometric schedule that starts at no rounds but floors at a positive count contradicts itself: the taper only ever descends from the root, so every depth would be lifted to the floor and the root would describe nothing. That combination is refused — which is why the root is faulted only when the floor is positive. | +| (`test:crate:collects-multiple-errors`) | config | Validation reports every violation it finds rather than stopping at the first, so a host with several bad fields learns of all of them in one pass instead of discovering them one restart at a time. Faulting several fields at once yields at least as many errors, each naming its own field. | +| (`test:crate:geometric-rounds-default`) | config | A geometric schedule scales its round count by the decay at each level of depth and then rests on its floor. Deeper cells are narrower and converge in fewer rounds, so tapering matches warm-up effort to the width a tracker actually has to learn, while the floor keeps even the deepest cells from being warmed with nothing. | +| (`test:crate:geometric-rounds-custom`) | config | cites (`claim:config:a-geometric-schedule-tapers-with-depth-and-then-rests-on-its-floor`) | +| (`test:crate:geometric-rounds-root-equals-min`) | config | cites (`claim:config:a-geometric-schedule-tapers-with-depth-and-then-rests-on-its-floor`) | +| (`test:crate:geometric-rounds-very-small-decay`) | config | cites (`claim:config:a-geometric-schedule-tapers-with-depth-and-then-rests-on-its-floor`) | +| (`test:crate:geometric-rounds-rounding-half-values`) | config | Round counts are whole, and a fractional product is rounded to nearest with halves going away from zero rather than truncated toward it. Truncation would bias every depth downward and compound with the taper, so a cell an exact half-round short is warmed the extra round instead of losing it. | +| (`test:crate:geometric-rounds-large-depth`) | config | At the deepest levels the G-tree can reach, the decayed product has long since collapsed toward zero, and the schedule still answers with its floor rather than with a degenerate number. The depth argument is bounded by the tree's own width, and the arithmetic stays well defined right up to that bound. | +| (`test:crate:geometric-rounds-extreme-depths-reach-the-floor`) | config | A public schedule remains tapered at depths beyond the exponent type's range. Extreme depths reach the floor, never exceed the schedule ceiling, and preserve the non-increasing shape across the conversion boundary. | +| (`test:crate:geometric-rounds-decay-one-is-constant`) | config | A decay of exactly one leaves the root untouched at every depth, giving a flat schedule that warms deep cells as heavily as shallow ones. This is what makes the upper endpoint worth admitting: uniform warm-up is expressible inside the geometric variant instead of needing a form of its own. | +| (`test:crate:geometric-rounds-root-zero-min-zero`) | config | A geometric schedule with neither root nor floor asks for no rounds at any depth. The variant can therefore express a complete absence of warm-up without switching to the explicit form, and it does so with no special casing: the taper of nothing is nothing, and a floor of nothing lifts it nowhere. | +| (`test:crate:explicit-rounds-for-depth`) | config | An explicit schedule is a direct lookup by depth, and depths past the end of the vector reuse its last entry rather than falling to zero or failing. The tail is the host's statement about all remaining depths, so a short vector still describes an unbounded tree. | +| (`test:crate:explicit-rounds-single-entry`) | config | cites (`claim:config:an-explicit-schedule-is-read-by-depth-and-clamps-to-its-last-entry-beyond-its-length`) | +| (`test:crate:explicit-rounds-non-monotonic`) | config | cites (`claim:config:an-explicit-schedule-is-read-by-depth-and-clamps-to-its-last-entry-beyond-its-length`) | +| (`test:crate:explicit-empty-is-disabled`) | config | An explicit schedule that can never yield a round counts as noise switched off, and the two facts agree: the round count is zero at every depth and the schedule reports itself disabled. That agreement is what lets other rules key off the disabled flag instead of re-deriving it. | +| (`test:crate:explicit-all-zeros-is-disabled`) | config | cites (`claim:config:an-explicit-schedule-that-can-never-yield-a-round-reports-itself-as-noise-switched-off`) | +| (`test:crate:explicit-single-zero-is-disabled`) | config | cites (`claim:config:an-explicit-schedule-that-can-never-yield-a-round-reports-itself-as-noise-switched-off`) | +| (`test:crate:explicit-mixed-zeros-not-disabled`) | config | A single non-zero entry anywhere keeps noise enabled, even where the shallow depths ask for none. A zero at a given depth is a statement about that depth alone, so a schedule may deliberately warm only the deeper cells and still counts as active. | +| (`test:crate:geometric-is-disabled-when-root-and-min-are-zero`) | config | A geometric schedule is disabled only when both its root and its floor are zero, because either one alone still produces rounds. The floor lifts every depth to at least its own count, and the root sets the count at the shallow depths before the taper has descended. Reading only one of the two calls a schedule silent that is still asking for warm-up. | +| (`test:crate:geometric-is-not-disabled-when-min-positive`) | config | cites (`claim:config:a-geometric-schedule-is-disabled-only-when-both-its-root-and-its-floor-are-zero`) | +| (`test:crate:geometric-is-not-disabled-when-root-positive-and-min-zero`) | config | cites (`claim:config:a-geometric-schedule-is-disabled-only-when-both-its-root-and-its-floor-are-zero`) | +| (`test:crate:max-rounds-geometric`) | config | The most a geometric schedule can ever ask for is its root, because the taper only descends from there. A host sizing buffers for warm-up can read the ceiling off the root alone, without evaluating the schedule at any depth. | +| (`test:crate:max-rounds-geometric-floor-above-root`) | config | cites (`claim:config:the-ceiling-of-a-geometric-schedule-is-its-root-because-the-taper-only-descends`) | +| (`test:crate:max-rounds-explicit`) | config | For an explicit schedule the ceiling is the largest entry it holds, not the first. Since the explicit form imposes no ordering, the depth-zero value carries no promise about the rest and the maximum has to be found rather than assumed. | +| (`test:crate:max-rounds-explicit-empty`) | config | cites (`claim:config:the-ceiling-of-an-explicit-schedule-is-its-largest-entry-not-its-first`) | +| (`test:crate:max-rounds-explicit-large`) | config | cites (`claim:config:the-ceiling-of-an-explicit-schedule-is-its-largest-entry-not-its-first`) | +| (`test:crate:noise-schedule-default-matches-doc`) | config | The shipped schedule is the geometric one its documentation describes, and it reports itself active. Its numbers are calibrated rather than arbitrary: the root sits a little above the worst-case baseline convergence measured at the default forgetting factor, and the floor covers deep cells whose convergence scales down with analysis width without vanishing. | +| (`test:crate:default-config-has-no-warnings`) | config | The shipped configuration draws no advisories, because the default schedule was calibrated against the default forgetting factor. Defaults that validated but warned would be an odd thing to ship, so the two sets of defaults are kept consistent with each other. | +| (`test:crate:warns-when-noise-root-too-low-for-lambda-099`) | config | A configuration whose warm-up rounds fall short of what its memory needs is advised, not refused: it is arithmetically sound, but baselines may not converge before real observations arrive, so early scores would be unreliable. Advice and refusal are separate channels — validation would pass this configuration unchanged, and only the warning list carries the concern. | +| (`test:crate:no-warning-when-noise-root-sufficient-for-lambda-095`) | config | How much warm-up is recommended falls with the forgetting factor, because a shorter memory converges sooner: a schedule too thin for a long-memory baseline is adequate for a shorter one. The same schedule draws advice or silence depending on the memory it is paired with, since the recommendation is a relation between the two rather than a property of either. | +| (`test:crate:warns-when-small-batch-and-lambda-095`) | config | Batch size enters the recommendation as well: with few synthetic samples per round, each round buys less convergence, so the same shorter memory demands markedly more rounds and a schedule that was adequate becomes advised against. Warm-up is really measured in observations rather than in rounds, and the recommendation reflects that. | +| (`test:crate:no-warning-for-explicit-schedule-with-enough-rounds`) | config | The advisory judges whatever the schedule actually yields at depth zero, whichever variant it is written in. An explicit schedule generous enough at the root passes the same check a geometric one would, so the recommendation is about warm-up delivered and not about how the host chose to express it. | +| (`test:crate:warning-display-is-informative`) | config | A rendered advisory carries the numbers a host needs in order to act on it: the root it found, the root it recommends, and the forgetting factor that set that recommendation. Advice naming only the problem would leave the reader to re-derive the target. | +| (`test:crate:rejects-coordinate-width-below-the-tracker-minimum`) | config | A coordinate width narrower than the smallest dimension a subspace tracker can model is refused at construction, with the same structured failure the configuration faults carry. The width is a parameter of the type rather than a field of the configuration, so validating the configuration alone can never see it, and the root tracker spans the whole width — at one dimension its lone basis vector spans the entire space, novelty is identically zero, and the tracker reports a settled model of everything while modelling nothing. Refusing is what lets the constructor's success mean the sentinel it returns can measure. | +| (`test:crate:accepts-the-narrowest-modellable-coordinate-width`) | config | cites (`claim:config:a-coordinate-width-below-the-tracker-minimum-is-refused-at-construction`) | +| (`test:crate:collects-a-width-fault-alongside-a-configuration-fault`) | config | cites (`claim:config:a-coordinate-width-below-the-tracker-minimum-is-refused-at-construction`) | +| (`test:crate:refuses-a-coordinate-width-above-the-centred-bit-ceiling`) | config | A coordinate width above what the centred bit vector can carry is refused at construction, the same way a width below the tracker minimum is. The bridge that turns a coordinate into centred bits is open to any implementor, and the spatial layer asks only that the width fit the coordinate type, so a host whose coordinates are wider than the vector can otherwise ask for a sentinel wider than the vector that feeds it. Nothing would fault: the slots past the vector's length come back as zeros, a centred bit is ±0.5 and never zero, and every dimension past the end would be modelled over a constant the coordinate stream never produced — a settled reading of data that does not exist, mixed into novelty, residual and rank alike. The refusal names the width and the ceiling, since those are what the host must reconcile. | +| (`test:crate:accepts-the-widest-modellable-coordinate-width`) | config | cites (`claim:config:a-coordinate-width-above-the-centred-bit-ceiling-is-refused-at-construction`) | +| (`test:crate:warming-thread-refusal-names-the-setting-and-the-environment`) | config | The refusal a host receives when the environment will not give the engine a warming thread names the setting that asked for one and quotes the operating system's own account of the refusal. Nothing in the configuration is wrong in that case, so a message that said only that a configuration was invalid would send an operator searching values that are all correct: naming the setting says which request to withdraw, and quoting the environment says whether withdrawing it is the right answer at all or whether the machine is simply out of threads. | +| (`test:crate:unrepresentable-noise-batch-is-rejected-before-construction`) | | — | +| (`test:crate:rejects-noise-matrix-size-even-when-the-outer-vector-fits`) | | — | +| (`test:crate:ignores-unallocated-noise-batch-size-when-disabled`) | | — | +| (`test:crate:clip-bias-is-negligible-at-steady-state`) | clipping | Rejecting the upper tail leaves the settled baseline where an unclipped run puts it. The clip removes a fraction of a percent of the mass of a right-skewed score distribution, so the downward bias it induces is smaller than the baseline's own steady-state jitter: outlier resistance is bought without moving the reference it protects. | +| (`test:crate:variance-estimate-unbiased-despite-clipping`) | clipping | The spread estimate survives tail truncation as well as the mean does. A second moment is far more sensitive to a missing tail than a first, so this is the tighter half of the same audit: a baseline built under clipping is calibrated and not merely correctly centred. | +| (`test:crate:graduated-exemption-prevents-bistable-attractor`) | clipping | Clip widths from tight to loose all settle on the same fixed point. A clipped baseline is a nonlinear filter with a second, biased fixed point where tight clipping keeps rejecting the very evidence that would loosen it; the graduated exemption widens the basin of the correct one during warm-up, so no configuration falls into the other. | +| (`test:crate:exemption-decay-does-not-cause-transient-instability`) | clipping | As the exemption is spent the effective clip tightens, and the baselines pass through that tightening without a spike or a dip — once the exemption has largely decayed, no rolling mean departs from the eventual steady state by more than the jitter envelope. The width is a smooth function of the exemption rather than a switch, which is what keeps a trajectory from being thrown across a basin boundary. | +| (`test:crate:clip-ceiling-stabilises-after-exemption-decay`) | clipping | The ceiling is the baseline's own mean and spread scaled by the clip width, so it settles when they do: once the exemption is spent its rolling variation stays within a few percent. A ceiling that kept swinging would clip in bursts and feed those bursts straight back into the mean and spread that define it. | +| (`test:crate:slow-ewma-clipping-does-not-bias-cusum-reference`) | clipping | The long-memory reference is filtered by the same ceiling as the short-memory baseline, and once the two have been seeded into agreement they stay in agreement across a long run. Clipping therefore adds no bias of its own on the reference side, so a gap between the two can be read as drift rather than as an artefact of filtering. | +| (`test:crate:cusum-bounded-through-clip-transitions`) | clipping | A tightening clip does not manufacture evidence of drift: through the whole stretch after seeding, the drift accumulators stay far below anything a host would act on. The moment the clip narrows is when the two baselines are most likely to disagree, so it is where a false alarm would appear if the mechanisms interfered with one another. | +| (`test:crate:rolling-mean-empty`) | convergence | Averaging an empty window yields zero rather than a division by zero, so a metric may ask for the mean of a stretch that turned out to hold nothing and still get an answer it can carry forward. | +| (`test:crate:rolling-mean-single`) | convergence | cites (`claim:convergence:the-window-average-is-the-plain-unweighted-mean-of-what-it-covers`) | +| (`test:crate:rolling-mean-known`) | convergence | The window average is the plain unweighted mean of the values it covers, with no decay of its own. The yardstick is deliberately unlike the baselines it measures: an instrument with a memory would judge a long-memory baseline by an equally sluggish standard. | +| (`test:crate:rolling-mean-negative`) | convergence | cites (`claim:convergence:the-window-average-is-the-plain-unweighted-mean-of-what-it-covers`) | +| (`test:crate:settled-all-within-tolerance`) | convergence | cites (`claim:convergence:settling-is-dated-from-the-round-after-the-last-violation`) | +| (`test:crate:settled-never`) | convergence | A trace still violating at its final round is not settled at all, and the answer is an absence rather than a round number. Settling is a claim about the whole remainder of a run, so it cannot be asserted while the run is still moving. | +| (`test:crate:settled-after-specific-round`) | convergence | Settling is dated from the round after the last violation, not from the first round that happened to fall inside tolerance. The search walks backwards from the end, so a trace that strays and returns is credited only from its return. | +| (`test:crate:settled-near-zero-reference-skipped`) | convergence | An axis whose reference level is effectively zero is passed over rather than failed. Tolerance is relative, so a near-zero reference would make every deviation enormous; an axis that never activated is treated as having nothing to say instead of as permanently unconverged. | +| (`test:crate:settled-single-trace`) | convergence | cites (`claim:convergence:settling-is-dated-from-the-round-after-the-last-violation`) | +| (`test:crate:settled-subset-of-axes`) | convergence | Only the axes actually asked about can hold settling back, so a wildly mismatched axis outside the requested set is invisible. The axes mature at very different rates, and a caller may need to know when the ones it depends on have settled without waiting on one it does not use. | +| (`test:crate:converged-too-few-values`) | convergence | A trace too short to hold both a window and a separate reference window yields no verdict better than its own length. With no room to compare an early stretch against a late one, the metric reports that convergence has not been demonstrated rather than guessing that it has. | +| (`test:crate:converged-already-stable`) | convergence | A trace that never departs from its final level is converged from its first round. The reference is the trace's own tail, so this metric answers when a run reached where it ended up — not whether that destination was the right one. | +| (`test:crate:converged-after-transient`) | convergence | A run that starts far from its eventual level is dated as converged after the transient and well before the end: the backward walk stops at the last window whose mean departed from the tail reference. Comparing windows rather than single rounds is what separates a genuine departure from ordinary jitter about a settled level. | +| (`test:crate:converged-near-zero-reference`) | convergence | A trace whose final level is effectively zero is reported as converged from the start rather than divided by. As with the settling metric, an inactive channel is excluded from the judgement instead of poisoning it. | +| (`test:crate:block-mean-late-near-zero`) | convergence | An axis whose late block is effectively zero yields no verdict at all rather than a ratio against nothing. An axis that never activated has no steady state to be stationary about, and saying so is more honest than reporting an enormous relative error. | +| (`test:crate:block-mean-identical-blocks`) | convergence | Stationarity is measured as the relative gap between an early block mean and a late one, so two blocks drawn from the same settled stretch differ by nothing. Averaging over blocks is what lets the test tell a baseline still moving from one merely jittering in place. | +| (`test:crate:block-mean-known-error`) | convergence | cites (`claim:convergence:stationarity-is-the-relative-gap-between-an-early-block-mean-and-a-late-one`) | +| (`test:crate:trailing-cv-too-few`) | convergence | Asking for the jitter of a window longer than the trace yields not a number, rather than a figure computed from whatever happened to be available. A short trace is not a quiet one, and the metric refuses to let the two be confused. | +| (`test:crate:trailing-cv-constant`) | convergence | Jitter is reported as a fraction of the level it sits on, so a flat tail has none whatever that level happens to be. Normalising by the mean is what makes the figure comparable across axes whose scores differ by orders of magnitude. | +| (`test:crate:trailing-cv-known`) | convergence | cites (`claim:convergence:jitter-is-reported-as-a-fraction-of-the-level-it-sits-on`) | +| (`test:crate:generate-noise-shape-and-values`) | noise | Injected noise arrives in exactly the shape the tracker expects, every entry plus or minus a half. Synthetic warm-up traffic therefore carries the same centring the real encoding produces, so a model warmed on noise is warmed on the same kind of thing it will later be asked to judge. | +| (`test:crate:as-slices-preserves-data`) | noise | Handing a generated batch to the tracker borrows it rather than transforming it: the same values arrive in the same order. Nothing is rescaled on the way in, so what a run does is attributable to the noise that was generated for it. | +| (`test:crate:cfg-test-is-valid`) | convergence | The configuration convergence is measured under passes the same validation any production configuration must. Results measured here are therefore statements about a legal sentinel rather than about a corner of the parameter space the crate would refuse to construct. | +| (`test:crate:cfg-b16-overrides-batch-size`) | convergence | The larger-batch variant differs from the standard one in batch size alone and is likewise valid, so a comparison between the two isolates the effect of batch size. Convergence claims made at two batch sizes are then about one system observed differently, not about two systems. | +| (`test:crate:cfg-production-overrides`) | convergence | The production-like variant changes three things together — a longer memory, larger batches and a slower rank cadence — and remains valid. These are the settings the shipped noise schedule is sized from, so they are exercised as a set rather than one at a time. | +| (`test:crate:convergence-rounds-table`) | convergence | How many noise rounds each axis needs, and what those rounds cost, can be re-derived on demand rather than taken on trust from the schedule that ships. The diagnostic asserts nothing: it exists so that a change in the scoring pipeline can be checked against the round budget the schedule was sized from. | +| (`test:crate:axis-drift-investigation`) | convergence | When an axis refuses to settle, a per-round trace of its scores, baseline, spread, clip pressure and ceiling is available beside the theory it is supposed to follow. Diagnosing a convergence failure means finding where the empirical trajectory leaves the predicted one, and that needs the trajectory itself rather than a pass or a fail. | +| (`test:crate:svd-timing-comparison`) | convergence | The cost of a warm-up run can be measured with the tracing layer installed and again without it, so the instrument's own overhead is separable from what it measures. A timing figure used to size the noise schedule would otherwise silently include the cost of having taken it. | +| (`test:crate:eta-starts-at-one-for-cold-tracker`) | noise | A tracker that has seen nothing counts as entirely noise-taught. Everything it will learn first comes from injected traffic, so the honest starting position is that none of its state yet reflects real observations. | +| (`test:crate:counters-start-at-zero-for-cold-tracker`) | noise | A fresh tracker claims no observations of either kind, and the total is exactly the two counts together. The counters are a record of what was fed in rather than an estimate, so nothing may be presumed before anything arrives. | +| (`test:crate:eta-tracks-theory-exactly`) | noise | The noise share follows the same recurrence as the model it describes. A real batch applies λ once, so after `k` batches the initial influence has been multiplied by λ exactly `k` times, independent of the rows in them. | +| (`test:crate:eta-decay-is-independent-of-batch-size`) | noise | One-sample and sixteen-sample batches each evolve the learned model once, so they must also apply the same single decay to its warm-up influence. The separate observation counters continue to record how many rows arrived. | +| (`test:crate:eta-maturity-threshold-uses-model-batch-count`) | noise | The maturity crossing count comes directly from repeatedly applying the configured forgetting factor until influence is strictly below the same threshold the tracker uses. No observed run supplies the expected count. | +| (`test:crate:eta-decreases-monotonically-under-real-data`) | noise | Every real batch lowers the noise share and none raises it, so warm-up influence is spent and never regained by ordinary operation. Monotonicity is what makes the share usable as a maturity signal: a threshold crossing means the same thing whenever it happens. | +| (`test:crate:eta-increases-monotonically-under-noise`) | noise | Injection pushes the share back up from wherever real data drove it, batch by batch and without reversal. A cell whose model is re-warmed is therefore re-declared immature rather than left claiming a maturity its state no longer has. | +| (`test:crate:eta-stays-in-unit-interval`) | noise | The share is a proportion and stays one under any interleaving of injected and real batches. Both updates are convex steps toward an endpoint inside the interval, so no mixture of workloads can carry it out of range and no consumer has to guard against a value that is not a fraction. | +| (`test:crate:eta-converges-to-one-under-noise`) | noise | Indefinite injection holds the share at exactly one, its fixed point: noise cannot make a model more than entirely noise-taught. A long warm-up therefore has a stable end state rather than an accumulating one. | +| (`test:crate:eta-decays-toward-zero-under-real-only`) | noise | A long enough run of real batches drives the share below the maturity threshold, so warm-up is forgotten on the same geometric cadence as the model rather than on a schedule determined by batch size. | +| (`test:crate:observation-counters-mixed-sequence`) | noise | The counters tally samples rather than batches and keep the two kinds apart: a stretch of injection moves only the noise count, a stretch of real traffic only the other, and the total is their sum. A host can therefore still tell how much of a model's experience was synthetic long after the noise share itself has decayed away. | +| (`test:crate:counters-track-real-only-sequence`) | noise | cites (`claim:noise:the-counters-tally-samples-not-batches-and-keep-the-two-kinds-apart`) | +| (`test:crate:ewma-cold-start-sets-mean-directly`) | convergence | The first batch seeds the baseline outright instead of being blended with the placeholder the baseline was constructed with. That placeholder is a stand-in chosen to keep early z-scores finite, not an observation, so blending against it would plant a bias that then has to decay away. | +| (`test:crate:ewma-cold-start-sets-variance-from-batch`) | convergence | cites (`claim:convergence:the-first-batch-seeds-the-baseline-outright-instead-of-blending-with-the-placeholder`) | +| (`test:crate:ewma-pure-convergence-rate`) | convergence | Fed a constant level, the baseline closes on it at the rate the forgetting factor dictates — the steps needed to reach a given relative error are what geometric decay predicts. Convergence time is therefore something that can be computed from configuration rather than discovered by running. | +| (`test:crate:ewma-higher-lambda-converges-slower`) | convergence | A longer memory buys steadiness by taking longer to arrive: the slower-decaying baseline needs strictly more steps to reach the same relative error. This is the trade the forgetting factor exists to express, and it is why a warm-up budget is sized against the configured factor rather than fixed at some number of rounds. | +| (`test:crate:ewma-convergence-independent-of-batch-size`) | convergence | A batch is one step of learning however many samples it carries: batches spanning a wide range of sizes all reach the same relative error in the same number of steps, because the update consumes the batch mean. Convergence is counted in rounds, not in samples. | +| (`test:crate:ewma-error-decreases-monotonically`) | convergence | Approaching a constant level the error never rebounds: each step is a convex move toward the target and cannot carry the mean past it. A baseline that oscillated on the way in would leave every convergence test ambiguous about when it had arrived. | +| (`test:crate:ewma-steady-state-is-stable`) | convergence | Once arrived, the baseline neither drifts nor oscillates — a long further run against the same level leaves it exactly there. Arrival is permanent while the input holds, so a later departure can be attributed to the input rather than to the estimator. | +| (`test:crate:ewma-tracks-step-change`) | convergence | After settling on one level the baseline re-converges when the input steps to another, within a time the forgetting factor bounds. A baseline models what is normal now, so a genuine change of regime has to be adopted rather than resisted indefinitely. | +| (`test:crate:ewma-variance-convergence`) | convergence | The spread converges on much the same schedule as the mean, since both are carried by the same decay factor. That matters because a z-score divides one by the other: a spread lagging far behind its mean would leave scores miscalibrated even after the level itself looked settled. | +| (`test:crate:ewma-clipping-slows-convergence`) | clipping | A ceiling can only slow the approach to a distant level, never hasten it: when the target sits far above the current mean, the clip rejects the very batches that would move it. This is the cost the warm-up exemption exists to avoid paying while a baseline has not yet found its level. | +| (`test:crate:graduated-clip-formula-is-smooth-and-monotonic`) | clipping | The effective clip width narrows smoothly and without reversal as the exemption is spent, from an effectively open ceiling while the model is entirely noise-taught down to the nominal width once it is not. There is no step anywhere along the transition, so no batch is judged by a much tighter rule than the batch before it. | +| (`test:crate:clip-exemption-eliminates-feedback-loop`) | clipping | No axis can clip itself into a baseline it then keeps rejecting the evidence against. The exemption holds the ceiling open while the noise share is high, so the first batches enter unfiltered and even the highest-variance axis settles well inside the round budget instead of being pinned by its own early ceiling. | +| (`test:crate:cold-warm-eliminates-surprise-nonstationarity`) | convergence | Because the latent spread is seeded from the first batch rather than starting at a placeholder far above its eventual value, the surprise axis is near stationary from its earliest rounds: early scores and late ones differ by well under a factor of two. Surprise divides by that spread, so a placeholder set too high would show up as a slow rise indistinguishable from a real trend. | +| (`test:crate:cusum-seeding-prevents-false-drift`) | convergence | Seeding the long-memory reference from the converged short-memory baseline at the hand-over from injected to real traffic keeps the switch from reading as drift: the accumulators stay far below anything actionable across a long real-data run. Left to converge on its own the long reference would lag for many hundreds of rounds, and the whole lag would be banked as evidence. | +| (`test:crate:per-axis-convergence-b4-within-bound`) | convergence | Every axis settles inside the round budget the noise schedule is sized from, each judged at its own tolerance. The axes differ by more than an order of magnitude in inherent jitter, so one shared tolerance would either excuse the quietest or condemn the noisiest; this per-axis bound is what makes a warm-up of bounded length sufficient. | +| (`test:crate:per-axis-convergence-b16-within-bound`) | convergence | cites (`claim:convergence:every-axis-settles-inside-the-round-budget-at-its-own-tolerance`) | +| (`test:crate:production-lambda-converges-within-bound`) | convergence | cites (`claim:convergence:every-axis-settles-inside-the-round-budget-at-its-own-tolerance`) | +| (`test:crate:per-axis-convergence-b4-robust-across-seeds`) | convergence | The budget holds across a spread of seeds and not merely for one lucky noise sequence. Two of the axes carry real seed-to-seed variance in when they settle, so a bound demonstrated once would not be a bound at all: sizing a shipped schedule needs the worst case over sequences. | +| (`test:crate:noise-baselines-converge-within-bound`) | convergence | A baseline counts as converged when an early block of rounds and a late one agree, not when its round-to-round jitter has stopped. Steady-state jitter is irreducible and differs by more than a hundredfold across the axes, so a single-round criterion tight enough for the quietest axis would report the noisiest as permanently unconverged long after it had reached its correct level. | +| (`test:crate:baseline-variance-converges-within-bound`) | convergence | cites (`claim:convergence:convergence-is-agreement-between-an-early-and-a-late-block-not-the-absence-of-jitter`) | +| (`test:crate:frozen-subspace-converges-at-least-as-fast`) | convergence | Holding the subspace still never makes the baselines settle later than letting it evolve. Scores are measured against the model, so a model that is itself still moving is a second source of variation on top of the input; removing it can only help. | +| (`test:crate:rank-reaches-max-within-expected-rounds`) | convergence | Rank climbs to its ceiling within a few adaptation intervals, one step at each. The model is therefore at full width long before the baselines settle, so the bulk of a warm-up is spent learning score distributions rather than discovering how many directions to keep. | +| (`test:crate:coherence-activates-at-rank-two`) | convergence | Coherence is a statement about pairs of latent directions, so below two directions there are no pairs and the axis does not exist. Its baseline is held at the cold placeholder rather than being fed the identically zero scores, and it enters through the cold-start path the moment a second direction appears — otherwise it would converge onto zero and then have to unlearn it. | +| (`test:crate:energy-ratio-stabilises-under-noise`) | convergence | The share of energy the retained directions capture settles onto a narrow plateau near the threshold that chose the rank. That ratio is the quantity rank adaptation reads, so its settling is what stops the rank from oscillating. | +| (`test:crate:report-baseline-is-pre-update-snapshot`) | convergence | A report carries the baseline the batch was scored against, never the one the batch produced: the first report shows the cold placeholder even though the internal state has already moved, and the second shows exactly what the first batch left behind. A batch can therefore never partly explain itself away, and a reader can reconstruct the comparison that produced the scores. | +| (`test:crate:z-scores-stay-bounded-at-steady-state`) | convergence | Measured against a settled baseline, ordinary traffic scores near zero on average and stays unremarkable on every single round. The baselines are calibrated and not merely stable: a systematic offset would mean every batch looked mildly anomalous, leaving no headroom to signal one that genuinely was. | +| (`test:crate:latent-variance-reaches-steady-state`) | convergence | The latent spread settles far below the value a freshly constructed tracker holds. That gap is why the first batch seeds the spread instead of blending toward it: starting an order of magnitude high would suppress the surprise axis for as long as the gap took to decay. | +| (`test:crate:latent-mean-stays-near-zero`) | convergence | Under centred input the latent mean stays at zero, so the value a fresh tracker starts from was already the right one. The encoding centres every bit for exactly this reason, and it is what lets the spread be measured about a fixed origin rather than a moving one. | +| (`test:crate:subspace-evolution-does-not-dominate-latvar-transient`) | convergence | The latent spread's approach to its steady state is governed by the forgetting factor, not by the subspace still moving underneath it: an evolving basis and an effectively frozen one land in the same neighbourhood. Warm-up length can therefore be reasoned about from the decay alone. | +| (`test:crate:production-noise-provides-reasonable-novelty-baseline`) | convergence | Novelty settles quickest of the four axes: a short schedule already places it within a few percent of where a far longer run leaves it. It is built from reconstruction error rather than from latent statistics, so it does not have to wait for the latent distribution to settle first — which is what makes a short warm-up useful before the other axes are ready. | +| (`test:crate:deterministic-under-same-seed`) | convergence | The same seed replays the same run bit for bit, baseline for baseline. Nothing in the pipeline depends on iteration order over an unordered structure or on timing, so a convergence figure is a property of the configuration and the seed rather than of the machine that measured it. | +| (`test:crate:starts-at-zero`) | cusum | A fresh accumulator holds no evidence and has taken no steps. Drift is something that must be accumulated from observations, so a newly created axis starts owing the host nothing to explain. | +| (`test:crate:accumulates-under-sustained-elevation`) | cusum | Evidence builds when batch means stay above the slow reference: an accumulator settled by a long run of ordinary batches grows once the scores are consistently elevated. Because each batch adds its remaining gap to the running sum, sustained elevation compounds — which is the point, since it separates a persistent shift from a single loud batch. | +| (`test:crate:steps-since-reset-increments`) | cusum | Every update advances the step count by exactly one, whatever the batch contained and whether or not the gap contributed anything. The count is how long evidence has been gathering, so a host can read an accumulator value against the number of chances it had to grow rather than against nothing. | +| (`test:crate:clamps-at-zero-when-below-baseline`) | cusum | A run of batches below the reference leaves the accumulator at zero rather than driving it negative. Quiet time banks no credit: the sum cannot go into debt during a lull and then have to be repaid before a genuine rise registers. Evidence of drift is always built from the present run, never netted against the past. | +| (`test:crate:allowance-absorbs-noise`) | cusum | The allowance is a dead band that ordinary variation does not cross: against a slow baseline with real spread, a generous allowance leaves slightly elevated batches accumulating essentially nothing. Because the band is scaled by the baseline's own deviation rather than being an absolute score, a noisy cell tolerates more before it counts as drifting than a quiet one does. | +| (`test:crate:allowance-uses-only-slow-variance`) | cusum | The dead band is exactly the configured sigma multiplier times the slow baseline's standard deviation. A known baseline therefore gives a known first step, with no unrelated stability constant widening the allowance. | +| (`test:crate:resets-to-zero`) | cusum | A reset discards the accumulated evidence and the count of steps that built it together. Neither outlives the other, so a host acknowledging a regime change is not left reading a fresh sum against a stale step count. | +| (`test:crate:reset-preserves-slow-baseline`) | cusum | What a reset does not touch is the slow baseline: its mean and spread come through unchanged. Acknowledging drift clears the evidence, not the reference the evidence was measured against — otherwise every acknowledgement would throw away a long-memory baseline that takes many batches to rebuild, and the axis would be blind while it re-converged. | +| (`test:crate:reset-cold-clears-everything`) | cusum | Clearing goes further than resetting: the evidence, the step count and the slow baseline all return to their freshly-constructed state, the baseline back to its placeholders rather than to whatever it had drifted to. This is the operation for an axis that has ceased to exist — coherence when the rank falls too low, say — where keeping a reference learned under a geometry that no longer holds would be worse than having none. | +| (`test:crate:seed-slow-from-aligns-baselines`) | cusum | The slow reference can be started from a fast baseline that has already converged, taking its mean and spread exactly. After noise is injected the two would otherwise disagree for a very long time — the slow baseline's memory is far too long to catch up — and every batch in between would register a gap that reflects the mismatch rather than any real drift. Seeding closes that gap in one step. | +| (`test:crate:update-filtered-matches-update-no-clip`) | cusum | The pre-filtered entry point differs from the ordinary one only in who applies the outlier filter: given a clip wide enough that none would be rejected, both reach the same accumulated value. Moving the filter out to a shared pipeline therefore changes where clipping happens and not what drift means. | +| (`test:crate:snapshot-reports-slow-baseline`) | cusum | The snapshot carries the slow reference itself, and that reference tracks the scores it was fed: after a run of batches at one level the reported mean sits near it. A host reading a report can therefore see what the drift was measured against, not merely how much of it accumulated. | +| (`test:crate:starts-cold`) | ewma | A newly constructed baseline is cold, and the mean and spread it reports are placeholders of one rather than anything measured. They are deliberately wide so that a value scored before any real data has arrived cannot come back with an extreme departure. | +| (`test:crate:first-update-warms`) | ewma | One update is the whole of warming: a single batch takes the baseline out of its cold state for good. There is no minimum sample count to reach and no separate warming phase to wait out at this level. | +| (`test:crate:first-update-sets-mean-to-batch-mean`) | ewma | The first batch is adopted outright: the mean becomes the batch's own mean exactly, with no trace of the placeholder blended in. Decaying towards the first real data instead would leave the baseline anchored for many batches to a value that was never an observation. | +| (`test:crate:first-update-single-value-keeps-unit-variance`) | ewma | A batch of one carries a mean but no spread, so the mean is taken and the variance is left exactly as it stood. Computing a deviation from a single sample would give zero — a spread the data does not support, and one that would make every subsequent value look like an outlier. | +| (`test:crate:first-update-multi-value-sets-sample-variance`) | ewma | A first batch of more than one sets the spread from its own mean squared deviation, taken about the batch mean and divided by the count with no correction applied. Both the centre and the spread the baseline starts from are therefore measurements rather than defaults. | +| (`test:crate:reset-cold-restores-placeholder-state`) | ewma | Resetting a warm baseline returns it to exactly the state construction left it in — placeholders restored and warmth withdrawn — rather than merely clearing a flag over learned numbers. What it learned is discarded, not hidden. | +| (`test:crate:reset-cold-allows-re-warming`) | ewma | After a reset the next batch is adopted outright, exactly as the very first one was: the mean lands on the new batch's value with nothing of the discarded baseline pulling it back. Resetting therefore genuinely re-starts the baseline rather than leaving it to decay out of its old position. | +| (`test:crate:seed-from-copies-warm-state`) | ewma | Seeding transfers both what a baseline learned and the fact that it learned it: the receiver takes the source's mean and spread and becomes warm. This is how a slow baseline is started from a fast one that has already converged, so the pair begin in agreement instead of the slow one spending its warm-up disagreeing with a baseline that is already right. | +| (`test:crate:seed-from-cold-source-does-not-warm-target`) | ewma | Warmth is never manufactured by seeding. A cold source hands over its placeholder numbers but leaves the receiver cold, so a baseline seeded before anything was learned still takes the cold path on its own first batch rather than blending against values nothing measured. | +| (`test:crate:seed-from-cold-source-withdraws-warmth-from-a-warm-receiver`) | ewma | A receiver that had learned something and is then seeded from a source that had not comes back cold, rather than keeping its own warmth over the placeholders it has just been handed. Warmth belongs to the baseline being transferred and not to the receiver: it is what says whether those two numbers were measured or were the pair a fresh baseline starts from. A receiver left warm over them would clip and score against a notion of normal nothing had observed, and the cold path that exists to replace exactly that state would never run again. | +| (`test:crate:update-empty-is-noop`) | ewma | A batch with nothing in it leaves both the mean and the spread exactly where they were. An idle interval is therefore not a data point: the baseline does not drift simply because time passed without observations. | +| (`test:crate:outliers-are-rejected`) | ewma | A warm baseline refuses values above its own ceiling before it learns anything, and a batch consisting entirely of such values moves it not at all. This is the poisoning defence: an attacker cannot walk the notion of normal upward by feeding extremes, because the extremes are precisely what never reaches the baseline. | +| (`test:crate:update-single-value-does-not-update-variance`) | ewma | cites (`claim:ewma:a-batch-of-one-carries-no-spread-so-the-variance-is-left-untouched`) | +| (`test:crate:update-skips-clipping-when-cold`) | ewma | While cold there is no ceiling to clip against, so nothing is rejected: a first batch far above the placeholder mean is accepted whole and becomes the baseline. The placeholders are not a real notion of normal, and filtering against them would let an arbitrary constant decide which of the first real observations the sentinel was allowed to see. | +| (`test:crate:update-raw-empty-is-noop`) | ewma | cites (`claim:ewma:an-empty-batch-teaches-the-baseline-nothing`) | +| (`test:crate:update-raw-matches-update-when-no-clipping`) | ewma | The unclipped path is the clipped one with only the filter removed: given a clip so wide that nothing could be rejected, the two produce the same mean and the same spread. Callers that do their own outlier rejection get identical arithmetic, so the two entry points cannot drift into disagreeing about what a batch means. | +| (`test:crate:decays-toward-new-data`) | ewma | Sustained new data pulls the baseline away from what it learned before: a mean established at one level and then fed a different one repeatedly ends up near the new level. The baseline tracks the present rather than averaging over all history, which is what makes it a moving notion of normal rather than a permanent one. | +| (`test:crate:higher-decay-forgets-slower`) | ewma | The decay factor is the length of the baseline's memory. Two baselines started at the same level and fed the same contradicting data diverge in the expected direction: the one with the higher factor still holds more of the original level. This is what lets a fast and a slow baseline over the same stream disagree usefully, which is the whole basis of drift detection. | +| (`test:crate:z-score-is-zero-at-mean`) | ewma | A value sitting at the baseline scores essentially nothing. The z-score measures signed departure from what the baseline expects, so agreement with it is the origin of the scale rather than a point somewhere along it. | +| (`test:crate:z-score-positive-above-mean`) | ewma | cites (`claim:ewma:the-z-score-is-signed-departure-from-the-baseline-and-zero-at-it`) | +| (`test:crate:z-score-negative-below-mean`) | ewma | cites (`claim:ewma:the-z-score-is-signed-departure-from-the-baseline-and-zero-at-it`) | +| (`test:crate:ceiling-returns-infinity-when-cold`) | ewma | cites (`claim:ewma:while-cold-there-is-no-ceiling-to-clip-against-so-nothing-is-rejected`) | +| (`test:crate:ceiling-returns-mean-plus-sigmas-when-warm`) | ewma | Once warm, the ceiling stands a fixed number of standard deviations above the mean — the requested multiple of the baseline's own square-rooted spread, added to its own centre. The threshold is therefore relative to what this cell has learned, not an absolute score chosen in advance for every cell alike. | +| (`test:crate:clip-sigmas-affects-ceiling`) | ewma | The clip setting is a monotone dial on how much the baseline is willing to learn from: given the same elevated value, a baseline clipping at a wide multiple moves at least as far as one clipping tightly. Tightening the setting can only ever admit less, so an operator turning it down is trading responsiveness for poisoning resistance and never the reverse. | +| (`test:crate:snapshot-matches-state`) | ewma | The snapshot a report carries holds the same mean and spread the baseline's own accessors report. What a caller reads out of a report is the state the engine is scoring against, not a rounded or separately derived summary of it. | +| (`test:crate:variance-floor-is-respected`) | ewma | A batch of identical values has no deviation at all, yet the spread does not reach zero: a floor holds it above. Without it a perfectly quiet period would collapse the spread, the ceiling would close onto the mean, and every subsequent value — however ordinary — would be rejected as an outlier, leaving the baseline permanently frozen at the quiet level. | +| (`test:crate:from-u128-zero-all-minus-half`) | bits | A clear bit becomes minus a half and a set bit plus a half. An all-zero coordinate therefore becomes minus a half throughout: the encoding is symmetric, while zero expected mean requires balanced bits. | +| (`test:crate:from-u128-max-all-plus-half`) | bits | cites (`claim:bits:a-clear-bit-becomes-minus-a-half-and-a-set-bit-plus-a-half`) | +| (`test:crate:from-u128-one-only-lsb-set`) | bits | cites (`claim:bits:the-most-significant-bit-stands-at-index-zero`) | +| (`test:crate:from-u128-msb-first-ordering`) | bits | Bits are stored most significant first: a value carrying only its top bit puts that bit at index zero and nothing else anywhere. This ordering is what lets a cell at depth `d` take its working observation by skipping the first `d` entries, because those are exactly the bits routing fixed. | +| (`test:crate:u128-custom-width-populates-n-bits`) | bits | The vector is backed by a fixed hundred-and-twenty-eight-slot array, but the requested width is what counts as populated: asking for eight bits fills eight slots and leaves the remainder at zero, and the observation reports its length as eight rather than as the array's size. A domain narrower than the backing store is therefore not padded with fabricated structure. | +| (`test:crate:u128-zero-width-gives-empty`) | bits | cites (`claim:bits:a-requested-width-populates-exactly-that-many-slots-and-leaves-the-rest-zero`) | +| (`test:crate:u128-width-capped-at-128`) | bits | cites (`claim:bits:a-width-wider-than-the-coordinate-is-capped-at-the-coordinates-own-width`) | +| (`test:crate:u64-max-all-plus-half`) | bits | The centring rule is a property of the conversion, not of the coordinate type: a sixty-four-bit value with every bit set produces sixty-four slots of plus a half, exactly as the wider coordinate does. A host working in a narrower domain gets the same representation, so the engine above the boundary need not know which width it was fed. | +| (`test:crate:u64-width-capped-at-64`) | bits | Asking a sixty-four-bit coordinate for a wider observation does not invent bits: the width is capped at what the value actually holds. A sentinel configured for the wider domain can therefore be handed narrower coordinates without the shift going out of range or the tail of the vector filling with structure that was never observed. | +| (`test:crate:from-coord-delegates-correctly`) | bits | The generic entry point the engine actually calls produces the same observation, bit for bit and length for length, as calling the conversion on the value directly. There is one encoding rather than two that happen to agree, so nothing can drift between the path tests exercise and the path production code takes. | +| (`test:crate:suffix-zero-is-full-vector`) | bits | A cell at the root of the G-tree has had no bits resolved by routing, so its working observation is the whole vector — not a copy of it, but the same values. The root tracker analyses the full domain width, which is the base case the depth arithmetic has to agree with. | +| (`test:crate:suffix-intermediate-depth`) | bits | At depth `d` the observation drops exactly its first `d` entries and keeps everything behind them. Those leading bits are constant across every value routed into the cell, so removing them leaves precisely the part that varies — the tracker's width is the domain width less its depth, and the bits it sees are the tail of the same vector rather than a re-derivation. | +| (`test:crate:suffix-at-len-is-empty`) | bits | A cell as deep as its domain is wide has nothing left to analyse: routing has resolved every bit, and the suffix is empty rather than an error. This holds at the full width and at a narrower configured one alike, which is why such cells are excluded from the analysis set by width rather than caught as a failure when a tracker tries to run on them. | +| (`test:crate:suffix-panics-beyond-len`) | bits | A depth past the observation's own width is treated as a programming error and not as a value to be tolerated. Empty is the answer at exactly the width; beyond it there is no honest answer, so the boundary between the degenerate case and the impossible one is drawn rather than blurred by silently clamping. | +| (`test:crate:suffix-norm-squared-is-width-over-four`) | bits | Because every centred bit has magnitude one half, a suffix's squared norm is a quarter of its width and nothing else — the same for every value and at every depth, checked here across saturated, sparse and arbitrary inputs at all depths. Observation magnitude therefore carries no information: a residual the tracker measures is a departure from learned structure, never an artefact of which value arrived. | +| (`test:crate:cold-maturity-is-fully-noisy`) | readout | A tracker that has seen nothing reports no observations of either kind and a baseline owed entirely to noise. The cold state is not "unknown" but a definite statement: whatever this tracker would score against, none of it came from real traffic, so a host can discount its scores rather than having to guess how new the model is. | +| (`test:crate:total-observations-sums-real-and-noise`) | readout | Total experience is the real and injected observations added together, and both counts stay separately readable beside it. The sum says how much the model has absorbed; the split says how much of that was manufactured during warm-up — a host needs the second to interpret the first, so the report offers the convenience without collapsing the distinction. | +| (`test:crate:novelty-not-saturated-when-residual-dof-positive`) | readout | Novelty is degenerate exactly when no residual degrees of freedom remain, and with residual dimensions still unexplained by the learned subspace it is not. Novelty measures the energy the model failed to account for, so while there is somewhere for that energy to live the axis is measuring something real. | +| (`test:crate:novelty-saturated-when-residual-dof-zero`) | readout | cites (`claim:readout:novelty-is-degenerate-exactly-when-no-residual-degrees-of-freedom-remain`) | +| (`test:crate:novelty-saturable-when-cap-ge-dim`) | readout | Saturability is a separate question from saturation: a tracker whose rank cap reaches its working dimension may still have residual room today, yet it is one of those that can lose the novelty axis as rank grows. The report distinguishes the two so a host can tell a temporary reading from a cell where novelty will eventually stop meaning anything. | +| (`test:crate:novelty-not-saturable-when-cap-lt-dim`) | readout | cites (`claim:readout:novelty-can-become-degenerate-whenever-the-rank-cap-reaches-the-working-dimension`) | +| (`test:crate:contour-snapshot-fields`) | readout | A contour snapshot carries the spatial shape, the accumulated volume, and the structural churn since the previous report side by side. Standing state and change-since-last-time are different questions about the spatial layer, and the snapshot answers both at once so a host need not difference successive reports to see the graph move. | +| (`test:crate:analysis-set-summary-empty`) | readout | A summary describing an analysis set with nothing selected still reports every field, its ranges zeroed rather than omitted, and still counts the root as a member of the full set. The shape a host parses does not change with how busy the sentinel is, and the permanent root tracker is visible even at the quietest extreme. | +| (`test:crate:analysis-set-summary-populated`) | readout | A populated summary reports its depth and importance spans as ordered pairs, low end first, and its three sizes widen as the definition of membership loosens: the cells that won the competition, the full set their ancestry closes over, and the investment set that also holds cells still warming. The nesting is what lets a host read the price of analysing a cell as well as the choice to analyse it. | +| (`test:crate:analysis-set-summary-with-degenerate-skips`) | readout | Cells too narrow to support a tracker are counted in the current selection snapshot rather than quietly dropped. Recomputing replaces the count instead of accumulating it, so the report describes the graph the host can inspect now while still exposing a persistently narrow configuration. | +| (`test:crate:member-score-has-cell-identity`) | readout | A member score names the cell it came from — a well-ordered interval and a depth — alongside a real number on each of the four axes and its standardised counterpart. Coordination scores describe a group, so without the identity a host could see that the group behaved oddly but not which part of the domain to look at. | +| (`test:crate:warming-targets-follow-current-selection`) | readout | Retained warming ancestors stop counting as competitive targets as soon as selection changes. | +| (`test:crate:new-starts-at-rank-one`) | subspace | A newly built cell model claims one direction, has counted no observations of either kind, and regards itself as entirely noise-taught. Starting at a single axis means the model asserts as little structure as it can and has to earn every further direction from energy it actually observes; starting at full noise influence means nothing it later reports is trusted until real traffic has displaced the warm-up that shaped it. | +| (`test:crate:new-accepts-min-tracker-dim`) | subspace | The narrowest width the engine admits is admitted, and the model it produces is an ordinary one starting at a single axis. The minimum is a boundary that is included rather than approached: cells right at the edge of being too deep to analyse still get a working model instead of a special case. | +| (`test:crate:new-panics-on-zero-dim-debug`) | subspace | A width below the minimum is treated as a caller's mistake, not as a state to accommodate: building a model for a cell with nothing left to analyse fails loudly in debug builds and names the offending width. Filtering such cells out is the selector's job, so a model that receives one has been handed something upstream should have excluded, and the fault is worth more than a degenerate model that would score nothing meaningfully. | +| (`test:crate:new-panics-on-dim-one-debug`) | subspace | cites (`claim:subspace:a-width-below-the-minimum-is-a-callers-fault-caught-in-debug-rather-than-a-state-to-accommodate`) | +| (`test:crate:dim-and-cap-reflect-construction`) | subspace | A cell's rank ceiling is the lesser of the configured maximum and its own width: a wide cell is capped by policy, a narrow one by geometry. There are no more independent directions than dimensions to hold them, so the width binds where it is the smaller of the two, and one configuration can serve cells of every depth without being retuned per depth. | +| (`test:crate:scoring-geometry-matches-state`) | subspace | A model reports the geometry its scores were computed in: the width it works over, the ceiling it may grow to, and the residual degrees of freedom left after the claimed directions are removed. That last figure is the divisor novelty is normalised by, so publishing it lets a host compare scores from cells of different depths and ranks instead of comparing numbers whose scale it cannot see. | +| (`test:crate:observe-returns-the-scoring-rank`) | subspace | A report carries the rank that was in force while the batch was scored, which on an unadapted model is the rank it was built with: adaptation happens after scoring, so a report never describes a model that did not produce it. The tracker is told which depth it serves but keeps that to itself — the depth reaches a host on the cell report, which is assembled from the cell rather than echoed back from the model. | +| (`test:crate:rank-change-report-describes-the-scoring-state`) | subspace | On a batch that changes rank, the report and the tracker's geometry snapshot keep the earlier rank and residual degrees of freedom that normalised novelty, while the current rank advances for the next batch. Multiplying novelty by the published residual degrees of freedom recovers the residual energy, so the geometry beside the score can be used to reconstruct its scale. | +| (`test:crate:non-adapting-batch-reports-current-rank-as-scoring-rank`) | subspace | When the adaptation interval does not fall on a batch, the current model and the scoring snapshot agree: the report rank equals the tracker's rank and its residual degrees of freedom are derived from that same rank. | +| (`test:crate:observe-per-sample-when-enabled`) | subspace | Per-row detail is produced only where a cell is configured to want it. Building it costs a standardisation of every axis for every row, which is worth paying when a host needs to know which observation in a batch was responsible and wasted when it only needs the batch's summary — so the choice is made per configuration rather than always. | +| (`test:crate:observe-no-per-sample-when-disabled`) | subspace | cites (`claim:subspace:per-row-detail-is-produced-only-where-it-is-configured-because-it-costs-work-per-row`) | +| (`test:crate:observe-report-batch-size-matches`) | subspace | Where per-row detail is produced there is exactly one entry for every row handed in, whether the batch was a single observation or many, and the same model gives both answers in turn. The correspondence is positional, so a host can attribute a score back to the observation that earned it without the model needing to know what that observation was. | +| (`test:crate:maturity-noise-only`) | subspace | Maturity is counted in observations rather than in calls: a batch of several injected rows advances the noise tally by that many and leaves the real tally untouched. Counting rows is what makes the figure comparable across cells fed at different batch sizes, and keeping the two tallies apart is what lets a host ask how much of what a cell knows it was taught deliberately. | +| (`test:crate:maturity-real-only`) | subspace | cites (`claim:subspace:maturity-counts-observations-row-by-row-and-keeps-the-injected-and-the-real-apart`) | +| (`test:crate:maturity-mixed-real-and-noise`) | subspace | cites (`claim:subspace:maturity-counts-observations-row-by-row-and-keeps-the-injected-and-the-real-apart`) | +| (`test:crate:noise-influence-decays-toward-zero-for-real`) | subspace | Sustained real traffic drives noise influence below the maturity threshold. The figure falls by the forgetting factor once per tracker batch, matching the model whose warm-up share it measures regardless of how many rows that batch carries. | +| (`test:crate:noise-influence-converges-toward-one-for-noise`) | subspace | Warm-up is re-enterable. A cell pushed part-way down by real traffic climbs back toward full influence when injection resumes, by the same geometric step run in the other direction. Cells are re-warmed after splits and long silences, so a figure that could only fall would leave a re-taught cell wrongly claiming its knowledge came from traffic it never saw. | +| (`test:crate:novelty-low-for-repeated-pattern`) | subspace | Novelty is whatever the learned directions fail to explain, divided by the room left over after those directions are removed. A pattern the model has been trained on lies almost inside its own axes, so what is left is nearly nothing and the pattern scores as unremarkable — the model reports familiarity by having nothing to report. | +| (`test:crate:novelty-high-for-unseen-pattern`) | subspace | cites (`claim:subspace:novelty-is-what-the-learned-directions-fail-to-explain-so-a-familiar-pattern-scores-low`) | +| (`test:crate:coherence-cold-at-rank-one`) | subspace | Coherence measures whether pairs of axes move together as they usually do, so at a single axis it does not exist: there is no pair, and the score is exactly zero on every batch rather than some small residue. Because those zeroes are an absence of the question and not an answer to it, the axis's baseline is deliberately left cold while rank stays at one — otherwise it would learn that zero is normal and treat the first genuine coherence value, once a second axis appears, as an alarm. | +| (`test:crate:rank-stays-bounded-by-max-rank`) | subspace | However long a cell runs and however strongly its traffic is structured, rank stays within a floor of one axis and the ceiling it was built with. The ceiling is what bounds the cost of every later step — the work per batch grows with rank — and the floor is what keeps a model from disappearing entirely during a quiet stretch and having to be rebuilt from nothing. | +| (`test:crate:rank-acquires-buffer-dimension`) | subspace | A cell keeps one axis more than the energy threshold strictly demands. Fed a single dominant pattern, the leading direction alone already captures the required share, yet the model settles at two directions rather than one. The spare axis is where a genuinely new direction first shows up: without it, novel structure would have to displace the established pattern before the model could represent it at all, and the arrival would be invisible until it was already dominant. | +| (`test:crate:energy-ratio-and-top-singular-value-evolve`) | subspace | A model that has been fed traffic reports a leading direction with real strength behind it and an energy share that is positive and cannot exceed the whole. The share is what the claimed axes explain out of everything the model holds, so it is bounded above by construction, and a leading value at zero would mean the model had learned nothing — the two figures together are how a host reads whether a cell's model has substance. | +| (`test:crate:report-energy-figures-describe-the-model-that-scored-the-batch`) | subspace | The energy share and leading singular value a report carries belong to the model that scored the batch, not to the model the batch left behind. Both are read off the same sigmas, and the batch replaces those sigmas before the report is assembled, so a figure read afterwards would describe a model that has not scored anything yet — and the energy share read afterwards is not even that, but the evolved sigmas divided by the rank that scored, a pairing no model ever held. The rank and the geometry beside them already describe the scoring model, so a host reading one report would be comparing an energy share against a rank drawn from a different moment. | +| (`test:crate:cusum-allowance-is-invariant-to-eps`) | subspace | Changing the denominator stability constant does not change a novelty CUSUM trajectory. The allowance belongs to the slow baseline variance alone, so two otherwise identical trackers accumulate the same drift even when their configured stability constants differ by the scale of that variance. | +| (`test:crate:cusum-reset-zeroes-steps`) | subspace | Clearing a cell's drift evidence restarts the count of batches that evidence was gathered over, so the very next batch is the first step of a new run rather than the next of an old one. Accumulated drift is only interpretable against how long it took to accumulate, and a fresh sum read against a stale count would look like a sudden collapse in drift rather than a deliberate acknowledgement of it. | +| (`test:crate:seed-cusum-slow-from-baselines-then-reset`) | subspace | Finishing warm-up is a two-step handover applied to every axis at once: the long-memory reference is seeded from the short-memory one that has already converged on the injected traffic, and only then is the evidence cleared. Done in that order, drift detection resumes from a state where the two references agree, so the first real batches are scored against a reference that is already current instead of registering the warm-up's own leftover gap as drift for as long as the slow memory takes to catch up. | +| (`test:crate:explicit-reset-clip-pressure`) | subspace | Clip pressure records how often a cell has lately been discarding scores as outliers, and it widens that cell's own outlier band while it is high. Warm-up is exactly when it runs high, since injected traffic is scored against a barely-formed model. Clearing it zeroes every axis together, so a cell entering production judges its first real batches by the ordinary band rather than by one still slackened by the noise it was taught with. | +| (`test:crate:eta-threshold-crossing-zeros-clip-pressure`) | subspace | cites (`claim:subspace:clearing-clip-pressure-zeroes-every-axis-so-warm-up-clipping-does-not-slacken-production-scoring`) | +| (`test:crate:a-declined-incremental-step-still-yields-a-usable-model`) | subspace | A step the incremental strategy declines is answered by the dense one, so what comes back is always a basis that was actually re-orthogonalised. The incremental path builds a small kernel and back-transforms through it, which needs spare dimensions to be stable, and at the coordination tier's width it declines every step on exactly those grounds. Declining is the honest answer, and the dispatcher's response to it is to run the strategy that does not need the step. That is the same response the incremental path now gives when its own corrective factorisation fails — the step that re-orthogonalises the basis — because the alternative is returning the basis from before that step under a field documented orthonormal, which every caller writes straight into a tracker and then relies on. The failure of that factorisation cannot be provoked from outside without a hook into the linear algebra, so what is exercised here is the fallback it now takes. | +| (`test:crate:cold-start-seeds-variance-from-first-batch`) | variance | The very first batch seeds the spread outright instead of being blended into the placeholder a fresh model was built with. One batch of centred noise is enough to leave every axis's spread near the value the geometry predicts for such traffic and well below the placeholder, so a cell begins scoring against something it observed rather than against a constant it was born holding and would take many batches to shake off. | +| (`test:crate:b1-surprise-bounded`) | variance | A cell fed one observation at a time keeps producing modest surprise scores over a long run of batches. Because each batch's contribution to the spread is measured from the mean carried in from before it, a single row still contributes a real squared departure — where measuring scatter within the batch would find none at all, drive the divisor to its floor, and turn ordinary traffic into scores several orders of magnitude too large. | +| (`test:crate:b1-latent-variance-stable`) | variance | cites (`claim:variance:centring-on-the-running-mean-keeps-a-single-row-batch-from-collapsing-the-spread`) | +| (`test:crate:b2-no-systematic-surprise-inflation`) | variance | Surprise carries no systematic bias from the batch size a cell is configured with: on ordinary traffic, once the model has settled, the average score sits around one. Measuring scatter within a small batch would understate the spread by a predictable fraction and inflate every score by its reciprocal, so a cell reading batches two at a time would look permanently twice as surprised as an identical cell reading them in larger groups. A score of about one has to mean "as expected" everywhere, or no threshold can be set once and applied across cells. | +| (`test:crate:batch-size-invariant-surprise-ratio`) | variance | cites (`claim:variance:the-surprise-ratio-carries-no-batch-size-bias-so-a-score-of-about-one-means-as-expected-everywhere`) | +| (`test:crate:runtime-floor-prevents-degenerate-collapse`) | variance | A stream in which every observation is identical genuinely has no spread, and centring on the running mean does not rescue it — the mean converges onto the repeated value and each batch's contribution goes to zero with it. A floor holds the divisor above a small positive value regardless, so the cell keeps scoring on a bounded scale. Without it, the first observation that differed at all would be divided by nothing and reported as unboundedly surprising, which says more about the arithmetic than about the traffic. | +| (`test:crate:variance-adapts-to-distribution-shift`) | variance | The spread follows the traffic. When a settled cell's input jumps to a substantially larger scale, the recorded spread on every axis climbs well past where it sat before, because it is a decaying average of what is arriving rather than a fixed property learned once. A shift in scale is therefore absorbed within a bounded stretch of batches instead of being reported as anomalous indefinitely — surprise is meant to answer "unusual for this cell lately", not "unusual for this cell when it was young". | +| (`test:crate:update-order-variance-before-mean`) | variance | Within a single batch the spread is measured before the mean moves, against the centre that batch arrived to find. The ordering is visible because the mean demonstrably shifts across the batch while the resulting spread stays in the range the earlier centre implies. Were the order reversed, a batch would be measured against a centre it had just pulled toward itself and would partly explain its own deviation away — the same reason scoring happens before the model absorbs the batch at all. | +| (`test:crate:shutdown-returns-under-repeated-spawn-and-stop-cycles`) | warmup | Shutting the warming thread down returns, every time, over a long run of spawn-and-stop cycles that does nothing else — the arrangement that puts the request at its most likely to land while the worker is between reading its predicate and sleeping on it. A shutdown that is lost in that window does not fail loudly: the worker sleeps on, the join waits for it, and the sentinel's own drop never completes, so what a host would see is a process that stops rather than an error it can act on. | +| (`test:crate:dropping-a-sentinel-consumes-a-failed-worker-join`) | warmup | A warming worker can fail before its owner is destroyed. Destruction still completes without unwinding, because the drop path records the failed join instead of turning a background failure into a destructor panic. | +| (`test:crate:reset-consumes-a-failed-worker-join`) | warmup | Reset follows the same host-preserving policy as destruction: a worker that has already failed is joined and recorded, then reset rebuilds the sentinel instead of panicking over a failure that happened in the background. | +| (`test:crate:selection-refresh-survives-warming-handoffs`) | warmup | Waiting, in-flight and ready cells keep the latest selection flag across both worker return paths. | \ No newline at end of file diff --git a/packages/sentinel/src/tests/analysis_set.rs b/packages/sentinel/src/tests/analysis_set.rs new file mode 100644 index 000000000..c40774981 --- /dev/null +++ b/packages/sentinel/src/tests/analysis_set.rs @@ -0,0 +1,519 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +//! # Test index +//! +//! | Test | Area | Claim | +//! |------|------|-------| +//! | [`empty_graph_produces_root_only`] | selection | A graph that has observed nothing has nothing worth competing over, so no cell is competitively selected and the set holds a single entry. Selection is driven by accumulated importance, and where there is none the sentinel invests in nothing rather than picking arbitrarily among equals. | +//! | [`root_always_present`] | selection | The root is in the full set unconditionally — here even when nothing has been observed and no cell competed at all. Ancestor closure walks upward from each selected cell, so the root's presence is what guarantees every such walk terminates at a cell that has a model rather than running off the top of the tree. | +//! | [`root_is_never_competitive`] | selection | However much traffic the graph has seen, and however generous the budget, the root is never competitively selected. It accumulates every observation by construction and would win any importance contest automatically, crowding out the cells whose behaviour is actually informative. Its place in the set is structural, and it is held apart from the cells that earned theirs. | +//! | [`competitive_set_respects_k`] | selection | The competitive set never exceeds the budget it was asked for, however many cells would qualify on their merits. The budget is what bounds the sentinel's modelling cost, so it is a ceiling rather than a target that a sufficiently busy graph could push past. | +//! | [`budget_of_one_selects_one_cell`] | selection | A budget of one selects a cell rather than nothing, because the root leaves the field before the cut rather than after it. Taken the other way round the root wins the only slot and is then discarded for being the root, so the smallest budget the configuration admits selects nothing at all however busy the graph is. The root wins that contest on a total it accumulated before its first split and stopped adding to at the split, while its children start from zero — so the emptiness persists until a child's own total passes a figure that is no longer growing, and every slot spent on the root is a slot spent on an entry that cannot be selected. | +//! | [`a_larger_budget_fills_every_slot_with_selectable_cells`] | selection | cites (´claim:selection:the-root-leaves-the-field-before-the-cut-so-every-slot-goes-to-a-selectable-cell´) | +//! | [`k_zero_yields_no_competitive_entries`] | selection | cites (´claim:selection:the-competitive-set-never-exceeds-the-budget-it-was-asked-for´) | +//! | [`depth_cutoff_zero_excludes_all_non_root`] | selection | The depth cutoff bounds the V-depth of every competitive cell: with the cutoff at zero, no selected cell sits deeper than zero however busy the graph. The bound is enforced while the tree is being walked rather than by discarding candidates afterwards, so the cutoff limits the work done as well as the cells returned. | +//! | [`overdeep_float_cells_are_excluded`] | selection | A floating-coordinate graph can contain cells deeper than its model width because splitting is gated by V-tree depth. Their modeled suffix has no remaining width, so selection excludes them rather than overflowing the width subtraction or admitting them as enormous candidates. | +//! | [`tie_breaking_is_deterministic`] | selection | Recomputing over an unchanged graph selects the same cells in the same order. Nothing in selection depends on iteration order, hashing, or timing, so two sentinels fed identical observations reach identical analysis sets — the foundation the reproducibility of every downstream score rests on. | +//! | [`competitive_ordering_by_importance_then_start`] | selection | Competitive cells come back in a total order: importance descending, and among cells of equal importance, interval start ascending. Ties are therefore settled by a property of the coordinate domain rather than by whatever order the tree walk happened to produce, which is what makes the ordering reproducible and not merely stable within one run. | +//! | [`internal_nodes_eligible_for_competitive_set`] | selection | A graph driven hard enough to split still yields competitive cells. The selector ranks by V-Tree importance alone and applies no filter on G-tree state, so a cell that has since become internal keeps its V-Tree position and remains eligible. Splitting refines the spatial structure; it does not silently remove cells from consideration. | +//! | [`full_set_is_superset_of_competitive`] | selection | Every competitive cell is also in the full set, and the full set is never the smaller of the two. Winning the competition confers membership rather than replacing it, so a cell can be looked up by either question without the two answers contradicting each other. | +//! | [`contains_returns_false_for_absent_node`] | selection | Membership is decided by what the set actually holds, not by whether a handle looks plausible: a fabricated handle the graph never allocated is simply absent. A caller holding a stale or invented cell identifier gets a negative answer rather than an accidental match on a reused slot. | +//! | [`is_competitive_true_for_selected_entries`] | selection | The competitiveness predicate agrees with the competitive list: every cell the set lists as competitive answers to that question as well. Asking by handle and reading the list are two views of one fact, so the two ways a caller can learn a cell's standing cannot disagree. | +//! | [`is_competitive_false_for_ancestor_only`] | selection | cites (´claim:selection:the-competitiveness-predicate-agrees-with-the-competitive-list´) | +//! | [`summary_empty_graph`] | selection | A summary of a set with nothing selected reports zeroes throughout — sizes, depth span, importance span and V-depth span alike — rather than omitting the ranges or filling them with sentinels. The full size is one, because the root is there. A host parsing summaries gets the same shape whether or not anything was selected. | +//! | [`summary_with_competitive_cells`] | selection | A summary counts the cells that competed and the cells the closure added as separate figures, and on a populated graph the full count strictly exceeds the competitive one. The cost of ancestry is therefore visible: a host can see how much modelling it is paying for beyond the cells it actually chose to invest in. | +//! | [`summary_online_keeps_the_investment_count_whole`] | selection | The producing sets shrink to whatever is online, but the investment does not: a cell still being warmed has been paid for and has produced nothing yet, and that gap is the whole difference between the two readings. A summary taken over the online cells therefore filters the producing count and leaves the investment count whole, so a host watching a warm-up sees what it has committed to as well as what is answering. | +//! | [`summary_depth_range_includes_root`] | selection | cites (´claim:selection:the-root-is-always-in-the-full-set-so-every-ancestor-chain-terminates´) | +//! | [`summary_importance_range_positive`] | selection | Where cells were selected at all, the least important of them still carries importance above zero, and the reported span runs the right way round. A cell can only win the competition on accumulated observation, so nothing with no traffic behind it appears in the summary as though it had been chosen. | +//! | [`summary_v_depth_range_nonzero`] | selection | cites (´claim:selection:the-root-is-never-competitive-however-important-it-is´) | + +//! Crate-level tests for **`AnalysisSet`** — which cells the sentinel spends +//! its modelling effort on. +//! +//! Selection happens in two movements. First a competition: V-Tree entries +//! within the depth cutoff and wide enough to support a tracker are ranked by +//! importance, and the top few win. Then a closure: every winner's G-tree +//! ancestors are pulled in whether or not they competed, so that each selected +//! cell has an unbroken chain of models back to the root. +//! +//! The two movements make two kinds of membership, and the accessors keep +//! them apart. A cell in the full set has a tracker; a cell in the +//! competitive set additionally earned its place. The root belongs to the +//! first and never to the second — it is present unconditionally so that +//! every ancestor chain terminates, which is a structural obligation rather +//! than a claim that the root deserved investment. +//! +//! Ranking is by V-Tree importance alone. The selector applies no filter on +//! G-tree state, so an internal cell is as eligible as a terminal one, and +//! ties are broken by interval start so that recomputing over an unchanged +//! graph returns the same cells in the same order. + +use std::collections::BTreeSet; + +use torrust_mudlark::{Config as GvConfig, GNodeId, GvGraph}; + +use crate::analysis_set::*; + +// ── Helpers ───────────────────────────────────────────── + +fn test_graph() -> GvGraph { + let cfg = GvConfig { + split_threshold: 100, + depth_create: 3, + depth_evict: 6, + budget: Some(100_000), + alpha_relax: 0.75, + bounded_eviction: true, + }; + GvGraph::new(cfg) +} + +/// Feed uniform traffic to force splits. +fn populated_graph() -> GvGraph { + let mut graph = test_graph(); + for i in 0u128..2000 { + graph.observe(i * (u128::MAX / 2000), 1u64); + } + graph +} + +// ── recompute() — construction ────────────────────────── + +/// A graph that has observed nothing has nothing worth competing over, so no +/// cell is competitively selected and the set holds a single entry. Selection +/// is driven by accumulated importance, and where there is none the sentinel +/// invests in nothing rather than picking arbitrarily among equals. +/// +/// ´claim:selection:an-unobserved-graph-selects-nothing-competitively´ +/// ´test:crate:empty-graph-produces-root-only´ +#[test] +fn empty_graph_produces_root_only() { + let graph = test_graph(); + let set = AnalysisSet::recompute(&graph, 10, 6); + assert_eq!(set.competitive_count(), 0); + assert_eq!(set.total_count(), 1); // root only +} + +/// The root is in the full set unconditionally — here even when nothing has +/// been observed and no cell competed at all. Ancestor closure walks upward +/// from each selected cell, so the root's presence is what guarantees every +/// such walk terminates at a cell that has a model rather than running off +/// the top of the tree. +/// +/// ´claim:selection:the-root-is-always-in-the-full-set-so-every-ancestor-chain-terminates´ +/// ´test:crate:root-always-present´ +#[test] +fn root_always_present() { + let graph = test_graph(); + let set = AnalysisSet::recompute(&graph, 10, 6); + assert!(set.contains(graph.g_root())); +} + +/// However much traffic the graph has seen, and however generous the budget, +/// the root is never competitively selected. It accumulates every observation +/// by construction and would win any importance contest automatically, +/// crowding out the cells whose behaviour is actually informative. Its place +/// in the set is structural, and it is held apart from the cells that earned +/// theirs. +/// +/// ´claim:selection:the-root-is-never-competitive-however-important-it-is´ +/// ´test:crate:root-is-never-competitive´ +#[test] +fn root_is_never_competitive() { + let graph = populated_graph(); + let set = AnalysisSet::recompute(&graph, 100, 6); + assert!( + !set.is_competitive(graph.g_root()), + "root must never appear in the competitive set (§ALGO S-8.1)", + ); +} + +/// The competitive set never exceeds the budget it was asked for, however +/// many cells would qualify on their merits. The budget is what bounds the +/// sentinel's modelling cost, so it is a ceiling rather than a target that a +/// sufficiently busy graph could push past. +/// +/// ´claim:selection:the-competitive-set-never-exceeds-the-budget-it-was-asked-for´ +/// ´test:crate:competitive-set-respects-k´ +#[test] +fn competitive_set_respects_k() { + let graph = populated_graph(); + let set = AnalysisSet::recompute(&graph, 2, 6); + assert!(set.competitive_count() <= 2); +} + +/// A budget of one selects a cell rather than nothing, because the root leaves +/// the field before the cut rather than after it. Taken the other way round the +/// root wins the only slot and is then discarded for being the root, so the +/// smallest budget the configuration admits selects nothing at all however busy +/// the graph is. The root wins that contest on a total it accumulated before +/// its first split and stopped adding to at the split, while its children start +/// from zero — so the emptiness persists until a child's own total passes a +/// figure that is no longer growing, and every slot spent on the root is a slot +/// spent on an entry that cannot be selected. +/// +/// ´claim:selection:the-root-leaves-the-field-before-the-cut-so-every-slot-goes-to-a-selectable-cell´ +/// ´test:crate:budget-of-one-selects-one-cell´ +#[test] +fn budget_of_one_selects_one_cell() { + let graph = populated_graph(); + let available = AnalysisSet::recompute(&graph, 1000, 6).competitive_count(); + assert!(available >= 1, "the populated graph must offer at least one candidate"); + + let set = AnalysisSet::recompute(&graph, 1, 6); + assert_eq!(set.competitive_count(), 1); + assert!(!set.is_competitive(graph.g_root())); +} + +/// The same rule at a budget the root could not have exhausted on its own: the +/// competitive set fills to the whole budget rather than to one less than it. +/// Removing the root after the cut would cost exactly one slot at every budget, +/// which is invisible at a large one and total at a budget of one. +/// +/// (´claim:selection:the-root-leaves-the-field-before-the-cut-so-every-slot-goes-to-a-selectable-cell´) +/// ´test:crate:a-larger-budget-fills-every-slot-with-selectable-cells´ +#[test] +fn a_larger_budget_fills_every_slot_with_selectable_cells() { + let graph = populated_graph(); + let available = AnalysisSet::recompute(&graph, 1000, 6).competitive_count(); + assert!(available >= 3, "the populated graph must offer at least three candidates"); + + let set = AnalysisSet::recompute(&graph, 3, 6); + assert_eq!(set.competitive_count(), 3); + assert!(!set.is_competitive(graph.g_root())); +} + +/// A budget of zero is the boundary of the same ceiling: a well-populated +/// graph yields no competitive cells at all, while the root remains in the +/// full set. Selection can be turned off entirely without the structural +/// guarantee going with it. +/// +/// (´claim:selection:the-competitive-set-never-exceeds-the-budget-it-was-asked-for´) +/// ´test:crate:k-zero-yields-no-competitive-entries´ +#[test] +fn k_zero_yields_no_competitive_entries() { + let graph = populated_graph(); + let set = AnalysisSet::recompute(&graph, 0, 6); + assert_eq!(set.competitive_count(), 0); + // Full set still has at least the root. + assert!(set.total_count() >= 1); + assert!(set.contains(graph.g_root())); +} + +/// The depth cutoff bounds the V-depth of every competitive cell: with the +/// cutoff at zero, no selected cell sits deeper than zero however busy the +/// graph. The bound is enforced while the tree is being walked rather than by +/// discarding candidates afterwards, so the cutoff limits the work done as +/// well as the cells returned. +/// +/// ´claim:selection:the-depth-cutoff-bounds-the-v-depth-of-every-competitive-cell´ +/// ´test:crate:depth-cutoff-zero-excludes-all-non-root´ +#[test] +fn depth_cutoff_zero_excludes_all_non_root() { + let graph = populated_graph(); + // depth_cutoff=0 means only V-depth 0 entries qualify. + // In practice this is restrictive enough that the + // competitive set should be small or empty. + let set = AnalysisSet::recompute(&graph, 100, 0); + // Root is always present but never competitive. + assert!(set.contains(graph.g_root())); + // All competitive entries (if any) must have v_depth == 0. + for entry in set.competitive() { + assert_eq!(entry.v_depth, 0, "depth_cutoff=0 should only admit v_depth=0"); + } +} + +/// A floating-coordinate graph can contain cells deeper than its model width because splitting is gated by V-tree depth. Their modeled suffix has no remaining width, so selection excludes them rather than overflowing the width subtraction or admitting them as enormous candidates. +/// +/// ´claim:selection:overdeep-float-cells-have-zero-remaining-width-and-are-excluded´ +/// ´test:crate:overdeep-float-cells-are-excluded´ +#[test] +fn overdeep_float_cells_are_excluded() { + const MODEL_WIDTH: u32 = 4; + let cfg = GvConfig { + split_threshold: 5.0, + depth_create: 8, + depth_evict: 16, + budget: None, + alpha_relax: 0.75, + bounded_eviction: true, + }; + let mut graph = GvGraph::::new(cfg); + for _ in 0..100 { + graph.observe(2.0, 10.0); + } + + let deepest = graph.layers().map(|(_, node)| node.depth).max().unwrap_or_default(); + assert!(deepest > MODEL_WIDTH, "the fixture must contain a node deeper than N"); + + let set = AnalysisSet::recompute(&graph, usize::MAX, usize::MAX); + assert!( + set.competitive().iter().all(|entry| entry.depth <= MODEL_WIDTH), + "selection must exclude every node deeper than N", + ); +} + +/// Recomputing over an unchanged graph selects the same cells in the same +/// order. Nothing in selection depends on iteration order, hashing, or +/// timing, so two sentinels fed identical observations reach identical +/// analysis sets — the foundation the reproducibility of every downstream +/// score rests on. +/// +/// ´claim:selection:recomputing-an-unchanged-graph-selects-the-same-cells-in-the-same-order´ +/// ´test:crate:tie-breaking-is-deterministic´ +#[test] +fn tie_breaking_is_deterministic() { + let graph = test_graph(); + let set1 = AnalysisSet::recompute(&graph, 3, 6); + let set2 = AnalysisSet::recompute(&graph, 3, 6); + assert_eq!( + set1.competitive().iter().map(|e| e.start).collect::>(), + set2.competitive().iter().map(|e| e.start).collect::>(), + ); +} + +/// Competitive cells come back in a total order: importance descending, +/// and among cells of equal importance, interval start ascending. Ties are +/// therefore settled by a property of the coordinate domain rather than by +/// whatever order the tree walk happened to produce, which is what makes the +/// ordering reproducible and not merely stable within one run. +/// +/// ´claim:selection:competitive-cells-are-ordered-by-importance-with-ties-broken-by-interval-start´ +/// ´test:crate:competitive-ordering-by-importance-then-start´ +#[test] +fn competitive_ordering_by_importance_then_start() { + let graph = populated_graph(); + let set = AnalysisSet::recompute(&graph, 20, 6); + let comp = set.competitive(); + for pair in comp.windows(2) { + let (a, b) = (&pair[0], &pair[1]); + assert!( + a.importance > b.importance || (a.importance == b.importance && a.start <= b.start), + "competitive entries must be ordered by importance desc, start asc", + ); + } +} + +/// A graph driven hard enough to split still yields competitive cells. The +/// selector ranks by V-Tree importance alone and applies no filter on G-tree +/// state, so a cell that has since become internal keeps its V-Tree position +/// and remains eligible. Splitting refines the spatial structure; it does not +/// silently remove cells from consideration. +/// +/// ´claim:selection:ranking-is-by-v-tree-importance-alone-so-internal-cells-stay-eligible´ +/// ´test:crate:internal-nodes-eligible-for-competitive-set´ +#[test] +fn internal_nodes_eligible_for_competitive_set() { + // After splits, internal G-Tree nodes retain their V-Tree + // position and frozen importance. The competitive set uses + // V-Tree ranking exclusively (§ALGO S-8.1) — no G-Tree state + // filter — so internal nodes with sufficient importance remain + // eligible. + let mut graph = test_graph(); + for _ in 0..500 { + graph.observe(0u128, 1u64); + graph.observe(u128::MAX / 2, 1u64); + } + + let set = AnalysisSet::recompute(&graph, 100, 20); + assert!( + set.competitive_count() > 0, + "graph with splits should have competitive entries", + ); +} + +// ── Accessor methods ──────────────────────────────────── + +/// Every competitive cell is also in the full set, and the full set is never +/// the smaller of the two. Winning the competition confers membership rather +/// than replacing it, so a cell can be looked up by either question without +/// the two answers contradicting each other. +/// +/// ´claim:selection:every-competitive-cell-is-also-in-the-full-set´ +/// ´test:crate:full-set-is-superset-of-competitive´ +#[test] +fn full_set_is_superset_of_competitive() { + let graph = populated_graph(); + let set = AnalysisSet::recompute(&graph, 10, 6); + for entry in set.competitive() { + assert!(set.contains(entry.gnode), "competitive entry must appear in full set"); + } + assert!(set.total_count() >= set.competitive_count()); +} + +/// Membership is decided by what the set actually holds, not by whether a +/// handle looks plausible: a fabricated handle the graph never allocated is +/// simply absent. A caller holding a stale or invented cell identifier gets a +/// negative answer rather than an accidental match on a reused slot. +/// +/// ´claim:selection:a-handle-the-graph-never-allocated-is-absent-from-the-set´ +/// ´test:crate:contains-returns-false-for-absent-node´ +#[test] +fn contains_returns_false_for_absent_node() { + let graph = test_graph(); + let set = AnalysisSet::recompute(&graph, 10, 6); + // GNodeId is an opaque arena handle. A fabricated id that was + // never allocated by the graph cannot appear in the set. + let bogus = GNodeId::from_parts(999_999, 0); + assert!(!set.contains(bogus)); +} + +/// The competitiveness predicate agrees with the competitive list: every cell +/// the set lists as competitive answers to that question as well. Asking by +/// handle and reading the list are two views of one fact, so the two ways a +/// caller can learn a cell's standing cannot disagree. +/// +/// ´claim:selection:the-competitiveness-predicate-agrees-with-the-competitive-list´ +/// ´test:crate:is-competitive-true-for-selected-entries´ +#[test] +fn is_competitive_true_for_selected_entries() { + let graph = populated_graph(); + let set = AnalysisSet::recompute(&graph, 10, 6); + for entry in set.competitive() { + assert!(set.is_competitive(entry.gnode)); + } +} + +/// The same agreement holds in the negative direction: a cell drawn in only +/// by ancestor closure is never reported as competitive. Cells the closure +/// added are therefore distinguishable from cells that earned their place, +/// which matters because the two were selected for entirely different +/// reasons. +/// +/// (´claim:selection:the-competitiveness-predicate-agrees-with-the-competitive-list´) +/// ´test:crate:is-competitive-false-for-ancestor-only´ +#[test] +fn is_competitive_false_for_ancestor_only() { + let graph = populated_graph(); + let set = AnalysisSet::recompute(&graph, 10, 6); + for entry in set.full() { + if !entry.is_competitive { + assert!( + !set.is_competitive(entry.gnode), + "ancestor-only entry must not be reported as competitive", + ); + } + } +} + +// ── summary() ─────────────────────────────────────────── + +/// A summary of a set with nothing selected reports zeroes throughout — sizes, +/// depth span, importance span and V-depth span alike — rather than omitting +/// the ranges or filling them with sentinels. The full size is one, because +/// the root is there. A host parsing summaries gets the same shape whether or +/// not anything was selected. +/// +/// ´claim:selection:a-summary-with-nothing-selected-reports-zeroed-ranges-rather-than-omitting-them´ +/// ´test:crate:summary-empty-graph´ +#[test] +fn summary_empty_graph() { + let graph = test_graph(); + let set = AnalysisSet::recompute(&graph, 10, 6); + let s = set.summary(); + assert_eq!(s.competitive_size, 0); + assert_eq!(s.full_size, 1); // root only + assert_eq!(s.depth_range, (0, 0)); + assert_eq!(s.importance_range, (0.0, 0.0)); + assert_eq!(s.v_depth_range, (0, 0)); +} + +/// A summary counts the cells that competed and the cells the closure added +/// as separate figures, and on a populated graph the full count strictly +/// exceeds the competitive one. The cost of ancestry is therefore visible: a +/// host can see how much modelling it is paying for beyond the cells it +/// actually chose to invest in. +/// +/// ´claim:selection:a-summary-counts-competition-and-closure-separately-so-the-cost-of-ancestry-is-visible´ +/// ´test:crate:summary-with-competitive-cells´ +#[test] +fn summary_with_competitive_cells() { + let graph = populated_graph(); + let set = AnalysisSet::recompute(&graph, 4, 6); + let s = set.summary(); + assert!(s.competitive_size > 0); + assert!(s.competitive_size <= 4); + assert!(s.full_size > s.competitive_size); // at least root + competitive +} + +/// The producing sets shrink to whatever is online, but the investment does +/// not: a cell still being warmed has been paid for and has produced nothing +/// yet, and that gap is the whole difference between the two readings. A +/// summary taken over the online cells therefore filters the producing count +/// and leaves the investment count whole, so a host watching a warm-up sees +/// what it has committed to as well as what is answering. +/// +/// ´claim:selection:a-summary-over-the-online-cells-leaves-the-investment-count-whole´ +/// ´test:crate:summary-online-keeps-the-investment-count-whole´ +#[test] +fn summary_online_keeps_the_investment_count_whole() { + let graph = populated_graph(); + let set = AnalysisSet::recompute(&graph, 4, 6); + assert!(set.total_count() > 1, "the fixture must select more than the root"); + + // One cell online; every other selected cell stands for one still being + // warmed in staging. + let online: BTreeSet = set.full().iter().map(|e| e.gnode).take(1).collect(); + let s = set.summary_online(&online); + + assert_eq!(s.full_size, 1, "the producing set is the online part of the selection"); + assert_eq!( + s.investment_set_size, + set.total_count(), + "the investment is the whole selection, warming cells included" + ); +} + +/// The depth span of a populated set always begins at zero, because the root +/// sits at depth zero and is always a member. The span therefore reports the +/// reach of the whole modelled chain rather than only the band the selected +/// cells happen to occupy. +/// +/// (´claim:selection:the-root-is-always-in-the-full-set-so-every-ancestor-chain-terminates´) +/// ´test:crate:summary-depth-range-includes-root´ +#[test] +fn summary_depth_range_includes_root() { + let graph = populated_graph(); + let set = AnalysisSet::recompute(&graph, 4, 6); + let s = set.summary(); + assert_eq!(s.depth_range.0, 0, "root at depth 0 is always in the full set"); +} + +/// Where cells were selected at all, the least important of them still +/// carries importance above zero, and the reported span runs the right way +/// round. A cell can only win the competition on accumulated observation, so +/// nothing with no traffic behind it appears in the summary as though it had +/// been chosen. +/// +/// ´claim:selection:a-selected-cell-carries-importance-above-zero-and-the-reported-span-runs-the-right-way-round´ +/// ´test:crate:summary-importance-range-positive´ +#[test] +fn summary_importance_range_positive() { + let graph = populated_graph(); + let set = AnalysisSet::recompute(&graph, 10, 6); + let s = set.summary(); + if s.competitive_size > 0 { + assert!(s.importance_range.0 > 0.0, "min importance should be positive"); + assert!(s.importance_range.1 >= s.importance_range.0, "max >= min"); + } +} + +/// Every competitively selected cell sits below the top of the V-Tree: the +/// reported V-depth span starts above zero. The only entry at depth zero is +/// the root, and it is barred from the competition, so what the summary +/// describes is genuinely the refined structure rather than the whole domain +/// counted once more. +/// +/// (´claim:selection:the-root-is-never-competitive-however-important-it-is´) +/// ´test:crate:summary-v-depth-range-nonzero´ +#[test] +fn summary_v_depth_range_nonzero() { + let graph = populated_graph(); + let set = AnalysisSet::recompute(&graph, 10, 6); + let s = set.summary(); + if s.competitive_size > 0 { + assert!(s.v_depth_range.0 > 0, "competitive entries should have v_depth > 0"); + assert!(s.v_depth_range.1 >= s.v_depth_range.0, "max >= min"); + } +} diff --git a/packages/sentinel/src/tests/config.rs b/packages/sentinel/src/tests/config.rs new file mode 100644 index 000000000..468ed87c4 --- /dev/null +++ b/packages/sentinel/src/tests/config.rs @@ -0,0 +1,1987 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +//! Tests for [`crate::config`] — the parameters that fix how the sentinel +//! observes, and the validation that decides which combinations of them are +//! coherent enough to run. +//! +//! The configuration is deliberately free of policy: every field controls how +//! the sentinel measures and learns, never what it thinks about what it sees. +//! That division is why validation can be purely local. Each bound exists +//! because a value outside it would make some piece of arithmetic meaningless +//! — a forgetting factor of one never forgets, a stability constant of zero +//! guards no denominator, a clip width of zero rejects every observation — +//! and not because the host's judgement was second-guessed. +//! +//! Three shapes recur. Rates and fractions live strictly inside the unit +//! interval, both endpoints being degenerate. Sizes and capacities must be at +//! least one, since a ceiling of zero leaves the sentinel nothing to work +//! with. And a few fields are bound to each other rather than to constants: +//! the CUSUM reference must be slower than the baseline it judges, eviction +//! depth must sit strictly deeper than creation depth, and the node budget +//! must exceed the headroom the gap between those depths implies. +//! +//! Validation collects every violation rather than stopping at the first, so +//! a host repairs its configuration in one pass. Advice is kept separate from +//! refusal: a combination that is arithmetically sound but empirically poor — +//! warm-up rounds too few for the configured memory — comes back as a warning +//! the host may ignore, never as an error that stops it. +//! +//! # Test index +//! +//! | Test | Area | Claim | +//! |------|------|-------| +//! | [`default_config_is_valid`] | config | The parameters the sentinel ships with satisfy every rule it checks them against. A host that configures nothing at all therefore starts from a coherent measurement setup rather than from a template it must first repair. | +//! | [`default_config_field_values`] | config | Each default holds the value its documentation promises — the long-memory forgetting factor, the modest rank ceiling, the analysis and graph budgets, the noise batch and its fixed seed. The defaults are a calibrated set rather than arbitrary placeholders, so documenting them and shipping them are held to be the same act. | +//! | [`rejects_max_rank_zero`] | config | A capacity ceiling of zero leaves the sentinel nothing to work with: a subspace tracker allowed no basis vectors can model nothing at all, so the value is refused rather than quietly read as a request for a disabled tracker. The refusal names that field and faults nothing else in an otherwise default configuration. | +//! | [`rejects_forgetting_factor_out_of_range`] | config | The forgetting factor must lie strictly inside the unit interval: at one the baseline never forgets and can never adapt, at zero it retains nothing, and outside the interval the exponential weighting stops being a weighting. Both endpoints are refused along with values beyond them, and the refusal carries back the value it saw so the host can tell which of its inputs was faulted. | +//! | [`accepts_forgetting_factor_near_boundaries`] | config | cites (´claim:config:a-rate-must-lie-strictly-inside-the-unit-interval-and-the-refusal-names-the-value-it-saw´) | +//! | [`rejects_rank_update_interval_zero`] | config | cites (´claim:config:a-capacity-of-zero-is-refused-because-it-leaves-the-sentinel-nothing-to-work-with´) | +//! | [`rejects_energy_threshold_out_of_range`] | config | cites (´claim:config:a-rate-must-lie-strictly-inside-the-unit-interval-and-the-refusal-names-the-value-it-saw´) | +//! | [`rejects_eps_not_positive`] | config | The stability constant exists to keep denominators away from zero, so a value at or below zero defeats the only thing it is for. Both are refused, and the refusal reports the offending value rather than silently substituting a workable one. | +//! | [`requires_finite_eps`] | config | A denominator guard must itself be finite: either infinity would collapse score and energy ratios rather than stabilise them, while the largest finite value remains a valid positive guard. | +//! | [`rejects_cusum_slow_decay_out_of_range`] | config | cites (´claim:config:a-rate-must-lie-strictly-inside-the-unit-interval-and-the-refusal-names-the-value-it-saw´) | +//! | [`rejects_cusum_slow_decay_below_forgetting`] | config | The CUSUM reference must have longer memory than the baseline it is measured against — strictly slower, not merely as slow. A reference adapting as fast as the baseline would follow a gradual drift instead of exposing it, and exposing exactly that drift is what the accumulator exists for. | +//! | [`accepts_cusum_slow_decay_just_above_forgetting`] | config | cites (´claim:config:the-cusum-reference-must-decay-strictly-slower-than-the-baseline-it-judges´) | +//! | [`rejects_cusum_coord_slow_decay_out_of_range`] | config | cites (´claim:config:a-rate-must-lie-strictly-inside-the-unit-interval-and-the-refusal-names-the-value-it-saw´) | +//! | [`rejects_cusum_coord_slow_decay_below_forgetting`] | config | cites (´claim:config:the-cusum-reference-must-decay-strictly-slower-than-the-baseline-it-judges´) | +//! | [`rejects_cusum_allowance_sigmas_negative`] | config | The CUSUM allowance is subtracted from each step's gap before anything accumulates, so a negative allowance would be added to every gap and would manufacture evidence of drift out of ordinary noise. Negative values are therefore refused. | +//! | [`accepts_cusum_allowance_sigmas_zero`] | config | cites (´claim:config:a-noise-allowance-may-never-be-negative-because-it-would-manufacture-the-drift-it-absorbs´) | +//! | [`rejects_clip_sigmas_not_positive`] | config | The clip width bounds how far above the mean an observation may sit and still update the baseline. At zero or below nothing would ever clear the bar, so the baseline would starve rather than be protected — the value is refused instead of being allowed to silence the very updates it guards. | +//! | [`rejects_clip_pressure_decay_out_of_range`] | config | cites (´claim:config:a-rate-must-lie-strictly-inside-the-unit-interval-and-the-refusal-names-the-value-it-saw´) | +//! | [`accepts_clip_pressure_decay_valid`] | config | cites (´claim:config:a-rate-must-lie-strictly-inside-the-unit-interval-and-the-refusal-names-the-value-it-saw´) | +//! | [`rejects_analysis_k_zero`] | config | cites (´claim:config:a-capacity-of-zero-is-refused-because-it-leaves-the-sentinel-nothing-to-work-with´) | +//! | [`accepts_analysis_k_one`] | config | cites (´claim:config:a-capacity-of-zero-is-refused-because-it-leaves-the-sentinel-nothing-to-work-with´) | +//! | [`accepts_analysis_depth_cutoff_zero`] | config | A depth cutoff of zero is admitted rather than refused, and it means something usable: only the V-Tree root remains eligible, which effectively turns adaptive selection off. Zero is a fault in a capacity but a legitimate setting for a depth gate, because a gate that admits nothing deep expresses a policy rather than an incoherence. | +//! | [`rejects_split_threshold_zero`] | config | A split threshold of zero would let a cell subdivide before accumulating any intensity at all, so the graph would fragment on its first observation instead of on evidence of sustained traffic. The threshold must be strictly positive. | +//! | [`rejects_d_create_zero`] | config | cites (´claim:config:a-capacity-of-zero-is-refused-because-it-leaves-the-sentinel-nothing-to-work-with´) | +//! | [`rejects_d_evict_not_greater_than_d_create`] | config | Eviction depth must sit strictly deeper than creation depth, and an equal pair is refused just as an inverted one is. The gap between them is the buffer zone — entries too deep to create children but not yet deep enough to be evicted — so collapsing it would leave a cell eligible for eviction the moment it stopped being eligible to grow. | +//! | [`accepts_d_evict_one_above_d_create`] | config | cites (´claim:config:the-eviction-depth-must-sit-strictly-deeper-than-the-creation-depth-so-a-buffer-zone-exists´) | +//! | [`rejects_budget_zero`] | config | cites (´claim:config:a-capacity-of-zero-is-refused-because-it-leaves-the-sentinel-nothing-to-work-with´) | +//! | [`rejects_budget_below_headroom`] | config | The node budget must exceed what the depth gates themselves imply: a buffer zone of a given width can hold a number of nodes growing as a power of three, and a budget merely equal to that leaves the graph no room to manoeuvre inside its own gates. Equality is refused, not merely shortfall. | +//! | [`accepts_budget_just_above_headroom`] | config | cites (´claim:config:the-node-budget-must-exceed-the-headroom-the-depth-gates-imply-and-equalling-it-is-not-enough´) | +//! | [`rejects_depth_buffer_whose_headroom_cannot_be_represented`] | config | Past a certain width the headroom the depth gates imply stops being a number the machine can hold, and the depth pair is refused on its own terms rather than measured against a figure that wrapped. The requirement grows as a power of three, so a buffer in the forties already exceeds the addressable range; computing it and comparing anyway would either abort the validation that promised to return its faults, or silently compare the budget against a small wrapped remainder and admit a configuration that cannot hold. The refusal names the two depths, since they are what the host must change. | +//! | [`reports_a_shortfall_at_the_widest_representable_depth_buffer`] | config | cites (´claim:config:a-depth-buffer-whose-headroom-cannot-be-represented-is-refused-on-its-own-terms´) | +//! | [`refuses_a_depth_pair_whose_headroom_exponent_cannot_be_represented`] | config | cites (´claim:config:a-depth-buffer-whose-headroom-cannot-be-represented-is-refused-on-its-own-terms´) | +//! | [`accepts_the_widest_representable_depth_pair_a_budget_can_clear`] | config | cites (´claim:config:a-depth-buffer-whose-headroom-cannot-be-represented-is-refused-on-its-own-terms´) | +//! | [`rejects_nan_in_every_floating_point_field`] | config | Every floating-point field refuses a non-number, because the ordered comparisons that police the other values cannot see one. A comparison against a non-number is false whichever way it is written, so a bound expressed as a pair of comparisons admits it silently — and the value then spreads, since every product and sum it enters returns a non-number too. A forgetting factor admitted this way reaches the baseline arithmetic and leaves every score afterwards unusable, with nothing in the report to say which field was responsible. The guard therefore sits ahead of the bound rather than inside it. | +//! | [`accepts_infinite_clip_width_and_refuses_infinite_rates`] | config | An infinite value is admitted where the interval is one-sided, because there it names a real limit rather than the absence of one. An infinite clip width is the unclipped configuration — the control arm the package's own clipping study runs against — and it compares correctly against every bound it is checked with, which is precisely what a non-number does not do. The fields whose intervals are two-sided still refuse it, and they refuse it through the bound they already carry rather than through a separate guard. | +//! | [`rejects_noise_batch_size_zero_when_enabled`] | config | A noise batch of no samples is a fault only when the schedule actually asks for rounds: with an active schedule the warm-up would run rounds that feed the tracker nothing. The check is conditional on the schedule rather than absolute, because zero samples per round is coherent when there are no rounds to run. | +//! | [`accepts_noise_batch_size_zero_when_disabled`] | config | cites (´claim:config:a-noise-batch-of-zero-is-faulted-only-when-the-schedule-actually-asks-for-rounds´) | +//! | [`rejects_noise_batch_size_zero_with_geometric_root_and_zero_floor`] | config | cites (´claim:config:a-noise-batch-of-zero-is-faulted-only-when-the-schedule-actually-asks-for-rounds´) | +//! | [`accepts_noise_batch_size_zero_with_geometric_root_and_floor_zero`] | config | cites (´claim:config:a-noise-batch-of-zero-is-faulted-only-when-the-schedule-actually-asks-for-rounds´) | +//! | [`accepts_noise_seed_none`] | config | An absent noise seed is valid and simply changes where the randomness comes from: with a seed the warm-up is reproducible across restarts, without one it is drawn from system entropy. Determinism is offered rather than required, so neither choice counts as a misconfiguration. | +//! | [`rejects_geometric_decay_out_of_range`] | config | The geometric decay is bound to a half-open interval rather than the open one other rates get: values at or below zero and above one are refused, but one itself is not. Zero would collapse the schedule onto its floor immediately, whereas a schedule that never tapers with depth is a legitimate thing to ask for. | +//! | [`rejects_geometric_decay_nan`] | config | A decay that is not a number is refused explicitly, because every comparison against it is false and a range check alone would let it through. The configuration is rejected before such a value could reach the exponentiation and turn every round count into nonsense. | +//! | [`rejects_geometric_root_zero_with_positive_min`] | config | A geometric schedule that starts at no rounds but floors at a positive count contradicts itself: the taper only ever descends from the root, so every depth would be lifted to the floor and the root would describe nothing. That combination is refused — which is why the root is faulted only when the floor is positive. | +//! | [`collects_multiple_errors`] | config | Validation reports every violation it finds rather than stopping at the first, so a host with several bad fields learns of all of them in one pass instead of discovering them one restart at a time. Faulting several fields at once yields at least as many errors, each naming its own field. | +//! | [`geometric_rounds_default`] | config | A geometric schedule scales its round count by the decay at each level of depth and then rests on its floor. Deeper cells are narrower and converge in fewer rounds, so tapering matches warm-up effort to the width a tracker actually has to learn, while the floor keeps even the deepest cells from being warmed with nothing. | +//! | [`geometric_rounds_custom`] | config | cites (´claim:config:a-geometric-schedule-tapers-with-depth-and-then-rests-on-its-floor´) | +//! | [`geometric_rounds_root_equals_min`] | config | cites (´claim:config:a-geometric-schedule-tapers-with-depth-and-then-rests-on-its-floor´) | +//! | [`geometric_rounds_very_small_decay`] | config | cites (´claim:config:a-geometric-schedule-tapers-with-depth-and-then-rests-on-its-floor´) | +//! | [`geometric_rounds_rounding_half_values`] | config | Round counts are whole, and a fractional product is rounded to nearest with halves going away from zero rather than truncated toward it. Truncation would bias every depth downward and compound with the taper, so a cell an exact half-round short is warmed the extra round instead of losing it. | +//! | [`geometric_rounds_large_depth`] | config | At the deepest levels the G-tree can reach, the decayed product has long since collapsed toward zero, and the schedule still answers with its floor rather than with a degenerate number. The depth argument is bounded by the tree's own width, and the arithmetic stays well defined right up to that bound. | +//! | [`geometric_rounds_extreme_depths_reach_the_floor`] | config | A public schedule remains tapered at depths beyond the exponent type's range. Extreme depths reach the floor, never exceed the schedule ceiling, and preserve the non-increasing shape across the conversion boundary. | +//! | [`geometric_rounds_decay_one_is_constant`] | config | A decay of exactly one leaves the root untouched at every depth, giving a flat schedule that warms deep cells as heavily as shallow ones. This is what makes the upper endpoint worth admitting: uniform warm-up is expressible inside the geometric variant instead of needing a form of its own. | +//! | [`geometric_rounds_root_zero_min_zero`] | config | A geometric schedule with neither root nor floor asks for no rounds at any depth. The variant can therefore express a complete absence of warm-up without switching to the explicit form, and it does so with no special casing: the taper of nothing is nothing, and a floor of nothing lifts it nowhere. | +//! | [`explicit_rounds_for_depth`] | config | An explicit schedule is a direct lookup by depth, and depths past the end of the vector reuse its last entry rather than falling to zero or failing. The tail is the host's statement about all remaining depths, so a short vector still describes an unbounded tree. | +//! | [`explicit_rounds_single_entry`] | config | cites (´claim:config:an-explicit-schedule-is-read-by-depth-and-clamps-to-its-last-entry-beyond-its-length´) | +//! | [`explicit_rounds_non_monotonic`] | config | cites (´claim:config:an-explicit-schedule-is-read-by-depth-and-clamps-to-its-last-entry-beyond-its-length´) | +//! | [`explicit_empty_is_disabled`] | config | An explicit schedule that can never yield a round counts as noise switched off, and the two facts agree: the round count is zero at every depth and the schedule reports itself disabled. That agreement is what lets other rules key off the disabled flag instead of re-deriving it. | +//! | [`explicit_all_zeros_is_disabled`] | config | cites (´claim:config:an-explicit-schedule-that-can-never-yield-a-round-reports-itself-as-noise-switched-off´) | +//! | [`explicit_single_zero_is_disabled`] | config | cites (´claim:config:an-explicit-schedule-that-can-never-yield-a-round-reports-itself-as-noise-switched-off´) | +//! | [`explicit_mixed_zeros_not_disabled`] | config | A single non-zero entry anywhere keeps noise enabled, even where the shallow depths ask for none. A zero at a given depth is a statement about that depth alone, so a schedule may deliberately warm only the deeper cells and still counts as active. | +//! | [`geometric_is_disabled_when_root_and_min_are_zero`] | config | A geometric schedule is disabled only when both its root and its floor are zero, because either one alone still produces rounds. The floor lifts every depth to at least its own count, and the root sets the count at the shallow depths before the taper has descended. Reading only one of the two calls a schedule silent that is still asking for warm-up. | +//! | [`geometric_is_not_disabled_when_min_positive`] | config | cites (´claim:config:a-geometric-schedule-is-disabled-only-when-both-its-root-and-its-floor-are-zero´) | +//! | [`geometric_is_not_disabled_when_root_positive_and_min_zero`] | config | cites (´claim:config:a-geometric-schedule-is-disabled-only-when-both-its-root-and-its-floor-are-zero´) | +//! | [`max_rounds_geometric`] | config | The most a geometric schedule can ever ask for is its root, because the taper only descends from there. A host sizing buffers for warm-up can read the ceiling off the root alone, without evaluating the schedule at any depth. | +//! | [`max_rounds_geometric_floor_above_root`] | config | cites (´claim:config:the-ceiling-of-a-geometric-schedule-is-its-root-because-the-taper-only-descends´) | +//! | [`max_rounds_explicit`] | config | For an explicit schedule the ceiling is the largest entry it holds, not the first. Since the explicit form imposes no ordering, the depth-zero value carries no promise about the rest and the maximum has to be found rather than assumed. | +//! | [`max_rounds_explicit_empty`] | config | cites (´claim:config:the-ceiling-of-an-explicit-schedule-is-its-largest-entry-not-its-first´) | +//! | [`max_rounds_explicit_large`] | config | cites (´claim:config:the-ceiling-of-an-explicit-schedule-is-its-largest-entry-not-its-first´) | +//! | [`noise_schedule_default_matches_doc`] | config | The shipped schedule is the geometric one its documentation describes, and it reports itself active. Its numbers are calibrated rather than arbitrary: the root sits a little above the worst-case baseline convergence measured at the default forgetting factor, and the floor covers deep cells whose convergence scales down with analysis width without vanishing. | +//! | [`default_config_has_no_warnings`] | config | The shipped configuration draws no advisories, because the default schedule was calibrated against the default forgetting factor. Defaults that validated but warned would be an odd thing to ship, so the two sets of defaults are kept consistent with each other. | +//! | [`warns_when_noise_root_too_low_for_lambda_099`] | config | A configuration whose warm-up rounds fall short of what its memory needs is advised, not refused: it is arithmetically sound, but baselines may not converge before real observations arrive, so early scores would be unreliable. Advice and refusal are separate channels — validation would pass this configuration unchanged, and only the warning list carries the concern. | +//! | [`no_warning_when_noise_root_sufficient_for_lambda_095`] | config | How much warm-up is recommended falls with the forgetting factor, because a shorter memory converges sooner: a schedule too thin for a long-memory baseline is adequate for a shorter one. The same schedule draws advice or silence depending on the memory it is paired with, since the recommendation is a relation between the two rather than a property of either. | +//! | [`warns_when_small_batch_and_lambda_095`] | config | Batch size enters the recommendation as well: with few synthetic samples per round, each round buys less convergence, so the same shorter memory demands markedly more rounds and a schedule that was adequate becomes advised against. Warm-up is really measured in observations rather than in rounds, and the recommendation reflects that. | +//! | [`no_warning_for_explicit_schedule_with_enough_rounds`] | config | The advisory judges whatever the schedule actually yields at depth zero, whichever variant it is written in. An explicit schedule generous enough at the root passes the same check a geometric one would, so the recommendation is about warm-up delivered and not about how the host chose to express it. | +//! | [`warning_display_is_informative`] | config | A rendered advisory carries the numbers a host needs in order to act on it: the root it found, the root it recommends, and the forgetting factor that set that recommendation. Advice naming only the problem would leave the reader to re-derive the target. | +//! | [`rejects_coordinate_width_below_the_tracker_minimum`] | config | A coordinate width narrower than the smallest dimension a subspace tracker can model is refused at construction, with the same structured failure the configuration faults carry. The width is a parameter of the type rather than a field of the configuration, so validating the configuration alone can never see it, and the root tracker spans the whole width — at one dimension its lone basis vector spans the entire space, novelty is identically zero, and the tracker reports a settled model of everything while modelling nothing. Refusing is what lets the constructor's success mean the sentinel it returns can measure. | +//! | [`accepts_the_narrowest_modellable_coordinate_width`] | config | cites (´claim:config:a-coordinate-width-below-the-tracker-minimum-is-refused-at-construction´) | +//! | [`collects_a_width_fault_alongside_a_configuration_fault`] | config | cites (´claim:config:a-coordinate-width-below-the-tracker-minimum-is-refused-at-construction´) | +//! | [`refuses_a_coordinate_width_above_the_centred_bit_ceiling`] | config | A coordinate width above what the centred bit vector can carry is refused at construction, the same way a width below the tracker minimum is. The bridge that turns a coordinate into centred bits is open to any implementor, and the spatial layer asks only that the width fit the coordinate type, so a host whose coordinates are wider than the vector can otherwise ask for a sentinel wider than the vector that feeds it. Nothing would fault: the slots past the vector's length come back as zeros, a centred bit is ±0.5 and never zero, and every dimension past the end would be modelled over a constant the coordinate stream never produced — a settled reading of data that does not exist, mixed into novelty, residual and rank alike. The refusal names the width and the ceiling, since those are what the host must reconcile. | +//! | [`accepts_the_widest_modellable_coordinate_width`] | config | cites (´claim:config:a-coordinate-width-above-the-centred-bit-ceiling-is-refused-at-construction´) | +//! | [`warming_thread_refusal_names_the_setting_and_the_environment`] | config | The refusal a host receives when the environment will not give the engine a warming thread names the setting that asked for one and quotes the operating system's own account of the refusal. Nothing in the configuration is wrong in that case, so a message that said only that a configuration was invalid would send an operator searching values that are all correct: naming the setting says which request to withdraw, and quoting the environment says whether withdrawing it is the right answer at all or whether the machine is simply out of threads. | + +//! | [`unrepresentable_noise_batch_is_rejected_before_construction`] | config | An enabled batch that cannot fit in the address space is rejected before construction can allocate it. | +//! | [`rejects_noise_matrix_size_even_when_the_outer_vector_fits`] | config | Batch validation includes the supported matrix width, not just the outer row vector. | +//! | [`ignores_unallocated_noise_batch_size_when_disabled`] | config | A disabled schedule never allocates its batch and therefore needs no allocation bound. | + +use crate::config::*; + +// ── Default config ────────────────────────────────────────── + +/// The parameters the sentinel ships with satisfy every rule it checks them +/// against. A host that configures nothing at all therefore starts from a +/// coherent measurement setup rather than from a template it must first +/// repair. +/// +/// ´claim:config:the-shipped-defaults-satisfy-every-rule-validation-checks´ +/// ´test:crate:default-config-is-valid´ +#[test] +fn default_config_is_valid() { + SentinelConfig::::default().validate().unwrap(); +} + +/// Each default holds the value its documentation promises — the long-memory +/// forgetting factor, the modest rank ceiling, the analysis and graph budgets, +/// the noise batch and its fixed seed. The defaults are a calibrated set rather +/// than arbitrary placeholders, so documenting them and shipping them are held +/// to be the same act. +/// +/// ´claim:config:every-default-field-holds-the-value-its-documentation-promises´ +/// ´test:crate:default-config-field-values´ +#[test] +fn default_config_field_values() { + let cfg = SentinelConfig::::default(); + cfg.validate().unwrap(); + + // Core subspace. + assert_eq!(cfg.max_rank, 16); + assert!((cfg.forgetting_factor - 0.99).abs() < f64::EPSILON); + assert_eq!(cfg.rank_update_interval, 100); + assert!((cfg.energy_threshold - 0.90).abs() < f64::EPSILON); + assert!((cfg.eps - 1e-6).abs() < f64::EPSILON); + + // CUSUM / EWMA. + assert!((cfg.cusum_slow_decay - 0.999).abs() < f64::EPSILON); + assert!((cfg.cusum_coord_slow_decay - 0.999).abs() < f64::EPSILON); + assert!((cfg.cusum_allowance_sigmas - 0.5).abs() < f64::EPSILON); + assert!((cfg.clip_sigmas - 3.0).abs() < f64::EPSILON); + assert!((cfg.clip_pressure_decay - 0.95).abs() < f64::EPSILON); + assert!(!cfg.per_sample_scores); + + // Analysis. + assert_eq!(cfg.analysis_k, 1024); + assert_eq!(cfg.analysis_depth_cutoff, 6); + + // G-V Graph. + assert_eq!(cfg.split_threshold, 100); + assert_eq!(cfg.d_create, 3); + assert_eq!(cfg.d_evict, 6); + assert_eq!(cfg.budget, 100_000); + + // Noise injection. + assert_eq!(cfg.noise_batch_size, 16); + assert_eq!(cfg.noise_seed, Some(42)); + assert!(!cfg.background_warming); +} + +// ── Per-field validation: core subspace fields ────────────── + +/// A capacity ceiling of zero leaves the sentinel nothing to work with: a +/// subspace tracker allowed no basis vectors can model nothing at all, so the +/// value is refused rather than quietly read as a request for a disabled +/// tracker. The refusal names that field and faults nothing else in an +/// otherwise default configuration. +/// +/// ´claim:config:a-capacity-of-zero-is-refused-because-it-leaves-the-sentinel-nothing-to-work-with´ +/// ´test:crate:rejects-max-rank-zero´ +#[test] +fn rejects_max_rank_zero() { + let cfg = SentinelConfig:: { + max_rank: 0, + ..SentinelConfig::::default() + }; + assert_eq!(cfg.validate(), Err(ConfigErrors(vec![ConfigError::MaxRankZero]))); +} + +/// The forgetting factor must lie strictly inside the unit interval: at one the +/// baseline never forgets and can never adapt, at zero it retains nothing, and +/// outside the interval the exponential weighting stops being a weighting. Both +/// endpoints are refused along with values beyond them, and the refusal carries +/// back the value it saw so the host can tell which of its inputs was +/// faulted. +/// +/// ´claim:config:a-rate-must-lie-strictly-inside-the-unit-interval-and-the-refusal-names-the-value-it-saw´ +/// ´test:crate:rejects-forgetting-factor-out-of-range´ +#[test] +fn rejects_forgetting_factor_out_of_range() { + for &bad in &[0.0, 1.0, -0.1, 1.5] { + let cfg = SentinelConfig:: { + forgetting_factor: bad, + // Keep slow decays above forgetting to avoid cascading errors. + cusum_slow_decay: 0.999, + cusum_coord_slow_decay: 0.999, + ..SentinelConfig::::default() + }; + let err = cfg.validate().unwrap_err(); + assert!( + err.0 + .iter() + .any(|e| matches!(e, ConfigError::ForgettingFactorOutOfRange(v) if (*v - bad).abs() < f64::EPSILON)), + "expected ForgettingFactorOutOfRange for {bad}" + ); + } +} + +/// The interval is open rather than narrowed by a margin of safety: a factor a +/// hair below one and one a hair above zero are both admitted, provided the +/// CUSUM references above them stay slower still. The line is drawn at the +/// endpoints themselves and nowhere short of them. +/// +/// (´claim:config:a-rate-must-lie-strictly-inside-the-unit-interval-and-the-refusal-names-the-value-it-saw´) +/// ´test:crate:accepts-forgetting-factor-near-boundaries´ +#[test] +fn accepts_forgetting_factor_near_boundaries() { + let cfg = SentinelConfig:: { + forgetting_factor: 0.9999, + cusum_slow_decay: 0.99999, + cusum_coord_slow_decay: 0.99999, + ..SentinelConfig::::default() + }; + cfg.validate().unwrap(); + + let cfg = SentinelConfig:: { + forgetting_factor: 0.0001, + cusum_slow_decay: 0.001, + cusum_coord_slow_decay: 0.001, + ..SentinelConfig::::default() + }; + cfg.validate().unwrap(); +} + +/// The same floor governs the cadence at which a tracker reassesses its rank: +/// an interval of zero asks for reassessment at no interval at all, so it is +/// refused rather than treated as reassessing constantly. +/// +/// (´claim:config:a-capacity-of-zero-is-refused-because-it-leaves-the-sentinel-nothing-to-work-with´) +/// ´test:crate:rejects-rank-update-interval-zero´ +#[test] +fn rejects_rank_update_interval_zero() { + let cfg = SentinelConfig:: { + rank_update_interval: 0, + ..SentinelConfig::::default() + }; + let err = cfg.validate().unwrap_err(); + assert!(err.0.contains(&ConfigError::RankUpdateIntervalZero)); +} + +/// The energy threshold obeys the same open interval, and for a kindred reason +/// at each end: retaining no fraction of the variance describes nothing, and +/// demanding the whole of it asks for a rank the data cannot justify. +/// +/// (´claim:config:a-rate-must-lie-strictly-inside-the-unit-interval-and-the-refusal-names-the-value-it-saw´) +/// ´test:crate:rejects-energy-threshold-out-of-range´ +#[test] +fn rejects_energy_threshold_out_of_range() { + for &bad in &[0.0, 1.0, -0.5, 1.1] { + let cfg = SentinelConfig:: { + energy_threshold: bad, + ..SentinelConfig::::default() + }; + let err = cfg.validate().unwrap_err(); + assert!( + err.0 + .iter() + .any(|e| matches!(e, ConfigError::EnergyThresholdOutOfRange(v) if (*v - bad).abs() < f64::EPSILON)), + "expected EnergyThresholdOutOfRange for {bad}" + ); + } +} + +/// The stability constant exists to keep denominators away from zero, so a +/// value at or below zero defeats the only thing it is for. Both are refused, +/// and the refusal reports the offending value rather than silently +/// substituting a workable one. +/// +/// ´claim:config:a-stability-constant-must-be-strictly-positive-because-a-zero-guard-guards-nothing´ +/// ´test:crate:rejects-eps-not-positive´ +#[test] +fn rejects_eps_not_positive() { + for &bad in &[0.0, -1e-6] { + let cfg = SentinelConfig:: { + eps: bad, + ..SentinelConfig::::default() + }; + let err = cfg.validate().unwrap_err(); + assert!( + err.0 + .iter() + .any(|e| matches!(e, ConfigError::EpsNotPositive(v) if (*v - bad).abs() < f64::EPSILON)), + "expected EpsNotPositive for {bad}" + ); + } +} + +/// A denominator guard must itself be finite: either infinity would make every +/// protected denominator infinite and collapse the resulting ratios to zero. +/// The largest finite value remains positive and is therefore admitted; the +/// validator enforces the stated domain without imposing a fitted upper bound. +/// +/// ´claim:config:the-denominator-guard-is-finite-and-positive´ +/// ´test:crate:requires-finite-eps´ +#[test] +fn requires_finite_eps() { + for &bad in &[f64::INFINITY, f64::NEG_INFINITY] { + let cfg = SentinelConfig:: { + eps: bad, + ..SentinelConfig::::default() + }; + let err = cfg.validate().unwrap_err(); + assert!( + err.0 + .iter() + .any(|e| matches!(e, ConfigError::EpsNotFinite(v) if v.to_bits() == bad.to_bits())), + "expected EpsNotFinite for {bad}" + ); + } + + let largest_finite = SentinelConfig:: { + eps: f64::MAX, + ..SentinelConfig::::default() + }; + assert_eq!(largest_finite.validate(), Ok(())); +} + +// ── Per-field validation: CUSUM / EWMA fields ─────────────── + +/// The slow CUSUM decay is a rate like any other and is held to the same open +/// interval. Its endpoints are refused before the separate question of whether +/// it is slower than the fast baseline is even reached. +/// +/// (´claim:config:a-rate-must-lie-strictly-inside-the-unit-interval-and-the-refusal-names-the-value-it-saw´) +/// ´test:crate:rejects-cusum-slow-decay-out-of-range´ +#[test] +fn rejects_cusum_slow_decay_out_of_range() { + for &bad in &[0.0, 1.0, -0.1, 1.5] { + let cfg = SentinelConfig:: { + cusum_slow_decay: bad, + ..SentinelConfig::::default() + }; + let err = cfg.validate().unwrap_err(); + assert!( + err.0 + .iter() + .any(|e| matches!(e, ConfigError::CusumSlowDecayOutOfRange(v) if (*v - bad).abs() < f64::EPSILON)), + "expected CusumSlowDecayOutOfRange for {bad}" + ); + } +} + +/// The CUSUM reference must have longer memory than the baseline it is measured +/// against — strictly slower, not merely as slow. A reference adapting as fast +/// as the baseline would follow a gradual drift instead of exposing it, and +/// exposing exactly that drift is what the accumulator exists for. +/// +/// ´claim:config:the-cusum-reference-must-decay-strictly-slower-than-the-baseline-it-judges´ +/// ´test:crate:rejects-cusum-slow-decay-below-forgetting´ +#[test] +fn rejects_cusum_slow_decay_below_forgetting() { + let cfg = SentinelConfig:: { + forgetting_factor: 0.99, + cusum_slow_decay: 0.98, + ..SentinelConfig::::default() + }; + let err = cfg.validate().unwrap_err(); + assert!(err.0.iter().any(|e| matches!(e, ConfigError::CusumSlowDecayTooLow { .. }))); +} + +/// Any strictly greater value satisfies the ordering; the margin need not be +/// generous. Only equality and below are refused, so the rule is about which +/// memory is the longer one and not about how much longer it is. +/// +/// (´claim:config:the-cusum-reference-must-decay-strictly-slower-than-the-baseline-it-judges´) +/// ´test:crate:accepts-cusum-slow-decay-just-above-forgetting´ +#[test] +fn accepts_cusum_slow_decay_just_above_forgetting() { + let cfg = SentinelConfig:: { + forgetting_factor: 0.95, + cusum_slow_decay: 0.951, + ..SentinelConfig::::default() + }; + cfg.validate().unwrap(); +} + +/// The coordination tier's slow decay is checked against the same open interval +/// as the per-tracker one. Splitting the two fields lets the host set different +/// drift sensitivities per tier without loosening what counts as a rate. +/// +/// (´claim:config:a-rate-must-lie-strictly-inside-the-unit-interval-and-the-refusal-names-the-value-it-saw´) +/// ´test:crate:rejects-cusum-coord-slow-decay-out-of-range´ +#[test] +fn rejects_cusum_coord_slow_decay_out_of_range() { + for &bad in &[0.0, 1.0, -0.1, 1.5] { + let cfg = SentinelConfig:: { + cusum_coord_slow_decay: bad, + ..SentinelConfig::::default() + }; + let err = cfg.validate().unwrap_err(); + assert!( + err.0 + .iter() + .any(|e| matches!(e, ConfigError::CusumCoordSlowDecayOutOfRange(v) if (*v - bad).abs() < f64::EPSILON)), + "expected CusumCoordSlowDecayOutOfRange for {bad}" + ); + } +} + +/// The ordering binds the coordination tier too: its reference is measured +/// against the same fast baseline and must likewise be slower than it. A second +/// tier buys independent sensitivity, not an exemption. +/// +/// (´claim:config:the-cusum-reference-must-decay-strictly-slower-than-the-baseline-it-judges´) +/// ´test:crate:rejects-cusum-coord-slow-decay-below-forgetting´ +#[test] +fn rejects_cusum_coord_slow_decay_below_forgetting() { + let cfg = SentinelConfig:: { + forgetting_factor: 0.99, + cusum_coord_slow_decay: 0.98, + ..SentinelConfig::::default() + }; + let err = cfg.validate().unwrap_err(); + assert!( + err.0 + .iter() + .any(|e| matches!(e, ConfigError::CusumCoordSlowDecayTooLow { .. })) + ); +} + +/// The CUSUM allowance is subtracted from each step's gap before anything +/// accumulates, so a negative allowance would be added to every gap and would +/// manufacture evidence of drift out of ordinary noise. Negative values are +/// therefore refused. +/// +/// ´claim:config:a-noise-allowance-may-never-be-negative-because-it-would-manufacture-the-drift-it-absorbs´ +/// ´test:crate:rejects-cusum-allowance-sigmas-negative´ +#[test] +fn rejects_cusum_allowance_sigmas_negative() { + let cfg = SentinelConfig:: { + cusum_allowance_sigmas: -0.1, + ..SentinelConfig::::default() + }; + let err = cfg.validate().unwrap_err(); + assert!(err.0.iter().any(|e| matches!(e, ConfigError::CusumAllowanceNegative(_)))); +} + +/// Zero sits on the permitted side of that line and means something definite: +/// no tolerance at all, so any positive gap accumulates. Declining to absorb +/// noise is a legitimate choice; inventing signal is not. +/// +/// (´claim:config:a-noise-allowance-may-never-be-negative-because-it-would-manufacture-the-drift-it-absorbs´) +/// ´test:crate:accepts-cusum-allowance-sigmas-zero´ +#[test] +fn accepts_cusum_allowance_sigmas_zero() { + let cfg = SentinelConfig:: { + cusum_allowance_sigmas: 0.0, + ..SentinelConfig::::default() + }; + cfg.validate().unwrap(); +} + +/// The clip width bounds how far above the mean an observation may sit and +/// still update the baseline. At zero or below nothing would ever clear the +/// bar, so the baseline would starve rather than be protected — the value is +/// refused instead of being allowed to silence the very updates it guards. +/// +/// ´claim:config:a-clip-width-must-be-strictly-positive-because-a-width-of-zero-would-reject-every-observation´ +/// ´test:crate:rejects-clip-sigmas-not-positive´ +#[test] +fn rejects_clip_sigmas_not_positive() { + for &bad in &[0.0, -1.0] { + let cfg = SentinelConfig:: { + clip_sigmas: bad, + ..SentinelConfig::::default() + }; + let err = cfg.validate().unwrap_err(); + assert!( + err.0.iter().any(|e| matches!(e, ConfigError::ClipSigmasNotPositive(_))), + "expected ClipSigmasNotPositive for {bad}" + ); + } +} + +/// The clip-pressure estimate adapts by the same kind of exponential rate and +/// is bound by the same open interval, so contamination tracking cannot be +/// configured either to never adapt or to never remember. +/// +/// (´claim:config:a-rate-must-lie-strictly-inside-the-unit-interval-and-the-refusal-names-the-value-it-saw´) +/// ´test:crate:rejects-clip-pressure-decay-out-of-range´ +#[test] +fn rejects_clip_pressure_decay_out_of_range() { + for &bad in &[0.0, 1.0, -0.1, 1.5] { + let cfg = SentinelConfig:: { + clip_pressure_decay: bad, + ..SentinelConfig::::default() + }; + let err = cfg.validate().unwrap_err(); + assert!( + err.0 + .iter() + .any(|e| matches!(e, ConfigError::ClipPressureDecayOutOfRange(v) if (*v - bad).abs() < f64::EPSILON)), + "expected ClipPressureDecayOutOfRange for {bad}" + ); + } +} + +/// An ordinary interior value passes without comment. The rule refuses only the +/// endpoints and what lies beyond them, so a rate chosen for a memory of a few +/// dozen batches is simply admitted. +/// +/// (´claim:config:a-rate-must-lie-strictly-inside-the-unit-interval-and-the-refusal-names-the-value-it-saw´) +/// ´test:crate:accepts-clip-pressure-decay-valid´ +#[test] +fn accepts_clip_pressure_decay_valid() { + let cfg = SentinelConfig:: { + clip_pressure_decay: 0.95, + ..SentinelConfig::::default() + }; + cfg.validate().unwrap(); +} + +// ── Per-field validation: analysis fields ─────────────────── + +/// The analysis ceiling is a capacity of the same kind: with no competitive +/// cells there is nothing to compete for and no tier to run, so zero is refused +/// rather than read as a request to switch analysis off. Here too the single +/// bad field yields its own error and no cascade. +/// +/// (´claim:config:a-capacity-of-zero-is-refused-because-it-leaves-the-sentinel-nothing-to-work-with´) +/// ´test:crate:rejects-analysis-k-zero´ +#[test] +fn rejects_analysis_k_zero() { + let cfg = SentinelConfig:: { + analysis_k: 0, + ..SentinelConfig::::default() + }; + assert_eq!(cfg.validate(), Err(ConfigErrors(vec![ConfigError::AnalysisKZero]))); +} + +/// The floor really is one rather than some larger practical minimum. A single +/// competitive cell is a coherent configuration, so the bound says that some +/// analysis must happen, not how much. +/// +/// (´claim:config:a-capacity-of-zero-is-refused-because-it-leaves-the-sentinel-nothing-to-work-with´) +/// ´test:crate:accepts-analysis-k-one´ +#[test] +fn accepts_analysis_k_one() { + let cfg = SentinelConfig:: { + analysis_k: 1, + ..SentinelConfig::::default() + }; + cfg.validate().unwrap(); +} + +/// A depth cutoff of zero is admitted rather than refused, and it means +/// something usable: only the V-Tree root remains eligible, which effectively +/// turns adaptive selection off. Zero is a fault in a capacity but a legitimate +/// setting for a depth gate, because a gate that admits nothing deep expresses +/// a policy rather than an incoherence. +/// +/// ´claim:config:a-depth-cutoff-of-zero-is-a-legal-way-to-disable-adaptive-selection-not-an-error´ +/// ´test:crate:accepts-analysis-depth-cutoff-zero´ +#[test] +fn accepts_analysis_depth_cutoff_zero() { + let cfg = SentinelConfig:: { + analysis_depth_cutoff: 0, + ..SentinelConfig::::default() + }; + cfg.validate().unwrap(); +} + +// ── Per-field validation: G-V Graph fields ────────────────── + +/// A split threshold of zero would let a cell subdivide before accumulating any +/// intensity at all, so the graph would fragment on its first observation +/// instead of on evidence of sustained traffic. The threshold must be strictly +/// positive. +/// +/// ´claim:config:a-split-threshold-must-be-positive-so-a-cell-subdivides-on-traffic-rather-than-immediately´ +/// ´test:crate:rejects-split-threshold-zero´ +#[test] +fn rejects_split_threshold_zero() { + let cfg = SentinelConfig:: { + split_threshold: 0, + ..SentinelConfig::::default() + }; + assert!(cfg.validate().is_err()); +} + +/// The creation depth is held to the same floor: a graph permitted to create at +/// no depth could never grow past its root, so zero is refused. +/// +/// (´claim:config:a-capacity-of-zero-is-refused-because-it-leaves-the-sentinel-nothing-to-work-with´) +/// ´test:crate:rejects-d-create-zero´ +#[test] +fn rejects_d_create_zero() { + let cfg = SentinelConfig:: { + d_create: 0, + ..SentinelConfig::::default() + }; + assert!(cfg.validate().is_err()); +} + +/// Eviction depth must sit strictly deeper than creation depth, and an equal +/// pair is refused just as an inverted one is. The gap between them is the +/// buffer zone — entries too deep to create children but not yet deep enough to +/// be evicted — so collapsing it would leave a cell eligible for eviction the +/// moment it stopped being eligible to grow. +/// +/// ´claim:config:the-eviction-depth-must-sit-strictly-deeper-than-the-creation-depth-so-a-buffer-zone-exists´ +/// ´test:crate:rejects-d-evict-not-greater-than-d-create´ +#[test] +fn rejects_d_evict_not_greater_than_d_create() { + // Equal. + let cfg = SentinelConfig:: { + d_create: 3, + d_evict: 3, + ..SentinelConfig::::default() + }; + let err = cfg.validate().unwrap_err(); + assert!( + err.0 + .contains(&ConfigError::DEvictNotGreaterThanDCreate { d_create: 3, d_evict: 3 }) + ); + + // Less. + let cfg = SentinelConfig:: { + d_create: 5, + d_evict: 3, + ..SentinelConfig::::default() + }; + let err = cfg.validate().unwrap_err(); + assert!( + err.0 + .contains(&ConfigError::DEvictNotGreaterThanDCreate { d_create: 5, d_evict: 3 }) + ); +} + +/// The narrowest possible buffer, a single level, satisfies the ordering. What +/// such a configuration still owes is headroom: a narrow buffer lowers the node +/// count the budget must exceed, and here the budget is set just above that +/// lowered requirement. +/// +/// (´claim:config:the-eviction-depth-must-sit-strictly-deeper-than-the-creation-depth-so-a-buffer-zone-exists´) +/// ´test:crate:accepts-d-evict-one-above-d-create´ +#[test] +fn accepts_d_evict_one_above_d_create() { + // d_evict = d_create + 1 is valid, but budget must satisfy headroom. + // buffer = 1, headroom = 3^2 = 9. + let cfg = SentinelConfig:: { + d_create: 3, + d_evict: 4, + budget: 10, // > 9 + ..SentinelConfig::::default() + }; + cfg.validate().unwrap(); +} + +/// A node budget of zero admits no live nodes at all. The sentinel always +/// operates in budgeted mode, so its ceiling cannot be read as the absence of +/// one. +/// +/// (´claim:config:a-capacity-of-zero-is-refused-because-it-leaves-the-sentinel-nothing-to-work-with´) +/// ´test:crate:rejects-budget-zero´ +#[test] +fn rejects_budget_zero() { + let cfg = SentinelConfig:: { + budget: 0, + ..SentinelConfig::::default() + }; + assert!(cfg.validate().is_err()); +} + +/// The node budget must exceed what the depth gates themselves imply: a buffer +/// zone of a given width can hold a number of nodes growing as a power of +/// three, and a budget merely equal to that leaves the graph no room to +/// manoeuvre inside its own gates. Equality is refused, not merely +/// shortfall. +/// +/// ´claim:config:the-node-budget-must-exceed-the-headroom-the-depth-gates-imply-and-equalling-it-is-not-enough´ +/// ´test:crate:rejects-budget-below-headroom´ +#[test] +fn rejects_budget_below_headroom() { + // With d_create=3, d_evict=6, buffer=3, headroom=3^4=81. + // Budget must be > 81. + let cfg = SentinelConfig:: { + d_create: 3, + d_evict: 6, + budget: 81, + ..SentinelConfig::::default() + }; + let err = cfg.validate().unwrap_err(); + assert!(err.0.iter().any(|e| matches!(e, ConfigError::BudgetTooSmall { .. }))); +} + +/// A single node above the requirement is enough, which pins the comparison as +/// strict rather than as a demand for margin. The headroom rule states the +/// minimum the gates force, and the host is free to sit immediately above +/// it. +/// +/// (´claim:config:the-node-budget-must-exceed-the-headroom-the-depth-gates-imply-and-equalling-it-is-not-enough´) +/// ´test:crate:accepts-budget-just-above-headroom´ +#[test] +fn accepts_budget_just_above_headroom() { + let cfg = SentinelConfig:: { + d_create: 3, + d_evict: 6, + budget: 82, + ..SentinelConfig::::default() + }; + cfg.validate().unwrap(); +} + +/// Past a certain width the headroom the depth gates imply stops being a +/// number the machine can hold, and the depth pair is refused on its own terms +/// rather than measured against a figure that wrapped. The requirement grows as +/// a power of three, so a buffer in the forties already exceeds the addressable +/// range; computing it and comparing anyway would either abort the validation +/// that promised to return its faults, or silently compare the budget against a +/// small wrapped remainder and admit a configuration that cannot hold. The +/// refusal names the two depths, since they are what the host must change. +/// +/// ´claim:config:a-depth-buffer-whose-headroom-cannot-be-represented-is-refused-on-its-own-terms´ +/// ´test:crate:rejects-depth-buffer-whose-headroom-cannot-be-represented´ +#[test] +fn rejects_depth_buffer_whose_headroom_cannot_be_represented() { + let cfg = SentinelConfig:: { + d_create: 1, + d_evict: 41, + budget: 1_000_000, + ..SentinelConfig::::default() + }; + let err = cfg.validate().unwrap_err(); + assert!(err.0.contains(&ConfigError::DepthBufferTooLarge { + d_create: 1, + d_evict: 41 + })); + assert!( + !err.0.iter().any(|e| matches!(e, ConfigError::BudgetTooSmall { .. })), + "an unrepresentable requirement is not reported as a budget shortfall" + ); +} + +/// The widest buffer whose headroom still fits is validated the ordinary way, +/// which fixes the boundary between the two refusals rather than leaving it to +/// be inferred. At this width the requirement is a real number, so a budget +/// below it comes back as a shortfall naming the figure it fell short of. +/// +/// (´claim:config:a-depth-buffer-whose-headroom-cannot-be-represented-is-refused-on-its-own-terms´) +/// ´test:crate:reports-a-shortfall-at-the-widest-representable-depth-buffer´ +#[test] +fn reports_a_shortfall_at_the_widest_representable_depth_buffer() { + let cfg = SentinelConfig:: { + d_create: 1, + d_evict: 40, + budget: 1_000_000, + ..SentinelConfig::::default() + }; + let err = cfg.validate().unwrap_err(); + assert!(err.0.iter().any(|e| matches!(e, ConfigError::BudgetTooSmall { .. }))); + assert!(!err.0.iter().any(|e| matches!(e, ConfigError::DepthBufferTooLarge { .. }))); +} + +/// The requirement is refused as unrepresentable when any step of it is, +/// including the step before the power. The exponent is one past the buffer, +/// and the buffer is a difference of two depths, so a creation depth of zero +/// against the widest eviction depth makes the exponent itself the step that +/// cannot be held: computing it in unchecked form aborts the validation where +/// arithmetic is checked, and wraps the exponent to zero where it is not — +/// which would measure the budget against a requirement of one and admit a pair +/// no budget can serve. A configuration already faulty for another reason is +/// where this arises, and it is exactly where validation must still return +/// faults rather than abort, so the zero creation depth comes back beside the +/// refusal in the same pass. +/// +/// (´claim:config:a-depth-buffer-whose-headroom-cannot-be-represented-is-refused-on-its-own-terms´) +/// ´test:crate:refuses-a-depth-pair-whose-headroom-exponent-cannot-be-represented´ +#[test] +fn refuses_a_depth_pair_whose_headroom_exponent_cannot_be_represented() { + let cfg = SentinelConfig:: { + d_create: 0, + d_evict: u32::MAX, + budget: 1_000_000, + ..SentinelConfig::::default() + }; + let err = cfg.validate().unwrap_err(); + assert!(err.0.contains(&ConfigError::DepthBufferTooLarge { + d_create: 0, + d_evict: u32::MAX + })); + assert!( + err.0.contains(&ConfigError::DCreateZero), + "the fault that made the pair invalid is reported alongside, not lost to an abort" + ); + assert!( + !err.0.iter().any(|e| matches!(e, ConfigError::BudgetTooSmall { .. })), + "a requirement no step of which could be computed is not reported as a budget shortfall" + ); +} + +/// The widest pair whose requirement can be computed is accepted when the +/// budget clears it, which fixes the boundary from the accepting side: the +/// refusals above say which pairs have no representable requirement, and this +/// says the pair one step inside that edge is served by a budget the machine +/// can hold. The particular depths are fixed by address arithmetic sixty-four +/// bits wide, as are those of the shortfall at the same buffer. +/// +/// (´claim:config:a-depth-buffer-whose-headroom-cannot-be-represented-is-refused-on-its-own-terms´) +/// ´test:crate:accepts-the-widest-representable-depth-pair-a-budget-can-clear´ +#[test] +fn accepts_the_widest_representable_depth_pair_a_budget_can_clear() { + let cfg = SentinelConfig:: { + d_create: 1, + d_evict: 40, + budget: usize::MAX, + ..SentinelConfig::::default() + }; + cfg.validate().unwrap(); +} + +// ── Per-field validation: non-numbers and infinities ──────── + +/// Every floating-point field refuses a non-number, because the ordered +/// comparisons that police the other values cannot see one. A comparison +/// against a non-number is false whichever way it is written, so a bound +/// expressed as a pair of comparisons admits it silently — and the value then +/// spreads, since every product and sum it enters returns a non-number too. A +/// forgetting factor admitted this way reaches the baseline arithmetic and +/// leaves every score afterwards unusable, with nothing in the report to say +/// which field was responsible. The guard therefore sits ahead of the bound +/// rather than inside it. +/// +/// ´claim:config:a-non-number-is-refused-in-every-floating-point-field-because-an-ordered-bound-cannot-see-one´ +/// ´test:crate:rejects-nan-in-every-floating-point-field´ +#[test] +fn rejects_nan_in_every_floating_point_field() { + let nan = f64::NAN; + let cases: [(&str, SentinelConfig); 8] = [ + ( + "forgetting_factor", + SentinelConfig { + forgetting_factor: nan, + ..SentinelConfig::::default() + }, + ), + ( + "energy_threshold", + SentinelConfig { + energy_threshold: nan, + ..SentinelConfig::::default() + }, + ), + ( + "eps", + SentinelConfig { + eps: nan, + ..SentinelConfig::::default() + }, + ), + ( + "cusum_slow_decay", + SentinelConfig { + cusum_slow_decay: nan, + ..SentinelConfig::::default() + }, + ), + ( + "cusum_coord_slow_decay", + SentinelConfig { + cusum_coord_slow_decay: nan, + ..SentinelConfig::::default() + }, + ), + ( + "cusum_allowance_sigmas", + SentinelConfig { + cusum_allowance_sigmas: nan, + ..SentinelConfig::::default() + }, + ), + ( + "clip_sigmas", + SentinelConfig { + clip_sigmas: nan, + ..SentinelConfig::::default() + }, + ), + ( + "clip_pressure_decay", + SentinelConfig { + clip_pressure_decay: nan, + ..SentinelConfig::::default() + }, + ), + ]; + + for (field, cfg) in cases { + assert!(cfg.validate().is_err(), "a non-number in {field} must be refused"); + } +} + +/// An infinite value is admitted where the interval is one-sided, because +/// there it names a real limit rather than the absence of one. An infinite +/// clip width is the unclipped configuration — the control arm the package's +/// own clipping study runs against — and it compares correctly against every +/// bound it is checked with, which is precisely what a non-number does not do. +/// The fields whose intervals are two-sided still refuse it, and they refuse it +/// through the bound they already carry rather than through a separate guard. +/// +/// ´claim:config:an-infinite-value-is-admitted-where-the-interval-is-one-sided-because-there-it-names-a-real-limit´ +/// ´test:crate:accepts-infinite-clip-width-and-refuses-infinite-rates´ +#[test] +fn accepts_infinite_clip_width_and_refuses_infinite_rates() { + let unclipped = SentinelConfig:: { + clip_sigmas: f64::INFINITY, + ..SentinelConfig::::default() + }; + unclipped.validate().unwrap(); + + let infinite_rate = SentinelConfig:: { + forgetting_factor: f64::INFINITY, + ..SentinelConfig::::default() + }; + let err = infinite_rate.validate().unwrap_err(); + assert!(err.0.iter().any(|e| matches!(e, ConfigError::ForgettingFactorOutOfRange(_)))); +} + +// ── Per-field validation: noise injection fields ──────────── + +/// A noise batch of no samples is a fault only when the schedule actually asks +/// for rounds: with an active schedule the warm-up would run rounds that feed +/// the tracker nothing. The check is conditional on the schedule rather than +/// absolute, because zero samples per round is coherent when there are no +/// rounds to run. +/// +/// ´claim:config:a-noise-batch-of-zero-is-faulted-only-when-the-schedule-actually-asks-for-rounds´ +/// ´test:crate:rejects-noise-batch-size-zero-when-enabled´ +#[test] +fn rejects_noise_batch_size_zero_when_enabled() { + let cfg = SentinelConfig:: { + noise_schedule: NoiseSchedule::Explicit(vec![5]), + noise_batch_size: 0, + ..SentinelConfig::::default() + }; + let err = cfg.validate().unwrap_err(); + assert!(err.0.contains(&ConfigError::NoiseBatchSizeZero)); +} + +/// With noise switched off entirely, a batch size of zero passes. The field +/// describes work that will never be requested, so validation has nothing to +/// object to. +/// +/// (´claim:config:a-noise-batch-of-zero-is-faulted-only-when-the-schedule-actually-asks-for-rounds´) +/// ´test:crate:accepts-noise-batch-size-zero-when-disabled´ +#[test] +fn accepts_noise_batch_size_zero_when_disabled() { + let cfg = SentinelConfig:: { + noise_schedule: NoiseSchedule::Explicit(vec![]), + noise_batch_size: 0, + ..SentinelConfig::::default() + }; + cfg.validate().unwrap(); +} + +/// The geometric form reaches the same check through its own reading of what +/// silence is. A positive root with a zero floor still asks for rounds at the +/// shallow depths, so a batch of no samples is faulted there exactly as it is +/// for an explicit schedule. This is the pairing that a floor-only reading of +/// disablement lets through, and letting it through is not a missing warning +/// but a broken tracker: the warm-up runs its rounds, divides by a batch of +/// nothing, and leaves the latent baseline unable to score anything for the +/// rest of the sentinel's life. +/// +/// (´claim:config:a-noise-batch-of-zero-is-faulted-only-when-the-schedule-actually-asks-for-rounds´) +/// ´test:crate:rejects-noise-batch-size-zero-with-geometric-root-and-zero-floor´ +#[test] +fn rejects_noise_batch_size_zero_with_geometric_root_and_zero_floor() { + let cfg = SentinelConfig:: { + noise_schedule: NoiseSchedule::Geometric { + root: 450, + decay: 0.5, + min: 0, + }, + noise_batch_size: 0, + ..SentinelConfig::::default() + }; + let err = cfg.validate().unwrap_err(); + assert!(err.0.contains(&ConfigError::NoiseBatchSizeZero)); +} + +/// The geometric form's own accepting side: with root and floor both zero the +/// schedule asks for nothing at any depth, so a batch size of zero describes +/// work that will never be requested and passes for the same reason the empty +/// explicit schedule does. +/// +/// (´claim:config:a-noise-batch-of-zero-is-faulted-only-when-the-schedule-actually-asks-for-rounds´) +/// ´test:crate:accepts-noise-batch-size-zero-with-geometric-root-and-floor-zero´ +#[test] +fn accepts_noise_batch_size_zero_with_geometric_root_and_floor_zero() { + let cfg = SentinelConfig:: { + noise_schedule: NoiseSchedule::Geometric { + root: 0, + decay: 0.5, + min: 0, + }, + noise_batch_size: 0, + ..SentinelConfig::::default() + }; + cfg.validate().unwrap(); +} + +/// An absent noise seed is valid and simply changes where the randomness comes +/// from: with a seed the warm-up is reproducible across restarts, without one +/// it is drawn from system entropy. Determinism is offered rather than +/// required, so neither choice counts as a misconfiguration. +/// +/// ´claim:config:an-absent-noise-seed-is-valid-and-trades-reproducibility-for-system-entropy´ +/// ´test:crate:accepts-noise-seed-none´ +#[test] +fn accepts_noise_seed_none() { + let cfg = SentinelConfig:: { + noise_seed: None, + ..SentinelConfig::::default() + }; + cfg.validate().unwrap(); +} + +/// The geometric decay is bound to a half-open interval rather than the open +/// one other rates get: values at or below zero and above one are refused, but +/// one itself is not. Zero would collapse the schedule onto its floor +/// immediately, whereas a schedule that never tapers with depth is a +/// legitimate thing to ask for. +/// +/// ´claim:config:the-noise-decay-is-bound-above-zero-and-at-most-one-because-a-schedule-that-never-tapers-is-legitimate´ +/// ´test:crate:rejects-geometric-decay-out-of-range´ +#[test] +fn rejects_geometric_decay_out_of_range() { + for &bad in &[0.0, -0.1, 1.5] { + let cfg = SentinelConfig:: { + noise_schedule: NoiseSchedule::Geometric { + root: 50, + decay: bad, + min: 10, + }, + ..SentinelConfig::::default() + }; + let err = cfg.validate().unwrap_err(); + assert!( + err.0 + .iter() + .any(|e| matches!(e, ConfigError::NoiseScheduleDecayOutOfRange(_))), + "expected NoiseScheduleDecayOutOfRange for decay={bad}" + ); + } +} + +/// A decay that is not a number is refused explicitly, because every comparison +/// against it is false and a range check alone would let it through. The +/// configuration is rejected before such a value could reach the exponentiation +/// and turn every round count into nonsense. +/// +/// ´claim:config:a-decay-that-is-not-a-number-is-refused-explicitly-because-range-comparisons-alone-would-admit-it´ +/// ´test:crate:rejects-geometric-decay-nan´ +#[test] +fn rejects_geometric_decay_nan() { + let cfg = SentinelConfig:: { + noise_schedule: NoiseSchedule::Geometric { + root: 50, + decay: f64::NAN, + min: 10, + }, + ..SentinelConfig::::default() + }; + assert!(cfg.validate().is_err()); +} + +/// A geometric schedule that starts at no rounds but floors at a positive count +/// contradicts itself: the taper only ever descends from the root, so every +/// depth would be lifted to the floor and the root would describe nothing. +/// That combination is refused — which is why the root is faulted only when +/// the floor is positive. +/// +/// ´claim:config:a-geometric-root-of-zero-under-a-positive-floor-is-refused-as-self-contradictory´ +/// ´test:crate:rejects-geometric-root-zero-with-positive-min´ +#[test] +fn rejects_geometric_root_zero_with_positive_min() { + let cfg = SentinelConfig:: { + noise_schedule: NoiseSchedule::Geometric { + root: 0, + decay: 0.5, + min: 10, + }, + ..SentinelConfig::::default() + }; + let err = cfg.validate().unwrap_err(); + assert!(err.0.contains(&ConfigError::NoiseScheduleRootZero)); +} + +// ── Error accumulation ────────────────────────────────────── + +/// Validation reports every violation it finds rather than stopping at the +/// first, so a host with several bad fields learns of all of them in one pass +/// instead of discovering them one restart at a time. Faulting several fields +/// at once yields at least as many errors, each naming its own field. +/// +/// ´claim:config:validation-reports-every-violation-it-finds-rather-than-stopping-at-the-first´ +/// ´test:crate:collects-multiple-errors´ +#[test] +fn collects_multiple_errors() { + let cfg = SentinelConfig:: { + max_rank: 0, + analysis_k: 0, + budget: 0, + ..SentinelConfig::::default() + }; + let err = cfg.validate().unwrap_err(); + assert!(err.0.len() >= 3, "expected at least 3 errors, got {}", err.0.len()); + assert!(err.0.contains(&ConfigError::MaxRankZero)); + assert!(err.0.contains(&ConfigError::AnalysisKZero)); + assert!(err.0.contains(&ConfigError::BudgetZero)); +} + +// ── NoiseSchedule::rounds_for_depth ───────────────────────── + +/// A geometric schedule scales its round count by the decay at each level of +/// depth and then rests on its floor. Deeper cells are narrower and converge in +/// fewer rounds, so tapering matches warm-up effort to the width a tracker +/// actually has to learn, while the floor keeps even the deepest cells from +/// being warmed with nothing. +/// +/// ´claim:config:a-geometric-schedule-tapers-with-depth-and-then-rests-on-its-floor´ +/// ´test:crate:geometric-rounds-default´ +#[test] +fn geometric_rounds_default() { + let schedule = NoiseSchedule::default(); + // Default: Geometric { root: 450, decay: 0.5, min: 50 } + assert_eq!(schedule.rounds_for_depth(0), 450); + assert_eq!(schedule.rounds_for_depth(1), 225); + assert_eq!(schedule.rounds_for_depth(2), 113); // 450 × 0.25 = 112.5 → 113 + assert_eq!(schedule.rounds_for_depth(3), 56); // 450 × 0.125 = 56.25 → 56 + assert_eq!(schedule.rounds_for_depth(4), 50); // 450 × 0.0625 = 28.125 → 28, but min=50 + assert_eq!(schedule.rounds_for_depth(10), 50); // floored to min + assert_eq!(schedule.rounds_for_depth(128), 50); // max depth → still min +} + +/// The shape is not tied to the shipped numbers: another root, decay and floor +/// taper the same way and settle onto the same kind of plateau. Those three are +/// the schedule's only degrees of freedom, and nothing else is baked in. +/// +/// (´claim:config:a-geometric-schedule-tapers-with-depth-and-then-rests-on-its-floor´) +/// ´test:crate:geometric-rounds-custom´ +#[test] +fn geometric_rounds_custom() { + let schedule = NoiseSchedule::geometric(400, 0.5, 100); + assert_eq!(schedule.rounds_for_depth(0), 400); + assert_eq!(schedule.rounds_for_depth(1), 200); + assert_eq!(schedule.rounds_for_depth(2), 100); + assert_eq!(schedule.rounds_for_depth(3), 100); // floored to min + assert_eq!(schedule.rounds_for_depth(20), 100); +} + +/// When root and floor coincide the taper is invisible: every depth returns the +/// same count, because the descent has nowhere to descend to. A constant +/// schedule is expressible without any special case. +/// +/// (´claim:config:a-geometric-schedule-tapers-with-depth-and-then-rests-on-its-floor´) +/// ´test:crate:geometric-rounds-root-equals-min´ +#[test] +fn geometric_rounds_root_equals_min() { + // When root == min, every depth yields the same value. + let schedule = NoiseSchedule::geometric(100, 0.5, 100); + for depth in 0..20 { + assert_eq!( + schedule.rounds_for_depth(depth), + 100, + "depth {depth} should yield root=min=100" + ); + } +} + +/// A steep decay reaches the floor within a level or two, and past that the +/// floor alone determines the answer. It is the floor and not the decay that +/// bounds a deep cell's warm-up, however aggressive the taper. +/// +/// (´claim:config:a-geometric-schedule-tapers-with-depth-and-then-rests-on-its-floor´) +/// ´test:crate:geometric-rounds-very-small-decay´ +#[test] +fn geometric_rounds_very_small_decay() { + // decay=0.01 → root × 0.01^depth, hits min almost immediately. + let schedule = NoiseSchedule::geometric(1000, 0.01, 5); + assert_eq!(schedule.rounds_for_depth(0), 1000); + assert_eq!(schedule.rounds_for_depth(1), 10); // 1000 × 0.01 = 10 + assert_eq!(schedule.rounds_for_depth(2), 5); // 1000 × 0.0001 → 0, but min=5 + assert_eq!(schedule.rounds_for_depth(10), 5); +} + +/// Round counts are whole, and a fractional product is rounded to nearest with +/// halves going away from zero rather than truncated toward it. Truncation +/// would bias every depth downward and compound with the taper, so a cell an +/// exact half-round short is warmed the extra round instead of losing it. +/// +/// ´claim:config:a-fractional-round-count-is-rounded-to-nearest-with-halves-away-from-zero-not-truncated´ +/// ´test:crate:geometric-rounds-rounding-half-values´ +#[test] +fn geometric_rounds_rounding_half_values() { + // f64::round() rounds half away from zero. + // root=100, decay=0.5 → 100, 50, 25, 12.5→13, 6.25→6, 3.125→3, 1.5625→2, 0.78→1 + let schedule = NoiseSchedule::geometric(100, 0.5, 1); + assert_eq!(schedule.rounds_for_depth(0), 100); + assert_eq!(schedule.rounds_for_depth(1), 50); + assert_eq!(schedule.rounds_for_depth(2), 25); + assert_eq!(schedule.rounds_for_depth(3), 13); // 12.5 rounds to 13 + assert_eq!(schedule.rounds_for_depth(4), 6); // 6.25 rounds to 6 + assert_eq!(schedule.rounds_for_depth(5), 3); // 3.125 rounds to 3 + assert_eq!(schedule.rounds_for_depth(6), 2); // 1.5625 rounds to 2 + assert_eq!(schedule.rounds_for_depth(7), 1); // 0.78125 rounds to 1 = min +} + +/// At the deepest levels the G-tree can reach, the decayed product has long +/// since collapsed toward zero, and the schedule still answers with its floor +/// rather than with a degenerate number. The depth argument is bounded by the +/// tree's own width, and the arithmetic stays well defined right up to that +/// bound. +/// +/// ´claim:config:the-schedule-stays-well-defined-at-the-deepest-depth-the-g-tree-can-reach´ +/// ´test:crate:geometric-rounds-large-depth´ +#[test] +fn geometric_rounds_large_depth() { + // G-tree depth is bounded ≤ 128. Verify numeric stability near that limit. + let schedule = NoiseSchedule::geometric(400, 0.5, 50); + assert_eq!(schedule.rounds_for_depth(126), 50); + assert_eq!(schedule.rounds_for_depth(127), 50); + assert_eq!(schedule.rounds_for_depth(128), 50); +} + +/// A public schedule remains tapered at depths beyond the exponent type's range. Extreme depths reach the floor, never exceed the schedule ceiling, and preserve the non-increasing shape across the conversion boundary. +/// +/// ´claim:config:a-geometric-schedule-reaches-its-floor-without-wrapping-at-any-public-depth´ +/// ´test:crate:geometric-rounds-extreme-depths-reach-the-floor´ +#[test] +fn geometric_rounds_extreme_depths_reach_the_floor() { + let schedule = NoiseSchedule::default(); + let signed_exponent_max = usize::try_from(i32::MAX).expect("the supported usize holds every non-negative i32"); + let depths = [0, 1, 128, signed_exponent_max, signed_exponent_max + 1, usize::MAX]; + let rounds: Vec<_> = depths.iter().map(|&depth| schedule.rounds_for_depth(depth)).collect(); + + assert_eq!(schedule.rounds_for_depth(usize::MAX), 50); + assert_eq!(schedule.rounds_for_depth(signed_exponent_max + 1), 50); + assert!( + rounds.windows(2).all(|pair| pair[0] >= pair[1]), + "the geometric schedule must remain non-increasing across the exponent boundary: {rounds:?}", + ); + assert!( + rounds.iter().all(|&count| count <= schedule.max_rounds()), + "every scheduled count must stay within the declared maximum: {rounds:?}", + ); +} + +/// A decay of exactly one leaves the root untouched at every depth, giving a +/// flat schedule that warms deep cells as heavily as shallow ones. This is what +/// makes the upper endpoint worth admitting: uniform warm-up is expressible +/// inside the geometric variant instead of needing a form of its own. +/// +/// ´claim:config:a-decay-of-one-yields-a-flat-schedule-that-warms-every-depth-alike´ +/// ´test:crate:geometric-rounds-decay-one-is-constant´ +#[test] +fn geometric_rounds_decay_one_is_constant() { + // decay=1.0 → root × 1.0^depth = root at every depth. + let schedule = NoiseSchedule::geometric(42, 1.0, 1); + for depth in [0, 1, 10, 50, 128] { + assert_eq!(schedule.rounds_for_depth(depth), 42); + } +} + +/// A geometric schedule with neither root nor floor asks for no rounds at any +/// depth. The variant can therefore express a complete absence of warm-up +/// without switching to the explicit form, and it does so with no special +/// casing: the taper of nothing is nothing, and a floor of nothing lifts it +/// nowhere. +/// +/// ´claim:config:a-geometric-schedule-with-no-root-and-no-floor-asks-for-no-rounds-at-any-depth´ +/// ´test:crate:geometric-rounds-root-zero-min-zero´ +#[test] +fn geometric_rounds_root_zero_min_zero() { + // Both root and min zero → always 0 rounds. + let schedule = NoiseSchedule::Geometric { + root: 0, + decay: 0.5, + min: 0, + }; + assert_eq!(schedule.rounds_for_depth(0), 0); + assert_eq!(schedule.rounds_for_depth(5), 0); + assert_eq!(schedule.rounds_for_depth(128), 0); +} + +/// An explicit schedule is a direct lookup by depth, and depths past the end of +/// the vector reuse its last entry rather than falling to zero or failing. The +/// tail is the host's statement about all remaining depths, so a short vector +/// still describes an unbounded tree. +/// +/// ´claim:config:an-explicit-schedule-is-read-by-depth-and-clamps-to-its-last-entry-beyond-its-length´ +/// ´test:crate:explicit-rounds-for-depth´ +#[test] +fn explicit_rounds_for_depth() { + let schedule = NoiseSchedule::Explicit(vec![50, 30, 10]); + assert_eq!(schedule.rounds_for_depth(0), 50); + assert_eq!(schedule.rounds_for_depth(1), 30); + assert_eq!(schedule.rounds_for_depth(2), 10); + // Beyond vector length → clamps to last entry. + assert_eq!(schedule.rounds_for_depth(3), 10); + assert_eq!(schedule.rounds_for_depth(99), 10); +} + +/// The degenerate case of that clamping: a lone entry answers for every depth, +/// which is how a uniform explicit schedule is written. +/// +/// (´claim:config:an-explicit-schedule-is-read-by-depth-and-clamps-to-its-last-entry-beyond-its-length´) +/// ´test:crate:explicit-rounds-single-entry´ +#[test] +fn explicit_rounds_single_entry() { + let schedule = NoiseSchedule::Explicit(vec![5]); + assert_eq!(schedule.rounds_for_depth(0), 5); + assert_eq!(schedule.rounds_for_depth(1), 5); + assert_eq!(schedule.rounds_for_depth(100), 5); +} + +/// Entries are taken exactly as given and in the order given — no monotonic +/// taper is imposed on the explicit form and none is inferred from it. The +/// clamp past the end still reuses the final entry, whatever its relation to +/// the ones before it. +/// +/// (´claim:config:an-explicit-schedule-is-read-by-depth-and-clamps-to-its-last-entry-beyond-its-length´) +/// ´test:crate:explicit-rounds-non-monotonic´ +#[test] +fn explicit_rounds_non_monotonic() { + // Explicit is a direct index — non-monotonic is fine. + let schedule = NoiseSchedule::Explicit(vec![10, 50, 5, 100, 1]); + assert_eq!(schedule.rounds_for_depth(0), 10); + assert_eq!(schedule.rounds_for_depth(1), 50); + assert_eq!(schedule.rounds_for_depth(2), 5); + assert_eq!(schedule.rounds_for_depth(3), 100); + assert_eq!(schedule.rounds_for_depth(4), 1); + // Beyond length → clamp to last entry. + assert_eq!(schedule.rounds_for_depth(5), 1); + assert_eq!(schedule.rounds_for_depth(999), 1); +} + +// ── NoiseSchedule::is_disabled ────────────────────────────── + +/// An explicit schedule that can never yield a round counts as noise switched +/// off, and the two facts agree: the round count is zero at every depth and the +/// schedule reports itself disabled. That agreement is what lets other rules +/// key off the disabled flag instead of re-deriving it. +/// +/// ´claim:config:an-explicit-schedule-that-can-never-yield-a-round-reports-itself-as-noise-switched-off´ +/// ´test:crate:explicit-empty-is-disabled´ +#[test] +fn explicit_empty_is_disabled() { + let schedule = NoiseSchedule::Explicit(vec![]); + assert_eq!(schedule.rounds_for_depth(0), 0); + assert_eq!(schedule.rounds_for_depth(5), 0); + assert!(schedule.is_disabled()); +} + +/// A vector of zeros is disabled just as an empty one is. Emptiness is not the +/// criterion — producing nothing is — so the two ways of writing no warm-up are +/// not held apart. +/// +/// (´claim:config:an-explicit-schedule-that-can-never-yield-a-round-reports-itself-as-noise-switched-off´) +/// ´test:crate:explicit-all-zeros-is-disabled´ +#[test] +fn explicit_all_zeros_is_disabled() { + let schedule = NoiseSchedule::Explicit(vec![0, 0, 0]); + assert_eq!(schedule.rounds_for_depth(0), 0); + assert!(schedule.is_disabled()); +} + +/// A lone zero entry disables the schedule at every depth, since the clamp past +/// the end reuses that same zero. A one-element vector cannot describe warm-up +/// that begins further down. +/// +/// (´claim:config:an-explicit-schedule-that-can-never-yield-a-round-reports-itself-as-noise-switched-off´) +/// ´test:crate:explicit-single-zero-is-disabled´ +#[test] +fn explicit_single_zero_is_disabled() { + let schedule = NoiseSchedule::Explicit(vec![0]); + assert_eq!(schedule.rounds_for_depth(0), 0); + assert_eq!(schedule.rounds_for_depth(10), 0); + assert!(schedule.is_disabled()); +} + +/// A single non-zero entry anywhere keeps noise enabled, even where the shallow +/// depths ask for none. A zero at a given depth is a statement about that depth +/// alone, so a schedule may deliberately warm only the deeper cells and still +/// counts as active. +/// +/// ´claim:config:one-non-zero-entry-anywhere-keeps-noise-enabled-however-many-depths-ask-for-none´ +/// ´test:crate:explicit-mixed-zeros-not-disabled´ +#[test] +fn explicit_mixed_zeros_not_disabled() { + // At least one non-zero entry → not disabled. + let schedule = NoiseSchedule::Explicit(vec![0, 0, 5]); + assert!(!schedule.is_disabled()); + assert_eq!(schedule.rounds_for_depth(0), 0); + assert_eq!(schedule.rounds_for_depth(1), 0); + assert_eq!(schedule.rounds_for_depth(2), 5); + assert_eq!(schedule.rounds_for_depth(10), 5); +} + +/// A geometric schedule is disabled only when both its root and its floor are +/// zero, because either one alone still produces rounds. The floor lifts every +/// depth to at least its own count, and the root sets the count at the shallow +/// depths before the taper has descended. Reading only one of the two calls a +/// schedule silent that is still asking for warm-up. +/// +/// ´claim:config:a-geometric-schedule-is-disabled-only-when-both-its-root-and-its-floor-are-zero´ +/// ´test:crate:geometric-is-disabled-when-root-and-min-are-zero´ +#[test] +fn geometric_is_disabled_when_root_and_min_are_zero() { + let schedule = NoiseSchedule::Geometric { + root: 0, + decay: 0.5, + min: 0, + }; + assert!(schedule.is_disabled()); + assert_eq!(schedule.rounds_for_depth(0), 0); +} + +/// One side of the same rule: a positive floor keeps the schedule active no +/// matter how steeply it tapers, since every depth is lifted to at least that +/// count. +/// +/// (´claim:config:a-geometric-schedule-is-disabled-only-when-both-its-root-and-its-floor-are-zero´) +/// ´test:crate:geometric-is-not-disabled-when-min-positive´ +#[test] +fn geometric_is_not_disabled_when_min_positive() { + let schedule = NoiseSchedule::Geometric { + root: 10, + decay: 0.5, + min: 1, + }; + assert!(!schedule.is_disabled()); +} + +/// The other side, and the one a floor-only reading gets wrong: a positive root +/// with a zero floor is not disabled, because the taper starts at the root and +/// the shallow depths are served from it. The depth-zero count is the root +/// itself, so a schedule reported silent here would be one that immediately +/// asks for hundreds of rounds. +/// +/// (´claim:config:a-geometric-schedule-is-disabled-only-when-both-its-root-and-its-floor-are-zero´) +/// ´test:crate:geometric-is-not-disabled-when-root-positive-and-min-zero´ +#[test] +fn geometric_is_not_disabled_when_root_positive_and_min_zero() { + let schedule = NoiseSchedule::Geometric { + root: 450, + decay: 0.5, + min: 0, + }; + assert!(!schedule.is_disabled()); + assert_eq!(schedule.rounds_for_depth(0), 450); +} + +// ── NoiseSchedule::max_rounds ─────────────────────────────── + +/// The most a geometric schedule can ever ask for is its root, because the +/// taper only descends from there. A host sizing buffers for warm-up can read +/// the ceiling off the root alone, without evaluating the schedule at any +/// depth. +/// +/// ´claim:config:the-ceiling-of-a-geometric-schedule-is-its-root-because-the-taper-only-descends´ +/// ´test:crate:max-rounds-geometric´ +#[test] +fn max_rounds_geometric() { + let schedule = NoiseSchedule::geometric(999, 0.1, 1); + assert_eq!(schedule.max_rounds(), 999); +} + +/// Where the floor stands above the root, the floor is the ceiling. The taper +/// descends from the root, so with a floor above it every depth is lifted to +/// the floor and the schedule yields that count everywhere — the root never +/// being reached at all. Reading the root alone understates what the schedule +/// asks for, and understates it at every depth rather than at some extreme, +/// which matters because this figure is what a host sizes warm-up buffers +/// from. The pairing is admitted by validation, so it is a configuration a +/// host can actually be holding. +/// +/// (´claim:config:the-ceiling-of-a-geometric-schedule-is-its-root-because-the-taper-only-descends´) +/// ´test:crate:max-rounds-geometric-floor-above-root´ +#[test] +fn max_rounds_geometric_floor_above_root() { + let schedule = NoiseSchedule::geometric(10, 0.5, 50); + let cfg = SentinelConfig:: { + noise_schedule: NoiseSchedule::geometric(10, 0.5, 50), + ..SentinelConfig::::default() + }; + cfg.validate().expect("a floor above the root is an accepted configuration"); + + assert_eq!(schedule.rounds_for_depth(0), 50); + assert_eq!(schedule.max_rounds(), 50); +} + +/// For an explicit schedule the ceiling is the largest entry it holds, not the +/// first. Since the explicit form imposes no ordering, the depth-zero value +/// carries no promise about the rest and the maximum has to be found rather +/// than assumed. +/// +/// ´claim:config:the-ceiling-of-an-explicit-schedule-is-its-largest-entry-not-its-first´ +/// ´test:crate:max-rounds-explicit´ +#[test] +fn max_rounds_explicit() { + // max_rounds returns the maximum value, not the first. + let schedule = NoiseSchedule::Explicit(vec![10, 50, 30]); + assert_eq!(schedule.max_rounds(), 50); +} + +/// A schedule with no entries has no largest entry, and its ceiling comes back +/// as zero rather than as an absence the caller must handle. Sizing a buffer +/// for a disabled schedule is asking for nothing. +/// +/// (´claim:config:the-ceiling-of-an-explicit-schedule-is-its-largest-entry-not-its-first´) +/// ´test:crate:max-rounds-explicit-empty´ +#[test] +fn max_rounds_explicit_empty() { + let schedule = NoiseSchedule::Explicit(vec![]); + assert_eq!(schedule.max_rounds(), 0); +} + +/// The ceiling is reported exactly, up to the widest count the round type can +/// hold, with nothing clamped or lost on the way out. A capacity hint that +/// quietly saturated would be worse than none at all. +/// +/// (´claim:config:the-ceiling-of-an-explicit-schedule-is-its-largest-entry-not-its-first´) +/// ´test:crate:max-rounds-explicit-large´ +#[test] +fn max_rounds_explicit_large() { + let schedule = NoiseSchedule::Explicit(vec![u32::MAX]); + assert_eq!(schedule.max_rounds(), u32::MAX); +} + +// ── NoiseSchedule::Default ────────────────────────────────── + +/// The shipped schedule is the geometric one its documentation describes, and +/// it reports itself active. Its numbers are calibrated rather than arbitrary: +/// the root sits a little above the worst-case baseline convergence measured at +/// the default forgetting factor, and the floor covers deep cells whose +/// convergence scales down with analysis width without vanishing. +/// +/// ´claim:config:the-shipped-noise-schedule-is-the-calibrated-geometric-one-and-it-is-active´ +/// ´test:crate:noise-schedule-default-matches-doc´ +#[test] +fn noise_schedule_default_matches_doc() { + let schedule = NoiseSchedule::default(); + match &schedule { + NoiseSchedule::Geometric { root, decay, min } => { + assert_eq!(*root, 450); + assert!((decay - 0.5).abs() < f64::EPSILON); + assert_eq!(*min, 50); + } + NoiseSchedule::Explicit(_) => panic!("default should be Geometric"), + } + assert!(!schedule.is_disabled()); +} + +// ── ConfigWarning ─────────────────────────────────────────── + +/// The shipped configuration draws no advisories, because the default schedule +/// was calibrated against the default forgetting factor. Defaults that +/// validated but warned would be an odd thing to ship, so the two sets of +/// defaults are kept consistent with each other. +/// +/// ´claim:config:the-shipped-defaults-draw-no-advisories-because-the-schedule-was-calibrated-for-the-default-memory´ +/// ´test:crate:default-config-has-no-warnings´ +#[test] +fn default_config_has_no_warnings() { + let cfg = SentinelConfig::::default(); + assert_eq!(cfg.warnings(), [] as [ConfigWarning; 0]); +} + +/// A configuration whose warm-up rounds fall short of what its memory needs is +/// advised, not refused: it is arithmetically sound, but baselines may not +/// converge before real observations arrive, so early scores would be +/// unreliable. Advice and refusal are separate channels — validation would pass +/// this configuration unchanged, and only the warning list carries the +/// concern. +/// +/// ´claim:config:insufficient-warm-up-is-advice-rather-than-refusal-because-the-configuration-still-runs´ +/// ´test:crate:warns-when-noise-root-too-low-for-lambda-099´ +#[test] +fn warns_when_noise_root_too_low_for_lambda_099() { + let cfg = SentinelConfig:: { + forgetting_factor: 0.99, + noise_schedule: NoiseSchedule::geometric(50, 0.5, 10), + ..SentinelConfig::::default() + }; + let warnings = cfg.warnings(); + assert_eq!(warnings.len(), 1); + assert!(matches!( + &warnings[0], + ConfigWarning::NoiseScheduleInsufficient { + root: 50, + recommended_root: 450, + lambda, + } if (*lambda - 0.99).abs() < f64::EPSILON + )); +} + +/// How much warm-up is recommended falls with the forgetting factor, because a +/// shorter memory converges sooner: a schedule too thin for a long-memory +/// baseline is adequate for a shorter one. The same schedule draws advice or +/// silence depending on the memory it is paired with, since the recommendation +/// is a relation between the two rather than a property of either. +/// +/// ´claim:config:the-recommended-warm-up-falls-with-the-forgetting-factor-because-a-shorter-memory-converges-sooner´ +/// ´test:crate:no-warning-when-noise-root-sufficient-for-lambda-095´ +#[test] +fn no_warning_when_noise_root_sufficient_for_lambda_095() { + let cfg = SentinelConfig:: { + forgetting_factor: 0.95, + noise_schedule: NoiseSchedule::geometric(50, 0.5, 10), + ..SentinelConfig::::default() + }; + assert_eq!(cfg.warnings(), [] as [ConfigWarning; 0]); +} + +/// Batch size enters the recommendation as well: with few synthetic samples per +/// round, each round buys less convergence, so the same shorter memory demands +/// markedly more rounds and a schedule that was adequate becomes advised +/// against. Warm-up is really measured in observations rather than in rounds, +/// and the recommendation reflects that. +/// +/// ´claim:config:a-smaller-noise-batch-raises-the-recommended-round-count-because-each-round-buys-less-convergence´ +/// ´test:crate:warns-when-small-batch-and-lambda-095´ +#[test] +fn warns_when_small_batch_and_lambda_095() { + // At λ=0.95, b=4, recommended root is 200. + let cfg = SentinelConfig:: { + forgetting_factor: 0.95, + noise_batch_size: 4, + noise_schedule: NoiseSchedule::geometric(50, 0.5, 10), + ..SentinelConfig::::default() + }; + let warnings = cfg.warnings(); + assert_eq!(warnings.len(), 1); + assert!(matches!( + &warnings[0], + ConfigWarning::NoiseScheduleInsufficient { + recommended_root: 200, + .. + } + )); +} + +/// The advisory judges whatever the schedule actually yields at depth zero, +/// whichever variant it is written in. An explicit schedule generous enough at +/// the root passes the same check a geometric one would, so the recommendation +/// is about warm-up delivered and not about how the host chose to express +/// it. +/// +/// ´claim:config:the-advisory-judges-the-rounds-a-schedule-actually-yields-at-depth-zero-whatever-its-variant´ +/// ´test:crate:no-warning-for-explicit-schedule-with-enough-rounds´ +#[test] +fn no_warning_for_explicit_schedule_with_enough_rounds() { + let cfg = SentinelConfig:: { + forgetting_factor: 0.99, + noise_schedule: NoiseSchedule::Explicit(vec![500, 300, 100]), + ..SentinelConfig::::default() + }; + assert_eq!(cfg.warnings(), [] as [ConfigWarning; 0]); +} + +/// A rendered advisory carries the numbers a host needs in order to act on it: +/// the root it found, the root it recommends, and the forgetting factor that +/// set that recommendation. Advice naming only the problem would leave the +/// reader to re-derive the target. +/// +/// ´claim:config:a-rendered-advisory-names-the-root-it-found-the-root-it-recommends-and-the-memory-that-set-it´ +/// ´test:crate:warning-display-is-informative´ +#[test] +fn warning_display_is_informative() { + let w = ConfigWarning::NoiseScheduleInsufficient { + root: 50, + recommended_root: 450, + lambda: 0.99, + }; + let msg = w.to_string(); + assert!(msg.contains("50")); + assert!(msg.contains("450")); + assert!(msg.contains("0.99")); +} + +// ── Construction refusal: coordinate width ────────────────── + +/// A coordinate width narrower than the smallest dimension a subspace tracker +/// can model is refused at construction, with the same structured failure the +/// configuration faults carry. The width is a parameter of the type rather than +/// a field of the configuration, so validating the configuration alone can +/// never see it, and the root tracker spans the whole width — at one dimension +/// its lone basis vector spans the entire space, novelty is identically zero, +/// and the tracker reports a settled model of everything while modelling +/// nothing. Refusing is what lets the constructor's success mean the sentinel +/// it returns can measure. +/// +/// ´claim:config:a-coordinate-width-below-the-tracker-minimum-is-refused-at-construction´ +/// ´test:crate:rejects-coordinate-width-below-the-tracker-minimum´ +#[test] +fn rejects_coordinate_width_below_the_tracker_minimum() { + use crate::SpectralSentinel; + + let Err(err) = SpectralSentinel::::new(SentinelConfig::::default()) else { + panic!("a coordinate width below the tracker minimum must be refused"); + }; + assert!(err.0.contains(&ConfigError::TrackerDimensionTooSmall { + width: 1, + minimum: crate::MIN_TRACKER_DIM + })); +} + +/// The narrowest width the tracker can model is admitted, which fixes the +/// boundary rather than leaving it to be inferred from the refusal alone. Two +/// dimensions leave one residual degree of freedom, which is the least that +/// makes a novelty reading mean anything. +/// +/// (´claim:config:a-coordinate-width-below-the-tracker-minimum-is-refused-at-construction´) +/// ´test:crate:accepts-the-narrowest-modellable-coordinate-width´ +#[test] +fn accepts_the_narrowest_modellable_coordinate_width() { + use crate::SpectralSentinel; + + let sentinel = SpectralSentinel::::new(SentinelConfig::::default()).unwrap(); + assert_eq!( + sentinel.cells_tracked(), + 1, + "the root tracker is built at the narrowest width" + ); +} + +/// A width fault and a configuration fault come back together rather than one +/// at a time, so a host repairing a sentinel that is wrong in both respects +/// learns both in a single pass. This is the collecting behaviour the +/// configuration's own validation promises, extended to the one fault that +/// validation cannot reach by itself. +/// +/// (´claim:config:a-coordinate-width-below-the-tracker-minimum-is-refused-at-construction´) +/// ´test:crate:collects-a-width-fault-alongside-a-configuration-fault´ +#[test] +fn collects_a_width_fault_alongside_a_configuration_fault() { + use crate::SpectralSentinel; + + let cfg = SentinelConfig:: { + max_rank: 0, + ..SentinelConfig::::default() + }; + let Err(err) = SpectralSentinel::::new(cfg) else { + panic!("a coordinate width below the tracker minimum must be refused"); + }; + assert!( + err.0 + .iter() + .any(|e| matches!(e, ConfigError::TrackerDimensionTooSmall { .. })) + ); + assert!(err.0.contains(&ConfigError::MaxRankZero)); +} + +// ── Construction refusal: the centred-bit ceiling ─────────── + +/// A coordinate type wider than the centred bit vector, standing in for the +/// downstream implementation the bridge trait is open to. +/// +/// No coordinate this crate ships can reach a width above the ceiling: the two +/// it implements the bridge for are sixty-four and a hundred and twenty-eight +/// bits wide, and the spatial layer settles at compile time that the width fit +/// the coordinate type — so a wider width over an in-crate coordinate never +/// reaches the constructor to be refused at all. A type declaring a wider +/// domain is what a host writes when its coordinates are wider, and it is +/// therefore the only way to put the constructor's ceiling to the question. +/// Its conversion delegates to the widest in-crate width, which is the honest +/// half of the dilemma the refusal removes: the alternative is a vector longer +/// than the array that carries it. Nothing past construction is exercised +/// through this type. +#[derive(Copy, Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord)] +struct WideCoordinate(u128); + +impl torrust_mudlark::Coordinate for WideCoordinate { + const BITS: u32 = 256; + + fn zero() -> Self { + Self(0) + } + + fn domain_max(n: u32) -> Self { + if n >= u128::BITS { Self(u128::MAX) } else { Self(1 << n) } + } + + fn midpoint(a: Self, b: Self) -> Self { + Self(a.0 + (b.0 - a.0) / 2) + } + + fn width(start: Self, end: Self) -> Self { + Self(end.0 - start.0) + } + + fn is_final(start: Self, end: Self, _depth: u32, _n: u32) -> bool { + end.0 - start.0 == 1 + } + + fn from_u64(v: u64) -> Self { + Self(u128::from(v)) + } + + fn next_value(self) -> Self { + Self(self.0 + 1) + } + + fn to_f64(self) -> f64 { + unreachable!("the stand-in never reaches the width ratio a range query would ask for") + } + + fn is_nan(self) -> bool { + false + } + + fn total_cmp(&self, other: &Self) -> std::cmp::Ordering { + self.0.cmp(&other.0) + } +} + +impl crate::CentredBitSource for WideCoordinate { + fn to_centred_bits(&self, n: u32) -> crate::CentredBits { + crate::CentredBitSource::to_centred_bits(&self.0, n) + } +} + +/// A coordinate width above what the centred bit vector can carry is refused +/// at construction, the same way a width below the tracker minimum is. The +/// bridge that turns a coordinate into centred bits is open to any +/// implementor, and the spatial layer asks only that the width fit the +/// coordinate type, so a host whose coordinates are wider than the vector can +/// otherwise ask for a sentinel wider than the vector that feeds it. Nothing +/// would fault: the slots past the vector's length come back as zeros, a +/// centred bit is ±0.5 and never zero, and every dimension past the end would +/// be modelled over a constant the coordinate stream never produced — a +/// settled reading of data that does not exist, mixed into novelty, residual +/// and rank alike. The refusal names the width and the ceiling, since those +/// are what the host must reconcile. +/// +/// ´claim:config:a-coordinate-width-above-the-centred-bit-ceiling-is-refused-at-construction´ +/// ´test:crate:refuses-a-coordinate-width-above-the-centred-bit-ceiling´ +#[test] +fn refuses_a_coordinate_width_above_the_centred_bit_ceiling() { + use crate::SpectralSentinel; + + let cfg = SentinelConfig:: { + noise_schedule: NoiseSchedule::Explicit(vec![]), + ..SentinelConfig::::default() + }; + let Err(err) = SpectralSentinel::::new(cfg) else { + panic!("a coordinate width above the centred bit ceiling must be refused"); + }; + assert!(err.0.contains(&ConfigError::TrackerDimensionTooLarge { + width: 200, + maximum: crate::MAX_TRACKER_DIM + })); +} + +/// The widest width the observation path can carry is admitted, which fixes +/// the ceiling from the accepting side. A hundred and twenty-eight bits is the +/// width the vector is built for and the one the crate's own default alias +/// stands at, so the refusal above must land strictly beyond it; with the +/// narrowest admitted width already pinned, both ends of the modellable range +/// are fixed by tests rather than inferred from the refusals alone. +/// +/// (´claim:config:a-coordinate-width-above-the-centred-bit-ceiling-is-refused-at-construction´) +/// ´test:crate:accepts-the-widest-modellable-coordinate-width´ +#[test] +fn accepts_the_widest_modellable_coordinate_width() { + use crate::SpectralSentinel; + + let sentinel = SpectralSentinel::::new(SentinelConfig::::default()).unwrap(); + assert_eq!( + sentinel.cells_tracked(), + 1, + "the root tracker is built at the widest modellable width" + ); +} + +// ── Construction refusal: the warming thread ──────────────── + +/// The refusal a host receives when the environment will not give the engine a +/// warming thread names the setting that asked for one and quotes the +/// operating system's own account of the refusal. Nothing in the configuration +/// is wrong in that case, so a message that said only that a configuration was +/// invalid would send an operator searching values that are all correct: +/// naming the setting says which request to withdraw, and quoting the +/// environment says whether withdrawing it is the right answer at all or +/// whether the machine is simply out of threads. +/// +/// ´claim:config:the-warming-thread-refusal-names-the-setting-that-asked-for-one-and-quotes-the-environment´ +/// ´test:crate:warming-thread-refusal-names-the-setting-and-the-environment´ +#[test] +fn warming_thread_refusal_names_the_setting_and_the_environment() { + let refusal = ConfigError::BackgroundWarmingThreadUnavailable { + reason: "Resource temporarily unavailable (os error 11)".to_owned(), + }; + + let rendered = refusal.to_string(); + + assert!( + rendered.contains("background_warming"), + "the refusal must name the setting that asked for the thread, got: {rendered}" + ); + assert!( + rendered.contains("Resource temporarily unavailable (os error 11)"), + "the refusal must quote the environment's own account, got: {rendered}" + ); +} + +#[test] +fn unrepresentable_noise_batch_is_rejected_before_construction() { + let config = SentinelConfig:: { + noise_schedule: NoiseSchedule::Explicit(vec![1]), + noise_batch_size: usize::MAX, + ..SentinelConfig::default() + }; + let validation = config.validate(); + // Keep construction before the assertions: the unvalidated configuration + // reaches the capacity-overflow panic in a constructor without this guard. + let construction = crate::SpectralSentinel::::new(config); + let errors = validation.expect_err("an unrepresentable noise batch must be rejected"); + assert_eq!(errors.0.len(), 1); + assert_eq!( + errors.to_string(), + format!("noise_batch_size ({}) exceeds representable allocation bounds", usize::MAX) + ); + assert!( + construction.is_err(), + "construction must return a structured configuration error" + ); +} + +#[test] +fn rejects_noise_matrix_size_even_when_the_outer_vector_fits() { + // One more row than fits in an unpadded, maximum-width f64 matrix; + // the outer Vec's much smaller row descriptor still fits. + let batch_size = isize::MAX.unsigned_abs() / (crate::MAX_TRACKER_DIM * size_of::()) + 1; + assert!( + batch_size + .checked_mul(size_of::>()) + .is_some_and(|bytes| bytes <= isize::MAX.unsigned_abs()) + ); + let config = SentinelConfig:: { + noise_schedule: NoiseSchedule::Explicit(vec![1]), + noise_batch_size: batch_size, + ..SentinelConfig::default() + }; + assert!(config.validate().is_err(), "matrix byte sizes must be representable"); +} + +#[test] +fn ignores_unallocated_noise_batch_size_when_disabled() { + let config = SentinelConfig:: { + noise_schedule: NoiseSchedule::Explicit(Vec::new()), + noise_batch_size: usize::MAX, + ..SentinelConfig::default() + }; + assert!(config.validate().is_ok()); +} diff --git a/packages/sentinel/src/tests/convergence_clipping.rs b/packages/sentinel/src/tests/convergence_clipping.rs new file mode 100644 index 000000000..a010cdda1 --- /dev/null +++ b/packages/sentinel/src/tests/convergence_clipping.rs @@ -0,0 +1,554 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +//! # Test index +//! +//! | Test | Area | Claim | +//! |------|------|-------| +//! | [`clip_bias_is_negligible_at_steady_state`] | clipping | Rejecting the upper tail leaves the settled baseline where an unclipped run puts it. The clip removes a fraction of a percent of the mass of a right-skewed score distribution, so the downward bias it induces is smaller than the baseline's own steady-state jitter: outlier resistance is bought without moving the reference it protects. | +//! | [`variance_estimate_unbiased_despite_clipping`] | clipping | The spread estimate survives tail truncation as well as the mean does. A second moment is far more sensitive to a missing tail than a first, so this is the tighter half of the same audit: a baseline built under clipping is calibrated and not merely correctly centred. | +//! | [`graduated_exemption_prevents_bistable_attractor`] | clipping | Clip widths from tight to loose all settle on the same fixed point. A clipped baseline is a nonlinear filter with a second, biased fixed point where tight clipping keeps rejecting the very evidence that would loosen it; the graduated exemption widens the basin of the correct one during warm-up, so no configuration falls into the other. | +//! | [`exemption_decay_does_not_cause_transient_instability`] | clipping | As the exemption is spent the effective clip tightens, and the baselines pass through that tightening without a spike or a dip — once the exemption has largely decayed, no rolling mean departs from the eventual steady state by more than the jitter envelope. The width is a smooth function of the exemption rather than a switch, which is what keeps a trajectory from being thrown across a basin boundary. | +//! | [`clip_ceiling_stabilises_after_exemption_decay`] | clipping | The ceiling is the baseline's own mean and spread scaled by the clip width, so it settles when they do: once the exemption is spent its rolling variation stays within a few percent. A ceiling that kept swinging would clip in bursts and feed those bursts straight back into the mean and spread that define it. | +//! | [`slow_ewma_clipping_does_not_bias_cusum_reference`] | clipping | The long-memory reference is filtered by the same ceiling as the short-memory baseline, and once the two have been seeded into agreement they stay in agreement across a long run. Clipping therefore adds no bias of its own on the reference side, so a gap between the two can be read as drift rather than as an artefact of filtering. | +//! | [`cusum_bounded_through_clip_transitions`] | clipping | A tightening clip does not manufacture evidence of drift: through the whole stretch after seeding, the drift accumulators stay far below anything a host would act on. The moment the clip narrows is when the two baselines are most likely to disagree, so it is where a false alarm would appear if the mechanisms interfered with one another. | + +//! An audit of what outlier rejection costs the converged model. +//! +//! Clipping exists so that a burst of inflated scores cannot poison a +//! baseline: only the upper tail is rejected, because anomaly scores are +//! non-negative and right-skewed and an attacker inflates them rather than +//! deflating them. But a baseline that filters against its own mean and +//! spread is a feedback loop, and a feedback loop can settle somewhere +//! other than where the data is. Three places in the pipeline apply the +//! same ceiling — the short-memory baseline, the long-memory reference the +//! drift accumulator measures against, and the exemption that widens the +//! ceiling while the model is still noise-taught — and each could, in +//! principle, bias the settled model or delay it. The tests here establish +//! that none of them does. +//! +//! Two properties do the work. The rejected tail is a fraction of a +//! percent of the mass at the nominal width, so the bias it induces is +//! smaller than the baseline's own steady-state jitter — for the spread as +//! well as for the level, which is the more demanding of the two. And the +//! exemption removes the loop's second, biased fixed point during warm-up +//! by holding the ceiling open until the model has a real baseline to +//! filter against; the width then narrows smoothly rather than switching, +//! so nothing is thrown across a basin boundary on the way down. +//! +//! The audit is therefore comparative: a clipped run against an unclipped +//! one, several clip widths against each other, and the trajectory +//! through the tightening against the steady state it ends at. + +use rand::SeedableRng; +use rand::rngs::SmallRng; + +use super::convergence_common::{AXIS_NAMES, as_slices, cfg_test, generate_noise, run_noise_trace}; +use crate::config::SentinelConfig; +use crate::sentinel::tracker::SubspaceTracker; + +/// Per-round snapshot of baseline mean, variance, and η. +struct RoundSnap { + baseline_mean: [f64; 4], + baseline_var: [f64; 4], + eta: f64, +} + +// ════════════════════════════════════════════════════════════ +// 1. Fast EWMA upper-tail clip (§ALGO S-6.1.1) +// ════════════════════════════════════════════════════════════ + +/// Rejecting the upper tail leaves the settled baseline where an unclipped run +/// puts it. The clip removes a fraction of a percent of the mass of a +/// right-skewed score distribution, so the downward bias it induces is smaller +/// than the baseline's own steady-state jitter: outlier resistance is bought +/// without moving the reference it protects. +/// +/// ´claim:clipping:outlier-rejection-leaves-the-settled-baseline-where-an-unclipped-run-puts-it´ +/// ´test:crate:clip-bias-is-negligible-at-steady-state´ +#[test] +fn clip_bias_is_negligible_at_steady_state() { + let dim = 128; + let total_rounds = 500; + let seed = 42; + + // Run with standard clipping. + let cfg_clipped = cfg_test(); // clip_sigmas = 3.0 + let traces_clipped = run_noise_trace(&cfg_clipped, dim, total_rounds, seed); + + // Run without clipping. + let cfg_unclipped = SentinelConfig { + clip_sigmas: f64::INFINITY, + ..cfg_test() + }; + let traces_unclipped = run_noise_trace(&cfg_unclipped, dim, total_rounds, seed); + + // Compare steady-state block means (last 100 rounds). + let block_start = total_rounds - 100; + + // Per-axis tolerances for the clip-vs-unclip comparison. + // + // These accommodate both the clip bias AND the EWMA jitter. + // The clip bias is < 0.3% of the mean (§4 theory), but the + // EWMA jitter adds uncertainty — especially for high-CV axes + // (coherence ~10%). We also use the same RNG seed, so the + // *input* noise sequence is identical; only the clipping + // creates divergence. But the graduated exemption causes the + // two runs to track differently from the start, so by round + // 400+ the EWMA trajectories have drifted apart by O(CV). + // + // Axis | expected bias | jitter CV | tolerance + // --------------|---------------|-----------|---------- + // Novelty | < 0.1% | 0.07% | 2% + // Displacement | < 0.5% | 3.4% | 10% + // Surprise | < 0.5% | 6.5% | 15% + // Coherence | < 0.5% | 10.0% | 20% + let tolerances = [0.02, 0.10, 0.15, 0.20]; + + for (ax, (name, tol)) in AXIS_NAMES.iter().zip(tolerances.iter()).enumerate() { + let clipped_mean: f64 = traces_clipped[block_start..] + .iter() + .map(|t| t.baseline_means[ax]) + .sum::() + / 100.0; + + let unclipped_mean: f64 = traces_unclipped[block_start..] + .iter() + .map(|t| t.baseline_means[ax]) + .sum::() + / 100.0; + + if unclipped_mean.abs() < 1e-12 { + continue; // axis inactive + } + + let rel_diff = (clipped_mean - unclipped_mean).abs() / unclipped_mean.abs(); + assert!( + rel_diff < *tol, + "{name}: clipped vs unclipped steady-state bias is {:.2}%, \ + tolerance is {:.0}% — clipping introduces excessive bias", + rel_diff * 100.0, + tol * 100.0, + ); + } +} + +/// The spread estimate survives tail truncation as well as the mean does. A +/// second moment is far more sensitive to a missing tail than a first, so this +/// is the tighter half of the same audit: a baseline built under clipping is +/// calibrated and not merely correctly centred. +/// +/// ´claim:clipping:the-spread-estimate-survives-tail-truncation-as-well-as-the-mean-does´ +/// ´test:crate:variance-estimate-unbiased-despite-clipping´ +#[test] +fn variance_estimate_unbiased_despite_clipping() { + let dim = 128; + let total_rounds = 500; + let seed = 42; + + // We need variance, not just means — run directly. + let run_variances = |cfg: &SentinelConfig| -> [f64; 4] { + let mut tracker = SubspaceTracker::new(dim, cfg, cfg.cusum_slow_decay); + let mut rng = SmallRng::seed_from_u64(seed); + let mut last_var = [0.0_f64; 4]; + for _ in 0..total_rounds { + let noise = generate_noise(dim, cfg.noise_batch_size, &mut rng); + let report = tracker.observe(&as_slices(&noise), 0, true); + last_var = [ + report.scores.novelty.baseline.variance, + report.scores.displacement.baseline.variance, + report.scores.surprise.baseline.variance, + report.scores.coherence.baseline.variance, + ]; + } + last_var + }; + + let cfg_clipped = cfg_test(); + let var_clipped = run_variances(&cfg_clipped); + + let cfg_unclipped = SentinelConfig { + clip_sigmas: f64::INFINITY, + ..cfg_test() + }; + let var_unclipped = run_variances(&cfg_unclipped); + + // Since variance is a second-moment estimate, it's more + // sensitive to tail truncation than the mean. But at 3σ + // the effect is still small. We use generous tolerances. + let tolerances = [0.05, 0.15, 0.25, 0.35]; + + for (ax, (name, tol)) in AXIS_NAMES.iter().zip(tolerances.iter()).enumerate() { + if var_unclipped[ax].abs() < 1e-12 { + continue; + } + let rel_diff = (var_clipped[ax] - var_unclipped[ax]).abs() / var_unclipped[ax]; + assert!( + rel_diff < *tol, + "{name}: clipped variance differs from unclipped by {:.2}%, \ + tolerance {:.0}% — clipping distorts the variance estimate", + rel_diff * 100.0, + tol * 100.0, + ); + } +} + +// ════════════════════════════════════════════════════════════ +// 2. Graduated clip-exemption (§ALGO S-6.4) +// ════════════════════════════════════════════════════════════ + +/// Clip widths from tight to loose all settle on the same fixed point. A +/// clipped baseline is a nonlinear filter with a second, biased fixed point +/// where tight clipping keeps rejecting the very evidence that would loosen it; +/// the graduated exemption widens the basin of the correct one during warm-up, +/// so no configuration falls into the other. +/// +/// ´claim:clipping:every-clip-width-settles-on-the-same-fixed-point´ +/// ´test:crate:graduated-exemption-prevents-bistable-attractor´ +#[test] +fn graduated_exemption_prevents_bistable_attractor() { + let dim = 128; + let total_rounds = 500; + let seed = 42; + + let clip_values = [2.0, 3.0, 5.0]; + let mut steady_states: Vec<[f64; 4]> = Vec::new(); + + for &clip in &clip_values { + let cfg = SentinelConfig { + clip_sigmas: clip, + ..cfg_test() + }; + let traces = run_noise_trace(&cfg, dim, total_rounds, seed); + + // Compute block mean of last 100 rounds. + let block_start = total_rounds - 100; + let mut means = [0.0_f64; 4]; + for (ax, mean) in means.iter_mut().enumerate() { + *mean = traces[block_start..].iter().map(|t| t.baseline_means[ax]).sum::() / 100.0; + } + steady_states.push(means); + } + + // Compare all pairs: each axis should agree within tolerance. + // + // The tolerance accounts for both the (small) clip bias + // difference between c = 2 and c = 5, and the EWMA trajectory + // divergence from different clipping histories. + let tolerances = [0.03, 0.12, 0.18, 0.30]; + + for i in 0..clip_values.len() { + for j in (i + 1)..clip_values.len() { + for (ax, (name, tol)) in AXIS_NAMES.iter().zip(tolerances.iter()).enumerate() { + let ref_val = f64::midpoint(steady_states[i][ax], steady_states[j][ax]); + if ref_val.abs() < 1e-12 { + continue; + } + let rel_diff = (steady_states[i][ax] - steady_states[j][ax]).abs() / ref_val.abs(); + assert!( + rel_diff < *tol, + "{name}: clip_sigmas {:.0} vs {:.0} differ by {:.2}%, tolerance {:.0}% — \ + graduated exemption may not be preventing bistable attractor", + clip_values[i], + clip_values[j], + rel_diff * 100.0, + tol * 100.0, + ); + } + } + } +} + +/// As the exemption is spent the effective clip tightens, and the baselines +/// pass through that tightening without a spike or a dip — once the exemption +/// has largely decayed, no rolling mean departs from the eventual steady state +/// by more than the jitter envelope. The width is a smooth function of the +/// exemption rather than a switch, which is what keeps a trajectory from being +/// thrown across a basin boundary. +/// +/// ´claim:clipping:the-baselines-cross-the-tightening-of-the-clip-without-a-transient´ +/// ´test:crate:exemption-decay-does-not-cause-transient-instability´ +#[test] +#[allow(clippy::cast_precision_loss)] +fn exemption_decay_does_not_cause_transient_instability() { + let cfg = cfg_test(); + let dim = 128; + let total_rounds = 400; + let traces = run_noise_trace(&cfg, dim, total_rounds, 42); + + let window = 20_usize; + + // Find the round where η drops below 0.1. + let eta_threshold_round = traces.iter().position(|t| t.noise_influence < 0.1).unwrap_or(total_rounds); + + // Compute the reference steady-state mean (last 50 rounds). + let ss_start = total_rounds - 50; + let mut ss_means = [0.0_f64; 4]; + for (ax, ss_mean) in ss_means.iter_mut().enumerate() { + *ss_mean = traces[ss_start..].iter().map(|t| t.baseline_means[ax]).sum::() / 50.0; + } + + // Per-axis tolerances for the post-exemption jitter envelope. + // These are wider than the block-mean tolerances in + // convergence_noise.rs because a 20-round rolling mean has + // higher variance than a 100-round block mean. + let tolerances = [0.05, 0.15, 0.25, 0.35]; + + let start_check = eta_threshold_round.max(window); + for round in start_check..=(total_rounds - window) { + let mut rolling = [0.0_f64; 4]; + for (ax, roll) in rolling.iter_mut().enumerate() { + *roll = traces[round..round + window] + .iter() + .map(|t| t.baseline_means[ax]) + .sum::() + / window as f64; + } + + for (ax, (name, tol)) in AXIS_NAMES.iter().zip(tolerances.iter()).enumerate() { + if ss_means[ax].abs() < 1e-12 { + continue; + } + let rel_dev = (rolling[ax] - ss_means[ax]).abs() / ss_means[ax].abs(); + assert!( + rel_dev < *tol, + "{name} at round {round}: rolling mean deviates {:.2}% from \ + steady state (tolerance {:.0}%) — transient instability \ + during exemption decay", + rel_dev * 100.0, + tol * 100.0, + ); + } + } +} + +/// The ceiling is the baseline's own mean and spread scaled by the clip width, +/// so it settles when they do: once the exemption is spent its rolling +/// variation stays within a few percent. A ceiling that kept swinging would +/// clip in bursts and feed those bursts straight back into the mean and spread +/// that define it. +/// +/// ´claim:clipping:the-ceiling-settles-into-a-narrow-band-so-no-clip-feedback-cycle-can-start´ +/// ´test:crate:clip-ceiling-stabilises-after-exemption-decay´ +#[test] +#[allow(clippy::cast_precision_loss)] +fn clip_ceiling_stabilises_after_exemption_decay() { + let cfg = cfg_test(); + let dim = 128; + let total_rounds = 400; + + let mut tracker = SubspaceTracker::new(dim, &cfg, cfg.cusum_slow_decay); + let mut rng = SmallRng::seed_from_u64(42); + + let mut snaps: Vec = Vec::with_capacity(total_rounds); + for _ in 0..total_rounds { + let noise = generate_noise(dim, cfg.noise_batch_size, &mut rng); + let report = tracker.observe(&as_slices(&noise), 0, true); + snaps.push(RoundSnap { + baseline_mean: [ + report.scores.novelty.baseline.mean, + report.scores.displacement.baseline.mean, + report.scores.surprise.baseline.mean, + report.scores.coherence.baseline.mean, + ], + baseline_var: [ + report.scores.novelty.baseline.variance, + report.scores.displacement.baseline.variance, + report.scores.surprise.baseline.variance, + report.scores.coherence.baseline.variance, + ], + eta: report.maturity.noise_influence, + }); + } + + // Compute ceiling series for each axis. + let clip = cfg.clip_sigmas; + let mut ceilings: [Vec; 4] = [ + Vec::with_capacity(total_rounds), + Vec::with_capacity(total_rounds), + Vec::with_capacity(total_rounds), + Vec::with_capacity(total_rounds), + ]; + + for snap in &snaps { + for (ax, ceil_vec) in ceilings.iter_mut().enumerate() { + ceil_vec.push(clip.mul_add(snap.baseline_var[ax].sqrt(), snap.baseline_mean[ax])); + } + } + + // Find round where η < 0.05. + let eta_settled = snaps.iter().position(|s| s.eta < 0.05).unwrap_or(total_rounds); + + // Check rolling CV of ceiling after exemption decay. + let window = 20_usize; + let cv_limit = 0.08; // 8% — generous for high-CV axes + + let start_check = eta_settled.max(window); + for (ax, ceil_series) in ceilings.iter().enumerate() { + for round in start_check..=(total_rounds - window) { + let block = &ceil_series[round..round + window]; + let mean = block.iter().sum::() / window as f64; + if mean.abs() < 1e-12 { + continue; + } + let var = block.iter().map(|v| (v - mean).powi(2)).sum::() / window as f64; + let cv = var.sqrt() / mean; + assert!( + cv < cv_limit, + "{}: clip ceiling CV at round {} is {:.2}% (limit {:.0}%) — \ + ceiling is not stabilising after exemption decay", + AXIS_NAMES[ax], + round, + cv * 100.0, + cv_limit * 100.0, + ); + } + } +} + +// ════════════════════════════════════════════════════════════ +// 3. Slow EWMA / CUSUM clip +// ════════════════════════════════════════════════════════════ + +/// The long-memory reference is filtered by the same ceiling as the +/// short-memory baseline, and once the two have been seeded into agreement they +/// stay in agreement across a long run. Clipping therefore adds no bias of its +/// own on the reference side, so a gap between the two can be read as drift +/// rather than as an artefact of filtering. +/// +/// ´claim:clipping:filtering-the-long-memory-reference-does-not-pull-it-away-from-the-short-memory-one´ +/// ´test:crate:slow-ewma-clipping-does-not-bias-cusum-reference´ +#[test] +fn slow_ewma_clipping_does_not_bias_cusum_reference() { + let cfg = cfg_test(); + let dim = 128; + let warmup_rounds = 200; + let post_seed_rounds = 300; + + let mut tracker = SubspaceTracker::new(dim, &cfg, cfg.cusum_slow_decay); + let mut rng = SmallRng::seed_from_u64(42); + + // Noise warm-up. + for _ in 0..warmup_rounds { + let noise = generate_noise(dim, cfg.noise_batch_size, &mut rng); + tracker.observe(&as_slices(&noise), 0, true); + } + + // Seed slow from fast (production workflow). + tracker.seed_cusum_slow_from_baselines(); + tracker.reset_cusum(); + + // Continue noise — at this point the slow EWMA starts from the + // fast EWMA's converged values. Both receive the same `clip_sigmas`. + let mut last_report = None; + for _ in 0..post_seed_rounds { + let noise = generate_noise(dim, cfg.noise_batch_size, &mut rng); + last_report = Some(tracker.observe(&as_slices(&noise), 0, true)); + } + + let report = last_report.unwrap(); + + // Compare fast vs slow baseline means. + // + // After 300 rounds post-seed at `λ_s` = 0.999: + // `λ_s`^300 ≈ 0.741 — slow has only decayed 26% from the seed. + // The fast EWMA at λ = 0.95 has fully forgotten the seed. + // + // So the slow baseline is ~74% seed + ~26% new data, while + // the fast is ~100% new data. Under i.i.d. noise, the seed + // value ≈ the new data's mean, so the gap is mainly from + // stochastic drift. Per-axis tolerances account for this. + // + // novelty: ~2%, displacement ~10%, surprise ~15%, coh ~25% + let tolerances = [0.02, 0.10, 0.15, 0.25]; + let axes = [ + ( + "novelty", + report.scores.novelty.baseline.mean, + report.scores.novelty.cusum.slow_baseline.mean, + ), + ( + "displacement", + report.scores.displacement.baseline.mean, + report.scores.displacement.cusum.slow_baseline.mean, + ), + ( + "surprise", + report.scores.surprise.baseline.mean, + report.scores.surprise.cusum.slow_baseline.mean, + ), + ( + "coherence", + report.scores.coherence.baseline.mean, + report.scores.coherence.cusum.slow_baseline.mean, + ), + ]; + + for ((name, fast, slow), tol) in axes.iter().zip(tolerances.iter()) { + if fast.abs() < 1e-12 { + continue; + } + let rel_diff = (fast - slow).abs() / fast.abs(); + assert!( + rel_diff < *tol, + "{name}: slow EWMA baseline differs from fast by {:.2}%, \ + tolerance {:.0}% — slow-EWMA clipping introduces bias \ + in CUSUM reference", + rel_diff * 100.0, + tol * 100.0, + ); + } +} + +/// A tightening clip does not manufacture evidence of drift: through the whole +/// stretch after seeding, the drift accumulators stay far below anything a host +/// would act on. The moment the clip narrows is when the two baselines are most +/// likely to disagree, so it is where a false alarm would appear if the +/// mechanisms interfered with one another. +/// +/// ´claim:clipping:a-tightening-clip-does-not-manufacture-evidence-of-drift´ +/// ´test:crate:cusum-bounded-through-clip-transitions´ +#[test] +fn cusum_bounded_through_clip_transitions() { + let cfg = cfg_test(); + let dim = 128; + let warmup_rounds = 200; + let transition_rounds = 300; + + let mut tracker = SubspaceTracker::new(dim, &cfg, cfg.cusum_slow_decay); + let mut rng = SmallRng::seed_from_u64(42); + + // Noise warm-up. + for _ in 0..warmup_rounds { + let noise = generate_noise(dim, cfg.noise_batch_size, &mut rng); + tracker.observe(&as_slices(&noise), 0, true); + } + + // Seed and reset (production workflow). + tracker.seed_cusum_slow_from_baselines(); + tracker.reset_cusum(); + + // Continue with noise — the clip width is now near production + // level (η should be small after 200 rounds at λ = 0.95: + // η = 0.95^200 ≈ 3.5e-5). + let mut max_cusum = [0.0_f64; 4]; + for _ in 0..transition_rounds { + let noise = generate_noise(dim, cfg.noise_batch_size, &mut rng); + let report = tracker.observe(&as_slices(&noise), 0, true); + + max_cusum[0] = max_cusum[0].max(report.scores.novelty.cusum.accumulator); + max_cusum[1] = max_cusum[1].max(report.scores.displacement.cusum.accumulator); + max_cusum[2] = max_cusum[2].max(report.scores.surprise.cusum.accumulator); + max_cusum[3] = max_cusum[3].max(report.scores.coherence.cusum.accumulator); + } + + let cusum_limit = 30.0; + for (ax, name) in AXIS_NAMES.iter().enumerate() { + assert!( + max_cusum[ax] < cusum_limit, + "{name}: CUSUM reached {:.2} during post-seed phase \ + (limit {cusum_limit}) — clip transitions causing false drift", + max_cusum[ax], + ); + } +} diff --git a/packages/sentinel/src/tests/convergence_common.rs b/packages/sentinel/src/tests/convergence_common.rs new file mode 100644 index 000000000..70498311a --- /dev/null +++ b/packages/sentinel/src/tests/convergence_common.rs @@ -0,0 +1,711 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +//! # Test index +//! +//! | Test | Area | Claim | +//! |------|------|-------| +//! | [`rolling_mean_empty`] | convergence | Averaging an empty window yields zero rather than a division by zero, so a metric may ask for the mean of a stretch that turned out to hold nothing and still get an answer it can carry forward. | +//! | [`rolling_mean_single`] | convergence | cites (´claim:convergence:the-window-average-is-the-plain-unweighted-mean-of-what-it-covers´) | +//! | [`rolling_mean_known`] | convergence | The window average is the plain unweighted mean of the values it covers, with no decay of its own. The yardstick is deliberately unlike the baselines it measures: an instrument with a memory would judge a long-memory baseline by an equally sluggish standard. | +//! | [`rolling_mean_negative`] | convergence | cites (´claim:convergence:the-window-average-is-the-plain-unweighted-mean-of-what-it-covers´) | +//! | [`settled_all_within_tolerance`] | convergence | cites (´claim:convergence:settling-is-dated-from-the-round-after-the-last-violation´) | +//! | [`settled_never`] | convergence | A trace still violating at its final round is not settled at all, and the answer is an absence rather than a round number. Settling is a claim about the whole remainder of a run, so it cannot be asserted while the run is still moving. | +//! | [`settled_after_specific_round`] | convergence | Settling is dated from the round after the last violation, not from the first round that happened to fall inside tolerance. The search walks backwards from the end, so a trace that strays and returns is credited only from its return. | +//! | [`settled_near_zero_reference_skipped`] | convergence | An axis whose reference level is effectively zero is passed over rather than failed. Tolerance is relative, so a near-zero reference would make every deviation enormous; an axis that never activated is treated as having nothing to say instead of as permanently unconverged. | +//! | [`settled_single_trace`] | convergence | cites (´claim:convergence:settling-is-dated-from-the-round-after-the-last-violation´) | +//! | [`settled_subset_of_axes`] | convergence | Only the axes actually asked about can hold settling back, so a wildly mismatched axis outside the requested set is invisible. The axes mature at very different rates, and a caller may need to know when the ones it depends on have settled without waiting on one it does not use. | +//! | [`converged_too_few_values`] | convergence | A trace too short to hold both a window and a separate reference window yields no verdict better than its own length. With no room to compare an early stretch against a late one, the metric reports that convergence has not been demonstrated rather than guessing that it has. | +//! | [`converged_already_stable`] | convergence | A trace that never departs from its final level is converged from its first round. The reference is the trace's own tail, so this metric answers when a run reached where it ended up — not whether that destination was the right one. | +//! | [`converged_after_transient`] | convergence | A run that starts far from its eventual level is dated as converged after the transient and well before the end: the backward walk stops at the last window whose mean departed from the tail reference. Comparing windows rather than single rounds is what separates a genuine departure from ordinary jitter about a settled level. | +//! | [`converged_near_zero_reference`] | convergence | A trace whose final level is effectively zero is reported as converged from the start rather than divided by. As with the settling metric, an inactive channel is excluded from the judgement instead of poisoning it. | +//! | [`block_mean_late_near_zero`] | convergence | An axis whose late block is effectively zero yields no verdict at all rather than a ratio against nothing. An axis that never activated has no steady state to be stationary about, and saying so is more honest than reporting an enormous relative error. | +//! | [`block_mean_identical_blocks`] | convergence | Stationarity is measured as the relative gap between an early block mean and a late one, so two blocks drawn from the same settled stretch differ by nothing. Averaging over blocks is what lets the test tell a baseline still moving from one merely jittering in place. | +//! | [`block_mean_known_error`] | convergence | cites (´claim:convergence:stationarity-is-the-relative-gap-between-an-early-block-mean-and-a-late-one´) | +//! | [`trailing_cv_too_few`] | convergence | Asking for the jitter of a window longer than the trace yields not a number, rather than a figure computed from whatever happened to be available. A short trace is not a quiet one, and the metric refuses to let the two be confused. | +//! | [`trailing_cv_constant`] | convergence | Jitter is reported as a fraction of the level it sits on, so a flat tail has none whatever that level happens to be. Normalising by the mean is what makes the figure comparable across axes whose scores differ by orders of magnitude. | +//! | [`trailing_cv_known`] | convergence | cites (´claim:convergence:jitter-is-reported-as-a-fraction-of-the-level-it-sits-on´) | +//! | [`generate_noise_shape_and_values`] | noise | Injected noise arrives in exactly the shape the tracker expects, every entry plus or minus a half. Synthetic warm-up traffic therefore carries the same centring the real encoding produces, so a model warmed on noise is warmed on the same kind of thing it will later be asked to judge. | +//! | [`as_slices_preserves_data`] | noise | Handing a generated batch to the tracker borrows it rather than transforming it: the same values arrive in the same order. Nothing is rescaled on the way in, so what a run does is attributable to the noise that was generated for it. | +//! | [`cfg_test_is_valid`] | convergence | The configuration convergence is measured under passes the same validation any production configuration must. Results measured here are therefore statements about a legal sentinel rather than about a corner of the parameter space the crate would refuse to construct. | +//! | [`cfg_b16_overrides_batch_size`] | convergence | The larger-batch variant differs from the standard one in batch size alone and is likewise valid, so a comparison between the two isolates the effect of batch size. Convergence claims made at two batch sizes are then about one system observed differently, not about two systems. | +//! | [`cfg_production_overrides`] | convergence | The production-like variant changes three things together — a longer memory, larger batches and a slower rank cadence — and remains valid. These are the settings the shipped noise schedule is sized from, so they are exercised as a set rather than one at a time. | + +//! Shared ground for the convergence tests: the configurations they run +//! under, the synthetic traffic they run on, and the metrics by which a +//! run is judged to have settled. The traffic is centred half-magnitude +//! noise — the same shape and centring the real encoding produces — so a +//! model warmed here is warmed on the kind of thing it will later judge. +//! +//! The metrics exist because "converged" is not obvious for an +//! exponentially-weighted estimator. Such an estimator never stops moving: +//! its output keeps jittering around the correct level for as long as the +//! input is stochastic, and how much it jitters differs by more than a +//! hundredfold across the four scoring axes. A criterion that waits for +//! the movement to stop would therefore never fire on the noisiest axis, +//! however correct that axis had become. Two answers live here. The +//! settling metric walks backwards from the end of a trace and dates +//! settling from the round after the last violation, so a stray excursion +//! cannot be forgiven by later good behaviour. The block metrics compare +//! an average over an early stretch against an average over a late one, +//! which averages the irreducible jitter down far enough that a baseline +//! still moving can be told from one merely fluctuating in place. +//! +//! Both families decline to judge a channel whose reference level is +//! effectively zero, rather than reporting it as broken. Tolerances are +//! relative, so they are meaningless there, and an axis that never +//! activated has no steady state it could have reached. +//! +//! The configurations come as a small family — a fast standard one, the +//! same at a larger batch size, and one resembling production's longer +//! memory — so that a convergence bound can be established at more than +//! one point and the effect of each setting seen separately. Each is +//! validated in its own right, because a bound measured under a +//! configuration the crate would refuse to build would be worth nothing. +//! +//! # §-references +//! +//! - §ALGO S-4.2 Phase 3 — Latent distribution cold→warm +//! - §ALGO S-6.1.1 — EWMA outlier filter / clipping ceiling +//! - §ALGO S-11.5 — Maturity tracking (noise influence η) +//! - ADR-S-013 — Warm-up convergence benchmark + +use rand::rngs::SmallRng; +use rand::{RngExt, SeedableRng}; + +use crate::config::SentinelConfig; +use crate::sentinel::tracker::SubspaceTracker; + +// ════════════════════════════════════════════════════════════ +// Configs +// ════════════════════════════════════════════════════════════ + +/// Standard test config (λ = 0.95, b = 4). +pub(super) fn cfg_test() -> SentinelConfig { + SentinelConfig { + max_rank: 2, + forgetting_factor: 0.95, + rank_update_interval: 5, + analysis_k: 16, + analysis_depth_cutoff: 6, + energy_threshold: 0.90, + eps: 1e-6, + per_sample_scores: false, + cusum_allowance_sigmas: 0.5, + cusum_slow_decay: 0.999, + cusum_coord_slow_decay: 0.999, + clip_sigmas: 3.0, + clip_pressure_decay: 0.95, + split_threshold: 100, + d_create: 3, + d_evict: 6, + budget: 100_000, + noise_schedule: crate::config::NoiseSchedule::Explicit(vec![5]), + noise_batch_size: 4, + noise_seed: Some(42), + background_warming: false, + svd_strategy: crate::maths::SvdStrategy::Brand, + } +} + +/// Test config with larger batch size (λ = 0.95, b = 16). +pub(super) fn cfg_b16() -> SentinelConfig { + SentinelConfig { + noise_batch_size: 16, + ..cfg_test() + } +} + +/// Production-like config (λ = 0.99, b = 16). +pub(super) fn cfg_production() -> SentinelConfig { + SentinelConfig { + forgetting_factor: 0.99, + rank_update_interval: 100, + noise_batch_size: 16, + noise_schedule: crate::config::NoiseSchedule::Explicit(vec![50]), + ..cfg_test() + } +} + +// ════════════════════════════════════════════════════════════ +// Noise generation +// ════════════════════════════════════════════════════════════ + +/// Generate a batch of random ±0.5 noise vectors. +pub(super) fn generate_noise(dim: usize, batch_size: usize, rng: &mut SmallRng) -> Vec> { + (0..batch_size) + .map(|_| (0..dim).map(|_| if rng.random_bool(0.5) { 0.5 } else { -0.5 }).collect()) + .collect() +} + +/// Convert owned vectors to a slice-of-slices for `tracker.observe()`. +pub(super) fn as_slices(vecs: &[Vec]) -> Vec<&[f64]> { + vecs.iter().map(Vec::as_slice).collect() +} + +// ════════════════════════════════════════════════════════════ +// Axis labels +// ════════════════════════════════════════════════════════════ + +pub(super) const AXIS_NAMES: [&str; 4] = ["novelty", "displacement", "surprise", "coherence"]; + +// ════════════════════════════════════════════════════════════ +// Convergence metrics +// ════════════════════════════════════════════════════════════ + +/// Multi-axis backward-walk convergence: finds the first round from +/// which ALL of the first `axes` channels stay within `tolerance` +/// of `reference` permanently. +pub(super) fn find_settled_round(traces: &[[f64; 4]], reference: &[f64; 4], tolerance: f64, axes: usize) -> Option { + let mut last_violation = None; + for (i, means) in traces.iter().enumerate().rev() { + let all_ok = (0..axes).all(|a| { + let ref_val = reference[a]; + if ref_val.abs() < 1e-12 { + true + } else { + (means[a] - ref_val).abs() < tolerance * ref_val.abs() + } + }); + if !all_ok { + last_violation = Some(i); + break; + } + } + match last_violation { + Some(v) if v + 1 < traces.len() => Some(v + 1), + None => Some(0), + _ => None, + } +} + +/// Windowed-mean convergence metric (ADR-S-013 §3a). +/// +/// Compares a rolling mean of a window-sized block against the +/// reference (rolling mean of the final `window` values). Returns +/// the first round where the metric permanently stays within +/// `tolerance` of the reference. +pub(super) fn find_converged_round(baselines: &[f64], window: usize, tolerance: f64) -> usize { + let n = baselines.len(); + if n < 2 * window { + return n; + } + let reference = rolling_mean(&baselines[n - window..]); + if reference.abs() < 1e-12 { + return 0; + } + for i in (window..n - window).rev() { + let local = rolling_mean(&baselines[i.saturating_sub(window)..i]); + if ((local - reference) / reference).abs() > tolerance { + return i + 1; + } + } + 0 +} + +/// Rolling mean of a slice. +#[allow(clippy::cast_precision_loss)] +pub(super) fn rolling_mean(slice: &[f64]) -> f64 { + if slice.is_empty() { + return 0.0; + } + slice.iter().sum::() / slice.len() as f64 +} + +/// Block-mean stationarity test for a single axis. +/// +/// Compares the mean of traces in `[early_start, early_end)` against +/// `[late_start, late_end)`. Returns `Some(relative_error)` if the +/// late-block mean is non-negligible, or `None` if the late-block mean +/// is effectively zero (axis inactive). +/// +/// The basis is derived here from the EWMA used by the tracker. For +/// stationary, uncorrelated batch-score innovations, the initial-condition +/// bias after `t` updates is proportional to `λ^t`; once that is below the +/// chosen transient tolerance, the expected baseline mean agrees with the +/// score distribution's expectation to the same tolerance. The stationary +/// EWMA has lag-`h` correlation `λ^h`, so a block of `L` consecutive baseline +/// snapshots has relative standard error approximately +/// `CV_EWMA × √((1+λ) / ((1−λ) × L))`. Widely separated blocks have negligible +/// cross-covariance, making the standard error of their difference `√2` times +/// that value. Callers derive their per-axis budgets beside each tolerance +/// from `CV_EWMA`, `λ`, the block length, and an explicit safety multiple. +pub(super) fn block_mean_relative_error( + traces: &[[f64; 4]], + axis: usize, + early_start: usize, + early_end: usize, + late_start: usize, + late_end: usize, +) -> Option { + let early_mean = rolling_mean(&traces[early_start..early_end].iter().map(|t| t[axis]).collect::>()); + let late_mean = rolling_mean(&traces[late_start..late_end].iter().map(|t| t[axis]).collect::>()); + if late_mean.abs() < 1e-12 { + return None; // axis inactive + } + Some((early_mean - late_mean).abs() / late_mean.abs()) +} + +/// Coefficient of variation of the last `window` values. +#[allow(clippy::cast_precision_loss)] +pub(super) fn trailing_cv(baselines: &[f64], window: usize) -> f64 { + let n = baselines.len(); + if n < window { + return f64::NAN; + } + let tail = &baselines[n - window..]; + let mean = rolling_mean(tail); + if mean.abs() < 1e-12 { + return 0.0; + } + let var = tail.iter().map(|v| (v - mean).powi(2)).sum::() / tail.len() as f64; + var.sqrt() / mean +} + +// ════════════════════════════════════════════════════════════ +// Trace collection +// ════════════════════════════════════════════════════════════ + +/// Per-round snapshot of baseline means, score means, and CUSUM state. +#[derive(Clone)] +#[allow(dead_code)] // Fields are for convergence test consumers. +pub(super) struct RoundTrace { + pub baseline_means: [f64; 4], + pub score_means: [f64; 4], + pub cusum_accumulators: [f64; 4], + pub slow_baseline_means: [f64; 4], + pub noise_influence: f64, +} + +/// Run `total_rounds` of noise injection and collect per-round traces. +pub(super) fn run_noise_trace(cfg: &SentinelConfig, dim: usize, total_rounds: usize, seed: u64) -> Vec { + let mut tracker = SubspaceTracker::new(dim, cfg, cfg.cusum_slow_decay); + let mut rng = SmallRng::seed_from_u64(seed); + let mut traces = Vec::with_capacity(total_rounds); + + for _ in 0..total_rounds { + let noise = generate_noise(dim, cfg.noise_batch_size, &mut rng); + let report = tracker.observe(&as_slices(&noise), 0, true); + + traces.push(RoundTrace { + baseline_means: [ + report.scores.novelty.baseline.mean, + report.scores.displacement.baseline.mean, + report.scores.surprise.baseline.mean, + report.scores.coherence.baseline.mean, + ], + score_means: [ + report.scores.novelty.mean, + report.scores.displacement.mean, + report.scores.surprise.mean, + report.scores.coherence.mean, + ], + cusum_accumulators: [ + report.scores.novelty.cusum.accumulator, + report.scores.displacement.cusum.accumulator, + report.scores.surprise.cusum.accumulator, + report.scores.coherence.cusum.accumulator, + ], + slow_baseline_means: [ + report.scores.novelty.cusum.slow_baseline.mean, + report.scores.displacement.cusum.slow_baseline.mean, + report.scores.surprise.cusum.slow_baseline.mean, + report.scores.coherence.cusum.slow_baseline.mean, + ], + noise_influence: report.maturity.noise_influence, + }); + } + + traces +} + +// ════════════════════════════════════════════════════════════ +// Unit tests +// ════════════════════════════════════════════════════════════ + +#[cfg(test)] +mod tests { + use super::*; + + // ── rolling_mean ──────────────────────────────────────── + + /// Averaging an empty window yields zero rather than a division by zero, so + /// a metric may ask for the mean of a stretch that turned out to hold + /// nothing and still get an answer it can carry forward. + /// + /// ´claim:convergence:an-empty-window-averages-to-zero-rather-than-failing´ + /// ´test:crate:rolling-mean-empty´ + #[test] + fn rolling_mean_empty() { + assert!(rolling_mean(&[]).abs() < f64::EPSILON); + } + + /// A window of one is that one value, since averaging cannot smooth what it + /// has seen only once. This pins the degenerate end of the same rule the + /// multi-value case fixes. + /// + /// (´claim:convergence:the-window-average-is-the-plain-unweighted-mean-of-what-it-covers´) + /// ´test:crate:rolling-mean-single´ + #[test] + fn rolling_mean_single() { + assert!((rolling_mean(&[7.0]) - 7.0).abs() < 1e-15); + } + + /// The window average is the plain unweighted mean of the values it covers, + /// with no decay of its own. The yardstick is deliberately unlike the + /// baselines it measures: an instrument with a memory would judge a + /// long-memory baseline by an equally sluggish standard. + /// + /// ´claim:convergence:the-window-average-is-the-plain-unweighted-mean-of-what-it-covers´ + /// ´test:crate:rolling-mean-known´ + #[test] + fn rolling_mean_known() { + // (1 + 2 + 3 + 4) / 4 = 2.5 + assert!((rolling_mean(&[1.0, 2.0, 3.0, 4.0]) - 2.5).abs() < 1e-15); + } + + /// Values below zero are averaged as they stand rather than by magnitude, + /// so a mean can itself come out negative. Convergence metrics compare + /// signed levels, and folding the sign away would make a trace that + /// overshoots look like one that undershoots. + /// + /// (´claim:convergence:the-window-average-is-the-plain-unweighted-mean-of-what-it-covers´) + /// ´test:crate:rolling-mean-negative´ + #[test] + fn rolling_mean_negative() { + // (-3 + 1) / 2 = -1.0 + assert!((rolling_mean(&[-3.0, 1.0]) - (-1.0)).abs() < 1e-15); + } + + // ── find_settled_round ────────────────────────────────── + + /// A trace that never leaves tolerance has no last violation at all, so it + /// counts as settled from its very first round. + /// + /// (´claim:convergence:settling-is-dated-from-the-round-after-the-last-violation´) + /// ´test:crate:settled-all-within-tolerance´ + #[test] + fn settled_all_within_tolerance() { + // All rounds identical to reference → settled from round 0. + let traces = vec![[1.0, 2.0, 3.0, 4.0]; 10]; + let reference = [1.0, 2.0, 3.0, 4.0]; + assert_eq!(find_settled_round(&traces, &reference, 0.05, 4), Some(0)); + } + + /// A trace still violating at its final round is not settled at all, and + /// the answer is an absence rather than a round number. Settling is a claim + /// about the whole remainder of a run, so it cannot be asserted while the + /// run is still moving. + /// + /// ´claim:convergence:a-trace-still-violating-at-its-last-round-is-never-settled´ + /// ´test:crate:settled-never´ + #[test] + fn settled_never() { + // Last round is a violation → cannot settle. + let mut traces = vec![[1.0, 2.0, 3.0, 4.0]; 10]; + traces[9] = [100.0, 2.0, 3.0, 4.0]; // violates axis 0 + assert_eq!(find_settled_round(&traces, &[1.0, 2.0, 3.0, 4.0], 0.05, 4), None); + } + + /// Settling is dated from the round after the last violation, not from the + /// first round that happened to fall inside tolerance. The search walks + /// backwards from the end, so a trace that strays and returns is credited + /// only from its return. + /// + /// ´claim:convergence:settling-is-dated-from-the-round-after-the-last-violation´ + /// ´test:crate:settled-after-specific-round´ + #[test] + fn settled_after_specific_round() { + // Violation at round 3, then stable from round 4 onward. + let mut traces = vec![[1.0, 2.0, 3.0, 4.0]; 10]; + traces[3] = [100.0, 2.0, 3.0, 4.0]; + assert_eq!(find_settled_round(&traces, &[1.0, 2.0, 3.0, 4.0], 0.05, 4), Some(4)); + } + + /// An axis whose reference level is effectively zero is passed over rather + /// than failed. Tolerance is relative, so a near-zero reference would make + /// every deviation enormous; an axis that never activated is treated as + /// having nothing to say instead of as permanently unconverged. + /// + /// ´claim:convergence:an-axis-with-a-near-zero-reference-is-passed-over-rather-than-failed´ + /// ´test:crate:settled-near-zero-reference-skipped´ + #[test] + fn settled_near_zero_reference_skipped() { + // Reference near zero → axis is always OK (skipped). + let traces = vec![[999.0, 0.0, 0.0, 0.0]; 5]; + let reference = [999.0, 0.0, 0.0, 0.0]; + assert_eq!(find_settled_round(&traces, &reference, 0.01, 4), Some(0)); + } + + /// A trace of a single round is settled when that round is within + /// tolerance: the backward walk finds nothing, so the degenerate case falls + /// out of the same rule rather than needing one of its own. + /// + /// (´claim:convergence:settling-is-dated-from-the-round-after-the-last-violation´) + /// ´test:crate:settled-single-trace´ + #[test] + fn settled_single_trace() { + let traces = vec![[1.0, 2.0, 3.0, 4.0]]; + let reference = [1.0, 2.0, 3.0, 4.0]; + assert_eq!(find_settled_round(&traces, &reference, 0.05, 4), Some(0)); + } + + /// Only the axes actually asked about can hold settling back, so a wildly + /// mismatched axis outside the requested set is invisible. The axes mature + /// at very different rates, and a caller may need to know when the ones it + /// depends on have settled without waiting on one it does not use. + /// + /// ´claim:convergence:only-the-axes-asked-about-can-hold-settling-back´ + /// ´test:crate:settled-subset-of-axes´ + #[test] + fn settled_subset_of_axes() { + // Only check first 2 axes; axis 2 huge mismatch is ignored. + let traces = vec![[1.0, 2.0, 999.0, 999.0]; 5]; + let reference = [1.0, 2.0, 3.0, 4.0]; + assert_eq!(find_settled_round(&traces, &reference, 0.05, 2), Some(0)); + } + + // ── find_converged_round ──────────────────────────────── + + /// A trace too short to hold both a window and a separate reference window + /// yields no verdict better than its own length. With no room to compare an + /// early stretch against a late one, the metric reports that convergence + /// has not been demonstrated rather than guessing that it has. + /// + /// ´claim:convergence:a-trace-too-short-for-two-windows-is-reported-as-not-yet-converged´ + /// ´test:crate:converged-too-few-values´ + #[test] + fn converged_too_few_values() { + // n < 2*window → returns n. + let data = vec![1.0; 5]; + assert_eq!(find_converged_round(&data, 10, 0.05), 5); + } + + /// A trace that never departs from its final level is converged from its + /// first round. The reference is the trace's own tail, so this metric + /// answers when a run reached where it ended up — not whether that + /// destination was the right one. + /// + /// ´claim:convergence:convergence-is-measured-against-the-traces-own-tail´ + /// ´test:crate:converged-already-stable´ + #[test] + fn converged_already_stable() { + // Constant input → converged from round 0. + let data = vec![3.0; 100]; + assert_eq!(find_converged_round(&data, 10, 0.05), 0); + } + + /// A run that starts far from its eventual level is dated as converged + /// after the transient and well before the end: the backward walk stops at + /// the last window whose mean departed from the tail reference. Comparing + /// windows rather than single rounds is what separates a genuine departure + /// from ordinary jitter about a settled level. + /// + /// ´claim:convergence:the-converged-round-is-the-one-after-the-last-window-that-departed-from-the-final-level´ + /// ´test:crate:converged-after-transient´ + #[test] + fn converged_after_transient() { + // Big values then settling to 1.0 — must converge after transient. + let mut data = vec![100.0; 20]; + data.extend(vec![1.0; 80]); + let round = find_converged_round(&data, 10, 0.05); + // Must be after the transient region but before the end. + assert!(round > 10, "should detect transient, got {round}"); + assert!(round < 50, "should converge well before end, got {round}"); + } + + /// A trace whose final level is effectively zero is reported as converged + /// from the start rather than divided by. As with the settling metric, an + /// inactive channel is excluded from the judgement instead of poisoning it. + /// + /// ´claim:convergence:a-trace-with-a-near-zero-final-level-is-treated-as-converged-rather-than-divided-by´ + /// ´test:crate:converged-near-zero-reference´ + #[test] + fn converged_near_zero_reference() { + // Near-zero final mean → returns 0. + let data = vec![0.0; 40]; + assert_eq!(find_converged_round(&data, 10, 0.05), 0); + } + + // ── block_mean_relative_error ─────────────────────────── + + /// An axis whose late block is effectively zero yields no verdict at all + /// rather than a ratio against nothing. An axis that never activated has no + /// steady state to be stationary about, and saying so is more honest than + /// reporting an enormous relative error. + /// + /// ´claim:convergence:an-axis-with-no-late-signal-yields-no-stationarity-verdict-at-all´ + /// ´test:crate:block-mean-late-near-zero´ + #[test] + fn block_mean_late_near_zero() { + // Late block mean near zero → axis inactive → None. + let traces = vec![[1.0, 0.0, 0.0, 0.0]; 20]; + assert!(block_mean_relative_error(&traces, 1, 0, 10, 10, 20).is_none()); + } + + /// Stationarity is measured as the relative gap between an early block mean + /// and a late one, so two blocks drawn from the same settled stretch differ + /// by nothing. Averaging over blocks is what lets the test tell a baseline + /// still moving from one merely jittering in place. + /// + /// ´claim:convergence:stationarity-is-the-relative-gap-between-an-early-block-mean-and-a-late-one´ + /// ´test:crate:block-mean-identical-blocks´ + #[test] + fn block_mean_identical_blocks() { + // Identical blocks → error is 0.0. + let traces = vec![[5.0, 5.0, 5.0, 5.0]; 20]; + let err = block_mean_relative_error(&traces, 0, 0, 10, 10, 20).unwrap(); + assert!(err < 1e-15, "expected ~0, got {err}"); + } + + /// The gap is normalised by the late block — the one taken as the reference + /// — so an early level at half the late one reports as a half. This pins + /// the scale of the figure that the per-axis tolerances are set against. + /// + /// (´claim:convergence:stationarity-is-the-relative-gap-between-an-early-block-mean-and-a-late-one´) + /// ´test:crate:block-mean-known-error´ + #[test] + fn block_mean_known_error() { + // Early block mean = 2.0, late block mean = 4.0 → error = 0.5. + let mut traces = Vec::new(); + for _ in 0..10 { + traces.push([2.0, 0.0, 0.0, 0.0]); + } + for _ in 0..10 { + traces.push([4.0, 0.0, 0.0, 0.0]); + } + let err = block_mean_relative_error(&traces, 0, 0, 10, 10, 20).unwrap(); + assert!((err - 0.5).abs() < 1e-15, "expected 0.5, got {err}"); + } + + // ── trailing_cv ───────────────────────────────────────── + + /// Asking for the jitter of a window longer than the trace yields not a + /// number, rather than a figure computed from whatever happened to be + /// available. A short trace is not a quiet one, and the metric refuses to + /// let the two be confused. + /// + /// ´claim:convergence:a-window-longer-than-the-trace-yields-no-jitter-figure-at-all´ + /// ´test:crate:trailing-cv-too-few´ + #[test] + fn trailing_cv_too_few() { + let data = vec![1.0; 3]; + assert!(trailing_cv(&data, 10).is_nan()); + } + + /// Jitter is reported as a fraction of the level it sits on, so a flat tail + /// has none whatever that level happens to be. Normalising by the mean is + /// what makes the figure comparable across axes whose scores differ by + /// orders of magnitude. + /// + /// ´claim:convergence:jitter-is-reported-as-a-fraction-of-the-level-it-sits-on´ + /// ´test:crate:trailing-cv-constant´ + #[test] + fn trailing_cv_constant() { + let data = vec![5.0; 20]; + assert!((trailing_cv(&data, 10)).abs() < 1e-15); + } + + /// A spread measured against a mean several times larger reports as that + /// ratio and not as the deviation itself, which pins the normalisation the + /// statement asserts. + /// + /// (´claim:convergence:jitter-is-reported-as-a-fraction-of-the-level-it-sits-on´) + /// ´test:crate:trailing-cv-known´ + #[test] + fn trailing_cv_known() { + // std([1,2,3,4,5]) / mean([1,2,3,4,5]) + // mean = 3.0, var = (4+1+0+1+4)/5 = 2.0, std = √2 + // CV = √2 / 3 ≈ 0.4714 + let data = vec![1.0, 2.0, 3.0, 4.0, 5.0]; + let cv = trailing_cv(&data, 5); + let expected = 2.0_f64.sqrt() / 3.0; + assert!((cv - expected).abs() < 1e-10, "expected {expected}, got {cv}"); + } + + // ── generate_noise / as_slices ────────────────────────── + + /// Injected noise arrives in exactly the shape the tracker expects, every + /// entry plus or minus a half. Synthetic warm-up traffic therefore carries + /// the same centring the real encoding produces, so a model warmed on noise + /// is warmed on the same kind of thing it will later be asked to judge. + /// + /// ´claim:noise:injected-noise-is-shaped-and-centred-exactly-like-real-observations´ + /// ´test:crate:generate-noise-shape-and-values´ + #[test] + fn generate_noise_shape_and_values() { + let mut rng = SmallRng::seed_from_u64(42); + let noise = generate_noise(8, 5, &mut rng); + assert_eq!(noise.len(), 5); + for row in &noise { + assert_eq!(row.len(), 8); + for &v in row { + assert!( + (v - 0.5).abs() < f64::EPSILON || (v + 0.5).abs() < f64::EPSILON, + "unexpected value {v}" + ); + } + } + } + + /// Handing a generated batch to the tracker borrows it rather than + /// transforming it: the same values arrive in the same order. Nothing is + /// rescaled on the way in, so what a run does is attributable to the noise + /// that was generated for it. + /// + /// ´claim:noise:handing-a-generated-batch-to-the-tracker-borrows-it-without-altering-it´ + /// ´test:crate:as-slices-preserves-data´ + #[test] + fn as_slices_preserves_data() { + let vecs = vec![vec![1.0, 2.0], vec![3.0, 4.0]]; + let slices = as_slices(&vecs); + assert_eq!(slices.len(), 2); + assert_eq!(slices[0], &[1.0, 2.0]); + assert_eq!(slices[1], &[3.0, 4.0]); + } + + // ── config constructors ───────────────────────────────── + + /// The configuration convergence is measured under passes the same + /// validation any production configuration must. Results measured here are + /// therefore statements about a legal sentinel rather than about a corner + /// of the parameter space the crate would refuse to construct. + /// + /// ´claim:convergence:the-configuration-convergence-is-measured-under-is-one-the-crate-would-accept´ + /// ´test:crate:cfg-test-is-valid´ + #[test] + fn cfg_test_is_valid() { + let cfg = cfg_test(); + assert!(cfg.validate().is_ok(), "cfg_test() must pass validation"); + } + + /// The larger-batch variant differs from the standard one in batch size + /// alone and is likewise valid, so a comparison between the two isolates + /// the effect of batch size. Convergence claims made at two batch sizes are + /// then about one system observed differently, not about two systems. + /// + /// ´claim:convergence:the-larger-batch-variant-differs-in-batch-size-alone´ + /// ´test:crate:cfg-b16-overrides-batch-size´ + #[test] + fn cfg_b16_overrides_batch_size() { + let cfg = cfg_b16(); + assert_eq!(cfg.noise_batch_size, 16); + assert!(cfg.validate().is_ok(), "cfg_b16() must pass validation"); + } + + /// The production-like variant changes three things together — a longer + /// memory, larger batches and a slower rank cadence — and remains valid. + /// These are the settings the shipped noise schedule is sized from, so they + /// are exercised as a set rather than one at a time. + /// + /// ´claim:convergence:the-production-like-variant-changes-memory-batch-size-and-rank-cadence-together´ + /// ´test:crate:cfg-production-overrides´ + #[test] + fn cfg_production_overrides() { + let cfg = cfg_production(); + assert!((cfg.forgetting_factor - 0.99).abs() < 1e-15); + assert_eq!(cfg.noise_batch_size, 16); + assert_eq!(cfg.rank_update_interval, 100); + assert!(cfg.validate().is_ok(), "cfg_production() must pass validation"); + } +} diff --git a/packages/sentinel/src/tests/convergence_diagnostics.rs b/packages/sentinel/src/tests/convergence_diagnostics.rs new file mode 100644 index 000000000..d8594b0c9 --- /dev/null +++ b/packages/sentinel/src/tests/convergence_diagnostics.rs @@ -0,0 +1,520 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +//! # Test index +//! +//! | Test | Area | Claim | +//! |------|------|-------| +//! | [`convergence_rounds_table`] | convergence | How many noise rounds each axis needs, and what those rounds cost, can be re-derived on demand rather than taken on trust from the schedule that ships. The diagnostic asserts nothing: it exists so that a change in the scoring pipeline can be checked against the round budget the schedule was sized from. | +//! | [`axis_drift_investigation`] | convergence | When an axis refuses to settle, a per-round trace of its scores, baseline, spread, clip pressure and ceiling is available beside the theory it is supposed to follow. Diagnosing a convergence failure means finding where the empirical trajectory leaves the predicted one, and that needs the trajectory itself rather than a pass or a fail. | +//! | [`svd_timing_comparison`] | convergence | The cost of a warm-up run can be measured with the tracing layer installed and again without it, so the instrument's own overhead is separable from what it measures. A timing figure used to size the noise schedule would otherwise silently include the cost of having taken it. | + +#![allow(clippy::print_stderr)] + +//! On-demand **convergence diagnostics** (run with `--ignored`). +//! +//! These tests produce detailed diagnostic tables for convergence +//! analysis and SVD timing. They have no assertions — they exist +//! to print human-readable tables when investigating performance +//! or tuning the noise schedule (`noise_schedule.rounds_for_depth()`). +//! +//! Ported from `convergence_benchmark.rs` tests that were pure +//! `eprintln!` diagnostic dumps with no assertions: +//! +//! - `noise_baselines_converge` → [`convergence_rounds_table`] +//! - `derived_noise_rounds_table` → (merged into `convergence_rounds_table`) +//! - `wall_clock_convergence_cost` → criterion `warmup_cost_detailed` group +//! - `svd_timing_diagnostic` → [`svd_timing_comparison`] +//! - (new) [`axis_drift_investigation`] — added for ADR-S-013 §1a +//! convergence-failure triage +//! +//! They are instruments rather than judgements, which is why they assert +//! nothing and are excluded from ordinary runs. A pass or a fail answers +//! whether a bound still holds; sizing the noise schedule, or working out +//! why an axis will not settle, needs the trajectory itself — the round at +//! which each axis converged, the cost of getting there, and the +//! per-round march of scores, baselines, spreads and ceilings against the +//! theory they are supposed to follow. Keeping the measurement separate +//! from the assertion also keeps a slow diagnostic out of the gate. +//! +//! # Running +//! +//! Run a single diagnostic: +//! +//! ```sh +//! cargo test -p torrust-sentinel -- --ignored convergence_rounds_table --nocapture +//! cargo test -p torrust-sentinel -- --ignored axis_drift_investigation --nocapture +//! cargo test -p torrust-sentinel -- --ignored svd_timing_comparison --nocapture +//! ``` +//! +//! Or all at once: +//! +//! ```sh +//! cargo test -p torrust-sentinel convergence_diagnostics -- --ignored --nocapture +//! ``` +//! +//! # §-references +//! +//! - §ALGO S-4.2 Phase 3 — Latent distribution cold→warm +//! - §ALGO S-6.1.1 — EWMA outlier filter / clipping ceiling +//! - ADR-S-013 — Warm-up convergence benchmark +//! - ADR-M-028 — Span-native tracing + +use std::time::Instant; + +use rand::SeedableRng; +use rand::rngs::SmallRng; + +use super::convergence_common::{ + AXIS_NAMES, as_slices, cfg_production, cfg_test, find_converged_round, generate_noise, trailing_cv, +}; +use crate::maths::bench_tracing::SpanTiming; +use crate::sentinel::tracker::SubspaceTracker; + +// ════════════════════════════════════════════════════════════ +// Convergence-rounds table +// ════════════════════════════════════════════════════════════ + +/// How many noise rounds each axis needs, and what those rounds cost, can be +/// re-derived on demand rather than taken on trust from the schedule that +/// ships. The diagnostic asserts nothing: it exists so that a change in the +/// scoring pipeline can be checked against the round budget the schedule was +/// sized from. +/// +/// ´claim:convergence:the-round-budget-behind-the-noise-schedule-can-be-re-derived-on-demand´ +/// ´test:crate:convergence-rounds-table´ +#[test] +#[ignore = "on-demand diagnostic — run with --ignored --nocapture"] +#[allow( + clippy::cast_precision_loss, + clippy::cast_possible_truncation, + clippy::cast_sign_loss, + clippy::similar_names, + clippy::too_many_lines +)] +fn convergence_rounds_table() { + let dim = 128; + let reference_rounds = 500; + let window = 20; + let tolerances = [0.01, 0.10, 0.20, 0.20]; // nov, disp, surp, coh + let (timing, _guard) = SpanTiming::install(); + + eprintln!("\n╔══════════════════════════════════════════════════════════╗"); + eprintln!("║ Convergence-rounds diagnostic (ADR-S-013 §3–§4) ║"); + eprintln!("╚══════════════════════════════════════════════════════════╝"); + + for (label, base_cfg) in [ + ("test (λ=0.95, b=4)", cfg_test()), + ("production (λ=0.99, b=16)", cfg_production()), + ] { + let batch_size = base_cfg.noise_batch_size; + let lambda = base_cfg.forgetting_factor; + + // ── Collect baseline traces ────────────────────── + timing.reset(); + let mut tracker = SubspaceTracker::new(dim, &base_cfg, base_cfg.cusum_slow_decay); + let mut rng = SmallRng::seed_from_u64(42); + + let mut baseline_traces: [Vec; 4] = [ + Vec::with_capacity(reference_rounds), + Vec::with_capacity(reference_rounds), + Vec::with_capacity(reference_rounds), + Vec::with_capacity(reference_rounds), + ]; + + for _ in 0..reference_rounds { + let noise = generate_noise(dim, batch_size, &mut rng); + let report = tracker.observe(&as_slices(&noise), 0, true); + + baseline_traces[0].push(report.scores.novelty.baseline.mean); + baseline_traces[1].push(report.scores.displacement.baseline.mean); + baseline_traces[2].push(report.scores.surprise.baseline.mean); + baseline_traces[3].push(report.scores.coherence.baseline.mean); + } + + // ── SVD per-round cost ─────────────────────────── + let brand_per_round_us = timing.total_ns("svd_brand") as f64 / 1_000.0 / reference_rounds as f64; + let naive_ns = timing.total_ns("svd_naive"); + let naive_per_round_us = if naive_ns > 0 { + naive_ns as f64 / 1_000.0 / reference_rounds as f64 + } else { + 0.0 + }; + + eprintln!("\n [{label}] — {reference_rounds} rounds, d={dim}, b={batch_size}"); + eprintln!(" SVD per-round: Brand {brand_per_round_us:.1} µs"); + if naive_ns > 0 { + let speedup = naive_per_round_us / brand_per_round_us; + eprintln!(" SVD per-round: Naïve {naive_per_round_us:.1} µs ({speedup:.2}× speedup)"); + } + + // ── Candidate rounds table ────────────────────── + eprintln!(); + if naive_ns > 0 { + eprintln!(" noise_rds | conv? | nov | disp | surp | coh | Brand (ms) | Naïve (ms)"); + eprintln!(" ----------|-------|------|------|------|------|------------|----------"); + } else { + eprintln!(" noise_rds | conv? | nov | disp | surp | coh | Brand (ms)"); + eprintln!(" ----------|-------|------|------|------|------|----------"); + } + + for candidate in [5, 10, 20, 50, 65, 100, 150, 200, 400] { + if candidate > reference_rounds { + continue; + } + + let mut all_converged = true; + let mut axis_rounds = [0usize; 4]; + + for i in 0..4 { + let slice = &baseline_traces[i][..candidate]; + let conv = find_converged_round(slice, window.min(candidate / 2), tolerances[i]); + axis_rounds[i] = conv; + if conv >= candidate.saturating_sub(window) { + all_converged = false; + } + } + + let brand_cost_ms = candidate as f64 * brand_per_round_us / 1000.0; + let status = if all_converged { "yes" } else { "NO " }; + if naive_ns > 0 { + let naive_cost_ms = candidate as f64 * naive_per_round_us / 1000.0; + eprintln!( + " {candidate:9} | {status:5} | {:4} | {:4} | {:4} | {:4} | {brand_cost_ms:10.1} | {naive_cost_ms:10.1}", + axis_rounds[0], axis_rounds[1], axis_rounds[2], axis_rounds[3], + ); + } else { + eprintln!( + " {candidate:9} | {status:5} | {:4} | {:4} | {:4} | {:4} | {brand_cost_ms:10.1}", + axis_rounds[0], axis_rounds[1], axis_rounds[2], axis_rounds[3], + ); + } + } + + // ── Derived recommended rounds ─────────────────── + let mut axis_convergence = [0usize; 4]; + for (i, trace) in baseline_traces.iter().enumerate() { + axis_convergence[i] = find_converged_round(trace, window, tolerances[i]); + } + let worst = *axis_convergence.iter().max().unwrap(); + let theoretical_eta_05 = 0.05_f64.log(lambda).ceil() as usize; + + eprintln!(); + eprintln!(" axis | converged | CV (last 100) | tolerance"); + eprintln!(" -------------|-----------|---------------|----------"); + for (i, name) in AXIS_NAMES.iter().enumerate() { + let cv = trailing_cv(&baseline_traces[i], 100) * 100.0; + eprintln!( + " {name:12} | {:9} | {cv:12.4}% | {:8}%", + axis_convergence[i], + tolerances[i] * 100.0 + ); + } + + let recommended = worst.max(theoretical_eta_05); + let with_margin = (recommended as f64 * 1.5).ceil() as usize; + let brand_per_round_ms = brand_per_round_us / 1000.0; + + eprintln!(); + eprintln!(" theoretical η < 0.05: {theoretical_eta_05} rounds"); + eprintln!(" worst-case axis: {worst} rounds"); + eprintln!( + " recommended: {recommended} rounds ({:.1} ms Brand)", + recommended as f64 * brand_per_round_ms + ); + eprintln!( + " + 50% margin: {with_margin} rounds ({:.1} ms Brand)", + with_margin as f64 * brand_per_round_ms + ); + eprintln!( + " current default: {} rounds ({:.1} ms Brand)", + base_cfg.noise_schedule.rounds_for_depth(0), + f64::from(base_cfg.noise_schedule.rounds_for_depth(0)) * brand_per_round_ms + ); + + let ratio = recommended as f64 / f64::from(base_cfg.noise_schedule.rounds_for_depth(0)); + if ratio > 1.0 { + eprintln!(" ⚠ current default is {ratio:.1}× too low!"); + } else { + eprintln!(" ✓ current default is sufficient ({ratio:.1}× of needed)"); + } + } +} + +// ════════════════════════════════════════════════════════════ +// Per-axis drift investigation +// ════════════════════════════════════════════════════════════ + +/// When an axis refuses to settle, a per-round trace of its scores, baseline, +/// spread, clip pressure and ceiling is available beside the theory it is +/// supposed to follow. Diagnosing a convergence failure means finding where the +/// empirical trajectory leaves the predicted one, and that needs the trajectory +/// itself rather than a pass or a fail. +/// +/// ´claim:convergence:a-per-round-trace-is-available-when-an-axis-refuses-to-settle´ +/// ´test:crate:axis-drift-investigation´ +#[test] +#[ignore = "on-demand diagnostic — run with --ignored --nocapture"] +#[allow(clippy::cast_precision_loss, clippy::too_many_lines)] +fn axis_drift_investigation() { + let cfg = cfg_test(); + let dim = 128; + let total_rounds = 500; + let mut tracker = SubspaceTracker::new(dim, &cfg, cfg.cusum_slow_decay); + let mut rng = SmallRng::seed_from_u64(42); + + // Collect full traces: [round][axis] for baseline means, score means, + // baseline variances, and clip-pressure EWMAs. + let mut bl_means: Vec<[f64; 4]> = Vec::with_capacity(total_rounds); + let mut sc_means: Vec<[f64; 4]> = Vec::with_capacity(total_rounds); + let mut bl_vars: Vec<[f64; 4]> = Vec::with_capacity(total_rounds); + let mut clip_pressures: Vec<[f64; 4]> = Vec::with_capacity(total_rounds); + let mut ranks: Vec = Vec::with_capacity(total_rounds); + let mut etas: Vec = Vec::with_capacity(total_rounds); + + for _ in 0..total_rounds { + let noise = generate_noise(dim, cfg.noise_batch_size, &mut rng); + let report = tracker.observe(&as_slices(&noise), 0, true); + + bl_means.push([ + report.scores.novelty.baseline.mean, + report.scores.displacement.baseline.mean, + report.scores.surprise.baseline.mean, + report.scores.coherence.baseline.mean, + ]); + sc_means.push([ + report.scores.novelty.mean, + report.scores.displacement.mean, + report.scores.surprise.mean, + report.scores.coherence.mean, + ]); + bl_vars.push([ + report.scores.novelty.baseline.variance, + report.scores.displacement.baseline.variance, + report.scores.surprise.baseline.variance, + report.scores.coherence.baseline.variance, + ]); + clip_pressures.push([ + report.scores.novelty.clip_pressure, + report.scores.displacement.clip_pressure, + report.scores.surprise.clip_pressure, + report.scores.coherence.clip_pressure, + ]); + ranks.push(report.rank); + etas.push(report.maturity.noise_influence); + } + + eprintln!("\n╔══════════════════════════════════════════════════════════════════════════════════╗"); + eprintln!( + "║ Axis drift investigation (λ={:.2}, b={}, d={dim}) ║", + cfg.forgetting_factor, cfg.noise_batch_size + ); + eprintln!("╚══════════════════════════════════════════════════════════════════════════════════╝"); + + // ── Per-axis detailed table ────────────────────────── + for (ax, name) in AXIS_NAMES.iter().enumerate() { + eprintln!("\n ─── {name} ───"); + eprintln!( + " round | rank | η | ρ̄ | score_mean | bl_mean | bl_var | clip_ceil | Δbl/bl (%)" + ); + eprintln!(" ------|------|----------|----------|--------------|--------------|--------------|--------------|----------"); + + let mut prev_bl = f64::NAN; + for r in 0..total_rounds { + if r < 30 || r % 10 == 0 || r == total_rounds - 1 { + let eta = etas[r]; + let cp = clip_pressures[r][ax]; + let p = eta.max(cp); + let eff_clip = cfg.clip_sigmas * (1.0 + p / (1.0 - p + cfg.eps)); + let clip_ceil = eff_clip.mul_add(bl_vars[r][ax].sqrt(), bl_means[r][ax]); + let delta_pct = if prev_bl.is_nan() || prev_bl.abs() < 1e-12 { + 0.0 + } else { + (bl_means[r][ax] - prev_bl) / prev_bl.abs() * 100.0 + }; + eprintln!( + " {:5} | {:4} | {:.6} | {:.6} | {:12.8} | {:12.8} | {:12.8} | {:12.4} | {:+8.4}", + r, ranks[r], eta, cp, sc_means[r][ax], bl_means[r][ax], bl_vars[r][ax], clip_ceil, delta_pct + ); + prev_bl = bl_means[r][ax]; + } + } + + // Stationarity test: compare first-half and second-half means + // of the score means (not baselines) post rank stabilisation. + let post_rank = 30; // well after rank reaches 2 + let mid = usize::midpoint(post_rank, total_rounds); + let first_half_mean: f64 = sc_means[post_rank..mid].iter().map(|s| s[ax]).sum::() / (mid - post_rank) as f64; + let second_half_mean: f64 = sc_means[mid..].iter().map(|s| s[ax]).sum::() / (total_rounds - mid) as f64; + let score_drift_pct = if first_half_mean.abs() > 1e-12 { + (second_half_mean - first_half_mean) / first_half_mean * 100.0 + } else { + 0.0 + }; + + // Same for baselines + let first_half_bl: f64 = bl_means[post_rank..mid].iter().map(|s| s[ax]).sum::() / (mid - post_rank) as f64; + let second_half_bl: f64 = bl_means[mid..].iter().map(|s| s[ax]).sum::() / (total_rounds - mid) as f64; + let bl_drift_pct = if first_half_bl.abs() > 1e-12 { + (second_half_bl - first_half_bl) / first_half_bl * 100.0 + } else { + 0.0 + }; + + let cv = trailing_cv(&bl_means.iter().map(|b| b[ax]).collect::>(), 100) * 100.0; + + eprintln!(); + eprintln!(" score mean drift (half1 vs half2): {score_drift_pct:+.4}%"); + eprintln!(" baseline drift (half1 vs half2): {bl_drift_pct:+.4}%"); + eprintln!(" baseline CV (last 100 rounds): {cv:.4}%"); + } + + // ── find_settled_round analysis ────────────────────── + // Reproduce the failing test's methodology and show which axis/round causes failure. + eprintln!("\n ─── find_settled_round breakdown (5% tol, reference = last round) ───"); + let reference = *bl_means.last().unwrap(); + for axes_count in [3, 4] { + eprintln!("\n checking {axes_count} axes:"); + for (i, means) in bl_means.iter().enumerate().rev() { + let violations: Vec = (0..axes_count) + .filter_map(|a| { + let ref_val = reference[a]; + if ref_val.abs() < 1e-12 { + None + } else { + let err = (means[a] - ref_val).abs() / ref_val.abs(); + if err >= 0.05 { + Some(format!("{}={:.4}%", AXIS_NAMES[a], err * 100.0)) + } else { + None + } + } + }) + .collect(); + if !violations.is_empty() { + eprintln!(" last violation at round {i}: {}", violations.join(", ")); + // Show 3 rounds around the violation + let start = i.saturating_sub(2); + let end = (i + 2).min(total_rounds - 1); + for (offset, bl_mean) in bl_means[start..=end].iter().enumerate() { + let r = start + offset; + let errs: Vec = (0..axes_count) + .map(|a| { + let ref_val = reference[a]; + let err = if ref_val.abs() < 1e-12 { + 0.0 + } else { + (bl_mean[a] - ref_val) / ref_val * 100.0 + }; + format!("{}={:+.3}%", AXIS_NAMES[a], err) + }) + .collect(); + let marker = if r == i { " ← violation" } else { "" }; + eprintln!(" round {:3}: {}{marker}", r, errs.join(", ")); + } + break; + } + } + } +} + +// ════════════════════════════════════════════════════════════ +// SVD timing comparison +// ════════════════════════════════════════════════════════════ + +/// The cost of a warm-up run can be measured with the tracing layer installed +/// and again without it, so the instrument's own overhead is separable from +/// what it measures. A timing figure used to size the noise schedule would +/// otherwise silently include the cost of having taken it. +/// +/// ´claim:convergence:warm-up-cost-can-be-measured-apart-from-the-cost-of-measuring-it´ +/// ´test:crate:svd-timing-comparison´ +#[test] +#[ignore = "on-demand diagnostic — run with --ignored --nocapture"] +#[allow(clippy::cast_precision_loss)] +fn svd_timing_comparison() { + let dim = 128; + let rounds = 500; + let n_reps = 5; + + eprintln!("\n╔══════════════════════════════════════════════════════════╗"); + eprintln!("║ SVD timing diagnostic: SpanTiming vs raw Instant ║"); + eprintln!("╚══════════════════════════════════════════════════════════╝"); + eprintln!(" cfg!(debug_assertions) = {}", cfg!(debug_assertions)); + + for (label, base_cfg) in [ + ("test (λ=0.95, b=4)", cfg_test()), + ("production (λ=0.99, b=16)", cfg_production()), + ] { + let batch_size = base_cfg.noise_batch_size; + + // Warm up: 2 full runs to stabilise CPU frequency. + for _ in 0..2 { + let mut t = SubspaceTracker::new(dim, &base_cfg, base_cfg.cusum_slow_decay); + let mut rng = SmallRng::seed_from_u64(42); + for _ in 0..rounds { + let noise = generate_noise(dim, batch_size, &mut rng); + t.observe(&as_slices(&noise), 0, true); + } + } + + // ── A: WITH subscriber (SpanTimingLayer) ──────── + let mut span_brand_sum = 0u128; + let mut span_naive_sum = 0u128; + let mut span_wall_sum = 0.0_f64; + + for _ in 0..n_reps { + let (timing, guard) = SpanTiming::install(); + timing.reset(); + let wall_start = Instant::now(); + { + let mut t = SubspaceTracker::new(dim, &base_cfg, base_cfg.cusum_slow_decay); + let mut rng = SmallRng::seed_from_u64(42); + for _ in 0..rounds { + let noise = generate_noise(dim, batch_size, &mut rng); + t.observe(&as_slices(&noise), 0, true); + } + } + span_wall_sum = wall_start.elapsed().as_secs_f64().mul_add(1000.0, span_wall_sum); + span_brand_sum += timing.total_ns("svd_brand"); + span_naive_sum += timing.total_ns("svd_naive"); + drop(guard); + } + + let oracle_on = span_naive_sum > 0; + let span_wall = span_wall_sum / f64::from(n_reps); + let span_brand = span_brand_sum as f64 / f64::from(n_reps) / 1_000_000.0; + let span_naive = span_naive_sum as f64 / f64::from(n_reps) / 1_000_000.0; + + // ── B: WITHOUT subscriber ─────────────────────── + let mut bare_sum = 0.0_f64; + + for _ in 0..n_reps { + let wall_start = Instant::now(); + { + let mut t = SubspaceTracker::new(dim, &base_cfg, base_cfg.cusum_slow_decay); + let mut rng = SmallRng::seed_from_u64(42); + for _ in 0..rounds { + let noise = generate_noise(dim, batch_size, &mut rng); + t.observe(&as_slices(&noise), 0, true); + } + } + bare_sum = wall_start.elapsed().as_secs_f64().mul_add(1000.0, bare_sum); + } + + let bare_wall = bare_sum / f64::from(n_reps); + + eprintln!("\n [{label}] — {rounds} rounds, d={dim}, b={batch_size}, {n_reps} reps each"); + eprintln!(" oracle active: {oracle_on}"); + eprintln!(" WITH subscriber (avg of {n_reps}):"); + eprintln!(" wall-clock: {span_wall:.2} ms"); + eprintln!(" span(brand): {span_brand:.2} ms"); + if oracle_on { + eprintln!(" span(naive): {span_naive:.2} ms"); + } + eprintln!(" WITHOUT subscriber (avg of {n_reps}):"); + eprintln!(" wall-clock: {bare_wall:.2} ms"); + eprintln!(" span(brand) / bare = {:.2}×", span_brand / bare_wall); + eprintln!(" sub-wall / bare = {:.2}×", span_wall / bare_wall); + } +} diff --git a/packages/sentinel/src/tests/convergence_eta.rs b/packages/sentinel/src/tests/convergence_eta.rs new file mode 100644 index 000000000..961e40b78 --- /dev/null +++ b/packages/sentinel/src/tests/convergence_eta.rs @@ -0,0 +1,406 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +//! How much of what a tracker knows came out of a bottle. +//! +//! A cell is warmed on synthetic noise before it ever judges real traffic, +//! and the noise share η is the tracker's own estimate of how much of its +//! state that warming still accounts for. It uses the same exponential +//! forgetting cadence as the model: an injected batch pulls the share toward +//! one and a real batch pulls it toward zero. Observation counters remain +//! sample counts, while the influence states how much of the batch-updated +//! model is still synthetic. +//! +//! Three properties make it usable rather than merely descriptive. It is +//! a proportion under every workload, so a consumer never has to guard +//! against a value that is not a fraction. It moves in one direction per +//! kind of input, so a threshold crossing means the same thing whenever it +//! happens — which is what lets the clip exemption and the end of warm-up +//! be keyed to it. It also matches its batch-indexed closed form exactly and +//! is independent of the number of samples within each batch. +//! +//! The observation counters run alongside and answer a different question. +//! They tally samples of each kind and never decay, so a host can still +//! see how much of a model's experience was synthetic long after the share +//! itself has been forgotten. +//! +//! # §-references +//! +//! - §ALGO S-11.5 — Maturity tracking (noise influence η) +//! - ADR-S-013 — Warm-up convergence benchmark +//! +//! # Test index +//! +//! | Test | Area | Claim | +//! |------|------|-------| +//! | [`eta_starts_at_one_for_cold_tracker`] | noise | A tracker that has seen nothing counts as entirely noise-taught. Everything it will learn first comes from injected traffic, so the honest starting position is that none of its state yet reflects real observations. | +//! | [`counters_start_at_zero_for_cold_tracker`] | noise | A fresh tracker claims no observations of either kind, and the total is exactly the two counts together. The counters are a record of what was fed in rather than an estimate, so nothing may be presumed before anything arrives. | +//! | [`eta_tracks_theory_exactly`] | noise | After each real batch the noise share equals its initial value times the model's forgetting factor raised to the number of batches, exactly matching the state it measures. | +//! | [`eta_decay_is_independent_of_batch_size`] | noise | One-sample and sixteen-sample batches apply the same decay to noise influence because each causes one model update. The observation counters still record their different sample counts. | +//! | [`eta_maturity_threshold_uses_model_batch_count`] | noise | The maturity threshold is crossed on the first batch for which the model's repeated forgetting factor takes influence below the threshold, with the count derived from that recurrence. | +//! | [`eta_decreases_monotonically_under_real_data`] | noise | Every real batch lowers the noise share and none raises it, so warm-up influence is spent and never regained by ordinary operation. Monotonicity is what makes the share usable as a maturity signal: a threshold crossing means the same thing whenever it happens. | +//! | [`eta_increases_monotonically_under_noise`] | noise | Injection pushes the share back up from wherever real data drove it, batch by batch and without reversal. A cell whose model is re-warmed is therefore re-declared immature rather than left claiming a maturity its state no longer has. | +//! | [`eta_stays_in_unit_interval`] | noise | The share is a proportion and stays one under any interleaving of injected and real batches. Both updates are convex steps toward an endpoint inside the interval, so no mixture of workloads can carry it out of range and no consumer has to guard against a value that is not a fraction. | +//! | [`eta_converges_to_one_under_noise`] | noise | Indefinite injection holds the share at exactly one, its fixed point: noise cannot make a model more than entirely noise-taught. A long warm-up therefore has a stable end state rather than an accumulating one. | +//! | [`eta_decays_toward_zero_under_real_only`] | noise | A long enough run of real batches drives the share below the maturity threshold, so warm-up is eventually forgotten on the same schedule as the model. | +//! | [`observation_counters_mixed_sequence`] | noise | The counters tally samples rather than batches and keep the two kinds apart: a stretch of injection moves only the noise count, a stretch of real traffic only the other, and the total is their sum. A host can therefore still tell how much of a model's experience was synthetic long after the noise share itself has decayed away. | +//! | [`counters_track_real_only_sequence`] | noise | cites (´claim:noise:the-counters-tally-samples-not-batches-and-keep-the-two-kinds-apart´) | + +use rand::SeedableRng; +use rand::rngs::SmallRng; + +use super::convergence_common::{as_slices, cfg_test, generate_noise}; +use crate::sentinel::tracker::SubspaceTracker; + +const MATURITY_THRESHOLD: f64 = 0.01; + +fn decay_crossing(initial: f64, lambda: f64) -> (usize, f64) { + (1_usize..=usize::MAX) + .scan(initial, |influence, batch| { + *influence *= lambda; + Some((batch, *influence)) + }) + .find(|(_, influence)| *influence < MATURITY_THRESHOLD) + .expect("a validated forgetting factor must cross the maturity threshold") +} + +// ════════════════════════════════════════════════════════════ +// Initial conditions +// ════════════════════════════════════════════════════════════ + +/// A tracker that has seen nothing counts as entirely noise-taught. Everything +/// it will learn first comes from injected traffic, so the honest starting +/// position is that none of its state yet reflects real observations. +/// +/// ´claim:noise:a-tracker-that-has-seen-nothing-counts-as-entirely-noise-taught´ +/// ´test:crate:eta-starts-at-one-for-cold-tracker´ +#[test] +fn eta_starts_at_one_for_cold_tracker() { + let cfg = cfg_test(); + let tracker = SubspaceTracker::new(128, &cfg, cfg.cusum_slow_decay); + assert!((tracker.maturity().noise_influence - 1.0).abs() < f64::EPSILON); +} + +/// A fresh tracker claims no observations of either kind, and the total is +/// exactly the two counts together. The counters are a record of what was fed +/// in rather than an estimate, so nothing may be presumed before anything +/// arrives. +/// +/// ´claim:noise:a-fresh-tracker-claims-no-observations-of-either-kind´ +/// ´test:crate:counters-start-at-zero-for-cold-tracker´ +#[test] +fn counters_start_at_zero_for_cold_tracker() { + let cfg = cfg_test(); + let tracker = SubspaceTracker::new(128, &cfg, cfg.cusum_slow_decay); + let m = tracker.maturity(); + assert_eq!(m.real_observations, 0); + assert_eq!(m.noise_observations, 0); + assert_eq!(m.total_observations(), 0); +} + +// ════════════════════════════════════════════════════════════ +// η recurrence +// ════════════════════════════════════════════════════════════ + +/// The noise share follows the same recurrence as the model it describes. A +/// real batch applies λ once, so after `k` batches the initial influence has +/// been multiplied by λ exactly `k` times, independent of the rows in them. +/// +/// ´claim:noise:the-noise-share-follows-the-models-batch-indexed-recurrence´ +/// ´test:crate:eta-tracks-theory-exactly´ +#[test] +fn eta_tracks_theory_exactly() { + let cfg = cfg_test(); + let lambda = cfg.forgetting_factor; + let mut tracker = SubspaceTracker::new(128, &cfg, cfg.cusum_slow_decay); + let mut rng = SmallRng::seed_from_u64(42); + let rows = generate_noise(128, 20, &mut rng); + let slices = as_slices(&rows); + let mut expected_eta = tracker.maturity().noise_influence; + + for batch in 1..=7 { + tracker.observe(&slices, 0, false); + expected_eta *= lambda; + assert_eq!( + tracker.maturity().noise_influence.to_bits(), + expected_eta.to_bits(), + "batch {batch} must apply one model-decay step" + ); + } +} + +/// One-sample and sixteen-sample batches each evolve the learned model once, +/// so they must also apply the same single decay to its warm-up influence. The +/// separate observation counters continue to record how many rows arrived. +/// +/// ´claim:noise:one-model-update-applies-one-noise-influence-decay-regardless-of-batch-size´ +/// ´test:crate:eta-decay-is-independent-of-batch-size´ +#[test] +fn eta_decay_is_independent_of_batch_size() { + let cfg = cfg_test(); + let mut one = SubspaceTracker::new(128, &cfg, cfg.cusum_slow_decay); + let mut sixteen = SubspaceTracker::new(128, &cfg, cfg.cusum_slow_decay); + let mut rng = SmallRng::seed_from_u64(43); + let one_row = generate_noise(128, 1, &mut rng); + let sixteen_rows = generate_noise(128, 16, &mut rng); + + one.observe(&as_slices(&one_row), 0, false); + sixteen.observe(&as_slices(&sixteen_rows), 0, false); + + assert_eq!(one.maturity().noise_influence.to_bits(), cfg.forgetting_factor.to_bits()); + assert_eq!(sixteen.maturity().noise_influence.to_bits(), cfg.forgetting_factor.to_bits()); + assert_eq!( + one.maturity().noise_influence.to_bits(), + sixteen.maturity().noise_influence.to_bits() + ); + assert_eq!(one.maturity().real_observations, 1); + assert_eq!(sixteen.maturity().real_observations, 16); +} + +/// The maturity crossing count comes directly from repeatedly applying the +/// configured forgetting factor until influence is strictly below the same +/// threshold the tracker uses. No observed run supplies the expected count. +/// +/// ´claim:noise:maturity-crosses-when-the-models-batch-decay-crosses-the-threshold´ +/// ´test:crate:eta-maturity-threshold-uses-model-batch-count´ +#[test] +fn eta_maturity_threshold_uses_model_batch_count() { + let cfg = cfg_test(); + let (crossing_batch, expected_eta) = decay_crossing(1.0, cfg.forgetting_factor); + + let mut tracker = SubspaceTracker::new(128, &cfg, cfg.cusum_slow_decay); + let mut rng = SmallRng::seed_from_u64(44); + let rows = generate_noise(128, 16, &mut rng); + let slices = as_slices(&rows); + + for batch in 1..=crossing_batch { + tracker.observe(&slices, 0, false); + if batch < crossing_batch { + assert!(tracker.maturity().noise_influence >= MATURITY_THRESHOLD); + } + } + + assert!(tracker.maturity().noise_influence < MATURITY_THRESHOLD); + assert_eq!(tracker.maturity().noise_influence.to_bits(), expected_eta.to_bits()); +} + +// ════════════════════════════════════════════════════════════ +// Monotonicity & bounds +// ════════════════════════════════════════════════════════════ + +/// Every real batch lowers the noise share and none raises it, so warm-up +/// influence is spent and never regained by ordinary operation. Monotonicity is +/// what makes the share usable as a maturity signal: a threshold crossing means +/// the same thing whenever it happens. +/// +/// ´claim:noise:every-real-batch-lowers-the-noise-share-and-none-raises-it´ +/// ´test:crate:eta-decreases-monotonically-under-real-data´ +#[test] +fn eta_decreases_monotonically_under_real_data() { + let cfg = cfg_test(); + let dim = 128; + let mut tracker = SubspaceTracker::new(dim, &cfg, cfg.cusum_slow_decay); + let mut rng = SmallRng::seed_from_u64(42); + + // Seed with noise. + for _ in 0..5 { + let noise = generate_noise(dim, cfg.noise_batch_size, &mut rng); + tracker.observe(&as_slices(&noise), 0, true); + } + + let mut prev_eta = tracker.maturity().noise_influence; + + // Real data — η must strictly decrease. + for batch in 0..50 { + let data = generate_noise(dim, 8, &mut rng); + tracker.observe(&as_slices(&data), 0, false); + let eta = tracker.maturity().noise_influence; + assert!( + eta < prev_eta + f64::EPSILON, + "batch {batch}: η increased: {prev_eta:.10} → {eta:.10}" + ); + prev_eta = eta; + } +} + +/// Injection pushes the share back up from wherever real data drove it, batch +/// by batch and without reversal. A cell whose model is re-warmed is therefore +/// re-declared immature rather than left claiming a maturity its state no +/// longer has. +/// +/// ´claim:noise:injection-pushes-the-noise-share-back-up-without-reversal´ +/// ´test:crate:eta-increases-monotonically-under-noise´ +#[test] +fn eta_increases_monotonically_under_noise() { + let cfg = cfg_test(); + let dim = 128; + let mut tracker = SubspaceTracker::new(dim, &cfg, cfg.cusum_slow_decay); + let mut rng = SmallRng::seed_from_u64(42); + + // Drive η below 1.0 with real data. + for _ in 0..30 { + let data = generate_noise(dim, 8, &mut rng); + tracker.observe(&as_slices(&data), 0, false); + } + + let eta_before_noise = tracker.maturity().noise_influence; + assert!( + eta_before_noise < 0.5, + "precondition: η should be well below 1.0, got {eta_before_noise}" + ); + + let mut prev_eta = eta_before_noise; + + // Noise — η must strictly increase toward 1.0. + for batch in 0..30 { + let noise = generate_noise(dim, cfg.noise_batch_size, &mut rng); + tracker.observe(&as_slices(&noise), 0, true); + let eta = tracker.maturity().noise_influence; + assert!( + eta > prev_eta - f64::EPSILON, + "batch {batch}: η decreased under noise: {prev_eta:.10} → {eta:.10}" + ); + prev_eta = eta; + } +} + +/// The share is a proportion and stays one under any interleaving of injected +/// and real batches. Both updates are convex steps toward an endpoint inside +/// the interval, so no mixture of workloads can carry it out of range and no +/// consumer has to guard against a value that is not a fraction. +/// +/// ´claim:noise:the-noise-share-is-a-proportion-under-any-interleaving-of-workloads´ +/// ´test:crate:eta-stays-in-unit-interval´ +#[test] +fn eta_stays_in_unit_interval() { + let cfg = cfg_test(); + let dim = 128; + let mut tracker = SubspaceTracker::new(dim, &cfg, cfg.cusum_slow_decay); + let mut rng = SmallRng::seed_from_u64(123); + + for round in 0..100 { + let is_noise = round % 3 == 0; // ~1/3 noise, ~2/3 real + let batch_size = if is_noise { cfg.noise_batch_size } else { 8 }; + let data = generate_noise(dim, batch_size, &mut rng); + tracker.observe(&as_slices(&data), 0, is_noise); + + let eta = tracker.maturity().noise_influence; + assert!((0.0..=1.0).contains(&eta), "round {round}: η out of [0, 1]: {eta}"); + } +} + +// ════════════════════════════════════════════════════════════ +// Fixed points & limits +// ════════════════════════════════════════════════════════════ + +/// Indefinite injection holds the share at exactly one, its fixed point: noise +/// cannot make a model more than entirely noise-taught. A long warm-up +/// therefore has a stable end state rather than an accumulating one. +/// +/// ´claim:noise:pure-injection-holds-the-noise-share-at-its-fixed-point-of-one´ +/// ´test:crate:eta-converges-to-one-under-noise´ +#[test] +fn eta_converges_to_one_under_noise() { + let cfg = cfg_test(); + let dim = 128; + let mut tracker = SubspaceTracker::new(dim, &cfg, cfg.cusum_slow_decay); + let mut rng = SmallRng::seed_from_u64(99); + + for _ in 1..=200 { + let noise = generate_noise(dim, cfg.noise_batch_size, &mut rng); + tracker.observe(&as_slices(&noise), 0, true); + } + + let final_eta = tracker.maturity().noise_influence; + assert!( + (final_eta - 1.0).abs() < 1e-10, + "η should stay at 1.0 under pure noise, got {final_eta}" + ); +} + +/// A long enough run of real batches drives the share below the maturity +/// threshold, so warm-up is forgotten on the same geometric cadence as the +/// model rather than on a schedule determined by batch size. +/// +/// ´claim:noise:a-long-run-of-real-data-drives-the-noise-share-to-nothing´ +/// ´test:crate:eta-decays-toward-zero-under-real-only´ +#[test] +fn eta_decays_toward_zero_under_real_only() { + let cfg = cfg_test(); + let dim = 128; + let mut tracker = SubspaceTracker::new(dim, &cfg, cfg.cusum_slow_decay); + let mut rng = SmallRng::seed_from_u64(77); + + let (crossing_batch, _) = decay_crossing(tracker.maturity().noise_influence, cfg.forgetting_factor); + for _ in 0..crossing_batch { + let data = generate_noise(dim, 8, &mut rng); + tracker.observe(&as_slices(&data), 0, false); + } + + let final_eta = tracker.maturity().noise_influence; + assert!( + final_eta < MATURITY_THRESHOLD, + "η should cross the model's maturity threshold, got {final_eta:.2e}" + ); +} + +// ════════════════════════════════════════════════════════════ +// Observation counters +// ════════════════════════════════════════════════════════════ + +/// The counters tally samples rather than batches and keep the two kinds apart: +/// a stretch of injection moves only the noise count, a stretch of real traffic +/// only the other, and the total is their sum. A host can therefore still tell +/// how much of a model's experience was synthetic long after the noise share +/// itself has decayed away. +/// +/// ´claim:noise:the-counters-tally-samples-not-batches-and-keep-the-two-kinds-apart´ +/// ´test:crate:observation-counters-mixed-sequence´ +#[test] +fn observation_counters_mixed_sequence() { + let cfg = cfg_test(); + let dim = 128; + let mut tracker = SubspaceTracker::new(dim, &cfg, cfg.cusum_slow_decay); + let mut rng = SmallRng::seed_from_u64(42); + + // 10 noise rounds of batch_size=4. + for _ in 0..10 { + let noise = generate_noise(dim, 4, &mut rng); + tracker.observe(&as_slices(&noise), 0, true); + } + assert_eq!(tracker.maturity().noise_observations, 40); + assert_eq!(tracker.maturity().real_observations, 0); + + // 5 real batches of batch_size=8. + for _ in 0..5 { + let data = generate_noise(dim, 8, &mut rng); + tracker.observe(&as_slices(&data), 0, false); + } + assert_eq!(tracker.maturity().noise_observations, 40); + assert_eq!(tracker.maturity().real_observations, 40); + assert_eq!(tracker.maturity().total_observations(), 80); +} + +/// With no injection at all the noise count stays at nothing while the real +/// count tracks every sample fed in, which pins the other end of the same +/// bookkeeping. +/// +/// (´claim:noise:the-counters-tally-samples-not-batches-and-keep-the-two-kinds-apart´) +/// ´test:crate:counters-track-real-only-sequence´ +#[test] +fn counters_track_real_only_sequence() { + let cfg = cfg_test(); + let dim = 128; + let mut tracker = SubspaceTracker::new(dim, &cfg, cfg.cusum_slow_decay); + let mut rng = SmallRng::seed_from_u64(55); + + for _ in 0..20 { + let data = generate_noise(dim, 10, &mut rng); + tracker.observe(&as_slices(&data), 0, false); + } + + let m = tracker.maturity(); + assert_eq!(m.real_observations, 200); + assert_eq!(m.noise_observations, 0); + assert_eq!(m.total_observations(), 200); +} diff --git a/packages/sentinel/src/tests/convergence_ewma.rs b/packages/sentinel/src/tests/convergence_ewma.rs new file mode 100644 index 000000000..4556adef2 --- /dev/null +++ b/packages/sentinel/src/tests/convergence_ewma.rs @@ -0,0 +1,394 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +//! The convergence guarantees everything above the baselines rests on, +//! established on the estimator alone — no subspace, no scoring pipeline, +//! no noise. +//! +//! A baseline is a running mean and spread that forget the past +//! geometrically, and almost everything the sentinel promises about +//! warm-up follows from what that single decay factor implies. Fed a +//! constant level the estimator closes on it at a rate the factor +//! dictates, so a warm-up budget can be computed from configuration +//! instead of discovered by running. The approach never rebounds and the +//! arrival is permanent while the input holds, so "converged" is a +//! well-defined moment rather than a lull. A longer memory buys steadiness +//! by taking longer to arrive, which is the trade the factor exists to +//! express, and a step change is adopted rather than resisted, because a +//! baseline models what is normal now. +//! +//! Two decisions are visible here rather than derived. The first batch +//! seeds the mean and the spread outright instead of blending against the +//! values a fresh estimator is constructed with: those are placeholders +//! chosen to keep early z-scores finite, and blending against them would +//! plant a bias that then has to decay away. And a batch is one step of +//! learning however many samples it carries, because the update consumes +//! the batch mean — which is why convergence is counted in rounds +//! throughout, and never in samples. +//! +//! Clipping appears once, at the end, for the property that motivates the +//! warm-up exemption: a ceiling can only slow the approach to a distant +//! level, never hasten it. +//! +//! # §-references +//! +//! - §ALGO S-6.1.1 — EWMA outlier filter +//! - ADR-S-013 — Warm-up convergence benchmark +//! +//! # Test index +//! +//! | Test | Area | Claim | +//! |------|------|-------| +//! | [`ewma_cold_start_sets_mean_directly`] | convergence | The first batch seeds the baseline outright instead of being blended with the placeholder the baseline was constructed with. That placeholder is a stand-in chosen to keep early z-scores finite, not an observation, so blending against it would plant a bias that then has to decay away. | +//! | [`ewma_cold_start_sets_variance_from_batch`] | convergence | cites (´claim:convergence:the-first-batch-seeds-the-baseline-outright-instead-of-blending-with-the-placeholder´) | +//! | [`ewma_pure_convergence_rate`] | convergence | Fed a constant level, the baseline closes on it at the rate the forgetting factor dictates — the steps needed to reach a given relative error are what geometric decay predicts. Convergence time is therefore something that can be computed from configuration rather than discovered by running. | +//! | [`ewma_higher_lambda_converges_slower`] | convergence | A longer memory buys steadiness by taking longer to arrive: the slower-decaying baseline needs strictly more steps to reach the same relative error. This is the trade the forgetting factor exists to express, and it is why a warm-up budget is sized against the configured factor rather than fixed at some number of rounds. | +//! | [`ewma_convergence_independent_of_batch_size`] | convergence | A batch is one step of learning however many samples it carries: batches spanning a wide range of sizes all reach the same relative error in the same number of steps, because the update consumes the batch mean. Convergence is counted in rounds, not in samples. | +//! | [`ewma_error_decreases_monotonically`] | convergence | Approaching a constant level the error never rebounds: each step is a convex move toward the target and cannot carry the mean past it. A baseline that oscillated on the way in would leave every convergence test ambiguous about when it had arrived. | +//! | [`ewma_steady_state_is_stable`] | convergence | Once arrived, the baseline neither drifts nor oscillates — a long further run against the same level leaves it exactly there. Arrival is permanent while the input holds, so a later departure can be attributed to the input rather than to the estimator. | +//! | [`ewma_tracks_step_change`] | convergence | After settling on one level the baseline re-converges when the input steps to another, within a time the forgetting factor bounds. A baseline models what is normal now, so a genuine change of regime has to be adopted rather than resisted indefinitely. | +//! | [`ewma_variance_convergence`] | convergence | The spread converges on much the same schedule as the mean, since both are carried by the same decay factor. That matters because a z-score divides one by the other: a spread lagging far behind its mean would leave scores miscalibrated even after the level itself looked settled. | +//! | [`ewma_clipping_slows_convergence`] | clipping | A ceiling can only slow the approach to a distant level, never hasten it: when the target sits far above the current mean, the clip rejects the very batches that would move it. This is the cost the warm-up exemption exists to avoid paying while a baseline has not yet found its level. | + +use crate::ewma::EwmaStats; + +// ════════════════════════════════════════════════════════════ +// 1. Cold start +// ════════════════════════════════════════════════════════════ + +/// The first batch seeds the baseline outright instead of being blended with +/// the placeholder the baseline was constructed with. That placeholder is a +/// stand-in chosen to keep early z-scores finite, not an observation, so +/// blending against it would plant a bias that then has to decay away. +/// +/// ´claim:convergence:the-first-batch-seeds-the-baseline-outright-instead-of-blending-with-the-placeholder´ +/// ´test:crate:ewma-cold-start-sets-mean-directly´ +#[test] +fn ewma_cold_start_sets_mean_directly() { + for lambda in [0.90, 0.95, 0.99] { + let mut ewma = EwmaStats::new(lambda); + assert!(!ewma.is_warm()); + + ewma.update(&[42.0, 42.0], f64::INFINITY); + assert!(ewma.is_warm()); + assert!( + (ewma.mean() - 42.0).abs() < 1e-10, + "λ={lambda}: cold start should set mean=42.0, got {}", + ewma.mean() + ); + } +} + +/// The same seeding governs the spread: a first batch with structure in it sets +/// the variance from that batch rather than leaving the constructed placeholder +/// standing. This is the half clipping depends on, since the ceiling is defined +/// from the spread. +/// +/// (´claim:convergence:the-first-batch-seeds-the-baseline-outright-instead-of-blending-with-the-placeholder´) +/// ´test:crate:ewma-cold-start-sets-variance-from-batch´ +#[test] +fn ewma_cold_start_sets_variance_from_batch() { + let mut ewma = EwmaStats::new(0.95); + assert!((ewma.variance() - 1.0).abs() < f64::EPSILON, "placeholder should be 1.0"); + + // Batch mean = 10.0, MSD = ((−1)² + 0² + 1²) / 3 = 2/3 + ewma.update(&[9.0, 10.0, 11.0], f64::INFINITY); + let expected_var = 2.0 / 3.0; + assert!( + (ewma.variance() - expected_var).abs() < 1e-10, + "cold start should set variance from batch: expected {expected_var}, got {}", + ewma.variance() + ); +} + +// ════════════════════════════════════════════════════════════ +// 2. Mean convergence rate +// ════════════════════════════════════════════════════════════ + +/// Fed a constant level, the baseline closes on it at the rate the forgetting +/// factor dictates — the steps needed to reach a given relative error are what +/// geometric decay predicts. Convergence time is therefore something that can +/// be computed from configuration rather than discovered by running. +/// +/// ´claim:convergence:the-baseline-closes-on-a-constant-level-at-the-rate-the-forgetting-factor-dictates´ +/// ´test:crate:ewma-pure-convergence-rate´ +#[test] +#[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] +fn ewma_pure_convergence_rate() { + let lambda = 0.95_f64; + let mut ewma = EwmaStats::new(lambda); + + let target = 5.0; + ewma.update(&[target; 4], f64::INFINITY); + assert!( + (ewma.mean() - target).abs() < 1e-10, + "cold→warm should set mean exactly, got {}", + ewma.mean() + ); + + let new_target = 10.0; + let new_values = [new_target; 4]; + + let mut steps_to_converge = None; + for step in 1..=200 { + ewma.update(&new_values, f64::INFINITY); + let error = (ewma.mean() - new_target).abs() / new_target; + if error < 0.05 && steps_to_converge.is_none() { + steps_to_converge = Some(step); + } + } + + let n = steps_to_converge.expect("should converge within 200 steps"); + let theory = 0.05_f64.log(lambda).ceil() as usize; + + assert!( + n <= theory + 2, + "pure EWMA should converge close to theory: got {n}, expected ~{theory}" + ); +} + +/// A longer memory buys steadiness by taking longer to arrive: the +/// slower-decaying baseline needs strictly more steps to reach the same +/// relative error. This is the trade the forgetting factor exists to express, +/// and it is why a warm-up budget is sized against the configured factor rather +/// than fixed at some number of rounds. +/// +/// ´claim:convergence:a-longer-memory-buys-steadiness-by-taking-longer-to-arrive´ +/// ´test:crate:ewma-higher-lambda-converges-slower´ +#[test] +fn ewma_higher_lambda_converges_slower() { + fn steps_to_converge(lambda: f64) -> usize { + let mut ewma = EwmaStats::new(lambda); + ewma.update(&[1.0; 4], f64::INFINITY); // cold start + + let target = 10.0; + let values = [target; 4]; + for step in 1..=1000 { + ewma.update(&values, f64::INFINITY); + if (ewma.mean() - target).abs() / target < 0.05 { + return step; + } + } + 1001 + } + + let fast = steps_to_converge(0.90); + let slow = steps_to_converge(0.99); + assert!(slow > fast, "λ=0.99 should be slower than λ=0.90: fast={fast}, slow={slow}"); +} + +/// A batch is one step of learning however many samples it carries: batches +/// spanning a wide range of sizes all reach the same relative error in the same +/// number of steps, because the update consumes the batch mean. Convergence is +/// counted in rounds, not in samples. +/// +/// ´claim:convergence:a-batch-is-one-step-of-learning-however-many-samples-it-carries´ +/// ´test:crate:ewma-convergence-independent-of-batch-size´ +#[test] +fn ewma_convergence_independent_of_batch_size() { + let lambda = 0.95_f64; + let target = 7.0; + + let mut results = Vec::new(); + for batch_size in [1, 4, 16, 64] { + let mut ewma = EwmaStats::new(lambda); + let values: Vec = vec![target; batch_size]; + + ewma.update(&[0.1], f64::INFINITY); + + let mut steps = None; + for step in 1..=200 { + ewma.update(&values, f64::INFINITY); + let error = (ewma.mean() - target).abs() / target; + if error < 0.05 && steps.is_none() { + steps = Some(step); + } + } + results.push(steps.expect("should converge")); + } + + let min = *results.iter().min().unwrap(); + let max = *results.iter().max().unwrap(); + assert!( + max - min <= 1, + "convergence should be independent of batch size: min={min}, max={max}, results={results:?}" + ); +} + +// ════════════════════════════════════════════════════════════ +// 3. Convergence quality +// ════════════════════════════════════════════════════════════ + +/// Approaching a constant level the error never rebounds: each step is a convex +/// move toward the target and cannot carry the mean past it. A baseline that +/// oscillated on the way in would leave every convergence test ambiguous about +/// when it had arrived. +/// +/// ´claim:convergence:the-approach-to-a-constant-level-never-rebounds´ +/// ´test:crate:ewma-error-decreases-monotonically´ +#[test] +fn ewma_error_decreases_monotonically() { + let lambda = 0.95_f64; + let mut ewma = EwmaStats::new(lambda); + ewma.update(&[1.0; 4], f64::INFINITY); // cold start at 1.0 + + let target = 10.0; + let values = [target; 4]; + let mut prev_error = f64::INFINITY; + + for step in 1..=100 { + ewma.update(&values, f64::INFINITY); + let error = (ewma.mean() - target).abs(); + assert!( + error <= prev_error + 1e-12, + "error increased at step {step}: {prev_error} → {error}" + ); + prev_error = error; + } +} + +/// Once arrived, the baseline neither drifts nor oscillates — a long further +/// run against the same level leaves it exactly there. Arrival is permanent +/// while the input holds, so a later departure can be attributed to the input +/// rather than to the estimator. +/// +/// ´claim:convergence:arrival-is-permanent-while-the-input-holds´ +/// ´test:crate:ewma-steady-state-is-stable´ +#[test] +fn ewma_steady_state_is_stable() { + let lambda = 0.95_f64; + let target = 7.5; + let mut ewma = EwmaStats::new(lambda); + + // Converge fully. + ewma.update(&[target; 4], f64::INFINITY); + for _ in 0..200 { + ewma.update(&[target; 4], f64::INFINITY); + } + + // Run 100 more steps and verify no drift. + for step in 1..=100 { + ewma.update(&[target; 4], f64::INFINITY); + let error = (ewma.mean() - target).abs(); + assert!( + error < 1e-10, + "steady state drifted at step {step}: mean={}, target={target}", + ewma.mean() + ); + } +} + +/// After settling on one level the baseline re-converges when the input steps +/// to another, within a time the forgetting factor bounds. A baseline models +/// what is normal now, so a genuine change of regime has to be adopted rather +/// than resisted indefinitely. +/// +/// ´claim:convergence:a-settled-baseline-re-converges-after-an-abrupt-level-shift´ +/// ´test:crate:ewma-tracks-step-change´ +#[test] +fn ewma_tracks_step_change() { + let lambda = 0.95_f64; + let mut ewma = EwmaStats::new(lambda); + + // Converge to 5.0. + ewma.update(&[5.0; 4], f64::INFINITY); + for _ in 0..200 { + ewma.update(&[5.0; 4], f64::INFINITY); + } + assert!((ewma.mean() - 5.0).abs() < 1e-10, "should have converged to 5.0"); + + // Step change to 15.0 — verify re-convergence. + let new_target = 15.0; + let mut reconverged = false; + for step in 1..=200 { + ewma.update(&[new_target; 4], f64::INFINITY); + if (ewma.mean() - new_target).abs() / new_target < 0.05 { + reconverged = true; + // Verify it happened in a reasonable number of steps. + assert!(step <= 80, "re-convergence took too long after step change: {step} steps"); + break; + } + } + assert!(reconverged, "EWMA did not re-converge to {new_target}"); +} + +// ════════════════════════════════════════════════════════════ +// 4. Variance convergence +// ════════════════════════════════════════════════════════════ + +/// The spread converges on much the same schedule as the mean, since both are +/// carried by the same decay factor. That matters because a z-score divides one +/// by the other: a spread lagging far behind its mean would leave scores +/// miscalibrated even after the level itself looked settled. +/// +/// ´claim:convergence:the-spread-converges-on-the-same-schedule-as-the-mean´ +/// ´test:crate:ewma-variance-convergence´ +#[test] +#[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] +fn ewma_variance_convergence() { + let lambda = 0.95_f64; + let mut ewma = EwmaStats::new(lambda); + + ewma.update(&[5.0, 5.0, 5.0, 5.0], f64::INFINITY); + + // Feed values with variance = 2/3 (values 9, 10, 11). + let target_var = 2.0 / 3.0; + let values = [9.0, 10.0, 11.0]; + + let mut steps_to_converge = None; + for step in 1..=200 { + ewma.update(&values, f64::INFINITY); + let error = (ewma.variance() - target_var).abs() / target_var; + if error < 0.10 && steps_to_converge.is_none() { + steps_to_converge = Some(step); + } + } + + let n = steps_to_converge.expect("variance should converge within 200 steps"); + let theory = 0.10_f64.log(lambda).ceil() as usize; + assert!(n <= theory + 5, "variance convergence too slow: got {n}, expected ~{theory}"); +} + +// ════════════════════════════════════════════════════════════ +// 5. Clipping interaction +// ════════════════════════════════════════════════════════════ + +/// A ceiling can only slow the approach to a distant level, never hasten it: +/// when the target sits far above the current mean, the clip rejects the very +/// batches that would move it. This is the cost the warm-up exemption exists to +/// avoid paying while a baseline has not yet found its level. +/// +/// ´claim:clipping:a-ceiling-can-only-slow-the-approach-to-a-distant-level-never-hasten-it´ +/// ´test:crate:ewma-clipping-slows-convergence´ +#[test] +fn ewma_clipping_slows_convergence() { + let lambda = 0.95_f64; + + let mut no_clip = EwmaStats::new(lambda); + let mut clipped = EwmaStats::new(lambda); + + no_clip.update(&[1.0, 1.0, 1.0], f64::INFINITY); + clipped.update(&[1.0, 1.0, 1.0], 3.0); + + let target = 10.0; + let values = [target; 4]; + + let mut no_clip_steps = None; + let mut clipped_steps = None; + + for step in 1..=500 { + no_clip.update(&values, f64::INFINITY); + clipped.update(&values, 3.0); + + if (no_clip.mean() - target).abs() / target < 0.05 && no_clip_steps.is_none() { + no_clip_steps = Some(step); + } + if (clipped.mean() - target).abs() / target < 0.05 && clipped_steps.is_none() { + clipped_steps = Some(step); + } + } + + let n_no_clip = no_clip_steps.expect("no_clip should converge"); + let n_clipped = clipped_steps.unwrap_or(500); + assert!(n_clipped >= n_no_clip, "clipping should not speed up convergence"); +} diff --git a/packages/sentinel/src/tests/convergence_fixes.rs b/packages/sentinel/src/tests/convergence_fixes.rs new file mode 100644 index 000000000..2a5c927f3 --- /dev/null +++ b/packages/sentinel/src/tests/convergence_fixes.rs @@ -0,0 +1,352 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +//! Why warm-up terminates, and the bound it terminates within. +//! +//! Three properties of the cold start had to hold before a warm-up of +//! bounded length could be promised at all, and each is guarded here. +//! Nothing about them is historical: they are the reasons the present +//! design converges. +//! +//! An axis must not be able to clip itself into a baseline it then keeps +//! rejecting the evidence against — so the ceiling is held open while the +//! model is still noise-taught, and narrows smoothly rather than +//! switching as that exemption is spent. The latent spread must be seeded +//! from the first batch rather than started at a placeholder far above +//! where it settles, because the surprise axis divides by that spread and +//! a placeholder set high would show up as a slow rise indistinguishable +//! from a real trend. And at the hand-over from injected to real traffic +//! the long-memory reference must be seeded from the converged +//! short-memory baseline: left to converge on its own it would lag for +//! many hundreds of rounds, and the whole lag would be banked as evidence +//! of drift that never happened. +//! +//! With those in place the remaining question is the size of the budget, +//! and it is asked at several points rather than once. Each axis is judged +//! at its own tolerance, since they differ by more than an order of +//! magnitude in inherent jitter; the bound is checked at two batch sizes +//! and at the production memory length, so it can be seen to scale with +//! configuration rather than being a constant; and it is checked across a +//! spread of seeds, because two of the axes carry real seed-to-seed +//! variance in when they settle and a bound shown once would be no bound. +//! +//! # §-references +//! +//! - §ALGO S-4.2 Phase 3 — Latent distribution cold→warm +//! - §ALGO S-6.4 — Clip-pressure EWMA / graduated clip formula +//! - §ALGO S-6.1.1 — EWMA outlier filter / clipping ceiling +//! - §ALGO S-11.5 — Maturity tracking (noise influence η) +//! - ADR-S-013 — Warm-up convergence benchmark +//! +//! # Test index +//! +//! | Test | Area | Claim | +//! |------|------|-------| +//! | [`graduated_clip_formula_is_smooth_and_monotonic`] | clipping | The effective clip width narrows smoothly and without reversal as the exemption is spent, from an effectively open ceiling while the model is entirely noise-taught down to the nominal width once it is not. There is no step anywhere along the transition, so no batch is judged by a much tighter rule than the batch before it. | +//! | [`clip_exemption_eliminates_feedback_loop`] | clipping | No axis can clip itself into a baseline it then keeps rejecting the evidence against. The exemption holds the ceiling open while the noise share is high, so the first batches enter unfiltered and even the highest-variance axis settles well inside the round budget instead of being pinned by its own early ceiling. | +//! | [`cold_warm_eliminates_surprise_nonstationarity`] | convergence | Because the latent spread is seeded from the first batch rather than starting at a placeholder far above its eventual value, the surprise axis is near stationary from its earliest rounds: early scores and late ones differ by well under a factor of two. Surprise divides by that spread, so a placeholder set too high would show up as a slow rise indistinguishable from a real trend. | +//! | [`cusum_seeding_prevents_false_drift`] | convergence | Seeding the long-memory reference from the converged short-memory baseline at the hand-over from injected to real traffic keeps the switch from reading as drift: the accumulators stay far below anything actionable across a long real-data run. Left to converge on its own the long reference would lag for many hundreds of rounds, and the whole lag would be banked as evidence. | +//! | [`per_axis_convergence_b4_within_bound`] | convergence | Every axis settles inside the round budget the noise schedule is sized from, each judged at its own tolerance. The axes differ by more than an order of magnitude in inherent jitter, so one shared tolerance would either excuse the quietest or condemn the noisiest; this per-axis bound is what makes a warm-up of bounded length sufficient. | +//! | [`per_axis_convergence_b16_within_bound`] | convergence | cites (´claim:convergence:every-axis-settles-inside-the-round-budget-at-its-own-tolerance´) | +//! | [`production_lambda_converges_within_bound`] | convergence | cites (´claim:convergence:every-axis-settles-inside-the-round-budget-at-its-own-tolerance´) | +//! | [`per_axis_convergence_b4_robust_across_seeds`] | convergence | The budget holds across a spread of seeds and not merely for one lucky noise sequence. Two of the axes carry real seed-to-seed variance in when they settle, so a bound demonstrated once would not be a bound at all: sizing a shipped schedule needs the worst case over sequences. | + +use rand::SeedableRng; +use rand::rngs::SmallRng; + +use super::convergence_common::{ + as_slices, cfg_b16, cfg_production, cfg_test, find_converged_round, generate_noise, run_noise_trace, +}; +use crate::sentinel::tracker::SubspaceTracker; + +// ════════════════════════════════════════════════════════════ +// Fix 1: Graduated clip-exemption (§ALGO S-6.4) +// ════════════════════════════════════════════════════════════ + +/// The effective clip width narrows smoothly and without reversal as the +/// exemption is spent, from an effectively open ceiling while the model is +/// entirely noise-taught down to the nominal width once it is not. There is no +/// step anywhere along the transition, so no batch is judged by a much tighter +/// rule than the batch before it. +/// +/// ´claim:clipping:the-effective-clip-width-narrows-smoothly-and-without-reversal-as-the-exemption-is-spent´ +/// ´test:crate:graduated-clip-formula-is-smooth-and-monotonic´ +#[test] +fn graduated_clip_formula_is_smooth_and_monotonic() { + let clip_sigmas = 3.0; + let eps = 1e-6; + + // Walk p from 1.0 → 0.0 and verify monotonicity (non-increasing). + let test_points = [1.0, 0.99, 0.95, 0.9, 0.8, 0.5, 0.3, 0.1, 0.01, 0.001, 0.0]; + let mut prev_eff: Option = None; + + for &p in &test_points { + let effective = clip_sigmas * (1.0 + p / (1.0 - p + eps)); + + if let Some(prev) = prev_eff { + assert!( + effective <= prev + 1e-6, + "effective clip should be non-increasing as p decreases: \ + p={p}, eff={effective}, prev_eff={prev}" + ); + } + prev_eff = Some(effective); + } + + // Boundary: at p = 0, effective ≈ clip_sigmas. + let at_zero = clip_sigmas * (1.0 + 0.0 / (1.0 - 0.0 + eps)); + assert!( + (at_zero - clip_sigmas).abs() < 1e-4, + "at p=0, effective clip should equal clip_sigmas: got {at_zero}" + ); + + // Boundary: at p = 1, effective is very large (open ceiling). + let at_one = clip_sigmas * (1.0 + 1.0 / (1.0 - 1.0 + eps)); + assert!(at_one > 1000.0, "at p=1, effective clip should be very large: got {at_one}"); +} + +/// No axis can clip itself into a baseline it then keeps rejecting the evidence +/// against. The exemption holds the ceiling open while the noise share is high, +/// so the first batches enter unfiltered and even the highest-variance axis +/// settles well inside the round budget instead of being pinned by its own +/// early ceiling. +/// +/// ´claim:clipping:an-axis-cannot-clip-itself-into-a-baseline-it-then-rejects-the-evidence-against´ +/// ´test:crate:clip-exemption-eliminates-feedback-loop´ +#[test] +fn clip_exemption_eliminates_feedback_loop() { + let cfg = cfg_test(); + let dim = 128; + let total_rounds = 500; + let traces = run_noise_trace(&cfg, dim, total_rounds, 42); + + let surprise_baselines: Vec = traces.iter().map(|t| t.baseline_means[2]).collect(); + + let window = 20; + let tolerance = 0.20; // 20% for surprise (high-CV axis) + let converged = find_converged_round(&surprise_baselines, window, tolerance); + + assert!( + converged <= 200, + "surprise should converge within 200 rounds with clip-exemption fix, got {converged}" + ); +} + +// ════════════════════════════════════════════════════════════ +// Fix 2: Cold→warm initialisation (§ALGO S-4.2 Phase 3) +// ════════════════════════════════════════════════════════════ + +/// Because the latent spread is seeded from the first batch rather than +/// starting at a placeholder far above its eventual value, the surprise axis is +/// near stationary from its earliest rounds: early scores and late ones differ +/// by well under a factor of two. Surprise divides by that spread, so a +/// placeholder set too high would show up as a slow rise indistinguishable from +/// a real trend. +/// +/// ´claim:convergence:seeding-the-latent-spread-from-the-first-batch-leaves-the-surprise-axis-stationary-from-the-start´ +/// ´test:crate:cold-warm-eliminates-surprise-nonstationarity´ +#[test] +fn cold_warm_eliminates_surprise_nonstationarity() { + let cfg = cfg_test(); + let dim = 128; + let total_rounds = 200; + let traces = run_noise_trace(&cfg, dim, total_rounds, 42); + + let surprise_scores: Vec = traces.iter().map(|t| t.score_means[2]).collect(); + + // Compare early (rounds 2–9) vs late (rounds 150+). + let early_avg: f64 = surprise_scores[2..10].iter().sum::() / 8.0; + let late_avg: f64 = surprise_scores[150..].iter().sum::() / 50.0; + let rise_factor = late_avg / early_avg; + + assert!( + rise_factor < 2.0, + "surprise rise factor should be < 2.0 with cold→warm fix, got {rise_factor:.2}×" + ); +} + +// ════════════════════════════════════════════════════════════ +// Fix 3: CUSUM slow-from-fast seeding (ADR-S-013 §6b) +// ════════════════════════════════════════════════════════════ + +/// Seeding the long-memory reference from the converged short-memory baseline +/// at the hand-over from injected to real traffic keeps the switch from reading +/// as drift: the accumulators stay far below anything actionable across a long +/// real-data run. Left to converge on its own the long reference would lag for +/// many hundreds of rounds, and the whole lag would be banked as evidence. +/// +/// ´claim:convergence:seeding-the-long-memory-reference-at-the-hand-over-keeps-the-switch-to-real-traffic-from-reading-as-drift´ +/// ´test:crate:cusum-seeding-prevents-false-drift´ +#[test] +fn cusum_seeding_prevents_false_drift() { + let cfg = cfg_test(); + let dim = 128; + let noise_rounds = 200; + let real_rounds = 300; + + let mut tracker = SubspaceTracker::new(dim, &cfg, cfg.cusum_slow_decay); + let mut rng = SmallRng::seed_from_u64(42); + + // Noise phase. + for _ in 0..noise_rounds { + let data = generate_noise(dim, cfg.noise_batch_size, &mut rng); + tracker.observe(&as_slices(&data), 0, true); + } + + // Transition: seed slow from fast, then reset CUSUM. + tracker.seed_cusum_slow_from_baselines(); + tracker.reset_cusum(); + + // Real-data phase (same noise-like data, but marked as real). + let mut max_cusum = [0.0_f64; 4]; + for _ in 0..real_rounds { + let data = generate_noise(dim, cfg.noise_batch_size, &mut rng); + let report = tracker.observe(&as_slices(&data), 0, false); + max_cusum[0] = max_cusum[0].max(report.scores.novelty.cusum.accumulator); + max_cusum[1] = max_cusum[1].max(report.scores.displacement.cusum.accumulator); + max_cusum[2] = max_cusum[2].max(report.scores.surprise.cusum.accumulator); + max_cusum[3] = max_cusum[3].max(report.scores.coherence.cusum.accumulator); + } + + let axis_names = ["novelty", "displacement", "surprise", "coherence"]; + for (i, &name) in axis_names.iter().enumerate() { + assert!( + max_cusum[i] < 20.0, + "{name}: CUSUM should stay < 20 with slow-from-fast seeding, \ + got {:.2} (surprise reaches ~198 without seeding)", + max_cusum[i], + ); + } +} + +// ════════════════════════════════════════════════════════════ +// Combined convergence bounds +// ════════════════════════════════════════════════════════════ + +/// Every axis settles inside the round budget the noise schedule is sized from, +/// each judged at its own tolerance. The axes differ by more than an order of +/// magnitude in inherent jitter, so one shared tolerance would either excuse +/// the quietest or condemn the noisiest; this per-axis bound is what makes a +/// warm-up of bounded length sufficient. +/// +/// ´claim:convergence:every-axis-settles-inside-the-round-budget-at-its-own-tolerance´ +/// ´test:crate:per-axis-convergence-b4-within-bound´ +#[test] +fn per_axis_convergence_b4_within_bound() { + let cfg = cfg_test(); + let dim = 128; + let total_rounds = 500; + let traces = run_noise_trace(&cfg, dim, total_rounds, 42); + + let tolerances = [0.01, 0.10, 0.20, 0.20]; + let window = 20; + + let mut worst_round = 0_usize; + for (axis, &tol) in tolerances.iter().enumerate() { + let baselines: Vec = traces.iter().map(|t| t.baseline_means[axis]).collect(); + let converged = find_converged_round(&baselines, window, tol); + worst_round = worst_round.max(converged); + } + + assert!( + worst_round <= 500, + "all axes should converge within 500 rounds at b=4, worst={worst_round}" + ); +} + +/// Larger batches settle inside a proportionally smaller budget, which pins the +/// batch-size end of the same bound: more samples per round buy a better +/// per-round estimate, not more rounds of learning. +/// +/// (´claim:convergence:every-axis-settles-inside-the-round-budget-at-its-own-tolerance´) +/// ´test:crate:per-axis-convergence-b16-within-bound´ +#[test] +fn per_axis_convergence_b16_within_bound() { + let cfg = cfg_b16(); + let dim = 128; + let total_rounds = 300; + let traces = run_noise_trace(&cfg, dim, total_rounds, 42); + + let tolerances = [0.01, 0.10, 0.20, 0.20]; + let window = 20; + + let mut worst_round = 0_usize; + for (axis, &tol) in tolerances.iter().enumerate() { + let baselines: Vec = traces.iter().map(|t| t.baseline_means[axis]).collect(); + let converged = find_converged_round(&baselines, window, tol); + worst_round = worst_round.max(converged); + } + + assert!( + worst_round <= 250, + "all axes should converge within 250 rounds at b=16, worst={worst_round}" + ); +} + +/// The shipped settings — a much longer memory and larger batches — settle +/// inside their own correspondingly larger budget, measured with a window +/// matched to that memory. This pins the long-memory end of the bound and shows +/// the budget scales with the forgetting factor rather than being a fixed +/// constant. +/// +/// (´claim:convergence:every-axis-settles-inside-the-round-budget-at-its-own-tolerance´) +/// ´test:crate:production-lambda-converges-within-bound´ +#[test] +fn production_lambda_converges_within_bound() { + let cfg = cfg_production(); + // Convergence speed depends on λ and b, not on dim. dim=32 + // halves the SVD cost vs 128 while preserving the bound + // (ADR-S-012). + let dim = 32; + let total_rounds = 1500; + let traces = run_noise_trace(&cfg, dim, total_rounds, 42); + + // At λ=0.99, use window = 1/α = 100. + let window = 100; + let tolerances = [0.01, 0.10, 0.20, 0.20]; + + let mut worst_round = 0_usize; + for (axis, &tol) in tolerances.iter().enumerate() { + let baselines: Vec = traces.iter().map(|t| t.baseline_means[axis]).collect(); + let converged = find_converged_round(&baselines, window, tol); + worst_round = worst_round.max(converged); + } + + assert!( + worst_round <= 1200, + "production config should converge within 1200 rounds, got {worst_round}" + ); +} + +/// The budget holds across a spread of seeds and not merely for one lucky noise +/// sequence. Two of the axes carry real seed-to-seed variance in when they +/// settle, so a bound demonstrated once would not be a bound at all: sizing a +/// shipped schedule needs the worst case over sequences. +/// +/// ´claim:convergence:the-round-budget-holds-across-seeds-and-not-merely-one-lucky-sequence´ +/// ´test:crate:per-axis-convergence-b4-robust-across-seeds´ +#[test] +fn per_axis_convergence_b4_robust_across_seeds() { + let cfg = cfg_test(); + // See production_lambda_converges_within_bound comment. + let dim = 32; + let total_rounds = 500; + let window = 20; + let tolerances = [0.01, 0.10, 0.20, 0.20]; + + for seed in [42, 123, 456, 789, 1024] { + let traces = run_noise_trace(&cfg, dim, total_rounds, seed); + + let mut worst_round = 0_usize; + for (axis, &tol) in tolerances.iter().enumerate() { + let baselines: Vec = traces.iter().map(|t| t.baseline_means[axis]).collect(); + let converged = find_converged_round(&baselines, window, tol); + worst_round = worst_round.max(converged); + } + + assert!( + worst_round <= 500, + "all axes should converge within 500 rounds at b=4 (seed={seed}), worst={worst_round}" + ); + } +} diff --git a/packages/sentinel/src/tests/convergence_noise.rs b/packages/sentinel/src/tests/convergence_noise.rs new file mode 100644 index 000000000..edbd66aca --- /dev/null +++ b/packages/sentinel/src/tests/convergence_noise.rs @@ -0,0 +1,651 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +//! # Test index +//! +//! | Test | Area | Claim | +//! |------|------|-------| +//! | [`noise_baselines_converge_within_bound`] | convergence | A baseline counts as converged when an early block of rounds and a late one agree, not when its round-to-round jitter has stopped. Steady-state jitter is irreducible and differs by more than a hundredfold across the axes, so a single-round criterion tight enough for the quietest axis would report the noisiest as permanently unconverged long after it had reached its correct level. | +//! | [`baseline_variance_converges_within_bound`] | convergence | cites (´claim:convergence:convergence-is-agreement-between-an-early-and-a-late-block-not-the-absence-of-jitter´) | +//! | [`frozen_subspace_converges_at_least_as_fast`] | convergence | Holding the subspace still never makes the baselines settle later than letting it evolve. Scores are measured against the model, so a model that is itself still moving is a second source of variation on top of the input; removing it can only help. | +//! | [`rank_reaches_max_within_expected_rounds`] | convergence | Rank climbs to its ceiling within a few adaptation intervals, one step at each. The model is therefore at full width long before the baselines settle, so the bulk of a warm-up is spent learning score distributions rather than discovering how many directions to keep. | +//! | [`coherence_activates_at_rank_two`] | convergence | Coherence is a statement about pairs of latent directions, so below two directions there are no pairs and the axis does not exist. Its baseline is held at the cold placeholder rather than being fed the identically zero scores, and it enters through the cold-start path the moment a second direction appears — otherwise it would converge onto zero and then have to unlearn it. | +//! | [`energy_ratio_stabilises_under_noise`] | convergence | The share of energy the retained directions capture settles onto a narrow plateau near the threshold that chose the rank. That ratio is the quantity rank adaptation reads, so its settling is what stops the rank from oscillating. | +//! | [`report_baseline_is_pre_update_snapshot`] | convergence | A report carries the baseline the batch was scored against, never the one the batch produced: the first report shows the cold placeholder even though the internal state has already moved, and the second shows exactly what the first batch left behind. A batch can therefore never partly explain itself away, and a reader can reconstruct the comparison that produced the scores. | +//! | [`z_scores_stay_bounded_at_steady_state`] | convergence | Measured against a settled baseline, ordinary traffic scores near zero on average and stays unremarkable on every single round. The baselines are calibrated and not merely stable: a systematic offset would mean every batch looked mildly anomalous, leaving no headroom to signal one that genuinely was. | +//! | [`latent_variance_reaches_steady_state`] | convergence | The latent spread settles far below the value a freshly constructed tracker holds. That gap is why the first batch seeds the spread instead of blending toward it: starting an order of magnitude high would suppress the surprise axis for as long as the gap took to decay. | +//! | [`latent_mean_stays_near_zero`] | convergence | Under centred input the latent mean stays at zero, so the value a fresh tracker starts from was already the right one. The encoding centres every bit for exactly this reason, and it is what lets the spread be measured about a fixed origin rather than a moving one. | +//! | [`subspace_evolution_does_not_dominate_latvar_transient`] | convergence | The latent spread's approach to its steady state is governed by the forgetting factor, not by the subspace still moving underneath it: an evolving basis and an effectively frozen one land in the same neighbourhood. Warm-up length can therefore be reasoned about from the decay alone. | +//! | [`production_noise_provides_reasonable_novelty_baseline`] | convergence | Novelty settles quickest of the four axes: a short schedule already places it within a few percent of where a far longer run leaves it. It is built from reconstruction error rather than from latent statistics, so it does not have to wait for the latent distribution to settle first — which is what makes a short warm-up useful before the other axes are ready. | +//! | [`deterministic_under_same_seed`] | convergence | The same seed replays the same run bit for bit, baseline for baseline. Nothing in the pipeline depends on iteration order over an unordered structure or on timing, so a convergence figure is a property of the configuration and the seed rather than of the machine that measured it. | + +//! What a whole tracker looks like once it has settled on noise. +//! +//! Warm-up feeds a fresh tracker synthetic traffic so that it arrives at +//! real work already knowing what ordinary looks like. That means more +//! than a mean per axis: the model has to have found how many directions +//! to keep, the latent distribution those directions are read against has +//! to have settled, and the four score baselines have to be calibrated +//! enough that ordinary traffic scores near zero and leaves headroom for +//! traffic that is not. +//! +//! The pieces settle in a definite order, and that order is why a bounded +//! warm-up works. Rank reaches its ceiling within a few adaptation +//! intervals and the captured energy plateaus near the threshold that +//! chose it, so the model is at full width long before the baselines are +//! anywhere near done — the bulk of a warm-up is spent learning score +//! distributions, not deciding on a geometry. The latent transient is +//! governed by the forgetting factor rather than by the subspace still +//! moving underneath it, so warm-up length can be reasoned about from the +//! decay alone. Coherence is the exception that proves the ordering: it +//! is a statement about pairs of directions, so it stays cold until there +//! are two to relate rather than converging onto the zeroes it would +//! otherwise be fed. +//! +//! Judging all this needs care about what convergence means. An +//! exponentially-weighted baseline never stops jittering, so settling is +//! established by agreement between an early block of rounds and a late +//! one, per axis and at that axis's own tolerance — for the spread as well +//! as the level, since a z-score divides by one and centres on the other. +//! And every measurement here is repeatable: the same seed replays the +//! same run bit for bit, so a convergence figure is a property of the +//! configuration rather than of the machine that took it. +//! +//! # §-references +//! +//! - §ALGO S-4.2 Phase 3 — Latent distribution cold→warm +//! - §ALGO S-6.1.1 — EWMA outlier filter +//! - §ALGO S-11.5 — Maturity tracking +//! - ADR-S-013 — Warm-up convergence benchmark +//! - ADR-S-014 — Subspace tracker visibility + +use rand::SeedableRng; +use rand::rngs::SmallRng; + +use super::convergence_common::{as_slices, block_mean_relative_error, cfg_test, find_settled_round, generate_noise}; +use crate::config::SentinelConfig; +use crate::sentinel::tracker::SubspaceTracker; + +// ════════════════════════════════════════════════════════════ +// Baseline convergence +// ════════════════════════════════════════════════════════════ + +/// A baseline counts as converged when an early block of rounds and a late one +/// agree, not when its round-to-round jitter has stopped. Steady-state jitter +/// is irreducible and differs by more than a hundredfold across the axes, so a +/// single-round criterion tight enough for the quietest axis would report the +/// noisiest as permanently unconverged long after it had reached its correct +/// level. +/// +/// ´claim:convergence:convergence-is-agreement-between-an-early-and-a-late-block-not-the-absence-of-jitter´ +/// ´test:crate:noise-baselines-converge-within-bound´ +#[test] +fn noise_baselines_converge_within_bound() { + let cfg = cfg_test(); + let dim = 128; + let mut tracker = SubspaceTracker::new(dim, &cfg, cfg.cusum_slow_decay); + let total_rounds = 300; + let mut rng = SmallRng::seed_from_u64(42); + + let mut traces: Vec<[f64; 4]> = Vec::with_capacity(total_rounds); + for _ in 0..total_rounds { + let noise = generate_noise(dim, cfg.noise_batch_size, &mut rng); + let report = tracker.observe(&as_slices(&noise), 0, true); + traces.push([ + report.scores.novelty.baseline.mean, + report.scores.displacement.baseline.mean, + report.scores.surprise.baseline.mean, + report.scores.coherence.baseline.mean, + ]); + } + + // Per-axis tolerances for the block-mean comparison. + // + // The budgets use the two-block difference scale derived beside + // `block_mean_relative_error`. With block length 100 and λ = 0.95, + // one block has factor √(39/100) ≈ 0.62 and two widely separated blocks + // have difference factor √2 × 0.62 ≈ 0.88. The stochastic-axis budgets + // exceed that scale by at least 2.5; novelty keeps a wider fixed minimum. + // + // Axis | EWMA CV | difference CV | tolerance + // --------------|---------|---------------|---------- + // Novelty | 0.07% | 0.06% | 2% + // Displacement | 3.4% | 3.00% | 10% + // Surprise | 6.5% | 5.73% | 15% + // Coherence | 10.0% | 8.83% | 25% + let axis_names = ["novelty", "displacement", "surprise", "coherence"]; + let tolerances = [0.02, 0.10, 0.15, 0.25]; + + // Early block: rounds [50, 150) — well past the EWMA transient + // (half-life ≈ 14 rounds at λ = 0.95, so by round 50 the bias + // is λ⁵⁰ ≈ 0.077 of its initial value). + // + // Late block: rounds [200, 300) — the reference steady state. + let early_start = 50; + let early_end = 150; + let late_start = 200; + let late_end = total_rounds; + + for (ax, (name, tol)) in axis_names.iter().zip(tolerances.iter()).enumerate() { + let err = block_mean_relative_error(&traces, ax, early_start, early_end, late_start, late_end); + if let Some(rel_err) = err { + assert!( + rel_err < *tol, + "{name} baseline not stationary: early vs late block \ + differ by {:.2}%, tolerance is {:.0}%", + rel_err * 100.0, + tol * 100.0, + ); + } + } +} + +/// The same block comparison holds for the spread and not only for the level. A +/// z-score divides by the spread, so a settled mean over an unsettled variance +/// would still be miscalibrated — this pins the half that calibration depends +/// on. +/// +/// (´claim:convergence:convergence-is-agreement-between-an-early-and-a-late-block-not-the-absence-of-jitter´) +/// ´test:crate:baseline-variance-converges-within-bound´ +#[test] +fn baseline_variance_converges_within_bound() { + let cfg = cfg_test(); + let dim = 128; + let mut tracker = SubspaceTracker::new(dim, &cfg, cfg.cusum_slow_decay); + let total_rounds = 300; + let mut rng = SmallRng::seed_from_u64(42); + + let mut traces: Vec<[f64; 4]> = Vec::with_capacity(total_rounds); + for _ in 0..total_rounds { + let noise = generate_noise(dim, cfg.noise_batch_size, &mut rng); + let report = tracker.observe(&as_slices(&noise), 0, true); + traces.push([ + report.scores.novelty.baseline.variance, + report.scores.displacement.baseline.variance, + report.scores.surprise.baseline.variance, + report.scores.coherence.baseline.variance, + ]); + } + + // Variance has higher CV than mean, so use more generous tolerances. + // Coherence variance is a 4th-order statistic (products of normals) + // with very high inherent variability. + let axis_names = ["novelty", "displacement", "surprise", "coherence"]; + let tolerances = [0.10, 0.20, 0.30, 0.50]; + + let early_start = 50; + let early_end = 150; + let late_start = 200; + let late_end = total_rounds; + + for (ax, (name, tol)) in axis_names.iter().zip(tolerances.iter()).enumerate() { + let err = block_mean_relative_error(&traces, ax, early_start, early_end, late_start, late_end); + if let Some(rel_err) = err { + assert!( + rel_err < *tol, + "{name} baseline variance not stationary: early vs late block \ + differ by {:.2}%, tolerance is {:.0}%", + rel_err * 100.0, + tol * 100.0, + ); + } + } +} + +/// Holding the subspace still never makes the baselines settle later than +/// letting it evolve. Scores are measured against the model, so a model that is +/// itself still moving is a second source of variation on top of the input; +/// removing it can only help. +/// +/// ´claim:convergence:a-moving-subspace-can-only-delay-the-baselines-never-hasten-them´ +/// ´test:crate:frozen-subspace-converges-at-least-as-fast´ +#[test] +fn frozen_subspace_converges_at_least_as_fast() { + let dim = 128; + let total_rounds = 300; + let seed = 42; + + let run = |cfg: &SentinelConfig| -> usize { + let mut tracker = SubspaceTracker::new(dim, cfg, cfg.cusum_slow_decay); + let mut rng = SmallRng::seed_from_u64(seed); + let mut traces = Vec::with_capacity(total_rounds); + for _ in 0..total_rounds { + let noise = generate_noise(dim, cfg.noise_batch_size, &mut rng); + let report = tracker.observe(&as_slices(&noise), 0, true); + traces.push([ + report.scores.novelty.baseline.mean, + report.scores.displacement.baseline.mean, + report.scores.surprise.baseline.mean, + report.scores.coherence.baseline.mean, + ]); + } + let reference = *traces.last().unwrap(); + find_settled_round(&traces, &reference, 0.05, 3).unwrap_or(total_rounds) + }; + + let cfg_frozen = SentinelConfig { + rank_update_interval: 10_000, + ..cfg_test() + }; + + let r_frozen = run(&cfg_frozen); + let r_normal = run(&cfg_test()); + + assert!( + r_frozen <= r_normal, + "frozen subspace should converge at least as fast: frozen={r_frozen}, normal={r_normal}" + ); +} + +// ════════════════════════════════════════════════════════════ +// Rank adaptation +// ════════════════════════════════════════════════════════════ + +/// Rank climbs to its ceiling within a few adaptation intervals, one step at +/// each. The model is therefore at full width long before the baselines settle, +/// so the bulk of a warm-up is spent learning score distributions rather than +/// discovering how many directions to keep. +/// +/// ´claim:convergence:rank-reaches-its-ceiling-long-before-the-baselines-settle´ +/// ´test:crate:rank-reaches-max-within-expected-rounds´ +#[test] +fn rank_reaches_max_within_expected_rounds() { + let cfg = cfg_test(); + let dim = 128; + let mut tracker = SubspaceTracker::new(dim, &cfg, cfg.cusum_slow_decay); + let mut rng = SmallRng::seed_from_u64(42); + + let mut ranks: Vec = Vec::with_capacity(100); + for _ in 0..100 { + let noise = generate_noise(dim, cfg.noise_batch_size, &mut rng); + let report = tracker.observe(&as_slices(&noise), 0, true); + ranks.push(report.rank); + } + + let first_rank2 = ranks.iter().position(|&r| r >= 2); + let r = first_rank2.expect("rank should reach 2 within 100 noise rounds"); + assert!(r <= 20, "rank should reach 2 within 20 noise rounds, got {r}"); +} + +/// Coherence is a statement about pairs of latent directions, so below two +/// directions there are no pairs and the axis does not exist. Its baseline is +/// held at the cold placeholder rather than being fed the identically zero +/// scores, and it enters through the cold-start path the moment a second +/// direction appears — otherwise it would converge onto zero and then have to +/// unlearn it. +/// +/// ´claim:convergence:the-coherence-baseline-stays-cold-until-there-are-two-directions-to-relate´ +/// ´test:crate:coherence-activates-at-rank-two´ +#[test] +fn coherence_activates_at_rank_two() { + let cfg = cfg_test(); + let dim = 128; + let mut tracker = SubspaceTracker::new(dim, &cfg, cfg.cusum_slow_decay); + let mut rng = SmallRng::seed_from_u64(42); + + let mut coherence_means: Vec = Vec::new(); + let mut ranks: Vec = Vec::new(); + + for _ in 0..100 { + let noise = generate_noise(dim, cfg.noise_batch_size, &mut rng); + let report = tracker.observe(&as_slices(&noise), 0, true); + coherence_means.push(report.scores.coherence.baseline.mean); + ranks.push(report.rank); + } + + // Before rank 2: coherence baseline stays at cold default. + let first_rank2 = ranks.iter().position(|&r| r >= 2).unwrap_or(100); + for r in 0..first_rank2.min(100) { + if ranks[r] < 2 { + assert!( + (coherence_means[r] - 1.0).abs() < 1e-10, + "round {r}: coherence mean should be cold (1.0) at rank {}, got {:.6}", + ranks[r], + coherence_means[r] + ); + } + } + + // After rank 2: coherence should evolve away from 1.0. + if first_rank2 + 5 < 100 { + let late_mean = coherence_means[first_rank2 + 5]; + assert!( + (late_mean - 1.0).abs() > 1e-6, + "coherence should evolve after reaching rank 2, still at {late_mean:.6}" + ); + } +} + +/// The share of energy the retained directions capture settles onto a narrow +/// plateau near the threshold that chose the rank. That ratio is the quantity +/// rank adaptation reads, so its settling is what stops the rank from +/// oscillating. +/// +/// ´claim:convergence:the-captured-energy-settles-onto-a-plateau-near-the-threshold-that-chose-the-rank´ +/// ´test:crate:energy-ratio-stabilises-under-noise´ +#[test] +fn energy_ratio_stabilises_under_noise() { + let cfg = cfg_test(); + let dim = 128; + let mut tracker = SubspaceTracker::new(dim, &cfg, cfg.cusum_slow_decay); + let mut rng = SmallRng::seed_from_u64(42); + + let total_rounds = 200; + let mut energies: Vec = Vec::with_capacity(total_rounds); + for _ in 0..total_rounds { + let noise = generate_noise(dim, cfg.noise_batch_size, &mut rng); + let report = tracker.observe(&as_slices(&noise), 0, true); + energies.push(report.energy_ratio); + } + + // After rank reaches max, energy ratio should be stable. + // Compare the last 50 rounds: max-min spread should be small. + let tail = &energies[total_rounds - 50..]; + let min_e = tail.iter().copied().fold(f64::INFINITY, f64::min); + let max_e = tail.iter().copied().fold(f64::NEG_INFINITY, f64::max); + let spread = max_e - min_e; + + assert!( + spread < 0.05, + "energy ratio not stable in last 50 rounds: spread = {spread:.4} (min={min_e:.4}, max={max_e:.4})" + ); + assert!( + min_e >= cfg.energy_threshold * 0.9, + "energy ratio ({min_e:.4}) should be near the threshold ({:.2})", + cfg.energy_threshold + ); +} + +// ════════════════════════════════════════════════════════════ +// Report snapshot semantics +// ════════════════════════════════════════════════════════════ + +/// A report carries the baseline the batch was scored against, never the one +/// the batch produced: the first report shows the cold placeholder even though +/// the internal state has already moved, and the second shows exactly what the +/// first batch left behind. A batch can therefore never partly explain itself +/// away, and a reader can reconstruct the comparison that produced the scores. +/// +/// ´claim:convergence:a-report-carries-the-baseline-the-batch-was-scored-against-not-the-one-it-produced´ +/// ´test:crate:report-baseline-is-pre-update-snapshot´ +#[test] +fn report_baseline_is_pre_update_snapshot() { + let cfg = cfg_test(); + let dim = 128; + let mut tracker = SubspaceTracker::new(dim, &cfg, cfg.cusum_slow_decay); + let mut rng = SmallRng::seed_from_u64(42); + + // First observe: EWMA is cold → report shows cold default (1.0). + let noise1 = generate_noise(dim, cfg.noise_batch_size, &mut rng); + let report1 = tracker.observe(&as_slices(&noise1), 0, true); + + assert!( + (report1.scores.novelty.baseline.mean - 1.0).abs() < 1e-10, + "first report should have cold baseline mean=1.0, got {}", + report1.scores.novelty.baseline.mean + ); + + // Internal EWMA is now warm. + let bl = tracker.axis_baselines(); + assert!( + (bl.novelty_mean - 1.0).abs() > 1e-6, + "after first observe, internal EWMA should have moved from 1.0, got {}", + bl.novelty_mean + ); + + // Second observe: report baseline matches post-first-update state. + let noise2 = generate_noise(dim, cfg.noise_batch_size, &mut rng); + let report2 = tracker.observe(&as_slices(&noise2), 0, true); + + assert!( + (report2.scores.novelty.baseline.mean - bl.novelty_mean).abs() < 1e-10, + "second report baseline ({:.6}) should match post-first-update state ({:.6})", + report2.scores.novelty.baseline.mean, + bl.novelty_mean + ); +} + +/// Measured against a settled baseline, ordinary traffic scores near zero on +/// average and stays unremarkable on every single round. The baselines are +/// calibrated and not merely stable: a systematic offset would mean every batch +/// looked mildly anomalous, leaving no headroom to signal one that genuinely +/// was. +/// +/// ´claim:convergence:against-a-settled-baseline-ordinary-traffic-scores-near-zero-and-never-extreme´ +/// ´test:crate:z-scores-stay-bounded-at-steady-state´ +#[test] +#[allow(clippy::cast_precision_loss)] +fn z_scores_stay_bounded_at_steady_state() { + let cfg = cfg_test(); + let dim = 128; + let mut tracker = SubspaceTracker::new(dim, &cfg, cfg.cusum_slow_decay); + let mut rng = SmallRng::seed_from_u64(42); + + // Warm up. + for _ in 0..200 { + let noise = generate_noise(dim, cfg.noise_batch_size, &mut rng); + tracker.observe(&as_slices(&noise), 0, true); + } + + // Collect z-scores over the next 100 rounds. + let mut z_traces: Vec<[f64; 4]> = Vec::with_capacity(100); + for _ in 0..100 { + let noise = generate_noise(dim, cfg.noise_batch_size, &mut rng); + let report = tracker.observe(&as_slices(&noise), 0, true); + z_traces.push([ + report.scores.novelty.mean_z_score, + report.scores.displacement.mean_z_score, + report.scores.surprise.mean_z_score, + report.scores.coherence.mean_z_score, + ]); + } + + let axis_names = ["novelty", "displacement", "surprise", "coherence"]; + for (ax, name) in axis_names.iter().enumerate() { + let mean_z: f64 = z_traces.iter().map(|t| t[ax]).sum::() / z_traces.len() as f64; + let max_abs_z: f64 = z_traces.iter().map(|t| t[ax].abs()).fold(0.0_f64, f64::max); + // Average z-score should be near zero over many rounds. + assert!(mean_z.abs() < 1.5, "{name} mean z-score = {mean_z:.2} — expected near zero"); + // Individual z-scores should not be extreme under noise. + assert!( + max_abs_z < 5.0, + "{name} max |z| = {max_abs_z:.2} — expected < 5.0 under noise" + ); + } +} + +// ════════════════════════════════════════════════════════════ +// Latent statistics steady state +// ════════════════════════════════════════════════════════════ + +/// The latent spread settles far below the value a freshly constructed tracker +/// holds. That gap is why the first batch seeds the spread instead of blending +/// toward it: starting an order of magnitude high would suppress the surprise +/// axis for as long as the gap took to decay. +/// +/// ´claim:convergence:the-latent-spread-settles-far-below-its-constructed-placeholder´ +/// ´test:crate:latent-variance-reaches-steady-state´ +#[test] +fn latent_variance_reaches_steady_state() { + let cfg = cfg_test(); + let dim = 128; + let mut tracker = SubspaceTracker::new(dim, &cfg, cfg.cusum_slow_decay); + let mut rng = SmallRng::seed_from_u64(42); + + for _ in 0..500 { + let noise = generate_noise(dim, cfg.noise_batch_size, &mut rng); + tracker.observe(&as_slices(&noise), 0, true); + } + + let lat_vars = tracker.latent_var(); + for (j, &v) in lat_vars.iter().enumerate() { + assert!(v < 0.5, "lat_var[{j}] = {v:.6} should be ≪ 1.0 at steady state"); + } +} + +/// Under centred input the latent mean stays at zero, so the value a fresh +/// tracker starts from was already the right one. The encoding centres every +/// bit for exactly this reason, and it is what lets the spread be measured +/// about a fixed origin rather than a moving one. +/// +/// ´claim:convergence:centred-input-leaves-the-latent-mean-where-a-fresh-tracker-starts-it´ +/// ´test:crate:latent-mean-stays-near-zero´ +#[test] +fn latent_mean_stays_near_zero() { + let cfg = cfg_test(); + let dim = 128; + let mut tracker = SubspaceTracker::new(dim, &cfg, cfg.cusum_slow_decay); + let mut rng = SmallRng::seed_from_u64(42); + + for _ in 0..500 { + let noise = generate_noise(dim, cfg.noise_batch_size, &mut rng); + tracker.observe(&as_slices(&noise), 0, true); + } + + let lat_means = tracker.latent_mean(); + for (j, &m) in lat_means.iter().enumerate() { + assert!(m.abs() < 0.1, "lat_mean[{j}] = {m:.6} should be near zero for centred noise"); + } +} + +/// The latent spread's approach to its steady state is governed by the +/// forgetting factor, not by the subspace still moving underneath it: an +/// evolving basis and an effectively frozen one land in the same neighbourhood. +/// Warm-up length can therefore be reasoned about from the decay alone. +/// +/// ´claim:convergence:the-latent-transient-is-governed-by-the-forgetting-factor-not-by-the-subspace-still-moving´ +/// ´test:crate:subspace-evolution-does-not-dominate-latvar-transient´ +#[test] +fn subspace_evolution_does_not_dominate_latvar_transient() { + let dim = 128; + let total_rounds = 200; + + let run = |rank_update_interval: u64| -> Vec { + let cfg = SentinelConfig { + rank_update_interval, + ..cfg_test() + }; + let mut tracker = SubspaceTracker::new(dim, &cfg, cfg.cusum_slow_decay); + let mut rng = SmallRng::seed_from_u64(42); + let mut var_trace = Vec::with_capacity(total_rounds); + for _ in 0..total_rounds { + let noise = generate_noise(dim, cfg.noise_batch_size, &mut rng); + tracker.observe(&as_slices(&noise), 0, true); + var_trace.push(tracker.latent_var().first().copied().unwrap_or(0.0)); + } + var_trace + }; + + let normal_trace = run(5); // normal rank adaptation + let frozen_trace = run(10_000); // effectively frozen + + let normal_ref = *normal_trace.last().unwrap(); + let frozen_ref = *frozen_trace.last().unwrap(); + let diff_pct = (normal_ref - frozen_ref).abs() / normal_ref.abs().max(1e-12) * 100.0; + + assert!( + diff_pct < 50.0, + "lat_var steady states should be similar: normal={normal_ref:.6}, frozen={frozen_ref:.6} ({diff_pct:.1}% diff)" + ); +} + +// ════════════════════════════════════════════════════════════ +// Production noise-rounds quality +// ════════════════════════════════════════════════════════════ + +/// Novelty settles quickest of the four axes: a short schedule already places +/// it within a few percent of where a far longer run leaves it. It is built +/// from reconstruction error rather than from latent statistics, so it does not +/// have to wait for the latent distribution to settle first — which is what +/// makes a short warm-up useful before the other axes are ready. +/// +/// ´claim:convergence:novelty-settles-quickest-because-it-does-not-wait-on-the-latent-distribution´ +/// ´test:crate:production-noise-provides-reasonable-novelty-baseline´ +#[test] +fn production_noise_provides_reasonable_novelty_baseline() { + let cfg = cfg_test(); + let dim = 128; + let mut tracker = SubspaceTracker::new(dim, &cfg, cfg.cusum_slow_decay); + let mut rng = SmallRng::seed_from_u64(42); + + let mut baselines_at_50 = [0.0_f64; 4]; + for round in 0..50 { + let noise = generate_noise(dim, cfg.noise_batch_size, &mut rng); + let report = tracker.observe(&as_slices(&noise), 0, true); + if round == 49 { + baselines_at_50 = [ + report.scores.novelty.baseline.mean, + report.scores.displacement.baseline.mean, + report.scores.surprise.baseline.mean, + report.scores.coherence.baseline.mean, + ]; + } + } + + // Run 250 more to get the reference. + for _ in 50..300 { + let noise = generate_noise(dim, cfg.noise_batch_size, &mut rng); + tracker.observe(&as_slices(&noise), 0, true); + } + + let bl = tracker.axis_baselines(); + let reference_novelty = bl.novelty_mean; + + if reference_novelty.abs() > 1e-12 { + let novelty_error = (baselines_at_50[0] - reference_novelty).abs() / reference_novelty.abs(); + assert!( + novelty_error < 0.05, + "novelty at round 50 is {:.1}% off — should be <5%", + novelty_error * 100.0 + ); + } +} + +// ════════════════════════════════════════════════════════════ +// Determinism +// ════════════════════════════════════════════════════════════ + +/// The same seed replays the same run bit for bit, baseline for baseline. +/// Nothing in the pipeline depends on iteration order over an unordered +/// structure or on timing, so a convergence figure is a property of the +/// configuration and the seed rather than of the machine that measured it. +/// +/// ´claim:convergence:the-same-seed-replays-the-same-run-bit-for-bit´ +/// ´test:crate:deterministic-under-same-seed´ +#[test] +fn deterministic_under_same_seed() { + let run = || { + let cfg = cfg_test(); + let dim = 128; + let mut tracker = SubspaceTracker::new(dim, &cfg, cfg.cusum_slow_decay); + let mut rng = SmallRng::seed_from_u64(42); + + let mut baselines = Vec::with_capacity(100); + for _ in 0..100 { + let noise = generate_noise(dim, cfg.noise_batch_size, &mut rng); + let report = tracker.observe(&as_slices(&noise), 0, true); + baselines.push([ + report.scores.novelty.baseline.mean, + report.scores.displacement.baseline.mean, + report.scores.surprise.baseline.mean, + report.scores.coherence.baseline.mean, + ]); + } + baselines + }; + + let a = run(); + let b = run(); + + assert_eq!(a.len(), b.len()); + for (i, (ra, rb)) in a.iter().zip(b.iter()).enumerate() { + for ax in 0..4 { + assert!( + (ra[ax] - rb[ax]).abs() == 0.0, + "round {i} axis {ax}: run A = {}, run B = {} — not bit-identical", + ra[ax], + rb[ax] + ); + } + } +} diff --git a/packages/sentinel/src/tests/cusum.rs b/packages/sentinel/src/tests/cusum.rs new file mode 100644 index 000000000..b4ad2bc74 --- /dev/null +++ b/packages/sentinel/src/tests/cusum.rs @@ -0,0 +1,351 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +//! # Test index +//! +//! | Test | Area | Claim | +//! |------|------|-------| +//! | [`starts_at_zero`] | cusum | A fresh accumulator holds no evidence and has taken no steps. Drift is something that must be accumulated from observations, so a newly created axis starts owing the host nothing to explain. | +//! | [`accumulates_under_sustained_elevation`] | cusum | Evidence builds when batch means stay above the slow reference: an accumulator settled by a long run of ordinary batches grows once the scores are consistently elevated. Because each batch adds its remaining gap to the running sum, sustained elevation compounds — which is the point, since it separates a persistent shift from a single loud batch. | +//! | [`steps_since_reset_increments`] | cusum | Every update advances the step count by exactly one, whatever the batch contained and whether or not the gap contributed anything. The count is how long evidence has been gathering, so a host can read an accumulator value against the number of chances it had to grow rather than against nothing. | +//! | [`clamps_at_zero_when_below_baseline`] | cusum | A run of batches below the reference leaves the accumulator at zero rather than driving it negative. Quiet time banks no credit: the sum cannot go into debt during a lull and then have to be repaid before a genuine rise registers. Evidence of drift is always built from the present run, never netted against the past. | +//! | [`allowance_absorbs_noise`] | cusum | The allowance is a dead band that ordinary variation does not cross: against a slow baseline with real spread, a generous allowance leaves slightly elevated batches accumulating essentially nothing. Because the band is scaled by the baseline's own deviation rather than being an absolute score, a noisy cell tolerates more before it counts as drifting than a quiet one does. | +//! | [`allowance_uses_only_slow_variance`] | cusum | The dead band is exactly the configured sigma multiplier times the slow baseline's standard deviation. A known baseline therefore gives a known first step, with no unrelated stability constant widening the allowance. | +//! | [`resets_to_zero`] | cusum | A reset discards the accumulated evidence and the count of steps that built it together. Neither outlives the other, so a host acknowledging a regime change is not left reading a fresh sum against a stale step count. | +//! | [`reset_preserves_slow_baseline`] | cusum | What a reset does not touch is the slow baseline: its mean and spread come through unchanged. Acknowledging drift clears the evidence, not the reference the evidence was measured against — otherwise every acknowledgement would throw away a long-memory baseline that takes many batches to rebuild, and the axis would be blind while it re-converged. | +//! | [`reset_cold_clears_everything`] | cusum | Clearing goes further than resetting: the evidence, the step count and the slow baseline all return to their freshly-constructed state, the baseline back to its placeholders rather than to whatever it had drifted to. This is the operation for an axis that has ceased to exist — coherence when the rank falls too low, say — where keeping a reference learned under a geometry that no longer holds would be worse than having none. | +//! | [`seed_slow_from_aligns_baselines`] | cusum | The slow reference can be started from a fast baseline that has already converged, taking its mean and spread exactly. After noise is injected the two would otherwise disagree for a very long time — the slow baseline's memory is far too long to catch up — and every batch in between would register a gap that reflects the mismatch rather than any real drift. Seeding closes that gap in one step. | +//! | [`update_filtered_matches_update_no_clip`] | cusum | The pre-filtered entry point differs from the ordinary one only in who applies the outlier filter: given a clip wide enough that none would be rejected, both reach the same accumulated value. Moving the filter out to a shared pipeline therefore changes where clipping happens and not what drift means. | +//! | [`snapshot_reports_slow_baseline`] | cusum | The snapshot carries the slow reference itself, and that reference tracks the scores it was fed: after a run of batches at one level the reported mean sits near it. A host reading a report can therefore see what the drift was measured against, not merely how much of it accumulated. | + +//! Tests for [`CusumAccumulator`](crate::sentinel::cusum::CusumAccumulator) — +//! the one-sided drift accumulator each scoring axis owns. +//! +//! A single elevated batch is not drift. The accumulator answers a different +//! question from the baseline: not whether this batch is unusual, but whether +//! a run of batches has been consistently above the reference for long +//! enough to be evidence rather than noise. It measures each batch's mean +//! against a slow baseline, subtracts a noise allowance scaled to that +//! baseline's own spread, and adds what remains to a running sum. +//! +//! Two decisions shape it. The sum is clamped at zero, so a period below the +//! reference banks no credit against a later rise — evidence has to be built +//! afresh rather than offset. And the gap is measured before the slow +//! baseline sees the batch, so a batch is always scored against the state +//! that preceded it and can never partly explain itself away. +//! +//! Resetting and clearing are deliberately different operations. A reset +//! discards the accumulated evidence but keeps what the slow baseline +//! learned; clearing returns both to their freshly-constructed state, which +//! is what an axis that has ceased to exist requires. + +use crate::sentinel::cusum::*; + +// ─── Construction ─────────────────────────────────────────── + +/// A fresh accumulator holds no evidence and has taken no steps. Drift is +/// something that must be accumulated from observations, so a newly created +/// axis starts owing the host nothing to explain. +/// +/// ´claim:cusum:a-fresh-accumulator-holds-no-evidence-and-has-taken-no-steps´ +/// ´test:crate:starts-at-zero´ +#[test] +fn starts_at_zero() { + let c = CusumAccumulator::new(0.999); + let snap = c.snapshot(); + assert!((snap.accumulator).abs() < f64::EPSILON); + assert_eq!(snap.steps_since_reset, 0); +} + +// ─── Core accumulation ───────────────────────────────────── + +/// Evidence builds when batch means stay above the slow reference: an +/// accumulator settled by a long run of ordinary batches grows once the +/// scores are consistently elevated. Because each batch adds its remaining +/// gap to the running sum, sustained elevation compounds — which is the +/// point, since it separates a persistent shift from a single loud batch. +/// +/// ´claim:cusum:sustained-elevation-above-the-slow-reference-accumulates-as-evidence´ +/// ´test:crate:accumulates-under-sustained-elevation´ +#[test] +fn accumulates_under_sustained_elevation() { + let mut c = CusumAccumulator::new(0.999); + // Warm the slow baseline with normal-ish scores. + for _ in 0..20 { + c.update(&[1.0, 1.0, 1.0], 1.0, 0.5, 3.0); + } + let before = c.snapshot().accumulator; + + // Now feed consistently elevated scores. + for _ in 0..10 { + c.update(&[5.0, 5.0, 5.0], 5.0, 0.5, 3.0); + } + assert!( + c.snapshot().accumulator > before, + "accumulator should grow under sustained elevation" + ); +} + +/// Every update advances the step count by exactly one, whatever the batch +/// contained and whether or not the gap contributed anything. The count is +/// how long evidence has been gathering, so a host can read an accumulator +/// value against the number of chances it had to grow rather than against +/// nothing. +/// +/// ´claim:cusum:every-update-advances-the-step-count-by-one-whatever-the-batch-contributed´ +/// ´test:crate:steps-since-reset-increments´ +#[test] +fn steps_since_reset_increments() { + let mut c = CusumAccumulator::new(0.999); + for i in 1..=5 { + c.update(&[1.0], 1.0, 0.5, 3.0); + assert_eq!(c.snapshot().steps_since_reset, i); + } +} + +// ─── Clamping ─────────────────────────────────────────────── + +/// A run of batches below the reference leaves the accumulator at zero rather +/// than driving it negative. Quiet time banks no credit: the sum cannot go +/// into debt during a lull and then have to be repaid before a genuine rise +/// registers. Evidence of drift is always built from the present run, never +/// netted against the past. +/// +/// ´claim:cusum:a-quiet-run-banks-no-credit-because-the-sum-is-clamped-at-zero´ +/// ´test:crate:clamps-at-zero-when-below-baseline´ +#[test] +fn clamps_at_zero_when_below_baseline() { + let mut c = CusumAccumulator::new(0.999); + // Warm with high values. + for _ in 0..20 { + c.update(&[10.0, 10.0], 10.0, 0.5, 3.0); + } + c.reset(); + + // Feed low values — gap is negative, accumulator stays at zero. + for _ in 0..10 { + c.update(&[0.1, 0.1], 0.1, 0.5, 3.0); + } + assert!( + (c.snapshot().accumulator).abs() < f64::EPSILON, + "accumulator should not go below zero" + ); +} + +// ─── Allowance ────────────────────────────────────────────── + +/// The allowance is a dead band that ordinary variation does not cross: +/// against a slow baseline with real spread, a generous allowance leaves +/// slightly elevated batches accumulating essentially nothing. Because the +/// band is scaled by the baseline's own deviation rather than being an +/// absolute score, a noisy cell tolerates more before it counts as drifting +/// than a quiet one does. +/// +/// ´claim:cusum:the-allowance-is-a-dead-band-scaled-to-the-baselines-own-spread´ +/// ´test:crate:allowance-absorbs-noise´ +#[test] +fn allowance_absorbs_noise() { + // With a large allowance, small deviations should not accumulate + // when the slow baseline has meaningful variance. + let mut c = CusumAccumulator::new(0.999); + + // Warm with varied data so the slow baseline has real variance. + for _ in 0..20 { + c.update(&[0.5, 1.0, 1.5], 1.0, 2.0, 3.0); + } + c.reset(); + + // Feed slightly elevated scores — allowance should absorb them. + for _ in 0..10 { + c.update(&[1.1, 1.2, 1.3], 1.2, 2.0, 3.0); + } + assert!( + c.snapshot().accumulator < 0.1, + "generous allowance should absorb small deviations, got {}", + c.snapshot().accumulator, + ); +} + +/// The dead band is exactly the configured sigma multiplier times the slow baseline's standard deviation. A known baseline therefore gives a known first step, with no unrelated stability constant widening the allowance. +/// +/// ´claim:cusum:the-allowance-is-exactly-the-sigma-multiplier-times-the-slow-baseline-standard-deviation´ +/// ´test:crate:allowance-uses-only-slow-variance´ +#[test] +fn allowance_uses_only_slow_variance() { + let mut c = CusumAccumulator::new(0.999); + + c.update_filtered(&[3.0, 3.0], 3.0, 0.5); + + assert_eq!( + c.snapshot().accumulator.to_bits(), + 1.5_f64.to_bits(), + "the cold slow baseline has mean one and variance one", + ); +} + +// ─── Reset ────────────────────────────────────────────────── + +/// A reset discards the accumulated evidence and the count of steps that +/// built it together. Neither outlives the other, so a host acknowledging a +/// regime change is not left reading a fresh sum against a stale step count. +/// +/// ´claim:cusum:a-reset-discards-the-evidence-and-the-count-that-built-it-together´ +/// ´test:crate:resets-to-zero´ +#[test] +fn resets_to_zero() { + let mut c = CusumAccumulator::new(0.999); + c.update(&[5.0, 5.0], 5.0, 0.0, 3.0); + c.update(&[5.0, 5.0], 5.0, 0.0, 3.0); + assert!(c.snapshot().accumulator > 0.0); + + c.reset(); + assert!((c.snapshot().accumulator).abs() < f64::EPSILON); + assert_eq!(c.snapshot().steps_since_reset, 0); +} + +/// What a reset does not touch is the slow baseline: its mean and spread come +/// through unchanged. Acknowledging drift clears the evidence, not the +/// reference the evidence was measured against — otherwise every +/// acknowledgement would throw away a long-memory baseline that takes many +/// batches to rebuild, and the axis would be blind while it re-converged. +/// +/// ´claim:cusum:a-reset-clears-the-evidence-without-discarding-the-reference-it-was-measured-against´ +/// ´test:crate:reset-preserves-slow-baseline´ +#[test] +fn reset_preserves_slow_baseline() { + let mut c = CusumAccumulator::new(0.999); + for _ in 0..20 { + c.update(&[5.0, 5.0], 5.0, 0.5, 3.0); + } + let baseline_before = c.snapshot().slow_baseline; + + c.reset(); + + let baseline_after = c.snapshot().slow_baseline; + assert!( + (baseline_before.mean - baseline_after.mean).abs() < f64::EPSILON, + "reset() should preserve the slow baseline mean" + ); + assert!( + (baseline_before.variance - baseline_after.variance).abs() < f64::EPSILON, + "reset() should preserve the slow baseline variance" + ); +} + +/// Clearing goes further than resetting: the evidence, the step count and the +/// slow baseline all return to their freshly-constructed state, the baseline +/// back to its placeholders rather than to whatever it had drifted to. This +/// is the operation for an axis that has ceased to exist — coherence when the +/// rank falls too low, say — where keeping a reference learned under a +/// geometry that no longer holds would be worse than having none. +/// +/// ´claim:cusum:clearing-returns-the-evidence-the-count-and-the-reference-all-to-their-constructed-state´ +/// ´test:crate:reset-cold-clears-everything´ +#[test] +fn reset_cold_clears_everything() { + let mut c = CusumAccumulator::new(0.999); + for _ in 0..20 { + c.update(&[5.0, 5.0], 5.0, 0.0, 3.0); + } + assert!(c.snapshot().accumulator > 0.0); + // Slow baseline should have drifted away from the cold defaults. + assert!((c.snapshot().slow_baseline.mean - 1.0).abs() > 0.1); + + c.reset_cold(); + + let snap = c.snapshot(); + assert!((snap.accumulator).abs() < f64::EPSILON); + assert_eq!(snap.steps_since_reset, 0); + // Cold defaults: mean = 1.0, variance = 1.0. + assert!( + (snap.slow_baseline.mean - 1.0).abs() < f64::EPSILON, + "reset_cold should restore cold-default mean" + ); + assert!( + (snap.slow_baseline.variance - 1.0).abs() < f64::EPSILON, + "reset_cold should restore cold-default variance" + ); +} + +// ─── Seeding ──────────────────────────────────────────────── + +/// The slow reference can be started from a fast baseline that has already +/// converged, taking its mean and spread exactly. After noise is injected the +/// two would otherwise disagree for a very long time — the slow baseline's +/// memory is far too long to catch up — and every batch in between would +/// register a gap that reflects the mismatch rather than any real drift. +/// Seeding closes that gap in one step. +/// +/// ´claim:cusum:the-slow-reference-can-be-seeded-from-a-converged-fast-baseline-so-the-two-start-in-agreement´ +/// ´test:crate:seed-slow-from-aligns-baselines´ +#[test] +fn seed_slow_from_aligns_baselines() { + use crate::ewma::EwmaStats; + + let mut fast = EwmaStats::new(0.95); + for _ in 0..20 { + fast.update(&[3.0, 3.5, 2.5], 3.0); + } + + let mut c = CusumAccumulator::new(0.999); + c.seed_slow_from(&fast); + + let snap = c.snapshot(); + assert!( + (snap.slow_baseline.mean - fast.mean()).abs() < f64::EPSILON, + "seed_slow_from should copy mean from fast EWMA" + ); + assert!( + (snap.slow_baseline.variance - fast.variance()).abs() < f64::EPSILON, + "seed_slow_from should copy variance from fast EWMA" + ); +} + +// ─── Filtered update ──────────────────────────────────────── + +/// The pre-filtered entry point differs from the ordinary one only in who +/// applies the outlier filter: given a clip wide enough that none would be +/// rejected, both reach the same accumulated value. Moving the filter out to +/// a shared pipeline therefore changes where clipping happens and not what +/// drift means. +/// +/// ´claim:cusum:the-pre-filtered-path-differs-from-the-ordinary-one-only-in-who-applies-the-filter´ +/// ´test:crate:update-filtered-matches-update-no-clip´ +#[test] +fn update_filtered_matches_update_no_clip() { + let mut a = CusumAccumulator::new(0.999); + let mut b = CusumAccumulator::new(0.999); + let scores = &[1.0, 1.1, 0.9, 1.05, 0.95]; + #[allow(clippy::cast_precision_loss)] + let mean = scores.iter().sum::() / scores.len() as f64; + + a.update(scores, mean, 0.5, 100.0); + b.update_filtered(scores, mean, 0.5); + + assert!((a.snapshot().accumulator - b.snapshot().accumulator).abs() < 1e-12); +} + +// ─── Snapshot ─────────────────────────────────────────────── + +/// The snapshot carries the slow reference itself, and that reference tracks +/// the scores it was fed: after a run of batches at one level the reported +/// mean sits near it. A host reading a report can therefore see what the +/// drift was measured against, not merely how much of it accumulated. +/// +/// ´claim:cusum:a-snapshot-carries-the-slow-reference-and-that-reference-tracks-the-scores-it-was-fed´ +/// ´test:crate:snapshot-reports-slow-baseline´ +#[test] +fn snapshot_reports_slow_baseline() { + let mut c = CusumAccumulator::new(0.999); + for _ in 0..20 { + c.update(&[4.0, 4.0], 4.0, 0.5, 3.0); + } + + let snap = c.snapshot(); + // After warming, baseline mean should be near 4.0. + assert!( + (snap.slow_baseline.mean - 4.0).abs() < 0.5, + "slow baseline mean should track input; got {}", + snap.slow_baseline.mean, + ); +} diff --git a/packages/sentinel/src/tests/ewma.rs b/packages/sentinel/src/tests/ewma.rs new file mode 100644 index 000000000..51863bf57 --- /dev/null +++ b/packages/sentinel/src/tests/ewma.rs @@ -0,0 +1,577 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +//! # Test index +//! +//! | Test | Area | Claim | +//! |------|------|-------| +//! | [`starts_cold`] | ewma | A newly constructed baseline is cold, and the mean and spread it reports are placeholders of one rather than anything measured. They are deliberately wide so that a value scored before any real data has arrived cannot come back with an extreme departure. | +//! | [`first_update_warms`] | ewma | One update is the whole of warming: a single batch takes the baseline out of its cold state for good. There is no minimum sample count to reach and no separate warming phase to wait out at this level. | +//! | [`first_update_sets_mean_to_batch_mean`] | ewma | The first batch is adopted outright: the mean becomes the batch's own mean exactly, with no trace of the placeholder blended in. Decaying towards the first real data instead would leave the baseline anchored for many batches to a value that was never an observation. | +//! | [`first_update_single_value_keeps_unit_variance`] | ewma | A batch of one carries a mean but no spread, so the mean is taken and the variance is left exactly as it stood. Computing a deviation from a single sample would give zero — a spread the data does not support, and one that would make every subsequent value look like an outlier. | +//! | [`first_update_multi_value_sets_sample_variance`] | ewma | A first batch of more than one sets the spread from its own mean squared deviation, taken about the batch mean and divided by the count with no correction applied. Both the centre and the spread the baseline starts from are therefore measurements rather than defaults. | +//! | [`reset_cold_restores_placeholder_state`] | ewma | Resetting a warm baseline returns it to exactly the state construction left it in — placeholders restored and warmth withdrawn — rather than merely clearing a flag over learned numbers. What it learned is discarded, not hidden. | +//! | [`reset_cold_allows_re_warming`] | ewma | After a reset the next batch is adopted outright, exactly as the very first one was: the mean lands on the new batch's value with nothing of the discarded baseline pulling it back. Resetting therefore genuinely re-starts the baseline rather than leaving it to decay out of its old position. | +//! | [`seed_from_copies_warm_state`] | ewma | Seeding transfers both what a baseline learned and the fact that it learned it: the receiver takes the source's mean and spread and becomes warm. This is how a slow baseline is started from a fast one that has already converged, so the pair begin in agreement instead of the slow one spending its warm-up disagreeing with a baseline that is already right. | +//! | [`seed_from_cold_source_does_not_warm_target`] | ewma | Warmth is never manufactured by seeding. A cold source hands over its placeholder numbers but leaves the receiver cold, so a baseline seeded before anything was learned still takes the cold path on its own first batch rather than blending against values nothing measured. | +//! | [`seed_from_cold_source_withdraws_warmth_from_a_warm_receiver`] | ewma | A receiver that had learned something and is then seeded from a source that had not comes back cold, rather than keeping its own warmth over the placeholders it has just been handed. Warmth belongs to the baseline being transferred and not to the receiver: it is what says whether those two numbers were measured or were the pair a fresh baseline starts from. A receiver left warm over them would clip and score against a notion of normal nothing had observed, and the cold path that exists to replace exactly that state would never run again. | +//! | [`update_empty_is_noop`] | ewma | A batch with nothing in it leaves both the mean and the spread exactly where they were. An idle interval is therefore not a data point: the baseline does not drift simply because time passed without observations. | +//! | [`outliers_are_rejected`] | ewma | A warm baseline refuses values above its own ceiling before it learns anything, and a batch consisting entirely of such values moves it not at all. This is the poisoning defence: an attacker cannot walk the notion of normal upward by feeding extremes, because the extremes are precisely what never reaches the baseline. | +//! | [`update_single_value_does_not_update_variance`] | ewma | cites (´claim:ewma:a-batch-of-one-carries-no-spread-so-the-variance-is-left-untouched´) | +//! | [`update_skips_clipping_when_cold`] | ewma | While cold there is no ceiling to clip against, so nothing is rejected: a first batch far above the placeholder mean is accepted whole and becomes the baseline. The placeholders are not a real notion of normal, and filtering against them would let an arbitrary constant decide which of the first real observations the sentinel was allowed to see. | +//! | [`update_raw_empty_is_noop`] | ewma | cites (´claim:ewma:an-empty-batch-teaches-the-baseline-nothing´) | +//! | [`update_raw_matches_update_when_no_clipping`] | ewma | The unclipped path is the clipped one with only the filter removed: given a clip so wide that nothing could be rejected, the two produce the same mean and the same spread. Callers that do their own outlier rejection get identical arithmetic, so the two entry points cannot drift into disagreeing about what a batch means. | +//! | [`decays_toward_new_data`] | ewma | Sustained new data pulls the baseline away from what it learned before: a mean established at one level and then fed a different one repeatedly ends up near the new level. The baseline tracks the present rather than averaging over all history, which is what makes it a moving notion of normal rather than a permanent one. | +//! | [`higher_decay_forgets_slower`] | ewma | The decay factor is the length of the baseline's memory. Two baselines started at the same level and fed the same contradicting data diverge in the expected direction: the one with the higher factor still holds more of the original level. This is what lets a fast and a slow baseline over the same stream disagree usefully, which is the whole basis of drift detection. | +//! | [`z_score_is_zero_at_mean`] | ewma | A value sitting at the baseline scores essentially nothing. The z-score measures signed departure from what the baseline expects, so agreement with it is the origin of the scale rather than a point somewhere along it. | +//! | [`z_score_positive_above_mean`] | ewma | cites (´claim:ewma:the-z-score-is-signed-departure-from-the-baseline-and-zero-at-it´) | +//! | [`z_score_negative_below_mean`] | ewma | cites (´claim:ewma:the-z-score-is-signed-departure-from-the-baseline-and-zero-at-it´) | +//! | [`ceiling_returns_infinity_when_cold`] | ewma | cites (´claim:ewma:while-cold-there-is-no-ceiling-to-clip-against-so-nothing-is-rejected´) | +//! | [`ceiling_returns_mean_plus_sigmas_when_warm`] | ewma | Once warm, the ceiling stands a fixed number of standard deviations above the mean — the requested multiple of the baseline's own square-rooted spread, added to its own centre. The threshold is therefore relative to what this cell has learned, not an absolute score chosen in advance for every cell alike. | +//! | [`clip_sigmas_affects_ceiling`] | ewma | The clip setting is a monotone dial on how much the baseline is willing to learn from: given the same elevated value, a baseline clipping at a wide multiple moves at least as far as one clipping tightly. Tightening the setting can only ever admit less, so an operator turning it down is trading responsiveness for poisoning resistance and never the reverse. | +//! | [`snapshot_matches_state`] | ewma | The snapshot a report carries holds the same mean and spread the baseline's own accessors report. What a caller reads out of a report is the state the engine is scoring against, not a rounded or separately derived summary of it. | +//! | [`variance_floor_is_respected`] | ewma | A batch of identical values has no deviation at all, yet the spread does not reach zero: a floor holds it above. Without it a perfectly quiet period would collapse the spread, the ceiling would close onto the mean, and every subsequent value — however ordinary — would be rejected as an outlier, leaving the baseline permanently frozen at the quiet level. | + +//! Unit tests for [`EwmaStats`](crate::ewma::EwmaStats) — the running mean +//! and spread every anomaly axis is scored against. +//! +//! A baseline has two lives. Cold, it holds placeholders rather than +//! measurements, and the first batch it sees is adopted outright: there is +//! nothing yet to blend with, and nothing to call an outlier against, so the +//! clip filter does not run. Warm, it blends each batch in by the decay +//! factor and refuses anything above its own ceiling first. +//! +//! That refusal is what makes the baseline hard to poison. Anomaly scores are +//! non-negative and right-skewed — an attacker inflates them and never +//! deflates them — so only the upper tail is clipped, and a batch consisting +//! entirely of outliers teaches the baseline nothing at all rather than +//! dragging it upward. The variance floor guards the same property from the +//! other side: a perfectly quiet period cannot collapse the spread to zero +//! and thereby turn every later value into an outlier. + +use crate::ewma::*; + +// ── Construction & initial state ──────────────────────────── + +/// A newly constructed baseline is cold, and the mean and spread it reports +/// are placeholders of one rather than anything measured. They are +/// deliberately wide so that a value scored before any real data has arrived +/// cannot come back with an extreme departure. +/// +/// ´claim:ewma:a-fresh-baseline-is-cold-and-holds-placeholders-not-measurements´ +/// ´test:crate:starts-cold´ +#[test] +fn starts_cold() { + let stats = EwmaStats::new(0.99); + assert!(!stats.is_warm()); + assert!((stats.mean() - 1.0).abs() < f64::EPSILON); + assert!((stats.variance() - 1.0).abs() < f64::EPSILON); +} + +// ── Cold → warm transition ────────────────────────────────── + +/// One update is the whole of warming: a single batch takes the baseline out +/// of its cold state for good. There is no minimum sample count to reach and +/// no separate warming phase to wait out at this level. +/// +/// ´claim:ewma:one-update-is-enough-to-turn-a-cold-baseline-warm´ +/// ´test:crate:first-update-warms´ +#[test] +fn first_update_warms() { + let mut stats = EwmaStats::new(0.99); + stats.update(&[2.0, 3.0, 4.0], 3.0); + assert!(stats.is_warm()); +} + +/// The first batch is adopted outright: the mean becomes the batch's own +/// mean exactly, with no trace of the placeholder blended in. Decaying +/// towards the first real data instead would leave the baseline anchored for +/// many batches to a value that was never an observation. +/// +/// ´claim:ewma:the-first-batch-is-adopted-outright-rather-than-blended-with-the-placeholder´ +/// ´test:crate:first-update-sets-mean-to-batch-mean´ +#[test] +fn first_update_sets_mean_to_batch_mean() { + let mut stats = EwmaStats::new(0.99); + stats.update(&[2.0, 4.0, 6.0], 3.0); + // Cold-path sets mean = batch mean = (2+4+6)/3 = 4.0 + assert!((stats.mean() - 4.0).abs() < f64::EPSILON); +} + +/// A batch of one carries a mean but no spread, so the mean is taken and the +/// variance is left exactly as it stood. Computing a deviation from a single +/// sample would give zero — a spread the data does not support, and one that +/// would make every subsequent value look like an outlier. +/// +/// ´claim:ewma:a-batch-of-one-carries-no-spread-so-the-variance-is-left-untouched´ +/// ´test:crate:first-update-single-value-keeps-unit-variance´ +#[test] +fn first_update_single_value_keeps_unit_variance() { + let mut stats = EwmaStats::new(0.99); + stats.update(&[5.0], 3.0); + assert!(stats.is_warm()); + assert!((stats.mean() - 5.0).abs() < f64::EPSILON); + // Single-element cold-path: variance stays at the 1.0 initial + // (the code only computes variance when normals.len() > 1). + assert!((stats.variance() - 1.0).abs() < f64::EPSILON); +} + +/// A first batch of more than one sets the spread from its own mean squared +/// deviation, taken about the batch mean and divided by the count with no +/// correction applied. Both the centre and the spread the baseline starts +/// from are therefore measurements rather than defaults. +/// +/// ´claim:ewma:a-first-batch-of-more-than-one-sets-the-spread-from-its-own-deviation´ +/// ´test:crate:first-update-multi-value-sets-sample-variance´ +#[test] +fn first_update_multi_value_sets_sample_variance() { + let mut stats = EwmaStats::new(0.99); + stats.update(&[0.0, 10.0], 3.0); + // mean = 5.0, variance = ((0-5)² + (10-5)²) / 2 = 25.0 + assert!((stats.mean() - 5.0).abs() < f64::EPSILON); + assert!((stats.variance() - 25.0).abs() < 1e-12); +} + +// ── reset_cold() ──────────────────────────────────────────── + +/// Resetting a warm baseline returns it to exactly the state construction +/// left it in — placeholders restored and warmth withdrawn — rather than +/// merely clearing a flag over learned numbers. What it learned is discarded, +/// not hidden. +/// +/// ´claim:ewma:resetting-cold-returns-a-baseline-to-the-state-construction-left-it-in´ +/// ´test:crate:reset-cold-restores-placeholder-state´ +#[test] +fn reset_cold_restores_placeholder_state() { + let mut stats = EwmaStats::new(0.95); + stats.update(&[10.0, 20.0, 30.0], 3.0); + assert!(stats.is_warm()); + + stats.reset_cold(); + assert!(!stats.is_warm()); + assert!((stats.mean() - 1.0).abs() < f64::EPSILON); + assert!((stats.variance() - 1.0).abs() < f64::EPSILON); +} + +/// After a reset the next batch is adopted outright, exactly as the very +/// first one was: the mean lands on the new batch's value with nothing of the +/// discarded baseline pulling it back. Resetting therefore genuinely +/// re-starts the baseline rather than leaving it to decay out of its old +/// position. +/// +/// ´claim:ewma:a-reset-baseline-re-warms-by-the-cold-path-not-by-decay´ +/// ´test:crate:reset-cold-allows-re-warming´ +#[test] +fn reset_cold_allows_re_warming() { + let mut stats = EwmaStats::new(0.95); + stats.update(&[10.0, 10.0, 10.0], 3.0); + stats.reset_cold(); + stats.update(&[42.0, 42.0, 42.0], 3.0); + assert!(stats.is_warm()); + assert!((stats.mean() - 42.0).abs() < f64::EPSILON); +} + +// ── seed_from() ───────────────────────────────────────────── + +/// Seeding transfers both what a baseline learned and the fact that it +/// learned it: the receiver takes the source's mean and spread and becomes +/// warm. This is how a slow baseline is started from a fast one that has +/// already converged, so the pair begin in agreement instead of the slow one +/// spending its warm-up disagreeing with a baseline that is already right. +/// +/// ´claim:ewma:seeding-copies-the-sources-baseline-and-its-warmth-together´ +/// ´test:crate:seed-from-copies-warm-state´ +#[test] +fn seed_from_copies_warm_state() { + let mut source = EwmaStats::new(0.99); + source.update(&[5.0, 10.0, 15.0], 3.0); + + let mut target = EwmaStats::new(0.99); + assert!(!target.is_warm()); + target.seed_from(&source); + + assert!(target.is_warm()); + assert!((target.mean() - source.mean()).abs() < f64::EPSILON); + assert!((target.variance() - source.variance()).abs() < f64::EPSILON); +} + +/// Warmth is never manufactured by seeding. A cold source hands over its +/// placeholder numbers but leaves the receiver cold, so a baseline seeded +/// before anything was learned still takes the cold path on its own first +/// batch rather than blending against values nothing measured. +/// +/// ´claim:ewma:seeding-from-a-cold-source-copies-placeholders-without-conferring-warmth´ +/// ´test:crate:seed-from-cold-source-does-not-warm-target´ +#[test] +fn seed_from_cold_source_does_not_warm_target() { + let source = EwmaStats::new(0.99); // never updated — cold + let mut target = EwmaStats::new(0.99); + target.seed_from(&source); + + assert!(!target.is_warm()); + // Still copies the placeholder values + assert!((target.mean() - 1.0).abs() < f64::EPSILON); + assert!((target.variance() - 1.0).abs() < f64::EPSILON); +} + +/// A receiver that had learned something and is then seeded from a source that +/// had not comes back cold, rather than keeping its own warmth over the +/// placeholders it has just been handed. Warmth belongs to the baseline being +/// transferred and not to the receiver: it is what says whether those two +/// numbers were measured or were the pair a fresh baseline starts from. A +/// receiver left warm over them would clip and score against a notion of +/// normal nothing had observed, and the cold path that exists to replace +/// exactly that state would never run again. +/// +/// ´claim:ewma:seeding-from-a-cold-source-withdraws-the-receivers-warmth-instead-of-leaving-it-over-placeholders´ +/// ´test:crate:seed-from-cold-source-withdraws-warmth-from-a-warm-receiver´ +#[test] +fn seed_from_cold_source_withdraws_warmth_from_a_warm_receiver() { + let source = EwmaStats::new(0.99); // never updated — cold + + let mut target = EwmaStats::new(0.99); + target.update(&[10.0, 20.0, 30.0], 3.0); + assert!(target.is_warm(), "the receiver is warm before it is seeded"); + + target.seed_from(&source); + + assert!(!target.is_warm(), "a cold source leaves the receiver cold"); + assert!((target.mean() - 1.0).abs() < f64::EPSILON); + assert!((target.variance() - 1.0).abs() < f64::EPSILON); + + // The cold path runs on the next batch: it is adopted outright rather + // than blended into the placeholders it would otherwise have decayed + // away from. + target.update(&[42.0, 42.0, 42.0], 3.0); + assert!(target.is_warm()); + assert!((target.mean() - 42.0).abs() < f64::EPSILON); +} + +// ── update() — clipped updates ────────────────────────────── + +/// A batch with nothing in it leaves both the mean and the spread exactly +/// where they were. An idle interval is therefore not a data point: the +/// baseline does not drift simply because time passed without observations. +/// +/// ´claim:ewma:an-empty-batch-teaches-the-baseline-nothing´ +/// ´test:crate:update-empty-is-noop´ +#[test] +fn update_empty_is_noop() { + let mut stats = EwmaStats::new(0.99); + stats.update(&[5.0, 5.0, 5.0], 3.0); + let mean_before = stats.mean(); + let var_before = stats.variance(); + + stats.update(&[], 3.0); + assert!((stats.mean() - mean_before).abs() < f64::EPSILON); + assert!((stats.variance() - var_before).abs() < f64::EPSILON); +} + +/// A warm baseline refuses values above its own ceiling before it learns +/// anything, and a batch consisting entirely of such values moves it not at +/// all. This is the poisoning defence: an attacker cannot walk the notion of +/// normal upward by feeding extremes, because the extremes are precisely what +/// never reaches the baseline. +/// +/// ´claim:ewma:a-batch-entirely-above-the-ceiling-cannot-move-the-baseline-it-would-poison´ +/// ´test:crate:outliers-are-rejected´ +#[test] +fn outliers_are_rejected() { + let mut stats = EwmaStats::new(0.99); + // Warm up with small values + stats.update(&[1.0, 1.0, 1.0], 3.0); + let mean_before = stats.mean(); + + // Feed extreme outlier — should be rejected + stats.update(&[1000.0], 3.0); + assert!( + (stats.mean() - mean_before).abs() < f64::EPSILON, + "mean should not change when all values are outliers" + ); +} + +/// The same restraint holds on the warm path as on the cold one: when only a +/// single value survives clipping, the mean moves and the spread does not. +/// The rule is about how much a batch can say about spread, not about which +/// stage of its life the baseline is in. +/// +/// (´claim:ewma:a-batch-of-one-carries-no-spread-so-the-variance-is-left-untouched´) +/// ´test:crate:update-single-value-does-not-update-variance´ +#[test] +fn update_single_value_does_not_update_variance() { + let mut stats = EwmaStats::new(0.95); + stats.update(&[5.0, 5.0, 5.0], 3.0); + let var_before = stats.variance(); + + // Single accepted value — variance path is skipped (normals.len() == 1). + stats.update(&[5.0], 3.0); + assert!( + (stats.variance() - var_before).abs() < f64::EPSILON, + "variance should not change from a single-element update" + ); +} + +/// While cold there is no ceiling to clip against, so nothing is rejected: a +/// first batch far above the placeholder mean is accepted whole and becomes +/// the baseline. The placeholders are not a real notion of normal, and +/// filtering against them would let an arbitrary constant decide which of the +/// first real observations the sentinel was allowed to see. +/// +/// ´claim:ewma:while-cold-there-is-no-ceiling-to-clip-against-so-nothing-is-rejected´ +/// ´test:crate:update-skips-clipping-when-cold´ +#[test] +fn update_skips_clipping_when_cold() { + let mut stats = EwmaStats::new(0.99); + // Even though 1000.0 is far from initial mean=1.0, cold-path accepts it. + stats.update(&[1000.0, 1000.0], 3.0); + assert!(stats.is_warm()); + assert!((stats.mean() - 1000.0).abs() < f64::EPSILON); +} + +// ── update_raw() — unclipped updates ──────────────────────── + +/// The unclipped path treats an empty batch the same way: nothing in, nothing +/// changed. Externalising the outlier filter does not turn absence of data +/// into evidence. +/// +/// (´claim:ewma:an-empty-batch-teaches-the-baseline-nothing´) +/// ´test:crate:update-raw-empty-is-noop´ +#[test] +fn update_raw_empty_is_noop() { + let mut stats = EwmaStats::new(0.99); + stats.update_raw(&[5.0, 5.0, 5.0]); + let mean_before = stats.mean(); + let var_before = stats.variance(); + + stats.update_raw(&[]); + assert!((stats.mean() - mean_before).abs() < f64::EPSILON); + assert!((stats.variance() - var_before).abs() < f64::EPSILON); +} + +/// The unclipped path is the clipped one with only the filter removed: given +/// a clip so wide that nothing could be rejected, the two produce the same +/// mean and the same spread. Callers that do their own outlier rejection get +/// identical arithmetic, so the two entry points cannot drift into +/// disagreeing about what a batch means. +/// +/// ´claim:ewma:the-unclipped-path-is-the-clipped-one-with-only-the-filter-removed´ +/// ´test:crate:update-raw-matches-update-when-no-clipping´ +#[test] +fn update_raw_matches_update_when_no_clipping() { + let mut a = EwmaStats::new(0.95); + let mut b = EwmaStats::new(0.95); + let values = &[1.0, 1.2, 0.8, 1.1, 0.9]; + + a.update(values, 100.0); // clip_sigmas so high nothing is clipped + b.update_raw(values); + + assert!((a.mean() - b.mean()).abs() < 1e-12); + assert!((a.variance() - b.variance()).abs() < 1e-12); +} + +// ── Decay behaviour ───────────────────────────────────────── + +/// Sustained new data pulls the baseline away from what it learned before: a +/// mean established at one level and then fed a different one repeatedly ends +/// up near the new level. The baseline tracks the present rather than +/// averaging over all history, which is what makes it a moving notion of +/// normal rather than a permanent one. +/// +/// ´claim:ewma:sustained-new-data-pulls-the-baseline-away-from-what-it-learned-before´ +/// ´test:crate:decays-toward-new-data´ +#[test] +fn decays_toward_new_data() { + let mut stats = EwmaStats::new(0.90); // fast decay + stats.update(&[10.0, 10.0, 10.0], 3.0); + assert!((stats.mean() - 10.0).abs() < f64::EPSILON); + + // Push toward 0.0 + for _ in 0..50 { + stats.update(&[0.0, 0.0, 0.0], 3.0); + } + assert!(stats.mean() < 1.0, "mean should have decayed toward 0.0"); +} + +/// The decay factor is the length of the baseline's memory. Two baselines +/// started at the same level and fed the same contradicting data diverge in +/// the expected direction: the one with the higher factor still holds more of +/// the original level. This is what lets a fast and a slow baseline over the +/// same stream disagree usefully, which is the whole basis of drift +/// detection. +/// +/// ´claim:ewma:a-higher-decay-factor-holds-the-past-longer´ +/// ´test:crate:higher-decay-forgets-slower´ +#[test] +fn higher_decay_forgets_slower() { + let mut slow = EwmaStats::new(0.99); // slow decay (long memory) + let mut fast = EwmaStats::new(0.90); // fast decay (short memory) + + // Both start at 10.0 + slow.update(&[10.0, 10.0, 10.0], 5.0); + fast.update(&[10.0, 10.0, 10.0], 5.0); + + // Push both toward 0.0 + for _ in 0..20 { + slow.update(&[0.0, 0.0, 0.0], 5.0); + fast.update(&[0.0, 0.0, 0.0], 5.0); + } + + // Slow retains more of the original 10.0 + assert!( + slow.mean() > fast.mean(), + "slow (λ=0.99) should retain more: slow={}, fast={}", + slow.mean(), + fast.mean() + ); +} + +// ── z_score() ─────────────────────────────────────────────── + +/// A value sitting at the baseline scores essentially nothing. The z-score +/// measures signed departure from what the baseline expects, so agreement +/// with it is the origin of the scale rather than a point somewhere along it. +/// +/// ´claim:ewma:the-z-score-is-signed-departure-from-the-baseline-and-zero-at-it´ +/// ´test:crate:z-score-is-zero-at-mean´ +#[test] +fn z_score_is_zero_at_mean() { + let mut stats = EwmaStats::new(0.99); + stats.update(&[5.0, 5.0, 5.0, 5.0], 3.0); + let z = stats.z_score(5.0, 1e-6); + assert!(z.abs() < 0.01); +} + +/// A value above the baseline scores positive, which is the direction that +/// matters: elevated anomaly scores are the ones the sentinel is watching for +/// and the ones an attacker would produce. +/// +/// (´claim:ewma:the-z-score-is-signed-departure-from-the-baseline-and-zero-at-it´) +/// ´test:crate:z-score-positive-above-mean´ +#[test] +fn z_score_positive_above_mean() { + let mut stats = EwmaStats::new(0.99); + stats.update(&[5.0, 5.0, 5.0], 3.0); + let z = stats.z_score(10.0, 1e-6); + assert!(z > 0.0, "z-score should be positive above mean, got {z}"); +} + +/// A value below the baseline scores negative rather than being folded to a +/// magnitude. The sign survives, so a caller can tell a quiet departure from +/// an elevated one instead of seeing both as equally unusual. +/// +/// (´claim:ewma:the-z-score-is-signed-departure-from-the-baseline-and-zero-at-it´) +/// ´test:crate:z-score-negative-below-mean´ +#[test] +fn z_score_negative_below_mean() { + let mut stats = EwmaStats::new(0.99); + stats.update(&[5.0, 5.0, 5.0], 3.0); + let z = stats.z_score(1.0, 1e-6); + assert!(z < 0.0, "z-score should be negative below mean, got {z}"); +} + +// ── ceiling() ─────────────────────────────────────────────── + +/// Asked for its ceiling while cold, a baseline answers with infinity — the +/// value that admits everything. The bypass on the cold update path is not a +/// special case hidden inside the update; it is visible in the ceiling +/// itself, so the two cannot disagree about whether clipping applies. +/// +/// (´claim:ewma:while-cold-there-is-no-ceiling-to-clip-against-so-nothing-is-rejected´) +/// ´test:crate:ceiling-returns-infinity-when-cold´ +#[test] +fn ceiling_returns_infinity_when_cold() { + let stats = EwmaStats::new(0.95); + assert!(stats.ceiling(3.0).is_infinite()); +} + +/// Once warm, the ceiling stands a fixed number of standard deviations above +/// the mean — the requested multiple of the baseline's own square-rooted +/// spread, added to its own centre. The threshold is therefore relative to +/// what this cell has learned, not an absolute score chosen in advance for +/// every cell alike. +/// +/// ´claim:ewma:a-warm-ceiling-stands-a-fixed-number-of-deviations-above-the-mean´ +/// ´test:crate:ceiling-returns-mean-plus-sigmas-when-warm´ +#[test] +fn ceiling_returns_mean_plus_sigmas_when_warm() { + let mut stats = EwmaStats::new(0.95); + stats.update(&[2.0, 2.0, 2.0], 3.0); + let ceil = stats.ceiling(3.0); + let expected = 3.0_f64.mul_add(stats.variance().sqrt(), stats.mean()); + assert!((ceil - expected).abs() < 1e-12); +} + +/// The clip setting is a monotone dial on how much the baseline is willing to +/// learn from: given the same elevated value, a baseline clipping at a wide +/// multiple moves at least as far as one clipping tightly. Tightening the +/// setting can only ever admit less, so an operator turning it down is +/// trading responsiveness for poisoning resistance and never the reverse. +/// +/// ´claim:ewma:widening-the-clip-admits-at-least-as-much-as-tightening-it´ +/// ´test:crate:clip-sigmas-affects-ceiling´ +#[test] +fn clip_sigmas_affects_ceiling() { + // With tight clip (1σ), more values are rejected. + let mut tight = EwmaStats::new(0.99); + tight.update(&[1.0, 1.0, 1.0], 1.0); + + // With wide clip (5σ), fewer values are rejected. + let mut wide = EwmaStats::new(0.99); + wide.update(&[1.0, 1.0, 1.0], 5.0); + + // Feed a moderately elevated value. + let elevated = &[3.0]; + let mean_before_tight = tight.mean(); + let mean_before_wide = wide.mean(); + tight.update(elevated, 1.0); + wide.update(elevated, 5.0); + + // Wide should have moved toward 3.0 more than tight + // (tight may reject 3.0 if it's beyond 1σ from mean ~1.0). + let tight_delta = (tight.mean() - mean_before_tight).abs(); + let wide_delta = (wide.mean() - mean_before_wide).abs(); + assert!( + wide_delta >= tight_delta, + "wide clip should accept more: wide_delta={wide_delta}, tight_delta={tight_delta}" + ); +} + +// ── snapshot() ────────────────────────────────────────────── + +/// The snapshot a report carries holds the same mean and spread the +/// baseline's own accessors report. What a caller reads out of a report is +/// the state the engine is scoring against, not a rounded or separately +/// derived summary of it. +/// +/// ´claim:ewma:a-snapshot-reports-the-baseline-the-accessors-report´ +/// ´test:crate:snapshot-matches-state´ +#[test] +fn snapshot_matches_state() { + let mut stats = EwmaStats::new(0.99); + stats.update(&[2.0, 4.0, 6.0], 3.0); + let snap = stats.snapshot(); + assert!((snap.mean - stats.mean()).abs() < f64::EPSILON); + assert!((snap.variance - stats.variance()).abs() < f64::EPSILON); +} + +// ── Variance floor ────────────────────────────────────────── + +/// A batch of identical values has no deviation at all, yet the spread does +/// not reach zero: a floor holds it above. Without it a perfectly quiet +/// period would collapse the spread, the ceiling would close onto the mean, +/// and every subsequent value — however ordinary — would be rejected as an +/// outlier, leaving the baseline permanently frozen at the quiet level. +/// +/// ´claim:ewma:the-spread-is-floored-so-a-quiet-period-cannot-freeze-the-baseline´ +/// ´test:crate:variance-floor-is-respected´ +#[test] +fn variance_floor_is_respected() { + let mut stats = EwmaStats::new(0.95); + // All identical values → zero deviation, but floor should apply. + stats.update(&[7.0, 7.0, 7.0, 7.0], 3.0); + assert!( + stats.variance() >= 1e-4, + "variance should be clamped to floor 1e-4, got {}", + stats.variance() + ); +} diff --git a/packages/sentinel/src/tests/mod.rs b/packages/sentinel/src/tests/mod.rs new file mode 100644 index 000000000..94530d714 --- /dev/null +++ b/packages/sentinel/src/tests/mod.rs @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +//! Collected crate tests for the sentinel crate. +//! +//! Tests that formerly lived as inline `#[cfg(test)] mod tests { … }` +//! blocks inside their parent modules are refactored here so the +//! production source files stay focused on production code. +//! +//! The convergence characterisation suite (ADR-S-013) also lives +//! under this tree. + +mod analysis_set; +mod config; +mod convergence_clipping; +mod convergence_common; +mod convergence_diagnostics; +mod convergence_eta; +mod convergence_ewma; +mod convergence_fixes; +mod convergence_noise; +mod cusum; +mod ewma; +mod observation; +mod report; +mod tracker; +mod variance_formula; +mod warming_thread; diff --git a/packages/sentinel/src/tests/observation.rs b/packages/sentinel/src/tests/observation.rs new file mode 100644 index 000000000..45ec8d86d --- /dev/null +++ b/packages/sentinel/src/tests/observation.rs @@ -0,0 +1,306 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +//! Tests for [`crate::observation`] — the boundary where a coordinate value +//! becomes the vector the subspace engine works on. +//! +//! The conversion is fixed by two decisions. A set bit is plus a half and a clear bit minus a half, so the encoded levels are symmetric about zero. Each dimension has zero expected mean under a uniform bit distribution (§ALGO S-2.3); arbitrary traffic need not have balanced bits. Bits are stored most-significant first, so that a cell's G-tree depth is a prefix length: the leading bits routing has already resolved sit at the front, and the suffix a tracker analyses is what remains behind them. +//! +//! Those two decisions together give the representation its one arithmetic +//! regularity. Every centred bit has magnitude one half whatever the value, +//! so a suffix of any width has squared norm equal to a quarter of that +//! width, at every depth and for every input. The tracker can therefore read +//! a residual as a departure from structure rather than as an artefact of +//! how large the observation happened to be. +//! +//! # Test index +//! +//! | Test | Area | Claim | +//! |------|------|-------| +//! | [`from_u128_zero_all_minus_half`] | bits | A clear bit becomes minus a half and a set bit plus a half. An all-zero coordinate therefore becomes minus a half throughout: the encoding is symmetric, while zero expected mean requires balanced bits. | +//! | [`from_u128_max_all_plus_half`] | bits | cites (´claim:bits:a-clear-bit-becomes-minus-a-half-and-a-set-bit-plus-a-half´) | +//! | [`from_u128_one_only_lsb_set`] | bits | cites (´claim:bits:the-most-significant-bit-stands-at-index-zero´) | +//! | [`from_u128_msb_first_ordering`] | bits | Bits are stored most significant first: a value carrying only its top bit puts that bit at index zero and nothing else anywhere. This ordering is what lets a cell at depth `d` take its working observation by skipping the first `d` entries, because those are exactly the bits routing fixed. | +//! | [`u128_custom_width_populates_n_bits`] | bits | The vector is backed by a fixed hundred-and-twenty-eight-slot array, but the requested width is what counts as populated: asking for eight bits fills eight slots and leaves the remainder at zero, and the observation reports its length as eight rather than as the array's size. A domain narrower than the backing store is therefore not padded with fabricated structure. | +//! | [`u128_zero_width_gives_empty`] | bits | cites (´claim:bits:a-requested-width-populates-exactly-that-many-slots-and-leaves-the-rest-zero´) | +//! | [`u128_width_capped_at_128`] | bits | cites (´claim:bits:a-width-wider-than-the-coordinate-is-capped-at-the-coordinates-own-width´) | +//! | [`u64_max_all_plus_half`] | bits | The centring rule is a property of the conversion, not of the coordinate type: a sixty-four-bit value with every bit set produces sixty-four slots of plus a half, exactly as the wider coordinate does. A host working in a narrower domain gets the same representation, so the engine above the boundary need not know which width it was fed. | +//! | [`u64_width_capped_at_64`] | bits | Asking a sixty-four-bit coordinate for a wider observation does not invent bits: the width is capped at what the value actually holds. A sentinel configured for the wider domain can therefore be handed narrower coordinates without the shift going out of range or the tail of the vector filling with structure that was never observed. | +//! | [`from_coord_delegates_correctly`] | bits | The generic entry point the engine actually calls produces the same observation, bit for bit and length for length, as calling the conversion on the value directly. There is one encoding rather than two that happen to agree, so nothing can drift between the path tests exercise and the path production code takes. | +//! | [`suffix_zero_is_full_vector`] | bits | A cell at the root of the G-tree has had no bits resolved by routing, so its working observation is the whole vector — not a copy of it, but the same values. The root tracker analyses the full domain width, which is the base case the depth arithmetic has to agree with. | +//! | [`suffix_intermediate_depth`] | bits | At depth `d` the observation drops exactly its first `d` entries and keeps everything behind them. Those leading bits are constant across every value routed into the cell, so removing them leaves precisely the part that varies — the tracker's width is the domain width less its depth, and the bits it sees are the tail of the same vector rather than a re-derivation. | +//! | [`suffix_at_len_is_empty`] | bits | A cell as deep as its domain is wide has nothing left to analyse: routing has resolved every bit, and the suffix is empty rather than an error. This holds at the full width and at a narrower configured one alike, which is why such cells are excluded from the analysis set by width rather than caught as a failure when a tracker tries to run on them. | +//! | [`suffix_panics_beyond_len`] | bits | A depth past the observation's own width is treated as a programming error and not as a value to be tolerated. Empty is the answer at exactly the width; beyond it there is no honest answer, so the boundary between the degenerate case and the impossible one is drawn rather than blurred by silently clamping. | +//! | [`suffix_norm_squared_is_width_over_four`] | bits | Because every centred bit has magnitude one half, a suffix's squared norm is a quarter of its width and nothing else — the same for every value and at every depth, checked here across saturated, sparse and arbitrary inputs at all depths. Observation magnitude therefore carries no information: a residual the tracker measures is a departure from learned structure, never an artefact of which value arrived. | + +use crate::observation::*; + +// ─── CentredBits::from_u128 ──────────────────────────────── + +/// A clear bit becomes minus a half and a set bit plus a half. An all-zero coordinate therefore becomes minus a half throughout: the encoding is symmetric, while zero expected mean requires balanced bits. +/// +/// ´claim:bits:a-clear-bit-becomes-minus-a-half-and-a-set-bit-plus-a-half´ +/// ´test:crate:from-u128-zero-all-minus-half´ +#[test] +fn from_u128_zero_all_minus_half() { + let cb = CentredBits::from_u128(0); + for (i, &b) in cb.bits.iter().enumerate() { + assert!((b - (-0.5)).abs() < f64::EPSILON, "bit {i}: expected -0.5, got {b}"); + } +} + +/// The opposite extreme of the same rule: a value with every bit set becomes +/// a vector of plus a half throughout. The two saturated inputs bracket the +/// encoding, so no bit pattern can produce a magnitude other than a half. +/// +/// (´claim:bits:a-clear-bit-becomes-minus-a-half-and-a-set-bit-plus-a-half´) +/// ´test:crate:from-u128-max-all-plus-half´ +#[test] +fn from_u128_max_all_plus_half() { + let cb = CentredBits::from_u128(u128::MAX); + for (i, &b) in cb.bits.iter().enumerate() { + assert!((b - 0.5).abs() < f64::EPSILON, "bit {i}: expected +0.5, got {b}"); + } +} + +/// The value one has only its least significant bit set, and that bit shows +/// up at the very last index rather than the first. Which end of the vector a +/// bit lands on is what makes a G-tree depth readable as a prefix length. +/// +/// (´claim:bits:the-most-significant-bit-stands-at-index-zero´) +/// ´test:crate:from-u128-one-only-lsb-set´ +#[test] +fn from_u128_one_only_lsb_set() { + let cb = CentredBits::from_u128(1); + for &b in &cb.bits[..127] { + assert!((b - (-0.5)).abs() < f64::EPSILON, "expected -0.5, got {b}"); + } + assert!((cb.bits[127] - 0.5).abs() < f64::EPSILON); +} + +/// Bits are stored most significant first: a value carrying only its top bit +/// puts that bit at index zero and nothing else anywhere. This ordering is +/// what lets a cell at depth `d` take its working observation by skipping the +/// first `d` entries, because those are exactly the bits routing fixed. +/// +/// ´claim:bits:the-most-significant-bit-stands-at-index-zero´ +/// ´test:crate:from-u128-msb-first-ordering´ +#[test] +fn from_u128_msb_first_ordering() { + // 0x80…0 has only the MSB set — index 0 should be +0.5. + let cb = CentredBits::from_u128(1_u128 << 127); + assert!((cb.bits[0] - 0.5).abs() < f64::EPSILON, "MSB should be at index 0"); + for &b in &cb.bits[1..] { + assert!((b - (-0.5)).abs() < f64::EPSILON, "remaining bits should be -0.5"); + } +} + +// ─── CentredBitSource for u128 — custom width ────────────── + +/// The vector is backed by a fixed hundred-and-twenty-eight-slot array, but +/// the requested width is what counts as populated: asking for eight bits +/// fills eight slots and leaves the remainder at zero, and the observation +/// reports its length as eight rather than as the array's size. A domain +/// narrower than the backing store is therefore not padded with fabricated +/// structure. +/// +/// ´claim:bits:a-requested-width-populates-exactly-that-many-slots-and-leaves-the-rest-zero´ +/// ´test:crate:u128-custom-width-populates-n-bits´ +#[test] +fn u128_custom_width_populates_n_bits() { + // 0xFF = 8 one-bits; ask for n=8 → first 8 slots are +0.5, rest 0.0. + let cb = 0xFF_u128.to_centred_bits(8); + for (i, &b) in cb.bits[..8].iter().enumerate() { + assert!((b - 0.5).abs() < f64::EPSILON, "bit {i}: expected +0.5, got {b}"); + } + for (i, &b) in cb.bits[8..].iter().enumerate() { + assert!(b.abs() < f64::EPSILON, "bit {}: expected 0.0, got {b}", i + 8); + } + // suffix(0) should return exactly 8 elements, not 128. + assert_eq!(cb.suffix(0).len(), 8); +} + +/// The degenerate end of the same rule: a width of zero populates nothing, so +/// the observation is empty and every slot stays at zero however large the +/// value handed in. A zero-width domain is admitted rather than rejected, and +/// it carries no bits of the value it came from. +/// +/// (´claim:bits:a-requested-width-populates-exactly-that-many-slots-and-leaves-the-rest-zero´) +/// ´test:crate:u128-zero-width-gives-empty´ +#[test] +fn u128_zero_width_gives_empty() { + let cb = 42_u128.to_centred_bits(0); + assert_eq!(cb.suffix(0).len(), 0); + for &b in &cb.bits { + assert!(b.abs() < f64::EPSILON, "all slots should be 0.0"); + } +} + +/// The cap is a property of the conversion rather than of the narrower +/// coordinate: asking the wider type for more bits than it holds returns its +/// own width, exactly as the narrower type does. The vector is backed by an +/// array of that same width, so an uncapped request walks off the end of it — +/// a host computing its width from a configured domain would get a panic out +/// of the observation boundary instead of an observation. +/// +/// (´claim:bits:a-width-wider-than-the-coordinate-is-capped-at-the-coordinates-own-width´) +/// ´test:crate:u128-width-capped-at-128´ +#[test] +fn u128_width_capped_at_128() { + // Requesting n=129 for a u128 should cap at 128. + let cb = u128::MAX.to_centred_bits(129); + assert_eq!(cb.suffix(0).len(), 128); + for (i, &b) in cb.bits.iter().enumerate() { + assert!((b - 0.5).abs() < f64::EPSILON, "bit {i}: expected +0.5, got {b}"); + } +} + +// ─── CentredBitSource for u64 ────────────────────────────── + +/// The centring rule is a property of the conversion, not of the coordinate +/// type: a sixty-four-bit value with every bit set produces sixty-four slots +/// of plus a half, exactly as the wider coordinate does. A host working in a +/// narrower domain gets the same representation, so the engine above the +/// boundary need not know which width it was fed. +/// +/// ´claim:bits:a-narrower-coordinate-encodes-by-the-same-rule-as-a-wider-one´ +/// ´test:crate:u64-max-all-plus-half´ +#[test] +fn u64_max_all_plus_half() { + let cb = u64::MAX.to_centred_bits(64); + assert_eq!(cb.suffix(0).len(), 64); + for (i, &b) in cb.bits[..64].iter().enumerate() { + assert!((b - 0.5).abs() < f64::EPSILON, "bit {i}: expected +0.5, got {b}"); + } +} + +/// Asking a sixty-four-bit coordinate for a wider observation does not +/// invent bits: the width is capped at what the value actually holds. A +/// sentinel configured for the wider domain can therefore be handed narrower +/// coordinates without the shift going out of range or the tail of the +/// vector filling with structure that was never observed. +/// +/// ´claim:bits:a-width-wider-than-the-coordinate-is-capped-at-the-coordinates-own-width´ +/// ´test:crate:u64-width-capped-at-64´ +#[test] +fn u64_width_capped_at_64() { + // Requesting n=128 for a u64 should cap at 64. + let cb = u64::MAX.to_centred_bits(128); + assert_eq!(cb.suffix(0).len(), 64); +} + +// ─── CentredBits::from_coord ─────────────────────────────── + +/// The generic entry point the engine actually calls produces the same +/// observation, bit for bit and length for length, as calling the conversion +/// on the value directly. There is one encoding rather than two that happen +/// to agree, so nothing can drift between the path tests exercise and the +/// path production code takes. +/// +/// ´claim:bits:the-generic-entry-point-produces-what-the-source-conversion-produces´ +/// ´test:crate:from-coord-delegates-correctly´ +#[test] +fn from_coord_delegates_correctly() { + let value = 0xDEAD_BEEF_u128; + let direct = value.to_centred_bits(128); + let via_coord = CentredBits::from_coord(&value, 128); + for (a, b) in direct.bits.iter().zip(&via_coord.bits) { + assert!((a - b).abs() < f64::EPSILON); + } + assert_eq!(direct.suffix(0).len(), via_coord.suffix(0).len()); +} + +// ─── CentredBits::suffix ─────────────────────────────────── + +/// A cell at the root of the G-tree has had no bits resolved by routing, so +/// its working observation is the whole vector — not a copy of it, but the +/// same values. The root tracker analyses the full domain width, which is the +/// base case the depth arithmetic has to agree with. +/// +/// ´claim:bits:the-suffix-at-depth-zero-is-the-whole-observation´ +/// ´test:crate:suffix-zero-is-full-vector´ +#[test] +fn suffix_zero_is_full_vector() { + let cb = CentredBits::from_u128(0xDEAD_BEEF); + let s = cb.suffix(0); + assert_eq!(s.len(), 128); + assert_eq!(s, &cb.bits[..]); +} + +/// At depth `d` the observation drops exactly its first `d` entries and keeps +/// everything behind them. Those leading bits are constant across every value +/// routed into the cell, so removing them leaves precisely the part that +/// varies — the tracker's width is the domain width less its depth, and the +/// bits it sees are the tail of the same vector rather than a re-derivation. +/// +/// ´claim:bits:a-suffix-drops-exactly-the-bits-routing-already-resolved´ +/// ´test:crate:suffix-intermediate-depth´ +#[test] +fn suffix_intermediate_depth() { + let cb = CentredBits::from_u128(1); + let s = cb.suffix(8); + assert_eq!(s.len(), 120); + assert_eq!(s, &cb.bits[8..128]); +} + +/// A cell as deep as its domain is wide has nothing left to analyse: routing +/// has resolved every bit, and the suffix is empty rather than an error. This +/// holds at the full width and at a narrower configured one alike, which is +/// why such cells are excluded from the analysis set by width rather than +/// caught as a failure when a tracker tries to run on them. +/// +/// ´claim:bits:a-cell-as-deep-as-its-domain-is-wide-has-an-empty-suffix´ +/// ´test:crate:suffix-at-len-is-empty´ +#[test] +fn suffix_at_len_is_empty() { + let cb = CentredBits::from_u128(42); + assert_eq!(cb.suffix(128).len(), 0); + + // Also verify for a smaller width. + let cb_small = 0xFF_u128.to_centred_bits(16); + assert_eq!(cb_small.suffix(16).len(), 0); +} + +/// A depth past the observation's own width is treated as a programming +/// error and not as a value to be tolerated. Empty is the answer at exactly +/// the width; beyond it there is no honest answer, so the boundary between +/// the degenerate case and the impossible one is drawn rather than blurred by +/// silently clamping. +/// +/// ´claim:bits:a-depth-past-the-observations-width-is-a-fault-not-a-value´ +/// ´test:crate:suffix-panics-beyond-len´ +#[test] +#[should_panic(expected = "slice index starts at 17 but ends at 16")] +fn suffix_panics_beyond_len() { + let cb = 0xFF_u128.to_centred_bits(16); + let _ = cb.suffix(17); // depth > len → panic +} + +/// Because every centred bit has magnitude one half, a suffix's squared norm +/// is a quarter of its width and nothing else — the same for every value and +/// at every depth, checked here across saturated, sparse and arbitrary inputs +/// at all depths. Observation magnitude therefore carries no information: a +/// residual the tracker measures is a departure from learned structure, never +/// an artefact of which value arrived. +/// +/// ´claim:bits:every-suffix-has-squared-norm-equal-to-a-quarter-of-its-width´ +/// ´test:crate:suffix-norm-squared-is-width-over-four´ +#[test] +fn suffix_norm_squared_is_width_over_four() { + let values: [u128; 5] = [0, 1, u128::MAX, 0xDEAD_BEEF, 0x1234_5678_9ABC_DEF0]; + + for &v in &values { + let cb = CentredBits::from_u128(v); + for d in 0..128_u8 { + let s = cb.suffix(d); + let w = s.len(); + let norm_sq: f64 = s.iter().map(|x| x * x).sum(); + #[allow(clippy::cast_precision_loss)] // width ≤ 128, well within f64 precision + let expected = w as f64 / 4.0; + assert!( + (norm_sq - expected).abs() < 1e-10, + "v={v:#x}, depth={d}: suffix norm²={norm_sq}, expected w/4={expected}" + ); + } + } +} diff --git a/packages/sentinel/src/tests/report.rs b/packages/sentinel/src/tests/report.rs new file mode 100644 index 000000000..4f4eee2fa --- /dev/null +++ b/packages/sentinel/src/tests/report.rs @@ -0,0 +1,335 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +//! Tests for [`crate::report`] — the records the sentinel hands back to the +//! host after each observation cycle. +//! +//! The reports exist because the sentinel measures and the host decides. +//! They carry raw statistics — scores, baselines, drift, maturity, geometry, +//! structural summaries — and never a threat level or a recommended action. +//! That division is what makes the small pieces of behaviour these types do +//! own worth pinning down: whatever logic lives in a report type is logic +//! the host will lean on when forming its own judgement. +//! +//! Most of the module is plain data, and the tests here cover the parts that +//! are not. Maturity distinguishes observations the tracker really saw from +//! the noise it was warmed with, and reports both rather than one blended +//! figure, so a host can tell a confident model from a fresh one. Geometry +//! answers whether an axis is structurally meaningful at all — novelty +//! measures leftover residual, so with no residual degrees of freedom there +//! is nothing left for it to measure, and a host reading a low novelty score +//! needs to know whether that means "nothing unusual" or "nothing +//! measurable". +//! +//! The remaining tests fix the shape of the summary records: what a contour +//! snapshot, an analysis-set summary, and a per-cell member score are +//! obliged to carry. A report consumer parses the same fields whether the +//! sentinel is idle or saturated, so the degenerate cases report zeroes +//! rather than omitting anything. +//! +//! # Test index +//! +//! | Test | Area | Claim | +//! |------|------|-------| +//! | [`warming_targets_follow_current_selection`] | readout | Retained warming ancestors stop counting as competitive targets as soon as selection changes. | +//! | [`cold_maturity_is_fully_noisy`] | readout | A tracker that has seen nothing reports no observations of either kind and a baseline owed entirely to noise. The cold state is not "unknown" but a definite statement: whatever this tracker would score against, none of it came from real traffic, so a host can discount its scores rather than having to guess how new the model is. | +//! | [`total_observations_sums_real_and_noise`] | readout | Total experience is the real and injected observations added together, and both counts stay separately readable beside it. The sum says how much the model has absorbed; the split says how much of that was manufactured during warm-up — a host needs the second to interpret the first, so the report offers the convenience without collapsing the distinction. | +//! | [`novelty_not_saturated_when_residual_dof_positive`] | readout | Novelty is degenerate exactly when no residual degrees of freedom remain, and with residual dimensions still unexplained by the learned subspace it is not. Novelty measures the energy the model failed to account for, so while there is somewhere for that energy to live the axis is measuring something real. | +//! | [`novelty_saturated_when_residual_dof_zero`] | readout | cites (´claim:readout:novelty-is-degenerate-exactly-when-no-residual-degrees-of-freedom-remain´) | +//! | [`novelty_saturable_when_cap_ge_dim`] | readout | Saturability is a separate question from saturation: a tracker whose rank cap reaches its working dimension may still have residual room today, yet it is one of those that can lose the novelty axis as rank grows. The report distinguishes the two so a host can tell a temporary reading from a cell where novelty will eventually stop meaning anything. | +//! | [`novelty_not_saturable_when_cap_lt_dim`] | readout | cites (´claim:readout:novelty-can-become-degenerate-whenever-the-rank-cap-reaches-the-working-dimension´) | +//! | [`contour_snapshot_fields`] | readout | A contour snapshot carries the spatial shape, the accumulated volume, and the structural churn since the previous report side by side. Standing state and change-since-last-time are different questions about the spatial layer, and the snapshot answers both at once so a host need not difference successive reports to see the graph move. | +//! | [`analysis_set_summary_empty`] | readout | A summary describing an analysis set with nothing selected still reports every field, its ranges zeroed rather than omitted, and still counts the root as a member of the full set. The shape a host parses does not change with how busy the sentinel is, and the permanent root tracker is visible even at the quietest extreme. | +//! | [`analysis_set_summary_populated`] | readout | A populated summary reports its depth and importance spans as ordered pairs, low end first, and its three sizes widen as the definition of membership loosens: the cells that won the competition, the full set their ancestry closes over, and the investment set that also holds cells still warming. The nesting is what lets a host read the price of analysing a cell as well as the choice to analyse it. | +//! | [`analysis_set_summary_with_degenerate_skips`] | readout | Cells too narrow to support a tracker are counted in the current selection snapshot rather than quietly dropped. Recomputing replaces the count instead of accumulating it, so the report describes the graph the host can inspect now while still exposing a persistently narrow configuration. | +//! | [`member_score_has_cell_identity`] | readout | A member score names the cell it came from — a well-ordered interval and a depth — alongside a real number on each of the four axes and its standardised counterpart. Coordination scores describe a group, so without the identity a host could see that the group behaved oddly but not which part of the domain to look at. | + +use crate::report::*; +use crate::{NoiseSchedule, SentinelConfig, SpectralSentinel}; + +// ── TrackerMaturity ───────────────────────────────────────── + +/// A tracker that has seen nothing reports no observations of either kind +/// and a baseline owed entirely to noise. The cold state is not "unknown" +/// but a definite statement: whatever this tracker would score against, none +/// of it came from real traffic, so a host can discount its scores rather +/// than having to guess how new the model is. +/// +/// ´claim:readout:a-cold-tracker-reports-a-baseline-owed-entirely-to-noise-rather-than-an-absent-one´ +/// ´test:crate:cold-maturity-is-fully-noisy´ +#[test] +fn cold_maturity_is_fully_noisy() { + let m = TrackerMaturity::cold(); + assert_eq!(m.real_observations, 0); + assert_eq!(m.noise_observations, 0); + assert!((m.noise_influence - 1.0).abs() < f64::EPSILON); + assert_eq!(m.total_observations(), 0); +} + +/// Total experience is the real and injected observations added together, +/// and both counts stay separately readable beside it. The sum says how much +/// the model has absorbed; the split says how much of that was manufactured +/// during warm-up — a host needs the second to interpret the first, so the +/// report offers the convenience without collapsing the distinction. +/// +/// ´claim:readout:total-experience-is-real-and-injected-observations-added-together-with-both-still-readable´ +/// ´test:crate:total-observations-sums-real-and-noise´ +#[test] +fn total_observations_sums_real_and_noise() { + let m = TrackerMaturity { + real_observations: 100, + noise_observations: 25, + noise_influence: 0.2, + }; + assert_eq!(m.total_observations(), 125); +} + +// ── ScoringGeometry ───────────────────────────────────────── + +/// Novelty is degenerate exactly when no residual degrees of freedom remain, +/// and with residual dimensions still unexplained by the learned subspace it +/// is not. Novelty measures the energy the model failed to account for, so +/// while there is somewhere for that energy to live the axis is measuring +/// something real. +/// +/// ´claim:readout:novelty-is-degenerate-exactly-when-no-residual-degrees-of-freedom-remain´ +/// ´test:crate:novelty-not-saturated-when-residual-dof-positive´ +#[test] +fn novelty_not_saturated_when_residual_dof_positive() { + let g = ScoringGeometry { + dim: 16, + cap: 8, + residual_dof: 12, + }; + assert!(!g.is_novelty_saturated()); +} + +/// The other side of the same test: once the subspace spans the whole +/// working dimension there is no residual left, and the geometry says so. +/// A host reading a novelty score of nothing here learns that the axis is +/// structurally silent rather than that the observation was ordinary. +/// +/// (´claim:readout:novelty-is-degenerate-exactly-when-no-residual-degrees-of-freedom-remain´) +/// ´test:crate:novelty-saturated-when-residual-dof-zero´ +#[test] +fn novelty_saturated_when_residual_dof_zero() { + let g = ScoringGeometry { + dim: 16, + cap: 16, + residual_dof: 0, + }; + assert!(g.is_novelty_saturated()); +} + +/// Saturability is a separate question from saturation: a tracker whose rank +/// cap reaches its working dimension may still have residual room today, yet +/// it is one of those that can lose the novelty axis as rank grows. The +/// report distinguishes the two so a host can tell a temporary reading from +/// a cell where novelty will eventually stop meaning anything. +/// +/// ´claim:readout:novelty-can-become-degenerate-whenever-the-rank-cap-reaches-the-working-dimension´ +/// ´test:crate:novelty-saturable-when-cap-ge-dim´ +#[test] +fn novelty_saturable_when_cap_ge_dim() { + let g = ScoringGeometry { + dim: 8, + cap: 8, + residual_dof: 4, + }; + assert!(g.is_novelty_saturable()); +} + +/// Where the cap sits below the working dimension the axis is safe for good: +/// however far rank adapts, residual dimensions remain. Capping rank below +/// the width is therefore not only a cost control but the thing that keeps +/// novelty measurable for the life of the cell. +/// +/// (´claim:readout:novelty-can-become-degenerate-whenever-the-rank-cap-reaches-the-working-dimension´) +/// ´test:crate:novelty-not-saturable-when-cap-lt-dim´ +#[test] +fn novelty_not_saturable_when_cap_lt_dim() { + let g = ScoringGeometry { + dim: 16, + cap: 8, + residual_dof: 8, + }; + assert!(!g.is_novelty_saturable()); +} + +// ── ContourSnapshot ───────────────────────────────────────── + +/// A contour snapshot carries the spatial shape, the accumulated volume, and +/// the structural churn since the previous report side by side. Standing +/// state and change-since-last-time are different questions about the +/// spatial layer, and the snapshot answers both at once so a host need not +/// difference successive reports to see the graph move. +/// +/// ´claim:readout:a-contour-snapshot-carries-standing-spatial-state-and-the-churn-since-the-last-report-together´ +/// ´test:crate:contour-snapshot-fields´ +#[test] +fn contour_snapshot_fields() { + let cs = ContourSnapshot { + plateau_count: 3, + cell_count: 12, + total_importance: 42_000.0, + splits_since_last_report: 0, + net_removals_since_last_report: 0, + }; + assert_eq!(cs.plateau_count, 3); + assert_eq!(cs.cell_count, 12); + assert!((cs.total_importance - 42_000.0).abs() < f64::EPSILON); + assert_eq!(cs.splits_since_last_report, 0); + assert_eq!(cs.net_removals_since_last_report, 0); +} + +// ── AnalysisSetSummary ────────────────────────────────────── + +/// A summary describing an analysis set with nothing selected still reports +/// every field, its ranges zeroed rather than omitted, and still counts the +/// root as a member of the full set. The shape a host parses does not change +/// with how busy the sentinel is, and the permanent root tracker is visible +/// even at the quietest extreme. +/// +/// ´claim:readout:an-unselected-analysis-set-summarises-with-zeroed-ranges-and-still-counts-the-root´ +/// ´test:crate:analysis-set-summary-empty´ +#[test] +fn analysis_set_summary_empty() { + let summary = AnalysisSetSummary { + competitive_size: 0, + full_size: 1, + investment_set_size: 1, + depth_range: (0, 0), + importance_range: (0.0, 0.0), + v_depth_range: (0, 0), + degenerate_cells_skipped: 0, + }; + assert_eq!(summary.competitive_size, 0); + assert_eq!(summary.full_size, 1); + assert_eq!(summary.investment_set_size, 1); + assert_eq!(summary.depth_range, (0, 0)); + assert_eq!(summary.importance_range, (0.0, 0.0)); + assert_eq!(summary.v_depth_range, (0, 0)); + assert_eq!(summary.degenerate_cells_skipped, 0); +} + +/// A populated summary reports its depth and importance spans as ordered +/// pairs, low end first, and its three sizes widen as the definition of +/// membership loosens: the cells that won the competition, the full set +/// their ancestry closes over, and the investment set that also holds cells +/// still warming. The nesting is what lets a host read the price of +/// analysing a cell as well as the choice to analyse it. +/// +/// ´claim:readout:a-summarys-three-sizes-widen-from-competition-through-closure-to-investment´ +/// ´test:crate:analysis-set-summary-populated´ +#[test] +fn analysis_set_summary_populated() { + let summary = AnalysisSetSummary { + competitive_size: 5, + full_size: 12, + investment_set_size: 15, + depth_range: (0, 4), + importance_range: (100.0, 5000.0), + v_depth_range: (1, 3), + degenerate_cells_skipped: 0, + }; + assert_eq!(summary.competitive_size, 5); + assert_eq!(summary.full_size, 12); + assert_eq!(summary.investment_set_size, 15); + assert!(summary.depth_range.0 < summary.depth_range.1); + assert!(summary.importance_range.0 < summary.importance_range.1); +} + +/// Cells too narrow to support a tracker are counted in the current selection snapshot rather than quietly dropped. Recomputing replaces the count instead of accumulating it, so the report describes the graph the host can inspect now while still exposing a persistently narrow configuration. +/// +/// ´claim:readout:cells-too-narrow-to-track-are-counted-rather-than-silently-dropped´ +/// ´test:crate:analysis-set-summary-with-degenerate-skips´ +#[test] +fn analysis_set_summary_with_degenerate_skips() { + let config = SentinelConfig:: { + analysis_k: 32, + analysis_depth_cutoff: 16, + split_threshold: 1, + d_create: 8, + d_evict: 16, + noise_schedule: NoiseSchedule::Explicit(Vec::new()), + ..SentinelConfig::default() + }; + let mut sentinel = SpectralSentinel::::new(config).unwrap(); + + let report = sentinel.ingest(&[0; 64]); + let expected = sentinel + .graph() + .layers_to(16) + .filter(|(_, node)| (4u32.saturating_sub(node.depth) as usize) < crate::MIN_TRACKER_DIM) + .count(); + + assert!(expected > 0, "fixture must create narrow selection candidates"); + assert_eq!(report.analysis_set_summary.degenerate_cells_skipped, expected); + assert_eq!(sentinel.degenerate_cells_skipped(), expected); +} + +// ── MemberScore ───────────────────────────────────────────── + +/// A member score names the cell it came from — a well-ordered interval and +/// a depth — alongside a real number on each of the four axes and its +/// standardised counterpart. Coordination scores describe a group, so +/// without the identity a host could see that the group behaved oddly but +/// not which part of the domain to look at. +/// +/// ´claim:readout:a-member-score-names-the-cell-it-came-from-so-a-group-finding-can-be-attributed´ +/// ´test:crate:member-score-has-cell-identity´ +#[test] +fn member_score_has_cell_identity() { + let ms = MemberScore:: { + cell_start: 0, + cell_end: u128::MAX / 2, + cell_depth: 1, + novelty: 0.5, + displacement: 0.3, + surprise: 0.7, + coherence: 0.9, + novelty_z: 1.0, + displacement_z: -0.5, + surprise_z: 2.1, + coherence_z: 0.0, + }; + assert!(ms.cell_start < ms.cell_end); + assert_eq!(ms.cell_depth, 1); + assert!(!ms.novelty.is_nan()); + assert!(!ms.displacement.is_nan()); + assert!(!ms.surprise.is_nan()); + assert!(!ms.coherence.is_nan()); +} + +/// Retained warming ancestors stop counting as competitive targets as soon as selection changes. +/// +/// ´claim:readout:warming-targets-follow-current-selection´ +/// ´test:crate:warming-targets-follow-current-selection´ +#[test] +fn warming_targets_follow_current_selection() { + let config = SentinelConfig:: { + analysis_k: 1, + split_threshold: 1, + d_create: 1, + d_evict: 2, + max_rank: 1, + noise_batch_size: 1, + noise_schedule: NoiseSchedule::Explicit(vec![0, 1_000_000]), + background_warming: true, + ..SentinelConfig::default() + }; + let mut sentinel = SpectralSentinel::::new(config).unwrap(); + for _ in 0..7 { + sentinel.ingest(&[0]); + } + + let selected = sentinel.analysis_set().competitive_count(); + let health = sentinel.health(); + assert_eq!(selected, 1, "the fixture must select a competitive target"); + let offline_competitive = selected - health.active_competitive_trackers; + assert!(offline_competitive > 0, "the long non-root schedule must still be warming"); + assert_eq!( + health.warming_competitive_targets, offline_competitive, + "warming classifications must describe the current selection" + ); +} diff --git a/packages/sentinel/src/tests/tracker.rs b/packages/sentinel/src/tests/tracker.rs new file mode 100644 index 000000000..ecef507c5 --- /dev/null +++ b/packages/sentinel/src/tests/tracker.rs @@ -0,0 +1,1080 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +//! # Test index +//! +//! | Test | Area | Claim | +//! |------|------|-------| +//! | [`new_starts_at_rank_one`] | subspace | A newly built cell model claims one direction, has counted no observations of either kind, and regards itself as entirely noise-taught. Starting at a single axis means the model asserts as little structure as it can and has to earn every further direction from energy it actually observes; starting at full noise influence means nothing it later reports is trusted until real traffic has displaced the warm-up that shaped it. | +//! | [`new_accepts_min_tracker_dim`] | subspace | The narrowest width the engine admits is admitted, and the model it produces is an ordinary one starting at a single axis. The minimum is a boundary that is included rather than approached: cells right at the edge of being too deep to analyse still get a working model instead of a special case. | +//! | [`new_panics_on_zero_dim_debug`] | subspace | A width below the minimum is treated as a caller's mistake, not as a state to accommodate: building a model for a cell with nothing left to analyse fails loudly in debug builds and names the offending width. Filtering such cells out is the selector's job, so a model that receives one has been handed something upstream should have excluded, and the fault is worth more than a degenerate model that would score nothing meaningfully. | +//! | [`new_panics_on_dim_one_debug`] | subspace | cites (´claim:subspace:a-width-below-the-minimum-is-a-callers-fault-caught-in-debug-rather-than-a-state-to-accommodate´) | +//! | [`dim_and_cap_reflect_construction`] | subspace | A cell's rank ceiling is the lesser of the configured maximum and its own width: a wide cell is capped by policy, a narrow one by geometry. There are no more independent directions than dimensions to hold them, so the width binds where it is the smaller of the two, and one configuration can serve cells of every depth without being retuned per depth. | +//! | [`scoring_geometry_matches_state`] | subspace | A model reports the geometry its scores were computed in: the width it works over, the ceiling it may grow to, and the residual degrees of freedom left after the claimed directions are removed. That last figure is the divisor novelty is normalised by, so publishing it lets a host compare scores from cells of different depths and ranks instead of comparing numbers whose scale it cannot see. | +//! | [`observe_returns_the_scoring_rank`] | subspace | A report carries the rank that was in force while the batch was scored, which on an unadapted model is the rank it was built with: adaptation happens after scoring, so a report never describes a model that did not produce it. The tracker is told which depth it serves but keeps that to itself — the depth reaches a host on the cell report, which is assembled from the cell rather than echoed back from the model. | +//! | [`rank_change_report_describes_the_scoring_state`] | subspace | On a batch that changes rank, the report and the tracker's geometry snapshot keep the earlier rank and residual degrees of freedom that normalised novelty, while the current rank advances for the next batch. Multiplying novelty by the published residual degrees of freedom recovers the residual energy, so the geometry beside the score can be used to reconstruct its scale. | +//! | [`non_adapting_batch_reports_current_rank_as_scoring_rank`] | subspace | When the adaptation interval does not fall on a batch, the current model and the scoring snapshot agree: the report rank equals the tracker's rank and its residual degrees of freedom are derived from that same rank. | +//! | [`observe_per_sample_when_enabled`] | subspace | Per-row detail is produced only where a cell is configured to want it. Building it costs a standardisation of every axis for every row, which is worth paying when a host needs to know which observation in a batch was responsible and wasted when it only needs the batch's summary — so the choice is made per configuration rather than always. | +//! | [`observe_no_per_sample_when_disabled`] | subspace | cites (´claim:subspace:per-row-detail-is-produced-only-where-it-is-configured-because-it-costs-work-per-row´) | +//! | [`observe_report_batch_size_matches`] | subspace | Where per-row detail is produced there is exactly one entry for every row handed in, whether the batch was a single observation or many, and the same model gives both answers in turn. The correspondence is positional, so a host can attribute a score back to the observation that earned it without the model needing to know what that observation was. | +//! | [`maturity_noise_only`] | subspace | Maturity is counted in observations rather than in calls: a batch of several injected rows advances the noise tally by that many and leaves the real tally untouched. Counting rows is what makes the figure comparable across cells fed at different batch sizes, and keeping the two tallies apart is what lets a host ask how much of what a cell knows it was taught deliberately. | +//! | [`maturity_real_only`] | subspace | cites (´claim:subspace:maturity-counts-observations-row-by-row-and-keeps-the-injected-and-the-real-apart´) | +//! | [`maturity_mixed_real_and_noise`] | subspace | cites (´claim:subspace:maturity-counts-observations-row-by-row-and-keeps-the-injected-and-the-real-apart´) | +//! | [`noise_influence_decays_toward_zero_for_real`] | subspace | Sustained real traffic drives noise influence below the maturity threshold one tracker batch at a time, matching the learned model's forgetting cadence regardless of the number of rows in each batch. | +//! | [`noise_influence_converges_toward_one_for_noise`] | subspace | Warm-up is re-enterable. A cell pushed part-way down by real traffic climbs back toward full influence when injection resumes, by the same geometric step run in the other direction. Cells are re-warmed after splits and long silences, so a figure that could only fall would leave a re-taught cell wrongly claiming its knowledge came from traffic it never saw. | +//! | [`novelty_low_for_repeated_pattern`] | subspace | Novelty is whatever the learned directions fail to explain, divided by the room left over after those directions are removed. A pattern the model has been trained on lies almost inside its own axes, so what is left is nearly nothing and the pattern scores as unremarkable — the model reports familiarity by having nothing to report. | +//! | [`novelty_high_for_unseen_pattern`] | subspace | cites (´claim:subspace:novelty-is-what-the-learned-directions-fail-to-explain-so-a-familiar-pattern-scores-low´) | +//! | [`coherence_cold_at_rank_one`] | subspace | Coherence measures whether pairs of axes move together as they usually do, so at a single axis it does not exist: there is no pair, and the score is exactly zero on every batch rather than some small residue. Because those zeroes are an absence of the question and not an answer to it, the axis's baseline is deliberately left cold while rank stays at one — otherwise it would learn that zero is normal and treat the first genuine coherence value, once a second axis appears, as an alarm. | +//! | [`rank_stays_bounded_by_max_rank`] | subspace | However long a cell runs and however strongly its traffic is structured, rank stays within a floor of one axis and the ceiling it was built with. The ceiling is what bounds the cost of every later step — the work per batch grows with rank — and the floor is what keeps a model from disappearing entirely during a quiet stretch and having to be rebuilt from nothing. | +//! | [`rank_acquires_buffer_dimension`] | subspace | A cell keeps one axis more than the energy threshold strictly demands. Fed a single dominant pattern, the leading direction alone already captures the required share, yet the model settles at two directions rather than one. The spare axis is where a genuinely new direction first shows up: without it, novel structure would have to displace the established pattern before the model could represent it at all, and the arrival would be invisible until it was already dominant. | +//! | [`energy_ratio_and_top_singular_value_evolve`] | subspace | A model that has been fed traffic reports a leading direction with real strength behind it and an energy share that is positive and cannot exceed the whole. The share is what the claimed axes explain out of everything the model holds, so it is bounded above by construction, and a leading value at zero would mean the model had learned nothing — the two figures together are how a host reads whether a cell's model has substance. | +//! | [`report_energy_figures_describe_the_model_that_scored_the_batch`] | subspace | The energy share and leading singular value a report carries belong to the model that scored the batch, not to the model the batch left behind. Both are read off the same sigmas, and the batch replaces those sigmas before the report is assembled, so a figure read afterwards would describe a model that has not scored anything yet — and the energy share read afterwards is not even that, but the evolved sigmas divided by the rank that scored, a pairing no model ever held. The rank and the geometry beside them already describe the scoring model, so a host reading one report would be comparing an energy share against a rank drawn from a different moment. | +//! | [`cusum_allowance_is_invariant_to_eps`] | subspace | Changing the denominator stability constant does not change a novelty CUSUM trajectory. The allowance belongs to the slow baseline variance alone, so two otherwise identical trackers accumulate the same drift even when their configured stability constants differ by the scale of that variance. | +//! | [`cusum_reset_zeroes_steps`] | subspace | Clearing a cell's drift evidence restarts the count of batches that evidence was gathered over, so the very next batch is the first step of a new run rather than the next of an old one. Accumulated drift is only interpretable against how long it took to accumulate, and a fresh sum read against a stale count would look like a sudden collapse in drift rather than a deliberate acknowledgement of it. | +//! | [`seed_cusum_slow_from_baselines_then_reset`] | subspace | Finishing warm-up is a two-step handover applied to every axis at once: the long-memory reference is seeded from the short-memory one that has already converged on the injected traffic, and only then is the evidence cleared. Done in that order, drift detection resumes from a state where the two references agree, so the first real batches are scored against a reference that is already current instead of registering the warm-up's own leftover gap as drift for as long as the slow memory takes to catch up. | +//! | [`explicit_reset_clip_pressure`] | subspace | Clip pressure records how often a cell has lately been discarding scores as outliers, and it widens that cell's own outlier band while it is high. Warm-up is exactly when it runs high, since injected traffic is scored against a barely-formed model. Clearing it zeroes every axis together, so a cell entering production judges its first real batches by the ordinary band rather than by one still slackened by the noise it was taught with. | +//! | [`eta_threshold_crossing_zeros_clip_pressure`] | subspace | cites (´claim:subspace:clearing-clip-pressure-zeroes-every-axis-so-warm-up-clipping-does-not-slacken-production-scoring´) | +//! | [`a_declined_incremental_step_still_yields_a_usable_model`] | subspace | A step the incremental strategy declines is answered by the dense one, so what comes back is always a basis that was actually re-orthogonalised. The incremental path builds a small kernel and back-transforms through it, which needs spare dimensions to be stable, and at the coordination tier's width it declines every step on exactly those grounds. Declining is the honest answer, and the dispatcher's response to it is to run the strategy that does not need the step. That is the same response the incremental path now gives when its own corrective factorisation fails — the step that re-orthogonalises the basis — because the alternative is returning the basis from before that step under a field documented orthonormal, which every caller writes straight into a tracker and then relies on. The failure of that factorisation cannot be provoked from outside without a hook into the linear algebra, so what is exercised here is the fallback it now takes. | + +//! Crate-level tests for [`SubspaceTracker`](crate::sentinel::tracker::SubspaceTracker). +//! +//! One of these belongs to each analysed cell, and it is the whole of what +//! that cell knows. It holds a small orthonormal basis — a handful of +//! directions that between them explain most of the traffic the cell has seen +//! — the strength of each direction, and a running picture of how a batch's +//! coordinates within those directions are usually distributed. Every batch is +//! scored against that model before the model absorbs it, so a score always +//! measures a departure from what was known beforehand and never from a state +//! the batch itself helped create. +//! +//! Three decisions shape the state. Rank is adaptive but heavily damped: it +//! moves by at most one axis at a time, toward the smallest number of +//! directions that captures the configured share of energy plus one spare, and +//! it never leaves the band between a single axis and the cell's ceiling. A +//! cell that has only ever seen injected noise must be distinguishable from one +//! taught by real traffic, so a noise-influence figure starts at full and +//! decays toward nothing with each real batch — climbing back if noise resumes, +//! because warm-up is re-enterable rather than a door that shuts once. +//! And the tracker knows nothing about cells, coordinates or the host's +//! domain: it takes rows of numbers and returns a report, which is what lets +//! the same engine serve every depth of the tree. +//! +//! Because the tracker owns no randomness, deterministic inputs make every +//! assertion here reproducible: the same rows produce the same model, the same +//! rank trajectory and the same scores on every run. + +use crate::config::{NoiseSchedule, SentinelConfig}; +use crate::sentinel::tracker::SubspaceTracker; + +// ════════════════════════════════════════════════════════════ +// Helpers +// ════════════════════════════════════════════════════════════ + +/// Config used by most tests: fast adaptation, per-sample scores on. +fn cfg_per_sample() -> SentinelConfig { + SentinelConfig { + max_rank: 4, + forgetting_factor: 0.95, + rank_update_interval: 10, + energy_threshold: 0.90, + eps: 1e-6, + per_sample_scores: true, + cusum_allowance_sigmas: 0.5, + ..SentinelConfig::default() + } +} + +/// Same as [`cfg_per_sample`] but with per-sample scores *disabled*. +fn cfg_no_per_sample() -> SentinelConfig { + SentinelConfig { + per_sample_scores: false, + ..cfg_per_sample() + } +} + +/// Generate a batch of centred bit vectors from `u128` values. +fn centred_rows(values: &[u128], depth: usize) -> Vec> { + values + .iter() + .map(|&v| { + (0..depth) + .map(|i| if (v >> (127 - i)) & 1 == 1 { 0.5 } else { -0.5 }) + .collect() + }) + .collect() +} + +fn as_slices(vecs: &[Vec]) -> Vec<&[f64]> { + vecs.iter().map(Vec::as_slice).collect() +} + +const MATURITY_THRESHOLD: f64 = 0.01; + +fn decay_crossing(initial: f64, lambda: f64) -> (usize, f64) { + (1_usize..=usize::MAX) + .scan(initial, |influence, batch| { + *influence *= lambda; + Some((batch, *influence)) + }) + .find(|(_, influence)| *influence < MATURITY_THRESHOLD) + .expect("a validated forgetting factor must cross the maturity threshold") +} + +// ════════════════════════════════════════════════════════════ +// Construction & accessors +// ════════════════════════════════════════════════════════════ + +/// A newly built cell model claims one direction, has counted no observations +/// of either kind, and regards itself as entirely noise-taught. Starting at a +/// single axis means the model asserts as little structure as it can and has +/// to earn every further direction from energy it actually observes; starting +/// at full noise influence means nothing it later reports is trusted until +/// real traffic has displaced the warm-up that shaped it. +/// +/// ´claim:subspace:a-new-cell-model-claims-one-direction-and-counts-itself-entirely-noise-taught´ +/// ´test:crate:new-starts-at-rank-one´ +#[test] +fn new_starts_at_rank_one() { + let cfg = cfg_per_sample(); + let t = SubspaceTracker::new(8, &cfg, 0.999); + + assert_eq!(t.rank(), 1); + assert_eq!(t.maturity().real_observations, 0); + assert_eq!(t.maturity().noise_observations, 0); + assert!((t.maturity().noise_influence - 1.0).abs() < f64::EPSILON); +} + +/// The narrowest width the engine admits is admitted, and the model it +/// produces is an ordinary one starting at a single axis. The minimum is a +/// boundary that is included rather than approached: cells right at the edge +/// of being too deep to analyse still get a working model instead of a special +/// case. +/// +/// ´claim:subspace:the-narrowest-admissible-width-yields-an-ordinary-model-rather-than-a-special-case´ +/// ´test:crate:new-accepts-min-tracker-dim´ +#[test] +fn new_accepts_min_tracker_dim() { + let cfg = cfg_per_sample(); + let t = SubspaceTracker::new(crate::MIN_TRACKER_DIM, &cfg, 0.999); + assert_eq!(t.rank(), 1); +} + +/// A width below the minimum is treated as a caller's mistake, not as a state +/// to accommodate: building a model for a cell with nothing left to analyse +/// fails loudly in debug builds and names the offending width. Filtering such +/// cells out is the selector's job, so a model that receives one has been +/// handed something upstream should have excluded, and the fault is worth more +/// than a degenerate model that would score nothing meaningfully. +/// +/// ´claim:subspace:a-width-below-the-minimum-is-a-callers-fault-caught-in-debug-rather-than-a-state-to-accommodate´ +/// ´test:crate:new-panics-on-zero-dim-debug´ +#[test] +#[cfg(debug_assertions)] +#[should_panic(expected = "SubspaceTracker::new() called with dim=0")] +fn new_panics_on_zero_dim_debug() { + let cfg = cfg_per_sample(); + drop(SubspaceTracker::new(0, &cfg, 0.999)); +} + +/// The rejection is not merely of the empty width: a single dimension is +/// refused too, and the message again names what was asked for. One direction +/// cannot be decomposed into structure and residual — there is no room left +/// over once an axis is claimed — so the boundary sits above zero rather than +/// at it. +/// +/// (´claim:subspace:a-width-below-the-minimum-is-a-callers-fault-caught-in-debug-rather-than-a-state-to-accommodate´) +/// ´test:crate:new-panics-on-dim-one-debug´ +#[test] +#[cfg(debug_assertions)] +#[should_panic(expected = "SubspaceTracker::new() called with dim=1")] +fn new_panics_on_dim_one_debug() { + let cfg = cfg_per_sample(); + drop(SubspaceTracker::new(1, &cfg, 0.999)); +} + +/// A cell's rank ceiling is the lesser of the configured maximum and its own +/// width: a wide cell is capped by policy, a narrow one by geometry. There are +/// no more independent directions than dimensions to hold them, so the width +/// binds where it is the smaller of the two, and one configuration can serve +/// cells of every depth without being retuned per depth. +/// +/// ´claim:subspace:the-rank-ceiling-is-the-lesser-of-the-configured-maximum-and-the-cells-own-width´ +/// ´test:crate:dim-and-cap-reflect-construction´ +#[test] +fn dim_and_cap_reflect_construction() { + let cfg = SentinelConfig { + max_rank: 6, + ..cfg_per_sample() + }; + + // When dim > max_rank, cap = max_rank. + let t = SubspaceTracker::new(16, &cfg, 0.999); + assert_eq!(t.dim(), 16); + assert_eq!(t.cap(), 6); + + // When dim < max_rank, cap = dim. + let t2 = SubspaceTracker::new(4, &cfg, 0.999); + assert_eq!(t2.dim(), 4); + assert_eq!(t2.cap(), 4); +} + +/// A model reports the geometry its scores were computed in: the width it +/// works over, the ceiling it may grow to, and the residual degrees of freedom +/// left after the claimed directions are removed. That last figure is the +/// divisor novelty is normalised by, so publishing it lets a host compare +/// scores from cells of different depths and ranks instead of comparing +/// numbers whose scale it cannot see. +/// +/// ´claim:subspace:a-model-publishes-the-residual-degrees-of-freedom-its-scores-were-normalised-by´ +/// ´test:crate:scoring-geometry-matches-state´ +#[test] +fn scoring_geometry_matches_state() { + let cfg = SentinelConfig { + max_rank: 4, + ..cfg_per_sample() + }; + let t = SubspaceTracker::new(16, &cfg, 0.999); + let g = t.scoring_geometry(); + + assert_eq!(g.dim, 16); + assert_eq!(g.cap, 4); + // rank starts at 1, so residual_dof = dim - rank = 15. + assert_eq!(g.residual_dof, 15); +} + +// ════════════════════════════════════════════════════════════ +// Observe — report structure +// ════════════════════════════════════════════════════════════ + +/// A report carries the rank that was in force while the batch was scored, +/// which on an unadapted model is the rank it was built with: adaptation +/// happens after scoring, so a report never describes a model that did not +/// produce it. The tracker is told which depth it serves but keeps that to +/// itself — the depth reaches a host on the cell report, which is assembled +/// from the cell rather than echoed back from the model. +/// +/// ´claim:subspace:a-report-carries-the-rank-that-scored-the-batch´ +/// ´test:crate:observe-returns-the-scoring-rank´ +#[test] +fn observe_returns_the_scoring_rank() { + let cfg = cfg_per_sample(); + let mut t = SubspaceTracker::new(8, &cfg, 0.999); + + let rows = centred_rows(&[0x0123_4567_89AB_CDEF_0123_4567_89AB_CDEF], 8); + let report = t.observe(&as_slices(&rows), 8, false); + + assert_eq!(report.rank, 1); // hasn't adapted yet +} + +/// On a batch that changes rank, the report and the tracker's geometry snapshot keep the earlier rank and residual degrees of freedom that normalised novelty, while the current rank advances for the next batch. Multiplying novelty by the published residual degrees of freedom recovers the residual energy, so the geometry beside the score can be used to reconstruct its scale. +/// +/// ´claim:subspace:a-rank-change-report-describes-the-state-that-scored-the-batch´ +/// ´test:crate:rank-change-report-describes-the-scoring-state´ +#[test] +fn rank_change_report_describes_the_scoring_state() { + let cfg = SentinelConfig { + max_rank: 4, + rank_update_interval: 1, + energy_threshold: 0.90, + noise_schedule: NoiseSchedule::Explicit(Vec::new()), + ..cfg_no_per_sample() + }; + let mut tracker = SubspaceTracker::new(8, &cfg, 0.999); + let rows = centred_rows(&[0], 8); + + let report = tracker.observe(&as_slices(&rows), 0, false); + let scoring_geometry = tracker.scoring_geometry(); + + assert_eq!(report.rank, 1, "the initial rank must score the first batch"); + assert_eq!(tracker.rank(), 2, "adaptation must prepare rank two for the next batch"); + assert_eq!(scoring_geometry.dim, 8); + assert_eq!(scoring_geometry.cap, 4); + assert_eq!(scoring_geometry.residual_dof, 7); + assert_eq!(report.geometry.dim, scoring_geometry.dim); + assert_eq!(report.geometry.cap, scoring_geometry.cap); + assert_eq!( + report.geometry.residual_dof, scoring_geometry.residual_dof, + "the tracker geometry snapshot must retain the scoring residual degrees of freedom" + ); + + let residual_dof = f64::from(u32::try_from(scoring_geometry.residual_dof).expect("the test geometry must fit in u32")); + let operation_scale = f64::from(u32::try_from(tracker.dim()).expect("the test dimension must fit in u32")); + let expected_novelty = 0.5_f64 * 0.5; + let expected_residual_energy = residual_dof * expected_novelty; + let tolerance = f64::EPSILON * operation_scale * expected_residual_energy.max(1.0); + let recovered_residual_energy = report.scores.novelty.mean * residual_dof; + + assert!( + (report.scores.novelty.mean - expected_novelty).abs() <= tolerance, + "novelty must be residual energy divided by the seven scoring residual degrees of freedom" + ); + assert!( + (recovered_residual_energy - expected_residual_energy).abs() <= tolerance, + "novelty times the reported scoring residual degrees of freedom must recover residual energy" + ); + assert_eq!( + report.scores.coherence.mean.to_bits(), + 0.0_f64.to_bits(), + "coherence does not exist at the scoring rank" + ); +} + +/// When the adaptation interval does not fall on a batch, the current model and the scoring snapshot agree: the report rank equals the tracker's rank and its residual degrees of freedom are derived from that same rank. +/// +/// ´claim:subspace:a-non-adapting-batch-reports-the-current-rank-as-its-scoring-rank´ +/// ´test:crate:non-adapting-batch-reports-current-rank-as-scoring-rank´ +#[test] +fn non_adapting_batch_reports_current_rank_as_scoring_rank() { + let cfg = SentinelConfig { + max_rank: 4, + rank_update_interval: 2, + noise_schedule: NoiseSchedule::Explicit(Vec::new()), + ..cfg_no_per_sample() + }; + let mut tracker = SubspaceTracker::new(8, &cfg, 0.999); + let rows = centred_rows(&[0], 8); + + let report = tracker.observe(&as_slices(&rows), 0, false); + let scoring_geometry = tracker.scoring_geometry(); + + assert_eq!( + tracker.rank(), + report.rank, + "rank must stay unchanged between adaptation steps" + ); + assert_eq!(scoring_geometry.residual_dof, tracker.dim() - tracker.rank()); + assert_eq!(report.geometry.residual_dof, scoring_geometry.residual_dof); +} + +/// Per-row detail is produced only where a cell is configured to want it. +/// Building it costs a standardisation of every axis for every row, which is +/// worth paying when a host needs to know which observation in a batch was +/// responsible and wasted when it only needs the batch's summary — so the +/// choice is made per configuration rather than always. +/// +/// ´claim:subspace:per-row-detail-is-produced-only-where-it-is-configured-because-it-costs-work-per-row´ +/// ´test:crate:observe-per-sample-when-enabled´ +#[test] +fn observe_per_sample_when_enabled() { + let cfg = cfg_per_sample(); + let mut t = SubspaceTracker::new(8, &cfg, 0.999); + + let rows = centred_rows(&[1, 2, 3], 8); + let report = t.observe(&as_slices(&rows), 8, false); + + let ps = report.per_sample.as_ref().expect("per_sample should be Some when enabled"); + assert_eq!(ps.len(), 3, "one SampleScore per input row"); +} + +/// The other end of the same choice: with per-row detail switched off the +/// report carries none, rather than carrying an empty list or zeroed entries. +/// Absent and empty are different answers, and a host reading a report can +/// tell that the detail was never asked for instead of concluding the batch +/// had nothing in it. +/// +/// (´claim:subspace:per-row-detail-is-produced-only-where-it-is-configured-because-it-costs-work-per-row´) +/// ´test:crate:observe-no-per-sample-when-disabled´ +#[test] +fn observe_no_per_sample_when_disabled() { + let cfg = cfg_no_per_sample(); + let mut t = SubspaceTracker::new(8, &cfg, 0.999); + + let rows = centred_rows(&[1, 2, 3], 8); + let report = t.observe(&as_slices(&rows), 8, false); + + assert!(report.per_sample.is_none(), "per_sample should be None when disabled"); +} + +/// Where per-row detail is produced there is exactly one entry for every row +/// handed in, whether the batch was a single observation or many, and the same +/// model gives both answers in turn. The correspondence is positional, so a +/// host can attribute a score back to the observation that earned it without +/// the model needing to know what that observation was. +/// +/// ´claim:subspace:there-is-exactly-one-per-row-score-for-every-row-in-the-batch-whatever-its-size´ +/// ´test:crate:observe-report-batch-size-matches´ +#[test] +fn observe_report_batch_size_matches() { + let cfg = cfg_per_sample(); + let mut t = SubspaceTracker::new(8, &cfg, 0.999); + + // Single sample. + let rows1 = centred_rows(&[1], 8); + let r1 = t.observe(&as_slices(&rows1), 8, false); + assert_eq!(r1.per_sample.as_ref().unwrap().len(), 1); + + // Larger batch. + let rows8 = centred_rows(&[1, 2, 3, 4, 5, 6, 7, 8], 8); + let r8 = t.observe(&as_slices(&rows8), 8, false); + assert_eq!(r8.per_sample.as_ref().unwrap().len(), 8); +} + +// ════════════════════════════════════════════════════════════ +// Maturity tracking +// ════════════════════════════════════════════════════════════ + +/// Maturity is counted in observations rather than in calls: a batch of +/// several injected rows advances the noise tally by that many and leaves the +/// real tally untouched. Counting rows is what makes the figure comparable +/// across cells fed at different batch sizes, and keeping the two tallies +/// apart is what lets a host ask how much of what a cell knows it was taught +/// deliberately. +/// +/// ´claim:subspace:maturity-counts-observations-row-by-row-and-keeps-the-injected-and-the-real-apart´ +/// ´test:crate:maturity-noise-only´ +#[test] +fn maturity_noise_only() { + let cfg = cfg_per_sample(); + let mut t = SubspaceTracker::new(8, &cfg, 0.999); + + let rows = centred_rows(&[1, 2, 3], 8); + t.observe(&as_slices(&rows), 8, true); + + assert_eq!(t.maturity().noise_observations, 3); + assert_eq!(t.maturity().real_observations, 0); +} + +/// The mirror case pins the other tally: real rows advance the real count by +/// one per row and leave the injected count at nothing. Which tally a batch +/// lands in is decided by the caller and not inferred from the data, because +/// synthetic and genuine observations can look identical and only the host +/// knows which it sent. +/// +/// (´claim:subspace:maturity-counts-observations-row-by-row-and-keeps-the-injected-and-the-real-apart´) +/// ´test:crate:maturity-real-only´ +#[test] +fn maturity_real_only() { + let cfg = cfg_per_sample(); + let mut t = SubspaceTracker::new(8, &cfg, 0.999); + + let rows = centred_rows(&[10, 20], 8); + t.observe(&as_slices(&rows), 8, false); + + assert_eq!(t.maturity().real_observations, 2); + assert_eq!(t.maturity().noise_observations, 0); +} + +/// The two tallies are independent accumulators, not two views of one figure: +/// the same rows sent first as injected and then as real leave both counts +/// standing at what each was given. What the real batch does move is the +/// influence figure, which drops below full the moment any genuine +/// observation arrives — so the record of how a cell was taught survives while +/// the weight given to that teaching starts falling immediately. +/// +/// (´claim:subspace:maturity-counts-observations-row-by-row-and-keeps-the-injected-and-the-real-apart´) +/// ´test:crate:maturity-mixed-real-and-noise´ +#[test] +fn maturity_mixed_real_and_noise() { + let cfg = cfg_per_sample(); + let mut t = SubspaceTracker::new(8, &cfg, 0.999); + + let rows = centred_rows(&[1, 2, 3], 8); + let slices = as_slices(&rows); + + t.observe(&slices, 8, true); // noise + assert_eq!(t.maturity().noise_observations, 3); + assert_eq!(t.maturity().real_observations, 0); + + t.observe(&slices, 8, false); // real + assert_eq!(t.maturity().real_observations, 3); + assert!(t.maturity().noise_influence < 1.0); +} + +/// Sustained real traffic drives noise influence below the maturity threshold. +/// The figure falls by the forgetting factor once per tracker batch, matching +/// the model whose warm-up share it measures regardless of how many rows that +/// batch carries. +/// +/// ´claim:subspace:sustained-real-traffic-drives-the-noise-influence-to-nothing-so-a-cell-can-declare-itself-warmed´ +/// ´test:crate:noise-influence-decays-toward-zero-for-real´ +#[test] +fn noise_influence_decays_toward_zero_for_real() { + let cfg = cfg_per_sample(); + let mut t = SubspaceTracker::new(8, &cfg, 0.999); + + let rows = centred_rows(&[1, 2, 3, 4], 8); + let slices = as_slices(&rows); + + let (crossing_batch, _) = decay_crossing(t.maturity().noise_influence, cfg.forgetting_factor); + for _ in 0..crossing_batch { + t.observe(&slices, 8, false); + } + + assert!( + t.maturity().noise_influence < MATURITY_THRESHOLD, + "η should decay toward 0 after many real batches, got {}", + t.maturity().noise_influence, + ); +} + +/// Warm-up is re-enterable. A cell pushed part-way down by real traffic climbs +/// back toward full influence when injection resumes, by the same geometric +/// step run in the other direction. Cells are re-warmed after splits and long +/// silences, so a figure that could only fall would leave a re-taught cell +/// wrongly claiming its knowledge came from traffic it never saw. +/// +/// ´claim:subspace:renewed-injection-drives-the-influence-back-up-so-warm-up-is-re-enterable-rather-than-a-one-way-door´ +/// ´test:crate:noise-influence-converges-toward-one-for-noise´ +#[test] +fn noise_influence_converges_toward_one_for_noise() { + let cfg = cfg_per_sample(); + let mut t = SubspaceTracker::new(8, &cfg, 0.999); + + // Start by feeding real data to push η below 1.0. + let rows = centred_rows(&[1, 2, 3, 4], 8); + let slices = as_slices(&rows); + for _ in 0..10 { + t.observe(&slices, 8, false); + } + let eta_after_real = t.maturity().noise_influence; + assert!(eta_after_real < 1.0); + + // Now feed noise — η should move back toward 1.0. + for _ in 0..30 { + t.observe(&slices, 8, true); + } + + assert!( + t.maturity().noise_influence > eta_after_real, + "η should increase toward 1 under noise, was {eta_after_real}, now {}", + t.maturity().noise_influence, + ); +} + +// ════════════════════════════════════════════════════════════ +// Scoring behaviour +// ════════════════════════════════════════════════════════════ + +/// Novelty is whatever the learned directions fail to explain, divided by the +/// room left over after those directions are removed. A pattern the model has +/// been trained on lies almost inside its own axes, so what is left is nearly +/// nothing and the pattern scores as unremarkable — the model reports +/// familiarity by having nothing to report. +/// +/// ´claim:subspace:novelty-is-what-the-learned-directions-fail-to-explain-so-a-familiar-pattern-scores-low´ +/// ´test:crate:novelty-low-for-repeated-pattern´ +#[test] +fn novelty_low_for_repeated_pattern() { + let cfg = cfg_per_sample(); + let mut t = SubspaceTracker::new(16, &cfg, 0.999); + + let pattern: u128 = 0xAAAA_0000_0000_0000_0000_0000_0000_0000; + let rows = centred_rows(&[pattern; 8], 16); + let slices = as_slices(&rows); + + // Train the subspace on a single repeated pattern. + for _ in 0..20 { + t.observe(&slices, 16, false); + } + + // Score the same pattern — novelty should be low. + let report = t.observe(&slices, 16, false); + assert!( + report.scores.novelty.mean < 1.0, + "novelty should be low for a learned pattern, got {}", + report.scores.novelty.mean, + ); +} + +/// The comparative end of the same statement, which is the end that matters +/// operationally: a pattern the model was never trained on scores strictly +/// higher than the one it was, on the same model in the same state. Novelty is +/// meaningful as a ranking against what this cell has learned rather than as +/// an absolute quantity, so what is claimed is the ordering between the two +/// and not a threshold either of them crosses. +/// +/// (´claim:subspace:novelty-is-what-the-learned-directions-fail-to-explain-so-a-familiar-pattern-scores-low´) +/// ´test:crate:novelty-high-for-unseen-pattern´ +#[test] +fn novelty_high_for_unseen_pattern() { + let cfg = cfg_per_sample(); + let mut t = SubspaceTracker::new(16, &cfg, 0.999); + + // Train on pattern A. + let pattern_a: u128 = 0xAAAA_0000_0000_0000_0000_0000_0000_0000; + let rows_a = centred_rows(&[pattern_a; 8], 16); + let slices_a = as_slices(&rows_a); + for _ in 0..20 { + t.observe(&slices_a, 16, false); + } + + // Novelty for trained pattern A. + let report_a = t.observe(&slices_a, 16, false); + + // Score a completely different pattern B (without further training). + let pattern_b: u128 = 0x5555_FFFF_0000_0000_0000_0000_0000_0000; + let rows_b = centred_rows(&[pattern_b; 8], 16); + let report_b = t.observe(&as_slices(&rows_b), 16, false); + + assert!( + report_b.scores.novelty.mean > report_a.scores.novelty.mean, + "unseen pattern should produce higher novelty ({}) than learned pattern ({})", + report_b.scores.novelty.mean, + report_a.scores.novelty.mean, + ); +} + +/// Coherence measures whether pairs of axes move together as they usually do, +/// so at a single axis it does not exist: there is no pair, and the score is +/// exactly zero on every batch rather than some small residue. Because those +/// zeroes are an absence of the question and not an answer to it, the axis's +/// baseline is deliberately left cold while rank stays at one — otherwise it +/// would learn that zero is normal and treat the first genuine coherence +/// value, once a second axis appears, as an alarm. +/// +/// ´claim:subspace:coherence-does-not-exist-at-a-single-axis-because-there-is-no-pair-to-be-coherent-about´ +/// ´test:crate:coherence-cold-at-rank-one´ +#[test] +fn coherence_cold_at_rank_one() { + // At rank 1 there are no pairs, so coherence should be zero. + let cfg = SentinelConfig { + max_rank: 1, + rank_update_interval: 1, + ..cfg_per_sample() + }; + let mut t = SubspaceTracker::new(8, &cfg, 0.999); + + let rows = centred_rows(&[0xFF00_0000_0000_0000_0000_0000_0000_0000; 4], 8); + let slices = as_slices(&rows); + + for _ in 0..10 { + let report = t.observe(&slices, 8, false); + assert!( + report.scores.coherence.mean.abs() < f64::EPSILON, + "coherence should be zero at rank 1, got {}", + report.scores.coherence.mean, + ); + } +} + +// ════════════════════════════════════════════════════════════ +// Rank adaptation +// ════════════════════════════════════════════════════════════ + +/// However long a cell runs and however strongly its traffic is structured, +/// rank stays within a floor of one axis and the ceiling it was built with. +/// The ceiling is what bounds the cost of every later step — the work per +/// batch grows with rank — and the floor is what keeps a model from +/// disappearing entirely during a quiet stretch and having to be rebuilt from +/// nothing. +/// +/// ´claim:subspace:rank-moves-within-a-floor-of-one-axis-and-the-configured-ceiling-and-never-outside-them´ +/// ´test:crate:rank-stays-bounded-by-max-rank´ +#[test] +fn rank_stays_bounded_by_max_rank() { + let cfg = SentinelConfig { + max_rank: 3, + rank_update_interval: 1, + ..cfg_per_sample() + }; + let mut t = SubspaceTracker::new(8, &cfg, 0.999); + + let rows = centred_rows(&[0xFF00_0000_0000_0000_0000_0000_0000_0000; 8], 8); + let slices = as_slices(&rows); + + for _ in 0..50 { + t.observe(&slices, 8, false); + } + + assert!(t.rank() >= 1); + assert!(t.rank() <= 3, "rank should not exceed max_rank, got {}", t.rank()); +} + +/// A cell keeps one axis more than the energy threshold strictly demands. Fed +/// a single dominant pattern, the leading direction alone already captures the +/// required share, yet the model settles at two directions rather than one. +/// The spare axis is where a genuinely new direction first shows up: without +/// it, novel structure would have to displace the established pattern before +/// the model could represent it at all, and the arrival would be invisible +/// until it was already dominant. +/// +/// ´claim:subspace:the-model-keeps-one-axis-beyond-what-the-energy-threshold-demands-so-a-new-direction-has-somewhere-to-land´ +/// ´test:crate:rank-acquires-buffer-dimension´ +#[test] +fn rank_acquires_buffer_dimension() { + // With one dominant pattern capturing ≥ 90% energy, the target is + // min{i : c_i ≥ 0.90} + 1 = 0 + 1 = 1 → +1 buffer → 2. + let cfg = SentinelConfig:: { + max_rank: 8, + rank_update_interval: 1, + energy_threshold: 0.90, + ..SentinelConfig::default() + }; + let mut t = SubspaceTracker::new(16, &cfg, 0.999); + + let pattern: u128 = 0xAAAA_BBBB_0000_0000_0000_0000_0000_0000; + let rows = centred_rows(&[pattern; 16], 16); + let slices = as_slices(&rows); + + for _ in 0..100 { + t.observe(&slices, 16, false); + } + + assert!( + t.rank() >= 2, + "rank should be at least 2 (buffer dimension), got {}", + t.rank(), + ); +} + +/// A model that has been fed traffic reports a leading direction with real +/// strength behind it and an energy share that is positive and cannot exceed +/// the whole. The share is what the claimed axes explain out of everything the +/// model holds, so it is bounded above by construction, and a leading value at +/// zero would mean the model had learned nothing — the two figures together +/// are how a host reads whether a cell's model has substance. +/// +/// ´claim:subspace:a-trained-model-reports-a-leading-direction-with-strength-and-an-energy-share-that-cannot-exceed-the-whole´ +/// ´test:crate:energy-ratio-and-top-singular-value-evolve´ +#[test] +fn energy_ratio_and_top_singular_value_evolve() { + let cfg = cfg_per_sample(); + let mut t = SubspaceTracker::new(8, &cfg, 0.999); + + let rows = centred_rows(&[0xABCD_EF01_2345_6789_ABCD_EF01_2345_6789; 4], 8); + let slices = as_slices(&rows); + + for _ in 0..20 { + t.observe(&slices, 8, false); + } + + let report = t.observe(&slices, 8, false); + + // After learning, the energy ratio should be positive and ≤ 1. + assert!(report.energy_ratio > 0.0, "energy_ratio should be positive"); + assert!(report.energy_ratio <= 1.0, "energy_ratio should be at most 1.0"); + + // Top singular value should be positive after observations. + assert!( + report.top_singular_value > 0.0, + "top_singular_value should be positive after training", + ); +} + +/// The energy share and leading singular value a report carries belong to the model that scored the batch, not to the model the batch left behind. Both are read off the same sigmas, and the batch replaces those sigmas before the report is assembled, so a figure read afterwards would describe a model that has not scored anything yet — and the energy share read afterwards is not even that, but the evolved sigmas divided by the rank that scored, a pairing no model ever held. The rank and the geometry beside them already describe the scoring model, so a host reading one report would be comparing an energy share against a rank drawn from a different moment. +/// +/// ´claim:subspace:the-reported-energy-share-and-leading-value-belong-to-the-model-that-scored-the-batch´ +/// ´test:crate:report-energy-figures-describe-the-model-that-scored-the-batch´ +#[test] +fn report_energy_figures_describe_the_model_that_scored_the_batch() { + let cfg = SentinelConfig { + max_rank: 4, + // No adaptation step falls inside this test, so the rank is constant + // throughout and every difference measured below is the subspace + // evolution's alone. + rank_update_interval: 1000, + noise_schedule: NoiseSchedule::Explicit(Vec::new()), + ..cfg_no_per_sample() + }; + let mut tracker = SubspaceTracker::new(8, &cfg, 0.999); + + // Teach one direction, so the single claimed axis holds nearly all the + // energy the model has and a batch off that axis has somewhere visible to + // move the share to. + let familiar = centred_rows(&[0x0000_0000_0000_0000_0000_0000_0000_0000; 4], 8); + for _ in 0..20 { + tracker.observe(&as_slices(&familiar), 8, false); + } + + // The model as it stands is the model the next batch will be scored + // against, so these are the figures that batch's report must carry. + let scoring_energy_ratio = tracker.energy_ratio(); + let scoring_top_singular_value = tracker.top_singular_value(); + + // The leading eight bits alternate where the familiar pattern was + // constant, which is orthogonal to it, so this batch puts energy into + // directions outside the claimed rank. + let novel = centred_rows(&[0xAAAA_AAAA_AAAA_AAAA_AAAA_AAAA_AAAA_AAAA; 4], 8); + let report = tracker.observe(&as_slices(&novel), 8, false); + + assert_eq!( + report.energy_ratio.to_bits(), + scoring_energy_ratio.to_bits(), + "the reported energy share must be the share the scoring model held" + ); + assert_eq!( + report.top_singular_value.to_bits(), + scoring_top_singular_value.to_bits(), + "the reported leading value must be the value the scoring model held" + ); + + // Both equalities would hold for the wrong reason against a model the + // batch had left untouched, so the evolution has to be shown to have + // happened at all. + assert!( + tracker.energy_ratio() < scoring_energy_ratio, + "the batch must move energy outside the claimed rank for the reported share to be worth checking" + ); + assert_ne!( + tracker.top_singular_value().to_bits(), + scoring_top_singular_value.to_bits(), + "the batch must move the leading value for the reported one to be worth checking" + ); + assert_eq!( + report.rank, + tracker.rank(), + "no adaptation step falls here, so the scoring rank is still the current one" + ); +} + +// ════════════════════════════════════════════════════════════ +// CUSUM & clip-pressure lifecycle +// ════════════════════════════════════════════════════════════ + +/// Changing the denominator stability constant does not change a novelty CUSUM trajectory. The allowance belongs to the slow baseline variance alone, so two otherwise identical trackers accumulate the same drift even when their configured stability constants differ by the scale of that variance. +/// +/// ´claim:subspace:the-cusum-allowance-does-not-depend-on-the-denominator-stability-constant´ +/// ´test:crate:cusum-allowance-is-invariant-to-eps´ +#[test] +fn cusum_allowance_is_invariant_to_eps() { + let config = |eps| SentinelConfig:: { + max_rank: 1, + rank_update_interval: u64::MAX, + eps, + per_sample_scores: false, + clip_sigmas: 1.0e6, + ..SentinelConfig::default() + }; + let mut default_eps = SubspaceTracker::new(2, &config(1.0e-6), 0.999); + let mut variance_scale_eps = SubspaceTracker::new(2, &config(1.0), 0.999); + let ordinary = [vec![0.5, 0.0], vec![-0.5, 0.0]]; + let elevated = [vec![0.0, 1.0], vec![0.0, -1.0]]; + let batches = [&ordinary[..], &elevated[..], &elevated[..]]; + let mut default_trajectory = Vec::new(); + let mut variance_scale_trajectory = Vec::new(); + + for batch in batches { + let slices = as_slices(batch); + default_trajectory.push( + default_eps + .observe(&slices, 0, false) + .scores + .novelty + .cusum + .accumulator + .to_bits(), + ); + variance_scale_trajectory.push( + variance_scale_eps + .observe(&slices, 0, false) + .scores + .novelty + .cusum + .accumulator + .to_bits(), + ); + } + + assert_eq!( + default_trajectory, variance_scale_trajectory, + "epsilon may stabilise denominators but must not widen the CUSUM allowance", + ); +} + +/// Clearing a cell's drift evidence restarts the count of batches that +/// evidence was gathered over, so the very next batch is the first step of a +/// new run rather than the next of an old one. Accumulated drift is only +/// interpretable against how long it took to accumulate, and a fresh sum read +/// against a stale count would look like a sudden collapse in drift rather +/// than a deliberate acknowledgement of it. +/// +/// ´claim:subspace:clearing-the-drift-evidence-restarts-the-step-count-so-the-next-batch-is-the-first-of-a-new-run´ +/// ´test:crate:cusum-reset-zeroes-steps´ +#[test] +fn cusum_reset_zeroes_steps() { + let cfg = cfg_per_sample(); + let mut t = SubspaceTracker::new(8, &cfg, 0.999); + + let rows = centred_rows(&[1, 2, 3, 4], 8); + let slices = as_slices(&rows); + + for _ in 0..5 { + t.observe(&slices, 8, false); + } + + t.reset_cusum(); + + let report = t.observe(&slices, 8, false); + assert_eq!(report.scores.novelty.cusum.steps_since_reset, 1); +} + +/// Finishing warm-up is a two-step handover applied to every axis at once: the +/// long-memory reference is seeded from the short-memory one that has already +/// converged on the injected traffic, and only then is the evidence cleared. +/// Done in that order, drift detection resumes from a state where the two +/// references agree, so the first real batches are scored against a reference +/// that is already current instead of registering the warm-up's own leftover +/// gap as drift for as long as the slow memory takes to catch up. +/// +/// ´claim:subspace:the-warm-up-handover-seeds-the-slow-reference-from-the-converged-fast-one-before-the-evidence-is-cleared´ +/// ´test:crate:seed-cusum-slow-from-baselines-then-reset´ +#[test] +fn seed_cusum_slow_from_baselines_then_reset() { + let cfg = cfg_per_sample(); + let mut t = SubspaceTracker::new(8, &cfg, 0.999); + + let rows = centred_rows(&[0xFF00_FF00_FF00_FF00_FF00_FF00_FF00_FF00; 8], 8); + let slices = as_slices(&rows); + + // Build up baselines with noise. + for _ in 0..20 { + t.observe(&slices, 8, true); + } + + // Seed slow from fast, then reset — the canonical warm-up + // completion sequence. + t.seed_cusum_slow_from_baselines(); + t.reset_cusum(); + + // After reset + one real step, CUSUM steps = 1 and accumulator + // should be close to zero (fast ≈ slow, so no drift). + let report = t.observe(&slices, 8, false); + assert_eq!(report.scores.novelty.cusum.steps_since_reset, 1); +} + +/// Clip pressure records how often a cell has lately been discarding scores as +/// outliers, and it widens that cell's own outlier band while it is high. +/// Warm-up is exactly when it runs high, since injected traffic is scored +/// against a barely-formed model. Clearing it zeroes every axis together, so a +/// cell entering production judges its first real batches by the ordinary band +/// rather than by one still slackened by the noise it was taught with. +/// +/// ´claim:subspace:clearing-clip-pressure-zeroes-every-axis-so-warm-up-clipping-does-not-slacken-production-scoring´ +/// ´test:crate:explicit-reset-clip-pressure´ +#[test] +fn explicit_reset_clip_pressure() { + let cfg = SentinelConfig:: { + max_rank: 4, + forgetting_factor: 0.95, + rank_update_interval: 10, + energy_threshold: 0.90, + clip_pressure_decay: 0.95, + ..SentinelConfig::default() + }; + let mut t = SubspaceTracker::new(8, &cfg, 0.999); + + // Inject noise to potentially build clip-pressure. + let rows = centred_rows(&[0xFF00_FF00_FF00_FF00_FF00_FF00_FF00_FF00; 8], 8); + let slices = as_slices(&rows); + for _ in 0..20 { + t.observe(&slices, 8, true); + } + + assert!( + t.maturity().noise_influence > 0.5, + "η should be high after noise injection, got {}", + t.maturity().noise_influence, + ); + + // Explicit reset. + t.seed_cusum_slow_from_baselines(); + t.reset_cusum(); + t.reset_clip_pressure(); + + let cp = t.clip_pressures(); + for (i, &v) in cp.iter().enumerate() { + assert!( + v.abs() < f64::EPSILON, + "axis {i} clip_pressure should be 0 after reset_clip_pressure(), got {v}", + ); + } +} + +/// The same clearing also happens of its own accord, at the moment the noise +/// influence falls through the threshold that marks warm-up complete — caught +/// here by stepping real batches in one at a time and looking at the exact +/// crossing. A cell whose host never makes the explicit call still leaves its +/// warm-up behind, because the condition that matters is that the influence +/// has decayed, not that anyone remembered to say so. +/// +/// (´claim:subspace:clearing-clip-pressure-zeroes-every-axis-so-warm-up-clipping-does-not-slacken-production-scoring´) +/// ´test:crate:eta-threshold-crossing-zeros-clip-pressure´ +#[test] +fn eta_threshold_crossing_zeros_clip_pressure() { + // When η crosses below MATURITY_THRESHOLD (0.01), clip-pressure + // is automatically zeroed even without an explicit reset call. + let cfg = SentinelConfig:: { + max_rank: 4, + forgetting_factor: 0.95, + rank_update_interval: 10, + energy_threshold: 0.90, + clip_pressure_decay: 0.95, + ..SentinelConfig::default() + }; + let mut t = SubspaceTracker::new(8, &cfg, 0.999); + + // Drive η high via noise. + let rows = centred_rows(&[0xABCD_EF01_2345_6789_ABCD_EF01_2345_6789; 4], 8); + let slices = as_slices(&rows); + for _ in 0..30 { + t.observe(&slices, 8, true); + } + assert!(t.maturity().noise_influence > 0.5); + + // Feed real traffic step-by-step, watching for the η threshold crossing. + let mut crossed = false; + for _ in 0..100 { + let old_eta = t.maturity().noise_influence; + t.observe(&slices, 8, false); + let new_eta = t.maturity().noise_influence; + + if old_eta >= MATURITY_THRESHOLD && new_eta < MATURITY_THRESHOLD { + let cp = t.clip_pressures(); + for (i, &v) in cp.iter().enumerate() { + assert!( + v.abs() < f64::EPSILON, + "axis {i} clip_pressure should be 0 at η threshold crossing, got {v}", + ); + } + crossed = true; + break; + } + } + assert!(crossed, "η never crossed the 0.01 threshold"); +} + +/// A step the incremental strategy declines is answered by the dense one, so +/// what comes back is always a basis that was actually re-orthogonalised. The +/// incremental path builds a small kernel and back-transforms through it, +/// which needs spare dimensions to be stable, and at the coordination tier's +/// width it declines every step on exactly those grounds. Declining is the +/// honest answer, and the dispatcher's response to it is to run the strategy +/// that does not need the step. That is the same response the incremental path +/// now gives when its own corrective factorisation fails — the step that +/// re-orthogonalises the basis — because the alternative is returning the +/// basis from before that step under a field documented orthonormal, which +/// every caller writes straight into a tracker and then relies on. The failure +/// of that factorisation cannot be provoked from outside without a hook into +/// the linear algebra, so what is exercised here is the fallback it now takes. +/// +/// ´claim:subspace:a-step-the-incremental-strategy-declines-is-answered-by-the-dense-one´ +/// ´test:crate:a-declined-incremental-step-still-yields-a-usable-model´ +#[test] +fn a_declined_incremental_step_still_yields_a_usable_model() { + let cfg = SentinelConfig:: { + svd_strategy: crate::SvdStrategy::Brand, + ..cfg_per_sample() + }; + + // The coordination tier's width, where the incremental kernel has no + // spare dimensions to work in and the strategy declines every step. + let mut tracker = SubspaceTracker::new(4, &cfg, 0.999); + + for round in 0..40 { + let a = f64::from(round % 7) / 10.0 - 0.3; + let b = f64::from(round % 5) / 10.0 - 0.2; + let rows: Vec> = vec![vec![a, b, -a, -b], vec![b, -a, a, -b]]; + let slices: Vec<&[f64]> = rows.iter().map(Vec::as_slice).collect(); + let report = tracker.observe(&slices, 0, false); + + assert!( + report.scores.novelty.mean.is_finite(), + "a declined step must still leave a model that can score" + ); + assert!(report.rank >= 1, "the model keeps at least one direction"); + assert!( + report.energy_ratio >= 0.0 && report.energy_ratio <= 1.0, + "the captured fraction stays a fraction, which an un-orthogonalised basis would not give" + ); + } +} diff --git a/packages/sentinel/src/tests/variance_formula.rs b/packages/sentinel/src/tests/variance_formula.rs new file mode 100644 index 000000000..192dd4cb5 --- /dev/null +++ b/packages/sentinel/src/tests/variance_formula.rs @@ -0,0 +1,426 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +//! Tests for the running spread the tracker keeps on each of its latent axes — +//! the figure a batch's deviation is divided by to become a surprise score. +//! +//! The spread is maintained as a decaying average, and the decision that shapes +//! it is what each batch's contribution is measured from. Measuring a batch's +//! scatter about its own mean makes the contribution depend on how many rows +//! happened to arrive: a single-row batch has no scatter about itself at all, +//! so the spread collapses toward nothing and every subsequent deviation +//! divided by it reads as enormous, while a two-row batch understates the +//! spread by half. Measuring instead from the running mean carried in from +//! before the batch removes that dependence — each row contributes its own +//! squared departure from an established centre — so a surprise of about one +//! means "as expected" whatever the batch size, and cells configured +//! differently produce comparable scores. +//! +//! Two further decisions guard the degenerate ends. The spread is seeded +//! outright from the first batch rather than blended against its placeholder, +//! because a cell that starts by blending spends a long stretch scoring against +//! a number it was born with rather than one it observed. And it is held above +//! a floor, so a perfectly constant stream — where the running mean converges +//! onto the value itself and genuinely leaves no spread — cannot drive the +//! divisor to zero and make the first different observation infinitely +//! surprising. +//! +//! # §-references +//! +//! - §ALGO S-4.2 Phase 3 — Evolve Latent Distribution +//! - §ALGO S-5.4 — Surprise scoring +//! - §ALGO S-11.2 — Cold-start latent seeding +//! - ADR-S-021 — EWMA-mean-centred latent variance +//! +//! # Test index +//! +//! | Test | Area | Claim | +//! |------|------|-------| +//! | [`cold_start_seeds_variance_from_first_batch`] | variance | The very first batch seeds the spread outright instead of being blended into the placeholder a fresh model was built with. One batch of centred noise is enough to leave every axis's spread near the value the geometry predicts for such traffic and well below the placeholder, so a cell begins scoring against something it observed rather than against a constant it was born holding and would take many batches to shake off. | +//! | [`b1_surprise_bounded`] | variance | A cell fed one observation at a time keeps producing modest surprise scores over a long run of batches. Because each batch's contribution to the spread is measured from the mean carried in from before it, a single row still contributes a real squared departure — where measuring scatter within the batch would find none at all, drive the divisor to its floor, and turn ordinary traffic into scores several orders of magnitude too large. | +//! | [`b1_latent_variance_stable`] | variance | cites (´claim:variance:centring-on-the-running-mean-keeps-a-single-row-batch-from-collapsing-the-spread´) | +//! | [`b2_no_systematic_surprise_inflation`] | variance | Surprise carries no systematic bias from the batch size a cell is configured with: on ordinary traffic, once the model has settled, the average score sits around one. Measuring scatter within a small batch would understate the spread by a predictable fraction and inflate every score by its reciprocal, so a cell reading batches two at a time would look permanently twice as surprised as an identical cell reading them in larger groups. A score of about one has to mean "as expected" everywhere, or no threshold can be set once and applied across cells. | +//! | [`batch_size_invariant_surprise_ratio`] | variance | cites (´claim:variance:the-surprise-ratio-carries-no-batch-size-bias-so-a-score-of-about-one-means-as-expected-everywhere´) | +//! | [`runtime_floor_prevents_degenerate_collapse`] | variance | A stream in which every observation is identical genuinely has no spread, and centring on the running mean does not rescue it — the mean converges onto the repeated value and each batch's contribution goes to zero with it. A floor holds the divisor above a small positive value regardless, so the cell keeps scoring on a bounded scale. Without it, the first observation that differed at all would be divided by nothing and reported as unboundedly surprising, which says more about the arithmetic than about the traffic. | +//! | [`variance_adapts_to_distribution_shift`] | variance | The spread follows the traffic. When a settled cell's input jumps to a substantially larger scale, the recorded spread on every axis climbs well past where it sat before, because it is a decaying average of what is arriving rather than a fixed property learned once. A shift in scale is therefore absorbed within a bounded stretch of batches instead of being reported as anomalous indefinitely — surprise is meant to answer "unusual for this cell lately", not "unusual for this cell when it was young". | +//! | [`update_order_variance_before_mean`] | variance | Within a single batch the spread is measured before the mean moves, against the centre that batch arrived to find. The ordering is visible because the mean demonstrably shifts across the batch while the resulting spread stays in the range the earlier centre implies. Were the order reversed, a batch would be measured against a centre it had just pulled toward itself and would partly explain its own deviation away — the same reason scoring happens before the model absorbs the batch at all. | + +use rand::SeedableRng; +use rand::rngs::SmallRng; + +use super::convergence_common::{as_slices, cfg_test, generate_noise}; +use crate::config::SentinelConfig; +use crate::sentinel::tracker::SubspaceTracker; + +// ════════════════════════════════════════════════════════════ +// Helpers +// ════════════════════════════════════════════════════════════ + +/// Build a config with a specific `noise_batch_size`. +fn cfg_with_batch_size(b: usize) -> SentinelConfig { + SentinelConfig { + noise_batch_size: b, + ..cfg_test() + } +} + +// ════════════════════════════════════════════════════════════ +// Cold-start seeding +// ════════════════════════════════════════════════════════════ + +/// The very first batch seeds the spread outright instead of being blended +/// into the placeholder a fresh model was built with. One batch of centred +/// noise is enough to leave every axis's spread near the value the geometry +/// predicts for such traffic and well below the placeholder, so a cell begins +/// scoring against something it observed rather than against a constant it was +/// born holding and would take many batches to shake off. +/// +/// ´claim:variance:the-first-batch-seeds-the-spread-outright-rather-than-being-blended-into-a-placeholder´ +/// ´test:crate:cold-start-seeds-variance-from-first-batch´ +#[test] +fn cold_start_seeds_variance_from_first_batch() { + let cfg = cfg_with_batch_size(16); + let dim = 128; + let mut tracker = SubspaceTracker::new(dim, &cfg, cfg.cusum_slow_decay); + let mut rng = SmallRng::seed_from_u64(42); + + let noise = generate_noise(dim, cfg.noise_batch_size, &mut rng); + tracker.observe(&as_slices(&noise), 0, true); + + let lat_vars = tracker.latent_var(); + for (j, &v) in lat_vars.iter().enumerate() { + // Should be near 0.25; upper bound excludes the 1.0 placeholder. + assert!( + (1e-2..0.8).contains(&v), + "cold-start lat_var[{j}] = {v:.6} should be near 0.25, not the 1.0 placeholder" + ); + } +} + +// ════════════════════════════════════════════════════════════ +// b = 1 tests +// ════════════════════════════════════════════════════════════ + +/// A cell fed one observation at a time keeps producing modest surprise scores +/// over a long run of batches. Because each batch's contribution to the spread +/// is measured from the mean carried in from before it, a single row still +/// contributes a real squared departure — where measuring scatter within the +/// batch would find none at all, drive the divisor to its floor, and turn +/// ordinary traffic into scores several orders of magnitude too large. +/// +/// ´claim:variance:centring-on-the-running-mean-keeps-a-single-row-batch-from-collapsing-the-spread´ +/// ´test:crate:b1-surprise-bounded´ +#[test] +fn b1_surprise_bounded() { + let cfg = cfg_with_batch_size(1); + let dim = 128; + let mut tracker = SubspaceTracker::new(dim, &cfg, cfg.cusum_slow_decay); + let mut rng = SmallRng::seed_from_u64(42); + let mut max_surprise = 0.0_f64; + + for _ in 0..200 { + let noise = generate_noise(dim, 1, &mut rng); + let report = tracker.observe(&as_slices(&noise), 0, true); + max_surprise = max_surprise.max(report.scores.surprise.mean); + } + + assert!( + max_surprise < 50.0, + "b=1 surprise should stay bounded; got max {max_surprise:.1} (old formula would exceed 10⁴)" + ); +} + +/// The same statement read on the quantity itself rather than on the score it +/// divides: after a long run of single-row batches, every axis's spread has +/// settled around the value the geometry of centred bit traffic predicts, not +/// against the floor. Each batch's contribution is an unbiased estimate of the +/// true spread even when the batch is one row, so the decaying average +/// converges on the right number instead of merely staying finite. +/// +/// (´claim:variance:centring-on-the-running-mean-keeps-a-single-row-batch-from-collapsing-the-spread´) +/// ´test:crate:b1-latent-variance-stable´ +#[test] +fn b1_latent_variance_stable() { + let cfg = cfg_with_batch_size(1); + let dim = 128; + let mut tracker = SubspaceTracker::new(dim, &cfg, cfg.cusum_slow_decay); + let mut rng = SmallRng::seed_from_u64(42); + + for _ in 0..500 { + let noise = generate_noise(dim, 1, &mut rng); + tracker.observe(&as_slices(&noise), 0, true); + } + + let lat_vars = tracker.latent_var(); + for (j, &v) in lat_vars.iter().enumerate() { + assert!( + (0.05..=0.5).contains(&v), + "b=1: lat_var[{j}] = {v:.6} should be in [0.05, 0.5] (≈0.25 expected)" + ); + } +} + +// ════════════════════════════════════════════════════════════ +// b = 2 test +// ════════════════════════════════════════════════════════════ + +/// Surprise carries no systematic bias from the batch size a cell is +/// configured with: on ordinary traffic, once the model has settled, the +/// average score sits around one. Measuring scatter within a small batch would +/// understate the spread by a predictable fraction and inflate every score by +/// its reciprocal, so a cell reading batches two at a time would look +/// permanently twice as surprised as an identical cell reading them in larger +/// groups. A score of about one has to mean "as expected" everywhere, or no +/// threshold can be set once and applied across cells. +/// +/// ´claim:variance:the-surprise-ratio-carries-no-batch-size-bias-so-a-score-of-about-one-means-as-expected-everywhere´ +/// ´test:crate:b2-no-systematic-surprise-inflation´ +#[test] +fn b2_no_systematic_surprise_inflation() { + let cfg = cfg_with_batch_size(2); + let dim = 128; + let mut tracker = SubspaceTracker::new(dim, &cfg, cfg.cusum_slow_decay); + let mut rng = SmallRng::seed_from_u64(42); + + // Warm up. + for _ in 0..200 { + let noise = generate_noise(dim, 2, &mut rng); + tracker.observe(&as_slices(&noise), 0, true); + } + + // Measure: collect surprise means over 200 more rounds. + let mut surprise_sum = 0.0; + let measurement_rounds: u32 = 200; + for _ in 0..measurement_rounds { + let noise = generate_noise(dim, 2, &mut rng); + let report = tracker.observe(&as_slices(&noise), 0, true); + surprise_sum += report.scores.surprise.mean; + } + let mean_surprise = surprise_sum / f64::from(measurement_rounds); + + assert!( + (0.5..=2.0).contains(&mean_surprise), + "b=2: mean surprise = {mean_surprise:.3} should be near 1.0 (old formula would give ≈2.0)" + ); +} + +// ════════════════════════════════════════════════════════════ +// Batch-size invariance +// ════════════════════════════════════════════════════════════ + +/// The general form of the same statement, swept over batch sizes spanning +/// two orders of magnitude from a single row to many. Two things are pinned +/// here that a single configuration cannot pin: each cell's average score sits +/// in a band around one, and the scores agree closely with each other across +/// the sweep. Self-consistency is the stronger half — a shared bias would move +/// every band together and pass the first check, but would show up at once as +/// a spread between them. +/// +/// (´claim:variance:the-surprise-ratio-carries-no-batch-size-bias-so-a-score-of-about-one-means-as-expected-everywhere´) +/// ´test:crate:batch-size-invariant-surprise-ratio´ +#[test] +fn batch_size_invariant_surprise_ratio() { + let batch_sizes = [1, 2, 4, 16, 64]; + // Surprise-ratio invariance is per-axis; dim=32 validates the + // property at ~4× less SVD cost than dim=128 (ADR-S-012). + let dim = 32; + let warmup_rounds = 100; + let measurement_rounds: u32 = 100; + + let mut ratios = Vec::with_capacity(batch_sizes.len()); + + for &b in &batch_sizes { + let cfg = cfg_with_batch_size(b); + let mut tracker = SubspaceTracker::new(dim, &cfg, cfg.cusum_slow_decay); + let mut rng = SmallRng::seed_from_u64(42); + + // Warm up. + for _ in 0..warmup_rounds { + let noise = generate_noise(dim, b, &mut rng); + tracker.observe(&as_slices(&noise), 0, true); + } + + // Measure. + let mut surprise_sum = 0.0; + for _ in 0..measurement_rounds { + let noise = generate_noise(dim, b, &mut rng); + let report = tracker.observe(&as_slices(&noise), 0, true); + surprise_sum += report.scores.surprise.mean; + } + let mean_surprise = surprise_sum / f64::from(measurement_rounds); + ratios.push((b, mean_surprise)); + } + + // Each ratio should be in a reasonable band around 1.0. + for &(b, ratio) in &ratios { + assert!( + (0.7..=1.5).contains(&ratio), + "b={b}: surprise ratio = {ratio:.3} should be in [0.7, 1.5]" + ); + } + + // Self-consistency: max - min < 0.5. + let max_r = ratios.iter().map(|(_, r)| *r).fold(f64::NEG_INFINITY, f64::max); + let min_r = ratios.iter().map(|(_, r)| *r).fold(f64::INFINITY, f64::min); + let spread = max_r - min_r; + + assert!( + spread < 0.5, + "surprise ratios should be self-consistent across batch sizes: \ + spread = {spread:.3} (ratios: {ratios:?})" + ); +} + +// ════════════════════════════════════════════════════════════ +// Runtime floor +// ════════════════════════════════════════════════════════════ + +/// A stream in which every observation is identical genuinely has no spread, +/// and centring on the running mean does not rescue it — the mean converges +/// onto the repeated value and each batch's contribution goes to zero with it. +/// A floor holds the divisor above a small positive value regardless, so the +/// cell keeps scoring on a bounded scale. Without it, the first observation +/// that differed at all would be divided by nothing and reported as +/// unboundedly surprising, which says more about the arithmetic than about the +/// traffic. +/// +/// ´claim:variance:a-floor-holds-the-spread-off-zero-so-a-perfectly-constant-stream-cannot-make-the-next-deviation-unbounded´ +/// ´test:crate:runtime-floor-prevents-degenerate-collapse´ +#[test] +fn runtime_floor_prevents_degenerate_collapse() { + let cfg = cfg_test(); // b = 4 + let dim = 128; + let mut tracker = SubspaceTracker::new(dim, &cfg, cfg.cusum_slow_decay); + let mut rng = SmallRng::seed_from_u64(42); + + // One batch of noise to initialise the basis. + let noise = generate_noise(dim, cfg.noise_batch_size, &mut rng); + tracker.observe(&as_slices(&noise), 0, true); + + // 200 batches of identical observations (all −0.5). + let constant_row: Vec = vec![-0.5; dim]; + let constant_batch: Vec> = (0..cfg.noise_batch_size).map(|_| constant_row.clone()).collect(); + + for _ in 0..200 { + tracker.observe(&as_slices(&constant_batch), 0, true); + } + + let lat_vars = tracker.latent_var(); + for (j, &v) in lat_vars.iter().enumerate() { + assert!(v >= 1e-2, "lat_var[{j}] = {v:.6e} should be ≥ 1e-2 (runtime floor)"); + } +} + +// ════════════════════════════════════════════════════════════ +// Distribution-shift adaptivity +// ════════════════════════════════════════════════════════════ + +/// The spread follows the traffic. When a settled cell's input jumps to a +/// substantially larger scale, the recorded spread on every axis climbs well +/// past where it sat before, because it is a decaying average of what is +/// arriving rather than a fixed property learned once. A shift in scale is +/// therefore absorbed within a bounded stretch of batches instead of being +/// reported as anomalous indefinitely — surprise is meant to answer "unusual +/// for this cell lately", not "unusual for this cell when it was young". +/// +/// ´claim:variance:the-spread-follows-the-traffic-so-a-shift-in-scale-is-absorbed-rather-than-reported-forever´ +/// ´test:crate:variance-adapts-to-distribution-shift´ +#[test] +fn variance_adapts_to_distribution_shift() { + let cfg = cfg_with_batch_size(4); + let dim = 128; + let mut tracker = SubspaceTracker::new(dim, &cfg, cfg.cusum_slow_decay); + let mut rng = SmallRng::seed_from_u64(42); + + // Warm up on standard ±0.5 noise. + for _ in 0..200 { + let noise = generate_noise(dim, cfg.noise_batch_size, &mut rng); + tracker.observe(&as_slices(&noise), 0, true); + } + + let pre_shift_var: Vec = tracker.latent_var().to_vec(); + + // Switch to 4× scaled noise (±2.0). + for _ in 0..100 { + let noise: Vec> = generate_noise(dim, cfg.noise_batch_size, &mut rng) + .into_iter() + .map(|row| row.into_iter().map(|x| x * 4.0).collect()) + .collect(); + tracker.observe(&as_slices(&noise), 0, true); + } + + let post_shift_var: Vec = tracker.latent_var().to_vec(); + + // lat_var should have increased substantially (16× theoretical). + for (j, (&pre, &post)) in pre_shift_var.iter().zip(post_shift_var.iter()).enumerate() { + assert!( + post > pre * 2.0, + "lat_var[{j}]: post-shift {post:.4} should be > 2× pre-shift {pre:.4}" + ); + } +} + +// ════════════════════════════════════════════════════════════ +// Update order (white-box) +// ════════════════════════════════════════════════════════════ + +/// Within a single batch the spread is measured before the mean moves, against +/// the centre that batch arrived to find. The ordering is visible because the +/// mean demonstrably shifts across the batch while the resulting spread stays +/// in the range the earlier centre implies. Were the order reversed, a batch +/// would be measured against a centre it had just pulled toward itself and +/// would partly explain its own deviation away — the same reason scoring +/// happens before the model absorbs the batch at all. +/// +/// ´claim:variance:the-spread-is-measured-against-the-centre-that-preceded-the-batch-so-a-batch-cannot-explain-itself-away´ +/// ´test:crate:update-order-variance-before-mean´ +#[test] +fn update_order_variance_before_mean() { + let cfg = SentinelConfig { + forgetting_factor: 0.9, + noise_batch_size: 4, + max_rank: 2, + rank_update_interval: 5, + ..cfg_test() + }; + let dim = 128; + let mut tracker = SubspaceTracker::new(dim, &cfg, cfg.cusum_slow_decay); + let mut rng = SmallRng::seed_from_u64(99); + + // Batch 1: cold→warm seeding. + let noise1 = generate_noise(dim, cfg.noise_batch_size, &mut rng); + tracker.observe(&as_slices(&noise1), 0, true); + + // Record pre-batch-2 `lat_mean` (this is what variance should + // have been computed against). + let pre_mean: Vec = tracker.latent_mean().to_vec(); + + // Batch 2: non-cold path. + let noise2 = generate_noise(dim, cfg.noise_batch_size, &mut rng); + tracker.observe(&as_slices(&noise2), 0, true); + + let post_mean: Vec = tracker.latent_mean().to_vec(); + let post_var: Vec = tracker.latent_var().to_vec(); + + // The key invariant: with variance-before-mean order, the variance + // was computed against `pre_mean`, then the mean was updated. + // If the order were reversed, the variance would be computed against + // `post_mean` (which would be slightly different). + // + // Verify indirectly: `pre_mean` ≠ `post_mean` (the mean moved), + // and `lat_var` is within a reasonable range (not stuck at ε or 1.0). + let mean_shifted = pre_mean.iter().zip(post_mean.iter()).any(|(&a, &b)| (a - b).abs() > 1e-8); + + assert!(mean_shifted, "lat_mean should have changed between batches 1 and 2"); + + for (j, &v) in post_var.iter().enumerate() { + // After batch 2, `lat_var` should be λ·seed_var + α·col_var + // where `col_var` was centred on `pre_mean` (≈ batch-1 mean). + // It should be in a reasonable range — not ε (which the old + // formula would give at b=1) and not wildly inflated. + assert!( + (1e-2..2.0).contains(&v), + "lat_var[{j}] = {v:.6} should be reasonable after 2 batches" + ); + } +} diff --git a/packages/sentinel/src/tests/warming_thread.rs b/packages/sentinel/src/tests/warming_thread.rs new file mode 100644 index 000000000..6910a3bb5 --- /dev/null +++ b/packages/sentinel/src/tests/warming_thread.rs @@ -0,0 +1,265 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +//! Tests for the background warming thread's shutdown handshake. +//! +//! The worker sleeps on a condition variable whose predicate is two facts — +//! whether the staging area holds warming work, and whether shutdown has been +//! asked for — and it holds the staging mutex from the moment it reads them +//! until the wait releases it. A writer that changes either fact outside that +//! mutex can place the change and its wake-up inside that window, where the +//! wake-up reaches a thread that has not yet begun to wait; the worker then +//! sleeps on a predicate that has already changed and nothing changes it +//! again. Shutdown is the transition where that costs a hang rather than a +//! delay, because the joining thread waits for a worker that will never look +//! at the flag again. +//! +//! The test here cannot make the window open on demand: it is a few +//! instructions wide and the scheduler decides. What it can do is take the +//! bet often enough that a lost wake-up shows up as a thread that never +//! finishes, and bound the wait so the failure arrives as a failed assertion +//! rather than as a suite that stops. +//! +//! # Test index +//! +//! | Test | Area | Claim | +//! |------|------|-------| +//! | [`shutdown_returns_under_repeated_spawn_and_stop_cycles`] | warmup | Shutting the warming thread down returns, every time, over a long run of spawn-and-stop cycles that does nothing else — the arrangement that puts the request at its most likely to land while the worker is between reading its predicate and sleeping on it. A shutdown that is lost in that window does not fail loudly: the worker sleeps on, the join waits for it, and the sentinel's own drop never completes, so what a host would see is a process that stops rather than an error it can act on. | +//! | [`dropping_a_sentinel_consumes_a_failed_worker_join`] | warmup | A warming worker can fail before its owner is destroyed. Destruction still completes without unwinding, because the drop path records the failed join instead of turning a background failure into a destructor panic. | +//! | [`reset_consumes_a_failed_worker_join`] | warmup | Reset follows the same host-preserving policy as destruction: a worker that has already failed is joined and recorded, then reset rebuilds the sentinel instead of panicking over a failure that happened in the background. | +//! | [`a_failed_warming_worker_still_brings_every_cell_online`] | warmup | A sentinel whose warming worker has died still brings every cell the selector pays for online: the cell the worker was holding when it went, and every cell staged afterwards. The dispatch keys on whether a worker is present, so a dead worker left standing would send every later reconciliation down the background branch — past the synchronous drain, into a notification nobody receives — and the engine would go on issuing reports from a cell set that had stopped growing. | +//! | [`selection_refresh_survives_warming_handoffs`] | warmup | Waiting, in-flight and ready cells keep the latest selection flag across both worker return paths. | + +use std::sync::mpsc::{self, RecvTimeoutError}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use crate::config::NoiseSchedule; +use crate::sentinel::staging::StagingArea; +use crate::sentinel::warming_thread::WarmingThreadHandle; +use crate::{SentinelConfig, SpectralSentinel}; + +/// How many spawn-and-stop cycles the witness runs. +const CYCLES: usize = 1_000; + +/// How long the witness waits for those cycles before calling the handshake +/// broken. Well inside the package's five-second per-test budget, and orders +/// of magnitude above the cycles' own cost. +const DEADLINE: Duration = Duration::from_secs(3); + +// ─── Shutdown handshake ───────────────────────────────────── + +/// Shutting the warming thread down returns, every time, over a long run of +/// spawn-and-stop cycles that does nothing else — the arrangement that puts +/// the request at its most likely to land while the worker is between reading +/// its predicate and sleeping on it. A shutdown that is lost in that window +/// does not fail loudly: the worker sleeps on, the join waits for it, and the +/// sentinel's own drop never completes, so what a host would see is a process +/// that stops rather than an error it can act on. +/// +/// ´claim:warmup:shutting-the-warming-thread-down-returns-however-the-request-races-the-worker-going-to-sleep´ +/// ´test:crate:shutdown-returns-under-repeated-spawn-and-stop-cycles´ +#[test] +fn shutdown_returns_under_repeated_spawn_and_stop_cycles() { + let (done, finished) = mpsc::channel(); + + // The cycles run on their own thread so that a lost wake-up is a + // deadline this thread can observe. Joining them directly would make + // the failure a hang, which no assertion can report. + let cycles = std::thread::spawn(move || { + for _ in 0..CYCLES { + let staging = Arc::new(Mutex::new(StagingArea::::new())); + let handle = WarmingThreadHandle::spawn(&staging, 4, Some(7)).expect("the environment must grant a warming thread"); + handle.shutdown(); + } + // The receiver is gone only if the witness has already reported the + // deadline, so there is nothing for this thread to do about it. + let _reported = done.send(()); + }); + + match finished.recv_timeout(DEADLINE) { + Ok(()) => {} + Err(RecvTimeoutError::Timeout) => { + panic!( + "{CYCLES} spawn-and-stop cycles did not finish within {DEADLINE:?}: a shutdown request was \ + stored and notified while the worker was between reading its predicate and sleeping on it, \ + so the worker never saw it and the join never returned" + ) + } + Err(RecvTimeoutError::Disconnected) => panic!("the cycling thread ended without reporting"), + } + + cycles.join().expect("cycling thread panicked"); +} + +/// A warming worker can fail before its owner is destroyed. Destruction +/// still completes without unwinding, because the drop path records the failed +/// join instead of turning a background failure into a destructor panic. +/// +/// ´claim:warmup:sentinel-destruction-consumes-a-failed-worker-join´ +/// ´test:crate:dropping-a-sentinel-consumes-a-failed-worker-join´ +#[test] +fn dropping_a_sentinel_consumes_a_failed_worker_join() { + let config = SentinelConfig:: { + noise_schedule: NoiseSchedule::Explicit(Vec::new()), + background_warming: true, + ..SentinelConfig::::default() + }; + let sentinel = SpectralSentinel::::new(config).unwrap(); + sentinel.fail_warming_worker_for_test(); + + let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| drop(sentinel))); + + assert!(outcome.is_ok(), "dropping the sentinel must consume the failed worker join"); +} + +/// Reset follows the same host-preserving policy as destruction: a worker +/// that has already failed is joined and recorded, then reset rebuilds the +/// sentinel instead of panicking over a failure that happened in the +/// background. +/// +/// ´claim:warmup:sentinel-reset-consumes-a-failed-worker-join´ +/// ´test:crate:reset-consumes-a-failed-worker-join´ +#[test] +fn reset_consumes_a_failed_worker_join() { + let config = SentinelConfig:: { + noise_schedule: NoiseSchedule::Explicit(Vec::new()), + background_warming: true, + ..SentinelConfig::::default() + }; + let mut sentinel = SpectralSentinel::::new(config).unwrap(); + sentinel.fail_warming_worker_for_test(); + + let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| sentinel.reset())); + + assert!(outcome.is_ok(), "reset must consume the failed worker join"); +} + +/// A sentinel whose warming worker has died still brings every cell the +/// selector pays for online: the cell the worker was holding when it went, and +/// every cell staged afterwards. The dispatch keys on whether a worker is +/// present, so a dead worker left standing would send every later +/// reconciliation down the background branch — past the synchronous drain, into +/// a notification nobody receives — and the engine would go on issuing reports +/// from a cell set that had stopped growing. +/// +/// ´claim:warmup:a-failed-warming-worker-still-brings-every-cell-online´ +/// ´test:crate:a-failed-warming-worker-still-brings-every-cell-online´ +#[test] +fn a_failed_warming_worker_still_brings_every_cell_online() { + let config = SentinelConfig:: { + analysis_k: 1, + split_threshold: 1, + d_create: 1, + d_evict: 2, + max_rank: 1, + noise_batch_size: 1, + noise_schedule: NoiseSchedule::Explicit(vec![0, 2]), + background_warming: false, + noise_seed: Some(7), + ..SentinelConfig::default() + }; + let mut sentinel = SpectralSentinel::::new(config).expect("the test configuration must be valid"); + + // The state the strand needs — a cell below the root that is online, so + // that killing a worker can take it off the producing set — is built here + // rather than waited for. The analysis set is a function of the value + // stream alone, so these eight batches decide which cells the selector + // names on every machine; what differs between the two warming modes is + // only whether those cells are online yet. With no worker the drain runs + // inside the reconciliation, so each call returns with every cell the + // schedule asks for already promoted, where a worker would have brought + // them online whenever it was next scheduled — and on a machine with few + // cores, that can be after this loop. + for step in 0..8_u64 { + sentinel.ingest(&[step * 4_096]); + } + + // The worker exists from here, so the strand below is a real one: the + // failure it induces runs through the worker's own poisoned-staging path, + // and the recovery measured afterwards is the one that follows a worker + // that died holding a cell. + sentinel.start_a_warming_worker_for_test(); + + let stranded = sentinel.strand_a_cell_on_a_failed_warming_worker_for_test(); + assert!( + !sentinel.cell_gnodes().contains(&stranded), + "the stranded cell must start off the producing set, or the witness proves nothing" + ); + + // One ingest is the whole recovery: the reconciliation it runs is where the + // dead worker is reaped, where the checkout record it left is dropped, and + // where the drain that replaces it runs. + sentinel.ingest(&[0]); + + assert!( + sentinel.cell_gnodes().contains(&stranded), + "the cell the worker was holding when it died must be built again and brought online, \ + not left stranded between the producing set and a checkout record nothing will redeem" + ); + + // Cells staged after the worker died have no one to warm them but the + // synchronous drain, which only runs if the dispatch can see that the + // worker is gone. + for step in 8..16_u64 { + sentinel.ingest(&[step * 4_096]); + } + + let producing = sentinel.cell_gnodes(); + let waiting: Vec<_> = sentinel + .analysis_set() + .full() + .iter() + .map(|entry| entry.gnode) + .filter(|gnode| !producing.contains(gnode)) + .collect(); + assert!( + waiting.is_empty(), + "every cell the selector pays for must be online after the worker died, but {waiting:?} are still waiting" + ); + assert_eq!( + sentinel.health().warming_trackers, + 0, + "no cell may be left in the staging area once the synchronous drain has taken over" + ); +} + +/// Waiting, in-flight and ready cells keep the latest selection flag across both worker return paths. +/// +/// ´claim:warmup:selection-refresh-survives-warming-handoffs´ +/// ´test:crate:selection-refresh-survives-warming-handoffs´ +#[test] +fn selection_refresh_survives_warming_handoffs() { + let config = crate::SentinelConfig::::default(); + let cell = crate::sentinel::CellState { + tracker: crate::sentinel::tracker::SubspaceTracker::new(4, &config, config.cusum_slow_decay), + depth: 1, + width: 4, + start: 0_u128, + end: 16, + is_competitive: false, + }; + let gnode = torrust_mudlark::GNodeId::from_parts(1, 0); + let mut staging = StagingArea::new(); + staging.enqueue(gnode, cell, 2); + staging.update_competitive(gnode, true); + assert_eq!(staging.warming_competitive_count(), 1); + + let (_, warming) = staging.take_highest_priority().unwrap(); + staging.update_competitive(gnode, false); + assert_eq!(staging.warming_competitive_count(), 0); + staging.return_warming(gnode, warming); + assert_eq!(staging.warming_competitive_count(), 0); + + let (_, warming) = staging.take_highest_priority().unwrap(); + assert!(!warming.cell.is_competitive); + staging.update_competitive(gnode, true); + assert_eq!(staging.warming_competitive_count(), 1); + staging.finish_warming(gnode, warming.cell); + let (_, ready) = staging.take_ready().pop().unwrap(); + assert!(ready.is_competitive); + + staging.enqueue(gnode, ready, 0); + staging.update_competitive(gnode, false); + assert!(!staging.take_ready().pop().unwrap().1.is_competitive); +} diff --git a/packages/sentinel/tests/README.md b/packages/sentinel/tests/README.md new file mode 100644 index 000000000..5c73118a5 --- /dev/null +++ b/packages/sentinel/tests/README.md @@ -0,0 +1,292 @@ +## Integration test matrix · `tab:sentinel:integration-test-matrix` + +**Table (Integration test matrix)** + +| Test | Area | Claim | +|------|------|-------| +| (`test:integration:empty-batch-produces-no-ancestor-reports`) | ancestry | A batch with no observations produces no ancestor reports, even from a sentinel already warmed and holding a chain of live models. Cells report what they saw, and a cell that saw nothing has nothing to say; the chain is a description of traffic rather than a periodic status broadcast, so a quiet interval costs a host no reports to filter out. | +| (`test:integration:root-ancestor-present-after-warm-up`) | ancestry | Traffic into a refined region is reported at both ends of the chain at once: the deep cells that earned selection appear as competitive, and the root appears alongside them as an ancestor. The same observations were analysed at both scales, which is what makes the two figures comparable in the first place — without the coarse reading there is nothing to judge the fine one against. | +| (`test:integration:root-sample-count-equals-batch-size`) | ancestry | An ancestor is credited with every observation that fell anywhere beneath it: the root's sample count for a batch drawn from two separate ranges is the whole batch. Delivery is by containment rather than by ownership, so nothing is consumed by the deepest cell that matched — this is what lets a coarse model hold a baseline for total volume that no individual cell could. | +| (`test:integration:ancestor-width-increases-toward-root`) | ancestry | Ordered by depth, the ancestors reported for a batch never narrow as one climbs toward the root: an ancestor analyses at least as wide a view as the cells below it. Ancestry and analysis breadth therefore point the same way, so a chain reads as a genuine sequence of scales — coarse above, specific below — rather than an arbitrary collection of models over the same region. | +| (`test:integration:ancestor-depths-cover-path-to-root`) | ancestry | The chain has no holes in it. The root is always reported, and where the selected cells sit well below it the intervening depths are present too — some as competitive cells in their own right, the rest drawn in by the closure. Reading the two report lists together therefore gives an unbroken path from the whole domain down to the finest cell, which is what allows a disturbance to be located at a scale rather than merely noticed at one. | +| (`test:integration:shared-ancestor-aggregates-disjoint-ranges`) | ancestry | cites (`claim:ancestry:an-ancestor-is-credited-with-every-observation-that-fell-beneath-it`) | +| (`test:integration:local-anomaly-detectable-in-hierarchy`) | ancestry | Structurally novel traffic entering one range while the others stay normal leaves a mark somewhere in the chain, measured against the same sentinel's own response to an ordinary batch a moment earlier. Which level catches it is not fixed — a narrow disturbance may barely move the root while standing out sharply in the cell containing it — so detection is a property of the chain as a whole rather than of any one model in it. | +| (`test:integration:global-anomaly-elevates-root-scores`) | ancestry | When every range turns anomalous at once, the root's own score rises above what the same sentinel produced for a normal batch. The coarse model is not merely a fallback for traffic too sparse to have earned a cell: it responds in its own right, and it responds to exactly the case no single cell can distinguish from its own local weather. | +| (`test:integration:global-anomaly-root-z-exceeds-local`) | ancestry | The root does not merely notice anomalies, it grades them by extent: with several ranges live, an anomaly in one of them moves the root less than the same anomaly in all of them, the sentinel having been returned to normal traffic in between so the two readings are of comparable states. Reach is thus legible in the score itself, and a host can separate a local incident from a system-wide shift without waiting to see how far it spreads. | +| (`test:integration:new-validates-config`) | engine | A configuration that could not produce a working model is rejected at construction rather than carried into the run: a rank budget of zero leaves the subspace tracker nothing to hold, and the constructor returns an error instead of a sentinel. Validation happens once, so every later method may assume its parameters are coherent. | +| (`test:integration:new-with-default-config-succeeds`) | engine | The configuration the crate ships as its default passes its own validation, so a host that supplies nothing of its own starts from a usable engine rather than from an error. Only validation is exercised here: the default noise schedule warms the root through many rounds, and paying for that construction adds nothing the rest of the suite does not already pay for. | +| (`test:integration:new-starts-with-root-cell`) | engine | A newly constructed sentinel holds exactly one tracker — the root — and has observed nothing. Cells below the root are created only when traffic justifies them, so construction commits to the single model that is structurally obligatory and to no other investment. | +| (`test:integration:empty-ingest-returns-empty-report`) | edge | A batch with no values in it is answered with a report that names no cells and no cross-cell contexts, and the observation counter does not move. The health section is still filled in — the root tracker is alive whether or not anything arrived — so an idle interval reads as an engine with nothing to say rather than as a gap in the record. | +| (`test:integration:single-value-ingest`) | engine | Every value is delivered to the root as well as to whatever cell it routes into, so even a lone observation produces an ancestor report and advances the lifetime count. The root is the one cell guaranteed to contain any coordinate, which is why a batch can never be scored against nothing. | +| (`test:integration:batch-counter-increments`) | engine | The lifetime counter accumulates real observations rather than batches: after a single value and then a batch of several it stands at their sum. Batch boundaries are a delivery convenience for the host and carry no weight in the record of what was actually seen. | +| (`test:integration:repeated-ingest-does-not-panic`) | engine | Feeding the same traffic round after round leaves every structural invariant standing at each step: the reported cells stay within the competitive budget, the two report vectors stay partitioned and ordered, widths still match depths, and no score turns into a non-number. The engine is a steady-state machine, so repetition is the ordinary case and not a stress case. | +| (`test:integration:config-accessor-returns-construction-config`) | engine | The configuration reads back exactly as it was handed in — rank budget, competitive budget and forgetting factor all unchanged. Construction validates the parameters but does not silently normalise or substitute them, so a host can trust the accessor as the authority on how this sentinel is behaving. | +| (`test:integration:graph-accessor-starts-with-single-root`) | engine | The spatial substrate underneath a fresh sentinel is a bare root with no accumulated importance at all. Construction does not seed the graph with anything, so the first real batch is also the first thing the spatial layer has ever ranked — there is no synthetic history for the selector to mistake for traffic. | +| (`test:integration:analysis-set-accessible`) | engine | The set of cells the sentinel is investing in is readable from outside, and after traffic it holds at least the root. The root's membership is unconditional — it is what every ancestor chain terminates at — so the set is never empty and a host inspecting it never has to handle the no-cells case. | +| (`test:integration:cell-gnodes-returns-all-tracked`) | engine | The list of cell handles and the count of tracked cells are two views of one map, never two records that could disagree. A host can enumerate the handles and know it has enumerated everything the engine is modelling. | +| (`test:integration:cells-tracked-matches-cell-gnodes-len`) | engine | cites (`claim:engine:the-listed-cell-handles-and-the-tracked-count-are-two-views-of-one-map`) | +| (`test:integration:lifetime-observations-reflects-real-input`) | engine | cites (`claim:engine:the-lifetime-counter-accumulates-observations-not-batches`) | +| (`test:integration:degenerate-cells-skipped-starts-at-zero`) | engine | A cell whose suffix is too narrow to support a subspace model is skipped and counted rather than modelled, and on a sentinel that has observed nothing that count is zero. The counter is therefore a record of something that happened, not a constant the engine carries around — a non-zero reading always means real traffic drove the domain that deep. | +| (`test:integration:health-accessible-on-fresh-sentinel`) | engine | A health snapshot can be taken before any observation arrives, and it describes the engine truthfully at that moment: one live tracker, nothing observed. Health is a readout of present state rather than a summary accumulated during ingestion, so a host may poll it on a schedule of its own without having to feed the engine first. | +| (`test:integration:inspect-cell-returns-state-for-root`) | engine | A tracked cell can be inspected individually, and what comes back describes that cell: its depth in the routing tree, the width its tracker analyses, and the rank of the subspace it has learned. The root reports depth zero and the full domain width because nothing has been resolved above it, and its rank is at least one from the moment it exists, since a tracker with no direction at all could produce no residual. | +| (`test:integration:inspect-cell-returns-none-for-unknown-gnode`) | engine | A cell handle is meaningful only to the sentinel that issued it. Handed a handle minted by a different sentinel, inspection returns nothing rather than the state of whichever local cell happens to sit at that index — the lookup is a membership question, so a stale or foreign handle is an absence and never a plausible-looking wrong answer. | +| (`test:integration:per-sample-scores-present-when-enabled`) | engine | Scores for individual values are attached to a cell's report only when the host asked for them, and then there is exactly one entry per observation in the batch. Per-sample detail costs memory proportional to the traffic, so it is opt-in rather than always paid for, and the one-to-one correspondence is what makes an entry attributable back to the value that produced it. | +| (`test:integration:per-sample-scores-absent-when-disabled`) | engine | cites (`claim:engine:per-sample-scores-appear-only-when-asked-for-and-then-carry-one-entry-per-observation`) | +| (`test:integration:reset-restores-initial-state`) | engine | Reset returns a used sentinel to the state it was constructed in: the spatial graph is rebuilt as a bare root, the observation counter is zero, and the root tracker alone is tracked. Learned structure is dropped wholesale rather than aged out, because reset exists for the case where the host knows the past no longer describes the future. The configuration is not part of what is cleared. | +| (`test:integration:centred-bits-built-from-raw-values`) | bits | The bit-vector type the conversion trait returns can be built from outside the crate, and what a caller builds is the same thing the crate's own conversion produces. The trait is published and open to a coordinate type the crate has never heard of, so an implementation of it has to be able to produce the value it is required to return: a type whose centred form is computed rather than shifted out of an integer has nothing here to delegate to, and without a constructor its implementation could not be written at all. | +| (`test:integration:centred-bits-refuses-a-length-past-the-array`) | bits | A length past the backing array is refused rather than clamped. The array is a hundred and twenty-eight slots and nothing wider can be represented, so an implementation asking for more has miscomputed its own width; handing back a shorter vector would let that mistake travel into the tracker as an observation narrower than the one its author believed it built. | +| (`test:integration:fresh-sentinel-has-zero-clip-pressure`) | pressure | A sentinel that has been shown nothing reports no rejection at all — not a nominal starting level, but zero across the smallest, largest and average axis alike. Rejection is a measurement of traffic, so with no traffic there is nothing to measure, and the ceiling starts as wide as it can be rather than pre-loaded against the first arrivals. | +| (`test:integration:clip-pressure-distribution-min-le-mean-le-max`) | pressure | The summary the sentinel publishes is a genuine summary of the per-axis values behind it: after a long clean run followed by a contaminated burst, where the axes have been driven apart, the smallest value still sits at or below the average and the average at or below the largest. An operator watching only the peak can therefore trust that no axis is being rejected harder than the number they are watching. | +| (`test:integration:per-axis-clip-pressure-in-cell-reports`) | pressure | Rejection is visible at the grain it actually happens: every batch report carries a separate, finite, non-negative figure for each of the four scoring axes of every cell reported. Each axis keeps its own ceiling and its own running rejection rate, so a caller can see which axis is under strain rather than only that something is. | +| (`test:integration:clip-pressure-stable-under-clean-traffic`) | pressure | Traffic that keeps its shape does not ratchet the rejection rate upward. Over a long clean run the second stretch of batches sits no higher than the first, and the average across axes stays well under half. A modest steady rejection is expected — a ceiling a few deviations out will always trim the occasional sample — but it settles at a level rather than climbing, which is what lets a rise be read as news. | +| (`test:integration:contamination-elevates-clip-pressure`) | pressure | Mixing a substantial minority of structurally novel values into an otherwise settled stream drives the peak rejection rate clearly above where clean traffic had left it. The rate is thus an observable symptom of contamination in its own right: the ceiling is doing its job of keeping those samples out of the baseline, and the pressure is the sentinel saying how hard it is having to work at it. | +| (`test:integration:clip-pressure-decays-after-contamination`) | pressure | When the contamination stops, the rejection rate comes back down: after a long stretch of clean traffic the peak sits below where the contaminated phase left it. The measure is a rolling one with a finite memory, so an episode ages out instead of marking the sentinel permanently — a system that never forgot would treat one past attack as grounds for a forever-loose ceiling. | +| (`test:integration:effective-ceiling-widens-under-pressure`) | pressure | The rejection rate is not merely reported, it feeds back into the ceiling. Recomputing the documented widening rule from the pressure a contaminated run actually leaves behind gives a ceiling meaningfully above the configured one. Refusal is therefore self-limiting by construction: the harder an axis has been rejecting, the more room it grants itself, so a genuine and lasting shift in the traffic can eventually be learned rather than clipped away for ever. | +| (`test:integration:faster-decay-recovers-sooner`) | pressure | How long the rejection rate remembers is a configured choice with an observable consequence: given identical contamination and an identical recovery window, the sentinel with the shorter memory ends up at a lower pressure than the one with the longer. The setting therefore trades how quickly a ceiling snaps back after an episode against how steadily it holds through a noisy one. | +| (`test:integration:warm-up-completion-resets-clip-pressure`) | pressure | Rejection accrued while a tracker still leaned on synthetic history is discarded at the moment that reliance falls away: shortly after the crossing, the peak rate across the sentinel is still low, having had only a handful of real batches in which to rebuild. Whatever the warming rounds caused the ceiling to refuse is therefore not allowed to widen the ceiling that production traffic will be judged against. | +| (`test:integration:sudden-single-cell-z-score`) | coverage | The sharpest corner of the matrix: a single range switching abruptly to structurally different values pushes the highest per-cell score above what the same sentinel measured on the preceding ordinary batch. One batch is enough here, because the disturbance is a departure from learned structure rather than a change in how often the range is visited. | +| (`test:integration:gradual-single-cell-cusum`) | coverage | Persistence is itself evidence. Where the anomalous traffic keeps arriving batch after batch, the accumulating statistic climbs above where it stood on the first such batch, so a disturbance that never grows louder still grows more certain. The comparison is taken only after the slow baseline has been allowed to settle, which is what separates genuine accumulation from the transient left over from warm-up. | +| (`test:integration:sudden-partial-per-cell`) | coverage | Normal traffic elsewhere does not dilute an anomaly. With several ranges live and only some of them turning anomalous, the highest per-cell score still rises above the steady-state batch — the untouched ranges contribute their own unremarkable readings and nothing else. Because scoring is per-cell rather than an average over the domain, an attacker gains nothing by keeping most of the traffic ordinary. | +| (`test:integration:gradual-partial-cusum`) | coverage | The two evasions combined — a drift rather than a jump, and in only part of the traffic — still accumulate: with one range going anomalous batch after batch while another stays ordinary, the accumulated statistic ends above where the first anomalous batch left it. Being quiet and being partial are not additive protections, because accumulation happens per cell and the ordinary range accumulates nothing to average it away. | +| (`test:integration:sudden-system-wide-root-catches`) | coverage | The case a per-cell view is least equipped for: every range changes at once, so no cell is unusual relative to its neighbours, and the reading that moves is the root's. Coverage of this corner is what the coarse end of the chain exists for — an attack that shifts the whole population is caught by the model whose region is the whole population. | +| (`test:integration:gradual-system-wide-root-cusum`) | coverage | The quietest corner of the matrix, and the one a self-adjusting baseline is most at risk of absorbing: a shift across all ranges that simply keeps happening. The accumulated statistic keeps climbing past its first-batch value rather than settling, so persistence still tells even where extent leaves no cell looking unusual against its neighbours and no batch looks unusual against the last. | +| (`test:integration:step2-invariants-hold-throughout-lifecycle`) | warmup | Deferring warm-up changes when a cell becomes live, not what a report is allowed to look like. Across a long run of splitting traffic — cells entering staging, warming, and being promoted mid-run — every batch report still satisfies the structural guarantees. A cell in staging is simply absent from the producing set until it is ready, so no half-built model can leak into the output. | +| (`test:integration:step2-lifecycle-new-cells-appear-after-splits`) | warmup | A sentinel that starts as root alone ends a run of splitting traffic with several live cells, and every one of them carries noise observations behind it. Promotion is conditional on the warm-up schedule being finished, so the live map never contains a cell that skipped its seeding — staging is a queue on the way in, not an alternative way in. | +| (`test:integration:step2-determinism-staging-path-matches-twin`) | warmup | Two sentinels built from the same configuration and the same noise seed, fed the same batches, agree on their reports down to the bits of the score means. The synchronous drain warms queued cells in a deterministic order from a seeded generator, so the staging detour introduces no freedom: an investigation can be replayed exactly rather than approximately. | +| (`test:integration:step2-ancestor-receives-observations-for-warming-cells`) | warmup | Deferring a cell costs no coverage. Every value routes to each live cell whose interval contains it, and the root's interval contains all of them, so the root reports a non-zero sample count whatever is queued below it. A region under construction is therefore still watched — at coarser resolution, by the ancestor chain — rather than unobserved until its cell is ready. | +| (`test:integration:step2-eviction-during-reconcile-no-panic`) | warmup | A cell can lose its place before it ever takes it. Under a tight budget and scattered traffic, cells are created and evicted continuously, and a cell still warming when its G-node leaves the analysis set is dropped from staging rather than promoted into a set it no longer belongs to. Work already spent on it is abandoned, which is cheaper than admitting a cell the selector has rejected. | +| (`test:integration:step2-reset-clears-staging-and-resumes`) | warmup | A reset returns the sentinel to the state it was constructed in — root alone, no accumulated observations — and that includes emptying the staging area, so no cell queued under the old structure can surface under the new one. What follows is a genuine second warm-up: further traffic splits the space and builds cells again from nothing. | +| (`test:integration:step3-invariants-hold-with-background-warming`) | warmup | cites (`claim:warmup:deferring-warm-up-through-a-staging-area-never-lets-a-half-built-cell-into-a-report`) | +| (`test:integration:step3-warmup-completes-eventually`) | warmup | cites (`claim:warmup:a-cell-reaches-the-live-map-only-after-its-noise-schedule-is-complete`) | +| (`test:integration:step3-scoring-works-after-background-warmup`) | warmup | A tracker warmed on another thread scores like any other: after a warmed run, every cell and ancestor in the batch reports finite score means. That is what the seeding is for — a model with no baseline would divide by a spread it does not have — and it holds whichever thread supplied the seed. | +| (`test:integration:step3-background-same-cell-structure-as-sync`) | warmup | Two sentinels fed identical traffic, one warming inline and one on a background thread, end with the same accumulated volume and the same set of tracked cells. Structure is decided by observation volume alone and never by anything the trackers compute, so warm-up scheduling — a modelling concern — cannot perturb it. Choosing the background path is a latency decision, not a modelling one. | +| (`test:integration:step3-higher-volume-cells-warm-via-priority`) | warmup | When warming capacity is scarce — a long noise schedule and small batches leave the queue in progress — the staging area serves its waiting cells in descending order of volume, so a heavily trafficked subtree gets cells into service before a quiet one. Warming effort is spent where the traffic is, which is the same principle that decided the cells were worth having. | +| (`test:integration:step3-concurrent-ingest-no-panic`) | warmup | Ingestion and background warming genuinely overlap: a long run of diverse traffic keeps creating cells while the thread is warming earlier ones, and the two never collide. A cell being warmed is checked out of the staging area for the duration, so the expensive work happens on a cell no other thread can reach, and the lock is held only for the queue operations around it. | +| (`test:integration:step3-eviction-during-background-warming-no-panic`) | warmup | cites (`claim:warmup:a-warming-cell-whose-node-leaves-the-analysis-set-is-discarded-rather-than-promoted`) | +| (`test:integration:step3-reset-restarts-background-thread`) | warmup | cites (`claim:warmup:a-reset-empties-the-staging-area-so-warm-up-begins-again-from-the-root-alone`) | +| (`test:integration:step3-drop-while-warming-no-hang`) | warmup | Dropping a sentinel with a long noise schedule still outstanding returns promptly instead of waiting for the queue to empty. The warming thread checks for shutdown between batches and abandons whatever remains, because cells nobody will ever read from are not worth finishing. Teardown costs at most one batch of work, not the rest of the schedule. | +| (`test:integration:identical-seed-produces-identical-reports`) | determinism | Two sentinels built from one configuration with one seed, and stepped through the same batches, produce the same report at every step: the same cells and the same cross-cell contexts in the same positions, carrying the same handles, intervals, depths, counts and ranks, and every figure they advertise equal in its bit pattern rather than merely near — each of the four axes with its extremes, its mean, both z-scores, its baseline, its drift evidence and its rejection rate, alongside the contour, the health section and the summary of what the sentinel is investing in. Comparing counts and a mean or two would pass two runs that had modelled different regions of the domain in the same number of cells. Agreement is checked batch by batch and not only at the end, so a divergence could not open and close again unnoticed. The one figure held out is the age of the oldest observation, which measures how long a batch waited rather than anything computed from it. | +| (`test:integration:deterministic-across-repeated-runs`) | engine | cites (`claim:engine:the-root-tracker-receives-every-observation-in-every-batch`) | +| (`test:integration:different-seeds-produce-different-scores`) | determinism | The seed is an input with observable consequences, not a formality: two sentinels differing only in their seed, fed identical values, disagree in at least one of the root's score means. The warming noise a tracker is primed with shapes the subspace it starts from, and that starting point is still visible in what the tracker measures once real traffic arrives — which is why reproducibility has to be stated in terms of the seed rather than of the data alone. | +| (`test:integration:report-ordering-is-deterministic`) | determinism | All three report vectors come out in the order their contract states rather than in the order the walk produced: the competitive cells and the ancestors ascend by node handle, and the cross-cell contexts come shallowest first with ties broken by the handle. Depth leads there because handles are recycled as cells are evicted and restored, so a correctly ordered run can carry a lower handle at a greater depth. Neither ordering is a property of traversal or of when a cell was created, so two runs list the same entries in the same positions and a reader may compare them index by index. Splitting is forced aggressively here so that each vector holds several entries and the ordering is actually put to the question. | +| (`test:integration:send-and-sync-bounds`) | engine | The engine type may be moved between threads and referenced from several at once — a statement about the type, discharged by the compiler when these bounds are demanded, not by anything the test executes at run time. It holds because the sentinel keeps no thread-bound state: its optional background warming lives behind a lock it owns. A host is therefore free to place a sentinel wherever its own concurrency model wants it. | +| (`test:integration:min-value-u128`) | edge | The bottom of the coordinate domain is an ordinary observation: a value with no bits set is counted and reported like any other, and every structural invariant survives it. Encoding centres each bit rather than taking it raw, so an all-zero value is a well-formed vector and not a degenerate one the geometry has to special-case. | +| (`test:integration:max-value-u128`) | edge | cites (`claim:edge:the-extremes-of-the-coordinate-domain-are-ordinary-observations`) | +| (`test:integration:min-and-max-together`) | edge | cites (`claim:edge:the-extremes-of-the-coordinate-domain-are-ordinary-observations`) | +| (`test:integration:all-nibbles-full-spread`) | edge | Traffic spread evenly over every leading region of the domain, batch after batch, gives the spatial layer no concentration to reward — and the engine keeps its invariants anyway, with the observation count equal to exactly what was fed in. Uniform traffic is the worst case for a selector that ranks by importance, and it produces a boring report rather than an unstable one. | +| (`test:integration:single-observation-batch`) | engine | cites (`claim:engine:the-root-tracker-receives-every-observation-in-every-batch`) | +| (`test:integration:very-large-batch`) | edge | There is no ceiling on batch size: a batch of many thousands of values spread across the domain is scored in one pass, the root's sample count equals the batch it was given, and the invariants hold. Ingestion walks the batch once and updates state as it goes, so a large batch costs time proportional to its size and nothing else. | +| (`test:integration:empty-then-burst`) | edge | A long idle stretch of empty batches leaves the engine exactly where it was — nothing observed, nothing reported — and the burst that follows is scored as though the idling had never happened. Empty batches do not accumulate into a state the engine has to recover from, because an empty batch returns before any of the observation machinery runs. | +| (`test:integration:single-observation-repeated`) | edge | One value hammered in over and over, under a split threshold low enough to make the spatial layer subdivide around it, is counted in full and leaves the invariants intact. Concentration is exactly what the spatial layer is built to notice, so the pathological case of total concentration is a case it handles rather than one that surprises it. | +| (`test:integration:all-same-value`) | edge | Identical observations teach the model no new directions, so after a long run of them the root's learned rank is still near its floor rather than having grown with the volume. Rank tracks how many directions the data actually spans, not how much data arrived, which is what keeps the measurement honest about a stream that carries no structure. | +| (`test:integration:alternating-two-values`) | edge | A stream that alternates strictly between two well-separated values still leaves the root with at least one learned direction and the invariants standing. Two points do span a direction, so this is the smallest non-trivial structure a tracker can be given — the case just above the one where nothing varies at all. | +| (`test:integration:reset-then-immediate-ingest`) | engine | A sentinel is usable the instant a reset returns: the very next batch is counted from zero and produces a root report, with no warm-up call or settling period in between. Reset rebuilds the root tracker as part of the operation rather than leaving the engine cell-less until traffic arrives. | +| (`test:integration:double-reset`) | engine | cites (`claim:engine:the-sentinel-is-immediately-usable-after-a-reset`) | +| (`test:integration:decay-to-zero-then-rebuild`) | edge | Decay severe enough to annihilate accumulated standing does not leave a dead engine: fresh traffic rebuilds cells from what it observes, the root is reported again, and trackers are live. Decay lowers what regions have earned rather than removing the machinery that earns it, so the recovery path is simply ordinary ingestion. | +| (`test:integration:rapid-decay-ingest-cycle`) | edge | Decay interleaved with ingestion round after round keeps producing valid reports with the invariants intact. Decay does not need to be rare or quiescent to be safe: it changes spatial standing between batches, and the analysis set is simply recomputed at the start of the next ingestion rather than being eagerly invalidated. | +| (`test:integration:analysis-k-equals-one`) | edge | Squeezing the competitive budget to a single cell squeezes exactly that: at most one cell is reported as having earned its place, and the root is still reported regardless. The budget governs investment, not structure, so the narrowest possible budget yields the smallest useful report rather than an empty one. The full-set bound is deliberately not asserted here — closure under ancestry keeps materialising chains the bound assumes a wider budget for. | +| (`test:integration:analysis-depth-cutoff-zero`) | edge | cites (`claim:edge:an-extreme-analysis-budget-narrows-what-can-be-chosen-but-never-empties-the-set`) | +| (`test:integration:per-sample-scores-large-batch`) | engine | cites (`claim:engine:per-sample-scores-appear-only-when-asked-for-and-then-carry-one-entry-per-observation`) | +| (`test:integration:edge-cases-degenerate-cells-skipped-starts-at-zero`) | engine | cites (`claim:engine:a-fresh-sentinel-has-skipped-no-cell-as-too-narrow-to-model`) | +| (`test:integration:degenerate-cells-skipped-in-report-normal-traffic`) | edge | Ordinary traffic never drives a cell narrow enough to be skipped: after a long run of ingestion the report still shows no skips at all. The guard is there for pathological splitting, not for everyday operation, so a non-zero reading in the field is a signal about the traffic rather than routine noise. | +| (`test:integration:degenerate-cells-skipped-after-reset`) | engine | cites (`claim:engine:a-fresh-sentinel-has-skipped-no-cell-as-too-narrow-to-model`) | +| (`test:integration:deep-spray-traffic-does-not-panic`) | edge | Splitting made as aggressive as the configuration allows, on traffic hammering a single point, drives the domain deep enough that some cells have almost no suffix left to analyse. Those cells are skipped and counted rather than given a tracker that could form no basis and produce no residual, and the run continues with the root still tracked. A cell too narrow to model is a case the engine declines, not a fault it crashes on. | +| (`test:integration:fresh-graph-has-one-node`) | routing | Before anything is observed the graph is a single cell spanning the whole coordinate domain. There is no partition worth choosing until traffic says where the boundaries should fall, so the sentinel starts with the one cell it can justify and refines outward from there. | +| (`test:integration:fresh-graph-has-one-terminal`) | routing | cites (`claim:routing:a-fresh-graph-is-one-root-cell-covering-the-whole-domain-with-nothing-accumulated`) | +| (`test:integration:fresh-graph-has-zero-total-sum`) | routing | cites (`claim:routing:a-fresh-graph-is-one-root-cell-covering-the-whole-domain-with-nothing-accumulated`) | +| (`test:integration:ingest-feeds-graph-with-delta-one`) | routing | Every value in a batch contributes exactly one unit of importance, whichever range it falls in, so the graph's total after successive batches into unrelated ranges is simply how many values were handed over. Importance is a count of arrivals rather than a weight the caller can set, which is what lets the selector read it as evidence of where traffic is. | +| (`test:integration:empty-ingest-does-not-observe`) | routing | A batch with nothing in it is not an event. The accumulated total stays where it was and no cell is created, so an interval in which nothing arrived neither adds evidence nor moves the partition. Quiet time is therefore invisible to the spatial layer rather than being recorded as an observation of emptiness. | +| (`test:integration:duplicate-values-each-contribute`) | routing | cites (`claim:routing:every-ingested-value-adds-exactly-one-unit-of-importance-wherever-it-lands`) | +| (`test:integration:graph-accumulates-across-batches`) | routing | cites (`claim:routing:every-ingested-value-adds-exactly-one-unit-of-importance-wherever-it-lands`) | +| (`test:integration:lifetime-observations-tracks-total-sum`) | routing | The counter the sentinel keeps and the total the graph accumulates stay equal batch after batch. They are one quantity read from two layers: real observations are the only thing that increments either, and the synthetic data used to warm trackers is deliberately kept out of both. A host can therefore read whichever is nearer to hand without learning which layer maintains it. | +| (`test:integration:concentrated-traffic-splits-nodes`) | routing | Traffic that keeps landing in one narrow range drives that range past the split threshold and the graph refines it, so the partition is bought with observations rather than configured up front. Where the traffic goes is where the resolution appears; the total meanwhile still counts exactly the values handed over, so refining a region does not manufacture evidence. | +| (`test:integration:concentrated-traffic-grows-terminals`) | routing | cites (`claim:routing:concentrated-traffic-buys-resolution-by-splitting-the-range-it-lands-in`) | +| (`test:integration:diverse-traffic-respects-budget`) | routing | Traffic spread thinly over many well-separated ranges asks the graph to refine everywhere at once, and the node budget is what keeps that from being unbounded: however many ranges are busy and however low the split threshold is set, the graph holds no more cells than the budget allows. The cost of modelling is a configured ceiling rather than a function of how widely an adversary chooses to scatter. | +| (`test:integration:reset-restores-fresh-graph-state`) | routing | Reset discards the partition as well as the evidence: a graph that had split under load comes back as the single root cell of a fresh sentinel, with nothing accumulated and nothing to land in but the root. Structure is derived from observations, so once the observations are dropped there is no refinement left worth preserving, and a reset sentinel cannot be distinguished from a new one by what its graph holds. | +| (`test:integration:top-of-domain-coordinate-routes-to-a-cell`) | routing | The topmost coordinate of the domain reaches a tracker rather than falling through every cell. Cell intervals are half-open, which has no upper edge case while the coordinate width is narrower than the coordinate type — the bound is then a representable value outside the domain. At the full width the domain's maximum is the type's maximum, there is no value above it to be excluded, and a half-open reading of the topmost interval therefore excludes a coordinate that is genuinely inside the domain. The spatial layer counts that observation either way, so the two readings would disagree: the accumulated total records an arrival that no tracker was ever shown. | +| (`test:integration:an-ordinary-coordinate-still-lands-in-one-cell`) | routing | cites (`claim:routing:the-domains-top-coordinate-reaches-a-tracker-rather-than-falling-through-every-cell`) | +| (`test:integration:fresh-has-one-root-tracker`) | health | A sentinel that has ingested nothing holds exactly one tracker, on the root cell, over a graph of a single node. The root exists unconditionally so that every ancestor chain has somewhere to terminate; every other tracker is bought only once traffic has justified it. | +| (`test:integration:fresh-has-zero-observations`) | health | The lifetime count reads zero on a fresh sentinel even though its root tracker has already absorbed synthetic seeding. Injected noise gives a tracker a usable model but is not evidence about traffic, so it is kept deliberately out of the number a host reads as how much the engine has actually seen. | +| (`test:integration:fresh-rank-distribution-is-uniform-at-one`) | health | With a single tracker the rank distribution degenerates: smallest, largest and mean all agree, and all sit at the starting rank of one, because a learned subspace begins with a single basis direction. The distribution summarises a fleet, and a fleet of one has no spread to report. | +| (`test:integration:fresh-maturity-is-cold`) | health | A tracker counts as cold for as long as its real-observation count is zero, whatever synthetic data it has already absorbed. Coldness measures exposure to the world rather than whether the model is populated, which is exactly the distinction a host needs when deciding how much a score from that tracker is worth. | +| (`test:integration:fresh-geometry-distribution`) | health | The geometry summary counts trackers whose axes are structurally unavailable rather than merely quiet. A rank-one tracker has no second direction for coherence to compare against and is counted inactive on that axis, while on a wide domain its rank is nowhere near the dimension so novelty is not saturated. A zero score means something different in each case, and this is where a host learns which case it is in. | +| (`test:integration:fresh-clip-pressure-is-zero`) | health | Clip pressure reads zero at every extreme on a sentinel that has seen no real data. Pressure accumulates only when observations actually press against the clipping bound, so it is a record of how often the engine had to hold data back — and an engine that has held nothing back reports none. | +| (`test:integration:fresh-coordination-is-empty`) | coordination | No coordination context exists until a batch has been observed, because a context is created only where cells on both sides of a split report in the same batch. The summary is nonetheless present and reads empty rather than being absent: the tier is always described, even when there is nothing in it. | +| (`test:integration:tracker-counts-are-consistent`) | health | The active trackers decompose exactly into the competitive cells, the ancestors pulled in to connect them, and the one permanent root. These are not three independent measurements but one partition reported three ways, so a host can read the shape of the engine's investment from them and any disagreement would be an accounting fault rather than a fact about traffic. | +| (`test:integration:investment-set-covers-active-plus-warming`) | health | The investment set is every tracker the engine is paying for, whether already scoring or still warming up in the staging pipeline. Warming cells consume memory and work before they produce anything, so a figure that counted only the online trackers would understate what the sentinel is spending. | +| (`test:integration:population-grows-after-divergent-ingest`) | health | Values whose leading bits diverge land in separate regions of the domain and the tracker population follows the traffic there. The two counts that describe that population stay in step: the cells the sentinel says it is tracking are exactly the cells with trackers behind them, so neither figure can drift into describing an investment that does not exist. | +| (`test:integration:lifetime-observations-accumulate`) | health | cites (`claim:health:lifetime-observations-counts-real-data-only-so-seeding-noise-leaves-it-at-zero`) | +| (`test:integration:rank-bounds-hold-after-ingest`) | health | Ranks reported after real traffic stay inside the configured ceiling from above and at the starting rank or better from below, with the mean lying between the two extremes it summarises. The ceiling is a spending decision the host made, so the snapshot is expected to respect it rather than announce a model larger than was authorised. | +| (`test:integration:noise-reduces-noise-influence`) | health | Noise influence is the share of a tracker's model still owed to its synthetic seeding, and real observations dilute it: once actual traffic has arrived the fleet's mean influence has fallen below the value it starts at. It is the measurement that tells a host how much of a score is still borrowed from data the engine invented for itself. | +| (`test:integration:cold-config-leaves-trackers-cold`) | health | With seeding switched off there is nothing to dilute, so real data leaves every tracker at the same maturity and the distribution collapses — its lowest value is no lower than its mean. Spread in maturity across the fleet comes from trackers being at different stages of shedding their synthetic prior, not from the traffic alone. | +| (`test:integration:health-evolves-over-repeated-batches`) | health | Across a run of warm-up batches the snapshot reflects the whole history rather than the batch just handled: observations total up across all of them, trackers remain live, and rank has had the chance to adapt above its starting value. Health describes the engine's state as it now stands, which is a running position and not a per-batch reading. | +| (`test:integration:coordination-starts-empty`) | coordination | cites (`claim:coordination:no-context-exists-until-a-batch-has-been-observed`) | +| (`test:integration:coordination-structurally-sound-after-noise`) | coordination | Whatever contexts a run happens to have created, the shape they report is internally consistent: the tier's fixed dimensionality, a capacity that exceeds neither the configured rank ceiling nor the four dimensions available, an influence share inside its unit range, and a rank between the starting value and that capacity. The check is conditional because contexts are created lazily where traffic happens to fall, and pretending otherwise would test the fixture rather than the engine. | +| (`test:integration:no-coordination-on-empty-batch`) | coordination | A batch carrying no values gives the tier nothing to group: no cell reports a score, no node can find both its subtrees contributing, and the coordination list comes back empty. Coordination is a statement about what several cells did together in one batch, so with no batch there is nothing to say. | +| (`test:integration:no-coordination-with-single-region`) | coordination | Traffic confined to one region leaves every reporting cell in the same subtree, so the both-subtrees condition is never met on its own account. Any context that does appear carries several member cells — which is what makes a coordination score a genuinely cross-cell measurement rather than a restatement of one cell's own score at a coarser scale. | +| (`test:integration:two-sibling-cells-fire-parent-context`) | coordination | Two well-separated regions reporting in the same batch turn the node above them into a live context: it sees members arriving from each side and begins modelling the pair as a group. Each report identifies itself by the dyadic interval of that node, whose lower bound always lies strictly below its upper bound, so a host can say which stretch of the domain the measurement covers. | +| (`test:integration:context-activation-requires-both-subtrees`) | coordination | After a run in which only one region reported, sending traffic to the opposite half of the domain can only add contexts, never take them away. Contexts are created where a split starts carrying reporters on both sides, so the arrival of a second region is precisely the event that brings the tier into existence. | +| (`test:integration:context-deactivation-on-subtree-loss`) | coordination | When a batch arrives from one region only, the nodes that had been spanning both stop qualifying, and the tier prunes them instead of carrying stale contexts forward. The count of live contexts therefore never rises when a subtree falls silent: what is reported describes the group structure of the batch in hand, not of some earlier one. | +| (`test:integration:nested-coordination-levels`) | coordination | cites (`claim:coordination:each-firing-node-appears-exactly-once-among-a-batchs-coordination-reports`) | +| (`test:integration:coordination-group-nesting-invariant`) | coordination | Membership nests with the tree. Wherever one context's interval contains another's, the containing context counts at least as many reporting cells, because the walk unions each subtree's members and passes the result upward. An ancestor therefore measures a superset of what its descendant measured, which is what lets the two readings be compared as the same pattern seen at two scales. | +| (`test:integration:root-context-sees-all-cells`) | coordination | cites (`claim:coordination:an-ancestor-context-counts-every-cell-its-descendant-counts`) | +| (`test:integration:semi-internal-node-passthrough`) | coordination | A node with only one contributing child poses no coordination question — there is no pair of subtrees to compare — so it fires nothing and simply forwards its members to its parent. Every report that does appear consequently spans a real split and carries a usable finite score, rather than the tier manufacturing a group out of a single branch. | +| (`test:integration:coordination-tracker-operates-at-w4`) | coordination | A member's contribution to a context is one value per scoring axis, so the tier works in four dimensions however wide the cells beneath it are. That fixed and tiny geometry is what makes a second tier affordable: its cost does not grow with the width of the domain the cells analyse. | +| (`test:integration:coordination-tracker-with-low-max-rank`) | coordination | A context's capacity is the configured rank ceiling, capped by the four dimensions actually available. Lowering that ceiling lowers the capacity with it while the dimensionality stays at four: capacity is a budget on how much structure the model may hold, not a change to what a member observation is. | +| (`test:integration:running-mean-cold-start`) | coordination | The first batch a context ever handles still yields finite numbers on all four axes. Members are centred against a running mean that starts at zero and is seeded outright from that batch's own column means rather than divided by an empty history, so activation costs the host no undefined readings to interpret. | +| (`test:integration:running-mean-ewma-update`) | coordination | A context that has activated goes on firing while both its subtrees keep reporting: repeating the same shape of batch produces coordination in nearly all of them. Its running mean is updated across those batches instead of being rebuilt each time, so what the tier measures stays a departure from the group's own recent pattern. | +| (`test:integration:only-competitive-cells-contribute`) | coordination | Membership is drawn only from competitive cells that actually observed something in this batch, so no context can report more members than there were such cells. The root is never competitive and so never joins a group: coordination measures the cells the engine chose to invest in, not the whole tree. | +| (`test:integration:cells-with-no-observations-excluded`) | coordination | cites (`claim:coordination:only-competitive-cells-that-observed-this-batch-become-members`) | +| (`test:integration:coordination-scores-are-finite`) | coordination | Everything a context publishes is a finite number: the four axis means, the drift accumulated on each, the fraction of variance the learned subspace captures and its largest singular value. Degenerate geometry at this tier is resolved inside the model rather than handed to the host as a not-a-number it would have to interpret for itself. | +| (`test:integration:per-member-scores-present-when-enabled`) | coordination | With per-sample scoring enabled a context also names its members: one entry per reporting cell, matching the membership count it declared, each carrying that cell's own four scores and the non-empty interval it covers. A host can therefore attribute a group-level reading to particular regions of the domain instead of seeing only the aggregate. | +| (`test:integration:coordination-survives-decay`) | coordination | Ageing the model down does not dismantle the tier. After a decay the sentinel still ingests and still produces reports on the next batch, because decay weakens the learned structure rather than removing the cells and contexts that carry it. | +| (`test:integration:inject-noise-warms-coordination`) | coordination | A new context is warmed with synthetic score vectors drawn to match its members' own baselines, which leaves it holding a usable model rather than an empty one. The warm-up then clears the drift accumulators, so the first real batch is that context's first step of evidence: what the engine invented about itself is never counted as evidence about traffic. | +| (`test:integration:deterministic-report-order`) | coordination | Two runs from the same seed over the same traffic produce the same reports: the same number of them, at the same nodes in the same order, with the same memberships and matching scores. The only randomness in the engine is the seeded warm-up, and pinning it makes two runs comparable — without that, no difference a host observed could be attributed. | +| (`test:integration:context-count-bounded-by-k-minus-1`) | coordination | Contexts are the internal nodes of a binary tree whose leaves are the competitive cells, so their number stays below the number of those cells. The tier's cost is thereby bounded by the analysis budget the host has already chosen and cannot grow on its own. | +| (`test:integration:coordination-reports-unique-gnodes`) | coordination | A batch produces at most one report per firing node. The walk visits each node once and a context is keyed by the node it sits at, so a nested hierarchy yields one measurement per level rather than repeated entries a host would first have to de-duplicate. | +| (`test:integration:batch-report-coordination-is-vec`) | coordination | The batch report always carries a list of coordination reports, empty when nothing fired, rather than an optional one. Absence of coordination is an ordinary outcome — a lone value in a batch simply produces none — so a host iterates the list without first having to test for presence. | +| (`test:integration:health-report-has-coordination-health`) | coordination | cites (`claim:coordination:the-tier-works-in-four-dimensions-because-a-member-contributes-one-value-per-scoring-axis`) | +| (`test:integration:config-cusum-coord-slow-decay-validated`) | coordination | The slow baseline the coordination tier measures drift against is validated like any other rate: strictly inside zero and one, and strictly slower than the fast forgetting factor. The separation is the whole point — the slow baseline is the reference the fast one is judged against — so a configuration where the two move at the same speed is rejected outright rather than quietly producing a meaningless reading. | +| (`test:integration:coordination-reports-are-ordered-by-depth-then-identifier`) | coordination | Coordination reports arrive shallowest first, ties broken by ascending identifier. The walk that produces them is bottom-up, which emits a strictly post-order sequence and puts the root — the shallowest context of all — last; that is deterministic but it is not the order either record states, and a reader taking the reports as a descent from the coarsest scale to the finest would have had the sequence exactly backwards. Depth is the ordering the output record describes and the identifier is the ordering this type's own documentation describes, so sorting on the pair satisfies both and is a total order besides, which sorting on depth alone would not be. | +| (`test:integration:contour-count-includes-semi-internal-nodes`) | coordination | The contour count is the whole contour: the terminal cells together with the semi-internal ones. A semi-internal node has one half subdivided and one that still accumulates locally, so that second half receives observations exactly as a terminal cell does and is part of the surface the snapshot describes. Counting only the terminals reported a resolution short by every half-subdivided node, which is a figure that drifts from the truth precisely while the structure is being reshaped. | +| (`test:integration:single-range-lifecycle`) | engine | A sentinel taken through a full life — constructed, seeded, warmed, driven at steady state, then handed a structurally unfamiliar batch — holds every structural invariant at each of those stages, not merely at the end. The health snapshot afterwards still describes a working engine: trackers are live, and the smallest learned rank is at least one, so no cell has collapsed to a model with no directions in it. | +| (`test:integration:multi-range-lifecycle`) | engine | Traffic concentrated in two well-separated regions of the domain drives the spatial layer to split, and the sentinel ends up tracking more than the root — separate models for separate structure, which is the whole point of a hierarchy. The invariants continue to hold through steady state and through an unfamiliar batch confined to one of the two regions, so the cells coexist rather than interfering. | +| (`test:integration:anomalous-batch-elevates-novelty`) | engine | Once a sentinel has settled on the structure it keeps being shown, a batch built on a different bit pattern scores higher than the ordinary batch immediately before it. The comparison is against that neighbour rather than against a fixed number, because the engine reports how far an observation departs from what this cell learned and leaves the question of how far is too far to the host. | +| (`test:integration:anomalous-batch-elevates-cusum`) | engine | Unfamiliarity that persists across many batches accumulates: the drift accumulator stands higher after a long run of unfamiliar traffic than it did at the baseline batch. A single surprising batch and a regime that has genuinely shifted look alike instant by instant, and the accumulator is what tells them apart — a small departure repeated is allowed to add up rather than being forgotten each round. | +| (`test:integration:decay-then-ingest-maintains-invariants`) | engine | Decay applied in the middle of a run attenuates accumulated spatial standing and nothing else: ingestion continues afterwards with every invariant intact and the observation counter still climbing. Temporal policy belongs to the host, and the engine implements it by lowering what cells have earned rather than by discarding what they have learned, so a decayed sentinel is a going concern and not a half-reset one. | +| (`test:integration:inspect-cells-after-multi-range-traffic`) | width | cites (`claim:width:a-cells-analysis-width-is-the-domain-width-less-its-depth`) | +| (`test:integration:coordination-activates-with-multi-range-traffic`) | engine | Once two separate regions are being modelled in earnest, a second tier of reporting appears without the host asking for it: a context that covers several cells at once and carries scores of its own, finite like any other. A pattern spread across sibling cells is invisible to each of them individually, so the engine models the pattern itself as soon as there are enough cells for one to exist. | +| (`test:integration:feed-forward-delta-one-normal`) | invariant | Each observation contributes exactly one unit to the structure's running total, checked after every batch of a long run: the total is always the number of values fed so far and never drifts from it. Importance is therefore a count of traffic rather than a derived score, which is what lets the weight a range carries be compared against another range's honestly. | +| (`test:integration:feed-forward-delta-one-anomalous`) | invariant | cites (`claim:invariant:the-running-total-counts-exactly-one-unit-per-observation`) | +| (`test:integration:feed-forward-delta-one-after-decay`) | invariant | cites (`claim:invariant:the-running-total-counts-exactly-one-unit-per-observation`) | +| (`test:integration:analysis-width-equals-128-minus-depth`) | invariant | Every cell and every ancestor in a report analyses exactly the domain less the levels its position has already fixed. The relation is arithmetic rather than incidental: the bits routing resolved are constant for everything arriving in that cell and carry no information, so what remains is precisely what a tracker there can learn from, and its declared analysis is that and nothing else. | +| (`test:integration:energy-ratio-bounded-zero-to-one`) | invariant | The fraction of structure a tracker has managed to capture is reported as a genuine fraction: it never falls below nothing and never exceeds everything, in any cell or ancestor of a report. Because it is bounded on both sides, a caller can read it directly as how much of what arrives the model explains, and can compare one cell's figure against another's. | +| (`test:integration:rank-bounded-by-max-rank`) | invariant | No tracker in a report has grown past the ceiling its configuration set, at any level of the tree. The ceiling is what makes a tracker's cost knowable in advance, so it has to be a hard limit on what the adaptation may reach for rather than a target it aims at — traffic complicated enough to justify more structure still does not get more. | +| (`test:integration:no-nan-scores-after-warmup`) | invariant | After seeding and a run of warming batches, no axis of any reported cell hands back a number that is not a number. This matters more than tidiness: such a value compares false against every threshold, so a single one would silently disarm the alerting that reads it, and the arithmetic guards its absence rather than callers being expected to check. | +| (`test:integration:noise-injected-before-real-observations`) | invariant | After a run that creates cells below the root, every cell being tracked carries synthetic observations — including those that came into being while traffic was already flowing. The ordering within the step is fixed rather than raced: preparation happens before a tracker is shown real data, so no cell is ever in the position of judging its first batch against nothing. | +| (`test:integration:graph-updated-before-analysis-set`) | invariant | The summary and the detail of a batch report describe the same moment: when the summary says cells are competing, the report already carries their entries. The structure is brought up to date before attention is reapportioned within the same batch, so a split does not leave a batch whose summary counts a cell that the report cannot show. | +| (`test:integration:root-always-in-analysis-set`) | invariant | Whatever range a batch is aimed at, the reported set of analysed cells still reaches back to the root — its shallowest member is the root in every batch of a run that cycles through all the leading ranges. The root's place is unconditional, so every chain of ancestors terminates and there is always a model covering traffic that belongs to no more specific cell. | +| (`test:integration:root-survives-extreme-decay`) | invariant | Ageing severe enough to annihilate the accumulated standing of everything else still leaves the root tracker in place, and traffic arriving afterwards is reported against it again. The root is permanent by construction rather than by having earned its standing, because a sentinel that could decay away its last model would have nothing to route to and no way to begin again. | +| (`test:integration:score-polarity-higher-is-more-anomalous`) | invariant | Two sentinels given identical settling traffic diverge in the expected direction once one of them is fed structurally novel batches: its peak drift reading ends up above the other's. Scores point one way — larger means more anomalous — so a caller may threshold and compare them without having to know which axis produced the number or which direction it runs in. | +| (`test:integration:cusum-accumulators-non-negative`) | invariant | Through a run of ordinary traffic, no drift accumulator on any axis of any reported cell ever goes below nothing. The accumulator is clamped at rest deliberately: a stretch of quieter-than-usual traffic must not bank negative standing that a later attack could spend, so a rise always starts from zero and means what it says. | +| (`test:integration:lifetime-observations-monotonically-increases`) | invariant | The lifetime observation count rises by exactly the size of each batch, for batches ranging from a single value to many. It is a plain census of what was handed in — never sampled, never rounded and never adjusted for what the values looked like — which is what makes it usable as the denominator when judging any rate the sentinel reports. | +| (`test:integration:cells-tracked-always-at-least-one`) | invariant | At no point in a sentinel's life is it tracking nothing: not at birth before any traffic, not through a run of ingestion, and not after ageing severe enough to strip away everything that had accumulated. There is always at least the root, so the question "what does the sentinel make of this value" always has an answer. | +| (`test:integration:assert-invariants-under-random-traffic`) | invariant | Traffic with no structure at all — values scattered across the whole domain, in batches whose size changes from one to the next — leaves every structural property standing, checked after each batch. The guarantees are not conditioned on the traffic being well behaved or on batches being uniform, which is the whole point of calling them guarantees. | +| (`test:integration:assert-invariants-after-decay-regrowth`) | invariant | A sentinel whose tree has been collapsed by severe ageing and then made to regrow under traffic spread across the ranges satisfies every structural property throughout the regrowth, batch by batch. The transient state of a system rebuilding itself is exactly where a bound is likeliest to slip, so the guarantees are asserted while it is in motion rather than once it has settled. | +| (`test:integration:a-single-arrival-outlives-the-projection-only-in-the-accumulator`) | invariant | The feed-forward count is checked in the accumulator's own domain rather than through a floating-point projection, because past a certain magnitude the projection cannot express a single arrival. The projection is lossy by its own documentation, and at the first magnitude where consecutive integers stop being separately representable, a total and that same total plus one arrival land on the same number while a total plus two lands two away. A check that projects both sides and allows them to differ by less than one arrival therefore rejects arithmetic that is exactly right. The accumulator keeps the distinction the projection loses, so the comparison belongs there; this test pins the property the choice rests on rather than the failure itself, which is some nine quadrillion observations away and not reachable by a test. | +| (`test:integration:root-warmed-at-construction`) | schedule | Constructing a sentinel already warms its root tracker: the root carries synthetic observations before any caller has had a chance to feed it. Those rounds push the tracker's reliance on synthetic history toward its maximum and hold it there, so a root that has seen nothing but warming still reports itself as fully synthetic — the reliance only falls once real traffic arrives to displace it. | +| (`test:integration:root-cold-when-schedule-empty`) | schedule | A schedule that specifies no rounds at any depth turns warming off rather than falling back to a default: the root is constructed cold, with no synthetic observations at all. Its reliance on synthetic history still reads as maximal, because that is the value a tracker is born with and warming is what would have started moving it. | +| (`test:integration:noise-does-not-count-as-real-observations`) | schedule | Warming rounds are not traffic and are not counted as traffic: a sentinel whose root has just been warmed still reports having observed nothing over its lifetime. The two ledgers are kept apart deliberately, so an operator reading the observation count sees what the world sent and never the sentinel's own preparation. | +| (`test:integration:new-cells-warmed-on-analysis-set-entry`) | schedule | Warming is not a construction-time favour granted to the root alone. After a run whose traffic is concentrated enough to split the tree, every cell the sentinel is tracking carries synthetic observations, including the ones that did not exist when the sentinel was built. A cell that arrives mid-run is therefore never asked to score its own first batch from an empty baseline. | +| (`test:integration:successive-cells-get-different-noise`) | schedule | cites (`claim:schedule:a-cell-born-mid-run-is-warmed-as-it-enters-the-analysis-set`) | +| (`test:integration:deeper-cells-receive-fewer-noise-rounds`) | schedule | Under a geometric schedule the round count falls off with depth, and the effect is visible in the trackers themselves: at least one cell below the root ends the run with fewer synthetic observations than the root has. The taper is the point of the schedule — a deeper cell works on a narrower slice and settles in fewer rounds, so spending root-sized warming on it would buy nothing. | +| (`test:integration:auto-inject-resets-cusum`) | schedule | Warming ends with the drift detectors wound back to zero, so the first real batch a tracker sees is its first step of drift accounting — every cell and ancestor in that batch's report says exactly one step has passed since its last reset. Without the reset, the drift a tracker accumulated while chasing synthetic rounds would be charged to whoever sent the first real request. | +| (`test:integration:deterministic-with-same-seed`) | schedule | Two sentinels built from the same seed and fed the same traffic come out with the same cells, warmed by the same number of rounds and left with the same reliance on synthetic history. Warming is a reproducible part of a run rather than a source of drift between two otherwise identical deployments, which is what makes a captured incident replayable at all. | +| (`test:integration:different-seeds-produce-different-baselines`) | schedule | The seed is not cosmetic: two sentinels warmed from different seeds and then fed byte-identical traffic score that traffic against different baselines. Warming leaves a real imprint on where a tracker starts, so an attacker who knew one deployment's warmed baseline would not thereby know another's. | +| (`test:integration:reset-reseeds-rng-and-warms-root`) | schedule | Resetting a sentinel that has been running for a long stretch puts its root back exactly where a freshly constructed one stands: the same number of warming rounds, the same reliance on synthetic history. Reset restores the generator to its seed and warms the rebuilt root again, so it is a genuine return to birth rather than a partial clearing that leaves a cold root behind. | +| (`test:integration:coordination-contexts-warmed-on-activation`) | schedule | The contexts that watch several cells at once are warmed on the same terms as the cell trackers: in a run where any of them became active, none of them is left cold. Their synthetic rounds are drawn to look like the score patterns they will actually be shown, sampled from the contributing cells' own baselines, because a context scores score vectors rather than raw coordinates. | +| (`test:integration:empty-ingest-produces-full-report`) | readout | An ingest with nothing in it still returns a complete report: the cell, ancestor and coordination lists are present and empty, the contour and health sections are populated, and the summary counts at least the root. A host polling a quiet sentinel therefore parses the same layout it parses under load, and can read structural state from a batch that carried no observations at all. | +| (`test:integration:cell-reports-are-competitive-only`) | readout | The report separates cells by how they earned their place: the cell list holds only competitively selected cells. A competitive cell was chosen because its traffic made it worth modelling, so a host reading that list is reading the sentinel's own investment decisions and nothing else. | +| (`test:integration:ancestor-reports-are-ancestor-only`) | readout | cites (`claim:readout:the-report-partitions-cells-by-how-they-earned-their-place-rather-than-listing-them-together`) | +| (`test:integration:cell-and-ancestor-cover-all-reported-cells`) | readout | cites (`claim:readout:the-report-partitions-cells-by-how-they-earned-their-place-rather-than-listing-them-together`) | +| (`test:integration:cell-reports-sorted-by-gnode-id`) | readout | Cell reports come back in strictly ascending handle order, never merely grouped. Nothing about the order reflects the sequence observations arrived in or how the internal maps happened to iterate, so two sentinels fed the same stream emit comparable reports and a difference between two readouts is a difference in the system. | +| (`test:integration:ancestor-reports-sorted-by-gnode-id`) | readout | cites (`claim:readout:report-lists-are-ordered-by-cell-handle-so-identical-runs-produce-identical-readouts`) | +| (`test:integration:coordination-reports-sorted-by-depth-then-gnode-id`) | readout | The tier above the cells is ordered too, but on a key of its own: coordination reports come shallowest first, with the node handle breaking ties among contexts at equal depth. Handles are recycled as cells are evicted and restored, so a correctly ordered run can carry a lower handle at a greater depth — which is why depth leads, and why an assertion on the handle alone would reject a readout that was right. Determinism is imposed on the readout as a whole rather than recovered separately wherever a list happens to be built, but each list states which order it is in. | +| (`test:integration:coordination-reports-have-unique-gnodes`) | readout | Each coordination context appears at most once in a batch. The contexts are found by walking a tree in which a node can be reached from several selected descendants, so uniqueness is a real obligation: without it a busy subtree would report the same group finding repeatedly and a host counting elevated contexts would over-count it. | +| (`test:integration:no-nan-in-score-fields`) | readout | Every score the readout carries is a number, on all four axes and across both competitive and ancestor cells. The scoring formulae divide by quantities that can legitimately reach zero — residual degrees of freedom, rank, baseline spread — so producing a number at the boundary is something the engine must arrange. A single non-number would poison every comparison a host makes downstream, silently rather than loudly. | +| (`test:integration:report-structure-per-sample-scores-present-when-enabled`) | readout | Per-observation detail is present in every cell report exactly when the host configured it, rather than appearing only where the engine found it convenient. The detail costs memory proportional to the batch, so it is optional — but an option that were honoured unevenly would be worse than none, since a host could not tell an absent field from an unremarkable cell. | +| (`test:integration:coordination-cells-reporting-is-nonzero`) | readout | A coordination report is emitted only where cells actually contributed to it: every one names a positive number of reporting cells. The tier exists to measure how a group of cells moves together, so a context with no contributors would be describing a group that did not exist this batch. | +| (`test:integration:member-score-identifies-cell`) | readout | cites (`claim:readout:a-member-score-names-the-cell-it-came-from-so-a-group-finding-can-be-attributed`) | +| (`test:integration:contour-reflects-graph-state`) | readout | The contour describes the spatial layer as it actually stands: after a batch of real traffic it reports accumulated importance above zero and at least one cell. It is read from the graph at report time rather than maintained alongside it, so it cannot drift out of step with the structure the trackers are attached to. | +| (`test:integration:contour-cell-count-grows-with-distinct-regions`) | readout | Traffic arriving in a well-separated second region never reduces the reported spatial resolution. New structure is added by bisection, and nothing about observing an unfamiliar region coarsens what the graph already learned elsewhere — so a host watching the cell count sees refinement accumulate rather than oscillate with the traffic mix. | +| (`test:integration:contour-reports-splits-since-last-report`) | readout | The split counter describes the interval since the previous report and is cleared with it: a heavy batch that forces bisection reports the splits it caused, and a quiet batch immediately after does not inherit them. The figure is a rate rather than a running total, which is what makes bursts of structural churn visible in the report that contained them. | +| (`test:integration:contour-mutation-counts-zero-on-empty-ingest`) | readout | cites (`claim:readout:structural-mutation-counts-describe-the-interval-since-the-previous-report-and-reset-with-it`) | +| (`test:integration:contour-after-decay`) | readout | Forgetting is visible in the readout: after the host applies decay, the reported importance is lower than before it. Temporal policy belongs to the host, which decides when history should count for less, and the contour is where that decision becomes observable — otherwise a host could not confirm that a decay it asked for had taken effect. | +| (`test:integration:health-inline-matches-standalone`) | readout | The health carried inside a batch is the same health a standalone query returns — lifetime observations, active trackers and node count all agree. There is one health computation rather than two that happen to coincide, so a host that reads health from reports and a host that polls for it cannot form different pictures of the same sentinel. | +| (`test:integration:health-tracker-breakdown-is-consistent`) | readout | The tracker breakdown accounts for the root apart from the two named categories: competitive and ancestor counts together fall short of the active total, the shortfall being the permanent root tracker. The root is present for structural reasons rather than because it competed or was closed over, and folding it into either count would misstate what the sentinel chose to invest in. The reported node total is likewise the graph's own count rather than a separately maintained tally. | +| (`test:integration:analysis-set-summary-matches-analysis-set`) | readout | The summary's counts are the analysis set's own counts, read from it rather than tallied a second time on the way into the report. A summary is offered so a host need not enumerate every cell; it would be worth little if enumerating the cells could contradict it. | +| (`test:integration:analysis-set-summary-depth-range-includes-root`) | readout | The reported depth span begins at the root and runs the right way round. Because ancestor closure always terminates at the root, the shallow end of the span is fixed at zero on a live sentinel, and the deep end describes the finest resolution currently being modelled — so the span is the reach of the whole chain, not the band the selected cells occupy. | +| (`test:integration:analysis-set-summary-investment-covers-full`) | readout | The investment set is never smaller than the set currently producing reports. Every cell that reports has a tracker, and some cells hold trackers that are still warming and not yet contributing — so the gap between the two figures is precisely the modelling the sentinel is paying for but not yet reading from. | +| (`test:integration:analysis-width-on-cell-report`) | readout | A cell's reported analysis width is always the domain width less its depth, for competitive and ancestor cells alike. The leading bits that routing already fixed are constant within the cell and carry no information, so the width states exactly how many bits the cell's model had left to work with — which is what a host needs to compare scores from cells at different depths. | +| (`test:integration:analysis-width-on-cell-inspection`) | readout | cites (`claim:readout:a-cells-reported-analysis-width-is-the-domain-width-less-its-depth`) | +| (`test:integration:batch-report-states-the-age-of-its-oldest-observation`) | readout | A report says how old its evidence was at the moment it was emitted: the batch is stamped as it arrives and the figure is read off as the report is assembled, so it is a positive interval that never exceeds the call that produced it. Both ends of the measurement are the sentinel's own monotonic clock, so a host learns the age of what it is holding without either side having to trust the other's idea of the time. | +| (`test:integration:batch-report-age-is-scoped-to-its-own-batch`) | readout | The age belongs to the batch that carried the observations rather than running from the sentinel's own beginning: after a silence, the next batch reports an age shorter than the silence that preceded it. An age that accumulated over uptime would answer how long the sentinel had been running, which is the wrong question — what a host needs is how stale the evidence in front of it is. | +| (`test:integration:empty-ingest-reports-no-observation-age`) | readout | A batch with no observations has no oldest observation, so it reports no age at all rather than an age of nothing. Absence and instantaneity are different facts about a report and a zero would have conflated them: a host watching for stale evidence has to be able to tell "nothing arrived" from "what arrived was fresh". | +| (`test:integration:new-with-test-config`) | width | The narrower alias constructs from an ordinary configuration and comes up tracking the root alone, exactly as the wider one does. Configuration carries no width of its own — width is a property of the type — so the same settings serve either domain. | +| (`test:integration:new-with-cold-config`) | width | cites (`claim:width:the-narrower-alias-constructs-under-any-supported-configuration-and-starts-with-the-root-alone`) | +| (`test:integration:new-with-integration-config`) | width | cites (`claim:width:the-narrower-alias-constructs-under-any-supported-configuration-and-starts-with-the-root-alone`) | +| (`test:integration:initial-state-has-root-only`) | width | cites (`claim:width:the-narrower-alias-constructs-under-any-supported-configuration-and-starts-with-the-root-alone`) | +| (`test:integration:empty-ingest-produces-report`) | edge | cites (`claim:edge:an-empty-batch-yields-a-report-with-no-cells-and-moves-no-counter`) | +| (`test:integration:ingest-returns-non-empty-report`) | engine | cites (`claim:engine:the-root-tracker-receives-every-observation-in-every-batch`) | +| (`test:integration:multiple-ingests-accumulate`) | engine | Ingestion accumulates rather than restarting: after many further batches of the same traffic the sentinel tracks no fewer cells than it did after the first. A batch is an increment to standing state, so cells already earned are not dropped merely because another batch arrived. | +| (`test:integration:analysis-widths-are-64-minus-depth`) | width | Every cell in the report, competitive or ancestor, analyses a width equal to the domain width less its own depth — here the narrower domain's width, at whatever depths the traffic reached. The bits routing has already resolved are constant within the cell and so carry no information for its tracker; what remains is the suffix, and its length is fixed by the depth. The width is read from the sentinel's type parameter rather than assumed, which is what makes the same arithmetic hold for either alias. | +| (`test:integration:cell-reports-are-competitive`) | engine | A report separates the cells that earned their modelling from the ones carried along to complete an ancestor chain, and the first vector holds only the former. The distinction is what tells a reader which measurements reflect a deliberate investment, so it is expressed as two vectors rather than as a flag to be filtered on. | +| (`test:integration:ancestor-reports-are-non-competitive`) | engine | cites (`claim:engine:cell-reports-hold-only-competitive-cells-and-ancestor-reports-only-non-competitive-ones`) | +| (`test:integration:reports-sorted-by-gnode-id`) | determinism | cites (`claim:determinism:every-report-vector-comes-out-in-the-order-its-contract-states-so-a-reader-never-depends-on-visit-order`) | +| (`test:integration:no-nan-in-scores`) | engine | Every reported score on every axis is a real number. The axes are ratios and standardised departures, so a variance that had collapsed to nothing or a basis that spanned no direction would surface as a non-number rather than as an obviously wrong value — which is why the absence of one is worth asserting across all four axes and both report vectors. | +| (`test:integration:creates-cells-on-split`) | engine | cites (`claim:engine:traffic-in-separate-regions-splits-the-domain-so-more-than-the-root-is-tracked`) | +| (`test:integration:cells-tracked-never-below-one`) | engine | The root tracker is permanent, before any traffic and after it. It is not selected on merit and cannot be displaced by the competition, because every ancestor chain has to terminate somewhere — so the tracked count has a floor of one and a host never meets a sentinel with nothing to report against. | +| (`test:integration:min-value`) | edge | cites (`claim:edge:the-extremes-of-the-coordinate-domain-are-ordinary-observations`) | +| (`test:integration:max-value`) | edge | cites (`claim:edge:the-extremes-of-the-coordinate-domain-are-ordinary-observations`) | +| (`test:integration:sentinel-u64-min-and-max-together`) | edge | cites (`claim:edge:the-extremes-of-the-coordinate-domain-are-ordinary-observations`) | +| (`test:integration:deterministic-output-for-same-input`) | determinism | cites (`claim:determinism:the-same-seed-and-the-same-data-reproduce-the-same-reports`) | +| (`test:integration:batch-report-json-round-trip`) | serde | A whole batch report survives being written out and read back with its structure intact: the cell, ancestor and coordination lists return at the same lengths, and the summary counts are unchanged. The nesting is deep and generic over the coordinate type, so this is the claim that the readout can be handed to a host in another process at all. | +| (`test:integration:empty-batch-report-json-round-trip`) | serde | cites (`claim:serde:a-whole-batch-report-survives-a-round-trip-with-its-structure-intact`) | +| (`test:integration:batch-report-without-age-field-deserializes`) | serde | A payload written before the age existed still reads back, its age absent rather than filled in: a report object simply missing that field deserialises with no age and everything else intact. Absence is the honest reading of an older payload — a number would claim a measurement the sender never made and could not have sent. | +| (`test:integration:analysis-set-summary-json-round-trip`) | serde | A report component can be carried on its own, not only inside the batch that produced it: the analysis-set summary alone round-trips with its sizes, its depth span and its skipped-cell count intact. A host forwarding structural state to one consumer and scores to another need not ship the whole readout to either. | +| (`test:integration:cell-report-json-round-trip`) | serde | Optional detail crosses the boundary as present or absent rather than collapsing: each cell report returns with its depth, counts and scores, and with per-sample detail still attached and still the same length when the host enabled it. An option that silently became absent in transit would look to the receiver exactly like a host that had never asked for it. | +| (`test:integration:contour-snapshot-json-round-trip`) | serde | cites (`claim:serde:every-report-component-can-be-carried-on-its-own-not-only-inside-the-batch-that-produced-it`) | +| (`test:integration:coordination-report-json-round-trip`) | serde | cites (`claim:serde:optional-detail-crosses-the-boundary-as-present-or-absent-rather-than-collapsing`) | +| (`test:integration:health-report-json-round-trip`) | serde | cites (`claim:serde:every-report-component-can-be-carried-on-its-own-not-only-inside-the-batch-that-produced-it`) | +| (`test:integration:cell-inspection-json-round-trip`) | serde | A cell inspection, taken by handle between batches rather than emitted by one, transports like any other readout: its depth, width and rank return exact and its per-axis baselines come back intact. Both ways of getting state out of the sentinel are equally publishable, so a host is not forced through the batch path merely to be able to forward what it learned. | +| (`test:integration:member-score-json-round-trip`) | serde | Integers cross exactly and floating-point values within one unit in the last place — the residue of passing through a decimal text form, checked here on a member score whose interval runs to the extreme of the coordinate domain. The cell identity is integral and so is preserved outright, while the scores are close enough that no threshold a host applies can turn on the difference. | +| (`test:integration:sample-score-json-round-trip`) | serde | cites (`claim:serde:a-float-returns-within-one-unit-in-the-last-place-of-the-value-that-was-sent`) | +| (`test:integration:decay-on-empty-graph-is-noop`) | decay | Decay scales what has been accumulated, so on a sentinel that has observed nothing there is nothing to scale and nothing to go wrong. A host can put decay on a timer before any traffic arrives without special-casing the empty graph. | +| (`test:integration:decay-at-attenuation-one-is-noop`) | decay | An attenuation of one leaves a populated graph exactly as it was. The identity is inside the parameter's range rather than outside it, so "decay by nothing this tick" is expressed with the same call as any other policy, and a host's schedule needs no branch around it. | +| (`test:integration:decay-reduces-total-sum`) | decay | A factor below one reduces the importance the graph has accumulated. This is the ordinary case and the reason the operation exists: standing earned by past traffic is worth less after a decay, so cells that have gone quiet drift down the ranking instead of holding their place forever on history. | +| (`test:integration:repeated-decay-eventually-zeroes-integer-counters`) | decay | Importance is held in integers, so repeated halving does not approach zero asymptotically — it arrives there. A long run of decays with no intervening traffic empties the graph completely, which means a region that stops being observed is eventually forgotten outright rather than leaving an ever-smaller residue that still outranks a genuinely new cell. | +| (`test:integration:decay-zero-attenuation-zeroes-graph`) | decay | Zero is the bottom of the attenuation range and reaches in one call what repeated halving reaches slowly: all accumulated importance is gone. It is the operation for declaring the spatial history worthless — after a suspected poisoning, say — without discarding the models that history produced. | +| (`test:integration:amplification-increases-total-sum`) | decay | The factor is not restricted to shrinking. Above one it increases the accumulated importance instead, which lets a host reinforce a subtree it knows to be worth watching. One operation therefore spans both temporal policies — letting the past fade and boosting a region's standing — with the identity sitting between them. | +| (`test:integration:selective-q-preserves-more-coarse-structure`) | decay | Selectivity makes the decay factor depend on a node's depth rather than applying one rate to the whole tree, so fine detail can be released while the coarse division of the domain is held. Two sentinels given identical traffic start from identical importance — the spatial layer is driven by volume alone — and diverge only through the selectivity each is then decayed with; both keep importance standing afterwards. | +| (`test:integration:max-selectivity-q-one-is-valid`) | decay | The selectivity range is closed at its top end: maximum selectivity is a valid request, not one step past the edge, and under an attenuating factor it cannot leave the graph holding more than it started with. The extreme of the parameter is usable rather than merely almost-reachable. | +| (`test:integration:decay-does-not-affect-tracker-count`) | decay | Decay reaches the spatial accounting and stops there. Halving every cell's importance destroys no tracker: the models a sentinel has built are not the same asset as the standing that justified building them, and forgetting the second does not throw away the first. This is the feed-forward invariant seen from the temporal side — importance flows into modelling decisions, never the reverse, so rescaling it cannot reach the models. | +| (`test:integration:decay-does-not-change-lifetime-observations`) | decay | cites (`claim:decay:decay-rescales-spatial-importance-only-and-never-the-models-or-the-record-of-what-was-observed`) | +| (`test:integration:decay-subtree-at-root-matches-global-decay`) | decay | There is one decay operation, not two. Targeting the subtree at the graph root produces exactly the importance a global decay produces, because the global call is defined as the subtree call made at the root. The targeted form is the general one, and the whole-graph form its degenerate case. | +| (`test:integration:decay-subtree-affects-only-targeted-subtree`) | decay | Aimed below the root, decay is bounded by its target: the region named loses standing while everything outside it keeps what it had, so more importance survives than the same factor applied globally. That containment is what makes the operation usable for a regime change in one part of the domain — the rest of the graph does not have to be punished to let one region re-form. | +| (`test:integration:decay-then-ingest-preserves-invariants`) | decay | Decay leaves the sentinel in a state the next batch can be scored from. It changes the rankings the selector reads but invalidates nothing eagerly; the analysis set is simply recomputed at the start of the following ingest, and the report that comes out satisfies every structural invariant. Forgetting is therefore composable with ordinary operation rather than something to be sequenced carefully around it. | +| (`test:integration:decay-panics-on-negative-attenuation`) | decay | A negative attenuation has no meaning — importance cannot be scaled through zero into a negative standing — and the call refuses it outright instead of clamping it to the nearest sensible value. Temporal policy is the host's, so a nonsensical factor is a bug in that policy and is reported as one. | +| (`test:integration:decay-panics-on-nan-attenuation`) | decay | cites (`claim:decay:an-attenuation-outside-the-non-negative-reals-is-refused-rather-than-quietly-repaired`) | +| (`test:integration:decay-panics-on-negative-q`) | decay | Selectivity runs from uniform to fully depth-weighted, and below that range there is nothing to mean. A negative request is refused rather than treated as uniform, because a host that computed it did not intend uniformity. | +| (`test:integration:decay-panics-on-q-above-one`) | decay | cites (`claim:decay:a-selectivity-outside-the-unit-interval-is-refused-rather-than-quietly-repaired`) | +| (`test:integration:decay-panics-on-nan-q`) | decay | cites (`claim:decay:a-selectivity-outside-the-unit-interval-is-refused-rather-than-quietly-repaired`) | +| (`test:integration:competitive-set-bounded-by-k`) | resistance | Spraying traffic across far more leading ranges than the sentinel is permitted to model does not enlarge the set of cells that compete for modelling effort: it stays within the configured cap. The cap is on attention, not on input, so an attacker who can address any part of the domain still cannot make the sentinel promise more work than it budgeted for. | +| (`test:integration:competitive-set-at-k-equals-one`) | resistance | cites (`claim:resistance:a-spray-across-many-ranges-cannot-enlarge-the-competitive-set-beyond-its-cap`) | +| (`test:integration:full-set-bounded-by-steiner`) | resistance | Capping the winners would be hollow if the ancestors pulled in to connect them to the root were unbounded, since each of those also carries a tracker. After a spray across many ranges the materialised set stays within the root plus the cap times the deepest level reached — the connecting chains are shared and counted, so the total cost of attention is a function of the cap and the depth alone, never of how many ranges were touched. | +| (`test:integration:g-nodes-bounded-by-budget-under-spray`) | resistance | Feeding a long run of one-value batches, each a distinct coordinate spread over the ranges, leaves the tree inside its node budget rather than growing a node per distinct value. The bound asserted is the budget itself, which is the figure the structure's own invariant refuses to exceed — a guard at twice it would let the test pass through states the structure calls violations. Memory is the resource an attacker would most like to exhaust, so the budget is enforced by eviction as the tree grows and is not merely a hint the structure is asked to respect. | +| (`test:integration:cells-tracked-bounded-under-diverse-traffic`) | resistance | Sustained traffic to every leading range at once leaves the number of live trackers bounded by roughly twice the competitive cap. Trackers are the expensive objects — each carries a learned subspace and its baselines — so what bounds them is the cap on attention rather than the diversity of the traffic. Diverse traffic that is not adversarial is held to the same bound as a spray, because the sentinel does not need to tell them apart to stay within budget. | +| (`test:integration:concentrated-range-survives-spray`) | resistance | A range carrying the great bulk of the traffic is still represented in the reports after a thin spray touches every other range — as a competitor in its own right or through an ancestor below the root that covers it. The root does not count towards that: it contains every coordinate and receives every batch, so a reading that accepted it would be satisfied by a report in which the concentrated range had lost every cell of its own. Attention is bought with accumulated weight rather than with novelty, which is what stops a cheap spray from evicting the model of the range an operator actually cares about. | +| (`test:integration:invariants-hold-under-spray`) | resistance | The structural guarantees are checked after every single batch of a wide spray and again through the concentrated burst that follows it, and none of them breaks. The bounds, the ordering of the reports and the presence of the root are not properties of a settled sentinel: they hold batch by batch while the tree is being churned by hostile traffic and while it is reconverging afterwards, which is the only time they matter. | +| (`test:integration:root-suffix-width-is-full-bit-width`) | suffix | cites (`claim:suffix:a-cells-analysis-width-is-the-domain-width-less-its-depth`) | +| (`test:integration:cell-analysis-width-equals-n-minus-depth`) | suffix | Across every cell a graph under traffic has produced, the width a cell analyses is exactly the domain width less its depth. The leading bits its depth stands for were fixed by routing and are identical for every value that reaches it, so modelling them would add a constant column and no information; the width is a consequence of position rather than a per-cell setting anyone can get wrong. | +| (`test:integration:geometry-dim-equals-analysis-width`) | suffix | The space a cell's model works in is its own suffix, not the domain: the dimension reported with its scoring geometry is the cell's analysis width at every depth the tree reaches. The two are not independently maintained numbers that happen to agree — the tracker is constructed at the cell's width — so a host reading the geometry is reading the same fact as one reading the width. | +| (`test:integration:geometry-cap-is-min-of-dim-and-max-rank`) | suffix | How much structure a cell may learn is limited by its own suffix as well as by configuration: the ceiling is whichever of the two is smaller. A model cannot hold more directions than the space it lives in has, so a cell deep enough to be narrower than the configured maximum is capped by its depth instead — the configured maximum is a budget, never a promise of capacity. | +| (`test:integration:residual-dof-equals-dim-minus-rank`) | suffix | What is left over for a cell to be surprised by is its width less the structure it has already learned. Those residual directions are the room in which an unexplained departure can register at all, so the same absolute departure means more in a narrow cell than in a wide one — and a cell whose model has grown to fill its width has no room left, which is precisely the degenerate case the geometry lets a host detect rather than hiding. | +| (`test:integration:deeper-cells-have-smaller-suffix-width`) | suffix | cites (`claim:suffix:a-cells-analysis-width-is-the-domain-width-less-its-depth`) | +| (`test:integration:cell-reports-carry-correct-suffix-widths`) | suffix | The width a cell analysed travels out with its numbers: each entry in a batch report states the width it was computed at, and that width still agrees with the cell's depth and with the geometry the scores came from. Scores from different depths are not commensurable, so a report that carried only the numbers would invite a host to compare them as though they were. | +| (`test:integration:scores-are-valid-across-suffix-widths`) | suffix | Every score axis yields a real number at every width the tree produced, across a graph warmed on separated ranges and then driven into one of them. Narrow geometries are where the divisions in the scoring arithmetic come closest to degenerating, so this is the property that lets a host treat a report as data rather than checking each figure for a non-number first. | +| (`test:integration:noise-injected-at-every-suffix-width`) | suffix | No cell begins scoring cold. Every cell the graph created under traffic has synthetic observations behind it, generated at that cell's own width, so the warm-up schedule reaches cells born deep in the tree and not only the root it started from. A cell that had never seen anything would find its first real batch infinitely surprising, and the sentinel would report the arrival of a new region as an anomaly in it. | +| (`test:integration:stage1-only-root-tracker`) | warmup | Traffic that has not yet reached the split threshold leaves the domain undivided: one cell, the root, and nothing competing for investment. Structure is bought with observation volume, so a sentinel that has seen only a handful of values has bought none of it yet. | +| (`test:integration:stage1-no-coordination`) | warmup | The coordination tier compares cells against each other, so it has nothing to say while there is only one. A pre-split batch therefore reports no coordination at all rather than a degenerate context over a single member: a context fires only where both sides of a G-node contribute cells. | +| (`test:integration:stage1-lifetime-observations-counted`) | warmup | The lifetime observation count starts at nothing and accrues one unit per input value, batch after batch, from the very first ingest. It is a record of what the host has fed in rather than a measure of what the sentinel has made of it, so it runs well before any structure exists to attribute it to. | +| (`test:integration:stage1-invariants-hold`) | warmup | The structural guarantees a report makes — the competitive cap, the root's presence in the full set, the separation of competitive from ancestor entries, sorted output, finite scores, widths matching depth — are not promises about steady state. They hold from the first batch, when the sentinel is a single cell and has almost nothing to report. | +| (`test:integration:stage2-cells-tracked-increases`) | warmup | Once a range has taken more traffic than the split threshold allows, the spatial layer divides it, and the newly exposed cells are picked up by the analysis set and given trackers of their own. Modelling effort follows the structure the traffic created rather than a shape chosen in advance. | +| (`test:integration:stage2-competitive-cells-appear`) | warmup | Having a tracker and being competitive are separate things, and the second arrives later. After a run of batches has given some cells enough accumulated importance to win the ranking, the competitive set becomes non-empty — so the transition out of the pre-split state is earned by volume, not conferred at creation. | +| (`test:integration:stage2-new-cells-have-high-noise-influence`) | warmup | A cell that has just come into service is mostly synthetic. Its tracker was seeded so that it could score at all, and until real batches have arrived to displace that seed, the maturity figure it publishes stays high. Every non-root cell with few real observations behind it says so, which lets a host discount a young cell's scores instead of trusting them equally. | +| (`test:integration:stage2-invariants-hold`) | warmup | cites (`claim:warmup:the-report-invariants-hold-at-every-stage-of-warm-up-not-merely-once-it-has-settled`) | +| (`test:integration:stage3-competitive-set-size-stabilises`) | warmup | Once the traffic pattern stops changing, the competitive set stops changing with it: repeated batches over the same ranges leave the number of selected cells varying only within a narrow band. Selection is recomputed from scratch on every batch, so stability here is a property of the ranking rather than of any memory the selector keeps. | +| (`test:integration:stage3-coordination-activates`) | warmup | cites (`claim:warmup:coordination-fires-only-where-two-subtrees-both-contribute-cells`) | +| (`test:integration:stage3-invariants-hold-throughout`) | warmup | cites (`claim:warmup:the-report-invariants-hold-at-every-stage-of-warm-up-not-merely-once-it-has-settled`) | +| (`test:integration:stage4-root-maturity-below-half`) | warmup | Real data displaces the synthetic seed geometrically: each real batch multiplies the synthetic share of a tracker's memory by the forgetting factor raised to the batch size. A modest run of warm-up batches is therefore enough to push the root well past the halfway mark, and the rate is a property of the configured forgetting factor rather than of the data. | +| (`test:integration:stage4-maturity-decreases-monotonically`) | warmup | Maturity only ever improves while real data is arriving: batch after batch, a cell's synthetic share is multiplied down and never rises again. It could rise only if further noise were injected, and nothing injects noise into a cell already in service. A host can therefore read the figure as a one-way progress measure rather than as something that might rebound. | +| (`test:integration:stage4-health-maturity-distribution`) | warmup | The health snapshot aggregates what individual cells know about their own maturity, and in steady state it says two things: the average tracker is no longer purely synthetic, and no tracker anywhere is still cold — every cell in service has seen real data. A cold entry in a warmed sentinel would mean a cell was being scored on noise alone. | +| (`test:integration:cold-start-noise-influence-is-one`) | warmup | Turning the noise schedule off shows what the seed was doing. The root then begins at a synthetic share of exactly one — the value a tracker with no information at all reports — and only real data moves it. Warm-up is thus an optional head start, not a precondition: the sentinel still runs without it, and simply says that everything it knows is unearned. | \ No newline at end of file diff --git a/packages/sentinel/tests/ancestor_chain.rs b/packages/sentinel/tests/ancestor_chain.rs new file mode 100644 index 000000000..bb0f1c590 --- /dev/null +++ b/packages/sentinel/tests/ancestor_chain.rs @@ -0,0 +1,414 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +//! The ancestor chain — what the models above a cell are for +//! (§ALGO S-16). +//! +//! Selecting a cell for analysis is never enough on its own. Every +//! competitively chosen cell is closed under G-tree ancestry, so the sentinel +//! also models each of its parents up to the root, and an observation is +//! delivered to every cell whose interval contains it rather than only to the +//! finest one. One arrival is therefore analysed several times over, once at +//! each scale it belongs to, and a report shows both the cells that earned +//! their place and the ancestors that were drawn in behind them. +//! +//! That redundancy is the point. A cell sees only its own narrow range and +//! cannot tell a change confined to it apart from a change happening +//! everywhere; an ancestor sees the union of its descendants and cannot +//! localise anything, but it can tell those two situations apart. Read +//! together, the chain gives the shape of a disturbance and not just its +//! presence: something that lifts one cell's scores while the root stays +//! ordinary is local, and something that lifts the root itself is not. +//! +//! The chain has to be intact for that reading to hold, which is what the +//! structural properties here pin down — the root always present, the depths +//! between it and the deepest selected cell not skipped, an ancestor's +//! analysis view never narrower than that of the cells beneath it, and an +//! ancestor's sample count accounting for every observation its descendants +//! saw. Where no observations arrive at all, no ancestor reports: the chain +//! describes traffic, and reports nothing when there is none. +//! +//! # Test index +//! +//! | Test | Area | Claim | +//! |------|------|-------| +//! | [`empty_batch_produces_no_ancestor_reports`] | ancestry | A batch with no observations produces no ancestor reports, even from a sentinel already warmed and holding a chain of live models. Cells report what they saw, and a cell that saw nothing has nothing to say; the chain is a description of traffic rather than a periodic status broadcast, so a quiet interval costs a host no reports to filter out. | +//! | [`root_ancestor_present_after_warm_up`] | ancestry | Traffic into a refined region is reported at both ends of the chain at once: the deep cells that earned selection appear as competitive, and the root appears alongside them as an ancestor. The same observations were analysed at both scales, which is what makes the two figures comparable in the first place — without the coarse reading there is nothing to judge the fine one against. | +//! | [`root_sample_count_equals_batch_size`] | ancestry | An ancestor is credited with every observation that fell anywhere beneath it: the root's sample count for a batch drawn from two separate ranges is the whole batch. Delivery is by containment rather than by ownership, so nothing is consumed by the deepest cell that matched — this is what lets a coarse model hold a baseline for total volume that no individual cell could. | +//! | [`ancestor_width_increases_toward_root`] | ancestry | Ordered by depth, the ancestors reported for a batch never narrow as one climbs toward the root: an ancestor analyses at least as wide a view as the cells below it. Ancestry and analysis breadth therefore point the same way, so a chain reads as a genuine sequence of scales — coarse above, specific below — rather than an arbitrary collection of models over the same region. | +//! | [`ancestor_depths_cover_path_to_root`] | ancestry | The chain has no holes in it. The root is always reported, and where the selected cells sit well below it the intervening depths are present too — some as competitive cells in their own right, the rest drawn in by the closure. Reading the two report lists together therefore gives an unbroken path from the whole domain down to the finest cell, which is what allows a disturbance to be located at a scale rather than merely noticed at one. | +//! | [`shared_ancestor_aggregates_disjoint_ranges`] | ancestry | cites (´claim:ancestry:an-ancestor-is-credited-with-every-observation-that-fell-beneath-it´) | +//! | [`local_anomaly_detectable_in_hierarchy`] | ancestry | Structurally novel traffic entering one range while the others stay normal leaves a mark somewhere in the chain, measured against the same sentinel's own response to an ordinary batch a moment earlier. Which level catches it is not fixed — a narrow disturbance may barely move the root while standing out sharply in the cell containing it — so detection is a property of the chain as a whole rather than of any one model in it. | +//! | [`global_anomaly_elevates_root_scores`] | ancestry | When every range turns anomalous at once, the root's own score rises above what the same sentinel produced for a normal batch. The coarse model is not merely a fallback for traffic too sparse to have earned a cell: it responds in its own right, and it responds to exactly the case no single cell can distinguish from its own local weather. | +//! | [`global_anomaly_root_z_exceeds_local`] | ancestry | The root does not merely notice anomalies, it grades them by extent: with several ranges live, an anomaly in one of them moves the root less than the same anomaly in all of them, the sentinel having been returned to normal traffic in between so the two readings are of comparable states. Reach is thus legible in the score itself, and a host can separate a local incident from a system-wide shift without waiting to see how far it spreads. | + +mod common; + +use common::{ScenarioBuilder, anomalous_values, assert_invariants, batches_to_maturity, cell_values, integration_config}; +use torrust_sentinel::{Sentinel128, SentinelConfig}; + +// ── Helpers ───────────────────────────────────────────────── + +/// Warmed sentinel with a single range and low split threshold. +fn single_range_sentinel() -> Sentinel128 { + ScenarioBuilder::new() + .config(SentinelConfig:: { + split_threshold: 10, + ..integration_config() + }) + .seed_range(0xA, 20) + .warm_batches(5) + .build() +} + +/// Warmed sentinel with two disjoint ranges. +fn dual_range_sentinel() -> Sentinel128 { + ScenarioBuilder::new() + .config(SentinelConfig:: { + split_threshold: 10, + ..integration_config() + }) + .seed_range(0xA, 20) + .seed_range(0xB, 20) + .warm_batches(5) + .build() +} + +/// Warmed sentinel with three disjoint ranges. +fn triple_range_sentinel() -> Sentinel128 { + ScenarioBuilder::new() + .config(SentinelConfig:: { + split_threshold: 10, + ..integration_config() + }) + .seed_range(0xA, 20) + .seed_range(0xB, 20) + .seed_range(0xC, 20) + .warm_batches(5) + .build() +} + +/// Extract root z-score from a report, defaulting to `0.0`. +fn root_novelty_z(report: &torrust_sentinel::BatchReport) -> f64 { + report + .ancestor_reports + .iter() + .find(|r| r.depth == 0) + .map_or(0.0, |r| r.scores.novelty.max_z_score) +} + +// ── 1. empty_batch_produces_no_ancestor_reports ───────────── + +/// A batch with no observations produces no ancestor reports, even from a +/// sentinel already warmed and holding a chain of live models. Cells report +/// what they saw, and a cell that saw nothing has nothing to say; the chain is +/// a description of traffic rather than a periodic status broadcast, so a +/// quiet interval costs a host no reports to filter out. +/// +/// ´claim:ancestry:a-batch-with-no-observations-produces-no-ancestor-reports-at-all´ +/// ´test:integration:empty-batch-produces-no-ancestor-reports´ +#[test] +fn empty_batch_produces_no_ancestor_reports() { + let mut s = single_range_sentinel(); + let report = s.ingest(&[]); + + assert!(report.ancestor_reports.is_empty()); + assert_invariants(&s, &report); +} + +// ── 2. root_ancestor_present_after_warm_up ────────────────── + +/// Traffic into a refined region is reported at both ends of the chain at +/// once: the deep cells that earned selection appear as competitive, and the +/// root appears alongside them as an ancestor. The same observations were +/// analysed at both scales, which is what makes the two figures comparable in +/// the first place — without the coarse reading there is nothing to judge the +/// fine one against. +/// +/// ´claim:ancestry:one-batch-is-reported-at-the-fine-scale-and-the-coarse-scale-together´ +/// ´test:integration:root-ancestor-present-after-warm-up´ +#[test] +fn root_ancestor_present_after_warm_up() { + let mut s = single_range_sentinel(); + let report = s.ingest(&cell_values(0xA, 8)); + + let root = report.ancestor_reports.iter().find(|r| r.depth == 0); + assert!(root.is_some(), "root tracker must produce a report"); + + // There should also be competitive cells at depth > 0 (the split + // threshold is low enough to create them). + assert!( + report.cell_reports.iter().any(|r| r.depth > 0), + "should have competitive cells at depth > 0", + ); + assert_invariants(&s, &report); +} + +// ── 3. root_sample_count_equals_batch_size ────────────────── + +/// An ancestor is credited with every observation that fell anywhere beneath +/// it: the root's sample count for a batch drawn from two separate ranges is +/// the whole batch. Delivery is by containment rather than by ownership, so +/// nothing is consumed by the deepest cell that matched — this is what lets a +/// coarse model hold a baseline for total volume that no individual cell +/// could. +/// +/// ´claim:ancestry:an-ancestor-is-credited-with-every-observation-that-fell-beneath-it´ +/// ´test:integration:root-sample-count-equals-batch-size´ +#[test] +fn root_sample_count_equals_batch_size() { + let mut s = dual_range_sentinel(); + + let batch = [cell_values(0xA, 5), cell_values(0xB, 7)].concat(); + let report = s.ingest(&batch); + + let root = report.ancestor_reports.iter().find(|r| r.depth == 0).unwrap(); + assert_eq!(root.sample_count, batch.len(), "root must see every observation"); + assert_invariants(&s, &report); +} + +// ── 4. ancestor_width_increases_toward_root ───────────────── + +/// Ordered by depth, the ancestors reported for a batch never narrow as one +/// climbs toward the root: an ancestor analyses at least as wide a view as the +/// cells below it. Ancestry and analysis breadth therefore point the same way, +/// so a chain reads as a genuine sequence of scales — coarse above, specific +/// below — rather than an arbitrary collection of models over the same region. +/// +/// ´claim:ancestry:an-ancestor-analyses-at-least-as-wide-a-view-as-the-cells-beneath-it´ +/// ´test:integration:ancestor-width-increases-toward-root´ +#[test] +fn ancestor_width_increases_toward_root() { + let mut s = single_range_sentinel(); + let report = s.ingest(&cell_values(0xA, 8)); + + // Sort ancestors by depth ascending; width should be non-increasing + // (width = 128 − depth, so shallower ⇒ wider). + let mut ancestors: Vec<_> = report.ancestor_reports.iter().collect(); + ancestors.sort_by_key(|r| r.depth); + + for window in ancestors.windows(2) { + assert!( + window[0].analysis_width >= window[1].analysis_width, + "ancestor at depth {} (width {}) should be >= depth {} (width {})", + window[0].depth, + window[0].analysis_width, + window[1].depth, + window[1].analysis_width, + ); + } + assert_invariants(&s, &report); +} + +// ── 5. ancestor_depths_cover_path_to_root ─────────────────── + +/// The chain has no holes in it. The root is always reported, and where the +/// selected cells sit well below it the intervening depths are present too — +/// some as competitive cells in their own right, the rest drawn in by the +/// closure. Reading the two report lists together therefore gives an unbroken +/// path from the whole domain down to the finest cell, which is what allows a +/// disturbance to be located at a scale rather than merely noticed at one. +/// +/// ´claim:ancestry:the-reported-chain-runs-unbroken-from-the-root-to-the-deepest-selected-cell´ +/// ´test:integration:ancestor-depths-cover-path-to-root´ +#[test] +fn ancestor_depths_cover_path_to_root() { + let mut s = single_range_sentinel(); + let report = s.ingest(&cell_values(0xA, 8)); + + // Root (depth 0) must always be present as an ancestor. + assert!( + report.ancestor_reports.iter().any(|a| a.depth == 0), + "root ancestor must be present", + ); + + // Collect depths from both competitive and ancestor reports. + // Intermediate nodes between root and deep cells may be competitive + // (in cell_reports) rather than in ancestor_reports. + let all_depths: std::collections::BTreeSet = report + .cell_reports + .iter() + .chain(report.ancestor_reports.iter()) + .map(|r| r.depth) + .collect(); + + // The combined depths should span from 0 up to the deepest + // competitive cell without large gaps — the Steiner tree + // connects them through the hierarchy. + let max_depth = report.cell_reports.iter().map(|c| c.depth).max().unwrap_or(0); + if max_depth > 1 { + assert!( + all_depths.len() > 2, + "depths 0..{max_depth} should include intermediate nodes, got {all_depths:?}", + ); + } + assert_invariants(&s, &report); +} + +// ── 6. shared_ancestor_aggregates_disjoint_ranges ─────────── + +/// The same crediting rule is what makes a shared ancestor a meeting point: +/// two ranges with nothing in common still both lie beneath the root, and its +/// count for the batch is their sum. Traffic that no single cell can see as +/// related is nonetheless seen together somewhere in the chain, which is the +/// mechanism by which a distributed pattern becomes visible at all. +/// +/// (´claim:ancestry:an-ancestor-is-credited-with-every-observation-that-fell-beneath-it´) +/// ´test:integration:shared-ancestor-aggregates-disjoint-ranges´ +#[test] +fn shared_ancestor_aggregates_disjoint_ranges() { + let mut s = dual_range_sentinel(); + + let batch_a = cell_values(0xA, 4); + let batch_b = cell_values(0xB, 6); + let combined = [batch_a, batch_b].concat(); + let report = s.ingest(&combined); + + let root = report.ancestor_reports.iter().find(|r| r.depth == 0).unwrap(); + assert_eq!(root.sample_count, 10, "root should aggregate traffic from both ranges"); + assert_invariants(&s, &report); +} + +// ── 7. local_anomaly_detectable_in_hierarchy ──────────────── + +/// Structurally novel traffic entering one range while the others stay normal +/// leaves a mark somewhere in the chain, measured against the same sentinel's +/// own response to an ordinary batch a moment earlier. Which level catches it +/// is not fixed — a narrow disturbance may barely move the root while standing +/// out sharply in the cell containing it — so detection is a property of the +/// chain as a whole rather than of any one model in it. +/// +/// ´claim:ancestry:an-anomaly-confined-to-one-range-still-registers-somewhere-in-the-chain´ +/// ´test:integration:local-anomaly-detectable-in-hierarchy´ +#[test] +fn local_anomaly_detectable_in_hierarchy() { + let mut s = triple_range_sentinel(); + + // Normal reference: root z-score under normal traffic. + let normal = s.ingest(&[cell_values(0xA, 8), cell_values(0xB, 8), cell_values(0xC, 8)].concat()); + let normal_root_z = root_novelty_z(&normal); + assert_invariants(&s, &normal); + + // Anomaly only in range A; B and C normal. + let report = s.ingest(&[anomalous_values(0xA, 8), cell_values(0xB, 8), cell_values(0xC, 8)].concat()); + + let anomaly_root_z = root_novelty_z(&report); + + let max_all_z = report + .cell_reports + .iter() + .chain(report.ancestor_reports.iter()) + .map(|cr| cr.scores.novelty.max_z_score) + .fold(f64::NEG_INFINITY, f64::max); + + assert!( + max_all_z > normal_root_z || anomaly_root_z > normal_root_z, + "local anomaly should be detectable at some hierarchical level: \ + max_z={max_all_z:.4}, anomaly_root_z={anomaly_root_z:.4}, \ + normal_root_z={normal_root_z:.4}", + ); + assert_invariants(&s, &report); +} + +// ── 8. global_anomaly_elevates_root_scores ────────────────── + +/// When every range turns anomalous at once, the root's own score rises above +/// what the same sentinel produced for a normal batch. The coarse model is not +/// merely a fallback for traffic too sparse to have earned a cell: it responds +/// in its own right, and it responds to exactly the case no single cell can +/// distinguish from its own local weather. +/// +/// ´claim:ancestry:an-anomaly-in-every-range-lifts-the-roots-own-score´ +/// ´test:integration:global-anomaly-elevates-root-scores´ +#[test] +fn global_anomaly_elevates_root_scores() { + let mut s = dual_range_sentinel(); + + // Normal reference. + let normal = s.ingest(&[cell_values(0xA, 8), cell_values(0xB, 8)].concat()); + let normal_root_z = root_novelty_z(&normal); + assert_invariants(&s, &normal); + + // System-wide anomaly. + let anomaly = s.ingest(&[anomalous_values(0xA, 8), anomalous_values(0xB, 8)].concat()); + let anomaly_root_z = root_novelty_z(&anomaly); + + assert!( + anomaly_root_z > normal_root_z, + "global anomaly should elevate root z-score: \ + anomaly={anomaly_root_z:.4}, normal={normal_root_z:.4}", + ); + assert_invariants(&s, &anomaly); +} + +// ── 9. global_anomaly_root_z_exceeds_local ────────────────── + +/// The root does not merely notice anomalies, it grades them by extent: with +/// several ranges live, an anomaly in one of them moves the root less than the +/// same anomaly in all of them, the sentinel having been returned to normal +/// traffic in between so the two readings are of comparable states. Reach is +/// thus legible in the score itself, and a host can separate a local incident +/// from a system-wide shift without waiting to see how far it spreads. +/// +/// ´claim:ancestry:the-root-grades-an-anomaly-by-how-much-of-the-domain-it-reaches´ +/// ´test:integration:global-anomaly-root-z-exceeds-local´ +#[test] +fn global_anomaly_root_z_exceeds_local() { + let config = SentinelConfig:: { + split_threshold: 10, + ..integration_config() + }; + let warm_batches = batches_to_maturity(config.forgetting_factor); + let mut s = ScenarioBuilder::new() + .config(config) + .seed_range(0xA, 20) + .seed_range(0xB, 20) + .seed_range(0xC, 20) + .seed_range(0xD, 20) + .warm_batches(warm_batches) + .build(); + + // Localised anomaly: only range A. + let local_report = s.ingest( + &[ + anomalous_values(0xA, 8), + cell_values(0xB, 8), + cell_values(0xC, 8), + cell_values(0xD, 8), + ] + .concat(), + ); + let local_root_z = root_novelty_z(&local_report); + assert_invariants(&s, &local_report); + + // Return to normal, then system-wide anomaly. + for _ in 0..3 { + s.ingest( + &[ + cell_values(0xA, 8), + cell_values(0xB, 8), + cell_values(0xC, 8), + cell_values(0xD, 8), + ] + .concat(), + ); + } + + let global_report = s.ingest( + &[ + anomalous_values(0xA, 8), + anomalous_values(0xB, 8), + anomalous_values(0xC, 8), + anomalous_values(0xD, 8), + ] + .concat(), + ); + let global_root_z = root_novelty_z(&global_report); + + assert!( + global_root_z > local_root_z, + "system-wide anomaly root z ({global_root_z:.4}) should exceed \ + localised anomaly root z ({local_root_z:.4})", + ); + assert_invariants(&s, &global_report); +} diff --git a/packages/sentinel/tests/api.rs b/packages/sentinel/tests/api.rs new file mode 100644 index 000000000..3c7d43d75 --- /dev/null +++ b/packages/sentinel/tests/api.rs @@ -0,0 +1,497 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +//! # Test index +//! +//! | Test | Area | Claim | +//! |------|------|-------| +//! | [`new_validates_config`] | engine | A configuration that could not produce a working model is rejected at construction rather than carried into the run: a rank budget of zero leaves the subspace tracker nothing to hold, and the constructor returns an error instead of a sentinel. Validation happens once, so every later method may assume its parameters are coherent. | +//! | [`new_with_default_config_succeeds`] | engine | The configuration the crate ships as its default passes its own validation, so a host that supplies nothing of its own starts from a usable engine rather than from an error. Only validation is exercised here: the default noise schedule warms the root through many rounds, and paying for that construction adds nothing the rest of the suite does not already pay for. | +//! | [`new_starts_with_root_cell`] | engine | A newly constructed sentinel holds exactly one tracker — the root — and has observed nothing. Cells below the root are created only when traffic justifies them, so construction commits to the single model that is structurally obligatory and to no other investment. | +//! | [`empty_ingest_returns_empty_report`] | edge | A batch with no values in it is answered with a report that names no cells and no cross-cell contexts, and the observation counter does not move. The health section is still filled in — the root tracker is alive whether or not anything arrived — so an idle interval reads as an engine with nothing to say rather than as a gap in the record. | +//! | [`single_value_ingest`] | engine | Every value is delivered to the root as well as to whatever cell it routes into, so even a lone observation produces an ancestor report and advances the lifetime count. The root is the one cell guaranteed to contain any coordinate, which is why a batch can never be scored against nothing. | +//! | [`batch_counter_increments`] | engine | The lifetime counter accumulates real observations rather than batches: after a single value and then a batch of several it stands at their sum. Batch boundaries are a delivery convenience for the host and carry no weight in the record of what was actually seen. | +//! | [`repeated_ingest_does_not_panic`] | engine | Feeding the same traffic round after round leaves every structural invariant standing at each step: the reported cells stay within the competitive budget, the two report vectors stay partitioned and ordered, widths still match depths, and no score turns into a non-number. The engine is a steady-state machine, so repetition is the ordinary case and not a stress case. | +//! | [`config_accessor_returns_construction_config`] | engine | The configuration reads back exactly as it was handed in — rank budget, competitive budget and forgetting factor all unchanged. Construction validates the parameters but does not silently normalise or substitute them, so a host can trust the accessor as the authority on how this sentinel is behaving. | +//! | [`graph_accessor_starts_with_single_root`] | engine | The spatial substrate underneath a fresh sentinel is a bare root with no accumulated importance at all. Construction does not seed the graph with anything, so the first real batch is also the first thing the spatial layer has ever ranked — there is no synthetic history for the selector to mistake for traffic. | +//! | [`analysis_set_accessible`] | engine | The set of cells the sentinel is investing in is readable from outside, and after traffic it holds at least the root. The root's membership is unconditional — it is what every ancestor chain terminates at — so the set is never empty and a host inspecting it never has to handle the no-cells case. | +//! | [`cell_gnodes_returns_all_tracked`] | engine | The list of cell handles and the count of tracked cells are two views of one map, never two records that could disagree. A host can enumerate the handles and know it has enumerated everything the engine is modelling. | +//! | [`cells_tracked_matches_cell_gnodes_len`] | engine | cites (´claim:engine:the-listed-cell-handles-and-the-tracked-count-are-two-views-of-one-map´) | +//! | [`lifetime_observations_reflects_real_input`] | engine | cites (´claim:engine:the-lifetime-counter-accumulates-observations-not-batches´) | +//! | [`degenerate_cells_skipped_starts_at_zero`] | engine | A cell whose suffix is too narrow to support a subspace model is skipped and counted rather than modelled, and on a sentinel that has observed nothing that count is zero. The counter is therefore a record of something that happened, not a constant the engine carries around — a non-zero reading always means real traffic drove the domain that deep. | +//! | [`health_accessible_on_fresh_sentinel`] | engine | A health snapshot can be taken before any observation arrives, and it describes the engine truthfully at that moment: one live tracker, nothing observed. Health is a readout of present state rather than a summary accumulated during ingestion, so a host may poll it on a schedule of its own without having to feed the engine first. | +//! | [`inspect_cell_returns_state_for_root`] | engine | A tracked cell can be inspected individually, and what comes back describes that cell: its depth in the routing tree, the width its tracker analyses, and the rank of the subspace it has learned. The root reports depth zero and the full domain width because nothing has been resolved above it, and its rank is at least one from the moment it exists, since a tracker with no direction at all could produce no residual. | +//! | [`inspect_cell_returns_none_for_unknown_gnode`] | engine | A cell handle is meaningful only to the sentinel that issued it. Handed a handle minted by a different sentinel, inspection returns nothing rather than the state of whichever local cell happens to sit at that index — the lookup is a membership question, so a stale or foreign handle is an absence and never a plausible-looking wrong answer. | +//! | [`per_sample_scores_present_when_enabled`] | engine | Scores for individual values are attached to a cell's report only when the host asked for them, and then there is exactly one entry per observation in the batch. Per-sample detail costs memory proportional to the traffic, so it is opt-in rather than always paid for, and the one-to-one correspondence is what makes an entry attributable back to the value that produced it. | +//! | [`per_sample_scores_absent_when_disabled`] | engine | cites (´claim:engine:per-sample-scores-appear-only-when-asked-for-and-then-carry-one-entry-per-observation´) | +//! | [`reset_restores_initial_state`] | engine | Reset returns a used sentinel to the state it was constructed in: the spatial graph is rebuilt as a bare root, the observation counter is zero, and the root tracker alone is tracked. Learned structure is dropped wholesale rather than aged out, because reset exists for the case where the host knows the past no longer describes the future. The configuration is not part of what is cleared. | +//! | [`centred_bits_built_from_raw_values`] | bits | The bit-vector type the conversion trait returns can be built from outside the crate, and what a caller builds is the same thing the crate's own conversion produces. The trait is published and open to a coordinate type the crate has never heard of, so an implementation of it has to be able to produce the value it is required to return: a type whose centred form is computed rather than shifted out of an integer has nothing here to delegate to, and without a constructor its implementation could not be written at all. | +//! | [`centred_bits_refuses_a_length_past_the_array`] | bits | A length past the backing array is refused rather than clamped. The array is a hundred and twenty-eight slots and nothing wider can be represented, so an implementation asking for more has miscomputed its own width; handing back a shorter vector would let that mistake travel into the tracker as an observation narrower than the one its author believed it built. | + +//! Contract tests for the sentinel's public operational surface — the +//! handful of methods a host actually calls: construct it, feed it batches, +//! read what it has measured, and reset it. +//! +//! That surface is deliberately narrow. The configuration is validated once, +//! at construction, so a sentinel that exists at all is one whose parameters +//! were coherent; nothing afterwards can put it into a state its config +//! forbade. Everything after construction either observes or reports. The +//! accessors are read-only views onto state the engine already holds — the +//! tracked cells, the spatial substrate, the analysis set, the health +//! snapshot — rather than computations a caller can perturb by asking. What +//! comes back is measurement: counts, scores, distributions, geometry, with +//! no verdict attached, because the sentinel measures and the host decides. +//! +//! Two structures anchor the whole surface. The root tracker is permanent: +//! it exists from construction, receives every observation as an ancestor of +//! whatever cell the value routed to, and is recreated by reset — so there +//! is always something to report against, and the tracked count never falls +//! to zero. And reset is a return to the freshly-constructed state rather +//! than a partial clearing: trackers, spatial graph and counters all go +//! back, while the configuration the sentinel was built with stays. + +mod common; + +use common::{assert_invariants, cell_values, seeded_sentinel, test_config}; +use torrust_sentinel::{CentredBitSource, CentredBits, Sentinel128, SentinelConfig}; + +// ═══════════════════════════════════════════════════════════ +// Construction +// ═══════════════════════════════════════════════════════════ + +/// A configuration that could not produce a working model is rejected at +/// construction rather than carried into the run: a rank budget of zero +/// leaves the subspace tracker nothing to hold, and the constructor returns +/// an error instead of a sentinel. Validation happens once, so every later +/// method may assume its parameters are coherent. +/// +/// ´claim:engine:construction-refuses-an-incoherent-configuration-instead-of-running-with-it´ +/// ´test:integration:new-validates-config´ +#[test] +fn new_validates_config() { + let bad = SentinelConfig:: { + max_rank: 0, + ..test_config() + }; + assert!(Sentinel128::new(bad).is_err()); +} + +/// The configuration the crate ships as its default passes its own +/// validation, so a host that supplies nothing of its own starts from a +/// usable engine rather than from an error. Only validation is exercised +/// here: the default noise schedule warms the root through many rounds, and +/// paying for that construction adds nothing the rest of the suite does not +/// already pay for. +/// +/// ´claim:engine:the-default-configuration-passes-its-own-validation´ +/// ´test:integration:new-with-default-config-succeeds´ +#[test] +fn new_with_default_config_succeeds() { + // Full construction with the default noise schedule (450 root rounds) + // takes several seconds. Validate instead — construction is tested + // end-to-end by every integration test that calls `Sentinel128::new()`. + let cfg = SentinelConfig::::default(); + assert!(cfg.validate().is_ok()); +} + +/// A newly constructed sentinel holds exactly one tracker — the root — and +/// has observed nothing. Cells below the root are created only when traffic +/// justifies them, so construction commits to the single model that is +/// structurally obligatory and to no other investment. +/// +/// ´claim:engine:a-fresh-sentinel-holds-the-root-tracker-alone-and-has-observed-nothing´ +/// ´test:integration:new-starts-with-root-cell´ +#[test] +fn new_starts_with_root_cell() { + let s = Sentinel128::new(test_config()).unwrap(); + assert_eq!(s.cells_tracked(), 1); + assert_eq!(s.lifetime_observations(), 0); +} + +// ═══════════════════════════════════════════════════════════ +// Ingest — basic contract +// ═══════════════════════════════════════════════════════════ + +/// A batch with no values in it is answered with a report that names no +/// cells and no cross-cell contexts, and the observation counter does not +/// move. The health section is still filled in — the root tracker is alive +/// whether or not anything arrived — so an idle interval reads as an engine +/// with nothing to say rather than as a gap in the record. +/// +/// ´claim:edge:an-empty-batch-yields-a-report-with-no-cells-and-moves-no-counter´ +/// ´test:integration:empty-ingest-returns-empty-report´ +#[test] +fn empty_ingest_returns_empty_report() { + let mut s = Sentinel128::new(test_config()).unwrap(); + let report = s.ingest(&[]); + + assert!(report.cell_reports.is_empty()); + assert!(report.coordination_reports.is_empty()); + assert_eq!(report.health.lifetime_observations, 0); + assert_eq!(report.health.active_trackers, 1); // root cell +} + +/// Every value is delivered to the root as well as to whatever cell it +/// routes into, so even a lone observation produces an ancestor report and +/// advances the lifetime count. The root is the one cell guaranteed to +/// contain any coordinate, which is why a batch can never be scored against +/// nothing. +/// +/// ´claim:engine:the-root-tracker-receives-every-observation-in-every-batch´ +/// ´test:integration:single-value-ingest´ +#[test] +fn single_value_ingest() { + let mut s = Sentinel128::new(test_config()).unwrap(); + let report = s.ingest(&[0xABCD_0000_0000_0000_0000_0000_0000_0001]); + + // Root cell always receives all observations (as an ancestor). + assert!(!report.ancestor_reports.is_empty()); + assert_eq!(report.health.lifetime_observations, 1); + assert_invariants(&s, &report); +} + +/// The lifetime counter accumulates real observations rather than batches: +/// after a single value and then a batch of several it stands at their sum. +/// Batch boundaries are a delivery convenience for the host and carry no +/// weight in the record of what was actually seen. +/// +/// ´claim:engine:the-lifetime-counter-accumulates-observations-not-batches´ +/// ´test:integration:batch-counter-increments´ +#[test] +fn batch_counter_increments() { + let mut s = Sentinel128::new(test_config()).unwrap(); + + s.ingest(&[1]); + assert_eq!(s.lifetime_observations(), 1); + + s.ingest(&[2, 3, 4]); + assert_eq!(s.lifetime_observations(), 4); +} + +/// Feeding the same traffic round after round leaves every structural +/// invariant standing at each step: the reported cells stay within the +/// competitive budget, the two report vectors stay partitioned and ordered, +/// widths still match depths, and no score turns into a non-number. The +/// engine is a steady-state machine, so repetition is the ordinary case and +/// not a stress case. +/// +/// ´claim:engine:repeated-ingestion-of-the-same-traffic-keeps-every-structural-invariant´ +/// ´test:integration:repeated-ingest-does-not-panic´ +#[test] +fn repeated_ingest_does_not_panic() { + let mut s = Sentinel128::new(test_config()).unwrap(); + let values = cell_values(0xA, 20); + + for _ in 0..10 { + let report = s.ingest(&values); + assert_invariants(&s, &report); + } +} + +// ═══════════════════════════════════════════════════════════ +// Accessors — read-only queries +// ═══════════════════════════════════════════════════════════ + +/// The configuration reads back exactly as it was handed in — rank budget, +/// competitive budget and forgetting factor all unchanged. Construction +/// validates the parameters but does not silently normalise or substitute +/// them, so a host can trust the accessor as the authority on how this +/// sentinel is behaving. +/// +/// ´claim:engine:the-configuration-reads-back-as-given-because-construction-never-rewrites-it´ +/// ´test:integration:config-accessor-returns-construction-config´ +#[test] +fn config_accessor_returns_construction_config() { + let cfg = test_config(); + let s = Sentinel128::new(cfg.clone()).unwrap(); + + assert_eq!(s.config().max_rank, cfg.max_rank); + assert_eq!(s.config().analysis_k, cfg.analysis_k); + assert!((s.config().forgetting_factor - cfg.forgetting_factor).abs() < f64::EPSILON); +} + +/// The spatial substrate underneath a fresh sentinel is a bare root with no +/// accumulated importance at all. Construction does not seed the graph with +/// anything, so the first real batch is also the first thing the spatial +/// layer has ever ranked — there is no synthetic history for the selector to +/// mistake for traffic. +/// +/// ´claim:engine:construction-leaves-the-spatial-substrate-a-bare-root-with-no-accumulated-importance´ +/// ´test:integration:graph-accessor-starts-with-single-root´ +#[test] +fn graph_accessor_starts_with_single_root() { + let s = Sentinel128::new(test_config()).unwrap(); + + assert_eq!(s.graph().node_count(), 1); + assert_eq!(s.graph().terminal_count(), 1); + assert_eq!(s.graph().total_sum(), 0u64); +} + +/// The set of cells the sentinel is investing in is readable from outside, +/// and after traffic it holds at least the root. The root's membership is +/// unconditional — it is what every ancestor chain terminates at — so the +/// set is never empty and a host inspecting it never has to handle the +/// no-cells case. +/// +/// ´claim:engine:the-analysis-set-is-readable-and-always-holds-at-least-the-root´ +/// ´test:integration:analysis-set-accessible´ +#[test] +fn analysis_set_accessible() { + let s = seeded_sentinel(); + + let aset = s.analysis_set(); + // The full set always includes the root. + assert!(aset.total_count() >= 1); +} + +/// The list of cell handles and the count of tracked cells are two views of +/// one map, never two records that could disagree. A host can enumerate the +/// handles and know it has enumerated everything the engine is modelling. +/// +/// ´claim:engine:the-listed-cell-handles-and-the-tracked-count-are-two-views-of-one-map´ +/// ´test:integration:cell-gnodes-returns-all-tracked´ +#[test] +fn cell_gnodes_returns_all_tracked() { + let s = seeded_sentinel(); + + let gnodes = s.cell_gnodes(); + assert_eq!(gnodes.len(), s.cells_tracked()); +} + +/// The same agreement holds on both sides of the event that could break it. +/// Traffic heavy enough to split the domain adds cells to the map, and the +/// count and the handle list move together through that churn rather than +/// one of them being refreshed a moment later than the other. +/// +/// (´claim:engine:the-listed-cell-handles-and-the-tracked-count-are-two-views-of-one-map´) +/// ´test:integration:cells-tracked-matches-cell-gnodes-len´ +#[test] +fn cells_tracked_matches_cell_gnodes_len() { + let mut s = Sentinel128::new(test_config()).unwrap(); + assert_eq!(s.cells_tracked(), s.cell_gnodes().len()); + + // After traffic that may create cells. + for _ in 0..5 { + s.ingest(&cell_values(0xF, 8)); + } + assert_eq!(s.cells_tracked(), s.cell_gnodes().len()); +} + +/// Pinning the other end of the same statement: the counter starts at zero +/// and each batch adds exactly as many observations as it carried, so its +/// value is the running total of real input and not of anything the engine +/// generated for itself while warming a tracker. +/// +/// (´claim:engine:the-lifetime-counter-accumulates-observations-not-batches´) +/// ´test:integration:lifetime-observations-reflects-real-input´ +#[test] +fn lifetime_observations_reflects_real_input() { + let mut s = Sentinel128::new(test_config()).unwrap(); + assert_eq!(s.lifetime_observations(), 0); + + let batch_a = cell_values(0xA, 5); + s.ingest(&batch_a); + assert_eq!(s.lifetime_observations(), 5); + + let batch_b = cell_values(0xB, 3); + s.ingest(&batch_b); + assert_eq!(s.lifetime_observations(), 8); +} + +/// A cell whose suffix is too narrow to support a subspace model is skipped +/// and counted rather than modelled, and on a sentinel that has observed +/// nothing that count is zero. The counter is therefore a record of +/// something that happened, not a constant the engine carries around — a +/// non-zero reading always means real traffic drove the domain that deep. +/// +/// ´claim:engine:a-fresh-sentinel-has-skipped-no-cell-as-too-narrow-to-model´ +/// ´test:integration:degenerate-cells-skipped-starts-at-zero´ +#[test] +fn degenerate_cells_skipped_starts_at_zero() { + let s = Sentinel128::new(test_config()).unwrap(); + assert_eq!(s.degenerate_cells_skipped(), 0); +} + +/// A health snapshot can be taken before any observation arrives, and it +/// describes the engine truthfully at that moment: one live tracker, nothing +/// observed. Health is a readout of present state rather than a summary +/// accumulated during ingestion, so a host may poll it on a schedule of its +/// own without having to feed the engine first. +/// +/// ´claim:engine:a-health-snapshot-is-available-before-any-observation-arrives´ +/// ´test:integration:health-accessible-on-fresh-sentinel´ +#[test] +fn health_accessible_on_fresh_sentinel() { + let s = Sentinel128::new(test_config()).unwrap(); + let h = s.health(); + + assert_eq!(h.active_trackers, 1); + assert_eq!(h.lifetime_observations, 0); +} + +/// A tracked cell can be inspected individually, and what comes back +/// describes that cell: its depth in the routing tree, the width its tracker +/// analyses, and the rank of the subspace it has learned. The root reports +/// depth zero and the full domain width because nothing has been resolved +/// above it, and its rank is at least one from the moment it exists, since a +/// tracker with no direction at all could produce no residual. +/// +/// ´claim:engine:a-tracked-cell-is-inspectable-and-carries-its-own-depth-width-and-rank´ +/// ´test:integration:inspect-cell-returns-state-for-root´ +#[test] +fn inspect_cell_returns_state_for_root() { + let s = Sentinel128::new(test_config()).unwrap(); + let root = s.graph().g_root(); + + let inspection = s.inspect_cell(root).expect("root cell should exist"); + assert_eq!(inspection.depth, 0); + assert_eq!(inspection.analysis_width, 128); + assert!(inspection.rank >= 1); +} + +/// A cell handle is meaningful only to the sentinel that issued it. Handed a +/// handle minted by a different sentinel, inspection returns nothing rather +/// than the state of whichever local cell happens to sit at that index — the +/// lookup is a membership question, so a stale or foreign handle is an +/// absence and never a plausible-looking wrong answer. +/// +/// ´claim:engine:a-handle-from-another-sentinel-inspects-to-nothing-rather-than-to-a-wrong-cell´ +/// ´test:integration:inspect-cell-returns-none-for-unknown-gnode´ +#[test] +fn inspect_cell_returns_none_for_unknown_gnode() { + let s = Sentinel128::new(test_config()).unwrap(); + // Build a second sentinel so its non-root GNodeIds are foreign. + let mut other = Sentinel128::new(test_config()).unwrap(); + other.ingest(&[0xF000_0000_0000_0000_0000_0000_0000_0001]); + + let other_gnodes = other.cell_gnodes(); + if other_gnodes.len() > 1 { + let non_root = other_gnodes.iter().find(|&&g| g != other.graph().g_root()).unwrap(); + assert!(s.inspect_cell(*non_root).is_none()); + } +} + +// ═══════════════════════════════════════════════════════════ +// Per-sample scores +// ═══════════════════════════════════════════════════════════ + +/// Scores for individual values are attached to a cell's report only when +/// the host asked for them, and then there is exactly one entry per +/// observation in the batch. Per-sample detail costs memory proportional to +/// the traffic, so it is opt-in rather than always paid for, and the +/// one-to-one correspondence is what makes an entry attributable back to the +/// value that produced it. +/// +/// ´claim:engine:per-sample-scores-appear-only-when-asked-for-and-then-carry-one-entry-per-observation´ +/// ´test:integration:per-sample-scores-present-when-enabled´ +#[test] +fn per_sample_scores_present_when_enabled() { + let mut s = Sentinel128::new(test_config()).unwrap(); // per_sample_scores = true + + let report = s.ingest(&[0xF000_0000_0000_0000_0000_0000_0000_0001]); + + let root_report = report.ancestor_reports.iter().find(|cr| cr.depth == 0).unwrap(); + assert!(root_report.per_sample.is_some()); + assert_eq!(root_report.per_sample.as_ref().unwrap().len(), 1); +} + +/// The other end of the same switch: with the option off the field is +/// absent, not an empty list. A host that did not ask cannot mistake missing +/// detail for a batch in which nothing scored, and the engine does not +/// allocate for detail nobody wanted. +/// +/// (´claim:engine:per-sample-scores-appear-only-when-asked-for-and-then-carry-one-entry-per-observation´) +/// ´test:integration:per-sample-scores-absent-when-disabled´ +#[test] +fn per_sample_scores_absent_when_disabled() { + let cfg = SentinelConfig:: { + per_sample_scores: false, + ..test_config() + }; + let mut s = Sentinel128::new(cfg).unwrap(); + + let report = s.ingest(&[0xF000_0000_0000_0000_0000_0000_0000_0001]); + let root_report = report.ancestor_reports.iter().find(|cr| cr.depth == 0).unwrap(); + assert!(root_report.per_sample.is_none()); +} + +// ═══════════════════════════════════════════════════════════ +// Reset +// ═══════════════════════════════════════════════════════════ + +/// Reset returns a used sentinel to the state it was constructed in: the +/// spatial graph is rebuilt as a bare root, the observation counter is zero, +/// and the root tracker alone is tracked. Learned structure is dropped +/// wholesale rather than aged out, because reset exists for the case where +/// the host knows the past no longer describes the future. The configuration +/// is not part of what is cleared. +/// +/// ´claim:engine:reset-returns-the-sentinel-to-its-freshly-constructed-state-and-keeps-the-configuration´ +/// ´test:integration:reset-restores-initial-state´ +#[test] +fn reset_restores_initial_state() { + let mut s = seeded_sentinel(); + + s.reset(); + + assert_eq!(s.graph().node_count(), 1); + assert_eq!(s.graph().total_sum(), 0u64); + assert_eq!(s.lifetime_observations(), 0); + assert_eq!(s.cells_tracked(), 1); +} + +// ═══════════════════════════════════════════════════════════ +// Observation boundary +// ═══════════════════════════════════════════════════════════ + +/// The bit-vector type the conversion trait returns can be built from +/// outside the crate, and what a caller builds is the same thing the crate's +/// own conversion produces. The trait is published and open to a coordinate +/// type the crate has never heard of, so an implementation of it has to be +/// able to produce the value it is required to return: a type whose centred +/// form is computed rather than shifted out of an integer has nothing here to +/// delegate to, and without a constructor its implementation could not be +/// written at all. +/// +/// ´claim:bits:the-published-bit-vector-can-be-built-from-outside-and-matches-the-crates-own-conversion´ +/// ´test:integration:centred-bits-built-from-raw-values´ +#[test] +fn centred_bits_built_from_raw_values() { + // 0b1010_1010 at width eight: set bits at the even positions, reading + // most significant first. + let mut raw = [0.0_f64; 128]; + for (i, slot) in raw[..8].iter_mut().enumerate() { + *slot = if i % 2 == 0 { 0.5 } else { -0.5 }; + } + + let built = CentredBits::new(raw, 8); + assert_eq!(built.len(), 8); + assert!(!built.is_empty()); + assert_eq!(built.suffix(0).len(), 8); + + let converted = 0b1010_1010_u128.to_centred_bits(8); + for (i, (&b, &c)) in built.suffix(0).iter().zip(converted.suffix(0).iter()).enumerate() { + assert!( + (b - c).abs() < f64::EPSILON, + "bit {i}: built {b}, converted {c} — the two constructions disagree" + ); + } + + let empty = CentredBits::new([0.0_f64; 128], 0); + assert!(empty.is_empty()); + assert_eq!(empty.suffix(0).len(), 0); +} + +/// A length past the backing array is refused rather than clamped. The array +/// is a hundred and twenty-eight slots and nothing wider can be represented, +/// so an implementation asking for more has miscomputed its own width; handing +/// back a shorter vector would let that mistake travel into the tracker as an +/// observation narrower than the one its author believed it built. +/// +/// ´claim:bits:a-length-past-the-backing-array-is-refused-rather-than-clamped´ +/// ´test:integration:centred-bits-refuses-a-length-past-the-array´ +#[test] +#[should_panic(expected = "centred bit length exceeds the 128-slot backing array")] +fn centred_bits_refuses_a_length_past_the_array() { + let _refused = CentredBits::new([0.0_f64; 128], 129); +} diff --git a/packages/sentinel/tests/clip_pressure.rs b/packages/sentinel/tests/clip_pressure.rs new file mode 100644 index 000000000..61672bf0b --- /dev/null +++ b/packages/sentinel/tests/clip_pressure.rs @@ -0,0 +1,475 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +//! # Test index +//! +//! | Test | Area | Claim | +//! |------|------|-------| +//! | [`fresh_sentinel_has_zero_clip_pressure`] | pressure | A sentinel that has been shown nothing reports no rejection at all — not a nominal starting level, but zero across the smallest, largest and average axis alike. Rejection is a measurement of traffic, so with no traffic there is nothing to measure, and the ceiling starts as wide as it can be rather than pre-loaded against the first arrivals. | +//! | [`clip_pressure_distribution_min_le_mean_le_max`] | pressure | The summary the sentinel publishes is a genuine summary of the per-axis values behind it: after a long clean run followed by a contaminated burst, where the axes have been driven apart, the smallest value still sits at or below the average and the average at or below the largest. An operator watching only the peak can therefore trust that no axis is being rejected harder than the number they are watching. | +//! | [`per_axis_clip_pressure_in_cell_reports`] | pressure | Rejection is visible at the grain it actually happens: every batch report carries a separate, finite, non-negative figure for each of the four scoring axes of every cell reported. Each axis keeps its own ceiling and its own running rejection rate, so a caller can see which axis is under strain rather than only that something is. | +//! | [`clip_pressure_stable_under_clean_traffic`] | pressure | Traffic that keeps its shape does not ratchet the rejection rate upward. Over a long clean run the second stretch of batches sits no higher than the first, and the average across axes stays well under half. A modest steady rejection is expected — a ceiling a few deviations out will always trim the occasional sample — but it settles at a level rather than climbing, which is what lets a rise be read as news. | +//! | [`contamination_elevates_clip_pressure`] | pressure | Mixing a substantial minority of structurally novel values into an otherwise settled stream drives the peak rejection rate clearly above where clean traffic had left it. The rate is thus an observable symptom of contamination in its own right: the ceiling is doing its job of keeping those samples out of the baseline, and the pressure is the sentinel saying how hard it is having to work at it. | +//! | [`clip_pressure_decays_after_contamination`] | pressure | When the contamination stops, the rejection rate comes back down: after a long stretch of clean traffic the peak sits below where the contaminated phase left it. The measure is a rolling one with a finite memory, so an episode ages out instead of marking the sentinel permanently — a system that never forgot would treat one past attack as grounds for a forever-loose ceiling. | +//! | [`effective_ceiling_widens_under_pressure`] | pressure | The rejection rate is not merely reported, it feeds back into the ceiling. Recomputing the documented widening rule from the pressure a contaminated run actually leaves behind gives a ceiling meaningfully above the configured one. Refusal is therefore self-limiting by construction: the harder an axis has been rejecting, the more room it grants itself, so a genuine and lasting shift in the traffic can eventually be learned rather than clipped away for ever. | +//! | [`faster_decay_recovers_sooner`] | pressure | How long the rejection rate remembers is a configured choice with an observable consequence: given identical contamination and an identical recovery window, the sentinel with the shorter memory ends up at a lower pressure than the one with the longer. The setting therefore trades how quickly a ceiling snaps back after an episode against how steadily it holds through a noisy one. | +//! | [`warm_up_completion_resets_clip_pressure`] | pressure | Rejection accrued while a tracker still leaned on synthetic history is discarded at the moment that reliance falls away: shortly after the crossing, the peak rate across the sentinel is still low, having had only a handful of real batches in which to rebuild. Whatever the warming rounds caused the ceiling to refuse is therefore not allowed to widen the ceiling that production traffic will be judged against. | + +//! Integration tests for **clip pressure** — the running measure of how much of +//! the incoming traffic each scoring axis is currently refusing to learn from. +//! +//! A baseline that learned from every sample would be led anywhere an attacker +//! wanted it to go, so each axis keeps a ceiling and drops the samples above it +//! before updating. Only the upper tail is filtered: anomaly scores are +//! non-negative and right-skewed, an attacker inflates them and never deflates +//! them, and a lower bound would throw away the honestly low scores of a quiet +//! period. Clip pressure is the fraction of samples that ceiling rejected, +//! smoothed over recent batches, and it is reported per axis and summarised +//! across the whole sentinel. +//! +//! The quantity exists because refusing to learn is only ever a temporary +//! answer. Persistent rejection means the traffic has genuinely moved and the +//! ceiling is now in the wrong place; so the ceiling is widened in proportion to +//! the pressure, which lets the sentinel eventually follow a real shift instead +//! of scoring forever against a world that no longer exists. The smoothing +//! constant sets how long that memory is, and rejection accumulated while a +//! tracker was still being warmed is discarded outright when warming ends, so +//! synthetic history cannot loosen the ceiling that real traffic will be judged +//! against. +//! +//! Most tests here start from a cold configuration, with no warming rounds, so +//! that the baselines they exercise were built from the score distributions the +//! test itself fed in; the one test concerned with the warming handover is the +//! exception. + +mod common; + +use common::{batches_to_maturity, cell_values, cold_config, test_config}; +use torrust_sentinel::{Sentinel128, SentinelConfig}; + +// ═══════════════════════════════════════════════════════════ +// Helpers +// ═══════════════════════════════════════════════════════════ + +/// Maximum `clip_pressure` from a health report across all +/// active tracker axes. +fn max_clip_pressure(s: &Sentinel128) -> f64 { + s.health().clip_pressure_distribution.max +} + +/// Mean `clip_pressure` from a health report. +fn mean_clip_pressure(s: &Sentinel128) -> f64 { + s.health().clip_pressure_distribution.mean +} + +/// Generate values routed to the same cell (leading nibble `nibble`) +/// but with moderate structural variety in the middle bits. +/// +/// Unlike [`cell_values()`] (sequential low bits only), this injects +/// variation across a wider bit range, producing per-batch score +/// variance that the EWMA can track meaningfully. +fn diverse_cell_values(nibble: u128, count: usize, batch_id: usize) -> Vec { + (0..count) + .map(|i| { + // Vary bits 16–31 based on batch_id, and bits 0–15 based on i. + let mid = ((batch_id as u128 * 7 + 3) % 0xFFFF) << 16; + let low = (i as u128) | ((i as u128 * 13 + batch_id as u128) % 0xFFFF); + (nibble << 124) | mid | low + }) + .collect() +} + +/// Generate contaminated values: same leading nibble for correct +/// routing, but with a dense middle-bit block that produces elevated +/// anomaly scores (structurally novel relative to [`diverse_cell_values()`]). +fn contaminated_values(nibble: u128, count: usize, offset: usize) -> Vec { + (0..count) + .map(|i| { + // Set bits 32–63, creating structural novelty relative to + // the normal diverse_cell_values pattern (which only varies 0–31). + let dense = 0x0000_0000_FFFF_FFFF_0000_0000_0000_0000_u128; + (nibble << 124) | dense | ((offset + i) as u128) + }) + .collect() +} + +/// Common cold-start config for clip-pressure tests: root-only cell, +/// deterministic decay. +fn clip_cold_config(decay: f64) -> SentinelConfig { + SentinelConfig:: { + clip_pressure_decay: decay, + clip_sigmas: 3.0, + split_threshold: 100_000, + ..cold_config() + } +} + +/// Advance a sentinel past maturity, then give it `n` diverse clean batches +/// on the given nibble. +fn warmed_sentinel(cfg: SentinelConfig, nibble: u128, batches: usize) -> Sentinel128 { + let maturity_batches = batches_to_maturity(cfg.forgetting_factor); + let mut s = Sentinel128::new(cfg).unwrap(); + for _ in 0..maturity_batches { + s.ingest(&diverse_cell_values(nibble, 16, 0)); + } + for batch_id in 0..batches { + s.ingest(&diverse_cell_values(nibble, 16, batch_id)); + } + s +} + +// ═══════════════════════════════════════════════════════════ +// Initial state +// ═══════════════════════════════════════════════════════════ + +/// A sentinel that has been shown nothing reports no rejection at all — not a +/// nominal starting level, but zero across the smallest, largest and average +/// axis alike. Rejection is a measurement of traffic, so with no traffic there +/// is nothing to measure, and the ceiling starts as wide as it can be rather +/// than pre-loaded against the first arrivals. +/// +/// ´claim:pressure:a-sentinel-that-has-seen-no-traffic-reports-no-rejection-at-all´ +/// ´test:integration:fresh-sentinel-has-zero-clip-pressure´ +#[test] +fn fresh_sentinel_has_zero_clip_pressure() { + let s = Sentinel128::new(cold_config()).unwrap(); + let cp = s.health().clip_pressure_distribution; + + assert!( + cp.min == 0.0 && cp.max == 0.0 && cp.mean == 0.0, + "fresh sentinel should have all-zero clip_pressure, got min={}, max={}, mean={}", + cp.min, + cp.max, + cp.mean, + ); +} + +// ═══════════════════════════════════════════════════════════ +// Distribution invariants +// ═══════════════════════════════════════════════════════════ + +/// The summary the sentinel publishes is a genuine summary of the per-axis +/// values behind it: after a long clean run followed by a contaminated burst, +/// where the axes have been driven apart, the smallest value still sits at or +/// below the average and the average at or below the largest. An operator +/// watching only the peak can therefore trust that no axis is being rejected +/// harder than the number they are watching. +/// +/// ´claim:pressure:the-published-summary-brackets-its-own-average-between-the-least-and-most-pressed-axis´ +/// ´test:integration:clip-pressure-distribution-min-le-mean-le-max´ +#[test] +fn clip_pressure_distribution_min_le_mean_le_max() { + // Ingest enough traffic that clip_pressure is non-trivially + // exercised, then verify the distribution ordering invariant. + let mut s = warmed_sentinel(clip_cold_config(0.95), 0xA, 50); + + // Inject a burst of contamination so min ≠ max is more likely + // when multiple axes are active. + for batch_id in 50..60 { + let mut batch = diverse_cell_values(0xA, 10, batch_id); + batch.extend(contaminated_values(0xA, 6, batch_id)); + s.ingest(&batch); + } + + let cp = s.health().clip_pressure_distribution; + + assert!( + cp.min <= cp.mean && cp.mean <= cp.max, + "invariant violated: min={} ≤ mean={} ≤ max={}", + cp.min, + cp.mean, + cp.max, + ); +} + +// ═══════════════════════════════════════════════════════════ +// Per-axis visibility +// ═══════════════════════════════════════════════════════════ + +/// Rejection is visible at the grain it actually happens: every batch report +/// carries a separate, finite, non-negative figure for each of the four scoring +/// axes of every cell reported. Each axis keeps its own ceiling and its own +/// running rejection rate, so a caller can see which axis is under strain +/// rather than only that something is. +/// +/// ´claim:pressure:every-scoring-axis-of-every-reported-cell-carries-its-own-finite-rejection-rate´ +/// ´test:integration:per-axis-clip-pressure-in-cell-reports´ +#[test] +fn per_axis_clip_pressure_in_cell_reports() { + // After ingestion, the batch report's cell_reports should + // carry finite, non-negative clip_pressure on every axis. + let mut s = warmed_sentinel(clip_cold_config(0.95), 0xA, 50); + + let report = s.ingest(&diverse_cell_values(0xA, 16, 50)); + + for cr in &report.cell_reports { + for (name, cp) in [ + ("novelty", cr.scores.novelty.clip_pressure), + ("displacement", cr.scores.displacement.clip_pressure), + ("surprise", cr.scores.surprise.clip_pressure), + ("coherence", cr.scores.coherence.clip_pressure), + ] { + assert!( + cp.is_finite() && cp >= 0.0, + "cell gnode {:?} axis {name}: clip_pressure should be finite and ≥ 0, got {cp}", + cr.gnode_id, + ); + } + } +} + +// ═══════════════════════════════════════════════════════════ +// Clean-traffic equilibrium (§ALGO S-6.4) +// ═══════════════════════════════════════════════════════════ + +/// Traffic that keeps its shape does not ratchet the rejection rate upward. Over +/// a long clean run the second stretch of batches sits no higher than the first, +/// and the average across axes stays well under half. A modest steady rejection +/// is expected — a ceiling a few deviations out will always trim the occasional +/// sample — but it settles at a level rather than climbing, which is what lets +/// a rise be read as news. +/// +/// ´claim:pressure:traffic-that-keeps-its-shape-holds-the-rejection-rate-at-a-modest-level-instead-of-climbing´ +/// ´test:integration:clip-pressure-stable-under-clean-traffic´ +#[test] +fn clip_pressure_stable_under_clean_traffic() { + let mut s = warmed_sentinel(clip_cold_config(0.95), 0xA, 200); + + // Collect clip_pressure over the next 100 batches. + let mut cp_values = Vec::with_capacity(100); + for batch_id in 200..300 { + s.ingest(&diverse_cell_values(0xA, 16, batch_id)); + cp_values.push(max_clip_pressure(&s)); + } + + // No upward trend: second half should not exceed first by much. + let first_half_mean: f64 = cp_values[..50].iter().sum::() / 50.0; + let second_half_mean: f64 = cp_values[50..].iter().sum::() / 50.0; + + assert!( + second_half_mean <= first_half_mean + 0.05, + "clip_pressure should not trend upward under clean traffic: \ + first_half={first_half_mean:.4}, second_half={second_half_mean:.4}" + ); + + // Mean should confirm stability. + let mean_cp = mean_clip_pressure(&s); + assert!( + mean_cp < 0.5, + "mean clip_pressure should be moderate under clean traffic, got {mean_cp:.4}" + ); +} + +// ═══════════════════════════════════════════════════════════ +// Contamination dynamics (§ALGO S-6.4.4) +// ═══════════════════════════════════════════════════════════ + +/// Mixing a substantial minority of structurally novel values into an otherwise +/// settled stream drives the peak rejection rate clearly above where clean +/// traffic had left it. The rate is thus an observable symptom of contamination +/// in its own right: the ceiling is doing its job of keeping those samples out +/// of the baseline, and the pressure is the sentinel saying how hard it is +/// having to work at it. +/// +/// ´claim:pressure:sustained-contamination-drives-the-rejection-rate-above-its-clean-level´ +/// ´test:integration:contamination-elevates-clip-pressure´ +#[test] +fn contamination_elevates_clip_pressure() { + let mut s = warmed_sentinel(clip_cold_config(0.95), 0xA, 200); + let cp_baseline = max_clip_pressure(&s); + + // 80 batches of mixed traffic: 60% normal + 40% outliers. + for batch_id in 200..280 { + let mut batch = diverse_cell_values(0xA, 10, batch_id); + batch.extend(contaminated_values(0xA, 6, batch_id)); + s.ingest(&batch); + } + + let cp_after = max_clip_pressure(&s); + + assert!( + cp_after > cp_baseline + 0.05, + "contamination should elevate clip_pressure: \ + baseline={cp_baseline:.4}, after={cp_after:.4}" + ); +} + +/// When the contamination stops, the rejection rate comes back down: after a +/// long stretch of clean traffic the peak sits below where the contaminated +/// phase left it. The measure is a rolling one with a finite memory, so an +/// episode ages out instead of marking the sentinel permanently — a system that +/// never forgot would treat one past attack as grounds for a forever-loose +/// ceiling. +/// +/// ´claim:pressure:the-rejection-rate-falls-back-once-the-contamination-stops´ +/// ´test:integration:clip-pressure-decays-after-contamination´ +#[test] +fn clip_pressure_decays_after_contamination() { + let mut s = warmed_sentinel(clip_cold_config(0.95), 0xA, 200); + + // Contamination phase. + for batch_id in 200..280 { + let mut batch = diverse_cell_values(0xA, 10, batch_id); + batch.extend(contaminated_values(0xA, 6, batch_id)); + s.ingest(&batch); + } + let cp_contaminated = max_clip_pressure(&s); + + // Recovery: 200 batches of clean traffic. + for batch_id in 280..480 { + s.ingest(&diverse_cell_values(0xA, 16, batch_id)); + } + let cp_recovered = max_clip_pressure(&s); + + assert!( + cp_recovered < cp_contaminated, + "clip_pressure should decrease after contamination ends: \ + contaminated={cp_contaminated:.4}, recovered={cp_recovered:.4}" + ); +} + +/// The rejection rate is not merely reported, it feeds back into the ceiling. +/// Recomputing the documented widening rule from the pressure a contaminated run +/// actually leaves behind gives a ceiling meaningfully above the configured one. +/// Refusal is therefore self-limiting by construction: the harder an axis has +/// been rejecting, the more room it grants itself, so a genuine and lasting +/// shift in the traffic can eventually be learned rather than clipped away for +/// ever. +/// +/// ´claim:pressure:a-raised-rejection-rate-widens-the-ceiling-so-refusal-cannot-become-permanent´ +/// ´test:integration:effective-ceiling-widens-under-pressure´ +#[test] +fn effective_ceiling_widens_under_pressure() { + let mut s = warmed_sentinel(clip_cold_config(0.95), 0xA, 200); + + // Contamination phase to elevate pressure. + for batch_id in 200..280 { + let mut batch = diverse_cell_values(0xA, 10, batch_id); + batch.extend(contaminated_values(0xA, 6, batch_id)); + s.ingest(&batch); + } + + let p = max_clip_pressure(&s); + let clip_sigmas = 3.0_f64; + let eps = 1e-6_f64; + + // §ALGO S-6.4 effective-clip formula: n_σ · (1 + ρ̄ / (1 − ρ̄ + ε)) + let effective_clip = clip_sigmas * (1.0 + p / (1.0 - p + eps)); + + assert!( + effective_clip > clip_sigmas * 1.01, + "effective clip {effective_clip:.4} should exceed base clip_sigmas {clip_sigmas}" + ); +} + +// ═══════════════════════════════════════════════════════════ +// Decay-rate effect +// ═══════════════════════════════════════════════════════════ + +/// How long the rejection rate remembers is a configured choice with an +/// observable consequence: given identical contamination and an identical +/// recovery window, the sentinel with the shorter memory ends up at a lower +/// pressure than the one with the longer. The setting therefore trades how +/// quickly a ceiling snaps back after an episode against how steadily it holds +/// through a noisy one. +/// +/// ´claim:pressure:a-shorter-memory-brings-the-rejection-rate-back-down-within-a-shorter-recovery-window´ +/// ´test:integration:faster-decay-recovers-sooner´ +#[test] +fn faster_decay_recovers_sooner() { + // Two sentinels, identical contamination, different λ_ρ. + // Lower λ_ρ → faster decay → lower pressure after recovery. + let fast_decay = 0.85; + let slow_decay = 0.98; + + let contaminate_and_recover = |decay: f64| -> f64 { + let mut s = warmed_sentinel(clip_cold_config(decay), 0xA, 200); + + for batch_id in 200..280 { + let mut batch = diverse_cell_values(0xA, 10, batch_id); + batch.extend(contaminated_values(0xA, 6, batch_id)); + s.ingest(&batch); + } + + // Fixed recovery window: 100 batches of clean traffic. + for batch_id in 280..380 { + s.ingest(&diverse_cell_values(0xA, 16, batch_id)); + } + + max_clip_pressure(&s) + }; + + let cp_fast = contaminate_and_recover(fast_decay); + let cp_slow = contaminate_and_recover(slow_decay); + + assert!( + cp_fast < cp_slow, + "faster decay (λ_ρ={fast_decay}) should recover to lower pressure \ + than slow decay (λ_ρ={slow_decay}): fast={cp_fast:.4}, slow={cp_slow:.4}" + ); +} + +// ═══════════════════════════════════════════════════════════ +// Warm-up reset (§ALGO S-11.4) +// ═══════════════════════════════════════════════════════════ + +/// Rejection accrued while a tracker still leaned on synthetic history is +/// discarded at the moment that reliance falls away: shortly after the +/// crossing, the peak rate across the sentinel is still low, having had only a +/// handful of real batches in which to rebuild. Whatever the warming rounds +/// caused the ceiling to refuse is therefore not allowed to widen the ceiling +/// that production traffic will be judged against. +/// +/// ´claim:pressure:the-rejection-rate-is-zeroed-when-a-tracker-stops-leaning-on-synthetic-history´ +/// ´test:integration:warm-up-completion-resets-clip-pressure´ +#[test] +fn warm_up_completion_resets_clip_pressure() { + // With noise injection, clip_pressure may rise during warm-up. + // After η crosses the warm-up threshold (0.01), the + // update_maturity() callback zeros clip_pressure on all axes. + // + // With λ=0.90 and batch_size=8, η < 0.01 requires ~44 batches + // (ln(0.01) / ln(0.90) ≈ 43.7). We run up to 100 batches. + let cfg = SentinelConfig:: { + clip_pressure_decay: 0.95, + clip_sigmas: 3.0, + split_threshold: 100_000, + forgetting_factor: 0.90, + ..test_config() + }; + let mut s = Sentinel128::new(cfg).unwrap(); + + // Seed traffic to create the root cell. + s.ingest(&cell_values(0xA, 16)); + + // Feed real traffic until η crosses below 0.01. + let root = s.graph().g_root(); + let mut crossed = false; + let mut post_cross_batches = 0_usize; + + for batch_id in 0..100 { + s.ingest(&diverse_cell_values(0xA, 16, batch_id)); + let insp = s.inspect_cell(root).unwrap(); + if !crossed && insp.maturity.noise_influence < 0.01 { + crossed = true; + } + if crossed { + post_cross_batches += 1; + if post_cross_batches == 5 { + // Clip_pressure was zeroed at crossing, then accumulated + // for only 5 batches. With λ_ρ=0.95, even if every + // batch clips 100%, ρ̄ ≤ 1 - 0.95^5 ≈ 0.23. + let h = s.health(); + assert!( + h.clip_pressure_distribution.max < 0.30, + "clip_pressure should be low shortly after warm-up \ + reset, got max={:.4} (5 batches post-crossing)", + h.clip_pressure_distribution.max + ); + break; + } + } + } + + assert!(crossed, "η should have crossed below 0.01 within 100 batches"); +} diff --git a/packages/sentinel/tests/common/README.md b/packages/sentinel/tests/common/README.md new file mode 100644 index 000000000..33bd5e17d --- /dev/null +++ b/packages/sentinel/tests/common/README.md @@ -0,0 +1,5 @@ +## Integration test matrix · `tab:sentinel:common-integration-test-matrix` + +**Table (Integration test matrix)** + +No integration tests in this folder. diff --git a/packages/sentinel/tests/common/assertions.rs b/packages/sentinel/tests/common/assertions.rs new file mode 100644 index 000000000..2c6059d59 --- /dev/null +++ b/packages/sentinel/tests/common/assertions.rs @@ -0,0 +1,563 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +//! Assertions and report-inspection helpers for tests. + +use std::collections::HashSet; + +use torrust_sentinel::{ + AnalysisSetSummary, AnomalyScores, BaselineSnapshot, BatchReport, CellReport, ClipPressureDistribution, ContourSnapshot, + CoordinationHealth, CoordinationReport, GeometryDistribution, HealthReport, MaturityDistribution, MemberScore, + RankDistribution, SampleScore, ScoreDistribution, ScoringGeometry, Sentinel128, TrackerMaturity, +}; + +// ─── assert_invariants() ─────────────────────────────────── + +/// Assert all structural invariants on a sentinel and its last report. +/// +/// Designed to be called after any `ingest()` in any test to verify +/// cross-cutting invariants. +pub fn assert_invariants(sentinel: &Sentinel128, report: &BatchReport) { + assert_steiner_tree_bound(report); + assert_competitive_cap(sentinel, report); + assert_root_in_full_set(report); + assert_cell_reports_competitive(report); + assert_ancestor_reports_non_competitive(report); + assert_cell_reports_sorted(report); + assert_ancestor_reports_sorted(report); + assert_coordination_reports_unique(report); + assert_no_nan_scores(report); + assert_analysis_widths(report); +} + +/// 1. Analysis set bounds: `full_size <= 1 + K * D_max` (§ALGO S-8.2). +/// +/// The *reduced* Steiner tree has at most `2K − 1` nodes, but the +/// materialised investment set includes degree-2 chain intermediaries +/// that push the count higher when competitive cells sit at varying +/// depths. The correct worst-case bound is `1 + K·D̄` (before +/// sharing); we use `1 + K·D_max` as a practical upper bound. +fn assert_steiner_tree_bound(report: &BatchReport) { + let summary = &report.analysis_set_summary; + let k = summary.competitive_size; + let d_max = summary.depth_range.1 as usize; + let bound = 1 + k * d_max; + assert!( + summary.full_size <= bound, + "materialised Steiner tree bound violated: full={}, competitive={}, d_max={}, bound={}", + summary.full_size, + k, + d_max, + bound, + ); +} + +/// 2. Competitive cap: `competitive_size <= analysis_k`. +fn assert_competitive_cap(sentinel: &Sentinel128, report: &BatchReport) { + let summary = &report.analysis_set_summary; + assert!( + summary.competitive_size <= sentinel.config().analysis_k, + "competitive set {} exceeds K={}", + summary.competitive_size, + sentinel.config().analysis_k, + ); +} + +/// 3. Root at depth 0 is always in the full set. +fn assert_root_in_full_set(report: &BatchReport) { + let summary = &report.analysis_set_summary; + if summary.full_size > 0 { + assert_eq!(summary.depth_range.0, 0, "root (depth 0) must be in the full analysis set"); + } +} + +/// 4. `cell_reports` entries are competitive only. +fn assert_cell_reports_competitive(report: &BatchReport) { + for cr in &report.cell_reports { + assert!( + cr.is_competitive, + "cell_reports entry at depth {} is not competitive", + cr.depth + ); + } +} + +/// 5. `ancestor_reports` entries are non-competitive. +fn assert_ancestor_reports_non_competitive(report: &BatchReport) { + for ar in &report.ancestor_reports { + assert!( + !ar.is_competitive, + "ancestor_reports entry at depth {} is competitive", + ar.depth + ); + } +} + +/// 6. Deterministic ordering: `cell_reports` sorted by `gnode_id` ascending. +fn assert_cell_reports_sorted(report: &BatchReport) { + for window in report.cell_reports.windows(2) { + assert!( + window[0].gnode_id < window[1].gnode_id, + "cell_reports not sorted by GNodeId: {:?} >= {:?}", + window[0].gnode_id, + window[1].gnode_id, + ); + } +} + +/// 7. Deterministic ordering: `ancestor_reports` sorted by `gnode_id` ascending. +fn assert_ancestor_reports_sorted(report: &BatchReport) { + for window in report.ancestor_reports.windows(2) { + assert!( + window[0].gnode_id < window[1].gnode_id, + "ancestor_reports not sorted by GNodeId: {:?} >= {:?}", + window[0].gnode_id, + window[1].gnode_id, + ); + } +} + +/// 8. Coordination reports have no duplicate `GNodeId`s. +fn assert_coordination_reports_unique(report: &BatchReport) { + let mut seen = HashSet::new(); + for cr in &report.coordination_reports { + assert!( + seen.insert(cr.gnode_id), + "duplicate GNodeId in coordination_reports: {:?}", + cr.gnode_id, + ); + } +} + +/// 9. No NaN in score fields. +fn assert_no_nan_scores(report: &BatchReport) { + for cr in report.cell_reports.iter().chain(report.ancestor_reports.iter()) { + assert!(!cr.scores.novelty.mean.is_nan(), "NaN novelty mean at depth {}", cr.depth); + assert!( + !cr.scores.displacement.mean.is_nan(), + "NaN displacement mean at depth {}", + cr.depth + ); + assert!(!cr.scores.surprise.mean.is_nan(), "NaN surprise mean at depth {}", cr.depth); + } +} + +/// 10. `analysis_width == 128 - depth` for every cell report. +fn assert_analysis_widths(report: &BatchReport) { + for cr in report.cell_reports.iter().chain(report.ancestor_reports.iter()) { + assert_eq!( + cr.analysis_width, + 128 - cr.depth as usize, + "analysis_width mismatch at depth {}", + cr.depth + ); + } +} + +/// Extract the maximum CUSUM accumulator value across all cell and +/// ancestor reports in a [`BatchReport`]. +pub fn max_cusum(report: &BatchReport) -> f64 { + report + .cell_reports + .iter() + .chain(report.ancestor_reports.iter()) + .flat_map(|cr| { + [ + cr.scores.novelty.cusum.accumulator, + cr.scores.displacement.cusum.accumulator, + cr.scores.surprise.cusum.accumulator, + cr.scores.coherence.cusum.accumulator, + ] + }) + .fold(0.0f64, f64::max) +} + +/// Extract the maximum novelty z-score across all cell and +/// ancestor reports in a [`BatchReport`]. +pub fn max_novelty_z(report: &BatchReport) -> f64 { + report + .cell_reports + .iter() + .chain(report.ancestor_reports.iter()) + .map(|cr| cr.scores.novelty.max_z_score) + .fold(0.0f64, f64::max) +} + +/// Extract the novelty z-score from the root (depth 0) ancestor +/// report, returning `0.0` if no root is present. +pub fn root_novelty_z(report: &BatchReport) -> f64 { + report + .ancestor_reports + .iter() + .find(|r| r.depth == 0) + .map_or(0.0, |r| r.scores.novelty.max_z_score) +} + +// ─── assert_reports_identical() ───────────────────────────── + +/// Assert that two batch reports are the same report, field for field. +/// +/// Every number is compared by its bit pattern rather than by value, because +/// the claim being checked is that two runs computed the same thing and not +/// merely that they landed near each other: equality on values would accept a +/// difference smaller than a printed digit, and would call the two signed +/// zeroes equal while calling two non-numbers different. +/// +/// One field is deliberately not compared for its value. The age of a +/// report's oldest observation is an interval between two readings of the +/// sentinel's own clock, so two runs differ in it by construction and no +/// arrangement of the inputs can make them agree. What is compared is whether +/// each report carries an age at all, which is the part that follows from the +/// batch rather than from when the batch happened to be processed. +pub fn assert_reports_identical(left: &BatchReport, right: &BatchReport) { + assert_eq!( + left.cell_reports.len(), + right.cell_reports.len(), + "the two runs reported different numbers of competitive cells" + ); + for (a, b) in left.cell_reports.iter().zip(&right.cell_reports) { + assert_cells_identical(a, b, "cell"); + } + + assert_eq!( + left.ancestor_reports.len(), + right.ancestor_reports.len(), + "the two runs reported different numbers of ancestor cells" + ); + for (a, b) in left.ancestor_reports.iter().zip(&right.ancestor_reports) { + assert_cells_identical(a, b, "ancestor"); + } + + assert_eq!( + left.coordination_reports.len(), + right.coordination_reports.len(), + "the two runs reported different numbers of coordination contexts" + ); + for (a, b) in left.coordination_reports.iter().zip(&right.coordination_reports) { + assert_coordination_identical(a, b); + } + + assert_contours_identical(&left.contour, &right.contour); + assert_health_identical(&left.health, &right.health); + assert_summaries_identical(&left.analysis_set_summary, &right.analysis_set_summary); + + assert_eq!( + left.oldest_observation_age_micros.is_some(), + right.oldest_observation_age_micros.is_some(), + "one run reported an observation age and the other reported none" + ); +} + +/// Assert two floating-point figures have the same bit pattern. +fn assert_bits(left: f64, right: f64, what: &str) { + assert_eq!(left.to_bits(), right.to_bits(), "{what} differs: {left} against {right}"); +} + +/// Every field of a cell report, for a competitive or an ancestor cell alike. +fn assert_cells_identical(left: &CellReport, right: &CellReport, tier: &str) { + let at = format!("{tier} {:?}", left.gnode_id); + + assert_eq!(left.gnode_id, right.gnode_id, "{at}: handle"); + assert_eq!(left.start, right.start, "{at}: interval start"); + assert_eq!(left.end, right.end, "{at}: interval end"); + assert_eq!(left.depth, right.depth, "{at}: depth"); + assert_eq!(left.analysis_width, right.analysis_width, "{at}: analysis width"); + assert_eq!(left.is_competitive, right.is_competitive, "{at}: competitiveness"); + assert_eq!(left.sample_count, right.sample_count, "{at}: sample count"); + assert_eq!(left.rank, right.rank, "{at}: rank"); + assert_bits(left.energy_ratio, right.energy_ratio, &format!("{at}: energy ratio")); + assert_bits( + left.top_singular_value, + right.top_singular_value, + &format!("{at}: top singular value"), + ); + + assert_scores_identical(&left.scores, &right.scores, &at); + assert_maturity_identical(&left.maturity, &right.maturity, &at); + assert_geometry_identical(&left.geometry, &right.geometry, &at); + + match (&left.per_sample, &right.per_sample) { + (None, None) => {} + (Some(a), Some(b)) => { + assert_eq!(a.len(), b.len(), "{at}: per-sample score count"); + for (i, (x, y)) in a.iter().zip(b).enumerate() { + assert_sample_scores_identical(x, y, &format!("{at}: sample {i}")); + } + } + _ => panic!("{at}: one run carried per-sample scores and the other did not"), + } +} + +/// Every field of a coordination report. +fn assert_coordination_identical(left: &CoordinationReport, right: &CoordinationReport) { + let at = format!("context {:?}", left.gnode_id); + + assert_eq!(left.gnode_id, right.gnode_id, "{at}: handle"); + assert_eq!(left.start, right.start, "{at}: interval start"); + assert_eq!(left.end, right.end, "{at}: interval end"); + assert_eq!(left.depth, right.depth, "{at}: depth"); + assert_eq!(left.cells_reporting, right.cells_reporting, "{at}: contributing cells"); + assert_eq!(left.rank, right.rank, "{at}: rank"); + assert_bits(left.energy_ratio, right.energy_ratio, &format!("{at}: energy ratio")); + assert_bits( + left.top_singular_value, + right.top_singular_value, + &format!("{at}: top singular value"), + ); + + assert_scores_identical(&left.scores, &right.scores, &at); + assert_maturity_identical(&left.maturity, &right.maturity, &at); + assert_geometry_identical(&left.geometry, &right.geometry, &at); + + match (&left.per_member, &right.per_member) { + (None, None) => {} + (Some(a), Some(b)) => { + assert_eq!(a.len(), b.len(), "{at}: member score count"); + for (i, (x, y)) in a.iter().zip(b).enumerate() { + assert_member_scores_identical(x, y, &format!("{at}: member {i}")); + } + } + _ => panic!("{at}: one run carried member scores and the other did not"), + } +} + +/// All four scoring axes. +fn assert_scores_identical(left: &AnomalyScores, right: &AnomalyScores, at: &str) { + assert_distributions_identical(&left.novelty, &right.novelty, &format!("{at}: novelty")); + assert_distributions_identical(&left.displacement, &right.displacement, &format!("{at}: displacement")); + assert_distributions_identical(&left.surprise, &right.surprise, &format!("{at}: surprise")); + assert_distributions_identical(&left.coherence, &right.coherence, &format!("{at}: coherence")); +} + +/// Every figure one axis publishes, baselines and drift evidence included. +fn assert_distributions_identical(left: &ScoreDistribution, right: &ScoreDistribution, at: &str) { + assert_bits(left.min, right.min, &format!("{at} minimum")); + assert_bits(left.max, right.max, &format!("{at} maximum")); + assert_bits(left.mean, right.mean, &format!("{at} mean")); + assert_bits(left.max_z_score, right.max_z_score, &format!("{at} maximum z-score")); + assert_bits(left.mean_z_score, right.mean_z_score, &format!("{at} mean z-score")); + assert_bits(left.clip_pressure, right.clip_pressure, &format!("{at} clip pressure")); + + assert_baselines_identical(&left.baseline, &right.baseline, &format!("{at} baseline")); + + assert_bits( + left.cusum.accumulator, + right.cusum.accumulator, + &format!("{at} cusum accumulator"), + ); + assert_eq!( + left.cusum.steps_since_reset, right.cusum.steps_since_reset, + "{at} cusum steps since reset" + ); + assert_baselines_identical( + &left.cusum.slow_baseline, + &right.cusum.slow_baseline, + &format!("{at} cusum slow baseline"), + ); +} + +/// A baseline's centre and spread. +fn assert_baselines_identical(left: &BaselineSnapshot, right: &BaselineSnapshot, at: &str) { + assert_bits(left.mean, right.mean, &format!("{at} mean")); + assert_bits(left.variance, right.variance, &format!("{at} variance")); +} + +/// What a tracker has seen, and how much of it was synthetic. +fn assert_maturity_identical(left: &TrackerMaturity, right: &TrackerMaturity, at: &str) { + assert_eq!( + left.real_observations, right.real_observations, + "{at}: real observation count" + ); + assert_eq!( + left.noise_observations, right.noise_observations, + "{at}: synthetic observation count" + ); + assert_bits(left.noise_influence, right.noise_influence, &format!("{at}: noise influence")); +} + +/// The shape of the space a tracker is working in. +fn assert_geometry_identical(left: &ScoringGeometry, right: &ScoringGeometry, at: &str) { + assert_eq!(left.dim, right.dim, "{at}: dimension"); + assert_eq!(left.cap, right.cap, "{at}: rank cap"); + assert_eq!(left.residual_dof, right.residual_dof, "{at}: residual degrees of freedom"); +} + +/// One observation's four axes, raw and standardised. +fn assert_sample_scores_identical(left: &SampleScore, right: &SampleScore, at: &str) { + assert_bits(left.novelty, right.novelty, &format!("{at}: novelty")); + assert_bits(left.displacement, right.displacement, &format!("{at}: displacement")); + assert_bits(left.surprise, right.surprise, &format!("{at}: surprise")); + assert_bits(left.coherence, right.coherence, &format!("{at}: coherence")); + assert_bits(left.novelty_z, right.novelty_z, &format!("{at}: novelty z-score")); + assert_bits( + left.displacement_z, + right.displacement_z, + &format!("{at}: displacement z-score"), + ); + assert_bits(left.surprise_z, right.surprise_z, &format!("{at}: surprise z-score")); + assert_bits(left.coherence_z, right.coherence_z, &format!("{at}: coherence z-score")); +} + +/// One contributing cell's four axes, and the cell they are attributed to. +fn assert_member_scores_identical(left: &MemberScore, right: &MemberScore, at: &str) { + assert_eq!(left.cell_start, right.cell_start, "{at}: cell interval start"); + assert_eq!(left.cell_end, right.cell_end, "{at}: cell interval end"); + assert_eq!(left.cell_depth, right.cell_depth, "{at}: cell depth"); + assert_bits(left.novelty, right.novelty, &format!("{at}: novelty")); + assert_bits(left.displacement, right.displacement, &format!("{at}: displacement")); + assert_bits(left.surprise, right.surprise, &format!("{at}: surprise")); + assert_bits(left.coherence, right.coherence, &format!("{at}: coherence")); + assert_bits(left.novelty_z, right.novelty_z, &format!("{at}: novelty z-score")); + assert_bits( + left.displacement_z, + right.displacement_z, + &format!("{at}: displacement z-score"), + ); + assert_bits(left.surprise_z, right.surprise_z, &format!("{at}: surprise z-score")); + assert_bits(left.coherence_z, right.coherence_z, &format!("{at}: coherence z-score")); +} + +/// The spatial layer as the report describes it. +fn assert_contours_identical(left: &ContourSnapshot, right: &ContourSnapshot) { + assert_eq!(left.plateau_count, right.plateau_count, "contour: plateau count"); + assert_eq!(left.cell_count, right.cell_count, "contour: cell count"); + assert_bits(left.total_importance, right.total_importance, "contour: total importance"); + assert_eq!( + left.splits_since_last_report, right.splits_since_last_report, + "contour: splits since the last report" + ); + assert_eq!( + left.net_removals_since_last_report, right.net_removals_since_last_report, + "contour: net removals since the last report" + ); +} + +/// Every operational figure the report carries. +fn assert_health_identical(left: &HealthReport, right: &HealthReport) { + assert_eq!(left.total_g_nodes, right.total_g_nodes, "health: total nodes"); + assert_eq!( + left.semi_internal_count, right.semi_internal_count, + "health: semi-internal nodes" + ); + assert_eq!(left.active_trackers, right.active_trackers, "health: active trackers"); + assert_eq!( + left.active_competitive_trackers, right.active_competitive_trackers, + "health: active competitive trackers" + ); + assert_eq!( + left.active_ancestor_trackers, right.active_ancestor_trackers, + "health: active ancestor trackers" + ); + assert_eq!( + left.active_coordination_contexts, right.active_coordination_contexts, + "health: active coordination contexts" + ); + assert_eq!( + left.investment_set_size, right.investment_set_size, + "health: investment set size" + ); + assert_eq!(left.warming_trackers, right.warming_trackers, "health: warming trackers"); + assert_eq!( + left.warming_competitive_targets, right.warming_competitive_targets, + "health: warming competitive targets" + ); + assert_eq!( + left.lifetime_observations, right.lifetime_observations, + "health: lifetime observations" + ); + assert_eq!(left.cells_tracked, right.cells_tracked, "health: cells tracked"); + + assert_ranks_identical(&left.rank_distribution, &right.rank_distribution, "health"); + assert_maturities_identical(&left.maturity_distribution, &right.maturity_distribution, "health"); + assert_geometries_identical(&left.geometry_distribution, &right.geometry_distribution, "health"); + assert_pressures_identical(&left.clip_pressure_distribution, &right.clip_pressure_distribution); + assert_coordination_health_identical(&left.coordination_health, &right.coordination_health); +} + +/// The rank spread over the live trackers. +fn assert_ranks_identical(left: &RankDistribution, right: &RankDistribution, at: &str) { + assert_eq!(left.min, right.min, "{at}: minimum rank"); + assert_eq!(left.max, right.max, "{at}: maximum rank"); + assert_bits(left.mean, right.mean, &format!("{at}: mean rank")); +} + +/// How much of what the live trackers hold is synthetic. +fn assert_maturities_identical(left: &MaturityDistribution, right: &MaturityDistribution, at: &str) { + assert_bits( + left.max_noise_influence, + right.max_noise_influence, + &format!("{at}: maximum noise influence"), + ); + assert_bits( + left.min_noise_influence, + right.min_noise_influence, + &format!("{at}: minimum noise influence"), + ); + assert_bits( + left.mean_noise_influence, + right.mean_noise_influence, + &format!("{at}: mean noise influence"), + ); + assert_eq!(left.cold_trackers, right.cold_trackers, "{at}: cold trackers"); +} + +/// Where the live trackers stand against their own geometric limits. +fn assert_geometries_identical(left: &GeometryDistribution, right: &GeometryDistribution, at: &str) { + assert_eq!(left.novelty_saturated, right.novelty_saturated, "{at}: novelty saturated"); + assert_eq!(left.novelty_saturable, right.novelty_saturable, "{at}: novelty saturable"); + assert_eq!(left.coherence_inactive, right.coherence_inactive, "{at}: coherence inactive"); +} + +/// The rejection rate spread over the live trackers. +fn assert_pressures_identical(left: &ClipPressureDistribution, right: &ClipPressureDistribution) { + assert_bits(left.min, right.min, "health: minimum clip pressure"); + assert_bits(left.max, right.max, "health: maximum clip pressure"); + assert_bits(left.mean, right.mean, "health: mean clip pressure"); +} + +/// The coordination tier's own health section. +fn assert_coordination_health_identical(left: &CoordinationHealth, right: &CoordinationHealth) { + assert_eq!( + left.active_contexts, right.active_contexts, + "coordination health: active contexts" + ); + assert_eq!(left.capacity, right.capacity, "coordination health: capacity"); + assert_eq!(left.dim, right.dim, "coordination health: dimension"); + assert_ranks_identical(&left.rank_distribution, &right.rank_distribution, "coordination health"); + assert_maturities_identical( + &left.maturity_distribution, + &right.maturity_distribution, + "coordination health", + ); + assert_geometries_identical( + &left.geometry_distribution, + &right.geometry_distribution, + "coordination health", + ); +} + +/// The summary of what the sentinel is currently investing in. +fn assert_summaries_identical(left: &AnalysisSetSummary, right: &AnalysisSetSummary) { + assert_eq!(left.competitive_size, right.competitive_size, "summary: competitive size"); + assert_eq!(left.full_size, right.full_size, "summary: full size"); + assert_eq!( + left.investment_set_size, right.investment_set_size, + "summary: investment set size" + ); + assert_eq!(left.depth_range, right.depth_range, "summary: depth range"); + assert_bits( + left.importance_range.0, + right.importance_range.0, + "summary: lowest importance", + ); + assert_bits( + left.importance_range.1, + right.importance_range.1, + "summary: highest importance", + ); + assert_eq!(left.v_depth_range, right.v_depth_range, "summary: v-depth range"); + assert_eq!( + left.degenerate_cells_skipped, right.degenerate_cells_skipped, + "summary: degenerate cells skipped" + ); +} diff --git a/packages/sentinel/tests/common/builders.rs b/packages/sentinel/tests/common/builders.rs new file mode 100644 index 000000000..62dd444d4 --- /dev/null +++ b/packages/sentinel/tests/common/builders.rs @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +//! Sentinel constructors and scenario builders for tests. + +use torrust_sentinel::{BatchReport, Sentinel128, SentinelConfig}; + +use super::config::{integration_config, test_config}; +use super::generators::cell_values; + +/// Pre-populate a sentinel with two distinct value ranges so the graph +/// creates cells and the analysis set is non-trivial. +pub fn seeded_sentinel() -> Sentinel128 { + let mut s = Sentinel128::new(test_config()).unwrap(); + // Two values with well-separated leading bits → distinct cells. + s.ingest(&[ + 0xF000_0000_0000_0000_0000_0000_0000_0001, + 0x1000_0000_0000_0000_0000_0000_0000_0002, + ]); + s +} + +// ─── ScenarioBuilder ─────────────────────────────────────── + +/// Builder for common test scenarios. +/// +/// Constructs a [`Sentinel128`] seeded with traffic across one +/// or more leading-nibble ranges and optionally warmed through +/// multiple batches. +pub struct ScenarioBuilder { + config: SentinelConfig, + seed_ranges: Vec<(u128, usize)>, + warm_batches: usize, +} + +impl ScenarioBuilder { + pub fn new() -> Self { + Self { + config: integration_config(), + seed_ranges: Vec::new(), + warm_batches: 0, + } + } + + pub fn config(mut self, cfg: SentinelConfig) -> Self { + self.config = cfg; + self + } + + /// Add a seed range: feed `count` sequential values with the + /// given leading `nibble` during initial seeding. + pub fn seed_range(mut self, nibble: u128, count: usize) -> Self { + self.seed_ranges.push((nibble, count)); + self + } + + /// Number of warm-up batches to run after seeding. + pub const fn warm_batches(mut self, n: usize) -> Self { + self.warm_batches = n; + self + } + + /// Build a sentinel that has been seeded and warmed. + pub fn build(self) -> Sentinel128 { + self.build_with_reports().0 + } + + /// Build and return both the sentinel and the warm-up reports. + pub fn build_with_reports(self) -> (Sentinel128, Vec>) { + let batch = self.make_batch(); + let mut s = Sentinel128::new(self.config).unwrap(); + let mut reports = Vec::new(); + + if !batch.is_empty() { + // Seed phase. + reports.push(s.ingest(&batch)); + + // Warm-up phase. + for _ in 0..self.warm_batches { + reports.push(s.ingest(&batch)); + } + } + + (s, reports) + } + + /// Flatten all seed ranges into a single batch of values. + fn make_batch(&self) -> Vec { + self.seed_ranges + .iter() + .flat_map(|&(nibble, count)| cell_values(nibble, count)) + .collect() + } +} diff --git a/packages/sentinel/tests/common/config.rs b/packages/sentinel/tests/common/config.rs new file mode 100644 index 000000000..0ab484e41 --- /dev/null +++ b/packages/sentinel/tests/common/config.rs @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +//! Sentinel configurations for tests. + +use torrust_sentinel::{NoiseSchedule, SentinelConfig, SvdStrategy}; + +const MATURITY_THRESHOLD: f64 = 0.01; + +/// Number of batch-indexed forgetting steps needed to leave warm-up. +pub fn batches_to_maturity(lambda: f64) -> usize { + (1_usize..=usize::MAX) + .scan(1.0, |influence, batch| { + *influence *= lambda; + Some((batch, *influence)) + }) + .find(|(_, influence)| *influence < MATURITY_THRESHOLD) + .map(|(batch, _)| batch) + .expect("a validated forgetting factor must cross the maturity threshold") +} + +/// Test config with faster EWMA parameters (ADR-S-012). +/// +/// Uses λ=0.90 (`forgetting_factor`) and `λ_s`=0.99 (`cusum_slow_decay`) +/// giving speed-separation ratio R = `W_s`/`W_f` = 100/10 = 10×, +/// matching the production ratio (λ=0.99, `λ_s`=0.999 → R=10×). +/// +/// Convergence properties at λ=0.90: +/// η < 0.50 after 7 batches (was 14 at λ=0.95) +/// η < 0.05 after 29 batches (was 59 at λ=0.95) +/// half-life `h_f` = 6.6 rounds (was 13.5) +/// +/// The slow EWMA at `λ_s`=0.99 has `h_s`=69 rounds, settling +/// to 87.5% after ~207 rounds (3× `h_s`). +pub fn test_config() -> SentinelConfig { + SentinelConfig:: { + max_rank: 4, + forgetting_factor: 0.90, + rank_update_interval: 10, + analysis_k: 16, + analysis_depth_cutoff: 6, + energy_threshold: 0.90, + eps: 1e-6, + per_sample_scores: true, + cusum_allowance_sigmas: 0.5, + cusum_slow_decay: 0.99, + cusum_coord_slow_decay: 0.99, + clip_sigmas: 3.0, + clip_pressure_decay: 0.95, + split_threshold: 100, + d_create: 3, + d_evict: 6, + budget: 100_000, + noise_schedule: NoiseSchedule::Explicit(vec![5]), + noise_batch_size: 4, + noise_seed: Some(42), + background_warming: false, + svd_strategy: SvdStrategy::Brand, + } +} + +/// Config with noise disabled — trackers start cold. +pub fn cold_config() -> SentinelConfig { + SentinelConfig:: { + noise_schedule: NoiseSchedule::Explicit(vec![]), + ..test_config() + } +} + +/// Config tuned for integration tests: tight rank so novelty +/// signals are clearer, deterministic noise, fast rank adaptation. +/// +/// Overrides from [`test_config()`]: +/// - `max_rank`: 4 → 2 +/// - `rank_update_interval`: 10 → 5 +/// - `per_sample_scores`: true → false +pub fn integration_config() -> SentinelConfig { + SentinelConfig:: { + max_rank: 2, + rank_update_interval: 5, + per_sample_scores: false, + ..test_config() + } +} diff --git a/packages/sentinel/tests/common/generators.rs b/packages/sentinel/tests/common/generators.rs new file mode 100644 index 000000000..57ba3efd9 --- /dev/null +++ b/packages/sentinel/tests/common/generators.rs @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +//! Value generators for test data. + +/// Generate `count` values in a narrow range with leading nibble +/// `nibble` and sequential low bits. +/// +/// The nibble is four bits wide, which is what the shift leaves room for: a +/// value of sixteen or above pushes its high bits off the top of the +/// coordinate and lands on the range sixteen below it, so a caller sweeping +/// past fifteen would revisit ranges it believed were new. The assertion +/// refuses that rather than letting the aliasing pass as traffic, and it +/// refuses it in every build: a release test run is a routine way to exercise +/// this suite, and a check that is compiled out of the build where the timings +/// are taken is no check at all — the aliasing would pass there as the very +/// concentrated traffic the caller was trying not to generate. +/// [`cell_values_prefix`] is the generator for a wider sweep. +pub fn cell_values(nibble: u128, count: usize) -> Vec { + assert!( + nibble < 16, + "cell_values takes a four-bit nibble; cell_values_prefix addresses a wider sweep" + ); + (0..count).map(|i| (nibble << 124) | (i as u128 + 1)).collect() +} + +/// Generate `count` values in a narrow range with leading six-bit `prefix` +/// and sequential low bits. +/// +/// Six bits address sixty-four leading ranges, each a $2^{122}$-wide +/// interval, so a sweep over `0..64` reaches sixty-four ranges that differ +/// inside their first six bits. The four-bit generator cannot express such a +/// sweep: past fifteen its ranges repeat, and a spray that believed it was +/// touching sixty-four ranges would be touching sixteen of them four times +/// each — traffic concentrated enough to build the very structure the spray +/// was meant to spread thin. A prefix past sixty-three aliases the same way +/// one bit up, so it is refused here rather than generated, in every build for +/// the reason [`cell_values`] gives. +pub fn cell_values_prefix(prefix: u128, count: usize) -> Vec { + assert!(prefix < 64, "cell_values_prefix takes a six-bit prefix"); + (0..count).map(|i| (prefix << 122) | (i as u128 + 1)).collect() +} + +/// Generate values with a dense bit pattern to create +/// structurally novel data relative to [`cell_values()`]. +/// +/// The leading nibble addresses ranges the same way it does there, and is +/// refused past fifteen for the same reason and in every build. +pub fn anomalous_values(nibble: u128, count: usize) -> Vec { + assert!(nibble < 16, "anomalous_values takes a four-bit nibble"); + // Set a dense block of high bits in the middle — structurally + // very different from the sparse sequential values above. + (0..count) + .map(|i| (nibble << 124) | 0x0FFF_FFFF_FFFF_FFFF_FFFF_FFFF_0000_0000 | (i as u128)) + .collect() +} diff --git a/packages/sentinel/tests/common/mod.rs b/packages/sentinel/tests/common/mod.rs new file mode 100644 index 000000000..4b528aeaa --- /dev/null +++ b/packages/sentinel/tests/common/mod.rs @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +//! Shared helpers for Sentinel integration tests. +//! +//! # Submodules +//! +//! | Module | Contents | +//! |-----------------|-------------------------------------------------------------| +//! | [`assertions`] | [`assert_invariants()`], [`assert_reports_identical()`], [`max_cusum()`], [`max_novelty_z()`], [`root_novelty_z()`] | +//! | [`builders`] | [`seeded_sentinel()`], [`ScenarioBuilder`] | +//! | [`config`] | [`test_config()`], [`cold_config()`], [`integration_config()`], [`batches_to_maturity()`] | +//! | [`generators`] | [`cell_values()`], [`cell_values_prefix()`], [`anomalous_values()`] | + +// Each integration test file includes this module independently, so +// not every test file uses every helper. +#![allow(dead_code, unused_imports)] + +mod assertions; +mod builders; +mod config; +mod generators; + +pub use assertions::{assert_invariants, assert_reports_identical, max_cusum, max_novelty_z, root_novelty_z}; +pub use builders::{ScenarioBuilder, seeded_sentinel}; +pub use config::{batches_to_maturity, cold_config, integration_config, test_config}; +pub use generators::{anomalous_values, cell_values, cell_values_prefix}; diff --git a/packages/sentinel/tests/coverage_matrix.rs b/packages/sentinel/tests/coverage_matrix.rs new file mode 100644 index 000000000..c6a0dc03e --- /dev/null +++ b/packages/sentinel/tests/coverage_matrix.rs @@ -0,0 +1,334 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +//! The coverage matrix — which combinations of structure and state the +//! sentinel is exercised against (§ALGO S-17.6). +//! +//! A disturbance can differ along two axes at once, and the two are +//! independent. It has an *extent* — confined to one range, spread over some +//! of them, or present everywhere — and it has an *onset*, either arriving all +//! at once or creeping in over a run of batches. Crossing the two gives the +//! modalities a deployed sentinel has to survive, and each of them stresses a +//! different part of the design: extent decides at which level of the chain +//! the disturbance is visible, while onset decides whether a single batch +//! comparison suffices or whether the evidence has to be carried forward and +//! summed. +//! +//! The matrix matters because the failure modes are complementary. A design +//! tuned only to per-cell comparison catches the sharp local case and is blind +//! to a slow shift that each batch alone could pass off as ordinary; a design +//! that only watches the whole domain misses what one range is doing. Covering +//! the grid is what says the two mechanisms compose rather than each covering +//! for the other's blind spot on the examples that happened to be written. +//! +//! Every case here asks the same question of the sentinel: is the measurement +//! after the disturbance higher than what this same sentinel produced under +//! steady traffic moments before? None asks whether a threshold was crossed. +//! What counts as alarming depends on the deployment and belongs to the host; +//! the sentinel's obligation is that the number move in the right direction +//! and by an amount the host can reason about (ADR-S-001). +//! +//! # Test index +//! +//! | Test | Area | Claim | +//! |------|------|-------| +//! | [`sudden_single_cell_z_score`] | coverage | The sharpest corner of the matrix: a single range switching abruptly to structurally different values pushes the highest per-cell score above what the same sentinel measured on the preceding ordinary batch. One batch is enough here, because the disturbance is a departure from learned structure rather than a change in how often the range is visited. | +//! | [`gradual_single_cell_cusum`] | coverage | Persistence is itself evidence. Where the anomalous traffic keeps arriving batch after batch, the accumulating statistic climbs above where it stood on the first such batch, so a disturbance that never grows louder still grows more certain. The comparison is taken only after the slow baseline has been allowed to settle, which is what separates genuine accumulation from the transient left over from warm-up. | +//! | [`sudden_partial_per_cell`] | coverage | Normal traffic elsewhere does not dilute an anomaly. With several ranges live and only some of them turning anomalous, the highest per-cell score still rises above the steady-state batch — the untouched ranges contribute their own unremarkable readings and nothing else. Because scoring is per-cell rather than an average over the domain, an attacker gains nothing by keeping most of the traffic ordinary. | +//! | [`gradual_partial_cusum`] | coverage | The two evasions combined — a drift rather than a jump, and in only part of the traffic — still accumulate: with one range going anomalous batch after batch while another stays ordinary, the accumulated statistic ends above where the first anomalous batch left it. Being quiet and being partial are not additive protections, because accumulation happens per cell and the ordinary range accumulates nothing to average it away. | +//! | [`sudden_system_wide_root_catches`] | coverage | The case a per-cell view is least equipped for: every range changes at once, so no cell is unusual relative to its neighbours, and the reading that moves is the root's. Coverage of this corner is what the coarse end of the chain exists for — an attack that shifts the whole population is caught by the model whose region is the whole population. | +//! | [`gradual_system_wide_root_cusum`] | coverage | The quietest corner of the matrix, and the one a self-adjusting baseline is most at risk of absorbing: a shift across all ranges that simply keeps happening. The accumulated statistic keeps climbing past its first-batch value rather than settling, so persistence still tells even where extent leaves no cell looking unusual against its neighbours and no batch looks unusual against the last. | + +mod common; + +use common::{ + ScenarioBuilder, anomalous_values, assert_invariants, cell_values, integration_config, max_cusum, max_novelty_z, + root_novelty_z, +}; +use torrust_sentinel::SentinelConfig; + +// ── 1. sudden_single_cell_z_score ────────────────────────── + +/// The sharpest corner of the matrix: a single range switching abruptly to +/// structurally different values pushes the highest per-cell score above what +/// the same sentinel measured on the preceding ordinary batch. One batch is +/// enough here, because the disturbance is a departure from learned structure +/// rather than a change in how often the range is visited. +/// +/// ´claim:coverage:a-sudden-change-in-one-range-shows-up-in-a-single-batch-comparison´ +/// ´test:integration:sudden-single-cell-z-score´ +#[test] +fn sudden_single_cell_z_score() { + let mut s = ScenarioBuilder::new() + .config(SentinelConfig:: { + split_threshold: 10, + ..integration_config() + }) + .seed_range(0xA, 20) + .warm_batches(5) + .build(); + + // Steady-state reference. + let normal = s.ingest(&cell_values(0xA, 8)); + assert_invariants(&s, &normal); + + // Anomalous batch — structurally different data. + let anomaly = s.ingest(&anomalous_values(0xA, 8)); + assert_invariants(&s, &anomaly); + + let normal_z = max_novelty_z(&normal); + let anomaly_z = max_novelty_z(&anomaly); + + assert!( + anomaly_z > normal_z, + "sudden single-cell anomaly should produce higher z-score: \ + anomaly={anomaly_z:.4}, normal={normal_z:.4}" + ); +} + +// ── 2. gradual_single_cell_cusum ─────────────────────────── + +/// Persistence is itself evidence. Where the anomalous traffic keeps arriving +/// batch after batch, the accumulating statistic climbs above where it stood +/// on the first such batch, so a disturbance that never grows louder still +/// grows more certain. The comparison is taken only after the slow baseline +/// has been allowed to settle, which is what separates genuine accumulation +/// from the transient left over from warm-up. +/// +/// ´claim:coverage:a-disturbance-that-persists-keeps-accumulating-even-when-it-never-grows-louder´ +/// ´test:integration:gradual-single-cell-cusum´ +#[test] +fn gradual_single_cell_cusum() { + let cfg = SentinelConfig:: { + max_rank: 1, + cusum_slow_decay: 0.96, + cusum_coord_slow_decay: 0.96, + split_threshold: 10, + ..integration_config() + }; + let mut s = ScenarioBuilder::new().config(cfg).seed_range(0xA, 20).warm_batches(8).build(); + + // Stabilise slow EWMA (λ_s=0.96, h_s≈17) before measuring. + for _ in 0..20 { + s.ingest(&cell_values(0xA, 8)); + } + + // Record CUSUM from first anomaly batch as baseline. + let first_report = s.ingest(&anomalous_values(0xA, 8)); + assert_invariants(&s, &first_report); + let cusum_first = max_cusum(&first_report); + + // Gradual anomaly: 4 more batches of anomalous traffic. + let mut cusum_after = cusum_first; + for _ in 0..4 { + let report = s.ingest(&anomalous_values(0xA, 8)); + cusum_after = max_cusum(&report); + } + + assert!( + cusum_after > cusum_first, + "CUSUM should accumulate under gradual anomaly: \ + first={cusum_first:.4}, last={cusum_after:.4}" + ); +} + +// ── 3. sudden_partial_per_cell ───────────────────────────── + +/// Normal traffic elsewhere does not dilute an anomaly. With several ranges +/// live and only some of them turning anomalous, the highest per-cell score +/// still rises above the steady-state batch — the untouched ranges contribute +/// their own unremarkable readings and nothing else. Because scoring is +/// per-cell rather than an average over the domain, an attacker gains nothing +/// by keeping most of the traffic ordinary. +/// +/// ´claim:coverage:normal-traffic-elsewhere-does-not-mask-an-anomaly-in-some-ranges´ +/// ´test:integration:sudden-partial-per-cell´ +#[test] +fn sudden_partial_per_cell() { + let mut s = ScenarioBuilder::new() + .config(SentinelConfig:: { + split_threshold: 10, + ..integration_config() + }) + .seed_range(0xA, 20) + .seed_range(0xB, 20) + .seed_range(0xC, 20) + .seed_range(0xD, 20) + .warm_batches(5) + .build(); + + // Steady-state reference across all ranges. + let normal = s.ingest( + &[ + cell_values(0xA, 8), + cell_values(0xB, 8), + cell_values(0xC, 8), + cell_values(0xD, 8), + ] + .concat(), + ); + assert_invariants(&s, &normal); + + // Partial anomaly: anomalous traffic only to ranges A and B. + let partial = s.ingest( + &[ + anomalous_values(0xA, 8), + anomalous_values(0xB, 8), + cell_values(0xC, 8), + cell_values(0xD, 8), + ] + .concat(), + ); + assert_invariants(&s, &partial); + + let normal_z = max_novelty_z(&normal); + let partial_z = max_novelty_z(&partial); + + assert!( + partial_z > normal_z, + "partial anomaly should produce elevated z-scores: \ + partial={partial_z:.4}, normal={normal_z:.4}" + ); +} + +// ── 4. gradual_partial_cusum ─────────────────────────────── + +/// The two evasions combined — a drift rather than a jump, and in only part of +/// the traffic — still accumulate: with one range going anomalous batch after +/// batch while another stays ordinary, the accumulated statistic ends above +/// where the first anomalous batch left it. Being quiet and being partial are +/// not additive protections, because accumulation happens per cell and the +/// ordinary range accumulates nothing to average it away. +/// +/// ´claim:coverage:being-both-gradual-and-partial-does-not-stop-the-evidence-accumulating´ +/// ´test:integration:gradual-partial-cusum´ +#[test] +fn gradual_partial_cusum() { + let cfg = SentinelConfig:: { + cusum_slow_decay: 0.96, + cusum_coord_slow_decay: 0.96, + split_threshold: 10, + ..integration_config() + }; + let mut s = ScenarioBuilder::new() + .config(cfg) + .seed_range(0xA, 20) + .seed_range(0xB, 20) + .warm_batches(8) + .build(); + + // Stabilise: let slow EWMA (λ_s=0.96, h_s≈17) catch up so CUSUM + // from the warm-up transient settles before we measure. + for _ in 0..20 { + s.ingest(&[cell_values(0xA, 8), cell_values(0xB, 8)].concat()); + } + + // Record CUSUM from first anomaly batch as baseline. + let first_anomaly = s.ingest(&[anomalous_values(0xA, 8), cell_values(0xB, 8)].concat()); + assert_invariants(&s, &first_anomaly); + let cusum_first = max_cusum(&first_anomaly); + + // Gradual partial: anomalous traffic to range A only, normal to B. + let mut cusum_after = cusum_first; + for _ in 0..4 { + let report = s.ingest(&[anomalous_values(0xA, 8), cell_values(0xB, 8)].concat()); + cusum_after = max_cusum(&report); + } + + assert!( + cusum_after > cusum_first, + "gradual partial anomaly should accumulate CUSUM: \ + first={cusum_first:.4}, last={cusum_after:.4}" + ); +} + +// ── 5. sudden_system_wide_root_catches ───────────────────── + +/// The case a per-cell view is least equipped for: every range changes at +/// once, so no cell is unusual relative to its neighbours, and the reading +/// that moves is the root's. Coverage of this corner is what the coarse end of +/// the chain exists for — an attack that shifts the whole population is caught +/// by the model whose region is the whole population. +/// +/// ´claim:coverage:a-change-in-every-range-at-once-is-caught-at-the-root´ +/// ´test:integration:sudden-system-wide-root-catches´ +#[test] +fn sudden_system_wide_root_catches() { + let mut s = ScenarioBuilder::new() + .config(SentinelConfig:: { + split_threshold: 10, + ..integration_config() + }) + .seed_range(0xA, 20) + .seed_range(0xB, 20) + .seed_range(0xC, 20) + .warm_batches(5) + .build(); + + // Normal reference. + let normal = s.ingest(&[cell_values(0xA, 8), cell_values(0xB, 8), cell_values(0xC, 8)].concat()); + assert_invariants(&s, &normal); + + // System-wide anomaly: all ranges anomalous. + let anomaly = s.ingest(&[anomalous_values(0xA, 8), anomalous_values(0xB, 8), anomalous_values(0xC, 8)].concat()); + assert_invariants(&s, &anomaly); + + let normal_root_z = root_novelty_z(&normal); + let anomaly_root_z = root_novelty_z(&anomaly); + + assert!( + anomaly_root_z > normal_root_z, + "system-wide anomaly should elevate root z-score: \ + anomaly={anomaly_root_z:.4}, normal={normal_root_z:.4}" + ); +} + +// ── 6. gradual_system_wide_root_cusum ────────────────────── + +/// The quietest corner of the matrix, and the one a self-adjusting baseline is +/// most at risk of absorbing: a shift across all ranges that simply keeps +/// happening. The accumulated statistic keeps climbing past its first-batch +/// value rather than settling, so persistence still tells even where extent +/// leaves no cell looking unusual against its neighbours and no batch looks +/// unusual against the last. +/// +/// ´claim:coverage:a-slow-shift-across-everything-accumulates-instead-of-becoming-the-new-normal´ +/// ´test:integration:gradual-system-wide-root-cusum´ +#[test] +fn gradual_system_wide_root_cusum() { + let cfg = SentinelConfig:: { + cusum_slow_decay: 0.96, + cusum_coord_slow_decay: 0.96, + split_threshold: 10, + ..integration_config() + }; + let mut s = ScenarioBuilder::new() + .config(cfg) + .seed_range(0xA, 20) + .seed_range(0xB, 20) + .warm_batches(8) + .build(); + + // Stabilise slow EWMA before measuring. + for _ in 0..20 { + s.ingest(&[cell_values(0xA, 8), cell_values(0xB, 8)].concat()); + } + + // Record CUSUM from first anomaly batch. + let first_report = s.ingest(&[anomalous_values(0xA, 8), anomalous_values(0xB, 8)].concat()); + assert_invariants(&s, &first_report); + let cusum_first = max_cusum(&first_report); + + // Gradual system-wide: all ranges anomalous for 4 more batches. + let mut cusum_after = cusum_first; + for _ in 0..4 { + let report = s.ingest(&[anomalous_values(0xA, 8), anomalous_values(0xB, 8)].concat()); + cusum_after = max_cusum(&report); + } + + assert!( + cusum_after > cusum_first, + "gradual system-wide anomaly should accumulate CUSUM: \ + first={cusum_first:.4}, last={cusum_after:.4}" + ); +} diff --git a/packages/sentinel/tests/deferred_warmup.rs b/packages/sentinel/tests/deferred_warmup.rs new file mode 100644 index 000000000..660e14720 --- /dev/null +++ b/packages/sentinel/tests/deferred_warmup.rs @@ -0,0 +1,781 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +//! # Test index +//! +//! | Test | Area | Claim | +//! |------|------|-------| +//! | [`step2_invariants_hold_throughout_lifecycle`] | warmup | Deferring warm-up changes when a cell becomes live, not what a report is allowed to look like. Across a long run of splitting traffic — cells entering staging, warming, and being promoted mid-run — every batch report still satisfies the structural guarantees. A cell in staging is simply absent from the producing set until it is ready, so no half-built model can leak into the output. | +//! | [`step2_lifecycle_new_cells_appear_after_splits`] | warmup | A sentinel that starts as root alone ends a run of splitting traffic with several live cells, and every one of them carries noise observations behind it. Promotion is conditional on the warm-up schedule being finished, so the live map never contains a cell that skipped its seeding — staging is a queue on the way in, not an alternative way in. | +//! | [`step2_determinism_staging_path_matches_twin`] | warmup | Two sentinels built from the same configuration and the same noise seed, fed the same batches, agree on their reports down to the bits of the score means. The synchronous drain warms queued cells in a deterministic order from a seeded generator, so the staging detour introduces no freedom: an investigation can be replayed exactly rather than approximately. | +//! | [`step2_ancestor_receives_observations_for_warming_cells`] | warmup | Deferring a cell costs no coverage. Every value routes to each live cell whose interval contains it, and the root's interval contains all of them, so the root reports a non-zero sample count whatever is queued below it. A region under construction is therefore still watched — at coarser resolution, by the ancestor chain — rather than unobserved until its cell is ready. | +//! | [`step2_eviction_during_reconcile_no_panic`] | warmup | A cell can lose its place before it ever takes it. Under a tight budget and scattered traffic, cells are created and evicted continuously, and a cell still warming when its G-node leaves the analysis set is dropped from staging rather than promoted into a set it no longer belongs to. Work already spent on it is abandoned, which is cheaper than admitting a cell the selector has rejected. | +//! | [`step2_reset_clears_staging_and_resumes`] | warmup | A reset returns the sentinel to the state it was constructed in — root alone, no accumulated observations — and that includes emptying the staging area, so no cell queued under the old structure can surface under the new one. What follows is a genuine second warm-up: further traffic splits the space and builds cells again from nothing. | +//! | [`step3_invariants_hold_with_background_warming`] | warmup | cites (´claim:warmup:deferring-warm-up-through-a-staging-area-never-lets-a-half-built-cell-into-a-report´) | +//! | [`step3_warmup_completes_eventually`] | warmup | cites (´claim:warmup:a-cell-reaches-the-live-map-only-after-its-noise-schedule-is-complete´) | +//! | [`step3_scoring_works_after_background_warmup`] | warmup | A tracker warmed on another thread scores like any other: after a warmed run, every cell and ancestor in the batch reports finite score means. That is what the seeding is for — a model with no baseline would divide by a spread it does not have — and it holds whichever thread supplied the seed. | +//! | [`step3_background_same_cell_structure_as_sync`] | warmup | Two sentinels fed identical traffic, one warming inline and one on a background thread, end with the same accumulated volume and the same set of tracked cells. Structure is decided by observation volume alone and never by anything the trackers compute, so warm-up scheduling — a modelling concern — cannot perturb it. Choosing the background path is a latency decision, not a modelling one. | +//! | [`step3_higher_volume_cells_warm_via_priority`] | warmup | When warming capacity is scarce — a long noise schedule and small batches leave the queue in progress — the staging area serves its waiting cells in descending order of volume, so a heavily trafficked subtree gets cells into service before a quiet one. Warming effort is spent where the traffic is, which is the same principle that decided the cells were worth having. | +//! | [`step3_concurrent_ingest_no_panic`] | warmup | Ingestion and background warming genuinely overlap: a long run of diverse traffic keeps creating cells while the thread is warming earlier ones, and the two never collide. A cell being warmed is checked out of the staging area for the duration, so the expensive work happens on a cell no other thread can reach, and the lock is held only for the queue operations around it. | +//! | [`step3_eviction_during_background_warming_no_panic`] | warmup | cites (´claim:warmup:a-warming-cell-whose-node-leaves-the-analysis-set-is-discarded-rather-than-promoted´) | +//! | [`step3_reset_restarts_background_thread`] | warmup | cites (´claim:warmup:a-reset-empties-the-staging-area-so-warm-up-begins-again-from-the-root-alone´) | +//! | [`step3_drop_while_warming_no_hang`] | warmup | Dropping a sentinel with a long noise schedule still outstanding returns promptly instead of waiting for the queue to empty. The warming thread checks for shutdown between batches and abandons whatever remains, because cells nobody will ever read from are not worth finishing. Teardown costs at most one batch of work, not the rest of the schedule. | + +//! Deferred warm-up (§ALGO S-11.6): paying for a new cell off the hot path. +//! +//! Seeding a new cell's tracker with synthetic noise is the expensive part of +//! creating one, and it comes due at exactly the wrong moment — a split is +//! triggered by a surge of traffic, so the cost lands while the sentinel is +//! busiest. Deferred warm-up moves it aside. A newly selected cell is not +//! built straight into the live map; it waits in a staging area, is warmed +//! round by round, and is promoted only once its schedule is complete. Until +//! then it counts towards the investment set but takes no real observations, +//! and the values that would have reached it are still seen by its ancestors. +//! +//! Two paths drain the staging area, and these tests exercise both end to +//! end. The synchronous drain runs inside reconciliation and finishes every +//! queued cell before the batch is scored, which keeps the whole pipeline +//! reproducible from a seed. The background thread instead takes the +//! highest-volume waiting cell out of the queue, injects noise without +//! holding the lock, and returns it — so warming overlaps ingestion rather +//! than blocking it, and the busiest cells come into service first. Which +//! path ran is meant to be invisible in the structure that results, because +//! that structure is decided by observation volume alone. +//! +//! The awkward moments are the ones worth pinning down: a cell evicted while +//! it is still warming, a reset that has to tear the thread down and stand it +//! back up, and a drop that must not wait on warming work nobody will read. + +mod common; + +use common::{cell_values, integration_config}; +use torrust_sentinel::{BatchReport, NoiseSchedule, Sentinel128, SentinelConfig}; + +// ════════════════════════════════════════════════════════════════ +// Helpers +// ════════════════════════════════════════════════════════════════ + +/// Invariant checks for deferred warm-up tests. +/// +/// This is a variant of [`common::assert_invariants`] that omits the +/// Steiner tree bound (`full_size <= 2 * competitive_size + 1`). +/// Deep-split configurations (low `split_threshold`) can transiently +/// violate the bound during rapid cell creation, so it is excluded +/// here. All other invariants from the common version are checked. +fn assert_deferred_invariants(sentinel: &Sentinel128, report: &BatchReport) { + let summary = &report.analysis_set_summary; + + // Competitive cap: competitive_size <= analysis_k. + assert!( + summary.competitive_size <= sentinel.config().analysis_k, + "competitive set {} exceeds K={}", + summary.competitive_size, + sentinel.config().analysis_k, + ); + + // Root at depth 0 is always in the full set. + if summary.full_size > 0 { + assert_eq!(summary.depth_range.0, 0, "root (depth 0) must be in the full analysis set"); + } + + // cell_reports are competitive only. + for cr in &report.cell_reports { + assert!( + cr.is_competitive, + "cell_reports entry at depth {} is not competitive", + cr.depth + ); + } + + // ancestor_reports are non-competitive. + for ar in &report.ancestor_reports { + assert!( + !ar.is_competitive, + "ancestor_reports entry at depth {} is competitive", + ar.depth + ); + } + + // Deterministic ordering: reports sorted by gnode_id ascending. + for window in report.cell_reports.windows(2) { + assert!( + window[0].gnode_id < window[1].gnode_id, + "cell_reports not sorted by GNodeId: {:?} >= {:?}", + window[0].gnode_id, + window[1].gnode_id, + ); + } + for window in report.ancestor_reports.windows(2) { + assert!( + window[0].gnode_id < window[1].gnode_id, + "ancestor_reports not sorted by GNodeId: {:?} >= {:?}", + window[0].gnode_id, + window[1].gnode_id, + ); + } + + // Coordination reports have no duplicate GNodeIds. + { + let mut seen = std::collections::HashSet::new(); + for cr in &report.coordination_reports { + assert!( + seen.insert(cr.gnode_id), + "duplicate GNodeId in coordination_reports: {:?}", + cr.gnode_id, + ); + } + } + + // No NaN in score fields (novelty, displacement, surprise). + for cr in report.cell_reports.iter().chain(report.ancestor_reports.iter()) { + assert!(!cr.scores.novelty.mean.is_nan(), "NaN novelty mean at depth {}", cr.depth); + assert!( + !cr.scores.displacement.mean.is_nan(), + "NaN displacement mean at depth {}", + cr.depth, + ); + assert!(!cr.scores.surprise.mean.is_nan(), "NaN surprise mean at depth {}", cr.depth); + } + + // analysis_width == 128 - depth for every cell/ancestor report. + for cr in report.cell_reports.iter().chain(report.ancestor_reports.iter()) { + assert_eq!( + cr.analysis_width, + 128 - cr.depth as usize, + "analysis_width mismatch at depth {}", + cr.depth, + ); + } +} + +/// Config that triggers splits quickly so new cells are created. +fn fast_split_config() -> SentinelConfig { + SentinelConfig:: { + split_threshold: 10, + analysis_k: 16, + analysis_depth_cutoff: 6, + noise_schedule: NoiseSchedule::Explicit(vec![3]), + noise_batch_size: 4, + noise_seed: Some(42), + background_warming: false, + ..integration_config() + } +} + +/// Config with background warming enabled (otherwise identical to +/// [`fast_split_config`]). +fn background_config() -> SentinelConfig { + SentinelConfig:: { + background_warming: true, + ..fast_split_config() + } +} + +// ════════════════════════════════════════════════════════════════ +// Step 2 — Synchronous Deferred Warm-Up +// ════════════════════════════════════════════════════════════════ + +// ── 2.8a: Invariants ──────────────────────────────────────── + +/// Deferring warm-up changes when a cell becomes live, not what a report is +/// allowed to look like. Across a long run of splitting traffic — cells +/// entering staging, warming, and being promoted mid-run — every batch report +/// still satisfies the structural guarantees. A cell in staging is simply +/// absent from the producing set until it is ready, so no half-built model +/// can leak into the output. +/// +/// ´claim:warmup:deferring-warm-up-through-a-staging-area-never-lets-a-half-built-cell-into-a-report´ +/// ´test:integration:step2-invariants-hold-throughout-lifecycle´ +#[test] +fn step2_invariants_hold_throughout_lifecycle() { + let mut s = Sentinel128::new(fast_split_config()).unwrap(); + + for _ in 0..15 { + let batch: Vec = [cell_values(0xA, 16), cell_values(0x5, 16)].concat(); + let report = s.ingest(&batch); + assert_deferred_invariants(&s, &report); + } +} + +// ── 2.8b: Lifecycle ───────────────────────────────────────── + +/// A sentinel that starts as root alone ends a run of splitting traffic with +/// several live cells, and every one of them carries noise observations +/// behind it. Promotion is conditional on the warm-up schedule being +/// finished, so the live map never contains a cell that skipped its seeding — +/// staging is a queue on the way in, not an alternative way in. +/// +/// ´claim:warmup:a-cell-reaches-the-live-map-only-after-its-noise-schedule-is-complete´ +/// ´test:integration:step2-lifecycle-new-cells-appear-after-splits´ +#[test] +fn step2_lifecycle_new_cells_appear_after_splits() { + let mut s = Sentinel128::new(fast_split_config()).unwrap(); + assert_eq!(s.cells_tracked(), 1, "initially only root"); + + // Feed diverse traffic to trigger splits and cell creation. + for _ in 0..15 { + let batch: Vec = [cell_values(0xA, 20), cell_values(0x5, 20)].concat(); + s.ingest(&batch); + } + + // After enough traffic, new cells should have been created via + // the staging area and promoted into the live cells map. + assert!( + s.cells_tracked() > 1, + "after splits, more than just root should be tracked (got {})", + s.cells_tracked(), + ); + + // All tracked cells should be noise-warmed. + for &gnode in &s.cell_gnodes() { + let insp = s.inspect_cell(gnode).unwrap(); + assert!( + insp.maturity.noise_observations > 0, + "cell {:?} at depth {} should have noise observations", + gnode, + insp.depth, + ); + } +} + +// ── 2.8c: Determinism ────────────────────────────────────── + +/// Two sentinels built from the same configuration and the same noise seed, +/// fed the same batches, agree on their reports down to the bits of the score +/// means. The synchronous drain warms queued cells in a deterministic order +/// from a seeded generator, so the staging detour introduces no freedom: an +/// investigation can be replayed exactly rather than approximately. +/// +/// ´claim:warmup:the-synchronous-staging-path-is-fully-determined-by-the-seed-so-twin-sentinels-agree-bit-for-bit´ +/// ´test:integration:step2-determinism-staging-path-matches-twin´ +#[test] +fn step2_determinism_staging_path_matches_twin() { + // Two sentinels with identical config and seed should produce + // bit-identical reports when using the synchronous staging path. + let cfg = fast_split_config(); + + let mut s1 = Sentinel128::new(cfg.clone()).unwrap(); + let mut s2 = Sentinel128::new(cfg).unwrap(); + + let values: Vec = [cell_values(0xA, 50), cell_values(0x5, 50)].concat(); + + for chunk in values.chunks(20) { + let r1 = s1.ingest(chunk); + let r2 = s2.ingest(chunk); + + assert_eq!(r1.cell_reports.len(), r2.cell_reports.len(), "cell report count mismatch"); + assert_eq!( + r1.ancestor_reports.len(), + r2.ancestor_reports.len(), + "ancestor report count mismatch", + ); + assert_eq!( + r1.coordination_reports.len(), + r2.coordination_reports.len(), + "coordination report count mismatch", + ); + assert_eq!( + r1.health.lifetime_observations, r2.health.lifetime_observations, + "lifetime observations mismatch", + ); + + // Bit-exact score comparison. + for (c1, c2) in r1.cell_reports.iter().zip(&r2.cell_reports) { + assert_eq!( + c1.scores.novelty.mean.to_bits(), + c2.scores.novelty.mean.to_bits(), + "novelty mean diverged", + ); + assert_eq!( + c1.scores.displacement.mean.to_bits(), + c2.scores.displacement.mean.to_bits(), + "displacement mean diverged", + ); + } + + for (a1, a2) in r1.ancestor_reports.iter().zip(&r2.ancestor_reports) { + assert_eq!( + a1.scores.novelty.mean.to_bits(), + a2.scores.novelty.mean.to_bits(), + "ancestor novelty mean diverged", + ); + } + } +} + +// ── 2.8d: Ancestor routing ───────────────────────────────── + +/// Deferring a cell costs no coverage. Every value routes to each live cell +/// whose interval contains it, and the root's interval contains all of them, +/// so the root reports a non-zero sample count whatever is queued below it. +/// A region under construction is therefore still watched — at coarser +/// resolution, by the ancestor chain — rather than unobserved until its cell +/// is ready. +/// +/// ´claim:warmup:no-region-goes-unwatched-while-its-cell-warms-because-the-ancestor-chain-still-observes-it´ +/// ´test:integration:step2-ancestor-receives-observations-for-warming-cells´ +#[test] +fn step2_ancestor_receives_observations_for_warming_cells() { + // With synchronous drain, warming cells are promoted within the + // same ingest() call. In all cases the root (an ancestor of + // everything) must receive every observation. + let mut s = Sentinel128::new(fast_split_config()).unwrap(); + + // Seed to create cells. + for _ in 0..10 { + s.ingest(&cell_values(0xA, 20)); + } + + // Now ingest a fresh batch and verify the root has observations. + let report = s.ingest(&cell_values(0xA, 8)); + let root_report = report.ancestor_reports.iter().find(|cr| cr.depth == 0); + assert!(root_report.is_some(), "root cell should receive observations as ancestor"); + assert!( + root_report.unwrap().sample_count > 0, + "root should have non-zero sample_count", + ); +} + +// ── 2.8e: Eviction ───────────────────────────────────────── + +/// A cell can lose its place before it ever takes it. Under a tight budget +/// and scattered traffic, cells are created and evicted continuously, and a +/// cell still warming when its G-node leaves the analysis set is dropped from +/// staging rather than promoted into a set it no longer belongs to. Work +/// already spent on it is abandoned, which is cheaper than admitting a cell +/// the selector has rejected. +/// +/// ´claim:warmup:a-warming-cell-whose-node-leaves-the-analysis-set-is-discarded-rather-than-promoted´ +/// ´test:integration:step2-eviction-during-reconcile-no-panic´ +#[test] +fn step2_eviction_during_reconcile_no_panic() { + // Create a sentinel with a small budget so cells get evicted. + let cfg = SentinelConfig:: { + split_threshold: 5, + budget: 50, + d_evict: 4, + ..fast_split_config() + }; + let mut s = Sentinel128::new(cfg).unwrap(); + + // Feed lots of diverse traffic — cells will be created and evicted. + for _ in 0..20 { + let batch: Vec = [ + cell_values(0xA, 10), + cell_values(0x5, 10), + cell_values(0x2, 10), + cell_values(0xE, 10), + ] + .concat(); + let report = s.ingest(&batch); + assert_deferred_invariants(&s, &report); + } +} + +// ── 2.8f: Reset ───────────────────────────────────────────── + +/// A reset returns the sentinel to the state it was constructed in — root +/// alone, no accumulated observations — and that includes emptying the +/// staging area, so no cell queued under the old structure can surface under +/// the new one. What follows is a genuine second warm-up: further traffic +/// splits the space and builds cells again from nothing. +/// +/// ´claim:warmup:a-reset-empties-the-staging-area-so-warm-up-begins-again-from-the-root-alone´ +/// ´test:integration:step2-reset-clears-staging-and-resumes´ +#[test] +fn step2_reset_clears_staging_and_resumes() { + let mut s = Sentinel128::new(fast_split_config()).unwrap(); + + // Build up state. + for _ in 0..10 { + s.ingest(&cell_values(0xA, 20)); + } + assert!(s.cells_tracked() > 1); + + s.reset(); + assert_eq!(s.cells_tracked(), 1, "after reset, only root"); + assert_eq!(s.lifetime_observations(), 0); + + // Resume ingestion — new cells should be created again. + for _ in 0..15 { + let batch: Vec = [cell_values(0xA, 20), cell_values(0x5, 20)].concat(); + s.ingest(&batch); + } + assert!( + s.cells_tracked() > 1, + "after reset + re-ingest, more than just root should be tracked (got {})", + s.cells_tracked(), + ); +} + +// ════════════════════════════════════════════════════════════════ +// Step 3 — Background Warming Thread +// ════════════════════════════════════════════════════════════════ + +// ── 3.6a: Invariants ──────────────────────────────────────── + +/// The same holds when a separate thread is doing the warming and cells may +/// be promoted between batches rather than within one. Whichever thread +/// finished a cell, the report published by an ingest describes only cells +/// already in service. +/// +/// (´claim:warmup:deferring-warm-up-through-a-staging-area-never-lets-a-half-built-cell-into-a-report´) +/// ´test:integration:step3-invariants-hold-with-background-warming´ +#[test] +fn step3_invariants_hold_with_background_warming() { + let mut s = Sentinel128::new(background_config()).unwrap(); + + // Give background thread time to warm cells between ingests. + for _ in 0..15 { + let batch: Vec = [cell_values(0xA, 16), cell_values(0x5, 16)].concat(); + let report = s.ingest(&batch); + assert_deferred_invariants(&s, &report); + std::thread::sleep(std::time::Duration::from_millis(2)); + } +} + +// ── 3.6b: Warm-up completes eventually ───────────────────── + +/// Handing warm-up to a background thread weakens the timing but not the +/// condition. Given a run of further batches to work through, every cell that +/// has reached the live map carries its noise observations, so the queue +/// drains rather than stalling: deferral postpones seeding, it does not +/// permit a cell to be promoted without it. +/// +/// (´claim:warmup:a-cell-reaches-the-live-map-only-after-its-noise-schedule-is-complete´) +/// ´test:integration:step3-warmup-completes-eventually´ +#[test] +fn step3_warmup_completes_eventually() { + let mut s = Sentinel128::new(background_config()).unwrap(); + + // Feed diverse traffic to trigger splits. + for _ in 0..15 { + let batch: Vec = [cell_values(0xA, 20), cell_values(0x5, 20)].concat(); + s.ingest(&batch); + } + + // Give the background thread time to finish warming. + // We spin-ingest a few more batches — each ingest promotes ready cells. + for _ in 0..20 { + s.ingest(&cell_values(0xA, 4)); + std::thread::sleep(std::time::Duration::from_millis(5)); + } + + // All tracked cells should have noise observations. + let gnodes = s.cell_gnodes(); + assert!(gnodes.len() > 1, "should have more than just root after splits"); + for &gnode in &gnodes { + let insp = s.inspect_cell(gnode).unwrap(); + assert!( + insp.maturity.noise_observations > 0, + "cell {:?} at depth {} should have noise observations after background warming", + gnode, + insp.depth, + ); + } +} + +// ── 3.6c: Scoring after warm-up ──────────────────────────── + +/// A tracker warmed on another thread scores like any other: after a warmed +/// run, every cell and ancestor in the batch reports finite score means. That +/// is what the seeding is for — a model with no baseline would divide by a +/// spread it does not have — and it holds whichever thread supplied the seed. +/// +/// ´claim:warmup:a-background-warmed-tracker-produces-finite-scores-from-its-first-real-batch´ +/// ´test:integration:step3-scoring-works-after-background-warmup´ +#[test] +fn step3_scoring_works_after_background_warmup() { + let cfg = SentinelConfig:: { + split_threshold: 10, + background_warming: true, + ..integration_config() + }; + + let mut s = Sentinel128::new(cfg).unwrap(); + + // Seed and warm with background thread. + for _ in 0..15 { + let batch: Vec = [cell_values(0xA, 20), cell_values(0x5, 20)].concat(); + s.ingest(&batch); + std::thread::sleep(std::time::Duration::from_millis(2)); + } + + // Steady-state: scoring should produce non-NaN values. + let report = s.ingest(&cell_values(0xA, 8)); + for cr in report.cell_reports.iter().chain(report.ancestor_reports.iter()) { + assert!(!cr.scores.novelty.mean.is_nan(), "novelty NaN at depth {}", cr.depth); + assert!( + !cr.scores.displacement.mean.is_nan(), + "displacement NaN at depth {}", + cr.depth, + ); + } +} + +// ── 3.6d: Cell structure matches sync path ────────────────── + +/// Two sentinels fed identical traffic, one warming inline and one on a +/// background thread, end with the same accumulated volume and the same set +/// of tracked cells. Structure is decided by observation volume alone and +/// never by anything the trackers compute, so warm-up scheduling — a +/// modelling concern — cannot perturb it. Choosing the background path is a +/// latency decision, not a modelling one. +/// +/// ´claim:warmup:the-choice-of-warm-up-path-cannot-change-the-cell-structure-because-splitting-is-driven-by-volume-alone´ +/// ´test:integration:step3-background-same-cell-structure-as-sync´ +#[test] +fn step3_background_same_cell_structure_as_sync() { + // Both modes with same seed should produce the same set of + // tracked cells (same GNodeIds), demonstrating that the staging + // pathway is consistent. + let base = SentinelConfig:: { + split_threshold: 10, + ..fast_split_config() + }; + + let sync_cfg = SentinelConfig:: { + background_warming: false, + ..base.clone() + }; + let bg_cfg = SentinelConfig:: { + background_warming: true, + ..base + }; + + let mut sync_s = Sentinel128::new(sync_cfg).unwrap(); + let mut bg_s = Sentinel128::new(bg_cfg).unwrap(); + + let values: Vec = [cell_values(0xA, 50), cell_values(0x5, 50)].concat(); + + for chunk in values.chunks(20) { + sync_s.ingest(chunk); + bg_s.ingest(chunk); + } + + // Give background thread time to finish. + for _ in 0..15 { + bg_s.ingest(&cell_values(0xA, 4)); + std::thread::sleep(std::time::Duration::from_millis(5)); + } + // Run same batches through sync too. + for _ in 0..15 { + sync_s.ingest(&cell_values(0xA, 4)); + } + + // The G-V Graph structure should be identical (same observations). + assert_eq!( + sync_s.graph().total_sum(), + bg_s.graph().total_sum(), + "total_sum should match between sync and background modes", + ); + + // Cell GNodeIds should match (same structural decisions). + let sync_gnodes = sync_s.cell_gnodes(); + let bg_gnodes = bg_s.cell_gnodes(); + assert_eq!(sync_gnodes, bg_gnodes, "cell gnodes should match between modes"); +} + +// ── 3.6e: Priority pipeline ──────────────────────────────── + +/// When warming capacity is scarce — a long noise schedule and small batches +/// leave the queue in progress — the staging area serves its waiting cells in +/// descending order of volume, so a heavily trafficked subtree gets cells into +/// service before a quiet one. Warming effort is spent where the traffic is, +/// which is the same principle that decided the cells were worth having. +/// +/// ´claim:warmup:the-staging-area-warms-the-busiest-waiting-cells-first´ +/// ´test:integration:step3-higher-volume-cells-warm-via-priority´ +#[test] +fn step3_higher_volume_cells_warm_via_priority() { + // The staging area serves cells in volume-descending order + // (§ALGO S-11.6.2). We use a slow noise schedule so that + // background warming is still in progress when we inspect, + // then verify that the high-volume subtree has promoted cells. + let cfg = SentinelConfig:: { + split_threshold: 5, + noise_schedule: NoiseSchedule::Explicit(vec![30]), + noise_batch_size: 2, + background_warming: true, + ..fast_split_config() + }; + let mut s = Sentinel128::new(cfg).unwrap(); + + // Feed heavy traffic to 0xA subtree so its cells have high volume. + for _ in 0..20 { + s.ingest(&cell_values(0xA, 30)); + } + + // Now seed a second subtree with much less traffic. + for _ in 0..5 { + s.ingest(&cell_values(0x5, 10)); + } + + // Allow partial background warming and trigger promotion sweeps. + for _ in 0..10 { + s.ingest(&cell_values(0xA, 2)); + std::thread::sleep(std::time::Duration::from_millis(10)); + } + + // At least one non-root cell should have been warmed and promoted + // via the priority pipeline, confirming the end-to-end path. + let gnodes = s.cell_gnodes(); + let warmed_non_root = gnodes + .iter() + .filter_map(|&gnode| s.inspect_cell(gnode)) + .filter(|insp| insp.depth > 0 && insp.maturity.noise_observations > 0) + .count(); + assert!( + warmed_non_root > 0, + "at least one non-root cell should have noise observations via background warming", + ); +} + +// ── 3.6f: Concurrency smoke test ──────────────────────────── + +/// Ingestion and background warming genuinely overlap: a long run of diverse +/// traffic keeps creating cells while the thread is warming earlier ones, and +/// the two never collide. A cell being warmed is checked out of the staging +/// area for the duration, so the expensive work happens on a cell no other +/// thread can reach, and the lock is held only for the queue operations +/// around it. +/// +/// ´claim:warmup:ingest-and-background-warming-overlap-safely-because-a-cell-being-warmed-is-checked-out-of-the-queue´ +/// ´test:integration:step3-concurrent-ingest-no-panic´ +#[test] +fn step3_concurrent_ingest_no_panic() { + // Ingest while the background warming thread runs concurrently. + // Uses max_rank=1 to avoid triggering the Brand SVD oracle on + // near-degenerate matrices in debug builds. + let cfg = SentinelConfig:: { + max_rank: 1, + ..background_config() + }; + let mut s = Sentinel128::new(cfg).unwrap(); + + // Feed diverse traffic — the background thread warms new cells + // concurrently with ingestion. + for _ in 0..30 { + let batch: Vec = [ + cell_values(0x3, 10), + cell_values(0x6, 10), + cell_values(0x9, 10), + cell_values(0xC, 10), + ] + .concat(); + let _report = s.ingest(&batch); + } +} + +// ── 3.6g: Eviction during background warming ─────────────── + +/// The harder end of the same statement: with a background thread running, a +/// cell can be evicted while it is checked out and physically absent from the +/// queue. Eviction still accounts for it, and the cell is discarded when the +/// thread hands it back rather than being restored into a set it has left. +/// +/// (´claim:warmup:a-warming-cell-whose-node-leaves-the-analysis-set-is-discarded-rather-than-promoted´) +/// ´test:integration:step3-eviction-during-background-warming-no-panic´ +#[test] +fn step3_eviction_during_background_warming_no_panic() { + let cfg = SentinelConfig:: { + split_threshold: 5, + budget: 50, + d_evict: 4, + background_warming: true, + ..fast_split_config() + }; + let mut s = Sentinel128::new(cfg).unwrap(); + + // Rapidly create and evict cells with diverse traffic. + for _ in 0..25 { + let batch: Vec = [ + cell_values(0xA, 10), + cell_values(0x5, 10), + cell_values(0x2, 10), + cell_values(0xE, 10), + ] + .concat(); + let report = s.ingest(&batch); + assert_deferred_invariants(&s, &report); + } +} + +// ── 3.6h: Reset restarts background thread ───────────────── + +/// This pins the case where the staging area has a second owner. A reset +/// shuts the warming thread down before clearing state and spawns a fresh one +/// afterwards, so the sentinel comes back to root alone with no thread still +/// holding cells from the previous life — and warming resumes normally on the +/// traffic that follows. +/// +/// (´claim:warmup:a-reset-empties-the-staging-area-so-warm-up-begins-again-from-the-root-alone´) +/// ´test:integration:step3-reset-restarts-background-thread´ +#[test] +fn step3_reset_restarts_background_thread() { + let mut s = Sentinel128::new(background_config()).unwrap(); + + // Build up state. + for _ in 0..10 { + s.ingest(&cell_values(0xA, 20)); + } + + // Reset — thread shuts down and restarts. + s.reset(); + assert_eq!(s.cells_tracked(), 1); + + // Resume with background warming — new cells should still warm. + for _ in 0..15 { + let batch: Vec = [cell_values(0xA, 20), cell_values(0x5, 20)].concat(); + s.ingest(&batch); + } + + // Give time for background warming. + for _ in 0..15 { + s.ingest(&cell_values(0xA, 4)); + std::thread::sleep(std::time::Duration::from_millis(5)); + } + + let gnodes = s.cell_gnodes(); + assert!(gnodes.len() > 1, "should have cells after reset + re-ingest"); +} + +// ── 3.6i: Drop — no hang, no leak ────────────────────────── + +/// Dropping a sentinel with a long noise schedule still outstanding returns +/// promptly instead of waiting for the queue to empty. The warming thread +/// checks for shutdown between batches and abandons whatever remains, because +/// cells nobody will ever read from are not worth finishing. Teardown costs +/// at most one batch of work, not the rest of the schedule. +/// +/// ´claim:warmup:dropping-a-sentinel-abandons-outstanding-warm-up-instead-of-waiting-for-it´ +/// ´test:integration:step3-drop-while-warming-no-hang´ +#[test] +fn step3_drop_while_warming_no_hang() { + // Create a sentinel with background warming and lots of cells + // being warmed, then drop it. Must not hang or panic. + let cfg = SentinelConfig:: { + split_threshold: 5, + noise_schedule: NoiseSchedule::Explicit(vec![100]), + background_warming: true, + ..fast_split_config() + }; + let mut s = Sentinel128::new(cfg).unwrap(); + + // Create cells that will still be warming at drop time. + for _ in 0..10 { + let batch: Vec = [cell_values(0xA, 20), cell_values(0x5, 20)].concat(); + s.ingest(&batch); + } + + // Drop the sentinel — this should shut down the background thread + // gracefully within a reasonable time. + let start = std::time::Instant::now(); + drop(s); + let elapsed = start.elapsed(); + + assert!( + elapsed < std::time::Duration::from_secs(5), + "drop should not hang: elapsed {elapsed:?}", + ); +} diff --git a/packages/sentinel/tests/determinism.rs b/packages/sentinel/tests/determinism.rs new file mode 100644 index 000000000..a406b4ead --- /dev/null +++ b/packages/sentinel/tests/determinism.rs @@ -0,0 +1,253 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +//! # Test index +//! +//! | Test | Area | Claim | +//! |------|------|-------| +//! | [`identical_seed_produces_identical_reports`] | determinism | Two sentinels built from one configuration with one seed, and stepped through the same batches, produce the same report at every step: the same cells and the same cross-cell contexts in the same positions, carrying the same handles, intervals, depths, counts and ranks, and every figure they advertise equal in its bit pattern rather than merely near — each of the four axes with its extremes, its mean, both z-scores, its baseline, its drift evidence and its rejection rate, alongside the contour, the health section and the summary of what the sentinel is investing in. Comparing counts and a mean or two would pass two runs that had modelled different regions of the domain in the same number of cells. Agreement is checked batch by batch and not only at the end, so a divergence could not open and close again unnoticed. The one figure held out is the age of the oldest observation, which measures how long a batch waited rather than anything computed from it. | +//! | [`deterministic_across_repeated_runs`] | engine | cites (´claim:engine:the-root-tracker-receives-every-observation-in-every-batch´) | +//! | [`different_seeds_produce_different_scores`] | determinism | The seed is an input with observable consequences, not a formality: two sentinels differing only in their seed, fed identical values, disagree in at least one of the root's score means. The warming noise a tracker is primed with shapes the subspace it starts from, and that starting point is still visible in what the tracker measures once real traffic arrives — which is why reproducibility has to be stated in terms of the seed rather than of the data alone. | +//! | [`report_ordering_is_deterministic`] | determinism | All three report vectors come out in the order their contract states rather than in the order the walk produced: the competitive cells and the ancestors ascend by node handle, and the cross-cell contexts come shallowest first with ties broken by the handle. Depth leads there because handles are recycled as cells are evicted and restored, so a correctly ordered run can carry a lower handle at a greater depth. Neither ordering is a property of traversal or of when a cell was created, so two runs list the same entries in the same positions and a reader may compare them index by index. Splitting is forced aggressively here so that each vector holds several entries and the ordering is actually put to the question. | +//! | [`send_and_sync_bounds`] | engine | The engine type may be moved between threads and referenced from several at once — a statement about the type, discharged by the compiler when these bounds are demanded, not by anything the test executes at run time. It holds because the sentinel keeps no thread-bound state: its optional background warming lives behind a lock it owns. A host is therefore free to place a sentinel wherever its own concurrency model wants it. | + +//! Reproducibility of the sentinel's output, and the bounds its type +//! carries. +//! +//! Everything the engine does to a batch is ordinary arithmetic in a fixed +//! order, and the one place randomness enters — the synthetic noise used to +//! warm a tracker before real traffic can teach it anything — is drawn from +//! a generator the configuration seeds. Two sentinels built from the same +//! configuration and fed the same values are therefore not merely close but +//! identical, down to the bit pattern of every figure they report. That is +//! what makes a report worth comparing across runs at all: a difference +//! between two runs is a difference in what they were given, never in the +//! order the machine happened to visit things. +//! +//! The seed settles the whole of it only with `background_warming` disabled +//! and on a fixed build — one target and one set of dependency versions — +//! because the generator behind the noise is chosen for speed rather than for +//! portability. The configuration these tests share leaves background warming +//! off, so what they exercise is the guarantee exactly as it is stated. Under +//! background warming the same seed and the same traffic still give the same +//! graph, the same investment set and the same report order, but neither the +//! baselines a tracker starts from nor the ingest cycle on which it first +//! scores: the warming worker draws from its own generator and takes +//! whichever staged cell leads on volume when it looks. +//! +//! For that guarantee to have content the seed has to be a real input. +//! Different seeds draw different warming noise, and the difference survives +//! into the scores rather than being averaged away, so a host that fixes the +//! seed is choosing a particular run and not merely satisfying a parameter. +//! +//! Ordering belongs to the same promise. Every vector in a report is emitted +//! in the order its own contract states — the competitive and ancestor lists +//! by ascending node handle, the cross-cell contexts shallowest first with +//! the handle breaking ties — so a reader compares two runs positionally +//! without depending on the order cells were visited in. Thread-safety is a +//! different kind of statement altogether — a property of the type rather +//! than of any run — and it is discharged by the compiler: the engine holds +//! no thread-bound state, so a host may move it between threads or share it +//! behind a lock of its own. + +mod common; + +use common::{ScenarioBuilder, assert_invariants, assert_reports_identical, cell_values, test_config}; +use torrust_sentinel::{Sentinel128, SentinelConfig}; + +// ── Reproducibility ───────────────────────────────────────── + +/// Two sentinels built from one configuration with one seed, and stepped +/// through the same batches, produce the same report at every step: the same +/// cells and the same cross-cell contexts in the same positions, carrying the +/// same handles, intervals, depths, counts and ranks, and every figure they +/// advertise equal in its bit pattern rather than merely near — each of the +/// four axes with its extremes, its mean, both z-scores, its baseline, its +/// drift evidence and its rejection rate, alongside the contour, the health +/// section and the summary of what the sentinel is investing in. Comparing +/// counts and a mean or two would pass two runs that had modelled different +/// regions of the domain in the same number of cells. Agreement is checked +/// batch by batch and not only at the end, so a divergence could not open and +/// close again unnoticed. The one figure held out is the age of the oldest +/// observation, which measures how long a batch waited rather than anything +/// computed from it. +/// +/// ´claim:determinism:the-same-seed-and-the-same-data-reproduce-the-same-reports´ +/// ´test:integration:identical-seed-produces-identical-reports´ +#[test] +fn identical_seed_produces_identical_reports() { + let cfg = SentinelConfig:: { + noise_seed: Some(42), + ..test_config() + }; + + let values = cell_values(0xA, 100); + + let mut s1 = Sentinel128::new(cfg.clone()).unwrap(); + let mut s2 = Sentinel128::new(cfg).unwrap(); + + for chunk in values.chunks(10) { + let r1 = s1.ingest(chunk); + let r2 = s2.ingest(chunk); + + assert_invariants(&s1, &r1); + assert_invariants(&s2, &r2); + + assert_reports_identical(&r1, &r2); + } +} + +/// A seeded run lands on stated figures rather than merely on some figure. +/// The root appears in the report, its sample count equals the size of the +/// batch, and the lifetime count agrees with it — every value reached the +/// root, none was counted twice, and the warming noise the seed generated +/// did not leak into the record of real observations. The root's scores are +/// numbers, not the non-number that an empty or singular update would leave +/// behind. +/// +/// (´claim:engine:the-root-tracker-receives-every-observation-in-every-batch´) +/// ´test:integration:deterministic-across-repeated-runs´ +#[test] +fn deterministic_across_repeated_runs() { + let cfg = SentinelConfig:: { + noise_seed: Some(12345), + ..test_config() + }; + let mut s = Sentinel128::new(cfg).unwrap(); + + let values = cell_values(0xA, 20); + let report = s.ingest(&values); + + assert_invariants(&s, &report); + + assert_eq!(report.health.lifetime_observations, 20); + + let root = report.ancestor_reports.iter().find(|r| r.depth == 0); + assert!(root.is_some(), "root (depth 0) must appear in ancestor reports"); + assert_eq!(root.unwrap().sample_count, 20); + + let root = root.unwrap(); + assert!(!root.scores.novelty.mean.is_nan()); + assert!(!root.scores.displacement.mean.is_nan()); +} + +// ── Seed sensitivity ──────────────────────────────────────── + +/// The seed is an input with observable consequences, not a formality: two +/// sentinels differing only in their seed, fed identical values, disagree in +/// at least one of the root's score means. The warming noise a tracker is +/// primed with shapes the subspace it starts from, and that starting point +/// is still visible in what the tracker measures once real traffic arrives — +/// which is why reproducibility has to be stated in terms of the seed rather +/// than of the data alone. +/// +/// ´claim:determinism:the-seed-is-visible-in-the-scores-so-two-seeds-do-not-coincide´ +/// ´test:integration:different-seeds-produce-different-scores´ +#[test] +fn different_seeds_produce_different_scores() { + let cfg_a = SentinelConfig:: { + noise_seed: Some(1), + ..test_config() + }; + let cfg_b = SentinelConfig:: { + noise_seed: Some(2), + ..test_config() + }; + + let values = cell_values(0xA, 40); + + let mut sa = Sentinel128::new(cfg_a).unwrap(); + let mut sb = Sentinel128::new(cfg_b).unwrap(); + + let ra = sa.ingest(&values); + let rb = sb.ingest(&values); + + assert_invariants(&sa, &ra); + assert_invariants(&sb, &rb); + + // At least one root-level score must differ because the noise + // sequences are seeded differently. + let root_a = ra.ancestor_reports.iter().find(|r| r.depth == 0).unwrap(); + let root_b = rb.ancestor_reports.iter().find(|r| r.depth == 0).unwrap(); + + let scores_identical = root_a.scores.novelty.mean.to_bits() == root_b.scores.novelty.mean.to_bits() + && root_a.scores.displacement.mean.to_bits() == root_b.scores.displacement.mean.to_bits() + && root_a.scores.surprise.mean.to_bits() == root_b.scores.surprise.mean.to_bits(); + + assert!(!scores_identical, "different seeds must produce different scores at root"); +} + +// ── Report ordering ───────────────────────────────────────── + +/// All three report vectors come out in the order their contract states +/// rather than in the order the walk produced: the competitive cells and the +/// ancestors ascend by node handle, and the cross-cell contexts come +/// shallowest first with ties broken by the handle. Depth leads there because +/// handles are recycled as cells are evicted and restored, so a correctly +/// ordered run can carry a lower handle at a greater depth. Neither ordering +/// is a property of traversal or of when a cell was created, so two runs list +/// the same entries in the same positions and a reader may compare them index +/// by index. Splitting is forced aggressively here so that each vector holds +/// several entries and the ordering is actually put to the question. +/// +/// ´claim:determinism:every-report-vector-comes-out-in-the-order-its-contract-states-so-a-reader-never-depends-on-visit-order´ +/// ´test:integration:report-ordering-is-deterministic´ +#[test] +fn report_ordering_is_deterministic() { + let cfg = SentinelConfig:: { + noise_seed: Some(42), + split_threshold: 10, + ..test_config() + }; + + let (mut s, _) = ScenarioBuilder::new() + .config(cfg) + .seed_range(0xA, 10) + .seed_range(0xB, 10) + .warm_batches(49) + .build_with_reports(); + + let report = s.ingest(&[cell_values(0xA, 4), cell_values(0xB, 4)].concat()); + + for window in report.cell_reports.windows(2) { + assert!(window[0].gnode_id < window[1].gnode_id, "cell_reports not in GNodeId order"); + } + + for window in report.ancestor_reports.windows(2) { + assert!( + window[0].gnode_id < window[1].gnode_id, + "ancestor_reports not in GNodeId order" + ); + } + + for window in report.coordination_reports.windows(2) { + assert!( + (window[0].depth, window[0].gnode_id) < (window[1].depth, window[1].gnode_id), + "coordination_reports not in (depth, GNodeId) order: ({}, {:?}) >= ({}, {:?})", + window[0].depth, + window[0].gnode_id, + window[1].depth, + window[1].gnode_id, + ); + } +} + +// ── Thread safety ─────────────────────────────────────────── + +/// The engine type may be moved between threads and referenced from several +/// at once — a statement about the type, discharged by the compiler when +/// these bounds are demanded, not by anything the test executes at run time. +/// It holds because the sentinel keeps no thread-bound state: its optional +/// background warming lives behind a lock it owns. A host is therefore free +/// to place a sentinel wherever its own concurrency model wants it. +/// +/// ´claim:engine:the-sentinel-type-carries-send-and-sync-so-a-host-may-own-it-across-threads´ +/// ´test:integration:send-and-sync-bounds´ +#[test] +fn send_and_sync_bounds() { + fn assert_send() {} + fn assert_sync() {} + + assert_send::(); + assert_sync::(); +} diff --git a/packages/sentinel/tests/edge_cases.rs b/packages/sentinel/tests/edge_cases.rs new file mode 100644 index 000000000..3197e7806 --- /dev/null +++ b/packages/sentinel/tests/edge_cases.rs @@ -0,0 +1,575 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +//! # Test index +//! +//! | Test | Area | Claim | +//! |------|------|-------| +//! | [`min_value_u128`] | edge | The bottom of the coordinate domain is an ordinary observation: a value with no bits set is counted and reported like any other, and every structural invariant survives it. Encoding centres each bit rather than taking it raw, so an all-zero value is a well-formed vector and not a degenerate one the geometry has to special-case. | +//! | [`max_value_u128`] | edge | cites (´claim:edge:the-extremes-of-the-coordinate-domain-are-ordinary-observations´) | +//! | [`min_and_max_together`] | edge | cites (´claim:edge:the-extremes-of-the-coordinate-domain-are-ordinary-observations´) | +//! | [`all_nibbles_full_spread`] | edge | Traffic spread evenly over every leading region of the domain, batch after batch, gives the spatial layer no concentration to reward — and the engine keeps its invariants anyway, with the observation count equal to exactly what was fed in. Uniform traffic is the worst case for a selector that ranks by importance, and it produces a boring report rather than an unstable one. | +//! | [`single_observation_batch`] | engine | cites (´claim:engine:the-root-tracker-receives-every-observation-in-every-batch´) | +//! | [`very_large_batch`] | edge | There is no ceiling on batch size: a batch of many thousands of values spread across the domain is scored in one pass, the root's sample count equals the batch it was given, and the invariants hold. Ingestion walks the batch once and updates state as it goes, so a large batch costs time proportional to its size and nothing else. | +//! | [`empty_then_burst`] | edge | A long idle stretch of empty batches leaves the engine exactly where it was — nothing observed, nothing reported — and the burst that follows is scored as though the idling had never happened. Empty batches do not accumulate into a state the engine has to recover from, because an empty batch returns before any of the observation machinery runs. | +//! | [`single_observation_repeated`] | edge | One value hammered in over and over, under a split threshold low enough to make the spatial layer subdivide around it, is counted in full and leaves the invariants intact. Concentration is exactly what the spatial layer is built to notice, so the pathological case of total concentration is a case it handles rather than one that surprises it. | +//! | [`all_same_value`] | edge | Identical observations teach the model no new directions, so after a long run of them the root's learned rank is still near its floor rather than having grown with the volume. Rank tracks how many directions the data actually spans, not how much data arrived, which is what keeps the measurement honest about a stream that carries no structure. | +//! | [`alternating_two_values`] | edge | A stream that alternates strictly between two well-separated values still leaves the root with at least one learned direction and the invariants standing. Two points do span a direction, so this is the smallest non-trivial structure a tracker can be given — the case just above the one where nothing varies at all. | +//! | [`reset_then_immediate_ingest`] | engine | A sentinel is usable the instant a reset returns: the very next batch is counted from zero and produces a root report, with no warm-up call or settling period in between. Reset rebuilds the root tracker as part of the operation rather than leaving the engine cell-less until traffic arrives. | +//! | [`double_reset`] | engine | cites (´claim:engine:the-sentinel-is-immediately-usable-after-a-reset´) | +//! | [`decay_to_zero_then_rebuild`] | edge | Decay severe enough to annihilate accumulated standing does not leave a dead engine: fresh traffic rebuilds cells from what it observes, the root is reported again, and trackers are live. Decay lowers what regions have earned rather than removing the machinery that earns it, so the recovery path is simply ordinary ingestion. | +//! | [`rapid_decay_ingest_cycle`] | edge | Decay interleaved with ingestion round after round keeps producing valid reports with the invariants intact. Decay does not need to be rare or quiescent to be safe: it changes spatial standing between batches, and the analysis set is simply recomputed at the start of the next ingestion rather than being eagerly invalidated. | +//! | [`analysis_k_equals_one`] | edge | Squeezing the competitive budget to a single cell squeezes exactly that: at most one cell is reported as having earned its place, and the root is still reported regardless. The budget governs investment, not structure, so the narrowest possible budget yields the smallest useful report rather than an empty one. The full-set bound is deliberately not asserted here — closure under ancestry keeps materialising chains the bound assumes a wider budget for. | +//! | [`analysis_depth_cutoff_zero`] | edge | cites (´claim:edge:an-extreme-analysis-budget-narrows-what-can-be-chosen-but-never-empties-the-set´) | +//! | [`per_sample_scores_large_batch`] | engine | cites (´claim:engine:per-sample-scores-appear-only-when-asked-for-and-then-carry-one-entry-per-observation´) | +//! | [`edge_cases_degenerate_cells_skipped_starts_at_zero`] | engine | cites (´claim:engine:a-fresh-sentinel-has-skipped-no-cell-as-too-narrow-to-model´) | +//! | [`degenerate_cells_skipped_in_report_normal_traffic`] | edge | Ordinary traffic never drives a cell narrow enough to be skipped: after a long run of ingestion the report still shows no skips at all. The guard is there for pathological splitting, not for everyday operation, so a non-zero reading in the field is a signal about the traffic rather than routine noise. | +//! | [`degenerate_cells_skipped_after_reset`] | engine | cites (´claim:engine:a-fresh-sentinel-has-skipped-no-cell-as-too-narrow-to-model´) | +//! | [`deep_spray_traffic_does_not_panic`] | edge | Splitting made as aggressive as the configuration allows, on traffic hammering a single point, drives the domain deep enough that some cells have almost no suffix left to analyse. Those cells are skipped and counted rather than given a tracker that could form no basis and produce no residual, and the run continues with the root still tracked. A cell too narrow to model is a case the engine declines, not a fault it crashes on. | + +//! Degenerate and boundary input, and what the engine does with it. +//! +//! A sentinel in service is fed whatever the host sees, not whatever suits +//! the model. Batches arrive empty, arrive one value at a time, and arrive +//! enormous. Values sit at the extremes of the coordinate domain, repeat +//! without variation, or alternate between two points indefinitely. None of +//! these is an error, so none of them is treated as one: the engine counts +//! what it was given, reports what it measured, and keeps its structural +//! invariants throughout. The boundary between a degenerate case and an +//! impossible one matters here — the degenerate cases are all admitted. +//! +//! Structureless traffic is the interesting degeneracy, because it is the +//! case where there is nothing to learn. Identical values present no new +//! direction, so the learned rank stays low instead of inflating on +//! repetition, and the measurement remains honest about how little variation +//! it has actually seen. +//! +//! Extreme configuration and extreme splitting are the other two edges. A +//! competitive budget of one, or a depth cutoff of zero, narrows what may be +//! selected but never empties the set, because the root's membership is +//! unconditional. And when aggressive splitting drives a cell's suffix too +//! narrow to support a subspace model at all, that cell is skipped and +//! counted rather than modelled badly or allowed to bring the run down. + +mod common; + +use common::{ScenarioBuilder, assert_invariants, cell_values, integration_config, test_config}; +use torrust_sentinel::{Sentinel128, SentinelConfig}; + +// ─── Boundary values ──────────────────────────────────────── + +/// The bottom of the coordinate domain is an ordinary observation: a value +/// with no bits set is counted and reported like any other, and every +/// structural invariant survives it. Encoding centres each bit rather than +/// taking it raw, so an all-zero value is a well-formed vector and not a +/// degenerate one the geometry has to special-case. +/// +/// ´claim:edge:the-extremes-of-the-coordinate-domain-are-ordinary-observations´ +/// ´test:integration:min-value-u128´ +#[test] +fn min_value_u128() { + let mut s = Sentinel128::new(test_config()).unwrap(); + let report = s.ingest(&[0u128]); + assert_eq!(report.health.lifetime_observations, 1); + assert_invariants(&s, &report); +} + +/// The top of the domain behaves the same way as the bottom — a saturated +/// value routes, scores and counts without arithmetic trouble at the far end +/// of the interval the root covers. +/// +/// (´claim:edge:the-extremes-of-the-coordinate-domain-are-ordinary-observations´) +/// ´test:integration:max-value-u128´ +#[test] +fn max_value_u128() { + let mut s = Sentinel128::new(test_config()).unwrap(); + let report = s.ingest(&[u128::MAX]); + assert_eq!(report.health.lifetime_observations, 1); + assert_invariants(&s, &report); +} + +/// The widest possible spread within one batch — both extremes at once — +/// is handled as a single ordinary batch. Values in a batch are scored +/// independently against the cells that contain them, so how far apart they +/// lie in the domain is not itself a difficulty. +/// +/// (´claim:edge:the-extremes-of-the-coordinate-domain-are-ordinary-observations´) +/// ´test:integration:min-and-max-together´ +#[test] +fn min_and_max_together() { + let mut s = Sentinel128::new(test_config()).unwrap(); + let report = s.ingest(&[0u128, u128::MAX]); + assert_eq!(report.health.lifetime_observations, 2); + assert_invariants(&s, &report); +} + +/// Traffic spread evenly over every leading region of the domain, batch +/// after batch, gives the spatial layer no concentration to reward — and the +/// engine keeps its invariants anyway, with the observation count equal to +/// exactly what was fed in. Uniform traffic is the worst case for a selector +/// that ranks by importance, and it produces a boring report rather than an +/// unstable one. +/// +/// ´claim:edge:traffic-spread-evenly-over-the-whole-domain-keeps-every-invariant-and-every-count´ +/// ´test:integration:all-nibbles-full-spread´ +#[test] +fn all_nibbles_full_spread() { + let mut s = Sentinel128::new(integration_config()).unwrap(); + + // Values spanning all 16 leading nibbles. + let values: Vec = (0..16u128).map(|nib| (nib << 124) | 1).collect(); + for _ in 0..20 { + s.ingest(&values); + } + let report = s.ingest(&values); + + assert_eq!(s.lifetime_observations(), 21 * 16); + assert_invariants(&s, &report); +} + +// ─── Batch-size extremes ──────────────────────────────────── + +/// The smallest non-empty batch, delivered over and over, produces a report +/// every time: each single observation reaches the root and is scored there, +/// and nothing is deferred until enough values have piled up. A host that +/// hands values over one at a time therefore gets the same record as one +/// that buffers them. +/// +/// (´claim:engine:the-root-tracker-receives-every-observation-in-every-batch´) +/// ´test:integration:single-observation-batch´ +#[test] +fn single_observation_batch() { + let mut s = Sentinel128::new(test_config()).unwrap(); + + // Single observation per batch for 50 batches. + let mut report = None; + for i in 0..50u128 { + let r = s.ingest(&[(0xA << 124) | i]); + assert!( + !r.ancestor_reports.is_empty() || !r.cell_reports.is_empty(), + "batch {i} should produce at least a root report" + ); + report = Some(r); + } + assert_eq!(s.lifetime_observations(), 50); + assert_invariants(&s, report.as_ref().unwrap()); +} + +/// There is no ceiling on batch size: a batch of many thousands of values +/// spread across the domain is scored in one pass, the root's sample count +/// equals the batch it was given, and the invariants hold. Ingestion walks +/// the batch once and updates state as it goes, so a large batch costs time +/// proportional to its size and nothing else. +/// +/// ´claim:edge:batch-size-is-unbounded-so-a-very-large-batch-is-scored-in-one-pass´ +/// ´test:integration:very-large-batch´ +#[test] +fn very_large_batch() { + let mut s = Sentinel128::new(test_config()).unwrap(); + + // 10K observations in one batch. + let values: Vec = (0..10_000u128).map(|i| ((i % 16) << 124) | (i + 1)).collect(); + let report = s.ingest(&values); + + assert_eq!(s.lifetime_observations(), 10_000); + let root = report.ancestor_reports.iter().find(|r| r.depth == 0); + assert!(root.is_some()); + assert_eq!(root.unwrap().sample_count, 10_000); + assert_invariants(&s, &report); +} + +/// A long idle stretch of empty batches leaves the engine exactly where it +/// was — nothing observed, nothing reported — and the burst that follows is +/// scored as though the idling had never happened. Empty batches do not +/// accumulate into a state the engine has to recover from, because an empty +/// batch returns before any of the observation machinery runs. +/// +/// ´claim:edge:idling-on-empty-batches-leaves-the-engine-ready-for-the-burst-that-follows´ +/// ´test:integration:empty-then-burst´ +#[test] +fn empty_then_burst() { + let mut s = Sentinel128::new(test_config()).unwrap(); + + // 100 empty ingests. + for _ in 0..100 { + let report = s.ingest(&[]); + assert!(report.cell_reports.is_empty()); + } + assert_eq!(s.lifetime_observations(), 0); + + // Then a large burst. + let report = s.ingest(&cell_values(0xA, 200)); + assert_eq!(s.lifetime_observations(), 200); + assert!( + !report.ancestor_reports.is_empty(), + "burst after empties should produce ancestor reports" + ); + assert_invariants(&s, &report); +} + +// ─── Traffic patterns ─────────────────────────────────────── + +/// One value hammered in over and over, under a split threshold low enough +/// to make the spatial layer subdivide around it, is counted in full and +/// leaves the invariants intact. Concentration is exactly what the spatial +/// layer is built to notice, so the pathological case of total concentration +/// is a case it handles rather than one that surprises it. +/// +/// ´claim:edge:a-single-value-repeated-without-variation-is-tolerated-and-still-counted-in-full´ +/// ´test:integration:single-observation-repeated´ +#[test] +fn single_observation_repeated() { + let mut s = Sentinel128::new(SentinelConfig:: { + split_threshold: 10, + ..test_config() + }) + .unwrap(); + + let value = 0xA000_0000_0000_0000_0000_0000_0000_0001u128; + let mut report = None; + for _ in 0..100 { + report = Some(s.ingest(&[value])); + } + + assert_eq!(s.lifetime_observations(), 100); + assert!(s.cells_tracked() >= 1); + assert_invariants(&s, report.as_ref().unwrap()); +} + +/// Identical observations teach the model no new directions, so after a long +/// run of them the root's learned rank is still near its floor rather than +/// having grown with the volume. Rank tracks how many directions the data +/// actually spans, not how much data arrived, which is what keeps the +/// measurement honest about a stream that carries no structure. +/// +/// ´claim:edge:identical-traffic-teaches-no-new-direction-so-the-learned-rank-stays-low´ +/// ´test:integration:all-same-value´ +#[test] +fn all_same_value() { + let mut s = Sentinel128::new(integration_config()).unwrap(); + + let value = 0xAAAA_BBBB_CCCC_DDDD_EEEE_FFFF_0000_1111u128; + let mut report = None; + for _ in 0..100 { + report = Some(s.ingest(&[value; 8])); + } + + // Rank should stay low — no structural variation. + let root = s.graph().g_root(); + let insp = s.inspect_cell(root).unwrap(); + assert!(insp.rank <= 2, "identical traffic should keep rank low, got {}", insp.rank); + assert_invariants(&s, report.as_ref().unwrap()); +} + +/// A stream that alternates strictly between two well-separated values still +/// leaves the root with at least one learned direction and the invariants +/// standing. Two points do span a direction, so this is the smallest +/// non-trivial structure a tracker can be given — the case just above the +/// one where nothing varies at all. +/// +/// ´claim:edge:a-stream-of-only-two-values-still-supports-at-least-one-learned-direction´ +/// ´test:integration:alternating-two-values´ +#[test] +fn alternating_two_values() { + let mut s = Sentinel128::new(integration_config()).unwrap(); + + let v1 = 0xF000_0000_0000_0000_0000_0000_0000_0001u128; + let v2 = 0x1000_0000_0000_0000_0000_0000_0000_0002u128; + + let mut report = None; + for i in 0..100 { + if i % 2 == 0 { + report = Some(s.ingest(&[v1; 4])); + } else { + report = Some(s.ingest(&[v2; 4])); + } + } + + assert!(s.cells_tracked() >= 1); + let root = s.graph().g_root(); + let insp = s.inspect_cell(root).unwrap(); + assert!(insp.rank >= 1); + assert_invariants(&s, report.as_ref().unwrap()); +} + +// ─── Reset behaviour ──────────────────────────────────────── + +/// A sentinel is usable the instant a reset returns: the very next batch is +/// counted from zero and produces a root report, with no warm-up call or +/// settling period in between. Reset rebuilds the root tracker as part of +/// the operation rather than leaving the engine cell-less until traffic +/// arrives. +/// +/// ´claim:engine:the-sentinel-is-immediately-usable-after-a-reset´ +/// ´test:integration:reset-then-immediate-ingest´ +#[test] +fn reset_then_immediate_ingest() { + let mut s = ScenarioBuilder::new().seed_range(0xA, 20).warm_batches(10).build(); + s.reset(); + + assert_eq!(s.lifetime_observations(), 0); + + let report = s.ingest(&cell_values(0xA, 16)); + assert_eq!(s.lifetime_observations(), 16); + assert!( + report.ancestor_reports.iter().any(|r| r.depth == 0), + "root must be reported after reset + ingest" + ); + assert_invariants(&s, &report); +} + +/// Resetting twice in a row is no different from resetting once — the second +/// call finds an already-fresh engine and leaves it fresh, counters included, +/// and the sentinel still ingests normally afterwards. A host may therefore +/// reset defensively without tracking whether it already did. +/// +/// (´claim:engine:the-sentinel-is-immediately-usable-after-a-reset´) +/// ´test:integration:double-reset´ +#[test] +fn double_reset() { + let mut s = ScenarioBuilder::new().seed_range(0xA, 20).warm_batches(10).build(); + s.reset(); + s.reset(); + + assert_eq!(s.lifetime_observations(), 0); + assert_eq!(s.degenerate_cells_skipped(), 0); + + // Must be usable after double reset. + let report = s.ingest(&cell_values(0xA, 8)); + assert_eq!(s.lifetime_observations(), 8); + assert_invariants(&s, &report); +} + +// ─── Decay edge cases ─────────────────────────────────────── + +/// Decay severe enough to annihilate accumulated standing does not leave a +/// dead engine: fresh traffic rebuilds cells from what it observes, the root +/// is reported again, and trackers are live. Decay lowers what regions have +/// earned rather than removing the machinery that earns it, so the recovery +/// path is simply ordinary ingestion. +/// +/// ´claim:edge:standing-decayed-almost-to-nothing-is-rebuilt-by-fresh-traffic-rather-than-leaving-a-dead-engine´ +/// ´test:integration:decay-to-zero-then-rebuild´ +#[test] +fn decay_to_zero_then_rebuild() { + let mut s = ScenarioBuilder::new().seed_range(0xA, 20).warm_batches(20).build(); + assert!(s.cells_tracked() > 1); + + // Annihilate everything. + s.decay(0.0001, 0.0); + + // Re-ingest: should rebuild cells from scratch. + for _ in 0..20 { + s.ingest(&cell_values(0xA, 16)); + } + let report = s.ingest(&cell_values(0xA, 8)); + + assert!(report.ancestor_reports.iter().any(|r| r.depth == 0)); + assert!(report.health.active_trackers >= 1); + assert_invariants(&s, &report); +} + +/// Decay interleaved with ingestion round after round keeps producing valid +/// reports with the invariants intact. Decay does not need to be rare or +/// quiescent to be safe: it changes spatial standing between batches, and +/// the analysis set is simply recomputed at the start of the next ingestion +/// rather than being eagerly invalidated. +/// +/// ´claim:edge:decay-interleaved-with-ingestion-round-after-round-keeps-the-reports-valid´ +/// ´test:integration:rapid-decay-ingest-cycle´ +#[test] +fn rapid_decay_ingest_cycle() { + let mut s = ScenarioBuilder::new().seed_range(0xA, 20).warm_batches(10).build(); + + for _ in 0..50 { + s.decay(0.8, 0.0); + let report = s.ingest(&cell_values(0xA, 8)); + assert!(!report.ancestor_reports.is_empty() || !report.cell_reports.is_empty()); + assert_invariants(&s, &report); + } +} + +// ─── Config edge cases ────────────────────────────────────── + +/// Squeezing the competitive budget to a single cell squeezes exactly that: +/// at most one cell is reported as having earned its place, and the root is +/// still reported regardless. The budget governs investment, not structure, +/// so the narrowest possible budget yields the smallest useful report rather +/// than an empty one. The full-set bound is deliberately not asserted here — +/// closure under ancestry keeps materialising chains the bound assumes a +/// wider budget for. +/// +/// ´claim:edge:an-extreme-analysis-budget-narrows-what-can-be-chosen-but-never-empties-the-set´ +/// ´test:integration:analysis-k-equals-one´ +#[test] +fn analysis_k_equals_one() { + let cfg = SentinelConfig:: { + analysis_k: 1, + split_threshold: 10, + ..integration_config() + }; + let mut s = Sentinel128::new(cfg).unwrap(); + + for _ in 0..50 { + s.ingest(&cell_values(0xA, 16)); + } + + let report = s.ingest(&cell_values(0xA, 8)); + assert!( + report.analysis_set_summary.competitive_size <= 1, + "with K=1, at most 1 competitive cell" + ); + assert!( + report.ancestor_reports.iter().any(|r| r.depth == 0), + "root must still be reported" + ); + // `assert_invariants` omitted: K=1 intentionally violates the + // Steiner tree bound (full_size >> 2*competitive_size+1). +} + +/// The other extreme configuration reaches the same floor. With the depth +/// cutoff at zero nothing below the top of the spatial tree can compete at +/// all, and the set still holds at least the root — the one member that is +/// there by obligation rather than by merit. +/// +/// (´claim:edge:an-extreme-analysis-budget-narrows-what-can-be-chosen-but-never-empties-the-set´) +/// ´test:integration:analysis-depth-cutoff-zero´ +#[test] +fn analysis_depth_cutoff_zero() { + let cfg = SentinelConfig:: { + analysis_depth_cutoff: 0, + split_threshold: 10, + ..integration_config() + }; + let mut s = Sentinel128::new(cfg).unwrap(); + + for _ in 0..50 { + s.ingest(&cell_values(0xA, 16)); + } + + let report = s.ingest(&cell_values(0xA, 8)); + // With depth cutoff 0, only the V-Tree root is eligible for + // competitive selection. + assert!( + report.analysis_set_summary.full_size >= 1, + "at least root must be in the full set" + ); + // `assert_invariants` omitted: depth_cutoff=0 may violate the + // Steiner tree bound by design. +} + +/// The one-entry-per-observation correspondence holds at scale, not just for +/// a batch of one: a batch of a couple of hundred values yields exactly that +/// many per-sample entries at the root. Nothing is sampled, truncated or +/// aggregated on the way out, so an entry can always be traced back to the +/// value that produced it however large the batch was. +/// +/// (´claim:engine:per-sample-scores-appear-only-when-asked-for-and-then-carry-one-entry-per-observation´) +/// ´test:integration:per-sample-scores-large-batch´ +#[test] +fn per_sample_scores_large_batch() { + let cfg = SentinelConfig:: { + per_sample_scores: true, + ..test_config() + }; + let mut s = Sentinel128::new(cfg).unwrap(); + + let report = s.ingest(&cell_values(0xA, 256)); + + let root = report.ancestor_reports.iter().find(|r| r.depth == 0); + assert!(root.is_some()); + let per_sample = root.unwrap().per_sample.as_ref(); + assert!(per_sample.is_some(), "per_sample should be Some when enabled"); + assert_eq!( + per_sample.unwrap().len(), + 256, + "per_sample should have one entry per observation" + ); + assert_invariants(&s, &report); +} + +// ─── ADR-S-011: Degenerate cell dimension guard ──────────── + +/// The skip counter starts at zero, which is what makes it evidence: any +/// later reading above zero was caused by traffic driving the domain deep +/// enough to produce a cell too narrow to model, and never by construction +/// itself. +/// +/// (´claim:engine:a-fresh-sentinel-has-skipped-no-cell-as-too-narrow-to-model´) +/// ´test:integration:edge-cases-degenerate-cells-skipped-starts-at-zero´ +#[test] +fn edge_cases_degenerate_cells_skipped_starts_at_zero() { + let s = Sentinel128::new(test_config()).unwrap(); + assert_eq!(s.degenerate_cells_skipped(), 0); +} + +/// Ordinary traffic never drives a cell narrow enough to be skipped: after a +/// long run of ingestion the report still shows no skips at all. The guard +/// is there for pathological splitting, not for everyday operation, so a +/// non-zero reading in the field is a signal about the traffic rather than +/// routine noise. +/// +/// ´claim:edge:ordinary-traffic-never-drives-a-cell-narrow-enough-to-be-skipped´ +/// ´test:integration:degenerate-cells-skipped-in-report-normal-traffic´ +#[test] +fn degenerate_cells_skipped_in_report_normal_traffic() { + let mut s = Sentinel128::new(integration_config()).unwrap(); + + for _ in 0..20 { + s.ingest(&cell_values(0xA, 16)); + } + let report = s.ingest(&cell_values(0xA, 8)); + assert_eq!( + report.analysis_set_summary.degenerate_cells_skipped, 0, + "normal traffic should not produce degenerate cells" + ); + assert_invariants(&s, &report); +} + +/// Reset restores the fresh reading of the skip counter along with +/// everything else, so the count belongs to the current life of the engine +/// and does not carry across a deliberate discarding of the past. +/// +/// (´claim:engine:a-fresh-sentinel-has-skipped-no-cell-as-too-narrow-to-model´) +/// ´test:integration:degenerate-cells-skipped-after-reset´ +#[test] +fn degenerate_cells_skipped_after_reset() { + let mut s = Sentinel128::new(integration_config()).unwrap(); + s.ingest(&cell_values(0xA, 8)); + s.reset(); + assert_eq!(s.degenerate_cells_skipped(), 0); +} + +/// Splitting made as aggressive as the configuration allows, on traffic +/// hammering a single point, drives the domain deep enough that some cells +/// have almost no suffix left to analyse. Those cells are skipped and +/// counted rather than given a tracker that could form no basis and produce +/// no residual, and the run continues with the root still tracked. A cell +/// too narrow to model is a case the engine declines, not a fault it +/// crashes on. +/// +/// ´claim:edge:a-cell-too-narrow-to-model-is-skipped-and-counted-rather-than-bringing-the-run-down´ +/// ´test:integration:deep-spray-traffic-does-not-panic´ +#[test] +fn deep_spray_traffic_does_not_panic() { + let cfg = SentinelConfig:: { + split_threshold: 2, // very aggressive splitting + d_create: 1, + d_evict: 2, + budget: 100_000, + analysis_k: 32, // large analysis set to pick up deep cells + analysis_depth_cutoff: 128, // don't filter by V-depth + ..integration_config() + }; + let mut s = Sentinel128::new(cfg).unwrap(); + + // Hammer a single value to force deep splits. + let value = 0xAAAA_BBBB_CCCC_DDDD_EEEE_FFFF_0000_1111u128; + for _ in 0..500 { + let _report = s.ingest(&[value]); + } + + // The sentinel must not panic. Any degenerate cells should be + // silently skipped and counted. + assert!(s.cells_tracked() >= 1, "at least root must be tracked"); +} diff --git a/packages/sentinel/tests/graph_routing.rs b/packages/sentinel/tests/graph_routing.rs new file mode 100644 index 000000000..7dafff210 --- /dev/null +++ b/packages/sentinel/tests/graph_routing.rs @@ -0,0 +1,743 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +//! # Test index +//! +//! | Test | Area | Claim | +//! |------|------|-------| +//! | [`full_width_u64_cells_keep_distinct_encodings`] | routing | Distinct values routed into one full-width cell have distinct suffix encodings. | +//! | [`full_width_u128_cells_keep_distinct_encodings`] | routing | The full-width routing correction also holds at the widest supported coordinate width. | +//! | [`fresh_graph_has_one_node`] | routing | Before anything is observed the graph is a single cell spanning the whole coordinate domain. There is no partition worth choosing until traffic says where the boundaries should fall, so the sentinel starts with the one cell it can justify and refines outward from there. | +//! | [`fresh_graph_has_one_terminal`] | routing | cites (´claim:routing:a-fresh-graph-is-one-root-cell-covering-the-whole-domain-with-nothing-accumulated´) | +//! | [`fresh_graph_has_zero_total_sum`] | routing | cites (´claim:routing:a-fresh-graph-is-one-root-cell-covering-the-whole-domain-with-nothing-accumulated´) | +//! | [`ingest_feeds_graph_with_delta_one`] | routing | Every value in a batch contributes exactly one unit of importance, whichever range it falls in, so the graph's total after successive batches into unrelated ranges is simply how many values were handed over. Importance is a count of arrivals rather than a weight the caller can set, which is what lets the selector read it as evidence of where traffic is. | +//! | [`empty_ingest_does_not_observe`] | routing | A batch with nothing in it is not an event. The accumulated total stays where it was and no cell is created, so an interval in which nothing arrived neither adds evidence nor moves the partition. Quiet time is therefore invisible to the spatial layer rather than being recorded as an observation of emptiness. | +//! | [`duplicate_values_each_contribute`] | routing | cites (´claim:routing:every-ingested-value-adds-exactly-one-unit-of-importance-wherever-it-lands´) | +//! | [`graph_accumulates_across_batches`] | routing | cites (´claim:routing:every-ingested-value-adds-exactly-one-unit-of-importance-wherever-it-lands´) | +//! | [`lifetime_observations_tracks_total_sum`] | routing | The counter the sentinel keeps and the total the graph accumulates stay equal batch after batch. They are one quantity read from two layers: real observations are the only thing that increments either, and the synthetic data used to warm trackers is deliberately kept out of both. A host can therefore read whichever is nearer to hand without learning which layer maintains it. | +//! | [`concentrated_traffic_splits_nodes`] | routing | Traffic that keeps landing in one narrow range drives that range past the split threshold and the graph refines it, so the partition is bought with observations rather than configured up front. Where the traffic goes is where the resolution appears; the total meanwhile still counts exactly the values handed over, so refining a region does not manufacture evidence. | +//! | [`concentrated_traffic_grows_terminals`] | routing | cites (´claim:routing:concentrated-traffic-buys-resolution-by-splitting-the-range-it-lands-in´) | +//! | [`diverse_traffic_respects_budget`] | routing | Traffic spread thinly over many well-separated ranges asks the graph to refine everywhere at once, and the node budget is what keeps that from being unbounded: however many ranges are busy and however low the split threshold is set, the graph holds no more cells than the budget allows. The cost of modelling is a configured ceiling rather than a function of how widely an adversary chooses to scatter. | +//! | [`reset_restores_fresh_graph_state`] | routing | Reset discards the partition as well as the evidence: a graph that had split under load comes back as the single root cell of a fresh sentinel, with nothing accumulated and nothing to land in but the root. Structure is derived from observations, so once the observations are dropped there is no refinement left worth preserving, and a reset sentinel cannot be distinguished from a new one by what its graph holds. | +//! | [`top_of_domain_coordinate_routes_to_a_cell`] | routing | The topmost coordinate of the domain reaches a tracker rather than falling through every cell. Cell intervals are half-open, which has no upper edge case while the coordinate width is narrower than the coordinate type — the bound is then a representable value outside the domain. At the full width the domain's maximum is the type's maximum, there is no value above it to be excluded, and a half-open reading of the topmost interval therefore excludes a coordinate that is genuinely inside the domain. The spatial layer counts that observation either way, so the two readings would disagree: the accumulated total records an arrival that no tracker was ever shown. | +//! | [`an_ordinary_coordinate_still_lands_in_one_cell`] | routing | cites (´claim:routing:the-domains-top-coordinate-reaches-a-tracker-rather-than-falling-through-every-cell´) | +//! | [`values_outside_the_domain_are_counted_nowhere`] | routing | A coordinate the domain cannot name is not an observation of it, and the three layers that would each read it differently are not left to disagree about that. The spatial layer accumulates such a value in the topmost cell, whose interval does not contain it; the encoder reads the low bits of the configured width, so it would hand a tracker the vector of the in-domain value the arrival is congruent to; and the interval scan matches no cell at all, not even the root. Deciding membership once, before any of them, is what keeps the counts one count: the value raises no total, moves no partition and reaches no tracker. | +//! | [`a_signed_coordinate_outside_the_domain_is_counted_nowhere_at_full_width`] | routing | Domain membership is decided by comparison against the domain's own bounds, so a coordinate type the crate does not ship is held to the same domain as the ones it does. The bridge into centred bits is public and nothing closes the set of its implementations, and the coordinate trait is implemented for the floats as well as the unsigned integers, so a host's coordinates may be signed and NaN-capable. Inferring from the bit width that every representable value is in the domain holds only for the unsigned types: at a width that fills a signed or floating type it would admit a value below the origin, a NaN and an infinity. The comparison is the root cell's own containment test and the root is permanent, so whatever the boundary admits the partition delivers, for any coordinate type a host may bring. | +//! | [`a_signed_coordinate_below_the_origin_is_counted_nowhere_at_a_narrow_width`] | routing | cites (´claim:routing:domain-membership-is-decided-by-comparison-for-every-coordinate-type´) | +//! | [`a_signed_coordinate_at_the_exclusive_bound_is_counted_nowhere_at_full_width`] | routing | cites (´claim:routing:domain-membership-is-decided-by-comparison-for-every-coordinate-type´) | + +//! Graph routing — how an observed value reaches the cell that will +//! analyse it. +//! +//! Ingestion has one job at the spatial layer: hand every raw value to the +//! G-V Graph as a single unit of importance. Nothing is weighted, discounted +//! for repetition, or batched away, so the graph's accumulated total is a +//! count of arrivals and nothing else. That is what makes the sentinel's own +//! lifetime counter and the graph's total two readings of one number rather +//! than two tallies that could drift apart. +//! +//! Structure then follows traffic. A range that keeps receiving values crosses +//! the split threshold and is refined into finer cells, which is how the +//! sentinel comes to model where traffic actually is rather than a partition +//! chosen in advance. Refinement is not free, so it is bounded: the node +//! budget caps how many cells the graph will hold however many distinct +//! ranges the traffic touches. Spreading traffic thinly across the domain +//! therefore costs a configured ceiling of memory rather than unbounded +//! growth, which matters because that spread is something an adversary +//! chooses. +//! +//! Both ends of that lifecycle are observable. A fresh sentinel is one root +//! cell spanning the whole domain with nothing accumulated, and `reset()` +//! returns it to exactly that state — the learned partition is derived from +//! observations, so discarding the observations leaves nothing worth keeping +//! behind. + +mod common; + +use common::{cell_values, test_config}; +use torrust_mudlark::Coordinate; +use torrust_sentinel::{CentredBitSource, CentredBits, Sentinel128, SentinelConfig, SpectralSentinel}; + +// ── Fresh state ───────────────────────────────────────────── + +/// Before anything is observed the graph is a single cell spanning the whole +/// coordinate domain. There is no partition worth choosing until traffic says +/// where the boundaries should fall, so the sentinel starts with the one cell +/// it can justify and refines outward from there. +/// +/// ´claim:routing:a-fresh-graph-is-one-root-cell-covering-the-whole-domain-with-nothing-accumulated´ +/// ´test:integration:fresh-graph-has-one-node´ +#[test] +fn fresh_graph_has_one_node() { + let s = Sentinel128::new(test_config()).unwrap(); + assert_eq!(s.graph().node_count(), 1); +} + +/// That single cell is also a leaf. Nothing has been split, so the one node +/// the graph holds is the one place an observation can land, and the count of +/// cells traffic can reach agrees with the count of nodes that exist. +/// +/// (´claim:routing:a-fresh-graph-is-one-root-cell-covering-the-whole-domain-with-nothing-accumulated´) +/// ´test:integration:fresh-graph-has-one-terminal´ +#[test] +fn fresh_graph_has_one_terminal() { + let s = Sentinel128::new(test_config()).unwrap(); + assert_eq!(s.graph().terminal_count(), 1); +} + +/// The other half of the fresh state: that root cell carries no accumulated +/// importance either. A cell exists because the domain has to be covered, not +/// because anything was seen in it, so structure and evidence start out +/// independent of one another. +/// +/// (´claim:routing:a-fresh-graph-is-one-root-cell-covering-the-whole-domain-with-nothing-accumulated´) +/// ´test:integration:fresh-graph-has-zero-total-sum´ +#[test] +fn fresh_graph_has_zero_total_sum() { + let s = Sentinel128::new(test_config()).unwrap(); + assert_eq!(s.graph().total_sum(), 0); +} + +// ── Basic routing ─────────────────────────────────────────── + +/// Every value in a batch contributes exactly one unit of importance, +/// whichever range it falls in, so the graph's total after successive batches +/// into unrelated ranges is simply how many values were handed over. +/// Importance is a count of arrivals rather than a weight the caller can set, +/// which is what lets the selector read it as evidence of where traffic is. +/// +/// ´claim:routing:every-ingested-value-adds-exactly-one-unit-of-importance-wherever-it-lands´ +/// ´test:integration:ingest-feeds-graph-with-delta-one´ +#[test] +fn ingest_feeds_graph_with_delta_one() { + let cfg = test_config(); + let mut s = Sentinel128::new(cfg).unwrap(); + + // First batch: 5 values. + s.ingest(&cell_values(0xA, 5)); + assert_eq!(s.graph().total_sum(), 5); + + // Second batch: 3 more values in a different range. + s.ingest(&cell_values(0x3, 3)); + assert_eq!(s.graph().total_sum(), 8); +} + +/// A batch with nothing in it is not an event. The accumulated total stays +/// where it was and no cell is created, so an interval in which nothing +/// arrived neither adds evidence nor moves the partition. Quiet time is +/// therefore invisible to the spatial layer rather than being recorded as an +/// observation of emptiness. +/// +/// ´claim:routing:an-empty-batch-routes-nothing-and-leaves-the-graph-exactly-as-it-was´ +/// ´test:integration:empty-ingest-does-not-observe´ +#[test] +fn empty_ingest_does_not_observe() { + let cfg = test_config(); + let mut s = Sentinel128::new(cfg).unwrap(); + + s.ingest(&[]); + assert_eq!(s.graph().total_sum(), 0); + assert_eq!(s.graph().node_count(), 1); // still just the root +} + +/// Repetition is traffic, not redundancy: one value handed over several times +/// in a batch counts several times over rather than collapsing into a single +/// arrival. The graph measures how often a region is visited, not how many +/// distinct values it has ever seen, which is why a flood from a single source +/// still moves the structure. +/// +/// (´claim:routing:every-ingested-value-adds-exactly-one-unit-of-importance-wherever-it-lands´) +/// ´test:integration:duplicate-values-each-contribute´ +#[test] +fn duplicate_values_each_contribute() { + let cfg = test_config(); + let mut s = Sentinel128::new(cfg).unwrap(); + + // Ingest three identical values — each should add 1 to total_sum. + let v = cell_values(0xB, 1)[0]; + s.ingest(&[v, v, v]); + assert_eq!(s.graph().total_sum(), 3); +} + +// ── Accumulation ──────────────────────────────────────────── + +/// The same unit contribution survives batch boundaries: a long run of +/// batches scattered over many ranges leaves a total equal to everything ever +/// handed over. A batch is a delivery convenience, not an accounting period, +/// so the graph reports lifetime evidence rather than the most recent window. +/// +/// (´claim:routing:every-ingested-value-adds-exactly-one-unit-of-importance-wherever-it-lands´) +/// ´test:integration:graph-accumulates-across-batches´ +#[test] +fn graph_accumulates_across_batches() { + let cfg = test_config(); + let mut s = Sentinel128::new(cfg).unwrap(); + + for nibble in 0..10u128 { + s.ingest(&cell_values(nibble, 100)); + } + + assert_eq!(s.graph().total_sum(), 1000); +} + +/// The counter the sentinel keeps and the total the graph accumulates stay +/// equal batch after batch. They are one quantity read from two layers: real +/// observations are the only thing that increments either, and the synthetic +/// data used to warm trackers is deliberately kept out of both. A host can +/// therefore read whichever is nearer to hand without learning which layer +/// maintains it. +/// +/// ´claim:routing:the-sentinels-lifetime-counter-and-the-graphs-total-are-one-quantity-read-twice´ +/// ´test:integration:lifetime-observations-tracks-total-sum´ +#[test] +fn lifetime_observations_tracks_total_sum() { + let cfg = test_config(); + let mut s = Sentinel128::new(cfg).unwrap(); + + s.ingest(&cell_values(0xC, 42)); + assert_eq!(s.lifetime_observations(), s.graph().total_sum()); + + // Still holds after a second batch. + s.ingest(&cell_values(0xD, 58)); + assert_eq!(s.lifetime_observations(), s.graph().total_sum()); + assert_eq!(s.lifetime_observations(), 100); +} + +// ── Structural evolution ──────────────────────────────────── + +/// Traffic that keeps landing in one narrow range drives that range past the +/// split threshold and the graph refines it, so the partition is bought with +/// observations rather than configured up front. Where the traffic goes is +/// where the resolution appears; the total meanwhile still counts exactly the +/// values handed over, so refining a region does not manufacture evidence. +/// +/// ´claim:routing:concentrated-traffic-buys-resolution-by-splitting-the-range-it-lands-in´ +/// ´test:integration:concentrated-traffic-splits-nodes´ +#[test] +fn concentrated_traffic_splits_nodes() { + let cfg = SentinelConfig:: { + split_threshold: 10, + ..test_config() + }; + let mut s = Sentinel128::new(cfg).unwrap(); + + // All values in the same nibble range — should concentrate + // into one cell and eventually split it. + s.ingest(&cell_values(0xA, 50)); + + assert_eq!(s.graph().total_sum(), 50); + assert!( + s.graph().node_count() > 1, + "expected splits from concentrated traffic, got {} nodes", + s.graph().node_count() + ); +} + +/// Refinement creates new places for traffic to land, not merely new interior +/// structure above the old cell: after a split the graph has more than one +/// leaf. Subsequent values in that range are therefore separated from one +/// another instead of continuing to pile into a single undifferentiated cell. +/// +/// (´claim:routing:concentrated-traffic-buys-resolution-by-splitting-the-range-it-lands-in´) +/// ´test:integration:concentrated-traffic-grows-terminals´ +#[test] +fn concentrated_traffic_grows_terminals() { + let cfg = SentinelConfig:: { + split_threshold: 10, + ..test_config() + }; + let mut s = Sentinel128::new(cfg).unwrap(); + + s.ingest(&cell_values(0xA, 50)); + + assert!( + s.graph().terminal_count() > 1, + "expected terminal_count > 1 after splits, got {}", + s.graph().terminal_count() + ); +} + +// ── Budget enforcement ────────────────────────────────────── + +/// Traffic spread thinly over many well-separated ranges asks the graph to +/// refine everywhere at once, and the node budget is what keeps that from +/// being unbounded: however many ranges are busy and however low the split +/// threshold is set, the graph holds no more cells than the budget allows. +/// The cost of modelling is a configured ceiling rather than a function of how +/// widely an adversary chooses to scatter. +/// +/// ´claim:routing:the-node-budget-caps-the-graph-however-widely-traffic-is-scattered´ +/// ´test:integration:diverse-traffic-respects-budget´ +#[test] +fn diverse_traffic_respects_budget() { + let cfg = SentinelConfig:: { + split_threshold: 5, + budget: 200, + ..test_config() + }; + let mut s = Sentinel128::new(cfg).unwrap(); + + // Spread observations across many distinct nibble ranges. + for nibble in 0..16u128 { + let values: Vec = (0u128..500).map(|i| (nibble << 124) | (i << 100)).collect(); + s.ingest(&values); + } + + assert!( + s.graph().node_count() <= 200, + "node count {} exceeds budget 200", + s.graph().node_count() + ); +} + +// ── Reset ─────────────────────────────────────────────────── + +/// Reset discards the partition as well as the evidence: a graph that had +/// split under load comes back as the single root cell of a fresh sentinel, +/// with nothing accumulated and nothing to land in but the root. Structure is +/// derived from observations, so once the observations are dropped there is no +/// refinement left worth preserving, and a reset sentinel cannot be +/// distinguished from a new one by what its graph holds. +/// +/// ´claim:routing:reset-returns-the-graph-to-the-single-root-cell-of-a-fresh-sentinel´ +/// ´test:integration:reset-restores-fresh-graph-state´ +#[test] +fn reset_restores_fresh_graph_state() { + let cfg = SentinelConfig:: { + split_threshold: 10, + ..test_config() + }; + let mut s = Sentinel128::new(cfg).unwrap(); + + // Ingest enough to split. + s.ingest(&cell_values(0xA, 100)); + assert!(s.graph().total_sum() > 0); + assert!(s.graph().node_count() > 1); + + s.reset(); + + assert_eq!(s.graph().total_sum(), 0); + assert_eq!(s.graph().node_count(), 1); + assert_eq!(s.graph().terminal_count(), 1); +} + +/// The topmost coordinate of the domain reaches a tracker rather than falling +/// through every cell. Cell intervals are half-open, which has no upper edge +/// case while the coordinate width is narrower than the coordinate type — the +/// bound is then a representable value outside the domain. At the full width +/// the domain's maximum is the type's maximum, there is no value above it to +/// be excluded, and a half-open reading of the topmost interval therefore +/// excludes a coordinate that is genuinely inside the domain. The spatial +/// layer counts that observation either way, so the two readings would +/// disagree: the accumulated total records an arrival that no tracker was ever +/// shown. +/// +/// ´claim:routing:the-domains-top-coordinate-reaches-a-tracker-rather-than-falling-through-every-cell´ +/// ´test:integration:top-of-domain-coordinate-routes-to-a-cell´ +#[test] +fn top_of_domain_coordinate_routes_to_a_cell() { + let cfg = test_config(); + let mut s = Sentinel128::new(cfg).unwrap(); + + let report = s.ingest(&[u128::MAX]); + + assert_eq!(s.graph().total_sum(), 1, "the spatial layer counts the observation"); + + // Competitive and ancestor cells are reported separately, and a fresh + // sentinel holds only the root, which is an ancestor by construction. + let routed: usize = report + .cell_reports + .iter() + .chain(report.ancestor_reports.iter()) + .map(|c| c.sample_count) + .sum(); + assert_eq!(routed, 1, "and a tracker must be shown the same observation"); +} + +/// The inclusive reading is confined to the top of the domain, so an ordinary +/// coordinate still lands in exactly one cell of the partition. Widening the +/// upper bound everywhere would put each boundary value in two sibling cells +/// at once and count it twice; widening it only where there is no successor +/// leaves every other boundary exactly as it was. +/// +/// (´claim:routing:the-domains-top-coordinate-reaches-a-tracker-rather-than-falling-through-every-cell´) +/// ´test:integration:an-ordinary-coordinate-still-lands-in-one-cell´ +#[test] +fn an_ordinary_coordinate_still_lands_in_one_cell() { + let cfg = test_config(); + let mut s = Sentinel128::new(cfg).unwrap(); + + let report = s.ingest(&cell_values(0x7, 1)); + + assert_eq!(s.graph().total_sum(), 1); + + let routed: usize = report + .cell_reports + .iter() + .chain(report.ancestor_reports.iter()) + .map(|c| c.sample_count) + .sum(); + assert_eq!(routed, 1, "one arrival is shown to one tracker"); +} + +/// A coordinate the domain cannot name is not an observation of it, and the +/// three layers that would each read it differently are not left to disagree +/// about that. The spatial layer accumulates such a value in the topmost cell, +/// whose interval does not contain it; the encoder reads the low bits of the +/// configured width, so it would hand a tracker the vector of the in-domain +/// value the arrival is congruent to; and the interval scan matches no cell at +/// all, not even the root. Deciding membership once, before any of them, is +/// what keeps the counts one count: the value raises no total, moves no +/// partition and reaches no tracker. +/// +/// The width here is narrower than the coordinate type, which for an unsigned +/// coordinate is the only shape in which a value outside the domain is +/// representable: at the full width every value `u64` can hold is inside the +/// domain, and the tests named `top_of_domain_coordinate_routes_to_a_cell`, +/// `an_ordinary_coordinate_still_lands_in_one_cell` and +/// `lifetime_observations_tracks_total_sum` hold that reading unchanged. A +/// coordinate type that is signed or NaN-capable has values outside the domain +/// at every width, which is what the two tests at the end of this file cover. +/// +/// ´claim:routing:a-coordinate-outside-the-domain-is-counted-in-no-total-and-reaches-no-tracker´ +/// ´test:integration:values-outside-the-domain-are-counted-nowhere´ +#[test] +fn values_outside_the_domain_are_counted_nowhere() { + // At N = 8 over a 64-bit coordinate the domain is [0, 256), so 256 is the + // first representable value outside it — and the value the 8-bit encoder + // would present as zero. + let mut s = torrust_sentinel::SpectralSentinel::::new(test_config()).unwrap(); + + let report = s.ingest(&[42, 256]); + + assert_eq!(s.graph().total_sum(), 1, "only the in-domain value moves the graph"); + assert_eq!( + s.lifetime_observations(), + 1, + "and only it is counted against the sentinel's lifetime" + ); + + let root = report + .cell_reports + .iter() + .chain(report.ancestor_reports.iter()) + .find(|cell| cell.depth == 0) + .expect("the root tracker is permanent and receives every observation"); + assert_eq!( + root.sample_count, 1, + "the root is shown the in-domain value alone — nothing aliased onto it" + ); + + // A batch of nothing but out-of-domain values is the same non-event as an + // empty batch: no total moves and the report describes no observation. + let empty = s.ingest(&[256, 257, u64::MAX]); + + assert_eq!(s.graph().total_sum(), 1, "the graph total is where the first batch left it"); + assert_eq!(s.lifetime_observations(), 1, "and so is the lifetime count"); + assert!( + empty.cell_reports.is_empty() && empty.ancestor_reports.is_empty(), + "no cell was shown anything, so no cell reports" + ); + assert!( + empty.oldest_observation_age_micros.is_none(), + "a batch with no observation in it has no oldest observation to age" + ); +} + +/// Check the first full-width midpoint and the maximum in a fresh engine. +fn assert_full_width_encodings() { + let config = SentinelConfig:: { + split_threshold: 1, + d_create: 1, + d_evict: 2, + analysis_k: 16, + max_rank: 1, + noise_schedule: torrust_sentinel::NoiseSchedule::Explicit(Vec::new()), + ..SentinelConfig::default() + }; + let top = C::domain_max(N); + let boundary = C::midpoint(C::zero(), top); + let mut sentinel = torrust_sentinel::SpectralSentinel::::new(config).unwrap(); + let report = sentinel.ingest(&[boundary, top]); + let boundary_bits = boundary.to_centred_bits(N); + let top_bits = top.to_centred_bits(N); + let cells: Vec<_> = report.cell_reports.iter().chain(&report.ancestor_reports).collect(); + assert!(cells.iter().any(|cell| cell.depth == 1), "the fixture must split the root"); + for cell in cells { + if cell.sample_count == 2 { + let depth = u8::try_from(cell.depth).unwrap(); + assert_ne!( + boundary_bits.suffix(depth), + top_bits.suffix(depth), + "distinct values in one cell must have distinct encodings at depth {depth}" + ); + } + } +} + +/// Distinct values routed into one full-width cell have distinct suffix encodings. +/// +/// ´claim:routing:full-width-cells-keep-distinct-encodings´ +/// ´test:integration:full-width-u64-cells-keep-distinct-encodings´ +#[test] +fn full_width_u64_cells_keep_distinct_encodings() { + assert_full_width_encodings::(); +} + +/// The full-width routing correction also holds at the widest supported coordinate width. +/// +/// ´test:integration:full-width-u128-cells-keep-distinct-encodings´ +#[test] +fn full_width_u128_cells_keep_distinct_encodings() { + assert_full_width_encodings::(); +} + +// ── A host's own coordinate type ──────────────────────────── + +/// A coordinate type of the kind a downstream host writes and this crate does +/// not ship: a newtype over `f64` whose every coordinate method is `f64`'s own. +/// +/// The bridge into centred bits is public and nothing closes the set of its +/// implementations, and the coordinate trait is implemented for the floats as +/// well as for the unsigned integers, so a host's coordinates may be signed and +/// NaN-capable. Nothing here narrows what such a host may write: each method +/// forwards to the implementation Mudlark already publishes for `f64`, so the +/// type claims no behaviour the trait does not already permit, and it carries +/// `f64`'s refusal of a successor value unchanged rather than inventing one the +/// float line does not have. A stand-in that quietly behaved better than `f64` +/// would prove something about itself rather than about what a host may bring. +/// +/// The conversion into centred bits reads the value's IEEE 754 bit pattern, +/// which is deterministic and needs no cast. Which bits a host derives from its +/// coordinates is its own affair, and the choice is not under test here: the +/// domain decision is taken before the encoder is reached, so no value these +/// tests expect to be rejected ever arrives at this conversion. +#[derive(Copy, Clone, Debug, Default, PartialEq, PartialOrd)] +struct SignedCoordinate(f64); + +impl Coordinate for SignedCoordinate { + const BITS: u32 = ::BITS; + + fn zero() -> Self { + Self(::zero()) + } + + fn domain_max(n: u32) -> Self { + Self(::domain_max(n)) + } + + fn midpoint(a: Self, b: Self) -> Self { + Self(::midpoint(a.0, b.0)) + } + + fn width(start: Self, end: Self) -> Self { + Self(::width(start.0, end.0)) + } + + fn is_final(start: Self, end: Self, depth: u32, n: u32) -> bool { + ::is_final(start.0, end.0, depth, n) + } + + fn from_u64(v: u64) -> Self { + Self(::from_u64(v)) + } + + fn next_value(self) -> Self { + Self(::next_value(self.0)) + } + + fn to_f64(self) -> f64 { + ::to_f64(self.0) + } + + fn is_nan(self) -> bool { + ::is_nan(self.0) + } + + fn total_cmp(&self, other: &Self) -> std::cmp::Ordering { + ::total_cmp(&self.0, &other.0) + } +} + +impl CentredBitSource for SignedCoordinate { + fn to_centred_bits(&self, n: u32) -> CentredBits { + self.0.to_bits().to_centred_bits(n) + } +} + +/// The count of observations the root tracker was shown in one report. +fn root_sample_count(report: &torrust_sentinel::BatchReport) -> usize { + report + .cell_reports + .iter() + .chain(report.ancestor_reports.iter()) + .find(|cell| cell.depth == 0) + .expect("the root tracker is permanent and receives every observation") + .sample_count +} + +/// Domain membership is decided by comparison against the domain's own bounds, +/// so a coordinate type the crate does not ship is held to the same domain as +/// the ones it does. Inferring from the bit width that every representable +/// value is in the domain holds only for the unsigned types: at a width that +/// fills a signed or floating type it would admit a value below the origin, a +/// NaN and an infinity, each of which the spatial layer, the encoder and the +/// interval scan would then read differently. The comparison is the root cell's +/// own containment test and the root is permanent, so whatever the boundary +/// admits the partition delivers — which is the mandatory delivery holding for +/// any coordinate type a host may bring, not only for the two shipped here. +/// +/// A NaN needs no case of its own. Every comparison with a NaN is false, so it +/// is neither at nor above the origin and both arms of the test refuse it. +/// +/// ´claim:routing:domain-membership-is-decided-by-comparison-for-every-coordinate-type´ +/// ´test:integration:a-signed-coordinate-outside-the-domain-is-counted-nowhere-at-full-width´ +#[test] +fn a_signed_coordinate_outside_the_domain_is_counted_nowhere_at_full_width() { + // At N = 64 the width fills the coordinate type, which is the shape in + // which an unsigned coordinate has nothing outside the domain at all. This + // type has three: below the origin, comparable with nothing, and above + // every bound. + let below = SignedCoordinate(-1.0); + let nan = SignedCoordinate(f64::NAN); + let above = SignedCoordinate(f64::INFINITY); + let outside = [below, nan, above]; + + let mut s = SpectralSentinel::::new(test_config()).unwrap(); + + let report = s.ingest(&[SignedCoordinate(42.0), below, nan, above]); + + assert_eq!(s.graph().total_sum(), 1, "only the in-domain value moves the graph"); + assert_eq!( + s.lifetime_observations(), + 1, + "and only it is counted against the sentinel's lifetime" + ); + assert_eq!( + root_sample_count(&report), + 1, + "the root is shown the in-domain value alone — nothing aliased onto it" + ); + + // A batch of nothing but such values is the same non-event as an empty + // batch, exactly as it is for an unsigned coordinate below the full width. + let empty = s.ingest(&outside); + + assert_eq!(s.graph().total_sum(), 1, "the graph total is where the first batch left it"); + assert_eq!(s.lifetime_observations(), 1, "and so is the lifetime count"); + assert!( + empty.cell_reports.is_empty() && empty.ancestor_reports.is_empty(), + "no cell was shown anything, so no cell reports" + ); + assert!( + empty.oldest_observation_age_micros.is_none(), + "a batch with no observation in it has no oldest observation to age" + ); + + // The domain's own values are untouched by the test that refuses those: + // the origin is inside the domain and so is any finite value below the + // upper bound the coordinate type names. + let accepted = s.ingest(&[SignedCoordinate(0.0), SignedCoordinate(1e18)]); + + assert_eq!(s.graph().total_sum(), 3, "the origin and an ordinary value are observations"); + assert_eq!(s.lifetime_observations(), 3, "and both are counted"); + assert_eq!(root_sample_count(&accepted), 2, "and both reach the root"); +} + +/// Below the full width the lower bound is what refuses a coordinate the domain +/// cannot name. The upper bound is a representable value there and stays +/// exclusive for every coordinate type, so a negative is rejected exactly as +/// the first value above the domain already is — a width narrower than the +/// coordinate type is not a case the old reading got right either, because a +/// comparison against the upper bound alone has no lower bound at all. +/// +/// (´claim:routing:domain-membership-is-decided-by-comparison-for-every-coordinate-type´) +/// ´test:integration:a-signed-coordinate-below-the-origin-is-counted-nowhere-at-a-narrow-width´ +#[test] +fn a_signed_coordinate_below_the_origin_is_counted_nowhere_at_a_narrow_width() { + // At N = 8 the domain is [0, 256): 256 is the first value above it and −1 + // the first below. + let mut s = SpectralSentinel::::new(test_config()).unwrap(); + + let report = s.ingest(&[SignedCoordinate(-1.0), SignedCoordinate(42.0), SignedCoordinate(256.0)]); + + assert_eq!(s.graph().total_sum(), 1, "only the in-domain value moves the graph"); + assert_eq!( + s.lifetime_observations(), + 1, + "and only it is counted against the sentinel's lifetime" + ); + assert_eq!( + root_sample_count(&report), + 1, + "the value below the origin is refused as the one above the bound is" + ); +} + +/// The inclusive reading of the domain's upper bound belongs to the integer +/// coordinates, where `domain_max` substitutes the type's maximum because +/// `2^N` is not representable: there the bound names the last value of the +/// domain and the topmost cell has to own it. A continuous coordinate carries +/// no such substitution — `2^N` is representable and `domain_max` returns it — +/// so at the full width it is the first value above the domain exactly as it +/// is at a narrow one. Taking the exception from the width alone does not tell +/// the two apart: it would admit that value, raise both totals and hand the +/// topmost cell an arrival its interval does not contain. Asking instead +/// whether the unit interval is indivisible at depth zero separates them by +/// the property that actually differs, so a type whose bound is exclusive +/// keeps it exclusive at every width. +/// +/// The integer half of the reading is unchanged and is held by two tests named +/// elsewhere in this file: `top_of_domain_coordinate_routes_to_a_cell`, where +/// the maximum of a full-width unsigned domain does reach a tracker, and +/// `values_outside_the_domain_are_counted_nowhere`, where the first value +/// above a narrower unsigned domain does not. +/// +/// (´claim:routing:domain-membership-is-decided-by-comparison-for-every-coordinate-type´) +/// ´test:integration:a-signed-coordinate-at-the-exclusive-bound-is-counted-nowhere-at-full-width´ +#[test] +fn a_signed_coordinate_at_the_exclusive_bound_is_counted_nowhere_at_full_width() { + // 18_446_744_073_709_551_616 is 2^64. A power of two is exact in a binary + // float whenever its exponent is in range, and 64 is far inside `f64`'s, + // so the literal is the bound itself rather than a neighbour of it and the + // equality the domain boundary tests is the exact one. The assertion below + // holds the literal to that: it is the same value the partition's topmost + // interval is built from. + let bound = SignedCoordinate(18_446_744_073_709_551_616.0); + assert_eq!( + bound, + ::domain_max(64), + "the literal is the upper bound of the domain at the full width" + ); + + let mut s = SpectralSentinel::::new(test_config()).unwrap(); + + let report = s.ingest(&[SignedCoordinate(42.0), bound]); + + assert_eq!(s.graph().total_sum(), 1, "only the in-domain value moves the graph"); + assert_eq!( + s.lifetime_observations(), + 1, + "and only it is counted against the sentinel's lifetime" + ); + assert_eq!( + root_sample_count(&report), + 1, + "the root is shown the in-domain value alone — no cell owns a bound that is outside the domain" + ); + + // A batch of nothing but the bound is the same non-event as an empty + // batch, exactly as a batch above a narrower domain is. + let empty = s.ingest(&[bound]); + + assert_eq!(s.graph().total_sum(), 1, "the graph total is where the first batch left it"); + assert_eq!(s.lifetime_observations(), 1, "and so is the lifetime count"); + assert!( + empty.cell_reports.is_empty() && empty.ancestor_reports.is_empty(), + "no cell was shown anything, so no cell reports" + ); + assert!( + empty.oldest_observation_age_micros.is_none(), + "a batch with no observation in it has no oldest observation to age" + ); +} diff --git a/packages/sentinel/tests/health.rs b/packages/sentinel/tests/health.rs new file mode 100644 index 000000000..52e81ad61 --- /dev/null +++ b/packages/sentinel/tests/health.rs @@ -0,0 +1,427 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +//! # Test index +//! +//! | Test | Area | Claim | +//! |------|------|-------| +//! | [`fresh_has_one_root_tracker`] | health | A sentinel that has ingested nothing holds exactly one tracker, on the root cell, over a graph of a single node. The root exists unconditionally so that every ancestor chain has somewhere to terminate; every other tracker is bought only once traffic has justified it. | +//! | [`fresh_has_zero_observations`] | health | The lifetime count reads zero on a fresh sentinel even though its root tracker has already absorbed synthetic seeding. Injected noise gives a tracker a usable model but is not evidence about traffic, so it is kept deliberately out of the number a host reads as how much the engine has actually seen. | +//! | [`fresh_rank_distribution_is_uniform_at_one`] | health | With a single tracker the rank distribution degenerates: smallest, largest and mean all agree, and all sit at the starting rank of one, because a learned subspace begins with a single basis direction. The distribution summarises a fleet, and a fleet of one has no spread to report. | +//! | [`fresh_maturity_is_cold`] | health | A tracker counts as cold for as long as its real-observation count is zero, whatever synthetic data it has already absorbed. Coldness measures exposure to the world rather than whether the model is populated, which is exactly the distinction a host needs when deciding how much a score from that tracker is worth. | +//! | [`fresh_geometry_distribution`] | health | The geometry summary counts trackers whose axes are structurally unavailable rather than merely quiet. A rank-one tracker has no second direction for coherence to compare against and is counted inactive on that axis, while on a wide domain its rank is nowhere near the dimension so novelty is not saturated. A zero score means something different in each case, and this is where a host learns which case it is in. | +//! | [`fresh_clip_pressure_is_zero`] | health | Clip pressure reads zero at every extreme on a sentinel that has seen no real data. Pressure accumulates only when observations actually press against the clipping bound, so it is a record of how often the engine had to hold data back — and an engine that has held nothing back reports none. | +//! | [`fresh_coordination_is_empty`] | coordination | No coordination context exists until a batch has been observed, because a context is created only where cells on both sides of a split report in the same batch. The summary is nonetheless present and reads empty rather than being absent: the tier is always described, even when there is nothing in it. | +//! | [`tracker_counts_are_consistent`] | health | The active trackers decompose exactly into the competitive cells, the ancestors pulled in to connect them, and the one permanent root. These are not three independent measurements but one partition reported three ways, so a host can read the shape of the engine's investment from them and any disagreement would be an accounting fault rather than a fact about traffic. | +//! | [`investment_set_covers_active_plus_warming`] | health | The investment set is every tracker the engine is paying for, whether already scoring or still warming up in the staging pipeline. Warming cells consume memory and work before they produce anything, so a figure that counted only the online trackers would understate what the sentinel is spending. | +//! | [`population_grows_after_divergent_ingest`] | health | Values whose leading bits diverge land in separate regions of the domain and the tracker population follows the traffic there. The two counts that describe that population stay in step: the cells the sentinel says it is tracking are exactly the cells with trackers behind them, so neither figure can drift into describing an investment that does not exist. | +//! | [`lifetime_observations_accumulate`] | health | cites (´claim:health:lifetime-observations-counts-real-data-only-so-seeding-noise-leaves-it-at-zero´) | +//! | [`rank_bounds_hold_after_ingest`] | health | Ranks reported after real traffic stay inside the configured ceiling from above and at the starting rank or better from below, with the mean lying between the two extremes it summarises. The ceiling is a spending decision the host made, so the snapshot is expected to respect it rather than announce a model larger than was authorised. | +//! | [`noise_reduces_noise_influence`] | health | Noise influence is the share of a tracker's model still owed to its synthetic seeding, and real observations dilute it: once actual traffic has arrived the fleet's mean influence has fallen below the value it starts at. It is the measurement that tells a host how much of a score is still borrowed from data the engine invented for itself. | +//! | [`cold_config_leaves_trackers_cold`] | health | With seeding switched off there is nothing to dilute, so real data leaves every tracker at the same maturity and the distribution collapses — its lowest value is no lower than its mean. Spread in maturity across the fleet comes from trackers being at different stages of shedding their synthetic prior, not from the traffic alone. | +//! | [`health_evolves_over_repeated_batches`] | health | Across a run of warm-up batches the snapshot reflects the whole history rather than the batch just handled: observations total up across all of them, trackers remain live, and rank has had the chance to adapt above its starting value. Health describes the engine's state as it now stands, which is a running position and not a per-batch reading. | +//! | [`coordination_starts_empty`] | coordination | cites (´claim:coordination:no-context-exists-until-a-batch-has-been-observed´) | +//! | [`coordination_structurally_sound_after_noise`] | coordination | Whatever contexts a run happens to have created, the shape they report is internally consistent: the tier's fixed dimensionality, a capacity that exceeds neither the configured rank ceiling nor the four dimensions available, an influence share inside its unit range, and a rank between the starting value and that capacity. The check is conditional because contexts are created lazily where traffic happens to fall, and pretending otherwise would test the fixture rather than the engine. | + +//! The health snapshot — what the sentinel reports about its own condition +//! rather than about the traffic it is watching. +//! +//! A host cannot read cell scores sensibly without knowing what state the +//! engine was in when it produced them, so the snapshot describes the +//! population of trackers it is paying for and how far along that population +//! is. The counts decompose exactly: the active trackers are the competitive +//! cells, plus the ancestors pulled in to connect them, plus the one +//! permanent root; the investment set is those together with whatever is +//! still warming in the staging pipeline. Rank, maturity, geometry and clip +//! pressure arrive as distributions rather than single values, because what +//! is interesting about a fleet of trackers is its extremes and its mean. +//! +//! Maturity is measured against the synthetic prior each tracker is seeded +//! with. A tracker that has seen only injected noise is cold: the noise buys +//! it a usable model but is not evidence about real traffic, so the lifetime +//! observation count ignores it entirely and the maturity figure records how +//! much of the model is still borrowed. Real data dilutes that share as it +//! arrives. +//! +//! The coordination tier appears here as a summary of its own, present and +//! reading empty before any context exists. Like every other field, none of +//! it is a verdict: health is a measurement about the engine, offered so the +//! host can judge how much weight the rest of the report deserves. + +mod common; + +use common::{ScenarioBuilder, cold_config, seeded_sentinel, test_config}; +use torrust_sentinel::Sentinel128; + +// ═══════════════════════════════════════════════════════════ +// Fresh sentinel +// ═══════════════════════════════════════════════════════════ + +/// A sentinel that has ingested nothing holds exactly one tracker, on the +/// root cell, over a graph of a single node. The root exists unconditionally +/// so that every ancestor chain has somewhere to terminate; every other +/// tracker is bought only once traffic has justified it. +/// +/// ´claim:health:a-fresh-sentinel-holds-exactly-the-root-tracker-over-a-single-node-graph´ +/// ´test:integration:fresh-has-one-root-tracker´ +#[test] +fn fresh_has_one_root_tracker() { + let s = Sentinel128::new(test_config()).unwrap(); + let h = s.health(); + + assert_eq!(h.active_trackers, 1, "only the root tracker exists"); + assert_eq!(h.cells_tracked, 1); + assert_eq!(h.total_g_nodes, 1); +} + +/// The lifetime count reads zero on a fresh sentinel even though its root +/// tracker has already absorbed synthetic seeding. Injected noise gives a +/// tracker a usable model but is not evidence about traffic, so it is kept +/// deliberately out of the number a host reads as how much the engine has +/// actually seen. +/// +/// ´claim:health:lifetime-observations-counts-real-data-only-so-seeding-noise-leaves-it-at-zero´ +/// ´test:integration:fresh-has-zero-observations´ +#[test] +fn fresh_has_zero_observations() { + let s = Sentinel128::new(test_config()).unwrap(); + let h = s.health(); + + assert_eq!(h.lifetime_observations, 0); +} + +/// With a single tracker the rank distribution degenerates: smallest, +/// largest and mean all agree, and all sit at the starting rank of one, +/// because a learned subspace begins with a single basis direction. The +/// distribution summarises a fleet, and a fleet of one has no spread to +/// report. +/// +/// ´claim:health:one-tracker-collapses-the-rank-distribution-to-a-single-value-at-the-starting-rank´ +/// ´test:integration:fresh-rank-distribution-is-uniform-at-one´ +#[test] +fn fresh_rank_distribution_is_uniform_at_one() { + let s = Sentinel128::new(test_config()).unwrap(); + let rd = s.health().rank_distribution; + + assert_eq!(rd.min, 1); + assert_eq!(rd.max, 1); + assert!((rd.mean - 1.0).abs() < f64::EPSILON); +} + +/// A tracker counts as cold for as long as its real-observation count is +/// zero, whatever synthetic data it has already absorbed. Coldness measures +/// exposure to the world rather than whether the model is populated, which +/// is exactly the distinction a host needs when deciding how much a score +/// from that tracker is worth. +/// +/// ´claim:health:a-tracker-that-has-seen-only-synthetic-data-is-still-counted-cold´ +/// ´test:integration:fresh-maturity-is-cold´ +#[test] +fn fresh_maturity_is_cold() { + let s = Sentinel128::new(test_config()).unwrap(); + let md = s.health().maturity_distribution; + + // Root tracker has received noise but no real observations, + // so it counts as cold (zero real observations). + assert_eq!(md.cold_trackers, 1); +} + +/// The geometry summary counts trackers whose axes are structurally +/// unavailable rather than merely quiet. A rank-one tracker has no second +/// direction for coherence to compare against and is counted inactive on +/// that axis, while on a wide domain its rank is nowhere near the dimension +/// so novelty is not saturated. A zero score means something different in +/// each case, and this is where a host learns which case it is in. +/// +/// ´claim:health:the-geometry-summary-counts-which-axes-are-structurally-available-rather-than-merely-quiet´ +/// ´test:integration:fresh-geometry-distribution´ +#[test] +fn fresh_geometry_distribution() { + let s = Sentinel128::new(test_config()).unwrap(); + let gd = s.health().geometry_distribution; + + // With a single rank-1 root tracker on a 128-wide space, + // novelty is not saturated and coherence is inactive (rank < 2). + assert_eq!(gd.novelty_saturated, 0); + assert_eq!(gd.coherence_inactive, 1, "rank-1 tracker cannot compute coherence"); +} + +/// Clip pressure reads zero at every extreme on a sentinel that has seen no +/// real data. Pressure accumulates only when observations actually press +/// against the clipping bound, so it is a record of how often the engine had +/// to hold data back — and an engine that has held nothing back reports +/// none. +/// +/// ´claim:health:clip-pressure-stays-at-zero-until-observations-press-against-the-bound´ +/// ´test:integration:fresh-clip-pressure-is-zero´ +#[test] +fn fresh_clip_pressure_is_zero() { + let s = Sentinel128::new(test_config()).unwrap(); + let cp = s.health().clip_pressure_distribution; + + assert!((cp.min).abs() < f64::EPSILON); + assert!((cp.max).abs() < f64::EPSILON); + assert!((cp.mean).abs() < f64::EPSILON); +} + +/// No coordination context exists until a batch has been observed, because a +/// context is created only where cells on both sides of a split report in +/// the same batch. The summary is nonetheless present and reads empty rather +/// than being absent: the tier is always described, even when there is +/// nothing in it. +/// +/// ´claim:coordination:no-context-exists-until-a-batch-has-been-observed´ +/// ´test:integration:fresh-coordination-is-empty´ +#[test] +fn fresh_coordination_is_empty() { + let s = Sentinel128::new(test_config()).unwrap(); + let ch = s.health().coordination_health; + + assert_eq!(ch.active_contexts, 0); +} + +// ═══════════════════════════════════════════════════════════ +// Tracker arithmetic +// ═══════════════════════════════════════════════════════════ + +/// The active trackers decompose exactly into the competitive cells, the +/// ancestors pulled in to connect them, and the one permanent root. These +/// are not three independent measurements but one partition reported three +/// ways, so a host can read the shape of the engine's investment from them +/// and any disagreement would be an accounting fault rather than a fact +/// about traffic. +/// +/// ´claim:health:active-trackers-decompose-exactly-into-competitive-cells-ancestors-and-the-permanent-root´ +/// ´test:integration:tracker-counts-are-consistent´ +#[test] +fn tracker_counts_are_consistent() { + let s = seeded_sentinel(); + let h = s.health(); + + // active = competitive + ancestor + 1 (root) + assert_eq!( + h.active_trackers, + h.active_competitive_trackers + h.active_ancestor_trackers + 1, + "active trackers = competitive + ancestor + root" + ); +} + +/// The investment set is every tracker the engine is paying for, whether +/// already scoring or still warming up in the staging pipeline. Warming +/// cells consume memory and work before they produce anything, so a figure +/// that counted only the online trackers would understate what the sentinel +/// is spending. +/// +/// ´claim:health:the-investment-set-is-the-online-trackers-plus-those-still-warming´ +/// ´test:integration:investment-set-covers-active-plus-warming´ +#[test] +fn investment_set_covers_active_plus_warming() { + let s = seeded_sentinel(); + let h = s.health(); + + assert_eq!( + h.investment_set_size, + h.active_trackers + h.warming_trackers, + "investment = active + warming" + ); +} + +// ═══════════════════════════════════════════════════════════ +// After ingestion +// ═══════════════════════════════════════════════════════════ + +/// Values whose leading bits diverge land in separate regions of the domain +/// and the tracker population follows the traffic there. The two counts that +/// describe that population stay in step: the cells the sentinel says it is +/// tracking are exactly the cells with trackers behind them, so neither +/// figure can drift into describing an investment that does not exist. +/// +/// ´claim:health:the-tracker-population-follows-the-traffic-and-cells-tracked-matches-the-trackers-behind-them´ +/// ´test:integration:population-grows-after-divergent-ingest´ +#[test] +fn population_grows_after_divergent_ingest() { + let mut s = Sentinel128::new(test_config()).unwrap(); + + // Two values with maximally separated leading bits. + s.ingest(&[ + 0xF000_0000_0000_0000_0000_0000_0000_0001, + 0x1000_0000_0000_0000_0000_0000_0000_0002, + ]); + + let h = s.health(); + assert!(h.active_trackers >= 1); + assert_eq!(h.cells_tracked, h.active_trackers); + assert_eq!(h.lifetime_observations, 2); +} + +/// Pins the accumulating end of the same count: every value in every batch +/// adds one, and the total carries across separate ingest calls instead of +/// describing only the batch just handled. That is what makes it readable as +/// the age of the engine's experience rather than a measure of the latest +/// traffic. +/// +/// (´claim:health:lifetime-observations-counts-real-data-only-so-seeding-noise-leaves-it-at-zero´) +/// ´test:integration:lifetime-observations-accumulate´ +#[test] +fn lifetime_observations_accumulate() { + let mut s = Sentinel128::new(test_config()).unwrap(); + + s.ingest(&[0x0000_0000_0000_0000_0000_0000_0000_0001]); + assert_eq!(s.health().lifetime_observations, 1); + + s.ingest(&[ + 0x0000_0000_0000_0000_0000_0000_0000_0002, + 0x0000_0000_0000_0000_0000_0000_0000_0003, + ]); + assert_eq!(s.health().lifetime_observations, 3); +} + +/// Ranks reported after real traffic stay inside the configured ceiling from +/// above and at the starting rank or better from below, with the mean lying +/// between the two extremes it summarises. The ceiling is a spending +/// decision the host made, so the snapshot is expected to respect it rather +/// than announce a model larger than was authorised. +/// +/// ´claim:health:reported-ranks-respect-the-configured-ceiling-and-the-mean-lies-between-the-extremes´ +/// ´test:integration:rank-bounds-hold-after-ingest´ +#[test] +fn rank_bounds_hold_after_ingest() { + let s = seeded_sentinel(); + let rd = s.health().rank_distribution; + + assert!(rd.min >= 1, "rank must be at least 1"); + assert!(rd.max <= s.config().max_rank, "rank must not exceed max_rank"); + #[allow(clippy::cast_precision_loss)] + { + assert!(rd.mean >= rd.min as f64); + assert!(rd.mean <= rd.max as f64); + } +} + +// ═══════════════════════════════════════════════════════════ +// Maturity +// ═══════════════════════════════════════════════════════════ + +/// Noise influence is the share of a tracker's model still owed to its +/// synthetic seeding, and real observations dilute it: once actual traffic +/// has arrived the fleet's mean influence has fallen below the value it +/// starts at. It is the measurement that tells a host how much of a score is +/// still borrowed from data the engine invented for itself. +/// +/// ´claim:health:real-observations-dilute-the-synthetic-share-so-noise-influence-falls-below-its-starting-value´ +/// ´test:integration:noise-reduces-noise-influence´ +#[test] +fn noise_reduces_noise_influence() { + let s = seeded_sentinel(); + let md = s.health().maturity_distribution; + + // Real observations dilute the noise baseline — mean influence + // drops below the 1.0 cold-start value. + assert!( + md.mean_noise_influence < 1.0, + "noise injection + real data should reduce mean noise_influence, got {}", + md.mean_noise_influence, + ); +} + +/// With seeding switched off there is nothing to dilute, so real data leaves +/// every tracker at the same maturity and the distribution collapses — its +/// lowest value is no lower than its mean. Spread in maturity across the +/// fleet comes from trackers being at different stages of shedding their +/// synthetic prior, not from the traffic alone. +/// +/// ´claim:health:with-seeding-disabled-the-maturity-distribution-collapses-to-a-single-level´ +/// ´test:integration:cold-config-leaves-trackers-cold´ +#[test] +fn cold_config_leaves_trackers_cold() { + let mut s = Sentinel128::new(cold_config()).unwrap(); + + // Even after real data, cold_config skips noise so trackers + // remain at maximum noise influence. + s.ingest(&[0xAAAA_BBBB_CCCC_DDDD_0000_0000_0000_0001]); + + let md = s.health().maturity_distribution; + assert!( + md.min_noise_influence >= md.mean_noise_influence, + "min should be >= mean (all at same level without noise)", + ); +} + +// ═══════════════════════════════════════════════════════════ +// Multi-batch evolution +// ═══════════════════════════════════════════════════════════ + +/// Across a run of warm-up batches the snapshot reflects the whole history +/// rather than the batch just handled: observations total up across all of +/// them, trackers remain live, and rank has had the chance to adapt above +/// its starting value. Health describes the engine's state as it now stands, +/// which is a running position and not a per-batch reading. +/// +/// ´claim:health:the-snapshot-describes-the-engine-as-it-stands-after-a-whole-run-not-just-the-latest-batch´ +/// ´test:integration:health-evolves-over-repeated-batches´ +#[test] +fn health_evolves_over_repeated_batches() { + let (s, _) = ScenarioBuilder::new() + .seed_range(0xA, 8) + .seed_range(0x5, 8) + .warm_batches(5) + .build_with_reports(); + + let h = s.health(); + + // After several warm-up batches the sentinel should have + // accumulated meaningful observations and the rank may have + // adapted above 1. + assert!(h.lifetime_observations >= 6 * 16, "6 batches × 16 values"); + assert!(h.rank_distribution.max >= 1); + assert!(h.active_trackers >= 1); +} + +// ═══════════════════════════════════════════════════════════ +// Coordination health +// ═══════════════════════════════════════════════════════════ + +/// Read directly from the coordination summary rather than through the wider +/// health report, a newly constructed sentinel still shows no active +/// contexts. Construction alone creates nothing at this tier; only observed +/// traffic can. +/// +/// (´claim:coordination:no-context-exists-until-a-batch-has-been-observed´) +/// ´test:integration:coordination-starts-empty´ +#[test] +fn coordination_starts_empty() { + let s = Sentinel128::new(test_config()).unwrap(); + let ch = s.health().coordination_health; + + assert_eq!(ch.active_contexts, 0); +} + +/// Whatever contexts a run happens to have created, the shape they report is +/// internally consistent: the tier's fixed dimensionality, a capacity that +/// exceeds neither the configured rank ceiling nor the four dimensions +/// available, an influence share inside its unit range, and a rank between +/// the starting value and that capacity. The check is conditional because +/// contexts are created lazily where traffic happens to fall, and pretending +/// otherwise would test the fixture rather than the engine. +/// +/// ´claim:coordination:a-contexts-reported-rank-and-capacity-stay-inside-the-bounds-it-was-built-with´ +/// ´test:integration:coordination-structurally-sound-after-noise´ +#[test] +fn coordination_structurally_sound_after_noise() { + let s = seeded_sentinel(); + let ch = s.health().coordination_health; + + // Coordination contexts are created lazily when enough + // competitive cells co-exist. If any were created, verify + // their structural properties. + if ch.active_contexts > 0 { + assert_eq!(ch.dim, 4, "coordination dimensionality is always 4"); + assert!(ch.capacity <= s.config().max_rank.min(4)); + assert!( + ch.maturity_distribution.max_noise_influence <= 1.0, + "noise_influence must be in [0, 1]" + ); + assert!(ch.rank_distribution.min >= 1); + assert!(ch.rank_distribution.max <= ch.capacity); + } +} diff --git a/packages/sentinel/tests/hierarchical_coordination.rs b/packages/sentinel/tests/hierarchical_coordination.rs new file mode 100644 index 000000000..bef178407 --- /dev/null +++ b/packages/sentinel/tests/hierarchical_coordination.rs @@ -0,0 +1,1064 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +//! Hierarchical coordination (§ALGO S-7) — the sentinel's second tier, which +//! watches how cells score *together* rather than how any one of them scores +//! alone. +//! +//! Each competitive cell contributes the four axis means it reported this +//! batch, and a context at an internal G-tree node models those vectors as a +//! group. A context exists only where the question is meaningful: both of the +//! node's subtrees must have cells reporting in the same batch, so a context +//! always spans a genuine split of the domain and never fires on one region +//! talking to itself. A node with a single contributing child asks no such +//! question and passes its members upward untouched. Contexts are therefore +//! the internal nodes of a binary tree whose leaves are the reporting cells, +//! which bounds their number below the number of those cells; and because the +//! walk unions each subtree's members on the way up, an ancestor's membership +//! is a superset of every descendant's. +//! +//! The tier is deliberately narrow. A member observation is one value per +//! scoring axis, so a context works in four dimensions however wide the cells +//! beneath it are, and its rank ceiling is the configured maximum capped at +//! those four. Members are centred against a running mean the context keeps +//! for itself, so what is measured is departure from the group's own recent +//! pattern rather than from any global reference. +//! +//! Everything a context publishes is a measurement: how many cells reported, +//! the interval they came from, the four axis statistics and their +//! accumulated drift, the geometry of the learned subspace. There are no +//! threat levels and no recommended actions here — the sentinel measures and +//! the host decides. +//! +//! # Test index +//! +//! | Test | Area | Claim | +//! |------|------|-------| +//! | [`no_coordination_on_empty_batch`] | coordination | A batch carrying no values gives the tier nothing to group: no cell reports a score, no node can find both its subtrees contributing, and the coordination list comes back empty. Coordination is a statement about what several cells did together in one batch, so with no batch there is nothing to say. | +//! | [`no_coordination_with_single_region`] | coordination | Traffic confined to one region leaves every reporting cell in the same subtree, so the both-subtrees condition is never met on its own account. Any context that does appear carries several member cells — which is what makes a coordination score a genuinely cross-cell measurement rather than a restatement of one cell's own score at a coarser scale. | +//! | [`two_sibling_cells_fire_parent_context`] | coordination | Two well-separated regions reporting in the same batch turn the node above them into a live context: it sees members arriving from each side and begins modelling the pair as a group. Each report identifies itself by the dyadic interval of that node, whose lower bound always lies strictly below its upper bound, so a host can say which stretch of the domain the measurement covers. | +//! | [`context_activation_requires_both_subtrees`] | coordination | After a run in which only one region reported, sending traffic to the opposite half of the domain can only add contexts, never take them away. Contexts are created where a split starts carrying reporters on both sides, so the arrival of a second region is precisely the event that brings the tier into existence. | +//! | [`context_deactivation_on_subtree_loss`] | coordination | When a batch arrives from one region only, the nodes that had been spanning both stop qualifying, and the tier prunes them instead of carrying stale contexts forward. The count of live contexts therefore never rises when a subtree falls silent: what is reported describes the group structure of the batch in hand, not of some earlier one. | +//! | [`nested_coordination_levels`] | coordination | cites (´claim:coordination:each-firing-node-appears-exactly-once-among-a-batchs-coordination-reports´) | +//! | [`coordination_group_nesting_invariant`] | coordination | Membership nests with the tree. Wherever one context's interval contains another's, the containing context counts at least as many reporting cells, because the walk unions each subtree's members and passes the result upward. An ancestor therefore measures a superset of what its descendant measured, which is what lets the two readings be compared as the same pattern seen at two scales. | +//! | [`root_context_sees_all_cells`] | coordination | cites (´claim:coordination:an-ancestor-context-counts-every-cell-its-descendant-counts´) | +//! | [`semi_internal_node_passthrough`] | coordination | A node with only one contributing child poses no coordination question — there is no pair of subtrees to compare — so it fires nothing and simply forwards its members to its parent. Every report that does appear consequently spans a real split and carries a usable finite score, rather than the tier manufacturing a group out of a single branch. | +//! | [`coordination_tracker_operates_at_w4`] | coordination | A member's contribution to a context is one value per scoring axis, so the tier works in four dimensions however wide the cells beneath it are. That fixed and tiny geometry is what makes a second tier affordable: its cost does not grow with the width of the domain the cells analyse. | +//! | [`coordination_tracker_with_low_max_rank`] | coordination | A context's capacity is the configured rank ceiling, capped by the four dimensions actually available. Lowering that ceiling lowers the capacity with it while the dimensionality stays at four: capacity is a budget on how much structure the model may hold, not a change to what a member observation is. | +//! | [`running_mean_cold_start`] | coordination | The first batch a context ever handles still yields finite numbers on all four axes. Members are centred against a running mean that starts at zero and is seeded outright from that batch's own column means rather than divided by an empty history, so activation costs the host no undefined readings to interpret. | +//! | [`running_mean_ewma_update`] | coordination | A context that has activated goes on firing while both its subtrees keep reporting: repeating the same shape of batch produces coordination in nearly all of them. Its running mean is updated across those batches instead of being rebuilt each time, so what the tier measures stays a departure from the group's own recent pattern. | +//! | [`only_competitive_cells_contribute`] | coordination | Membership is drawn only from competitive cells that actually observed something in this batch, so no context can report more members than there were such cells. The root is never competitive and so never joins a group: coordination measures the cells the engine chose to invest in, not the whole tree. | +//! | [`cells_with_no_observations_excluded`] | coordination | cites (´claim:coordination:only-competitive-cells-that-observed-this-batch-become-members´) | +//! | [`coordination_scores_are_finite`] | coordination | Everything a context publishes is a finite number: the four axis means, the drift accumulated on each, the fraction of variance the learned subspace captures and its largest singular value. Degenerate geometry at this tier is resolved inside the model rather than handed to the host as a not-a-number it would have to interpret for itself. | +//! | [`per_member_scores_present_when_enabled`] | coordination | With per-sample scoring enabled a context also names its members: one entry per reporting cell, matching the membership count it declared, each carrying that cell's own four scores and the non-empty interval it covers. A host can therefore attribute a group-level reading to particular regions of the domain instead of seeing only the aggregate. | +//! | [`coordination_survives_decay`] | coordination | Ageing the model down does not dismantle the tier. After a decay the sentinel still ingests and still produces reports on the next batch, because decay weakens the learned structure rather than removing the cells and contexts that carry it. | +//! | [`inject_noise_warms_coordination`] | coordination | A new context is warmed with synthetic score vectors drawn to match its members' own baselines, which leaves it holding a usable model rather than an empty one. The warm-up then clears the drift accumulators, so the first real batch is that context's first step of evidence: what the engine invented about itself is never counted as evidence about traffic. | +//! | [`deterministic_report_order`] | coordination | Two runs from the same seed over the same traffic produce the same reports: the same number of them, at the same nodes in the same order, with the same memberships and matching scores. The only randomness in the engine is the seeded warm-up, and pinning it makes two runs comparable — without that, no difference a host observed could be attributed. | +//! | [`context_count_bounded_by_k_minus_1`] | coordination | Contexts are the internal nodes of a binary tree whose leaves are the competitive cells, so their number stays below the number of those cells. The tier's cost is thereby bounded by the analysis budget the host has already chosen and cannot grow on its own. | +//! | [`coordination_reports_unique_gnodes`] | coordination | A batch produces at most one report per firing node. The walk visits each node once and a context is keyed by the node it sits at, so a nested hierarchy yields one measurement per level rather than repeated entries a host would first have to de-duplicate. | +//! | [`batch_report_coordination_is_vec`] | coordination | The batch report always carries a list of coordination reports, empty when nothing fired, rather than an optional one. Absence of coordination is an ordinary outcome — a lone value in a batch simply produces none — so a host iterates the list without first having to test for presence. | +//! | [`health_report_has_coordination_health`] | coordination | cites (´claim:coordination:the-tier-works-in-four-dimensions-because-a-member-contributes-one-value-per-scoring-axis´) | +//! | [`config_cusum_coord_slow_decay_validated`] | coordination | The slow baseline the coordination tier measures drift against is validated like any other rate: strictly inside zero and one, and strictly slower than the fast forgetting factor. The separation is the whole point — the slow baseline is the reference the fast one is judged against — so a configuration where the two move at the same speed is rejected outright rather than quietly producing a meaningless reading. | +//! | [`coordination_reports_are_ordered_by_depth_then_identifier`] | coordination | Coordination reports arrive shallowest first, ties broken by ascending identifier. The walk that produces them is bottom-up, which emits a strictly post-order sequence and puts the root — the shallowest context of all — last; that is deterministic but it is not the order either record states, and a reader taking the reports as a descent from the coarsest scale to the finest would have had the sequence exactly backwards. Depth is the ordering the output record describes and the identifier is the ordering this type's own documentation describes, so sorting on the pair satisfies both and is a total order besides, which sorting on depth alone would not be. | +//! | [`contour_count_includes_semi_internal_nodes`] | coordination | The contour count is the whole contour: the terminal cells together with the semi-internal ones. A semi-internal node has one half subdivided and one that still accumulates locally, so that second half receives observations exactly as a terminal cell does and is part of the surface the snapshot describes. Counting only the terminals reported a resolution short by every half-subdivided node, which is a figure that drifts from the truth precisely while the structure is being reshaped. | + +mod common; + +use std::collections::BTreeSet; + +use common::{ScenarioBuilder, assert_invariants, cell_values, seeded_sentinel, test_config}; +use torrust_sentinel::{GNodeId, Sentinel128, SentinelConfig}; + +// ═══════════════════════════════════════════════════════════ +// Activation & deactivation (§ALGO S-7.7) +// ═══════════════════════════════════════════════════════════ + +// ── no_coordination_on_empty_batch ────────────────────────── + +/// A batch carrying no values gives the tier nothing to group: no cell +/// reports a score, no node can find both its subtrees contributing, and the +/// coordination list comes back empty. Coordination is a statement about +/// what several cells did together in one batch, so with no batch there is +/// nothing to say. +/// +/// ´claim:coordination:a-batch-with-no-observations-produces-no-coordination-report´ +/// ´test:integration:no-coordination-on-empty-batch´ +#[test] +fn no_coordination_on_empty_batch() { + let mut s = Sentinel128::new(test_config()).unwrap(); + let report = s.ingest(&[]); + + assert!( + report.coordination_reports.is_empty(), + "empty batch should produce no coordination reports" + ); + assert_invariants(&s, &report); +} + +// ── no_coordination_with_single_region ────────────────────── + +/// Traffic confined to one region leaves every reporting cell in the same +/// subtree, so the both-subtrees condition is never met on its own account. +/// Any context that does appear carries several member cells — which is what +/// makes a coordination score a genuinely cross-cell measurement rather than +/// a restatement of one cell's own score at a coarser scale. +/// +/// ´claim:coordination:a-context-carries-at-least-two-member-cells-because-both-subtrees-must-contribute´ +/// ´test:integration:no-coordination-with-single-region´ +#[test] +fn no_coordination_with_single_region() { + let mut s = ScenarioBuilder::new() + .config(test_config()) + .seed_range(0xF, 8) + .warm_batches(5) + .build(); + + let report = s.ingest(&cell_values(0xF, 8)); + + // All competitive cells are in one subtree → no both-subtree + // condition → coordination should not fire. + for cr in &report.coordination_reports { + assert!( + cr.cells_reporting >= 2, + "coordination should only fire with ≥ 2 cells from both subtrees" + ); + } + assert_invariants(&s, &report); +} + +// ── two_sibling_cells_fire_parent_context ─────────────────── + +/// Two well-separated regions reporting in the same batch turn the node +/// above them into a live context: it sees members arriving from each side +/// and begins modelling the pair as a group. Each report identifies itself +/// by the dyadic interval of that node, whose lower bound always lies +/// strictly below its upper bound, so a host can say which stretch of the +/// domain the measurement covers. +/// +/// ´claim:coordination:two-populated-sibling-subtrees-turn-their-common-ancestor-into-a-live-context´ +/// ´test:integration:two-sibling-cells-fire-parent-context´ +#[test] +fn two_sibling_cells_fire_parent_context() { + let cfg = SentinelConfig:: { + split_threshold: 10, + ..test_config() + }; + let mut s = ScenarioBuilder::new() + .config(cfg) + .seed_range(0xF, 4) + .seed_range(0x1, 4) + .warm_batches(14) + .build(); + + let report = s.ingest(&[cell_values(0xF, 4), cell_values(0x1, 4)].concat()); + + assert!( + !report.coordination_reports.is_empty(), + "two distinct cell regions should produce coordination" + ); + for cr in &report.coordination_reports { + assert!(cr.cells_reporting >= 2, "coordination needs ≥ 2 cells"); + assert!(cr.start < cr.end, "start should be < end"); + } + assert_invariants(&s, &report); +} + +// ── context_activation_requires_both_subtrees ─────────────── + +/// After a run in which only one region reported, sending traffic to the +/// opposite half of the domain can only add contexts, never take them away. +/// Contexts are created where a split starts carrying reporters on both +/// sides, so the arrival of a second region is precisely the event that +/// brings the tier into existence. +/// +/// ´claim:coordination:a-second-reporting-region-can-only-add-contexts-never-remove-them´ +/// ´test:integration:context-activation-requires-both-subtrees´ +#[test] +fn context_activation_requires_both_subtrees() { + let mut s = ScenarioBuilder::new() + .config(test_config()) + .seed_range(0xF, 8) + .warm_batches(9) + .build(); + + let ch_one = s.health().coordination_health; + + // Now add a second region in the opposite half of the domain. + for _ in 0..5 { + s.ingest(&[cell_values(0xF, 4), cell_values(0x1, 4)].concat()); + } + let ch_both = s.health().coordination_health; + + assert!( + ch_both.active_contexts >= ch_one.active_contexts, + "adding a second region should activate coordination: \ + before={}, after={}", + ch_one.active_contexts, + ch_both.active_contexts + ); +} + +// ── context_deactivation_on_subtree_loss ──────────────────── + +/// When a batch arrives from one region only, the nodes that had been +/// spanning both stop qualifying, and the tier prunes them instead of +/// carrying stale contexts forward. The count of live contexts therefore +/// never rises when a subtree falls silent: what is reported describes the +/// group structure of the batch in hand, not of some earlier one. +/// +/// ´claim:coordination:a-context-whose-subtree-falls-silent-is-pruned-rather-than-carried-forward´ +/// ´test:integration:context-deactivation-on-subtree-loss´ +#[test] +fn context_deactivation_on_subtree_loss() { + let mut s = ScenarioBuilder::new() + .config(test_config()) + .seed_range(0xF, 4) + .seed_range(0x1, 4) + .warm_batches(9) + .build(); + + let ch_both = s.health().coordination_health; + + // Ingest only one region — the other region's cells may still + // be competitive but have zero observations this batch. + let _report = s.ingest(&cell_values(0xF, 4)); + let ch_one = s.health().coordination_health; + + assert!( + ch_one.active_contexts <= ch_both.active_contexts, + "losing a subtree should deactivate contexts: \ + both={}, one={}", + ch_both.active_contexts, + ch_one.active_contexts + ); +} + +// ═══════════════════════════════════════════════════════════ +// Hierarchy & nesting (§ALGO S-7.1 and §ALGO S-7.4) +// ═══════════════════════════════════════════════════════════ + +// ── nested_coordination_levels ────────────────────────────── + +/// Several regions spread across the domain make the hierarchy fire at more +/// than one level at once: a context over each neighbouring pair and wider +/// ones above them. The levels stay distinct — every report in a batch +/// stands at its own node, so a nested pair is two measurements at two +/// scales rather than the same measurement counted twice. +/// +/// (´claim:coordination:each-firing-node-appears-exactly-once-among-a-batchs-coordination-reports´) +/// ´test:integration:nested-coordination-levels´ +#[test] +fn nested_coordination_levels() { + let cfg = SentinelConfig:: { + analysis_k: 16, + split_threshold: 10, + ..test_config() + }; + let mut s = ScenarioBuilder::new() + .config(cfg) + .seed_range(0x1, 4) + .seed_range(0x3, 4) + .seed_range(0x9, 4) + .seed_range(0xF, 4) + .warm_batches(19) + .build(); + + let report = s.ingest( + &[ + cell_values(0x1, 4), + cell_values(0x3, 4), + cell_values(0x9, 4), + cell_values(0xF, 4), + ] + .concat(), + ); + + // With 4 cell regions, hierarchy should fire at multiple levels. + if report.coordination_reports.len() > 1 { + let gnodes: BTreeSet<_> = report.coordination_reports.iter().map(|cr| cr.gnode_id).collect(); + assert!( + gnodes.len() == report.coordination_reports.len(), + "each coordination report should be at a unique gnode" + ); + } + assert_invariants(&s, &report); +} + +// ── coordination_group_nesting_invariant ──────────────────── + +/// Membership nests with the tree. Wherever one context's interval contains +/// another's, the containing context counts at least as many reporting +/// cells, because the walk unions each subtree's members and passes the +/// result upward. An ancestor therefore measures a superset of what its +/// descendant measured, which is what lets the two readings be compared as +/// the same pattern seen at two scales. +/// +/// ´claim:coordination:an-ancestor-context-counts-every-cell-its-descendant-counts´ +/// ´test:integration:coordination-group-nesting-invariant´ +#[test] +fn coordination_group_nesting_invariant() { + let cfg = SentinelConfig:: { + analysis_k: 16, + split_threshold: 10, + ..test_config() + }; + let mut s = ScenarioBuilder::new() + .config(cfg) + .seed_range(0x1, 4) + .seed_range(0x5, 4) + .seed_range(0x9, 4) + .seed_range(0xF, 4) + .warm_batches(14) + .build(); + + let report = s.ingest( + &[ + cell_values(0x1, 4), + cell_values(0x5, 4), + cell_values(0x9, 4), + cell_values(0xF, 4), + ] + .concat(), + ); + + // For any two coordination reports where one's interval + // contains the other, the parent should have ≥ cells_reporting. + for a in &report.coordination_reports { + for b in &report.coordination_reports { + if a.gnode_id == b.gnode_id { + continue; + } + if a.start <= b.start && a.end >= b.end { + assert!( + a.cells_reporting >= b.cells_reporting, + "ancestor context [{}..{}) (depth {}, cells {}) \ + should have ≥ cells than descendant [{}..{}) (depth {}, cells {})", + a.start, + a.end, + a.depth, + a.cells_reporting, + b.start, + b.end, + b.depth, + b.cells_reporting + ); + } + } + } + assert_invariants(&s, &report); +} + +// ── root_context_sees_all_cells ───────────────────────────── + +/// Pins the top of that nesting: when several competitive cells report +/// together, the widest live context is the one that has accumulated them, +/// so at least one report shows a membership larger than a single cell. +/// Which node that is depends on where the traffic actually split, not on +/// any privileged root context. +/// +/// (´claim:coordination:an-ancestor-context-counts-every-cell-its-descendant-counts´) +/// ´test:integration:root-context-sees-all-cells´ +#[test] +fn root_context_sees_all_cells() { + let mut s = ScenarioBuilder::new() + .config(test_config()) + .seed_range(0xF, 4) + .seed_range(0x1, 4) + .warm_batches(9) + .build(); + + let report = s.ingest(&[cell_values(0xF, 4), cell_values(0x1, 4)].concat()); + + let competitive_with_obs = report + .cell_reports + .iter() + .filter(|c| c.is_competitive && c.sample_count > 0) + .count(); + + if !report.coordination_reports.is_empty() && competitive_with_obs >= 2 { + let max_reporting = report + .coordination_reports + .iter() + .map(|cr| cr.cells_reporting) + .max() + .unwrap_or(0); + assert!( + max_reporting >= 2, + "at least one coordination context should see multiple cells" + ); + } + assert_invariants(&s, &report); +} + +// ── semi_internal_node_passthrough ────────────────────────── + +/// A node with only one contributing child poses no coordination question — +/// there is no pair of subtrees to compare — so it fires nothing and simply +/// forwards its members to its parent. Every report that does appear +/// consequently spans a real split and carries a usable finite score, rather +/// than the tier manufacturing a group out of a single branch. +/// +/// ´claim:coordination:a-node-with-one-contributing-child-forwards-its-cells-without-firing´ +/// ´test:integration:semi-internal-node-passthrough´ +#[test] +fn semi_internal_node_passthrough() { + let mut s = ScenarioBuilder::new() + .config(test_config()) + .seed_range(0xF, 4) + .seed_range(0x1, 4) + .warm_batches(9) + .build(); + + let report = s.ingest(&[cell_values(0xF, 4), cell_values(0x1, 4)].concat()); + + // Semi-internal nodes (one child) should not fire coordination; + // only nodes with both subtrees contributing should appear. + for cr in &report.coordination_reports { + assert!(cr.cells_reporting >= 2); + assert!(cr.scores.novelty.mean.is_finite()); + } + assert_invariants(&s, &report); +} + +// ═══════════════════════════════════════════════════════════ +// Scoring & tracking (§§ALGO S-7.2–7.5) +// ═══════════════════════════════════════════════════════════ + +// ── coordination_tracker_operates_at_w4 ───────────────────── + +/// A member's contribution to a context is one value per scoring axis, so +/// the tier works in four dimensions however wide the cells beneath it are. +/// That fixed and tiny geometry is what makes a second tier affordable: its +/// cost does not grow with the width of the domain the cells analyse. +/// +/// ´claim:coordination:the-tier-works-in-four-dimensions-because-a-member-contributes-one-value-per-scoring-axis´ +/// ´test:integration:coordination-tracker-operates-at-w4´ +#[test] +fn coordination_tracker_operates_at_w4() { + let s = ScenarioBuilder::new() + .config(test_config()) + .seed_range(0xF, 4) + .seed_range(0x1, 4) + .warm_batches(9) + .build(); + + let ch = s.health().coordination_health; + if ch.active_contexts > 0 { + assert_eq!(ch.dim, 4, "coordination trackers should operate at w=4"); + // cap = min(4, max_rank). test_config has max_rank=4. + assert_eq!(ch.capacity, 4, "cap should be min(4, max_rank)"); + } +} + +// ── coordination_tracker_with_low_max_rank ────────────────── + +/// A context's capacity is the configured rank ceiling, capped by the four +/// dimensions actually available. Lowering that ceiling lowers the capacity +/// with it while the dimensionality stays at four: capacity is a budget on +/// how much structure the model may hold, not a change to what a member +/// observation is. +/// +/// ´claim:coordination:a-contexts-capacity-is-the-configured-rank-ceiling-capped-at-the-four-available-dimensions´ +/// ´test:integration:coordination-tracker-with-low-max-rank´ +#[test] +fn coordination_tracker_with_low_max_rank() { + let cfg = SentinelConfig:: { + max_rank: 2, + ..test_config() + }; + let s = ScenarioBuilder::new() + .config(cfg) + .seed_range(0xF, 4) + .seed_range(0x1, 4) + .warm_batches(9) + .build(); + + let ch = s.health().coordination_health; + if ch.active_contexts > 0 { + assert_eq!(ch.dim, 4, "coordination dim is always 4"); + assert_eq!(ch.capacity, 2, "cap should be min(4, max_rank=2) = 2"); + } +} + +// ── running_mean_cold_start ───────────────────────────────── + +/// The first batch a context ever handles still yields finite numbers on all +/// four axes. Members are centred against a running mean that starts at zero +/// and is seeded outright from that batch's own column means rather than +/// divided by an empty history, so activation costs the host no undefined +/// readings to interpret. +/// +/// ´claim:coordination:a-newly-activated-context-scores-its-first-batch-in-finite-numbers´ +/// ´test:integration:running-mean-cold-start´ +#[test] +fn running_mean_cold_start() { + let mut s = ScenarioBuilder::new() + .config(test_config()) + .seed_range(0xF, 4) + .seed_range(0x1, 4) + .warm_batches(9) + .build(); + + let ch = s.health().coordination_health; + if ch.active_contexts > 0 { + let report = s.ingest(&[cell_values(0xF, 4), cell_values(0x1, 4)].concat()); + for cr in &report.coordination_reports { + assert!(cr.scores.novelty.mean.is_finite()); + assert!(cr.scores.displacement.mean.is_finite()); + assert!(cr.scores.surprise.mean.is_finite()); + assert!(cr.scores.coherence.mean.is_finite()); + } + assert_invariants(&s, &report); + } +} + +// ── running_mean_ewma_update ──────────────────────────────── + +/// A context that has activated goes on firing while both its subtrees keep +/// reporting: repeating the same shape of batch produces coordination in +/// nearly all of them. Its running mean is updated across those batches +/// instead of being rebuilt each time, so what the tier measures stays a +/// departure from the group's own recent pattern. +/// +/// ´claim:coordination:an-active-context-keeps-firing-while-both-its-subtrees-keep-reporting´ +/// ´test:integration:running-mean-ewma-update´ +#[test] +fn running_mean_ewma_update() { + let mut s = ScenarioBuilder::new() + .config(test_config()) + .seed_range(0xF, 4) + .seed_range(0x1, 4) + .warm_batches(14) + .build(); + + let batch = [cell_values(0xF, 4), cell_values(0x1, 4)].concat(); + let mut reports = Vec::new(); + for _ in 0..5 { + reports.push(s.ingest(&batch)); + } + + let non_empty: usize = reports.iter().filter(|r| !r.coordination_reports.is_empty()).count(); + assert!(non_empty >= 3, "most batches should produce coordination: got {non_empty}/5"); +} + +// ── only_competitive_cells_contribute ─────────────────────── + +/// Membership is drawn only from competitive cells that actually observed +/// something in this batch, so no context can report more members than there +/// were such cells. The root is never competitive and so never joins a +/// group: coordination measures the cells the engine chose to invest in, not +/// the whole tree. +/// +/// ´claim:coordination:only-competitive-cells-that-observed-this-batch-become-members´ +/// ´test:integration:only-competitive-cells-contribute´ +#[test] +fn only_competitive_cells_contribute() { + let mut s = ScenarioBuilder::new() + .config(test_config()) + .seed_range(0xF, 4) + .seed_range(0x1, 4) + .warm_batches(4) + .build(); + + let report = s.ingest(&[cell_values(0xF, 4), cell_values(0x1, 4)].concat()); + + // Root is never competitive (§ALGO S-8.1). coordination + // cells_reporting should only count competitive cells. + let competitive_count = report + .cell_reports + .iter() + .filter(|c| c.is_competitive && c.sample_count > 0) + .count(); + + for cr in &report.coordination_reports { + assert!( + cr.cells_reporting <= competitive_count, + "cells_reporting ({}) should not exceed competitive cells with observations ({})", + cr.cells_reporting, + competitive_count + ); + } + assert_invariants(&s, &report); +} + +// ── cells_with_no_observations_excluded ───────────────────── + +/// Pins the other end of that rule: a cell can remain competitive and still +/// contribute nothing, because it saw no values this time. Sending traffic +/// to one region only leaves the opposite region's cells silent, and a +/// subtree of silent cells does not count as contributing, so nothing fires +/// on their behalf. +/// +/// (´claim:coordination:only-competitive-cells-that-observed-this-batch-become-members´) +/// ´test:integration:cells-with-no-observations-excluded´ +#[test] +fn cells_with_no_observations_excluded() { + let mut s = ScenarioBuilder::new() + .config(test_config()) + .seed_range(0xF, 4) + .seed_range(0x1, 4) + .warm_batches(9) + .build(); + + // Ingest only one region — the other region's competitive + // cells get zero observations this batch. + let report = s.ingest(&cell_values(0xF, 4)); + + // §ALGO S-7.2: cells with no observations are excluded. If only one + // subtree has observations, coordination should not fire. + for cr in &report.coordination_reports { + assert!(cr.cells_reporting >= 2, "coordination should need cells from both subtrees"); + } + assert_invariants(&s, &report); +} + +// ── coordination_scores_are_finite ────────────────────────── + +/// Everything a context publishes is a finite number: the four axis means, +/// the drift accumulated on each, the fraction of variance the learned +/// subspace captures and its largest singular value. Degenerate geometry at +/// this tier is resolved inside the model rather than handed to the host as +/// a not-a-number it would have to interpret for itself. +/// +/// ´claim:coordination:every-published-coordination-measurement-is-a-finite-number´ +/// ´test:integration:coordination-scores-are-finite´ +#[test] +fn coordination_scores_are_finite() { + let mut s = ScenarioBuilder::new() + .config(test_config()) + .seed_range(0xF, 4) + .seed_range(0x1, 4) + .warm_batches(14) + .build(); + + let report = s.ingest(&[cell_values(0xF, 4), cell_values(0x1, 4)].concat()); + + for cr in &report.coordination_reports { + assert!(cr.scores.novelty.mean.is_finite(), "NaN/Inf novelty mean"); + assert!(cr.scores.displacement.mean.is_finite(), "NaN/Inf displacement mean"); + assert!(cr.scores.surprise.mean.is_finite(), "NaN/Inf surprise mean"); + assert!(cr.scores.coherence.mean.is_finite(), "NaN/Inf coherence mean"); + + assert!(!cr.scores.novelty.mean.is_nan(), "NaN novelty"); + assert!(!cr.scores.displacement.mean.is_nan(), "NaN displacement"); + assert!(!cr.scores.surprise.mean.is_nan(), "NaN surprise"); + assert!(!cr.scores.coherence.mean.is_nan(), "NaN coherence"); + + // CUSUM accumulators must be finite. + assert!(cr.scores.novelty.cusum.accumulator.is_finite()); + assert!(cr.scores.displacement.cusum.accumulator.is_finite()); + assert!(cr.scores.surprise.cusum.accumulator.is_finite()); + assert!(cr.scores.coherence.cusum.accumulator.is_finite()); + + // Energy ratio and singular values must be sane. + assert!(cr.energy_ratio.is_finite()); + assert!(cr.top_singular_value.is_finite()); + } + assert_invariants(&s, &report); +} + +// ── per_member_scores_present_when_enabled ────────────────── + +/// With per-sample scoring enabled a context also names its members: one +/// entry per reporting cell, matching the membership count it declared, each +/// carrying that cell's own four scores and the non-empty interval it +/// covers. A host can therefore attribute a group-level reading to +/// particular regions of the domain instead of seeing only the aggregate. +/// +/// ´claim:coordination:per-member-scores-attribute-a-group-reading-to-the-cells-that-produced-it´ +/// ´test:integration:per-member-scores-present-when-enabled´ +#[test] +fn per_member_scores_present_when_enabled() { + // test_config() has per_sample_scores = true. + let mut s = ScenarioBuilder::new() + .config(test_config()) + .seed_range(0xF, 4) + .seed_range(0x1, 4) + .warm_batches(14) + .build(); + + let report = s.ingest(&[cell_values(0xF, 4), cell_values(0x1, 4)].concat()); + + for cr in &report.coordination_reports { + let members = cr + .per_member + .as_ref() + .expect("per_member should be Some when per_sample_scores is enabled"); + assert_eq!( + members.len(), + cr.cells_reporting, + "per_member count should equal cells_reporting" + ); + for ms in members { + assert!(ms.novelty.is_finite()); + assert!(ms.displacement.is_finite()); + assert!(ms.surprise.is_finite()); + assert!(ms.coherence.is_finite()); + assert!(ms.cell_start < ms.cell_end, "member cell interval must be non-empty"); + } + } + assert_invariants(&s, &report); +} + +// ═══════════════════════════════════════════════════════════ +// Stability & resilience +// ═══════════════════════════════════════════════════════════ + +// ── coordination_survives_decay ───────────────────────────── + +/// Ageing the model down does not dismantle the tier. After a decay the +/// sentinel still ingests and still produces reports on the next batch, +/// because decay weakens the learned structure rather than removing the +/// cells and contexts that carry it. +/// +/// ´claim:coordination:decaying-the-model-leaves-the-tier-able-to-report-on-the-next-batch´ +/// ´test:integration:coordination-survives-decay´ +#[test] +fn coordination_survives_decay() { + let mut s = ScenarioBuilder::new() + .config(test_config()) + .seed_range(0xF, 4) + .seed_range(0x1, 4) + .warm_batches(9) + .build(); + + s.decay(0.5, 0.0); + + // After decay, new observations should still produce functional reports. + let report = s.ingest(&[cell_values(0xF, 4), cell_values(0x1, 4)].concat()); + assert!( + !report.cell_reports.is_empty() || !report.ancestor_reports.is_empty(), + "coordination tier should remain functional after decay" + ); + assert_invariants(&s, &report); +} + +// ── inject_noise_warms_coordination ───────────────────────── + +/// A new context is warmed with synthetic score vectors drawn to match its +/// members' own baselines, which leaves it holding a usable model rather +/// than an empty one. The warm-up then clears the drift accumulators, so the +/// first real batch is that context's first step of evidence: what the +/// engine invented about itself is never counted as evidence about traffic. +/// +/// ´claim:coordination:warming-leaves-a-context-experienced-but-with-its-drift-evidence-cleared´ +/// ´test:integration:inject-noise-warms-coordination´ +#[test] +fn inject_noise_warms_coordination() { + let mut s = seeded_sentinel(); + + let ch = s.health().coordination_health; + + if ch.active_contexts > 0 { + assert!( + ch.maturity_distribution.max_noise_influence < 1.0, + "coordination should have warmed after noise" + ); + } + + // Verify CUSUM reset: ingest one batch and check steps_since_reset. + let report = s.ingest(&[ + 0xF000_0000_0000_0000_0000_0000_0000_AAAA, + 0x1000_0000_0000_0000_0000_0000_0000_BBBB, + ]); + for cr in &report.coordination_reports { + assert_eq!( + cr.scores.novelty.cusum.steps_since_reset, 1, + "coordination CUSUM should have been reset after noise" + ); + } + assert_invariants(&s, &report); +} + +// ── deterministic_report_order ────────────────────────────── + +/// Two runs from the same seed over the same traffic produce the same +/// reports: the same number of them, at the same nodes in the same order, +/// with the same memberships and matching scores. The only randomness in the +/// engine is the seeded warm-up, and pinning it makes two runs comparable — +/// without that, no difference a host observed could be attributed. +/// +/// ´claim:coordination:the-same-seed-and-traffic-give-the-same-reports-in-the-same-order´ +/// ´test:integration:deterministic-report-order´ +#[test] +fn deterministic_report_order() { + let batch: Vec = [cell_values(0xF, 4), cell_values(0x1, 4)].concat(); + + let run = |seed: u64| { + let cfg = SentinelConfig:: { + analysis_k: 16, + noise_seed: Some(seed), + ..test_config() + }; + let mut s = Sentinel128::new(cfg).unwrap(); + for _ in 0..10 { + s.ingest(&batch); + } + s.ingest(&batch) + }; + + let r1 = run(42); + let r2 = run(42); + + assert_eq!(r1.coordination_reports.len(), r2.coordination_reports.len()); + for (a, b) in r1.coordination_reports.iter().zip(r2.coordination_reports.iter()) { + assert_eq!(a.gnode_id, b.gnode_id, "reports should be in same order"); + assert_eq!(a.depth, b.depth); + assert_eq!(a.cells_reporting, b.cells_reporting); + assert!( + (a.scores.novelty.mean - b.scores.novelty.mean).abs() < 1e-10, + "deterministic runs should produce identical scores" + ); + } +} + +// ═══════════════════════════════════════════════════════════ +// Invariants (§ALGO S-7.1) +// ═══════════════════════════════════════════════════════════ + +// ── context_count_bounded_by_k_minus_1 ────────────────────── + +/// Contexts are the internal nodes of a binary tree whose leaves are the +/// competitive cells, so their number stays below the number of those cells. +/// The tier's cost is thereby bounded by the analysis budget the host has +/// already chosen and cannot grow on its own. +/// +/// ´claim:coordination:contexts-are-internal-nodes-of-a-binary-tree-so-they-stay-fewer-than-the-competitive-cells´ +/// ´test:integration:context-count-bounded-by-k-minus-1´ +#[test] +fn context_count_bounded_by_k_minus_1() { + let cfg = SentinelConfig:: { + analysis_k: 16, + split_threshold: 10, + ..test_config() + }; + let s = ScenarioBuilder::new() + .config(cfg) + .seed_range(0x1, 4) + .seed_range(0x5, 4) + .seed_range(0x9, 4) + .seed_range(0xF, 4) + .warm_batches(14) + .build(); + + let competitive_cells = s.analysis_set().competitive().len(); + let active = s.health().coordination_health.active_contexts; + + // Binary tree internal node bound: |contexts| ≤ K-1 + // where K = number of competitive cells (§ALGO S-7.1). + if competitive_cells > 0 { + assert!( + active <= competitive_cells.saturating_sub(1).max(1), + "active contexts ({active}) should be ≤ K-1 ({}) (§ALGO S-7.1)", + competitive_cells.saturating_sub(1) + ); + } +} + +// ── coordination_reports_unique_gnodes ────────────────────── + +/// A batch produces at most one report per firing node. The walk visits each +/// node once and a context is keyed by the node it sits at, so a nested +/// hierarchy yields one measurement per level rather than repeated entries a +/// host would first have to de-duplicate. +/// +/// ´claim:coordination:each-firing-node-appears-exactly-once-among-a-batchs-coordination-reports´ +/// ´test:integration:coordination-reports-unique-gnodes´ +#[test] +fn coordination_reports_unique_gnodes() { + let mut s = ScenarioBuilder::new() + .config(test_config()) + .seed_range(0xF, 4) + .seed_range(0x1, 4) + .warm_batches(14) + .build(); + + let report = s.ingest(&[cell_values(0xF, 4), cell_values(0x1, 4)].concat()); + + let gnodes: BTreeSet<_> = report.coordination_reports.iter().map(|cr| cr.gnode_id).collect(); + assert_eq!( + gnodes.len(), + report.coordination_reports.len(), + "no duplicate GNodeIds in coordination_reports" + ); + assert_invariants(&s, &report); +} + +// ═══════════════════════════════════════════════════════════ +// API surface +// ═══════════════════════════════════════════════════════════ + +// ── batch_report_coordination_is_vec ──────────────────────── + +/// The batch report always carries a list of coordination reports, empty +/// when nothing fired, rather than an optional one. Absence of coordination +/// is an ordinary outcome — a lone value in a batch simply produces none — +/// so a host iterates the list without first having to test for presence. +/// +/// ´claim:coordination:coordination-arrives-as-a-possibly-empty-list-rather-than-an-optional-value´ +/// ´test:integration:batch-report-coordination-is-vec´ +#[test] +fn batch_report_coordination_is_vec() { + let mut s = Sentinel128::new(test_config()).unwrap(); + let report = s.ingest(&[42]); + + // The coordination field is Vec, not Option. + let _: &Vec> = &report.coordination_reports; + assert!(report.coordination_reports.is_empty()); +} + +// ── health_report_has_coordination_health ──────────────────── + +/// The health snapshot always includes the coordination summary, and that +/// summary declares the tier's fixed dimensionality even on a sentinel that +/// has ingested nothing and holds no contexts at all. The field describes +/// the tier itself, not merely whichever contexts happen to exist at the +/// moment of the reading. +/// +/// (´claim:coordination:the-tier-works-in-four-dimensions-because-a-member-contributes-one-value-per-scoring-axis´) +/// ´test:integration:health-report-has-coordination-health´ +#[test] +fn health_report_has_coordination_health() { + let s = Sentinel128::new(test_config()).unwrap(); + let h = s.health(); + + let ch = &h.coordination_health; + assert_eq!(ch.active_contexts, 0); + assert_eq!(ch.dim, 4); +} + +// ── config_cusum_coord_slow_decay_validated ────────────────── + +/// The slow baseline the coordination tier measures drift against is +/// validated like any other rate: strictly inside zero and one, and strictly +/// slower than the fast forgetting factor. The separation is the whole +/// point — the slow baseline is the reference the fast one is judged +/// against — so a configuration where the two move at the same speed is +/// rejected outright rather than quietly producing a meaningless reading. +/// +/// ´claim:coordination:the-tiers-slow-baseline-must-be-strictly-slower-than-the-fast-one-and-inside-zero-and-one´ +/// ´test:integration:config-cusum-coord-slow-decay-validated´ +#[test] +fn config_cusum_coord_slow_decay_validated() { + // Valid value. + let cfg = SentinelConfig:: { + cusum_coord_slow_decay: 0.999, + ..SentinelConfig::::default() + }; + cfg.validate().unwrap(); + + // Out of range: exactly 1.0. + let cfg = SentinelConfig:: { + cusum_coord_slow_decay: 1.0, + ..SentinelConfig::::default() + }; + assert!(cfg.validate().is_err()); + + // Out of range: exactly 0.0. + let cfg = SentinelConfig:: { + cusum_coord_slow_decay: 0.0, + ..SentinelConfig::::default() + }; + assert!(cfg.validate().is_err()); + + // Too low: must be > forgetting_factor. + let cfg = SentinelConfig:: { + forgetting_factor: 0.99, + cusum_coord_slow_decay: 0.99, + ..SentinelConfig::::default() + }; + assert!(cfg.validate().is_err()); +} + +/// Coordination reports arrive shallowest first, ties broken by ascending +/// identifier. The walk that produces them is bottom-up, which emits a +/// strictly post-order sequence and puts the root — the shallowest context of +/// all — last; that is deterministic but it is not the order either record +/// states, and a reader taking the reports as a descent from the coarsest +/// scale to the finest would have had the sequence exactly backwards. Depth +/// is the ordering the output record describes and the identifier is the +/// ordering this type's own documentation describes, so sorting on the pair +/// satisfies both and is a total order besides, which sorting on depth alone +/// would not be. +/// +/// ´claim:coordination:reports-arrive-shallowest-first-with-ties-broken-by-identifier´ +/// ´test:integration:coordination-reports-are-ordered-by-depth-then-identifier´ +#[test] +fn coordination_reports_are_ordered_by_depth_then_identifier() { + let cfg = SentinelConfig:: { + analysis_k: 16, + split_threshold: 10, + ..test_config() + }; + let mut s = ScenarioBuilder::new() + .config(cfg) + .seed_range(0x1, 4) + .seed_range(0x3, 4) + .seed_range(0x9, 4) + .seed_range(0xF, 4) + .warm_batches(19) + .build(); + + let report = s.ingest( + &[ + cell_values(0x1, 4), + cell_values(0x3, 4), + cell_values(0x9, 4), + cell_values(0xF, 4), + ] + .concat(), + ); + + assert!( + report.coordination_reports.len() > 1, + "nested contexts are needed for an ordering to be observable" + ); + let keys: Vec<(u32, GNodeId)> = report.coordination_reports.iter().map(|cr| (cr.depth, cr.gnode_id)).collect(); + let mut sorted = keys.clone(); + sorted.sort_unstable(); + assert_eq!(keys, sorted, "reports run shallowest first, ties by ascending identifier"); + + assert_invariants(&s, &report); +} + +/// The contour count is the whole contour: the terminal cells together with +/// the semi-internal ones. A semi-internal node has one half subdivided and +/// one that still accumulates locally, so that second half receives +/// observations exactly as a terminal cell does and is part of the surface the +/// snapshot describes. Counting only the terminals reported a resolution +/// short by every half-subdivided node, which is a figure that drifts from the +/// truth precisely while the structure is being reshaped. +/// +/// ´claim:coordination:the-contour-count-is-the-terminals-together-with-the-semi-internal-nodes´ +/// ´test:integration:contour-count-includes-semi-internal-nodes´ +#[test] +fn contour_count_includes_semi_internal_nodes() { + let cfg = SentinelConfig:: { + split_threshold: 5, + budget: 200, + ..test_config() + }; + let mut s = Sentinel128::new(cfg).unwrap(); + + let mut saw_semi_internal = false; + for nibble in 0..16u128 { + let batch: Vec = (0u128..500).map(|i| (nibble << 124) | (i << 100)).collect(); + let report = s.ingest(&batch); + + let terminals = s.graph().terminal_count() as usize; + let semi_internal = report.health.semi_internal_count; + assert_eq!( + report.contour.cell_count, + terminals + semi_internal, + "the contour is the terminals together with the semi-internal nodes" + ); + if semi_internal > 0 { + saw_semi_internal = true; + } + } + + assert!( + saw_semi_internal, + "this run must reach a half-subdivided node for the sum to be distinguishable" + ); +} diff --git a/packages/sentinel/tests/integration.rs b/packages/sentinel/tests/integration.rs new file mode 100644 index 000000000..28399c104 --- /dev/null +++ b/packages/sentinel/tests/integration.rs @@ -0,0 +1,343 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +//! End-to-end integration tests for the sentinel. +//! +//! These tests exercise multi-subsystem interactions that span the full +//! sentinel lifecycle. Each test constructs a sentinel, feeds realistic +//! traffic, and verifies cross-cutting concerns — things that only become +//! visible when construction, ingestion, scoring, coordination, decay, +//! and health reporting all run together. +//! +//! Focused unit-level concerns live in dedicated files: +//! +//! | Concern | File | +//! |-----------------------|---------------------------------| +//! | Public API contracts | `api.rs` | +//! | Report structure | `report_structure.rs` | +//! | Reset behaviour | `api.rs`, `edge_cases.rs`, `noise.rs` | +//! | CUSUM accumulation | `coverage_matrix.rs` | +//! | Warm-vs-cold scoring | `noise.rs`, `health.rs` | +//! | Spatial decay | `spatial_decay.rs` | +//! | Ancestor chains | `ancestor_chain.rs` | +//! | Invariants | `invariants.rs` | +//! | Determinism | `determinism.rs` | +//! +//! What a lifecycle test is for is agreement between the layers. The spatial +//! substrate splits the domain as traffic concentrates; the selector invests +//! in the cells that earned it and closes them under ancestry; each cell's +//! tracker scores the suffix left to it; and a second tier scores the pattern +//! across cells. A disagreement between those layers — a report naming a cell +//! the engine no longer tracks, a width that does not match a depth, an +//! ancestor listed as though it had competed — is invisible to any one +//! subsystem's own tests and shows up only when they run together over a +//! whole run. +//! +//! Because the sentinel measures and the host decides, the end-to-end claims +//! here are comparative rather than categorical. An unfamiliar batch scores +//! higher than the familiar one immediately before it; unfamiliarity +//! sustained over many batches accumulates instead of being forgotten; +//! decay lowers spatial standing without disturbing the observation record or +//! the trackers' learned state. No threshold is asserted anywhere, because +//! the engine does not own one. +//! +//! # Test index +//! +//! | Test | Area | Claim | +//! |------|------|-------| +//! | [`single_range_lifecycle`] | engine | A sentinel taken through a full life — constructed, seeded, warmed, driven at steady state, then handed a structurally unfamiliar batch — holds every structural invariant at each of those stages, not merely at the end. The health snapshot afterwards still describes a working engine: trackers are live, and the smallest learned rank is at least one, so no cell has collapsed to a model with no directions in it. | +//! | [`multi_range_lifecycle`] | engine | Traffic concentrated in two well-separated regions of the domain drives the spatial layer to split, and the sentinel ends up tracking more than the root — separate models for separate structure, which is the whole point of a hierarchy. The invariants continue to hold through steady state and through an unfamiliar batch confined to one of the two regions, so the cells coexist rather than interfering. | +//! | [`anomalous_batch_elevates_novelty`] | engine | Once a sentinel has settled on the structure it keeps being shown, a batch built on a different bit pattern scores higher than the ordinary batch immediately before it. The comparison is against that neighbour rather than against a fixed number, because the engine reports how far an observation departs from what this cell learned and leaves the question of how far is too far to the host. | +//! | [`anomalous_batch_elevates_cusum`] | engine | Unfamiliarity that persists across many batches accumulates: the drift accumulator stands higher after a long run of unfamiliar traffic than it did at the baseline batch. A single surprising batch and a regime that has genuinely shifted look alike instant by instant, and the accumulator is what tells them apart — a small departure repeated is allowed to add up rather than being forgotten each round. | +//! | [`decay_then_ingest_maintains_invariants`] | engine | Decay applied in the middle of a run attenuates accumulated spatial standing and nothing else: ingestion continues afterwards with every invariant intact and the observation counter still climbing. Temporal policy belongs to the host, and the engine implements it by lowering what cells have earned rather than by discarding what they have learned, so a decayed sentinel is a going concern and not a half-reset one. | +//! | [`inspect_cells_after_multi_range_traffic`] | width | cites (´claim:width:a-cells-analysis-width-is-the-domain-width-less-its-depth´) | +//! | [`coordination_activates_with_multi_range_traffic`] | engine | Once two separate regions are being modelled in earnest, a second tier of reporting appears without the host asking for it: a context that covers several cells at once and carries scores of its own, finite like any other. A pattern spread across sibling cells is invisible to each of them individually, so the engine models the pattern itself as soon as there are enough cells for one to exist. | + +mod common; + +use common::{ScenarioBuilder, anomalous_values, assert_invariants, cell_values, integration_config, max_cusum, max_novelty_z}; +use torrust_sentinel::SentinelConfig; + +// ── Full lifecycle ────────────────────────────────────────── + +/// A sentinel taken through a full life — constructed, seeded, warmed, +/// driven at steady state, then handed a structurally unfamiliar batch — +/// holds every structural invariant at each of those stages, not merely at +/// the end. The health snapshot afterwards still describes a working engine: +/// trackers are live, and the smallest learned rank is at least one, so no +/// cell has collapsed to a model with no directions in it. +/// +/// ´claim:engine:a-sentinel-holds-its-invariants-through-a-whole-lifecycle-from-first-seed-to-anomaly´ +/// ´test:integration:single-range-lifecycle´ +#[test] +fn single_range_lifecycle() { + let (mut s, warm_reports) = ScenarioBuilder::new() + .seed_range(0xF, 8) + .warm_batches(15) + .build_with_reports(); + + // Invariants hold throughout warm-up. + for r in &warm_reports { + assert_invariants(&s, r); + } + assert!(s.cells_tracked() >= 1); + + // Steady-state ingestion. + for _ in 0..10 { + let report = s.ingest(&cell_values(0xF, 8)); + assert_invariants(&s, &report); + } + + // Anomalous batch. + let anomaly_report = s.ingest(&anomalous_values(0xF, 8)); + assert_invariants(&s, &anomaly_report); + assert!(!anomaly_report.ancestor_reports.is_empty()); + + // Health check. + let h = s.health(); + assert!(h.active_trackers >= 1); + assert!(h.rank_distribution.min >= 1); +} + +/// Traffic concentrated in two well-separated regions of the domain drives +/// the spatial layer to split, and the sentinel ends up tracking more than +/// the root — separate models for separate structure, which is the whole +/// point of a hierarchy. The invariants continue to hold through steady +/// state and through an unfamiliar batch confined to one of the two regions, +/// so the cells coexist rather than interfering. +/// +/// ´claim:engine:traffic-in-separate-regions-splits-the-domain-so-more-than-the-root-is-tracked´ +/// ´test:integration:multi-range-lifecycle´ +#[test] +fn multi_range_lifecycle() { + let mut s = ScenarioBuilder::new() + .config(SentinelConfig:: { + split_threshold: 10, + ..integration_config() + }) + .seed_range(0xF, 12) + .seed_range(0x1, 12) + .warm_batches(15) + .build(); + + // Two disjoint ranges should produce more than just the root cell. + assert!(s.cells_tracked() > 1, "expected multiple cells from disjoint ranges"); + + // Steady-state. + for _ in 0..10 { + let batch: Vec = [cell_values(0xF, 8), cell_values(0x1, 8)].concat(); + let report = s.ingest(&batch); + assert_invariants(&s, &report); + } + + // Anomaly in one range only. + let mixed: Vec = [anomalous_values(0xF, 8), cell_values(0x1, 8)].concat(); + let report = s.ingest(&mixed); + assert_invariants(&s, &report); + + let h = s.health(); + assert!(h.lifetime_observations > 0); + assert!(h.active_trackers >= 2); +} + +// ── Anomaly detection (end-to-end) ────────────────────────── + +/// Once a sentinel has settled on the structure it keeps being shown, a +/// batch built on a different bit pattern scores higher than the ordinary +/// batch immediately before it. The comparison is against that neighbour +/// rather than against a fixed number, because the engine reports how far an +/// observation departs from what this cell learned and leaves the question +/// of how far is too far to the host. +/// +/// ´claim:engine:a-structurally-unfamiliar-batch-scores-higher-than-the-familiar-one-that-preceded-it´ +/// ´test:integration:anomalous-batch-elevates-novelty´ +#[test] +fn anomalous_batch_elevates_novelty() { + let mut s = ScenarioBuilder::new().seed_range(0xA, 16).warm_batches(20).build(); + + // Final normal batch as baseline. + let normal = s.ingest(&cell_values(0xA, 8)); + assert_invariants(&s, &normal); + + // Anomalous batch. + let anomaly = s.ingest(&anomalous_values(0xA, 8)); + assert_invariants(&s, &anomaly); + + let normal_z = max_novelty_z(&normal); + let anomaly_z = max_novelty_z(&anomaly); + assert!( + anomaly_z > normal_z, + "anomalous batch should produce higher novelty z-score: \ + anomaly={anomaly_z:.4}, normal={normal_z:.4}" + ); +} + +/// Unfamiliarity that persists across many batches accumulates: the drift +/// accumulator stands higher after a long run of unfamiliar traffic than it +/// did at the baseline batch. A single surprising batch and a regime that +/// has genuinely shifted look alike instant by instant, and the accumulator +/// is what tells them apart — a small departure repeated is allowed to add +/// up rather than being forgotten each round. +/// +/// ´claim:engine:sustained-unfamiliarity-accumulates-instead-of-being-forgotten-batch-by-batch´ +/// ´test:integration:anomalous-batch-elevates-cusum´ +#[test] +fn anomalous_batch_elevates_cusum() { + let mut s = ScenarioBuilder::new() + .config(SentinelConfig:: { + max_rank: 1, + cusum_slow_decay: 0.96, + cusum_coord_slow_decay: 0.96, + ..integration_config() + }) + .seed_range(0xF, 8) + .warm_batches(30) + .build(); + + let baseline = s.ingest(&cell_values(0xF, 8)); + assert_invariants(&s, &baseline); + let cusum_before = max_cusum(&baseline); + + // Sustained anomalous traffic. + let mut last_report = baseline; + for _ in 0..15 { + last_report = s.ingest(&anomalous_values(0xF, 8)); + assert_invariants(&s, &last_report); + } + + let cusum_after = max_cusum(&last_report); + assert!( + cusum_after > cusum_before, + "CUSUM should grow under sustained anomalous traffic: \ + before={cusum_before:.6}, after={cusum_after:.6}" + ); +} + +// ── Decay → ingest round-trip ─────────────────────────────── + +/// Decay applied in the middle of a run attenuates accumulated spatial +/// standing and nothing else: ingestion continues afterwards with every +/// invariant intact and the observation counter still climbing. Temporal +/// policy belongs to the host, and the engine implements it by lowering what +/// cells have earned rather than by discarding what they have learned, so a +/// decayed sentinel is a going concern and not a half-reset one. +/// +/// ´claim:engine:decay-attenuates-standing-only-so-ingestion-continues-and-the-observation-record-keeps-growing´ +/// ´test:integration:decay-then-ingest-maintains-invariants´ +#[test] +fn decay_then_ingest_maintains_invariants() { + let mut s = ScenarioBuilder::new() + .seed_range(0xF, 8) + .seed_range(0x1, 8) + .warm_batches(10) + .build(); + + // Pre-decay steady state. + for _ in 0..5 { + let batch: Vec = [cell_values(0xF, 4), cell_values(0x1, 4)].concat(); + let r = s.ingest(&batch); + assert_invariants(&s, &r); + } + + let obs_before = s.lifetime_observations(); + + // Decay: uniform 50% attenuation. + s.decay(0.5, 0.0); + + // Post-decay ingestion must succeed with invariants intact. + for _ in 0..10 { + let batch: Vec = [cell_values(0xF, 4), cell_values(0x1, 4)].concat(); + let r = s.ingest(&batch); + assert_invariants(&s, &r); + } + + assert!( + s.lifetime_observations() > obs_before, + "observations should continue accumulating after decay" + ); +} + +// ── Inspection after realistic traffic ────────────────────── + +/// After traffic deep enough to create cells at several depths, every handle +/// the engine lists is still inspectable, and each cell's analysis width is +/// the domain width less its own depth — the rule holds at whatever depths +/// real splitting produced, not just at the root. Each has at least one +/// learned direction, and the root is among them, as it must be for the +/// ancestor chains to terminate. +/// +/// (´claim:width:a-cells-analysis-width-is-the-domain-width-less-its-depth´) +/// ´test:integration:inspect-cells-after-multi-range-traffic´ +#[test] +fn inspect_cells_after_multi_range_traffic() { + let mut s = ScenarioBuilder::new() + .seed_range(0xF, 8) + .seed_range(0x1, 8) + .warm_batches(10) + .build(); + + for _ in 0..10 { + s.ingest(&[cell_values(0xF, 4), cell_values(0x1, 4)].concat()); + } + + let gnodes = s.cell_gnodes(); + assert_ne!(gnodes, [] as [torrust_sentinel::GNodeId; 0]); + + let mut found_root = false; + for gnode in &gnodes { + let insp = s.inspect_cell(*gnode).expect("tracked cell must be inspectable"); + assert_eq!(insp.analysis_width, 128 - insp.depth as usize); + assert!(insp.rank >= 1, "tracked cell should have rank >= 1"); + if insp.depth == 0 { + found_root = true; + } + } + assert!(found_root, "root cell (depth 0) must always be tracked"); +} + +// ── Coordination (end-to-end) ─────────────────────────────── + +/// Once two separate regions are being modelled in earnest, a second tier of +/// reporting appears without the host asking for it: a context that covers +/// several cells at once and carries scores of its own, finite like any +/// other. A pattern spread across sibling cells is invisible to each of them +/// individually, so the engine models the pattern itself as soon as there +/// are enough cells for one to exist. +/// +/// ´claim:engine:a-second-tier-of-reporting-appears-once-several-cells-are-modelled-together´ +/// ´test:integration:coordination-activates-with-multi-range-traffic´ +#[test] +fn coordination_activates_with_multi_range_traffic() { + let mut s = ScenarioBuilder::new() + .config(SentinelConfig:: { + split_threshold: 10, + per_sample_scores: true, + ..integration_config() + }) + .seed_range(0xF, 20) + .seed_range(0x1, 20) + .warm_batches(20) + .build(); + + // Continue feeding both ranges to keep competitive cells active. + let mut saw_coordination = false; + for _ in 0..30 { + let batch: Vec = [cell_values(0xF, 8), cell_values(0x1, 8)].concat(); + let report = s.ingest(&batch); + + if !report.coordination_reports.is_empty() { + saw_coordination = true; + // Coordination reports should have valid scores. + for cr in &report.coordination_reports { + assert!(cr.cells_reporting >= 2); + assert!(!cr.scores.novelty.mean.is_nan()); + } + break; + } + } + + assert!( + saw_coordination, + "coordination should activate with two disjoint competitive ranges" + ); +} diff --git a/packages/sentinel/tests/invariants.rs b/packages/sentinel/tests/invariants.rs new file mode 100644 index 000000000..758e7035e --- /dev/null +++ b/packages/sentinel/tests/invariants.rs @@ -0,0 +1,626 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +//! # Test index +//! +//! | Test | Area | Claim | +//! |------|------|-------| +//! | [`feed_forward_delta_one_normal`] | invariant | Each observation contributes exactly one unit to the structure's running total, checked after every batch of a long run: the total is always the number of values fed so far and never drifts from it. Importance is therefore a count of traffic rather than a derived score, which is what lets the weight a range carries be compared against another range's honestly. | +//! | [`feed_forward_delta_one_anomalous`] | invariant | cites (´claim:invariant:the-running-total-counts-exactly-one-unit-per-observation´) | +//! | [`feed_forward_delta_one_after_decay`] | invariant | cites (´claim:invariant:the-running-total-counts-exactly-one-unit-per-observation´) | +//! | [`analysis_width_equals_128_minus_depth`] | invariant | Every cell and every ancestor in a report analyses exactly the domain less the levels its position has already fixed. The relation is arithmetic rather than incidental: the bits routing resolved are constant for everything arriving in that cell and carry no information, so what remains is precisely what a tracker there can learn from, and its declared analysis is that and nothing else. | +//! | [`energy_ratio_bounded_zero_to_one`] | invariant | The fraction of structure a tracker has managed to capture is reported as a genuine fraction: it never falls below nothing and never exceeds everything, in any cell or ancestor of a report. Because it is bounded on both sides, a caller can read it directly as how much of what arrives the model explains, and can compare one cell's figure against another's. | +//! | [`rank_bounded_by_max_rank`] | invariant | No tracker in a report has grown past the ceiling its configuration set, at any level of the tree. The ceiling is what makes a tracker's cost knowable in advance, so it has to be a hard limit on what the adaptation may reach for rather than a target it aims at — traffic complicated enough to justify more structure still does not get more. | +//! | [`no_nan_scores_after_warmup`] | invariant | After seeding and a run of warming batches, no axis of any reported cell hands back a number that is not a number. This matters more than tidiness: such a value compares false against every threshold, so a single one would silently disarm the alerting that reads it, and the arithmetic guards its absence rather than callers being expected to check. | +//! | [`noise_injected_before_real_observations`] | invariant | After a run that creates cells below the root, every cell being tracked carries synthetic observations — including those that came into being while traffic was already flowing. The ordering within the step is fixed rather than raced: preparation happens before a tracker is shown real data, so no cell is ever in the position of judging its first batch against nothing. | +//! | [`graph_updated_before_analysis_set`] | invariant | The summary and the detail of a batch report describe the same moment: when the summary says cells are competing, the report already carries their entries. The structure is brought up to date before attention is reapportioned within the same batch, so a split does not leave a batch whose summary counts a cell that the report cannot show. | +//! | [`root_always_in_analysis_set`] | invariant | Whatever range a batch is aimed at, the reported set of analysed cells still reaches back to the root — its shallowest member is the root in every batch of a run that cycles through all the leading ranges. The root's place is unconditional, so every chain of ancestors terminates and there is always a model covering traffic that belongs to no more specific cell. | +//! | [`root_survives_extreme_decay`] | invariant | Ageing severe enough to annihilate the accumulated standing of everything else still leaves the root tracker in place, and traffic arriving afterwards is reported against it again. The root is permanent by construction rather than by having earned its standing, because a sentinel that could decay away its last model would have nothing to route to and no way to begin again. | +//! | [`score_polarity_higher_is_more_anomalous`] | invariant | Two sentinels given identical settling traffic diverge in the expected direction once one of them is fed structurally novel batches: its peak drift reading ends up above the other's. Scores point one way — larger means more anomalous — so a caller may threshold and compare them without having to know which axis produced the number or which direction it runs in. | +//! | [`cusum_accumulators_non_negative`] | invariant | Through a run of ordinary traffic, no drift accumulator on any axis of any reported cell ever goes below nothing. The accumulator is clamped at rest deliberately: a stretch of quieter-than-usual traffic must not bank negative standing that a later attack could spend, so a rise always starts from zero and means what it says. | +//! | [`lifetime_observations_monotonically_increases`] | invariant | The lifetime observation count rises by exactly the size of each batch, for batches ranging from a single value to many. It is a plain census of what was handed in — never sampled, never rounded and never adjusted for what the values looked like — which is what makes it usable as the denominator when judging any rate the sentinel reports. | +//! | [`cells_tracked_always_at_least_one`] | invariant | At no point in a sentinel's life is it tracking nothing: not at birth before any traffic, not through a run of ingestion, and not after ageing severe enough to strip away everything that had accumulated. There is always at least the root, so the question "what does the sentinel make of this value" always has an answer. | +//! | [`assert_invariants_under_random_traffic`] | invariant | Traffic with no structure at all — values scattered across the whole domain, in batches whose size changes from one to the next — leaves every structural property standing, checked after each batch. The guarantees are not conditioned on the traffic being well behaved or on batches being uniform, which is the whole point of calling them guarantees. | +//! | [`assert_invariants_after_decay_regrowth`] | invariant | A sentinel whose tree has been collapsed by severe ageing and then made to regrow under traffic spread across the ranges satisfies every structural property throughout the regrowth, batch by batch. The transient state of a system rebuilding itself is exactly where a bound is likeliest to slip, so the guarantees are asserted while it is in motion rather than once it has settled. | +//! | [`a_single_arrival_outlives_the_projection_only_in_the_accumulator`] | invariant | The feed-forward count is checked in the accumulator's own domain rather than through a floating-point projection, because past a certain magnitude the projection cannot express a single arrival. The projection is lossy by its own documentation, and at the first magnitude where consecutive integers stop being separately representable, a total and that same total plus one arrival land on the same number while a total plus two lands two away. A check that projects both sides and allows them to differ by less than one arrival therefore rejects arithmetic that is exactly right. The accumulator keeps the distinction the projection loses, so the comparison belongs there; this test pins the property the choice rests on rather than the failure itself, which is some nine quadrillion observations away and not reachable by a test. | + +//! Integration tests for the properties the sentinel is required to hold at +//! **all times, whatever the traffic** — the statements a reader of any report +//! is entitled to assume without checking. +//! +//! They fall into a few families. Accounting: each observation adds exactly one +//! to the structure's running total, whatever the batch contained, so importance +//! measures weight of traffic and can never be inflated by the content of a +//! request. Geometry: a cell's analysis is exactly as wide as the domain less +//! the levels routing has already resolved, its captured-energy fraction lies in +//! the unit interval, its rank never exceeds the configured ceiling, and no +//! score is ever reported as not-a-number. Ordering within a batch: a tracker is +//! warmed before it is asked about real traffic, and a cell the summary counts +//! as competing is already reported in the same batch that created it. +//! +//! Two of the families exist because of what a caller does with a report. The +//! root is present in every analysis set, so every chain of ancestors +//! terminates and there is always somewhere to attribute traffic that belongs +//! nowhere else; and it survives decay severe enough to collapse everything +//! else, because a sentinel with no root would have nowhere to begin again. +//! Score polarity runs one way — higher means more anomalous, on every axis — +//! so a caller may compare and threshold without asking which direction this +//! particular number points in, and the drift accumulators never go negative, +//! so a quiet period cannot bank credit against a future attack. +//! +//! The last tests are cross-cutting: they assert the whole suite batch by batch +//! under unstructured traffic of varying size, and across a collapse and +//! regrowth of the tree, on the principle that an invariant is only worth the +//! name if it holds while the system is in motion. + +mod common; + +use common::{ScenarioBuilder, anomalous_values, assert_invariants, cell_values, integration_config, max_cusum, test_config}; +use torrust_mudlark::Inspectable; +use torrust_sentinel::{NoiseSchedule, Sentinel128, SentinelConfig}; + +// ── G1: Feed-forward invariant ────────────────────────────── + +/// Each observation contributes exactly one unit to the structure's running +/// total, checked after every batch of a long run: the total is always the +/// number of values fed so far and never drifts from it. Importance is +/// therefore a count of traffic rather than a derived score, which is what lets +/// the weight a range carries be compared against another range's honestly. +/// +/// ´claim:invariant:the-running-total-counts-exactly-one-unit-per-observation´ +/// ´test:integration:feed-forward-delta-one-normal´ +#[test] +fn feed_forward_delta_one_normal() { + let mut s = Sentinel128::new(test_config()).unwrap(); + + for batch_num in 1..=50u64 { + let values = cell_values(0xA, 8); + s.ingest(&values); + assert_eq!( + s.graph().total_sum(), + batch_num * 8, + "total_sum must equal cumulative observation count" + ); + } +} + +/// The accounting is blind to content: a batch of structurally novel values +/// arriving after a settled run raises the total by its own size and no more. +/// This is the security-relevant half of the rule — how anomalous a request +/// looks buys it no extra standing in the structure, and no attacker can win +/// attention for a region by making its traffic look strange. +/// +/// (´claim:invariant:the-running-total-counts-exactly-one-unit-per-observation´) +/// ´test:integration:feed-forward-delta-one-anomalous´ +#[test] +fn feed_forward_delta_one_anomalous() { + let mut s = Sentinel128::new(test_config()).unwrap(); + + // Normal traffic. + for _ in 0..20 { + s.ingest(&cell_values(0xA, 8)); + } + let before = s.graph().total_sum(); + + // Anomalous traffic — graph total_sum should still increase by exactly n. + let anomalous = anomalous_values(0xA, 8); + s.ingest(&anomalous); + assert_eq!( + s.graph().total_sum(), + before + 8, + "anomalous traffic should not change delta-one accounting" + ); +} + +/// Ageing the structure down rescales what has accumulated, but it does not +/// disturb the increment: a batch arriving afterwards adds exactly its own size +/// on top of the reduced total. The two mechanisms compose cleanly, so a +/// long-lived sentinel can forget old traffic without its counting of new +/// traffic going wrong. +/// +/// (´claim:invariant:the-running-total-counts-exactly-one-unit-per-observation´) +/// ´test:integration:feed-forward-delta-one-after-decay´ +#[test] +fn feed_forward_delta_one_after_decay() { + let mut s = Sentinel128::new(test_config()).unwrap(); + + s.ingest(&cell_values(0xA, 100)); + s.decay(0.5, 0.0); + let after_decay = s.graph().total_sum(); + + s.ingest(&cell_values(0xA, 10)); + assert_eq!( + s.graph().total_sum(), + after_decay + 10, + "after decay + ingest, total_sum reflects only the new ingest count" + ); +} + +// ── G2: Constant-norm geometry ────────────────────────────── + +/// Every cell and every ancestor in a report analyses exactly the domain less +/// the levels its position has already fixed. The relation is arithmetic rather +/// than incidental: the bits routing resolved are constant for everything +/// arriving in that cell and carry no information, so what remains is precisely +/// what a tracker there can learn from, and its declared analysis is that and +/// nothing else. +/// +/// ´claim:invariant:a-cells-analysis-covers-the-domain-less-the-levels-routing-already-fixed´ +/// ´test:integration:analysis-width-equals-128-minus-depth´ +#[test] +fn analysis_width_equals_128_minus_depth() { + let mut s = ScenarioBuilder::new() + .config(SentinelConfig:: { + max_rank: 2, + split_threshold: 10, + ..integration_config() + }) + .seed_range(0xA, 20) + .warm_batches(15) + .build(); + + let report = s.ingest(&cell_values(0xA, 8)); + + for cr in report.cell_reports.iter().chain(report.ancestor_reports.iter()) { + assert_eq!( + cr.analysis_width, + 128 - cr.depth as usize, + "analysis_width mismatch at depth {}", + cr.depth + ); + } +} + +/// The fraction of structure a tracker has managed to capture is reported as a +/// genuine fraction: it never falls below nothing and never exceeds everything, +/// in any cell or ancestor of a report. Because it is bounded on both sides, a +/// caller can read it directly as how much of what arrives the model explains, +/// and can compare one cell's figure against another's. +/// +/// ´claim:invariant:the-captured-structure-fraction-stays-inside-the-unit-interval´ +/// ´test:integration:energy-ratio-bounded-zero-to-one´ +#[test] +fn energy_ratio_bounded_zero_to_one() { + let mut s = ScenarioBuilder::new() + .config(SentinelConfig:: { + max_rank: 2, + split_threshold: 10, + ..integration_config() + }) + .seed_range(0xA, 20) + .warm_batches(15) + .build(); + + let report = s.ingest(&cell_values(0xA, 8)); + + for cr in report.cell_reports.iter().chain(report.ancestor_reports.iter()) { + assert!( + (0.0..=1.0).contains(&cr.energy_ratio), + "energy_ratio {} out of [0.0, 1.0] at depth {}", + cr.energy_ratio, + cr.depth + ); + } +} + +/// No tracker in a report has grown past the ceiling its configuration set, at +/// any level of the tree. The ceiling is what makes a tracker's cost knowable in +/// advance, so it has to be a hard limit on what the adaptation may reach for +/// rather than a target it aims at — traffic complicated enough to justify more +/// structure still does not get more. +/// +/// ´claim:invariant:no-tracker-grows-past-the-ceiling-its-configuration-set´ +/// ´test:integration:rank-bounded-by-max-rank´ +#[test] +fn rank_bounded_by_max_rank() { + let cfg = SentinelConfig:: { + max_rank: 2, + split_threshold: 10, + ..integration_config() + }; + let max_rank = cfg.max_rank; + + let mut s = ScenarioBuilder::new() + .config(cfg) + .seed_range(0xA, 20) + .warm_batches(15) + .build(); + + let report = s.ingest(&cell_values(0xA, 8)); + + for cr in report.cell_reports.iter().chain(report.ancestor_reports.iter()) { + assert!( + cr.rank <= max_rank, + "rank {} exceeds max_rank {} at depth {}", + cr.rank, + max_rank, + cr.depth + ); + } +} + +/// After seeding and a run of warming batches, no axis of any reported cell +/// hands back a number that is not a number. This matters more than tidiness: +/// such a value compares false against every threshold, so a single one would +/// silently disarm the alerting that reads it, and the arithmetic guards its +/// absence rather than callers being expected to check. +/// +/// ´claim:invariant:no-scoring-axis-ever-hands-back-a-value-that-is-not-a-number´ +/// ´test:integration:no-nan-scores-after-warmup´ +#[test] +fn no_nan_scores_after_warmup() { + let mut s = ScenarioBuilder::new() + .config(SentinelConfig:: { + max_rank: 2, + split_threshold: 10, + ..integration_config() + }) + .seed_range(0xA, 20) + .warm_batches(15) + .build(); + + let report = s.ingest(&cell_values(0xA, 8)); + + for cr in report.cell_reports.iter().chain(report.ancestor_reports.iter()) { + assert!(!cr.scores.novelty.mean.is_nan(), "NaN novelty at depth {}", cr.depth); + assert!( + !cr.scores.displacement.mean.is_nan(), + "NaN displacement at depth {}", + cr.depth + ); + assert!(!cr.scores.surprise.mean.is_nan(), "NaN surprise at depth {}", cr.depth); + } +} + +// ── G3: Step ordering ────────────────────────────────────── + +/// After a run that creates cells below the root, every cell being tracked +/// carries synthetic observations — including those that came into being while +/// traffic was already flowing. The ordering within the step is fixed rather +/// than raced: preparation happens before a tracker is shown real data, so no +/// cell is ever in the position of judging its first batch against nothing. +/// +/// ´claim:invariant:a-tracker-is-prepared-before-it-is-ever-shown-real-traffic´ +/// ´test:integration:noise-injected-before-real-observations´ +#[test] +fn noise_injected_before_real_observations() { + let cfg = SentinelConfig:: { + noise_schedule: NoiseSchedule::Explicit(vec![5]), + noise_batch_size: 4, + split_threshold: 10, + ..integration_config() + }; + let mut s = Sentinel128::new(cfg).unwrap(); + + // Feed enough traffic to create at least one competitive cell. + for _ in 0..20 { + s.ingest(&cell_values(0xA, 20)); + } + + // Every tracker should have noise_observations > 0 (noise ran first). + for &gnode in &s.cell_gnodes() { + let insp = s.inspect_cell(gnode).unwrap(); + assert!( + insp.maturity.noise_observations > 0, + "cell at depth {} should have noise observations", + insp.depth + ); + } +} + +/// The summary and the detail of a batch report describe the same moment: when +/// the summary says cells are competing, the report already carries their +/// entries. The structure is brought up to date before attention is +/// reapportioned within the same batch, so a split does not leave a batch whose +/// summary counts a cell that the report cannot show. +/// +/// ´claim:invariant:a-cell-the-summary-counts-as-competing-is-already-detailed-in-the-same-report´ +/// ´test:integration:graph-updated-before-analysis-set´ +#[test] +fn graph_updated_before_analysis_set() { + let cfg = SentinelConfig:: { + split_threshold: 10, + ..integration_config() + }; + let mut s = Sentinel128::new(cfg).unwrap(); + + // Feed concentrated traffic to force a split. + for _ in 0..5 { + s.ingest(&cell_values(0xA, 20)); + } + + // After enough traffic to split, the new cell should appear in + // the analysis set in the same batch's report. + let report = s.ingest(&cell_values(0xA, 20)); + let summary = &report.analysis_set_summary; + + // If competitive cells exist, they must already be represented. + if summary.competitive_size > 0 { + assert!( + !report.cell_reports.is_empty(), + "competitive cells in summary but no cell_reports" + ); + } +} + +// ── Root invariants ───────────────────────────────────────── + +/// Whatever range a batch is aimed at, the reported set of analysed cells still +/// reaches back to the root — its shallowest member is the root in every batch +/// of a run that cycles through all the leading ranges. The root's place is +/// unconditional, so every chain of ancestors terminates and there is always a +/// model covering traffic that belongs to no more specific cell. +/// +/// ´claim:invariant:the-root-is-in-every-analysed-set-so-every-ancestor-chain-terminates´ +/// ´test:integration:root-always-in-analysis-set´ +#[test] +fn root_always_in_analysis_set() { + let mut s = Sentinel128::new(integration_config()).unwrap(); + + for i in 0..30u128 { + let report = s.ingest(&cell_values(i % 16, 8)); + assert_eq!( + report.analysis_set_summary.depth_range.0, 0, + "root (depth 0) must always be in the analysis set" + ); + } +} + +/// Ageing severe enough to annihilate the accumulated standing of everything +/// else still leaves the root tracker in place, and traffic arriving afterwards +/// is reported against it again. The root is permanent by construction rather +/// than by having earned its standing, because a sentinel that could decay away +/// its last model would have nothing to route to and no way to begin again. +/// +/// ´claim:invariant:the-root-tracker-outlives-any-amount-of-ageing´ +/// ´test:integration:root-survives-extreme-decay´ +#[test] +fn root_survives_extreme_decay() { + let mut s = ScenarioBuilder::new().seed_range(0xA, 20).warm_batches(20).build(); + + assert!(s.cells_tracked() > 1); + + // Annihilate everything. + s.decay(0.0001, 0.0); + + // Root must still be present. + let root = s.graph().g_root(); + assert!(s.inspect_cell(root).is_some(), "root tracker must survive decay"); + + // Re-ingest — root should still produce reports. + let report = s.ingest(&cell_values(0xA, 8)); + assert!( + report.ancestor_reports.iter().any(|r| r.depth == 0), + "root must appear in ancestor_reports after decay + re-ingest" + ); +} + +// ── Score invariants ──────────────────────────────────────── + +/// Two sentinels given identical settling traffic diverge in the expected +/// direction once one of them is fed structurally novel batches: its peak drift +/// reading ends up above the other's. Scores point one way — larger means more +/// anomalous — so a caller may threshold and compare them without having to +/// know which axis produced the number or which direction it runs in. +/// +/// ´claim:invariant:scores-run-one-way-a-larger-reading-always-means-more-anomalous´ +/// ´test:integration:score-polarity-higher-is-more-anomalous´ +#[test] +fn score_polarity_higher_is_more_anomalous() { + // Use a simple setup: repeated cell_values warm-up, then compare + // max_cusum() under continued normal vs. anomalous traffic. + let batch = cell_values(0xA, 8); + + let cfg = SentinelConfig:: { + max_rank: 2, + split_threshold: 10, + ..integration_config() + }; + + // ── Run A: continued normal traffic ── + let mut sa = Sentinel128::new(cfg.clone()).unwrap(); + for _ in 0..20 { + sa.ingest(&batch); + } + let normal_report = sa.ingest(&batch); + let normal_cusum = max_cusum(&normal_report); + + // ── Run B: anomalous traffic after identical warm-up ── + let mut sb = Sentinel128::new(cfg).unwrap(); + for _ in 0..20 { + sb.ingest(&batch); + } + let anom_batch = anomalous_values(0xA, 8); + for _ in 0..5 { + sb.ingest(&anom_batch); + } + let anomaly_report = sb.ingest(&anom_batch); + let anomaly_cusum = max_cusum(&anomaly_report); + + assert!( + anomaly_cusum > normal_cusum, + "anomalous data should produce higher CUSUM: anomaly={anomaly_cusum:.6}, normal={normal_cusum:.6}" + ); +} + +/// Through a run of ordinary traffic, no drift accumulator on any axis of any +/// reported cell ever goes below nothing. The accumulator is clamped at rest +/// deliberately: a stretch of quieter-than-usual traffic must not bank negative +/// standing that a later attack could spend, so a rise always starts from zero +/// and means what it says. +/// +/// ´claim:invariant:a-drift-accumulator-never-falls-below-zero-so-quiet-traffic-banks-no-credit´ +/// ´test:integration:cusum-accumulators-non-negative´ +#[test] +fn cusum_accumulators_non_negative() { + let mut s = ScenarioBuilder::new().seed_range(0xA, 20).warm_batches(20).build(); + + for _ in 0..10 { + let report = s.ingest(&cell_values(0xA, 8)); + + for cr in report.cell_reports.iter().chain(report.ancestor_reports.iter()) { + for (name, acc) in [ + ("novelty", cr.scores.novelty.cusum.accumulator), + ("displacement", cr.scores.displacement.cusum.accumulator), + ("surprise", cr.scores.surprise.cusum.accumulator), + ("coherence", cr.scores.coherence.cusum.accumulator), + ] { + assert!( + acc >= 0.0, + "CUSUM {name} accumulator is negative ({acc}) at depth {}", + cr.depth + ); + } + } + } +} + +// ── Counters ──────────────────────────────────────────────── + +/// The lifetime observation count rises by exactly the size of each batch, for +/// batches ranging from a single value to many. It is a plain census of what +/// was handed in — never sampled, never rounded and never adjusted for what the +/// values looked like — which is what makes it usable as the denominator when +/// judging any rate the sentinel reports. +/// +/// ´claim:invariant:the-lifetime-count-rises-by-exactly-the-size-of-each-batch´ +/// ´test:integration:lifetime-observations-monotonically-increases´ +#[test] +fn lifetime_observations_monotonically_increases() { + let mut s = Sentinel128::new(integration_config()).unwrap(); + let mut prev = 0u64; + + for batch_size in [1, 5, 20, 3, 50] { + s.ingest(&cell_values(0xA, batch_size)); + let current = s.lifetime_observations(); + assert_eq!( + current, + prev + batch_size as u64, + "lifetime_observations should grow by batch size" + ); + prev = current; + } +} + +/// At no point in a sentinel's life is it tracking nothing: not at birth before +/// any traffic, not through a run of ingestion, and not after ageing severe +/// enough to strip away everything that had accumulated. There is always at +/// least the root, so the question "what does the sentinel make of this value" +/// always has an answer. +/// +/// ´claim:invariant:at-no-point-in-its-life-is-the-sentinel-tracking-nothing´ +/// ´test:integration:cells-tracked-always-at-least-one´ +#[test] +fn cells_tracked_always_at_least_one() { + let mut s = Sentinel128::new(integration_config()).unwrap(); + + assert!(s.cells_tracked() >= 1, "fresh sentinel must track at least the root"); + + for _ in 0..20 { + s.ingest(&cell_values(0xA, 8)); + assert!(s.cells_tracked() >= 1, "cells_tracked must never drop below 1"); + } + + s.decay(0.0001, 0.0); + assert!(s.cells_tracked() >= 1, "cells_tracked must stay ≥ 1 after extreme decay"); +} + +// ── Cross-cutting ─────────────────────────────────────────── + +/// Traffic with no structure at all — values scattered across the whole domain, +/// in batches whose size changes from one to the next — leaves every structural +/// property standing, checked after each batch. The guarantees are not +/// conditioned on the traffic being well behaved or on batches being uniform, +/// which is the whole point of calling them guarantees. +/// +/// ´claim:invariant:the-whole-suite-holds-under-unstructured-traffic-in-batches-of-varying-size´ +/// ´test:integration:assert-invariants-under-random-traffic´ +#[test] +fn assert_invariants_under_random_traffic() { + use std::hash::{DefaultHasher, Hash, Hasher}; + + let mut s = Sentinel128::new(SentinelConfig:: { + split_threshold: 10, + ..integration_config() + }) + .unwrap(); + + // Use a simple deterministic hash-based "random" generator. + for batch_idx in 0..60u64 { + let batch_size = (batch_idx % 63) as usize + 1; // 1–63 + let values: Vec = (0..batch_size) + .map(|i| { + let mut h = DefaultHasher::new(); + (batch_idx, i).hash(&mut h); + u128::from(h.finish()) | (u128::from(h.finish()) << 64) + }) + .collect(); + + let report = s.ingest(&values); + assert_invariants(&s, &report); + } +} + +/// A sentinel whose tree has been collapsed by severe ageing and then made to +/// regrow under traffic spread across the ranges satisfies every structural +/// property throughout the regrowth, batch by batch. The transient state of a +/// system rebuilding itself is exactly where a bound is likeliest to slip, so +/// the guarantees are asserted while it is in motion rather than once it has +/// settled. +/// +/// ´claim:invariant:the-guarantees-hold-throughout-a-collapse-and-the-regrowth-that-follows´ +/// ´test:integration:assert-invariants-after-decay-regrowth´ +#[test] +fn assert_invariants_after_decay_regrowth() { + let mut s = ScenarioBuilder::new() + .seed_range(0xA, 20) + .seed_range(0x5, 20) + .warm_batches(20) + .build(); + + // Severe decay to collapse most cells. + s.decay(0.001, 0.0); + + // Regrow with spread traffic so the tree re-balances. + for i in 0..20u128 { + let report = s.ingest(&cell_values(i % 16, 20)); + assert_invariants(&s, &report); + } +} + +/// The feed-forward count is checked in the accumulator's own domain rather +/// than through a floating-point projection, because past a certain magnitude +/// the projection cannot express a single arrival. The projection is lossy by +/// its own documentation, and at the first magnitude where consecutive +/// integers stop being separately representable, a total and that same total +/// plus one arrival land on the same number while a total plus two lands two +/// away. A check that projects both sides and allows them to differ by less +/// than one arrival therefore rejects arithmetic that is exactly right. The +/// accumulator keeps the distinction the projection loses, so the comparison +/// belongs there; this test pins the property the choice rests on rather than +/// the failure itself, which is some nine quadrillion observations away and +/// not reachable by a test. +/// +/// ´claim:invariant:the-feed-forward-count-is-compared-in-the-accumulator-domain-because-the-projection-loses-a-single-arrival´ +/// ´test:integration:a-single-arrival-outlives-the-projection-only-in-the-accumulator´ +#[test] +fn a_single_arrival_outlives_the_projection_only_in_the_accumulator() { + // The first magnitude at which consecutive integers stop being separately + // representable in the projection's format. + let total: u64 = 1u64 << 53; + let one_more: u64 = total + 1; + let two_more: u64 = total + 2; + + // The projection cannot tell one arrival from none at this magnitude. + assert_eq!( + total.to_f64_approx().to_bits(), + one_more.to_f64_approx().to_bits(), + "the projection separated one arrival" + ); + // The accumulator's own comparison can. + assert_ne!(total, one_more); + + // And the gap the projection does report can exceed a single arrival, so + // an absolute tolerance of one arrival is not a safe reading of it. + assert!((two_more.to_f64_approx() - one_more.to_f64_approx()).abs() > 1.0); +} diff --git a/packages/sentinel/tests/noise.rs b/packages/sentinel/tests/noise.rs new file mode 100644 index 000000000..87ccf6ba2 --- /dev/null +++ b/packages/sentinel/tests/noise.rs @@ -0,0 +1,407 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +//! Integration tests for the **warming schedule** — the configured policy that +//! decides how many rounds of synthetic observations a tracker is fed before it +//! is allowed to judge real traffic, and what those rounds leave behind. +//! +//! A tracker with no history has no baseline to score against, so its first +//! real batch would find everything extreme. The schedule closes that window by +//! warming a tracker at the moment it comes into being — the root when the +//! sentinel is constructed, a deeper cell when it enters the analysis set — so +//! nothing is ever asked a question before it can hold an answer. Synthetic +//! rounds are booked apart from real ones: they raise the tracker's own +//! maturity counters and leave the sentinel's traffic counters untouched, +//! because they are not traffic and must not be reported as though they were. +//! +//! Two decisions shape the policy. The round count tapers with depth, because a +//! deeper cell analyses a narrower slice of the domain and settles sooner, so +//! paying root-sized warming everywhere would be waste; a schedule can also be +//! written out per depth by hand, and an empty one turns warming off entirely. +//! And the draws come from a single seeded generator that advances as it moves +//! from cell to cell, which makes a whole run reproducible from its seed while +//! still giving every tracker its own draws. Because warming does leave a mark, +//! the handover is deliberate: the slow baselines are seeded from the fast +//! ones, then the drift accumulators and the rejection state are zeroed, so a +//! tracker begins its real life at rest rather than carrying synthetic history +//! into its first verdict. +//! +//! # Test index +//! +//! | Test | Area | Claim | +//! |------|------|-------| +//! | [`root_warmed_at_construction`] | schedule | Constructing a sentinel already warms its root tracker: the root carries synthetic observations before any caller has had a chance to feed it. Those rounds push the tracker's reliance on synthetic history toward its maximum and hold it there, so a root that has seen nothing but warming still reports itself as fully synthetic — the reliance only falls once real traffic arrives to displace it. | +//! | [`root_cold_when_schedule_empty`] | schedule | A schedule that specifies no rounds at any depth turns warming off rather than falling back to a default: the root is constructed cold, with no synthetic observations at all. Its reliance on synthetic history still reads as maximal, because that is the value a tracker is born with and warming is what would have started moving it. | +//! | [`noise_does_not_count_as_real_observations`] | schedule | Warming rounds are not traffic and are not counted as traffic: a sentinel whose root has just been warmed still reports having observed nothing over its lifetime. The two ledgers are kept apart deliberately, so an operator reading the observation count sees what the world sent and never the sentinel's own preparation. | +//! | [`new_cells_warmed_on_analysis_set_entry`] | schedule | Warming is not a construction-time favour granted to the root alone. After a run whose traffic is concentrated enough to split the tree, every cell the sentinel is tracking carries synthetic observations, including the ones that did not exist when the sentinel was built. A cell that arrives mid-run is therefore never asked to score its own first batch from an empty baseline. | +//! | [`successive_cells_get_different_noise`] | schedule | cites (´claim:schedule:a-cell-born-mid-run-is-warmed-as-it-enters-the-analysis-set´) | +//! | [`deeper_cells_receive_fewer_noise_rounds`] | schedule | Under a geometric schedule the round count falls off with depth, and the effect is visible in the trackers themselves: at least one cell below the root ends the run with fewer synthetic observations than the root has. The taper is the point of the schedule — a deeper cell works on a narrower slice and settles in fewer rounds, so spending root-sized warming on it would buy nothing. | +//! | [`auto_inject_resets_cusum`] | schedule | Warming ends with the drift detectors wound back to zero, so the first real batch a tracker sees is its first step of drift accounting — every cell and ancestor in that batch's report says exactly one step has passed since its last reset. Without the reset, the drift a tracker accumulated while chasing synthetic rounds would be charged to whoever sent the first real request. | +//! | [`deterministic_with_same_seed`] | schedule | Two sentinels built from the same seed and fed the same traffic come out with the same cells, warmed by the same number of rounds and left with the same reliance on synthetic history. Warming is a reproducible part of a run rather than a source of drift between two otherwise identical deployments, which is what makes a captured incident replayable at all. | +//! | [`different_seeds_produce_different_baselines`] | schedule | The seed is not cosmetic: two sentinels warmed from different seeds and then fed byte-identical traffic score that traffic against different baselines. Warming leaves a real imprint on where a tracker starts, so an attacker who knew one deployment's warmed baseline would not thereby know another's. | +//! | [`reset_reseeds_rng_and_warms_root`] | schedule | Resetting a sentinel that has been running for a long stretch puts its root back exactly where a freshly constructed one stands: the same number of warming rounds, the same reliance on synthetic history. Reset restores the generator to its seed and warms the rebuilt root again, so it is a genuine return to birth rather than a partial clearing that leaves a cold root behind. | +//! | [`coordination_contexts_warmed_on_activation`] | schedule | The contexts that watch several cells at once are warmed on the same terms as the cell trackers: in a run where any of them became active, none of them is left cold. Their synthetic rounds are drawn to look like the score patterns they will actually be shown, sampled from the contributing cells' own baselines, because a context scores score vectors rather than raw coordinates. | + +mod common; + +use common::{ScenarioBuilder, assert_invariants, cell_values, cold_config, seeded_sentinel, test_config}; +use torrust_sentinel::{NoiseSchedule, Sentinel128, SentinelConfig}; + +// ── Root Warming at Construction ──────────────────────────── + +/// Constructing a sentinel already warms its root tracker: the root carries +/// synthetic observations before any caller has had a chance to feed it. Those +/// rounds push the tracker's reliance on synthetic history toward its maximum +/// and hold it there, so a root that has seen nothing but warming still reports +/// itself as fully synthetic — the reliance only falls once real traffic +/// arrives to displace it. +/// +/// ´claim:schedule:the-root-tracker-is-already-warmed-when-the-sentinel-is-handed-back´ +/// ´test:integration:root-warmed-at-construction´ +#[test] +fn root_warmed_at_construction() { + let s = Sentinel128::new(test_config()).unwrap(); + let root = s.graph().g_root(); + let insp = s.inspect_cell(root).expect("root should exist"); + + assert!( + insp.maturity.noise_observations > 0, + "root tracker should have received noise observations" + ); + // noise_influence starts at 1.0 and noise pushes toward 1.0, + // so a noise-only tracker remains at 1.0. It decays only + // after real observations. + assert!( + (insp.maturity.noise_influence - 1.0).abs() < f64::EPSILON, + "noise_influence should stay at 1.0 with only noise" + ); +} + +/// A schedule that specifies no rounds at any depth turns warming off rather +/// than falling back to a default: the root is constructed cold, with no +/// synthetic observations at all. Its reliance on synthetic history still reads +/// as maximal, because that is the value a tracker is born with and warming is +/// what would have started moving it. +/// +/// ´claim:schedule:an-empty-schedule-switches-warming-off-and-leaves-the-tracker-cold´ +/// ´test:integration:root-cold-when-schedule-empty´ +#[test] +fn root_cold_when_schedule_empty() { + let s = Sentinel128::new(cold_config()).unwrap(); + let root = s.graph().g_root(); + let insp = s.inspect_cell(root).unwrap(); + + assert_eq!(insp.maturity.noise_observations, 0); + assert!( + (insp.maturity.noise_influence - 1.0).abs() < f64::EPSILON, + "cold root should have default noise_influence of 1.0" + ); +} + +/// Warming rounds are not traffic and are not counted as traffic: a sentinel +/// whose root has just been warmed still reports having observed nothing over +/// its lifetime. The two ledgers are kept apart deliberately, so an operator +/// reading the observation count sees what the world sent and never the +/// sentinel's own preparation. +/// +/// ´claim:schedule:synthetic-rounds-never-enter-the-observed-traffic-count´ +/// ´test:integration:noise-does-not-count-as-real-observations´ +#[test] +fn noise_does_not_count_as_real_observations() { + let s = Sentinel128::new(test_config()).unwrap(); + assert_eq!( + s.lifetime_observations(), + 0, + "noise injection at construction should not increment lifetime_observations" + ); +} + +// ── Child Cell Warming ────────────────────────────────────── + +/// Warming is not a construction-time favour granted to the root alone. After a +/// run whose traffic is concentrated enough to split the tree, every cell the +/// sentinel is tracking carries synthetic observations, including the ones that +/// did not exist when the sentinel was built. A cell that arrives mid-run is +/// therefore never asked to score its own first batch from an empty baseline. +/// +/// ´claim:schedule:a-cell-born-mid-run-is-warmed-as-it-enters-the-analysis-set´ +/// ´test:integration:new-cells-warmed-on-analysis-set-entry´ +#[test] +fn new_cells_warmed_on_analysis_set_entry() { + let (s, _reports) = ScenarioBuilder::new() + .config(SentinelConfig:: { + split_threshold: 10, + analysis_k: 16, + ..test_config() + }) + .seed_range(0xF, 20) + .warm_batches(19) + .build_with_reports(); + + // All cells should be warm. + for &gnode in &s.cell_gnodes() { + let insp = s.inspect_cell(gnode).unwrap(); + assert!(insp.maturity.noise_observations > 0, "cell {gnode:?} should be noise-warmed"); + } +} + +/// Traffic split across two well-separated ranges produces several non-root +/// cells in one run, and each of them is warmed in turn. This pins the +/// many-cells end of the rule: the generator is a single persistent one that +/// advances as it serves each cell, so warming a second cell neither replays +/// the first cell's draws nor exhausts the supply. +/// +/// (´claim:schedule:a-cell-born-mid-run-is-warmed-as-it-enters-the-analysis-set´) +/// ´test:integration:successive-cells-get-different-noise´ +#[test] +fn successive_cells_get_different_noise() { + let (s, _) = ScenarioBuilder::new() + .config(SentinelConfig:: { + split_threshold: 10, + analysis_k: 16, + ..test_config() + }) + .seed_range(0xF, 4) + .seed_range(0x1, 4) + .warm_batches(14) + .build_with_reports(); + + let root = s.graph().g_root(); + let non_root: Vec<_> = s.cell_gnodes().into_iter().filter(|&g| g != root).collect(); + + // We need at least two non-root cells to compare. + assert!( + non_root.len() >= 2, + "expected at least 2 non-root cells, got {}", + non_root.len() + ); + + // Both cells must be warmed. + for &gnode in &non_root { + let insp = s.inspect_cell(gnode).unwrap(); + assert!(insp.maturity.noise_observations > 0, "cell {gnode:?} should be noise-warmed"); + } +} + +// ── Depth-Tiered Noise ────────────────────────────────────── + +/// Under a geometric schedule the round count falls off with depth, and the +/// effect is visible in the trackers themselves: at least one cell below the +/// root ends the run with fewer synthetic observations than the root has. The +/// taper is the point of the schedule — a deeper cell works on a narrower slice +/// and settles in fewer rounds, so spending root-sized warming on it would buy +/// nothing. +/// +/// ´claim:schedule:a-geometric-schedule-spends-fewer-rounds-on-deeper-cells-than-on-the-root´ +/// ´test:integration:deeper-cells-receive-fewer-noise-rounds´ +#[test] +fn deeper_cells_receive_fewer_noise_rounds() { + // Use a geometric schedule with aggressive decay so the + // difference between root (depth 0) and child cells is obvious. + let cfg = SentinelConfig:: { + split_threshold: 10, + analysis_k: 16, + noise_schedule: NoiseSchedule::geometric(100, 0.5, 5), + ..test_config() + }; + + let (s, _) = ScenarioBuilder::new() + .config(cfg) + .seed_range(0xF, 20) + .warm_batches(19) + .build_with_reports(); + + let root = s.graph().g_root(); + let root_insp = s.inspect_cell(root).unwrap(); + + // At least one child cell should have fewer noise observations + // than the root (deeper → fewer rounds via geometric decay). + let non_root: Vec<_> = s.cell_gnodes().into_iter().filter(|&g| g != root).collect(); + if !non_root.is_empty() { + let any_fewer = non_root.iter().any(|&gnode| { + let insp = s.inspect_cell(gnode).unwrap(); + insp.maturity.noise_observations < root_insp.maturity.noise_observations + }); + assert!( + any_fewer, + "at least one deeper cell should have fewer noise rounds than root ({})", + root_insp.maturity.noise_observations + ); + } +} + +// ── CUSUM Reset After Noise ───────────────────────────────── + +/// Warming ends with the drift detectors wound back to zero, so the first real +/// batch a tracker sees is its first step of drift accounting — every cell and +/// ancestor in that batch's report says exactly one step has passed since its +/// last reset. Without the reset, the drift a tracker accumulated while chasing +/// synthetic rounds would be charged to whoever sent the first real request. +/// +/// ´claim:schedule:warming-hands-over-a-tracker-whose-drift-accounting-starts-at-the-first-real-batch´ +/// ´test:integration:auto-inject-resets-cusum´ +#[test] +fn auto_inject_resets_cusum() { + let mut s = Sentinel128::new(test_config()).unwrap(); + + // Root was auto-injected at construction; CUSUM should be reset. + // Ingest one real batch — steps_since_reset should be 1. + let report = s.ingest(&[ + 0xF000_0000_0000_0000_0000_0000_0000_AAAA, + 0x1000_0000_0000_0000_0000_0000_0000_BBBB, + ]); + + assert_invariants(&s, &report); + + for cr in report.cell_reports.iter().chain(report.ancestor_reports.iter()) { + assert_eq!( + cr.scores.novelty.cusum.steps_since_reset, 1, + "CUSUM should have been reset after auto noise injection" + ); + } +} + +// ── Noise Determinism ─────────────────────────────────────── + +/// Two sentinels built from the same seed and fed the same traffic come out +/// with the same cells, warmed by the same number of rounds and left with the +/// same reliance on synthetic history. Warming is a reproducible part of a run +/// rather than a source of drift between two otherwise identical deployments, +/// which is what makes a captured incident replayable at all. +/// +/// ´claim:schedule:a-fixed-seed-makes-warming-reproduce-cell-for-cell´ +/// ´test:integration:deterministic-with-same-seed´ +#[test] +fn deterministic_with_same_seed() { + let s1 = seeded_sentinel(); + let s2 = seeded_sentinel(); + + let gnodes1 = s1.cell_gnodes(); + let gnodes2 = s2.cell_gnodes(); + assert_eq!(gnodes1, gnodes2); + + for &gnode in &gnodes1 { + let insp1 = s1.inspect_cell(gnode).unwrap(); + let insp2 = s2.inspect_cell(gnode).unwrap(); + assert_eq!(insp1.maturity.noise_observations, insp2.maturity.noise_observations); + assert!( + (insp1.maturity.noise_influence - insp2.maturity.noise_influence).abs() < f64::EPSILON, + "noise influence should be identical with same seed" + ); + } +} + +/// The seed is not cosmetic: two sentinels warmed from different seeds and then +/// fed byte-identical traffic score that traffic against different baselines. +/// Warming leaves a real imprint on where a tracker starts, so an attacker who +/// knew one deployment's warmed baseline would not thereby know another's. +/// +/// ´claim:schedule:the-seed-decides-which-baseline-a-warmed-tracker-settles-on´ +/// ´test:integration:different-seeds-produce-different-baselines´ +#[test] +fn different_seeds_produce_different_baselines() { + let cfg1 = SentinelConfig:: { + noise_seed: Some(42), + ..test_config() + }; + let cfg2 = SentinelConfig:: { + noise_seed: Some(999), + ..test_config() + }; + + let values = cell_values(0xA, 20); + + let mut s1 = Sentinel128::new(cfg1).unwrap(); + let mut s2 = Sentinel128::new(cfg2).unwrap(); + + // Seed with identical traffic. + s1.ingest(&values); + s2.ingest(&values); + + // Probe with the same batch. + let r1 = s1.ingest(&values); + let r2 = s2.ingest(&values); + + assert_invariants(&s1, &r1); + assert_invariants(&s2, &r2); + + let means1: Vec = r1 + .cell_reports + .iter() + .chain(r1.ancestor_reports.iter()) + .map(|cr| cr.scores.novelty.mean) + .collect(); + let means2: Vec = r2 + .cell_reports + .iter() + .chain(r2.ancestor_reports.iter()) + .map(|cr| cr.scores.novelty.mean) + .collect(); + + assert_ne!(means1, means2, "different seeds should produce different baselines"); +} + +// ── Reset Behaviour ───────────────────────────────────────── + +/// Resetting a sentinel that has been running for a long stretch puts its root +/// back exactly where a freshly constructed one stands: the same number of +/// warming rounds, the same reliance on synthetic history. Reset restores the +/// generator to its seed and warms the rebuilt root again, so it is a genuine +/// return to birth rather than a partial clearing that leaves a cold root +/// behind. +/// +/// ´claim:schedule:a-reset-rebuilds-and-rewarms-the-root-to-the-state-a-fresh-sentinel-has´ +/// ´test:integration:reset-reseeds-rng-and-warms-root´ +#[test] +fn reset_reseeds_rng_and_warms_root() { + let cfg = test_config(); + let fresh = Sentinel128::new(cfg.clone()).unwrap(); + + let mut reset_s = Sentinel128::new(cfg).unwrap(); + reset_s.ingest(&cell_values(0xA, 100)); + reset_s.reset(); + + let fresh_insp = fresh.inspect_cell(fresh.graph().g_root()).unwrap(); + let reset_insp = reset_s.inspect_cell(reset_s.graph().g_root()).unwrap(); + + assert_eq!(fresh_insp.maturity.noise_observations, reset_insp.maturity.noise_observations); + assert!( + (fresh_insp.maturity.noise_influence - reset_insp.maturity.noise_influence).abs() < f64::EPSILON, + "noise influence after reset should match fresh sentinel" + ); +} + +// ── Coordination Warming ──────────────────────────────────── + +/// The contexts that watch several cells at once are warmed on the same terms +/// as the cell trackers: in a run where any of them became active, none of them +/// is left cold. Their synthetic rounds are drawn to look like the score +/// patterns they will actually be shown, sampled from the contributing cells' +/// own baselines, because a context scores score vectors rather than raw +/// coordinates. +/// +/// ´claim:schedule:a-context-that-activates-is-warmed-from-the-baselines-of-the-cells-it-watches´ +/// ´test:integration:coordination-contexts-warmed-on-activation´ +#[test] +fn coordination_contexts_warmed_on_activation() { + let (s, _) = ScenarioBuilder::new() + .config(SentinelConfig:: { + split_threshold: 10, + ..test_config() + }) + .seed_range(0xF, 4) + .seed_range(0x1, 4) + .warm_batches(14) + .build_with_reports(); + + let ch = s.health().coordination_health; + if ch.active_contexts > 0 { + // Coordination contexts should have been warmed on activation + // via Gamma-sampled synthetic noise (§ALGO S-11.7). + assert!( + ch.maturity_distribution.cold_trackers == 0, + "all active coordination contexts should be warmed, found {} cold", + ch.maturity_distribution.cold_trackers, + ); + } +} diff --git a/packages/sentinel/tests/pedagogy.rs b/packages/sentinel/tests/pedagogy.rs new file mode 100644 index 000000000..d31619f98 --- /dev/null +++ b/packages/sentinel/tests/pedagogy.rs @@ -0,0 +1,907 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +#![allow(clippy::print_stdout)] + +//! # End-to-End Pedagogy Test for the Spectral Sentinel +//! +//! This custom-harness test walks through the Sentinel lifecycle as a +//! narrative: construction, feed-forward observation, spatial refinement, +//! analysis-set closure, hierarchical coordination, score interpretation, +//! host-controlled decay, and reset. Every assertion corresponds to a +//! public contract from the algorithm, API reference, or ADRs. +//! +//! ## What is the Spectral Sentinel? +//! +//! The Sentinel is a three-layer online anomaly spectrometer for +//! positionally structured coordinate streams: +//! +//! - **Layer 1: Spatial substrate.** A Mudlark G-V Graph adapts a dyadic +//! contour over the coordinate domain using pure observation volume. +//! - **Layer 2: Analysis selector.** The top competitive cells are closed +//! under G-tree ancestry so every selected region has a full model chain +//! back to the root. +//! - **Layer 3: Analysis engine.** Per-cell subspace trackers score suffix +//! bit vectors on four raw axes, and coordination trackers model +//! cross-cell score patterns. +//! +//! The central teaching point is the design principle from ADR-S-001 and +//! ADR-S-002: **the Sentinel measures; the host decides**. The spatial +//! graph receives `Delta = 1` per input value. Scores flow outward in +//! reports; they never flow back into spatial importance. +//! +//! ## What this test demonstrates +//! +//! Steps 0-2 show construction, automatic noise warm-up, the feed-forward +//! invariant, and spatial refinement under concentrated traffic. Steps 3-4 +//! show analysis-set ancestry, suffix widths, report structure, raw score +//! polarity, and hierarchical coordination. Steps 5-6 demonstrate that +//! temporal policy is host-controlled through `decay()` and that `reset()` +//! returns the system to a fresh, warmed root. +//! +//! Run with: +//! +//! ```sh +//! cargo test -p torrust-sentinel --test pedagogy +//! ``` +//! +//! # Test Index +//! +//! ## Construction & Warm-Up (§ALGO S-11, ADR-S-007) +//! +//! | Step | Function | What it teaches | +//! |------|----------|-----------------| +//! | 0 | [`step_0_fresh`] | Fresh Sentinel = one spatial root, one warmed tracker, no real observations | +//! +//! ## Feed-Forward Observation (§ALGO S-9.1, ADR-S-002) +//! +//! | Step | Function | What it teaches | +//! |------|----------|-----------------| +//! | 1 | [`step_1_first_batch`] | Each raw value increments the graph by exactly one unit | +//! | 2 | [`step_2_concentrated_traffic`] | Volume alone drives spatial splitting and analysis selection | +//! +//! ## Multi-Scale Analysis (§ALGO S-8, §ALGO S-9) +//! +//! | Step | Function | What it teaches | +//! |------|----------|-----------------| +//! | 3 | [`step_3_multiscale_reports`] | Competitive cells plus ancestors form a complete reporting chain | +//! | 4 | [`step_4_scores_are_measurements`] | Scores are finite, higher-polarity measurements, not verdicts | +//! +//! ## Host Policy & Lifecycle (§ALGO S-10, §ALGO S-13.4) +//! +//! | Step | Function | What it teaches | +//! |------|----------|-----------------| +//! | 5 | [`step_5_host_controlled_decay`] | Decay changes spatial importance, not tracker count or lifetime observations | +//! | 6 | [`step_6_reset`] | Reset clears learned state while preserving the configured warm-up lifecycle | + +mod common; + +use common::{anomalous_values, assert_invariants, cell_values}; +use torrust_sentinel::{ + AnomalyScores, BatchReport, CellReport, CoordinationReport, MemberScore, NoiseSchedule, ScoreDistribution, Sentinel128, + SentinelConfig, SvdStrategy, +}; + +// -- Configuration --------------------------------------------------------- + +/// Small deterministic configuration for a narrative end-to-end run. +/// +/// The values are intentionally close to the shared integration-test +/// presets, but written out here so the test explains itself. The low +/// split threshold makes spatial refinement visible after a few batches; +/// explicit noise keeps warm-up fast and deterministic. +fn pedagogy_config() -> SentinelConfig { + SentinelConfig:: { + max_rank: 2, + forgetting_factor: 0.90, + rank_update_interval: 5, + analysis_k: 16, + analysis_depth_cutoff: 6, + energy_threshold: 0.90, + eps: 1e-6, + per_sample_scores: true, + cusum_allowance_sigmas: 0.5, + cusum_slow_decay: 0.99, + cusum_coord_slow_decay: 0.99, + clip_sigmas: 3.0, + clip_pressure_decay: 0.95, + split_threshold: 10, + d_create: 3, + d_evict: 6, + budget: 100_000, + noise_schedule: NoiseSchedule::Explicit(vec![5]), + noise_batch_size: 4, + noise_seed: Some(2026), + background_warming: false, + svd_strategy: SvdStrategy::Brand, + } +} + +// -- Helpers --------------------------------------------------------------- + +fn heading(s: &str) { + let rule = "=".repeat(72); + println!("\n{rule}"); + println!(" {s}"); + println!("{rule}"); +} + +fn subheading(s: &str) { + println!("\n-- {s} --"); +} + +fn all_cell_reports(report: &BatchReport) -> impl Iterator> { + report.cell_reports.iter().chain(report.ancestor_reports.iter()) +} + +fn root_ancestor(report: &BatchReport) -> &CellReport { + report + .ancestor_reports + .iter() + .find(|cell| cell.depth == 0) + .expect("root tracker must report as an ancestor when the batch is non-empty") +} + +fn max_novelty_z(report: &BatchReport) -> f64 { + all_cell_reports(report) + .map(|cell| cell.scores.novelty.max_z_score) + .fold(0.0_f64, f64::max) +} + +fn assert_score_distribution(name: &str, score: &ScoreDistribution) { + assert!(score.min.is_finite(), "{name}.min must be finite"); + assert!(score.max.is_finite(), "{name}.max must be finite"); + assert!(score.mean.is_finite(), "{name}.mean must be finite"); + assert!(score.max_z_score.is_finite(), "{name}.max_z_score must be finite"); + assert!(score.mean_z_score.is_finite(), "{name}.mean_z_score must be finite"); + assert!(score.baseline.mean.is_finite(), "{name}.baseline.mean must be finite"); + assert!(score.baseline.variance.is_finite(), "{name}.baseline.variance must be finite"); + assert!(score.cusum.accumulator.is_finite(), "{name}.cusum.accumulator must be finite"); + assert!( + score.cusum.slow_baseline.mean.is_finite(), + "{name}.cusum.slow_baseline.mean must be finite" + ); + assert!( + score.cusum.slow_baseline.variance.is_finite(), + "{name}.cusum.slow_baseline.variance must be finite" + ); + assert!(score.clip_pressure.is_finite(), "{name}.clip_pressure must be finite"); + + assert!(score.max + 1e-12 >= score.min, "{name}.max must be >= min"); + assert!(score.mean + 1e-12 >= score.min, "{name}.mean must be >= min"); + assert!(score.mean <= score.max + 1e-12, "{name}.mean must be <= max"); + assert!( + score.baseline.variance >= -1e-12, + "{name}.baseline variance must be non-negative" + ); + assert!( + score.cusum.slow_baseline.variance >= -1e-12, + "{name}.slow baseline variance must be non-negative" + ); + assert!( + score.cusum.accumulator >= -1e-12, + "{name}.CUSUM is one-sided and non-negative" + ); + assert!( + (-1e-12..=1.0 + 1e-12).contains(&score.clip_pressure), + "{name}.clip_pressure must be in [0, 1]" + ); +} + +fn assert_scores_are_measurements(scores: &AnomalyScores) { + assert_score_distribution("novelty", &scores.novelty); + assert_score_distribution("displacement", &scores.displacement); + assert_score_distribution("surprise", &scores.surprise); + assert_score_distribution("coherence", &scores.coherence); + + assert!( + scores.novelty.min >= -1e-12, + "novelty is residual energy and must be non-negative" + ); + assert!( + scores.displacement.min >= -1e-12 && scores.displacement.max <= 1.0 + 1e-12, + "displacement is bounded in [0, 1] (closed under float tolerance)" + ); + assert!(scores.surprise.min >= -1e-12, "surprise must be non-negative"); + assert!(scores.coherence.min >= -1e-12, "coherence must be non-negative"); +} + +/// Scalar polarity contract for a single cell's contribution to a +/// coordination report. +/// +/// Unlike [`assert_scores_are_measurements`], which checks a full +/// [`ScoreDistribution`] per axis, a [`MemberScore`] carries only one +/// scalar value (plus a z-score) per axis. We assert the same polarity +/// invariants as the distribution form: finiteness, non-negativity for +/// novelty/surprise/coherence, and the closed-tolerance `[0, 1]` bound +/// for displacement. +fn assert_member_polarity(member: &MemberScore) { + assert!(member.cell_start < member.cell_end, "member score identifies a real cell"); + + for (name, value) in [ + ("novelty", member.novelty), + ("displacement", member.displacement), + ("surprise", member.surprise), + ("coherence", member.coherence), + ("novelty_z", member.novelty_z), + ("displacement_z", member.displacement_z), + ("surprise_z", member.surprise_z), + ("coherence_z", member.coherence_z), + ] { + assert!(value.is_finite(), "member.{name} must be finite"); + } + + assert!(member.novelty >= -1e-12, "member novelty must be non-negative"); + assert!( + member.displacement >= -1e-12 && member.displacement <= 1.0 + 1e-12, + "member displacement is bounded in [0, 1] (closed under float tolerance)" + ); + assert!(member.surprise >= -1e-12, "member surprise must be non-negative"); + assert!(member.coherence >= -1e-12, "member coherence must be non-negative"); +} + +fn assert_cell_report_contract(cell: &CellReport, config: &SentinelConfig) { + assert!(cell.start < cell.end, "cell interval must be non-empty"); + assert_eq!( + cell.analysis_width, + 128 - cell.depth as usize, + "analysis width is the suffix width N - depth" + ); + assert_eq!(cell.geometry.dim, cell.analysis_width, "geometry dim tracks suffix width"); + assert_eq!( + cell.geometry.cap, + cell.analysis_width.min(config.max_rank), + "rank cap is min(width, max_rank)" + ); + assert!(cell.rank >= 1, "trackers keep at least one active basis vector"); + assert!(cell.rank <= cell.geometry.cap, "rank must respect the geometry cap"); + assert_eq!( + cell.geometry.residual_dof, + cell.geometry.dim - cell.rank, + "residual degrees of freedom are dim - rank" + ); + assert!(cell.sample_count > 0, "reported cells received observations this batch"); + assert!( + cell.maturity.total_observations() > 0, + "reported cells have a maturity history" + ); + assert!( + (0.0..=1.0).contains(&cell.maturity.noise_influence), + "noise influence is a fraction" + ); + + if config.per_sample_scores { + let per_sample = cell.per_sample.as_ref().expect("per-sample scores are enabled"); + assert_eq!(per_sample.len(), cell.sample_count, "one sample score per routed observation"); + } else { + assert!(cell.per_sample.is_none(), "per-sample scores are disabled"); + } + + assert_scores_are_measurements(&cell.scores); +} + +fn assert_coordination_contract(coordination: &CoordinationReport, config: &SentinelConfig) { + assert!( + coordination.start < coordination.end, + "coordination interval must be non-empty" + ); + assert!( + coordination.cells_reporting >= 2, + "coordination needs cells from both subtrees" + ); + assert_eq!( + coordination.geometry.dim, 4, + "coordination trackers operate on four score axes" + ); + assert_eq!( + coordination.geometry.cap, + config.max_rank.min(4), + "coordination rank cap is min(4, max_rank)" + ); + assert!(coordination.rank >= 1, "coordination rank must be at least one"); + assert!( + coordination.rank <= coordination.geometry.cap, + "coordination rank respects cap" + ); + assert_eq!( + coordination.geometry.residual_dof, + coordination.geometry.dim - coordination.rank, + "coordination residual DOF is dim - rank" + ); + + if config.per_sample_scores { + let members = coordination.per_member.as_ref().expect("per-member scores are enabled"); + assert_eq!( + members.len(), + coordination.cells_reporting, + "one member score per contributing cell" + ); + for member in members { + assert_member_polarity(member); + } + } + + assert_scores_are_measurements(&coordination.scores); +} + +fn assert_report_contract(sentinel: &Sentinel128, report: &BatchReport, batch_size: usize) { + let config = sentinel.config(); + assert_invariants(sentinel, report); + + assert_eq!( + report.health.lifetime_observations, + sentinel.lifetime_observations(), + "inline health mirrors the sentinel counter" + ); + assert_eq!( + report.health.cells_tracked, + sentinel.cells_tracked(), + "inline health mirrors active cells" + ); + assert_eq!( + report.analysis_set_summary.competitive_size, + sentinel.analysis_set().competitive_count(), + "summary mirrors the current producing competitive set" + ); + assert_eq!( + report.analysis_set_summary.full_size, + sentinel.analysis_set().total_count(), + "summary mirrors the current producing full set" + ); + assert!( + report.analysis_set_summary.competitive_size <= config.analysis_k, + "competitive analysis set respects K" + ); + assert!( + report.analysis_set_summary.investment_set_size >= report.analysis_set_summary.full_size, + "investment set covers online plus warming cells" + ); + assert_eq!( + report.analysis_set_summary.depth_range.0, 0, + "depth range starts at the root: the producing set is anchored at \ + depth 0 via ancestor closure (§ALGO S-8.2)" + ); + + for cell in &report.cell_reports { + assert!(cell.is_competitive, "cell_reports are the producing competitive set"); + assert_cell_report_contract(cell, config); + } + for cell in &report.ancestor_reports { + assert!(!cell.is_competitive, "ancestor_reports are ancestor-only cells"); + assert_cell_report_contract(cell, config); + } + for coordination in &report.coordination_reports { + assert_coordination_contract(coordination, config); + } + + let root = root_ancestor(report); + assert_eq!(root.sample_count, batch_size, "root receives every observation in the batch"); +} + +fn mixed_normal_batch() -> Vec { + [ + cell_values(0x1, 8), + cell_values(0x5, 8), + cell_values(0xA, 8), + cell_values(0xF, 8), + ] + .concat() +} + +/// Maximum number of multi-region batches Step 3 will ingest while +/// waiting for hierarchical coordination to activate. Far above the +/// typical activation round under [`pedagogy_config`]; a panic at this +/// bound indicates a real regression, not flakiness. +const MULTISCALE_MAX_ROUNDS: usize = 30; + +/// Function-pointer alias used in Step 4 to project an [`AnomalyScores`] +/// value onto one of its four axis distributions without repeating the +/// full type signature. +type ScoreAxis = fn(&AnomalyScores) -> &ScoreDistribution; + +// ======================================================================== +// STEP 0: CONSTRUCTION AND AUTOMATIC ROOT WARM-UP +// §ALGO S-11, ADR-S-007, ADR-S-001 +// ======================================================================== + +/// A fresh Sentinel has one spatial G-node and one permanent root tracker. +/// +/// The root tracker is warmed automatically with synthetic noise. Noise is +/// tracker-local: it does not count as a real observation and does not touch +/// the spatial G-V Graph. This is the first visible consequence of the +/// feed-forward design. +fn step_0_fresh(sentinel: &Sentinel128) { + heading("Step 0: Construction and Automatic Root Warm-Up"); + println!(" (§ALGO S-11, ADR-S-007, ADR-S-001)"); + + assert_eq!(sentinel.graph().node_count(), 1, "fresh graph has one G-node"); + assert_eq!(sentinel.graph().terminal_count(), 1, "fresh graph has one terminal cell"); + assert_eq!(sentinel.graph().total_sum(), 0, "fresh graph has no real volume"); + assert_eq!(sentinel.cells_tracked(), 1, "only the root tracker is active"); + assert_eq!(sentinel.lifetime_observations(), 0, "noise is not real traffic"); + assert_eq!(sentinel.analysis_set().competitive_count(), 0, "root is never competitive"); + assert_eq!( + sentinel.analysis_set().total_count(), + 1, + "full analysis set contains the root" + ); + + let root = sentinel.graph().g_root(); + let root_cell = sentinel.inspect_cell(root).expect("root cell is always inspectable"); + assert_eq!(root_cell.depth, 0); + assert_eq!(root_cell.analysis_width, 128); + assert!(!root_cell.is_competitive, "root provides context, not a competitive target"); + assert!( + root_cell.maturity.noise_observations > 0, + "root tracker is noise-warmed at construction" + ); + assert_eq!(root_cell.maturity.real_observations, 0, "root has no real observations yet"); + + let health = sentinel.health(); + assert_eq!(health.active_trackers, 1); + assert_eq!(health.active_coordination_contexts, 0, "coordination is demand-driven"); + assert_eq!( + health.maturity_distribution.cold_trackers, 1, + "noise-only trackers are still cold" + ); + + println!("Fresh state:"); + println!(" spatial graph: one root, total importance = 0"); + println!(" analysis set: root only, no competitive cells yet"); + println!(" root tracker: warmed with synthetic noise, real observations = 0"); + println!(" design point: the Sentinel measures; the host decides"); +} + +// ======================================================================== +// STEP 1: FIRST REAL BATCH -- FEED-FORWARD OBSERVATION +// §ALGO S-9.1, ADR-S-002 +// ======================================================================== + +/// The first real batch proves the feed-forward invariant at the public +/// surface: the graph's total importance and the lifetime observation +/// counter increase by exactly the number of submitted values. A second +/// probe of the same length but very different bit structure produces +/// the same accounting delta, showing that score content does not feed +/// back into spatial importance. +fn step_1_first_batch(sentinel: &mut Sentinel128) { + heading("Step 1: First Real Batch -- Feed-Forward Observation"); + println!(" (§ALGO S-9.1, ADR-S-002)"); + + // Probe A: a structurally simple batch. + let values = cell_values(0xA, 8); + let before_sum = sentinel.graph().total_sum(); + let report = sentinel.ingest(&values); + + assert_eq!(sentinel.graph().total_sum(), before_sum + values.len() as u64); + assert_eq!(sentinel.lifetime_observations(), values.len() as u64); + assert_report_contract(sentinel, &report, values.len()); + + let root = root_ancestor(&report); + let probe_a_novelty_z = max_novelty_z(&report); + subheading("Root tracker report"); + println!(" sample_count = {}", root.sample_count); + println!(" novelty mean = {:.6}", root.scores.novelty.mean); + println!(" displacement mean = {:.6}", root.scores.displacement.mean); + println!(" surprise mean = {:.6}", root.scores.surprise.mean); + println!(" coherence mean = {:.6}", root.scores.coherence.mean); + + // Probe B: a same-sized but structurally very different batch. + // + // The feed-forward invariant says the graph receives Delta = 1 per + // input value, regardless of what scores those values produce. We + // verify it directly: submit a structurally anomalous batch of the + // same length and assert the same spatial accounting delta. The two + // batches will differ in score magnitudes; they must not differ in + // their effect on graph importance per input value. + let sum_before_b = sentinel.graph().total_sum(); + let obs_before_b = sentinel.lifetime_observations(); + let anomalous = anomalous_values(0xA, values.len()); + let anomaly_report = sentinel.ingest(&anomalous); + + assert_eq!( + sentinel.graph().total_sum() - sum_before_b, + values.len() as u64, + "graph importance increments by batch length irrespective of score content" + ); + assert_eq!( + sentinel.lifetime_observations() - obs_before_b, + values.len() as u64, + "lifetime observations count input values, not score magnitudes" + ); + assert_report_contract(sentinel, &anomaly_report, anomalous.len()); + + subheading("Same volume, different content"); + println!( + " probe A: {} values, max novelty z = {:.6}, graph delta = {}", + values.len(), + probe_a_novelty_z, + values.len(), + ); + println!( + " probe B: {} values, max novelty z = {:.6}, graph delta = {}", + anomalous.len(), + max_novelty_z(&anomaly_report), + anomalous.len(), + ); + println!(" -> identical graph deltas confirm scores do not feed back into importance"); + + println!(); + println!("What happened:"); + println!(" 1. Every coordinate was observed by the G-V Graph with Delta = 1."); + println!(" 2. Two batches of equal length produced the same spatial accounting"); + println!(" delta despite very different score profiles."); + println!(" 3. The report contains raw measurements, not a verdict or action."); +} + +// ======================================================================== +// STEP 2: CONCENTRATED TRAFFIC CREATES SPATIAL STRUCTURE +// §ALGO S-3, §ALGO S-8, ADR-S-006 +// ======================================================================== + +/// Concentrated traffic crosses the spatial split threshold. The graph +/// refines under volume alone, then the analysis selector recomputes the +/// competitive targets and closes them under ancestry. +fn step_2_concentrated_traffic(sentinel: &mut Sentinel128) { + heading("Step 2: Concentrated Traffic -- Spatial Refinement"); + println!(" (§ALGO S-3, §ALGO S-8, ADR-S-006)"); + + let values = cell_values(0xA, 40); + let nodes_before = sentinel.graph().node_count(); + let terminals_before = sentinel.graph().terminal_count(); + let sum_before = sentinel.graph().total_sum(); + + let report = sentinel.ingest(&values); + + assert_eq!(sentinel.graph().total_sum(), sum_before + values.len() as u64); + assert!( + sentinel.graph().node_count() > nodes_before, + "concentrated traffic should split G-nodes" + ); + assert!( + sentinel.graph().terminal_count() > terminals_before, + "spatial contour should gain terminal cells" + ); + assert!( + report.contour.splits_since_last_report > 0, + "report exposes the structural split event" + ); + assert!( + report.analysis_set_summary.competitive_size > 0, + "non-root cells become competitive targets" + ); + assert!( + report.cell_reports.iter().any(|cell| cell.depth > 0), + "at least one non-root competitive cell reports" + ); + assert_report_contract(sentinel, &report, values.len()); + + subheading("Contour snapshot"); + println!(" plateaus = {}", report.contour.plateau_count); + println!(" terminal cells = {}", report.contour.cell_count); + println!(" total importance = {:.0}", report.contour.total_importance); + println!(" splits since previous report = {}", report.contour.splits_since_last_report); + + subheading("Analysis set summary"); + println!(" competitive size = {}", report.analysis_set_summary.competitive_size); + println!(" full size = {}", report.analysis_set_summary.full_size); + println!(" investment size = {}", report.analysis_set_summary.investment_set_size); + println!(" depth range = {:?}", report.analysis_set_summary.depth_range); + + println!(); + println!("Key point: the analysis tier follows spatial volume; it does not steer it."); +} + +// ======================================================================== +// STEP 3: MULTI-SCALE REPORTS AND HIERARCHICAL COORDINATION +// §ALGO S-8.2, §ALGO S-9, ADR-S-019 +// ======================================================================== + +/// Multi-region traffic creates multiple competitive cells. Their G-tree +/// ancestors provide shared context, and internal nodes whose left and +/// right subtrees both contribute competitive scores can emit coordination +/// reports. +fn step_3_multiscale_reports(sentinel: &mut Sentinel128) { + heading("Step 3: Multi-Scale Reports and Coordination"); + println!(" (§ALGO S-8.2, §ALGO S-9, ADR-S-019)"); + + let values = mixed_normal_batch(); + let mut report = sentinel.ingest(&values); + assert_report_contract(sentinel, &report, values.len()); + + let mut activated_at: Option = None; + for round in 1..=MULTISCALE_MAX_ROUNDS { + if !report.coordination_reports.is_empty() && report.cell_reports.len() >= 2 { + activated_at = Some(round); + break; + } + report = sentinel.ingest(&values); + assert_report_contract(sentinel, &report, values.len()); + } + + let activated_at = activated_at.unwrap_or_else(|| { + panic!( + "coordination did not activate within {MULTISCALE_MAX_ROUNDS} multi-region batches: \ + cell_reports={}, coordination_reports={}", + report.cell_reports.len(), + report.coordination_reports.len(), + ) + }); + println!("Coordination activated after {activated_at} multi-region batch(es)."); + + assert!( + report.cell_reports.len() >= 2, + "multi-region traffic should produce several competitive reports" + ); + assert!( + !report.coordination_reports.is_empty(), + "competitive cells in both subtrees should activate coordination" + ); + + let depths: std::collections::BTreeSet<_> = all_cell_reports(&report).map(|cell| cell.depth).collect(); + assert!(depths.contains(&0), "combined reports include the root depth"); + assert!(depths.len() >= 2, "combined reports show at least two spatial scales"); + + subheading("Competitive reports"); + for cell in &report.cell_reports { + println!( + " GNode {:?}: [{:#034x}, {:#034x}) depth={} width={} samples={}", + cell.gnode_id, cell.start, cell.end, cell.depth, cell.analysis_width, cell.sample_count + ); + } + + subheading("Ancestor reports"); + for cell in &report.ancestor_reports { + println!( + " GNode {:?}: [{:#034x}, {:#034x}) depth={} width={} samples={}", + cell.gnode_id, cell.start, cell.end, cell.depth, cell.analysis_width, cell.sample_count + ); + } + + subheading("Coordination reports"); + for coordination in &report.coordination_reports { + println!( + " GNode {:?}: depth={} cells_reporting={} novelty mean={:.6}", + coordination.gnode_id, coordination.depth, coordination.cells_reporting, coordination.scores.novelty.mean + ); + } + + println!(); + println!("Key point: cell reports are local, ancestors are multi-scale context,"); + println!("and coordination reports measure cross-cell score patterns."); +} + +// ======================================================================== +// STEP 4: SCORES ARE MEASUREMENTS, NOT OPINIONS +// §ALGO S-6, ADR-S-001 +// ======================================================================== + +/// The four score axes share a uniform polarity: higher means more +/// anomalous. This step uses a fresh, focused scoring probe so the +/// comparison is about the score contract rather than the long-running +/// narrative sentinel's mixed traffic history. Novelty is the strongest +/// indicator for this kind of structural anomaly and is asserted +/// strictly; the other three axes are reported for inspection. +fn step_4_scores_are_measurements() { + heading("Step 4: Scores Are Measurements, Not Opinions"); + println!(" (§ALGO S-6, ADR-S-001)"); + + let mut scoring_config = pedagogy_config(); + scoring_config.noise_seed = Some(42); + let mut scoring_sentinel = Sentinel128::new(scoring_config).expect("valid scoring config constructs"); + + let seed = cell_values(0xA, 20); + let seed_report = scoring_sentinel.ingest(&seed); + assert_report_contract(&scoring_sentinel, &seed_report, seed.len()); + for _ in 0..5 { + let warm_report = scoring_sentinel.ingest(&seed); + assert_report_contract(&scoring_sentinel, &warm_report, seed.len()); + } + + let normal = cell_values(0xA, 8); + let normal_report = scoring_sentinel.ingest(&normal); + assert_report_contract(&scoring_sentinel, &normal_report, normal.len()); + + let anomaly = anomalous_values(0xA, 8); + let anomaly_report = scoring_sentinel.ingest(&anomaly); + assert_report_contract(&scoring_sentinel, &anomaly_report, anomaly.len()); + + let normal_z = max_novelty_z(&normal_report); + let anomaly_z = max_novelty_z(&anomaly_report); + assert!( + anomaly_z > normal_z, + "structurally dense batch should elevate novelty: anomaly={anomaly_z:.6}, normal={normal_z:.6}" + ); + + subheading("Novelty comparison (asserted)"); + println!(" normal max novelty z = {normal_z:.6}"); + println!(" anomalous max novelty z = {anomaly_z:.6}"); + + // The other three axes share the same *raw-score* polarity invariant + // (higher means more anomalous departure), but the *z-scores* below + // compare a batch against each cell's current baseline -- and the + // seed batch above shaped that baseline. A "normal" batch can + // therefore show higher z-scores on displacement, surprise, or + // coherence than the structurally anomalous batch when the + // anomalous batch happens to land closer to the seeded baseline + // along those axes. This is not a polarity violation -- both + // batches' raw scores are non-negative -- it is a reminder that + // z-scores are relative to whatever the cell has learned. We surface + // the numbers without asserting an ordering. + let max_z_per_axis = |report: &BatchReport, axis: ScoreAxis| -> f64 { + all_cell_reports(report) + .map(|cell| axis(&cell.scores).max_z_score) + .fold(0.0_f64, f64::max) + }; + let axes: &[(&str, ScoreAxis)] = &[ + ("displacement", |s| &s.displacement), + ("surprise", |s| &s.surprise), + ("coherence", |s| &s.coherence), + ]; + subheading("Other axes (reported, not asserted)"); + for (name, axis) in axes { + let normal_max = max_z_per_axis(&normal_report, *axis); + let anomaly_max = max_z_per_axis(&anomaly_report, *axis); + println!(" {name:<13} normal max z = {normal_max:>10.6} anomalous max z = {anomaly_max:>10.6}"); + } + + subheading("Score-axis contract"); + println!(" novelty: residual energy per residual degree of freedom, non-negative"); + println!(" displacement: bounded cell displacement, in [0, 1)"); + println!(" surprise: diagonal latent deviation, non-negative"); + println!(" coherence: off-diagonal latent deviation, non-negative"); + + println!(); + println!("There is no alert threshold here. The host reads these measurements"); + println!("and decides what, if anything, they mean in its own domain."); +} + +// ======================================================================== +// STEP 5: HOST-CONTROLLED TEMPORAL POLICY +// §ALGO S-10, §ALGO S-13.4, ADR-S-002 +// ======================================================================== + +/// Decay is the host's temporal policy hook. The call itself rescales +/// spatial importance in the G-V Graph; it does not count as real +/// traffic, and it does not directly destroy trackers. Tracker churn +/// happens later, through normal selection reconciliation when cells +/// lose enough standing to drop out of the competitive top-K. +fn step_5_host_controlled_decay(sentinel: &mut Sentinel128) { + heading("Step 5: Host-Controlled Decay"); + println!(" (§ALGO S-10, §ALGO S-13.4, ADR-S-002)"); + + let sum_before = sentinel.graph().total_sum(); + let observations_before = sentinel.lifetime_observations(); + let trackers_before = sentinel.cells_tracked(); + + sentinel.decay(0.5, 0.0); + + let sum_after_decay = sentinel.graph().total_sum(); + let trackers_after_decay = sentinel.cells_tracked(); + assert!( + sum_after_decay < sum_before, + "uniform attenuation should reduce spatial importance" + ); + assert_eq!( + sentinel.lifetime_observations(), + observations_before, + "decay is not real traffic" + ); + assert_eq!( + trackers_after_decay, trackers_before, + "decay() itself does not destroy trackers; lifecycle changes happen via later reconciliation" + ); + + let values = cell_values(0xF, 8); + let report = sentinel.ingest(&values); + let trackers_after_ingest = sentinel.cells_tracked(); + assert_eq!(sentinel.graph().total_sum(), sum_after_decay + values.len() as u64); + assert_eq!(sentinel.lifetime_observations(), observations_before + values.len() as u64); + assert_report_contract(sentinel, &report, values.len()); + + subheading("Decay effect"); + println!(" total importance: before decay = {sum_before}, after decay = {sum_after_decay}"); + println!( + " active trackers: before decay = {trackers_before}, after decay = {trackers_after_decay} \ + (unchanged), after subsequent ingest = {trackers_after_ingest}" + ); + println!( + " lifetime observations: before = {observations_before}, after ingest = {}", + sentinel.lifetime_observations() + ); + + println!(); + println!("Key point: decay rescales spatial weight; selection reconciliation"); + println!("decides downstream tracker lifecycle. Feedback, when desired,"); + println!("belongs in host policy."); +} + +// ======================================================================== +// STEP 6: RESET +// Public API lifecycle contract +// ======================================================================== + +/// Reset clears the spatial graph, trackers, coordination contexts, and +/// counters, then recreates the warmed root according to the same config. +fn step_6_reset(sentinel: &mut Sentinel128) { + heading("Step 6: Reset Restores the Fresh Lifecycle"); + + assert!( + sentinel.graph().total_sum() > 0, + "the narrative produced real traffic before reset" + ); + assert!( + sentinel.lifetime_observations() > 0, + "the narrative counted real observations before reset" + ); + + sentinel.reset(); + + assert_eq!(sentinel.graph().node_count(), 1); + assert_eq!(sentinel.graph().terminal_count(), 1); + assert_eq!(sentinel.graph().total_sum(), 0); + assert_eq!(sentinel.cells_tracked(), 1); + assert_eq!(sentinel.lifetime_observations(), 0); + assert_eq!(sentinel.health().active_coordination_contexts, 0); + + let root = sentinel.graph().g_root(); + let root_cell = sentinel.inspect_cell(root).expect("root is recreated on reset"); + assert_eq!(root_cell.depth, 0); + assert_eq!(root_cell.analysis_width, 128); + assert!(root_cell.maturity.noise_observations > 0, "reset recreates the warmed root"); + + println!("After reset:"); + println!(" graph: one root, total importance = 0"); + println!(" trackers: root only"); + println!(" coordination contexts: none"); + println!(" root warm-up: reapplied from the configured noise schedule"); +} + +// ======================================================================== +// MAIN +// ======================================================================== + +fn main() { + // Support `--list --format terse` so cargo-nextest can enumerate this + // custom-harness test binary. With `--ignored`, output nothing. + let args: Vec = std::env::args().collect(); + if args.iter().any(|arg| arg == "--list") { + if !args.iter().any(|arg| arg == "--ignored") { + println!("pedagogy: test"); + } + return; + } + + let config = pedagogy_config(); + config.validate().expect("pedagogy config must be valid"); + + let mut sentinel = Sentinel128::new(config).expect("valid config constructs a Sentinel"); + + step_0_fresh(&sentinel); + step_1_first_batch(&mut sentinel); + step_2_concentrated_traffic(&mut sentinel); + step_3_multiscale_reports(&mut sentinel); + step_4_scores_are_measurements(); + step_5_host_controlled_decay(&mut sentinel); + step_6_reset(&mut sentinel); + + heading("All Claims Verified"); + println!(); + println!(" Feed-forward invariant:"); + println!(" [ok] graph importance increases by one per input value"); + println!(" [ok] same-length batches produce identical spatial accounting"); + println!(" deltas regardless of score content"); + println!(); + println!(" Analysis lifecycle:"); + println!(" [ok] automatic noise warm-up is tracker-local"); + println!(" [ok] volume-driven spatial splits create competitive analysis cells"); + println!(" [ok] ancestors keep a complete multi-scale chain to the root"); + println!(" [ok] coordination activates from cross-cell score patterns"); + println!(); + println!(" Reporting principle:"); + println!(" [ok] report values are raw, finite statistical measurements"); + println!(" [ok] no test asserts a policy verdict or host action"); + println!(); + println!(" Host-controlled lifecycle:"); + println!(" [ok] decay changes spatial importance without counting traffic"); + println!(" [ok] reset restores a fresh graph and warmed root tracker"); +} diff --git a/packages/sentinel/tests/pedagogy_advanced.rs b/packages/sentinel/tests/pedagogy_advanced.rs new file mode 100644 index 000000000..4fc6213fd --- /dev/null +++ b/packages/sentinel/tests/pedagogy_advanced.rs @@ -0,0 +1,857 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +#![allow(clippy::print_stdout)] + +//! # Advanced Pedagogy Test — Companion to [`pedagogy.rs`] +//! +//! The basic pedagogy test walks through the Sentinel **lifecycle +//! surface**: construction, feed-forward observation, spatial refinement, +//! coordination, host decay, and reset. This companion test exercises the +//! **inspection surface** — the readouts, summaries, and cross-checks a +//! host uses after the lifecycle has produced a non-trivial model. +//! +//! ## Why a separate test? +//! +//! The Sentinel deliberately separates measurement from policy. The basic +//! test shows how measurements are produced. This test shows how to read +//! them without smuggling in decisions: analysis-set membership, ancestor +//! closure, per-sample payloads, geometry flags, coordination contexts, +//! scale profiles, and temporal separation between spatial weight and +//! tracker state. +//! +//! ## What this test demonstrates +//! +//! ### Inspection-oriented configuration (Step 0) +//! +//! The test uses deterministic foreground warm-up so every tracker created +//! by investment reconciliation is online in the same `ingest()` call. That +//! keeps the inspection surface stable: `investment_set_size == full_size` +//! once the graph has settled, and no asynchronous warm-up race can obscure +//! the fields being demonstrated. +//! +//! ### Analysis-set anatomy (Step 1) +//! +//! The public [`AnalysisSet`] is checked against the report summary. Every +//! competitive cell's G-tree parent chain is present in the full set, and +//! the root is permanent context, never a competitive target. +//! +//! ### Report and inspection consistency (Steps 2-3) +//! +//! [`CellReport`] values are cross-checked with `inspect_cell()`. Optional +//! per-sample payloads are verified as raw finite measurements whose counts +//! match the routing counts. +//! +//! ### Geometry edge flags (Step 4) +//! +//! The scoring geometry record is not decoration. It tells the host when an +//! axis is structurally inactive or saturable. A rank-one configuration +//! demonstrates coherence inactivity; a tiny four-bit Sentinel demonstrates +//! novelty-saturability without relying on fragile score values. +//! +//! ### Coordination read surface (Step 5) +//! +//! Coordination contexts are inspected as reports over competitive-cell +//! score vectors. Per-member records are verified against the contributing +//! competitive cells. +//! +//! ### Scale profile and temporal separation (Steps 6-7) +//! +//! The same batch yields measurements at several depths; the test prints a +//! depth-indexed novelty profile without asserting a policy verdict. Then +//! host-controlled decay is shown to rescale spatial importance without +//! mutating tracker baselines or counting traffic. +//! +//! ### Reset read surface (Step 8) +//! +//! Reset drops learned state and recreates the warmed root, clearing the +//! inspection surfaces back to the fresh lifecycle. +//! +//! Run with: +//! +//! ```sh +//! cargo test -p torrust-sentinel --test pedagogy_advanced +//! ``` +//! +//! # Test Index +//! +//! ## Inspection Setup (§ALGO S-11, ADR-S-019) +//! +//! | Step | Function | What it teaches | +//! |------|----------|-----------------| +//! | 0 | [`step_0_inspection_setup`] | Foreground warm-up gives deterministic readouts | +//! +//! ## Analysis-Set Read Surface (§ALGO S-8, ADR-S-019) +//! +//! | Step | Function | What it teaches | +//! |------|----------|-----------------| +//! | 1 | [`step_1_analysis_set_anatomy`] | Competitive targets plus ancestors form the full set | +//! | 2 | [`step_2_inspection_mirrors_reports`] | `inspect_cell()` and batch reports expose the same tracker facts | +//! +//! ## Report Payloads and Geometry (§ALGO S-14) +//! +//! | Step | Function | What it teaches | +//! |------|----------|-----------------| +//! | 3 | [`step_3_payload_records`] | Per-sample payloads are count-aligned raw measurements | +//! | 4 | [`step_4_scoring_geometry_edges`] | Geometry flags explain inactive and saturable axes | +//! +//! ## Coordination and Scale (§ALGO S-7, §ALGO S-9) +//! +//! | Step | Function | What it teaches | +//! |------|----------|-----------------| +//! | 5 | [`step_5_coordination_read_surface`] | Coordination reports measure cross-cell score patterns | +//! | 6 | [`step_6_scale_profile`] | Depth-indexed scores are diagnostic measurements, not verdicts | +//! +//! ## Host Policy and Lifecycle (§ALGO S-10, ADR-S-002) +//! +//! | Step | Function | What it teaches | +//! |------|----------|-----------------| +//! | 7 | [`step_7_temporal_separation`] | Spatial decay does not mutate tracker baselines | +//! | 8 | [`step_8_reset_read_surface`] | Reset clears readouts and recreates the warmed root | + +mod common; + +use std::collections::{BTreeMap, BTreeSet}; + +use common::{anomalous_values, assert_invariants, cell_values}; +use torrust_sentinel::{ + AxisBaselineSnapshots, BaselineSnapshot, BatchReport, CellInspection, CellReport, CoordinationReport, MemberScore, + NoiseSchedule, SampleScore, ScoreDistribution, Sentinel128, SentinelConfig, SpectralSentinel, SvdStrategy, +}; + +// -- Configuration --------------------------------------------------------- + +/// Deterministic configuration tuned for inspection rather than suspense. +/// +/// Compared with the basic pedagogy test, this uses a slightly larger +/// analysis budget and rank cap so the read surface has more structure to +/// inspect. Background warming stays disabled: this file teaches the public +/// readouts, not scheduler timing. +fn advanced_config() -> SentinelConfig { + SentinelConfig:: { + max_rank: 4, + forgetting_factor: 0.90, + rank_update_interval: 4, + analysis_k: 24, + analysis_depth_cutoff: 6, + energy_threshold: 0.90, + eps: 1e-6, + per_sample_scores: true, + cusum_allowance_sigmas: 0.5, + cusum_slow_decay: 0.99, + cusum_coord_slow_decay: 0.99, + clip_sigmas: 3.0, + clip_pressure_decay: 0.95, + split_threshold: 8, + d_create: 3, + d_evict: 6, + budget: 100_000, + noise_schedule: NoiseSchedule::Explicit(vec![6]), + noise_batch_size: 4, + noise_seed: Some(2026), + background_warming: false, + svd_strategy: SvdStrategy::Brand, + } +} + +fn rank_one_config() -> SentinelConfig { + SentinelConfig:: { + max_rank: 1, + noise_seed: Some(7), + ..advanced_config() + } +} + +fn tiny_geometry_config() -> SentinelConfig { + SentinelConfig:: { + max_rank: 4, + rank_update_interval: 1, + noise_seed: Some(11), + ..advanced_config() + } +} + +// -- Helpers --------------------------------------------------------------- + +const MULTISCALE_MAX_ROUNDS: usize = 40; + +type TinySentinel = SpectralSentinel; + +fn heading(s: &str) { + let rule = "=".repeat(72); + println!("\n{rule}"); + println!(" {s}"); + println!("{rule}"); +} + +fn subheading(s: &str) { + println!("\n-- {s} --"); +} + +fn mixed_normal_batch() -> Vec { + [ + cell_values(0x1, 8), + cell_values(0x5, 8), + cell_values(0xA, 8), + cell_values(0xF, 8), + ] + .concat() +} + +fn all_cell_reports(report: &BatchReport) -> impl Iterator> { + report.cell_reports.iter().chain(report.ancestor_reports.iter()) +} + +fn root_report(report: &BatchReport) -> &CellReport { + report + .ancestor_reports + .iter() + .find(|cell| cell.depth == 0) + .expect("non-empty batch reports include the root ancestor") +} + +fn assert_score_distribution(name: &str, score: &ScoreDistribution) { + for (field, value) in [ + ("min", score.min), + ("max", score.max), + ("mean", score.mean), + ("max_z_score", score.max_z_score), + ("mean_z_score", score.mean_z_score), + ("baseline.mean", score.baseline.mean), + ("baseline.variance", score.baseline.variance), + ("cusum.accumulator", score.cusum.accumulator), + ("cusum.slow_baseline.mean", score.cusum.slow_baseline.mean), + ("cusum.slow_baseline.variance", score.cusum.slow_baseline.variance), + ("clip_pressure", score.clip_pressure), + ] { + assert!(value.is_finite(), "{name}.{field} must be finite"); + } + + assert!(score.max + 1e-12 >= score.min, "{name}.max must be >= min"); + assert!(score.mean + 1e-12 >= score.min, "{name}.mean must be >= min"); + assert!(score.mean <= score.max + 1e-12, "{name}.mean must be <= max"); + assert!(score.baseline.variance >= -1e-12, "{name}.baseline variance is non-negative"); + assert!( + score.cusum.slow_baseline.variance >= -1e-12, + "{name}.slow baseline variance is non-negative" + ); + assert!(score.cusum.accumulator >= -1e-12, "{name}.CUSUM is one-sided"); + assert!( + (-1e-12..=1.0 + 1e-12).contains(&score.clip_pressure), + "{name}.clip pressure is a fraction" + ); +} + +fn assert_cell_scores(cell: &CellReport) { + assert_score_distribution("novelty", &cell.scores.novelty); + assert_score_distribution("displacement", &cell.scores.displacement); + assert_score_distribution("surprise", &cell.scores.surprise); + assert_score_distribution("coherence", &cell.scores.coherence); + + assert!(cell.scores.novelty.min >= -1e-12, "novelty is non-negative"); + assert!( + cell.scores.displacement.min >= -1e-12 && cell.scores.displacement.max <= 1.0 + 1e-12, + "displacement is bounded in [0, 1]" + ); + assert!(cell.scores.surprise.min >= -1e-12, "surprise is non-negative"); + assert!(cell.scores.coherence.min >= -1e-12, "coherence is non-negative"); +} + +fn assert_sample_score(sample: &SampleScore) { + for (name, value) in [ + ("novelty", sample.novelty), + ("displacement", sample.displacement), + ("surprise", sample.surprise), + ("coherence", sample.coherence), + ("novelty_z", sample.novelty_z), + ("displacement_z", sample.displacement_z), + ("surprise_z", sample.surprise_z), + ("coherence_z", sample.coherence_z), + ] { + assert!(value.is_finite(), "sample.{name} must be finite"); + } + + assert!(sample.novelty >= -1e-12, "sample novelty is non-negative"); + assert!( + sample.displacement >= -1e-12 && sample.displacement <= 1.0 + 1e-12, + "sample displacement is bounded in [0, 1]" + ); + assert!(sample.surprise >= -1e-12, "sample surprise is non-negative"); + assert!(sample.coherence >= -1e-12, "sample coherence is non-negative"); +} + +fn assert_member_score(member: &MemberScore) { + assert!(member.cell_start < member.cell_end, "member cell interval is non-empty"); + for (name, value) in [ + ("novelty", member.novelty), + ("displacement", member.displacement), + ("surprise", member.surprise), + ("coherence", member.coherence), + ("novelty_z", member.novelty_z), + ("displacement_z", member.displacement_z), + ("surprise_z", member.surprise_z), + ("coherence_z", member.coherence_z), + ] { + assert!(value.is_finite(), "member.{name} must be finite"); + } + assert!(member.novelty >= -1e-12, "member novelty is non-negative"); + assert!( + member.displacement >= -1e-12 && member.displacement <= 1.0 + 1e-12, + "member displacement is bounded in [0, 1]" + ); + assert!(member.surprise >= -1e-12, "member surprise is non-negative"); + assert!(member.coherence >= -1e-12, "member coherence is non-negative"); +} + +fn assert_cell_geometry(cell: &CellReport, config: &SentinelConfig) { + assert_eq!(cell.analysis_width, 128 - cell.depth as usize); + assert_eq!(cell.geometry.dim, cell.analysis_width); + assert_eq!(cell.geometry.cap, cell.analysis_width.min(config.max_rank)); + assert!(cell.rank >= 1); + assert!(cell.rank <= cell.geometry.cap); + assert_eq!(cell.geometry.residual_dof, cell.geometry.dim - cell.rank); +} + +fn assert_report_surface(sentinel: &Sentinel128, report: &BatchReport, batch_size: usize) { + let config = sentinel.config(); + + assert_invariants(sentinel, report); + assert_eq!(report.health.lifetime_observations, sentinel.lifetime_observations()); + assert_eq!(report.health.cells_tracked, sentinel.cells_tracked()); + assert_eq!(report.health.active_trackers, sentinel.cell_gnodes().len()); + assert_eq!( + report.analysis_set_summary.competitive_size, + sentinel.analysis_set().competitive_count() + ); + assert_eq!(report.analysis_set_summary.full_size, sentinel.analysis_set().total_count()); + assert!(report.analysis_set_summary.competitive_size <= config.analysis_k); + assert!(report.analysis_set_summary.investment_set_size >= report.analysis_set_summary.full_size); + + let root = root_report(report); + assert_eq!(root.sample_count, batch_size, "root receives the full ingestion batch"); + + for cell in &report.cell_reports { + assert!(cell.is_competitive, "cell_reports are competitive reports"); + assert!(cell.sample_count > 0); + assert_cell_geometry(cell, config); + assert_cell_scores(cell); + } + for cell in &report.ancestor_reports { + assert!(!cell.is_competitive, "ancestor_reports are non-competitive reports"); + assert!(cell.sample_count > 0); + assert_cell_geometry(cell, config); + assert_cell_scores(cell); + } +} + +fn assert_inspection_matches_report(sentinel: &Sentinel128, report: &CellReport) { + let inspection = sentinel + .inspect_cell(report.gnode_id) + .expect("reported cells are inspectable after the batch"); + + assert_eq!(inspection.gnode_id, report.gnode_id); + assert_eq!((inspection.start, inspection.end), (report.start, report.end)); + assert_eq!(inspection.depth, report.depth); + assert_eq!(inspection.analysis_width, report.analysis_width); + assert_eq!(inspection.is_competitive, report.is_competitive); + assert_eq!(inspection.rank, report.rank); + assert_eq!(inspection.geometry.dim, report.geometry.dim); + assert_eq!(inspection.geometry.cap, report.geometry.cap); + assert_eq!(inspection.geometry.residual_dof, report.geometry.residual_dof); + assert_eq!(inspection.maturity.real_observations, report.maturity.real_observations); + assert_eq!(inspection.maturity.noise_observations, report.maturity.noise_observations); +} + +fn assert_coordination_report(coordination: &CoordinationReport, config: &SentinelConfig) { + assert!(coordination.start < coordination.end); + assert!(coordination.cells_reporting >= 2); + assert_eq!(coordination.geometry.dim, 4); + assert_eq!(coordination.geometry.cap, config.max_rank.min(4)); + assert!(coordination.rank >= 1); + assert!(coordination.rank <= coordination.geometry.cap); + assert_eq!( + coordination.geometry.residual_dof, + coordination.geometry.dim - coordination.rank + ); + assert_score_distribution("coord.novelty", &coordination.scores.novelty); + assert_score_distribution("coord.displacement", &coordination.scores.displacement); + assert_score_distribution("coord.surprise", &coordination.scores.surprise); + assert_score_distribution("coord.coherence", &coordination.scores.coherence); +} + +fn assert_baseline_same(name: &str, before: BaselineSnapshot, after: BaselineSnapshot) { + assert_eq!( + before.mean.to_bits(), + after.mean.to_bits(), + "{name} mean changed during spatial decay" + ); + assert_eq!( + before.variance.to_bits(), + after.variance.to_bits(), + "{name} variance changed during spatial decay" + ); +} + +fn assert_baselines_same(before: AxisBaselineSnapshots, after: AxisBaselineSnapshots) { + assert_baseline_same("novelty", before.novelty, after.novelty); + assert_baseline_same("displacement", before.displacement, after.displacement); + assert_baseline_same("surprise", before.surprise, after.surprise); + assert_baseline_same("coherence", before.coherence, after.coherence); +} + +fn drive_to_multiscale(sentinel: &mut Sentinel128) -> BatchReport { + let seed = cell_values(0xA, 16); + let seed_report = sentinel.ingest(&seed); + assert_report_surface(sentinel, &seed_report, seed.len()); + + let concentrated = cell_values(0xA, 48); + let concentrated_report = sentinel.ingest(&concentrated); + assert_report_surface(sentinel, &concentrated_report, concentrated.len()); + + let values = mixed_normal_batch(); + let mut report = sentinel.ingest(&values); + assert_report_surface(sentinel, &report, values.len()); + + for round in 1..=MULTISCALE_MAX_ROUNDS { + if report.cell_reports.len() >= 2 && !report.coordination_reports.is_empty() { + println!("Multiscale state reached after {round} mixed batch(es)."); + return report; + } + report = sentinel.ingest(&values); + assert_report_surface(sentinel, &report, values.len()); + } + + panic!( + "coordination did not activate within {MULTISCALE_MAX_ROUNDS} rounds: cell_reports={}, coordination_reports={}", + report.cell_reports.len(), + report.coordination_reports.len(), + ); +} + +// ======================================================================== +// STEP 0: INSPECTION-ORIENTED SETUP +// §ALGO S-11, ADR-S-019 +// ======================================================================== + +fn step_0_inspection_setup(sentinel: &Sentinel128) { + heading("Step 0: Inspection-Oriented Setup"); + println!(" (§ALGO S-11, ADR-S-019)"); + + let health = sentinel.health(); + assert_eq!(sentinel.graph().node_count(), 1); + assert_eq!(sentinel.graph().total_sum(), 0); + assert_eq!(sentinel.cells_tracked(), 1); + assert_eq!(sentinel.analysis_set().competitive_count(), 0); + assert_eq!(sentinel.analysis_set().total_count(), 1); + assert_eq!(health.warming_trackers, 0, "foreground warm-up leaves no staging backlog"); + assert_eq!(health.investment_set_size, 1); + + let root = sentinel + .inspect_cell(sentinel.graph().g_root()) + .expect("fresh Sentinel exposes its root tracker"); + assert_eq!(root.depth, 0); + assert_eq!(root.analysis_width, 128); + assert!(!root.is_competitive); + assert!(root.maturity.noise_observations > 0); + assert_eq!(root.maturity.real_observations, 0); + + println!("Fresh inspection surface:"); + println!(" graph nodes = {}", sentinel.graph().node_count()); + println!(" active trackers = {}", health.active_trackers); + println!(" warming trackers = {}", health.warming_trackers); + println!(" root noise observations = {}", root.maturity.noise_observations); + println!(" foreground warm-up makes tracker readouts immediately inspectable"); +} + +// ======================================================================== +// STEP 1: ANALYSIS SET ANATOMY +// §ALGO S-8, ADR-S-019 +// ======================================================================== + +fn step_1_analysis_set_anatomy(sentinel: &Sentinel128, report: &BatchReport) { + heading("Step 1: Analysis-Set Anatomy"); + println!(" (§ALGO S-8, ADR-S-019)"); + + let analysis = sentinel.analysis_set(); + let summary = report.analysis_set_summary; + + assert_eq!(analysis.competitive_count(), summary.competitive_size); + assert_eq!(analysis.total_count(), summary.full_size); + assert!(analysis.total_count() > analysis.competitive_count()); + + let root = sentinel.graph().g_root(); + assert!(analysis.contains(root), "root is always in the full analysis set"); + assert!( + !analysis.is_competitive(root), + "root provides context, not a competitive slot" + ); + + for entry in analysis.competitive() { + let mut current = Some(entry.gnode); + while let Some(gnode) = current { + assert!( + analysis.contains(gnode), + "ancestor closure missing GNode {:?} for competitive [{:#034x}, {:#034x})", + gnode, + entry.start, + entry.end, + ); + let info = sentinel.graph().gnode_info(gnode).expect("analysis cells are live G-nodes"); + current = info.parent; + } + } + + subheading("Set sizes"); + println!(" competitive targets = {}", analysis.competitive_count()); + println!(" full analysis set = {}", analysis.total_count()); + println!(" investment set = {}", summary.investment_set_size); + println!(" depth range = {:?}", summary.depth_range); + + subheading("Competitive cells"); + for entry in analysis.competitive() { + println!( + " [{:#034x}, {:#034x}) depth={} v_depth={} importance={}", + entry.start, entry.end, entry.depth, entry.v_depth, entry.importance, + ); + } + + println!(); + println!("Every competitive cell's parent chain is present in the full set."); + println!("The host can inspect local cells and their shared ancestors separately."); +} + +// ======================================================================== +// STEP 2: INSPECTION MIRRORS REPORTS +// §ALGO S-14, ADR-S-014 +// ======================================================================== + +fn step_2_inspection_mirrors_reports(sentinel: &Sentinel128, report: &BatchReport) { + heading("Step 2: inspect_cell Mirrors Batch Reports"); + println!(" (§ALGO S-14, ADR-S-014)"); + + let mut checked = 0usize; + for cell in all_cell_reports(report) { + assert_inspection_matches_report(sentinel, cell); + checked += 1; + } + + let inspected_ids: BTreeSet<_> = sentinel.cell_gnodes().into_iter().collect(); + assert_eq!(inspected_ids.len(), sentinel.cells_tracked()); + for id in inspected_ids { + let inspection: CellInspection = sentinel.inspect_cell(id).expect("cell_gnodes are inspectable"); + assert_eq!(inspection.geometry.dim, inspection.analysis_width); + assert_eq!( + inspection.geometry.cap, + inspection.analysis_width.min(sentinel.config().max_rank) + ); + } + + println!("Checked {checked} reported cell snapshots against inspect_cell()."); + println!("`inspect_cell()` is the stable read path for tracker metadata between batches."); +} + +// ======================================================================== +// STEP 3: PAYLOAD RECORDS +// §ALGO S-14.8, §ALGO S-14.9 +// ======================================================================== + +fn step_3_payload_records(report: &BatchReport) { + heading("Step 3: Per-Sample and Per-Member Payload Records"); + println!(" (§ALGO S-14.8, §ALGO S-14.9)"); + + let mut sample_records = 0usize; + for cell in all_cell_reports(report) { + let samples = cell.per_sample.as_ref().expect("advanced config enables per-sample scores"); + assert_eq!(samples.len(), cell.sample_count); + for sample in samples { + assert_sample_score(sample); + } + sample_records += samples.len(); + } + + let mut member_records = 0usize; + for coordination in &report.coordination_reports { + let members = coordination + .per_member + .as_ref() + .expect("advanced config enables per-member coordination scores"); + assert_eq!(members.len(), coordination.cells_reporting); + for member in members { + assert_member_score(member); + } + member_records += members.len(); + } + + println!("Per-sample records checked: {sample_records}"); + println!("Per-member coordination records checked: {member_records}"); + println!("These payloads are raw measurements aligned with routing, not decisions."); +} + +// ======================================================================== +// STEP 4: SCORING GEOMETRY EDGE FLAGS +// §ALGO S-5.2, §ALGO S-14.7 +// ======================================================================== + +fn step_4_scoring_geometry_edges() { + heading("Step 4: Scoring Geometry Edge Flags"); + println!(" (§ALGO S-5.2, §ALGO S-14.7)"); + + let rank_one = Sentinel128::new(rank_one_config()).expect("rank-one config is valid"); + let rank_one_health = rank_one.health(); + assert_eq!( + rank_one_health.geometry_distribution.coherence_inactive, + rank_one_health.active_trackers + ); + + let root = rank_one + .inspect_cell(rank_one.graph().g_root()) + .expect("rank-one root is inspectable"); + assert_eq!(root.rank, 1); + assert_eq!(root.geometry.cap, 1); + assert_eq!(root.geometry.residual_dof, root.geometry.dim - 1); + assert!(!root.geometry.is_novelty_saturated()); + + let tiny = TinySentinel::new(tiny_geometry_config()).expect("tiny geometry config is valid"); + let tiny_root = tiny.inspect_cell(tiny.graph().g_root()).expect("tiny root is inspectable"); + assert_eq!(tiny_root.geometry.dim, 4); + assert_eq!(tiny_root.geometry.cap, 4); + assert!(tiny_root.geometry.is_novelty_saturable()); + + println!("Rank-one Sentinel:"); + println!(" active trackers = {}", rank_one_health.active_trackers); + println!( + " coherence-inactive trackers = {}", + rank_one_health.geometry_distribution.coherence_inactive + ); + println!( + " root dim = {}, cap = {}, residual DOF = {}", + root.geometry.dim, root.geometry.cap, root.geometry.residual_dof + ); + + println!("Tiny four-bit Sentinel:"); + println!(" root dim = {}, cap = {}", tiny_root.geometry.dim, tiny_root.geometry.cap); + println!(" novelty-saturable = {}", tiny_root.geometry.is_novelty_saturable()); + println!("Geometry fields tell the host whether an axis is structurally meaningful."); +} + +// ======================================================================== +// STEP 5: COORDINATION READ SURFACE +// §ALGO S-7, §ALGO S-9 +// ======================================================================== + +fn step_5_coordination_read_surface(sentinel: &Sentinel128, report: &BatchReport) { + heading("Step 5: Coordination Read Surface"); + println!(" (§ALGO S-7, §ALGO S-9)"); + + assert!( + !report.coordination_reports.is_empty(), + "multiscale setup activated coordination" + ); + assert!(sentinel.health().active_coordination_contexts >= report.coordination_reports.len()); + + let competitive_cells: BTreeSet<_> = report + .cell_reports + .iter() + .map(|cell| (cell.start, cell.end, cell.depth)) + .collect(); + + for coordination in &report.coordination_reports { + assert_coordination_report(coordination, sentinel.config()); + let members = coordination.per_member.as_ref().expect("per-member payloads are enabled"); + for member in members { + assert!(member.cell_start >= coordination.start); + assert!(member.cell_end <= coordination.end); + assert!( + competitive_cells.contains(&(member.cell_start, member.cell_end, member.cell_depth)), + "coordination member must be a reporting competitive cell" + ); + } + } + + subheading("Coordination contexts"); + for coordination in &report.coordination_reports { + println!( + " [{:#034x}, {:#034x}) depth={} cells={} rank={} novelty_mean={:.6}", + coordination.start, + coordination.end, + coordination.depth, + coordination.cells_reporting, + coordination.rank, + coordination.scores.novelty.mean, + ); + } + + println!(); + println!("Coordination rows are competitive-cell score summaries."); + println!("The tier measures cross-cell structure; it does not classify it."); +} + +// ======================================================================== +// STEP 6: SCALE PROFILE +// §ALGO S-9.4, §ALGO S-16.5 +// ======================================================================== + +fn step_6_scale_profile(sentinel: &mut Sentinel128) { + heading("Step 6: Depth-Indexed Scale Profile"); + println!(" (§ALGO S-9.4, §ALGO S-16.5)"); + + let values = anomalous_values(0xA, 8); + let report = sentinel.ingest(&values); + assert_report_surface(sentinel, &report, values.len()); + + let mut max_novelty_by_depth: BTreeMap = BTreeMap::new(); + for cell in all_cell_reports(&report) { + let entry = max_novelty_by_depth.entry(cell.depth).or_insert(0.0); + *entry = (*entry).max(cell.scores.novelty.max_z_score); + } + + assert!(max_novelty_by_depth.contains_key(&0), "root depth participates"); + assert!( + max_novelty_by_depth.keys().any(|depth| *depth > 0), + "non-root depths participate" + ); + assert!(max_novelty_by_depth.len() >= 2, "scale profile has multiple depths"); + + subheading("Max novelty z-score by G-tree depth"); + for (depth, z) in &max_novelty_by_depth { + assert!(z.is_finite()); + println!(" depth {depth}: {z:.6}"); + } + + println!(); + println!("The host can compare depth-local and root-level measurements."); + println!("This test asserts the profile exists, not what policy should conclude."); +} + +// ======================================================================== +// STEP 7: TEMPORAL SEPARATION +// §ALGO S-10, ADR-S-002 +// ======================================================================== + +fn step_7_temporal_separation(sentinel: &mut Sentinel128) { + heading("Step 7: Temporal Separation"); + println!(" (§ALGO S-10, ADR-S-002)"); + + let root = sentinel.graph().g_root(); + let before_sum = sentinel.graph().total_sum(); + let before_lifetime = sentinel.lifetime_observations(); + let before_cells = sentinel.cells_tracked(); + let before_root = sentinel.inspect_cell(root).expect("root is inspectable before decay"); + let before_baselines = before_root.baselines; + + sentinel.decay(0.5, 0.0); + + let after_sum = sentinel.graph().total_sum(); + let after_root = sentinel.inspect_cell(root).expect("root remains inspectable after decay"); + assert!(after_sum < before_sum, "uniform attenuation reduces spatial importance"); + assert_eq!(sentinel.lifetime_observations(), before_lifetime, "decay is not traffic"); + assert_eq!( + sentinel.cells_tracked(), + before_cells, + "decay does not directly destroy trackers" + ); + assert_baselines_same(before_baselines, after_root.baselines); + + println!("Spatial importance: {before_sum} -> {after_sum}"); + println!( + "Lifetime observations: {before_lifetime} -> {}", + sentinel.lifetime_observations() + ); + println!("Active trackers: {before_cells} -> {}", sentinel.cells_tracked()); + println!("Root tracker baselines unchanged by spatial decay. ✓"); +} + +// ======================================================================== +// STEP 8: RESET READ SURFACE +// Public lifecycle contract +// ======================================================================== + +fn step_8_reset_read_surface(sentinel: &mut Sentinel128) { + heading("Step 8: Reset Read Surface"); + + assert!(sentinel.graph().total_sum() > 0); + assert!(sentinel.lifetime_observations() > 0); + + sentinel.reset(); + + let health = sentinel.health(); + assert_eq!(sentinel.graph().node_count(), 1); + assert_eq!(sentinel.graph().terminal_count(), 1); + assert_eq!(sentinel.graph().total_sum(), 0); + assert_eq!(sentinel.cells_tracked(), 1); + assert_eq!(sentinel.lifetime_observations(), 0); + assert_eq!(sentinel.analysis_set().competitive_count(), 0); + assert_eq!(sentinel.analysis_set().total_count(), 1); + assert_eq!(health.active_coordination_contexts, 0); + assert_eq!(health.warming_trackers, 0); + + let root = sentinel + .inspect_cell(sentinel.graph().g_root()) + .expect("reset recreates the inspectable root"); + assert_eq!(root.depth, 0); + assert_eq!(root.analysis_width, 128); + assert!(root.maturity.noise_observations > 0); + assert_eq!(root.maturity.real_observations, 0); + + println!("After reset:"); + println!(" graph nodes = {}", sentinel.graph().node_count()); + println!(" active trackers = {}", health.active_trackers); + println!(" coordination contexts = {}", health.active_coordination_contexts); + println!(" root noise observations = {}", root.maturity.noise_observations); +} + +// ======================================================================== +// MAIN +// ======================================================================== + +fn main() { + // Support `--list --format terse` so cargo-nextest can enumerate this + // custom-harness test binary. With `--ignored`, output nothing. + let args: Vec = std::env::args().collect(); + if args.iter().any(|arg| arg == "--list") { + if !args.iter().any(|arg| arg == "--ignored") { + println!("pedagogy_advanced: test"); + } + return; + } + + let config = advanced_config(); + config.validate().expect("advanced pedagogy config must be valid"); + + let mut sentinel = Sentinel128::new(config).expect("valid config constructs a Sentinel"); + + step_0_inspection_setup(&sentinel); + let report = drive_to_multiscale(&mut sentinel); + step_1_analysis_set_anatomy(&sentinel, &report); + step_2_inspection_mirrors_reports(&sentinel, &report); + step_3_payload_records(&report); + step_4_scoring_geometry_edges(); + step_5_coordination_read_surface(&sentinel, &report); + step_6_scale_profile(&mut sentinel); + step_7_temporal_separation(&mut sentinel); + step_8_reset_read_surface(&mut sentinel); + + heading("All Advanced Claims Verified"); + println!(); + println!(" Inspection setup:"); + println!(" [ok] foreground warm-up makes tracker readouts deterministic"); + println!(" [ok] fresh Sentinel exposes a warmed, non-competitive root"); + println!(); + println!(" Analysis-set read surface:"); + println!(" [ok] report summary matches AnalysisSet accessors"); + println!(" [ok] competitive cells are closed under G-tree ancestry"); + println!(" [ok] inspect_cell mirrors reported tracker metadata"); + println!(); + println!(" Report payloads and geometry:"); + println!(" [ok] per-sample payloads match routed sample counts"); + println!(" [ok] per-member coordination payloads match reporting cells"); + println!(" [ok] geometry flags expose inactive and saturable axes"); + println!(); + println!(" Coordination and scale:"); + println!(" [ok] coordination contexts measure cross-cell score patterns"); + println!(" [ok] depth-indexed profiles give scale diagnostics"); + println!(); + println!(" Host policy and lifecycle:"); + println!(" [ok] spatial decay leaves tracker baselines untouched"); + println!(" [ok] reset restores the fresh read surface and warmed root"); +} diff --git a/packages/sentinel/tests/report_structure.rs b/packages/sentinel/tests/report_structure.rs new file mode 100644 index 000000000..925e22ee3 --- /dev/null +++ b/packages/sentinel/tests/report_structure.rs @@ -0,0 +1,784 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +//! # Test index +//! +//! | Test | Area | Claim | +//! |------|------|-------| +//! | [`empty_ingest_produces_full_report`] | readout | An ingest with nothing in it still returns a complete report: the cell, ancestor and coordination lists are present and empty, the contour and health sections are populated, and the summary counts at least the root. A host polling a quiet sentinel therefore parses the same layout it parses under load, and can read structural state from a batch that carried no observations at all. | +//! | [`cell_reports_are_competitive_only`] | readout | The report separates cells by how they earned their place: the cell list holds only competitively selected cells. A competitive cell was chosen because its traffic made it worth modelling, so a host reading that list is reading the sentinel's own investment decisions and nothing else. | +//! | [`ancestor_reports_are_ancestor_only`] | readout | cites (´claim:readout:the-report-partitions-cells-by-how-they-earned-their-place-rather-than-listing-them-together´) | +//! | [`cell_and_ancestor_cover_all_reported_cells`] | readout | cites (´claim:readout:the-report-partitions-cells-by-how-they-earned-their-place-rather-than-listing-them-together´) | +//! | [`cell_reports_sorted_by_gnode_id`] | readout | Cell reports come back in strictly ascending handle order, never merely grouped. Nothing about the order reflects the sequence observations arrived in or how the internal maps happened to iterate, so two sentinels fed the same stream emit comparable reports and a difference between two readouts is a difference in the system. | +//! | [`ancestor_reports_sorted_by_gnode_id`] | readout | cites (´claim:readout:report-lists-are-ordered-by-cell-handle-so-identical-runs-produce-identical-readouts´) | +//! | [`coordination_reports_sorted_by_depth_then_gnode_id`] | readout | The tier above the cells is ordered too, but on a key of its own: coordination reports come shallowest first, with the node handle breaking ties among contexts at equal depth. Handles are recycled as cells are evicted and restored, so a correctly ordered run can carry a lower handle at a greater depth — which is why depth leads, and why an assertion on the handle alone would reject a readout that was right. Determinism is imposed on the readout as a whole rather than recovered separately wherever a list happens to be built, but each list states which order it is in. | +//! | [`coordination_reports_have_unique_gnodes`] | readout | Each coordination context appears at most once in a batch. The contexts are found by walking a tree in which a node can be reached from several selected descendants, so uniqueness is a real obligation: without it a busy subtree would report the same group finding repeatedly and a host counting elevated contexts would over-count it. | +//! | [`no_nan_in_score_fields`] | readout | Every score the readout carries is a number, on all four axes and across both competitive and ancestor cells. The scoring formulae divide by quantities that can legitimately reach zero — residual degrees of freedom, rank, baseline spread — so producing a number at the boundary is something the engine must arrange. A single non-number would poison every comparison a host makes downstream, silently rather than loudly. | +//! | [`report_structure_per_sample_scores_present_when_enabled`] | readout | Per-observation detail is present in every cell report exactly when the host configured it, rather than appearing only where the engine found it convenient. The detail costs memory proportional to the batch, so it is optional — but an option that were honoured unevenly would be worse than none, since a host could not tell an absent field from an unremarkable cell. | +//! | [`coordination_cells_reporting_is_nonzero`] | readout | A coordination report is emitted only where cells actually contributed to it: every one names a positive number of reporting cells. The tier exists to measure how a group of cells moves together, so a context with no contributors would be describing a group that did not exist this batch. | +//! | [`member_score_identifies_cell`] | readout | cites (´claim:readout:a-member-score-names-the-cell-it-came-from-so-a-group-finding-can-be-attributed´) | +//! | [`contour_reflects_graph_state`] | readout | The contour describes the spatial layer as it actually stands: after a batch of real traffic it reports accumulated importance above zero and at least one cell. It is read from the graph at report time rather than maintained alongside it, so it cannot drift out of step with the structure the trackers are attached to. | +//! | [`contour_cell_count_grows_with_distinct_regions`] | readout | Traffic arriving in a well-separated second region never reduces the reported spatial resolution. New structure is added by bisection, and nothing about observing an unfamiliar region coarsens what the graph already learned elsewhere — so a host watching the cell count sees refinement accumulate rather than oscillate with the traffic mix. | +//! | [`contour_reports_splits_since_last_report`] | readout | The split counter describes the interval since the previous report and is cleared with it: a heavy batch that forces bisection reports the splits it caused, and a quiet batch immediately after does not inherit them. The figure is a rate rather than a running total, which is what makes bursts of structural churn visible in the report that contained them. | +//! | [`contour_mutation_counts_zero_on_empty_ingest`] | readout | cites (´claim:readout:structural-mutation-counts-describe-the-interval-since-the-previous-report-and-reset-with-it´) | +//! | [`contour_after_decay`] | readout | Forgetting is visible in the readout: after the host applies decay, the reported importance is lower than before it. Temporal policy belongs to the host, which decides when history should count for less, and the contour is where that decision becomes observable — otherwise a host could not confirm that a decay it asked for had taken effect. | +//! | [`health_inline_matches_standalone`] | readout | The health carried inside a batch is the same health a standalone query returns — lifetime observations, active trackers and node count all agree. There is one health computation rather than two that happen to coincide, so a host that reads health from reports and a host that polls for it cannot form different pictures of the same sentinel. | +//! | [`health_tracker_breakdown_is_consistent`] | readout | The tracker breakdown accounts for the root apart from the two named categories: competitive and ancestor counts together fall short of the active total, the shortfall being the permanent root tracker. The root is present for structural reasons rather than because it competed or was closed over, and folding it into either count would misstate what the sentinel chose to invest in. The reported node total is likewise the graph's own count rather than a separately maintained tally. | +//! | [`analysis_set_summary_matches_analysis_set`] | readout | The summary's counts are the analysis set's own counts, read from it rather than tallied a second time on the way into the report. A summary is offered so a host need not enumerate every cell; it would be worth little if enumerating the cells could contradict it. | +//! | [`analysis_set_summary_depth_range_includes_root`] | readout | The reported depth span begins at the root and runs the right way round. Because ancestor closure always terminates at the root, the shallow end of the span is fixed at zero on a live sentinel, and the deep end describes the finest resolution currently being modelled — so the span is the reach of the whole chain, not the band the selected cells occupy. | +//! | [`analysis_set_summary_investment_covers_full`] | readout | The investment set is never smaller than the set currently producing reports. Every cell that reports has a tracker, and some cells hold trackers that are still warming and not yet contributing — so the gap between the two figures is precisely the modelling the sentinel is paying for but not yet reading from. | +//! | [`analysis_width_on_cell_report`] | readout | A cell's reported analysis width is always the domain width less its depth, for competitive and ancestor cells alike. The leading bits that routing already fixed are constant within the cell and carry no information, so the width states exactly how many bits the cell's model had left to work with — which is what a host needs to compare scores from cells at different depths. | +//! | [`analysis_width_on_cell_inspection`] | readout | cites (´claim:readout:a-cells-reported-analysis-width-is-the-domain-width-less-its-depth´) | +//! | [`batch_report_states_the_age_of_its_oldest_observation`] | readout | A report says how old its evidence was at the moment it was emitted: the batch is stamped as it arrives and the figure is read off as the report is assembled, so it is a positive interval that never exceeds the call that produced it. Both ends of the measurement are the sentinel's own monotonic clock, so a host learns the age of what it is holding without either side having to trust the other's idea of the time. | +//! | [`batch_report_age_is_scoped_to_its_own_batch`] | readout | The age belongs to the batch that carried the observations rather than running from the sentinel's own beginning: after a silence, the next batch reports an age shorter than the silence that preceded it. An age that accumulated over uptime would answer how long the sentinel had been running, which is the wrong question — what a host needs is how stale the evidence in front of it is. | +//! | [`empty_ingest_reports_no_observation_age`] | readout | A batch with no observations has no oldest observation, so it reports no age at all rather than an age of nothing. Absence and instantaneity are different facts about a report and a zero would have conflated them: a host watching for stale evidence has to be able to tell "nothing arrived" from "what arrived was fresh". | + +//! Integration tests for the shape of the readout a live sentinel hands +//! back — what a batch report is obliged to contain, and how it is +//! arranged, when it comes from a sentinel that has actually observed +//! traffic rather than from a hand-built struct. +//! +//! The report is the whole interface between a component that measures and a +//! host that decides, so its structure is load-bearing in ways the numbers +//! are not. Cells are partitioned by how they earned their place — +//! competitively selected on one side, drawn in by ancestor closure on the +//! other — because the two were chosen for different reasons and a host +//! weighing a finding needs to know which it is reading. Every list is +//! ordered by a key of its own rather than by whatever order a walk produced +//! — the two cell lists by handle, the cross-cell contexts shallowest first +//! with the handle breaking ties — so two sentinels fed the same stream emit +//! byte-comparable output and a diff between reports means a difference in +//! the system rather than in iteration order. +//! +//! Two further commitments run through these tests. The report's shape does +//! not depend on how busy the sentinel is: an ingest with nothing in it +//! still returns every section, with empty lists and zeroed counters, so a +//! consumer parses one layout rather than several. And the report is +//! self-consistent with the state it describes — the health it carries is +//! the health a standalone query returns, the summary counts are the +//! analysis set's own counts, and each cell's analysis width is its domain +//! width less its depth — so a host never has to reconcile two views of one +//! sentinel. +//! +//! Alongside the standing state, the contour reports the structural churn +//! since the previous report: splits and net removals, counted over the +//! interval and reset with it, so growth and eviction are visible without +//! differencing successive snapshots. + +mod common; + +use std::time::{Duration, Instant}; + +use common::{ScenarioBuilder, assert_invariants, cell_values, test_config}; +use torrust_sentinel::Sentinel128; + +// ── Helper ────────────────────────────────────────────────── + +/// Build a sentinel with enough traffic that both competitive and +/// ancestor cells exist, using `ScenarioBuilder`. +fn multi_cell_sentinel() -> Sentinel128 { + ScenarioBuilder::new() + .config(test_config()) + .seed_range(0xF, 10) + .seed_range(0x1, 10) + .warm_batches(5) + .build() +} + +// ═══════════════════════════════════════════════════════════ +// Empty report +// ═══════════════════════════════════════════════════════════ + +/// An ingest with nothing in it still returns a complete report: the cell, +/// ancestor and coordination lists are present and empty, the contour and +/// health sections are populated, and the summary counts at least the root. +/// A host polling a quiet sentinel therefore parses the same layout it +/// parses under load, and can read structural state from a batch that +/// carried no observations at all. +/// +/// ´claim:readout:an-empty-ingest-still-returns-a-complete-report-with-empty-lists-rather-than-nothing´ +/// ´test:integration:empty-ingest-produces-full-report´ +#[test] +fn empty_ingest_produces_full_report() { + let mut s = Sentinel128::new(test_config()).unwrap(); + let report = s.ingest(&[]); + + assert!(report.cell_reports.is_empty()); + assert!(report.ancestor_reports.is_empty()); + assert!(report.coordination_reports.is_empty()); + // Contour is valid even on empty ingest. + assert!(report.contour.cell_count > 0 || report.contour.plateau_count == 0); + // Health is populated. + assert!(report.health.active_trackers > 0 || report.health.lifetime_observations == 0); + // Summary sizes are consistent — at least the root. + assert!(report.analysis_set_summary.full_size >= 1); + + assert_invariants(&s, &report); +} + +// ═══════════════════════════════════════════════════════════ +// Cell-report partition +// ═══════════════════════════════════════════════════════════ + +/// The report separates cells by how they earned their place: the cell list +/// holds only competitively selected cells. A competitive cell was chosen +/// because its traffic made it worth modelling, so a host reading that list +/// is reading the sentinel's own investment decisions and nothing else. +/// +/// ´claim:readout:the-report-partitions-cells-by-how-they-earned-their-place-rather-than-listing-them-together´ +/// ´test:integration:cell-reports-are-competitive-only´ +#[test] +fn cell_reports_are_competitive_only() { + let mut s = multi_cell_sentinel(); + let report = s.ingest(&cell_values(0xF, 4)); + + for cr in &report.cell_reports { + assert!( + cr.is_competitive, + "cell_reports must only contain competitive cells, got depth={}", + cr.depth + ); + } + assert_invariants(&s, &report); +} + +/// The complementary half: the ancestor list holds only cells that were +/// never competitively selected. These exist because closure required a +/// model chain back to the root, and they supply coarser context rather than +/// a judgement that the region deserved attention — which is exactly why +/// they are kept out of the competitive list. +/// +/// (´claim:readout:the-report-partitions-cells-by-how-they-earned-their-place-rather-than-listing-them-together´) +/// ´test:integration:ancestor-reports-are-ancestor-only´ +#[test] +fn ancestor_reports_are_ancestor_only() { + let mut s = multi_cell_sentinel(); + let report = s.ingest(&cell_values(0xF, 4)); + + for ar in &report.ancestor_reports { + assert!( + !ar.is_competitive, + "ancestor_reports must only contain ancestor cells, got depth={}", + ar.depth + ); + } + assert_invariants(&s, &report); +} + +/// The two lists are genuinely a partition and not merely two views: no cell +/// handle appears in both. A host can therefore concatenate them to see +/// everything that reported this batch, or count them separately, without +/// double-counting any cell. +/// +/// (´claim:readout:the-report-partitions-cells-by-how-they-earned-their-place-rather-than-listing-them-together´) +/// ´test:integration:cell-and-ancestor-cover-all-reported-cells´ +#[test] +fn cell_and_ancestor_cover_all_reported_cells() { + let mut s = multi_cell_sentinel(); + let report = s.ingest(&cell_values(0xF, 4)); + + // No gnode_id appears in both partitions. + let competitive_ids: std::collections::HashSet<_> = report.cell_reports.iter().map(|cr| cr.gnode_id).collect(); + let ancestor_ids: std::collections::HashSet<_> = report.ancestor_reports.iter().map(|ar| ar.gnode_id).collect(); + + assert!( + competitive_ids.is_disjoint(&ancestor_ids), + "cell_reports and ancestor_reports must be disjoint" + ); + + assert_invariants(&s, &report); +} + +// ═══════════════════════════════════════════════════════════ +// Ordering and uniqueness +// ═══════════════════════════════════════════════════════════ + +/// Cell reports come back in strictly ascending handle order, never merely +/// grouped. Nothing about the order reflects the sequence observations +/// arrived in or how the internal maps happened to iterate, so two sentinels +/// fed the same stream emit comparable reports and a difference between two +/// readouts is a difference in the system. +/// +/// ´claim:readout:report-lists-are-ordered-by-cell-handle-so-identical-runs-produce-identical-readouts´ +/// ´test:integration:cell-reports-sorted-by-gnode-id´ +#[test] +fn cell_reports_sorted_by_gnode_id() { + let mut s = multi_cell_sentinel(); + let report = s.ingest(&cell_values(0xF, 4)); + + for window in report.cell_reports.windows(2) { + assert!( + window[0].gnode_id < window[1].gnode_id, + "cell_reports not sorted: {:?} >= {:?}", + window[0].gnode_id, + window[1].gnode_id, + ); + } +} + +/// The ordering rule is a property of the readout rather than of the +/// competitive list: ancestor reports are sorted by the same key, strictly +/// ascending. Cells the closure added are as reproducibly placed as cells +/// that were chosen. +/// +/// (´claim:readout:report-lists-are-ordered-by-cell-handle-so-identical-runs-produce-identical-readouts´) +/// ´test:integration:ancestor-reports-sorted-by-gnode-id´ +#[test] +fn ancestor_reports_sorted_by_gnode_id() { + let mut s = multi_cell_sentinel(); + let report = s.ingest(&cell_values(0xF, 4)); + + for window in report.ancestor_reports.windows(2) { + assert!( + window[0].gnode_id < window[1].gnode_id, + "ancestor_reports not sorted: {:?} >= {:?}", + window[0].gnode_id, + window[1].gnode_id, + ); + } +} + +/// The tier above the cells is ordered too, but on a key of its own: +/// coordination reports come shallowest first, with the node handle breaking +/// ties among contexts at equal depth. Handles are recycled as cells are +/// evicted and restored, so a correctly ordered run can carry a lower handle +/// at a greater depth — which is why depth leads, and why an assertion on the +/// handle alone would reject a readout that was right. Determinism is imposed +/// on the readout as a whole rather than recovered separately wherever a list +/// happens to be built, but each list states which order it is in. +/// +/// ´claim:readout:coordination-reports-come-shallowest-first-with-the-handle-breaking-ties´ +/// ´test:integration:coordination-reports-sorted-by-depth-then-gnode-id´ +#[test] +fn coordination_reports_sorted_by_depth_then_gnode_id() { + let mut s = multi_cell_sentinel(); + let report = s.ingest(&cell_values(0xF, 4)); + + for window in report.coordination_reports.windows(2) { + assert!( + (window[0].depth, window[0].gnode_id) < (window[1].depth, window[1].gnode_id), + "coordination_reports not in (depth, handle) order: ({}, {:?}) >= ({}, {:?})", + window[0].depth, + window[0].gnode_id, + window[1].depth, + window[1].gnode_id, + ); + } +} + +/// Each coordination context appears at most once in a batch. The +/// contexts are found by walking a tree in which a node can be reached from +/// several selected descendants, so uniqueness is a real obligation: without +/// it a busy subtree would report the same group finding repeatedly and a +/// host counting elevated contexts would over-count it. +/// +/// ´claim:readout:each-coordination-context-appears-at-most-once-in-a-batch´ +/// ´test:integration:coordination-reports-have-unique-gnodes´ +#[test] +fn coordination_reports_have_unique_gnodes() { + let mut s = multi_cell_sentinel(); + let report = s.ingest(&cell_values(0xF, 4)); + + let mut seen = std::collections::HashSet::new(); + for cr in &report.coordination_reports { + assert!( + seen.insert(cr.gnode_id), + "duplicate GNodeId in coordination_reports: {:?}", + cr.gnode_id, + ); + } +} + +// ═══════════════════════════════════════════════════════════ +// Scores +// ═══════════════════════════════════════════════════════════ + +/// Every score the readout carries is a number, on all four axes and across +/// both competitive and ancestor cells. The scoring formulae divide by +/// quantities that can legitimately reach zero — residual degrees of +/// freedom, rank, baseline spread — so producing a number at the boundary is +/// something the engine must arrange. A single non-number would poison every +/// comparison a host makes downstream, silently rather than loudly. +/// +/// ´claim:readout:every-reported-score-is-a-number-so-a-degenerate-model-never-leaks-a-non-number-into-the-readout´ +/// ´test:integration:no-nan-in-score-fields´ +#[test] +fn no_nan_in_score_fields() { + let mut s = multi_cell_sentinel(); + let report = s.ingest(&cell_values(0xF, 4)); + + for cr in report.cell_reports.iter().chain(report.ancestor_reports.iter()) { + assert!(!cr.scores.novelty.mean.is_nan(), "NaN novelty mean at depth {}", cr.depth); + assert!( + !cr.scores.displacement.mean.is_nan(), + "NaN displacement mean at depth {}", + cr.depth + ); + assert!(!cr.scores.surprise.mean.is_nan(), "NaN surprise mean at depth {}", cr.depth); + assert!(!cr.scores.coherence.mean.is_nan(), "NaN coherence mean at depth {}", cr.depth); + } + + assert_invariants(&s, &report); +} + +/// Per-observation detail is present in every cell report exactly when the +/// host configured it, rather than appearing only where the engine found it +/// convenient. The detail costs memory proportional to the batch, so it is +/// optional — but an option that were honoured unevenly would be worse than +/// none, since a host could not tell an absent field from an unremarkable +/// cell. +/// +/// ´claim:readout:per-observation-detail-is-present-exactly-where-the-host-asked-for-it´ +/// ´test:integration:report-structure-per-sample-scores-present-when-enabled´ +#[test] +fn report_structure_per_sample_scores_present_when_enabled() { + // test_config() has per_sample_scores = true. + let mut s = multi_cell_sentinel(); + let report = s.ingest(&cell_values(0xF, 4)); + + for cr in &report.cell_reports { + assert!( + cr.per_sample.is_some(), + "per_sample should be Some when per_sample_scores is enabled, depth={}", + cr.depth, + ); + } + + assert_invariants(&s, &report); +} + +// ═══════════════════════════════════════════════════════════ +// Coordination reports +// ═══════════════════════════════════════════════════════════ + +/// A coordination report is emitted only where cells actually contributed to +/// it: every one names a positive number of reporting cells. The tier exists +/// to measure how a group of cells moves together, so a context with no +/// contributors would be describing a group that did not exist this batch. +/// +/// ´claim:readout:a-coordination-report-appears-only-where-cells-actually-contributed-to-it´ +/// ´test:integration:coordination-cells-reporting-is-nonzero´ +#[test] +fn coordination_cells_reporting_is_nonzero() { + let mut s = multi_cell_sentinel(); + let report = s.ingest(&cell_values(0xF, 4)); + + for cr in &report.coordination_reports { + assert!( + cr.cells_reporting > 0, + "coordination report at depth {} has zero cells_reporting", + cr.depth, + ); + } +} + +/// The identity carried by a member score holds for scores the engine +/// actually produced, not only for ones built by hand: every member of every +/// coordination group names a non-empty interval. The group's own report +/// says a subtree behaved unusually; these entries are what let a host +/// narrow that to a region of the domain. +/// +/// (´claim:readout:a-member-score-names-the-cell-it-came-from-so-a-group-finding-can-be-attributed´) +/// ´test:integration:member-score-identifies-cell´ +#[test] +fn member_score_identifies_cell() { + let mut s = multi_cell_sentinel(); + let report = s.ingest(&cell_values(0xF, 4)); + + for cr in &report.coordination_reports { + if let Some(members) = &cr.per_member { + for ms in members { + assert!(ms.cell_start < ms.cell_end, "MemberScore must have cell_start < cell_end"); + } + } + } + + assert_invariants(&s, &report); +} + +// ═══════════════════════════════════════════════════════════ +// Contour snapshot +// ═══════════════════════════════════════════════════════════ + +/// The contour describes the spatial layer as it actually stands: after a +/// batch of real traffic it reports accumulated importance above zero and at +/// least one cell. It is read from the graph at report time rather than +/// maintained alongside it, so it cannot drift out of step with the +/// structure the trackers are attached to. +/// +/// ´claim:readout:the-contour-describes-the-spatial-layer-as-it-actually-stands-at-report-time´ +/// ´test:integration:contour-reflects-graph-state´ +#[test] +fn contour_reflects_graph_state() { + let mut s = Sentinel128::new(test_config()).unwrap(); + let values = cell_values(0xA, 20); + let report = s.ingest(&values); + + assert!(report.contour.total_importance > 0.0); + assert!(report.contour.cell_count >= 1); + + assert_invariants(&s, &report); +} + +/// Traffic arriving in a well-separated second region never reduces the +/// reported spatial resolution. New structure is added by bisection, and +/// nothing about observing an unfamiliar region coarsens what the graph +/// already learned elsewhere — so a host watching the cell count sees +/// refinement accumulate rather than oscillate with the traffic mix. +/// +/// ´claim:readout:spatial-resolution-never-falls-back-when-a-new-well-separated-region-arrives´ +/// ´test:integration:contour-cell-count-grows-with-distinct-regions´ +#[test] +fn contour_cell_count_grows_with_distinct_regions() { + let mut s = Sentinel128::new(test_config()).unwrap(); + + let r1 = s.ingest(&cell_values(0xA, 20)); + let count_after_one = r1.contour.cell_count; + + // Add a well-separated region. + let r2 = s.ingest(&cell_values(0x5, 20)); + assert!( + r2.contour.cell_count >= count_after_one, + "cell_count should not shrink when adding distinct regions" + ); + + assert_invariants(&s, &r2); +} + +/// The split counter describes the interval since the previous report and is +/// cleared with it: a heavy batch that forces bisection reports the splits it +/// caused, and a quiet batch immediately after does not inherit them. The +/// figure is a rate rather than a running total, which is what makes bursts +/// of structural churn visible in the report that contained them. +/// +/// ´claim:readout:structural-mutation-counts-describe-the-interval-since-the-previous-report-and-reset-with-it´ +/// ´test:integration:contour-reports-splits-since-last-report´ +#[test] +fn contour_reports_splits_since_last_report() { + let mut s = Sentinel128::new(test_config()).unwrap(); + // The threshold is 100 and the graph splits only above it, so the last + // observation creates exactly the root's two children. + let r1 = s.ingest(&cell_values(0xA, 101)); + assert_eq!(r1.contour.splits_since_last_report, 2); + assert_eq!(r1.contour.net_removals_since_last_report, 0); + assert_eq!(s.graph().node_count(), 3); + assert_eq!(s.graph().terminal_count(), 2); + + // A quiet interval has no spatial events and cannot inherit the first + // report's child-creation count. + let r2 = s.ingest(&[]); + assert_eq!(r2.contour.splits_since_last_report, 0); + assert_eq!(r2.contour.net_removals_since_last_report, 0); +} + +/// The floor of the same rule, on a sentinel that has already been driven +/// hard: an ingest with no observations reports no splits and no removals. +/// Structural change is caused by observations, so an interval with none +/// reports zeroed churn rather than repeating the last non-empty batch's +/// figures. +/// +/// (´claim:readout:structural-mutation-counts-describe-the-interval-since-the-previous-report-and-reset-with-it´) +/// ´test:integration:contour-mutation-counts-zero-on-empty-ingest´ +#[test] +fn contour_mutation_counts_zero_on_empty_ingest() { + let mut s = Sentinel128::new(test_config()).unwrap(); + // Prime the sentinel with some data first. + s.ingest(&cell_values(0xA, 20)); + // Empty ingest should report zero mutations. + let r = s.ingest(&[]); + assert_eq!(r.contour.splits_since_last_report, 0); + assert_eq!(r.contour.net_removals_since_last_report, 0); +} + +/// Forgetting is visible in the readout: after the host applies decay, the +/// reported importance is lower than before it. Temporal policy belongs to +/// the host, which decides when history should count for less, and the +/// contour is where that decision becomes observable — otherwise a host +/// could not confirm that a decay it asked for had taken effect. +/// +/// ´claim:readout:host-applied-decay-shows-up-in-the-reported-importance-so-forgetting-is-observable´ +/// ´test:integration:contour-after-decay´ +#[test] +fn contour_after_decay() { + let mut s = Sentinel128::new(test_config()).unwrap(); + s.ingest(&cell_values(0xA, 20)); + let before = s.ingest(&cell_values(0xA, 5)); + let before_importance = before.contour.total_importance; + + s.decay(0.5, 0.0); + + let after = s.ingest(&cell_values(0xA, 1)); + assert!( + after.contour.total_importance < before_importance, + "total_importance should decrease after decay" + ); +} + +// ═══════════════════════════════════════════════════════════ +// Health inline +// ═══════════════════════════════════════════════════════════ + +/// The health carried inside a batch is the same health a standalone query +/// returns — lifetime observations, active trackers and node count all +/// agree. There is one health computation rather than two that happen to +/// coincide, so a host that reads health from reports and a host that polls +/// for it cannot form different pictures of the same sentinel. +/// +/// ´claim:readout:the-health-carried-in-a-batch-is-the-same-health-a-standalone-query-returns´ +/// ´test:integration:health-inline-matches-standalone´ +#[test] +fn health_inline_matches_standalone() { + let mut s = multi_cell_sentinel(); + let report = s.ingest(&cell_values(0xF, 4)); + let standalone = s.health(); + + assert_eq!(report.health.lifetime_observations, standalone.lifetime_observations); + assert_eq!(report.health.active_trackers, standalone.active_trackers); + assert_eq!(report.health.total_g_nodes, standalone.total_g_nodes); +} + +/// The tracker breakdown accounts for the root apart from the two named +/// categories: competitive and ancestor counts together fall short of the +/// active total, the shortfall being the permanent root tracker. The root is +/// present for structural reasons rather than because it competed or was +/// closed over, and folding it into either count would misstate what the +/// sentinel chose to invest in. The reported node total is likewise the +/// graph's own count rather than a separately maintained tally. +/// +/// ´claim:readout:the-tracker-breakdown-accounts-for-the-permanent-root-apart-from-competitive-and-ancestor-cells´ +/// ´test:integration:health-tracker-breakdown-is-consistent´ +#[test] +fn health_tracker_breakdown_is_consistent() { + let mut s = multi_cell_sentinel(); + let report = s.ingest(&cell_values(0xF, 4)); + let h = &report.health; + + // competitive + ancestor + root (1) ≤ active_trackers. + assert!( + h.active_competitive_trackers + h.active_ancestor_trackers < h.active_trackers, + "competitive ({}) + ancestor ({}) should be less than active ({}) (root counted separately)", + h.active_competitive_trackers, + h.active_ancestor_trackers, + h.active_trackers, + ); + assert_eq!(h.total_g_nodes, s.graph().node_count() as usize); +} + +// ═══════════════════════════════════════════════════════════ +// Analysis-set summary +// ═══════════════════════════════════════════════════════════ + +/// The summary's counts are the analysis set's own counts, read from it +/// rather than tallied a second time on the way into the report. A summary +/// is offered so a host need not enumerate every cell; it would be worth +/// little if enumerating the cells could contradict it. +/// +/// ´claim:readout:the-summary-counts-are-the-analysis-sets-own-counts-and-not-a-second-tally´ +/// ´test:integration:analysis-set-summary-matches-analysis-set´ +#[test] +fn analysis_set_summary_matches_analysis_set() { + let s = multi_cell_sentinel(); + let aset = s.analysis_set(); + let summary = aset.summary(); + + assert_eq!(summary.competitive_size, aset.competitive_count()); + assert_eq!(summary.full_size, aset.total_count()); +} + +/// The reported depth span begins at the root and runs the right way round. +/// Because ancestor closure always terminates at the root, the shallow end +/// of the span is fixed at zero on a live sentinel, and the deep end +/// describes the finest resolution currently being modelled — so the span is +/// the reach of the whole chain, not the band the selected cells occupy. +/// +/// ´claim:readout:a-reported-depth-span-starts-at-the-root-and-runs-the-right-way-round´ +/// ´test:integration:analysis-set-summary-depth-range-includes-root´ +#[test] +fn analysis_set_summary_depth_range_includes_root() { + let mut s = multi_cell_sentinel(); + let report = s.ingest(&cell_values(0xF, 4)); + let summary = &report.analysis_set_summary; + + assert_eq!(summary.depth_range.0, 0, "depth_range min must be 0 (the root)"); + assert!( + summary.depth_range.1 >= summary.depth_range.0, + "depth_range max must be >= min" + ); + + assert_invariants(&s, &report); +} + +/// The investment set is never smaller than the set currently producing +/// reports. Every cell that reports has a tracker, and some cells hold +/// trackers that are still warming and not yet contributing — so the gap +/// between the two figures is precisely the modelling the sentinel is paying +/// for but not yet reading from. +/// +/// ´claim:readout:the-investment-set-is-never-smaller-than-the-set-currently-producing-reports´ +/// ´test:integration:analysis-set-summary-investment-covers-full´ +#[test] +fn analysis_set_summary_investment_covers_full() { + let mut s = multi_cell_sentinel(); + let report = s.ingest(&cell_values(0xF, 4)); + let summary = &report.analysis_set_summary; + + assert!( + summary.investment_set_size >= summary.full_size, + "investment_set_size ({}) must be >= full_size ({})", + summary.investment_set_size, + summary.full_size, + ); + + assert_invariants(&s, &report); +} + +// ═══════════════════════════════════════════════════════════ +// Analysis width +// ═══════════════════════════════════════════════════════════ + +/// A cell's reported analysis width is always the domain width less its +/// depth, for competitive and ancestor cells alike. The leading bits that +/// routing already fixed are constant within the cell and carry no +/// information, so the width states exactly how many bits the cell's model +/// had left to work with — which is what a host needs to compare scores from +/// cells at different depths. +/// +/// ´claim:readout:a-cells-reported-analysis-width-is-the-domain-width-less-its-depth´ +/// ´test:integration:analysis-width-on-cell-report´ +#[test] +fn analysis_width_on_cell_report() { + let mut s = multi_cell_sentinel(); + let report = s.ingest(&cell_values(0xF, 4)); + + for cr in report.cell_reports.iter().chain(report.ancestor_reports.iter()) { + assert_eq!( + cr.analysis_width, + 128 - cr.depth as usize, + "analysis_width mismatch at depth {}", + cr.depth, + ); + } +} + +/// The same relation holds on the inspection path, which reaches cells +/// directly by handle rather than through a batch. A host can therefore +/// interrogate a cell between batches and read its width the same way, +/// without the two routes into the sentinel disagreeing about the geometry +/// of a single cell. +/// +/// (´claim:readout:a-cells-reported-analysis-width-is-the-domain-width-less-its-depth´) +/// ´test:integration:analysis-width-on-cell-inspection´ +#[test] +fn analysis_width_on_cell_inspection() { + let s = multi_cell_sentinel(); + + for &gnode in &s.cell_gnodes() { + let inspection = s.inspect_cell(gnode).unwrap(); + assert_eq!( + inspection.analysis_width, + 128 - inspection.depth as usize, + "analysis_width mismatch on inspection at depth {}", + inspection.depth, + ); + } +} + +// ═══════════════════════════════════════════════════════════ +// Observation age +// ═══════════════════════════════════════════════════════════ + +/// A report says how old its evidence was at the moment it was emitted: the +/// batch is stamped as it arrives and the figure is read off as the report is +/// assembled, so it is a positive interval that never exceeds the call that +/// produced it. Both ends of the measurement are the sentinel's own monotonic +/// clock, so a host learns the age of what it is holding without either side +/// having to trust the other's idea of the time. +/// +/// ´claim:readout:a-report-states-how-old-its-oldest-observation-was-when-the-report-was-emitted´ +/// ´test:integration:batch-report-states-the-age-of-its-oldest-observation´ +#[test] +fn batch_report_states_the_age_of_its_oldest_observation() { + let mut s = multi_cell_sentinel(); + + let call_start = Instant::now(); + let report = s.ingest(&cell_values(0xF, 128)); + let call_micros = u64::try_from(call_start.elapsed().as_micros()).unwrap(); + + let age = report + .oldest_observation_age_micros + .expect("a batch carrying observations has an oldest one"); + + assert!(age > 0, "the age should span the work the call did, got {age} micros"); + assert!( + age <= call_micros, + "age ({age}) cannot exceed the call it was measured inside ({call_micros})", + ); +} + +/// The age belongs to the batch that carried the observations rather than +/// running from the sentinel's own beginning: after a silence, the next batch +/// reports an age shorter than the silence that preceded it. An age that +/// accumulated over uptime would answer how long the sentinel had been +/// running, which is the wrong question — what a host needs is how stale the +/// evidence in front of it is. +/// +/// ´claim:readout:the-reported-age-belongs-to-its-own-batch-rather-than-running-from-the-sentinels-beginning´ +/// ´test:integration:batch-report-age-is-scoped-to-its-own-batch´ +#[test] +fn batch_report_age_is_scoped_to_its_own_batch() { + let mut s = multi_cell_sentinel(); + + let _first = s.ingest(&cell_values(0xF, 64)); + + // A silence, so that an age running from the sentinel's own beginning + // would have this stretch of quiet inside it. + let quiet = Duration::from_millis(200); + std::thread::sleep(quiet); + + let call_start = Instant::now(); + let second = s.ingest(&cell_values(0x1, 64)); + let call_micros = u64::try_from(call_start.elapsed().as_micros()).unwrap(); + + let age = second + .oldest_observation_age_micros + .expect("a batch carrying observations has an oldest one"); + let quiet_micros = u64::try_from(quiet.as_micros()).unwrap(); + + // An age scoped to its own batch is bounded by the call that produced the + // batch, whatever happened before the call. + assert!( + age <= call_micros, + "age ({age}) cannot exceed the call it was measured inside ({call_micros})", + ); + // The same bound stated against the silence: excluding the quiet means the + // age stays under it once the call's own duration is accounted for. + assert!( + age < quiet_micros + call_micros, + "age ({age}) should exclude the {quiet_micros} micros of silence before the batch arrived, allowing the {call_micros} micros the call itself took", + ); +} + +/// A batch with no observations has no oldest observation, so it reports no +/// age at all rather than an age of nothing. Absence and instantaneity are +/// different facts about a report and a zero would have conflated them: a host +/// watching for stale evidence has to be able to tell "nothing arrived" from +/// "what arrived was fresh". +/// +/// ´claim:readout:an-empty-batch-reports-no-age-at-all-rather-than-an-age-of-zero´ +/// ´test:integration:empty-ingest-reports-no-observation-age´ +#[test] +fn empty_ingest_reports_no_observation_age() { + let mut s = multi_cell_sentinel(); + + let quiet = s.ingest(&[]); + assert!( + quiet.oldest_observation_age_micros.is_none(), + "an empty batch has no oldest observation to be any age", + ); + + let busy = s.ingest(&cell_values(0xF, 8)); + assert!( + busy.oldest_observation_age_micros.is_some(), + "a batch that carried observations states their age", + ); +} diff --git a/packages/sentinel/tests/sentinel_u64.rs b/packages/sentinel/tests/sentinel_u64.rs new file mode 100644 index 000000000..be061259b --- /dev/null +++ b/packages/sentinel/tests/sentinel_u64.rs @@ -0,0 +1,455 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +//! Integration tests validating the **64-bit sentinel** path end-to-end. +//! +//! Constructs a [`Sentinel64`], feeds `u64` values, and verifies +//! that the generic machinery works correctly for a non-`u128` +//! coordinate type. +//! +//! The engine is generic over its coordinate type and its bit width, and the +//! crate ships two aliases over that one implementation — a sentinel for a +//! 128-bit domain and one for a 64-bit domain. Nothing about the narrower +//! alias is a separate code path, which is precisely why it needs its own +//! exercise: a width hard-coded somewhere instead of derived from the type +//! parameter would pass unnoticed at the width the rest of the suite happens +//! to use, and would show up here as a report that describes cells the +//! narrower domain does not have. +//! +//! The rule the width has to satisfy is that a cell's analysis width is the +//! domain width less its depth in the routing tree: the leading bits routing +//! has already resolved are not part of what that cell's tracker analyses. +//! Everything else the narrower sentinel promises is the same promise the +//! wider one makes — construction from any supported configuration, a +//! permanent root, reports partitioned into competitive cells and ancestors +//! and ordered by node handle, finite scores, and reproducibility from a +//! fixed seed. Parity, not a second contract. +//! +//! # Test index +//! +//! | Test | Area | Claim | +//! |------|------|-------| +//! | [`new_with_test_config`] | width | The narrower alias constructs from an ordinary configuration and comes up tracking the root alone, exactly as the wider one does. Configuration carries no width of its own — width is a property of the type — so the same settings serve either domain. | +//! | [`new_with_cold_config`] | width | cites (´claim:width:the-narrower-alias-constructs-under-any-supported-configuration-and-starts-with-the-root-alone´) | +//! | [`new_with_integration_config`] | width | cites (´claim:width:the-narrower-alias-constructs-under-any-supported-configuration-and-starts-with-the-root-alone´) | +//! | [`initial_state_has_root_only`] | width | cites (´claim:width:the-narrower-alias-constructs-under-any-supported-configuration-and-starts-with-the-root-alone´) | +//! | [`empty_ingest_produces_report`] | edge | cites (´claim:edge:an-empty-batch-yields-a-report-with-no-cells-and-moves-no-counter´) | +//! | [`ingest_returns_non_empty_report`] | engine | cites (´claim:engine:the-root-tracker-receives-every-observation-in-every-batch´) | +//! | [`multiple_ingests_accumulate`] | engine | Ingestion accumulates rather than restarting: after many further batches of the same traffic the sentinel tracks no fewer cells than it did after the first. A batch is an increment to standing state, so cells already earned are not dropped merely because another batch arrived. | +//! | [`analysis_widths_are_64_minus_depth`] | width | Every cell in the report, competitive or ancestor, analyses a width equal to the domain width less its own depth — here the narrower domain's width, at whatever depths the traffic reached. The bits routing has already resolved are constant within the cell and so carry no information for its tracker; what remains is the suffix, and its length is fixed by the depth. The width is read from the sentinel's type parameter rather than assumed, which is what makes the same arithmetic hold for either alias. | +//! | [`cell_reports_are_competitive`] | engine | A report separates the cells that earned their modelling from the ones carried along to complete an ancestor chain, and the first vector holds only the former. The distinction is what tells a reader which measurements reflect a deliberate investment, so it is expressed as two vectors rather than as a flag to be filtered on. | +//! | [`ancestor_reports_are_non_competitive`] | engine | cites (´claim:engine:cell-reports-hold-only-competitive-cells-and-ancestor-reports-only-non-competitive-ones´) | +//! | [`reports_sorted_by_gnode_id`] | determinism | cites (´claim:determinism:every-report-vector-comes-out-in-the-order-its-contract-states-so-a-reader-never-depends-on-visit-order´) | +//! | [`no_nan_in_scores`] | engine | Every reported score on every axis is a real number. The axes are ratios and standardised departures, so a variance that had collapsed to nothing or a basis that spanned no direction would surface as a non-number rather than as an obviously wrong value — which is why the absence of one is worth asserting across all four axes and both report vectors. | +//! | [`creates_cells_on_split`] | engine | cites (´claim:engine:traffic-in-separate-regions-splits-the-domain-so-more-than-the-root-is-tracked´) | +//! | [`cells_tracked_never_below_one`] | engine | The root tracker is permanent, before any traffic and after it. It is not selected on merit and cannot be displaced by the competition, because every ancestor chain has to terminate somewhere — so the tracked count has a floor of one and a host never meets a sentinel with nothing to report against. | +//! | [`min_value`] | edge | cites (´claim:edge:the-extremes-of-the-coordinate-domain-are-ordinary-observations´) | +//! | [`max_value`] | edge | cites (´claim:edge:the-extremes-of-the-coordinate-domain-are-ordinary-observations´) | +//! | [`sentinel_u64_min_and_max_together`] | edge | cites (´claim:edge:the-extremes-of-the-coordinate-domain-are-ordinary-observations´) | +//! | [`deterministic_output_for_same_input`] | determinism | cites (´claim:determinism:the-same-seed-and-the-same-data-reproduce-the-same-reports´) | + +mod common; + +use common::{cold_config, integration_config, test_config}; +use torrust_sentinel::{Sentinel64, SentinelConfig}; + +// ─── Helpers (u64-specific) ───────────────────────────────── + +/// Generate `count` values with the given leading nibble and +/// sequential low bits, analogous to `common::cell_values()` but +/// for the 64-bit domain. +fn cell_values_u64(nibble: u64, count: usize) -> Vec { + (0..count).map(|i| (nibble << 60) | (i as u64 + 1)).collect() +} + +// ── Construction ──────────────────────────────────────────── + +/// The narrower alias constructs from an ordinary configuration and comes up +/// tracking the root alone, exactly as the wider one does. Configuration +/// carries no width of its own — width is a property of the type — so the +/// same settings serve either domain. +/// +/// ´claim:width:the-narrower-alias-constructs-under-any-supported-configuration-and-starts-with-the-root-alone´ +/// ´test:integration:new-with-test-config´ +#[test] +fn new_with_test_config() { + let s = Sentinel64::new(test_config()).unwrap(); + assert_eq!(s.cells_tracked(), 1); +} + +/// The same holds with warming noise switched off entirely, which is the +/// configuration most likely to expose a width assumption: the root tracker +/// is built at the narrower width and never primed, and construction still +/// succeeds with one cell tracked. +/// +/// (´claim:width:the-narrower-alias-constructs-under-any-supported-configuration-and-starts-with-the-root-alone´) +/// ´test:integration:new-with-cold-config´ +#[test] +fn new_with_cold_config() { + let s = Sentinel64::new(cold_config()).unwrap(); + assert_eq!(s.cells_tracked(), 1); +} + +/// And again with a tighter rank budget and faster rank adaptation — the +/// settings that most directly govern the geometry a tracker maintains. +/// None of them is width-dependent, so the narrower sentinel accepts them +/// unchanged. +/// +/// (´claim:width:the-narrower-alias-constructs-under-any-supported-configuration-and-starts-with-the-root-alone´) +/// ´test:integration:new-with-integration-config´ +#[test] +fn new_with_integration_config() { + let s = Sentinel64::new(integration_config()).unwrap(); + assert_eq!(s.cells_tracked(), 1); +} + +/// Stating the initial condition on its own: before any traffic the +/// narrower sentinel tracks the root and nothing else, so every cell that +/// appears later was created by something observed. +/// +/// (´claim:width:the-narrower-alias-constructs-under-any-supported-configuration-and-starts-with-the-root-alone´) +/// ´test:integration:initial-state-has-root-only´ +#[test] +fn initial_state_has_root_only() { + let s = Sentinel64::new(test_config()).unwrap(); + assert_eq!(s.cells_tracked(), 1, "fresh sentinel should have only the root cell"); +} + +// ── Ingestion basics ──────────────────────────────────────── + +/// An empty batch at the narrower width is answered the same way as at the +/// wider one: a report naming no cells and no ancestors, rather than an +/// error or an absent report. The early return on an empty batch precedes +/// everything width-dependent, so the two aliases cannot diverge here. +/// +/// (´claim:edge:an-empty-batch-yields-a-report-with-no-cells-and-moves-no-counter´) +/// ´test:integration:empty-ingest-produces-report´ +#[test] +fn empty_ingest_produces_report() { + let mut s = Sentinel64::new(test_config()).unwrap(); + let report = s.ingest(&[]); + // Empty batch is valid — no observations, but struct is populated. + assert!(report.cell_reports.is_empty()); + assert!(report.ancestor_reports.is_empty()); +} + +/// A non-empty batch of narrow coordinates produces at least one report +/// entry, because the root contains every value in its domain whatever that +/// domain's width happens to be. +/// +/// (´claim:engine:the-root-tracker-receives-every-observation-in-every-batch´) +/// ´test:integration:ingest-returns-non-empty-report´ +#[test] +fn ingest_returns_non_empty_report() { + let mut s = Sentinel64::new(test_config()).unwrap(); + let values = cell_values_u64(0xF, 4); + let report = s.ingest(&values); + assert!( + !report.cell_reports.is_empty() || !report.ancestor_reports.is_empty(), + "ingesting values should produce at least one report entry", + ); +} + +/// Ingestion accumulates rather than restarting: after many further batches +/// of the same traffic the sentinel tracks no fewer cells than it did after +/// the first. A batch is an increment to standing state, so cells already +/// earned are not dropped merely because another batch arrived. +/// +/// ´claim:engine:repeated-ingestion-adds-to-standing-state-and-does-not-lose-cells-already-tracked´ +/// ´test:integration:multiple-ingests-accumulate´ +#[test] +fn multiple_ingests_accumulate() { + let mut s = Sentinel64::new(test_config()).unwrap(); + + s.ingest(&cell_values_u64(0xF, 8)); + let cells_after_first = s.cells_tracked(); + + for _ in 0..10 { + s.ingest(&cell_values_u64(0xF, 8)); + } + + assert!( + s.cells_tracked() >= cells_after_first, + "repeated ingestion should not lose cells", + ); +} + +// ── Report structure ──────────────────────────────────────── + +/// Every cell in the report, competitive or ancestor, analyses a width equal +/// to the domain width less its own depth — here the narrower domain's +/// width, at whatever depths the traffic reached. The bits routing has +/// already resolved are constant within the cell and so carry no +/// information for its tracker; what remains is the suffix, and its length +/// is fixed by the depth. The width is read from the sentinel's type +/// parameter rather than assumed, which is what makes the same arithmetic +/// hold for either alias. +/// +/// ´claim:width:a-cells-analysis-width-is-the-domain-width-less-its-depth´ +/// ´test:integration:analysis-widths-are-64-minus-depth´ +#[test] +fn analysis_widths_are_64_minus_depth() { + let mut s = Sentinel64::new(test_config()).unwrap(); + + // Feed diverse traffic to populate multiple depths. + for _ in 0..10 { + s.ingest(&[cell_values_u64(0xF, 4), cell_values_u64(0x1, 4)].concat()); + } + let report = s.ingest(&cell_values_u64(0xF, 8)); + + for cr in report.cell_reports.iter().chain(report.ancestor_reports.iter()) { + assert_eq!( + cr.analysis_width, + 64 - cr.depth as usize, + "analysis_width mismatch at depth {}", + cr.depth, + ); + } +} + +/// A report separates the cells that earned their modelling from the ones +/// carried along to complete an ancestor chain, and the first vector holds +/// only the former. The distinction is what tells a reader which +/// measurements reflect a deliberate investment, so it is expressed as two +/// vectors rather than as a flag to be filtered on. +/// +/// ´claim:engine:cell-reports-hold-only-competitive-cells-and-ancestor-reports-only-non-competitive-ones´ +/// ´test:integration:cell-reports-are-competitive´ +#[test] +fn cell_reports_are_competitive() { + let mut s = Sentinel64::new(test_config()).unwrap(); + + for _ in 0..10 { + s.ingest(&[cell_values_u64(0xA, 4), cell_values_u64(0x5, 4)].concat()); + } + let report = s.ingest(&cell_values_u64(0xA, 8)); + + for cr in &report.cell_reports { + assert!( + cr.is_competitive, + "cell_reports entry at depth {} is not competitive", + cr.depth + ); + } +} + +/// The complementary half of the partition: the ancestor vector holds no +/// cell that competed. A cell appears in exactly one of the two vectors, so +/// nothing is counted twice and nothing falls between them. +/// +/// (´claim:engine:cell-reports-hold-only-competitive-cells-and-ancestor-reports-only-non-competitive-ones´) +/// ´test:integration:ancestor-reports-are-non-competitive´ +#[test] +fn ancestor_reports_are_non_competitive() { + let mut s = Sentinel64::new(test_config()).unwrap(); + + for _ in 0..10 { + s.ingest(&[cell_values_u64(0xA, 4), cell_values_u64(0x5, 4)].concat()); + } + let report = s.ingest(&cell_values_u64(0xA, 8)); + + for ar in &report.ancestor_reports { + assert!( + !ar.is_competitive, + "ancestor_reports entry at depth {} is competitive", + ar.depth + ); + } +} + +/// Node-handle ordering is a property of the report and not of the domain +/// width: at the narrower width both vectors still come out strictly +/// ascending, so a reader compares runs positionally here exactly as it does +/// at the wider one. +/// +/// (´claim:determinism:every-report-vector-comes-out-in-the-order-its-contract-states-so-a-reader-never-depends-on-visit-order´) +/// ´test:integration:reports-sorted-by-gnode-id´ +#[test] +fn reports_sorted_by_gnode_id() { + let mut s = Sentinel64::new(test_config()).unwrap(); + + for _ in 0..10 { + s.ingest(&[cell_values_u64(0xF, 4), cell_values_u64(0x1, 4)].concat()); + } + let report = s.ingest(&cell_values_u64(0xF, 8)); + + for window in report.cell_reports.windows(2) { + assert!( + window[0].gnode_id < window[1].gnode_id, + "cell_reports not sorted: {:?} >= {:?}", + window[0].gnode_id, + window[1].gnode_id, + ); + } + for window in report.ancestor_reports.windows(2) { + assert!( + window[0].gnode_id < window[1].gnode_id, + "ancestor_reports not sorted: {:?} >= {:?}", + window[0].gnode_id, + window[1].gnode_id, + ); + } +} + +/// Every reported score on every axis is a real number. The axes are ratios +/// and standardised departures, so a variance that had collapsed to nothing +/// or a basis that spanned no direction would surface as a non-number rather +/// than as an obviously wrong value — which is why the absence of one is +/// worth asserting across all four axes and both report vectors. +/// +/// ´claim:engine:every-reported-score-is-a-real-number-and-never-a-non-number´ +/// ´test:integration:no-nan-in-scores´ +#[test] +fn no_nan_in_scores() { + let mut s = Sentinel64::new(test_config()).unwrap(); + + for _ in 0..10 { + s.ingest(&[cell_values_u64(0xF, 4), cell_values_u64(0x1, 4)].concat()); + } + let report = s.ingest(&cell_values_u64(0xF, 8)); + + for cr in report.cell_reports.iter().chain(report.ancestor_reports.iter()) { + assert!(!cr.scores.novelty.mean.is_nan(), "NaN novelty mean at depth {}", cr.depth); + assert!( + !cr.scores.displacement.mean.is_nan(), + "NaN displacement mean at depth {}", + cr.depth + ); + assert!(!cr.scores.surprise.mean.is_nan(), "NaN surprise mean at depth {}", cr.depth); + assert!(!cr.scores.coherence.mean.is_nan(), "NaN coherence mean at depth {}", cr.depth); + } +} + +// ── Cell management ───────────────────────────────────────── + +/// Splitting works the same way in the narrower domain: sustained traffic in +/// two separated regions, under a low split threshold, leaves the sentinel +/// tracking more than the root. Subdivision is driven by observation volume +/// against that threshold, and neither quantity depends on how wide the +/// coordinates are. +/// +/// (´claim:engine:traffic-in-separate-regions-splits-the-domain-so-more-than-the-root-is-tracked´) +/// ´test:integration:creates-cells-on-split´ +#[test] +fn creates_cells_on_split() { + let mut s = Sentinel64::new(SentinelConfig:: { + split_threshold: 10, + ..test_config() + }) + .unwrap(); + + // Feed diverse traffic across two leading nibbles to trigger splits. + for _ in 0..15 { + s.ingest(&[cell_values_u64(0xA, 20), cell_values_u64(0x5, 20)].concat()); + } + + assert!(s.cells_tracked() >= 2, "should have split beyond the root cell"); +} + +/// The root tracker is permanent, before any traffic and after it. It is not +/// selected on merit and cannot be displaced by the competition, because +/// every ancestor chain has to terminate somewhere — so the tracked count +/// has a floor of one and a host never meets a sentinel with nothing to +/// report against. +/// +/// ´claim:engine:the-root-tracker-is-permanent-so-the-tracked-count-never-falls-below-one´ +/// ´test:integration:cells-tracked-never-below-one´ +#[test] +fn cells_tracked_never_below_one() { + let mut s = Sentinel64::new(test_config()).unwrap(); + assert!(s.cells_tracked() >= 1, "root cell must always exist"); + + s.ingest(&cell_values_u64(0xF, 4)); + assert!(s.cells_tracked() >= 1, "root cell must persist after ingestion"); +} + +// ── Boundary values ───────────────────────────────────────── + +/// The bottom of the narrower domain is an ordinary observation too, counted +/// like any other. The extreme is defined by the width the sentinel was +/// instantiated at, so each alias has its own extremes and handles them the +/// same way. +/// +/// (´claim:edge:the-extremes-of-the-coordinate-domain-are-ordinary-observations´) +/// ´test:integration:min-value´ +#[test] +fn min_value() { + let mut s = Sentinel64::new(test_config()).unwrap(); + let report = s.ingest(&[0u64]); + assert_eq!(report.health.lifetime_observations, 1); +} + +/// A saturated narrow coordinate is likewise routed and counted normally — +/// the value that sets every bit the domain has, at the top of the interval +/// the root covers. +/// +/// (´claim:edge:the-extremes-of-the-coordinate-domain-are-ordinary-observations´) +/// ´test:integration:max-value´ +#[test] +fn max_value() { + let mut s = Sentinel64::new(test_config()).unwrap(); + let report = s.ingest(&[u64::MAX]); + assert_eq!(report.health.lifetime_observations, 1); +} + +/// Both extremes of the narrower domain in one batch are counted as two +/// ordinary observations, so the widest spread the domain admits is not a +/// case the engine treats specially. +/// +/// (´claim:edge:the-extremes-of-the-coordinate-domain-are-ordinary-observations´) +/// ´test:integration:sentinel-u64-min-and-max-together´ +#[test] +fn sentinel_u64_min_and_max_together() { + let mut s = Sentinel64::new(test_config()).unwrap(); + let report = s.ingest(&[0u64, u64::MAX]); + assert_eq!(report.health.lifetime_observations, 2); +} + +// ── Determinism ───────────────────────────────────────────── + +/// Reproducibility holds at the narrower width as well, and over structure +/// as well as over scores: two runs of one seeded configuration on one batch +/// agree batch by batch on how many cells were reported, on each cell's +/// depth, analysed width and learned rank, and on the scores themselves. The +/// shape of a run is as reproducible as its numbers. +/// +/// (´claim:determinism:the-same-seed-and-the-same-data-reproduce-the-same-reports´) +/// ´test:integration:deterministic-output-for-same-input´ +#[test] +fn deterministic_output_for_same_input() { + let cfg = SentinelConfig:: { + noise_seed: Some(42), + ..test_config() + }; + + let batch = cell_values_u64(0xF, 8); + + let run = |cfg: SentinelConfig| { + let mut s = Sentinel64::new(cfg).unwrap(); + let mut reports = Vec::new(); + for _ in 0..10 { + reports.push(s.ingest(&batch)); + } + reports + }; + + let a = run(cfg.clone()); + let b = run(cfg); + + assert_eq!(a.len(), b.len()); + for (ra, rb) in a.iter().zip(b.iter()) { + assert_eq!(ra.cell_reports.len(), rb.cell_reports.len()); + assert_eq!(ra.ancestor_reports.len(), rb.ancestor_reports.len()); + + for (ca, cb) in ra.cell_reports.iter().zip(rb.cell_reports.iter()) { + assert_eq!(ca.depth, cb.depth); + assert_eq!(ca.analysis_width, cb.analysis_width); + assert_eq!(ca.rank, cb.rank); + assert!( + (ca.scores.novelty.mean - cb.scores.novelty.mean).abs() < f64::EPSILON, + "novelty diverged at depth {}", + ca.depth, + ); + } + } +} diff --git a/packages/sentinel/tests/serde_roundtrip.rs b/packages/sentinel/tests/serde_roundtrip.rs new file mode 100644 index 000000000..7ef0f8832 --- /dev/null +++ b/packages/sentinel/tests/serde_roundtrip.rs @@ -0,0 +1,441 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +//! # Test index +//! +//! | Test | Area | Claim | +//! |------|------|-------| +//! | [`batch_report_json_round_trip`] | serde | A whole batch report survives being written out and read back with its structure intact: the cell, ancestor and coordination lists return at the same lengths, and the summary counts are unchanged. The nesting is deep and generic over the coordinate type, so this is the claim that the readout can be handed to a host in another process at all. | +//! | [`empty_batch_report_json_round_trip`] | serde | cites (´claim:serde:a-whole-batch-report-survives-a-round-trip-with-its-structure-intact´) | +//! | [`batch_report_without_age_field_deserializes`] | serde | A payload written before the age existed still reads back, its age absent rather than filled in: a report object simply missing that field deserialises with no age and everything else intact. Absence is the honest reading of an older payload — a number would claim a measurement the sender never made and could not have sent. | +//! | [`analysis_set_summary_json_round_trip`] | serde | A report component can be carried on its own, not only inside the batch that produced it: the analysis-set summary alone round-trips with its sizes, its depth span and its skipped-cell count intact. A host forwarding structural state to one consumer and scores to another need not ship the whole readout to either. | +//! | [`cell_report_json_round_trip`] | serde | Optional detail crosses the boundary as present or absent rather than collapsing: each cell report returns with its depth, counts and scores, and with per-sample detail still attached and still the same length when the host enabled it. An option that silently became absent in transit would look to the receiver exactly like a host that had never asked for it. | +//! | [`contour_snapshot_json_round_trip`] | serde | cites (´claim:serde:every-report-component-can-be-carried-on-its-own-not-only-inside-the-batch-that-produced-it´) | +//! | [`coordination_report_json_round_trip`] | serde | cites (´claim:serde:optional-detail-crosses-the-boundary-as-present-or-absent-rather-than-collapsing´) | +//! | [`health_report_json_round_trip`] | serde | cites (´claim:serde:every-report-component-can-be-carried-on-its-own-not-only-inside-the-batch-that-produced-it´) | +//! | [`cell_inspection_json_round_trip`] | serde | A cell inspection, taken by handle between batches rather than emitted by one, transports like any other readout: its depth, width and rank return exact and its per-axis baselines come back intact. Both ways of getting state out of the sentinel are equally publishable, so a host is not forced through the batch path merely to be able to forward what it learned. | +//! | [`member_score_json_round_trip`] | serde | Integers cross exactly and floating-point values within one unit in the last place — the residue of passing through a decimal text form, checked here on a member score whose interval runs to the extreme of the coordinate domain. The cell identity is integral and so is preserved outright, while the scores are close enough that no threshold a host applies can turn on the difference. | +//! | [`sample_score_json_round_trip`] | serde | cites (´claim:serde:a-float-returns-within-one-unit-in-the-last-place-of-the-value-that-was-sent´) | + +//! Transport tests for the readouts the sentinel hands back: what survives +//! being written out and read back in, under the optional serialisation +//! feature. +//! +//! The sentinel measures and the host decides, and the host is often +//! somewhere else — another process, a log, a queue, a store consulted long +//! after the batch that produced the numbers. Serialisation is therefore not +//! a convenience bolted onto the report types but part of how the +//! measurement reaches whoever acts on it, and a field that does not survive +//! the trip is a measurement that was never really published. +//! +//! Two properties are worth stating plainly. Every report component +//! serialises on its own as well as inside the batch that produced it, so a +//! host can forward just the health snapshot or just one cell without +//! carrying the whole readout. And the values that come back are the values +//! that went out — floating-point figures within one unit in the last place, +//! the residue of passing through a decimal text form, which is close enough +//! that no comparison a host makes can turn on the difference. +//! +//! The reports are generated from a sentinel driven only far enough to +//! produce a structurally complete readout — cells, ancestors, coordination +//! contexts and per-sample detail all populated. What is under test here is +//! the shape of what crosses the boundary, not whether the statistics behind +//! it have converged. + +#![cfg(feature = "serde")] + +mod common; + +use common::{ScenarioBuilder, cell_values, integration_config}; +use torrust_sentinel::{ + AnalysisSetSummary, BatchReport, CellInspection, CellReport, ContourSnapshot, CoordinationReport, HealthReport, MemberScore, + SampleScore, Sentinel128, SentinelConfig, +}; + +/// Assert two f64s are within 1 ULP (unit in the last place) of each other. +/// +/// JSON round-trips through decimal representation can lose 1 ULP for +/// certain edge-case values at the boundary of shortest-representation +/// algorithms (e.g. when the 15-digit and 16-digit decimal forms straddle +/// a float boundary). +fn assert_f64_ulp(label: &str, a: f64, b: f64) { + let a_bits = a.to_bits(); + let b_bits = b.to_bits(); + let ulp_dist = a_bits.abs_diff(b_bits); + assert!( + ulp_dist <= 1, + "{label}: {a} vs {b}, ULP distance = {ulp_dist} (bits: {a_bits} vs {b_bits})" + ); +} + +/// Produce a report with enough structure for round-trip testing. +/// +/// Only 5 warm-up batches — serde tests need a structurally complete +/// report (cells, ancestors, coordination), not a statistically +/// converged one. See ADR-S-012. +fn rich_report() -> BatchReport { + let cfg = SentinelConfig:: { + split_threshold: 10, + per_sample_scores: true, + ..integration_config() + }; + let (mut s, _) = ScenarioBuilder::new() + .config(cfg) + .seed_range(0xA, 16) + .seed_range(0xB, 16) + .warm_batches(4) + .build_with_reports(); + + s.ingest(&[cell_values(0xA, 8), cell_values(0xB, 8)].concat()) +} + +// ─── Composite reports ────────────────────────────────────── + +/// A whole batch report survives being written out and read back with its +/// structure intact: the cell, ancestor and coordination lists return at the +/// same lengths, and the summary counts are unchanged. The nesting is deep +/// and generic over the coordinate type, so this is the claim that the +/// readout can be handed to a host in another process at all. +/// +/// ´claim:serde:a-whole-batch-report-survives-a-round-trip-with-its-structure-intact´ +/// ´test:integration:batch-report-json-round-trip´ +#[test] +fn batch_report_json_round_trip() { + let report = rich_report(); + + let json = serde_json::to_string(&report).unwrap(); + let deser: BatchReport = serde_json::from_str(&json).unwrap(); + + assert_eq!(report.cell_reports.len(), deser.cell_reports.len()); + assert_eq!(report.ancestor_reports.len(), deser.ancestor_reports.len()); + assert_eq!(report.coordination_reports.len(), deser.coordination_reports.len()); + assert_eq!( + report.analysis_set_summary.competitive_size, + deser.analysis_set_summary.competitive_size, + ); + assert_eq!(report.analysis_set_summary.full_size, deser.analysis_set_summary.full_size); + assert_eq!(report.oldest_observation_age_micros, deser.oldest_observation_age_micros); +} + +/// The empty end of the same trip: a report from an ingest with no +/// observations comes back with its lists present and empty rather than +/// missing. Emptiness is carried across the boundary as a fact, so a +/// receiving host distinguishes "nothing reported" from a truncated or +/// malformed readout. +/// +/// (´claim:serde:a-whole-batch-report-survives-a-round-trip-with-its-structure-intact´) +/// ´test:integration:empty-batch-report-json-round-trip´ +#[test] +fn empty_batch_report_json_round_trip() { + let mut s = Sentinel128::new(integration_config()).unwrap(); + let report = s.ingest(&[]); + + let json = serde_json::to_string(&report).unwrap(); + let deser: BatchReport = serde_json::from_str(&json).unwrap(); + + assert!(deser.cell_reports.is_empty()); + assert!(deser.ancestor_reports.is_empty()); + assert!(deser.coordination_reports.is_empty()); + assert!(deser.oldest_observation_age_micros.is_none()); +} + +/// A payload written before the age existed still reads back, its age absent +/// rather than filled in: a report object simply missing that field +/// deserialises with no age and everything else intact. Absence is the honest +/// reading of an older payload — a number would claim a measurement the sender +/// never made and could not have sent. +/// +/// ´claim:serde:a-payload-lacking-the-observation-age-reads-back-with-no-age-rather-than-a-fabricated-one´ +/// ´test:integration:batch-report-without-age-field-deserializes´ +#[test] +fn batch_report_without_age_field_deserializes() { + let report = rich_report(); + assert!( + report.oldest_observation_age_micros.is_some(), + "a live batch states its age, so there is something to take away", + ); + + // Cut the field back out of the text. The surgery is textual because a + // 128-bit coordinate does not fit `serde_json::Value`'s number, so the + // payload cannot be taken apart as a tree and put back together. + let json = serde_json::to_string(&report).unwrap(); + let key = ",\"oldest_observation_age_micros\":"; + let at = json.find(key).expect("a live batch's payload carries the age field"); + let closing = json.rfind('}').expect("a batch report serialises as a JSON object"); + let cut = &json[at + key.len()..closing]; + assert!( + cut.parse::().is_ok(), + "the age should be the object's last member, but {cut} stands between it and the end", + ); + let older_payload = format!("{}{}", &json[..at], &json[closing..]); + + let deser: BatchReport = serde_json::from_str(&older_payload).unwrap(); + + assert!(deser.oldest_observation_age_micros.is_none()); + assert_eq!(report.cell_reports.len(), deser.cell_reports.len()); + assert_eq!(report.ancestor_reports.len(), deser.ancestor_reports.len()); + assert_eq!(report.analysis_set_summary.full_size, deser.analysis_set_summary.full_size,); +} + +// ─── Component reports (alphabetical) ─────────────────────── + +/// A report component can be carried on its own, not only inside the batch +/// that produced it: the analysis-set summary alone round-trips with its +/// sizes, its depth span and its skipped-cell count intact. A host +/// forwarding structural state to one consumer and scores to another need +/// not ship the whole readout to either. +/// +/// ´claim:serde:every-report-component-can-be-carried-on-its-own-not-only-inside-the-batch-that-produced-it´ +/// ´test:integration:analysis-set-summary-json-round-trip´ +#[test] +fn analysis_set_summary_json_round_trip() { + let report = rich_report(); + + let json = serde_json::to_string(&report.analysis_set_summary).unwrap(); + let deser: AnalysisSetSummary = serde_json::from_str(&json).unwrap(); + + assert_eq!(report.analysis_set_summary.competitive_size, deser.competitive_size); + assert_eq!(report.analysis_set_summary.full_size, deser.full_size); + assert_eq!(report.analysis_set_summary.investment_set_size, deser.investment_set_size); + assert_eq!(report.analysis_set_summary.depth_range, deser.depth_range); + assert_eq!( + report.analysis_set_summary.degenerate_cells_skipped, + deser.degenerate_cells_skipped, + ); +} + +/// Optional detail crosses the boundary as present or absent rather than +/// collapsing: each cell report returns with its depth, counts and scores, +/// and with per-sample detail still attached and still the same length when +/// the host enabled it. An option that silently became absent in transit +/// would look to the receiver exactly like a host that had never asked for +/// it. +/// +/// ´claim:serde:optional-detail-crosses-the-boundary-as-present-or-absent-rather-than-collapsing´ +/// ´test:integration:cell-report-json-round-trip´ +#[test] +fn cell_report_json_round_trip() { + let report = rich_report(); + + for cr in &report.cell_reports { + let json = serde_json::to_string(cr).unwrap(); + let deser: CellReport = serde_json::from_str(&json).unwrap(); + + assert_eq!(cr.depth, deser.depth); + assert_eq!(cr.sample_count, deser.sample_count); + assert_eq!(cr.rank, deser.rank); + assert_eq!(cr.is_competitive, deser.is_competitive); + assert_f64_ulp("novelty.mean", cr.scores.novelty.mean, deser.scores.novelty.mean); + assert_f64_ulp( + "novelty.clip_pressure", + cr.scores.novelty.clip_pressure, + deser.scores.novelty.clip_pressure, + ); + // per_sample populated when per_sample_scores is enabled. + assert_eq!(cr.per_sample.is_some(), deser.per_sample.is_some()); + if let (Some(orig), Some(rt)) = (&cr.per_sample, &deser.per_sample) { + assert_eq!(orig.len(), rt.len()); + } + } +} + +/// The contour travels alone as well, its plateau and cell counts exact and +/// its accumulated importance intact. This is the section a host watching +/// spatial structure over time would forward on its own, so it has to be +/// meaningful detached from the scores it shipped beside. +/// +/// (´claim:serde:every-report-component-can-be-carried-on-its-own-not-only-inside-the-batch-that-produced-it´) +/// ´test:integration:contour-snapshot-json-round-trip´ +#[test] +fn contour_snapshot_json_round_trip() { + let report = rich_report(); + + let json = serde_json::to_string(&report.contour).unwrap(); + let deser: ContourSnapshot = serde_json::from_str(&json).unwrap(); + + assert_eq!(report.contour.plateau_count, deser.plateau_count); + assert_eq!(report.contour.cell_count, deser.cell_count); + assert_f64_ulp("total_importance", report.contour.total_importance, deser.total_importance); +} + +/// The optional-detail rule holds a tier up, where the attachment is a list +/// of per-member scores rather than per-sample ones: each coordination +/// report returns with its group size and rank, and with its membership +/// still present and the same length. The identity of who was in the group +/// is what makes a group finding actionable, so it must not be the part that +/// transport drops. +/// +/// (´claim:serde:optional-detail-crosses-the-boundary-as-present-or-absent-rather-than-collapsing´) +/// ´test:integration:coordination-report-json-round-trip´ +#[test] +fn coordination_report_json_round_trip() { + let report = rich_report(); + + for coord in &report.coordination_reports { + let json = serde_json::to_string(coord).unwrap(); + let deser: CoordinationReport = serde_json::from_str(&json).unwrap(); + + assert_eq!(coord.depth, deser.depth); + assert_eq!(coord.cells_reporting, deser.cells_reporting); + assert_eq!(coord.rank, deser.rank); + assert_f64_ulp("novelty.mean", coord.scores.novelty.mean, deser.scores.novelty.mean); + // per_member populated when per_sample_scores is enabled. + assert_eq!(coord.per_member.is_some(), deser.per_member.is_some()); + if let (Some(orig), Some(rt)) = (&coord.per_member, &deser.per_member) { + assert_eq!(orig.len(), rt.len()); + } + } +} + +/// Health travels alone too, and it is the component with the most nesting: +/// the tracker counts return exactly and the clip-pressure distribution +/// inside it survives with its ends and its mean. Operational state is +/// typically the piece a host routes to a different consumer from the +/// scores, which is precisely why it has to stand on its own. +/// +/// (´claim:serde:every-report-component-can-be-carried-on-its-own-not-only-inside-the-batch-that-produced-it´) +/// ´test:integration:health-report-json-round-trip´ +#[test] +fn health_report_json_round_trip() { + let report = rich_report(); + + let json = serde_json::to_string(&report.health).unwrap(); + let deser: HealthReport = serde_json::from_str(&json).unwrap(); + + assert_eq!(report.health.lifetime_observations, deser.lifetime_observations); + assert_eq!(report.health.active_trackers, deser.active_trackers); + assert_eq!(report.health.active_competitive_trackers, deser.active_competitive_trackers); + assert_eq!(report.health.active_ancestor_trackers, deser.active_ancestor_trackers); + assert_eq!(report.health.investment_set_size, deser.investment_set_size); + assert_f64_ulp( + "clip_pressure_distribution.min", + report.health.clip_pressure_distribution.min, + deser.clip_pressure_distribution.min, + ); + assert_f64_ulp( + "clip_pressure_distribution.max", + report.health.clip_pressure_distribution.max, + deser.clip_pressure_distribution.max, + ); + assert_f64_ulp( + "clip_pressure_distribution.mean", + report.health.clip_pressure_distribution.mean, + deser.clip_pressure_distribution.mean, + ); +} + +// ─── Inspection types ─────────────────────────────────────── + +/// A cell inspection, taken by handle between batches rather than emitted by +/// one, transports like any other readout: its depth, width and rank return +/// exact and its per-axis baselines come back intact. Both ways of getting +/// state out of the sentinel are equally publishable, so a host is not +/// forced through the batch path merely to be able to forward what it +/// learned. +/// +/// ´claim:serde:an-inspection-taken-outside-the-batch-path-transports-like-any-other-readout´ +/// ´test:integration:cell-inspection-json-round-trip´ +#[test] +fn cell_inspection_json_round_trip() { + let cfg = SentinelConfig:: { + split_threshold: 10, + ..integration_config() + }; + let (s, _) = ScenarioBuilder::new() + .config(cfg) + .seed_range(0xA, 20) + .warm_batches(4) + .build_with_reports(); + + let root = s.graph().g_root(); + let inspection = s.inspect_cell(root).unwrap(); + + let json = serde_json::to_string(&inspection).unwrap(); + let deser: CellInspection = serde_json::from_str(&json).unwrap(); + + assert_eq!(inspection.depth, deser.depth); + assert_eq!(inspection.analysis_width, deser.analysis_width); + assert_eq!(inspection.rank, deser.rank); + assert_f64_ulp( + "baselines.novelty.mean", + inspection.baselines.novelty.mean, + deser.baselines.novelty.mean, + ); +} + +// ─── Leaf types ───────────────────────────────────────────── + +/// Integers cross exactly and floating-point values within one unit in the +/// last place — the residue of passing through a decimal text form, checked +/// here on a member score whose interval runs to the extreme of the +/// coordinate domain. The cell identity is integral and so is preserved +/// outright, while the scores are close enough that no threshold a host +/// applies can turn on the difference. +/// +/// ´claim:serde:a-float-returns-within-one-unit-in-the-last-place-of-the-value-that-was-sent´ +/// ´test:integration:member-score-json-round-trip´ +#[test] +fn member_score_json_round_trip() { + let ms = MemberScore:: { + cell_start: 0, + cell_end: u128::MAX / 2, + cell_depth: 1, + novelty: 0.5, + displacement: 0.3, + surprise: 0.7, + coherence: 0.9, + novelty_z: 1.0, + displacement_z: -0.5, + surprise_z: 2.1, + coherence_z: 0.0, + }; + + let json = serde_json::to_string(&ms).unwrap(); + let deser: MemberScore = serde_json::from_str(&json).unwrap(); + + assert_eq!(ms.cell_start, deser.cell_start); + assert_eq!(ms.cell_end, deser.cell_end); + assert_eq!(ms.cell_depth, deser.cell_depth); + assert_f64_ulp("novelty", ms.novelty, deser.novelty); + assert_f64_ulp("displacement", ms.displacement, deser.displacement); + assert_f64_ulp("surprise", ms.surprise, deser.surprise); + assert_f64_ulp("coherence", ms.coherence, deser.coherence); + assert_f64_ulp("novelty_z", ms.novelty_z, deser.novelty_z); + assert_f64_ulp("displacement_z", ms.displacement_z, deser.displacement_z); + assert_f64_ulp("surprise_z", ms.surprise_z, deser.surprise_z); + assert_f64_ulp("coherence_z", ms.coherence_z, deser.coherence_z); +} + +/// The same tolerance holds for values the engine actually computed rather +/// than ones chosen by hand: every per-observation score the sentinel +/// produced, raw and standardised on all four axes, returns within a unit in +/// the last place. Real scores are arbitrary bit patterns rather than tidy +/// decimals, which is the case a text encoding is most likely to round. +/// +/// (´claim:serde:a-float-returns-within-one-unit-in-the-last-place-of-the-value-that-was-sent´) +/// ´test:integration:sample-score-json-round-trip´ +#[test] +fn sample_score_json_round_trip() { + let report = rich_report(); + + // Extract SampleScores from the first cell with per-sample data. + let samples = report + .cell_reports + .iter() + .find_map(|cr| cr.per_sample.as_ref()) + .expect("per_sample_scores is enabled; at least one cell should have samples"); + + for ss in samples { + let json = serde_json::to_string(ss).unwrap(); + let deser: SampleScore = serde_json::from_str(&json).unwrap(); + + assert_f64_ulp("novelty", ss.novelty, deser.novelty); + assert_f64_ulp("displacement", ss.displacement, deser.displacement); + assert_f64_ulp("surprise", ss.surprise, deser.surprise); + assert_f64_ulp("coherence", ss.coherence, deser.coherence); + assert_f64_ulp("novelty_z", ss.novelty_z, deser.novelty_z); + assert_f64_ulp("displacement_z", ss.displacement_z, deser.displacement_z); + assert_f64_ulp("surprise_z", ss.surprise_z, deser.surprise_z); + assert_f64_ulp("coherence_z", ss.coherence_z, deser.coherence_z); + } +} diff --git a/packages/sentinel/tests/spatial_decay.rs b/packages/sentinel/tests/spatial_decay.rs new file mode 100644 index 000000000..e1cece78e --- /dev/null +++ b/packages/sentinel/tests/spatial_decay.rs @@ -0,0 +1,422 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +//! Spatial decay — the only way the sentinel forgets. +//! +//! Importance in the spatial layer only ever accumulates. Every observed +//! value adds one unit of volume, and nothing in the scoring path ever +//! subtracts from it, so the graph on its own has no notion of the past +//! mattering less than the present. Forgetting is an operation the host +//! performs deliberately: [`decay()`] rescales the whole graph, and +//! [`decay_subtree()`] rescales one region, which is what a regime change or +//! a suspected poisoning confined to part of the domain calls for. +//! +//! One control covers both directions. A factor below one fades accumulated +//! standing, a factor above one reinforces it, and exactly one is the +//! identity — so a host can put decay on a fixed schedule and let the factor +//! decide whether anything happens on a given tick. The second parameter +//! varies that factor by depth, allowing coarse structure to be held while +//! fine detail is let go. Neither parameter is clamped: an out-of-range +//! request means the host's temporal policy is wrong, and quietly repairing +//! it would hide the mistake rather than surface it. +//! +//! Decay stops at the spatial layer, and that is the feed-forward invariant +//! seen from its other side. Scores never flow back into importance; decayed +//! importance never reaches forward into the models. No tracker is destroyed, +//! no subspace or baseline is touched, and the record of how much has been +//! ingested is untouched — what changes is only the ranking that decides +//! where future modelling effort goes, and that change is picked up at the +//! next ingest rather than applied eagerly. +//! +//! [`decay()`]: torrust_sentinel::SpectralSentinel::decay +//! [`decay_subtree()`]: torrust_sentinel::SpectralSentinel::decay_subtree +//! +//! # Test index +//! +//! | Test | Area | Claim | +//! |------|------|-------| +//! | [`decay_on_empty_graph_is_noop`] | decay | Decay scales what has been accumulated, so on a sentinel that has observed nothing there is nothing to scale and nothing to go wrong. A host can put decay on a timer before any traffic arrives without special-casing the empty graph. | +//! | [`decay_at_attenuation_one_is_noop`] | decay | An attenuation of one leaves a populated graph exactly as it was. The identity is inside the parameter's range rather than outside it, so "decay by nothing this tick" is expressed with the same call as any other policy, and a host's schedule needs no branch around it. | +//! | [`decay_reduces_total_sum`] | decay | A factor below one reduces the importance the graph has accumulated. This is the ordinary case and the reason the operation exists: standing earned by past traffic is worth less after a decay, so cells that have gone quiet drift down the ranking instead of holding their place forever on history. | +//! | [`repeated_decay_eventually_zeroes_integer_counters`] | decay | Importance is held in integers, so repeated halving does not approach zero asymptotically — it arrives there. A long run of decays with no intervening traffic empties the graph completely, which means a region that stops being observed is eventually forgotten outright rather than leaving an ever-smaller residue that still outranks a genuinely new cell. | +//! | [`decay_zero_attenuation_zeroes_graph`] | decay | Zero is the bottom of the attenuation range and reaches in one call what repeated halving reaches slowly: all accumulated importance is gone. It is the operation for declaring the spatial history worthless — after a suspected poisoning, say — without discarding the models that history produced. | +//! | [`amplification_increases_total_sum`] | decay | The factor is not restricted to shrinking. Above one it increases the accumulated importance instead, which lets a host reinforce a subtree it knows to be worth watching. One operation therefore spans both temporal policies — letting the past fade and boosting a region's standing — with the identity sitting between them. | +//! | [`selective_q_preserves_more_coarse_structure`] | decay | Selectivity makes the decay factor depend on a node's depth rather than applying one rate to the whole tree, so fine detail can be released while the coarse division of the domain is held. Two sentinels given identical traffic start from identical importance — the spatial layer is driven by volume alone — and diverge only through the selectivity each is then decayed with; both keep importance standing afterwards. | +//! | [`max_selectivity_q_one_is_valid`] | decay | The selectivity range is closed at its top end: maximum selectivity is a valid request, not one step past the edge, and under an attenuating factor it cannot leave the graph holding more than it started with. The extreme of the parameter is usable rather than merely almost-reachable. | +//! | [`decay_does_not_affect_tracker_count`] | decay | Decay reaches the spatial accounting and stops there. Halving every cell's importance destroys no tracker: the models a sentinel has built are not the same asset as the standing that justified building them, and forgetting the second does not throw away the first. This is the feed-forward invariant seen from the temporal side — importance flows into modelling decisions, never the reverse, so rescaling it cannot reach the models. | +//! | [`decay_does_not_change_lifetime_observations`] | decay | cites (´claim:decay:decay-rescales-spatial-importance-only-and-never-the-models-or-the-record-of-what-was-observed´) | +//! | [`decay_subtree_at_root_matches_global_decay`] | decay | There is one decay operation, not two. Targeting the subtree at the graph root produces exactly the importance a global decay produces, because the global call is defined as the subtree call made at the root. The targeted form is the general one, and the whole-graph form its degenerate case. | +//! | [`decay_subtree_affects_only_targeted_subtree`] | decay | Aimed below the root, decay is bounded by its target: the region named loses standing while everything outside it keeps what it had, so more importance survives than the same factor applied globally. That containment is what makes the operation usable for a regime change in one part of the domain — the rest of the graph does not have to be punished to let one region re-form. | +//! | [`decay_then_ingest_preserves_invariants`] | decay | Decay leaves the sentinel in a state the next batch can be scored from. It changes the rankings the selector reads but invalidates nothing eagerly; the analysis set is simply recomputed at the start of the following ingest, and the report that comes out satisfies every structural invariant. Forgetting is therefore composable with ordinary operation rather than something to be sequenced carefully around it. | +//! | [`decay_panics_on_negative_attenuation`] | decay | A negative attenuation has no meaning — importance cannot be scaled through zero into a negative standing — and the call refuses it outright instead of clamping it to the nearest sensible value. Temporal policy is the host's, so a nonsensical factor is a bug in that policy and is reported as one. | +//! | [`decay_panics_on_nan_attenuation`] | decay | cites (´claim:decay:an-attenuation-outside-the-non-negative-reals-is-refused-rather-than-quietly-repaired´) | +//! | [`decay_panics_on_negative_q`] | decay | Selectivity runs from uniform to fully depth-weighted, and below that range there is nothing to mean. A negative request is refused rather than treated as uniform, because a host that computed it did not intend uniformity. | +//! | [`decay_panics_on_q_above_one`] | decay | cites (´claim:decay:a-selectivity-outside-the-unit-interval-is-refused-rather-than-quietly-repaired´) | +//! | [`decay_panics_on_nan_q`] | decay | cites (´claim:decay:a-selectivity-outside-the-unit-interval-is-refused-rather-than-quietly-repaired´) | + +mod common; + +use common::{assert_invariants, seeded_sentinel, test_config}; +use torrust_sentinel::{Sentinel128, SentinelConfig}; + +// ── Identity & no-ops ────────────────────────────────────── + +/// Decay scales what has been accumulated, so on a sentinel that has observed +/// nothing there is nothing to scale and nothing to go wrong. A host can put +/// decay on a timer before any traffic arrives without special-casing the +/// empty graph. +/// +/// ´claim:decay:decaying-a-graph-that-has-observed-nothing-leaves-nothing-to-scale´ +/// ´test:integration:decay-on-empty-graph-is-noop´ +#[test] +fn decay_on_empty_graph_is_noop() { + let mut s = Sentinel128::new(test_config()).unwrap(); + assert_eq!(s.graph().total_sum(), 0); + + s.decay(0.5, 0.0); + assert_eq!(s.graph().total_sum(), 0); +} + +/// An attenuation of one leaves a populated graph exactly as it was. The +/// identity is inside the parameter's range rather than outside it, so +/// "decay by nothing this tick" is expressed with the same call as any other +/// policy, and a host's schedule needs no branch around it. +/// +/// ´claim:decay:an-attenuation-of-one-is-the-identity-so-a-scheduled-decay-needs-no-special-case-for-doing-nothing´ +/// ´test:integration:decay-at-attenuation-one-is-noop´ +#[test] +fn decay_at_attenuation_one_is_noop() { + let mut s = seeded_sentinel(); + let before = s.graph().total_sum(); + + s.decay(1.0, 0.0); + assert_eq!(s.graph().total_sum(), before); +} + +// ── Attenuation (0 < att < 1) ────────────────────────────── + +/// A factor below one reduces the importance the graph has accumulated. This +/// is the ordinary case and the reason the operation exists: standing earned +/// by past traffic is worth less after a decay, so cells that have gone quiet +/// drift down the ranking instead of holding their place forever on history. +/// +/// ´claim:decay:a-factor-below-one-reduces-accumulated-importance-so-quiet-cells-lose-standing´ +/// ´test:integration:decay-reduces-total-sum´ +#[test] +fn decay_reduces_total_sum() { + let mut s = seeded_sentinel(); + let before = s.graph().total_sum(); + assert!(before > 0); + + s.decay(0.5, 0.0); + assert!(s.graph().total_sum() < before); +} + +/// Importance is held in integers, so repeated halving does not approach zero +/// asymptotically — it arrives there. A long run of decays with no +/// intervening traffic empties the graph completely, which means a region +/// that stops being observed is eventually forgotten outright rather than +/// leaving an ever-smaller residue that still outranks a genuinely new cell. +/// +/// ´claim:decay:repeated-attenuation-of-integer-importance-reaches-exactly-zero-rather-than-an-endless-tail´ +/// ´test:integration:repeated-decay-eventually-zeroes-integer-counters´ +#[test] +fn repeated_decay_eventually_zeroes_integer_counters() { + let mut s = seeded_sentinel(); + assert!(s.graph().total_sum() > 0); + + for _ in 0..100 { + s.decay(0.5, 0.0); + } + assert_eq!(s.graph().total_sum(), 0); +} + +// ── Zero attenuation ─────────────────────────────────────── + +/// Zero is the bottom of the attenuation range and reaches in one call what +/// repeated halving reaches slowly: all accumulated importance is gone. It is +/// the operation for declaring the spatial history worthless — after a +/// suspected poisoning, say — without discarding the models that history +/// produced. +/// +/// ´claim:decay:an-attenuation-of-zero-erases-all-accumulated-importance-in-a-single-call´ +/// ´test:integration:decay-zero-attenuation-zeroes-graph´ +#[test] +fn decay_zero_attenuation_zeroes_graph() { + let mut s = seeded_sentinel(); + assert!(s.graph().total_sum() > 0); + + s.decay(0.0, 0.0); + assert_eq!(s.graph().total_sum(), 0); +} + +// ── Amplification (att > 1) ──────────────────────────────── + +/// The factor is not restricted to shrinking. Above one it increases the +/// accumulated importance instead, which lets a host reinforce a subtree it +/// knows to be worth watching. One operation therefore spans both temporal +/// policies — letting the past fade and boosting a region's standing — with +/// the identity sitting between them. +/// +/// ´claim:decay:a-factor-above-one-amplifies-instead-of-attenuating-so-one-operation-covers-both-directions´ +/// ´test:integration:amplification-increases-total-sum´ +#[test] +fn amplification_increases_total_sum() { + let mut s = seeded_sentinel(); + let before = s.graph().total_sum(); + + s.decay(2.0, 0.0); + assert!(s.graph().total_sum() > before); +} + +// ── Depth selectivity (q parameter) ──────────────────────── + +/// Selectivity makes the decay factor depend on a node's depth rather than +/// applying one rate to the whole tree, so fine detail can be released while +/// the coarse division of the domain is held. Two sentinels given identical +/// traffic start from identical importance — the spatial layer is driven by +/// volume alone — and diverge only through the selectivity each is then +/// decayed with; both keep importance standing afterwards. +/// +/// ´claim:decay:depth-selectivity-varies-the-factor-by-depth-so-coarse-structure-can-outlive-fine-detail´ +/// ´test:integration:selective-q-preserves-more-coarse-structure´ +#[test] +fn selective_q_preserves_more_coarse_structure() { + let cfg = SentinelConfig:: { + split_threshold: 5, + ..test_config() + }; + + let mut s_uniform = Sentinel128::new(cfg.clone()).unwrap(); + let mut s_selective = Sentinel128::new(cfg).unwrap(); + + let values: Vec = (0..100).collect(); + s_uniform.ingest(&values); + s_selective.ingest(&values); + + assert_eq!(s_uniform.graph().total_sum(), s_selective.graph().total_sum()); + + // Only meaningful if splits occurred (otherwise q has no effect). + if s_uniform.graph().node_count() > 1 { + s_uniform.decay(0.5, 0.0); + s_selective.decay(0.5, 0.5); + + // Both should decay, but selective decay may produce a + // different total because per-depth factors differ. + assert!(s_uniform.graph().total_sum() > 0); + assert!(s_selective.graph().total_sum() > 0); + } +} + +/// The selectivity range is closed at its top end: maximum selectivity is a +/// valid request, not one step past the edge, and under an attenuating factor +/// it cannot leave the graph holding more than it started with. The extreme +/// of the parameter is usable rather than merely almost-reachable. +/// +/// ´claim:decay:the-selectivity-range-is-closed-so-full-selectivity-is-a-valid-request´ +/// ´test:integration:max-selectivity-q-one-is-valid´ +#[test] +fn max_selectivity_q_one_is_valid() { + let mut s = seeded_sentinel(); + let before = s.graph().total_sum(); + + // q=1.0 is the maximum-selectivity boundary — must not panic. + s.decay(0.5, 1.0); + assert!(s.graph().total_sum() <= before); +} + +// ── Feed-forward invariant ───────────────────────────────── + +/// Decay reaches the spatial accounting and stops there. Halving every cell's +/// importance destroys no tracker: the models a sentinel has built are not +/// the same asset as the standing that justified building them, and forgetting +/// the second does not throw away the first. This is the feed-forward +/// invariant seen from the temporal side — importance flows into modelling +/// decisions, never the reverse, so rescaling it cannot reach the models. +/// +/// ´claim:decay:decay-rescales-spatial-importance-only-and-never-the-models-or-the-record-of-what-was-observed´ +/// ´test:integration:decay-does-not-affect-tracker-count´ +#[test] +fn decay_does_not_affect_tracker_count() { + let mut s = seeded_sentinel(); + let trackers_before = s.cells_tracked(); + + s.decay(0.5, 0.0); + assert_eq!(s.cells_tracked(), trackers_before); +} + +/// The other thing decay leaves alone is the ledger. The lifetime observation +/// count records what the host actually fed in and is not a decayable +/// quantity, so it survives a decay unchanged — a host can still say how much +/// data has passed through after any amount of forgetting. +/// +/// (´claim:decay:decay-rescales-spatial-importance-only-and-never-the-models-or-the-record-of-what-was-observed´) +/// ´test:integration:decay-does-not-change-lifetime-observations´ +#[test] +fn decay_does_not_change_lifetime_observations() { + let mut s = seeded_sentinel(); + let before = s.health().lifetime_observations; + + s.decay(0.5, 0.0); + assert_eq!(s.health().lifetime_observations, before); +} + +// ── decay_subtree ────────────────────────────────────────── + +/// There is one decay operation, not two. Targeting the subtree at the graph +/// root produces exactly the importance a global decay produces, because the +/// global call is defined as the subtree call made at the root. The targeted +/// form is the general one, and the whole-graph form its degenerate case. +/// +/// ´claim:decay:global-decay-is-exactly-subtree-decay-applied-at-the-root´ +/// ´test:integration:decay-subtree-at-root-matches-global-decay´ +#[test] +fn decay_subtree_at_root_matches_global_decay() { + let cfg = test_config(); + + let mut s_global = Sentinel128::new(cfg.clone()).unwrap(); + s_global.ingest(&[42, 43, 44, 45, 46]); + s_global.decay(0.5, 0.0); + + let mut s_subtree = Sentinel128::new(cfg).unwrap(); + s_subtree.ingest(&[42, 43, 44, 45, 46]); + let root = s_subtree.graph().g_root(); + s_subtree.decay_subtree(root, 0.5, 0.0); + + assert_eq!(s_global.graph().total_sum(), s_subtree.graph().total_sum()); +} + +/// Aimed below the root, decay is bounded by its target: the region named +/// loses standing while everything outside it keeps what it had, so more +/// importance survives than the same factor applied globally. That +/// containment is what makes the operation usable for a regime change in one +/// part of the domain — the rest of the graph does not have to be punished to +/// let one region re-form. +/// +/// ´claim:decay:a-subtree-decay-spends-its-whole-effect-inside-the-target-and-leaves-the-rest-of-the-graph-standing´ +/// ´test:integration:decay-subtree-affects-only-targeted-subtree´ +#[test] +fn decay_subtree_affects_only_targeted_subtree() { + let cfg = SentinelConfig:: { + split_threshold: 5, + ..test_config() + }; + + let mut s_global = Sentinel128::new(cfg.clone()).unwrap(); + let mut s_partial = Sentinel128::new(cfg).unwrap(); + + let values: Vec = (0..100).collect(); + s_global.ingest(&values); + s_partial.ingest(&values); + + let before = s_partial.graph().total_sum(); + + // Find a non-root cell to use as subtree target. + let root = s_partial.graph().g_root(); + let non_root: Vec<_> = s_partial.cell_gnodes().into_iter().filter(|&id| id != root).collect(); + + if !non_root.is_empty() { + s_global.decay(0.5, 0.0); + s_partial.decay_subtree(non_root[0], 0.5, 0.0); + + // Subtree decay touches fewer nodes → more total_sum remains. + assert!(s_partial.graph().total_sum() > s_global.graph().total_sum()); + // But the targeted subtree was still reduced. + assert!(s_partial.graph().total_sum() <= before); + } +} + +// ── Composition ──────────────────────────────────────────── + +/// Decay leaves the sentinel in a state the next batch can be scored from. It +/// changes the rankings the selector reads but invalidates nothing eagerly; +/// the analysis set is simply recomputed at the start of the following +/// ingest, and the report that comes out satisfies every structural +/// invariant. Forgetting is therefore composable with ordinary operation +/// rather than something to be sequenced carefully around it. +/// +/// ´claim:decay:the-analysis-set-catches-up-with-decayed-importance-at-the-next-ingest-rather-than-being-invalidated-eagerly´ +/// ´test:integration:decay-then-ingest-preserves-invariants´ +#[test] +fn decay_then_ingest_preserves_invariants() { + let mut s = seeded_sentinel(); + s.decay(0.5, 0.0); + + let report = s.ingest(&[45, 46, 47]); + assert_invariants(&s, &report); +} + +// ── Panics — invalid attenuation ─────────────────────────── + +/// A negative attenuation has no meaning — importance cannot be scaled +/// through zero into a negative standing — and the call refuses it outright +/// instead of clamping it to the nearest sensible value. Temporal policy is +/// the host's, so a nonsensical factor is a bug in that policy and is +/// reported as one. +/// +/// ´claim:decay:an-attenuation-outside-the-non-negative-reals-is-refused-rather-than-quietly-repaired´ +/// ´test:integration:decay-panics-on-negative-attenuation´ +#[test] +#[should_panic(expected = "attenuation")] +fn decay_panics_on_negative_attenuation() { + let mut s = seeded_sentinel(); + s.decay(-1.0, 0.0); +} + +/// The other way a factor can be meaningless is by being no number at all, +/// typically the result of an arithmetic accident upstream in the host's +/// policy. It is rejected on the same footing as a negative factor, so a +/// silent propagation cannot poison the graph's importance. +/// +/// (´claim:decay:an-attenuation-outside-the-non-negative-reals-is-refused-rather-than-quietly-repaired´) +/// ´test:integration:decay-panics-on-nan-attenuation´ +#[test] +#[should_panic(expected = "attenuation")] +fn decay_panics_on_nan_attenuation() { + let mut s = seeded_sentinel(); + s.decay(f64::NAN, 0.0); +} + +// ── Panics — invalid q ──────────────────────────────────── + +/// Selectivity runs from uniform to fully depth-weighted, and below that +/// range there is nothing to mean. A negative request is refused rather than +/// treated as uniform, because a host that computed it did not intend +/// uniformity. +/// +/// ´claim:decay:a-selectivity-outside-the-unit-interval-is-refused-rather-than-quietly-repaired´ +/// ´test:integration:decay-panics-on-negative-q´ +#[test] +#[should_panic(expected = "q must be")] +fn decay_panics_on_negative_q() { + let mut s = seeded_sentinel(); + s.decay(0.5, -0.1); +} + +/// The upper end is closed at maximum selectivity, and just past it the +/// request is refused. This is what makes the boundary meaningful: full +/// selectivity is accepted and anything beyond it is an error, rather than +/// the range trailing off into values silently treated as the maximum. +/// +/// (´claim:decay:a-selectivity-outside-the-unit-interval-is-refused-rather-than-quietly-repaired´) +/// ´test:integration:decay-panics-on-q-above-one´ +#[test] +#[should_panic(expected = "q must be")] +fn decay_panics_on_q_above_one() { + let mut s = seeded_sentinel(); + s.decay(0.5, 1.1); +} + +/// A selectivity that is not a number compares false against both ends of the +/// range, so a bounds check written carelessly would let it through. It is +/// refused explicitly, which is what keeps the range genuinely closed rather +/// than closed only for values that compare. +/// +/// (´claim:decay:a-selectivity-outside-the-unit-interval-is-refused-rather-than-quietly-repaired´) +/// ´test:integration:decay-panics-on-nan-q´ +#[test] +#[should_panic(expected = "q must be")] +fn decay_panics_on_nan_q() { + let mut s = seeded_sentinel(); + s.decay(0.5, f64::NAN); +} diff --git a/packages/sentinel/tests/spray_resistance.rs b/packages/sentinel/tests/spray_resistance.rs new file mode 100644 index 000000000..4ec6ff599 --- /dev/null +++ b/packages/sentinel/tests/spray_resistance.rs @@ -0,0 +1,342 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +//! # Test index +//! +//! | Test | Area | Claim | +//! |------|------|-------| +//! | [`competitive_set_bounded_by_k`] | resistance | Spraying traffic across far more leading ranges than the sentinel is permitted to model does not enlarge the set of cells that compete for modelling effort: it stays within the configured cap. The cap is on attention, not on input, so an attacker who can address any part of the domain still cannot make the sentinel promise more work than it budgeted for. | +//! | [`competitive_set_at_k_equals_one`] | resistance | cites (´claim:resistance:a-spray-across-many-ranges-cannot-enlarge-the-competitive-set-beyond-its-cap´) | +//! | [`full_set_bounded_by_steiner`] | resistance | Capping the winners would be hollow if the ancestors pulled in to connect them to the root were unbounded, since each of those also carries a tracker. After a spray across many ranges the materialised set stays within the root plus the cap times the deepest level reached — the connecting chains are shared and counted, so the total cost of attention is a function of the cap and the depth alone, never of how many ranges were touched. | +//! | [`g_nodes_bounded_by_budget_under_spray`] | resistance | Feeding a long run of one-value batches, each a distinct coordinate spread over the ranges, leaves the tree inside its node budget rather than growing a node per distinct value. The bound asserted is the budget itself, which is the figure the structure's own invariant refuses to exceed — a guard at twice it would let the test pass through states the structure calls violations. Memory is the resource an attacker would most like to exhaust, so the budget is enforced by eviction as the tree grows and is not merely a hint the structure is asked to respect. | +//! | [`cells_tracked_bounded_under_diverse_traffic`] | resistance | Sustained traffic to every leading range at once leaves the number of live trackers bounded by roughly twice the competitive cap. Trackers are the expensive objects — each carries a learned subspace and its baselines — so what bounds them is the cap on attention rather than the diversity of the traffic. Diverse traffic that is not adversarial is held to the same bound as a spray, because the sentinel does not need to tell them apart to stay within budget. | +//! | [`concentrated_range_survives_spray`] | resistance | A range carrying the great bulk of the traffic is still represented in the reports after a thin spray touches every other range — as a competitor in its own right or through an ancestor below the root that covers it. The root does not count towards that: it contains every coordinate and receives every batch, so a reading that accepted it would be satisfied by a report in which the concentrated range had lost every cell of its own. Attention is bought with accumulated weight rather than with novelty, which is what stops a cheap spray from evicting the model of the range an operator actually cares about. | +//! | [`invariants_hold_under_spray`] | resistance | The structural guarantees are checked after every single batch of a wide spray and again through the concentrated burst that follows it, and none of them breaks. The bounds, the ordering of the reports and the presence of the root are not properties of a settled sentinel: they hold batch by batch while the tree is being churned by hostile traffic and while it is reconverging afterwards, which is the only time they matter. | + +//! Integration tests for the sentinel's **resistance to spray** — traffic +//! spread thinly and deliberately across the whole coordinate domain rather +//! than concentrated where real activity lives. +//! +//! A spray is cheap for an attacker and expensive for a naive watcher. Every +//! fresh region looks like something new worth modelling, so a design that let +//! attention follow novelty could be made to allocate a tracker per sprayed +//! region until it ran out of memory, or to evict the model of the range that +//! actually mattered. The sentinel answers with hard caps rather than +//! heuristics: only a bounded number of cells may compete for modelling effort, +//! the ancestors dragged in to connect those winners to the root are bounded in +//! turn by that count times the depth reached, the tree beneath it all is held +//! under an explicit node budget, and the number of live trackers is bounded by +//! the competitive cap. None of those bounds is a function of how many distinct +//! values arrived, which is precisely why spraying more of them buys nothing. +//! +//! The caps alone would be a poor defence if they were satisfied by dropping +//! the traffic that matters, so the other half of the property is that +//! attention is bought with weight, not with variety: a range carrying the bulk +//! of the traffic is still represented after a thin spray across many ranges. +//! The tests here drive the sentinel through spray, through spray followed by a +//! concentrated burst, and through the boundary where only a single cell may +//! ever compete. + +mod common; + +use common::{assert_invariants, cell_values, cell_values_prefix, integration_config, test_config}; +use torrust_sentinel::{CellReport, Sentinel128, SentinelConfig}; + +// ═══════════════════════════════════════════════════════════ +// Analysis-set bounds +// ═══════════════════════════════════════════════════════════ + +/// Spraying traffic across far more leading ranges than the sentinel is +/// permitted to model does not enlarge the set of cells that compete for +/// modelling effort: it stays within the configured cap. The cap is on +/// attention, not on input, so an attacker who can address any part of the +/// domain still cannot make the sentinel promise more work than it budgeted +/// for. +/// +/// ´claim:resistance:a-spray-across-many-ranges-cannot-enlarge-the-competitive-set-beyond-its-cap´ +/// ´test:integration:competitive-set-bounded-by-k´ +#[test] +fn competitive_set_bounded_by_k() { + let k = 8; + let cfg = SentinelConfig:: { + analysis_k: k, + split_threshold: 10, + budget: 10_000, + ..test_config() + }; + let mut s = Sentinel128::new(cfg).unwrap(); + + // Spray traffic across 64 distinct leading six-bit prefixes. + for prefix in 0..64u128 { + s.ingest(&cell_values_prefix(prefix, 20)); + } + + let report = s.ingest(&cell_values_prefix(0, 4)); + assert_invariants(&s, &report); + assert!( + report.analysis_set_summary.competitive_size <= k, + "competitive set {} exceeds K={}", + report.analysis_set_summary.competitive_size, + k, + ); +} + +/// This pins the tightest end of the cap. Configured to model a single +/// competitor and then fed repeated traffic to every leading range in turn, the +/// sentinel still admits at most one — and it does not answer the pressure by +/// emptying out, since the set it materialises always retains at least the +/// root. A cap of one is an ordinary setting rather than a degenerate case that +/// collapses the analysis set. +/// +/// (´claim:resistance:a-spray-across-many-ranges-cannot-enlarge-the-competitive-set-beyond-its-cap´) +/// ´test:integration:competitive-set-at-k-equals-one´ +#[test] +fn competitive_set_at_k_equals_one() { + let cfg = SentinelConfig:: { + analysis_k: 1, + split_threshold: 10, + budget: 10_000, + ..integration_config() + }; + let mut s = Sentinel128::new(cfg).unwrap(); + + for nibble in 0..16u128 { + for _ in 0..8 { + s.ingest(&cell_values(nibble, 16)); + } + } + + let report = s.ingest(&cell_values(0xA, 4)); + assert_invariants(&s, &report); + assert!( + report.analysis_set_summary.competitive_size <= 1, + "with K=1, competitive set should be at most 1, got {}", + report.analysis_set_summary.competitive_size, + ); + assert!( + report.analysis_set_summary.full_size >= 1, + "full set should have at least root" + ); +} + +/// Capping the winners would be hollow if the ancestors pulled in to connect +/// them to the root were unbounded, since each of those also carries a tracker. +/// After a spray across many ranges the materialised set stays within the root +/// plus the cap times the deepest level reached — the connecting chains are +/// shared and counted, so the total cost of attention is a function of the cap +/// and the depth alone, never of how many ranges were touched. +/// +/// ´claim:resistance:the-ancestors-connecting-the-winners-are-bounded-by-the-cap-times-the-depth-reached´ +/// ´test:integration:full-set-bounded-by-steiner´ +#[test] +fn full_set_bounded_by_steiner() { + let k = 8; + let cfg = SentinelConfig:: { + analysis_k: k, + split_threshold: 10, + budget: 10_000, + ..test_config() + }; + let mut s = Sentinel128::new(cfg).unwrap(); + + for prefix in 0..64u128 { + s.ingest(&cell_values_prefix(prefix, 20)); + } + + let report = s.ingest(&cell_values_prefix(0, 4)); + assert_invariants(&s, &report); + + let summary = &report.analysis_set_summary; + let d_max = summary.depth_range.1 as usize; + let bound = 1 + k * d_max; + assert!( + summary.full_size <= bound, + "full set {} exceeds Steiner bound 1+K·d_max={} (K={}, d_max={})", + summary.full_size, + bound, + k, + d_max, + ); +} + +// ═══════════════════════════════════════════════════════════ +// Resource bounds +// ═══════════════════════════════════════════════════════════ + +/// Feeding a long run of one-value batches, each a distinct coordinate spread +/// over the ranges, leaves the tree inside its node budget rather than +/// growing a node per distinct value. The bound asserted is the budget +/// itself, which is the figure the structure's own invariant refuses to +/// exceed — a guard at twice it would let the test pass through states the +/// structure calls violations. Memory is the resource an attacker would most +/// like to exhaust, so the budget is enforced by eviction as the tree grows +/// and is not merely a hint the structure is asked to respect. +/// +/// ´claim:resistance:a-stream-of-distinct-sprayed-values-cannot-grow-the-tree-past-its-node-budget´ +/// ´test:integration:g-nodes-bounded-by-budget-under-spray´ +#[test] +fn g_nodes_bounded_by_budget_under_spray() { + let budget = 500; + let cfg = SentinelConfig:: { + budget, + split_threshold: 10, + d_create: 3, + d_evict: 6, + ..test_config() + }; + let mut s = Sentinel128::new(cfg).unwrap(); + + // Spray many distinct values across nibble ranges. + for i in 0..500u128 { + let nibble = i % 16; + s.ingest(&[(nibble << 124) | (i + 1)]); + } + + let g_nodes = s.health().total_g_nodes; + assert!( + g_nodes <= budget, + "G-tree node count {g_nodes} exceeds the node budget ({budget})", + ); +} + +/// Sustained traffic to every leading range at once leaves the number of live +/// trackers bounded by roughly twice the competitive cap. Trackers are the +/// expensive objects — each carries a learned subspace and its baselines — so +/// what bounds them is the cap on attention rather than the diversity of the +/// traffic. Diverse traffic that is not adversarial is held to the same bound +/// as a spray, because the sentinel does not need to tell them apart to stay +/// within budget. +/// +/// ´claim:resistance:the-number-of-live-trackers-stays-near-the-competitive-cap-however-diverse-the-traffic´ +/// ´test:integration:cells-tracked-bounded-under-diverse-traffic´ +#[test] +fn cells_tracked_bounded_under_diverse_traffic() { + let k = 8; + let cfg = SentinelConfig:: { + analysis_k: k, + split_threshold: 10, + budget: 10_000, + ..test_config() + }; + let mut s = Sentinel128::new(cfg).unwrap(); + + // Send traffic to all 16 leading-nibble ranges. + for nibble in 0..16u128 { + for _ in 0..12 { + s.ingest(&cell_values(nibble, 16)); + } + } + + assert!( + s.cells_tracked() <= 2 * k + 1, + "cells_tracked {} exceeds 2K+1={} (K={})", + s.cells_tracked(), + 2 * k + 1, + k, + ); +} + +// ═══════════════════════════════════════════════════════════ +// Fairness +// ═══════════════════════════════════════════════════════════ + +/// A range carrying the great bulk of the traffic is still represented in the +/// reports after a thin spray touches every other range — as a competitor in +/// its own right or through an ancestor below the root that covers it. The +/// root does not count towards that: it contains every coordinate and +/// receives every batch, so a reading that accepted it would be satisfied by +/// a report in which the concentrated range had lost every cell of its own. +/// Attention is bought with accumulated weight rather than with novelty, +/// which is what stops a cheap spray from evicting the model of the range an +/// operator actually cares about. +/// +/// ´claim:resistance:a-thin-spray-does-not-displace-the-range-that-carries-the-weight-of-the-traffic´ +/// ´test:integration:concentrated-range-survives-spray´ +#[test] +fn concentrated_range_survives_spray() { + let cfg = SentinelConfig:: { + analysis_k: 4, + split_threshold: 50, + budget: 10_000, + max_rank: 2, + ..integration_config() + }; + let mut s = Sentinel128::new(cfg).unwrap(); + + // 90% traffic to range A. + for _ in 0..20 { + s.ingest(&cell_values(0xA, 40)); + } + + // 10% spray across many ranges. + for nibble in 0..16u128 { + s.ingest(&cell_values(nibble, 3)); + } + + let report = s.ingest(&cell_values(0xA, 8)); + assert_invariants(&s, &report); + + // The concentrated range should still be represented — either as a + // competitive cell or through an ancestor below the root that covers it. + // The root is excluded: it contains every coordinate and receives every + // batch, so accepting it would accept a report in which range A had lost + // every cell of its own. + let range_a = cell_values(0xA, 1)[0]; + let covers_range_a = |cr: &CellReport| cr.depth > 0 && cr.sample_count > 0 && cr.start <= range_a && range_a < cr.end; + let has_range_a = report + .cell_reports + .iter() + .chain(report.ancestor_reports.iter()) + .any(covers_range_a); + + assert!( + has_range_a, + "no report below the root covers range A with samples: cells {:?}, ancestors {:?}", + report + .cell_reports + .iter() + .map(|cr| (cr.depth, cr.start, cr.end, cr.sample_count)) + .collect::>(), + report + .ancestor_reports + .iter() + .map(|cr| (cr.depth, cr.start, cr.end, cr.sample_count)) + .collect::>(), + ); +} + +// ═══════════════════════════════════════════════════════════ +// Cross-cutting invariants +// ═══════════════════════════════════════════════════════════ + +/// The structural guarantees are checked after every single batch of a wide +/// spray and again through the concentrated burst that follows it, and none of +/// them breaks. The bounds, the ordering of the reports and the presence of the +/// root are not properties of a settled sentinel: they hold batch by batch +/// while the tree is being churned by hostile traffic and while it is +/// reconverging afterwards, which is the only time they matter. +/// +/// ´claim:resistance:the-structural-guarantees-hold-batch-by-batch-through-a-spray-and-the-burst-that-follows-it´ +/// ´test:integration:invariants-hold-under-spray´ +#[test] +fn invariants_hold_under_spray() { + let cfg = SentinelConfig:: { + analysis_k: 8, + split_threshold: 10, + budget: 10_000, + ..test_config() + }; + let mut s = Sentinel128::new(cfg).unwrap(); + + // Phase 1: wide spray across 64 distinct leading six-bit prefixes. + for prefix in 0..64u128 { + let report = s.ingest(&cell_values_prefix(prefix, 20)); + assert_invariants(&s, &report); + } + + // Phase 2: concentrated burst after spray. + for _ in 0..10 { + let report = s.ingest(&cell_values(0xA, 50)); + assert_invariants(&s, &report); + } +} diff --git a/packages/sentinel/tests/suffix_analysis.rs b/packages/sentinel/tests/suffix_analysis.rs new file mode 100644 index 000000000..1708a80ab --- /dev/null +++ b/packages/sentinel/tests/suffix_analysis.rs @@ -0,0 +1,312 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +//! # Test index +//! +//! | Test | Area | Claim | +//! |------|------|-------| +//! | [`root_suffix_width_is_full_bit_width`] | suffix | cites (´claim:suffix:a-cells-analysis-width-is-the-domain-width-less-its-depth´) | +//! | [`cell_analysis_width_equals_n_minus_depth`] | suffix | Across every cell a graph under traffic has produced, the width a cell analyses is exactly the domain width less its depth. The leading bits its depth stands for were fixed by routing and are identical for every value that reaches it, so modelling them would add a constant column and no information; the width is a consequence of position rather than a per-cell setting anyone can get wrong. | +//! | [`geometry_dim_equals_analysis_width`] | suffix | The space a cell's model works in is its own suffix, not the domain: the dimension reported with its scoring geometry is the cell's analysis width at every depth the tree reaches. The two are not independently maintained numbers that happen to agree — the tracker is constructed at the cell's width — so a host reading the geometry is reading the same fact as one reading the width. | +//! | [`geometry_cap_is_min_of_dim_and_max_rank`] | suffix | How much structure a cell may learn is limited by its own suffix as well as by configuration: the ceiling is whichever of the two is smaller. A model cannot hold more directions than the space it lives in has, so a cell deep enough to be narrower than the configured maximum is capped by its depth instead — the configured maximum is a budget, never a promise of capacity. | +//! | [`residual_dof_equals_dim_minus_rank`] | suffix | What is left over for a cell to be surprised by is its width less the structure it has already learned. Those residual directions are the room in which an unexplained departure can register at all, so the same absolute departure means more in a narrow cell than in a wide one — and a cell whose model has grown to fill its width has no room left, which is precisely the degenerate case the geometry lets a host detect rather than hiding. | +//! | [`deeper_cells_have_smaller_suffix_width`] | suffix | cites (´claim:suffix:a-cells-analysis-width-is-the-domain-width-less-its-depth´) | +//! | [`cell_reports_carry_correct_suffix_widths`] | suffix | The width a cell analysed travels out with its numbers: each entry in a batch report states the width it was computed at, and that width still agrees with the cell's depth and with the geometry the scores came from. Scores from different depths are not commensurable, so a report that carried only the numbers would invite a host to compare them as though they were. | +//! | [`scores_are_valid_across_suffix_widths`] | suffix | Every score axis yields a real number at every width the tree produced, across a graph warmed on separated ranges and then driven into one of them. Narrow geometries are where the divisions in the scoring arithmetic come closest to degenerating, so this is the property that lets a host treat a report as data rather than checking each figure for a non-number first. | +//! | [`noise_injected_at_every_suffix_width`] | suffix | No cell begins scoring cold. Every cell the graph created under traffic has synthetic observations behind it, generated at that cell's own width, so the warm-up schedule reaches cells born deep in the tree and not only the root it started from. A cell that had never seen anything would find its first real batch infinitely surprising, and the sentinel would report the arrival of a new region as an anomaly in it. | + +//! Suffix analysis — which part of an observation a cell actually models +//! (§ALGO S-2.4). +//! +//! A cell's position in the G-tree is a prefix of the coordinate: routing has +//! already decided the leading bits by the time a value arrives, and within +//! the cell those bits are the same for every observation. They therefore +//! carry no statistical content and are not handed to the cell's model at all. +//! What the model sees is the suffix behind them, so a cell's analysis width +//! is the domain width less its depth — the deeper a cell, the narrower and +//! more specialised the thing it is modelling, and the shallower a cell, the +//! wider the view it keeps. +//! +//! That single identity propagates outward rather than being recomputed +//! anywhere. It fixes the working dimension of the cell's subspace tracker, +//! and through it the geometry the scores are read against: the rank a cell +//! can reach is capped by its own width where that is narrower than the +//! configured maximum, and the residual degrees of freedom against which +//! novelty is judged are whatever the width leaves once the modelled rank is +//! taken out. A narrow cell is thus not a wide cell with fewer observations; +//! it is a smaller geometry, and its scores mean something correspondingly +//! smaller. +//! +//! Because the width varies from cell to cell, it has to travel with the +//! numbers. Every cell report carries the width it was computed at, so a host +//! comparing two cells knows it is comparing different geometries, and the +//! machinery around the model — warming a new cell with synthetic data, +//! producing finite scores on every axis — has to work at every width the tree +//! can produce rather than only at the root's. + +mod common; + +use common::{ScenarioBuilder, assert_invariants, cell_values, seeded_sentinel, test_config}; +use torrust_sentinel::Sentinel128; + +// ═══════════════════════════════════════════════════════════ +// Suffix width identity +// ═══════════════════════════════════════════════════════════ + +/// The base case of the depth identity: the root has had nothing resolved by +/// routing, so it models the entire domain width. A sentinel that has observed +/// nothing therefore already has one cell watching everything, which is why +/// there is never a value the sentinel cannot score. +/// +/// (´claim:suffix:a-cells-analysis-width-is-the-domain-width-less-its-depth´) +/// ´test:integration:root-suffix-width-is-full-bit-width´ +#[test] +fn root_suffix_width_is_full_bit_width() { + let s = Sentinel128::new(test_config()).unwrap(); + + let gnodes = s.cell_gnodes(); + assert!(!gnodes.is_empty(), "sentinel must have at least the root cell"); + + let root = s.inspect_cell(gnodes[0]).unwrap(); + assert_eq!(root.depth, 0); + assert_eq!(root.analysis_width, 128); +} + +/// Across every cell a graph under traffic has produced, the width a cell +/// analyses is exactly the domain width less its depth. The leading bits its +/// depth stands for were fixed by routing and are identical for every value +/// that reaches it, so modelling them would add a constant column and no +/// information; the width is a consequence of position rather than a +/// per-cell setting anyone can get wrong. +/// +/// ´claim:suffix:a-cells-analysis-width-is-the-domain-width-less-its-depth´ +/// ´test:integration:cell-analysis-width-equals-n-minus-depth´ +#[test] +fn cell_analysis_width_equals_n_minus_depth() { + let mut s = seeded_sentinel(); + s.ingest(&cell_values(0xA, 200)); + + for &gnode in &s.cell_gnodes() { + let ins = s.inspect_cell(gnode).unwrap(); + let expected = 128 - ins.depth as usize; + assert_eq!( + ins.analysis_width, expected, + "depth {}: expected analysis_width={expected}, got {}", + ins.depth, ins.analysis_width, + ); + } +} + +/// The space a cell's model works in is its own suffix, not the domain: the +/// dimension reported with its scoring geometry is the cell's analysis width +/// at every depth the tree reaches. The two are not independently maintained +/// numbers that happen to agree — the tracker is constructed at the cell's +/// width — so a host reading the geometry is reading the same fact as one +/// reading the width. +/// +/// ´claim:suffix:a-cells-model-works-in-its-own-suffix-space-and-not-the-whole-domain´ +/// ´test:integration:geometry-dim-equals-analysis-width´ +#[test] +fn geometry_dim_equals_analysis_width() { + let mut s = seeded_sentinel(); + s.ingest(&cell_values(0xA, 200)); + + for &gnode in &s.cell_gnodes() { + let ins = s.inspect_cell(gnode).unwrap(); + assert_eq!( + ins.geometry.dim, ins.analysis_width, + "depth {}: geometry.dim={} != analysis_width={}", + ins.depth, ins.geometry.dim, ins.analysis_width, + ); + } +} + +// ═══════════════════════════════════════════════════════════ +// Geometry cap +// ═══════════════════════════════════════════════════════════ + +/// How much structure a cell may learn is limited by its own suffix as well as +/// by configuration: the ceiling is whichever of the two is smaller. A model +/// cannot hold more directions than the space it lives in has, so a cell deep +/// enough to be narrower than the configured maximum is capped by its depth +/// instead — the configured maximum is a budget, never a promise of capacity. +/// +/// ´claim:suffix:a-cells-structural-ceiling-is-its-own-width-when-that-is-narrower-than-the-configured-maximum´ +/// ´test:integration:geometry-cap-is-min-of-dim-and-max-rank´ +#[test] +fn geometry_cap_is_min_of_dim_and_max_rank() { + let cfg = test_config(); + let max_rank = cfg.max_rank; + let mut s = Sentinel128::new(cfg).unwrap(); + s.ingest(&cell_values(0xF, 200)); + + for &gnode in &s.cell_gnodes() { + let ins = s.inspect_cell(gnode).unwrap(); + let expected_cap = ins.geometry.dim.min(max_rank); + assert_eq!( + ins.geometry.cap, expected_cap, + "depth {}: expected cap={expected_cap}, got {}", + ins.depth, ins.geometry.cap, + ); + } +} + +/// What is left over for a cell to be surprised by is its width less the +/// structure it has already learned. Those residual directions are the room in +/// which an unexplained departure can register at all, so the same absolute +/// departure means more in a narrow cell than in a wide one — and a cell whose +/// model has grown to fill its width has no room left, which is precisely the +/// degenerate case the geometry lets a host detect rather than hiding. +/// +/// ´claim:suffix:what-a-cell-can-still-be-surprised-by-is-its-width-less-the-structure-it-has-learned´ +/// ´test:integration:residual-dof-equals-dim-minus-rank´ +#[test] +fn residual_dof_equals_dim_minus_rank() { + let mut s = seeded_sentinel(); + s.ingest(&cell_values(0xA, 200)); + + for &gnode in &s.cell_gnodes() { + let ins = s.inspect_cell(gnode).unwrap(); + let expected_dof = ins.geometry.dim - ins.rank; + assert_eq!( + ins.geometry.residual_dof, expected_dof, + "depth {}: expected residual_dof={expected_dof}, got {}", + ins.depth, ins.geometry.residual_dof, + ); + } +} + +// ═══════════════════════════════════════════════════════════ +// Deeper cells have narrower suffixes +// ═══════════════════════════════════════════════════════════ + +/// The ordering consequence of the same identity: sort a graph's cells by +/// depth and their widths fall strictly the other way. Refinement is a trade — +/// each split buys a more specific region at the cost of a narrower view +/// inside it — so no cell in the tree is both deeper and wider than another. +/// +/// (´claim:suffix:a-cells-analysis-width-is-the-domain-width-less-its-depth´) +/// ´test:integration:deeper-cells-have-smaller-suffix-width´ +#[test] +fn deeper_cells_have_smaller_suffix_width() { + let mut s = seeded_sentinel(); + s.ingest(&cell_values(0xF, 200)); + + let mut widths: Vec<(u32, usize)> = s + .cell_gnodes() + .iter() + .map(|&g| { + let ins = s.inspect_cell(g).unwrap(); + (ins.depth, ins.analysis_width) + }) + .collect(); + widths.sort_by_key(|&(depth, _)| depth); + + for pair in widths.windows(2) { + let (d1, w1) = pair[0]; + let (d2, w2) = pair[1]; + if d2 > d1 { + assert!( + w2 < w1, + "depth {d1} (width {w1}) should be wider than depth {d2} (width {w2})", + ); + } + } +} + +// ═══════════════════════════════════════════════════════════ +// Report suffix widths +// ═══════════════════════════════════════════════════════════ + +/// The width a cell analysed travels out with its numbers: each entry in a +/// batch report states the width it was computed at, and that width still +/// agrees with the cell's depth and with the geometry the scores came from. +/// Scores from different depths are not commensurable, so a report that +/// carried only the numbers would invite a host to compare them as though they +/// were. +/// +/// ´claim:suffix:a-batch-report-states-the-width-each-cell-was-scored-at´ +/// ´test:integration:cell-reports-carry-correct-suffix-widths´ +#[test] +fn cell_reports_carry_correct_suffix_widths() { + let mut s = seeded_sentinel(); + let report = s.ingest(&cell_values(0xF, 100)); + + assert_invariants(&s, &report); + + for cr in &report.cell_reports { + let expected = 128 - cr.depth as usize; + assert_eq!( + cr.analysis_width, expected, + "cell report depth {}: expected analysis_width={expected}, got {}", + cr.depth, cr.analysis_width, + ); + assert_eq!( + cr.geometry.dim, expected, + "cell report depth {}: geometry.dim={} != {expected}", + cr.depth, cr.geometry.dim, + ); + } +} + +/// Every score axis yields a real number at every width the tree produced, +/// across a graph warmed on separated ranges and then driven into one of them. +/// Narrow geometries are where the divisions in the scoring arithmetic come +/// closest to degenerating, so this is the property that lets a host treat a +/// report as data rather than checking each figure for a non-number first. +/// +/// ´claim:suffix:every-score-axis-yields-a-real-number-at-every-width-the-tree-produces´ +/// ´test:integration:scores-are-valid-across-suffix-widths´ +#[test] +fn scores_are_valid_across_suffix_widths() { + let (mut s, _) = ScenarioBuilder::new() + .config(test_config()) + .seed_range(0xF, 50) + .seed_range(0x1, 50) + .warm_batches(3) + .build_with_reports(); + + let report = s.ingest(&cell_values(0xF, 100)); + + assert_invariants(&s, &report); + + for cr in &report.cell_reports { + assert!(!cr.scores.novelty.mean.is_nan(), "NaN novelty at depth {}", cr.depth); + assert!( + !cr.scores.displacement.mean.is_nan(), + "NaN displacement at depth {}", + cr.depth + ); + assert!(!cr.scores.surprise.mean.is_nan(), "NaN surprise at depth {}", cr.depth); + assert!(!cr.scores.coherence.mean.is_nan(), "NaN coherence at depth {}", cr.depth); + } +} + +// ═══════════════════════════════════════════════════════════ +// Noise at all suffix widths +// ═══════════════════════════════════════════════════════════ + +/// No cell begins scoring cold. Every cell the graph created under traffic has +/// synthetic observations behind it, generated at that cell's own width, so +/// the warm-up schedule reaches cells born deep in the tree and not only the +/// root it started from. A cell that had never seen anything would find its +/// first real batch infinitely surprising, and the sentinel would report the +/// arrival of a new region as an anomaly in it. +/// +/// ´claim:suffix:no-cell-starts-scoring-cold-however-narrow-its-suffix´ +/// ´test:integration:noise-injected-at-every-suffix-width´ +#[test] +fn noise_injected_at_every_suffix_width() { + let mut s = seeded_sentinel(); + s.ingest(&cell_values(0xA, 200)); + + for &gnode in &s.cell_gnodes() { + let ins = s.inspect_cell(gnode).unwrap(); + assert!( + ins.maturity.noise_observations > 0, + "cell at depth {} (width {}) received no noise", + ins.depth, + ins.analysis_width, + ); + } +} diff --git a/packages/sentinel/tests/warm_up.rs b/packages/sentinel/tests/warm_up.rs new file mode 100644 index 000000000..68508e8d1 --- /dev/null +++ b/packages/sentinel/tests/warm_up.rs @@ -0,0 +1,498 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// SPDX-FileCopyrightText: 2026 Torrust project contributors + +//! # Test index +//! +//! | Test | Area | Claim | +//! |------|------|-------| +//! | [`stage1_only_root_tracker`] | warmup | Traffic that has not yet reached the split threshold leaves the domain undivided: one cell, the root, and nothing competing for investment. Structure is bought with observation volume, so a sentinel that has seen only a handful of values has bought none of it yet. | +//! | [`stage1_no_coordination`] | warmup | The coordination tier compares cells against each other, so it has nothing to say while there is only one. A pre-split batch therefore reports no coordination at all rather than a degenerate context over a single member: a context fires only where both sides of a G-node contribute cells. | +//! | [`stage1_lifetime_observations_counted`] | warmup | The lifetime observation count starts at nothing and accrues one unit per input value, batch after batch, from the very first ingest. It is a record of what the host has fed in rather than a measure of what the sentinel has made of it, so it runs well before any structure exists to attribute it to. | +//! | [`stage1_invariants_hold`] | warmup | The structural guarantees a report makes — the competitive cap, the root's presence in the full set, the separation of competitive from ancestor entries, sorted output, finite scores, widths matching depth — are not promises about steady state. They hold from the first batch, when the sentinel is a single cell and has almost nothing to report. | +//! | [`stage2_cells_tracked_increases`] | warmup | Once a range has taken more traffic than the split threshold allows, the spatial layer divides it, and the newly exposed cells are picked up by the analysis set and given trackers of their own. Modelling effort follows the structure the traffic created rather than a shape chosen in advance. | +//! | [`stage2_competitive_cells_appear`] | warmup | Having a tracker and being competitive are separate things, and the second arrives later. After a run of batches has given some cells enough accumulated importance to win the ranking, the competitive set becomes non-empty — so the transition out of the pre-split state is earned by volume, not conferred at creation. | +//! | [`stage2_new_cells_have_high_noise_influence`] | warmup | A cell that has just come into service is mostly synthetic. Its tracker was seeded so that it could score at all, and until real batches have arrived to displace that seed, the maturity figure it publishes stays high. Every non-root cell with few real observations behind it says so, which lets a host discount a young cell's scores instead of trusting them equally. | +//! | [`stage2_invariants_hold`] | warmup | cites (´claim:warmup:the-report-invariants-hold-at-every-stage-of-warm-up-not-merely-once-it-has-settled´) | +//! | [`stage3_competitive_set_size_stabilises`] | warmup | Once the traffic pattern stops changing, the competitive set stops changing with it: repeated batches over the same ranges leave the number of selected cells varying only within a narrow band. Selection is recomputed from scratch on every batch, so stability here is a property of the ranking rather than of any memory the selector keeps. | +//! | [`stage3_coordination_activates`] | warmup | cites (´claim:warmup:coordination-fires-only-where-two-subtrees-both-contribute-cells´) | +//! | [`stage3_invariants_hold_throughout`] | warmup | cites (´claim:warmup:the-report-invariants-hold-at-every-stage-of-warm-up-not-merely-once-it-has-settled´) | +//! | [`stage4_root_maturity_below_half`] | warmup | Real data displaces the synthetic seed geometrically: each real batch multiplies the synthetic share of a tracker's memory by the forgetting factor raised to the batch size. A modest run of warm-up batches is therefore enough to push the root well past the halfway mark, and the rate is a property of the configured forgetting factor rather than of the data. | +//! | [`stage4_maturity_decreases_monotonically`] | warmup | Maturity only ever improves while real data is arriving: batch after batch, a cell's synthetic share is multiplied down and never rises again. It could rise only if further noise were injected, and nothing injects noise into a cell already in service. A host can therefore read the figure as a one-way progress measure rather than as something that might rebound. | +//! | [`stage4_health_maturity_distribution`] | warmup | The health snapshot aggregates what individual cells know about their own maturity, and in steady state it says two things: the average tracker is no longer purely synthetic, and no tracker anywhere is still cold — every cell in service has seen real data. A cold entry in a warmed sentinel would mean a cell was being scored on noise alone. | +//! | [`cold_start_noise_influence_is_one`] | warmup | Turning the noise schedule off shows what the seed was doing. The root then begins at a synthetic share of exactly one — the value a tracker with no information at all reports — and only real data moves it. Warm-up is thus an optional head start, not a precondition: the sentinel still runs without it, and simply says that everything it knows is unearned. | + +//! §ALGO S-11.8 — **Warm-up sequence**: how a sentinel comes into service. +//! +//! A sentinel arrives knowing nothing. It begins as a single root cell +//! covering the whole domain, and everything else — the division of that +//! domain into cells, the competition that decides which of them are worth +//! modelling, the coordination tier that compares them — has to be earned +//! from observed traffic. These tests walk that sequence from the first batch +//! to steady state and fix what is true at each point along it. +//! +//! Two quantities move in opposite directions during warm-up. Structure +//! grows: volume accumulates, cells split off, and the competitive set fills +//! and then settles once traffic has stopped telling the graph anything new. +//! Maturity falls: every tracker is seeded with synthetic noise so that it +//! has a baseline to score against before it has seen anything real, and the +//! synthetic share of its memory is multiplied down by the forgetting factor +//! with each real batch. A cell is therefore never unable to score. It is +//! only more or less made of noise, and the maturity figures are what say +//! which. +//! +//! The two are independent by construction. Splitting is driven by +//! observation volume alone and never by scores, so warm-up cannot chase its +//! own tail; and the structural invariants the reports must satisfy hold at +//! every stage rather than only once the sentinel has settled. + +mod common; + +use common::{ScenarioBuilder, assert_invariants, cell_values, cold_config, integration_config, test_config}; +use torrust_sentinel::{NoiseSchedule, Sentinel128, SentinelConfig}; + +// ── Stage 1: Pre-split ────────────────────────────────────── + +/// Traffic that has not yet reached the split threshold leaves the domain +/// undivided: one cell, the root, and nothing competing for investment. +/// Structure is bought with observation volume, so a sentinel that has seen +/// only a handful of values has bought none of it yet. +/// +/// ´claim:warmup:before-the-first-split-the-root-is-the-only-cell-and-nothing-is-competing´ +/// ´test:integration:stage1-only-root-tracker´ +#[test] +fn stage1_only_root_tracker() { + let cfg = SentinelConfig:: { + split_threshold: 100, + ..test_config() + }; + let mut s = Sentinel128::new(cfg).unwrap(); + + // Feed fewer than split_threshold observations — all to same range. + let report = s.ingest(&cell_values(0xA, 50)); + + assert_eq!( + report.analysis_set_summary.competitive_size, 0, + "pre-split: no competitive cells yet" + ); + assert_eq!(s.cells_tracked(), 1, "only root tracker"); +} + +/// The coordination tier compares cells against each other, so it has nothing +/// to say while there is only one. A pre-split batch therefore reports no +/// coordination at all rather than a degenerate context over a single member: +/// a context fires only where both sides of a G-node contribute cells. +/// +/// ´claim:warmup:coordination-fires-only-where-two-subtrees-both-contribute-cells´ +/// ´test:integration:stage1-no-coordination´ +#[test] +fn stage1_no_coordination() { + let cfg = SentinelConfig:: { + split_threshold: 100, + ..test_config() + }; + let mut s = Sentinel128::new(cfg).unwrap(); + + let report = s.ingest(&cell_values(0xA, 50)); + assert!( + report.coordination_reports.is_empty(), + "pre-split: coordination should be empty" + ); +} + +/// The lifetime observation count starts at nothing and accrues one unit per +/// input value, batch after batch, from the very first ingest. It is a record +/// of what the host has fed in rather than a measure of what the sentinel has +/// made of it, so it runs well before any structure exists to attribute it to. +/// +/// ´claim:warmup:the-lifetime-observation-count-accrues-from-the-first-batch-even-before-any-structure-exists´ +/// ´test:integration:stage1-lifetime-observations-counted´ +#[test] +fn stage1_lifetime_observations_counted() { + let cfg = SentinelConfig:: { + split_threshold: 100, + ..test_config() + }; + let mut s = Sentinel128::new(cfg).unwrap(); + + assert_eq!(s.lifetime_observations(), 0); + s.ingest(&cell_values(0xA, 50)); + assert_eq!(s.lifetime_observations(), 50); + s.ingest(&cell_values(0xA, 30)); + assert_eq!(s.lifetime_observations(), 80); +} + +/// The structural guarantees a report makes — the competitive cap, the root's +/// presence in the full set, the separation of competitive from ancestor +/// entries, sorted output, finite scores, widths matching depth — are not +/// promises about steady state. They hold from the first batch, when the +/// sentinel is a single cell and has almost nothing to report. +/// +/// ´claim:warmup:the-report-invariants-hold-at-every-stage-of-warm-up-not-merely-once-it-has-settled´ +/// ´test:integration:stage1-invariants-hold´ +#[test] +fn stage1_invariants_hold() { + let cfg = SentinelConfig:: { + split_threshold: 100, + ..test_config() + }; + let mut s = Sentinel128::new(cfg).unwrap(); + + let report = s.ingest(&cell_values(0xA, 50)); + assert_invariants(&s, &report); +} + +// ── Stage 2: Spatial formation ────────────────────────────── + +/// Once a range has taken more traffic than the split threshold allows, the +/// spatial layer divides it, and the newly exposed cells are picked up by the +/// analysis set and given trackers of their own. Modelling effort follows the +/// structure the traffic created rather than a shape chosen in advance. +/// +/// ´claim:warmup:traffic-past-the-split-threshold-divides-the-space-and-each-new-cell-earns-its-own-tracker´ +/// ´test:integration:stage2-cells-tracked-increases´ +#[test] +fn stage2_cells_tracked_increases() { + let cfg = SentinelConfig:: { + split_threshold: 10, + ..integration_config() + }; + let mut s = Sentinel128::new(cfg).unwrap(); + + for _ in 0..5 { + s.ingest(&cell_values(0xA, 20)); + } + + assert!(s.cells_tracked() > 1, "splits should create new cells"); +} + +/// Having a tracker and being competitive are separate things, and the second +/// arrives later. After a run of batches has given some cells enough +/// accumulated importance to win the ranking, the competitive set becomes +/// non-empty — so the transition out of the pre-split state is earned by +/// volume, not conferred at creation. +/// +/// ´claim:warmup:a-cell-becomes-competitive-only-once-it-has-accumulated-enough-importance-to-win-the-ranking´ +/// ´test:integration:stage2-competitive-cells-appear´ +#[test] +fn stage2_competitive_cells_appear() { + let cfg = SentinelConfig:: { + split_threshold: 10, + ..integration_config() + }; + let mut s = Sentinel128::new(cfg).unwrap(); + + // Feed enough to trigger splits (> split_threshold per range). + for _ in 0..5 { + s.ingest(&cell_values(0xA, 20)); + } + + let report = s.ingest(&cell_values(0xA, 8)); + assert!( + report.analysis_set_summary.competitive_size > 0, + "after sufficient traffic, competitive cells should appear" + ); +} + +/// A cell that has just come into service is mostly synthetic. Its tracker +/// was seeded so that it could score at all, and until real batches have +/// arrived to displace that seed, the maturity figure it publishes stays +/// high. Every non-root cell with few real observations behind it says so, +/// which lets a host discount a young cell's scores instead of trusting them +/// equally. +/// +/// ´claim:warmup:a-freshly-promoted-cell-is-still-mostly-synthetic-and-publishes-that-fact´ +/// ´test:integration:stage2-new-cells-have-high-noise-influence´ +#[test] +fn stage2_new_cells_have_high_noise_influence() { + let cfg = SentinelConfig:: { + split_threshold: 10, + noise_schedule: NoiseSchedule::Explicit(vec![5]), + noise_batch_size: 4, + ..integration_config() + }; + let mut s = Sentinel128::new(cfg).unwrap(); + + // Feed just enough to create new cells. + for _ in 0..5 { + s.ingest(&cell_values(0xA, 20)); + } + + // Newly created cells should have high noise influence (η > 0.3) + // because they've had very few real observations. + let root = s.graph().g_root(); + for &gnode in &s.cell_gnodes() { + if gnode == root { + continue; // root is special — skip. + } + let insp = s.inspect_cell(gnode).unwrap(); + // New cells may not have processed many real batches yet, + // so η should be high. + if insp.maturity.real_observations < 20 { + assert!( + insp.maturity.noise_influence > 0.3, + "early cell at depth {} should have high noise influence, got {:.4}", + insp.depth, + insp.maturity.noise_influence + ); + } + } +} + +/// The same guarantees survive the most turbulent part of warm-up. Splitting +/// is creating cells, trackers are being built and some of them discarded +/// again, and the report emitted in the middle of that still satisfies every +/// structural invariant. +/// +/// (´claim:warmup:the-report-invariants-hold-at-every-stage-of-warm-up-not-merely-once-it-has-settled´) +/// ´test:integration:stage2-invariants-hold´ +#[test] +fn stage2_invariants_hold() { + let cfg = SentinelConfig:: { + split_threshold: 10, + ..integration_config() + }; + let mut s = Sentinel128::new(cfg).unwrap(); + + for _ in 0..5 { + s.ingest(&cell_values(0xA, 20)); + } + + let report = s.ingest(&cell_values(0xA, 8)); + assert_invariants(&s, &report); +} + +// ── Stage 3: Stabilisation ────────────────────────────────── + +/// Once the traffic pattern stops changing, the competitive set stops +/// changing with it: repeated batches over the same ranges leave the number +/// of selected cells varying only within a narrow band. Selection is +/// recomputed from scratch on every batch, so stability here is a property of +/// the ranking rather than of any memory the selector keeps. +/// +/// ´claim:warmup:a-settled-traffic-pattern-settles-the-competitive-set-size-even-though-selection-is-recomputed-each-batch´ +/// ´test:integration:stage3-competitive-set-size-stabilises´ +#[test] +fn stage3_competitive_set_size_stabilises() { + let cfg = SentinelConfig:: { + split_threshold: 10, + ..integration_config() + }; + + let (mut s, _) = ScenarioBuilder::new() + .config(cfg) + .seed_range(0xA, 20) + .seed_range(0xB, 20) + .warm_batches(8) + .build_with_reports(); + + // Run 10 more batches and check variance of competitive_size. + let mut sizes = Vec::new(); + for _ in 0..10 { + let report = s.ingest(&[cell_values(0xA, 10), cell_values(0xB, 10)].concat()); + sizes.push(report.analysis_set_summary.competitive_size); + } + + // Stabilisation: low variance in competitive set size. + let min = *sizes.iter().min().unwrap(); + let max = *sizes.iter().max().unwrap(); + assert!(max - min <= 2, "competitive set size should stabilise (min={min}, max={max})"); +} + +/// The far end of the same rule. Seeding two well-separated ranges and +/// warming until several cells compete gives a common ancestor two +/// contributing subtrees, and coordination becomes active — either as reports +/// in the batch or as live contexts in the health snapshot. What switches +/// coordination on is the arrival of cells on both sides, which volume alone +/// decides. +/// +/// (´claim:warmup:coordination-fires-only-where-two-subtrees-both-contribute-cells´) +/// ´test:integration:stage3-coordination-activates´ +#[test] +fn stage3_coordination_activates() { + let mut s = ScenarioBuilder::new() + .config(SentinelConfig:: { + split_threshold: 10, + ..integration_config() + }) + .seed_range(0xA, 20) + .seed_range(0xB, 20) + .warm_batches(12) + .build(); + + let report = s.ingest(&[cell_values(0xA, 10), cell_values(0xB, 10)].concat()); + + // If there are multiple competitive cells, coordination contexts + // should have appeared. Cell creation depends on traffic volume + // (split_threshold), not λ — needs enough warm batches for the + // coordination context to observe both subtrees. + if report.analysis_set_summary.competitive_size >= 2 { + assert!( + !report.coordination_reports.is_empty() || report.health.coordination_health.active_contexts > 0, + "with ≥2 competitive cells, coordination should be active" + ); + } +} + +/// This pins the strongest reading of the guarantee: not that some sampled +/// report is well formed, but that every report is, across the whole warming +/// run and the batches that follow it. A transient violation between two +/// checked batches would be a violation. +/// +/// (´claim:warmup:the-report-invariants-hold-at-every-stage-of-warm-up-not-merely-once-it-has-settled´) +/// ´test:integration:stage3-invariants-hold-throughout´ +#[test] +fn stage3_invariants_hold_throughout() { + let cfg = SentinelConfig:: { + split_threshold: 10, + ..integration_config() + }; + + let (mut s, reports) = ScenarioBuilder::new() + .config(cfg) + .seed_range(0xA, 20) + .seed_range(0xB, 20) + .warm_batches(8) + .build_with_reports(); + + // Verify invariants on the last warm-up report. + if let Some(last) = reports.last() { + assert_invariants(&s, last); + } + + // Verify invariants on 5 further batches. + for _ in 0..5 { + let report = s.ingest(&[cell_values(0xA, 10), cell_values(0xB, 10)].concat()); + assert_invariants(&s, &report); + } +} + +// ── Stage 4: Steady state ─────────────────────────────────── + +/// Real data displaces the synthetic seed geometrically: each real batch +/// multiplies the synthetic share of a tracker's memory by the forgetting +/// factor raised to the batch size. A modest run of warm-up batches is +/// therefore enough to push the root well past the halfway mark, and the rate +/// is a property of the configured forgetting factor rather than of the data. +/// +/// ´claim:warmup:real-batches-displace-the-synthetic-seed-geometrically-so-maturity-arrives-within-a-predictable-number-of-batches´ +/// ´test:integration:stage4-root-maturity-below-half´ +#[test] +fn stage4_root_maturity_below_half() { + let s = ScenarioBuilder::new() + .config(SentinelConfig:: { + split_threshold: 10, + ..integration_config() + }) + .seed_range(0xA, 20) + .warm_batches(15) + .build(); + + // After sufficient batches, root tracker should be mature. + // 15 batches at λ=0.90 gives 0.90^15 = 0.206 fractional + // weight from noise — well under 0.5. See ADR-S-012. + let root = s.graph().g_root(); + let insp = s.inspect_cell(root).unwrap(); + assert!( + insp.maturity.noise_influence < 0.5, + "after 15 warm-up batches, root noise_influence should be < 0.5, got {:.4}", + insp.maturity.noise_influence + ); +} + +/// Maturity only ever improves while real data is arriving: batch after +/// batch, a cell's synthetic share is multiplied down and never rises again. +/// It could rise only if further noise were injected, and nothing injects +/// noise into a cell already in service. A host can therefore read the figure +/// as a one-way progress measure rather than as something that might rebound. +/// +/// ´claim:warmup:the-synthetic-share-of-a-serving-cell-never-rises-again´ +/// ´test:integration:stage4-maturity-decreases-monotonically´ +#[test] +fn stage4_maturity_decreases_monotonically() { + let mut s = Sentinel128::new(SentinelConfig:: { + split_threshold: 10, + ..integration_config() + }) + .unwrap(); + + let root = s.graph().g_root(); + let mut prev_ni = s.inspect_cell(root).unwrap().maturity.noise_influence; + + for _ in 0..10 { + s.ingest(&cell_values(0xA, 8)); + let ni = s.inspect_cell(root).unwrap().maturity.noise_influence; + assert!( + ni <= prev_ni + f64::EPSILON, + "noise_influence should not increase: prev={prev_ni:.6}, current={ni:.6}" + ); + prev_ni = ni; + } + + // After 10 batches, it should have decreased significantly. + assert!(prev_ni < 1.0, "noise_influence should decrease from 1.0 after real batches"); +} + +/// The health snapshot aggregates what individual cells know about their own +/// maturity, and in steady state it says two things: the average tracker is +/// no longer purely synthetic, and no tracker anywhere is still cold — every +/// cell in service has seen real data. A cold entry in a warmed sentinel +/// would mean a cell was being scored on noise alone. +/// +/// ´claim:warmup:a-warmed-sentinel-reports-no-cold-trackers-because-every-serving-cell-has-seen-real-data´ +/// ´test:integration:stage4-health-maturity-distribution´ +#[test] +fn stage4_health_maturity_distribution() { + let mut s = ScenarioBuilder::new() + .config(SentinelConfig:: { + split_threshold: 10, + ..integration_config() + }) + .seed_range(0xA, 20) + .warm_batches(15) + .build(); + + let report = s.ingest(&cell_values(0xA, 10)); + let maturity = &report.health.maturity_distribution; + + assert!( + maturity.mean_noise_influence < 1.0, + "mean noise influence should be below 1.0 in steady state, got {:.4}", + maturity.mean_noise_influence + ); + assert_eq!(maturity.cold_trackers, 0, "no cold trackers should remain in steady state"); + assert_invariants(&s, &report); +} + +// ── Cold start ────────────────────────────────────────────── + +/// Turning the noise schedule off shows what the seed was doing. The root +/// then begins at a synthetic share of exactly one — the value a tracker with +/// no information at all reports — and only real data moves it. Warm-up is +/// thus an optional head start, not a precondition: the sentinel still runs +/// without it, and simply says that everything it knows is unearned. +/// +/// ´claim:warmup:with-the-noise-schedule-empty-a-tracker-begins-fully-uninformed-and-only-real-data-moves-it´ +/// ´test:integration:cold-start-noise-influence-is-one´ +#[test] +fn cold_start_noise_influence_is_one() { + let mut s = Sentinel128::new(cold_config()).unwrap(); + + // With noise disabled, the root tracker starts completely cold. + let root = s.graph().g_root(); + let insp = s.inspect_cell(root).unwrap(); + assert!( + (insp.maturity.noise_influence - 1.0).abs() < f64::EPSILON, + "cold start: root noise_influence should be 1.0, got {:.4}", + insp.maturity.noise_influence + ); + + // After one batch of real data, noise_influence should drop. + let report = s.ingest(&cell_values(0xA, 20)); + let insp = s.inspect_cell(root).unwrap(); + assert!( + insp.maturity.noise_influence < 1.0, + "after real data, noise_influence should drop below 1.0, got {:.4}", + insp.maturity.noise_influence + ); + assert_invariants(&s, &report); +}