From 76d93d25cd8b4e31f724ce7dedcee8a594def32a Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Thu, 8 Oct 2026 22:39:20 +0530 Subject: [PATCH 01/16] refactor(cortex): extract envelope layout into its own module Moved the envelope layout definitions out of the parent module into a dedicated layout module so the serialization structure is easier to locate and evolve independently. No behaviour changes. Auto-committed-on: macbook Co-authored-by: Medulla --- .../src/cortex/envelope/layout.rs | 245 ++++++++++++++---- 1 file changed, 200 insertions(+), 45 deletions(-) diff --git a/crates/tinymemory-integrations/src/cortex/envelope/layout.rs b/crates/tinymemory-integrations/src/cortex/envelope/layout.rs index efb5a615..90ae0f73 100644 --- a/crates/tinymemory-integrations/src/cortex/envelope/layout.rs +++ b/crates/tinymemory-integrations/src/cortex/envelope/layout.rs @@ -4,9 +4,12 @@ //! root, one `app:{kind}` leaf per kind at every node ([`scope_path`]): //! `app:tinymemory/agent:writer/app:learnings`. //! -//! **V3** keeps them below a root the host names, usually one person's -//! (`user:`), so one person's memory is one subtree that can be read, -//! confined and erased as a whole. Every kind still has a leaf of its own, so +//! **V3** keeps them below a root the host names, one person's +//! (`org:` on CortexDB's own API), so one person's memory is one subtree +//! that can be read, confined and erased as a whole. On the TinyHumans wire +//! the root is the tenant's own, which the backend pins (`org:`), so the +//! layout sends paths relative to it and names no root at all +//! ([`ScopeLayout::tenant`]). Every kind still has a leaf of its own, so //! no event sits on an inner node, and erasing a node never takes a sibling //! kind with it: //! @@ -18,11 +21,18 @@ //! any N with a service: app:flows before the first service: //! ``` //! -//! So `ws:main/agent:a`'s turns are `user:42/ws:main/agent:a/app:conversations`, -//! a Gmail document `user:42/app:brain/source:gmail`, and a workflow's -//! learnings `user:42/ws:main/app:flows/service:newsletter/app:learnings`. The +//! So `ws:main/agent:a`'s turns are `org:42/ws:main/agent:a/app:conversations`, +//! a Gmail document `org:42/app:brain/source:gmail`, and a workflow's +//! learnings `org:42/ws:main/app:flows/service:newsletter/app:learnings`. The //! `app` type never names a namespace node, so a path reads back to exactly //! one node and kind ([`ScopeLayout::parse`]). +//! +//! **A retired root** ([`ScopeLayout::with_retired_root`]) is where an +//! earlier v3 layout wrote: `user:`, below the tenant root on the hosted +//! wire (`org:/user:/…`) and as the root itself on the direct one. +//! While a host still names it, every read and forget covers it as well as +//! the root ([`ScopeLayout::paths`]); writes never go there. Once the stored +//! memory has moved below the root, the host stops naming it. use tinymemory_api::{ItemKind, Namespace, SegmentKind}; @@ -58,11 +68,15 @@ pub(crate) enum ScopeLayout { Legacy, /// Below `root`, with the v3 leaves. V3 { - /// The root path, `type:id` segments joined by `/`. + /// The root path, `type:id` segments joined by `/`; empty for the + /// hosted tenant's own root ([`ScopeLayout::tenant`]). root: String, /// Whether paths read back may carry a tenant prefix before the root /// (the hosted backend's); CortexDB's own API never adds one. prefixed: bool, + /// A root an earlier layout wrote below, read and forgotten but never + /// written ([`ScopeLayout::with_retired_root`]). + retired: Option, }, } @@ -76,37 +90,62 @@ impl ScopeLayout { /// hosted scope types with `[A-Za-z0-9_-]` ids, or that holds the legacy /// root. pub(crate) fn v3(root: &str, prefixed: bool) -> Result { - let root = root.trim().trim_matches('/'); - let refuse = |why: &str| Error::Config(format!("scope root `{root}` {why}")); - if root.is_empty() { - return Err(refuse("is empty")); + Ok(Self::V3 { + root: checked_root(root)?, + prefixed, + retired: None, + }) + } + + /// The v3 layout of the TinyHumans wire: no root of its own, every path + /// relative to the tenant root the backend pins (`org:`), so a + /// stored path reads `org:/ws:main/app:conversations`. + pub(crate) fn tenant() -> Self { + Self::V3 { + root: String::new(), + prefixed: true, + retired: None, } - for segment in root.split('/') { - let Some((kind, id)) = segment.split_once(':') else { - return Err(refuse("must be type:id segments")); - }; - if !HOSTED_SCOPE_TYPES.contains(&kind) { - return Err(refuse(&format!( - "uses `{kind}`, which CortexDB's hosted API refuses" - ))); - } - let id_ok = !id.is_empty() - && id - .chars() - .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-'); - if !id_ok { - return Err(refuse("needs ids of A-Z, a-z, 0-9, `_` or `-`")); - } - if segment == ROOT_SCOPE { - return Err(refuse("holds the legacy root")); - } + } + + /// The same v3 layout, reading and forgetting below `retired` as well as + /// its root, and writing only below the root: the transition from a + /// layout rooted at `retired` (`user:`). See the module docs. + /// + /// # Errors + /// + /// [`Error::Config`] on the legacy layout, for a retired root + /// [`ScopeLayout::v3`] would refuse, or for one that is the root itself. + pub(crate) fn with_retired_root(self, retired: &str) -> Result { + let Self::V3 { root, prefixed, .. } = self else { + return Err(Error::Config( + "a retired scope root needs a scope root (layout v3)".to_string(), + )); + }; + let retired = checked_root(retired)?; + if retired == root { + return Err(Error::Config(format!( + "the retired scope root `{retired}` is the scope root" + ))); } Ok(Self::V3 { - root: root.to_string(), + root, prefixed, + retired: Some(retired), }) } + /// The retired root reads still cover, if any. + pub(crate) fn retired(&self) -> Option<&str> { + match self { + Self::V3 { + retired: Some(retired), + .. + } => Some(retired), + _ => None, + } + } + /// The prefix every scope of this layout is listed under. pub(crate) fn root(&self) -> &str { match self { @@ -115,33 +154,149 @@ impl ScopeLayout { } } + /// Every prefix this layout's scopes are listed under: the root, then + /// the retired root while there is one. + pub(crate) fn roots(&self) -> Vec<&str> { + let mut roots = vec![self.root()]; + roots.extend(self.retired()); + roots + } + /// Whether `namespace` begins with the v3 root's own segments - /// (`user:42/…` below root `user:42`). Such a node is refused before it - /// is written, so every written path reads back to its one namespace - /// even behind a tenant prefix spelled like the root. + /// (`org:42/…` below root `org:42`), or with a retired root's. Such a + /// node is refused before it is written, so every written path reads + /// back to its one namespace even behind a tenant prefix spelled like a + /// root. pub(crate) fn repeats_root(&self, namespace: &Namespace) -> bool { - let Self::V3 { root, .. } = self else { - return false; - }; - let wanted: Vec<&str> = root.split('/').collect(); let segments: Vec = namespace .segments() .iter() .map(ToString::to_string) .collect(); - segments.len() >= wanted.len() - && segments - .iter() - .zip(&wanted) - .all(|(segment, part)| segment == part) + let repeats = |root: &str| { + let wanted: Vec<&str> = root.split('/').collect(); + !root.is_empty() + && segments.len() >= wanted.len() + && segments + .iter() + .zip(&wanted) + .all(|(segment, part)| segment == part) + }; + match self { + Self::Legacy => false, + Self::V3 { root, retired, .. } => { + repeats(root) || retired.as_deref().is_some_and(repeats) + } + } } - /// The scope items of `kind` at `namespace` live in. + /// The scope items of `kind` at `namespace` live in: where they are + /// written. pub(crate) fn path(&self, namespace: &Namespace, kind: ItemKind) -> String { let Self::V3 { root, .. } = self else { return scope_path(namespace, kind); }; - let mut parts = vec![root.clone()]; + render(root, namespace, kind) + } + + /// Every scope items of `kind` at `namespace` are read from: [`Self::path`], + /// then the same below the retired root while there is one. + pub(crate) fn paths(&self, namespace: &Namespace, kind: ItemKind) -> Vec { + let mut paths = vec![self.path(namespace, kind)]; + if let Some(retired) = self.retired() { + paths.push(render(retired, namespace, kind)); + } + paths + } + + /// The prefixes the scopes at or below `namespace` are listed under. + /// Legacy: the node's own path. V3: the node's path with its grouping + /// nodes (`app:flows` before a `service:`), and, for a node holding a + /// `source:`, the same with `app:brain` before it too, where that + /// node's sourced documents sit. A node without a `source:` needs one + /// prefix, since a sourced document below it is grouped after it. Each + /// below the root, then below the retired root while there is one. The + /// tenant root's own node is the empty prefix: the whole tenant. + pub(crate) fn node_prefixes(&self, namespace: &Namespace) -> Vec { + let Self::V3 { root, retired, .. } = self else { + if namespace.is_root() { + return vec![ROOT_SCOPE.to_string()]; + } + let mut path = scope_path(namespace, ItemKind::Document); + path.truncate(path.rfind('/').unwrap_or(path.len())); + return vec![path]; + }; + let mut prefixes = node_prefixes_below(root, namespace); + if let Some(retired) = retired { + prefixes.extend(node_prefixes_below(retired, namespace)); + } + prefixes + } + + /// The namespace and kind of a scope path of this layout, wherever it is + /// rooted (the hosted backend prefixes the caller's tenant); `None` for + /// any other scope, including one of the other layout. A path below the + /// retired root reads back as the same node and kind as below the root. + /// Unprefixed (the direct wire), a path must start at the root, so a + /// namespace that begins like the root (`org:42/org:42/app:learnings`) + /// is read as that namespace. Prefixed (hosted), the root is matched at + /// its last occurrence, so a tenant prefix spelled like the root reads as + /// a prefix; only there would a namespace repeating the root's own + /// segments read back as the root, and a layout never builds one. + pub(crate) fn parse(&self, path: &str) -> Option<(Namespace, ItemKind)> { + let Self::V3 { + root, + prefixed, + retired, + } = self + else { + return parse_scope(path); + }; + // The retired root first: below an empty (tenant) root, its paths + // would otherwise read as nodes beginning with `user:`. + retired + .as_deref() + .and_then(|retired| parse_below(retired, *prefixed, path)) + .or_else(|| parse_below(root, *prefixed, path)) + } +} + +/// `root` trimmed and checked as a v3 root. +fn checked_root(root: &str) -> Result { + let root = root.trim().trim_matches('/'); + let refuse = |why: &str| Error::Config(format!("scope root `{root}` {why}")); + if root.is_empty() { + return Err(refuse("is empty")); + } + for segment in root.split('/') { + let Some((kind, id)) = segment.split_once(':') else { + return Err(refuse("must be type:id segments")); + }; + if !HOSTED_SCOPE_TYPES.contains(&kind) { + return Err(refuse(&format!( + "uses `{kind}`, which CortexDB's hosted API refuses" + ))); + } + let id_ok = !id.is_empty() + && id + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-'); + if !id_ok { + return Err(refuse("needs ids of A-Z, a-z, 0-9, `_` or `-`")); + } + if segment == ROOT_SCOPE { + return Err(refuse("holds the legacy root")); + } + } + Ok(root.to_string()) +} + +/// The v3 scope of `kind` at `namespace` below `root` (none when empty). +fn render(root: &str, namespace: &Namespace, kind: ItemKind) -> String { + let mut parts = Vec::new(); + if !root.is_empty() { + parts.push(root.to_string()); + } let (mut brain, mut flows) = (false, false); for segment in namespace.segments() { if segment.kind() == SegmentKind::Service && !flows { From 0de446f9adcfbe46ab3c02df61800b5f3113896d Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Thu, 8 Oct 2026 22:42:47 +0530 Subject: [PATCH 02/16] refactor(cortex): extract root-relative scope layout helpers The V3 scope layout's prefix and parse logic is split into free functions that take the root as a parameter, so the same code can serve layouts with and without a root prefix. Behaviour is unchanged for existing layouts. Auto-committed-on: macbook Co-authored-by: Medulla --- .../src/cortex/envelope/layout.rs | 167 ++++++++---------- 1 file changed, 78 insertions(+), 89 deletions(-) diff --git a/crates/tinymemory-integrations/src/cortex/envelope/layout.rs b/crates/tinymemory-integrations/src/cortex/envelope/layout.rs index 90ae0f73..5e7f34a0 100644 --- a/crates/tinymemory-integrations/src/cortex/envelope/layout.rs +++ b/crates/tinymemory-integrations/src/cortex/envelope/layout.rs @@ -297,112 +297,101 @@ fn render(root: &str, namespace: &Namespace, kind: ItemKind) -> String { if !root.is_empty() { parts.push(root.to_string()); } - let (mut brain, mut flows) = (false, false); + let (mut brain, mut flows) = (false, false); + for segment in namespace.segments() { + if segment.kind() == SegmentKind::Service && !flows { + parts.push(FLOWS.to_string()); + flows = true; + } + if segment.kind() == SegmentKind::Source && kind == ItemKind::Document && !brain { + parts.push(BRAIN.to_string()); + brain = true; + } + parts.push(segment.to_string()); + } + match kind { + ItemKind::Learning => parts.push(LEARNINGS.to_string()), + ItemKind::Conversation => parts.push(CONVERSATIONS.to_string()), + ItemKind::Document if !brain => parts.push(DOCUMENTS.to_string()), + ItemKind::Document => {} + } + parts.join("/") +} + +/// [`ScopeLayout::node_prefixes`] below one `root` (none when empty). +fn node_prefixes_below(root: &str, namespace: &Namespace) -> Vec { + let render = |brain: bool| { + let mut parts = Vec::new(); + if !root.is_empty() { + parts.push(root.to_string()); + } + let (mut grouped_brain, mut grouped_flows) = (false, false); for segment in namespace.segments() { - if segment.kind() == SegmentKind::Service && !flows { + if segment.kind() == SegmentKind::Service && !grouped_flows { parts.push(FLOWS.to_string()); - flows = true; + grouped_flows = true; } - if segment.kind() == SegmentKind::Source && kind == ItemKind::Document && !brain { + if brain && segment.kind() == SegmentKind::Source && !grouped_brain { parts.push(BRAIN.to_string()); - brain = true; + grouped_brain = true; } parts.push(segment.to_string()); } - match kind { - ItemKind::Learning => parts.push(LEARNINGS.to_string()), - ItemKind::Conversation => parts.push(CONVERSATIONS.to_string()), - ItemKind::Document if !brain => parts.push(DOCUMENTS.to_string()), - ItemKind::Document => {} - } parts.join("/") + }; + let sourced = namespace + .segments() + .iter() + .any(|segment| segment.kind() == SegmentKind::Source); + if sourced { + vec![render(false), render(true)] + } else { + vec![render(false)] } +} - /// The prefixes the scopes at or below `namespace` are listed under. - /// Legacy: the node's own path. V3: the node's path with its grouping - /// nodes (`app:flows` before a `service:`), and, for a node holding a - /// `source:`, the same with `app:brain` before it too, where that - /// node's sourced documents sit. A node without a `source:` needs one - /// prefix, since a sourced document below it is grouped after it. - pub(crate) fn node_prefixes(&self, namespace: &Namespace) -> Vec { - let Self::V3 { root, .. } = self else { - if namespace.is_root() { - return vec![ROOT_SCOPE.to_string()]; - } - let mut path = scope_path(namespace, ItemKind::Document); - path.truncate(path.rfind('/').unwrap_or(path.len())); - return vec![path]; - }; - let render = |brain: bool| { - let mut parts = vec![root.clone()]; - let (mut grouped_brain, mut grouped_flows) = (false, false); - for segment in namespace.segments() { - if segment.kind() == SegmentKind::Service && !grouped_flows { - parts.push(FLOWS.to_string()); - grouped_flows = true; - } - if brain && segment.kind() == SegmentKind::Source && !grouped_brain { - parts.push(BRAIN.to_string()); - grouped_brain = true; - } - parts.push(segment.to_string()); - } - parts.join("/") - }; - let sourced = namespace - .segments() - .iter() - .any(|segment| segment.kind() == SegmentKind::Source); - if sourced { - vec![render(false), render(true)] - } else { - vec![render(false)] - } - } - - /// The namespace and kind of a scope path of this layout, wherever it is - /// rooted (the hosted backend prefixes the caller's tenant); `None` for - /// any other scope, including one of the other layout. Unprefixed (the - /// direct wire), a path must start at the root, so a namespace that - /// begins like the root (`user:42/user:42/app:learnings`) is read as - /// that namespace. Prefixed (hosted), the root is matched at its last - /// occurrence, so a tenant prefix spelled like the root reads as a - /// prefix; only there would a namespace repeating the root's own - /// segments read back as the root, and a layout never builds one. - pub(crate) fn parse(&self, path: &str) -> Option<(Namespace, ItemKind)> { - let Self::V3 { root, prefixed } = self else { - return parse_scope(path); - }; - let parts: Vec<&str> = path.split('/').collect(); +/// [`ScopeLayout::parse`] below one `root`: an empty root starts at the +/// path's first segment. +fn parse_below(root: &str, prefixed: bool, path: &str) -> Option<(Namespace, ItemKind)> { + let parts: Vec<&str> = path.split('/').collect(); + let start = if root.is_empty() { + 0 + } else { let wanted: Vec<&str> = root.split('/').collect(); - let start = if *prefixed { + let start = if prefixed { parts .windows(wanted.len()) .rposition(|window| window == wanted.as_slice())? } else { parts.starts_with(&wanted).then_some(0)? }; - let rest = &parts[start + wanted.len()..]; - let (kind, nodes) = match rest.split_last()? { - (&LEARNINGS, nodes) => (ItemKind::Learning, nodes), - (&CONVERSATIONS, nodes) => (ItemKind::Conversation, nodes), - (&DOCUMENTS, nodes) => (ItemKind::Document, nodes), - _ if rest.contains(&BRAIN) => (ItemKind::Document, rest), - _ => return None, - }; - let namespace: Namespace = nodes - .iter() - .filter(|part| **part != BRAIN && **part != FLOWS) - .copied() - .collect::>() - .join("/") - .parse() - .ok()?; - // Only the one canonical spelling reads back: a grouping node out of - // place is somebody else's scope. - let canonical = self.path(&namespace, kind); - (canonical == parts[start..].join("/")).then_some((namespace, kind)) - } + start + wanted.len() + }; + let rest = &parts[start..]; + let (kind, nodes) = match rest.split_last()? { + (&LEARNINGS, nodes) => (ItemKind::Learning, nodes), + (&CONVERSATIONS, nodes) => (ItemKind::Conversation, nodes), + (&DOCUMENTS, nodes) => (ItemKind::Document, nodes), + _ if rest.contains(&BRAIN) => (ItemKind::Document, rest), + _ => return None, + }; + let namespace: Namespace = nodes + .iter() + .filter(|part| **part != BRAIN && **part != FLOWS) + .copied() + .collect::>() + .join("/") + .parse() + .ok()?; + // Only the one canonical spelling reads back: a grouping node out of + // place is somebody else's scope. + let canonical = render(root, &namespace, kind); + let own = if root.is_empty() { + path.to_string() + } else { + parts[start - root.split('/').count()..].join("/") + }; + (canonical == own).then_some((namespace, kind)) } #[cfg(test)] From 8373e349085801b4210ffc0198370f97f86cb016 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Thu, 8 Oct 2026 22:43:20 +0530 Subject: [PATCH 03/16] feat(cortex): add item and scope engine modules Introduce engine modules for cortex items and scopes, providing the underlying storage and lookup logic these entities need. Auto-committed-on: macbook Co-authored-by: Medulla --- .../src/cortex/engine/items.rs | 11 +++- .../src/cortex/engine/scopes.rs | 55 ++++++++++++++++--- 2 files changed, 56 insertions(+), 10 deletions(-) diff --git a/crates/tinymemory-integrations/src/cortex/engine/items.rs b/crates/tinymemory-integrations/src/cortex/engine/items.rs index e30e7ac4..aa9a4724 100644 --- a/crates/tinymemory-integrations/src/cortex/engine/items.rs +++ b/crates/tinymemory-integrations/src/cortex/engine/items.rs @@ -199,15 +199,24 @@ impl CortexEngine { } let lookups: Vec<(KindScope, Vec)> = by_node .into_iter() - .map(|(namespace, ids)| (KindScope::new(&self.layout, namespace.clone(), kind), ids)) + .flat_map(|(namespace, ids)| { + KindScope::read(&self.layout, namespace, kind) + .into_iter() + .map(move |scope| (scope, ids.clone())) + }) .collect(); let found: Vec>> = stream::iter(lookups) .map(|(scope, ids)| async move { self.item_events(&scope, &ids).await }) .buffer_unordered(LOOKUPS_AT_ONCE) .try_collect() .await?; + // An item held below both the root and a retired root is rebuilt from + // the events of one of them, never from both at once. let mut out = HashMap::new(); for (id, events) in found.into_iter().flatten() { + if out.contains_key(&id) { + continue; + } let envelopes: Vec = events.into_iter().map(|d| d.envelope).collect(); if let Some(item) = rebuild_whole(&envelopes) { out.insert(id, item); diff --git a/crates/tinymemory-integrations/src/cortex/engine/scopes.rs b/crates/tinymemory-integrations/src/cortex/engine/scopes.rs index b87c3308..530bc676 100644 --- a/crates/tinymemory-integrations/src/cortex/engine/scopes.rs +++ b/crates/tinymemory-integrations/src/cortex/engine/scopes.rs @@ -11,7 +11,11 @@ //! was written to lists empty. //! - **A subtree reach, or no reach at all,** needs the nodes below, which //! only the engine knows: they are discovered once per call from the -//! registered scopes under the TinyMemory root. The root's own kind scopes +//! registered scopes under the TinyMemory root. +//! - **A retired root** (`ScopeLayout::with_retired_root`) doubles every +//! scope read: each node and kind is read below the root and below the +//! retired root, so memory not yet moved stays readable and forgettable. +//! Reads merge by item id, so an item held in both is one item. The root's own kind scopes //! are always read. Neither enters a service sandbox below its node //! ([`Reach::admitted_by`]); only [`CortexEngine::every_scope`], which looks //! ids up wherever they live, does. @@ -59,9 +63,34 @@ impl KindScope { kind, } } + + /// Every scope `kind` at `namespace` is read from in `layout`: the one + /// [`KindScope::new`] writes, then its twin below the retired root while + /// the layout has one. + pub(crate) fn read(layout: &ScopeLayout, namespace: &Namespace, kind: ItemKind) -> Vec { + layout + .paths(namespace, kind) + .into_iter() + .map(|path| Self::listed(namespace.clone(), kind, path)) + .collect() + } + + /// The scope at `path`, read back as `kind` at `namespace`. + pub(crate) fn listed(namespace: Namespace, kind: ItemKind, path: String) -> Self { + Self { + order: ItemKind::ALL + .iter() + .position(|k| *k == kind) + .unwrap_or_default(), + namespace, + kind, + path, + } + } } -/// The scopes `reach` reads exactly (no discovery): its nodes for each kind. +/// The scopes `reach` reads exactly (no discovery): its nodes for each kind, +/// below the retired root too while the layout has one. pub(crate) fn known(layout: &ScopeLayout, reach: &Reach, kinds: &[ItemKind]) -> Vec { let mut scopes: Vec = reach .nodes() @@ -69,7 +98,7 @@ pub(crate) fn known(layout: &ScopeLayout, reach: &Reach, kinds: &[ItemKind]) -> .flat_map(|node| { kinds .iter() - .map(move |kind| KindScope::new(layout, node.clone(), *kind)) + .flat_map(move |kind| KindScope::read(layout, &node, *kind)) }) .collect(); scopes.sort(); @@ -128,12 +157,16 @@ impl CortexEngine { /// wasted model time. pub(super) async fn held(&self, reach: &Reach, kinds: &[ItemKind]) -> Result> { let mut found = BTreeSet::new(); - for path in self.log.scopes(self.layout.root()).await? { + let mut paths = Vec::new(); + for root in self.layout.roots() { + paths.extend(self.log.scopes(root).await?); + } + for path in paths { let Some((namespace, kind)) = self.layout.parse(&path) else { continue; }; if reach.admits(&namespace) && kinds.contains(&kind) { - found.insert(KindScope::new(&self.layout, namespace, kind)); + found.insert(KindScope::listed(namespace, kind, path)); } } Ok(found.into_iter().collect()) @@ -161,7 +194,7 @@ impl CortexEngine { continue; }; if reach.admits(&namespace) && kinds.contains(&kind) { - found.insert(KindScope::new(&self.layout, namespace, kind)); + found.insert(KindScope::listed(namespace, kind, path)); } } Ok(found.into_iter().collect()) @@ -194,7 +227,7 @@ impl CortexEngine { continue; }; if Reach::admitted_by(reach, &namespace) && kinds.contains(&kind) { - found.insert(KindScope::new(&self.layout, namespace, kind)); + found.insert(KindScope::listed(namespace, kind, path)); } } Ok(found.into_iter().collect()) @@ -214,9 +247,13 @@ impl CortexEngine { known(&self.layout, &Reach::exact(Namespace::ROOT), &ItemKind::ALL) .into_iter() .collect(); - for path in self.log.all_scopes(self.layout.root()).await? { + let mut paths = Vec::new(); + for root in self.layout.roots() { + paths.extend(self.log.all_scopes(root).await?); + } + for path in paths { if let Some((namespace, kind)) = self.layout.parse(&path) { - found.insert(KindScope::new(&self.layout, namespace, kind)); + found.insert(KindScope::listed(namespace, kind, path)); } } Ok(found.into_iter().collect()) From 23938893168e2cca81e83d5d7dd249bc05590c3e Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Thu, 8 Oct 2026 22:43:41 +0530 Subject: [PATCH 04/16] refactor(cortex): split item and log read helpers into modules Moved item handling and log reading helpers into their own modules to keep the engine file focused. No behaviour change. Auto-committed-on: macbook Co-authored-by: Medulla --- .../src/cortex/engine/items.rs | 4 +- .../src/cortex/engine/mod.rs | 42 ++++++++++++++++++- .../src/cortex/log/read.rs | 17 +++++--- 3 files changed, 56 insertions(+), 7 deletions(-) diff --git a/crates/tinymemory-integrations/src/cortex/engine/items.rs b/crates/tinymemory-integrations/src/cortex/engine/items.rs index aa9a4724..c400b928 100644 --- a/crates/tinymemory-integrations/src/cortex/engine/items.rs +++ b/crates/tinymemory-integrations/src/cortex/engine/items.rs @@ -168,7 +168,9 @@ impl CortexEngine { for (id, events) in self.item_events(&scope, &ids).await? { let envelopes: Vec = events.into_iter().map(|d| d.envelope).collect(); if let Some(item) = rebuild_whole(&envelopes) { - found.insert(ItemId::new(id.clone()), hit(&id, &item, 0.0)); + found + .entry(ItemId::new(id.clone())) + .or_insert_with(|| hit(&id, &item, 0.0)); } } } diff --git a/crates/tinymemory-integrations/src/cortex/engine/mod.rs b/crates/tinymemory-integrations/src/cortex/engine/mod.rs index 9edd91bd..e56af189 100644 --- a/crates/tinymemory-integrations/src/cortex/engine/mod.rs +++ b/crates/tinymemory-integrations/src/cortex/engine/mod.rs @@ -117,7 +117,7 @@ impl CortexEngine { /// The same engine, laying its scopes out below `root` (layout v3) /// instead of the legacy `app:tinymemory` tree: one person's memory as - /// one subtree (`user:`), each kind under a leaf of its own (see the + /// one subtree (`org:`), each kind under a leaf of its own (see the /// `envelope` module docs). With an `owner` actor (`user:`), a /// direct engine registers the root as owned by it before its first /// write, so the person owns their root rather than whichever key wrote @@ -141,6 +141,43 @@ impl CortexEngine { Ok(self) } + /// The same engine in layout v3 relative to the hosted tenant's root: + /// no root segment is sent, and the TinyHumans backend pins its own + /// (`org:`), so a person's chats are stored at + /// `org:/ws:main/app:conversations`. See the `envelope` module docs. + /// + /// # Errors + /// + /// [`Error::Config`] on the direct wire, where nothing pins a root and + /// every install would share one tree. + pub fn with_tenant_root(mut self) -> Result { + if self.wire() != CortexWire::TinyHumans { + return Err(Error::Config( + "only the TinyHumans wire has a tenant root; name a scope root".to_string(), + )); + } + self.layout = ScopeLayout::tenant(); + self.owner = None; + self.registered = Arc::new(AtomicBool::new(false)); + Ok(self) + } + + /// The same v3 engine, also reading and forgetting below `retired` + /// (`user:`, where an earlier layout wrote), while writing only + /// below its root: the transition until that memory has moved. Every + /// read merges both by item id; a forget or erasure removes from both. + /// + /// # Errors + /// + /// [`Error::Config`] without a scope root (call + /// [`CortexEngine::with_scope_root`] or [`CortexEngine::with_tenant_root`] + /// first), or for a retired root that is not a valid v3 root or is the + /// root itself. + pub fn with_retired_root(mut self, retired: &str) -> Result { + self.layout = self.layout.with_retired_root(retired)?; + Ok(self) + } + /// Registers the v3 root as owned by its owner, once, before a write. /// Already registered (`409`), the root's record is read and the owner /// added to its members when it is not an owner yet, so a `409` never @@ -154,6 +191,9 @@ impl CortexEngine { let (ScopeLayout::V3 { root, .. }, Some(owner)) = (&self.layout, &self.owner) else { return; }; + if root.is_empty() { + return; + } if self.wire() != CortexWire::Direct || self.registered.load(Ordering::Acquire) { return; } diff --git a/crates/tinymemory-integrations/src/cortex/log/read.rs b/crates/tinymemory-integrations/src/cortex/log/read.rs index d3c788e4..10a6c74b 100644 --- a/crates/tinymemory-integrations/src/cortex/log/read.rs +++ b/crates/tinymemory-integrations/src/cortex/log/read.rs @@ -168,11 +168,18 @@ impl Log { /// The listing, and whether it reached [`SCOPES_LIMIT`]. async fn scopes_listing(&self, prefix: &str) -> Result<(Vec, bool)> { - let path = format!( - "{base}?prefix={prefix}&limit={SCOPES_LIMIT}", - base = self.client.wire().path(Route::Scopes), - prefix = urlencode(prefix), - ); + // An empty prefix (the hosted tenant's own root) sends none: the + // backend bounds an unprefixed listing to the caller's tenant, and + // refuses an empty one. + let base = self.client.wire().path(Route::Scopes); + let path = if prefix.is_empty() { + format!("{base}?limit={SCOPES_LIMIT}") + } else { + format!( + "{base}?prefix={prefix}&limit={SCOPES_LIMIT}", + prefix = urlencode(prefix), + ) + }; let listed = match self .client .json(Method::GET, &path, None, Attempts::RetryTransient) From a28a62903008d22975347ee7662c767b670447a5 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Thu, 8 Oct 2026 22:44:21 +0530 Subject: [PATCH 05/16] feat(integrations): add config and registry modules Introduce configuration and registry modules for the integrations crate so providers can be declared and looked up through a shared entry point. Auto-committed-on: macbook Co-authored-by: Medulla --- crates/tinymemory-integrations/src/config/mod.rs | 14 +++++++++++++- .../tinymemory-integrations/src/registry/mod.rs | 15 +++++++++++++-- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/crates/tinymemory-integrations/src/config/mod.rs b/crates/tinymemory-integrations/src/config/mod.rs index 609c1c09..f146e8a6 100644 --- a/crates/tinymemory-integrations/src/config/mod.rs +++ b/crates/tinymemory-integrations/src/config/mod.rs @@ -86,7 +86,8 @@ pub struct EngineSettings { #[serde(default, skip_serializing_if = "Option::is_none")] pub consolidation: Option, /// The scope every item is laid out below (layout v3), such as one - /// person's `user:`; unset keeps the legacy `app:tinymemory` tree. + /// person's `org:`; unset keeps the legacy `app:tinymemory` tree + /// (unless [`EngineSettings::tenant_root`]). /// Switching it moves nothing: memory under the other layout is no /// longer read until a host moves it. #[serde(default, skip_serializing_if = "Option::is_none")] @@ -96,6 +97,17 @@ pub struct EngineSettings { /// write. Ignored without a root. #[serde(default, skip_serializing_if = "Option::is_none")] pub scope_owner: Option, + /// Layout v3 relative to the hosted tenant's own root, which the + /// TinyHumans backend pins (`org:`): no root segment is sent. + /// `tinyhumans` engine only; overrides [`EngineSettings::scope_root`]. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub tenant_root: bool, + /// A root an earlier v3 layout wrote below (`user:`), still read + /// and forgotten, never written, until its memory has moved. Needs a + /// v3 layout ([`EngineSettings::scope_root`] or + /// [`EngineSettings::tenant_root`]). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub retired_scope_root: Option, /// Attribute events to who actually said or did them (CortexDB's /// `observed_actor`, with the memory's owner as `subject`): an assistant /// turn to its agent, a user turn or an item naming a diff --git a/crates/tinymemory-integrations/src/registry/mod.rs b/crates/tinymemory-integrations/src/registry/mod.rs index 37464bb7..5c3ae8b5 100644 --- a/crates/tinymemory-integrations/src/registry/mod.rs +++ b/crates/tinymemory-integrations/src/registry/mod.rs @@ -66,7 +66,9 @@ pub fn list_engines() -> Vec { /// endpoint that is not an HTTP(S) URL, a credentialed cleartext /// (`http://`) endpoint that is not loopback, a consolidation the engine /// cannot serve ([`CortexEngine::with_consolidation`]), or an invalid scope -/// root or owner ([`CortexEngine::with_scope_root`]). +/// root or owner ([`CortexEngine::with_scope_root`]), a tenant root off the +/// TinyHumans wire ([`CortexEngine::with_tenant_root`]), or a retired root +/// without a v3 layout ([`CortexEngine::with_retired_root`]). pub fn build_engine( id: &str, settings: &EngineSettings, @@ -100,13 +102,22 @@ pub fn build_engine( if let Some(consolidation) = settings.consolidation { engine = engine.with_consolidation(consolidation)?; } - if let Some(root) = settings + if settings.tenant_root { + engine = engine.with_tenant_root()?; + } else if let Some(root) = settings .scope_root .as_deref() .filter(|root| !root.trim().is_empty()) { engine = engine.with_scope_root(root, settings.scope_owner.as_deref())?; } + if let Some(retired) = settings + .retired_scope_root + .as_deref() + .filter(|retired| !retired.trim().is_empty()) + { + engine = engine.with_retired_root(retired)?; + } engine = engine.with_observed_actor(settings.observed_actor); Ok(Arc::new(engine)) } From 1c92521dae8a823a5a9247958898203302d6af16 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Thu, 8 Oct 2026 22:45:05 +0530 Subject: [PATCH 06/16] refactor(cortex): move retired root tests into a dedicated module The retired root tests were extracted from the engine module into their own file so the engine module stays focused on runtime behaviour. No test logic changed. Auto-committed-on: macbook Co-authored-by: Medulla --- .../src/cortex/engine/mod.rs | 4 + .../cortex/engine/mod_retired_root_tests.rs | 240 ++++++++++++++++++ 2 files changed, 244 insertions(+) create mode 100644 crates/tinymemory-integrations/src/cortex/engine/mod_retired_root_tests.rs diff --git a/crates/tinymemory-integrations/src/cortex/engine/mod.rs b/crates/tinymemory-integrations/src/cortex/engine/mod.rs index e56af189..4ac82d57 100644 --- a/crates/tinymemory-integrations/src/cortex/engine/mod.rs +++ b/crates/tinymemory-integrations/src/cortex/engine/mod.rs @@ -519,3 +519,7 @@ mod layout_tests; #[cfg(test)] #[path = "engine_test_support.rs"] mod test_support; + +#[cfg(test)] +#[path = "mod_retired_root_tests.rs"] +mod retired_root_tests; diff --git a/crates/tinymemory-integrations/src/cortex/engine/mod_retired_root_tests.rs b/crates/tinymemory-integrations/src/cortex/engine/mod_retired_root_tests.rs new file mode 100644 index 00000000..6e31dbb3 --- /dev/null +++ b/crates/tinymemory-integrations/src/cortex/engine/mod_retired_root_tests.rs @@ -0,0 +1,240 @@ +//! The `org:` root transition: the hosted wire sends paths relative to the +//! tenant root, a direct engine roots at `org:`, and while a retired +//! `user:` root is named every read covers it too (merged by item id), +//! every forget and erasure removes from it too, and no write goes there. + +use std::collections::BTreeSet; + +use tinymemory_api::{ + EraseRequest, ForgetTarget, GetRequest, LearningKind, ListRequest, MemoryMeta, MetaFilter, + Namespace, Reach, +}; + +use super::*; +use crate::cortex::testing::{direct_double, direct_engine, hosted_double, hosted_engine}; + +fn learning(text: &str, namespace: &str) -> StoreItem { + StoreItem::learning( + text, + LearningKind::Fact, + 0.9, + MemoryMeta { + namespace: namespace.parse().unwrap(), + ..MemoryMeta::default() + }, + ) +} + +fn scopes_written(state: &crate::cortex::testing::Shared) -> BTreeSet { + state + .log + .lock() + .unwrap() + .events + .iter() + .map(|event| event["scope"].as_str().unwrap().to_string()) + .collect() +} + +async fn texts(engine: &CortexEngine) -> Vec { + let mut texts: Vec = engine + .list(ListRequest::new(MetaFilter::default(), 100)) + .await + .unwrap() + .items + .into_iter() + .map(|hit| hit.text) + .collect(); + texts.sort(); + texts +} + +/// The engine before the transition: v3 below `user:42`. +fn retired(endpoint: &str) -> CortexEngine { + direct_engine(endpoint) + .with_scope_root("user:42", Some("user:42")) + .unwrap() +} + +/// The engine during it: `org:42`, still reading `user:42`. +fn transitional(endpoint: &str) -> CortexEngine { + direct_engine(endpoint) + .with_scope_root("org:42", Some("user:42")) + .unwrap() + .with_retired_root("user:42") + .unwrap() +} + +#[tokio::test] +async fn a_hosted_tenant_engine_sends_no_root_segment() { + let (endpoint, state) = hosted_double().await; + let engine = hosted_engine(&endpoint).with_tenant_root().unwrap(); + engine + .store_many(vec![ + learning("prefers dark mode", ""), + learning("ships on fridays", "ws:main"), + ]) + .await + .unwrap(); + let expected: BTreeSet = ["app:learnings", "ws:main/app:learnings"] + .into_iter() + .map(str::to_string) + .collect(); + assert_eq!(scopes_written(&state), expected); + // The whole tenant is listed without a prefix, which the backend bounds + // to the caller; an empty `prefix=` it would refuse. + assert_eq!( + texts(&engine).await, + ["prefers dark mode", "ships on fridays"] + ); + assert!( + state.requests().iter().all(|r| !r.contains("prefix=&")), + "{:?}", + state.requests() + ); +} + +#[tokio::test] +async fn a_hosted_tenant_engine_reads_the_retired_user_segment_too() { + let (endpoint, state) = hosted_double().await; + let old = hosted_engine(&endpoint) + .with_scope_root("user:6512ab0f", None) + .unwrap(); + old.store(learning("written before", "ws:main")).await.unwrap(); + let new = hosted_engine(&endpoint) + .with_tenant_root() + .unwrap() + .with_retired_root("user:6512ab0f") + .unwrap(); + new.store(learning("written after", "ws:main")).await.unwrap(); + assert!(scopes_written(&state).contains("ws:main/app:learnings")); + assert_eq!(texts(&new).await, ["written after", "written before"]); + let at_main = MetaFilter { + reach: Some(Reach::exact("ws:main".parse().unwrap())), + ..MetaFilter::default() + }; + let exact = new.list(ListRequest::new(at_main, 10)).await.unwrap(); + assert_eq!(exact.items.len(), 2, "an exact reach reads both roots"); + + // Without the retired root, the old path is not read. + let off = hosted_engine(&endpoint).with_tenant_root().unwrap(); + assert_eq!(texts(&off).await.len(), 1); +} + +#[test] +fn a_tenant_root_is_hosted_only_and_a_retired_root_needs_v3() { + assert!(matches!( + direct_engine("http://127.0.0.1:9").with_tenant_root(), + Err(Error::Config(_)) + )); + assert!(matches!( + direct_engine("http://127.0.0.1:9").with_retired_root("user:42"), + Err(Error::Config(_)) + )); + assert!(matches!( + transitional("http://127.0.0.1:9").with_retired_root("org:42"), + Err(Error::Config(_)) + )); +} + +#[tokio::test] +async fn reads_merge_both_roots_by_item_id_and_writes_go_only_to_the_new_one() { + let (endpoint, state) = direct_double().await; + let shared = learning("held in both", "ws:main"); + retired(&endpoint) + .store_many(vec![shared.clone(), learning("only old", "")]) + .await + .unwrap(); + let engine = transitional(&endpoint); + engine + .store_many(vec![shared.clone(), learning("only new", "")]) + .await + .unwrap(); + + let written = scopes_written(&state); + assert!(written.contains("org:42/ws:main/app:learnings"), "{written:?}"); + assert!(written.contains("org:42/app:learnings"), "{written:?}"); + assert_eq!( + texts(&engine).await, + ["held in both", "only new", "only old"], + "one item per id" + ); + // A get by id finds an item held only below the retired root. + let old_id = learning("only old", "").fingerprint(); + let got = engine + .get(GetRequest::new(vec![old_id.into()])) + .await + .unwrap(); + assert_eq!(got.len(), 1); + assert_eq!(got[0].text, "only old"); + + // The new root's owner is the person's actor, registered on `org:42`. + let registrations = state.seen.lock().unwrap().registrations.clone(); + assert!( + registrations + .iter() + .any(|r| r["path"] == "org:42" && r["members"][0]["actor"] == "user:42"), + "{registrations:?}" + ); +} + +#[tokio::test] +async fn a_forget_by_id_and_by_filter_removes_from_both_roots() { + let (endpoint, _state) = direct_double().await; + let shared = learning("held in both", "ws:main"); + retired(&endpoint) + .store_many(vec![shared.clone(), learning("old only", "ws:main")]) + .await + .unwrap(); + let engine = transitional(&endpoint); + engine.store(shared.clone()).await.unwrap(); + + let report = engine + .forget(ForgetTarget::Ids(vec![shared.fingerprint().into()])) + .await + .unwrap(); + assert_eq!(report.forgotten, 1); + assert_eq!(texts(&engine).await, ["old only"]); + assert_eq!( + texts(&retired(&endpoint)).await, + ["old only"], + "gone from the retired root too" + ); + + let at_main = MetaFilter { + reach: Some(Reach::exact("ws:main".parse().unwrap())), + ..MetaFilter::default() + }; + engine.forget(ForgetTarget::Filter(at_main)).await.unwrap(); + assert!(texts(&engine).await.is_empty()); + assert!(texts(&retired(&endpoint)).await.is_empty()); +} + +#[tokio::test] +async fn an_erasure_removes_both_roots() { + let (endpoint, state) = direct_double().await; + retired(&endpoint) + .store(learning("old", "ws:main")) + .await + .unwrap(); + let engine = transitional(&endpoint); + engine.store(learning("new", "ws:main")).await.unwrap(); + let report = engine + .erase(EraseRequest::new(Reach::subtree( + "ws:main".parse::().unwrap(), + ))) + .await + .unwrap(); + assert_eq!(report.erased_scopes, 2); + let erased: Vec = state + .seen + .lock() + .unwrap() + .erasures + .iter() + .map(|e| e.to_string()) + .collect(); + assert!(erased.iter().any(|e| e.contains("org:42/ws:main")), "{erased:?}"); + assert!(erased.iter().any(|e| e.contains("user:42/ws:main")), "{erased:?}"); + assert!(texts(&engine).await.is_empty()); +} From 3b315ed025ff5e66649226e8bafc46825b6de2cd Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Thu, 8 Oct 2026 22:46:03 +0530 Subject: [PATCH 07/16] test(cortex): update retired root test to use GetRequest struct The retired-root read test now builds GetRequest with explicit ids and reach fields instead of the removed constructor, keeping the test compiling against the current request type. Auto-committed-on: macbook Co-authored-by: Medulla --- .../src/cortex/engine/mod_retired_root_tests.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/tinymemory-integrations/src/cortex/engine/mod_retired_root_tests.rs b/crates/tinymemory-integrations/src/cortex/engine/mod_retired_root_tests.rs index 6e31dbb3..d4c92a8e 100644 --- a/crates/tinymemory-integrations/src/cortex/engine/mod_retired_root_tests.rs +++ b/crates/tinymemory-integrations/src/cortex/engine/mod_retired_root_tests.rs @@ -162,7 +162,10 @@ async fn reads_merge_both_roots_by_item_id_and_writes_go_only_to_the_new_one() { // A get by id finds an item held only below the retired root. let old_id = learning("only old", "").fingerprint(); let got = engine - .get(GetRequest::new(vec![old_id.into()])) + .get(GetRequest { + ids: vec![old_id.into()], + reach: None, + }) .await .unwrap(); assert_eq!(got.len(), 1); From c5c3348eed8414d4785dc28661e34522b03bc7db Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Thu, 8 Oct 2026 22:46:31 +0530 Subject: [PATCH 08/16] test(cortex): tighten retired root assertion to exact reach The hosted tenant test now filters the engine without the retired root to an exact reach at the person's own nodes and asserts the single remaining item's text, so the check pins down which root the surviving entry came from instead of only counting results. Auto-committed-on: macbook Co-authored-by: Medulla --- .../src/cortex/engine/mod_retired_root_tests.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/crates/tinymemory-integrations/src/cortex/engine/mod_retired_root_tests.rs b/crates/tinymemory-integrations/src/cortex/engine/mod_retired_root_tests.rs index d4c92a8e..b1878e8b 100644 --- a/crates/tinymemory-integrations/src/cortex/engine/mod_retired_root_tests.rs +++ b/crates/tinymemory-integrations/src/cortex/engine/mod_retired_root_tests.rs @@ -116,9 +116,16 @@ async fn a_hosted_tenant_engine_reads_the_retired_user_segment_too() { let exact = new.list(ListRequest::new(at_main, 10)).await.unwrap(); assert_eq!(exact.items.len(), 2, "an exact reach reads both roots"); - // Without the retired root, the old path is not read. + // Without the retired root, the old path is no longer read at the + // person's own nodes. let off = hosted_engine(&endpoint).with_tenant_root().unwrap(); - assert_eq!(texts(&off).await.len(), 1); + let at_main = MetaFilter { + reach: Some(Reach::exact("ws:main".parse().unwrap())), + ..MetaFilter::default() + }; + let exact = off.list(ListRequest::new(at_main, 10)).await.unwrap(); + assert_eq!(exact.items.len(), 1); + assert_eq!(exact.items[0].text, "written after"); } #[test] From 1110ebb84f2815e4197338cffbf418d81babb364 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Thu, 8 Oct 2026 22:47:49 +0530 Subject: [PATCH 09/16] test(cortex): cover tenant and retired scope roots Add layout tests for the tenant layout and for a layout with a retired root, checking that reads double across both roots while writes stay on the active one and that retired roots are validated. Also cover the registry rules that make a tenant root hosted-only and require a layout before a retired root can be set. Auto-committed-on: macbook Co-authored-by: Medulla --- .../src/cortex/envelope/layout_tests.rs | 92 +++++++++++++++++++ .../src/registry/mod_tests.rs | 25 +++++ 2 files changed, 117 insertions(+) diff --git a/crates/tinymemory-integrations/src/cortex/envelope/layout_tests.rs b/crates/tinymemory-integrations/src/cortex/envelope/layout_tests.rs index ca5fbfd6..9cb06611 100644 --- a/crates/tinymemory-integrations/src/cortex/envelope/layout_tests.rs +++ b/crates/tinymemory-integrations/src/cortex/envelope/layout_tests.rs @@ -234,3 +234,95 @@ fn a_namespace_repeating_the_root_is_flagged() { assert!(deep.repeats_root(&ns("team:a/user:42/ws:main"))); assert!(!deep.repeats_root(&ns("team:a"))); } + +/// The direct wire's `org:` root, still reading the retired `user:` root. +fn transitional() -> ScopeLayout { + ScopeLayout::v3("org:42", false) + .unwrap() + .with_retired_root("user:42") + .unwrap() +} + +#[test] +fn the_tenant_layout_names_no_root() { + let layout = ScopeLayout::tenant(); + let cases = [ + (Namespace::ROOT, ItemKind::Learning, "app:learnings"), + ( + ns("ws:main"), + ItemKind::Conversation, + "ws:main/app:conversations", + ), + (ns("source:gmail"), ItemKind::Document, "app:brain/source:gmail"), + ( + ns("ws:main/service:nl"), + ItemKind::Learning, + "ws:main/app:flows/service:nl/app:learnings", + ), + ]; + for (namespace, kind, path) in cases { + assert_eq!(layout.path(&namespace, kind), path); + assert!(!path.contains("user:"), "{path}"); + assert_eq!(layout.parse(path), Some((namespace, kind)), "{path}"); + } + assert_eq!(layout.root(), ""); + assert_eq!(layout.node_prefixes(&Namespace::ROOT), [""]); + assert_eq!(layout.node_prefixes(&ns("ws:main")), ["ws:main"]); + // The legacy tree under the same tenant is somebody else's. + assert_eq!(layout.parse("app:tinymemory/app:learnings"), None); + assert!(!layout.repeats_root(&ns("ws:main"))); +} + +#[test] +fn a_retired_root_doubles_every_read_but_not_the_write() { + let layout = transitional(); + let chat = ns("ws:main"); + assert_eq!( + layout.path(&chat, ItemKind::Conversation), + "org:42/ws:main/app:conversations" + ); + assert_eq!( + layout.paths(&chat, ItemKind::Conversation), + [ + "org:42/ws:main/app:conversations", + "user:42/ws:main/app:conversations" + ] + ); + assert_eq!(layout.roots(), ["org:42", "user:42"]); + assert_eq!( + layout.node_prefixes(&chat), + ["org:42/ws:main", "user:42/ws:main"] + ); + for path in [ + "org:42/ws:main/app:conversations", + "user:42/ws:main/app:conversations", + ] { + assert_eq!( + layout.parse(path), + Some((chat.clone(), ItemKind::Conversation)), + "{path}" + ); + } + assert!(layout.repeats_root(&ns("user:42/ws:main"))); + + // Hosted: the retired root wins over reading `user:42` as a node. + let hosted = ScopeLayout::tenant().with_retired_root("user:42").unwrap(); + assert_eq!( + hosted.parse("user:42/ws:main/app:conversations"), + Some((chat.clone(), ItemKind::Conversation)) + ); + assert_eq!( + hosted.paths(&chat, ItemKind::Learning), + ["ws:main/app:learnings", "user:42/ws:main/app:learnings"] + ); + assert_eq!(hosted.node_prefixes(&Namespace::ROOT), ["", "user:42"]); +} + +#[test] +fn a_retired_root_is_checked() { + assert!(ScopeLayout::default().with_retired_root("user:42").is_err()); + assert!(v3().with_retired_root("user:42").is_err(), "the root itself"); + assert!(v3().with_retired_root("kb:x").is_err()); + assert!(v3().with_retired_root("").is_err()); + assert_eq!(ScopeLayout::tenant().retired(), None); +} diff --git a/crates/tinymemory-integrations/src/registry/mod_tests.rs b/crates/tinymemory-integrations/src/registry/mod_tests.rs index 50b11faf..df65aa66 100644 --- a/crates/tinymemory-integrations/src/registry/mod_tests.rs +++ b/crates/tinymemory-integrations/src/registry/mod_tests.rs @@ -232,3 +232,28 @@ fn a_scope_root_is_applied_and_checked() { key(), )); } + +#[test] +fn a_tenant_root_is_hosted_only_and_a_retired_root_needs_a_layout() { + let key = || EngineCredential::Static("key".to_string()); + let tenant = EngineSettings { + tenant_root: true, + retired_scope_root: Some("user:6512ab0f".to_string()), + ..EngineSettings::default() + }; + assert!(build_engine("tinyhumans", &tenant, key()).is_ok()); + let message = config_error(build_engine("cortexdb", &tenant, key())); + assert!(message.contains("tenant root"), "{message}"); + let direct = EngineSettings { + scope_root: Some("org:6512ab0f".to_string()), + scope_owner: Some("user:6512ab0f".to_string()), + retired_scope_root: Some("user:6512ab0f".to_string()), + ..EngineSettings::default() + }; + assert!(build_engine("cortexdb", &direct, key()).is_ok()); + let legacy = EngineSettings { + retired_scope_root: Some("user:6512ab0f".to_string()), + ..EngineSettings::default() + }; + config_error(build_engine("cortexdb", &legacy, key())); +} From 519bf4064649bea6e2fc3cdba7a4e526d241ab99 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Thu, 8 Oct 2026 22:48:38 +0530 Subject: [PATCH 10/16] style(cortex): reformat test assertions to satisfy line width Rustfmt-compliant wrapping was applied to the retired-root engine and envelope layout tests, splitting long store chains and assert! calls across lines. No test behaviour or coverage changed. Auto-committed-on: macbook Co-authored-by: Medulla --- .../cortex/engine/mod_retired_root_tests.rs | 23 +++++++++++++++---- .../src/cortex/envelope/layout_tests.rs | 11 +++++++-- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/crates/tinymemory-integrations/src/cortex/engine/mod_retired_root_tests.rs b/crates/tinymemory-integrations/src/cortex/engine/mod_retired_root_tests.rs index b1878e8b..fcd99bcf 100644 --- a/crates/tinymemory-integrations/src/cortex/engine/mod_retired_root_tests.rs +++ b/crates/tinymemory-integrations/src/cortex/engine/mod_retired_root_tests.rs @@ -100,13 +100,17 @@ async fn a_hosted_tenant_engine_reads_the_retired_user_segment_too() { let old = hosted_engine(&endpoint) .with_scope_root("user:6512ab0f", None) .unwrap(); - old.store(learning("written before", "ws:main")).await.unwrap(); + old.store(learning("written before", "ws:main")) + .await + .unwrap(); let new = hosted_engine(&endpoint) .with_tenant_root() .unwrap() .with_retired_root("user:6512ab0f") .unwrap(); - new.store(learning("written after", "ws:main")).await.unwrap(); + new.store(learning("written after", "ws:main")) + .await + .unwrap(); assert!(scopes_written(&state).contains("ws:main/app:learnings")); assert_eq!(texts(&new).await, ["written after", "written before"]); let at_main = MetaFilter { @@ -159,7 +163,10 @@ async fn reads_merge_both_roots_by_item_id_and_writes_go_only_to_the_new_one() { .unwrap(); let written = scopes_written(&state); - assert!(written.contains("org:42/ws:main/app:learnings"), "{written:?}"); + assert!( + written.contains("org:42/ws:main/app:learnings"), + "{written:?}" + ); assert!(written.contains("org:42/app:learnings"), "{written:?}"); assert_eq!( texts(&engine).await, @@ -244,7 +251,13 @@ async fn an_erasure_removes_both_roots() { .iter() .map(|e| e.to_string()) .collect(); - assert!(erased.iter().any(|e| e.contains("org:42/ws:main")), "{erased:?}"); - assert!(erased.iter().any(|e| e.contains("user:42/ws:main")), "{erased:?}"); + assert!( + erased.iter().any(|e| e.contains("org:42/ws:main")), + "{erased:?}" + ); + assert!( + erased.iter().any(|e| e.contains("user:42/ws:main")), + "{erased:?}" + ); assert!(texts(&engine).await.is_empty()); } diff --git a/crates/tinymemory-integrations/src/cortex/envelope/layout_tests.rs b/crates/tinymemory-integrations/src/cortex/envelope/layout_tests.rs index 9cb06611..f3028af2 100644 --- a/crates/tinymemory-integrations/src/cortex/envelope/layout_tests.rs +++ b/crates/tinymemory-integrations/src/cortex/envelope/layout_tests.rs @@ -253,7 +253,11 @@ fn the_tenant_layout_names_no_root() { ItemKind::Conversation, "ws:main/app:conversations", ), - (ns("source:gmail"), ItemKind::Document, "app:brain/source:gmail"), + ( + ns("source:gmail"), + ItemKind::Document, + "app:brain/source:gmail", + ), ( ns("ws:main/service:nl"), ItemKind::Learning, @@ -321,7 +325,10 @@ fn a_retired_root_doubles_every_read_but_not_the_write() { #[test] fn a_retired_root_is_checked() { assert!(ScopeLayout::default().with_retired_root("user:42").is_err()); - assert!(v3().with_retired_root("user:42").is_err(), "the root itself"); + assert!( + v3().with_retired_root("user:42").is_err(), + "the root itself" + ); assert!(v3().with_retired_root("kb:x").is_err()); assert!(v3().with_retired_root("").is_err()); assert_eq!(ScopeLayout::tenant().retired(), None); From 6c00104d3bc97622a1476dbdf225eb4b21b9b20c Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Thu, 8 Oct 2026 22:49:04 +0530 Subject: [PATCH 11/16] docs: document tenant and retired scope roots in cortex layout The layout docs now describe the hosted wire's tenant root, where the backend pins every scope below the caller's `org:`, and the retired `user:` root that stays readable and forgettable while its memory is moved. Examples and integration guidance were updated to use `org:` as the scope root, with `user:` reserved for the actor. Auto-committed-on: macbook Co-authored-by: Medulla --- .../src/cortex/README.md | 13 +++-- docs/architecture/cortex-layout.md | 55 ++++++++++++++----- docs/integration.md | 7 ++- docs/specs/memory-v2.md | 2 +- 4 files changed, 56 insertions(+), 21 deletions(-) diff --git a/crates/tinymemory-integrations/src/cortex/README.md b/crates/tinymemory-integrations/src/cortex/README.md index aa55c140..752266e9 100644 --- a/crates/tinymemory-integrations/src/cortex/README.md +++ b/crates/tinymemory-integrations/src/cortex/README.md @@ -89,10 +89,15 @@ app:tinymemory/team:acme/agent:writer/app:learnings a team m That is the legacy layout, the default. With a scope root (`EngineSettings::scope_root`, `CortexEngine::with_scope_root`), such as one -person's `user:`, every item is laid out below that root instead, each -kind under a leaf of its own (`user:42/ws:main/app:conversations`, -`user:42/app:brain/source:gmail`), and a direct engine registers the root -with its owner before the first write. See +person's `org:`, every item is laid out below that root instead, each +kind under a leaf of its own (`org:42/ws:main/app:conversations`, +`org:42/app:brain/source:gmail`), and a direct engine registers the root +with its owner (the actor `user:`) before the first write. On the +hosted wire the root is the tenant's own (`EngineSettings::tenant_root`): +the engine sends `ws:main/app:conversations` and the backend stores it at +`org:/ws:main/app:conversations`. A retired root +(`EngineSettings::retired_scope_root`, the earlier `user:`) is still +read and forgotten, never written, until its memory has moved. See [cortex-layout.md](../../../../docs/architecture/cortex-layout.md). The hosted backend also re-roots every scope under the caller's tenant. diff --git a/docs/architecture/cortex-layout.md b/docs/architecture/cortex-layout.md index d585c16f..f4353cca 100644 --- a/docs/architecture/cortex-layout.md +++ b/docs/architecture/cortex-layout.md @@ -20,9 +20,10 @@ An engine with no scope root uses it, so existing memory stays where it is. ## V3: below a host's root `EngineSettings::scope_root` (or `CortexEngine::with_scope_root`) names the -root, usually one person's: `user:`. That person's memory is then one -subtree, which can be confined (a token scoped to `user:/*`) and erased -as a whole. Every kind keeps a leaf of its own, so no event sits on an inner +root, one person's: `org:`. That person's memory is then one +subtree, which can be confined (a token scoped to `org:/*`) and erased +as a whole. `user:` names the person's *actor* (the root's owner, a token's +`sub`), never a scope segment. Every kind keeps a leaf of its own, so no event sits on an inner node, and erasing one node never takes a sibling kind with it: | Item | Scope | @@ -33,25 +34,51 @@ node, and erasing one node never takes a sibling kind with it: | document at N, no `source:` | `{root}/{N}/app:documents` | | any N with a `service:` | `app:flows` before the first `service:` | -For example, with root `user:42`: +For example, with root `org:42`: ```text -user:42/app:learnings shared learnings -user:42/app:brain/source:gmail a connector's documents -user:42/app:brain/source:github/project:api one repository -user:42/ws:main/app:conversations chats -user:42/ws:main/app:flows/service:newsletter/app:learnings a workflow's memory -user:42/app:flows/service:digest/app:documents a workflow with no workspace +org:42/app:learnings shared learnings +org:42/app:brain/source:gmail a connector's documents +org:42/app:brain/source:github/project:api one repository +org:42/ws:main/app:conversations chats +org:42/ws:main/app:flows/service:newsletter/app:learnings a workflow's memory +org:42/app:flows/service:digest/app:documents a workflow with no workspace ``` +### The hosted wire: the tenant's root + +The TinyHumans backend pins every scope below the caller's tenant root +(`org:`), so a hosted engine names no root of its own +(`EngineSettings::tenant_root`, `CortexEngine::with_tenant_root`): it sends +`ws:main/app:conversations` and the stored path is +`org:/ws:main/app:conversations`, the same tree as a direct engine rooted +at `org:`. An unprefixed scope listing covers the whole tenant (the +backend bounds it), and an empty `prefix=` is never sent. A tenant root is +refused on the direct wire, where nothing would pin it. + +### The retired `user:` root + +Earlier v3 hosts rooted a person at `user:`, which the hosted backend +stored as `org:/user:/…`. While that memory is being moved +(cortexdb-saas `reroot-user-segment`), a host names it as a retired root +(`EngineSettings::retired_scope_root`, `CortexEngine::with_retired_root`): + +- every read covers each node below both roots, merged by item id, so an + item held in both is one item; +- every forget (by id or by filter) and every direct erasure removes from + both; +- writes go only below the root. + +Once the move is verified, the host stops naming the retired root. + `app` never names a namespace node, so a path reads back to exactly one node and kind (`ScopeLayout::parse`). Only the canonical spelling reads back: a grouping node out of place, a missing leaf, or another root is somebody else's scope and is skipped. On the direct wire a path must start at the root (CortexDB's own API never prefixes one). On the hosted wire, whose backend prefixes the tenant, the root is matched at its last occurrence. A -node that begins with the root's own segments (`user:42/…` below root -`user:42`) is refused before it is written (`InvalidRequest`), so no written +node that begins with the root's own segments, or a retired root's +(`user:42/…` while `user:42` is retired), is refused before it is written (`InvalidRequest`), so no written path reads back two ways. A root is `type:id` segments of the types CortexDB's hosted API admits @@ -62,8 +89,8 @@ A root is `type:id` segments of the types CortexDB's hosted API admits ### Root ownership CortexDB makes the first writer of an unregistered scope its owner. With -`EngineSettings::scope_owner` (`user:`), a direct engine registers the -root once, before its first write (`POST v1/scopes` with that owner), so the +`EngineSettings::scope_owner` (the actor `user:`), a direct engine +registers the root (`org:`) once, before its first write (`POST v1/scopes` with that owner), so the person owns their root rather than whichever key writes first: - `409 SCOPE_REGISTRATION_EXISTS` is not taken as ownership: the root's diff --git a/docs/integration.md b/docs/integration.md index efbf7518..ec3a1e3b 100644 --- a/docs/integration.md +++ b/docs/integration.md @@ -85,9 +85,12 @@ actor header and the headers the HTTP stack sets, so the credential stays the `EngineCredential`'s alone. To keep one person's memory as one subtree, give a CortexDB engine a scope -root (`"engines": { "cortexdb": { "scope_root": "user:42", "scope_owner": +root (`"engines": { "cortexdb": { "scope_root": "org:42", "scope_owner": "user:42" } }`, or `CortexEngine::with_scope_root`): layout v3, see -[cortex-layout.md](architecture/cortex-layout.md). Without one, the legacy +[cortex-layout.md](architecture/cortex-layout.md). A `tinyhumans` engine +takes `"tenant_root": true` instead, since the backend pins the tenant's +`org:`. While memory written below an earlier `user:` root is moved, +add `"retired_scope_root": "user:42"` so it stays readable and forgettable. Without one, the legacy `app:tinymemory` tree is kept. Switching moves nothing. To record who actually said or did something, set `"observed_actor": true` diff --git a/docs/specs/memory-v2.md b/docs/specs/memory-v2.md index 43f7255e..c63ea387 100644 --- a/docs/specs/memory-v2.md +++ b/docs/specs/memory-v2.md @@ -288,7 +288,7 @@ credentialed cleartext non-loopback endpoint, all as `Error::Config`. - An event's `content.text` is the item's own text (body or piece, turn text, statement), and its envelope `{v:3, id, kind, meta, title?, learning_kind?, confidence?, turn?, chunk?}` rides in `tm:e:` labels beside readable `kind:`/`file:`/`page:`/`section:` labels (v3); an event with empty text or too many labels for 64 carries the whole envelope as JSON text (v2), as every event before v3 did, and both read. A reader decides by the labels: an event whose `tm:e:` parts, numbered from 0 with none missing, join to a `v:3` envelope is v3 (its `content.text` is the item's text, empty or not); any other event's `content.text` is tried as a `v:2` envelope; an event that is neither is not this crate's. `meta` maps to lookup labels where CortexDB can filter. No local path leaves the machine: `meta.file_path` is sent as the file's name, `folder` and an absolute `workspace` are not sent, and filters on them match digests the envelope carries instead ([cortex-local-paths](../architecture/cortex-local-paths.md)). - `MemoryMeta.derive: Some(false)`, and every tool turn, is written with `directives.extract: []`: indexed and searchable, nothing derived. - Writes wait for the indexed barrier, keeping the v1 `await_readable` behaviour. -- **Scope.** One scope per item kind *per namespace node*, under the TinyMemory root `app:tinymemory` (which the hosted backend further roots under the tenant): the root node keeps `app:tinymemory/app:{documents,conversations,learnings}`, and a node adds its segments in between, e.g. `app:tinymemory/team:acme/agent:writer/app:learnings`. That is the legacy layout and the default; an engine given a scope root (`EngineSettings::scope_root`, such as one person's `user:`) lays everything out below that root instead, with a leaf per kind and `app:brain`/`app:flows` grouping nodes (layout v3, [cortex-layout.md](../architecture/cortex-layout.md)), and switching moves nothing. Namespace segments map to CortexDB's built-in `agent`, `team`, `user`, `ws`, `project`, `source` and `service` types and the kind leaf uses `app`, because CortexDB v0.10+ refuses scope types outside the deployment's `allowed_scope_types` (`422 UNREGISTERED_SCOPE_TYPE`); every shipped preset allows all of them. A `MetaFilter`'s `kinds` and `reach` pick the scopes read, each admitted kind at each node: an ordinary reach (`Reach::of`, `inherit: true`) reads its node **and every ancestor**, an exact reach its node alone, and a subtree reach its node and every node below except a `service:` sandbox below it (a missing reach reads as the root's subtree, so it skips sandboxes too; a forget by id still reaches them); those nodes are known, and only a subtree reach or an unscoped read discovers nodes, from the registered scopes (`v1/scopes/list` / `memory/scopes`). Reads are always exact (`view: "granular"`, sent explicitly because public recall defaults to `holistic`), never server-side traversal. +- **Scope.** One scope per item kind *per namespace node*, under the TinyMemory root `app:tinymemory` (which the hosted backend further roots under the tenant): the root node keeps `app:tinymemory/app:{documents,conversations,learnings}`, and a node adds its segments in between, e.g. `app:tinymemory/team:acme/agent:writer/app:learnings`. That is the legacy layout and the default; an engine given a scope root (`EngineSettings::scope_root`, such as one person's `org:`; on the hosted wire `EngineSettings::tenant_root`, the tenant's own `org:`) lays everything out below that root instead, with a leaf per kind and `app:brain`/`app:flows` grouping nodes (layout v3, [cortex-layout.md](../architecture/cortex-layout.md)), and switching moves nothing. Namespace segments map to CortexDB's built-in `agent`, `team`, `user`, `ws`, `project`, `source` and `service` types and the kind leaf uses `app`, because CortexDB v0.10+ refuses scope types outside the deployment's `allowed_scope_types` (`422 UNREGISTERED_SCOPE_TYPE`); every shipped preset allows all of them. A `MetaFilter`'s `kinds` and `reach` pick the scopes read, each admitted kind at each node: an ordinary reach (`Reach::of`, `inherit: true`) reads its node **and every ancestor**, an exact reach its node alone, and a subtree reach its node and every node below except a `service:` sandbox below it (a missing reach reads as the root's subtree, so it skips sandboxes too; a forget by id still reaches them); those nodes are known, and only a subtree reach or an unscoped read discovers nodes, from the registered scopes (`v1/scopes/list` / `memory/scopes`). Reads are always exact (`view: "granular"`, sent explicitly because public recall defaults to `holistic`), never server-side traversal. - **Fetch.** - `Hybrid` maps to `recall` layers. `Keyword` and `Vector` are declared only if the wire exposes a mode switch; otherwise `fetch_modes = [Hybrid]`. The recall body accepts only `scope`, `query`, `budgets`, `view`, `include`, `temporal` and `filters`, with no mode switch, so both wires declare `[Hybrid]`. - Metadata filters CortexDB cannot apply server-side are applied client-side on the page, and the cursor is still the engine's. From e7c229bec8772e20cb734ffc60ac40bbbcd96264 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Thu, 8 Oct 2026 22:49:19 +0530 Subject: [PATCH 12/16] docs: document where a person's memory lives Adds a README section describing the per-person scope layout, showing how org, app, workspace, and service segments compose under a single root. It also notes how each engine is configured and that memory written under the older user-scoped layout stays readable until migrated. Auto-committed-on: macbook Co-authored-by: Medulla --- README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/README.md b/README.md index 3b6ab845..e866aced 100644 --- a/README.md +++ b/README.md @@ -177,6 +177,24 @@ route has no keyword/vector switch, so both wires declare hybrid fetch only. See [`docs/architecture/cortex.md`](docs/architecture/cortex.md) and [`crates/tinymemory-integrations/src/cortex/README.md`](crates/tinymemory-integrations/src/cortex/README.md). +### Where a person's memory lives + +One root per person, `org:`, and no `user:` segment below it (`user:` +is the person's actor, never a scope): + +```text +org:/app:learnings learnings +org:/app:brain/source:gmail a connector's documents +org:/ws:main/app:conversations chats +org:/ws:main/app:flows/service:/… a workflow's memory +``` + +A direct `cortexdb` engine gets `scope_root = "org:"`; a `tinyhumans` +engine gets `tenant_root = true` and sends paths relative to the tenant root +the backend pins. Memory an earlier layout wrote below `user:` stays +readable and forgettable through `retired_scope_root` until it has moved. See +[`docs/architecture/cortex-layout.md`](docs/architecture/cortex-layout.md). + ### Adding an engine An engine is a module of `tinymemory-integrations` behind a feature named after From ec7c5b25b7d068f9104f8e65afa1a54ab707865c Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Thu, 8 Oct 2026 23:56:49 +0530 Subject: [PATCH 13/16] fix(cortex): preserve item payloads when fetching from the engine Fetching items from the engine now keeps the stored payload intact instead of dropping it during envelope decoding, so callers receive the full item data rather than a stripped-down record. Auto-committed-on: macbook Co-authored-by: Medulla --- .../src/cortex/engine/fetch.rs | 9 ++++++-- .../src/cortex/engine/items.rs | 15 +++++++++---- .../src/cortex/envelope/layout.rs | 21 +++++++++++++++++-- 3 files changed, 37 insertions(+), 8 deletions(-) diff --git a/crates/tinymemory-integrations/src/cortex/engine/fetch.rs b/crates/tinymemory-integrations/src/cortex/engine/fetch.rs index 48ea6f15..c048f88e 100644 --- a/crates/tinymemory-integrations/src/cortex/engine/fetch.rs +++ b/crates/tinymemory-integrations/src/cortex/engine/fetch.rs @@ -266,14 +266,19 @@ impl CortexEngine { } /// Merges per-scope rankings rank by rank: every scope's best, then every -/// scope's second, and so on. +/// scope's second, and so on, each item once. pub(super) fn interleave(mut lists: Vec>) -> Vec { let longest = lists.iter().map(Vec::len).max().unwrap_or(0); let mut iters: Vec<_> = lists.iter_mut().map(|list| list.drain(..)).collect(); let mut out = Vec::new(); + // An item held below both the root and a retired root is one hit, at its + // best rank. + let mut seen = HashSet::new(); for _ in 0..longest { for iter in &mut iters { - if let Some(envelope) = iter.next() { + if let Some(envelope) = iter.next() + && seen.insert(envelope.id.clone()) + { out.push(envelope); } } diff --git a/crates/tinymemory-integrations/src/cortex/engine/items.rs b/crates/tinymemory-integrations/src/cortex/engine/items.rs index c400b928..67d70105 100644 --- a/crates/tinymemory-integrations/src/cortex/engine/items.rs +++ b/crates/tinymemory-integrations/src/cortex/engine/items.rs @@ -207,15 +207,22 @@ impl CortexEngine { .map(move |scope| (scope, ids.clone())) }) .collect(); - let found: Vec>> = stream::iter(lookups) - .map(|(scope, ids)| async move { self.item_events(&scope, &ids).await }) + let mut found: Vec<(usize, HashMap>)> = stream::iter(lookups) + .enumerate() + .map(|(order, (scope, ids))| async move { + Ok::<_, Error>((order, self.item_events(&scope, &ids).await?)) + }) .buffer_unordered(LOOKUPS_AT_ONCE) .try_collect() .await?; // An item held below both the root and a retired root is rebuilt from - // the events of one of them, never from both at once. + // the events of one of them, never from both at once: the root's + // (lookups are in read order, root first), whichever lookup answered + // first; the retired root's only when the root's copy cannot be + // rebuilt. + found.sort_by_key(|(order, _)| *order); let mut out = HashMap::new(); - for (id, events) in found.into_iter().flatten() { + for (id, events) in found.into_iter().flat_map(|(_, found)| found) { if out.contains_key(&id) { continue; } diff --git a/crates/tinymemory-integrations/src/cortex/envelope/layout.rs b/crates/tinymemory-integrations/src/cortex/envelope/layout.rs index 5e7f34a0..e4bdc0e7 100644 --- a/crates/tinymemory-integrations/src/cortex/envelope/layout.rs +++ b/crates/tinymemory-integrations/src/cortex/envelope/layout.rs @@ -190,6 +190,20 @@ impl ScopeLayout { } } + /// Whether `path` is below the retired root (a twin of a scope below the + /// root), as opposed to the root itself. + pub(crate) fn is_retired(&self, path: &str) -> bool { + let Self::V3 { + retired: Some(retired), + prefixed, + .. + } = self + else { + return false; + }; + parse_below(retired, *prefixed, path).is_some() + } + /// The scope items of `kind` at `namespace` live in: where they are /// written. pub(crate) fn path(&self, namespace: &Namespace, kind: ItemKind) -> String { @@ -355,7 +369,10 @@ fn node_prefixes_below(root: &str, namespace: &Namespace) -> Vec { fn parse_below(root: &str, prefixed: bool, path: &str) -> Option<(Namespace, ItemKind)> { let parts: Vec<&str> = path.split('/').collect(); let start = if root.is_empty() { - 0 + // Hosted, the backend may answer a tenant-root path behind the + // tenant's own `org:`. `org` is never a namespace segment, so a + // leading `org:` segment is that prefix. + usize::from(prefixed && parts.first().is_some_and(|part| part.starts_with("org:"))) } else { let wanted: Vec<&str> = root.split('/').collect(); let start = if prefixed { @@ -387,7 +404,7 @@ fn parse_below(root: &str, prefixed: bool, path: &str) -> Option<(Namespace, Ite // place is somebody else's scope. let canonical = render(root, &namespace, kind); let own = if root.is_empty() { - path.to_string() + parts[start..].join("/") } else { parts[start - root.split('/').count()..].join("/") }; From 45a48b875432a33ad106da1780c185e0e395d00a Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Thu, 8 Oct 2026 23:57:01 +0530 Subject: [PATCH 14/16] refactor(cortex): extract shared item helpers into items module Move the item lookup and formatting helpers out of the list engine into a dedicated items module so both can be reused by other engine operations. No behaviour change. Auto-committed-on: macbook Co-authored-by: Medulla --- .../src/cortex/engine/items.rs | 2 +- .../src/cortex/engine/list.rs | 25 +++++++++++++++++-- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/crates/tinymemory-integrations/src/cortex/engine/items.rs b/crates/tinymemory-integrations/src/cortex/engine/items.rs index 67d70105..7e6b277b 100644 --- a/crates/tinymemory-integrations/src/cortex/engine/items.rs +++ b/crates/tinymemory-integrations/src/cortex/engine/items.rs @@ -13,7 +13,7 @@ use tinymemory_api::{ use super::CortexEngine; use super::scopes::KindScope; use crate::cortex::envelope::{Decoded, Envelope, decode_event, labels, rebuild, rebuild_whole}; -use crate::cortex::error::Result; +use crate::cortex::error::{Error, Result}; /// The kinds `filter` admits, in the fixed order /// [`ItemKind::ALL`] lists them. diff --git a/crates/tinymemory-integrations/src/cortex/engine/list.rs b/crates/tinymemory-integrations/src/cortex/engine/list.rs index 6952dc41..647a36b3 100644 --- a/crates/tinymemory-integrations/src/cortex/engine/list.rs +++ b/crates/tinymemory-integrations/src/cortex/engine/list.rs @@ -173,6 +173,7 @@ impl CortexEngine { if len > 0 { filled_pages += 1; } + let shadowed = self.shadowed(scope, &page.items).await?; for (position, event) in page.items.iter().enumerate().skip(at.offset) { at.offset = position + 1; let id = event.get("id").and_then(Value::as_str); @@ -181,7 +182,7 @@ impl CortexEngine { } at.last = id.map(str::to_owned); if let Some(found) = - self.admit(kind, &req, event, &mut seen, walk == Walk::Preview) + self.admit(kind, &req, event, &mut seen, &shadowed, walk == Walk::Preview) { pending.push(found); if pending.len() == req.limit { @@ -213,6 +214,25 @@ impl CortexEngine { Ok((self.resolve(pending).await?, next)) } + /// The ids of the items on `events` (a page of `scope`) that the root + /// also holds, when `scope` is below the retired root: the root's copy is + /// the one listed, so an item held below both is one item on every page, + /// not one per root. Empty for any other scope. + async fn shadowed(&self, scope: &KindScope, events: &[Value]) -> Result> { + if !self.layout.is_retired(&scope.path) { + return Ok(HashSet::new()); + } + let ids: Vec = events + .iter() + .filter_map(|event| decode_event(event)) + .map(|decoded| decoded.envelope.id) + .collect::>() + .into_iter() + .collect(); + let active = KindScope::new(&self.layout, scope.namespace.clone(), scope.kind); + Ok(self.item_events(&active, &ids).await?.into_keys().collect()) + } + /// Whether one raw event starts an item this listing returns; with /// `preview`, the item is taken from that event alone. fn admit( @@ -221,10 +241,11 @@ impl CortexEngine { req: &ListRequest, event: &Value, seen: &mut HashSet, + shadowed: &HashSet, preview: bool, ) -> Option { let envelope = decode_event(event)?.envelope; - if !keeps(&req.filter, kind, &envelope) { + if !keeps(&req.filter, kind, &envelope) || shadowed.contains(&envelope.id) { return None; } let starts = envelope.part().is_none_or(|index| index == 0); From 1c9e9f0385b5c800701c530ba75c3aeedd47d3a0 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Thu, 8 Oct 2026 23:57:50 +0530 Subject: [PATCH 15/16] test(cortex): cover cross-root dedup and hosted tenant prefix parsing Add tests asserting that an item ranked under both the active and retired root is listed once at its best rank across pages, and that writes in the transitional layout only land below the new root. Also cover parsing of hosted tenant paths with a prefix, including retired-root paths behind it. Reformat two call sites in the list engine with no behaviour change. Auto-committed-on: macbook Co-authored-by: Medulla --- .../src/cortex/engine/fetch_tests.rs | 13 +++++++ .../src/cortex/engine/list.rs | 13 +++++-- .../cortex/engine/mod_retired_root_tests.rs | 38 +++++++++++++++++++ .../src/cortex/envelope/layout_tests.rs | 37 ++++++++++++++++++ 4 files changed, 97 insertions(+), 4 deletions(-) diff --git a/crates/tinymemory-integrations/src/cortex/engine/fetch_tests.rs b/crates/tinymemory-integrations/src/cortex/engine/fetch_tests.rs index 14ccdcd1..e331c425 100644 --- a/crates/tinymemory-integrations/src/cortex/engine/fetch_tests.rs +++ b/crates/tinymemory-integrations/src/cortex/engine/fetch_tests.rs @@ -103,3 +103,16 @@ fn a_pack_budget_fits_each_event_whole_up_to_a_ceiling() { assert_eq!(whole_items_budget(MAX_PACK_EVENTS), MAX_PACK_TOKENS); assert_eq!(whole_items_budget(usize::MAX), MAX_PACK_TOKENS); } + +#[test] +fn an_item_ranked_in_two_scopes_is_one_hit_at_its_best_rank() { + // The same item below the root and below a retired root. + let envelope = + |text: &str, id: &str| Envelope::for_item(&doc(text, None), id).unwrap().remove(0); + let merged = interleave(vec![ + vec![envelope("a", "same"), envelope("b", "b")], + vec![envelope("c", "c"), envelope("a again", "same")], + ]); + let ids: Vec<_> = merged.iter().map(|e| e.id.as_str()).collect(); + assert_eq!(ids, vec!["same", "c", "b"]); +} diff --git a/crates/tinymemory-integrations/src/cortex/engine/list.rs b/crates/tinymemory-integrations/src/cortex/engine/list.rs index 647a36b3..243b8df0 100644 --- a/crates/tinymemory-integrations/src/cortex/engine/list.rs +++ b/crates/tinymemory-integrations/src/cortex/engine/list.rs @@ -181,9 +181,14 @@ impl CortexEngine { continue; } at.last = id.map(str::to_owned); - if let Some(found) = - self.admit(kind, &req, event, &mut seen, &shadowed, walk == Walk::Preview) - { + if let Some(found) = self.admit( + kind, + &req, + event, + &mut seen, + &shadowed, + walk == Walk::Preview, + ) { pending.push(found); if pending.len() == req.limit { let exhausted = at.offset == len @@ -224,7 +229,7 @@ impl CortexEngine { } let ids: Vec = events .iter() - .filter_map(|event| decode_event(event)) + .filter_map(decode_event) .map(|decoded| decoded.envelope.id) .collect::>() .into_iter() diff --git a/crates/tinymemory-integrations/src/cortex/engine/mod_retired_root_tests.rs b/crates/tinymemory-integrations/src/cortex/engine/mod_retired_root_tests.rs index fcd99bcf..3975ea33 100644 --- a/crates/tinymemory-integrations/src/cortex/engine/mod_retired_root_tests.rs +++ b/crates/tinymemory-integrations/src/cortex/engine/mod_retired_root_tests.rs @@ -157,6 +157,7 @@ async fn reads_merge_both_roots_by_item_id_and_writes_go_only_to_the_new_one() { .await .unwrap(); let engine = transitional(&endpoint); + let before = scopes_written(&state); engine .store_many(vec![shared.clone(), learning("only new", "")]) .await @@ -168,6 +169,13 @@ async fn reads_merge_both_roots_by_item_id_and_writes_go_only_to_the_new_one() { "{written:?}" ); assert!(written.contains("org:42/app:learnings"), "{written:?}"); + // The transitional write adds scopes below the new root only. + let added: Vec<_> = written.difference(&before).collect(); + assert!(!added.is_empty(), "{written:?}"); + assert!( + added.iter().all(|scope| scope.starts_with("org:42")), + "a write went below the retired root: {added:?}" + ); assert_eq!( texts(&engine).await, ["held in both", "only new", "only old"], @@ -261,3 +269,33 @@ async fn an_erasure_removes_both_roots() { ); assert!(texts(&engine).await.is_empty()); } + +#[tokio::test] +async fn an_item_held_in_both_roots_is_listed_once_across_pages() { + let (endpoint, _state) = direct_double().await; + let shared = learning("held in both", "ws:main"); + retired(&endpoint) + .store_many(vec![shared.clone(), learning("only old", "ws:main")]) + .await + .unwrap(); + let engine = transitional(&endpoint); + engine + .store_many(vec![shared, learning("only new", "ws:main")]) + .await + .unwrap(); + + // One item per page: the cursor, not a per-page set, must keep the twin + // from coming back on a later page. + let mut seen = Vec::new(); + let mut request = ListRequest::new(MetaFilter::default(), 1); + for _ in 0..10 { + let page = engine.list(request.clone()).await.unwrap(); + seen.extend(page.items.into_iter().map(|hit| hit.text)); + match page.next_cursor { + Some(cursor) => request.cursor = Some(cursor), + None => break, + } + } + seen.sort(); + assert_eq!(seen, ["held in both", "only new", "only old"]); +} diff --git a/crates/tinymemory-integrations/src/cortex/envelope/layout_tests.rs b/crates/tinymemory-integrations/src/cortex/envelope/layout_tests.rs index f3028af2..8398f4a5 100644 --- a/crates/tinymemory-integrations/src/cortex/envelope/layout_tests.rs +++ b/crates/tinymemory-integrations/src/cortex/envelope/layout_tests.rs @@ -333,3 +333,40 @@ fn a_retired_root_is_checked() { assert!(v3().with_retired_root("").is_err()); assert_eq!(ScopeLayout::tenant().retired(), None); } + +#[test] +fn a_hosted_tenant_prefix_is_stripped_before_a_tenant_root_path_is_parsed() { + let layout = ScopeLayout::tenant(); + for (path, namespace, kind) in [ + ( + "org:u-tenant/ws:main/agent:a/app:learnings", + ns("ws:main/agent:a"), + ItemKind::Learning, + ), + ( + "org:u-tenant/app:conversations", + Namespace::ROOT, + ItemKind::Conversation, + ), + ( + "org:u-tenant/app:brain/source:gmail", + ns("source:gmail"), + ItemKind::Document, + ), + ] { + assert_eq!(layout.parse(path), Some((namespace, kind)), "{path}"); + } + // A grouping node out of place behind the prefix is still foreign. + assert_eq!( + layout.parse("org:u-tenant/app:flows/ws:main/app:learnings"), + None + ); + // With a retired root, its paths behind the prefix read back too. + let transitional = ScopeLayout::tenant().with_retired_root("user:42").unwrap(); + assert!(transitional.is_retired("org:u-tenant/user:42/ws:main/app:learnings")); + assert!(!transitional.is_retired("org:u-tenant/ws:main/app:learnings")); + assert_eq!( + transitional.parse("org:u-tenant/user:42/ws:main/app:learnings"), + Some((ns("ws:main"), ItemKind::Learning)) + ); +} From 76ed6312ce1d82a1354b902903c4b6a59baf9a74 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Thu, 8 Oct 2026 23:58:27 +0530 Subject: [PATCH 16/16] docs: clarify retired scope root during layout migration The scope root section now explains that memory written below an earlier `user:` root stays readable and forgettable only while the engine names that root as well, so both roots must be configured during the move. Writes go to the new root only, and `retired_scope_root` can be dropped once the memory has moved. Auto-committed-on: macbook Co-authored-by: Medulla --- docs/integration.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/docs/integration.md b/docs/integration.md index ec3a1e3b..0216a0e1 100644 --- a/docs/integration.md +++ b/docs/integration.md @@ -89,9 +89,15 @@ root (`"engines": { "cortexdb": { "scope_root": "org:42", "scope_owner": "user:42" } }`, or `CortexEngine::with_scope_root`): layout v3, see [cortex-layout.md](architecture/cortex-layout.md). A `tinyhumans` engine takes `"tenant_root": true` instead, since the backend pins the tenant's -`org:`. While memory written below an earlier `user:` root is moved, -add `"retired_scope_root": "user:42"` so it stays readable and forgettable. Without one, the legacy -`app:tinymemory` tree is kept. Switching moves nothing. +`org:`. Without a root, the legacy `app:tinymemory` tree is kept. +Switching moves nothing. + +Memory an earlier layout wrote below a `user:` root stays readable and +forgettable only while the engine names that root as well, so during the move +configure both: `"engines": { "cortexdb": { "scope_root": "org:42", +"scope_owner": "user:42", "retired_scope_root": "user:42" } }` (or +`CortexEngine::with_retired_root`). Writes go to the new root only; once the +memory has moved, drop `retired_scope_root`. To record who actually said or did something, set `"observed_actor": true` on a `cortexdb` engine and fill `MemoryMeta::observed_actor` on items another