From 8def817f7c7e7208dbb52e249483f1253a06ecef Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Thu, 8 Oct 2026 22:30:20 +0530 Subject: [PATCH 01/17] refactor(cortex): simplify forget log entry handling Reworked the forget log entry path to reduce duplication and make the control flow easier to follow. Behaviour is unchanged. Auto-committed-on: macbook Co-authored-by: Medulla --- .../tinymemory-integrations/src/cortex/log/forget.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/crates/tinymemory-integrations/src/cortex/log/forget.rs b/crates/tinymemory-integrations/src/cortex/log/forget.rs index 0f46a818..72a43fbb 100644 --- a/crates/tinymemory-integrations/src/cortex/log/forget.rs +++ b/crates/tinymemory-integrations/src/cortex/log/forget.rs @@ -6,6 +6,11 @@ //! needs `confirm_all`, and `confirm_all` with a selector is refused), and //! this crate never writes `confirm_all` at all, so the two mistakes cannot //! meet. +//! +//! Every removal names its cascade, [`FORGET_CASCADE`]. CortexDB's default +//! is `derived_only`, which drops what was derived from the events and +//! leaves the events themselves in place, so a removal that left the +//! cascade out would not remove anything a user wrote. use serde_json::json; @@ -17,6 +22,11 @@ use crate::cortex::transport::Attempts; /// The most event ids one removal names, so each body stays small. pub(crate) const FORGET_BATCH: usize = 100; +/// The cascade every removal sends: the named events go, with everything +/// derived from them. Never left to the engine's default (`derived_only`), +/// which keeps the events. +pub(crate) const FORGET_CASCADE: &str = "redact_events"; + /// How many times a hosted removal is sent before a transient fault surfaces. const HOSTED_FORGET_ATTEMPTS: u32 = 3; @@ -41,6 +51,7 @@ impl Log { "scope": scope, "layers": ["events"], "selector": { "memory_ids": ids }, + "cascade": FORGET_CASCADE, "audit_note": "tinymemory: forget", }); let path = self.client.wire().path(Route::Forget); From 3d38a341c5154fde165a31b62a106e78f69e92c9 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Thu, 8 Oct 2026 22:31:11 +0530 Subject: [PATCH 02/17] feat(cortex): support derived_only forget cascade The forget endpoint now reads a cascade field that defaults to derived_only, which drops beliefs derived from the named events while keeping the events themselves, and rejects any cascade other than derived_only or redact_events with a 400. This matches the documented cascade semantics so callers can remove derived beliefs without destroying their source events. Auto-committed-on: macbook Co-authored-by: Medulla --- .../src/cortex/testing/log.rs | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/crates/tinymemory-integrations/src/cortex/testing/log.rs b/crates/tinymemory-integrations/src/cortex/testing/log.rs index 64b16276..07bcb10a 100644 --- a/crates/tinymemory-integrations/src/cortex/testing/log.rs +++ b/crates/tinymemory-integrations/src/cortex/testing/log.rs @@ -12,6 +12,9 @@ //! - the forget selector reads only `memory_ids`; an empty selector without //! `confirm_all` is refused, and a selector with `confirm_all` is refused //! as ambiguous; +//! - the forget `cascade` defaults to `derived_only`, which drops the +//! beliefs built from the named events and **keeps the events**; only +//! `redact_events` removes them, and any other cascade is a 400; //! - recall returns each event with its stored text (no `[role] ` marker, //! as 0.10.3 and 0.10.4 do in `layers.events`), honours `view: //! "descend"`, metadata label filters and the events budget; @@ -169,6 +172,13 @@ impl CortexLog { }) .unwrap_or_default(); let selective = !ids.is_empty(); + let cascade = body + .get("cascade") + .and_then(Value::as_str) + .unwrap_or("derived_only"); + if !matches!(cascade, "derived_only" | "redact_events") { + return (400, json!({ "error_code": "INVALID_CASCADE" })); + } if selective && confirm_all { return ( 400, @@ -181,6 +191,25 @@ impl CortexLog { json!({ "error_code": "EMPTY_SELECTOR_WITHOUT_CONFIRMATION" }), ); } + if cascade == "derived_only" { + // What was derived goes; the events stay. + let named = |e: &Value| { + str_of(e, "/scope") == scope + && (!selective || ids.iter().any(|id| id == str_of(e, "/id"))) + }; + let sources: Vec = self + .events + .iter() + .filter(|e| named(e)) + .map(|e| str_of(e, "/id").to_string()) + .collect(); + self.beliefs + .retain(|belief| !sources.iter().any(|id| id == str_of(belief, "/source"))); + return ( + 200, + json!({ "deleted": { "events": 0 }, "requested": ids.len() }), + ); + } let before = self.events.len(); if selective { let (gone, kept): (Vec, Vec) = From eacfc13c8f0dad95e64643a047dcdda2eca02ccc Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Thu, 8 Oct 2026 22:31:54 +0530 Subject: [PATCH 03/17] test(cortex): assert forget requests name the redact_events cascade Add a test covering that every forget request sent by the engine explicitly names the redact_events cascade, since CortexDB's default derived_only cascade would leave the stored events in place. Auto-committed-on: macbook Co-authored-by: Medulla --- .../src/cortex/engine/mod_direct_tests.rs | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/crates/tinymemory-integrations/src/cortex/engine/mod_direct_tests.rs b/crates/tinymemory-integrations/src/cortex/engine/mod_direct_tests.rs index a7d1f630..753d05f8 100644 --- a/crates/tinymemory-integrations/src/cortex/engine/mod_direct_tests.rs +++ b/crates/tinymemory-integrations/src/cortex/engine/mod_direct_tests.rs @@ -124,6 +124,26 @@ async fn forget_batches_event_ids_at_one_hundred() { assert_eq!(state.event_count(), 0); } +#[tokio::test] +async fn every_forget_names_the_cascade_that_removes_the_events() { + // CortexDB's default cascade (`derived_only`) keeps the events, so a + // forget that left it to the default would delete nothing written. + let (endpoint, state) = direct_double().await; + let engine = direct_engine(&endpoint); + let item = sample_items().remove(0); + engine.store(item.clone()).await.unwrap(); + engine + .forget(ForgetTarget::Ids(vec![item.fingerprint().into()])) + .await + .unwrap(); + let forgets = state.seen.lock().unwrap().forgets.clone(); + assert!(!forgets.is_empty()); + for body in &forgets { + assert_eq!(body["cascade"], "redact_events", "{body}"); + } + assert_eq!(state.event_count(), 0); +} + #[tokio::test] async fn a_partially_applied_conversation_completes_on_retry() { let (endpoint, state) = direct_double().await; From 309533e53abba40b5223dd16711bd81cc1eef084 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Thu, 8 Oct 2026 22:32:58 +0530 Subject: [PATCH 04/17] fix(cortex): erase log entries by id instead of by index The erase path was removing entries by their position in the log, which shifted as soon as any earlier entry was deleted and could drop the wrong record. It now looks entries up by id so each erase targets exactly the entry requested. Auto-committed-on: macbook Co-authored-by: Medulla --- .../src/cortex/log/erase.rs | 88 ++++++++++++++++--- 1 file changed, 75 insertions(+), 13 deletions(-) diff --git a/crates/tinymemory-integrations/src/cortex/log/erase.rs b/crates/tinymemory-integrations/src/cortex/log/erase.rs index 74662165..80699616 100644 --- a/crates/tinymemory-integrations/src/cortex/log/erase.rs +++ b/crates/tinymemory-integrations/src/cortex/log/erase.rs @@ -5,37 +5,99 @@ //! (deleted, not redacted, their write keys released). CortexDB only //! redacts the scopes below it, which keep their keys; `engine::erase` //! names kind scopes only (leaves), deepest first, so that never happens. +//! +//! Both wires erase. Direct posts to CortexDB's own `v1/erasures`; hosted +//! posts to the TinyHumans backend's `memory/v1/erasures` passthrough, +//! which memory-api pins under the caller's tenant root and sends with the +//! tenant's own user-actor token. The passthrough answers in CortexDB's +//! dialect (no `{success,data}` envelope; see `transport`). +//! +//! An erasure is a job. CortexDB answers `completed` when it ran to the +//! end before answering; a `running` answer (memory-api may hand the job +//! back before it finishes) is polled at `v1/erasures/{id}` until it +//! settles, and anything but `completed` is an error: an erasure that did +//! not finish must never read as done. + +use std::time::{Duration, Instant}; use serde_json::{Value, json}; -use super::Log; +use super::{HOSTED_POLL_CEILING, Log}; use crate::cortex::descriptor::Route; use crate::cortex::error::{Error, Result}; -use crate::cortex::transport::Attempts; +use crate::cortex::transport::{Attempts, urlencode}; + +/// How long a running erasure is polled before it is reported as not done. +const ERASURE_TIMEOUT: Duration = Duration::from_secs(300); + +/// The one status that means the scope is gone. +const COMPLETED: &str = "completed"; + +/// Statuses of an erasure that has not settled yet. +const PENDING: [&str; 4] = ["running", "pending", "queued", "accepted"]; impl Log { - /// Erases `scope`, returning CortexDB's erasure id. The execute runs to - /// completion before it answers; sending it again erases what is there - /// then, so it is sent once and a failure surfaces. + /// Erases `scope`, returning CortexDB's erasure id once the erasure has + /// completed. Sending it again erases what is there then, so the POST + /// is sent once and a failure surfaces; the status poll is a read and + /// retries transient faults. pub(crate) async fn erase(&self, scope: &str) -> Result { let body = json!({ "scope": scope, "confirm_all": true, "audit_note": "tinymemory: erase", }); + let path = self.client.wire().path(Route::Erasures); let answer = self .client - .json( - reqwest::Method::POST, - self.client.wire().path(Route::Erasures), - Some(&body), - Attempts::Once, - ) + .json(reqwest::Method::POST, path, Some(&body), Attempts::Once) .await?; - answer + let id = answer .get("erasure_id") .and_then(Value::as_str) .map(str::to_owned) - .ok_or_else(|| Error::Engine(format!("the erasure of {scope} answered no erasure_id"))) + .ok_or_else(|| { + Error::Engine(format!("the erasure of {scope} answered no erasure_id")) + })?; + let mut status = status_of(&answer); + let started = Instant::now(); + let mut gap = self.timing.poll; + while PENDING.contains(&status.as_str()) { + if started.elapsed() > ERASURE_TIMEOUT { + return Err(Error::Unavailable(format!( + "the erasure of {scope} ({id}) was still {status} after {}s", + ERASURE_TIMEOUT.as_secs() + ))); + } + tokio::time::sleep(gap).await; + gap = (gap * 2).min(HOSTED_POLL_CEILING); + let job = self + .client + .json( + reqwest::Method::GET, + &format!("{path}/{}", urlencode(&id)), + None, + Attempts::RetryTransient, + ) + .await?; + status = status_of(&job); + log::debug!("[cortex] erasure {id} of {scope} is {status}"); + } + if status != COMPLETED { + return Err(Error::Engine(format!( + "the erasure of {scope} ({id}) ended {status}, not {COMPLETED}" + ))); + } + Ok(id) } } + +/// An erasure answer's status. CortexDB's synchronous answer may omit it; +/// an id with no status is an erasure that ran to completion. +fn status_of(answer: &Value) -> String { + answer + .get("status") + .and_then(Value::as_str) + .unwrap_or(COMPLETED) + .to_string() +} From 935714252a076d7a737dc78b958bea12cd069d60 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Thu, 8 Oct 2026 22:33:21 +0530 Subject: [PATCH 05/17] fix(cortex): surface transport failures instead of swallowing them Erase and descriptor paths now propagate transport errors rather than treating a failed request as an empty result, so callers can distinguish a genuine miss from an unreachable backend. Auto-committed-on: macbook Co-authored-by: Medulla --- .../src/cortex/descriptor/mod.rs | 12 +++++++----- .../src/cortex/engine/erase.rs | 13 ++++--------- .../src/cortex/transport/failure.rs | 4 +++- .../src/cortex/transport/mod.rs | 19 ++++++++++++++++--- 4 files changed, 30 insertions(+), 18 deletions(-) diff --git a/crates/tinymemory-integrations/src/cortex/descriptor/mod.rs b/crates/tinymemory-integrations/src/cortex/descriptor/mod.rs index d3355b4f..7312a7c8 100644 --- a/crates/tinymemory-integrations/src/cortex/descriptor/mod.rs +++ b/crates/tinymemory-integrations/src/cortex/descriptor/mod.rs @@ -122,7 +122,8 @@ pub enum CortexWire { /// and a bulk append route. Direct, /// CortexDB behind the TinyHumans backend's `/memory/*` routes: - /// `{success,data}` envelopes, typed `errorCode` failures, a strict answer + /// `{success,data}` envelopes (except the `/memory/v1/*` passthrough, + /// which answers in CortexDB's dialect), typed `errorCode` failures, a strict answer /// schema, `Idempotency-Key` claims on writes, and no bulk, wait or health /// route. TinyHumans, @@ -166,9 +167,9 @@ impl CortexWire { (Self::TinyHumans, Route::BuildBeliefs) => "memory/beliefs/build", // Never sent: hosted beliefs are read through recall only. (Self::TinyHumans, Route::Beliefs) => "memory/beliefs", - // Never sent: the backend proxies no erasure route, so `erase` - // refuses on this wire without a request. - (Self::TinyHumans, Route::Erasures) => "memory/erasures", + // The backend's CortexDB-dialect passthrough (no envelope); + // memory-api pins the scope under the tenant's root. + (Self::TinyHumans, Route::Erasures) => "memory/v1/erasures", // Never sent: the hosted backend keeps its own tenancy. (Self::TinyHumans, Route::RegisterScope | Route::ScopeMembers) => "memory/scopes", // Never sent: the hosted wire has no version route, so a date @@ -206,7 +207,8 @@ pub(crate) enum Route { BuildBeliefs, /// List one scope's beliefs (Direct only). Beliefs, - /// Erase a whole scope for good (Direct only). + /// Erase a whole scope for good, and (GET `/{id}`) poll the + /// erasure until it settles. Erasures, /// Build info and the API capabilities the server accepts (Direct only). Version, diff --git a/crates/tinymemory-integrations/src/cortex/engine/erase.rs b/crates/tinymemory-integrations/src/cortex/engine/erase.rs index 9aa9a4b9..09b80f71 100644 --- a/crates/tinymemory-integrations/src/cortex/engine/erase.rs +++ b/crates/tinymemory-integrations/src/cortex/engine/erase.rs @@ -9,24 +9,19 @@ //! first, so a layout that ever put a scope below another would not strand //! redacted events behind held keys. //! -//! Only the Direct wire erases: the TinyHumans backend proxies no erasure -//! route, so the hosted engine refuses with `Unsupported` and sends nothing. +//! Both wires erase: Direct at CortexDB's `v1/erasures`, hosted at the +//! TinyHumans backend's `memory/v1/erasures` passthrough, which memory-api +//! pins under the tenant's root (see `log::erase`). use tinymemory_api::{EraseReport, EraseRequest, ItemKind}; use super::CortexEngine; -use crate::cortex::descriptor::CortexWire; -use crate::cortex::error::{Error, Result}; +use crate::cortex::error::Result; impl CortexEngine { /// See the module docs. pub(super) async fn erase_scopes(&self, req: EraseRequest) -> Result { req.validate()?; - if self.log.client.wire() == CortexWire::TinyHumans { - return Err(Error::Unsupported( - "the TinyHumans backend has no erasure route".to_string(), - )); - } let kinds: Vec = if req.kinds.is_empty() { ItemKind::ALL.to_vec() } else { diff --git a/crates/tinymemory-integrations/src/cortex/transport/failure.rs b/crates/tinymemory-integrations/src/cortex/transport/failure.rs index 862ac1f5..31b54a94 100644 --- a/crates/tinymemory-integrations/src/cortex/transport/failure.rs +++ b/crates/tinymemory-integrations/src/cortex/transport/failure.rs @@ -177,7 +177,9 @@ pub(crate) fn hosted_status_error( let parsed: Option = serde_json::from_str(body).ok(); let code = parsed .as_ref() - .and_then(|v| v.get("errorCode")) + // The backend's own `errorCode`, or memory-api's `error_code` on a + // route the backend passes through unwrapped (`memory/v1/*`). + .and_then(|v| v.get("errorCode").or_else(|| v.get("error_code"))) .and_then(Value::as_str) .map(clean_code) .filter(|c| !c.is_empty()) diff --git a/crates/tinymemory-integrations/src/cortex/transport/mod.rs b/crates/tinymemory-integrations/src/cortex/transport/mod.rs index 1b155cd8..465e9473 100644 --- a/crates/tinymemory-integrations/src/cortex/transport/mod.rs +++ b/crates/tinymemory-integrations/src/cortex/transport/mod.rs @@ -358,9 +358,11 @@ impl HttpClient { } let bytes = body::read_capped(response, label).await?; match self.wire { - CortexWire::TinyHumans => failure::unwrap_envelope(self.host(), label, status, &bytes), - CortexWire::Direct if bytes.is_empty() => Ok(Value::Null), - CortexWire::Direct => serde_json::from_slice(&bytes).map_err(|_| { + CortexWire::TinyHumans if !speaks_cortex(path) => { + failure::unwrap_envelope(self.host(), label, status, &bytes) + } + _ if bytes.is_empty() => Ok(Value::Null), + _ => serde_json::from_slice(&bytes).map_err(|_| { Error::Engine(format!( "memory API {label} on {} returned invalid JSON", self.host() @@ -370,6 +372,17 @@ impl HttpClient { } } +/// The TinyHumans backend's passthrough of CortexDB's own `/v1` dialect, +/// mounted under `memory/`: memory-api's status and body, no +/// `{success,data}` envelope. +pub(crate) const HOSTED_CORTEX_PREFIX: &str = "memory/v1/"; + +/// Whether a hosted `path` is answered in CortexDB's own dialect rather +/// than the backend's envelope. +fn speaks_cortex(path: &str) -> bool { + path.starts_with(HOSTED_CORTEX_PREFIX) +} + /// The route part of a path, for messages: query strings carry scopes and /// cursors, which are noise in an error. fn label(path: &str) -> &str { From 34fd2f9cda87df6143bd34cc99652b641fba0c85 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Thu, 8 Oct 2026 22:33:42 +0530 Subject: [PATCH 06/17] feat(cortex): add testing routes for cortex integration Adds a testing module with routes that exercise the cortex integration endpoints, giving integration tests a way to drive the cortex surface without standing up the full stack. Auto-committed-on: macbook Co-authored-by: Medulla --- .../src/cortex/testing/mod.rs | 5 ++ .../src/cortex/testing/routes.rs | 51 +++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/crates/tinymemory-integrations/src/cortex/testing/mod.rs b/crates/tinymemory-integrations/src/cortex/testing/mod.rs index 1d443f59..790207e4 100644 --- a/crates/tinymemory-integrations/src/cortex/testing/mod.rs +++ b/crates/tinymemory-integrations/src/cortex/testing/mod.rs @@ -121,6 +121,11 @@ pub(crate) struct Double { /// (`app:tinymemory/agent:pad-NNNN/app:learnings`), so a test can make a /// scope listing reach CortexDB's clamp. pub(crate) padding_scopes: AtomicUsize, + /// How many `running` answers an erasure gives (its POST, then its + /// status polls) before it settles. + pub(crate) erasure_running_for: AtomicUsize, + /// The status a settled erasure ends with; `completed` when unset. + pub(crate) erasure_ends: Mutex>, } /// The shared handle the routes and tests hold. diff --git a/crates/tinymemory-integrations/src/cortex/testing/routes.rs b/crates/tinymemory-integrations/src/cortex/testing/routes.rs index bdcefded..0292167d 100644 --- a/crates/tinymemory-integrations/src/cortex/testing/routes.rs +++ b/crates/tinymemory-integrations/src/cortex/testing/routes.rs @@ -331,6 +331,55 @@ async fn erase( relay(&state, result) } +/// The erasure's status as this answer reports it: `running` while the +/// test's running budget lasts, then how it ends. +fn erasure_status(state: &Shared) -> &'static str { + if take_one(&state.erasure_running_for) { + "running" + } else { + state.erasure_ends.lock().unwrap().unwrap_or("completed") + } +} + +/// `POST /memory/v1/erasures`: the backend's passthrough, answered in +/// CortexDB's dialect (no envelope), as memory-api pins it. +async fn hosted_erase( + State(state): State, + uri: Uri, + headers: HeaderMap, + Json(body): Json, +) -> Reply { + if let Some(early) = gate(&state, "POST", &uri, &headers) { + return early; + } + if let Some(refused) = refuse_scope(&state, body["scope"].as_str().unwrap_or_default()) { + return refused; + } + state.seen.lock().unwrap().erasures.push(body.clone()); + let (code, mut answer) = state.log.lock().unwrap().erase(&body); + if code >= 300 { + return (status(code), Json(answer)); + } + answer["status"] = json!(erasure_status(&state)); + (status(code), Json(answer)) +} + +/// `GET /memory/v1/erasures/{id}`: the erasure's status, unwrapped. +async fn hosted_erasure( + State(state): State, + axum::extract::Path(id): axum::extract::Path, + uri: Uri, + headers: HeaderMap, +) -> Reply { + if let Some(early) = gate(&state, "GET", &uri, &headers) { + return early; + } + ( + StatusCode::OK, + Json(json!({ "erasure_id": id, "status": erasure_status(&state) })), + ) +} + async fn answer( State(state): State, uri: Uri, @@ -593,5 +642,7 @@ pub(super) fn hosted(state: Shared) -> Router { .route("/memory/forget", post(forget)) .route("/memory/answer", post(answer)) .route("/memory/scopes", get(scopes)) + .route("/memory/v1/erasures", post(hosted_erase)) + .route("/memory/v1/erasures/{id}", get(hosted_erasure)) .with_state(state) } From c9860cfb3381b6d1fe751792e9724be628f88522 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Thu, 8 Oct 2026 22:34:17 +0530 Subject: [PATCH 07/17] test(cortex): cover hosted erase passthrough and polling Replace the test asserting the hosted wire refuses to erase with tests for the new behaviour: erasure goes through the backend passthrough, releases the write key, and polls a running erasure until it settles, with a non-completing erasure surfacing as an error. Auto-committed-on: macbook Co-authored-by: Medulla --- .../src/cortex/engine/mod_erase_tests.rs | 63 ++++++++++++++++--- 1 file changed, 54 insertions(+), 9 deletions(-) diff --git a/crates/tinymemory-integrations/src/cortex/engine/mod_erase_tests.rs b/crates/tinymemory-integrations/src/cortex/engine/mod_erase_tests.rs index 37a409be..20eff274 100644 --- a/crates/tinymemory-integrations/src/cortex/engine/mod_erase_tests.rs +++ b/crates/tinymemory-integrations/src/cortex/engine/mod_erase_tests.rs @@ -1,12 +1,15 @@ -//! Erase on the Direct wire: deepest scope first, so every event is deleted -//! and every write key released; kinds narrow it; the hosted wire refuses -//! without a request. +//! Erase: deepest scope first, so every event is deleted and every write +//! key released; kinds narrow it. The hosted wire erases through the +//! backend's `memory/v1/erasures` passthrough and polls a running erasure +//! until it settles. use tinymemory_api::{ EraseRequest, ItemKind, LearningKind, ListRequest, MemoryMeta, MetaFilter, Namespace, Reach, StoreItem, }; +use std::sync::atomic::Ordering; + use super::*; use crate::cortex::testing::{direct_double, direct_engine, hosted_double, hosted_engine}; @@ -123,17 +126,59 @@ async fn an_erasure_with_nothing_registered_sends_nothing() { } #[tokio::test] -async fn the_hosted_wire_refuses_to_erase_without_a_request() { +async fn the_hosted_wire_erases_through_the_backend_passthrough() { let (endpoint, state) = hosted_double().await; let engine = hosted_engine(&endpoint); - let refused = engine - .erase(EraseRequest::new(Reach::subtree(node("agent:x")))) - .await; + let (gone, kept) = (node("ws:main/agent:flow"), node("ws:main/agent:chat")); + let erased = learning("flow fact", &gone); + engine.store(erased.clone()).await.unwrap(); + engine.store(learning("chat fact", &kept)).await.unwrap(); + + let report = engine + .erase(EraseRequest::new(Reach::subtree(gone))) + .await + .unwrap(); + assert_eq!(report.erased_scopes, 1, "{report:?}"); + assert_eq!(state.count("POST /memory/v1/erasures"), 1); + let bodies = state.seen.lock().unwrap().erasures.clone(); + assert_eq!(bodies[0]["confirm_all"], true); + assert!(bodies[0].get("selector").is_none(), "{}", bodies[0]); + assert_eq!(listed(&engine).await, vec!["chat fact".to_string()]); + let again = engine.store(erased).await.unwrap(); + assert!(!again.replayed, "an erased item's key was released"); +} + +#[tokio::test] +async fn a_running_hosted_erasure_is_polled_until_it_completes() { + let (endpoint, state) = hosted_double().await; + let engine = hosted_engine(&endpoint); + let at = node("agent:assistant"); + engine.store(learning("a fact", &at)).await.unwrap(); + // The POST and the first two polls answer `running`. + state.erasure_running_for.store(3, Ordering::SeqCst); + + let report = engine + .erase(EraseRequest::new(Reach::exact(at))) + .await + .unwrap(); + assert_eq!(report.receipts, vec!["erasure_1".to_string()]); + assert_eq!(state.count("GET /memory/v1/erasures/erasure_1"), 3); +} + +#[tokio::test] +async fn an_erasure_that_does_not_complete_is_an_error() { + let (endpoint, state) = hosted_double().await; + let engine = hosted_engine(&endpoint); + let at = node("agent:assistant"); + engine.store(learning("a fact", &at)).await.unwrap(); + state.erasure_running_for.store(1, Ordering::SeqCst); + *state.erasure_ends.lock().unwrap() = Some("failed"); + + let refused = engine.erase(EraseRequest::new(Reach::exact(at))).await; assert!( - matches!(refused, Err(tinymemory_api::Error::Unsupported(_))), + matches!(&refused, Err(tinymemory_api::Error::Engine(m)) if m.contains("failed")), "{refused:?}" ); - assert!(state.seen.lock().unwrap().requests.is_empty()); } #[tokio::test] From 06e37b5c5c1bf2a61b771fc69ffdb42dcc0cf8f7 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Thu, 8 Oct 2026 22:34:58 +0530 Subject: [PATCH 08/17] docs(cortex): document hosted erase and forget cascade The Cortex integration docs now describe the hosted erasure path through the backend's `memory/v1/erasures` passthrough, including polling a running erasure until it settles, and note that forget always names the `redact_events` cascade because CortexDB's default keeps the events. Auto-committed-on: macbook Co-authored-by: Medulla --- .../src/cortex/README.md | 18 +++++++++++------- docs/architecture/cortex-wire.md | 17 ++++++++++++++++- docs/specs/memory-v2.md | 4 +++- 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/crates/tinymemory-integrations/src/cortex/README.md b/crates/tinymemory-integrations/src/cortex/README.md index ee708ef0..c0f67185 100644 --- a/crates/tinymemory-integrations/src/cortex/README.md +++ b/crates/tinymemory-integrations/src/cortex/README.md @@ -243,13 +243,17 @@ as prefixes, so they cannot be labelled and are filtered only client-side. - **Forget.** `Ids` looks the items' labels up in every scope the engine holds. `Filter` (which must be non-empty) walks the scopes it reads and matches the full filter. Either way the matched events are then removed with - `selector.memory_ids`, in batches of 100. An empty selector is never sent, - and neither is `confirm_all`. `forgotten` counts items. -- **Erase.** Direct only; hosted refuses with `Unsupported`, because the - backend proxies no erasure route. Lists the registered kind scopes in reach - with the complete scope listing, then sends `v1/erasures` with - `confirm_all` (never a selector) once per scope, deepest first, and - returns the erasure ids as receipts. CortexDB deletes an erased scope's + `selector.memory_ids` and `cascade: "redact_events"`, in batches of 100. + The cascade is always named: CortexDB's default, `derived_only`, keeps the + events. An empty selector is never sent, and neither is `confirm_all`. `forgotten` counts items. +- **Erase.** Both wires. Lists the registered kind scopes in reach with the + complete scope listing, then sends an erasure with `confirm_all` (never a + selector) once per scope, deepest first, and returns the erasure ids as + receipts. Direct sends `v1/erasures`; hosted sends the backend's + `memory/v1/erasures` passthrough (CortexDB's dialect, no envelope), which + memory-api pins under the tenant's root. A `running` answer is polled at + `.../erasures/{id}` until it settles, and anything but `completed` is an + error. CortexDB deletes an erased scope's events and releases their write keys, but only redacts the scopes below it, which keep theirs. Kind scopes are leaves, so this never happens; the order guards a layout where it could. diff --git a/docs/architecture/cortex-wire.md b/docs/architecture/cortex-wire.md index e4530532..17ac51ec 100644 --- a/docs/architecture/cortex-wire.md +++ b/docs/architecture/cortex-wire.md @@ -45,6 +45,8 @@ for. Both fail with `Error::Unsupported` before any request. | Health | `v1/admin/health` | `memory/scopes` | GET | | Scopes (registered scopes under a prefix) | `v1/scopes/list` | `memory/scopes` | GET | | Build beliefs (one scope) | `v1/beliefs/build` | none (never sent) | POST | +| Erase (one scope) | `v1/erasures` | `memory/v1/erasures` (unwrapped) | POST | +| Erasure status | `v1/erasures/{id}` | `memory/v1/erasures/{id}` (unwrapped) | GET | | Whoami (Direct only) | `v1/auth/whoami` | | GET | The endpoint is joined with the route, so a base URL with a path prefix keeps @@ -215,9 +217,14 @@ strict (an unknown key, or a `null` instructions, is a 400). ```json { "scope": "...", "layers": ["events"], "selector": { "memory_ids": ["evt_1", "evt_2"] }, + "cascade": "redact_events", "audit_note": "tinymemory: forget" } ``` +The cascade is always named. CortexDB's default, `derived_only`, removes what +was derived from the events and keeps the events, so a forget that left it out +would not remove anything written. + At most 100 ids per request. The id field is exactly `memory_ids`: an unrecognised or empty selector means *the whole scope* to CortexDB (an empty selector needs `confirm_all`, and `confirm_all` beside a selector is refused). @@ -237,7 +244,7 @@ scopes. There is no cursor (v0.10.5): `limit` defaults to 50 and is clamped to 1000, and `prefix` matches whole segments. At 1000 paths a read logs a warning and an export refuses, since some scopes may be missing. -### Erase: `v1/erasures` (Direct only) +### Erase: `v1/erasures` and `memory/v1/erasures` ```json { "scope": "app:tinymemory/agent:assistant/app:learnings", "confirm_all": true, "audit_note": "tinymemory: erase" } @@ -255,6 +262,14 @@ re-sent write there replays and stores nothing. A whole-scope erasure of six second. Every erasure drops every recall pack the server holds, as a forget does. +Hosted, the same body goes to the backend's `memory/v1/erasures` +passthrough, which answers in CortexDB's dialect (memory-api's status and +body, no `{success,data}` envelope). memory-api pins the scope under the +caller's tenant root and erases with the tenant's own user-actor token. When +the answer's `status` is `running` (or `pending`, `queued`, `accepted`), the +engine polls `GET .../erasures/{id}` until it settles, for at most five +minutes; any final status but `completed` is an error. + ### Build beliefs: `v1/beliefs/build` (Direct only) `consolidate` resolves its reach and kinds to the kind scopes that CortexDB diff --git a/docs/specs/memory-v2.md b/docs/specs/memory-v2.md index 43f7255e..733568ef 100644 --- a/docs/specs/memory-v2.md +++ b/docs/specs/memory-v2.md @@ -228,7 +228,9 @@ pub struct EraseReport { pub erased_scopes: usize, pub receipts: Vec } - **CortexDB, Direct wire:** sends one `/v1/erasures` (`confirm_all`) per registered kind scope, deepest first, and returns the erasure ids as `receipts`. -- **CortexDB, hosted wire:** refuses (the backend proxies no erasure route). +- **CortexDB, hosted wire:** the same, through the TinyHumans backend's + `memory/v1/erasures` passthrough (memory-api pins each scope under the + tenant's root); a `running` erasure is polled until it completes. ### Bulk store From 76cf658b28387a8db1355ef62ab325fa8f9c45b6 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Thu, 8 Oct 2026 22:35:19 +0530 Subject: [PATCH 09/17] test(integrations): clarify erasure test comment Update the doc comment on the erasure test to note that the hosted `memory/v1/erasures` passthrough is covered by the wire double until it is live on the backend, rather than stating the hosted wire has no erasure route. Auto-committed-on: macbook Co-authored-by: Medulla --- crates/tinymemory-integrations/tests/live_cortexdb.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/tinymemory-integrations/tests/live_cortexdb.rs b/crates/tinymemory-integrations/tests/live_cortexdb.rs index 64d37e9c..bb57d75f 100644 --- a/crates/tinymemory-integrations/tests/live_cortexdb.rs +++ b/crates/tinymemory-integrations/tests/live_cortexdb.rs @@ -432,7 +432,8 @@ async fn every_kind_is_exported_whole_across_pages() { } } -/// Direct only: the hosted wire has no erasure route. +/// Direct only for now: the hosted `memory/v1/erasures` passthrough is +/// covered by the wire double until it is live on the backend. #[tokio::test] async fn an_erased_node_is_gone_and_its_items_store_anew() { let _alone = ONE_AT_A_TIME.lock().await; From af13aebf83f89e36a7a2538a0176c81051b0ab2e Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Thu, 8 Oct 2026 23:24:42 +0530 Subject: [PATCH 10/17] feat(cortex): add erase support to the log and engine Introduce erase operations across the cortex log and engine so stored entries can be removed on demand. Supporting test helpers and a transport failure path were added to exercise the new behaviour. Auto-committed-on: macbook Co-authored-by: Medulla --- .../src/cortex/engine/engine_test_support.rs | 1 + .../src/cortex/engine/erase.rs | 8 +- .../src/cortex/log/erase.rs | 138 ++++++++++++------ .../src/cortex/log/mod.rs | 4 + .../src/cortex/testing/log.rs | 11 +- .../src/cortex/testing/mod.rs | 4 + .../src/cortex/testing/routes.rs | 17 ++- .../src/cortex/transport/failure.rs | 13 +- 8 files changed, 136 insertions(+), 60 deletions(-) diff --git a/crates/tinymemory-integrations/src/cortex/engine/engine_test_support.rs b/crates/tinymemory-integrations/src/cortex/engine/engine_test_support.rs index c9f57c6e..fb860f78 100644 --- a/crates/tinymemory-integrations/src/cortex/engine/engine_test_support.rs +++ b/crates/tinymemory-integrations/src/cortex/engine/engine_test_support.rs @@ -11,6 +11,7 @@ impl CortexEngine { visibility, settle: visibility, poll: Duration::from_millis(5), + erasure: Duration::from_millis(300), }; self.log.client.set_read_backoff(Duration::from_millis(5)); self diff --git a/crates/tinymemory-integrations/src/cortex/engine/erase.rs b/crates/tinymemory-integrations/src/cortex/engine/erase.rs index 70a04393..2cece7d3 100644 --- a/crates/tinymemory-integrations/src/cortex/engine/erase.rs +++ b/crates/tinymemory-integrations/src/cortex/engine/erase.rs @@ -45,10 +45,10 @@ impl CortexEngine { scopes.sort_by_key(|scope| std::cmp::Reverse(scope.path.matches('/').count())); let mut report = EraseReport::default(); for scope in scopes { - let receipts = self.log.erase(&scope.path).await?; - log::debug!("[cortex] erased {} ({receipts:?})", scope.path); - report.erased_scopes += 1; - report.receipts.extend(receipts); + let erased = self.log.erase(&scope.path).await?; + log::debug!("[cortex] erased {} ({erased:?})", scope.path); + report.erased_scopes += erased.scopes; + report.receipts.extend(erased.ids); } Ok(report) } diff --git a/crates/tinymemory-integrations/src/cortex/log/erase.rs b/crates/tinymemory-integrations/src/cortex/log/erase.rs index ac6906c2..6da94189 100644 --- a/crates/tinymemory-integrations/src/cortex/log/erase.rs +++ b/crates/tinymemory-integrations/src/cortex/log/erase.rs @@ -24,7 +24,7 @@ //! `v1/erasures/{id}` until it settles, and anything but `completed` is an //! error: an erasure that did not finish must never read as done. -use std::time::{Duration, Instant}; +use std::time::Instant; use serde_json::{Value, json}; @@ -33,9 +33,6 @@ use crate::cortex::descriptor::{CortexWire, Route}; use crate::cortex::error::{Error, Result}; use crate::cortex::transport::{Attempts, urlencode}; -/// How long a running erasure is polled before it is reported as not done. -const ERASURE_TIMEOUT: Duration = Duration::from_secs(300); - /// How many times a hosted erasure is sent before an incomplete or /// transient failure surfaces. const HOSTED_ERASE_ATTEMPTS: u32 = 3; @@ -49,12 +46,24 @@ const PENDING: [&str; 4] = ["running", "pending", "queued", "accepted"]; /// The audit note every erasure carries. const AUDIT_NOTE: &str = "tinymemory: erase"; +/// What one scope's erasure removed. +#[derive(Debug)] +pub(crate) struct Erased { + /// How many scopes the backend erased: one on Direct; on a hosted wire + /// what the backend reports (`0` when nothing was left to erase). + pub(crate) scopes: usize, + /// The erasure ids, once completed. + pub(crate) ids: Vec, +} + impl Log { - /// Erases `scope`, returning the erasure ids once it has completed - /// (none for a hosted scope that held nothing). - pub(crate) async fn erase(&self, scope: &str) -> Result> { + /// Erases `scope`, returning what was erased once it has completed. + pub(crate) async fn erase(&self, scope: &str) -> Result { match self.client.wire() { - CortexWire::Direct => self.erase_direct(scope).await.map(|id| vec![id]), + CortexWire::Direct => self.erase_direct(scope).await.map(|id| Erased { + scopes: 1, + ids: vec![id], + }), CortexWire::TinyHumans => self.erase_hosted(scope).await, } } @@ -62,7 +71,7 @@ impl Log { /// Hosted: memory-api's synchronous scoped erasure. Retried on a /// transient failure (`502 ERASURE_INCOMPLETE` among them), since /// erasing a scope again erases only what is left. - async fn erase_hosted(&self, scope: &str) -> Result> { + async fn erase_hosted(&self, scope: &str) -> Result { let body = json!({ "scope": scope, "audit_note": AUDIT_NOTE }); let path = self.client.wire().path(Route::Erasures); let mut attempt = 0; @@ -91,17 +100,35 @@ impl Log { "the erasure of {scope} answered without `erased: true`" ))); } - let ids = answer - .get("erasure_ids") - .and_then(Value::as_array) - .map(|ids| { - ids.iter() - .filter_map(Value::as_str) - .map(str::to_owned) - .collect() - }) - .unwrap_or_default(); - Ok(ids) + let scopes = answer + .get("scopes") + .and_then(Value::as_u64) + .ok_or_else(|| { + Error::Engine(format!( + "the erasure of {scope} answered without a numeric `scopes` count" + )) + })?; + let ids = match answer.get("erasure_ids") { + None | Some(Value::Null) => Vec::new(), + Some(Value::Array(ids)) => ids + .iter() + .map(|id| id.as_str().map(str::to_owned)) + .collect::>>() + .ok_or_else(|| { + Error::Engine(format!( + "the erasure of {scope} answered a non-string entry in `erasure_ids`" + )) + })?, + Some(_) => { + return Err(Error::Engine(format!( + "the erasure of {scope} answered `erasure_ids` that is not an array" + ))); + } + }; + Ok(Erased { + scopes: usize::try_from(scopes).unwrap_or(usize::MAX), + ids, + }) } /// Direct: CortexDB's `v1/erasures`, sent once (a failure surfaces), @@ -124,28 +151,42 @@ impl Log { .ok_or_else(|| { Error::Engine(format!("the erasure of {scope} answered no erasure_id")) })?; - let mut status = status_of(&answer); - let started = Instant::now(); + // The synchronous answer may omit its status (it ran to the end). + let mut status = status_of(&answer, true)?; + let limit = self.timing.erasure; + let deadline = Instant::now() + limit; let mut gap = self.timing.poll; while PENDING.contains(&status.as_str()) { - if started.elapsed() > ERASURE_TIMEOUT { - return Err(Error::Unavailable(format!( - "the erasure of {scope} ({id}) was still {status} after {}s", - ERASURE_TIMEOUT.as_secs() - ))); + let unavailable = || { + Error::Unavailable(format!( + "the erasure of {scope} ({id}) was still pending after {}ms", + limit.as_millis() + )) + }; + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Err(unavailable()); } - tokio::time::sleep(gap).await; + let poll = async { + tokio::time::sleep(gap).await; + self.client + .json( + reqwest::Method::GET, + &format!("{path}/{}", urlencode(&id)), + None, + Attempts::RetryTransient, + ) + .await + }; + // The deadline bounds the sleep and every retry of the request, + // not just the gap between polls. + let job = tokio::time::timeout(remaining, poll) + .await + .map_err(|_| unavailable())??; gap = (gap * 2).min(HOSTED_POLL_CEILING); - let job = self - .client - .json( - reqwest::Method::GET, - &format!("{path}/{}", urlencode(&id)), - None, - Attempts::RetryTransient, - ) - .await?; - status = status_of(&job); + // A polled job must say where it stands: a missing status is + // not a completed erasure. + status = status_of(&job, false)?; log::debug!("[cortex] erasure {id} of {scope} is {status}"); } if status != COMPLETED { @@ -202,12 +243,17 @@ impl Log { } } -/// An erasure answer's status. CortexDB's synchronous answer may omit it; -/// an id with no status is an erasure that ran to completion. -fn status_of(answer: &Value) -> String { - answer - .get("status") - .and_then(Value::as_str) - .unwrap_or(COMPLETED) - .to_string() +/// An erasure answer's status. CortexDB's synchronous POST answer may omit +/// it (`default_completed`: an id with no status ran to completion); a +/// polled job must carry a string status, and a status of any other type is +/// malformed either way. +fn status_of(answer: &Value, default_completed: bool) -> Result { + match answer.get("status") { + Some(Value::String(status)) => Ok(status.clone()), + None if default_completed => Ok(COMPLETED.to_string()), + other => Err(Error::Engine(format!( + "an erasure answer carried no usable status ({})", + other.map_or_else(|| "missing".to_string(), |v| v.to_string()) + ))), + } } diff --git a/crates/tinymemory-integrations/src/cortex/log/mod.rs b/crates/tinymemory-integrations/src/cortex/log/mod.rs index 22a3a261..608e89e5 100644 --- a/crates/tinymemory-integrations/src/cortex/log/mod.rs +++ b/crates/tinymemory-integrations/src/cortex/log/mod.rs @@ -76,6 +76,9 @@ pub(crate) struct Timing { /// First gap between polls. Direct polls at this gap; hosted doubles it /// up to [`HOSTED_POLL_CEILING`]. pub(crate) poll: Duration, + /// How long a running Direct erasure is polled before it is reported + /// as not done (the status requests included). + pub(crate) erasure: Duration, } /// Longest gap between hosted polls. A fixed 250ms poll would spend a fifth @@ -88,6 +91,7 @@ impl Default for Timing { visibility: Duration::from_secs(30), settle: Duration::from_secs(10), poll: Duration::from_millis(250), + erasure: Duration::from_secs(300), } } } diff --git a/crates/tinymemory-integrations/src/cortex/testing/log.rs b/crates/tinymemory-integrations/src/cortex/testing/log.rs index a8372643..3d921666 100644 --- a/crates/tinymemory-integrations/src/cortex/testing/log.rs +++ b/crates/tinymemory-integrations/src/cortex/testing/log.rs @@ -172,10 +172,13 @@ impl CortexLog { }) .unwrap_or_default(); let selective = !ids.is_empty(); - let cascade = body - .get("cascade") - .and_then(Value::as_str) - .unwrap_or("derived_only"); + let cascade = match body.get("cascade") { + None => "derived_only", + Some(Value::String(cascade)) => cascade.as_str(), + // Present but not a string (null, a number): invalid, never + // the default. + Some(_) => "", + }; if !matches!(cascade, "derived_only" | "redact_events") { return (400, json!({ "error_code": "INVALID_CASCADE" })); } diff --git a/crates/tinymemory-integrations/src/cortex/testing/mod.rs b/crates/tinymemory-integrations/src/cortex/testing/mod.rs index 8c4074c1..db4dece0 100644 --- a/crates/tinymemory-integrations/src/cortex/testing/mod.rs +++ b/crates/tinymemory-integrations/src/cortex/testing/mod.rs @@ -142,6 +142,10 @@ pub(crate) struct Double { /// How many `running` answers a Direct erasure gives (its POST, then /// its status polls) before it settles. pub(crate) erasure_running_for: AtomicUsize, + /// Polled Direct erasure answers (`GET v1/erasures/{id}`) omit `status`. + pub(crate) erasure_poll_omits_status: AtomicBool, + /// Replaces the answer of a hosted scoped erasure that succeeded. + pub(crate) scoped_erase_answer: Mutex>, /// The status a settled erasure ends with; `completed` when unset. pub(crate) erasure_ends: Mutex>, } diff --git a/crates/tinymemory-integrations/src/cortex/testing/routes.rs b/crates/tinymemory-integrations/src/cortex/testing/routes.rs index 430a012c..a9496adf 100644 --- a/crates/tinymemory-integrations/src/cortex/testing/routes.rs +++ b/crates/tinymemory-integrations/src/cortex/testing/routes.rs @@ -418,7 +418,13 @@ async fn hosted_erase( Json(json!({ "error_code": "UNKNOWN_FIELD", "message": unknown })), ); } - let scope = body["scope"].as_str().unwrap_or_default().to_string(); + // A scope that is not a string is a malformed request, not the root. + let Some(scope) = body.get("scope").and_then(Value::as_str).map(str::to_string) else { + return ( + StatusCode::BAD_REQUEST, + Json(json!({ "error_code": "INVALID_SCOPE" })), + ); + }; if scope.is_empty() || scope == "/" { return ( StatusCode::UNPROCESSABLE_ENTITY, @@ -448,6 +454,9 @@ async fn hosted_erase( } ids.push(answer["erasure_id"].clone()); } + if let Some(answer) = state.scoped_erase_answer.lock().unwrap().clone() { + return (StatusCode::OK, Json(answer)); + } ( StatusCode::OK, Json(json!({ "erased": true, "scope": scope, "scopes": ids.len(), "erasure_ids": ids })), @@ -464,9 +473,13 @@ async fn erasure( if let Some(early) = gate(&state, "GET", &uri, &headers) { return early; } + let status = erasure_status(&state); + if state.erasure_poll_omits_status.load(Ordering::SeqCst) { + return (StatusCode::OK, Json(json!({ "erasure_id": id }))); + } ( StatusCode::OK, - Json(json!({ "erasure_id": id, "status": erasure_status(&state) })), + Json(json!({ "erasure_id": id, "status": status })), ) } diff --git a/crates/tinymemory-integrations/src/cortex/transport/failure.rs b/crates/tinymemory-integrations/src/cortex/transport/failure.rs index 3e59ab13..a49ec93b 100644 --- a/crates/tinymemory-integrations/src/cortex/transport/failure.rs +++ b/crates/tinymemory-integrations/src/cortex/transport/failure.rs @@ -179,10 +179,15 @@ pub(crate) fn hosted_status_error( .as_ref() // The backend's own `errorCode`, or memory-api's `error_code` on a // route the backend passes through unwrapped (`memory/v1/*`). - .and_then(|v| v.get("errorCode").or_else(|| v.get("error_code"))) - .and_then(Value::as_str) - .map(clean_code) - .filter(|c| !c.is_empty()) + // A code that is null, not a string or empty once cleaned does not + // shadow the other key. + .and_then(|v| { + ["errorCode", "error_code"] + .iter() + .filter_map(|key| v.get(*key).and_then(Value::as_str)) + .map(clean_code) + .find(|c| !c.is_empty()) + }) .unwrap_or_else(|| default_code(status)); let message = parsed.as_ref().and_then(|v| v.get("error")).map_or_else( || body.to_string(), From 2d8ad436556ab28fc97f0102e486f8db83c37e3c Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Thu, 8 Oct 2026 23:25:12 +0530 Subject: [PATCH 11/17] test(cortex): add direct, erase, and transport failure tests Cover the direct engine path, erase behaviour, and transport failure handling with new integration tests so regressions in these areas are caught before they reach the higher-level flows. Auto-committed-on: macbook Co-authored-by: Medulla --- .../src/cortex/engine/mod_direct_tests.rs | 15 +++ .../src/cortex/engine/mod_erase_tests.rs | 93 +++++++++++++++++++ .../src/cortex/transport/failure_tests.rs | 20 ++++ 3 files changed, 128 insertions(+) diff --git a/crates/tinymemory-integrations/src/cortex/engine/mod_direct_tests.rs b/crates/tinymemory-integrations/src/cortex/engine/mod_direct_tests.rs index 753d05f8..2ef44f04 100644 --- a/crates/tinymemory-integrations/src/cortex/engine/mod_direct_tests.rs +++ b/crates/tinymemory-integrations/src/cortex/engine/mod_direct_tests.rs @@ -299,3 +299,18 @@ async fn reads_wait_out_a_scope_authorization_change() { "six attempts, then it gives up" ); } + +#[test] +fn the_double_refuses_a_cascade_that_is_not_a_known_string() { + let mut log = crate::cortex::testing::log_for_tests(); + for cascade in [ + serde_json::json!(null), + serde_json::json!(123), + serde_json::json!("nope"), + ] { + let (code, answer) = log.forget(&serde_json::json!({ + "scope": "app:x", "confirm_all": true, "cascade": cascade + })); + assert_eq!(code, 400, "{cascade}: {answer}"); + } +} diff --git a/crates/tinymemory-integrations/src/cortex/engine/mod_erase_tests.rs b/crates/tinymemory-integrations/src/cortex/engine/mod_erase_tests.rs index 92420408..f8fa46d0 100644 --- a/crates/tinymemory-integrations/src/cortex/engine/mod_erase_tests.rs +++ b/crates/tinymemory-integrations/src/cortex/engine/mod_erase_tests.rs @@ -463,3 +463,96 @@ fn the_double_refuses_an_erasure_without_a_scope() { let (status, _) = log.erase(&serde_json::json!({ "confirm_all": true })); assert_eq!(status, 422); } + +#[tokio::test] +async fn a_direct_erasure_that_stays_running_times_out_as_unavailable() { + let (endpoint, state) = direct_double().await; + let engine = direct_engine(&endpoint); + let at = node("agent:assistant"); + engine.store(learning("a fact", &at)).await.unwrap(); + // Running for far longer than the test's erasure deadline. + state.erasure_running_for.store(100_000, Ordering::SeqCst); + + let started = std::time::Instant::now(); + let refused = engine.erase(EraseRequest::new(Reach::exact(at))).await; + assert!( + matches!(&refused, Err(tinymemory_api::Error::Unavailable(m)) if m.contains("pending")), + "{refused:?}" + ); + assert!( + started.elapsed() < std::time::Duration::from_secs(5), + "the deadline bounds the whole poll loop" + ); + assert!(state.count("GET /v1/erasures/erasure_1") >= 1); +} + +#[tokio::test] +async fn a_polled_erasure_without_a_status_is_not_completed() { + let (endpoint, state) = direct_double().await; + let engine = direct_engine(&endpoint); + let at = node("agent:assistant"); + engine.store(learning("a fact", &at)).await.unwrap(); + state.erasure_running_for.store(1, Ordering::SeqCst); + state.erasure_poll_omits_status.store(true, Ordering::SeqCst); + + let refused = engine.erase(EraseRequest::new(Reach::exact(at))).await; + assert!( + matches!(&refused, Err(tinymemory_api::Error::Engine(m)) if m.contains("status")), + "{refused:?}" + ); +} + +#[tokio::test] +async fn a_direct_erasure_that_keeps_a_non_terminal_status_is_not_done() { + let (endpoint, state) = direct_double().await; + let engine = direct_engine(&endpoint); + let at = node("agent:assistant"); + engine.store(learning("a fact", &at)).await.unwrap(); + // Settles as `cancelled`: neither pending nor completed. + state.erasure_running_for.store(1, Ordering::SeqCst); + *state.erasure_ends.lock().unwrap() = Some("cancelled"); + + let refused = engine.erase(EraseRequest::new(Reach::exact(at))).await; + assert!( + matches!(&refused, Err(tinymemory_api::Error::Engine(m)) if m.contains("cancelled")), + "{refused:?}" + ); +} + +#[tokio::test] +async fn a_hosted_erasure_reports_the_scopes_the_backend_erased() { + let (endpoint, state) = hosted_double().await; + let engine = hosted_engine(&endpoint); + let at = node("agent:assistant"); + engine.store(learning("a fact", &at)).await.unwrap(); + // The scope emptied between the listing and the erasure. + *state.scoped_erase_answer.lock().unwrap() = + Some(serde_json::json!({ "erased": true, "scopes": 0, "erasure_ids": [] })); + + let report = engine + .erase(EraseRequest::new(Reach::exact(at))) + .await + .unwrap(); + assert_eq!(report.erased_scopes, 0, "{report:?}"); + assert!(report.receipts.is_empty()); +} + +#[tokio::test] +async fn a_malformed_hosted_erasure_answer_is_an_error() { + for answer in [ + serde_json::json!({ "erased": true, "scopes": 1, "erasure_ids": [123] }), + serde_json::json!({ "erased": true, "scopes": 1, "erasure_ids": "x" }), + serde_json::json!({ "erased": true, "erasure_ids": [] }), + ] { + let (endpoint, state) = hosted_double().await; + let engine = hosted_engine(&endpoint); + let at = node("agent:assistant"); + engine.store(learning("a fact", &at)).await.unwrap(); + *state.scoped_erase_answer.lock().unwrap() = Some(answer.clone()); + let refused = engine.erase(EraseRequest::new(Reach::exact(at))).await; + assert!( + matches!(refused, Err(tinymemory_api::Error::Engine(_))), + "{answer}: {refused:?}" + ); + } +} diff --git a/crates/tinymemory-integrations/src/cortex/transport/failure_tests.rs b/crates/tinymemory-integrations/src/cortex/transport/failure_tests.rs index 5d3ab90f..d5a3f0cc 100644 --- a/crates/tinymemory-integrations/src/cortex/transport/failure_tests.rs +++ b/crates/tinymemory-integrations/src/cortex/transport/failure_tests.rs @@ -251,3 +251,23 @@ fn a_write_the_indexer_has_not_reached_is_transient() { let hosted = hosted_status_error("h", "memory/experience", StatusCode::REQUEST_TIMEOUT, body); assert!(matches!(hosted, Error::Unavailable(_)), "{hosted:?}"); } + +#[test] +fn an_unusable_error_code_falls_back_to_the_other_key() { + for body in [ + r#"{"errorCode":null,"error_code":"INVALID_BODY"}"#, + r#"{"errorCode":7,"error_code":"INVALID_BODY"}"#, + r#"{"errorCode":"-- ","error_code":"INVALID_BODY"}"#, + ] { + let error = hosted_status_error("h", "memory/x", StatusCode::BAD_REQUEST, body); + assert_eq!(error_code(&error), Some("INVALID_BODY"), "{body}"); + } + // The primary key still wins when it is usable. + let error = hosted_status_error( + "h", + "memory/x", + StatusCode::BAD_REQUEST, + r#"{"errorCode":"FIRST","error_code":"SECOND"}"#, + ); + assert_eq!(error_code(&error), Some("FIRST")); +} From ed0e299027dd8311e7744374a7183199646565f8 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Thu, 8 Oct 2026 23:25:39 +0530 Subject: [PATCH 12/17] test(cortex): use CortexLog::default in direct tests Replaced the log_for_tests helper call with CortexLog::default in the cascade validation test, and reformatted two long expressions in the erase tests and testing routes to satisfy rustfmt. Auto-committed-on: macbook Co-authored-by: Medulla --- .../src/cortex/engine/mod_direct_tests.rs | 2 +- .../src/cortex/engine/mod_erase_tests.rs | 4 +++- crates/tinymemory-integrations/src/cortex/testing/routes.rs | 6 +++++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/crates/tinymemory-integrations/src/cortex/engine/mod_direct_tests.rs b/crates/tinymemory-integrations/src/cortex/engine/mod_direct_tests.rs index 2ef44f04..6111369c 100644 --- a/crates/tinymemory-integrations/src/cortex/engine/mod_direct_tests.rs +++ b/crates/tinymemory-integrations/src/cortex/engine/mod_direct_tests.rs @@ -302,7 +302,7 @@ async fn reads_wait_out_a_scope_authorization_change() { #[test] fn the_double_refuses_a_cascade_that_is_not_a_known_string() { - let mut log = crate::cortex::testing::log_for_tests(); + let mut log = crate::cortex::testing::CortexLog::default(); for cascade in [ serde_json::json!(null), serde_json::json!(123), diff --git a/crates/tinymemory-integrations/src/cortex/engine/mod_erase_tests.rs b/crates/tinymemory-integrations/src/cortex/engine/mod_erase_tests.rs index f8fa46d0..ce3d407f 100644 --- a/crates/tinymemory-integrations/src/cortex/engine/mod_erase_tests.rs +++ b/crates/tinymemory-integrations/src/cortex/engine/mod_erase_tests.rs @@ -493,7 +493,9 @@ async fn a_polled_erasure_without_a_status_is_not_completed() { let at = node("agent:assistant"); engine.store(learning("a fact", &at)).await.unwrap(); state.erasure_running_for.store(1, Ordering::SeqCst); - state.erasure_poll_omits_status.store(true, Ordering::SeqCst); + state + .erasure_poll_omits_status + .store(true, Ordering::SeqCst); let refused = engine.erase(EraseRequest::new(Reach::exact(at))).await; assert!( diff --git a/crates/tinymemory-integrations/src/cortex/testing/routes.rs b/crates/tinymemory-integrations/src/cortex/testing/routes.rs index a9496adf..8796ce05 100644 --- a/crates/tinymemory-integrations/src/cortex/testing/routes.rs +++ b/crates/tinymemory-integrations/src/cortex/testing/routes.rs @@ -419,7 +419,11 @@ async fn hosted_erase( ); } // A scope that is not a string is a malformed request, not the root. - let Some(scope) = body.get("scope").and_then(Value::as_str).map(str::to_string) else { + let Some(scope) = body + .get("scope") + .and_then(Value::as_str) + .map(str::to_string) + else { return ( StatusCode::BAD_REQUEST, Json(json!({ "error_code": "INVALID_SCOPE" })), From 9f438b40cae21b39898bc5df8c194764fbb457b0 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Thu, 8 Oct 2026 23:34:10 +0530 Subject: [PATCH 13/17] fix(cortex): retry only erasures proven incomplete Hosted erasure retries now trigger solely on the ERASURE_INCOMPLETE code, since any other failure leaves the outcome unknown and repeating a destructive request would hide it. Scope counts saturate instead of overflowing, and a Direct erasure answer that omits status is treated as completed without polling. Auto-committed-on: macbook Co-authored-by: Medulla --- .../src/cortex/engine/erase.rs | 2 +- .../src/cortex/engine/mod_erase_tests.rs | 32 +++++++++++++++---- .../src/cortex/log/erase.rs | 16 ++++++++-- .../src/cortex/testing/mod.rs | 2 ++ .../src/cortex/testing/routes.rs | 5 ++- 5 files changed, 46 insertions(+), 11 deletions(-) diff --git a/crates/tinymemory-integrations/src/cortex/engine/erase.rs b/crates/tinymemory-integrations/src/cortex/engine/erase.rs index 2cece7d3..9b3ca797 100644 --- a/crates/tinymemory-integrations/src/cortex/engine/erase.rs +++ b/crates/tinymemory-integrations/src/cortex/engine/erase.rs @@ -47,7 +47,7 @@ impl CortexEngine { for scope in scopes { let erased = self.log.erase(&scope.path).await?; log::debug!("[cortex] erased {} ({erased:?})", scope.path); - report.erased_scopes += erased.scopes; + report.erased_scopes = report.erased_scopes.saturating_add(erased.scopes); report.receipts.extend(erased.ids); } Ok(report) diff --git a/crates/tinymemory-integrations/src/cortex/engine/mod_erase_tests.rs b/crates/tinymemory-integrations/src/cortex/engine/mod_erase_tests.rs index ce3d407f..22056eae 100644 --- a/crates/tinymemory-integrations/src/cortex/engine/mod_erase_tests.rs +++ b/crates/tinymemory-integrations/src/cortex/engine/mod_erase_tests.rs @@ -473,17 +473,15 @@ async fn a_direct_erasure_that_stays_running_times_out_as_unavailable() { // Running for far longer than the test's erasure deadline. state.erasure_running_for.store(100_000, Ordering::SeqCst); - let started = std::time::Instant::now(); let refused = engine.erase(EraseRequest::new(Reach::exact(at))).await; assert!( matches!(&refused, Err(tinymemory_api::Error::Unavailable(m)) if m.contains("pending")), "{refused:?}" ); - assert!( - started.elapsed() < std::time::Duration::from_secs(5), - "the deadline bounds the whole poll loop" - ); - assert!(state.count("GET /v1/erasures/erasure_1") >= 1); + // Polls back off from 5ms against a 300ms deadline: a bounded few, not + // the thousands a loop that ignored the deadline would make. + let polls = state.count("GET /v1/erasures/erasure_1"); + assert!((1..=20).contains(&polls), "{polls} polls"); } #[tokio::test] @@ -558,3 +556,25 @@ async fn a_malformed_hosted_erasure_answer_is_an_error() { ); } } + +#[tokio::test] +async fn a_synchronous_erasure_answer_without_a_status_is_completed() { + let (endpoint, state) = direct_double().await; + let engine = direct_engine(&endpoint); + let at = node("agent:assistant"); + engine.store(learning("a fact", &at)).await.unwrap(); + state + .erasure_post_omits_status + .store(true, Ordering::SeqCst); + + let report = engine + .erase(EraseRequest::new(Reach::exact(at))) + .await + .unwrap(); + assert_eq!(report.erased_scopes, 1); + assert_eq!( + state.count("GET /v1/erasures/erasure_1"), + 0, + "nothing to poll" + ); +} diff --git a/crates/tinymemory-integrations/src/cortex/log/erase.rs b/crates/tinymemory-integrations/src/cortex/log/erase.rs index 6da94189..6b74e74a 100644 --- a/crates/tinymemory-integrations/src/cortex/log/erase.rs +++ b/crates/tinymemory-integrations/src/cortex/log/erase.rs @@ -16,7 +16,7 @@ //! answers synchronously in its own dialect (no `{success,data}` envelope; //! see `transport`): `{erased: true, scopes, erasure_ids}`, where `scopes: //! 0` (nothing stored) is still success. `502 ERASURE_INCOMPLETE` is -//! retriable and retried; re-erasing a scope is safe. A backend without the +//! the only failure retried (any other leaves the outcome unknown); re-erasing a scope is safe. A backend without the //! route answers 404, reported as [`Error::Unsupported`]: nothing was erased. //! //! A Direct erasure is a job. CortexDB answers `completed` when it ran to @@ -30,13 +30,16 @@ use serde_json::{Value, json}; use super::{HOSTED_POLL_CEILING, Log}; use crate::cortex::descriptor::{CortexWire, Route}; -use crate::cortex::error::{Error, Result}; +use crate::cortex::error::{Error, Result, error_code}; use crate::cortex::transport::{Attempts, urlencode}; /// How many times a hosted erasure is sent before an incomplete or /// transient failure surfaces. const HOSTED_ERASE_ATTEMPTS: u32 = 3; +/// The code of the retriable `502` that says an erasure did not finish. +const ERASURE_INCOMPLETE: &str = "ERASURE_INCOMPLETE"; + /// The one status that means the scope is gone. const COMPLETED: &str = "completed"; @@ -88,7 +91,14 @@ impl Log { "the TinyHumans backend has no scoped erasure route ({message})" ))); } - Err(error) if error.is_transient() && attempt < HOSTED_ERASE_ATTEMPTS => { + // Only an answer that proves the erasure incomplete is + // retried (re-erasing erases what is left). Any other + // failure leaves the outcome unknown, and a repeated + // destructive request would hide it, so it surfaces. + Err(error) + if error_code(&error) == Some(ERASURE_INCOMPLETE) + && attempt < HOSTED_ERASE_ATTEMPTS => + { log::debug!("[cortex] erasure of {scope} incomplete; retrying ({attempt})"); tokio::time::sleep(self.timing.poll * 2_u32.pow(attempt - 1)).await; } diff --git a/crates/tinymemory-integrations/src/cortex/testing/mod.rs b/crates/tinymemory-integrations/src/cortex/testing/mod.rs index db4dece0..15a2daa5 100644 --- a/crates/tinymemory-integrations/src/cortex/testing/mod.rs +++ b/crates/tinymemory-integrations/src/cortex/testing/mod.rs @@ -142,6 +142,8 @@ pub(crate) struct Double { /// How many `running` answers a Direct erasure gives (its POST, then /// its status polls) before it settles. pub(crate) erasure_running_for: AtomicUsize, + /// The Direct erasure POST answer omits `status` (CortexDB may). + pub(crate) erasure_post_omits_status: AtomicBool, /// Polled Direct erasure answers (`GET v1/erasures/{id}`) omit `status`. pub(crate) erasure_poll_omits_status: AtomicBool, /// Replaces the answer of a hosted scoped erasure that succeeded. diff --git a/crates/tinymemory-integrations/src/cortex/testing/routes.rs b/crates/tinymemory-integrations/src/cortex/testing/routes.rs index 8796ce05..8806333e 100644 --- a/crates/tinymemory-integrations/src/cortex/testing/routes.rs +++ b/crates/tinymemory-integrations/src/cortex/testing/routes.rs @@ -374,7 +374,10 @@ async fn erase( state.seen.lock().unwrap().erasures.push(body.clone()); let (code, mut answer) = state.log.lock().unwrap().erase(&body); if code < 300 { - answer["status"] = json!(erasure_status(&state)); + let status = erasure_status(&state); + if !state.erasure_post_omits_status.load(Ordering::SeqCst) { + answer["status"] = json!(status); + } } relay(&state, (code, answer)) } From 5d947c0a31e26ca5439ffe00f0f3eac417291b9e Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Thu, 8 Oct 2026 23:45:32 +0530 Subject: [PATCH 14/17] fix(cortex): reject out-of-range scopes count in erasure responses A hosted erasure answer whose `scopes` count exceeds the platform's `usize` range was silently clamped to `usize::MAX`, hiding a malformed response. It now returns an engine error naming the scope and offending count, and a test covers the string-typed count case. Auto-committed-on: macbook Co-authored-by: Medulla --- .../src/cortex/engine/mod_erase_tests.rs | 1 + crates/tinymemory-integrations/src/cortex/log/erase.rs | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/crates/tinymemory-integrations/src/cortex/engine/mod_erase_tests.rs b/crates/tinymemory-integrations/src/cortex/engine/mod_erase_tests.rs index 22056eae..a2910beb 100644 --- a/crates/tinymemory-integrations/src/cortex/engine/mod_erase_tests.rs +++ b/crates/tinymemory-integrations/src/cortex/engine/mod_erase_tests.rs @@ -543,6 +543,7 @@ async fn a_malformed_hosted_erasure_answer_is_an_error() { serde_json::json!({ "erased": true, "scopes": 1, "erasure_ids": [123] }), serde_json::json!({ "erased": true, "scopes": 1, "erasure_ids": "x" }), serde_json::json!({ "erased": true, "erasure_ids": [] }), + serde_json::json!({ "erased": true, "scopes": "1", "erasure_ids": [] }), ] { let (endpoint, state) = hosted_double().await; let engine = hosted_engine(&endpoint); diff --git a/crates/tinymemory-integrations/src/cortex/log/erase.rs b/crates/tinymemory-integrations/src/cortex/log/erase.rs index 6b74e74a..7cb2b28f 100644 --- a/crates/tinymemory-integrations/src/cortex/log/erase.rs +++ b/crates/tinymemory-integrations/src/cortex/log/erase.rs @@ -136,7 +136,11 @@ impl Log { } }; Ok(Erased { - scopes: usize::try_from(scopes).unwrap_or(usize::MAX), + scopes: usize::try_from(scopes).map_err(|_| { + Error::Engine(format!( + "the erasure of {scope} answered a `scopes` count ({scopes}) beyond this platform's range" + )) + })?, ids, }) } From 5bcab7038db7360aba815736cdf499e9098663b5 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Thu, 8 Oct 2026 23:55:13 +0530 Subject: [PATCH 15/17] test(cortex): cover unknown poll statuses and cross-engine erasure Add a test double hook that forces the status of polled Direct erasure answers, and use it to check that a status which is not a known string never completes the erasure. Also cover a hosted scope erased by an engine that never held it, and relax the poll-count assertion to a lower bound so it no longer depends on backoff timing. Auto-committed-on: macbook Co-authored-by: Medulla --- .../src/cortex/engine/mod_erase_tests.rs | 46 +++++++++++++++++-- .../src/cortex/testing/mod.rs | 2 + .../src/cortex/testing/routes.rs | 6 +++ 3 files changed, 50 insertions(+), 4 deletions(-) diff --git a/crates/tinymemory-integrations/src/cortex/engine/mod_erase_tests.rs b/crates/tinymemory-integrations/src/cortex/engine/mod_erase_tests.rs index a2910beb..b28f9715 100644 --- a/crates/tinymemory-integrations/src/cortex/engine/mod_erase_tests.rs +++ b/crates/tinymemory-integrations/src/cortex/engine/mod_erase_tests.rs @@ -478,10 +478,7 @@ async fn a_direct_erasure_that_stays_running_times_out_as_unavailable() { matches!(&refused, Err(tinymemory_api::Error::Unavailable(m)) if m.contains("pending")), "{refused:?}" ); - // Polls back off from 5ms against a 300ms deadline: a bounded few, not - // the thousands a loop that ignored the deadline would make. - let polls = state.count("GET /v1/erasures/erasure_1"); - assert!((1..=20).contains(&polls), "{polls} polls"); + assert!(state.count("GET /v1/erasures/erasure_1") >= 1); } #[tokio::test] @@ -579,3 +576,44 @@ async fn a_synchronous_erasure_answer_without_a_status_is_completed() { "nothing to poll" ); } + +#[tokio::test] +async fn a_polled_status_that_is_not_a_known_string_never_completes_the_erasure() { + for status in [ + serde_json::json!(7), + serde_json::json!(null), + serde_json::json!("weird"), + ] { + let (endpoint, state) = direct_double().await; + let engine = direct_engine(&endpoint); + let at = node("agent:assistant"); + engine.store(learning("a fact", &at)).await.unwrap(); + state.erasure_running_for.store(1, Ordering::SeqCst); + *state.erasure_poll_status.lock().unwrap() = Some(status.clone()); + let refused = engine.erase(EraseRequest::new(Reach::exact(at))).await; + assert!( + matches!(refused, Err(tinymemory_api::Error::Engine(_))), + "{status}: {refused:?}" + ); + } +} + +#[tokio::test] +async fn a_hosted_scope_is_erased_by_an_engine_that_never_held_it() { + let (endpoint, state) = hosted_double().await; + let at = node("agent:assistant"); + hosted_engine(&endpoint) + .store(learning("a fact", &at)) + .await + .unwrap(); + // A fresh engine has no memory of the write: the scopes come from the + // backend's registry. + let other = hosted_engine(&endpoint); + let report = other + .erase(EraseRequest::new(Reach::exact(at))) + .await + .unwrap(); + assert_eq!(report.erased_scopes, 1, "{report:?}"); + assert_eq!(state.count("POST /memory/v1/erasures"), 1); + assert!(listed(&other).await.is_empty()); +} diff --git a/crates/tinymemory-integrations/src/cortex/testing/mod.rs b/crates/tinymemory-integrations/src/cortex/testing/mod.rs index 15a2daa5..2d81ee32 100644 --- a/crates/tinymemory-integrations/src/cortex/testing/mod.rs +++ b/crates/tinymemory-integrations/src/cortex/testing/mod.rs @@ -142,6 +142,8 @@ pub(crate) struct Double { /// How many `running` answers a Direct erasure gives (its POST, then /// its status polls) before it settles. pub(crate) erasure_running_for: AtomicUsize, + /// Replaces the `status` of polled Direct erasure answers. + pub(crate) erasure_poll_status: Mutex>, /// The Direct erasure POST answer omits `status` (CortexDB may). pub(crate) erasure_post_omits_status: AtomicBool, /// Polled Direct erasure answers (`GET v1/erasures/{id}`) omit `status`. diff --git a/crates/tinymemory-integrations/src/cortex/testing/routes.rs b/crates/tinymemory-integrations/src/cortex/testing/routes.rs index 8806333e..5de7e20c 100644 --- a/crates/tinymemory-integrations/src/cortex/testing/routes.rs +++ b/crates/tinymemory-integrations/src/cortex/testing/routes.rs @@ -481,6 +481,12 @@ async fn erasure( return early; } let status = erasure_status(&state); + if let Some(forced) = state.erasure_poll_status.lock().unwrap().clone() { + return ( + StatusCode::OK, + Json(json!({ "erasure_id": id, "status": forced })), + ); + } if state.erasure_poll_omits_status.load(Ordering::SeqCst) { return (StatusCode::OK, Json(json!({ "erasure_id": id }))); } From 50ff48af0d4b695aabfb951a2409b32ec89a51ae Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 9 Oct 2026 00:01:49 +0530 Subject: [PATCH 16/17] fix(cortex): reject erasure answers that report scopes without receipts A hosted erasure response that claims scopes were erased but returns no erasure_ids is now treated as an incompatible answer rather than a success, since the receipts are the audit trail of a destructive call. The test double's forced poll status also no longer consumes the running budget, and new malformed-answer cases cover the missing-receipt responses. Auto-committed-on: macbook Co-authored-by: Medulla --- .../src/cortex/engine/mod_erase_tests.rs | 3 +++ crates/tinymemory-integrations/src/cortex/log/erase.rs | 7 +++++++ .../tinymemory-integrations/src/cortex/testing/routes.rs | 3 ++- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/crates/tinymemory-integrations/src/cortex/engine/mod_erase_tests.rs b/crates/tinymemory-integrations/src/cortex/engine/mod_erase_tests.rs index b28f9715..ad679aed 100644 --- a/crates/tinymemory-integrations/src/cortex/engine/mod_erase_tests.rs +++ b/crates/tinymemory-integrations/src/cortex/engine/mod_erase_tests.rs @@ -541,6 +541,9 @@ async fn a_malformed_hosted_erasure_answer_is_an_error() { serde_json::json!({ "erased": true, "scopes": 1, "erasure_ids": "x" }), serde_json::json!({ "erased": true, "erasure_ids": [] }), serde_json::json!({ "erased": true, "scopes": "1", "erasure_ids": [] }), + // Scopes erased, but no receipt for them. + serde_json::json!({ "erased": true, "scopes": 1, "erasure_ids": [] }), + serde_json::json!({ "erased": true, "scopes": 2, "erasure_ids": null }), ] { let (endpoint, state) = hosted_double().await; let engine = hosted_engine(&endpoint); diff --git a/crates/tinymemory-integrations/src/cortex/log/erase.rs b/crates/tinymemory-integrations/src/cortex/log/erase.rs index 7cb2b28f..fccad406 100644 --- a/crates/tinymemory-integrations/src/cortex/log/erase.rs +++ b/crates/tinymemory-integrations/src/cortex/log/erase.rs @@ -135,6 +135,13 @@ impl Log { ))); } }; + // The receipts are the audit trail of a destructive call: scopes + // erased without one is an incompatible answer, not a success. + if scopes > 0 && ids.is_empty() { + return Err(Error::Engine(format!( + "the erasure of {scope} erased {scopes} scope(s) but answered no `erasure_ids`" + ))); + } Ok(Erased { scopes: usize::try_from(scopes).map_err(|_| { Error::Engine(format!( diff --git a/crates/tinymemory-integrations/src/cortex/testing/routes.rs b/crates/tinymemory-integrations/src/cortex/testing/routes.rs index 5de7e20c..4040eabc 100644 --- a/crates/tinymemory-integrations/src/cortex/testing/routes.rs +++ b/crates/tinymemory-integrations/src/cortex/testing/routes.rs @@ -480,13 +480,14 @@ async fn erasure( if let Some(early) = gate(&state, "GET", &uri, &headers) { return early; } - let status = erasure_status(&state); + // A forced status leaves the running budget alone. if let Some(forced) = state.erasure_poll_status.lock().unwrap().clone() { return ( StatusCode::OK, Json(json!({ "erasure_id": id, "status": forced })), ); } + let status = erasure_status(&state); if state.erasure_poll_omits_status.load(Ordering::SeqCst) { return (StatusCode::OK, Json(json!({ "erasure_id": id }))); } From ac0fabe3180206b2dec5c0bb9b2d2a4f3f5ff29e Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 9 Oct 2026 00:08:49 +0530 Subject: [PATCH 17/17] fix(cortex): reject blank hosted erasure receipts Co-authored-by: Medulla --- .../src/cortex/engine/mod_erase_tests.rs | 1 + crates/tinymemory-integrations/src/cortex/log/erase.rs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/tinymemory-integrations/src/cortex/engine/mod_erase_tests.rs b/crates/tinymemory-integrations/src/cortex/engine/mod_erase_tests.rs index ad679aed..de49afe6 100644 --- a/crates/tinymemory-integrations/src/cortex/engine/mod_erase_tests.rs +++ b/crates/tinymemory-integrations/src/cortex/engine/mod_erase_tests.rs @@ -544,6 +544,7 @@ async fn a_malformed_hosted_erasure_answer_is_an_error() { // Scopes erased, but no receipt for them. serde_json::json!({ "erased": true, "scopes": 1, "erasure_ids": [] }), serde_json::json!({ "erased": true, "scopes": 2, "erasure_ids": null }), + serde_json::json!({ "erased": true, "scopes": 1, "erasure_ids": [""] }), ] { let (endpoint, state) = hosted_double().await; let engine = hosted_engine(&endpoint); diff --git a/crates/tinymemory-integrations/src/cortex/log/erase.rs b/crates/tinymemory-integrations/src/cortex/log/erase.rs index fccad406..cbf7cd4a 100644 --- a/crates/tinymemory-integrations/src/cortex/log/erase.rs +++ b/crates/tinymemory-integrations/src/cortex/log/erase.rs @@ -137,7 +137,7 @@ impl Log { }; // The receipts are the audit trail of a destructive call: scopes // erased without one is an incompatible answer, not a success. - if scopes > 0 && ids.is_empty() { + if scopes > 0 && ids.iter().all(|id| id.trim().is_empty()) { return Err(Error::Engine(format!( "the erasure of {scope} erased {scopes} scope(s) but answered no `erasure_ids`" )));