Skip to content

refactor(safety): delegate store::safety to the shared tinymemory-safety crate #178

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
senamakel merged 3 commits into from
Sep 30, 2026
Merged

refactor(safety): delegate store::safety to the shared tinymemory-safety crate #178

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
2718338
refactor(safety): delegate store::safety to the shared tinymemory-saf…
senamakel Sep 30, 2026
629371e
fix(safety): update doc reference to use full module path
senamakel Sep 30, 2026
6958497
fix(docs): update cross-reference to use full path for has_likely_pii
senamakel Sep 30, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

8 changes: 8 additions & 0 deletions Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,14 @@ people = ["tokio"]
contacts = ["people", "dep:objc2", "dep:objc2-foundation", "dep:objc2-contacts", "dep:block2"]

[dependencies]
# The shared secret and PII scrubber (`memory::store::safety` is a thin wrapper
# that pins this engine's policy). By git rev for the same reason
# `tinymemory-api` is — neither crate is published — and a host that vendors
# tinymemory patches it to its checkout:
#
# [patch."https://github.com/tinyhumansai/tinymemory"]
# tinymemory-safety = { path = "crates/tinymemory-safety" }
tinymemory-safety = { git = "https://github.com/tinyhumansai/tinymemory", rev = "b49650e1eac6d55a41f641f4ec8a683a25df9518" }
anyhow = "1"
log = "0.4"
futures = "0.3"
Expand Down
333 changes: 31 additions & 302 deletions src/memory/store/safety/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,324 +1,53 @@
//! Secret-detection and redaction helpers for memory writes.
//!
//! Ported from OpenHuman's `memory_store::safety`. Conservative by design — it
//! prefers false positives over leaking credentials into long-lived stores.
//! The scrubbers themselves — credential patterns, the sensitive-key
//! classifier, the JSON depth cap and the checksum-gated multilingual
//! national-ID PII module — live in the shared `tinymemory-safety` crate, which
//! the OpenHuman host and `tinymemory-core` use as well (it used to be copied
//! three times). This module keeps the engine's public `safety::*` paths and
//! pins the engine's one policy choice: a *bare* (separator-less) Luhn-valid
//! digit run is only redacted as a credit card when corroborated by a real
//! network IIN or a nearby card keyword, so 13-digit epoch-millisecond
//! timestamps in stored JSON envelopes are not corrupted (opencompany#1201).
//!
//! The exhaustive multilingual national-ID PII module (`safety::pii`, ~1k lines
//! of checksum logic) is ported from OpenHuman and runs as part of
//! `sanitize_text`. The write-rejection boundary stays stricter than content
//! scrubbing: formatted national IDs are rejected, while phone/email-like text is
//! scrubbed from content without rejecting every write that mentions them.
//! The write-rejection boundary ([`has_likely_pii`](tinymemory_safety::has_likely_pii)) stays stricter than
//! content scrubbing: formatted national IDs are rejected, while phone/email-like
//! text is scrubbed from content without rejecting every write that mentions
//! them.

use std::sync::LazyLock;

use regex::Regex;
use serde_json::Value;

/// Exhaustive checksum-gated multilingual national-ID PII module (ported from
/// OpenHuman). Content scrubbing runs from `sanitize_text`; the boundary
/// check is re-exported as [`has_likely_pii`].
pub mod pii;

pub use pii::{has_likely_email, has_likely_pii};

const REDACTED_SECRET: &str = "[REDACTED_SECRET]";
const REDACTED_PRIVATE_KEY: &str = "[REDACTED_PRIVATE_KEY]";
const MAX_JSON_SANITIZE_DEPTH: usize = 128;

/// Tally of what a sanitization pass changed.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct SanitizationReport {
/// Count of secret/token pattern matches rewritten in string text by the
/// text-pattern redaction pass.
pub text_redactions: usize,
/// Count of JSON object entries dropped wholesale because their key was
/// classified as sensitive by the key classifier.
pub key_redactions: usize,
/// Count of full private-key blocks replaced; these are
/// the most severe hits since the entire block is removed.
pub blocked_secret_hits: usize,
/// Count of nodes collapsed because JSON nesting reached
/// the JSON traversal depth cap; the subtree is replaced rather than walked.
pub depth_redactions: usize,
/// Count of personal-identifier matches replaced by the
/// lightweight PII screen.
pub pii_redactions: usize,
}

impl SanitizationReport {
/// True when any field recorded a redaction.
pub fn changed(&self) -> bool {
self.text_redactions > 0
|| self.key_redactions > 0
|| self.blocked_secret_hits > 0
|| self.depth_redactions > 0
|| self.pii_redactions > 0
}

/// Sum two reports field-wise.
pub fn merge(self, rhs: Self) -> Self {
Self {
text_redactions: self.text_redactions + rhs.text_redactions,
key_redactions: self.key_redactions + rhs.key_redactions,
blocked_secret_hits: self.blocked_secret_hits + rhs.blocked_secret_hits,
depth_redactions: self.depth_redactions + rhs.depth_redactions,
pii_redactions: self.pii_redactions + rhs.pii_redactions,
}
}
}

/// A sanitized value plus the [`SanitizationReport`] describing the changes.
#[derive(Debug, Clone)]
pub struct Sanitized<T> {
/// The cleaned value with secrets and PII removed.
pub value: T,
/// Tally of what the sanitization pass changed to produce `value`.
pub report: SanitizationReport,
}

static BLOCK_PATTERNS: LazyLock<Vec<Regex>> = LazyLock::new(|| {
vec![
Regex::new(
r"(?is)-----BEGIN(?: [A-Z]+)? PRIVATE KEY-----.*?-----END(?: [A-Z]+)? PRIVATE KEY-----",
)
.expect("valid private key block"),
Regex::new(r"(?is)-----BEGIN OPENSSH PRIVATE KEY-----.*?-----END OPENSSH PRIVATE KEY-----")
.expect("valid openssh private key block"),
Regex::new(
r"(?is)-----BEGIN PGP PRIVATE KEY BLOCK-----.*?-----END PGP PRIVATE KEY BLOCK-----",
)
.expect("valid pgp private key block"),
]
});
pub use tinymemory_safety::{
has_likely_email, has_likely_pii, has_likely_secret, BareCardGate, Policy, SanitizationReport,
Sanitized,
};

static REDACTION_PATTERNS: LazyLock<Vec<(Regex, &'static str)>> = LazyLock::new(|| {
vec![
(
Regex::new(r"(?i)(bearer\s+)[A-Za-z0-9._~+/=-]{8,}").expect("valid bearer redaction"),
"${1}[REDACTED]",
),
(
Regex::new(r#"(?i)(api[_-]?key\s*[=:\s]\s*["']?)[^\s"']+"#)
.expect("valid api key redaction"),
"${1}[REDACTED]",
),
(
Regex::new(
r#"(?i)\b(token|access[_-]?token|refresh[_-]?token|client[_-]?secret|password|secret)\b\s*[=:\s]\s*["']?[^\s"'&]+"#,
)
.expect("valid token redaction"),
"[REDACTED]",
),
(
Regex::new(r"\bsk-[A-Za-z0-9]{20,}\b").expect("valid openai key redaction"),
"[REDACTED]",
),
(
Regex::new(r"\bgh[pousr]_[A-Za-z0-9_]{20,}\b").expect("valid github token redaction"),
"[REDACTED]",
),
(
Regex::new(r"\bAKIA[0-9A-Z]{16}\b").expect("valid aws key redaction"),
"[REDACTED]",
),
(
Regex::new(r"\bASIA[0-9A-Z]{16}\b").expect("valid aws sts key redaction"),
"[REDACTED]",
),
(
Regex::new(r"\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9._-]{8,}\.[A-Za-z0-9._-]{8,}\b")
.expect("valid jwt redaction"),
"[REDACTED]",
),
(
Regex::new(
r#"(?i)\b(access_token|refresh_token|id_token|authorization_code|code_verifier|code_challenge)\b\s*[=:\s]\s*["']?[^\s"'&]+"#,
)
.expect("valid oauth token redaction"),
"[REDACTED]",
),
(
Regex::new(r"\bAIza[0-9A-Za-z\-_]{35}\b").expect("valid google api key redaction"),
"[REDACTED]",
),
(
Regex::new(r"\bsk-ant-[A-Za-z0-9\-_]{16,}\b").expect("valid anthropic key redaction"),
"[REDACTED]",
),
(
Regex::new(r"\bsk-(?:proj|org)-[A-Za-z0-9\-_]{12,}\b")
.expect("valid openai scoped key redaction"),
"[REDACTED]",
),
(
Regex::new(r"\b(?:sk|rk)_(?:live|test)_[A-Za-z0-9]{16,}\b")
.expect("valid stripe key redaction"),
"[REDACTED]",
),
(
Regex::new(r"\bxox(?:a|b|p|s|r)-[A-Za-z0-9-]{10,}\b")
.expect("valid slack token redaction"),
"[REDACTED]",
),
(
Regex::new(r"\bgithub_pat_[A-Za-z0-9_]{20,}\b").expect("valid github pat redaction"),
"[REDACTED]",
),
(
Regex::new(r"\bglpat-[A-Za-z0-9\-_]{16,}\b").expect("valid gitlab pat redaction"),
"[REDACTED]",
),
(
Regex::new(r"\bnpm_[A-Za-z0-9]{20,}\b").expect("valid npm token redaction"),
"[REDACTED]",
),
(
Regex::new(r"\bSG\.[A-Za-z0-9_\-]{16,}\.[A-Za-z0-9_\-]{16,}\b")
.expect("valid sendgrid key redaction"),
"[REDACTED]",
),
]
});

/// True when `value` looks like it contains a credential.
pub fn has_likely_secret(value: &str) -> bool {
BLOCK_PATTERNS.iter().any(|p| p.is_match(value))
|| REDACTION_PATTERNS.iter().any(|(p, _)| p.is_match(value))
}
/// The scrubbing policy every engine write uses.
const ENGINE_POLICY: Policy = Policy::corroborated();

/// Scrub secrets and PII from free text, returning the cleaned text plus a
/// [`SanitizationReport`].
pub fn sanitize_text(value: &str) -> Sanitized<String> {
let mut out = value.to_string();
let mut report = SanitizationReport::default();

for pattern in BLOCK_PATTERNS.iter() {
let hits = pattern.find_iter(&out).count();
if hits > 0 {
report.blocked_secret_hits += hits;
out = pattern.replace_all(&out, REDACTED_PRIVATE_KEY).into_owned();
}
}

for (pattern, replacement) in REDACTION_PATTERNS.iter() {
let hits = pattern.find_iter(&out).count();
if hits > 0 {
report.text_redactions += hits;
out = pattern.replace_all(&out, *replacement).into_owned();
}
}

// Full multilingual national-ID PII scrub (checksum-gated, normalization
// pre-pass) — runs after secret redaction so every call site that scrubs
// secrets also scrubs PII.
let pii = pii::redact_pii(&out);
report = report.merge(pii.report);
out = pii.value;

Sanitized { value: out, report }
tinymemory_safety::sanitize_text_with(value, ENGINE_POLICY)
}

/// Recursively scrub a JSON value: sensitive keys are replaced wholesale and
/// every string value runs through `sanitize_text`.
/// every string value runs through [`sanitize_text`].
pub fn sanitize_json(value: &Value) -> Sanitized<Value> {
sanitize_json_inner(value, 0)
tinymemory_safety::sanitize_json_with(value, ENGINE_POLICY)
}

/// Recursive worker behind [`sanitize_json`].
///
/// `depth` counts nesting from the call in `sanitize_json` (which starts at
/// `0`); once it reaches [`MAX_JSON_SANITIZE_DEPTH`] the whole subtree at that
/// point is replaced by a single redaction marker rather than walked further,
/// bounding recursion against pathologically deep or adversarial JSON.
fn sanitize_json_inner(value: &Value, depth: usize) -> Sanitized<Value> {
if depth >= MAX_JSON_SANITIZE_DEPTH {
return Sanitized {
value: Value::String(REDACTED_SECRET.to_string()),
report: SanitizationReport {
depth_redactions: 1,
..SanitizationReport::default()
},
};
}

match value {
Value::Object(map) => {
let mut out = serde_json::Map::new();
let mut report = SanitizationReport::default();
for (key, value) in map {
if is_sensitive_key(key) {
report.key_redactions += 1;
out.insert(key.clone(), Value::String(REDACTED_SECRET.to_string()));
continue;
}
let sanitized = sanitize_json_inner(value, depth + 1);
report = report.merge(sanitized.report);
out.insert(key.clone(), sanitized.value);
}
Sanitized {
value: Value::Object(out),
report,
}
}
Value::Array(items) => {
let mut out = Vec::with_capacity(items.len());
let mut report = SanitizationReport::default();
for item in items {
let sanitized = sanitize_json_inner(item, depth + 1);
report = report.merge(sanitized.report);
out.push(sanitized.value);
}
Sanitized {
value: Value::Array(out),
report,
}
}
Value::String(value) => {
let sanitized = sanitize_text(value);
Sanitized {
value: Value::String(sanitized.value),
report: sanitized.report,
}
}
_ => Sanitized {
value: value.clone(),
report: SanitizationReport::default(),
},
}
}
/// Personal-PII detection and redaction (national IDs, financial identifiers,
/// international phone), on-device, regex + checksum only.
pub mod pii {
use super::{Sanitized, ENGINE_POLICY};

/// True when a JSON object key's name itself suggests it holds a secret
/// (`api_key`, `token`, `password`, …), independent of the value's contents.
///
/// Matching keys are redacted wholesale in [`sanitize_json_inner`] — the
/// value is replaced rather than scanned, since a key named e.g. `password`
/// is assumed sensitive even if its value doesn't match any
/// [`REDACTION_PATTERNS`] regex. Matching is on the key with all
/// non-alphanumeric characters stripped and lowercased, so `API-Key`,
/// `api_key`, and `apiKey` are all treated identically.
fn is_sensitive_key(key: &str) -> bool {
let normalized: String = key
.chars()
.filter(|c| c.is_ascii_alphanumeric())
.map(|c| c.to_ascii_lowercase())
.collect();
pub use tinymemory_safety::pii::{has_likely_email, has_likely_pii};

matches!(
normalized.as_str(),
"apikey"
| "token"
| "accesstoken"
| "refreshtoken"
| "authorization"
| "password"
| "secret"
| "clientsecret"
) || normalized.ends_with("token")
|| normalized.ends_with("apikey")
|| normalized.ends_with("clientsecret")
|| normalized.contains("password")
|| normalized.contains("secret")
|| normalized.ends_with("key")
/// Redact format-based multilingual PII from `text` under the engine policy.
pub fn redact_pii(text: &str) -> Sanitized<String> {
tinymemory_safety::pii::redact_pii_with(text, ENGINE_POLICY)
}
}

#[cfg(test)]
Expand Down
Loading
Loading