From 9b244743b74cf40825ec39de69971ac0d9124ea3 Mon Sep 17 00:00:00 2001 From: James Sadler Date: Wed, 19 Aug 2026 15:03:45 +1000 Subject: [PATCH 01/11] feat(proxy): accept inbound EQL ciphertext payloads Signed-off-by: James Sadler --- CHANGELOG.md | 4 + packages/cipherstash-proxy/src/error.rs | 7 + .../src/postgresql/inbound_eql.rs | 159 ++++++++++++++++++ .../src/postgresql/middleware/frontend.rs | 136 +++++++++++---- .../cipherstash-proxy/src/postgresql/mod.rs | 1 + .../src/postgresql/rewrite/bind.rs | 47 +++++- 6 files changed, 319 insertions(+), 35 deletions(-) create mode 100644 packages/cipherstash-proxy/src/postgresql/inbound_eql.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 3bcde7a4..3ac7488c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ## [Unreleased] +### Added + +- **Pre-encrypted EQL values in statements**: SQL literals and bound parameters may now carry EQL v3 storage payloads produced by an application. Proxy validates their wire shape, version, destination column and required SEM terms, authenticates their ciphertext with the connection's active keyset, and forwards them without encrypting them again. Invalid payloads fail closed with one generic error so validation details cannot be used as an oracle. + ### Changed - **Upstream TLS verification for client traffic**: Connections with `with_tls_verification` enabled now use a cached snapshot of the system root certificates, loaded once when Proxy starts. Unlike Proxy's background database connections, pg-proto client-traffic connections do not apply operating-system revocation checks or enterprise verification policy. Restart Proxy after changing the system trust store. diff --git a/packages/cipherstash-proxy/src/error.rs b/packages/cipherstash-proxy/src/error.rs index 94463720..18bd9e42 100644 --- a/packages/cipherstash-proxy/src/error.rs +++ b/packages/cipherstash-proxy/src/error.rs @@ -75,6 +75,7 @@ impl Error { // stores plaintext in a column its operator believes is encrypted // (CIP-3688). No configuration may turn that back on. Error::Mapping(MappingError::UnmappableEncryptedColumn { .. }) + | Error::Encrypt(EncryptError::InvalidInboundCiphertext) ) } } @@ -255,6 +256,12 @@ pub enum TlsConfigError { #[derive(Error, Debug)] pub enum EncryptError { + /// Deliberately contains no payload or validation detail: inbound + /// ciphertext failures are attacker-controlled and detailed responses can + /// become an oracle. + #[error("Invalid encrypted value")] + InvalidInboundCiphertext, + #[error(transparent)] CiphertextCouldNotBeSerialised(#[from] serde_json::Error), diff --git a/packages/cipherstash-proxy/src/postgresql/inbound_eql.rs b/packages/cipherstash-proxy/src/postgresql/inbound_eql.rs new file mode 100644 index 00000000..3301b189 --- /dev/null +++ b/packages/cipherstash-proxy/src/postgresql/inbound_eql.rs @@ -0,0 +1,159 @@ +use crate::{error::EncryptError, postgresql::Column, EqlCiphertext}; +use cipherstash_client::{ + eql::{EncryptedPayloadV3, EQL_SCHEMA_VERSION_V3}, + schema::column::IndexType, +}; +use serde_json::Value; + +/// Parse a value only when it advertises itself as an EQL storage payload. +/// Ordinary JSON remains plaintext; malformed payload-shaped JSON fails closed. +pub fn parse(bytes: &[u8], column: &Column) -> Result, EncryptError> { + let Ok(value) = serde_json::from_slice::(bytes) else { + return Ok(None); + }; + let Some(object) = value.as_object() else { + return Ok(None); + }; + + let payload_shaped = object.contains_key("c") + || object.contains_key("h") + || object.contains_key("sv") && object.contains_key("i"); + if !payload_shaped { + return Ok(None); + } + + let ciphertext: EqlCiphertext = + serde_json::from_value(value).map_err(|_| EncryptError::InvalidInboundCiphertext)?; + validate_metadata(&ciphertext, column)?; + Ok(Some(ciphertext)) +} + +fn validate_metadata(ciphertext: &EqlCiphertext, column: &Column) -> Result<(), EncryptError> { + if ciphertext.version() != EQL_SCHEMA_VERSION_V3 + || ciphertext.identifier() != &column.identifier + { + return Err(EncryptError::InvalidInboundCiphertext); + } + + match ciphertext { + EqlCiphertext::Encrypted(payload) => validate_scalar_terms(payload, column), + EqlCiphertext::SteVec(payload) => { + let configured = column + .config + .indexes + .iter() + .any(|index| matches!(index.index_type, IndexType::SteVec { .. })); + if !configured || payload.ste_vec.is_empty() { + return Err(EncryptError::InvalidInboundCiphertext); + } + Ok(()) + } + } +} + +fn validate_scalar_terms( + payload: &EncryptedPayloadV3, + column: &Column, +) -> Result<(), EncryptError> { + let mut hmac = false; + let mut bloom = false; + let mut ore = false; + let mut ope = false; + for index in &column.config.indexes { + match index.index_type { + IndexType::Unique { .. } => hmac = true, + IndexType::Match { .. } => bloom = true, + IndexType::Ore => ore = true, + IndexType::Ope => ope = true, + IndexType::SteVec { .. } => return Err(EncryptError::InvalidInboundCiphertext), + } + } + + if payload.hmac_256.is_some() != hmac + || payload.bloom_filter.is_some() != bloom + || payload.ore_block_u64_8_256.is_some() != ore + || payload.ope_cllw.is_some() != ope + { + return Err(EncryptError::InvalidInboundCiphertext); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use cipherstash_client::schema::{ColumnConfig, ColumnMode, ColumnType}; + use cipherstash_client::zerokms::EncryptedRecord; + use eql_mapper::EqlTermVariant; + use uuid::Uuid; + + fn column() -> Column { + Column { + identifier: crate::Identifier::new("users", "email"), + config: ColumnConfig { + name: "email".into(), + in_place: true, + cast_type: ColumnType::Text, + indexes: vec![], + mode: ColumnMode::Encrypted, + }, + postgres_type: postgres_types::Type::TEXT, + eql_term: EqlTermVariant::Full, + } + } + + fn payload(identifier: crate::Identifier) -> EqlCiphertext { + EqlCiphertext::Encrypted(EncryptedPayloadV3 { + version: EQL_SCHEMA_VERSION_V3, + identifier, + ciphertext: EncryptedRecord { + iv: Default::default(), + ciphertext: vec![1; 16], + tag: vec![2; 16], + descriptor: "email".into(), + keyset_id: Some(Uuid::nil()), + decryption_policy: None, + }, + hmac_256: None, + bloom_filter: None, + ore_block_u64_8_256: None, + ope_cllw: None, + }) + } + + #[test] + fn ordinary_json_is_plaintext() { + assert!(parse(br#"{"name":"Ada"}"#, &column()).unwrap().is_none()); + } + + #[test] + fn malformed_payload_shape_fails_closed() { + assert!(matches!( + parse(br#"{"v":3,"i":"users.email","c":"bad"}"#, &column()), + Err(EncryptError::InvalidInboundCiphertext) + )); + } + + #[test] + fn destination_identifier_must_match() { + let ciphertext = payload(crate::Identifier::new("users", "phone")); + assert!(matches!( + validate_metadata(&ciphertext, &column()), + Err(EncryptError::InvalidInboundCiphertext) + )); + } + + #[test] + fn configured_sem_terms_must_be_present() { + let mut column = column(); + column + .config + .indexes + .push(cipherstash_client::schema::column::Index::new_unique()); + let ciphertext = payload(column.identifier.clone()); + assert!(matches!( + validate_metadata(&ciphertext, &column), + Err(EncryptError::InvalidInboundCiphertext) + )); + } +} diff --git a/packages/cipherstash-proxy/src/postgresql/middleware/frontend.rs b/packages/cipherstash-proxy/src/postgresql/middleware/frontend.rs index e63a1d01..24de2217 100644 --- a/packages/cipherstash-proxy/src/postgresql/middleware/frontend.rs +++ b/packages/cipherstash-proxy/src/postgresql/middleware/frontend.rs @@ -14,6 +14,7 @@ use crate::postgresql::context::Portal; use crate::postgresql::data::{ compose_json_selector_path, json_value_selector_plaintext, literal_from_sql, literal_json_value, }; +use crate::postgresql::inbound_eql; use crate::postgresql::rewrite::Name; use crate::postgresql::rewrite::UNSPECIFIED_TYPE_OID; use crate::prometheus::{ @@ -551,7 +552,18 @@ impl Frontend { return Ok(vec![]); } - let plaintexts = literals_to_plaintext(typed_statement, literal_columns)?; + let inbound = literal_values + .iter() + .zip(literal_columns) + .map(|((_, literal), column)| { + let (Some(column), Some(value)) = (column, (*literal).clone().into_string()) else { + return Ok(None); + }; + inbound_eql::parse(value.as_bytes(), column).map_err(Error::from) + }) + .collect::, Error>>()?; + let skip = inbound.iter().map(Option::is_some).collect::>(); + let plaintexts = literals_to_plaintext_skipping(typed_statement, literal_columns, &skip)?; let start = Instant::now(); @@ -563,6 +575,9 @@ impl Frontend { counter!(ENCRYPTION_ERROR_TOTAL).increment(1); })?; + self.authenticate_and_merge_inbound(&mut encrypted, inbound) + .await?; + for ((_, literal), encrypted) in literal_values.iter().zip(encrypted.iter_mut()) { project_query_operand( typed_statement.query_operands.contains_literal(literal), @@ -1066,8 +1081,13 @@ impl Frontend { bind: &Bind, statement: &Statement, ) -> Result>, Error> { - let plaintexts = - bind.to_plaintext(&statement.output_params, &statement.postgres_param_types)?; + let inbound = bind.inbound_ciphertexts(&statement.output_params)?; + let skip = inbound.iter().map(Option::is_some).collect::>(); + let plaintexts = bind.to_plaintext_skipping( + &statement.output_params, + &statement.postgres_param_types, + &skip, + )?; // Encryption is positional over the OUTPUT params — the values actually // sent — not over what the client bound. @@ -1089,6 +1109,9 @@ impl Frontend { counter!(ENCRYPTION_ERROR_TOTAL).increment(1); })?; + self.authenticate_and_merge_inbound(&mut encrypted, inbound) + .await?; + for (output, encrypted) in statement.output_params.iter().zip(encrypted.iter_mut()) { project_query_operand(output.query_operand, encrypted); } @@ -1115,6 +1138,37 @@ impl Frontend { Ok(encrypted) } + /// Authenticate inbound ciphertext with this connection's scoped cipher. + /// Any parse, metadata, key or AEAD failure is collapsed to one response. + async fn authenticate_and_merge_inbound( + &self, + encrypted: &mut [Option], + inbound: Vec>, + ) -> Result<(), Error> { + let positions = inbound + .iter() + .enumerate() + .filter_map(|(index, ciphertext)| ciphertext.as_ref().map(|ct| (index, ct.clone()))) + .collect::>(); + if positions.is_empty() { + return Ok(()); + } + + let ciphertexts = positions + .iter() + .map(|(_, ciphertext)| Some(ciphertext.clone())) + .collect(); + self.context + .decrypt(ciphertexts) + .await + .map_err(|_| EncryptError::InvalidInboundCiphertext)?; + + for (index, ciphertext) in positions { + encrypted[index] = Some(EqlOutput::Store(ciphertext)); + } + Ok(()) + } + fn type_check<'a>( &self, statement: &'a ast::Statement, @@ -1230,45 +1284,59 @@ fn project_query_operand(query_operand: bool, encrypted: &mut Option) fn literals_to_plaintext( typed_statement: &TypeCheckedStatement<'_>, literal_columns: &Vec>, +) -> Result>, Error> { + literals_to_plaintext_skipping(typed_statement, literal_columns, &[]) +} + +fn literals_to_plaintext_skipping( + typed_statement: &TypeCheckedStatement<'_>, + literal_columns: &Vec>, + skip: &[bool], ) -> Result>, Error> { let literals = typed_statement.literal_values(); let plaintexts = literals .iter() .zip(literal_columns) - .map(|((eql_term, val), col)| match col { - Some(col) => { - let plaintext = match eql_term.variant() { - EqlTermVariant::JsonValueSelector => { - json_value_selector_literal_plaintext(typed_statement, val) - } - // A selector that carries a collapsed chain keys the composed - // path, not the one segment it spells. Only a selector the - // mapper recorded a chain for: a single access has no record - // and takes the ordinary single-segment route below. - EqlTermVariant::JsonAccessor - if typed_statement - .json_accessor_paths - .for_literal(val) - .is_some() => - { - json_accessor_path_literal_plaintext(typed_statement, val) - } - _ => literal_from_sql(val, col.eql_term(), col.cast_type()), - }; + .enumerate() + .map(|(index, ((eql_term, val), col))| { + if skip.get(index).copied().unwrap_or(false) { + return Ok(None); + } + match col { + Some(col) => { + let plaintext = match eql_term.variant() { + EqlTermVariant::JsonValueSelector => { + json_value_selector_literal_plaintext(typed_statement, val) + } + // A selector that carries a collapsed chain keys the composed + // path, not the one segment it spells. Only a selector the + // mapper recorded a chain for: a single access has no record + // and takes the ordinary single-segment route below. + EqlTermVariant::JsonAccessor + if typed_statement + .json_accessor_paths + .for_literal(val) + .is_some() => + { + json_accessor_path_literal_plaintext(typed_statement, val) + } + _ => literal_from_sql(val, col.eql_term(), col.cast_type()), + }; - plaintext.map_err(|err| { - debug!( - target: MAPPER, - msg = "Could not convert literal value", - value = ?val, - cast_type = ?col.cast_type(), - error = err.to_string() - ); - MappingError::InvalidParameter(Box::new(col.to_owned())).into() - }) + plaintext.map_err(|err| { + debug!( + target: MAPPER, + msg = "Could not convert literal value", + value = ?val, + cast_type = ?col.cast_type(), + error = err.to_string() + ); + MappingError::InvalidParameter(Box::new(col.to_owned())).into() + }) + } + None => Ok(None), } - None => Ok(None), }) .collect::, Error>>()?; Ok(plaintexts) diff --git a/packages/cipherstash-proxy/src/postgresql/mod.rs b/packages/cipherstash-proxy/src/postgresql/mod.rs index a3995b97..c3751ba1 100644 --- a/packages/cipherstash-proxy/src/postgresql/mod.rs +++ b/packages/cipherstash-proxy/src/postgresql/mod.rs @@ -5,6 +5,7 @@ mod diagnostics; mod driver; mod error_handler; mod format_code; +mod inbound_eql; mod middleware; mod parser; mod rewrite; diff --git a/packages/cipherstash-proxy/src/postgresql/rewrite/bind.rs b/packages/cipherstash-proxy/src/postgresql/rewrite/bind.rs index ce151567..6561d301 100644 --- a/packages/cipherstash-proxy/src/postgresql/rewrite/bind.rs +++ b/packages/cipherstash-proxy/src/postgresql/rewrite/bind.rs @@ -11,6 +11,7 @@ use crate::postgresql::data::{ json_value_selector_plaintext, }; use crate::postgresql::format_code::FormatCode; +use crate::postgresql::inbound_eql; use crate::{EqlOutput, EqlQueryPayload}; use bytes::{BufMut, BytesMut}; use cipherstash_client::encryption::Plaintext; @@ -64,10 +65,23 @@ impl Bind { &self, output_params: &[OutputParam], param_types: &[i32], + ) -> Result>, Error> { + self.to_plaintext_skipping(output_params, param_types, &[]) + } + + pub fn to_plaintext_skipping( + &self, + output_params: &[OutputParam], + param_types: &[i32], + skip: &[bool], ) -> Result>, Error> { output_params .iter() - .map(|output| { + .enumerate() + .map(|(output_index, output)| { + if skip.get(output_index).copied().unwrap_or(false) { + return Ok(None); + } let Some(col) = &output.column else { // Native param: forwarded verbatim, nothing to encrypt. return Ok(None); @@ -110,6 +124,37 @@ impl Bind { .collect() } + /// Detect already-encrypted storage payloads before decoding parameters as + /// their configured plaintext PostgreSQL types. + pub fn inbound_ciphertexts( + &self, + output_params: &[OutputParam], + ) -> Result>, Error> { + output_params + .iter() + .map(|output| { + let Some(column) = &output.column else { + return Ok(None); + }; + let OutputParamSource::Input(input) = output.source else { + return Ok(None); + }; + let Some(param) = self.param_values.get(input) else { + return Ok(None); + }; + if param.is_null() { + return Ok(None); + } + let bytes = if param.is_binary() && param.bytes.first() == Some(&1) { + param.json_bytes() + } else { + ¶m.bytes + }; + inbound_eql::parse(bytes, column).map_err(Error::from) + }) + .collect() + } + /// Composes `{"path", "value"}` — the input to `SteVecValueSelector` — from /// the operands of a JSON field equality. /// From 7df197e26bed7ba1f5ba9c50d2921cdde6a1c47f Mon Sep 17 00:00:00 2001 From: James Sadler Date: Wed, 19 Aug 2026 16:17:17 +1000 Subject: [PATCH 02/11] test(proxy): cover inbound EQL payloads end to end Signed-off-by: James Sadler --- Cargo.lock | 2 + .../src/inbound_ciphertext.rs | 140 ++++++++++++++++++ .../cipherstash-proxy-integration/src/lib.rs | 1 + packages/showcase/Cargo.toml | 2 + packages/showcase/README.md | 26 +++- packages/showcase/src/main.rs | 3 + packages/showcase/src/pre_encrypted.rs | 112 ++++++++++++++ 7 files changed, 279 insertions(+), 7 deletions(-) create mode 100644 packages/cipherstash-proxy-integration/src/inbound_ciphertext.rs create mode 100644 packages/showcase/src/pre_encrypted.rs diff --git a/Cargo.lock b/Cargo.lock index cab338f9..6d3f842e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4387,6 +4387,8 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" name = "showcase" version = "3.0.1" dependencies = [ + "cipherstash-client", + "cipherstash-config", "rand 0.9.2", "rustls", "serde", diff --git a/packages/cipherstash-proxy-integration/src/inbound_ciphertext.rs b/packages/cipherstash-proxy-integration/src/inbound_ciphertext.rs new file mode 100644 index 00000000..ae42d0bc --- /dev/null +++ b/packages/cipherstash-proxy-integration/src/inbound_ciphertext.rs @@ -0,0 +1,140 @@ +//! End-to-end coverage for application-encrypted EQL payloads entering Proxy. + +#[cfg(test)] +mod tests { + use crate::common::{clear_with_client, connect_with_tls, random_id, PROXY}; + use cipherstash_client::{ + encryption::{Plaintext, ScopedCipher}, + eql::{ + encrypt_eql_v3, EqlEncryptOpts, EqlOperation, EqlOutputV3, Identifier, + PreparedPlaintext, + }, + schema::{column::Index, ColumnConfig, ColumnType}, + zerokms::{ClientKey, ZeroKMSBuilder}, + AutoStrategy, IdentifiedBy, + }; + use std::{borrow::Cow, sync::Arc}; + use uuid::Uuid; + + async fn cipher() -> Arc> { + let client_id = env("CS_CLIENT_ID", "CS_ENCRYPT__CLIENT_ID") + .parse() + .expect("CS_CLIENT_ID must be a UUID"); + let client_key = + ClientKey::from_hex_v1(client_id, &env("CS_CLIENT_KEY", "CS_ENCRYPT__CLIENT_KEY")) + .expect("CS_CLIENT_KEY must be valid"); + let zerokms = ZeroKMSBuilder::auto() + .expect("ZeroKMS credentials must be configured") + .with_client_key(client_key) + .build() + .expect("ZeroKMS client must initialize"); + let keyset_id: Uuid = env("CS_DEFAULT_KEYSET_ID", "CS_ENCRYPT__DEFAULT_KEYSET_ID") + .parse() + .expect("CS_DEFAULT_KEYSET_ID must be a UUID"); + Arc::new( + ScopedCipher::init(Arc::new(zerokms), Some(IdentifiedBy::Uuid(keyset_id))) + .await + .expect("scoped cipher must initialize"), + ) + } + + fn env(primary: &str, nested: &str) -> String { + std::env::var(primary) + .or_else(|_| std::env::var(nested)) + .unwrap_or_else(|_| panic!("{primary} must be configured")) + } + + fn text_search_config(column: &str) -> ColumnConfig { + ColumnConfig::build(column) + .casts_as(ColumnType::Text) + .add_index(Index::new_unique()) + .add_index(Index::new_ope()) + .add_index(Index::new_match()) + } + + async fn encrypt_text(table: &str, column: &str, plaintext: &str) -> String { + let prepared = PreparedPlaintext::new( + Cow::Owned(text_search_config(column)), + Identifier::new(table, column), + Plaintext::from(plaintext), + EqlOperation::Store, + ); + let mut outputs = + encrypt_eql_v3(cipher().await, vec![prepared], &EqlEncryptOpts::default()) + .await + .expect("application-side encryption must succeed"); + let EqlOutputV3::Store(ciphertext) = outputs.remove(0) else { + panic!("store encryption must return a stored payload"); + }; + serde_json::to_string(&ciphertext).unwrap() + } + + #[tokio::test] + async fn accepts_pre_encrypted_parameter_for_storage_and_search() { + let client = connect_with_tls(*PROXY).await; + clear_with_client(&client).await; + let id = random_id(); + let plaintext = "encrypted in the application"; + let payload = encrypt_text("encrypted", "encrypted_text", plaintext).await; + + client + .execute( + "INSERT INTO encrypted (id, encrypted_text) VALUES ($1, $2)", + &[&id, &payload], + ) + .await + .unwrap(); + + let rows = client + .query( + "SELECT encrypted_text FROM encrypted WHERE encrypted_text = $1", + &[&payload], + ) + .await + .unwrap(); + assert_eq!(rows[0].get::<_, String>(0), plaintext); + } + + #[tokio::test] + async fn accepts_pre_encrypted_literal_for_storage() { + let client = connect_with_tls(*PROXY).await; + clear_with_client(&client).await; + let id = random_id(); + let plaintext = "application encrypted literal"; + let payload = encrypt_text("encrypted", "encrypted_text", plaintext).await; + let payload = payload.replace('\'', "''"); + + client + .simple_query(&format!( + "INSERT INTO encrypted (id, encrypted_text) VALUES ({id}, '{payload}')" + )) + .await + .unwrap(); + + let row = client + .query_one("SELECT encrypted_text FROM encrypted WHERE id = $1", &[&id]) + .await + .unwrap(); + assert_eq!(row.get::<_, String>(0), plaintext); + } + + #[tokio::test] + async fn rejects_payload_for_a_different_destination_with_generic_error() { + let client = connect_with_tls(*PROXY).await; + clear_with_client(&client).await; + let id = random_id(); + let payload = encrypt_text("some_other_table", "encrypted_text", "secret").await; + + let error = client + .execute( + "INSERT INTO encrypted (id, encrypted_text) VALUES ($1, $2)", + &[&id, &payload], + ) + .await + .expect_err("destination mismatch must fail closed"); + assert_eq!( + error.as_db_error().unwrap().message(), + "Invalid encrypted value" + ); + } +} diff --git a/packages/cipherstash-proxy-integration/src/lib.rs b/packages/cipherstash-proxy-integration/src/lib.rs index 756ab8d2..d2a713d5 100644 --- a/packages/cipherstash-proxy-integration/src/lib.rs +++ b/packages/cipherstash-proxy-integration/src/lib.rs @@ -7,6 +7,7 @@ mod empty_result; mod encryption_sanity; mod eql_regression; mod extended_protocol_error_messages; +mod inbound_ciphertext; mod insert; mod legacy_v2_column; mod map_concat; diff --git a/packages/showcase/Cargo.toml b/packages/showcase/Cargo.toml index 5881d1ad..d8a7f124 100644 --- a/packages/showcase/Cargo.toml +++ b/packages/showcase/Cargo.toml @@ -5,6 +5,8 @@ edition.workspace = true description = "Healthcare data model demonstrating EQL v3 searchable encryption with realistic encrypted application patterns" [dependencies] +cipherstash-client = { workspace = true, features = ["tokio"] } +cipherstash-config = { workspace = true } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" uuid = { version = "1.11.0", features = ["serde", "v4"] } diff --git a/packages/showcase/README.md b/packages/showcase/README.md index 6dae109d..7394425e 100644 --- a/packages/showcase/README.md +++ b/packages/showcase/README.md @@ -466,12 +466,24 @@ mise run test:integration:showcase The showcase will execute and display: -1. **Original Healthcare Query**: Aspirin prescription lookup -2. **Field Access Operations**: Testing `->` and `->>` -3. **Containment Operations**: Testing `@>` and `<@` -4. **JSONPath Functions**: Testing `jsonb_path_*` functions -5. **Comparison Operations**: Numeric, string, date, and float comparisons -6. **Complex Nested Queries**: JOINs, aggregations, and subqueries +1. **Application-side Encryption**: Insert pre-encrypted EQL payloads as a bound parameter and a SQL literal +2. **Original Healthcare Query**: Aspirin prescription lookup +3. **Field Access Operations**: Testing `->` and `->>` +4. **Containment Operations**: Testing `@>` and `<@` +5. **JSONPath Functions**: Testing `jsonb_path_*` functions +6. **Comparison Operations**: Numeric, string, date, and float comparisons +7. **Complex Nested Queries**: JOINs, aggregations, and subqueries + +### Application-side Encryption + +The `pre_encrypted` example constructs the same `eql_v3_json_search` column configuration declared by `patients.pii`, encrypts patient PII with `cipherstash-client`, and sends the resulting EQL payload through Proxy. It demonstrates both supported input forms: + +```sql +INSERT INTO patients (id, pii) VALUES ($1, $2); -- payload parameter +INSERT INTO patients (id, pii) VALUES ('...', '{...}'); -- payload literal +``` + +Proxy parses and authenticates each payload, checks that its identifier and SEM shape match `patients.pii`, and forwards it without double encryption. Selecting the rows through Proxy returns the original plaintext JSON. Each test section provides detailed output showing: - ✅ Successful query execution @@ -499,4 +511,4 @@ Examples: ⚠️ **Chained Operators**: The `->` operator cannot be chained on `ste_vec` encrypted columns. Use JSONPath functions like `jsonb_path_query_first()` for deep nested access instead. -This showcase proves that EQL v3 provides comprehensive JSONB support for encrypted data, enabling sophisticated healthcare applications while maintaining strong privacy protections. \ No newline at end of file +This showcase proves that EQL v3 provides comprehensive JSONB support for encrypted data, enabling sophisticated healthcare applications while maintaining strong privacy protections. diff --git a/packages/showcase/src/main.rs b/packages/showcase/src/main.rs index 9cc67097..f6c76226 100644 --- a/packages/showcase/src/main.rs +++ b/packages/showcase/src/main.rs @@ -53,6 +53,7 @@ mod common; mod data; mod model; +mod pre_encrypted; mod schema; use common::{connect_with_tls, trace, PROXY}; @@ -75,6 +76,7 @@ async fn main() -> Result<(), Box> { setup_schema().await; insert_test_data().await; create_enhanced_jsonb_test_data().await; + pre_encrypted::run_examples().await?; let client = connect_with_tls(*PROXY).await; @@ -156,6 +158,7 @@ async fn main() -> Result<(), Box> { println!(" • Healthcare-compliant database schema with proper foreign keys"); println!(" • Realistic medical data with nested objects, arrays, and mixed data types"); println!(" • Secure querying of encrypted data while maintaining privacy"); + println!(" • Application-side encryption passed through Proxy as parameters and literals"); println!(); println!("✨ EQL v3 provides comprehensive JSONB support for encrypted healthcare data!"); diff --git a/packages/showcase/src/pre_encrypted.rs b/packages/showcase/src/pre_encrypted.rs new file mode 100644 index 00000000..d4c65061 --- /dev/null +++ b/packages/showcase/src/pre_encrypted.rs @@ -0,0 +1,112 @@ +//! Application-side encryption examples for Stash-style ingestion. +//! +//! Proxy accepts the resulting EQL storage payload as either a bound parameter +//! or a SQL literal, authenticates it, and avoids encrypting it a second time. + +use crate::common::{connect_with_tls, PROXY}; +use cipherstash_client::{ + encryption::{Plaintext, ScopedCipher}, + eql::{ + encrypt_eql_v3, EqlEncryptOpts, EqlOperation, EqlOutputV3, Identifier, PreparedPlaintext, + }, + schema::{ColumnConfig, ColumnType}, + zerokms::{ClientKey, ZeroKMSBuilder}, + AutoStrategy, IdentifiedBy, +}; +use cipherstash_config::column::{ArrayIndexMode, Index, IndexType, SteVecMode}; +use serde_json::{json, Value}; +use std::{borrow::Cow, sync::Arc}; +use uuid::Uuid; + +pub async fn run_examples() -> Result<(), Box> { + println!("\n🔐 === Application-side EQL encryption ==="); + let client = connect_with_tls(*PROXY).await; + + // Example 1: bind an application-encrypted payload as a parameter. + let parameter_id = Uuid::parse_str("a1b2c3d4-e5f6-4a5b-8c9d-123456789021")?; + let parameter_pii = json!({ + "first_name": "Ada", + "last_name": "Lovelace", + "email": "ada@example.com", + "date_of_birth": "1815-12-10" + }); + let parameter_payload = encrypt_patient_pii(parameter_pii.clone()).await?; + client + .execute( + "INSERT INTO patients (id, pii) VALUES ($1, $2)", + &[¶meter_id, ¶meter_payload], + ) + .await?; + println!("✅ Inserted application-encrypted PII as a bound parameter"); + + // Example 2: the same wire payload can be supplied as a SQL literal. + let literal_id = Uuid::parse_str("a1b2c3d4-e5f6-4a5b-8c9d-123456789022")?; + let literal_pii = json!({ + "first_name": "Grace", + "last_name": "Hopper", + "email": "grace@example.com", + "date_of_birth": "1906-12-09" + }); + let literal_payload = encrypt_patient_pii(literal_pii.clone()).await?; + let literal_payload = literal_payload.to_string().replace('\'', "''"); + client + .simple_query(&format!( + "INSERT INTO patients (id, pii) VALUES ('{literal_id}', '{literal_payload}')" + )) + .await?; + println!("✅ Inserted application-encrypted PII as a SQL literal"); + + // Both rows still decrypt normally when selected through Proxy. + for (id, expected) in [(parameter_id, parameter_pii), (literal_id, literal_pii)] { + let row = client + .query_one("SELECT pii FROM patients WHERE id = $1", &[&id]) + .await?; + assert_eq!(row.get::<_, Value>(0), expected); + } + println!("✅ Proxy authenticated and decrypted both application-encrypted values"); + Ok(()) +} + +async fn encrypt_patient_pii(value: Value) -> Result> { + let config = ColumnConfig::build("pii") + .casts_as(ColumnType::Json) + .add_index(Index::new(IndexType::SteVec { + prefix: "patients/pii".into(), + term_filters: Vec::new(), + array_index_mode: ArrayIndexMode::ALL, + mode: SteVecMode::default(), + })); + let prepared = PreparedPlaintext::new( + Cow::Owned(config), + Identifier::new("patients", "pii"), + Plaintext::Json(Some(value)), + EqlOperation::Store, + ); + let mut outputs = encrypt_eql_v3( + scoped_cipher().await?, + vec![prepared], + &EqlEncryptOpts::default(), + ) + .await?; + let EqlOutputV3::Store(ciphertext) = outputs.remove(0) else { + return Err("store encryption returned a query payload".into()); + }; + Ok(serde_json::to_value(ciphertext)?) +} + +async fn scoped_cipher() -> Result>, Box> { + let client_id = env("CS_CLIENT_ID", "CS_ENCRYPT__CLIENT_ID")?.parse()?; + let client_key = + ClientKey::from_hex_v1(client_id, &env("CS_CLIENT_KEY", "CS_ENCRYPT__CLIENT_KEY")?)?; + let zerokms = ZeroKMSBuilder::auto()? + .with_client_key(client_key) + .build()?; + let keyset_id: Uuid = env("CS_DEFAULT_KEYSET_ID", "CS_ENCRYPT__DEFAULT_KEYSET_ID")?.parse()?; + Ok(Arc::new( + ScopedCipher::init(Arc::new(zerokms), Some(IdentifiedBy::Uuid(keyset_id))).await?, + )) +} + +fn env(primary: &str, nested: &str) -> Result { + std::env::var(primary).or_else(|_| std::env::var(nested)) +} From 74c91b48811fbf985f3270952a52ac2f0357fddb Mon Sep 17 00:00:00 2001 From: James Sadler Date: Thu, 20 Aug 2026 10:51:05 +1000 Subject: [PATCH 03/11] fix(proxy): authenticate inbound EQL metadata Signed-off-by: James Sadler --- CHANGELOG.md | 2 +- .../src/inbound_ciphertext.rs | 42 ++++++++++- .../src/postgresql/inbound_eql.rs | 71 +++++++++++++++++-- .../src/postgresql/middleware/frontend.rs | 67 ++++++++++++++--- packages/showcase/README.md | 4 +- packages/showcase/src/pre_encrypted.rs | 2 +- 6 files changed, 165 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ac7488c..8fa36c3a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Added -- **Pre-encrypted EQL values in statements**: SQL literals and bound parameters may now carry EQL v3 storage payloads produced by an application. Proxy validates their wire shape, version, destination column and required SEM terms, authenticates their ciphertext with the connection's active keyset, and forwards them without encrypting them again. Invalid payloads fail closed with one generic error so validation details cannot be used as an oracle. +- **Pre-encrypted EQL values in statements**: SQL literals and bound parameters may now carry EQL v3 storage payloads produced by an application. Proxy validates their wire shape and version, requires the authenticated ciphertext descriptor to name the inferred destination column, authenticates the ciphertext with the connection's active keyset, and independently re-derives every SEM term from the plaintext before forwarding it without encrypting it again. Invalid payloads fail closed with one generic error so validation details cannot be used as an oracle. ### Changed diff --git a/packages/cipherstash-proxy-integration/src/inbound_ciphertext.rs b/packages/cipherstash-proxy-integration/src/inbound_ciphertext.rs index ae42d0bc..f64eeae3 100644 --- a/packages/cipherstash-proxy-integration/src/inbound_ciphertext.rs +++ b/packages/cipherstash-proxy-integration/src/inbound_ciphertext.rs @@ -44,8 +44,8 @@ mod tests { .unwrap_or_else(|_| panic!("{primary} must be configured")) } - fn text_search_config(column: &str) -> ColumnConfig { - ColumnConfig::build(column) + fn text_search_config(table: &str, column: &str) -> ColumnConfig { + ColumnConfig::build(format!("{table}/{column}")) .casts_as(ColumnType::Text) .add_index(Index::new_unique()) .add_index(Index::new_ope()) @@ -54,7 +54,7 @@ mod tests { async fn encrypt_text(table: &str, column: &str, plaintext: &str) -> String { let prepared = PreparedPlaintext::new( - Cow::Owned(text_search_config(column)), + Cow::Owned(text_search_config(table, column)), Identifier::new(table, column), Plaintext::from(plaintext), EqlOperation::Store, @@ -124,6 +124,9 @@ mod tests { clear_with_client(&client).await; let id = random_id(); let payload = encrypt_text("some_other_table", "encrypted_text", "secret").await; + let mut payload: serde_json::Value = serde_json::from_str(&payload).unwrap(); + payload["i"]["t"] = "encrypted".into(); + let payload = payload.to_string(); let error = client .execute( @@ -137,4 +140,37 @@ mod tests { "Invalid encrypted value" ); } + + #[tokio::test] + async fn rejects_sem_terms_spliced_from_another_plaintext() { + let client = connect_with_tls(*PROXY).await; + clear_with_client(&client).await; + let id = random_id(); + let x: serde_json::Value = serde_json::from_str( + &encrypt_text("encrypted", "encrypted_text", "indexed as x").await, + ) + .unwrap(); + let mut y: serde_json::Value = serde_json::from_str( + &encrypt_text("encrypted", "encrypted_text", "decrypts as y").await, + ) + .unwrap(); + for term in ["hm", "bf", "ob", "op"] { + if let Some(value) = x.get(term) { + y[term] = value.clone(); + } + } + let payload = y.to_string(); + + let error = client + .execute( + "INSERT INTO encrypted (id, encrypted_text) VALUES ($1, $2)", + &[&id, &payload], + ) + .await + .expect_err("spliced SEM terms must fail closed"); + assert_eq!( + error.as_db_error().unwrap().message(), + "Invalid encrypted value" + ); + } } diff --git a/packages/cipherstash-proxy/src/postgresql/inbound_eql.rs b/packages/cipherstash-proxy/src/postgresql/inbound_eql.rs index 3301b189..5ebd95e1 100644 --- a/packages/cipherstash-proxy/src/postgresql/inbound_eql.rs +++ b/packages/cipherstash-proxy/src/postgresql/inbound_eql.rs @@ -5,8 +5,9 @@ use cipherstash_client::{ }; use serde_json::Value; -/// Parse a value only when it advertises itself as an EQL storage payload. -/// Ordinary JSON remains plaintext; malformed payload-shaped JSON fails closed. +/// Parse a value only when its version, identifier, and storage fields advertise +/// it as an EQL payload. Ordinary JSON (including an object with a `c` key) +/// remains plaintext; malformed advertised payloads fail closed. pub fn parse(bytes: &[u8], column: &Column) -> Result, EncryptError> { let Ok(value) = serde_json::from_slice::(bytes) else { return Ok(None); @@ -15,9 +16,9 @@ pub fn parse(bytes: &[u8], column: &Column) -> Result, Enc return Ok(None); }; - let payload_shaped = object.contains_key("c") - || object.contains_key("h") - || object.contains_key("sv") && object.contains_key("i"); + let payload_shaped = object.contains_key("v") + && object.contains_key("i") + && (object.contains_key("c") || object.contains_key("h") || object.contains_key("sv")); if !payload_shaped { return Ok(None); } @@ -35,6 +36,18 @@ fn validate_metadata(ciphertext: &EqlCiphertext, column: &Column) -> Result<(), return Err(EncryptError::InvalidInboundCiphertext); } + // The descriptor is covered by the encrypted record's AEAD tag. Requiring + // the canonical table/column descriptor cryptographically binds a payload + // to its claimed destination, unlike the self-reported `i` field alone. + let expected_descriptor = format!("{}/{}", column.identifier.table, column.identifier.column); + let descriptor = match ciphertext { + EqlCiphertext::Encrypted(payload) => &payload.ciphertext.descriptor, + EqlCiphertext::SteVec(payload) => &payload.key_header.descriptor, + }; + if descriptor != &expected_descriptor { + return Err(EncryptError::InvalidInboundCiphertext); + } + match ciphertext { EqlCiphertext::Encrypted(payload) => validate_scalar_terms(payload, column), EqlCiphertext::SteVec(payload) => { @@ -51,6 +64,20 @@ fn validate_metadata(ciphertext: &EqlCiphertext, column: &Column) -> Result<(), } } +/// Compare all searchable metadata after the plaintext has been authenticated +/// and independently re-encrypted for the inferred destination column. +/// `into_query_operand` removes only record ciphertext/key material, leaving +/// the identifier and every scalar or SteVec SEM term for an exact comparison. +pub fn sem_terms_match(inbound: &EqlCiphertext, derived: EqlCiphertext) -> bool { + match ( + serde_json::to_value(inbound.clone().into_query_operand()), + serde_json::to_value(derived.into_query_operand()), + ) { + (Ok(inbound), Ok(derived)) => inbound == derived, + _ => false, + } +} + fn validate_scalar_terms( payload: &EncryptedPayloadV3, column: &Column, @@ -110,7 +137,7 @@ mod tests { iv: Default::default(), ciphertext: vec![1; 16], tag: vec![2; 16], - descriptor: "email".into(), + descriptor: "users/email".into(), keyset_id: Some(Uuid::nil()), decryption_policy: None, }, @@ -126,6 +153,13 @@ mod tests { assert!(parse(br#"{"name":"Ada"}"#, &column()).unwrap().is_none()); } + #[test] + fn ordinary_json_with_a_c_key_is_plaintext() { + assert!(parse(br#"{"c":"customer code"}"#, &column()) + .unwrap() + .is_none()); + } + #[test] fn malformed_payload_shape_fails_closed() { assert!(matches!( @@ -143,6 +177,19 @@ mod tests { )); } + #[test] + fn authenticated_descriptor_must_match_destination() { + let mut ciphertext = payload(crate::Identifier::new("users", "email")); + let EqlCiphertext::Encrypted(payload) = &mut ciphertext else { + unreachable!() + }; + payload.ciphertext.descriptor = "accounts/email".into(); + assert!(matches!( + validate_metadata(&ciphertext, &column()), + Err(EncryptError::InvalidInboundCiphertext) + )); + } + #[test] fn configured_sem_terms_must_be_present() { let mut column = column(); @@ -156,4 +203,16 @@ mod tests { Err(EncryptError::InvalidInboundCiphertext) )); } + + #[test] + fn independently_derived_sem_terms_must_match() { + let derived = payload(crate::Identifier::new("users", "email")); + let mut spliced = derived.clone(); + let EqlCiphertext::Encrypted(payload) = &mut spliced else { + unreachable!() + }; + payload.hmac_256 = Some("term from another plaintext".into()); + + assert!(!sem_terms_match(&spliced, derived)); + } } diff --git a/packages/cipherstash-proxy/src/postgresql/middleware/frontend.rs b/packages/cipherstash-proxy/src/postgresql/middleware/frontend.rs index 24de2217..cf39dea5 100644 --- a/packages/cipherstash-proxy/src/postgresql/middleware/frontend.rs +++ b/packages/cipherstash-proxy/src/postgresql/middleware/frontend.rs @@ -4,7 +4,7 @@ use super::super::error_handler::PostgreSqlErrorHandler; use super::super::parser::SqlParser; use super::super::rewrite::bind::Bind; use crate::error::{EncryptError, Error, MappingError}; -use crate::log::{MAPPER, PROTOCOL}; +use crate::log::{ENCRYPT, MAPPER, PROTOCOL}; use crate::postgresql::context::column::Column; use crate::postgresql::context::statement::{ output_params_from_plan, OutputParam, OutputParamSource, @@ -575,7 +575,7 @@ impl Frontend { counter!(ENCRYPTION_ERROR_TOTAL).increment(1); })?; - self.authenticate_and_merge_inbound(&mut encrypted, inbound) + self.authenticate_and_merge_inbound(&mut encrypted, inbound, literal_columns) .await?; for ((_, literal), encrypted) in literal_values.iter().zip(encrypted.iter_mut()) { @@ -1109,7 +1109,7 @@ impl Frontend { counter!(ENCRYPTION_ERROR_TOTAL).increment(1); })?; - self.authenticate_and_merge_inbound(&mut encrypted, inbound) + self.authenticate_and_merge_inbound(&mut encrypted, inbound, &output_param_columns) .await?; for (output, encrypted) in statement.output_params.iter().zip(encrypted.iter_mut()) { @@ -1144,11 +1144,15 @@ impl Frontend { &self, encrypted: &mut [Option], inbound: Vec>, + columns: &[Option], ) -> Result<(), Error> { let positions = inbound - .iter() + .into_iter() + .zip(columns) .enumerate() - .filter_map(|(index, ciphertext)| ciphertext.as_ref().map(|ct| (index, ct.clone()))) + .filter_map(|(index, (ciphertext, column))| { + Some((index, ciphertext?, column.as_ref()?.clone())) + }) .collect::>(); if positions.is_empty() { return Ok(()); @@ -1156,14 +1160,57 @@ impl Frontend { let ciphertexts = positions .iter() - .map(|(_, ciphertext)| Some(ciphertext.clone())) + .map(|(_, ciphertext, _)| Some(ciphertext.clone())) .collect(); - self.context - .decrypt(ciphertexts) + let plaintexts = self.context.decrypt(ciphertexts).await.map_err(|err| { + warn!( + target: ENCRYPT, + client_id = self.context.client_id, + msg = "Inbound EQL ciphertext authentication failed", + error = ?err, + ); + EncryptError::InvalidInboundCiphertext + })?; + + // Re-encrypt the authenticated plaintext for the inferred destination + // and compare every derived SEM term. This detects term splicing: the + // AEAD tag authenticates `c`, but the searchable metadata sits outside + // it in the EQL envelope. + let verification_columns = positions + .iter() + .map(|(_, _, column)| Some(column.clone())) + .collect::>(); + let derived = self + .context + .encrypt(plaintexts, &verification_columns) .await - .map_err(|_| EncryptError::InvalidInboundCiphertext)?; + .map_err(|err| { + warn!( + target: ENCRYPT, + client_id = self.context.client_id, + msg = "Inbound EQL ciphertext metadata verification failed", + error = ?err, + ); + EncryptError::InvalidInboundCiphertext + })?; - for (index, ciphertext) in positions { + for ((index, ciphertext, _), derived) in positions.into_iter().zip(derived) { + let Some(EqlOutput::Store(derived)) = derived else { + warn!( + target: ENCRYPT, + client_id = self.context.client_id, + msg = "Inbound EQL ciphertext SEM terms did not match plaintext", + ); + return Err(EncryptError::InvalidInboundCiphertext.into()); + }; + if !inbound_eql::sem_terms_match(&ciphertext, derived) { + warn!( + target: ENCRYPT, + client_id = self.context.client_id, + msg = "Inbound EQL ciphertext SEM terms did not match plaintext", + ); + return Err(EncryptError::InvalidInboundCiphertext.into()); + } encrypted[index] = Some(EqlOutput::Store(ciphertext)); } Ok(()) diff --git a/packages/showcase/README.md b/packages/showcase/README.md index 7394425e..996898e8 100644 --- a/packages/showcase/README.md +++ b/packages/showcase/README.md @@ -476,14 +476,14 @@ The showcase will execute and display: ### Application-side Encryption -The `pre_encrypted` example constructs the same `eql_v3_json_search` column configuration declared by `patients.pii`, encrypts patient PII with `cipherstash-client`, and sends the resulting EQL payload through Proxy. It demonstrates both supported input forms: +The `pre_encrypted` example constructs the same `eql_v3_json_search` column configuration declared by `patients.pii`, encrypts patient PII with `cipherstash-client`, and sends the resulting EQL payload through Proxy. The application-side `ColumnConfig` uses the canonical `patients/pii` descriptor; this authenticated descriptor binds the ciphertext to its destination. The example demonstrates both supported input forms: ```sql INSERT INTO patients (id, pii) VALUES ($1, $2); -- payload parameter INSERT INTO patients (id, pii) VALUES ('...', '{...}'); -- payload literal ``` -Proxy parses and authenticates each payload, checks that its identifier and SEM shape match `patients.pii`, and forwards it without double encryption. Selecting the rows through Proxy returns the original plaintext JSON. +Proxy parses and authenticates each payload, checks that its identifier and authenticated descriptor match `patients.pii`, and independently re-derives every SEM term from the decrypted plaintext before forwarding it without double encryption. Selecting the rows through Proxy returns the original plaintext JSON. Each test section provides detailed output showing: - ✅ Successful query execution diff --git a/packages/showcase/src/pre_encrypted.rs b/packages/showcase/src/pre_encrypted.rs index d4c65061..1de6f66b 100644 --- a/packages/showcase/src/pre_encrypted.rs +++ b/packages/showcase/src/pre_encrypted.rs @@ -68,7 +68,7 @@ pub async fn run_examples() -> Result<(), Box> { } async fn encrypt_patient_pii(value: Value) -> Result> { - let config = ColumnConfig::build("pii") + let config = ColumnConfig::build("patients/pii") .casts_as(ColumnType::Json) .add_index(Index::new(IndexType::SteVec { prefix: "patients/pii".into(), From 37c0914e32b3a15dfa1b06a820c122db6a07f471 Mon Sep 17 00:00:00 2001 From: James Sadler Date: Thu, 20 Aug 2026 14:55:05 +1000 Subject: [PATCH 04/11] fix(proxy): honor configured default keyset Scope the ZeroKMS cipher to CS_DEFAULT_KEYSET_ID whenever a connection has not selected an override. Previously Proxy only checked that the setting existed, then passed no identifier to ScopedCipher and could silently use the client's account default instead. Application-encrypted payloads use the configured keyset explicitly. When the account and configured defaults differ, Proxy derived searchable-encryption metadata with another index key and rejected valid inbound ciphertext during authentication. Preserve connection-level keyset precedence while making the configured fallback effective. Signed-off-by: James Sadler --- .../cipherstash-proxy/src/proxy/zerokms/zerokms.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/packages/cipherstash-proxy/src/proxy/zerokms/zerokms.rs b/packages/cipherstash-proxy/src/proxy/zerokms/zerokms.rs index bb563c19..b602006d 100644 --- a/packages/cipherstash-proxy/src/proxy/zerokms/zerokms.rs +++ b/packages/cipherstash-proxy/src/proxy/zerokms/zerokms.rs @@ -16,7 +16,7 @@ use cipherstash_client::{ PreparedPlaintext, }, schema::column::IndexType, - zerokms::{Decryptable, EncryptedRecord, RecordWithNonce, RetrieveKeyPayload}, + zerokms::{Decryptable, EncryptedRecord, IdentifiedBy, RecordWithNonce, RetrieveKeyPayload}, }; use eql_mapper::EqlTermVariant; use metrics::{counter, histogram}; @@ -164,7 +164,15 @@ impl ZeroKms { info!(target: ZEROKMS, msg = "Initializing ZeroKMS ScopedCipher (cache miss)", ?keyset_id); counter!(KEYSET_CIPHER_CACHE_MISS_TOTAL).increment(1); - let identified_by = keyset_id.as_ref().map(|id| id.0.clone()); + // A connection-level keyset takes precedence. Otherwise, scope the + // cipher to Proxy's configured default instead of passing `None` and + // silently falling back to the ZeroKMS client's account default. The + // two defaults are not required to be the same, and using the account + // default would derive different searchable-encryption terms. + let identified_by = keyset_id + .as_ref() + .map(|id| id.0.clone()) + .or_else(|| self.default_keyset_id.map(IdentifiedBy::Uuid)); let start = Instant::now(); let result = ScopedCipher::init(zerokms_client, identified_by).await; From 221bde8fac15ddfeb81200ec93a71bed963bcb0a Mon Sep 17 00:00:00 2001 From: James Sadler Date: Thu, 20 Aug 2026 15:10:43 +1000 Subject: [PATCH 05/11] fix(proxy): compare bloom-filter terms as sets Inbound EQL authentication independently re-encrypts plaintext and compares its searchable-encryption metadata with the supplied payload. Match-index generation does not guarantee a stable ordering for Bloom-filter bit positions, so comparing serialized query operands rejected valid ciphertext whenever equivalent positions were emitted in another order. Compare scalar metadata field by field and normalize Bloom-filter positions before equality. Continue comparing identifiers, exact-match terms, ordered terms, versions, and structured SteVec operands exactly so altered metadata still fails closed. Add a regression test covering reordered equivalent Bloom-filter terms. Signed-off-by: James Sadler --- .../src/postgresql/inbound_eql.rs | 45 ++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/packages/cipherstash-proxy/src/postgresql/inbound_eql.rs b/packages/cipherstash-proxy/src/postgresql/inbound_eql.rs index 5ebd95e1..546eb7c9 100644 --- a/packages/cipherstash-proxy/src/postgresql/inbound_eql.rs +++ b/packages/cipherstash-proxy/src/postgresql/inbound_eql.rs @@ -67,8 +67,20 @@ fn validate_metadata(ciphertext: &EqlCiphertext, column: &Column) -> Result<(), /// Compare all searchable metadata after the plaintext has been authenticated /// and independently re-encrypted for the inferred destination column. /// `into_query_operand` removes only record ciphertext/key material, leaving -/// the identifier and every scalar or SteVec SEM term for an exact comparison. +/// the identifier and every scalar or SteVec SEM term. Bloom-filter positions +/// are compared without regard to order; all other terms compare exactly. pub fn sem_terms_match(inbound: &EqlCiphertext, derived: EqlCiphertext) -> bool { + if let (EqlCiphertext::Encrypted(inbound), EqlCiphertext::Encrypted(derived)) = + (inbound, &derived) + { + return inbound.version == derived.version + && inbound.identifier == derived.identifier + && inbound.hmac_256 == derived.hmac_256 + && bloom_filters_match(&inbound.bloom_filter, &derived.bloom_filter) + && inbound.ore_block_u64_8_256 == derived.ore_block_u64_8_256 + && inbound.ope_cllw == derived.ope_cllw; + } + match ( serde_json::to_value(inbound.clone().into_query_operand()), serde_json::to_value(derived.into_query_operand()), @@ -78,6 +90,23 @@ pub fn sem_terms_match(inbound: &EqlCiphertext, derived: EqlCiphertext) -> bool } } +fn bloom_filters_match(inbound: &Option>, derived: &Option>) -> bool { + match (inbound, derived) { + (Some(inbound), Some(derived)) => { + // Bloom-filter positions are a set. Their generation order is not + // stable, so comparing the serialized arrays directly rejects + // equivalent terms produced by independent encryptions. + let mut inbound = inbound.clone(); + let mut derived = derived.clone(); + inbound.sort_unstable(); + derived.sort_unstable(); + inbound == derived + } + (None, None) => true, + _ => false, + } +} + fn validate_scalar_terms( payload: &EncryptedPayloadV3, column: &Column, @@ -215,4 +244,18 @@ mod tests { assert!(!sem_terms_match(&spliced, derived)); } + + #[test] + fn bloom_filter_order_does_not_affect_sem_term_matching() { + let mut inbound = payload(crate::Identifier::new("users", "email")); + let mut derived = inbound.clone(); + if let EqlCiphertext::Encrypted(payload) = &mut inbound { + payload.bloom_filter = Some(vec![3, 1, 2]); + } + if let EqlCiphertext::Encrypted(payload) = &mut derived { + payload.bloom_filter = Some(vec![1, 2, 3]); + } + + assert!(sem_terms_match(&inbound, derived)); + } } From e329453eb319186326c45c83c73f2c0955a3b0d0 Mon Sep 17 00:00:00 2001 From: James Sadler Date: Thu, 20 Aug 2026 16:08:02 +1000 Subject: [PATCH 06/11] feat(proxy): accept inbound EQL query operands Signed-off-by: James Sadler --- CHANGELOG.md | 2 +- .../src/inbound_ciphertext.rs | 154 +++++++++- .../src/postgresql/context/mod.rs | 18 ++ .../src/postgresql/inbound_eql.rs | 263 ++++++++++++++++-- .../src/postgresql/middleware/frontend.rs | 46 +-- .../src/postgresql/rewrite/bind.rs | 10 +- .../src/inference/infer_type_impls/expr.rs | 7 +- packages/eql-mapper/src/lib.rs | 36 ++- packages/eql-mapper/src/query_operands.rs | 6 +- .../src/transformation_rules/helpers.rs | 6 +- .../rewrite_containment_ops.rs | 14 +- packages/showcase/README.md | 8 +- packages/showcase/src/pre_encrypted.rs | 88 +++++- 13 files changed, 565 insertions(+), 93 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8fa36c3a..d01c7077 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Added -- **Pre-encrypted EQL values in statements**: SQL literals and bound parameters may now carry EQL v3 storage payloads produced by an application. Proxy validates their wire shape and version, requires the authenticated ciphertext descriptor to name the inferred destination column, authenticates the ciphertext with the connection's active keyset, and independently re-derives every SEM term from the plaintext before forwarding it without encrypting it again. Invalid payloads fail closed with one generic error so validation details cannot be used as an oracle. +- **Application-generated EQL values in statements**: SQL literals and bound parameters may now carry EQL v3 storage payloads or query-only SEM operands produced by an application. Proxy authenticates stored ciphertext, requires its authenticated descriptor to name the inferred destination column, and independently re-derives every SEM term before forwarding it without double encryption. Query-only operands contain no ciphertext to authenticate, so Proxy instead validates their version, identifier, term shape, column capabilities, and syntactic query role; they are rejected in storage positions. Invalid payloads fail closed with one generic error so validation details cannot be used as an oracle. ### Changed diff --git a/packages/cipherstash-proxy-integration/src/inbound_ciphertext.rs b/packages/cipherstash-proxy-integration/src/inbound_ciphertext.rs index f64eeae3..be289f51 100644 --- a/packages/cipherstash-proxy-integration/src/inbound_ciphertext.rs +++ b/packages/cipherstash-proxy-integration/src/inbound_ciphertext.rs @@ -4,15 +4,16 @@ mod tests { use crate::common::{clear_with_client, connect_with_tls, random_id, PROXY}; use cipherstash_client::{ - encryption::{Plaintext, ScopedCipher}, + encryption::{Plaintext, QueryOp, ScopedCipher}, eql::{ - encrypt_eql_v3, EqlEncryptOpts, EqlOperation, EqlOutputV3, Identifier, + encrypt_eql_v3, EqlCiphertextV3, EqlEncryptOpts, EqlOperation, EqlOutputV3, Identifier, PreparedPlaintext, }, schema::{column::Index, ColumnConfig, ColumnType}, zerokms::{ClientKey, ZeroKMSBuilder}, AutoStrategy, IdentifiedBy, }; + use cipherstash_config::column::{ArrayIndexMode, IndexType, SteVecMode}; use std::{borrow::Cow, sync::Arc}; use uuid::Uuid; @@ -52,6 +53,17 @@ mod tests { .add_index(Index::new_match()) } + fn json_search_config(table: &str, column: &str) -> ColumnConfig { + ColumnConfig::build(format!("{table}/{column}")) + .casts_as(ColumnType::Json) + .add_index(Index::new(IndexType::SteVec { + prefix: format!("{table}/{column}"), + term_filters: Vec::new(), + array_index_mode: ArrayIndexMode::ALL, + mode: SteVecMode::default(), + })) + } + async fn encrypt_text(table: &str, column: &str, plaintext: &str) -> String { let prepared = PreparedPlaintext::new( Cow::Owned(text_search_config(table, column)), @@ -69,6 +81,35 @@ mod tests { serde_json::to_string(&ciphertext).unwrap() } + async fn query_text(table: &str, column: &str, plaintext: &str) -> String { + let stored: EqlCiphertextV3 = + serde_json::from_str(&encrypt_text(table, column, plaintext).await).unwrap(); + serde_json::to_string(&stored.into_query_operand()).unwrap() + } + + async fn query_json( + table: &str, + column: &str, + plaintext: serde_json::Value, + ) -> serde_json::Value { + let config = json_search_config(table, column); + let index_type = config.indexes[0].index_type.clone(); + let prepared = PreparedPlaintext::new( + Cow::Owned(config), + Identifier::new(table, column), + Plaintext::Json(Some(plaintext)), + EqlOperation::Query(&index_type, QueryOp::Default), + ); + let mut outputs = + encrypt_eql_v3(cipher().await, vec![prepared], &EqlEncryptOpts::default()) + .await + .expect("application-side query encryption must succeed"); + let EqlOutputV3::Query(query) = outputs.remove(0) else { + panic!("query encryption must return a query-only payload"); + }; + serde_json::to_value(query).unwrap() + } + #[tokio::test] async fn accepts_pre_encrypted_parameter_for_storage_and_search() { let client = connect_with_tls(*PROXY).await; @@ -118,6 +159,115 @@ mod tests { assert_eq!(row.get::<_, String>(0), plaintext); } + #[tokio::test] + async fn accepts_query_only_parameter_for_search() { + let client = connect_with_tls(*PROXY).await; + clear_with_client(&client).await; + let id = random_id(); + let plaintext = "queried with application SEM terms"; + + client + .execute( + "INSERT INTO encrypted (id, encrypted_text) VALUES ($1, $2)", + &[&id, &plaintext], + ) + .await + .unwrap(); + + let payload = query_text("encrypted", "encrypted_text", plaintext).await; + let rows = client + .query( + "SELECT encrypted_text FROM encrypted WHERE encrypted_text = $1", + &[&payload], + ) + .await + .unwrap(); + assert_eq!(rows[0].get::<_, String>(0), plaintext); + } + + #[tokio::test] + async fn accepts_query_only_literal_for_search() { + let client = connect_with_tls(*PROXY).await; + clear_with_client(&client).await; + let id = random_id(); + let plaintext = "queried with literal SEM terms"; + + client + .execute( + "INSERT INTO encrypted (id, encrypted_text) VALUES ($1, $2)", + &[&id, &plaintext], + ) + .await + .unwrap(); + + let payload = query_text("encrypted", "encrypted_text", plaintext) + .await + .replace('\'', "''"); + let rows = client + .simple_query(&format!( + "SELECT encrypted_text FROM encrypted WHERE encrypted_text = '{payload}'" + )) + .await + .unwrap(); + let row = rows + .iter() + .find_map(|message| match message { + tokio_postgres::SimpleQueryMessage::Row(row) => Some(row), + _ => None, + }) + .expect("query-only literal must match one row"); + assert_eq!(row.get(0), Some(plaintext)); + } + + #[tokio::test] + async fn rejects_query_only_parameter_for_storage() { + let client = connect_with_tls(*PROXY).await; + clear_with_client(&client).await; + let id = random_id(); + let payload = query_text("encrypted", "encrypted_text", "not writable").await; + + let error = client + .execute( + "INSERT INTO encrypted (id, encrypted_text) VALUES ($1, $2)", + &[&id, &payload], + ) + .await + .expect_err("query-only payloads must not be accepted for storage"); + assert_eq!( + error.as_db_error().unwrap().message(), + "Invalid encrypted value" + ); + } + + #[tokio::test] + async fn accepts_query_only_ste_vec_parameter_for_json_search() { + let client = connect_with_tls(*PROXY).await; + clear_with_client(&client).await; + let id = random_id(); + let plaintext = serde_json::json!({ + "patient": { "name": "Ada Lovelace" }, + "active": true + }); + + client + .execute( + "INSERT INTO encrypted (id, encrypted_jsonb) VALUES ($1, $2)", + &[&id, &plaintext], + ) + .await + .unwrap(); + + let payload = query_json("encrypted", "encrypted_jsonb", plaintext.clone()).await; + let rows = client + .query( + "SELECT encrypted_jsonb FROM encrypted WHERE encrypted_jsonb @> $1", + &[&payload], + ) + .await + .unwrap(); + assert_eq!(rows[0].get::<_, serde_json::Value>(0), plaintext); + } + #[tokio::test] async fn rejects_payload_for_a_different_destination_with_generic_error() { let client = connect_with_tls(*PROXY).await; diff --git a/packages/cipherstash-proxy/src/postgresql/context/mod.rs b/packages/cipherstash-proxy/src/postgresql/context/mod.rs index 206326bd..9797d20c 100644 --- a/packages/cipherstash-proxy/src/postgresql/context/mod.rs +++ b/packages/cipherstash-proxy/src/postgresql/context/mod.rs @@ -779,6 +779,12 @@ where plaintexts: Vec>, columns: &[Option], ) -> Result>, Error> { + if plaintexts.iter().all(Option::is_none) { + return Ok(std::iter::repeat_with(|| None) + .take(plaintexts.len()) + .collect()); + } + let keyset_id = self.keyset_identifier(); self.encryption @@ -1122,6 +1128,18 @@ mod tests { assert!(context.take_schema_changed()); } + #[tokio::test] + async fn empty_plaintext_batch_does_not_call_encryption_service() { + let context = create_context(); + let output = context + .encrypt(vec![None, None], &[None, None]) + .await + .unwrap(); + + assert_eq!(output.len(), 2); + assert!(output.iter().all(Option::is_none)); + } + fn statement() -> Statement { Statement { param_columns: vec![], diff --git a/packages/cipherstash-proxy/src/postgresql/inbound_eql.rs b/packages/cipherstash-proxy/src/postgresql/inbound_eql.rs index 546eb7c9..0c7bb3b0 100644 --- a/packages/cipherstash-proxy/src/postgresql/inbound_eql.rs +++ b/packages/cipherstash-proxy/src/postgresql/inbound_eql.rs @@ -1,14 +1,31 @@ -use crate::{error::EncryptError, postgresql::Column, EqlCiphertext}; +use crate::{error::EncryptError, postgresql::Column, EqlCiphertext, EqlQueryPayload}; use cipherstash_client::{ eql::{EncryptedPayloadV3, EQL_SCHEMA_VERSION_V3}, schema::column::IndexType, }; +use eql_mapper::EqlTermVariant; use serde_json::Value; -/// Parse a value only when its version, identifier, and storage fields advertise -/// it as an EQL payload. Ordinary JSON (including an object with a `c` key) -/// remains plaintext; malformed advertised payloads fail closed. -pub fn parse(bytes: &[u8], column: &Column) -> Result, EncryptError> { +/// An application-generated EQL value entering Proxy. +#[derive(Debug)] +pub enum InboundEql { + /// A stored payload carrying source ciphertext. This must be authenticated + /// and have its SEM terms independently verified before it can be used. + Store(EqlCiphertext), + /// A query operand carrying SEM terms only. It can never be written and has + /// no source ciphertext with which to authenticate its metadata. + Query(EqlQueryPayload), +} + +/// Parse a value only when its fields advertise it as an EQL storage payload or +/// query operand. Query-only payloads are valid exclusively in syntactic query +/// positions. Ordinary JSON (including an object with a `c` key) remains +/// plaintext; malformed advertised payloads fail closed. +pub fn parse( + bytes: &[u8], + column: &Column, + query_operand: bool, +) -> Result, EncryptError> { let Ok(value) = serde_json::from_slice::(bytes) else { return Ok(None); }; @@ -16,20 +33,39 @@ pub fn parse(bytes: &[u8], column: &Column) -> Result, Enc return Ok(None); }; - let payload_shaped = object.contains_key("v") + let storage_shaped = object.contains_key("v") && object.contains_key("i") && (object.contains_key("c") || object.contains_key("h") || object.contains_key("sv")); - if !payload_shaped { + if storage_shaped { + let ciphertext: EqlCiphertext = + serde_json::from_value(value).map_err(|_| EncryptError::InvalidInboundCiphertext)?; + validate_storage_metadata(&ciphertext, column)?; + return Ok(Some(InboundEql::Store(ciphertext))); + } + + let scalar_query_shaped = object.contains_key("v") + && object.contains_key("i") + && ["hm", "bf", "ob", "op"] + .iter() + .any(|term| object.contains_key(*term)); + let ste_vec_query_shaped = object.len() == 1 && object.contains_key("sv"); + if !scalar_query_shaped && !ste_vec_query_shaped { return Ok(None); } + if !query_operand { + return Err(EncryptError::InvalidInboundCiphertext); + } - let ciphertext: EqlCiphertext = + let query: EqlQueryPayload = serde_json::from_value(value).map_err(|_| EncryptError::InvalidInboundCiphertext)?; - validate_metadata(&ciphertext, column)?; - Ok(Some(ciphertext)) + validate_query_metadata(&query, column)?; + Ok(Some(InboundEql::Query(query))) } -fn validate_metadata(ciphertext: &EqlCiphertext, column: &Column) -> Result<(), EncryptError> { +fn validate_storage_metadata( + ciphertext: &EqlCiphertext, + column: &Column, +) -> Result<(), EncryptError> { if ciphertext.version() != EQL_SCHEMA_VERSION_V3 || ciphertext.identifier() != &column.identifier { @@ -64,6 +100,64 @@ fn validate_metadata(ciphertext: &EqlCiphertext, column: &Column) -> Result<(), } } +fn validate_query_metadata(query: &EqlQueryPayload, column: &Column) -> Result<(), EncryptError> { + match query { + EqlQueryPayload::Encrypted(payload) => { + if payload.version != EQL_SCHEMA_VERSION_V3 || payload.identifier != column.identifier { + return Err(EncryptError::InvalidInboundCiphertext); + } + + match column.eql_term { + EqlTermVariant::Full | EqlTermVariant::Partial | EqlTermVariant::Tokenized => { + validate_scalar_term_presence( + payload.hmac_256.is_some(), + payload.bloom_filter.is_some(), + payload.ore_block_u64_8_256.is_some(), + payload.ope_cllw.is_some(), + column, + ) + } + EqlTermVariant::JsonOrd => { + let ste_vec_configured = column + .config + .indexes + .iter() + .any(|index| matches!(index.index_type, IndexType::SteVec { .. })); + if !ste_vec_configured + || payload.hmac_256.is_some() + || payload.bloom_filter.is_some() + || payload.ore_block_u64_8_256.is_some() + || payload.ope_cllw.is_none() + { + return Err(EncryptError::InvalidInboundCiphertext); + } + Ok(()) + } + _ => Err(EncryptError::InvalidInboundCiphertext), + } + } + EqlQueryPayload::SteVec(payload) => { + let configured = column + .config + .indexes + .iter() + .any(|index| matches!(index.index_type, IndexType::SteVec { .. })); + let query_shape = matches!( + column.eql_term, + EqlTermVariant::Full | EqlTermVariant::Partial | EqlTermVariant::JsonValueSelector + ); + if !configured || !query_shape || payload.ste_vec.is_empty() { + return Err(EncryptError::InvalidInboundCiphertext); + } + Ok(()) + } + // Bare selector hashes are indistinguishable from ordinary plaintext + // text on the PostgreSQL wire, so they cannot safely advertise + // themselves as pre-computed query operands. + EqlQueryPayload::Selector(_) => Err(EncryptError::InvalidInboundCiphertext), + } +} + /// Compare all searchable metadata after the plaintext has been authenticated /// and independently re-encrypted for the inferred destination column. /// `into_query_operand` removes only record ciphertext/key material, leaving @@ -110,6 +204,22 @@ fn bloom_filters_match(inbound: &Option>, derived: &Option>) - fn validate_scalar_terms( payload: &EncryptedPayloadV3, column: &Column, +) -> Result<(), EncryptError> { + validate_scalar_term_presence( + payload.hmac_256.is_some(), + payload.bloom_filter.is_some(), + payload.ore_block_u64_8_256.is_some(), + payload.ope_cllw.is_some(), + column, + ) +} + +fn validate_scalar_term_presence( + has_hmac: bool, + has_bloom: bool, + has_ore: bool, + has_ope: bool, + column: &Column, ) -> Result<(), EncryptError> { let mut hmac = false; let mut bloom = false; @@ -125,11 +235,7 @@ fn validate_scalar_terms( } } - if payload.hmac_256.is_some() != hmac - || payload.bloom_filter.is_some() != bloom - || payload.ore_block_u64_8_256.is_some() != ore - || payload.ope_cllw.is_some() != ope - { + if has_hmac != hmac || has_bloom != bloom || has_ore != ore || has_ope != ope { return Err(EncryptError::InvalidInboundCiphertext); } Ok(()) @@ -140,6 +246,7 @@ mod tests { use super::*; use cipherstash_client::schema::{ColumnConfig, ColumnMode, ColumnType}; use cipherstash_client::zerokms::EncryptedRecord; + use cipherstash_config::column::{ArrayIndexMode, Index, SteVecMode}; use eql_mapper::EqlTermVariant; use uuid::Uuid; @@ -177,14 +284,29 @@ mod tests { }) } + fn ste_vec_column() -> Column { + let mut column = column(); + column.config.cast_type = ColumnType::Json; + column.config.indexes.push(Index::new(IndexType::SteVec { + prefix: "users/email".into(), + term_filters: Vec::new(), + array_index_mode: ArrayIndexMode::ALL, + mode: SteVecMode::default(), + })); + column.postgres_type = postgres_types::Type::JSONB; + column + } + #[test] fn ordinary_json_is_plaintext() { - assert!(parse(br#"{"name":"Ada"}"#, &column()).unwrap().is_none()); + assert!(parse(br#"{"name":"Ada"}"#, &column(), false) + .unwrap() + .is_none()); } #[test] fn ordinary_json_with_a_c_key_is_plaintext() { - assert!(parse(br#"{"c":"customer code"}"#, &column()) + assert!(parse(br#"{"c":"customer code"}"#, &column(), false) .unwrap() .is_none()); } @@ -192,7 +314,7 @@ mod tests { #[test] fn malformed_payload_shape_fails_closed() { assert!(matches!( - parse(br#"{"v":3,"i":"users.email","c":"bad"}"#, &column()), + parse(br#"{"v":3,"i":"users.email","c":"bad"}"#, &column(), false), Err(EncryptError::InvalidInboundCiphertext) )); } @@ -201,7 +323,7 @@ mod tests { fn destination_identifier_must_match() { let ciphertext = payload(crate::Identifier::new("users", "phone")); assert!(matches!( - validate_metadata(&ciphertext, &column()), + validate_storage_metadata(&ciphertext, &column()), Err(EncryptError::InvalidInboundCiphertext) )); } @@ -214,7 +336,7 @@ mod tests { }; payload.ciphertext.descriptor = "accounts/email".into(); assert!(matches!( - validate_metadata(&ciphertext, &column()), + validate_storage_metadata(&ciphertext, &column()), Err(EncryptError::InvalidInboundCiphertext) )); } @@ -228,7 +350,7 @@ mod tests { .push(cipherstash_client::schema::column::Index::new_unique()); let ciphertext = payload(column.identifier.clone()); assert!(matches!( - validate_metadata(&ciphertext, &column), + validate_storage_metadata(&ciphertext, &column), Err(EncryptError::InvalidInboundCiphertext) )); } @@ -258,4 +380,101 @@ mod tests { assert!(sem_terms_match(&inbound, derived)); } + + #[test] + fn query_only_scalar_payload_is_accepted_for_a_query_operand() { + let mut column = column(); + column + .config + .indexes + .push(cipherstash_client::schema::column::Index::new_unique()); + let mut ciphertext = payload(column.identifier.clone()); + let EqlCiphertext::Encrypted(payload) = &mut ciphertext else { + unreachable!() + }; + payload.hmac_256 = Some("application-generated SEM term".into()); + let query = serde_json::to_vec(&ciphertext.into_query_operand()).unwrap(); + + assert!(matches!( + parse(&query, &column, true), + Ok(Some(InboundEql::Query(_))) + )); + } + + #[test] + fn query_only_scalar_payload_is_rejected_for_storage() { + let mut column = column(); + column + .config + .indexes + .push(cipherstash_client::schema::column::Index::new_unique()); + let mut ciphertext = payload(column.identifier.clone()); + let EqlCiphertext::Encrypted(payload) = &mut ciphertext else { + unreachable!() + }; + payload.hmac_256 = Some("application-generated SEM term".into()); + let query = serde_json::to_vec(&ciphertext.into_query_operand()).unwrap(); + + assert!(matches!( + parse(&query, &column, false), + Err(EncryptError::InvalidInboundCiphertext) + )); + } + + #[test] + fn query_only_scalar_identifier_must_match_destination() { + let mut column = column(); + column + .config + .indexes + .push(cipherstash_client::schema::column::Index::new_unique()); + let mut ciphertext = payload(crate::Identifier::new("users", "phone")); + let EqlCiphertext::Encrypted(payload) = &mut ciphertext else { + unreachable!() + }; + payload.hmac_256 = Some("application-generated SEM term".into()); + let query = serde_json::to_vec(&ciphertext.into_query_operand()).unwrap(); + + assert!(matches!( + parse(&query, &column, true), + Err(EncryptError::InvalidInboundCiphertext) + )); + } + + #[test] + fn query_only_json_ordering_term_is_accepted() { + let mut column = ste_vec_column(); + column.eql_term = EqlTermVariant::JsonOrd; + let query = serde_json::to_vec(&serde_json::json!({ + "v": EQL_SCHEMA_VERSION_V3, + "i": { "t": "users", "c": "email" }, + "op": "application-generated ordering term" + })) + .unwrap(); + + assert!(matches!( + parse(&query, &column, true), + Ok(Some(InboundEql::Query(EqlQueryPayload::Encrypted(_)))) + )); + } + + #[test] + fn query_only_ste_vec_payload_is_accepted_for_a_query_operand() { + let query = br#"{"sv":[{"s":"application-generated selector"}]}"#; + + assert!(matches!( + parse(query, &ste_vec_column(), true), + Ok(Some(InboundEql::Query(EqlQueryPayload::SteVec(_)))) + )); + } + + #[test] + fn query_only_ste_vec_payload_is_rejected_for_storage() { + let query = br#"{"sv":[{"s":"application-generated selector"}]}"#; + + assert!(matches!( + parse(query, &ste_vec_column(), false), + Err(EncryptError::InvalidInboundCiphertext) + )); + } } diff --git a/packages/cipherstash-proxy/src/postgresql/middleware/frontend.rs b/packages/cipherstash-proxy/src/postgresql/middleware/frontend.rs index cf39dea5..06c4493d 100644 --- a/packages/cipherstash-proxy/src/postgresql/middleware/frontend.rs +++ b/packages/cipherstash-proxy/src/postgresql/middleware/frontend.rs @@ -559,7 +559,12 @@ impl Frontend { let (Some(column), Some(value)) = (column, (*literal).clone().into_string()) else { return Ok(None); }; - inbound_eql::parse(value.as_bytes(), column).map_err(Error::from) + inbound_eql::parse( + value.as_bytes(), + column, + typed_statement.query_operands.contains_literal(literal), + ) + .map_err(Error::from) }) .collect::, Error>>()?; let skip = inbound.iter().map(Option::is_some).collect::>(); @@ -575,7 +580,7 @@ impl Frontend { counter!(ENCRYPTION_ERROR_TOTAL).increment(1); })?; - self.authenticate_and_merge_inbound(&mut encrypted, inbound, literal_columns) + self.merge_inbound_eql(&mut encrypted, inbound, literal_columns) .await?; for ((_, literal), encrypted) in literal_values.iter().zip(encrypted.iter_mut()) { @@ -1081,7 +1086,7 @@ impl Frontend { bind: &Bind, statement: &Statement, ) -> Result>, Error> { - let inbound = bind.inbound_ciphertexts(&statement.output_params)?; + let inbound = bind.inbound_eql(&statement.output_params)?; let skip = inbound.iter().map(Option::is_some).collect::>(); let plaintexts = bind.to_plaintext_skipping( &statement.output_params, @@ -1109,7 +1114,7 @@ impl Frontend { counter!(ENCRYPTION_ERROR_TOTAL).increment(1); })?; - self.authenticate_and_merge_inbound(&mut encrypted, inbound, &output_param_columns) + self.merge_inbound_eql(&mut encrypted, inbound, &output_param_columns) .await?; for (output, encrypted) in statement.output_params.iter().zip(encrypted.iter_mut()) { @@ -1138,22 +1143,31 @@ impl Frontend { Ok(encrypted) } - /// Authenticate inbound ciphertext with this connection's scoped cipher. - /// Any parse, metadata, key or AEAD failure is collapsed to one response. - async fn authenticate_and_merge_inbound( + /// Merge application-generated EQL values into the encryption output. + /// Stored payloads are authenticated and independently verified. Query-only + /// payloads have no ciphertext to authenticate and are accepted only after + /// role-aware structural validation in `inbound_eql::parse`. + async fn merge_inbound_eql( &self, encrypted: &mut [Option], - inbound: Vec>, + inbound: Vec>, columns: &[Option], ) -> Result<(), Error> { - let positions = inbound - .into_iter() - .zip(columns) - .enumerate() - .filter_map(|(index, (ciphertext, column))| { - Some((index, ciphertext?, column.as_ref()?.clone())) - }) - .collect::>(); + let mut positions = Vec::new(); + for (index, (payload, column)) in inbound.into_iter().zip(columns).enumerate() { + match payload { + Some(inbound_eql::InboundEql::Query(query)) => { + encrypted[index] = Some(EqlOutput::Query(query)); + } + Some(inbound_eql::InboundEql::Store(ciphertext)) => { + let Some(column) = column else { + return Err(EncryptError::InvalidInboundCiphertext.into()); + }; + positions.push((index, ciphertext, column.clone())); + } + None => {} + } + } if positions.is_empty() { return Ok(()); } diff --git a/packages/cipherstash-proxy/src/postgresql/rewrite/bind.rs b/packages/cipherstash-proxy/src/postgresql/rewrite/bind.rs index 6561d301..569830ff 100644 --- a/packages/cipherstash-proxy/src/postgresql/rewrite/bind.rs +++ b/packages/cipherstash-proxy/src/postgresql/rewrite/bind.rs @@ -124,12 +124,12 @@ impl Bind { .collect() } - /// Detect already-encrypted storage payloads before decoding parameters as - /// their configured plaintext PostgreSQL types. - pub fn inbound_ciphertexts( + /// Detect application-generated storage or query payloads before decoding + /// parameters as their configured plaintext PostgreSQL types. + pub fn inbound_eql( &self, output_params: &[OutputParam], - ) -> Result>, Error> { + ) -> Result>, Error> { output_params .iter() .map(|output| { @@ -150,7 +150,7 @@ impl Bind { } else { ¶m.bytes }; - inbound_eql::parse(bytes, column).map_err(Error::from) + inbound_eql::parse(bytes, column, output.query_operand).map_err(Error::from) }) .collect() } diff --git a/packages/eql-mapper/src/inference/infer_type_impls/expr.rs b/packages/eql-mapper/src/inference/infer_type_impls/expr.rs index 525fc6ec..17773b51 100644 --- a/packages/eql-mapper/src/inference/infer_type_impls/expr.rs +++ b/packages/eql-mapper/src/inference/infer_type_impls/expr.rs @@ -414,9 +414,8 @@ impl<'ast> InferType<'ast, Expr> for TypeInferencer<'ast> { // The operands of a predicate reach PostgreSQL as query // operands — terms only, never a ciphertext. Record them so the - // proxy projects their payloads accordingly. Containment - // (`@>`/`<@`) is deliberately excluded: its needle is a whole - // document and keeps its full payload. + // proxy projects their payloads accordingly. JSON containment + // uses a SteVec query needle rather than a stored document. if matches!( op, BinaryOperator::Eq @@ -426,6 +425,8 @@ impl<'ast> InferType<'ast, Expr> for TypeInferencer<'ast> { | BinaryOperator::Gt | BinaryOperator::GtEq | BinaryOperator::AtAt + | BinaryOperator::AtArrow + | BinaryOperator::ArrowAt ) { self.record_query_operands([&**left, &**right]); } diff --git a/packages/eql-mapper/src/lib.rs b/packages/eql-mapper/src/lib.rs index 28dbfbbe..4de54147 100644 --- a/packages/eql-mapper/src/lib.rs +++ b/packages/eql-mapper/src/lib.rs @@ -2624,7 +2624,7 @@ mod test { tables: { patients: { id, - notes (EQL: JsonLike + Contain), + notes (EQL("eql_v3_json_search"): JsonLike + Contain), } } }); @@ -2635,14 +2635,18 @@ mod test { match type_check(schema, &statement) { Ok(typed) => { + if matches!(op, "@>" | "<@") { + let literal = typed.literal_values()[0].1; + assert!(typed.query_operands.contains_literal(literal)); + } match typed.transform(test_helpers::dummy_encrypted_json_selector( &statement, vec![ast::Value::SingleQuotedString("medications".to_owned())], )) { Ok(statement) => { let expected = match op { - "@>" => "SELECT id, eql_v3.jsonb_contains(notes, ''::JSONB::public.eql_v3_text_search) AS meds FROM patients".to_string(), - "<@" => "SELECT id, eql_v3.jsonb_contained_by(notes, ''::JSONB::public.eql_v3_text_search) AS meds FROM patients".to_string(), + "@>" => "SELECT id, eql_v3.jsonb_contains(notes, ''::JSONB::eql_v3.query_json) AS meds FROM patients".to_string(), + "<@" => "SELECT id, eql_v3.jsonb_contained_by(notes, ''::JSONB::eql_v3.query_json) AS meds FROM patients".to_string(), // -> / ->> field access: functionalised to eql_v3."->"/"->>", // with the field selector passed as encrypted text. "->" => "SELECT id, eql_v3.\"->\"(notes, '') AS meds FROM patients".to_string(), @@ -2664,7 +2668,7 @@ mod test { tables: { patients: { id, - notes (EQL: JsonLike + Contain), + notes (EQL("eql_v3_json_search"): JsonLike + Contain), } } }); @@ -2685,7 +2689,7 @@ mod test { tables: { patients: { id, - notes (EQL: JsonLike + Contain), + notes (EQL("eql_v3_json_search"): JsonLike + Contain), } } }); @@ -2775,7 +2779,7 @@ mod test { tables: { patients: { id, - notes (EQL: JsonLike + Contain), + notes (EQL("eql_v3_json_search"): JsonLike + Contain), } } }); @@ -2795,12 +2799,14 @@ mod test { "Expected @> to be transformed to eql_v3.jsonb_contains, got: {sql}" ); - // CRITICAL: Verify the parameter is cast to enable GIN index usage - // The cast ::JSONB::public.eql_v3_text_search is required for GIN indexes to work + // A containment needle is a term-only query operand. It must use the + // query_json domain so no source ciphertext is required. assert!( - sql.contains("::JSONB::public.eql_v3_text_search") || sql.contains("::jsonb::public.eql_v3_text_search"), - "Expected parameter to be cast as ::JSONB::public.eql_v3_text_search for GIN index support, got: {sql}" + sql.contains("::JSONB::eql_v3.query_json") + || sql.contains("::jsonb::eql_v3.query_json"), + "Expected parameter to be cast as ::JSONB::eql_v3.query_json, got: {sql}" ); + assert!(transformed.params.outputs()[0].query_operand); } #[test] @@ -2809,7 +2815,7 @@ mod test { tables: { patients: { id, - notes (EQL: JsonLike + Contain), + notes (EQL("eql_v3_json_search"): JsonLike + Contain), } } }); @@ -2829,11 +2835,13 @@ mod test { "Expected <@ to be transformed to eql_v3.jsonb_contained_by, got: {sql}" ); - // CRITICAL: Verify the parameter is cast to enable GIN index usage + // The contained value is also a term-only query operand. assert!( - sql.contains("::JSONB::public.eql_v3_text_search") || sql.contains("::jsonb::public.eql_v3_text_search"), - "Expected parameter to be cast as ::JSONB::public.eql_v3_text_search for GIN index support, got: {sql}" + sql.contains("::JSONB::eql_v3.query_json") + || sql.contains("::jsonb::eql_v3.query_json"), + "Expected parameter to be cast as ::JSONB::eql_v3.query_json, got: {sql}" ); + assert!(transformed.params.outputs()[0].query_operand); } #[test] diff --git a/packages/eql-mapper/src/query_operands.rs b/packages/eql-mapper/src/query_operands.rs index f167e83a..0d876428 100644 --- a/packages/eql-mapper/src/query_operands.rs +++ b/packages/eql-mapper/src/query_operands.rs @@ -27,9 +27,9 @@ use crate::Param; /// /// Membership is decided syntactically, by the predicate an operand belongs to /// — the same contexts whose rewrite rules cast to a `eql_v3.query_*` twin: -/// comparisons (`=`, `<>`, `<`, `<=`, `>`, `>=`), `LIKE`/`ILIKE` and `@@`. -/// Everything else — `INSERT` values, `UPDATE` assignments, containment needles -/// — is a stored value and keeps its full payload. +/// comparisons (`=`, `<>`, `<`, `<=`, `>`, `>=`), containment (`@>`/`<@`), +/// `LIKE`/`ILIKE` and `@@`. Everything else — including `INSERT` values and +/// `UPDATE` assignments — is a stored value and keeps its full payload. #[derive(Debug, Default)] pub struct QueryOperands<'ast> { params: HashSet, diff --git a/packages/eql-mapper/src/transformation_rules/helpers.rs b/packages/eql-mapper/src/transformation_rules/helpers.rs index 81461ac1..c30625c3 100644 --- a/packages/eql-mapper/src/transformation_rules/helpers.rs +++ b/packages/eql-mapper/src/transformation_rules/helpers.rs @@ -64,9 +64,9 @@ pub(crate) fn query_operand_domain(eql_term: &EqlTerm) -> Option<(String, String /// own domain, carrying the ciphertext plus every search term the column /// indexes. /// -/// This is what an `INSERT` value, an `UPDATE` assignment and a containment -/// needle all need — as opposed to a predicate operand, which needs only the -/// terms of [`query_operand_domain`]. +/// This is what an `INSERT` value or an `UPDATE` assignment needs — as opposed +/// to a predicate operand, which needs only the terms of +/// [`query_operand_domain`]. /// /// Returns `None` for a JSON selector, which is bare text in every position. pub(crate) fn full_payload_domain(eql_term: &EqlTerm) -> Option<(String, String)> { diff --git a/packages/eql-mapper/src/transformation_rules/rewrite_containment_ops.rs b/packages/eql-mapper/src/transformation_rules/rewrite_containment_ops.rs index 0acfe477..2eda2d32 100644 --- a/packages/eql-mapper/src/transformation_rules/rewrite_containment_ops.rs +++ b/packages/eql-mapper/src/transformation_rules/rewrite_containment_ops.rs @@ -13,7 +13,7 @@ use sqltk::{NodeKey, NodePath, Visitable}; use crate::unifier::{Type, Value}; use crate::EqlMapperError; -use super::helpers::{cast_encrypted_operand, full_payload_domain}; +use super::helpers::{cast_encrypted_operand, query_operand_domain}; use super::TransformationRule; /// Rewrites JSON binary operators on encrypted columns to `eql_v3` function @@ -115,17 +115,15 @@ impl<'ast> TransformationRule<'ast> for RewriteContainmentOps<'ast> { _ => return Ok(false), }; - // A containment needle is a whole encrypted document, so it - // casts to the column domain, not to a query twin. A `->`/`->>` - // selector takes no cast at all — `full_payload_domain` returns - // `None` for it — because `eql_v3."->"(json, text)` wants the - // bare encrypted selector text. - cast_encrypted_operand(&self.node_types, original_left, left, full_payload_domain); + // Containment uses a term-only SteVec query needle. A `->`/`->>` + // selector also remains query-only but takes no cast because + // `eql_v3."->"(json, text)` wants the bare encrypted selector. + cast_encrypted_operand(&self.node_types, original_left, left, query_operand_domain); cast_encrypted_operand( &self.node_types, original_right, right, - full_payload_domain, + query_operand_domain, ); // Use mem::replace to move (not copy) the original nodes, diff --git a/packages/showcase/README.md b/packages/showcase/README.md index 996898e8..5bb245bf 100644 --- a/packages/showcase/README.md +++ b/packages/showcase/README.md @@ -466,7 +466,7 @@ mise run test:integration:showcase The showcase will execute and display: -1. **Application-side Encryption**: Insert pre-encrypted EQL payloads as a bound parameter and a SQL literal +1. **Application-side EQL**: Insert pre-encrypted storage payloads and search with query-only SEM payloads, using both parameters and SQL literals 2. **Original Healthcare Query**: Aspirin prescription lookup 3. **Field Access Operations**: Testing `->` and `->>` 4. **Containment Operations**: Testing `@>` and `<@` @@ -476,14 +476,16 @@ The showcase will execute and display: ### Application-side Encryption -The `pre_encrypted` example constructs the same `eql_v3_json_search` column configuration declared by `patients.pii`, encrypts patient PII with `cipherstash-client`, and sends the resulting EQL payload through Proxy. The application-side `ColumnConfig` uses the canonical `patients/pii` descriptor; this authenticated descriptor binds the ciphertext to its destination. The example demonstrates both supported input forms: +The `pre_encrypted` example constructs the same `eql_v3_json_search` column configuration declared by `patients.pii`, encrypts patient PII or its query SEM terms with `cipherstash-client`, and sends the resulting EQL payload through Proxy. The application-side `ColumnConfig` uses the canonical `patients/pii` descriptor; this authenticated descriptor binds stored ciphertext to its destination. The example demonstrates both supported input forms and both payload roles: ```sql INSERT INTO patients (id, pii) VALUES ($1, $2); -- payload parameter INSERT INTO patients (id, pii) VALUES ('...', '{...}'); -- payload literal +SELECT id FROM patients WHERE pii @> $1; -- query-only SEM parameter +SELECT id FROM patients WHERE pii @> '{...}'; -- query-only SEM literal ``` -Proxy parses and authenticates each payload, checks that its identifier and authenticated descriptor match `patients.pii`, and independently re-derives every SEM term from the decrypted plaintext before forwarding it without double encryption. Selecting the rows through Proxy returns the original plaintext JSON. +For stored payloads, Proxy authenticates the ciphertext, checks that its identifier and authenticated descriptor match `patients.pii`, and independently re-derives every SEM term from the decrypted plaintext. Query-only payloads deliberately contain no source ciphertext, so authentication is impossible and unnecessary: Proxy validates that their shape is valid for `patients.pii` and accepts them only in predicate positions, where incorrect terms can only produce incorrect query results and cannot poison stored data. Both forms are forwarded without double encryption. Each test section provides detailed output showing: - ✅ Successful query execution diff --git a/packages/showcase/src/pre_encrypted.rs b/packages/showcase/src/pre_encrypted.rs index 1de6f66b..9aa5ab8b 100644 --- a/packages/showcase/src/pre_encrypted.rs +++ b/packages/showcase/src/pre_encrypted.rs @@ -1,11 +1,12 @@ -//! Application-side encryption examples for Stash-style ingestion. +//! Application-side EQL examples for storage and search. //! -//! Proxy accepts the resulting EQL storage payload as either a bound parameter -//! or a SQL literal, authenticates it, and avoids encrypting it a second time. +//! Proxy accepts storage and query-only payloads as either bound parameters or +//! SQL literals, applies role-appropriate validation, and avoids encrypting +//! them a second time. use crate::common::{connect_with_tls, PROXY}; use cipherstash_client::{ - encryption::{Plaintext, ScopedCipher}, + encryption::{Plaintext, QueryOp, ScopedCipher}, eql::{ encrypt_eql_v3, EqlEncryptOpts, EqlOperation, EqlOutputV3, Identifier, PreparedPlaintext, }, @@ -57,25 +58,54 @@ pub async fn run_examples() -> Result<(), Box> { println!("✅ Inserted application-encrypted PII as a SQL literal"); // Both rows still decrypt normally when selected through Proxy. - for (id, expected) in [(parameter_id, parameter_pii), (literal_id, literal_pii)] { + for (id, expected) in [ + (parameter_id, parameter_pii.clone()), + (literal_id, literal_pii.clone()), + ] { let row = client .query_one("SELECT pii FROM patients WHERE id = $1", &[&id]) .await?; assert_eq!(row.get::<_, Value>(0), expected); } println!("✅ Proxy authenticated and decrypted both application-encrypted values"); + + // Example 3: a query-only EQL payload contains SteVec SEM terms but no + // source ciphertext. Proxy validates its query role and forwards it without + // attempting authentication or encrypting it a second time. + let parameter_query = query_patient_pii(parameter_pii).await?; + let row = client + .query_one( + "SELECT id FROM patients WHERE pii @> $1", + &[¶meter_query], + ) + .await?; + assert_eq!(row.get::<_, Uuid>(0), parameter_id); + println!("✅ Queried with application-generated SEM terms as a bound parameter"); + + // Example 4: query-only payloads are also accepted as SQL literals in + // predicate positions. + let literal_query = query_patient_pii(literal_pii) + .await? + .to_string() + .replace('\'', "''"); + let rows = client + .simple_query(&format!( + "SELECT id FROM patients WHERE pii @> '{literal_query}'" + )) + .await?; + let matched = rows.iter().any(|message| match message { + tokio_postgres::SimpleQueryMessage::Row(row) => { + row.get(0) == Some(literal_id.to_string().as_str()) + } + _ => false, + }); + assert!(matched); + println!("✅ Queried with application-generated SEM terms as a SQL literal"); Ok(()) } async fn encrypt_patient_pii(value: Value) -> Result> { - let config = ColumnConfig::build("patients/pii") - .casts_as(ColumnType::Json) - .add_index(Index::new(IndexType::SteVec { - prefix: "patients/pii".into(), - term_filters: Vec::new(), - array_index_mode: ArrayIndexMode::ALL, - mode: SteVecMode::default(), - })); + let config = patient_pii_config(); let prepared = PreparedPlaintext::new( Cow::Owned(config), Identifier::new("patients", "pii"), @@ -94,6 +124,38 @@ async fn encrypt_patient_pii(value: Value) -> Result Result> { + let config = patient_pii_config(); + let index_type = config.indexes[0].index_type.clone(); + let prepared = PreparedPlaintext::new( + Cow::Owned(config), + Identifier::new("patients", "pii"), + Plaintext::Json(Some(value)), + EqlOperation::Query(&index_type, QueryOp::Default), + ); + let mut outputs = encrypt_eql_v3( + scoped_cipher().await?, + vec![prepared], + &EqlEncryptOpts::default(), + ) + .await?; + let EqlOutputV3::Query(query) = outputs.remove(0) else { + return Err("query encryption returned a storage payload".into()); + }; + Ok(serde_json::to_value(query)?) +} + +fn patient_pii_config() -> ColumnConfig { + ColumnConfig::build("patients/pii") + .casts_as(ColumnType::Json) + .add_index(Index::new(IndexType::SteVec { + prefix: "patients/pii".into(), + term_filters: Vec::new(), + array_index_mode: ArrayIndexMode::ALL, + mode: SteVecMode::default(), + })) +} + async fn scoped_cipher() -> Result>, Box> { let client_id = env("CS_CLIENT_ID", "CS_ENCRYPT__CLIENT_ID")?.parse()?; let client_key = From f1a68646c3dd9c157ef85d8b8ccff22bc386680f Mon Sep 17 00:00:00 2001 From: James Sadler Date: Thu, 20 Aug 2026 16:34:37 +1000 Subject: [PATCH 07/11] feat(proxy): accept inbound selector hashes Signed-off-by: James Sadler --- CHANGELOG.md | 2 +- .../src/inbound_ciphertext.rs | 68 ++++++++++++++++++ .../src/postgresql/inbound_eql.rs | 71 +++++++++++++++++-- .../src/postgresql/middleware/frontend.rs | 30 ++++++-- packages/showcase/README.md | 6 +- packages/showcase/src/pre_encrypted.rs | 54 ++++++++++++++ 6 files changed, 219 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d01c7077..28949522 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Added -- **Application-generated EQL values in statements**: SQL literals and bound parameters may now carry EQL v3 storage payloads or query-only SEM operands produced by an application. Proxy authenticates stored ciphertext, requires its authenticated descriptor to name the inferred destination column, and independently re-derives every SEM term before forwarding it without double encryption. Query-only operands contain no ciphertext to authenticate, so Proxy instead validates their version, identifier, term shape, column capabilities, and syntactic query role; they are rejected in storage positions. Invalid payloads fail closed with one generic error so validation details cannot be used as an oracle. +- **Application-generated EQL values in statements**: SQL literals and bound parameters may now carry EQL v3 storage payloads or query-only SEM operands produced by an application. Proxy authenticates stored ciphertext, requires its authenticated descriptor to name the inferred destination column, and independently re-derives every SEM term before forwarding it without double encryption. Query-only operands contain no ciphertext to authenticate, so Proxy instead validates their version, identifier, term shape, column capabilities, and syntactic query role; they are rejected in storage positions. This includes bare SteVec selector hashes matching `^[0-9a-f]{32}$`: in JSON selector query positions a match is treated as already hashed, while a non-match remains plaintext and is encrypted normally. A matching plaintext selector is inherently ambiguous and is intentionally treated as already hashed. Invalid payloads fail closed with one generic error so validation details cannot be used as an oracle. ### Changed diff --git a/packages/cipherstash-proxy-integration/src/inbound_ciphertext.rs b/packages/cipherstash-proxy-integration/src/inbound_ciphertext.rs index be289f51..84356665 100644 --- a/packages/cipherstash-proxy-integration/src/inbound_ciphertext.rs +++ b/packages/cipherstash-proxy-integration/src/inbound_ciphertext.rs @@ -110,6 +110,28 @@ mod tests { serde_json::to_value(query).unwrap() } + async fn query_json_selector(table: &str, column: &str, path: &str) -> String { + let config = json_search_config(table, column); + let index_type = config.indexes[0].index_type.clone(); + let prepared = PreparedPlaintext::new( + Cow::Owned(config), + Identifier::new(table, column), + Plaintext::from(path), + EqlOperation::Query(&index_type, QueryOp::SteVecSelector), + ); + let mut outputs = + encrypt_eql_v3(cipher().await, vec![prepared], &EqlEncryptOpts::default()) + .await + .expect("application-side selector encryption must succeed"); + let EqlOutputV3::Query(query) = outputs.remove(0) else { + panic!("selector encryption must return a query-only payload"); + }; + let serde_json::Value::String(selector) = serde_json::to_value(query).unwrap() else { + panic!("selector encryption must return a bare selector hash"); + }; + selector + } + #[tokio::test] async fn accepts_pre_encrypted_parameter_for_storage_and_search() { let client = connect_with_tls(*PROXY).await; @@ -268,6 +290,52 @@ mod tests { assert_eq!(rows[0].get::<_, serde_json::Value>(0), plaintext); } + #[tokio::test] + async fn accepts_bare_selector_hashes_as_parameters_and_literals() { + let client = connect_with_tls(*PROXY).await; + clear_with_client(&client).await; + let id = random_id(); + let plaintext = serde_json::json!({ + "patient": { "name": "Ada Lovelace" } + }); + + client + .execute( + "INSERT INTO encrypted (id, encrypted_jsonb) VALUES ($1, $2)", + &[&id, &plaintext], + ) + .await + .unwrap(); + + let selector = query_json_selector("encrypted", "encrypted_jsonb", "$.patient.name").await; + let row = client + .query_one( + "SELECT encrypted_jsonb -> $1 FROM encrypted WHERE id = $2", + &[&selector, &id], + ) + .await + .unwrap(); + assert_eq!( + row.get::<_, serde_json::Value>(0), + serde_json::json!("Ada Lovelace") + ); + + let row = client + .simple_query(&format!( + "SELECT jsonb_path_query_first(encrypted_jsonb, '{selector}') \ + FROM encrypted WHERE id = '{id}'" + )) + .await + .unwrap() + .into_iter() + .find_map(|message| match message { + tokio_postgres::SimpleQueryMessage::Row(row) => row.get(0).map(str::to_owned), + _ => None, + }) + .expect("bare selector literal must return an extracted value"); + assert_eq!(row, "\"Ada Lovelace\""); + } + #[tokio::test] async fn rejects_payload_for_a_different_destination_with_generic_error() { let client = connect_with_tls(*PROXY).await; diff --git a/packages/cipherstash-proxy/src/postgresql/inbound_eql.rs b/packages/cipherstash-proxy/src/postgresql/inbound_eql.rs index 0c7bb3b0..83246c0e 100644 --- a/packages/cipherstash-proxy/src/postgresql/inbound_eql.rs +++ b/packages/cipherstash-proxy/src/postgresql/inbound_eql.rs @@ -6,6 +6,13 @@ use cipherstash_client::{ use eql_mapper::EqlTermVariant; use serde_json::Value; +/// Tokenized SteVec selectors are 16 bytes rendered as lowercase hexadecimal. +/// +/// Plaintext selectors can also match this format. In a JSON selector query +/// position that ambiguity is intentionally resolved in favour of treating a +/// match as an application-generated query operand. +const SELECTOR_HASH_LEN: usize = 32; + /// An application-generated EQL value entering Proxy. #[derive(Debug)] pub enum InboundEql { @@ -26,6 +33,20 @@ pub fn parse( column: &Column, query_operand: bool, ) -> Result, EncryptError> { + if query_operand + && matches!( + column.eql_term, + EqlTermVariant::JsonAccessor | EqlTermVariant::JsonPath + ) + && is_selector_hash(bytes) + { + let selector = String::from_utf8(bytes.to_vec()) + .map_err(|_| EncryptError::InvalidInboundCiphertext)?; + let query = EqlQueryPayload::Selector(selector); + validate_query_metadata(&query, column)?; + return Ok(Some(InboundEql::Query(query))); + } + let Ok(value) = serde_json::from_slice::(bytes) else { return Ok(None); }; @@ -62,6 +83,14 @@ pub fn parse( Ok(Some(InboundEql::Query(query))) } +/// Equivalent to the static selector-hash regex `^[0-9a-f]{32}$`. +fn is_selector_hash(bytes: &[u8]) -> bool { + bytes.len() == SELECTOR_HASH_LEN + && bytes + .iter() + .all(|byte| byte.is_ascii_digit() || matches!(byte, b'a'..=b'f')) +} + fn validate_storage_metadata( ciphertext: &EqlCiphertext, column: &Column, @@ -151,10 +180,21 @@ fn validate_query_metadata(query: &EqlQueryPayload, column: &Column) -> Result<( } Ok(()) } - // Bare selector hashes are indistinguishable from ordinary plaintext - // text on the PostgreSQL wire, so they cannot safely advertise - // themselves as pre-computed query operands. - EqlQueryPayload::Selector(_) => Err(EncryptError::InvalidInboundCiphertext), + EqlQueryPayload::Selector(selector) => { + let configured = column + .config + .indexes + .iter() + .any(|index| matches!(index.index_type, IndexType::SteVec { .. })); + let query_shape = matches!( + column.eql_term, + EqlTermVariant::JsonAccessor | EqlTermVariant::JsonPath + ); + if !configured || !query_shape || !is_selector_hash(selector.as_bytes()) { + return Err(EncryptError::InvalidInboundCiphertext); + } + Ok(()) + } } } @@ -477,4 +517,27 @@ mod tests { Err(EncryptError::InvalidInboundCiphertext) )); } + + #[test] + fn bare_selector_hash_is_accepted_for_a_json_accessor_query_operand() { + let mut column = ste_vec_column(); + column.eql_term = EqlTermVariant::JsonAccessor; + + assert!(matches!( + parse(b"0123456789abcdef0123456789abcdef", &column, true), + Ok(Some(InboundEql::Query(EqlQueryPayload::Selector(selector)))) + if selector == "0123456789abcdef0123456789abcdef" + )); + } + + #[test] + fn selector_that_does_not_match_hash_format_remains_plaintext() { + let mut column = ste_vec_column(); + column.eql_term = EqlTermVariant::JsonAccessor; + + assert!(parse(b"patient.name", &column, true).unwrap().is_none()); + assert!(parse(b"0123456789ABCDEF0123456789ABCDEF", &column, true) + .unwrap() + .is_none()); + } } diff --git a/packages/cipherstash-proxy/src/postgresql/middleware/frontend.rs b/packages/cipherstash-proxy/src/postgresql/middleware/frontend.rs index 06c4493d..4937a0c6 100644 --- a/packages/cipherstash-proxy/src/postgresql/middleware/frontend.rs +++ b/packages/cipherstash-proxy/src/postgresql/middleware/frontend.rs @@ -562,7 +562,7 @@ impl Frontend { inbound_eql::parse( value.as_bytes(), column, - typed_statement.query_operands.contains_literal(literal), + literal_is_query_operand(typed_statement, literal, column), ) .map_err(Error::from) }) @@ -583,11 +583,15 @@ impl Frontend { self.merge_inbound_eql(&mut encrypted, inbound, literal_columns) .await?; - for ((_, literal), encrypted) in literal_values.iter().zip(encrypted.iter_mut()) { - project_query_operand( - typed_statement.query_operands.contains_literal(literal), - encrypted, - ); + for (((_, literal), column), encrypted) in literal_values + .iter() + .zip(literal_columns) + .zip(encrypted.iter_mut()) + { + let query_operand = column + .as_ref() + .is_some_and(|column| literal_is_query_operand(typed_statement, literal, column)); + project_query_operand(query_operand, encrypted); } debug!(target: MAPPER, @@ -1342,6 +1346,20 @@ fn project_query_operand(query_operand: bool, encrypted: &mut Option) } } +/// JSON path/accessor literals are query operands even though they are passed +/// as bare text and therefore need no query-domain cast. All other literals +/// use the predicate roles recorded by EQL Mapper. +fn literal_is_query_operand( + typed_statement: &TypeCheckedStatement<'_>, + literal: &ast::Value, + column: &Column, +) -> bool { + matches!( + column.eql_term, + EqlTermVariant::JsonAccessor | EqlTermVariant::JsonPath + ) || typed_statement.query_operands.contains_literal(literal) +} + fn literals_to_plaintext( typed_statement: &TypeCheckedStatement<'_>, literal_columns: &Vec>, diff --git a/packages/showcase/README.md b/packages/showcase/README.md index 5bb245bf..aa1297d7 100644 --- a/packages/showcase/README.md +++ b/packages/showcase/README.md @@ -483,9 +483,13 @@ INSERT INTO patients (id, pii) VALUES ($1, $2); -- payload parameter INSERT INTO patients (id, pii) VALUES ('...', '{...}'); -- payload literal SELECT id FROM patients WHERE pii @> $1; -- query-only SEM parameter SELECT id FROM patients WHERE pii @> '{...}'; -- query-only SEM literal +SELECT pii -> $1 FROM patients; -- bare selector-hash parameter +SELECT jsonb_path_query_first(pii, '') FROM patients; -- selector-hash literal ``` -For stored payloads, Proxy authenticates the ciphertext, checks that its identifier and authenticated descriptor match `patients.pii`, and independently re-derives every SEM term from the decrypted plaintext. Query-only payloads deliberately contain no source ciphertext, so authentication is impossible and unnecessary: Proxy validates that their shape is valid for `patients.pii` and accepts them only in predicate positions, where incorrect terms can only produce incorrect query results and cannot poison stored data. Both forms are forwarded without double encryption. +For stored payloads, Proxy authenticates the ciphertext, checks that its identifier and authenticated descriptor match `patients.pii`, and independently re-derives every SEM term from the decrypted plaintext. Query-only payloads deliberately contain no source ciphertext, so authentication is impossible and unnecessary: Proxy validates that their shape is valid for `patients.pii` and accepts them only in query positions, where incorrect terms can only produce incorrect query results and cannot poison stored data. Both forms are forwarded without double encryption. + +A SteVec path selector is a bare tokenized-selector hash matching `^[0-9a-f]{32}$`. In a JSON selector query position, Proxy treats a matching value as already hashed; anything not matching that format is treated as plaintext and encrypted normally. This is intentionally ambiguous: plaintext selectors are a superset of the hash format, so a genuine plaintext selector consisting of exactly 32 lowercase hexadecimal characters is also treated as already hashed. Applications with such a field name must currently query it using an application-generated selector hash. Each test section provides detailed output showing: - ✅ Successful query execution diff --git a/packages/showcase/src/pre_encrypted.rs b/packages/showcase/src/pre_encrypted.rs index 9aa5ab8b..cdd6bff5 100644 --- a/packages/showcase/src/pre_encrypted.rs +++ b/packages/showcase/src/pre_encrypted.rs @@ -101,6 +101,36 @@ pub async fn run_examples() -> Result<(), Box> { }); assert!(matched); println!("✅ Queried with application-generated SEM terms as a SQL literal"); + + // Example 5: SteVec path selectors are bare, 32-character lowercase hex + // query terms. Proxy recognises and forwards an application-generated hash + // instead of hashing it again. + let parameter_selector = query_patient_selector("$.first_name").await?; + let row = client + .query_one( + "SELECT pii -> $1 FROM patients WHERE id = $2", + &[¶meter_selector, ¶meter_id], + ) + .await?; + assert_eq!(row.get::<_, Value>(0), json!("Ada")); + println!("✅ Queried with an application-generated selector hash parameter"); + + // Example 6: selector hashes work as literals too. A plaintext selector + // matching the same format is ambiguous and is intentionally interpreted + // as already hashed; see the showcase README for the compatibility rule. + let literal_selector = query_patient_selector("$.first_name").await?; + let rows = client + .simple_query(&format!( + "SELECT jsonb_path_query_first(pii, '{literal_selector}') \ + FROM patients WHERE id = '{literal_id}'" + )) + .await?; + let selected = rows.iter().find_map(|message| match message { + tokio_postgres::SimpleQueryMessage::Row(row) => row.get(0), + _ => None, + }); + assert_eq!(selected, Some("\"Grace\"")); + println!("✅ Queried with an application-generated selector hash literal"); Ok(()) } @@ -145,6 +175,30 @@ async fn query_patient_pii(value: Value) -> Result Result> { + let config = patient_pii_config(); + let index_type = config.indexes[0].index_type.clone(); + let prepared = PreparedPlaintext::new( + Cow::Owned(config), + Identifier::new("patients", "pii"), + Plaintext::from(path), + EqlOperation::Query(&index_type, QueryOp::SteVecSelector), + ); + let mut outputs = encrypt_eql_v3( + scoped_cipher().await?, + vec![prepared], + &EqlEncryptOpts::default(), + ) + .await?; + let EqlOutputV3::Query(query) = outputs.remove(0) else { + return Err("selector encryption returned a storage payload".into()); + }; + let Value::String(selector) = serde_json::to_value(query)? else { + return Err("selector encryption returned a non-selector query payload".into()); + }; + Ok(selector) +} + fn patient_pii_config() -> ColumnConfig { ColumnConfig::build("patients/pii") .casts_as(ColumnType::Json) From e7db1e0e0bdfb3fc2c4c9430d5d4e70a4a7a69a9 Mon Sep 17 00:00:00 2001 From: James Sadler Date: Sun, 23 Aug 2026 22:24:28 +1000 Subject: [PATCH 08/11] fix(proxy): close inbound EQL validation gaps Signed-off-by: James Sadler --- CHANGELOG.md | 4 +- docs/errors.md | 48 ++++++++ .../src/inbound_ciphertext.rs | 108 +++++++++++++++++- packages/cipherstash-proxy/src/error.rs | 5 +- .../src/postgresql/diagnostics.rs | 9 ++ .../src/postgresql/error_handler.rs | 18 ++- .../src/postgresql/inbound_eql.rs | 37 +++--- .../src/postgresql/middleware/frontend.rs | 38 +++++- 8 files changed, 236 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 28949522..7b079e61 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Added -- **Application-generated EQL values in statements**: SQL literals and bound parameters may now carry EQL v3 storage payloads or query-only SEM operands produced by an application. Proxy authenticates stored ciphertext, requires its authenticated descriptor to name the inferred destination column, and independently re-derives every SEM term before forwarding it without double encryption. Query-only operands contain no ciphertext to authenticate, so Proxy instead validates their version, identifier, term shape, column capabilities, and syntactic query role; they are rejected in storage positions. This includes bare SteVec selector hashes matching `^[0-9a-f]{32}$`: in JSON selector query positions a match is treated as already hashed, while a non-match remains plaintext and is encrypted normally. A matching plaintext selector is inherently ambiguous and is intentionally treated as already hashed. Invalid payloads fail closed with one generic error so validation details cannot be used as an oracle. +- **Application-generated EQL values in statements**: SQL literals and bound parameters may now carry EQL v3 storage payloads or query-only SEM operands produced by an application. Proxy authenticates every stored ciphertext, requires its authenticated descriptor to name the inferred destination column, and independently re-derives every SEM term before forwarding it without double encryption. Query-only operands contain no ciphertext to authenticate, so Proxy instead validates their version, identifier, term shape, column capabilities, and syntactic query role; they are rejected in storage positions. This includes bare SteVec selector hashes matching `^[0-9a-f]{32}$`: in JSON selector query positions a match is treated as already hashed, while a non-match remains plaintext and is encrypted normally. A matching plaintext selector is inherently ambiguous and is intentionally treated as already hashed. Invalid payloads fail closed with one generic, transaction-aborting error so validation details cannot be used as an oracle. + + Compatibility note: on encrypted columns, a JSON object with top-level `v` and `i` keys plus at least one of `c`, `h`, or `sv` is reserved as an advertised EQL storage payload. If it is not valid EQL, Proxy rejects it instead of encrypting it as plaintext, and there is no opt-out. Before upgrading, audit plaintext application writes for this key combination; [Invalid encrypted value](docs/errors.md#encrypt-invalid-inbound-ciphertext) includes a `jsonb` scan predicate. ### Changed diff --git a/docs/errors.md b/docs/errors.md index 53b84444..c735245d 100644 --- a/docs/errors.md +++ b/docs/errors.md @@ -16,6 +16,7 @@ - [Internal Error](#mapping-internal-error) - Encrypt errors: + - [Invalid encrypted value](#encrypt-invalid-inbound-ciphertext) - [Column could not be encrypted](#encrypt-column-could-not-be-encrypted) - [Could not decrypt data for keyset](#encrypt-could-not-decrypt-data-for-keyset) - [KeysetId could not be parsed](#encrypt-keyset-id-could-not-be-parsed) @@ -345,6 +346,53 @@ If the error persists, please contact CipherStash [support](https://cipherstash. # Encrypt errors +## Invalid encrypted value + +CipherStash Proxy rejected an application-generated EQL storage payload or +query operand because it was malformed, unauthentic, intended for another +column, carried unexpected searchable encrypted metadata (SEM), or appeared in +an invalid statement position. The response deliberately does not identify +which validation failed. + +### Error message + +``` +Invalid encrypted value. For help visit https://github.com/cipherstash/proxy/blob/main/docs/errors.md#encrypt-invalid-inbound-ciphertext +``` + +### How to fix + +Regenerate the payload using the same column configuration, keyset, and +credentials as Proxy. Storage payloads must target the inferred destination +column and carry ciphertext plus exactly its configured SEM terms. Query-only +payloads must be used only in query positions. + +### Plaintext compatibility + +On an encrypted column, Proxy treats a JSON object as an advertised EQL storage +payload when it has top-level `v` and `i` keys together with at least one of +`c`, `h`, or `sv`. An object matching that key pattern which is not a valid EQL +payload is rejected rather than encrypted as plaintext. There is no opt-out for +this fail-closed check. + +Before upgrading, audit JSON values supplied as plaintext to encrypted columns. +For a `jsonb` source column named `value`, this predicate identifies the +ambiguous shape: + +```sql +WHERE value ? 'v' + AND value ? 'i' + AND value ?| ARRAY['c', 'h', 'sv'] +``` + +For text sources, first restrict the scan to values that your application knows +are valid JSON, then apply the same predicate after casting them to `jsonb`. +Rename one of these top-level keys or generate the value as an EQL payload +before sending it through Proxy. + + + + ## Column could not be encrypted The column could not be encrypted. diff --git a/packages/cipherstash-proxy-integration/src/inbound_ciphertext.rs b/packages/cipherstash-proxy-integration/src/inbound_ciphertext.rs index 84356665..8e0fb18d 100644 --- a/packages/cipherstash-proxy-integration/src/inbound_ciphertext.rs +++ b/packages/cipherstash-proxy-integration/src/inbound_ciphertext.rs @@ -15,8 +15,12 @@ mod tests { }; use cipherstash_config::column::{ArrayIndexMode, IndexType, SteVecMode}; use std::{borrow::Cow, sync::Arc}; + use tokio_postgres::error::SqlState; use uuid::Uuid; + const INVALID_INBOUND_PAYLOAD: &str = "Invalid encrypted value. For help visit \ + https://github.com/cipherstash/proxy/blob/main/docs/errors.md#encrypt-invalid-inbound-ciphertext"; + async fn cipher() -> Arc> { let client_id = env("CS_CLIENT_ID", "CS_ENCRYPT__CLIENT_ID") .parse() @@ -81,6 +85,27 @@ mod tests { serde_json::to_string(&ciphertext).unwrap() } + async fn encrypt_json( + table: &str, + column: &str, + plaintext: serde_json::Value, + ) -> serde_json::Value { + let prepared = PreparedPlaintext::new( + Cow::Owned(json_search_config(table, column)), + Identifier::new(table, column), + Plaintext::Json(Some(plaintext)), + EqlOperation::Store, + ); + let mut outputs = + encrypt_eql_v3(cipher().await, vec![prepared], &EqlEncryptOpts::default()) + .await + .expect("application-side JSON encryption must succeed"); + let EqlOutputV3::Store(ciphertext) = outputs.remove(0) else { + panic!("JSON encryption must return a stored payload"); + }; + serde_json::to_value(ciphertext).unwrap() + } + async fn query_text(table: &str, column: &str, plaintext: &str) -> String { let stored: EqlCiphertextV3 = serde_json::from_str(&encrypt_text(table, column, plaintext).await).unwrap(); @@ -257,7 +282,7 @@ mod tests { .expect_err("query-only payloads must not be accepted for storage"); assert_eq!( error.as_db_error().unwrap().message(), - "Invalid encrypted value" + INVALID_INBOUND_PAYLOAD ); } @@ -336,6 +361,83 @@ mod tests { assert_eq!(row, "\"Ada Lovelace\""); } + #[tokio::test] + async fn rejects_a_ste_vec_payload_with_tampered_non_root_ciphertext() { + let client = connect_with_tls(*PROXY).await; + clear_with_client(&client).await; + let id = random_id(); + let mut payload = encrypt_json( + "encrypted", + "encrypted_jsonb", + serde_json::json!({ + "patient": { "name": "Ada Lovelace", "active": true } + }), + ) + .await; + + let entries = payload["sv"].as_array_mut().unwrap(); + assert!(entries.len() > 1); + let mut ciphertext = entries[1]["c"] + .as_str() + .unwrap() + .chars() + .collect::>(); + let different = ciphertext + .iter() + .position(|candidate| *candidate != ciphertext[0]) + .unwrap(); + ciphertext.swap(0, different); + entries[1]["c"] = ciphertext.into_iter().collect::().into(); + + let error = client + .execute( + "INSERT INTO encrypted (id, encrypted_jsonb) VALUES ($1, $2)", + &[&id, &payload], + ) + .await + .expect_err("tampered non-root ciphertext must be rejected"); + assert_eq!( + error.as_db_error().unwrap().message(), + INVALID_INBOUND_PAYLOAD + ); + } + + #[tokio::test] + async fn invalid_payload_aborts_only_the_current_transaction() { + let mut client = connect_with_tls(*PROXY).await; + clear_with_client(&client).await; + let id = random_id(); + let malformed = serde_json::json!({ + "v": 3, + "i": { "t": "encrypted", "c": "encrypted_jsonb" }, + "c": "not a ciphertext" + }); + let transaction = client.transaction().await.unwrap(); + + let error = transaction + .execute( + "INSERT INTO encrypted (id, encrypted_jsonb) VALUES ($1, $2)", + &[&id, &malformed], + ) + .await + .expect_err("invalid payload must abort the statement"); + assert_eq!(error.code(), Some(&SqlState::INVALID_TEXT_REPRESENTATION)); + assert_eq!( + error.as_db_error().unwrap().message(), + INVALID_INBOUND_PAYLOAD + ); + + let aborted = transaction + .query_one("SELECT 1", &[]) + .await + .expect_err("transaction must remain aborted until rollback"); + assert_eq!(aborted.code(), Some(&SqlState::IN_FAILED_SQL_TRANSACTION)); + transaction.rollback().await.unwrap(); + + let row = client.query_one("SELECT 1", &[]).await.unwrap(); + assert_eq!(row.get::<_, i32>(0), 1); + } + #[tokio::test] async fn rejects_payload_for_a_different_destination_with_generic_error() { let client = connect_with_tls(*PROXY).await; @@ -355,7 +457,7 @@ mod tests { .expect_err("destination mismatch must fail closed"); assert_eq!( error.as_db_error().unwrap().message(), - "Invalid encrypted value" + INVALID_INBOUND_PAYLOAD ); } @@ -388,7 +490,7 @@ mod tests { .expect_err("spliced SEM terms must fail closed"); assert_eq!( error.as_db_error().unwrap().message(), - "Invalid encrypted value" + INVALID_INBOUND_PAYLOAD ); } } diff --git a/packages/cipherstash-proxy/src/error.rs b/packages/cipherstash-proxy/src/error.rs index 18bd9e42..92bc2678 100644 --- a/packages/cipherstash-proxy/src/error.rs +++ b/packages/cipherstash-proxy/src/error.rs @@ -259,7 +259,10 @@ pub enum EncryptError { /// Deliberately contains no payload or validation detail: inbound /// ciphertext failures are attacker-controlled and detailed responses can /// become an oracle. - #[error("Invalid encrypted value")] + #[error( + "Invalid encrypted value. For help visit {}#encrypt-invalid-inbound-ciphertext", + ERROR_DOC_BASE_URL + )] InvalidInboundCiphertext, #[error(transparent)] diff --git a/packages/cipherstash-proxy/src/postgresql/diagnostics.rs b/packages/cipherstash-proxy/src/postgresql/diagnostics.rs index 3bd0a8dd..ab44421b 100644 --- a/packages/cipherstash-proxy/src/postgresql/diagnostics.rs +++ b/packages/cipherstash-proxy/src/postgresql/diagnostics.rs @@ -70,6 +70,15 @@ pub fn invalid_parameter(message: String, table: &str, column: &str) -> Diagnost ]) } +/// Invalid application-generated EQL payload. +/// +/// This is a statement error rather than a connection failure: PostgreSQL +/// aborts the current transaction, while the connection remains usable after +/// the client rolls it back. +pub fn invalid_encrypted_value(message: String) -> DiagnosticResponse { + standard("ERROR", CODE_INVALID_TEXT_REPRESENTATION, message) +} + pub fn unknown_column(message: String, table: &str, column: &str) -> DiagnosticResponse { response([ (b'S', "ERROR".to_owned()), diff --git a/packages/cipherstash-proxy/src/postgresql/error_handler.rs b/packages/cipherstash-proxy/src/postgresql/error_handler.rs index 2478f7b1..b0db884d 100644 --- a/packages/cipherstash-proxy/src/postgresql/error_handler.rs +++ b/packages/cipherstash-proxy/src/postgresql/error_handler.rs @@ -27,6 +27,7 @@ pub trait PostgreSqlErrorHandler { /// - `EncryptError::UnknownColumn` -> Unknown column error /// - `EncryptError::CouldNotDecryptDataForKeyset` -> System error /// - `EncryptError::UnknownKeysetIdentifier` -> System error + /// - `EncryptError::InvalidInboundCiphertext` -> Invalid encrypted value error /// - `Error::ConnectionTimeout` -> Idle session timeout error /// - All others -> System error /// @@ -53,6 +54,9 @@ pub trait PostgreSqlErrorHandler { Error::Encrypt(EncryptError::UnknownKeysetIdentifier { .. }) => { diagnostics::system_error(err.to_string()) } + Error::Encrypt(EncryptError::InvalidInboundCiphertext) => { + diagnostics::invalid_encrypted_value(err.to_string()) + } Error::ConnectionTimeout { .. } => diagnostics::connection_timeout(err.to_string()), Error::Protocol( ProtocolError::HeldDataRowMissingOperation @@ -70,7 +74,9 @@ pub trait PostgreSqlErrorHandler { #[cfg(test)] mod tests { use super::*; - use crate::postgresql::diagnostics::{CODE_IDLE_SESSION_TIMEOUT, CODE_SYSTEM_ERROR}; + use crate::postgresql::diagnostics::{ + self, CODE_IDLE_SESSION_TIMEOUT, CODE_INVALID_TEXT_REPRESENTATION, CODE_SYSTEM_ERROR, + }; use std::time::Duration; /// Minimal implementation of PostgreSqlErrorHandler for testing the default method. @@ -128,4 +134,14 @@ mod tests { )) ); } + + #[test] + fn invalid_inbound_ciphertext_maps_to_a_nonfatal_statement_error() { + let handler = TestHandler; + let err = Error::Encrypt(EncryptError::InvalidInboundCiphertext); + let response = handler.error_to_response(err); + + assert_eq!(field(&response, b'C'), Some(CODE_INVALID_TEXT_REPRESENTATION)); + assert!(!diagnostics::is_fatal(&response)); + } } diff --git a/packages/cipherstash-proxy/src/postgresql/inbound_eql.rs b/packages/cipherstash-proxy/src/postgresql/inbound_eql.rs index 83246c0e..d674ae6d 100644 --- a/packages/cipherstash-proxy/src/postgresql/inbound_eql.rs +++ b/packages/cipherstash-proxy/src/postgresql/inbound_eql.rs @@ -116,12 +116,7 @@ fn validate_storage_metadata( match ciphertext { EqlCiphertext::Encrypted(payload) => validate_scalar_terms(payload, column), EqlCiphertext::SteVec(payload) => { - let configured = column - .config - .indexes - .iter() - .any(|index| matches!(index.index_type, IndexType::SteVec { .. })); - if !configured || payload.ste_vec.is_empty() { + if !has_ste_vec_terms(column) || payload.ste_vec.is_empty() { return Err(EncryptError::InvalidInboundCiphertext); } Ok(()) @@ -147,12 +142,7 @@ fn validate_query_metadata(query: &EqlQueryPayload, column: &Column) -> Result<( ) } EqlTermVariant::JsonOrd => { - let ste_vec_configured = column - .config - .indexes - .iter() - .any(|index| matches!(index.index_type, IndexType::SteVec { .. })); - if !ste_vec_configured + if !has_ste_vec_terms(column) || payload.hmac_256.is_some() || payload.bloom_filter.is_some() || payload.ore_block_u64_8_256.is_some() @@ -166,31 +156,22 @@ fn validate_query_metadata(query: &EqlQueryPayload, column: &Column) -> Result<( } } EqlQueryPayload::SteVec(payload) => { - let configured = column - .config - .indexes - .iter() - .any(|index| matches!(index.index_type, IndexType::SteVec { .. })); let query_shape = matches!( column.eql_term, EqlTermVariant::Full | EqlTermVariant::Partial | EqlTermVariant::JsonValueSelector ); - if !configured || !query_shape || payload.ste_vec.is_empty() { + if !has_ste_vec_terms(column) || !query_shape || payload.ste_vec.is_empty() { return Err(EncryptError::InvalidInboundCiphertext); } Ok(()) } EqlQueryPayload::Selector(selector) => { - let configured = column - .config - .indexes - .iter() - .any(|index| matches!(index.index_type, IndexType::SteVec { .. })); let query_shape = matches!( column.eql_term, EqlTermVariant::JsonAccessor | EqlTermVariant::JsonPath ); - if !configured || !query_shape || !is_selector_hash(selector.as_bytes()) { + if !has_ste_vec_terms(column) || !query_shape || !is_selector_hash(selector.as_bytes()) + { return Err(EncryptError::InvalidInboundCiphertext); } Ok(()) @@ -198,6 +179,14 @@ fn validate_query_metadata(query: &EqlQueryPayload, column: &Column) -> Result<( } } +fn has_ste_vec_terms(column: &Column) -> bool { + column + .config + .indexes + .iter() + .any(|index| matches!(index.index_type, IndexType::SteVec { .. })) +} + /// Compare all searchable metadata after the plaintext has been authenticated /// and independently re-encrypted for the inferred destination column. /// `into_query_operand` removes only record ciphertext/key material, leaving diff --git a/packages/cipherstash-proxy/src/postgresql/middleware/frontend.rs b/packages/cipherstash-proxy/src/postgresql/middleware/frontend.rs index 4937a0c6..90420764 100644 --- a/packages/cipherstash-proxy/src/postgresql/middleware/frontend.rs +++ b/packages/cipherstash-proxy/src/postgresql/middleware/frontend.rs @@ -24,7 +24,7 @@ use crate::prometheus::{ STATEMENTS_UNMAPPABLE_TOTAL, }; use crate::proxy::EncryptionService; -use crate::{EqlOutput, EqlQueryPayload}; +use crate::{EqlCiphertext, EqlOutput, EqlQueryPayload}; use cipherstash_client::encryption::Plaintext; use eql_mapper::{self, EqlMapperError, EqlTermVariant, JsonSelectorSegment, TypeCheckedStatement}; use metrics::{counter, histogram}; @@ -1190,6 +1190,42 @@ impl Frontend { EncryptError::InvalidInboundCiphertext })?; + // A SteVec document's root entry decrypts to the complete plaintext, + // but every other entry carries independent AEAD ciphertext. Verify + // those entries too before accepting the document for storage. Making + // each entry the sole member of a cloned envelope reuses the ordinary + // SteVec decrypt path, which binds its selector as nonce and AAD. + let non_root_entries = positions + .iter() + .flat_map(|(_, ciphertext, _)| match ciphertext { + EqlCiphertext::SteVec(document) => document + .ste_vec + .iter() + .skip(1) + .map(|entry| { + let mut document = document.clone(); + document.ste_vec = vec![entry.clone()]; + Some(EqlCiphertext::SteVec(document)) + }) + .collect::>(), + EqlCiphertext::Encrypted(_) => Vec::new(), + }) + .collect::>(); + if !non_root_entries.is_empty() { + self.context + .decrypt(non_root_entries) + .await + .map_err(|err| { + warn!( + target: ENCRYPT, + client_id = self.context.client_id, + msg = "Inbound EQL SteVec entry authentication failed", + error = ?err, + ); + EncryptError::InvalidInboundCiphertext + })?; + } + // Re-encrypt the authenticated plaintext for the inferred destination // and compare every derived SEM term. This detects term splicing: the // AEAD tag authenticates `c`, but the searchable metadata sits outside From 01f4163553a13e1392fe87fbfd2ee37c8b3016a8 Mon Sep 17 00:00:00 2001 From: James Sadler Date: Sun, 23 Aug 2026 22:42:15 +1000 Subject: [PATCH 09/11] fix(proxy): authenticate complete SteVec payloads Signed-off-by: James Sadler --- .../src/postgresql/middleware/frontend.rs | 38 +-------- .../src/proxy/zerokms/zerokms.rs | 81 ++++++++++--------- 2 files changed, 43 insertions(+), 76 deletions(-) diff --git a/packages/cipherstash-proxy/src/postgresql/middleware/frontend.rs b/packages/cipherstash-proxy/src/postgresql/middleware/frontend.rs index 90420764..4937a0c6 100644 --- a/packages/cipherstash-proxy/src/postgresql/middleware/frontend.rs +++ b/packages/cipherstash-proxy/src/postgresql/middleware/frontend.rs @@ -24,7 +24,7 @@ use crate::prometheus::{ STATEMENTS_UNMAPPABLE_TOTAL, }; use crate::proxy::EncryptionService; -use crate::{EqlCiphertext, EqlOutput, EqlQueryPayload}; +use crate::{EqlOutput, EqlQueryPayload}; use cipherstash_client::encryption::Plaintext; use eql_mapper::{self, EqlMapperError, EqlTermVariant, JsonSelectorSegment, TypeCheckedStatement}; use metrics::{counter, histogram}; @@ -1190,42 +1190,6 @@ impl Frontend { EncryptError::InvalidInboundCiphertext })?; - // A SteVec document's root entry decrypts to the complete plaintext, - // but every other entry carries independent AEAD ciphertext. Verify - // those entries too before accepting the document for storage. Making - // each entry the sole member of a cloned envelope reuses the ordinary - // SteVec decrypt path, which binds its selector as nonce and AAD. - let non_root_entries = positions - .iter() - .flat_map(|(_, ciphertext, _)| match ciphertext { - EqlCiphertext::SteVec(document) => document - .ste_vec - .iter() - .skip(1) - .map(|entry| { - let mut document = document.clone(); - document.ste_vec = vec![entry.clone()]; - Some(EqlCiphertext::SteVec(document)) - }) - .collect::>(), - EqlCiphertext::Encrypted(_) => Vec::new(), - }) - .collect::>(); - if !non_root_entries.is_empty() { - self.context - .decrypt(non_root_entries) - .await - .map_err(|err| { - warn!( - target: ENCRYPT, - client_id = self.context.client_id, - msg = "Inbound EQL SteVec entry authentication failed", - error = ?err, - ); - EncryptError::InvalidInboundCiphertext - })?; - } - // Re-encrypt the authenticated plaintext for the inferred destination // and compare every derived SEM term. This detects term splicing: the // AEAD tag authenticates `c`, but the searchable metadata sits outside diff --git a/packages/cipherstash-proxy/src/proxy/zerokms/zerokms.rs b/packages/cipherstash-proxy/src/proxy/zerokms/zerokms.rs index b602006d..be52166f 100644 --- a/packages/cipherstash-proxy/src/proxy/zerokms/zerokms.rs +++ b/packages/cipherstash-proxy/src/proxy/zerokms/zerokms.rs @@ -46,9 +46,9 @@ enum V3Record { /// A scalar payload's `c` — self-describing, nonce derived from the data /// key's IV, nothing bound into the AAD. Scalar(EncryptedRecord), - /// A SteVec document's root entry, reassembled from the document's `h` - /// header. Nonce and AAD both derive from the entry's selector. - SteVecRoot(RecordWithNonce), + /// A SteVec entry, reassembled from the document's `h` header. Nonce and + /// AAD both derive from the entry's selector. + SteVecEntry(RecordWithNonce), } impl Decryptable for V3Record { @@ -57,35 +57,35 @@ impl Decryptable for V3Record { fn keyset_id(&self) -> Option { match self { V3Record::Scalar(record) => record.keyset_id(), - V3Record::SteVecRoot(record) => record.keyset_id(), + V3Record::SteVecEntry(record) => record.keyset_id(), } } fn retrieve_key_payload(&self) -> Result, Self::Error> { match self { V3Record::Scalar(record) => record.retrieve_key_payload(), - V3Record::SteVecRoot(record) => record.retrieve_key_payload(), + V3Record::SteVecEntry(record) => record.retrieve_key_payload(), } } fn into_encrypted_record(self) -> Result { match self { V3Record::Scalar(record) => record.into_encrypted_record(), - V3Record::SteVecRoot(record) => record.into_encrypted_record(), + V3Record::SteVecEntry(record) => record.into_encrypted_record(), } } fn nonce_override(&self) -> Option<[u8; 12]> { match self { V3Record::Scalar(_) => None, - V3Record::SteVecRoot(record) => record.nonce_override(), + V3Record::SteVecEntry(record) => record.nonce_override(), } } fn aad_selector(&self) -> Option<[u8; 16]> { match self { V3Record::Scalar(_) => None, - V3Record::SteVecRoot(record) => record.aad_selector(), + V3Record::SteVecEntry(record) => record.aad_selector(), } } } @@ -379,7 +379,8 @@ impl EncryptionService for ZeroKms { let cipher = self.init_cipher(keyset_id.clone()).await?; - // Collect indices and the root records for non-None values. + // Collect decryptable records and identify which decrypted records + // contain the plaintext returned to the caller. // // cipherstash-client has no `decrypt_eql_v3` counterpart to // `encrypt_eql_v3` — the v2 `decrypt_eql` only accepts `EqlCiphertext`. @@ -390,36 +391,39 @@ impl EncryptionService for ZeroKms { // unwrapped, and `EncryptedRecord` is `Decryptable`. // // SteVec: the document holds the key material once in the `h` header - // and each entry carries only raw AEAD bytes, so the record has to be - // reassembled from the header plus the ROOT entry (`sv[0]`, the same - // decryption-root invariant v2 had). The selector is the AEAD binding — - // its first 12 bytes are the nonce and all 16 go into the AAD — which is - // why the reassembled record is a `RecordWithNonce`. - let mut indices: Vec = Vec::new(); + // and each entry carries only raw AEAD bytes, so every record has to be + // reassembled from the header plus that entry. The selector is the AEAD + // binding — its first 12 bytes are the nonce and all 16 go into the AAD + // — which is why the records are `RecordWithNonce`. All entries are + // decrypted to authenticate them, but only the root entry (`sv[0]`) + // contains the complete plaintext returned to the caller. Value entries + // intentionally decrypt to a sentinel that is not a legal `Plaintext`. + let mut result_positions: Vec> = Vec::new(); let mut records_to_decrypt: Vec = Vec::new(); for (idx, ct_opt) in ciphertexts.iter().enumerate() { if let Some(ct) = ct_opt { - let record = match ct { + match ct { EqlCiphertextV3::Encrypted(payload) => { - V3Record::Scalar(payload.ciphertext.clone()) + records_to_decrypt.push(V3Record::Scalar(payload.ciphertext.clone())); + result_positions.push(Some(idx)); } EqlCiphertextV3::SteVec(document) => { - let root = document - .ste_vec - .first() - .ok_or(EncryptError::SteVecMissingRootEntry)?; - - let selector = decode_ste_vec_selector(&root.selector)?; - V3Record::SteVecRoot( - document - .key_header - .record_with_selector(root.ciphertext.clone(), selector), - ) + if document.ste_vec.is_empty() { + return Err(EncryptError::SteVecMissingRootEntry.into()); + } + + for (entry_index, entry) in document.ste_vec.iter().enumerate() { + let selector = decode_ste_vec_selector(&entry.selector)?; + records_to_decrypt.push(V3Record::SteVecEntry( + document + .key_header + .record_with_selector(entry.ciphertext.clone(), selector), + )); + result_positions.push((entry_index == 0).then_some(idx)); + } } - }; - indices.push(idx); - records_to_decrypt.push(record); + } } } @@ -437,18 +441,17 @@ impl EncryptionService for ZeroKms { let decrypted = cipher .decrypt(records_to_decrypt, &opts) .await - .map_err(ZeroKMSError::from)? - .into_iter() - .map(|bytes| Plaintext::from_slice(&bytes)) - .collect::, _>>() - .map_err(EncryptError::from)?; + .map_err(ZeroKMSError::from)?; let decrypt_duration = decrypt_start.elapsed(); debug!(target: ENCRYPT, msg="Decrypt completed", count = decrypted.len(), duration_ms = decrypt_duration.as_millis()); - // Reconstruct the result vector with None values in the right places + // Reconstruct the result vector from scalar and SteVec-root plaintexts. + // Non-root SteVec bytes were decrypted only to authenticate them. let mut result: Vec> = vec![None; ciphertexts.len()]; - for (idx, plaintext) in indices.into_iter().zip(decrypted.into_iter()) { - result[idx] = Some(plaintext); + for (result_position, bytes) in result_positions.into_iter().zip(decrypted) { + if let Some(idx) = result_position { + result[idx] = Some(Plaintext::from_slice(&bytes).map_err(EncryptError::from)?); + } } Ok(result) From 96f22cd784d376fe1404de2f985f3b94817942e8 Mon Sep 17 00:00:00 2001 From: James Sadler Date: Wed, 26 Aug 2026 13:06:32 +1000 Subject: [PATCH 10/11] style(proxy): format rebased diagnostic test Signed-off-by: James Sadler --- packages/cipherstash-proxy/src/postgresql/error_handler.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/cipherstash-proxy/src/postgresql/error_handler.rs b/packages/cipherstash-proxy/src/postgresql/error_handler.rs index b0db884d..68290884 100644 --- a/packages/cipherstash-proxy/src/postgresql/error_handler.rs +++ b/packages/cipherstash-proxy/src/postgresql/error_handler.rs @@ -141,7 +141,10 @@ mod tests { let err = Error::Encrypt(EncryptError::InvalidInboundCiphertext); let response = handler.error_to_response(err); - assert_eq!(field(&response, b'C'), Some(CODE_INVALID_TEXT_REPRESENTATION)); + assert_eq!( + field(&response, b'C'), + Some(CODE_INVALID_TEXT_REPRESENTATION) + ); assert!(!diagnostics::is_fatal(&response)); } } From 7b58470e9084fd3601c15424c26fad8b63089bb4 Mon Sep 17 00:00:00 2001 From: James Sadler Date: Wed, 26 Aug 2026 14:30:54 +1000 Subject: [PATCH 11/11] fix: address inbound EQL review feedback Signed-off-by: James Sadler --- CHANGELOG.md | 4 +- docs/errors.md | 45 ++-- .../src/inbound_ciphertext.rs | 169 ++++++++++++++- packages/cipherstash-proxy/src/error.rs | 12 +- .../src/postgresql/context/mod.rs | 18 ++ .../src/postgresql/error_handler.rs | 8 +- .../src/postgresql/inbound_eql.rs | 196 ++++++++++++++---- .../src/postgresql/middleware/backend.rs | 8 + .../src/postgresql/middleware/frontend.rs | 86 ++++++-- packages/cipherstash-proxy/src/proxy/mod.rs | 8 + .../src/proxy/zerokms/zerokms.rs | 173 +++++++++++----- 11 files changed, 591 insertions(+), 136 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b079e61..e7d00852 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). - **Application-generated EQL values in statements**: SQL literals and bound parameters may now carry EQL v3 storage payloads or query-only SEM operands produced by an application. Proxy authenticates every stored ciphertext, requires its authenticated descriptor to name the inferred destination column, and independently re-derives every SEM term before forwarding it without double encryption. Query-only operands contain no ciphertext to authenticate, so Proxy instead validates their version, identifier, term shape, column capabilities, and syntactic query role; they are rejected in storage positions. This includes bare SteVec selector hashes matching `^[0-9a-f]{32}$`: in JSON selector query positions a match is treated as already hashed, while a non-match remains plaintext and is encrypted normally. A matching plaintext selector is inherently ambiguous and is intentionally treated as already hashed. Invalid payloads fail closed with one generic, transaction-aborting error so validation details cannot be used as an oracle. - Compatibility note: on encrypted columns, a JSON object with top-level `v` and `i` keys plus at least one of `c`, `h`, or `sv` is reserved as an advertised EQL storage payload. If it is not valid EQL, Proxy rejects it instead of encrypting it as plaintext, and there is no opt-out. Before upgrading, audit plaintext application writes for this key combination; [Invalid encrypted value](docs/errors.md#encrypt-invalid-inbound-ciphertext) includes a `jsonb` scan predicate. + Compatibility note: on every encrypted column type, Proxy reserves three JSON object shapes for application-generated EQL: storage payloads with top-level `v` and `i` plus at least one of `c`, `h`, or `sv`; scalar query payloads with `v` and `i` plus at least one of `hm`, `bf`, `ob`, or `op`; and SteVec query payloads whose only top-level key is `sv`. An object matching one of these shapes is validated as EQL rather than encrypted as plaintext, and there is no opt-out. Query-only shapes are rejected in storage positions. Before upgrading, audit plaintext application writes for these key combinations; [Invalid encrypted value](docs/errors.md#encrypt-invalid-inbound-eql-payload) includes a `jsonb` scan predicate. ### Changed @@ -18,6 +18,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Fixed +- **Configured default keyset selection**: when a connection has not selected a keyset explicitly, Proxy now scopes encryption and decryption to `CS_DEFAULT_KEYSET_ID`. Previously it passed no keyset to ZeroKMS and could silently use the account default instead, deriving different searchable-encryption terms when the two defaults differed. Before upgrading, verify that the configured and account defaults are intentional; values written by an affected version under the unintended account default must be decrypted with that old keyset and re-encrypted under the configured default. + - **PostgreSQL protocol error handling after the pg-proto migration**: Proxy now rejects `require_tls` configurations that omit a certificate, preserves PostgreSQL transaction state when statement mapping fails, returns decryption failures as PostgreSQL errors without dropping the connection, and reloads changed schemas only after PostgreSQL confirms the transaction boundary. Prepared-statement replacement also preserves existing portals and overlapping statement metrics. ## [3.0.1] - 2026-08-05 diff --git a/docs/errors.md b/docs/errors.md index c735245d..d7267cce 100644 --- a/docs/errors.md +++ b/docs/errors.md @@ -16,7 +16,7 @@ - [Internal Error](#mapping-internal-error) - Encrypt errors: - - [Invalid encrypted value](#encrypt-invalid-inbound-ciphertext) + - [Invalid encrypted value](#encrypt-invalid-inbound-eql-payload) - [Column could not be encrypted](#encrypt-column-could-not-be-encrypted) - [Could not decrypt data for keyset](#encrypt-could-not-decrypt-data-for-keyset) - [KeysetId could not be parsed](#encrypt-keyset-id-could-not-be-parsed) @@ -346,7 +346,7 @@ If the error persists, please contact CipherStash [support](https://cipherstash. # Encrypt errors -## Invalid encrypted value +## Invalid encrypted value CipherStash Proxy rejected an application-generated EQL storage payload or query operand because it was malformed, unauthentic, intended for another @@ -357,7 +357,7 @@ which validation failed. ### Error message ``` -Invalid encrypted value. For help visit https://github.com/cipherstash/proxy/blob/main/docs/errors.md#encrypt-invalid-inbound-ciphertext +Invalid encrypted value. For help visit https://github.com/cipherstash/proxy/blob/main/docs/errors.md#encrypt-invalid-inbound-eql-payload ``` ### How to fix @@ -369,20 +369,39 @@ payloads must be used only in query positions. ### Plaintext compatibility -On an encrypted column, Proxy treats a JSON object as an advertised EQL storage -payload when it has top-level `v` and `i` keys together with at least one of -`c`, `h`, or `sv`. An object matching that key pattern which is not a valid EQL -payload is rejected rather than encrypted as plaintext. There is no opt-out for -this fail-closed check. +On every encrypted column type, Proxy reserves three JSON object shapes for +application-generated EQL: + +- a storage payload with top-level `v` and `i` keys plus at least one of `c`, + `h`, or `sv`; +- a scalar query payload with top-level `v` and `i` keys plus at least one of + `hm`, `bf`, `ob`, or `op`; or +- a SteVec query payload whose only top-level key is `sv`. + +An object matching one of these patterns is validated as EQL rather than +encrypted as plaintext. Invalid EQL is rejected, and query-only payloads are +rejected in storage positions. There is no opt-out for this fail-closed check. Before upgrading, audit JSON values supplied as plaintext to encrypted columns. -For a `jsonb` source column named `value`, this predicate identifies the -ambiguous shape: +For a `jsonb` source column named `value`, this predicate identifies all three +reserved shapes: ```sql -WHERE value ? 'v' - AND value ? 'i' - AND value ?| ARRAY['c', 'h', 'sv'] +WHERE CASE + WHEN jsonb_typeof(value) <> 'object' THEN false + ELSE ( + value ? 'v' + AND value ? 'i' + AND value ?| ARRAY['c', 'h', 'sv', 'hm', 'bf', 'ob', 'op'] + ) OR ( + value ? 'sv' + AND NOT EXISTS ( + SELECT 1 + FROM jsonb_object_keys(value) AS keys(candidate_key) + WHERE candidate_key <> 'sv' + ) + ) +END ``` For text sources, first restrict the scan to values that your application knows diff --git a/packages/cipherstash-proxy-integration/src/inbound_ciphertext.rs b/packages/cipherstash-proxy-integration/src/inbound_ciphertext.rs index 8e0fb18d..624b5c6b 100644 --- a/packages/cipherstash-proxy-integration/src/inbound_ciphertext.rs +++ b/packages/cipherstash-proxy-integration/src/inbound_ciphertext.rs @@ -19,7 +19,7 @@ mod tests { use uuid::Uuid; const INVALID_INBOUND_PAYLOAD: &str = "Invalid encrypted value. For help visit \ - https://github.com/cipherstash/proxy/blob/main/docs/errors.md#encrypt-invalid-inbound-ciphertext"; + https://github.com/cipherstash/proxy/blob/main/docs/errors.md#encrypt-invalid-inbound-eql-payload"; async fn cipher() -> Arc> { let client_id = env("CS_CLIENT_ID", "CS_ENCRYPT__CLIENT_ID") @@ -158,7 +158,7 @@ mod tests { } #[tokio::test] - async fn accepts_pre_encrypted_parameter_for_storage_and_search() { + async fn accepts_pre_encrypted_parameter_with_the_configured_default_keyset() { let client = connect_with_tls(*PROXY).await; clear_with_client(&client).await; let id = random_id(); @@ -183,6 +183,35 @@ mod tests { assert_eq!(rows[0].get::<_, String>(0), plaintext); } + #[tokio::test] + async fn accepts_pre_encrypted_ste_vec_parameter_for_storage_and_readback() { + let client = connect_with_tls(*PROXY).await; + clear_with_client(&client).await; + let id = random_id(); + let plaintext = serde_json::json!({ + "patient": { "name": "Ada Lovelace" }, + "allergies": ["pollen", "latex"] + }); + let payload = encrypt_json("encrypted", "encrypted_jsonb", plaintext.clone()).await; + + client + .execute( + "INSERT INTO encrypted (id, encrypted_jsonb) VALUES ($1, $2)", + &[&id, &payload], + ) + .await + .unwrap(); + + let row = client + .query_one( + "SELECT encrypted_jsonb FROM encrypted WHERE id = $1", + &[&id], + ) + .await + .unwrap(); + assert_eq!(row.get::<_, serde_json::Value>(0), plaintext); + } + #[tokio::test] async fn accepts_pre_encrypted_literal_for_storage() { let client = connect_with_tls(*PROXY).await; @@ -402,6 +431,142 @@ mod tests { ); } + #[tokio::test] + async fn rejects_a_ste_vec_payload_with_a_tampered_array_marker() { + let client = connect_with_tls(*PROXY).await; + clear_with_client(&client).await; + let id = random_id(); + let mut payload = encrypt_json( + "encrypted", + "encrypted_jsonb", + serde_json::json!({ "allergies": ["pollen", "latex"] }), + ) + .await; + let entry = payload["sv"] + .as_array_mut() + .unwrap() + .iter_mut() + .find(|entry| entry.get("a").is_some()) + .expect("an array value must produce an array-marked SteVec entry"); + entry["a"] = false.into(); + + let error = client + .execute( + "INSERT INTO encrypted (id, encrypted_jsonb) VALUES ($1, $2)", + &[&id, &payload], + ) + .await + .expect_err("tampered SteVec array metadata must be rejected"); + assert_eq!( + error.as_db_error().unwrap().message(), + INVALID_INBOUND_PAYLOAD + ); + } + + #[tokio::test] + async fn rejects_ste_vec_terms_spliced_from_another_plaintext() { + let client = connect_with_tls(*PROXY).await; + clear_with_client(&client).await; + let id = random_id(); + let x = encrypt_json( + "encrypted", + "encrypted_jsonb", + serde_json::json!({ "patient": { "name": "indexed as x" } }), + ) + .await; + let mut y = encrypt_json( + "encrypted", + "encrypted_jsonb", + serde_json::json!({ "patient": { "name": "decrypts as y" } }), + ) + .await; + let x_term = x["sv"] + .as_array() + .unwrap() + .iter() + .find_map(|entry| entry.get("op").cloned()) + .expect("ordered string entries must carry an ordering term"); + let y_entry = y["sv"] + .as_array_mut() + .unwrap() + .iter_mut() + .find(|entry| entry.get("op").is_some()) + .expect("ordered string entries must carry an ordering term"); + y_entry["op"] = x_term; + + let error = client + .execute( + "INSERT INTO encrypted (id, encrypted_jsonb) VALUES ($1, $2)", + &[&id, &y], + ) + .await + .expect_err("spliced SteVec SEM terms must be rejected"); + assert_eq!( + error.as_db_error().unwrap().message(), + INVALID_INBOUND_PAYLOAD + ); + } + + #[tokio::test] + async fn rejects_mismatched_keyset_metadata_for_scalar_and_ste_vec_payloads() { + let client = connect_with_tls(*PROXY).await; + clear_with_client(&client).await; + + let mut scalar: EqlCiphertextV3 = serde_json::from_str( + &encrypt_text( + "encrypted", + "encrypted_text", + "wrong scalar keyset metadata", + ) + .await, + ) + .unwrap(); + let EqlCiphertextV3::Encrypted(scalar) = &mut scalar else { + unreachable!() + }; + scalar.ciphertext.keyset_id = Some(Uuid::new_v4()); + let scalar = serde_json::to_string(&scalar).unwrap(); + + let scalar_error = client + .execute( + "INSERT INTO encrypted (id, encrypted_text) VALUES ($1, $2)", + &[&random_id(), &scalar], + ) + .await + .expect_err("false scalar keyset metadata must be rejected"); + assert_eq!( + scalar_error.as_db_error().unwrap().message(), + INVALID_INBOUND_PAYLOAD + ); + + let mut ste_vec: EqlCiphertextV3 = serde_json::from_value( + encrypt_json( + "encrypted", + "encrypted_jsonb", + serde_json::json!({ "patient": { "name": "wrong SteVec keyset metadata" } }), + ) + .await, + ) + .unwrap(); + let EqlCiphertextV3::SteVec(payload) = &mut ste_vec else { + unreachable!() + }; + payload.key_header.keyset_id = Some(Uuid::new_v4()); + let ste_vec = serde_json::to_value(&ste_vec).unwrap(); + + let ste_vec_error = client + .execute( + "INSERT INTO encrypted (id, encrypted_jsonb) VALUES ($1, $2)", + &[&random_id(), &ste_vec], + ) + .await + .expect_err("false SteVec keyset metadata must be rejected"); + assert_eq!( + ste_vec_error.as_db_error().unwrap().message(), + INVALID_INBOUND_PAYLOAD + ); + } + #[tokio::test] async fn invalid_payload_aborts_only_the_current_transaction() { let mut client = connect_with_tls(*PROXY).await; diff --git a/packages/cipherstash-proxy/src/error.rs b/packages/cipherstash-proxy/src/error.rs index 92bc2678..c1b6ec5f 100644 --- a/packages/cipherstash-proxy/src/error.rs +++ b/packages/cipherstash-proxy/src/error.rs @@ -75,7 +75,7 @@ impl Error { // stores plaintext in a column its operator believes is encrypted // (CIP-3688). No configuration may turn that back on. Error::Mapping(MappingError::UnmappableEncryptedColumn { .. }) - | Error::Encrypt(EncryptError::InvalidInboundCiphertext) + | Error::Encrypt(EncryptError::InvalidInboundEqlPayload) ) } } @@ -256,14 +256,14 @@ pub enum TlsConfigError { #[derive(Error, Debug)] pub enum EncryptError { - /// Deliberately contains no payload or validation detail: inbound - /// ciphertext failures are attacker-controlled and detailed responses can - /// become an oracle. + /// Deliberately contains no payload or validation detail: inbound EQL + /// payloads are attacker-controlled and detailed responses can become an + /// oracle. #[error( - "Invalid encrypted value. For help visit {}#encrypt-invalid-inbound-ciphertext", + "Invalid encrypted value. For help visit {}#encrypt-invalid-inbound-eql-payload", ERROR_DOC_BASE_URL )] - InvalidInboundCiphertext, + InvalidInboundEqlPayload, #[error(transparent)] CiphertextCouldNotBeSerialised(#[from] serde_json::Error), diff --git a/packages/cipherstash-proxy/src/postgresql/context/mod.rs b/packages/cipherstash-proxy/src/postgresql/context/mod.rs index 9797d20c..6d2b3436 100644 --- a/packages/cipherstash-proxy/src/postgresql/context/mod.rs +++ b/packages/cipherstash-proxy/src/postgresql/context/mod.rs @@ -800,6 +800,16 @@ where self.encryption.decrypt(keyset_id, ciphertexts).await } + pub async fn decrypt_inbound_eql( + &self, + ciphertexts: Vec>, + ) -> Result>, Error> { + let keyset_id = self.keyset_identifier(); + self.encryption + .decrypt_inbound_eql(keyset_id, ciphertexts) + .await + } + pub async fn reload_schema(&self) -> bool { let (responder, receiver) = oneshot::channel(); match self @@ -1041,6 +1051,14 @@ mod tests { ) -> Result>, Error> { Ok(vec![]) } + + async fn decrypt_inbound_eql( + &self, + _keyset_id: Option, + _ciphertexts: Vec>, + ) -> Result>, Error> { + Ok(vec![]) + } } fn create_context() -> Context { diff --git a/packages/cipherstash-proxy/src/postgresql/error_handler.rs b/packages/cipherstash-proxy/src/postgresql/error_handler.rs index 68290884..29297cfe 100644 --- a/packages/cipherstash-proxy/src/postgresql/error_handler.rs +++ b/packages/cipherstash-proxy/src/postgresql/error_handler.rs @@ -27,7 +27,7 @@ pub trait PostgreSqlErrorHandler { /// - `EncryptError::UnknownColumn` -> Unknown column error /// - `EncryptError::CouldNotDecryptDataForKeyset` -> System error /// - `EncryptError::UnknownKeysetIdentifier` -> System error - /// - `EncryptError::InvalidInboundCiphertext` -> Invalid encrypted value error + /// - `EncryptError::InvalidInboundEqlPayload` -> Invalid encrypted value error /// - `Error::ConnectionTimeout` -> Idle session timeout error /// - All others -> System error /// @@ -54,7 +54,7 @@ pub trait PostgreSqlErrorHandler { Error::Encrypt(EncryptError::UnknownKeysetIdentifier { .. }) => { diagnostics::system_error(err.to_string()) } - Error::Encrypt(EncryptError::InvalidInboundCiphertext) => { + Error::Encrypt(EncryptError::InvalidInboundEqlPayload) => { diagnostics::invalid_encrypted_value(err.to_string()) } Error::ConnectionTimeout { .. } => diagnostics::connection_timeout(err.to_string()), @@ -136,9 +136,9 @@ mod tests { } #[test] - fn invalid_inbound_ciphertext_maps_to_a_nonfatal_statement_error() { + fn invalid_inbound_eql_payload_maps_to_a_nonfatal_statement_error() { let handler = TestHandler; - let err = Error::Encrypt(EncryptError::InvalidInboundCiphertext); + let err = Error::Encrypt(EncryptError::InvalidInboundEqlPayload); let response = handler.error_to_response(err); assert_eq!( diff --git a/packages/cipherstash-proxy/src/postgresql/inbound_eql.rs b/packages/cipherstash-proxy/src/postgresql/inbound_eql.rs index d674ae6d..10f319ee 100644 --- a/packages/cipherstash-proxy/src/postgresql/inbound_eql.rs +++ b/packages/cipherstash-proxy/src/postgresql/inbound_eql.rs @@ -41,7 +41,7 @@ pub fn parse( && is_selector_hash(bytes) { let selector = String::from_utf8(bytes.to_vec()) - .map_err(|_| EncryptError::InvalidInboundCiphertext)?; + .map_err(|_| EncryptError::InvalidInboundEqlPayload)?; let query = EqlQueryPayload::Selector(selector); validate_query_metadata(&query, column)?; return Ok(Some(InboundEql::Query(query))); @@ -59,7 +59,7 @@ pub fn parse( && (object.contains_key("c") || object.contains_key("h") || object.contains_key("sv")); if storage_shaped { let ciphertext: EqlCiphertext = - serde_json::from_value(value).map_err(|_| EncryptError::InvalidInboundCiphertext)?; + serde_json::from_value(value).map_err(|_| EncryptError::InvalidInboundEqlPayload)?; validate_storage_metadata(&ciphertext, column)?; return Ok(Some(InboundEql::Store(ciphertext))); } @@ -74,11 +74,11 @@ pub fn parse( return Ok(None); } if !query_operand { - return Err(EncryptError::InvalidInboundCiphertext); + return Err(EncryptError::InvalidInboundEqlPayload); } let query: EqlQueryPayload = - serde_json::from_value(value).map_err(|_| EncryptError::InvalidInboundCiphertext)?; + serde_json::from_value(value).map_err(|_| EncryptError::InvalidInboundEqlPayload)?; validate_query_metadata(&query, column)?; Ok(Some(InboundEql::Query(query))) } @@ -98,7 +98,7 @@ fn validate_storage_metadata( if ciphertext.version() != EQL_SCHEMA_VERSION_V3 || ciphertext.identifier() != &column.identifier { - return Err(EncryptError::InvalidInboundCiphertext); + return Err(EncryptError::InvalidInboundEqlPayload); } // The descriptor is covered by the encrypted record's AEAD tag. Requiring @@ -110,14 +110,14 @@ fn validate_storage_metadata( EqlCiphertext::SteVec(payload) => &payload.key_header.descriptor, }; if descriptor != &expected_descriptor { - return Err(EncryptError::InvalidInboundCiphertext); + return Err(EncryptError::InvalidInboundEqlPayload); } match ciphertext { EqlCiphertext::Encrypted(payload) => validate_scalar_terms(payload, column), EqlCiphertext::SteVec(payload) => { if !has_ste_vec_terms(column) || payload.ste_vec.is_empty() { - return Err(EncryptError::InvalidInboundCiphertext); + return Err(EncryptError::InvalidInboundEqlPayload); } Ok(()) } @@ -128,7 +128,7 @@ fn validate_query_metadata(query: &EqlQueryPayload, column: &Column) -> Result<( match query { EqlQueryPayload::Encrypted(payload) => { if payload.version != EQL_SCHEMA_VERSION_V3 || payload.identifier != column.identifier { - return Err(EncryptError::InvalidInboundCiphertext); + return Err(EncryptError::InvalidInboundEqlPayload); } match column.eql_term { @@ -148,11 +148,11 @@ fn validate_query_metadata(query: &EqlQueryPayload, column: &Column) -> Result<( || payload.ore_block_u64_8_256.is_some() || payload.ope_cllw.is_none() { - return Err(EncryptError::InvalidInboundCiphertext); + return Err(EncryptError::InvalidInboundEqlPayload); } Ok(()) } - _ => Err(EncryptError::InvalidInboundCiphertext), + _ => Err(EncryptError::InvalidInboundEqlPayload), } } EqlQueryPayload::SteVec(payload) => { @@ -161,7 +161,7 @@ fn validate_query_metadata(query: &EqlQueryPayload, column: &Column) -> Result<( EqlTermVariant::Full | EqlTermVariant::Partial | EqlTermVariant::JsonValueSelector ); if !has_ste_vec_terms(column) || !query_shape || payload.ste_vec.is_empty() { - return Err(EncryptError::InvalidInboundCiphertext); + return Err(EncryptError::InvalidInboundEqlPayload); } Ok(()) } @@ -172,7 +172,7 @@ fn validate_query_metadata(query: &EqlQueryPayload, column: &Column) -> Result<( ); if !has_ste_vec_terms(column) || !query_shape || !is_selector_hash(selector.as_bytes()) { - return Err(EncryptError::InvalidInboundCiphertext); + return Err(EncryptError::InvalidInboundEqlPayload); } Ok(()) } @@ -189,25 +189,42 @@ fn has_ste_vec_terms(column: &Column) -> bool { /// Compare all searchable metadata after the plaintext has been authenticated /// and independently re-encrypted for the inferred destination column. -/// `into_query_operand` removes only record ciphertext/key material, leaving -/// the identifier and every scalar or SteVec SEM term. Bloom-filter positions -/// are compared without regard to order; all other terms compare exactly. +/// Record ciphertext and key material are excluded. The identifier, every SEM +/// term, and the SteVec array marker are compared. Bloom-filter positions are +/// compared without regard to order; all other metadata compares exactly. pub fn sem_terms_match(inbound: &EqlCiphertext, derived: EqlCiphertext) -> bool { - if let (EqlCiphertext::Encrypted(inbound), EqlCiphertext::Encrypted(derived)) = - (inbound, &derived) - { - return inbound.version == derived.version - && inbound.identifier == derived.identifier - && inbound.hmac_256 == derived.hmac_256 - && bloom_filters_match(&inbound.bloom_filter, &derived.bloom_filter) - && inbound.ore_block_u64_8_256 == derived.ore_block_u64_8_256 - && inbound.ope_cllw == derived.ope_cllw; - } - - match ( - serde_json::to_value(inbound.clone().into_query_operand()), - serde_json::to_value(derived.into_query_operand()), - ) { + match (inbound, derived) { + (EqlCiphertext::Encrypted(inbound), EqlCiphertext::Encrypted(derived)) => { + inbound.version == derived.version + && inbound.identifier == derived.identifier + && inbound.hmac_256 == derived.hmac_256 + && bloom_filters_match(&inbound.bloom_filter, &derived.bloom_filter) + && inbound.ore_block_u64_8_256 == derived.ore_block_u64_8_256 + && inbound.ope_cllw == derived.ope_cllw + } + (EqlCiphertext::SteVec(inbound), EqlCiphertext::SteVec(derived)) => { + inbound.version == derived.version + && inbound.identifier == derived.identifier + && inbound.ste_vec.len() == derived.ste_vec.len() + && inbound + .ste_vec + .iter() + .zip(derived.ste_vec) + .all(|(inbound, derived)| { + inbound.selector == derived.selector + && inbound.is_array == derived.is_array + && ste_vec_terms_match(&inbound.term, &derived.term) + }) + } + _ => false, + } +} + +fn ste_vec_terms_match( + inbound: &Option, + derived: &Option, +) -> bool { + match (serde_json::to_value(inbound), serde_json::to_value(derived)) { (Ok(inbound), Ok(derived)) => inbound == derived, _ => false, } @@ -260,12 +277,12 @@ fn validate_scalar_term_presence( IndexType::Match { .. } => bloom = true, IndexType::Ore => ore = true, IndexType::Ope => ope = true, - IndexType::SteVec { .. } => return Err(EncryptError::InvalidInboundCiphertext), + IndexType::SteVec { .. } => return Err(EncryptError::InvalidInboundEqlPayload), } } if has_hmac != hmac || has_bloom != bloom || has_ore != ore || has_ope != ope { - return Err(EncryptError::InvalidInboundCiphertext); + return Err(EncryptError::InvalidInboundEqlPayload); } Ok(()) } @@ -273,8 +290,9 @@ fn validate_scalar_term_presence( #[cfg(test)] mod tests { use super::*; + use cipherstash_client::eql::{SteVecEntryV3, SteVecKind, SteVecPayloadV3}; use cipherstash_client::schema::{ColumnConfig, ColumnMode, ColumnType}; - use cipherstash_client::zerokms::EncryptedRecord; + use cipherstash_client::zerokms::{EncryptedRecord, KeyHeader}; use cipherstash_config::column::{ArrayIndexMode, Index, SteVecMode}; use eql_mapper::EqlTermVariant; use uuid::Uuid; @@ -326,6 +344,27 @@ mod tests { column } + fn ste_vec_payload(is_array: Option) -> EqlCiphertext { + EqlCiphertext::SteVec(SteVecPayloadV3 { + version: EQL_SCHEMA_VERSION_V3, + kind: SteVecKind::SteVec, + identifier: crate::Identifier::new("users", "email"), + key_header: KeyHeader { + iv: Default::default(), + tag: vec![2; 16], + descriptor: "users/email".into(), + keyset_id: Some(Uuid::nil()), + decryption_policy: None, + }, + ste_vec: vec![SteVecEntryV3 { + selector: "0123456789abcdef0123456789abcdef".into(), + ciphertext: vec![1; 16], + is_array, + term: None, + }], + }) + } + #[test] fn ordinary_json_is_plaintext() { assert!(parse(br#"{"name":"Ada"}"#, &column(), false) @@ -344,7 +383,7 @@ mod tests { fn malformed_payload_shape_fails_closed() { assert!(matches!( parse(br#"{"v":3,"i":"users.email","c":"bad"}"#, &column(), false), - Err(EncryptError::InvalidInboundCiphertext) + Err(EncryptError::InvalidInboundEqlPayload) )); } @@ -353,7 +392,7 @@ mod tests { let ciphertext = payload(crate::Identifier::new("users", "phone")); assert!(matches!( validate_storage_metadata(&ciphertext, &column()), - Err(EncryptError::InvalidInboundCiphertext) + Err(EncryptError::InvalidInboundEqlPayload) )); } @@ -366,7 +405,7 @@ mod tests { payload.ciphertext.descriptor = "accounts/email".into(); assert!(matches!( validate_storage_metadata(&ciphertext, &column()), - Err(EncryptError::InvalidInboundCiphertext) + Err(EncryptError::InvalidInboundEqlPayload) )); } @@ -380,7 +419,7 @@ mod tests { let ciphertext = payload(column.identifier.clone()); assert!(matches!( validate_storage_metadata(&ciphertext, &column), - Err(EncryptError::InvalidInboundCiphertext) + Err(EncryptError::InvalidInboundEqlPayload) )); } @@ -410,6 +449,36 @@ mod tests { assert!(sem_terms_match(&inbound, derived)); } + #[test] + fn bloom_filter_presence_mismatch_does_not_match() { + let mut inbound = payload(crate::Identifier::new("users", "email")); + let derived = inbound.clone(); + let EqlCiphertext::Encrypted(payload) = &mut inbound else { + unreachable!() + }; + payload.bloom_filter = Some(vec![1, 2, 3]); + + assert!(!sem_terms_match(&inbound, derived)); + } + + #[test] + fn ste_vec_array_marker_must_match_independent_derivation() { + let inbound = ste_vec_payload(Some(true)); + let derived = ste_vec_payload(None); + + assert!(!sem_terms_match(&inbound, derived)); + } + + #[test] + fn scalar_storage_payload_is_rejected_for_a_ste_vec_column() { + let ciphertext = payload(crate::Identifier::new("users", "email")); + + assert!(matches!( + validate_storage_metadata(&ciphertext, &ste_vec_column()), + Err(EncryptError::InvalidInboundEqlPayload) + )); + } + #[test] fn query_only_scalar_payload_is_accepted_for_a_query_operand() { let mut column = column(); @@ -446,7 +515,7 @@ mod tests { assert!(matches!( parse(&query, &column, false), - Err(EncryptError::InvalidInboundCiphertext) + Err(EncryptError::InvalidInboundEqlPayload) )); } @@ -466,7 +535,7 @@ mod tests { assert!(matches!( parse(&query, &column, true), - Err(EncryptError::InvalidInboundCiphertext) + Err(EncryptError::InvalidInboundEqlPayload) )); } @@ -487,6 +556,45 @@ mod tests { )); } + #[test] + fn query_only_json_ordering_term_rejects_stray_scalar_terms() { + let mut column = ste_vec_column(); + column.eql_term = EqlTermVariant::JsonOrd; + let query = serde_json::to_vec(&serde_json::json!({ + "v": EQL_SCHEMA_VERSION_V3, + "i": { "t": "users", "c": "email" }, + "hm": "unexpected scalar term", + "op": "ordering term" + })) + .unwrap(); + + assert!(matches!( + parse(&query, &column, true), + Err(EncryptError::InvalidInboundEqlPayload) + )); + } + + #[test] + fn query_only_scalar_payload_rejects_unsupported_eql_term() { + let mut column = column(); + column + .config + .indexes + .push(cipherstash_client::schema::column::Index::new_unique()); + column.eql_term = EqlTermVariant::JsonValueSelector; + let mut ciphertext = payload(column.identifier.clone()); + let EqlCiphertext::Encrypted(payload) = &mut ciphertext else { + unreachable!() + }; + payload.hmac_256 = Some("application-generated SEM term".into()); + let query = serde_json::to_vec(&ciphertext.into_query_operand()).unwrap(); + + assert!(matches!( + parse(&query, &column, true), + Err(EncryptError::InvalidInboundEqlPayload) + )); + } + #[test] fn query_only_ste_vec_payload_is_accepted_for_a_query_operand() { let query = br#"{"sv":[{"s":"application-generated selector"}]}"#; @@ -503,7 +611,15 @@ mod tests { assert!(matches!( parse(query, &ste_vec_column(), false), - Err(EncryptError::InvalidInboundCiphertext) + Err(EncryptError::InvalidInboundEqlPayload) + )); + } + + #[test] + fn query_only_ste_vec_payload_rejects_empty_terms() { + assert!(matches!( + parse(br#"{"sv":[]}"#, &ste_vec_column(), true), + Err(EncryptError::InvalidInboundEqlPayload) )); } diff --git a/packages/cipherstash-proxy/src/postgresql/middleware/backend.rs b/packages/cipherstash-proxy/src/postgresql/middleware/backend.rs index 9b7399f1..a8c9aa36 100644 --- a/packages/cipherstash-proxy/src/postgresql/middleware/backend.rs +++ b/packages/cipherstash-proxy/src/postgresql/middleware/backend.rs @@ -744,6 +744,14 @@ mod tests { ) -> Result>, Error> { Ok(vec![]) } + + async fn decrypt_inbound_eql( + &self, + _keyset_id: Option, + _ciphertexts: Vec>, + ) -> Result>, Error> { + Ok(vec![]) + } } fn create_backend() -> Backend { diff --git a/packages/cipherstash-proxy/src/postgresql/middleware/frontend.rs b/packages/cipherstash-proxy/src/postgresql/middleware/frontend.rs index 4937a0c6..cb6db2d5 100644 --- a/packages/cipherstash-proxy/src/postgresql/middleware/frontend.rs +++ b/packages/cipherstash-proxy/src/postgresql/middleware/frontend.rs @@ -1165,7 +1165,7 @@ impl Frontend { } Some(inbound_eql::InboundEql::Store(ciphertext)) => { let Some(column) = column else { - return Err(EncryptError::InvalidInboundCiphertext.into()); + return Err(EncryptError::InvalidInboundEqlPayload.into()); }; positions.push((index, ciphertext, column.clone())); } @@ -1180,15 +1180,19 @@ impl Frontend { .iter() .map(|(_, ciphertext, _)| Some(ciphertext.clone())) .collect(); - let plaintexts = self.context.decrypt(ciphertexts).await.map_err(|err| { - warn!( - target: ENCRYPT, - client_id = self.context.client_id, - msg = "Inbound EQL ciphertext authentication failed", - error = ?err, - ); - EncryptError::InvalidInboundCiphertext - })?; + let plaintexts = self + .context + .decrypt_inbound_eql(ciphertexts) + .await + .map_err(|err| { + warn!( + target: ENCRYPT, + client_id = self.context.client_id, + msg = "Inbound EQL ciphertext authentication failed", + error = ?err, + ); + EncryptError::InvalidInboundEqlPayload + })?; // Re-encrypt the authenticated plaintext for the inferred destination // and compare every derived SEM term. This detects term splicing: the @@ -1209,7 +1213,7 @@ impl Frontend { msg = "Inbound EQL ciphertext metadata verification failed", error = ?err, ); - EncryptError::InvalidInboundCiphertext + EncryptError::InvalidInboundEqlPayload })?; for ((index, ciphertext, _), derived) in positions.into_iter().zip(derived) { @@ -1219,7 +1223,7 @@ impl Frontend { client_id = self.context.client_id, msg = "Inbound EQL ciphertext SEM terms did not match plaintext", ); - return Err(EncryptError::InvalidInboundCiphertext.into()); + return Err(EncryptError::InvalidInboundEqlPayload.into()); }; if !inbound_eql::sem_terms_match(&ciphertext, derived) { warn!( @@ -1227,7 +1231,7 @@ impl Frontend { client_id = self.context.client_id, msg = "Inbound EQL ciphertext SEM terms did not match plaintext", ); - return Err(EncryptError::InvalidInboundCiphertext.into()); + return Err(EncryptError::InvalidInboundEqlPayload.into()); } encrypted[index] = Some(EqlOutput::Store(ciphertext)); } @@ -1360,13 +1364,6 @@ fn literal_is_query_operand( ) || typed_statement.query_operands.contains_literal(literal) } -fn literals_to_plaintext( - typed_statement: &TypeCheckedStatement<'_>, - literal_columns: &Vec>, -) -> Result>, Error> { - literals_to_plaintext_skipping(typed_statement, literal_columns, &[]) -} - fn literals_to_plaintext_skipping( typed_statement: &TypeCheckedStatement<'_>, literal_columns: &Vec>, @@ -1596,11 +1593,14 @@ impl Frontend { mod tests { use super::{quote_literal, Frontend}; use crate::config::TandemConfig; - use crate::error::{Error, MappingError}; + use crate::error::{EncryptError, Error, MappingError}; use crate::postgresql::context::{Context, KeysetIdentifier}; use crate::postgresql::error_handler::PostgreSqlErrorHandler; + use crate::postgresql::inbound_eql::InboundEql; use crate::postgresql::Column; use crate::proxy::{EncryptConfig, EncryptionService}; + use cipherstash_client::eql::{EncryptedPayloadV3, EQL_SCHEMA_VERSION_V3}; + use cipherstash_client::zerokms::EncryptedRecord; use eql_mapper::Schema; use pg_proto::{Bind, FrontendMessage, Parse}; use std::sync::Arc; @@ -1626,6 +1626,14 @@ mod tests { ) -> Result>, Error> { Ok(Vec::new()) } + + async fn decrypt_inbound_eql( + &self, + _keyset_id: Option, + _ciphertexts: Vec>, + ) -> Result>, Error> { + Ok(Vec::new()) + } } fn frontend() -> Frontend { @@ -1642,6 +1650,25 @@ mod tests { Frontend::new(context) } + fn inbound_storage_payload() -> crate::EqlCiphertext { + crate::EqlCiphertext::Encrypted(EncryptedPayloadV3 { + version: EQL_SCHEMA_VERSION_V3, + identifier: crate::Identifier::new("users", "email"), + ciphertext: EncryptedRecord { + iv: Default::default(), + ciphertext: vec![1; 16], + tag: vec![2; 16], + descriptor: "users/email".into(), + keyset_id: Some(uuid::Uuid::nil()), + decryption_policy: None, + }, + hmac_256: None, + bloom_filter: None, + ore_block_u64_8_256: None, + ope_cllw: None, + }) + } + #[test] fn exception_literals_escape_quotes_and_backslashes() { assert_eq!(quote_literal("can't"), "'can''t'"); @@ -1687,4 +1714,21 @@ mod tests { FrontendMessage::Bind(_) )); } + + #[tokio::test] + async fn stored_inbound_eql_without_a_destination_column_fails_closed() { + let mut encrypted = vec![None]; + let result = frontend() + .merge_inbound_eql( + &mut encrypted, + vec![Some(InboundEql::Store(inbound_storage_payload()))], + &[None], + ) + .await; + + assert!(matches!( + result, + Err(Error::Encrypt(EncryptError::InvalidInboundEqlPayload)) + )); + } } diff --git a/packages/cipherstash-proxy/src/proxy/mod.rs b/packages/cipherstash-proxy/src/proxy/mod.rs index 9a1bc0e7..8bf051e0 100644 --- a/packages/cipherstash-proxy/src/proxy/mod.rs +++ b/packages/cipherstash-proxy/src/proxy/mod.rs @@ -180,6 +180,14 @@ pub trait EncryptionService: Send + Sync { keyset_id: Option, ciphertexts: Vec>, ) -> Result>, Error>; + + /// Authenticate every ciphertext component of application-generated EQL + /// storage payloads and return their root plaintexts for SEM verification. + async fn decrypt_inbound_eql( + &self, + keyset_id: Option, + ciphertexts: Vec>, + ) -> Result>, Error>; } #[cfg(test)] diff --git a/packages/cipherstash-proxy/src/proxy/zerokms/zerokms.rs b/packages/cipherstash-proxy/src/proxy/zerokms/zerokms.rs index be52166f..a35d82a8 100644 --- a/packages/cipherstash-proxy/src/proxy/zerokms/zerokms.rs +++ b/packages/cipherstash-proxy/src/proxy/zerokms/zerokms.rs @@ -13,7 +13,7 @@ use cipherstash_client::{ encryption::{DecryptOptions, Plaintext, QueryOp}, eql::{ encrypt_eql_v3, EqlCiphertextV3, EqlEncryptOpts, EqlOperation, EqlOutputV3, - PreparedPlaintext, + PreparedPlaintext, SteVecEntryV3, }, schema::column::IndexType, zerokms::{Decryptable, EncryptedRecord, IdentifiedBy, RecordWithNonce, RetrieveKeyPayload}, @@ -51,6 +51,12 @@ enum V3Record { SteVecEntry(RecordWithNonce), } +#[derive(Clone, Copy)] +enum SteVecAuthentication { + RootOnly, + AllEntries, +} + impl Decryptable for V3Record { type Error = Infallible; @@ -369,73 +375,84 @@ impl EncryptionService for ZeroKms { &self, keyset_id: Option, ciphertexts: Vec>, + ) -> Result>, Error> { + self.decrypt_eql(keyset_id, ciphertexts, SteVecAuthentication::RootOnly) + .await + } + + async fn decrypt_inbound_eql( + &self, + keyset_id: Option, + ciphertexts: Vec>, + ) -> Result>, Error> { + self.decrypt_eql(keyset_id, ciphertexts, SteVecAuthentication::AllEntries) + .await + } +} + +impl ZeroKms { + async fn decrypt_eql( + &self, + keyset_id: Option, + ciphertexts: Vec>, + ste_vec_authentication: SteVecAuthentication, ) -> Result>, Error> { debug!(target: ENCRYPT, msg="Decrypt", ?keyset_id, default_keyset_id = ?self.default_keyset_id); - // A keyset is required if no default keyset has been configured if self.default_keyset_id.is_none() && keyset_id.is_none() { return Err(EncryptError::MissingKeysetIdentifier.into()); } - let cipher = self.init_cipher(keyset_id.clone()).await?; + let cipher = self.init_cipher(keyset_id).await?; + if matches!(ste_vec_authentication, SteVecAuthentication::AllEntries) + && ciphertexts + .iter() + .flatten() + .any(|ciphertext| ciphertext_keyset_id(ciphertext) != Some(cipher.keyset_id())) + { + return Err(EncryptError::InvalidInboundEqlPayload.into()); + } - // Collect decryptable records and identify which decrypted records - // contain the plaintext returned to the caller. - // - // cipherstash-client has no `decrypt_eql_v3` counterpart to - // `encrypt_eql_v3` — the v2 `decrypt_eql` only accepts `EqlCiphertext`. - // We assemble the decryptable record ourselves, which is what - // protect-ffi does too (`encrypted_record_from_value`). - // - // Scalar: `c` is already the `EncryptedRecord` the v2 path would have - // unwrapped, and `EncryptedRecord` is `Decryptable`. - // - // SteVec: the document holds the key material once in the `h` header - // and each entry carries only raw AEAD bytes, so every record has to be - // reassembled from the header plus that entry. The selector is the AEAD - // binding — its first 12 bytes are the nonce and all 16 go into the AAD - // — which is why the records are `RecordWithNonce`. All entries are - // decrypted to authenticate them, but only the root entry (`sv[0]`) - // contains the complete plaintext returned to the caller. Value entries - // intentionally decrypt to a sentinel that is not a legal `Plaintext`. + // `decryption_policy` needs no parallel structural check here. For + // tag-version 1 records ZeroKMS supplies and verifies the policy MAC + // during key retrieval, so a forged policy fails authentication below. + + // Ordinary database reads authenticate only the root SteVec entry, + // whose ciphertext contains the complete plaintext. Inbound storage + // validation authenticates every entry because the whole application- + // supplied document is about to become stored state. let mut result_positions: Vec> = Vec::new(); let mut records_to_decrypt: Vec = Vec::new(); - for (idx, ct_opt) in ciphertexts.iter().enumerate() { - if let Some(ct) = ct_opt { - match ct { - EqlCiphertextV3::Encrypted(payload) => { - records_to_decrypt.push(V3Record::Scalar(payload.ciphertext.clone())); - result_positions.push(Some(idx)); - } - EqlCiphertextV3::SteVec(document) => { - if document.ste_vec.is_empty() { - return Err(EncryptError::SteVecMissingRootEntry.into()); - } - - for (entry_index, entry) in document.ste_vec.iter().enumerate() { - let selector = decode_ste_vec_selector(&entry.selector)?; - records_to_decrypt.push(V3Record::SteVecEntry( - document - .key_header - .record_with_selector(entry.ciphertext.clone(), selector), - )); - result_positions.push((entry_index == 0).then_some(idx)); - } + for (idx, ciphertext) in ciphertexts.iter().enumerate() { + match ciphertext { + Some(EqlCiphertextV3::Encrypted(payload)) => { + records_to_decrypt.push(V3Record::Scalar(payload.ciphertext.clone())); + result_positions.push(Some(idx)); + } + Some(EqlCiphertextV3::SteVec(document)) => { + let entries = + ste_vec_entries_to_authenticate(&document.ste_vec, ste_vec_authentication)?; + for (entry_index, entry) in entries.iter().enumerate() { + let selector = decode_ste_vec_selector(&entry.selector)?; + records_to_decrypt.push(V3Record::SteVecEntry( + document + .key_header + .record_with_selector(entry.ciphertext.clone(), selector), + )); + result_positions.push((entry_index == 0).then_some(idx)); } } + None => {} } } - // If no ciphertexts to decrypt, return all None if records_to_decrypt.is_empty() { return Ok(vec![None; ciphertexts.len()]); } - // Default opts: the cipher is already scoped to the right keyset, and - // Proxy does not set a lock context. + // The cipher is already scoped to the active keyset. let opts = DecryptOptions::default(); - debug!(target: ENCRYPT, msg="Decrypting EQL v3 records", count = records_to_decrypt.len()); let decrypt_start = Instant::now(); let decrypted = cipher @@ -445,8 +462,8 @@ impl EncryptionService for ZeroKms { let decrypt_duration = decrypt_start.elapsed(); debug!(target: ENCRYPT, msg="Decrypt completed", count = decrypted.len(), duration_ms = decrypt_duration.as_millis()); - // Reconstruct the result vector from scalar and SteVec-root plaintexts. - // Non-root SteVec bytes were decrypted only to authenticate them. + // Non-root entries are authenticated but intentionally have no output + // position: their decrypted sentinels are not legal Plaintext values. let mut result: Vec> = vec![None; ciphertexts.len()]; for (result_position, bytes) in result_positions.into_iter().zip(decrypted) { if let Some(idx) = result_position { @@ -457,3 +474,61 @@ impl EncryptionService for ZeroKms { Ok(result) } } + +fn ste_vec_entries_to_authenticate( + entries: &[SteVecEntryV3], + authentication: SteVecAuthentication, +) -> Result<&[SteVecEntryV3], EncryptError> { + let root = entries + .first() + .ok_or(EncryptError::SteVecMissingRootEntry)?; + Ok(match authentication { + SteVecAuthentication::RootOnly => std::slice::from_ref(root), + SteVecAuthentication::AllEntries => entries, + }) +} + +fn ciphertext_keyset_id(ciphertext: &EqlCiphertextV3) -> Option { + match ciphertext { + EqlCiphertextV3::Encrypted(payload) => payload.ciphertext.keyset_id, + EqlCiphertextV3::SteVec(document) => document.key_header.keyset_id, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn ste_vec_entry(selector: &str) -> SteVecEntryV3 { + SteVecEntryV3 { + selector: selector.into(), + ciphertext: vec![1; 16], + is_array: None, + term: None, + } + } + + #[test] + fn ordinary_decryption_authenticates_only_the_ste_vec_root() { + let entries = vec![ste_vec_entry("root"), ste_vec_entry("nested")]; + + assert_eq!( + ste_vec_entries_to_authenticate(&entries, SteVecAuthentication::RootOnly) + .unwrap() + .len(), + 1 + ); + } + + #[test] + fn inbound_validation_authenticates_every_ste_vec_entry() { + let entries = vec![ste_vec_entry("root"), ste_vec_entry("nested")]; + + assert_eq!( + ste_vec_entries_to_authenticate(&entries, SteVecAuthentication::AllEntries) + .unwrap() + .len(), + entries.len() + ); + } +}