diff --git a/sds/src/match_validation/http_validator_v2.rs b/sds/src/match_validation/http_validator_v2.rs index ae936d68..f4e4b7dd 100644 --- a/sds/src/match_validation/http_validator_v2.rs +++ b/sds/src/match_validation/http_validator_v2.rs @@ -556,6 +556,7 @@ mod tests { suppressions: None, precedence: Precedence::default(), is_supporting_rule: false, + sds_rule_name: None, } } @@ -1369,6 +1370,7 @@ calls: suppressions: None, precedence: Precedence::default(), is_supporting_rule: false, + sds_rule_name: None, }, RootCompiledRule { inner: Box::new(MockCompiledRule), @@ -1387,6 +1389,7 @@ calls: suppressions: None, precedence: Precedence::default(), is_supporting_rule: false, + sds_rule_name: None, }, ]; @@ -1496,6 +1499,7 @@ match_pairing: suppressions: None, precedence: Precedence::default(), is_supporting_rule: false, + sds_rule_name: None, }, RootCompiledRule { inner: Box::new(MockCompiledRule), @@ -1514,6 +1518,7 @@ match_pairing: suppressions: None, precedence: Precedence::default(), is_supporting_rule: false, + sds_rule_name: None, }, ]; @@ -1710,6 +1715,7 @@ match_pairing: suppressions: None, precedence: Precedence::default(), is_supporting_rule: false, + sds_rule_name: None, }, RootCompiledRule { inner: Box::new(MockCompiledRule), @@ -1728,6 +1734,7 @@ match_pairing: suppressions: None, precedence: Precedence::default(), is_supporting_rule: false, + sds_rule_name: None, }, ]; diff --git a/sds/src/scanner/metrics.rs b/sds/src/scanner/metrics.rs index a50d1528..91599776 100644 --- a/sds/src/scanner/metrics.rs +++ b/sds/src/scanner/metrics.rs @@ -1,6 +1,26 @@ use crate::Labels; use metrics::{Counter, counter}; +/// Looks up the value of a `"key:value"` tag entry by key, matching the format used by +/// standard rule definitions (e.g. `sensitive_data:travis_ci_access_token`). +fn get_tag_value<'a>(tags: &'a [String], key: &str) -> Option<&'a str> { + let prefix_len = key.len() + 1; + tags.iter().find_map(|tag| { + (tag.len() > prefix_len && tag.starts_with(key) && tag.as_bytes()[key.len()] == b':') + .then(|| &tag[prefix_len..]) + }) +} + +/// Computes the `"{sensitive_data_category}/{sensitive_data}"` tag value for a rule from its +/// `tags`. Missing `sensitive_data_category` falls back to `"missing_category"`. Missing +/// `sensitive_data` yields `None`, since there's nothing meaningful to tag with. +pub fn compute_sds_rule_name(tags: &[String]) -> Option { + let sensitive_data = get_tag_value(tags, "sensitive_data")?; + let sensitive_data_category = + get_tag_value(tags, "sensitive_data_category").unwrap_or("missing_category"); + Some(format!("{sensitive_data_category}/{sensitive_data}")) +} + #[derive(Clone)] pub struct RuleMetrics { /// Pre-initialized counter for the fast path (debug observability disabled). @@ -49,3 +69,54 @@ impl ScannerMetrics { } } } + +#[cfg(test)] +mod test { + use super::compute_sds_rule_name; + + #[test] + fn compute_sds_rule_name_returns_none_for_empty_tags() { + assert_eq!(compute_sds_rule_name(&[]), None); + } + + #[test] + fn compute_sds_rule_name_concatenates_category_and_data() { + let tags = vec![ + "sensitive_data_category:credentials".to_string(), + "sensitive_data:travis_ci_access_token".to_string(), + ]; + assert_eq!( + compute_sds_rule_name(&tags), + Some("credentials/travis_ci_access_token".to_string()) + ); + } + + #[test] + fn compute_sds_rule_name_returns_none_when_category_present_but_data_missing() { + let tags = vec!["sensitive_data_category:credentials".to_string()]; + assert_eq!(compute_sds_rule_name(&tags), None); + } + + #[test] + fn compute_sds_rule_name_falls_back_to_missing_category_when_category_absent() { + let tags = vec!["sensitive_data:travis_ci_access_token".to_string()]; + assert_eq!( + compute_sds_rule_name(&tags), + Some("missing_category/travis_ci_access_token".to_string()) + ); + } + + #[test] + fn compute_sds_rule_name_does_not_confuse_sensitive_data_category_with_sensitive_data() { + // "sensitive_data_category" starts with the "sensitive_data" key, so the lookup + // must not mistake one tag for the other. + let tags = vec!["sensitive_data_category:credentials".to_string()]; + assert_eq!(compute_sds_rule_name(&tags), None); + } + + #[test] + fn compute_sds_rule_name_ignores_malformed_tags_without_a_colon() { + let tags = vec!["sensitive_data".to_string()]; + assert_eq!(compute_sds_rule_name(&tags), None); + } +} diff --git a/sds/src/scanner/mod.rs b/sds/src/scanner/mod.rs index 834ce3ce..48753945 100644 --- a/sds/src/scanner/mod.rs +++ b/sds/src/scanner/mod.rs @@ -27,6 +27,7 @@ pub use crate::secondary_validation::Validator; use crate::stats::GLOBAL_STATS; use crate::tokio::TOKIO_RUNTIME; use crate::{CreateScannerError, EncodeIndices, MatchAction, Path, ScannerError}; +use ::metrics::counter; use ahash::AHashMap; use futures::executor::block_on; use serde::{Deserialize, Serialize}; @@ -107,6 +108,11 @@ pub struct RootRuleConfig { precedence: Precedence, #[serde(default)] pub is_supporting_rule: bool, + /// Raw `"key:value"` strings, matching the format used by standard rule definitions + /// (e.g. `sensitive_data:travis_ci_access_token`). Not parsed into a map since no + /// producer of these values does so either; consumers parse the entries they need. + #[serde(default)] + pub tags: Vec, #[serde(flatten)] pub inner: T, } @@ -135,6 +141,7 @@ impl RootRuleConfig { suppressions: None, precedence: Precedence::default(), is_supporting_rule: false, + tags: Vec::new(), inner, } } @@ -149,6 +156,7 @@ impl RootRuleConfig { suppressions: self.suppressions, precedence: self.precedence, is_supporting_rule: self.is_supporting_rule, + tags: self.tags, inner: func(self.inner), } } @@ -186,6 +194,11 @@ impl RootRuleConfig { self } + pub fn tags(mut self, tags: Vec) -> Self { + self.tags = tags; + self + } + pub fn get_suppressions(&self) -> Option<&Suppressions> { self.suppressions.as_ref() } @@ -213,6 +226,10 @@ pub struct RootCompiledRule { pub suppressions: Option, pub precedence: Precedence, pub is_supporting_rule: bool, + /// Precomputed `"{sensitive_data_category}/{sensitive_data}"` tag value derived from + /// the rule's `tags`, used to tag `scanning.match_count`. `None` when the rule has no + /// `sensitive_data` tag. + pub sds_rule_name: Option, } impl RootCompiledRule { @@ -589,10 +606,26 @@ impl Scanner { ) { // Add number of scanned events self.metrics.num_scanned_events.increment(1); - // Add number of matches - self.metrics - .match_count - .increment(output_rule_matches.len() as u64); + // Add number of matches, tagged per rule so `sds_rule_name` can break down match counts + // by the rule's sensitive_data_category/sensitive_data tags. + let mut match_counts_by_rule: AHashMap = AHashMap::new(); + for rule_match in output_rule_matches { + *match_counts_by_rule + .entry(rule_match.rule_index) + .or_default() += 1; + } + for (rule_index, count) in match_counts_by_rule { + match self.rules[rule_index].sds_rule_name.as_deref() { + Some(sds_rule_name) => { + let labels = self.labels.clone_with_labels(Labels::new(&[( + "sds_rule_name", + sds_rule_name.to_string(), + )])); + counter!("scanning.match_count", labels).increment(count); + } + None => self.metrics.match_count.increment(count), + } + } if let Some(io_duration) = io_duration { let total_duration = start.elapsed(); @@ -1137,6 +1170,7 @@ impl ScannerBuilder<'_> { suppressions: compiled_suppressions, precedence: config.precedence, is_supporting_rule: config.is_supporting_rule, + sds_rule_name: metrics::compute_sds_rule_name(&config.tags), }) }) .collect::, CreateScannerError>>()?; diff --git a/sds/src/scanner/test/metrics.rs b/sds/src/scanner/test/metrics.rs index 55f33c2e..c6ac2e0c 100644 --- a/sds/src/scanner/test/metrics.rs +++ b/sds/src/scanner/test/metrics.rs @@ -61,6 +61,167 @@ fn should_submit_scanning_metrics() { } } +#[test] +fn should_submit_match_count_metric_tagged_with_sds_rule_name() { + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + + metrics::with_local_recorder(&recorder, || { + let rule_0 = RootRuleConfig::new(RegexRuleConfig::new("secret").build()) + .match_action(MatchAction::None) + .tags(vec![ + "sensitive_data_category:credentials".to_string(), + "sensitive_data:travis_ci_access_token".to_string(), + ]); + + let scanner = ScannerBuilder::new(&[rule_0]).build().unwrap(); + let mut content = SimpleEvent::Map(BTreeMap::from([( + "key1".to_string(), + SimpleEvent::String("secret".to_string()), + )])); + + scanner.scan(&mut content).unwrap(); + }); + + let snapshot = snapshotter.snapshot().into_hashmap(); + + let metric_name = "scanning.match_count"; + let labels = vec![Label::new( + "sds_rule_name", + "credentials/travis_ci_access_token", + )]; + let metric_value = snapshot + .get(&CompositeKey::new( + Counter, + Key::from_parts(metric_name, labels), + )) + .expect("tagged match_count metric not found"); + + assert_eq!(metric_value, &(None, None, DebugValue::Counter(1))); +} + +#[test] +fn should_submit_match_count_metric_with_missing_category_tag() { + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + + metrics::with_local_recorder(&recorder, || { + let rule_0 = RootRuleConfig::new(RegexRuleConfig::new("secret").build()) + .match_action(MatchAction::None) + .tags(vec!["sensitive_data:travis_ci_access_token".to_string()]); + + let scanner = ScannerBuilder::new(&[rule_0]).build().unwrap(); + let mut content = SimpleEvent::Map(BTreeMap::from([( + "key1".to_string(), + SimpleEvent::String("secret".to_string()), + )])); + + scanner.scan(&mut content).unwrap(); + }); + + let snapshot = snapshotter.snapshot().into_hashmap(); + + let metric_name = "scanning.match_count"; + let labels = vec![Label::new( + "sds_rule_name", + "missing_category/travis_ci_access_token", + )]; + let metric_value = snapshot + .get(&CompositeKey::new( + Counter, + Key::from_parts(metric_name, labels), + )) + .expect("tagged match_count metric not found"); + + assert_eq!(metric_value, &(None, None, DebugValue::Counter(1))); +} + +#[test] +fn should_submit_untagged_match_count_metric_when_sensitive_data_tag_is_missing() { + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + + metrics::with_local_recorder(&recorder, || { + let rule_0 = RootRuleConfig::new(RegexRuleConfig::new("secret").build()) + .match_action(MatchAction::None) + .tags(vec!["sensitive_data_category:credentials".to_string()]); + + let scanner = ScannerBuilder::new(&[rule_0]).build().unwrap(); + let mut content = SimpleEvent::Map(BTreeMap::from([( + "key1".to_string(), + SimpleEvent::String("secret".to_string()), + )])); + + scanner.scan(&mut content).unwrap(); + }); + + let snapshot = snapshotter.snapshot().into_hashmap(); + + let metric_name = "scanning.match_count"; + let metric_value = snapshot + .get(&CompositeKey::new(Counter, Key::from_name(metric_name))) + .expect("untagged match_count metric not found"); + + assert_eq!(metric_value, &(None, None, DebugValue::Counter(1))); +} + +#[test] +fn should_submit_separately_tagged_match_count_metrics_for_multiple_rules() { + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + + metrics::with_local_recorder(&recorder, || { + let rule_0 = RootRuleConfig::new(RegexRuleConfig::new("secret").build()) + .match_action(MatchAction::None) + .tags(vec![ + "sensitive_data_category:credentials".to_string(), + "sensitive_data:travis_ci_access_token".to_string(), + ]); + let rule_1 = RootRuleConfig::new(RegexRuleConfig::new("foo").build()) + .match_action(MatchAction::None) + .tags(vec![ + "sensitive_data_category:pii".to_string(), + "sensitive_data:email".to_string(), + ]); + + let scanner = ScannerBuilder::new(&[rule_0, rule_1]).build().unwrap(); + let mut content = SimpleEvent::Map(BTreeMap::from([( + "key1".to_string(), + SimpleEvent::String("secret foo".to_string()), + )])); + + scanner.scan(&mut content).unwrap(); + }); + + let snapshot = snapshotter.snapshot().into_hashmap(); + + let metric_name = "scanning.match_count"; + + let credentials_labels = vec![Label::new( + "sds_rule_name", + "credentials/travis_ci_access_token", + )]; + let credentials_metric_value = snapshot + .get(&CompositeKey::new( + Counter, + Key::from_parts(metric_name, credentials_labels), + )) + .expect("credentials match_count metric not found"); + assert_eq!( + credentials_metric_value, + &(None, None, DebugValue::Counter(1)) + ); + + let pii_labels = vec![Label::new("sds_rule_name", "pii/email")]; + let pii_metric_value = snapshot + .get(&CompositeKey::new( + Counter, + Key::from_parts(metric_name, pii_labels), + )) + .expect("pii match_count metric not found"); + assert_eq!(pii_metric_value, &(None, None, DebugValue::Counter(1))); +} + #[test] fn should_submit_excluded_match_metric() { let recorder = DebuggingRecorder::new();