From e6a0cfec919d0ae0400ab86fb11887cd0e6e481b Mon Sep 17 00:00:00 2001 From: "david.ioinovitch" Date: Thu, 24 Sep 2026 15:55:47 -0400 Subject: [PATCH] Align VIN checksum tests with existing validator patterns --- sds-go/go/scanner_test.go | 57 ++++++++ sds-go/go/vin_checksum_test.go | 79 ---------- sds/src/scanner/test/mod.rs | 1 - sds/src/scanner/test/vin_checksum.rs | 146 ------------------- sds/src/secondary_validation/vin_checksum.rs | 106 ++++++++++++++ 5 files changed, 163 insertions(+), 226 deletions(-) delete mode 100644 sds-go/go/vin_checksum_test.go delete mode 100644 sds/src/scanner/test/vin_checksum.rs diff --git a/sds-go/go/scanner_test.go b/sds-go/go/scanner_test.go index 877787ac..b2858c53 100644 --- a/sds-go/go/scanner_test.go +++ b/sds-go/go/scanner_test.go @@ -578,6 +578,63 @@ func TestAustrianChecksumSecondaryValidator(t *testing.T) { runTest(t, scannerWithChecksum, testData, false) } +func TestVinChecksumSecondaryValidator(t *testing.T) { + scannerWithoutChecksum, err := CreateScanner([]RuleConfig{ + RegexRuleConfig{Id: "vin_rule", Pattern: `[A-Za-z0-9]{17}`, MatchAction: MatchAction{Type: MatchActionRedact, RedactionValue: "[redacted]"}}, + }) + if err != nil { + t.Fatal("failed to create the scanner wo checksum:", err.Error()) + } + defer scannerWithoutChecksum.Delete() + scannerWithChecksum, err := CreateScanner([]RuleConfig{ + RegexRuleConfig{Id: "vin_rule", Pattern: `[A-Za-z0-9]{17}`, MatchAction: MatchAction{Type: MatchActionRedact, RedactionValue: "[redacted]"}, SecondaryValidator: NewSecondaryValidator("VinChecksum")}, + }) + if err != nil { + t.Fatal("failed to create the scanner with checksum:", err.Error()) + } + defer scannerWithChecksum.Delete() + + testData := map[string]testResult{ + "5YJ3E1EAXHF000316 3D7KA28693G723011": { + mutated: true, + str: "[redacted] [redacted]", + rules: []RuleMatch{ + { + RuleIdx: 0, + StartIndex: 0, + ReplacementType: ReplacementTypePlaceholder, + EndIndexExclusive: 10, + ShiftOffset: -7, + }, { + RuleIdx: 0, + StartIndex: 11, + ReplacementType: ReplacementTypePlaceholder, + EndIndexExclusive: 21, + ShiftOffset: -14, + }, + }, + }, + } + runTest(t, scannerWithoutChecksum, testData, false) + + testData = map[string]testResult{ + "5YJ3E1EAXHF000316 3D7KA28693G723011": { + mutated: true, + str: "[redacted] 3D7KA28693G723011", + rules: []RuleMatch{ + { + RuleIdx: 0, + StartIndex: 0, + ReplacementType: ReplacementTypePlaceholder, + EndIndexExclusive: 10, + ShiftOffset: -7, + }, + }, + }, + } + runTest(t, scannerWithChecksum, testData, false) +} + func TestJWTSecondaryValidator(t *testing.T) { scannerWithoutChecksum, err := CreateScanner([]RuleConfig{ RegexRuleConfig{Id: "rule_oauth_test", Pattern: `ey[\w=-]+\.ey[\w=-]+\.[\w-]+`, MatchAction: MatchAction{Type: MatchActionRedact, RedactionValue: "[redacted]"}}, diff --git a/sds-go/go/vin_checksum_test.go b/sds-go/go/vin_checksum_test.go deleted file mode 100644 index 87f12fbf..00000000 --- a/sds-go/go/vin_checksum_test.go +++ /dev/null @@ -1,79 +0,0 @@ -package dd_sds - -import ( - "encoding/json" - "testing" -) - -func TestVinChecksumSecondaryValidator(t *testing.T) { - validator := NewSecondaryValidator("VinChecksum") - encoded, err := json.Marshal(validator) - if err != nil { - t.Fatal("failed to serialize VIN validator:", err) - } - if string(encoded) != `{"type":"VinChecksum"}` { - t.Fatalf("unexpected VIN validator JSON: %s", encoded) - } - - scanner, err := CreateScanner([]RuleConfig{ - RegexRuleConfig{ - Id: "vin", - Pattern: "[A-Za-z0-9]+", - MatchAction: MatchAction{Type: MatchActionRedact, RedactionValue: "[REDACTED]"}, - SecondaryValidator: validator, - }, - }) - if err != nil { - t.Fatal("failed to create scanner with VIN checksum through FFI:", err) - } - defer scanner.Delete() - - const validUS = "5YJ3E1EAXHF000316" - const validChina = "LFWADRJF011002346" - const invalidUS = "3D7KA28693G723011" - const invalidChina = "L5BGA2V58NG590409" - - cases := []struct { - name string - input string - output string - matchStart uint32 - expectedMatch bool - }{ - {name: "US X check digit", input: validUS, output: "[REDACTED]", expectedMatch: true}, - {name: "China zero check digit", input: validChina, output: "[REDACTED]", expectedMatch: true}, - {name: "lowercase VIN", input: "5yj3e1eaxhf000316", output: "[REDACTED]", expectedMatch: true}, - {name: "historical US false positive", input: invalidUS, output: invalidUS}, - {name: "historical China false positive", input: invalidChina, output: invalidChina}, - {name: "invalid letter in check position", input: "5YJ3E1EAAHF000316", output: "5YJ3E1EAAHF000316"}, - {name: "short input", input: "5YJ3E1EAXHF00031", output: "5YJ3E1EAXHF00031"}, - { - name: "continue from invalid to valid", - input: invalidChina + " " + validChina, - output: invalidChina + " [REDACTED]", - matchStart: 18, - expectedMatch: true, - }, - { - name: "leave invalid after valid unchanged", - input: validUS + " " + invalidUS, - output: "[REDACTED] " + invalidUS, - expectedMatch: true, - }, - } - for _, testCase := range cases { - t.Run(testCase.name, func(t *testing.T) { - expected := testResult{str: testCase.output, mutated: testCase.expectedMatch} - if testCase.expectedMatch { - expected.rules = []RuleMatch{{ - RuleIdx: 0, - ReplacementType: ReplacementTypePlaceholder, - StartIndex: testCase.matchStart, - EndIndexExclusive: testCase.matchStart + 10, - ShiftOffset: -7, - }} - } - runTest(t, scanner, map[string]testResult{testCase.input: expected}, false) - }) - } -} diff --git a/sds/src/scanner/test/mod.rs b/sds/src/scanner/test/mod.rs index 13c353a5..d351d8ad 100644 --- a/sds/src/scanner/test/mod.rs +++ b/sds/src/scanner/test/mod.rs @@ -6,7 +6,6 @@ mod overlapping_matches; mod parallel_scan; mod supporting_rule; mod validators; -mod vin_checksum; use super::*; use super::{ScannerBuilder, StringMatch}; diff --git a/sds/src/scanner/test/vin_checksum.rs b/sds/src/scanner/test/vin_checksum.rs deleted file mode 100644 index dbcfa8c1..00000000 --- a/sds/src/scanner/test/vin_checksum.rs +++ /dev/null @@ -1,146 +0,0 @@ -use crate::{MatchAction, RegexRuleConfig, RootRuleConfig, Scanner, SecondaryValidator}; - -fn validator() -> SecondaryValidator { - serde_json::from_str(r#"{"type":"VinChecksum"}"#).expect("VIN checksum must be supported") -} - -#[test] -fn vin_checksum_accepts_independent_vectors_and_ascii_case() { - let validator = validator().compile(); - // GB 16735-2019 Appendix A (zero) and published VIN examples (numeric/X). - for vin in [ - "LFWADRJF011002346", - "5YJ3E1EAXHF000316", - "1HGBH41JXMN109186", - "LZPTCAP2561500278", - "LJSKA3BF3CD820005", - "5yj3e1eaxhf000316", - "lFwAdRjF011002346", - ] { - assert!(validator.is_valid_match(vin), "rejected {vin}"); - } -} - -#[test] -fn vin_checksum_rejects_historical_invalid_examples_and_malformed_input() { - let validator = validator().compile(); - for vin in [ - "L5BGA2V58NG590409", - "3D7KA28693G723011", - "2FMDK36C18BA04895", - "5YJ3E1EA0HF000316", - "5YJ3E1EAXHF000317", - "LFWADRJFA11002346", - "", - "LFWADRJF01100234", - "LFWADRJF0110023460", - "LFWADRJF 11002346", - "LFWADRJF-11002346", - "LFWADRJF_11002346", - "LFWADRJF/11002346", - "LFWADRJF01100234é", - "LFWADRJF0110023é", - "ＬFWADRJF011002346", - ] { - assert!(!validator.is_valid_match(vin), "accepted {vin}"); - } - for forbidden in [b'I', b'O', b'Q', b'i', b'o', b'q'] { - for position in 0..17 { - let mut vin = b"LFWADRJF011002346".to_vec(); - vin[position] = forbidden; - let vin = String::from_utf8(vin).unwrap(); - assert!(!validator.is_valid_match(&vin), "accepted {vin}"); - } - } -} - -#[test] -fn vin_checksum_accepts_only_the_expected_ninth_character() { - let validator = validator().compile(); - for vin in ["LFWADRJF011002346", "5YJ3E1EAXHF000316"] { - for character in b"0123456789ABCDEFGHJKLMNPRSTUVWXYZ" { - let mut candidate = vin.as_bytes().to_vec(); - candidate[8] = *character; - let candidate = String::from_utf8(candidate).unwrap(); - assert_eq!( - validator.is_valid_match(&candidate), - candidate == vin, - "{candidate}" - ); - } - } -} - -#[test] -fn vin_checksum_transliterates_every_allowed_character() { - let validator = validator().compile(); - // With only position one nonzero, its weight of eight determines the check digit. - for (characters, check_digit) in [ - ("0", b'0'), - ("1AJ", b'8'), - ("2BKS", b'5'), - ("3CLT", b'2'), - ("4DMU", b'X'), - ("5ENV", b'7'), - ("6FW", b'4'), - ("7GPX", b'1'), - ("8HY", b'9'), - ("9RZ", b'6'), - ] { - for character in characters.bytes() { - let mut vin = *b"00000000000000000"; - vin[0] = character; - vin[8] = check_digit; - let vin = std::str::from_utf8(&vin).unwrap(); - assert!(validator.is_valid_match(vin), "rejected {vin}"); - } - } -} - -#[test] -fn vin_checksum_round_trips_configuration() { - assert_eq!( - serde_json::to_string(&validator()).unwrap(), - r#"{"type":"VinChecksum"}"# - ); -} - -#[test] -fn vin_checksum_checks_integrity_not_vehicle_issuance() { - let validator = validator().compile(); - for vin in ["00000000000000000", "11111111111111111"] { - assert!(validator.is_valid_match(vin)); - } -} - -#[test] -fn vin_checksum_filters_candidates_and_continues_scanning() { - let config: RegexRuleConfig = - serde_json::from_str(r#"{"pattern":"[A-Za-z0-9]+","validator":{"type":"VinChecksum"}}"#) - .expect("VIN checksum configuration must deserialize"); - let scanner = Scanner::builder(&[RootRuleConfig::new(config.build()).match_action( - MatchAction::Redact { - replacement: "[REDACTED]".into(), - }, - )]) - .build() - .unwrap(); - for (input, output, matches) in [ - ("L5BGA2V58NG590409", "L5BGA2V58NG590409", 0), - ( - "L5BGA2V58NG590409 LFWADRJF011002346", - "L5BGA2V58NG590409 [REDACTED]", - 1, - ), - ( - "5YJ3E1EAXHF000316 3D7KA28693G723011", - "[REDACTED] 3D7KA28693G723011", - 1, - ), - ] { - let mut event = input.to_string(); - let result = scanner.scan(&mut event).unwrap(); - assert_eq!(event, output); - assert_eq!(result.len(), matches); - } -} diff --git a/sds/src/secondary_validation/vin_checksum.rs b/sds/src/secondary_validation/vin_checksum.rs index e0df4cfe..581eba30 100644 --- a/sds/src/secondary_validation/vin_checksum.rs +++ b/sds/src/secondary_validation/vin_checksum.rs @@ -36,3 +36,109 @@ impl Validator for VinChecksum { bytes[8].to_ascii_uppercase() == check_digit } } + +#[cfg(test)] +mod test { + use crate::secondary_validation::*; + + #[test] + fn vin_checksum_accepts_independent_vectors_and_ascii_case() { + let validator = VinChecksum; + // GB 16735-2019 Appendix A (zero) and published VIN examples (numeric/X). + for vin in [ + "LFWADRJF011002346", + "5YJ3E1EAXHF000316", + "1HGBH41JXMN109186", + "LZPTCAP2561500278", + "LJSKA3BF3CD820005", + "5yj3e1eaxhf000316", + "lFwAdRjF011002346", + ] { + assert!(validator.is_valid_match(vin), "rejected {vin}"); + } + } + + #[test] + fn vin_checksum_rejects_historical_invalid_examples_and_malformed_input() { + let validator = VinChecksum; + for vin in [ + "L5BGA2V58NG590409", + "3D7KA28693G723011", + "2FMDK36C18BA04895", + "5YJ3E1EA0HF000316", + "5YJ3E1EAXHF000317", + "LFWADRJFA11002346", + "", + "LFWADRJF01100234", + "LFWADRJF0110023460", + "LFWADRJF 11002346", + "LFWADRJF-11002346", + "LFWADRJF_11002346", + "LFWADRJF/11002346", + "LFWADRJF01100234é", + "LFWADRJF0110023é", + "ＬFWADRJF011002346", + ] { + assert!(!validator.is_valid_match(vin), "accepted {vin}"); + } + for forbidden in [b'I', b'O', b'Q', b'i', b'o', b'q'] { + for position in 0..17 { + let mut vin = b"LFWADRJF011002346".to_vec(); + vin[position] = forbidden; + let vin = String::from_utf8(vin).unwrap(); + assert!(!validator.is_valid_match(&vin), "accepted {vin}"); + } + } + } + + #[test] + fn vin_checksum_accepts_only_the_expected_ninth_character() { + let validator = VinChecksum; + for vin in ["LFWADRJF011002346", "5YJ3E1EAXHF000316"] { + for character in b"0123456789ABCDEFGHJKLMNPRSTUVWXYZ" { + let mut candidate = vin.as_bytes().to_vec(); + candidate[8] = *character; + let candidate = String::from_utf8(candidate).unwrap(); + assert_eq!( + validator.is_valid_match(&candidate), + candidate == vin, + "{candidate}" + ); + } + } + } + + #[test] + fn vin_checksum_transliterates_every_allowed_character() { + let validator = VinChecksum; + // With only position one nonzero, its weight of eight determines the check digit. + for (characters, check_digit) in [ + ("0", b'0'), + ("1AJ", b'8'), + ("2BKS", b'5'), + ("3CLT", b'2'), + ("4DMU", b'X'), + ("5ENV", b'7'), + ("6FW", b'4'), + ("7GPX", b'1'), + ("8HY", b'9'), + ("9RZ", b'6'), + ] { + for character in characters.bytes() { + let mut vin = *b"00000000000000000"; + vin[0] = character; + vin[8] = check_digit; + let vin = std::str::from_utf8(&vin).unwrap(); + assert!(validator.is_valid_match(vin), "rejected {vin}"); + } + } + } + + #[test] + fn vin_checksum_checks_integrity_not_vehicle_issuance() { + let validator = VinChecksum; + for vin in ["00000000000000000", "11111111111111111"] { + assert!(validator.is_valid_match(vin)); + } + } +}