diff --git a/README.md b/README.md index bdc310b..5de487a 100644 --- a/README.md +++ b/README.md @@ -42,10 +42,13 @@ and do not establish a presumption of conformity. - `applies_if` supports the paths `$.` and `$.[?search(@, '')]` or `[?match(@, '')]`. The regular expression of `search()` and `match()` is an I-Regexp (RFC 9485) as - RFC 9535 requires: `match()` needs the entire value, `search()` a substring, - and a pattern that does not conform selects nothing. `matches` of the - condition itself is ECMA-262 as above. A condition dpplint cannot evaluate - skips the criterion with the reason. + RFC 9535 requires: `match()` needs the entire value, `search()` a substring. + The pattern is checked before the JSONPath is evaluated; an invalid I-Regexp + or one with `^` or `$` outside a character class skips the criterion with + the reason. `matches` of the condition itself is ECMA-262 as above and holds + only for JSON strings (numbers, booleans, null, arrays and objects never + satisfy it). Any other condition dpplint cannot evaluate also skips the + criterion with the reason. - Criteria with `check.type: did` send the DIDs of the passport to [didlint](https://didlint.ownyourdata.eu) (DID Core, DID Resolution) and check linked verifiable presentations for the VC Data Model 2.0 context. The diff --git a/app/services/applies_if.rb b/app/services/applies_if.rb index 40297d1..2924f57 100644 --- a/app/services/applies_if.rb +++ b/app/services/applies_if.rb @@ -9,12 +9,16 @@ # The string literal may be single- or double-quoted with the escapes of # RFC 9535, 2.3.1.1. The filter selects the elements of an array (or the # member values of an object) for which the function holds; search() and -# match() evaluate their pattern as I-Regexp (RFC 9485, see IRegexp), are false -# for values that are not strings and, following RFC 9535, false for a pattern -# that does not conform to RFC 9485. +# match() evaluate their pattern as I-Regexp (RFC 9485, see IRegexp) and are +# false for values that are not strings. The pattern is checked before the +# JSONPath is evaluated: an invalid I-Regexp or one with `^` or `$` outside a +# character class makes the criterion skipped (see problem), as +# CRITERIA-FORMAT.md requires, instead of letting the function return false. # # `matches` of the assertion itself is an ECMA-262 regular expression -# (see EcmaRegexp), as for all JSON assertions. +# (see EcmaRegexp), as for all JSON assertions. It holds only for JSON +# strings; numbers, booleans, null, arrays and objects never satisfy it and +# are not converted to text. class AppliesIf MEMBER = /[A-Za-z_\u0080-\u{10FFFF}][A-Za-z0-9_\u0080-\u{10FFFF}]*/ SIMPLE = /\A\$\.(#{MEMBER})\z/ @@ -31,7 +35,10 @@ def self.holds?(conditions, passport) # with such a condition is skipped. def self.problem(conditions) Array(conditions).each do |c| - parse_path(c["path"].to_s) + _member, function, pattern = parse_path(c["path"].to_s) + if function && (reason = IRegexp.problem(pattern)) + return "regular expression #{pattern.inspect} of #{function}() in applies_if path #{c['path']} #{reason}" + end next unless c.key?("matches") reason = EcmaRegexp.problem(c["matches"]) @@ -112,7 +119,7 @@ def holds? return !values.empty? == @c["exists"] if @c.key?("exists") return values.any? { |v| v == @c["equals"] } if @c.key?("equals") return values.any? { |v| @c["in"].include?(v) } if @c.key?("in") - return values.any? { |v| EcmaRegexp.search?(@c["matches"], v.to_s) } if @c.key?("matches") + return values.any? { |v| v.is_a?(String) && EcmaRegexp.search?(@c["matches"], v) } if @c.key?("matches") !values.empty? end diff --git a/app/services/i_regexp.rb b/app/services/i_regexp.rb index 181952b..ff3dcaa 100644 --- a/app/services/i_regexp.rb +++ b/app/services/i_regexp.rb @@ -3,21 +3,25 @@ # of dpp-criteria requires for regular expressions inside a JSONPath expression. # # The pattern is parsed with the ABNF of RFC 9485, section 3, and translated -# into an equivalent Ruby Regexp. `^` and `$` outside a character class are -# anchors at the start and end of the whole value: the ABNF lists them as -# ordinary characters, but the mappings of RFC 9485, section 5, and the -# JSONPath Compliance Test Suite (match(): "explicit caret", "explicit dollar") -# treat them as anchors, and dpplint follows that. Further differences to Ruby -# and ECMA-262 that matter: `.` matches any character except LF and CR; only the escapes \( \) \* \+ \- \. \? \[ \\ \] -# \^ \n \r \t \{ \| \} and the category escapes \p{..} and \P{..} exist (no \d, +# into an equivalent Ruby Regexp. CRITERIA-FORMAT.md does not allow `^` or `$` +# outside a character class (RFC 9485 lists them as ordinary characters, its +# mappings in section 5 and the JSONPath Compliance Test Suite treat them as +# anchors): such a pattern is unusable (Anchored), like an invalid one +# (Invalid); `\^` counts as well. Differences to Ruby and ECMA-262 that +# matter: `.` matches any character except LF and CR; only the escapes \( \) \* +# \+ \- \. \? \[ \\ \] \n \r \t \{ \| \} and the category escapes \p{..} and \P{..} exist (no \d, # \w, \s); there are no lazy quantifiers, no {,m} and no non-capturing or other # special groups. Matching works on code points and is case-sensitive. # -# match() holds if the entire value matches, search() if a substring does. -# Following RFC 9535, the result is false if the value is not a string or the -# pattern does not conform to RFC 9485. +# match() holds if the entire value matches, search() if a substring does; +# both are false if the value is not a string. Callers check patterns with +# `problem` before evaluating the JSONPath: an unusable pattern makes the +# criterion skipped ("Results"). match?/search? return false for one, as a +# safeguard only. class IRegexp - class Invalid < StandardError; end + class Unusable < StandardError; end + class Invalid < Unusable; end + class Anchored < Unusable; end CATEGORIES = %w[L Ll Lm Lo Lt Lu M Mc Me Mn N Nd Nl No P Pc Pd Pe Pf Pi Po Ps Z Zl Zp Zs S Sc Sk Sm So C Cc Cf Cn Co].freeze SINGLE_ESCAPES = { "n" => 0x0A, "r" => 0x0D, "t" => 0x09 }.freeze @@ -31,7 +35,7 @@ def self.match?(pattern, value) return false unless value.is_a?(String) Regexp.new("\\A(?:#{translate(pattern)})\\z").match?(value) - rescue Invalid + rescue Unusable false end @@ -40,16 +44,18 @@ def self.search?(pattern, value) return false unless value.is_a?(String) Regexp.new(translate(pattern)).match?(value) - rescue Invalid + rescue Unusable false end - # nil for a pattern that conforms to RFC 9485, otherwise a reason. + # nil for a usable pattern, otherwise the reason. def self.problem(pattern) translate(pattern) nil rescue Invalid => e "is not a valid I-Regexp (RFC 9485): #{e.message}" + rescue Anchored => e + "contains #{e.message} outside a character class, which CRITERIA-FORMAT.md does not allow in JSONPath (use match() for a whole-value match)" end def self.translate(pattern) @@ -111,8 +117,7 @@ def atom when "(" then group when "." then @out << "[^\\n\\r]" when "[" then char_class - when "^" then @out << "\\A" - when "$" then @out << "\\z" + when "^", "$" then raise Anchored, c when "\\" then @out << escape_outside else raise Invalid, "#{c} is not allowed here" if SPECIAL.include?(c) @@ -150,6 +155,7 @@ def escape_outside c = take return category(c) if %w[p P].include?(c) + raise Anchored, "\\^" if c == "^" char(single_escape(c)) end diff --git a/test/services/applies_if_test.rb b/test/services/applies_if_test.rb index 0862fe1..a74df42 100644 --- a/test/services/applies_if_test.rb +++ b/test/services/applies_if_test.rb @@ -36,12 +36,17 @@ def holds?(conditions, passport) = AppliesIf.holds?(conditions, passport) refute holds?(BAT_002, { "contentSpecificationIds" => "Battery" }) end - test "a pattern that does not conform to RFC 9485 selects nothing (RFC 9535)" do - conditions = [{ "path" => "$.contentSpecificationIds[?search(@, '\\\\d+')]", "exists" => true }] - assert_nil AppliesIf.problem(conditions) - refute holds?(conditions, { "contentSpecificationIds" => ["59040"] }) - absent = [{ "path" => "$.contentSpecificationIds[?search(@, '\\\\d+')]", "exists" => false }] - assert holds?(absent, { "contentSpecificationIds" => ["59040"] }) + test "an invalid I-Regexp is found before the JSONPath is evaluated (dpp-criteria 4b17bb8)" do + conditions = [{ "path" => "$.contentSpecificationIds[?search(@, '\\\\d+')]", "exists" => false }] + assert_match(/\Aregular expression "\\\\d\+" of search\(\) in applies_if path .* is not a valid I-Regexp \(RFC 9485\)/, + AppliesIf.problem(conditions)) + end + + test "^ or $ outside a character class in match() or search() is unusable" do + ["$.contentSpecificationIds[?search(@, '^[Bb]atter')]", "$.contentSpecificationIds[?match(@, 'pcds$')]"].each do |path| + assert_match(/in applies_if path .* contains [$^] outside a character class/, AppliesIf.problem([{ "path" => path, "exists" => true }]), path) + end + assert_nil AppliesIf.problem([{ "path" => "$.contentSpecificationIds[?match(@, '[^x]+')]", "exists" => true }]) end test "JSONPath string literals: double quotes and escapes (RFC 9535, 2.3.1.1)" do @@ -60,6 +65,18 @@ def holds?(conditions, passport) = AppliesIf.holds?(conditions, passport) assert holds?([{ "path" => "$.facilityId", "exists" => false }], {}) end + test "matches holds only for JSON strings; other values are not converted to text" do + condition = [{ "path" => "$.v", "matches" => "^(1|true|null)$" }] + assert holds?(condition, { "v" => "1" }) + refute holds?(condition, { "v" => 1 }) + refute holds?(condition, { "v" => true }) + refute holds?(condition, { "v" => nil }) + refute holds?([{ "path" => "$.v", "matches" => "a" }], { "v" => ["a"] }) + refute holds?([{ "path" => "$.v", "matches" => "a" }], { "v" => { "k" => "a" } }) + refute holds?([{ "path" => "$.v", "matches" => "59040" }], { "v" => 59040.0 }) + assert holds?([{ "path" => "$.contentSpecificationIds[?search(@, '59040')]", "matches" => "DIN" }], { "contentSpecificationIds" => [59040, "DIN SPEC 59040"] }) + end + test "matches of the assertion is ECMA-262, searched in the value" do assert holds?([{ "path" => "$.granularity", "matches" => "^(item|batch)$" }], { "granularity" => "item" }) refute holds?([{ "path" => "$.granularity", "matches" => "^(item|batch)$" }], { "granularity" => "model\nitem" }) diff --git a/test/services/i_regexp_test.rb b/test/services/i_regexp_test.rb index 679300a..65dfefc 100644 --- a/test/services/i_regexp_test.rb +++ b/test/services/i_regexp_test.rb @@ -21,13 +21,21 @@ class IRegexpTest < ActiveSupport::TestCase refute IRegexp.search?("59040|PCDS|pcds", "Pcds") end - test "^ and $ anchor the whole value (Compliance Test Suite: explicit caret, explicit dollar)" do - assert IRegexp.match?("^ab.*", "abc") - refute IRegexp.match?("^ab.*", "xab") - assert IRegexp.match?(".*bc$", "abc") - refute IRegexp.match?(".*bc$", "abcx") - refute IRegexp.search?("^b", "a\nb") - refute IRegexp.search?("a$", "a\nb") + test "^ or $ outside a character class makes the pattern unusable (dpp-criteria 4b17bb8)" do + ["^ab.*", ".*bc$", "^[Bb]atter", "59040|PCDS$", "(^a)", "a\\^b"].each do |pattern| + assert_match(/\Acontains (\^|\$|\\\^) outside a character class/, IRegexp.problem(pattern), pattern) + refute IRegexp.search?(pattern, pattern.delete("^$\\")), pattern + end + end + + test "^ and $ inside a character class are allowed" do + assert_nil IRegexp.problem("[^a]") + assert_nil IRegexp.problem("[a^]") + assert_nil IRegexp.problem("[$]") + assert_nil IRegexp.problem("[\\^]") + assert IRegexp.match?("[$]", "$") + assert IRegexp.match?("[a^]", "^") + refute IRegexp.match?("[^a]", "a") end test ". matches any character except LF and CR" do diff --git a/test/services/passport_linter_test.rb b/test/services/passport_linter_test.rb index 94b5eb4..7623fd1 100644 --- a/test/services/passport_linter_test.rb +++ b/test/services/passport_linter_test.rb @@ -82,6 +82,18 @@ def lint(responses, check = CHECK) assert_match(/\Aregular expression "a\*\*" in applies_if is not a valid ECMA-262 regular expression/, result[:reason]) end + test "applies_if with ^ in search() skips the criterion with a reason instead of evaluating it" do + criterion = { "id" => "DPP-BAT-002", "title" => "t", "level" => "MUST", "target" => "passport", "method" => "automated", + "applies_if" => [{ "path" => "$.contentSpecificationIds[?search(@, '^[Bb]atter')]", "exists" => false }], + "check" => { "type" => "did", "paths" => ["$.facilityId"] } } + report = PassportLinter.new(catalogue: FakeCatalogue.new([criterion]), resolver: FakeResolver.new({})) + .run(passport: { "contentSpecificationIds" => ["Battery"] }) + result = report[:criteria].first + assert_equal "skipped", result[:result] + assert_match(/of search\(\) in applies_if path .* contains \^ outside a character class/, result[:reason]) + assert_equal "0 of 0 automated checks passed", report[:summary][:text] + end + test "applies_if with search() whose condition does not hold skips the criterion" do criterion = { "id" => "DPP-BAT-002", "title" => "t", "level" => "MUST", "target" => "passport", "method" => "automated", "applies_if" => [{ "path" => "$.contentSpecificationIds[?search(@, '[Bb]atter')]", "exists" => true }],