diff --git a/.gitignore b/.gitignore index 82902c3..d66d6e0 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ db/scopes.db db/scopes.db-wal db/scopes.db-shm spec/examples.txt +/tmp/test.db* diff --git a/CHANGELOG.md b/CHANGELOG.md index 83c15da..ff72b94 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,23 @@ # Changelog +## Version 2.1.3 - Malformed Scope Handling + +### ✨ Features + +- Add `GET /malformed-scopes` to list invalid-format assets ordered by program. +- Expand explicit YesWeHack hostname alternatives in parentheses or brackets, including multi-part TLDs and URL paths. + +### 🐛 Bug Fixes + +- Restrict hostname expansion to safe host alternatives so query parameters and paths cannot produce unintended targets. +- Hide malformed assets belonging to programs that have been removed. +- Clear previously rejected assets after a successful sync when they are no longer malformed. +- Allow the `extra_data` migration to complete when the column exists but the migration version was not recorded. +- Isolate tests from the application database and preserve migration metadata during test cleanup. +- Update the CLI version and HTTP User-Agent to `2.1.3`. + +--- + ## Version 2.1.2 - Scope State Synchronization ### 🐛 Bug Fixes diff --git a/README.md b/README.md index 51e9908..6e4a999 100644 --- a/README.md +++ b/README.md @@ -383,6 +383,37 @@ curl -H "X-API-KEY: your_api_key" "http://localhost:4567/exclusions" +
+GET /malformed-scopes - List scopes rejected for an invalid format + +Returns only assets recorded with an `Invalid format` reason for programs that still exist, ordered by program slug, platform, then scope value. Programs with the same slug on different platforms remain identifiable by the `platform` field. + +### Example Request + +```bash +curl -H "X-API-KEY: your_api_key" "http://localhost:4567/malformed-scopes" +``` + +### Example Response + +```json +{ + "malformed_scopes": [ + { + "id": 1, + "platform": "hackerone", + "program_slug": "example-program", + "value": "example.com (production only)", + "reason": "Invalid format for web scope", + "created_at": "2026-01-09T10:00:00Z" + } + ], + "count": 1 +} +``` + +
+ ## Notifications Two notifiers are available and can be enabled independently (both at once if needed): @@ -513,8 +544,10 @@ Scopes are automatically categorized based on pattern matching, overriding platf Each platform has custom normalization rules to handle their scope formats: **YesWeHack** -- Expands multi-TLD patterns: `example.{fr,com}` → `example.fr`, `example.com` -- Handles prefix patterns: `{www,api}.example.com` → `www.example.com`, `api.example.com` +- Expands pipe-separated hostname alternatives in parentheses or brackets: `(www|api).example.com` → `www.example.com`, `api.example.com` +- Preserves URL paths: `https://api-(eu|sg).example.com/connect` → `https://api-eu.example.com/connect`, `https://api-sg.example.com/connect` +- Removes soft hyphens from alternatives and expands multi-part TLDs such as `example.(com|co.uk)` +- Expands only named entries when a list contains `…`; it does not infer additional domains **HackerOne** - Replaces `.*` with `.com`: `example.*` → `example.com` @@ -546,6 +579,7 @@ Applied to all scopes regardless of platform: Validation Rules Scopes are validated before being added to the database. Invalid scopes trigger `ignored_asset` notifications. +On a later successful sync, scopes that no longer fail validation are removed from the malformed-scopes list. **Rejected patterns:** - Values without dots (unless IP addresses) diff --git a/config/settings.yml b/config/settings.yml index 5e38d7b..85c86a1 100644 --- a/config/settings.yml +++ b/config/settings.yml @@ -7,7 +7,7 @@ app: # HTTP client configuration http: - user_agent: "ScopesExtractor/2.1.2 (Ruby; +github.com/JoshuaMart/ScopesExtractor)" + user_agent: "ScopesExtractor/2.1.3 (Ruby; +github.com/JoshuaMart/ScopesExtractor)" proxy: null # Optional: "http://proxy.example.com:8080" timeout: 30 # seconds diff --git a/db/migrations/002_add_extra_data_to_history.rb b/db/migrations/002_add_extra_data_to_history.rb index 8fb16f3..e14d6ba 100644 --- a/db/migrations/002_add_extra_data_to_history.rb +++ b/db/migrations/002_add_extra_data_to_history.rb @@ -2,10 +2,12 @@ Sequel.migration do up do - add_column :history, :extra_data, String, text: true + unless schema(:history).any? { |column, _| column == :extra_data } + add_column :history, :extra_data, String, text: true + end end down do - drop_column :history, :extra_data + drop_column :history, :extra_data if schema(:history).any? { |column, _| column == :extra_data } end end diff --git a/lib/scopes_extractor/api.rb b/lib/scopes_extractor/api.rb index c2f9621..2edddda 100644 --- a/lib/scopes_extractor/api.rb +++ b/lib/scopes_extractor/api.rb @@ -166,6 +166,25 @@ class API < Sinatra::Base { error: e.message }.to_json end + # GET /malformed-scopes - List scopes rejected for an invalid format, sorted by program + get '/malformed-scopes' do + results = ScopesExtractor.db[:ignored_assets] + .join(:programs, + Sequel[:programs][:platform] => Sequel[:ignored_assets][:platform], + Sequel[:programs][:slug] => Sequel[:ignored_assets][:program_slug]) + .select_all(:ignored_assets) + .where(Sequel.like(Sequel[:ignored_assets][:reason], 'Invalid format%')) + .order(Sequel[:ignored_assets][:program_slug], + Sequel[:ignored_assets][:platform], + Sequel[:ignored_assets][:value]) + .all + + { malformed_scopes: results, count: results.size }.to_json + rescue StandardError => e + status 500 + { error: e.message }.to_json + end + # Error handlers error 404 do content_type :json diff --git a/lib/scopes_extractor/cli.rb b/lib/scopes_extractor/cli.rb index 75f74fd..90f06c1 100644 --- a/lib/scopes_extractor/cli.rb +++ b/lib/scopes_extractor/cli.rb @@ -10,7 +10,7 @@ def self.exit_on_failure? desc 'version', 'Display version information' def version - puts 'ScopesExtractor version 2.1.2' + puts 'ScopesExtractor version 2.1.3' end desc 'reset', 'Reset the database (WARNING: deletes all data)' diff --git a/lib/scopes_extractor/config.rb b/lib/scopes_extractor/config.rb index a6deae8..c20bf08 100644 --- a/lib/scopes_extractor/config.rb +++ b/lib/scopes_extractor/config.rb @@ -29,7 +29,7 @@ def http end def user_agent - http[:user_agent] || 'ScopesExtractor/2.1.2' + http[:user_agent] || 'ScopesExtractor/2.1.3' end def proxy diff --git a/lib/scopes_extractor/diff_engine.rb b/lib/scopes_extractor/diff_engine.rb index d0496fb..7624fea 100644 --- a/lib/scopes_extractor/diff_engine.rb +++ b/lib/scopes_extractor/diff_engine.rb @@ -147,6 +147,7 @@ def scope_identity(scope) def filter_and_normalize_scopes(platform_name, fetched_program) valid_scopes = [] + invalid_values = [] fetched_program.scopes.each do |scope| # Normalize the scope value @@ -155,6 +156,7 @@ def filter_and_normalize_scopes(platform_name, fetched_program) normalized_values.each do |normalized_value| # Validate unless Validator.valid_web_target?(normalized_value, scope.type) + invalid_values << normalized_value handle_ignored_asset(platform_name, fetched_program, normalized_value, scope.type) next end @@ -168,6 +170,13 @@ def filter_and_normalize_scopes(platform_name, fetched_program) end end + # Keep ignored assets in sync with the current malformed scopes. + ignored = @db[:ignored_assets] + .where(platform: platform_name, program_slug: fetched_program.slug) + .where(Sequel.like(:reason, 'Invalid format%')) + ignored = ignored.exclude(value: invalid_values.uniq) unless invalid_values.empty? + ignored.delete + valid_scopes end diff --git a/lib/scopes_extractor/normalizer.rb b/lib/scopes_extractor/normalizer.rb index 43d8a4f..bc0402f 100644 --- a/lib/scopes_extractor/normalizer.rb +++ b/lib/scopes_extractor/normalizer.rb @@ -2,7 +2,15 @@ module ScopesExtractor module Normalizer - MULTI_TLDS_REGEX = %r{(?https?://|wss?://|\*\.)?(?[\w.-]+\.)\((?[a-z0-9.\\/|_-]+)\)} + HOST_ALTERNATIVES_REGEX = / + \A(?[^()\[\]]*) + (?\(|\[) + (?[^()\[\]]+) + (?\)|\]) + (?[^()\[\]]*)\z + /x + ALTERNATIVE_REGEX = %r{\A[a-z0-9][a-z0-9._/-]*\z} + MAX_ALTERNATIVES = 100 def self.normalize(platform, value) value = global_normalization(value) @@ -20,7 +28,7 @@ def self.normalize(platform, value) end def self.global_normalization(value) - value = value.to_s.strip + value = value.to_s.strip.delete("\u00AD") # Remove protocol only if it's a wildcard (not a valid URL anymore) value = value.sub(%r{^https?://}, '') if value.include?('*') @@ -51,14 +59,40 @@ def self.global_end_strip(value) end def self.normalize_yeswehack(value) - if (match = value.match(MULTI_TLDS_REGEX)) - prefix = match[:prefix] || '' - middle = match[:middle] - tlds = match[:tlds].split('|') - tlds.map { |tld| "#{prefix}#{middle}#{tld}" } - else - [value] - end + match = value.match(HOST_ALTERNATIVES_REGEX) + return [value] unless expandable_host_pattern?(match) + + explicit_options = host_options(match) + return [value] if explicit_options.empty? + + explicit_options.map { |option| "#{match[:prefix]}#{option}#{match[:suffix]}" } + end + + def self.expandable_host_pattern?(match) + return false unless match + return false unless { '(' => ')', '[' => ']' }[match[:open]] == match[:close] + + # The group must appear in the hostname, before a path, query, fragment, port, or userinfo. + host_prefix = match[:prefix].sub(%r{\A[a-z][a-z0-9+.-]*://}, '') + !host_prefix.match?(%r{[/?#@:]}) + end + + def self.host_options(match) + options = match[:options].split('|', -1).map(&:strip) + return [] unless options.size.between?(2, MAX_ALTERNATIVES) + + explicit_options = options.reject { |option| %w[… ...].include?(option) } + return [] unless explicit_options.all? { |option| valid_host_option?(option, match) } + + explicit_options + end + + def self.valid_host_option?(option, match) + return false unless option.match?(ALTERNATIVE_REGEX) + return true unless option.include?('/') + + # Existing YesWeHack patterns may include a path in a complete TLD alternative. + match[:prefix].end_with?('.') && match[:suffix].empty? end def self.normalize_intigriti(value) diff --git a/spec/scopes_extractor/api_spec.rb b/spec/scopes_extractor/api_spec.rb index 51cc74c..cfbbaeb 100644 --- a/spec/scopes_extractor/api_spec.rb +++ b/spec/scopes_extractor/api_spec.rb @@ -474,6 +474,68 @@ def authenticated_header end end + describe 'GET /malformed-scopes' do + context 'with active and orphan assets' do + before do + %w[a-program z-program].each do |slug| + ScopesExtractor.db[:programs].insert( + platform: 'hackerone', slug: slug, name: slug, bounty: true, last_updated: Time.now + ) + end + + [ + ['z-program', 'z.example', 'Invalid format for web scope'], + ['a-program', 'b.example', 'Invalid format for api scope'], + ['a-program', 'a.example', 'Invalid format'], + ['a-program', 'excluded.example', 'Manually excluded'] + ].each do |slug, value, reason| + ScopesExtractor.db[:ignored_assets].insert( + platform: 'hackerone', program_slug: slug, value: value, + reason: reason, created_at: Time.now + ) + end + ScopesExtractor.db[:ignored_assets].insert( + platform: 'yeswehack', program_slug: 'a-program', value: 'orphan.example', + reason: 'Invalid format for web scope', created_at: Time.now + ) + end + + it 'returns only invalid-format scopes ordered by program and value' do + get '/malformed-scopes', {}, authenticated_header + + expect(last_response).to be_ok + data = JSON.parse(last_response.body) + expect(data['count']).to eq(3) + expect(data['malformed_scopes'].map { |scope| [scope['program_slug'], scope['value']] }).to eq( + [['a-program', 'a.example'], ['a-program', 'b.example'], ['z-program', 'z.example']] + ) + end + end + + it 'omits malformed scopes when their program has been removed' do + program_id = ScopesExtractor.db[:programs].insert( + platform: 'yeswehack', slug: 'removed-program', name: 'Removed', bounty: true, last_updated: Time.now + ) + ScopesExtractor.db[:ignored_assets].insert( + platform: 'yeswehack', program_slug: 'removed-program', value: 'invalid', + reason: 'Invalid format for web scope', created_at: Time.now + ) + ScopesExtractor.db[:programs].where(id: program_id).delete + + get '/malformed-scopes', {}, authenticated_header + + expect(last_response).to be_ok + expect(JSON.parse(last_response.body)).to include('malformed_scopes' => [], 'count' => 0) + end + + it 'returns an empty list when no malformed scopes exist' do + get '/malformed-scopes', {}, authenticated_header + + expect(last_response).to be_ok + expect(JSON.parse(last_response.body)).to include('malformed_scopes' => [], 'count' => 0) + end + end + describe 'Error handling' do it 'returns 404 for unknown routes' do get '/unknown', {}, authenticated_header diff --git a/spec/scopes_extractor/config_spec.rb b/spec/scopes_extractor/config_spec.rb index 2703721..e39aff2 100644 --- a/spec/scopes_extractor/config_spec.rb +++ b/spec/scopes_extractor/config_spec.rb @@ -28,6 +28,7 @@ describe '.database_path' do it 'returns the configured database path' do + allow(described_class).to receive(:database_path).and_call_original expect(described_class.database_path).to eq('db/scopes.db') end end @@ -36,7 +37,7 @@ describe 'http settings' do describe '.user_agent' do it 'returns the configured user agent' do - expect(described_class.user_agent).to include('ScopesExtractor/2.1.2') + expect(described_class.user_agent).to include('ScopesExtractor/2.1.3') expect(described_class.user_agent).to include('Ruby') expect(described_class.user_agent).to include('github.com') end diff --git a/spec/scopes_extractor/database_spec.rb b/spec/scopes_extractor/database_spec.rb index 8835609..0f2fb17 100644 --- a/spec/scopes_extractor/database_spec.rb +++ b/spec/scopes_extractor/database_spec.rb @@ -83,6 +83,19 @@ expect(ScopesExtractor.logger).to receive(:info).with('Database is up to date') described_class.migrate end + + it 'completes a migration when extra_data exists but its version was not recorded' do + Sequel.extension :migration + migrations_path = File.join(ScopesExtractor.root, 'db', 'migrations') + Sequel::Migrator.run(ScopesExtractor.db, migrations_path, target: 1) + ScopesExtractor.db.alter_table(:history) { add_column :extra_data, String, text: true } + ScopesExtractor.db[:history].insert(event_type: 'remove_program', extra_data: '{"slug":"test"}') + + described_class.migrate + + expect(ScopesExtractor.db[:schema_info].get(:version)).to eq(2) + expect(ScopesExtractor.db[:history].get(:extra_data)).to eq('{"slug":"test"}') + end end describe '.cleanup_old_history' do diff --git a/spec/scopes_extractor/diff_engine_spec.rb b/spec/scopes_extractor/diff_engine_spec.rb index 584262c..d1124df 100644 --- a/spec/scopes_extractor/diff_engine_spec.rb +++ b/spec/scopes_extractor/diff_engine_spec.rb @@ -238,6 +238,31 @@ ignored = ScopesExtractor.db[:ignored_assets].all expect(ignored.count).to eq(1) end + + it 'replaces a previously ignored alternative pattern with valid scopes' do + source = "www.unibet.(com\u00AD|it|se|co.uk|be|nl|dk|ro|ee|ie|com.au|mt)" + ScopesExtractor.db[:ignored_assets].insert( + platform: 'yeswehack', program_slug: 'test-program', value: source, + reason: 'Invalid format for web scope', created_at: Time.now + ) + ScopesExtractor.db[:ignored_assets].insert( + platform: 'yeswehack', program_slug: 'test-program', value: 'manual.example.com', + reason: 'Manually excluded', created_at: Time.now + ) + program = ScopesExtractor::Models::Program.new( + slug: 'test-program', platform: 'yeswehack', name: 'Test Program', bounty: true, + scopes: [ + ScopesExtractor::Models::Scope.new(value: source, type: 'web', is_in_scope: true), + ScopesExtractor::Models::Scope.new(value: 'invalid', type: 'web', is_in_scope: true) + ] + ) + + diff_engine.process_program('yeswehack', program) + + expect(ScopesExtractor.db[:scopes].select_map(:value)).to include('www.unibet.com', 'www.unibet.com.au') + expect(ScopesExtractor.db[:scopes].count).to eq(12) + expect(ScopesExtractor.db[:ignored_assets].select_map(:value)).to contain_exactly('invalid', 'manual.example.com') + end end context 'when a scope changes' do diff --git a/spec/scopes_extractor/normalizer_spec.rb b/spec/scopes_extractor/normalizer_spec.rb index 58ab2c2..7ba871a 100644 --- a/spec/scopes_extractor/normalizer_spec.rb +++ b/spec/scopes_extractor/normalizer_spec.rb @@ -60,6 +60,56 @@ input = 'example.(com|net|org)' expect(described_class.normalize('yeswehack', input)).to eq(['example.com', 'example.net', 'example.org']) end + + it 'expands alternatives at the beginning of a hostname' do + input = '(navigate|engage|checkout|internal|cached-api|api).shoppingapp.decathlon.com' + expect(described_class.normalize('yeswehack', input)).to eq( + %w[navigate engage checkout internal cached-api api].map { |name| "#{name}.shoppingapp.decathlon.com" } + ) + + expect(described_class.normalize('yeswehack', '(se|fi|uk).bingo.com')).to eq( + %w[se.bingo.com fi.bingo.com uk.bingo.com] + ) + end + + it 'expands alternatives inside a hostname while preserving the URL path' do + input = 'https://api-(global|eu|sg|cn).decathlon.net/connect' + expect(described_class.normalize('yeswehack', input)).to eq( + %w[global eu sg cn].map { |region| "https://api-#{region}.decathlon.net/connect" } + ) + end + + it 'removes soft hyphens before expanding multi-part TLDs' do + tlds = %w[com it se co.uk be nl dk ro ee ie com.au mt] + %w[payment www].each do |subdomain| + input = "#{subdomain}.unibet.(com\u00AD|it|se|co.uk|be|nl|dk|ro|ee|ie|com.au|mt)" + expect(described_class.normalize('yeswehack', input)).to eq( + tlds.map { |tld| "#{subdomain}.unibet.#{tld}" } + ) + end + end + + it 'expands only the explicit entries in a bracketed list' do + input = 'https://[it | fr | us | sg | …].louisvuitton.com' + expect(described_class.normalize('yeswehack', input)).to eq( + %w[it fr us sg].map { |country| "https://#{country}.louisvuitton.com" } + ) + end + + it 'does not expand alternatives outside a hostname or across a hostname boundary' do + expect(described_class.normalize('yeswehack', 'https://example.com/(one|two)')).to eq( + ['https://example.com/(one|two)'] + ) + expect(described_class.normalize('yeswehack', 'https://example.com?q=(a|b)')).to eq( + ['https://example.com?q=(a|b)'] + ) + expect(described_class.normalize('yeswehack', 'https://api-(foo/bar|baz).example.com')).to eq( + ['https://api-(foo/bar|baz).example.com'] + ) + expect(described_class.normalize('yeswehack', 'example.com (production only)')).to eq( + ['example.com (production only)'] + ) + end end context 'with Intigriti platform' do diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 9b5a6f2..28d10e6 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -12,6 +12,11 @@ ScopesExtractor.logger.level = Logger::FATAL RSpec.configure do |config| + config.before do + # Database specs reset tables; keep every example away from the configured application database. + allow(ScopesExtractor::Config).to receive(:database_path).and_return('tmp/test.db') + end + config.expect_with :rspec do |expectations| expectations.include_chain_clauses_in_custom_matcher_descriptions = true end @@ -35,6 +40,8 @@ begin ScopesExtractor.db.tables.each do |table| + next if table == :schema_info + ScopesExtractor.db[table].delete rescue Sequel::DatabaseError # Ignore errors on readonly databases or missing tables