diff --git a/.gitignore b/.gitignore
index 82902c3..d66d6e0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,3 +4,4 @@ db/scopes.db
db/scopes.db-wal
db/scopes.db-shm
spec/examples.txt
+/tmp/test.db*
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 83c15da..ff72b94 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,23 @@
# Changelog
+## Version 2.1.3 - Malformed Scope Handling
+
+### ✨ Features
+
+- Add `GET /malformed-scopes` to list invalid-format assets ordered by program.
+- Expand explicit YesWeHack hostname alternatives in parentheses or brackets, including multi-part TLDs and URL paths.
+
+### 🐛 Bug Fixes
+
+- Restrict hostname expansion to safe host alternatives so query parameters and paths cannot produce unintended targets.
+- Hide malformed assets belonging to programs that have been removed.
+- Clear previously rejected assets after a successful sync when they are no longer malformed.
+- Allow the `extra_data` migration to complete when the column exists but the migration version was not recorded.
+- Isolate tests from the application database and preserve migration metadata during test cleanup.
+- Update the CLI version and HTTP User-Agent to `2.1.3`.
+
+---
+
## Version 2.1.2 - Scope State Synchronization
### 🐛 Bug Fixes
diff --git a/README.md b/README.md
index 51e9908..6e4a999 100644
--- a/README.md
+++ b/README.md
@@ -383,6 +383,37 @@ curl -H "X-API-KEY: your_api_key" "http://localhost:4567/exclusions"
+
+GET /malformed-scopes - List scopes rejected for an invalid format
+
+Returns only assets recorded with an `Invalid format` reason for programs that still exist, ordered by program slug, platform, then scope value. Programs with the same slug on different platforms remain identifiable by the `platform` field.
+
+### Example Request
+
+```bash
+curl -H "X-API-KEY: your_api_key" "http://localhost:4567/malformed-scopes"
+```
+
+### Example Response
+
+```json
+{
+ "malformed_scopes": [
+ {
+ "id": 1,
+ "platform": "hackerone",
+ "program_slug": "example-program",
+ "value": "example.com (production only)",
+ "reason": "Invalid format for web scope",
+ "created_at": "2026-01-09T10:00:00Z"
+ }
+ ],
+ "count": 1
+}
+```
+
+
+
## Notifications
Two notifiers are available and can be enabled independently (both at once if needed):
@@ -513,8 +544,10 @@ Scopes are automatically categorized based on pattern matching, overriding platf
Each platform has custom normalization rules to handle their scope formats:
**YesWeHack**
-- Expands multi-TLD patterns: `example.{fr,com}` → `example.fr`, `example.com`
-- Handles prefix patterns: `{www,api}.example.com` → `www.example.com`, `api.example.com`
+- Expands pipe-separated hostname alternatives in parentheses or brackets: `(www|api).example.com` → `www.example.com`, `api.example.com`
+- Preserves URL paths: `https://api-(eu|sg).example.com/connect` → `https://api-eu.example.com/connect`, `https://api-sg.example.com/connect`
+- Removes soft hyphens from alternatives and expands multi-part TLDs such as `example.(com|co.uk)`
+- Expands only named entries when a list contains `…`; it does not infer additional domains
**HackerOne**
- Replaces `.*` with `.com`: `example.*` → `example.com`
@@ -546,6 +579,7 @@ Applied to all scopes regardless of platform:
Validation Rules
Scopes are validated before being added to the database. Invalid scopes trigger `ignored_asset` notifications.
+On a later successful sync, scopes that no longer fail validation are removed from the malformed-scopes list.
**Rejected patterns:**
- Values without dots (unless IP addresses)
diff --git a/config/settings.yml b/config/settings.yml
index 5e38d7b..85c86a1 100644
--- a/config/settings.yml
+++ b/config/settings.yml
@@ -7,7 +7,7 @@ app:
# HTTP client configuration
http:
- user_agent: "ScopesExtractor/2.1.2 (Ruby; +github.com/JoshuaMart/ScopesExtractor)"
+ user_agent: "ScopesExtractor/2.1.3 (Ruby; +github.com/JoshuaMart/ScopesExtractor)"
proxy: null # Optional: "http://proxy.example.com:8080"
timeout: 30 # seconds
diff --git a/db/migrations/002_add_extra_data_to_history.rb b/db/migrations/002_add_extra_data_to_history.rb
index 8fb16f3..e14d6ba 100644
--- a/db/migrations/002_add_extra_data_to_history.rb
+++ b/db/migrations/002_add_extra_data_to_history.rb
@@ -2,10 +2,12 @@
Sequel.migration do
up do
- add_column :history, :extra_data, String, text: true
+ unless schema(:history).any? { |column, _| column == :extra_data }
+ add_column :history, :extra_data, String, text: true
+ end
end
down do
- drop_column :history, :extra_data
+ drop_column :history, :extra_data if schema(:history).any? { |column, _| column == :extra_data }
end
end
diff --git a/lib/scopes_extractor/api.rb b/lib/scopes_extractor/api.rb
index c2f9621..2edddda 100644
--- a/lib/scopes_extractor/api.rb
+++ b/lib/scopes_extractor/api.rb
@@ -166,6 +166,25 @@ class API < Sinatra::Base
{ error: e.message }.to_json
end
+ # GET /malformed-scopes - List scopes rejected for an invalid format, sorted by program
+ get '/malformed-scopes' do
+ results = ScopesExtractor.db[:ignored_assets]
+ .join(:programs,
+ Sequel[:programs][:platform] => Sequel[:ignored_assets][:platform],
+ Sequel[:programs][:slug] => Sequel[:ignored_assets][:program_slug])
+ .select_all(:ignored_assets)
+ .where(Sequel.like(Sequel[:ignored_assets][:reason], 'Invalid format%'))
+ .order(Sequel[:ignored_assets][:program_slug],
+ Sequel[:ignored_assets][:platform],
+ Sequel[:ignored_assets][:value])
+ .all
+
+ { malformed_scopes: results, count: results.size }.to_json
+ rescue StandardError => e
+ status 500
+ { error: e.message }.to_json
+ end
+
# Error handlers
error 404 do
content_type :json
diff --git a/lib/scopes_extractor/cli.rb b/lib/scopes_extractor/cli.rb
index 75f74fd..90f06c1 100644
--- a/lib/scopes_extractor/cli.rb
+++ b/lib/scopes_extractor/cli.rb
@@ -10,7 +10,7 @@ def self.exit_on_failure?
desc 'version', 'Display version information'
def version
- puts 'ScopesExtractor version 2.1.2'
+ puts 'ScopesExtractor version 2.1.3'
end
desc 'reset', 'Reset the database (WARNING: deletes all data)'
diff --git a/lib/scopes_extractor/config.rb b/lib/scopes_extractor/config.rb
index a6deae8..c20bf08 100644
--- a/lib/scopes_extractor/config.rb
+++ b/lib/scopes_extractor/config.rb
@@ -29,7 +29,7 @@ def http
end
def user_agent
- http[:user_agent] || 'ScopesExtractor/2.1.2'
+ http[:user_agent] || 'ScopesExtractor/2.1.3'
end
def proxy
diff --git a/lib/scopes_extractor/diff_engine.rb b/lib/scopes_extractor/diff_engine.rb
index d0496fb..7624fea 100644
--- a/lib/scopes_extractor/diff_engine.rb
+++ b/lib/scopes_extractor/diff_engine.rb
@@ -147,6 +147,7 @@ def scope_identity(scope)
def filter_and_normalize_scopes(platform_name, fetched_program)
valid_scopes = []
+ invalid_values = []
fetched_program.scopes.each do |scope|
# Normalize the scope value
@@ -155,6 +156,7 @@ def filter_and_normalize_scopes(platform_name, fetched_program)
normalized_values.each do |normalized_value|
# Validate
unless Validator.valid_web_target?(normalized_value, scope.type)
+ invalid_values << normalized_value
handle_ignored_asset(platform_name, fetched_program, normalized_value, scope.type)
next
end
@@ -168,6 +170,13 @@ def filter_and_normalize_scopes(platform_name, fetched_program)
end
end
+ # Keep ignored assets in sync with the current malformed scopes.
+ ignored = @db[:ignored_assets]
+ .where(platform: platform_name, program_slug: fetched_program.slug)
+ .where(Sequel.like(:reason, 'Invalid format%'))
+ ignored = ignored.exclude(value: invalid_values.uniq) unless invalid_values.empty?
+ ignored.delete
+
valid_scopes
end
diff --git a/lib/scopes_extractor/normalizer.rb b/lib/scopes_extractor/normalizer.rb
index 43d8a4f..bc0402f 100644
--- a/lib/scopes_extractor/normalizer.rb
+++ b/lib/scopes_extractor/normalizer.rb
@@ -2,7 +2,15 @@
module ScopesExtractor
module Normalizer
- MULTI_TLDS_REGEX = %r{(?https?://|wss?://|\*\.)?(?[\w.-]+\.)\((?[a-z0-9.\\/|_-]+)\)}
+ HOST_ALTERNATIVES_REGEX = /
+ \A(?[^()\[\]]*)
+ (?\(|\[)
+ (?[^()\[\]]+)
+ (?\)|\])
+ (?[^()\[\]]*)\z
+ /x
+ ALTERNATIVE_REGEX = %r{\A[a-z0-9][a-z0-9._/-]*\z}
+ MAX_ALTERNATIVES = 100
def self.normalize(platform, value)
value = global_normalization(value)
@@ -20,7 +28,7 @@ def self.normalize(platform, value)
end
def self.global_normalization(value)
- value = value.to_s.strip
+ value = value.to_s.strip.delete("\u00AD")
# Remove protocol only if it's a wildcard (not a valid URL anymore)
value = value.sub(%r{^https?://}, '') if value.include?('*')
@@ -51,14 +59,40 @@ def self.global_end_strip(value)
end
def self.normalize_yeswehack(value)
- if (match = value.match(MULTI_TLDS_REGEX))
- prefix = match[:prefix] || ''
- middle = match[:middle]
- tlds = match[:tlds].split('|')
- tlds.map { |tld| "#{prefix}#{middle}#{tld}" }
- else
- [value]
- end
+ match = value.match(HOST_ALTERNATIVES_REGEX)
+ return [value] unless expandable_host_pattern?(match)
+
+ explicit_options = host_options(match)
+ return [value] if explicit_options.empty?
+
+ explicit_options.map { |option| "#{match[:prefix]}#{option}#{match[:suffix]}" }
+ end
+
+ def self.expandable_host_pattern?(match)
+ return false unless match
+ return false unless { '(' => ')', '[' => ']' }[match[:open]] == match[:close]
+
+ # The group must appear in the hostname, before a path, query, fragment, port, or userinfo.
+ host_prefix = match[:prefix].sub(%r{\A[a-z][a-z0-9+.-]*://}, '')
+ !host_prefix.match?(%r{[/?#@:]})
+ end
+
+ def self.host_options(match)
+ options = match[:options].split('|', -1).map(&:strip)
+ return [] unless options.size.between?(2, MAX_ALTERNATIVES)
+
+ explicit_options = options.reject { |option| %w[… ...].include?(option) }
+ return [] unless explicit_options.all? { |option| valid_host_option?(option, match) }
+
+ explicit_options
+ end
+
+ def self.valid_host_option?(option, match)
+ return false unless option.match?(ALTERNATIVE_REGEX)
+ return true unless option.include?('/')
+
+ # Existing YesWeHack patterns may include a path in a complete TLD alternative.
+ match[:prefix].end_with?('.') && match[:suffix].empty?
end
def self.normalize_intigriti(value)
diff --git a/spec/scopes_extractor/api_spec.rb b/spec/scopes_extractor/api_spec.rb
index 51cc74c..cfbbaeb 100644
--- a/spec/scopes_extractor/api_spec.rb
+++ b/spec/scopes_extractor/api_spec.rb
@@ -474,6 +474,68 @@ def authenticated_header
end
end
+ describe 'GET /malformed-scopes' do
+ context 'with active and orphan assets' do
+ before do
+ %w[a-program z-program].each do |slug|
+ ScopesExtractor.db[:programs].insert(
+ platform: 'hackerone', slug: slug, name: slug, bounty: true, last_updated: Time.now
+ )
+ end
+
+ [
+ ['z-program', 'z.example', 'Invalid format for web scope'],
+ ['a-program', 'b.example', 'Invalid format for api scope'],
+ ['a-program', 'a.example', 'Invalid format'],
+ ['a-program', 'excluded.example', 'Manually excluded']
+ ].each do |slug, value, reason|
+ ScopesExtractor.db[:ignored_assets].insert(
+ platform: 'hackerone', program_slug: slug, value: value,
+ reason: reason, created_at: Time.now
+ )
+ end
+ ScopesExtractor.db[:ignored_assets].insert(
+ platform: 'yeswehack', program_slug: 'a-program', value: 'orphan.example',
+ reason: 'Invalid format for web scope', created_at: Time.now
+ )
+ end
+
+ it 'returns only invalid-format scopes ordered by program and value' do
+ get '/malformed-scopes', {}, authenticated_header
+
+ expect(last_response).to be_ok
+ data = JSON.parse(last_response.body)
+ expect(data['count']).to eq(3)
+ expect(data['malformed_scopes'].map { |scope| [scope['program_slug'], scope['value']] }).to eq(
+ [['a-program', 'a.example'], ['a-program', 'b.example'], ['z-program', 'z.example']]
+ )
+ end
+ end
+
+ it 'omits malformed scopes when their program has been removed' do
+ program_id = ScopesExtractor.db[:programs].insert(
+ platform: 'yeswehack', slug: 'removed-program', name: 'Removed', bounty: true, last_updated: Time.now
+ )
+ ScopesExtractor.db[:ignored_assets].insert(
+ platform: 'yeswehack', program_slug: 'removed-program', value: 'invalid',
+ reason: 'Invalid format for web scope', created_at: Time.now
+ )
+ ScopesExtractor.db[:programs].where(id: program_id).delete
+
+ get '/malformed-scopes', {}, authenticated_header
+
+ expect(last_response).to be_ok
+ expect(JSON.parse(last_response.body)).to include('malformed_scopes' => [], 'count' => 0)
+ end
+
+ it 'returns an empty list when no malformed scopes exist' do
+ get '/malformed-scopes', {}, authenticated_header
+
+ expect(last_response).to be_ok
+ expect(JSON.parse(last_response.body)).to include('malformed_scopes' => [], 'count' => 0)
+ end
+ end
+
describe 'Error handling' do
it 'returns 404 for unknown routes' do
get '/unknown', {}, authenticated_header
diff --git a/spec/scopes_extractor/config_spec.rb b/spec/scopes_extractor/config_spec.rb
index 2703721..e39aff2 100644
--- a/spec/scopes_extractor/config_spec.rb
+++ b/spec/scopes_extractor/config_spec.rb
@@ -28,6 +28,7 @@
describe '.database_path' do
it 'returns the configured database path' do
+ allow(described_class).to receive(:database_path).and_call_original
expect(described_class.database_path).to eq('db/scopes.db')
end
end
@@ -36,7 +37,7 @@
describe 'http settings' do
describe '.user_agent' do
it 'returns the configured user agent' do
- expect(described_class.user_agent).to include('ScopesExtractor/2.1.2')
+ expect(described_class.user_agent).to include('ScopesExtractor/2.1.3')
expect(described_class.user_agent).to include('Ruby')
expect(described_class.user_agent).to include('github.com')
end
diff --git a/spec/scopes_extractor/database_spec.rb b/spec/scopes_extractor/database_spec.rb
index 8835609..0f2fb17 100644
--- a/spec/scopes_extractor/database_spec.rb
+++ b/spec/scopes_extractor/database_spec.rb
@@ -83,6 +83,19 @@
expect(ScopesExtractor.logger).to receive(:info).with('Database is up to date')
described_class.migrate
end
+
+ it 'completes a migration when extra_data exists but its version was not recorded' do
+ Sequel.extension :migration
+ migrations_path = File.join(ScopesExtractor.root, 'db', 'migrations')
+ Sequel::Migrator.run(ScopesExtractor.db, migrations_path, target: 1)
+ ScopesExtractor.db.alter_table(:history) { add_column :extra_data, String, text: true }
+ ScopesExtractor.db[:history].insert(event_type: 'remove_program', extra_data: '{"slug":"test"}')
+
+ described_class.migrate
+
+ expect(ScopesExtractor.db[:schema_info].get(:version)).to eq(2)
+ expect(ScopesExtractor.db[:history].get(:extra_data)).to eq('{"slug":"test"}')
+ end
end
describe '.cleanup_old_history' do
diff --git a/spec/scopes_extractor/diff_engine_spec.rb b/spec/scopes_extractor/diff_engine_spec.rb
index 584262c..d1124df 100644
--- a/spec/scopes_extractor/diff_engine_spec.rb
+++ b/spec/scopes_extractor/diff_engine_spec.rb
@@ -238,6 +238,31 @@
ignored = ScopesExtractor.db[:ignored_assets].all
expect(ignored.count).to eq(1)
end
+
+ it 'replaces a previously ignored alternative pattern with valid scopes' do
+ source = "www.unibet.(com\u00AD|it|se|co.uk|be|nl|dk|ro|ee|ie|com.au|mt)"
+ ScopesExtractor.db[:ignored_assets].insert(
+ platform: 'yeswehack', program_slug: 'test-program', value: source,
+ reason: 'Invalid format for web scope', created_at: Time.now
+ )
+ ScopesExtractor.db[:ignored_assets].insert(
+ platform: 'yeswehack', program_slug: 'test-program', value: 'manual.example.com',
+ reason: 'Manually excluded', created_at: Time.now
+ )
+ program = ScopesExtractor::Models::Program.new(
+ slug: 'test-program', platform: 'yeswehack', name: 'Test Program', bounty: true,
+ scopes: [
+ ScopesExtractor::Models::Scope.new(value: source, type: 'web', is_in_scope: true),
+ ScopesExtractor::Models::Scope.new(value: 'invalid', type: 'web', is_in_scope: true)
+ ]
+ )
+
+ diff_engine.process_program('yeswehack', program)
+
+ expect(ScopesExtractor.db[:scopes].select_map(:value)).to include('www.unibet.com', 'www.unibet.com.au')
+ expect(ScopesExtractor.db[:scopes].count).to eq(12)
+ expect(ScopesExtractor.db[:ignored_assets].select_map(:value)).to contain_exactly('invalid', 'manual.example.com')
+ end
end
context 'when a scope changes' do
diff --git a/spec/scopes_extractor/normalizer_spec.rb b/spec/scopes_extractor/normalizer_spec.rb
index 58ab2c2..7ba871a 100644
--- a/spec/scopes_extractor/normalizer_spec.rb
+++ b/spec/scopes_extractor/normalizer_spec.rb
@@ -60,6 +60,56 @@
input = 'example.(com|net|org)'
expect(described_class.normalize('yeswehack', input)).to eq(['example.com', 'example.net', 'example.org'])
end
+
+ it 'expands alternatives at the beginning of a hostname' do
+ input = '(navigate|engage|checkout|internal|cached-api|api).shoppingapp.decathlon.com'
+ expect(described_class.normalize('yeswehack', input)).to eq(
+ %w[navigate engage checkout internal cached-api api].map { |name| "#{name}.shoppingapp.decathlon.com" }
+ )
+
+ expect(described_class.normalize('yeswehack', '(se|fi|uk).bingo.com')).to eq(
+ %w[se.bingo.com fi.bingo.com uk.bingo.com]
+ )
+ end
+
+ it 'expands alternatives inside a hostname while preserving the URL path' do
+ input = 'https://api-(global|eu|sg|cn).decathlon.net/connect'
+ expect(described_class.normalize('yeswehack', input)).to eq(
+ %w[global eu sg cn].map { |region| "https://api-#{region}.decathlon.net/connect" }
+ )
+ end
+
+ it 'removes soft hyphens before expanding multi-part TLDs' do
+ tlds = %w[com it se co.uk be nl dk ro ee ie com.au mt]
+ %w[payment www].each do |subdomain|
+ input = "#{subdomain}.unibet.(com\u00AD|it|se|co.uk|be|nl|dk|ro|ee|ie|com.au|mt)"
+ expect(described_class.normalize('yeswehack', input)).to eq(
+ tlds.map { |tld| "#{subdomain}.unibet.#{tld}" }
+ )
+ end
+ end
+
+ it 'expands only the explicit entries in a bracketed list' do
+ input = 'https://[it | fr | us | sg | …].louisvuitton.com'
+ expect(described_class.normalize('yeswehack', input)).to eq(
+ %w[it fr us sg].map { |country| "https://#{country}.louisvuitton.com" }
+ )
+ end
+
+ it 'does not expand alternatives outside a hostname or across a hostname boundary' do
+ expect(described_class.normalize('yeswehack', 'https://example.com/(one|two)')).to eq(
+ ['https://example.com/(one|two)']
+ )
+ expect(described_class.normalize('yeswehack', 'https://example.com?q=(a|b)')).to eq(
+ ['https://example.com?q=(a|b)']
+ )
+ expect(described_class.normalize('yeswehack', 'https://api-(foo/bar|baz).example.com')).to eq(
+ ['https://api-(foo/bar|baz).example.com']
+ )
+ expect(described_class.normalize('yeswehack', 'example.com (production only)')).to eq(
+ ['example.com (production only)']
+ )
+ end
end
context 'with Intigriti platform' do
diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb
index 9b5a6f2..28d10e6 100644
--- a/spec/spec_helper.rb
+++ b/spec/spec_helper.rb
@@ -12,6 +12,11 @@
ScopesExtractor.logger.level = Logger::FATAL
RSpec.configure do |config|
+ config.before do
+ # Database specs reset tables; keep every example away from the configured application database.
+ allow(ScopesExtractor::Config).to receive(:database_path).and_return('tmp/test.db')
+ end
+
config.expect_with :rspec do |expectations|
expectations.include_chain_clauses_in_custom_matcher_descriptions = true
end
@@ -35,6 +40,8 @@
begin
ScopesExtractor.db.tables.each do |table|
+ next if table == :schema_info
+
ScopesExtractor.db[table].delete
rescue Sequel::DatabaseError
# Ignore errors on readonly databases or missing tables