fix: malformed scopes + migration fix

.gitignore

@@ -4,3 +4,4 @@ db/scopes.db
 db/scopes.db-wal
 db/scopes.db-shm
 spec/examples.txt
+/tmp/test.db*

CHANGELOG.md

@@ -1,5 +1,23 @@
 # Changelog
 
+## Version 2.1.3 - Malformed Scope Handling
+
+### ✨ Features
+
+- Add `GET /malformed-scopes` to list invalid-format assets ordered by program.
+- Expand explicit YesWeHack hostname alternatives in parentheses or brackets, including multi-part TLDs and URL paths.
+
+### 🐛 Bug Fixes
+
+- Restrict hostname expansion to safe host alternatives so query parameters and paths cannot produce unintended targets.
+- Hide malformed assets belonging to programs that have been removed.
+- Clear previously rejected assets after a successful sync when they are no longer malformed.
+- Allow the `extra_data` migration to complete when the column exists but the migration version was not recorded.
+- Isolate tests from the application database and preserve migration metadata during test cleanup.
+- Update the CLI version and HTTP User-Agent to `2.1.3`.
+
+---
+
 ## Version 2.1.2 - Scope State Synchronization
 
 ### 🐛 Bug Fixes

README.md

@@ -383,6 +383,37 @@ curl -H "X-API-KEY: your_api_key" "http://localhost:4567/exclusions"
 
 </details>
 
+<details>
+<summary><strong>GET /malformed-scopes</strong> - List scopes rejected for an invalid format</summary>
+
+Returns only assets recorded with an `Invalid format` reason for programs that still exist, ordered by program slug, platform, then scope value. Programs with the same slug on different platforms remain identifiable by the `platform` field.
+
+### Example Request
+
+```bash
+curl -H "X-API-KEY: your_api_key" "http://localhost:4567/malformed-scopes"
+```
+
+### Example Response
+
+```json
+{
+  "malformed_scopes": [
+    {
+      "id": 1,
+      "platform": "hackerone",
+      "program_slug": "example-program",
+      "value": "example.com (production only)",
+      "reason": "Invalid format for web scope",
+      "created_at": "2026-01-09T10:00:00Z"
+    }
+  ],
+  "count": 1
+}
+```
+
+</details>
+
 ## Notifications
 
 Two notifiers are available and can be enabled independently (both at once if needed):
@@ -513,8 +544,10 @@ Scopes are automatically categorized based on pattern matching, overriding platf
 Each platform has custom normalization rules to handle their scope formats:
 
 **YesWeHack**
-- Expands multi-TLD patterns: `example.{fr,com}` → `example.fr`, `example.com`
-- Handles prefix patterns: `{www,api}.example.com` → `www.example.com`, `api.example.com`
+- Expands pipe-separated hostname alternatives in parentheses or brackets: `(www|api).example.com` → `www.example.com`, `api.example.com`
+- Preserves URL paths: `https://api-(eu|sg).example.com/connect` → `https://api-eu.example.com/connect`, `https://api-sg.example.com/connect`
+- Removes soft hyphens from alternatives and expands multi-part TLDs such as `example.(com|co.uk)`
+- Expands only named entries when a list contains `…`; it does not infer additional domains
 
 **HackerOne**
 - Replaces `.*` with `.com`: `example.*` → `example.com`
@@ -546,6 +579,7 @@ Applied to all scopes regardless of platform:
 <summary><strong>Validation Rules</strong></summary>
 
 Scopes are validated before being added to the database. Invalid scopes trigger `ignored_asset` notifications.
+On a later successful sync, scopes that no longer fail validation are removed from the malformed-scopes list.
 
 **Rejected patterns:**
 - Values without dots (unless IP addresses)

config/settings.yml

@@ -7,7 +7,7 @@ app:
 
 # HTTP client configuration
 http:
-  user_agent: "ScopesExtractor/2.1.2 (Ruby; +github.com/JoshuaMart/ScopesExtractor)"
+  user_agent: "ScopesExtractor/2.1.3 (Ruby; +github.com/JoshuaMart/ScopesExtractor)"
   proxy: null  # Optional: "http://proxy.example.com:8080"
   timeout: 30  # seconds
 

db/migrations/002_add_extra_data_to_history.rb

@@ -2,10 +2,12 @@
 
 Sequel.migration do
   up do
-    add_column :history, :extra_data, String, text: true
+    unless schema(:history).any? { |column, _| column == :extra_data }
+      add_column :history, :extra_data, String, text: true
+    end
   end
 
   down do
-    drop_column :history, :extra_data
+    drop_column :history, :extra_data if schema(:history).any? { |column, _| column == :extra_data }
   end
 end

lib/scopes_extractor/api.rb

@@ -166,6 +166,25 @@ class API < Sinatra::Base
       { error: e.message }.to_json
     end
 
+    # GET /malformed-scopes - List scopes rejected for an invalid format, sorted by program
+    get '/malformed-scopes' do
+      results = ScopesExtractor.db[:ignored_assets]
+                               .join(:programs,
+                                     Sequel[:programs][:platform] => Sequel[:ignored_assets][:platform],
+                                     Sequel[:programs][:slug] => Sequel[:ignored_assets][:program_slug])
+                               .select_all(:ignored_assets)
+                               .where(Sequel.like(Sequel[:ignored_assets][:reason], 'Invalid format%'))
+                               .order(Sequel[:ignored_assets][:program_slug],
+                                      Sequel[:ignored_assets][:platform],
+                                      Sequel[:ignored_assets][:value])
+                               .all
+
+      { malformed_scopes: results, count: results.size }.to_json
+    rescue StandardError => e
+      status 500
+      { error: e.message }.to_json
+    end
+
     # Error handlers
     error 404 do
       content_type :json

lib/scopes_extractor/cli.rb

@@ -10,7 +10,7 @@ def self.exit_on_failure?
 
     desc 'version', 'Display version information'
     def version
-      puts 'ScopesExtractor version 2.1.2'
+      puts 'ScopesExtractor version 2.1.3'
     end
 
     desc 'reset', 'Reset the database (WARNING: deletes all data)'

lib/scopes_extractor/config.rb

@@ -29,7 +29,7 @@ def http
       end
 
       def user_agent
-        http[:user_agent] || 'ScopesExtractor/2.1.2'
+        http[:user_agent] || 'ScopesExtractor/2.1.3'
       end
 
       def proxy

lib/scopes_extractor/diff_engine.rb

@@ -147,6 +147,7 @@ def scope_identity(scope)
 
     def filter_and_normalize_scopes(platform_name, fetched_program)
       valid_scopes = []
+      invalid_values = []
 
       fetched_program.scopes.each do |scope|
         # Normalize the scope value
@@ -155,6 +156,7 @@ def filter_and_normalize_scopes(platform_name, fetched_program)
         normalized_values.each do |normalized_value|
           # Validate
           unless Validator.valid_web_target?(normalized_value, scope.type)
+            invalid_values << normalized_value
             handle_ignored_asset(platform_name, fetched_program, normalized_value, scope.type)
             next
           end
@@ -168,6 +170,13 @@ def filter_and_normalize_scopes(platform_name, fetched_program)
         end
       end
 
+      # Keep ignored assets in sync with the current malformed scopes.
+      ignored = @db[:ignored_assets]
+                .where(platform: platform_name, program_slug: fetched_program.slug)
+                .where(Sequel.like(:reason, 'Invalid format%'))
+      ignored = ignored.exclude(value: invalid_values.uniq) unless invalid_values.empty?
+      ignored.delete
+
       valid_scopes
     end
 

lib/scopes_extractor/normalizer.rb

@@ -2,7 +2,15 @@
 
 module ScopesExtractor
   module Normalizer
-    MULTI_TLDS_REGEX = %r{(?<prefix>https?://|wss?://|\*\.)?(?<middle>[\w.-]+\.)\((?<tlds>[a-z0-9.\\/|_-]+)\)}
+    HOST_ALTERNATIVES_REGEX = /
+      \A(?<prefix>[^()\[\]]*)
+      (?<open>\(|\[)
+      (?<options>[^()\[\]]+)
+      (?<close>\)|\])
+      (?<suffix>[^()\[\]]*)\z
+    /x
+    ALTERNATIVE_REGEX = %r{\A[a-z0-9][a-z0-9._/-]*\z}
+    MAX_ALTERNATIVES = 100
 
     def self.normalize(platform, value)
       value = global_normalization(value)
@@ -20,7 +28,7 @@ def self.normalize(platform, value)
     end
 
     def self.global_normalization(value)
-      value = value.to_s.strip
+      value = value.to_s.strip.delete("\u00AD")
 
       # Remove protocol only if it's a wildcard (not a valid URL anymore)
       value = value.sub(%r{^https?://}, '') if value.include?('*')
@@ -51,14 +59,40 @@ def self.global_end_strip(value)
     end
 
     def self.normalize_yeswehack(value)
-      if (match = value.match(MULTI_TLDS_REGEX))
-        prefix = match[:prefix] || ''
-        middle = match[:middle]
-        tlds = match[:tlds].split('|')
-        tlds.map { |tld| "#{prefix}#{middle}#{tld}" }
-      else
-        [value]
-      end
+      match = value.match(HOST_ALTERNATIVES_REGEX)
+      return [value] unless expandable_host_pattern?(match)
+
+      explicit_options = host_options(match)
+      return [value] if explicit_options.empty?
+
+      explicit_options.map { |option| "#{match[:prefix]}#{option}#{match[:suffix]}" }
+    end
+
+    def self.expandable_host_pattern?(match)
+      return false unless match
+      return false unless { '(' => ')', '[' => ']' }[match[:open]] == match[:close]
+
+      # The group must appear in the hostname, before a path, query, fragment, port, or userinfo.
+      host_prefix = match[:prefix].sub(%r{\A[a-z][a-z0-9+.-]*://}, '')
+      !host_prefix.match?(%r{[/?#@:]})
+    end
+
+    def self.host_options(match)
+      options = match[:options].split('|', -1).map(&:strip)
+      return [] unless options.size.between?(2, MAX_ALTERNATIVES)
+
+      explicit_options = options.reject { |option| %w[… ...].include?(option) }
+      return [] unless explicit_options.all? { |option| valid_host_option?(option, match) }
+
+      explicit_options
+    end
+
+    def self.valid_host_option?(option, match)
+      return false unless option.match?(ALTERNATIVE_REGEX)
+      return true unless option.include?('/')
+
+      # Existing YesWeHack patterns may include a path in a complete TLD alternative.
+      match[:prefix].end_with?('.') && match[:suffix].empty?
     end
 
     def self.normalize_intigriti(value)

spec/scopes_extractor/api_spec.rb

@@ -474,6 +474,68 @@ def authenticated_header
     end
   end
 
+  describe 'GET /malformed-scopes' do
+    context 'with active and orphan assets' do
+      before do
+        %w[a-program z-program].each do |slug|
+          ScopesExtractor.db[:programs].insert(
+            platform: 'hackerone', slug: slug, name: slug, bounty: true, last_updated: Time.now
+          )
+        end
+
+        [
+          ['z-program', 'z.example', 'Invalid format for web scope'],
+          ['a-program', 'b.example', 'Invalid format for api scope'],
+          ['a-program', 'a.example', 'Invalid format'],
+          ['a-program', 'excluded.example', 'Manually excluded']
+        ].each do |slug, value, reason|
+          ScopesExtractor.db[:ignored_assets].insert(
+            platform: 'hackerone', program_slug: slug, value: value,
+            reason: reason, created_at: Time.now
+          )
+        end
+        ScopesExtractor.db[:ignored_assets].insert(
+          platform: 'yeswehack', program_slug: 'a-program', value: 'orphan.example',
+          reason: 'Invalid format for web scope', created_at: Time.now
+        )
+      end
+
+      it 'returns only invalid-format scopes ordered by program and value' do
+        get '/malformed-scopes', {}, authenticated_header
+
+        expect(last_response).to be_ok
+        data = JSON.parse(last_response.body)
+        expect(data['count']).to eq(3)
+        expect(data['malformed_scopes'].map { |scope| [scope['program_slug'], scope['value']] }).to eq(
+          [['a-program', 'a.example'], ['a-program', 'b.example'], ['z-program', 'z.example']]
+        )
+      end
+    end
+
+    it 'omits malformed scopes when their program has been removed' do
+      program_id = ScopesExtractor.db[:programs].insert(
+        platform: 'yeswehack', slug: 'removed-program', name: 'Removed', bounty: true, last_updated: Time.now
+      )
+      ScopesExtractor.db[:ignored_assets].insert(
+        platform: 'yeswehack', program_slug: 'removed-program', value: 'invalid',
+        reason: 'Invalid format for web scope', created_at: Time.now
+      )
+      ScopesExtractor.db[:programs].where(id: program_id).delete
+
+      get '/malformed-scopes', {}, authenticated_header
+
+      expect(last_response).to be_ok
+      expect(JSON.parse(last_response.body)).to include('malformed_scopes' => [], 'count' => 0)
+    end
+
+    it 'returns an empty list when no malformed scopes exist' do
+      get '/malformed-scopes', {}, authenticated_header
+
+      expect(last_response).to be_ok
+      expect(JSON.parse(last_response.body)).to include('malformed_scopes' => [], 'count' => 0)
+    end
+  end
+
   describe 'Error handling' do
     it 'returns 404 for unknown routes' do
       get '/unknown', {}, authenticated_header

spec/scopes_extractor/config_spec.rb

@@ -28,6 +28,7 @@
 
     describe '.database_path' do
       it 'returns the configured database path' do
+        allow(described_class).to receive(:database_path).and_call_original
         expect(described_class.database_path).to eq('db/scopes.db')
       end
     end
@@ -36,7 +37,7 @@
   describe 'http settings' do
     describe '.user_agent' do
       it 'returns the configured user agent' do
-        expect(described_class.user_agent).to include('ScopesExtractor/2.1.2')
+        expect(described_class.user_agent).to include('ScopesExtractor/2.1.3')
         expect(described_class.user_agent).to include('Ruby')
         expect(described_class.user_agent).to include('github.com')
       end

spec/scopes_extractor/database_spec.rb

@@ -83,6 +83,19 @@
       expect(ScopesExtractor.logger).to receive(:info).with('Database is up to date')
       described_class.migrate
     end
+
+    it 'completes a migration when extra_data exists but its version was not recorded' do
+      Sequel.extension :migration
+      migrations_path = File.join(ScopesExtractor.root, 'db', 'migrations')
+      Sequel::Migrator.run(ScopesExtractor.db, migrations_path, target: 1)
+      ScopesExtractor.db.alter_table(:history) { add_column :extra_data, String, text: true }
+      ScopesExtractor.db[:history].insert(event_type: 'remove_program', extra_data: '{"slug":"test"}')
+
+      described_class.migrate
+
+      expect(ScopesExtractor.db[:schema_info].get(:version)).to eq(2)
+      expect(ScopesExtractor.db[:history].get(:extra_data)).to eq('{"slug":"test"}')
+    end
   end
 
   describe '.cleanup_old_history' do

spec/scopes_extractor/diff_engine_spec.rb

@@ -238,6 +238,31 @@
         ignored = ScopesExtractor.db[:ignored_assets].all
         expect(ignored.count).to eq(1)
       end
+
+      it 'replaces a previously ignored alternative pattern with valid scopes' do
+        source = "www.unibet.(com\u00AD|it|se|co.uk|be|nl|dk|ro|ee|ie|com.au|mt)"
+        ScopesExtractor.db[:ignored_assets].insert(
+          platform: 'yeswehack', program_slug: 'test-program', value: source,
+          reason: 'Invalid format for web scope', created_at: Time.now
+        )
+        ScopesExtractor.db[:ignored_assets].insert(
+          platform: 'yeswehack', program_slug: 'test-program', value: 'manual.example.com',
+          reason: 'Manually excluded', created_at: Time.now
+        )
+        program = ScopesExtractor::Models::Program.new(
+          slug: 'test-program', platform: 'yeswehack', name: 'Test Program', bounty: true,
+          scopes: [
+            ScopesExtractor::Models::Scope.new(value: source, type: 'web', is_in_scope: true),
+            ScopesExtractor::Models::Scope.new(value: 'invalid', type: 'web', is_in_scope: true)
+          ]
+        )
+
+        diff_engine.process_program('yeswehack', program)
+
+        expect(ScopesExtractor.db[:scopes].select_map(:value)).to include('www.unibet.com', 'www.unibet.com.au')
+        expect(ScopesExtractor.db[:scopes].count).to eq(12)
+        expect(ScopesExtractor.db[:ignored_assets].select_map(:value)).to contain_exactly('invalid', 'manual.example.com')
+      end
     end
 
     context 'when a scope changes' do