fix: scope state synchronization

CHANGELOG.md

@@ -1,5 +1,16 @@
 # Changelog
 
+## Version 2.1.2 - Scope State Synchronization
+
+### 🐛 Bug Fixes
+
+- Compare scopes by value, type, and in-scope status so changes to either property update the stored scope and the `/` API response.
+- Record scope state and type transitions as `remove_scope` followed by `add_scope` events in history.
+- Remove scopes by their database ID so entries sharing a value do not delete one another.
+- Update the CLI `version` command and HTTP User-Agent to report the current release.
+
+---
+
 ## Version 2.1.1 - YesWeHack Inactive Programs
 
 ### 🐛 Bug Fixes

config/settings.yml

@@ -7,7 +7,7 @@ app:
 
 # HTTP client configuration
 http:
-  user_agent: "ScopesExtractor/2.1.1 (Ruby; +github.com/JoshuaMart/ScopesExtractor)"
+  user_agent: "ScopesExtractor/2.1.2 (Ruby; +github.com/JoshuaMart/ScopesExtractor)"
   proxy: null  # Optional: "http://proxy.example.com:8080"
   timeout: 30  # seconds
 

lib/scopes_extractor/cli.rb

@@ -10,7 +10,7 @@ def self.exit_on_failure?
 
     desc 'version', 'Display version information'
     def version
-      puts 'ScopesExtractor version 2.1.1'
+      puts 'ScopesExtractor version 2.1.2'
     end
 
     desc 'reset', 'Reset the database (WARNING: deletes all data)'

lib/scopes_extractor/config.rb

@@ -29,7 +29,7 @@ def http
       end
 
       def user_agent
-        http[:user_agent] || 'ScopesExtractor/2.1.1'
+        http[:user_agent] || 'ScopesExtractor/2.1.2'
       end
 
       def proxy

lib/scopes_extractor/diff_engine.rb

@@ -118,25 +118,31 @@ def update_program_if_changed(program_id, existing_program, fetched_program)
 
     def sync_scopes(program_id, platform_name, fetched_program, skip_notifications: false)
       existing_scopes = @db[:scopes].where(program_id: program_id).all
-      existing_values = existing_scopes.map { |s| s[:value] }
 
-      # Normalize and validate scopes
+      # Normalize and validate scopes before comparing all three identity fields.
+      # A status or type change becomes a removal followed by an addition.
       filtered_scopes = filter_and_normalize_scopes(platform_name, fetched_program)
-      fetched_values = filtered_scopes.map(&:value).uniq
+                        .uniq { |scope| scope_identity(scope) }
+      existing_identities = existing_scopes.to_set { |scope| scope_identity(scope) }
+      fetched_identities = filtered_scopes.to_set { |scope| scope_identity(scope) }
 
-      scope_stats = process_added_scopes(
+      removed_scopes = existing_scopes.reject { |scope| fetched_identities.include?(scope_identity(scope)) }
+      added_scopes = filtered_scopes.reject { |scope| existing_identities.include?(scope_identity(scope)) }
+
+      process_removed_scopes(program_id, platform_name, fetched_program, removed_scopes,
+                             skip_notifications: skip_notifications)
+
+      process_added_scopes(
         program_id,
         platform_name,
         fetched_program,
-        existing_values,
-        filtered_scopes,
+        added_scopes,
         skip_notifications: skip_notifications
       )
+    end
 
-      process_removed_scopes(program_id, platform_name, fetched_program, existing_values, fetched_values,
-                             existing_scopes, skip_notifications: skip_notifications)
-
-      scope_stats
+    def scope_identity(scope)
+      scope.to_h.values_at(:value, :type, :is_in_scope)
     end
 
     def filter_and_normalize_scopes(platform_name, fetched_program)
@@ -165,30 +171,27 @@ def filter_and_normalize_scopes(platform_name, fetched_program)
       valid_scopes
     end
 
-    def process_added_scopes(program_id, platform_name, fetched_program, existing_values, filtered_scopes,
-                             skip_notifications: false)
-      fetched_values = filtered_scopes.map(&:value).uniq
-      added = fetched_values - existing_values
-
+    def process_added_scopes(program_id, platform_name, fetched_program, added_scopes, skip_notifications: false)
       # Count scopes by type for new program notification
       scope_stats = Hash.new(0)
 
-      added.each do |val|
-        scope_obj = filtered_scopes.find { |s| s.value == val }
+      added_scopes.each do |scope_obj|
         insert_scope(program_id, scope_obj)
 
         # Count by type
         scope_stats[scope_obj.type] += 1
 
         # Skip individual notifications for new programs
-        @notifier.notify_new_scope(platform_name, fetched_program.name, val, scope_obj.type) unless skip_notifications
+        unless skip_notifications
+          @notifier.notify_new_scope(platform_name, fetched_program.name, scope_obj.value, scope_obj.type)
+        end
 
         log_event(
           program_id: program_id,
           platform_name: platform_name,
           program_name: fetched_program.name,
           event_type: 'add_scope',
-          details: val,
+          details: scope_obj.value,
           scope_type: scope_obj.is_in_scope ? 'in' : 'out',
           category: scope_obj.type
         )
@@ -197,23 +200,19 @@ def process_added_scopes(program_id, platform_name, fetched_program, existing_va
       scope_stats
     end
 
-    def process_removed_scopes(program_id, platform_name, fetched_program, existing_values, fetched_values,
-                               existing_scopes, skip_notifications: false)
-      removed = existing_values - fetched_values
-
-      removed.each do |val|
-        existing_scope = existing_scopes.find { |s| s[:value] == val }
-        next unless existing_scope
-
-        delete_scope(program_id, val)
-        @notifier.notify_removed_scope(platform_name, fetched_program.name, val) unless skip_notifications
+    def process_removed_scopes(program_id, platform_name, fetched_program, removed_scopes, skip_notifications: false)
+      removed_scopes.each do |existing_scope|
+        delete_scope(existing_scope[:id])
+        unless skip_notifications
+          @notifier.notify_removed_scope(platform_name, fetched_program.name, existing_scope[:value])
+        end
 
         log_event(
           program_id: program_id,
           platform_name: platform_name,
           program_name: fetched_program.name,
           event_type: 'remove_scope',
-          details: val,
+          details: existing_scope[:value],
           scope_type: existing_scope[:is_in_scope] ? 'in' : 'out',
           category: existing_scope[:type]
         )
@@ -230,8 +229,8 @@ def insert_scope(program_id, scope_obj)
       )
     end
 
-    def delete_scope(program_id, value)
-      @db[:scopes].where(program_id: program_id, value: value).delete
+    def delete_scope(scope_id)
+      @db[:scopes].where(id: scope_id).delete
     end
 
     def handle_ignored_asset(platform_name, fetched_program, value, type)

spec/scopes_extractor/api_spec.rb

@@ -127,6 +127,24 @@ def authenticated_header
         expect(data['scopes'].size).to eq(2)
       end
 
+      it 'shows the new scope state after synchronization' do
+        notifier = instance_double(ScopesExtractor::Notifiers::Discord,
+                                   notify_removed_scope: nil, notify_new_scope: nil)
+        program = ScopesExtractor::Models::Program.new(
+          slug: 'test-program', platform: 'yeswehack', name: 'Test Program', bounty: true,
+          scopes: [
+            ScopesExtractor::Models::Scope.new(value: '*.example.com', type: 'web', is_in_scope: false),
+            ScopesExtractor::Models::Scope.new(value: 'com.example.app', type: 'mobile', is_in_scope: false)
+          ]
+        )
+        ScopesExtractor::DiffEngine.new(notifier: notifier).process_program('yeswehack', program)
+
+        get '/', { slug: 'test-program', type: 'web' }, authenticated_header
+
+        data = JSON.parse(last_response.body)
+        expect(data['scopes'].first['is_in_scope']).to be false
+      end
+
       it 'filters by platform' do
         get '/', { platform: 'yeswehack' }, authenticated_header
         expect(last_response).to be_ok

spec/scopes_extractor/config_spec.rb

@@ -36,7 +36,7 @@
   describe 'http settings' do
     describe '.user_agent' do
       it 'returns the configured user agent' do
-        expect(described_class.user_agent).to include('ScopesExtractor/2.1.1')
+        expect(described_class.user_agent).to include('ScopesExtractor/2.1.2')
         expect(described_class.user_agent).to include('Ruby')
         expect(described_class.user_agent).to include('github.com')
       end

spec/scopes_extractor/diff_engine_spec.rb

@@ -239,6 +239,79 @@
         expect(ignored.count).to eq(1)
       end
     end
+
+    context 'when a scope changes' do
+      let(:program_id) do
+        ScopesExtractor.db[:programs].insert(
+          slug: 'existing-program', platform: 'yeswehack', name: 'Existing Program',
+          bounty: true, last_updated: Time.now
+        )
+      end
+
+      before do
+        ScopesExtractor.db[:scopes].insert(
+          program_id: program_id, value: 'example.com', type: 'web',
+          is_in_scope: true, created_at: Time.now
+        )
+      end
+
+      def program_with(*scopes)
+        ScopesExtractor::Models::Program.new(
+          slug: 'existing-program', platform: 'yeswehack', name: 'Existing Program',
+          bounty: true, scopes: scopes
+        )
+      end
+
+      def scope(value: 'example.com', type: 'web', is_in_scope: true)
+        ScopesExtractor::Models::Scope.new(value: value, type: type, is_in_scope: is_in_scope)
+      end
+
+      it 'records an in-to-out transition and updates the stored scope' do
+        diff_engine.process_program('yeswehack', program_with(scope(is_in_scope: false)))
+
+        expect(ScopesExtractor.db[:scopes].where(program_id: program_id).all)
+          .to contain_exactly(include(value: 'example.com', type: 'web', is_in_scope: false))
+        changes = ScopesExtractor.db[:history].order(:id).all
+        expect(changes.map { |change| [change[:event_type], change[:scope_type]] })
+          .to eq([%w[remove_scope in], %w[add_scope out]])
+      end
+
+      it 'records an out-to-in transition' do
+        ScopesExtractor.db[:scopes].where(program_id: program_id).update(is_in_scope: false)
+
+        diff_engine.process_program('yeswehack', program_with(scope))
+
+        expect(ScopesExtractor.db[:scopes].where(program_id: program_id).first[:is_in_scope]).to be true
+        expect(ScopesExtractor.db[:history].order(:id).map { |change| [change[:event_type], change[:scope_type]] })
+          .to eq([%w[remove_scope out], %w[add_scope in]])
+      end
+
+      it 'records a type change for the same value' do
+        diff_engine.process_program('yeswehack', program_with(scope(type: 'mobile')))
+
+        expect(ScopesExtractor.db[:scopes].where(program_id: program_id).first[:type]).to eq('mobile')
+        expect(ScopesExtractor.db[:history].order(:id).map { |change| [change[:event_type], change[:category]] })
+          .to eq([%w[remove_scope web], %w[add_scope mobile]])
+      end
+
+      it 'does not create events or duplicate rows on an unchanged sync' do
+        existing_id = ScopesExtractor.db[:scopes].where(program_id: program_id).first[:id]
+
+        diff_engine.process_program('yeswehack', program_with(scope(value: 'EXAMPLE.COM'), scope))
+
+        expect(ScopesExtractor.db[:scopes].where(program_id: program_id).select_map(:id)).to eq([existing_id])
+        expect(ScopesExtractor.db[:history].count).to eq(0)
+      end
+
+      it 'removes only the matching entry when a value has both scope states' do
+        diff_engine.process_program('yeswehack', program_with(scope, scope(is_in_scope: false)))
+        expect(ScopesExtractor.db[:scopes].where(program_id: program_id).count).to eq(2)
+
+        diff_engine.process_program('yeswehack', program_with(scope(is_in_scope: false)))
+
+        expect(ScopesExtractor.db[:scopes].where(program_id: program_id).select_map(:is_in_scope)).to eq([false])
+      end
+    end
   end
 
   describe '#process_program when fetch failed' do