From 22f72ff5e8ad8b0515d4412126bd247fc860d9c0 Mon Sep 17 00:00:00 2001 From: Joshua MARTINELLE Date: Wed, 23 Sep 2026 21:31:15 +0200 Subject: [PATCH] Fix scope state synchronization and bump version to 2.1.2 --- CHANGELOG.md | 11 ++++ config/settings.yml | 2 +- lib/scopes_extractor/cli.rb | 2 +- lib/scopes_extractor/config.rb | 2 +- lib/scopes_extractor/diff_engine.rb | 63 ++++++++++--------- spec/scopes_extractor/api_spec.rb | 18 ++++++ spec/scopes_extractor/config_spec.rb | 2 +- spec/scopes_extractor/diff_engine_spec.rb | 73 +++++++++++++++++++++++ 8 files changed, 137 insertions(+), 36 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c08b5b..83c15da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +## Version 2.1.2 - Scope State Synchronization + +### 🐛 Bug Fixes + +- Compare scopes by value, type, and in-scope status so changes to either property update the stored scope and the `/` API response. +- Record scope state and type transitions as `remove_scope` followed by `add_scope` events in history. +- Remove scopes by their database ID so entries sharing a value do not delete one another. +- Update the CLI `version` command and HTTP User-Agent to report the current release. + +--- + ## Version 2.1.1 - YesWeHack Inactive Programs ### 🐛 Bug Fixes diff --git a/config/settings.yml b/config/settings.yml index bbd168d..5e38d7b 100644 --- a/config/settings.yml +++ b/config/settings.yml @@ -7,7 +7,7 @@ app: # HTTP client configuration http: - user_agent: "ScopesExtractor/2.1.1 (Ruby; +github.com/JoshuaMart/ScopesExtractor)" + user_agent: "ScopesExtractor/2.1.2 (Ruby; +github.com/JoshuaMart/ScopesExtractor)" proxy: null # Optional: "http://proxy.example.com:8080" timeout: 30 # seconds diff --git a/lib/scopes_extractor/cli.rb b/lib/scopes_extractor/cli.rb index 855fe0f..75f74fd 100644 --- a/lib/scopes_extractor/cli.rb +++ b/lib/scopes_extractor/cli.rb @@ -10,7 +10,7 @@ def self.exit_on_failure? desc 'version', 'Display version information' def version - puts 'ScopesExtractor version 2.1.1' + puts 'ScopesExtractor version 2.1.2' end desc 'reset', 'Reset the database (WARNING: deletes all data)' diff --git a/lib/scopes_extractor/config.rb b/lib/scopes_extractor/config.rb index 34caa35..a6deae8 100644 --- a/lib/scopes_extractor/config.rb +++ b/lib/scopes_extractor/config.rb @@ -29,7 +29,7 @@ def http end def user_agent - http[:user_agent] || 'ScopesExtractor/2.1.1' + http[:user_agent] || 'ScopesExtractor/2.1.2' end def proxy diff --git a/lib/scopes_extractor/diff_engine.rb b/lib/scopes_extractor/diff_engine.rb index 0a2876f..d0496fb 100644 --- a/lib/scopes_extractor/diff_engine.rb +++ b/lib/scopes_extractor/diff_engine.rb @@ -118,25 +118,31 @@ def update_program_if_changed(program_id, existing_program, fetched_program) def sync_scopes(program_id, platform_name, fetched_program, skip_notifications: false) existing_scopes = @db[:scopes].where(program_id: program_id).all - existing_values = existing_scopes.map { |s| s[:value] } - # Normalize and validate scopes + # Normalize and validate scopes before comparing all three identity fields. + # A status or type change becomes a removal followed by an addition. filtered_scopes = filter_and_normalize_scopes(platform_name, fetched_program) - fetched_values = filtered_scopes.map(&:value).uniq + .uniq { |scope| scope_identity(scope) } + existing_identities = existing_scopes.to_set { |scope| scope_identity(scope) } + fetched_identities = filtered_scopes.to_set { |scope| scope_identity(scope) } - scope_stats = process_added_scopes( + removed_scopes = existing_scopes.reject { |scope| fetched_identities.include?(scope_identity(scope)) } + added_scopes = filtered_scopes.reject { |scope| existing_identities.include?(scope_identity(scope)) } + + process_removed_scopes(program_id, platform_name, fetched_program, removed_scopes, + skip_notifications: skip_notifications) + + process_added_scopes( program_id, platform_name, fetched_program, - existing_values, - filtered_scopes, + added_scopes, skip_notifications: skip_notifications ) + end - process_removed_scopes(program_id, platform_name, fetched_program, existing_values, fetched_values, - existing_scopes, skip_notifications: skip_notifications) - - scope_stats + def scope_identity(scope) + scope.to_h.values_at(:value, :type, :is_in_scope) end def filter_and_normalize_scopes(platform_name, fetched_program) @@ -165,30 +171,27 @@ def filter_and_normalize_scopes(platform_name, fetched_program) valid_scopes end - def process_added_scopes(program_id, platform_name, fetched_program, existing_values, filtered_scopes, - skip_notifications: false) - fetched_values = filtered_scopes.map(&:value).uniq - added = fetched_values - existing_values - + def process_added_scopes(program_id, platform_name, fetched_program, added_scopes, skip_notifications: false) # Count scopes by type for new program notification scope_stats = Hash.new(0) - added.each do |val| - scope_obj = filtered_scopes.find { |s| s.value == val } + added_scopes.each do |scope_obj| insert_scope(program_id, scope_obj) # Count by type scope_stats[scope_obj.type] += 1 # Skip individual notifications for new programs - @notifier.notify_new_scope(platform_name, fetched_program.name, val, scope_obj.type) unless skip_notifications + unless skip_notifications + @notifier.notify_new_scope(platform_name, fetched_program.name, scope_obj.value, scope_obj.type) + end log_event( program_id: program_id, platform_name: platform_name, program_name: fetched_program.name, event_type: 'add_scope', - details: val, + details: scope_obj.value, scope_type: scope_obj.is_in_scope ? 'in' : 'out', category: scope_obj.type ) @@ -197,23 +200,19 @@ def process_added_scopes(program_id, platform_name, fetched_program, existing_va scope_stats end - def process_removed_scopes(program_id, platform_name, fetched_program, existing_values, fetched_values, - existing_scopes, skip_notifications: false) - removed = existing_values - fetched_values - - removed.each do |val| - existing_scope = existing_scopes.find { |s| s[:value] == val } - next unless existing_scope - - delete_scope(program_id, val) - @notifier.notify_removed_scope(platform_name, fetched_program.name, val) unless skip_notifications + def process_removed_scopes(program_id, platform_name, fetched_program, removed_scopes, skip_notifications: false) + removed_scopes.each do |existing_scope| + delete_scope(existing_scope[:id]) + unless skip_notifications + @notifier.notify_removed_scope(platform_name, fetched_program.name, existing_scope[:value]) + end log_event( program_id: program_id, platform_name: platform_name, program_name: fetched_program.name, event_type: 'remove_scope', - details: val, + details: existing_scope[:value], scope_type: existing_scope[:is_in_scope] ? 'in' : 'out', category: existing_scope[:type] ) @@ -230,8 +229,8 @@ def insert_scope(program_id, scope_obj) ) end - def delete_scope(program_id, value) - @db[:scopes].where(program_id: program_id, value: value).delete + def delete_scope(scope_id) + @db[:scopes].where(id: scope_id).delete end def handle_ignored_asset(platform_name, fetched_program, value, type) diff --git a/spec/scopes_extractor/api_spec.rb b/spec/scopes_extractor/api_spec.rb index 1de5d39..51cc74c 100644 --- a/spec/scopes_extractor/api_spec.rb +++ b/spec/scopes_extractor/api_spec.rb @@ -127,6 +127,24 @@ def authenticated_header expect(data['scopes'].size).to eq(2) end + it 'shows the new scope state after synchronization' do + notifier = instance_double(ScopesExtractor::Notifiers::Discord, + notify_removed_scope: nil, notify_new_scope: nil) + program = ScopesExtractor::Models::Program.new( + slug: 'test-program', platform: 'yeswehack', name: 'Test Program', bounty: true, + scopes: [ + ScopesExtractor::Models::Scope.new(value: '*.example.com', type: 'web', is_in_scope: false), + ScopesExtractor::Models::Scope.new(value: 'com.example.app', type: 'mobile', is_in_scope: false) + ] + ) + ScopesExtractor::DiffEngine.new(notifier: notifier).process_program('yeswehack', program) + + get '/', { slug: 'test-program', type: 'web' }, authenticated_header + + data = JSON.parse(last_response.body) + expect(data['scopes'].first['is_in_scope']).to be false + end + it 'filters by platform' do get '/', { platform: 'yeswehack' }, authenticated_header expect(last_response).to be_ok diff --git a/spec/scopes_extractor/config_spec.rb b/spec/scopes_extractor/config_spec.rb index 336bdee..2703721 100644 --- a/spec/scopes_extractor/config_spec.rb +++ b/spec/scopes_extractor/config_spec.rb @@ -36,7 +36,7 @@ describe 'http settings' do describe '.user_agent' do it 'returns the configured user agent' do - expect(described_class.user_agent).to include('ScopesExtractor/2.1.1') + expect(described_class.user_agent).to include('ScopesExtractor/2.1.2') expect(described_class.user_agent).to include('Ruby') expect(described_class.user_agent).to include('github.com') end diff --git a/spec/scopes_extractor/diff_engine_spec.rb b/spec/scopes_extractor/diff_engine_spec.rb index d60f0df..584262c 100644 --- a/spec/scopes_extractor/diff_engine_spec.rb +++ b/spec/scopes_extractor/diff_engine_spec.rb @@ -239,6 +239,79 @@ expect(ignored.count).to eq(1) end end + + context 'when a scope changes' do + let(:program_id) do + ScopesExtractor.db[:programs].insert( + slug: 'existing-program', platform: 'yeswehack', name: 'Existing Program', + bounty: true, last_updated: Time.now + ) + end + + before do + ScopesExtractor.db[:scopes].insert( + program_id: program_id, value: 'example.com', type: 'web', + is_in_scope: true, created_at: Time.now + ) + end + + def program_with(*scopes) + ScopesExtractor::Models::Program.new( + slug: 'existing-program', platform: 'yeswehack', name: 'Existing Program', + bounty: true, scopes: scopes + ) + end + + def scope(value: 'example.com', type: 'web', is_in_scope: true) + ScopesExtractor::Models::Scope.new(value: value, type: type, is_in_scope: is_in_scope) + end + + it 'records an in-to-out transition and updates the stored scope' do + diff_engine.process_program('yeswehack', program_with(scope(is_in_scope: false))) + + expect(ScopesExtractor.db[:scopes].where(program_id: program_id).all) + .to contain_exactly(include(value: 'example.com', type: 'web', is_in_scope: false)) + changes = ScopesExtractor.db[:history].order(:id).all + expect(changes.map { |change| [change[:event_type], change[:scope_type]] }) + .to eq([%w[remove_scope in], %w[add_scope out]]) + end + + it 'records an out-to-in transition' do + ScopesExtractor.db[:scopes].where(program_id: program_id).update(is_in_scope: false) + + diff_engine.process_program('yeswehack', program_with(scope)) + + expect(ScopesExtractor.db[:scopes].where(program_id: program_id).first[:is_in_scope]).to be true + expect(ScopesExtractor.db[:history].order(:id).map { |change| [change[:event_type], change[:scope_type]] }) + .to eq([%w[remove_scope out], %w[add_scope in]]) + end + + it 'records a type change for the same value' do + diff_engine.process_program('yeswehack', program_with(scope(type: 'mobile'))) + + expect(ScopesExtractor.db[:scopes].where(program_id: program_id).first[:type]).to eq('mobile') + expect(ScopesExtractor.db[:history].order(:id).map { |change| [change[:event_type], change[:category]] }) + .to eq([%w[remove_scope web], %w[add_scope mobile]]) + end + + it 'does not create events or duplicate rows on an unchanged sync' do + existing_id = ScopesExtractor.db[:scopes].where(program_id: program_id).first[:id] + + diff_engine.process_program('yeswehack', program_with(scope(value: 'EXAMPLE.COM'), scope)) + + expect(ScopesExtractor.db[:scopes].where(program_id: program_id).select_map(:id)).to eq([existing_id]) + expect(ScopesExtractor.db[:history].count).to eq(0) + end + + it 'removes only the matching entry when a value has both scope states' do + diff_engine.process_program('yeswehack', program_with(scope, scope(is_in_scope: false))) + expect(ScopesExtractor.db[:scopes].where(program_id: program_id).count).to eq(2) + + diff_engine.process_program('yeswehack', program_with(scope(is_in_scope: false))) + + expect(ScopesExtractor.db[:scopes].where(program_id: program_id).select_map(:is_in_scope)).to eq([false]) + end + end end describe '#process_program when fetch failed' do