From f6271f62368d168f58b8b85eb739b23795c12ede Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E7=A8=8B=E6=A1=94?= Date: Mon, 21 Sep 2026 11:06:50 +0800 Subject: [PATCH 1/3] Fix cross-project bot throttling and event cleanup --- .../RemoveBotEventsWorkItemHandler.cs | 14 ++- .../Default/0_ThrottleBotsPlugin.cs | 30 +++--- .../RemoveBotEventsWorkItemHandlerTests.cs | 73 ++++++++++++++ .../Plugins/ThrottleBotsPluginTests.cs | 95 +++++++++++++++++++ 4 files changed, 199 insertions(+), 13 deletions(-) create mode 100644 tests/Exceptionless.Tests/Jobs/WorkItemHandlers/RemoveBotEventsWorkItemHandlerTests.cs create mode 100644 tests/Exceptionless.Tests/Plugins/ThrottleBotsPluginTests.cs diff --git a/src/Exceptionless.Core/Jobs/WorkItemHandlers/RemoveBotEventsWorkItemHandler.cs b/src/Exceptionless.Core/Jobs/WorkItemHandlers/RemoveBotEventsWorkItemHandler.cs index 0e852ee03d..f6a161ae97 100644 --- a/src/Exceptionless.Core/Jobs/WorkItemHandlers/RemoveBotEventsWorkItemHandler.cs +++ b/src/Exceptionless.Core/Jobs/WorkItemHandlers/RemoveBotEventsWorkItemHandler.cs @@ -1,7 +1,10 @@ +using Exceptionless.Core.Models; using Exceptionless.Core.Models.WorkItems; using Exceptionless.Core.Repositories; +using Exceptionless.Core.Repositories.Configuration; using Foundatio.Jobs; using Foundatio.Lock; +using Foundatio.Repositories; using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Jobs.WorkItemHandlers; @@ -27,12 +30,21 @@ public RemoveBotEventsWorkItemHandler(IEventRepository eventRepository, ILockPro public override async Task HandleItemAsync(WorkItemContext context) { var wi = context.GetData()!; + ArgumentException.ThrowIfNullOrWhiteSpace(wi.OrganizationId); + ArgumentException.ThrowIfNullOrWhiteSpace(wi.ProjectId); + ArgumentException.ThrowIfNullOrWhiteSpace(wi.ClientIpAddress); using var _ = Log.BeginScope(new ExceptionlessState().Organization(wi.OrganizationId).Project(wi.ProjectId).Tag("Delete").Tag("Bot")); Log.LogInformation("Received remove bot events work item OrganizationId={OrganizationId} ProjectId={ProjectId}, ClientIpAddress={ClientIpAddress}, UtcStartDate={UtcStartDate}, UtcEndDate={UtcEndDate}", wi.OrganizationId, wi.ProjectId, wi.ClientIpAddress, wi.UtcStartDate, wi.UtcEndDate); await context.ReportProgressAsync(0, $"Starting deleting of bot events... OrganizationId={wi.OrganizationId}"); - long deleted = await _eventRepository.RemoveAllAsync(wi.OrganizationId, wi.ClientIpAddress, wi.UtcStartDate, wi.UtcEndDate); + var query = new RepositoryQuery() + .Organization(wi.OrganizationId) + .Project(wi.ProjectId) + .DateRange(wi.UtcStartDate, wi.UtcEndDate, (PersistentEvent e) => e.Date) + .Index(wi.UtcStartDate, wi.UtcEndDate) + .FieldEquals(EventIndex.Alias.IpAddress, wi.ClientIpAddress); + long deleted = await _eventRepository.RemoveAllAsync(q => query); await context.ReportProgressAsync(100, $"Bot events deleted: {deleted} OrganizationId={wi.OrganizationId}"); Log.LogInformation("Removed {Deleted} bot events OrganizationId={OrganizationId} ProjectId={ProjectId}, ClientIpAddress={ClientIpAddress}, UtcStartDate={UtcStartDate}, UtcEndDate={UtcEndDate}", deleted, wi.OrganizationId, wi.ProjectId, wi.ClientIpAddress, wi.UtcStartDate, wi.UtcEndDate); } diff --git a/src/Exceptionless.Core/Plugins/EventProcessor/Default/0_ThrottleBotsPlugin.cs b/src/Exceptionless.Core/Plugins/EventProcessor/Default/0_ThrottleBotsPlugin.cs index 3b18c78bcd..9c988ae0ab 100644 --- a/src/Exceptionless.Core/Plugins/EventProcessor/Default/0_ThrottleBotsPlugin.cs +++ b/src/Exceptionless.Core/Plugins/EventProcessor/Default/0_ThrottleBotsPlugin.cs @@ -33,19 +33,25 @@ public override async Task EventBatchProcessingAsync(ICollection c if (_options.AppMode == AppMode.Development) return; - var firstContext = contexts.First(); - if (!firstContext.Project.DeleteBotDataEnabled || !firstContext.IncludePrivateInformation) - return; - - // Throttle errors by client ip address to no more than X every 5 minutes. - var clientIpAddressGroups = contexts.GroupBy(c => c.Event.GetRequestInfo(_serializer, _logger)?.ClientIpAddress); + // Keep each project's client IP counters and cleanup tasks isolated. + var clientIpAddressGroups = contexts + .Where(c => c.Project.DeleteBotDataEnabled && c.IncludePrivateInformation) + .GroupBy(c => new + { + c.Event.OrganizationId, + c.Event.ProjectId, + ClientIpAddress = c.Event.GetRequestInfo(_serializer, _logger)?.ClientIpAddress + }); foreach (var clientIpAddressGroup in clientIpAddressGroups) { - if (String.IsNullOrEmpty(clientIpAddressGroup.Key) || clientIpAddressGroup.Key.IsPrivateNetwork()) + var scope = clientIpAddressGroup.Key; + if (String.IsNullOrEmpty(scope.ClientIpAddress) || scope.ClientIpAddress.IsPrivateNetwork()) + { continue; + } var clientIpContexts = clientIpAddressGroup.ToList(); - string throttleCacheKey = String.Concat("bot:", clientIpAddressGroup.Key, ":", _timeProvider.GetUtcNow().UtcDateTime.Floor(_throttlingPeriod).Ticks); + string throttleCacheKey = $"bot:{scope.OrganizationId}:{scope.ProjectId}:{scope.ClientIpAddress}:{_timeProvider.GetUtcNow().UtcDateTime.Floor(_throttlingPeriod).Ticks}"; int? requestCount = await _cache.GetAsync(throttleCacheKey, null); if (requestCount.HasValue) { @@ -62,14 +68,14 @@ public override async Task EventBatchProcessingAsync(ICollection c continue; var utcNow = _timeProvider.GetUtcNow().UtcDateTime; - _logger.LogInformation("Bot throttle triggered. IP: {IP} Time: {ThrottlingPeriod} Project: {ProjectId}", clientIpAddressGroup.Key, utcNow.Floor(_throttlingPeriod), firstContext.Event.ProjectId); + _logger.LogInformation("Bot throttle triggered. IP: {IP} Time: {ThrottlingPeriod} Organization: {OrganizationId} Project: {ProjectId}", scope.ClientIpAddress, utcNow.Floor(_throttlingPeriod), scope.OrganizationId, scope.ProjectId); // The throttle was triggered, go and delete all the errors that triggered the throttle to reduce bot noise in the system await _workItemQueue.EnqueueAsync(new RemoveBotEventsWorkItem { - OrganizationId = firstContext.Event.OrganizationId, - ProjectId = firstContext.Event.ProjectId, - ClientIpAddress = clientIpAddressGroup.Key, + OrganizationId = scope.OrganizationId, + ProjectId = scope.ProjectId, + ClientIpAddress = scope.ClientIpAddress, UtcStartDate = utcNow.Floor(_throttlingPeriod), UtcEndDate = utcNow.Ceiling(_throttlingPeriod) }); diff --git a/tests/Exceptionless.Tests/Jobs/WorkItemHandlers/RemoveBotEventsWorkItemHandlerTests.cs b/tests/Exceptionless.Tests/Jobs/WorkItemHandlers/RemoveBotEventsWorkItemHandlerTests.cs new file mode 100644 index 0000000000..a049ac6605 --- /dev/null +++ b/tests/Exceptionless.Tests/Jobs/WorkItemHandlers/RemoveBotEventsWorkItemHandlerTests.cs @@ -0,0 +1,73 @@ +using System.Reflection; +using Exceptionless.Core.Jobs.WorkItemHandlers; +using Exceptionless.Core.Models; +using Exceptionless.Core.Models.WorkItems; +using Exceptionless.Core.Repositories; +using Exceptionless.Core.Repositories.Options; +using Foundatio.Jobs; +using Foundatio.Repositories; +using Foundatio.Utility; +using Microsoft.Extensions.Logging.Abstractions; +using Xunit; + +namespace Exceptionless.Tests.Jobs.WorkItemHandlers; + +public sealed class RemoveBotEventsWorkItemHandlerTests +{ + [Fact] + public async Task HandleItemAsync_CleanupTask_RestrictsDeletionToOwningProject() + { + var repository = DispatchProxy.Create(); + var handler = new RemoveBotEventsWorkItemHandler(repository, null!, NullLoggerFactory.Instance); + await handler.HandleItemAsync(CreateContext(CreateWorkItem())); + + var query = Assert.IsAssignableFrom(((RecordingRepository)repository).Query); + Assert.Equal(["organization-a"], query.GetOrganizations()); + Assert.Equal(["project-a"], query.GetProjects()); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + public async Task HandleItemAsync_LegacyTaskWithoutProject_RejectsUnscopedDeletion(string? projectId) + { + var repository = DispatchProxy.Create(); + var handler = new RemoveBotEventsWorkItemHandler(repository, null!, NullLoggerFactory.Instance); + var item = CreateWorkItem() with { ProjectId = projectId! }; + await Assert.ThrowsAnyAsync(() => handler.HandleItemAsync(CreateContext(item))); + Assert.Equal(0, ((RecordingRepository)repository).DeleteCalls); + } + + private static RemoveBotEventsWorkItem CreateWorkItem() => new() + { + OrganizationId = "organization-a", + ProjectId = "project-a", + ClientIpAddress = "203.0.113.10", + UtcStartDate = new DateTime(2026, 9, 21, 2, 25, 0, DateTimeKind.Utc), + UtcEndDate = new DateTime(2026, 9, 21, 2, 30, 0, DateTimeKind.Utc) + }; + + private static WorkItemContext CreateContext(RemoveBotEventsWorkItem item) => + new(item, "test-job", EmptyLock.Empty, TestContext.Current.CancellationToken, static (_, _) => Task.CompletedTask); + + private class RecordingRepository : DispatchProxy + { + public IRepositoryQuery? Query { get; private set; } + public int DeleteCalls { get; private set; } + + protected override object? Invoke(MethodInfo? targetMethod, object?[]? args) + { + if (targetMethod?.Name == nameof(IEventRepository.RemoveAllAsync)) + { + DeleteCalls++; + Query = args![0] is RepositoryQueryDescriptor descriptor + ? descriptor(new RepositoryQuery()) + : args[0] as IRepositoryQuery; + return Task.FromResult(1L); + } + + throw new NotSupportedException(targetMethod?.Name); + } + } +} diff --git a/tests/Exceptionless.Tests/Plugins/ThrottleBotsPluginTests.cs b/tests/Exceptionless.Tests/Plugins/ThrottleBotsPluginTests.cs new file mode 100644 index 0000000000..586fe8ae4f --- /dev/null +++ b/tests/Exceptionless.Tests/Plugins/ThrottleBotsPluginTests.cs @@ -0,0 +1,95 @@ +using System.Text.Json; +using Exceptionless.Core; +using Exceptionless.Core.Models; +using Exceptionless.Core.Models.Data; +using Exceptionless.Core.Plugins.EventProcessor; +using Exceptionless.Core.Serialization; +using Foundatio.Caching; +using Foundatio.Jobs; +using Foundatio.Queues; +using Foundatio.Serializer; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using Xunit; + +namespace Exceptionless.Tests.Plugins; + +public sealed class ThrottleBotsPluginTests : IDisposable +{ + private readonly FakeTimeProvider _time = new(new DateTimeOffset(2026, 9, 21, 2, 25, 0, TimeSpan.Zero)); + private readonly InMemoryCacheClient _cache; + private readonly InMemoryQueue _queue = new(); + private readonly ThrottleBotsPlugin _plugin; + + public ThrottleBotsPluginTests() + { + _cache = new InMemoryCacheClient(new InMemoryCacheClientOptions { TimeProvider = _time }); + var configuration = new ConfigurationBuilder().AddInMemoryCollection(new Dictionary + { + [nameof(AppOptions.BaseURL)] = "http://localhost", + [nameof(AppOptions.AppMode)] = "Production", + [nameof(AppOptions.BotThrottleLimit)] = "2" + }).Build(); + var serializer = new SystemTextJsonSerializer(new JsonSerializerOptions().ConfigureExceptionlessDefaults()); + _plugin = new ThrottleBotsPlugin(_cache, _queue, serializer, _time, + AppOptions.ReadFromConfiguration(configuration), NullLoggerFactory.Instance); + } + + [Theory] + [InlineData("organization-a", "project-b")] + [InlineData("organization-b", "project-a")] + public async Task EventBatchProcessingAsync_SharedIpAcrossScopes_KeepsCountersSeparate(string organizationId, string projectId) + { + await _plugin.EventBatchProcessingAsync([CreateContext()]); + var other = CreateContext(organizationId, projectId); + await _plugin.EventBatchProcessingAsync([other]); + Assert.False(other.IsDiscarded); + Assert.Equal(0, (await _queue.GetQueueStatsAsync()).Queued); + + var repeated = CreateContext(); + await _plugin.EventBatchProcessingAsync([repeated]); + Assert.True(repeated.IsDiscarded); + Assert.True(repeated.IsCancelled); + Assert.Equal(1, (await _queue.GetQueueStatsAsync()).Queued); + } + + [Fact] + public async Task EventBatchProcessingAsync_MixedProjects_OnlyDiscardsEnabledProjectAtLimit() + { + var disabled = CreateContext(projectId: "disabled", enabled: false); + var first = CreateContext(); + var second = CreateContext(); + var other = CreateContext(projectId: "other"); + await _plugin.EventBatchProcessingAsync([disabled, first, second, other]); + Assert.True(first.IsDiscarded); + Assert.True(second.IsDiscarded); + Assert.False(disabled.IsDiscarded); + Assert.False(other.IsDiscarded); + Assert.Equal(1, (await _queue.GetQueueStatsAsync()).Queued); + } + + [Fact] + public async Task EventBatchProcessingAsync_NextWindow_StartsNewCounter() + { + await _plugin.EventBatchProcessingAsync([CreateContext(), CreateContext()]); + _time.Advance(TimeSpan.FromMinutes(5)); + var next = CreateContext(); + await _plugin.EventBatchProcessingAsync([next]); + Assert.False(next.IsDiscarded); + } + + private static EventContext CreateContext(string organizationId = "organization-a", string projectId = "project-a", bool enabled = true) + { + var ev = new PersistentEvent { Type = Event.KnownTypes.Error }; + ev.AddRequestInfo(new RequestInfo { ClientIpAddress = "203.0.113.10" }); + return new EventContext(ev, new Organization { Id = organizationId }, + new Project { Id = projectId, OrganizationId = organizationId, DeleteBotDataEnabled = enabled }); + } + + public void Dispose() + { + _queue.Dispose(); + _cache.Dispose(); + } +} From 12a6e5c96ef9c84c92a6011557d0e7fe3620e52f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E7=A8=8B=E6=A1=94?= Date: Tue, 22 Sep 2026 10:25:33 +0800 Subject: [PATCH 2/3] Address project-scoped cleanup review feedback --- .../RemoveBotEventsWorkItemHandler.cs | 11 +---- .../Default/0_ThrottleBotsPlugin.cs | 5 ++- .../Repositories/EventRepository.cs | 22 ++++++++++ .../Interfaces/IEventRepository.cs | 1 + .../RemoveBotEventsWorkItemHandlerTests.cs | 21 +++++----- .../Plugins/ThrottleBotsPluginTests.cs | 16 ++++---- .../Repositories/EventRepositoryTests.cs | 41 +++++++++++++++++++ 7 files changed, 88 insertions(+), 29 deletions(-) diff --git a/src/Exceptionless.Core/Jobs/WorkItemHandlers/RemoveBotEventsWorkItemHandler.cs b/src/Exceptionless.Core/Jobs/WorkItemHandlers/RemoveBotEventsWorkItemHandler.cs index f6a161ae97..6ea6855625 100644 --- a/src/Exceptionless.Core/Jobs/WorkItemHandlers/RemoveBotEventsWorkItemHandler.cs +++ b/src/Exceptionless.Core/Jobs/WorkItemHandlers/RemoveBotEventsWorkItemHandler.cs @@ -1,10 +1,7 @@ -using Exceptionless.Core.Models; using Exceptionless.Core.Models.WorkItems; using Exceptionless.Core.Repositories; -using Exceptionless.Core.Repositories.Configuration; using Foundatio.Jobs; using Foundatio.Lock; -using Foundatio.Repositories; using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Jobs.WorkItemHandlers; @@ -38,13 +35,7 @@ public override async Task HandleItemAsync(WorkItemContext context) Log.LogInformation("Received remove bot events work item OrganizationId={OrganizationId} ProjectId={ProjectId}, ClientIpAddress={ClientIpAddress}, UtcStartDate={UtcStartDate}, UtcEndDate={UtcEndDate}", wi.OrganizationId, wi.ProjectId, wi.ClientIpAddress, wi.UtcStartDate, wi.UtcEndDate); await context.ReportProgressAsync(0, $"Starting deleting of bot events... OrganizationId={wi.OrganizationId}"); - var query = new RepositoryQuery() - .Organization(wi.OrganizationId) - .Project(wi.ProjectId) - .DateRange(wi.UtcStartDate, wi.UtcEndDate, (PersistentEvent e) => e.Date) - .Index(wi.UtcStartDate, wi.UtcEndDate) - .FieldEquals(EventIndex.Alias.IpAddress, wi.ClientIpAddress); - long deleted = await _eventRepository.RemoveAllAsync(q => query); + long deleted = await _eventRepository.RemoveAllAsync(wi.OrganizationId, wi.ProjectId, wi.ClientIpAddress, wi.UtcStartDate, wi.UtcEndDate); await context.ReportProgressAsync(100, $"Bot events deleted: {deleted} OrganizationId={wi.OrganizationId}"); Log.LogInformation("Removed {Deleted} bot events OrganizationId={OrganizationId} ProjectId={ProjectId}, ClientIpAddress={ClientIpAddress}, UtcStartDate={UtcStartDate}, UtcEndDate={UtcEndDate}", deleted, wi.OrganizationId, wi.ProjectId, wi.ClientIpAddress, wi.UtcStartDate, wi.UtcEndDate); } diff --git a/src/Exceptionless.Core/Plugins/EventProcessor/Default/0_ThrottleBotsPlugin.cs b/src/Exceptionless.Core/Plugins/EventProcessor/Default/0_ThrottleBotsPlugin.cs index 9c988ae0ab..fb6c46d3b3 100644 --- a/src/Exceptionless.Core/Plugins/EventProcessor/Default/0_ThrottleBotsPlugin.cs +++ b/src/Exceptionless.Core/Plugins/EventProcessor/Default/0_ThrottleBotsPlugin.cs @@ -28,6 +28,9 @@ public ThrottleBotsPlugin(ICacheClient cacheClient, IQueue workIte _timeProvider = timeProvider; } + private static string CacheKey(string organizationId, string projectId, string clientIpAddress, long period) => + String.Concat("Organization:", organizationId, ":Project:", projectId, ":bot:", period, ":", clientIpAddress); + public override async Task EventBatchProcessingAsync(ICollection contexts) { if (_options.AppMode == AppMode.Development) @@ -51,7 +54,7 @@ public override async Task EventBatchProcessingAsync(ICollection c } var clientIpContexts = clientIpAddressGroup.ToList(); - string throttleCacheKey = $"bot:{scope.OrganizationId}:{scope.ProjectId}:{scope.ClientIpAddress}:{_timeProvider.GetUtcNow().UtcDateTime.Floor(_throttlingPeriod).Ticks}"; + string throttleCacheKey = CacheKey(scope.OrganizationId, scope.ProjectId, scope.ClientIpAddress, _timeProvider.GetUtcNow().UtcDateTime.Floor(_throttlingPeriod).Ticks); int? requestCount = await _cache.GetAsync(throttleCacheKey, null); if (requestCount.HasValue) { diff --git a/src/Exceptionless.Core/Repositories/EventRepository.cs b/src/Exceptionless.Core/Repositories/EventRepository.cs index 3fba37a69d..c28d1907a9 100644 --- a/src/Exceptionless.Core/Repositories/EventRepository.cs +++ b/src/Exceptionless.Core/Repositories/EventRepository.cs @@ -77,6 +77,28 @@ public Task RemoveAllAsync(string organizationId, string? clientIpAddress, return RemoveAllAsync(q => query, options); } + public Task RemoveAllAsync(string organizationId, string projectId, string clientIpAddress, DateTime? utcStart, DateTime? utcEnd, CommandOptionsDescriptor? options = null) + { + ArgumentException.ThrowIfNullOrWhiteSpace(organizationId); + ArgumentException.ThrowIfNullOrWhiteSpace(projectId); + ArgumentException.ThrowIfNullOrWhiteSpace(clientIpAddress); + + var query = new RepositoryQuery() + .Organization(organizationId) + .Project(projectId) + .FieldEquals(EventIndex.Alias.IpAddress, clientIpAddress); + if (utcStart.HasValue || utcEnd.HasValue) + { + query = query.DateRange(utcStart, utcEnd, InferField(e => e.Date)); + } + if (utcStart.HasValue && utcEnd.HasValue) + { + query = query.Index(utcStart, utcEnd); + } + + return RemoveAllAsync(q => query, options); + } + public Task> GetByReferenceIdAsync(string projectId, string referenceId) { return FindAsync(q => q.Project(projectId).FieldEquals(e => e.ReferenceId, referenceId).SortDescending(e => e.Date), o => o.PageLimit(10)); diff --git a/src/Exceptionless.Core/Repositories/Interfaces/IEventRepository.cs b/src/Exceptionless.Core/Repositories/Interfaces/IEventRepository.cs index c7c2272cbc..49ae487bf8 100644 --- a/src/Exceptionless.Core/Repositories/Interfaces/IEventRepository.cs +++ b/src/Exceptionless.Core/Repositories/Interfaces/IEventRepository.cs @@ -12,6 +12,7 @@ public interface IEventRepository : IRepositoryOwnedByOrganizationAndProject> GetOpenSessionsAsync(DateTime createdBeforeUtc, CommandOptionsDescriptor? options = null); Task UpdateSessionStartLastActivityAsync(string id, DateTime lastActivityUtc, bool isSessionEnd = false, bool hasError = false, bool sendNotifications = true); Task RemoveAllAsync(string organizationId, string? clientIpAddress, DateTime? utcStart, DateTime? utcEnd, CommandOptionsDescriptor? options = null); + Task RemoveAllAsync(string organizationId, string projectId, string clientIpAddress, DateTime? utcStart, DateTime? utcEnd, CommandOptionsDescriptor? options = null); Task RemoveAllByStackIdsAsync(string[] stackIds); } diff --git a/tests/Exceptionless.Tests/Jobs/WorkItemHandlers/RemoveBotEventsWorkItemHandlerTests.cs b/tests/Exceptionless.Tests/Jobs/WorkItemHandlers/RemoveBotEventsWorkItemHandlerTests.cs index a049ac6605..e3532cfcdf 100644 --- a/tests/Exceptionless.Tests/Jobs/WorkItemHandlers/RemoveBotEventsWorkItemHandlerTests.cs +++ b/tests/Exceptionless.Tests/Jobs/WorkItemHandlers/RemoveBotEventsWorkItemHandlerTests.cs @@ -1,11 +1,8 @@ using System.Reflection; using Exceptionless.Core.Jobs.WorkItemHandlers; -using Exceptionless.Core.Models; using Exceptionless.Core.Models.WorkItems; using Exceptionless.Core.Repositories; -using Exceptionless.Core.Repositories.Options; using Foundatio.Jobs; -using Foundatio.Repositories; using Foundatio.Utility; using Microsoft.Extensions.Logging.Abstractions; using Xunit; @@ -19,11 +16,15 @@ public async Task HandleItemAsync_CleanupTask_RestrictsDeletionToOwningProject() { var repository = DispatchProxy.Create(); var handler = new RemoveBotEventsWorkItemHandler(repository, null!, NullLoggerFactory.Instance); - await handler.HandleItemAsync(CreateContext(CreateWorkItem())); + var item = CreateWorkItem(); + await handler.HandleItemAsync(CreateContext(item)); - var query = Assert.IsAssignableFrom(((RecordingRepository)repository).Query); - Assert.Equal(["organization-a"], query.GetOrganizations()); - Assert.Equal(["project-a"], query.GetProjects()); + var arguments = Assert.IsType(((RecordingRepository)repository).DeleteArguments); + Assert.Equal(item.OrganizationId, arguments[0]); + Assert.Equal(item.ProjectId, arguments[1]); + Assert.Equal(item.ClientIpAddress, arguments[2]); + Assert.Equal(item.UtcStartDate, arguments[3]); + Assert.Equal(item.UtcEndDate, arguments[4]); } [Theory] @@ -53,7 +54,7 @@ private static WorkItemContext CreateContext(RemoveBotEventsWorkItem item) => private class RecordingRepository : DispatchProxy { - public IRepositoryQuery? Query { get; private set; } + public object?[]? DeleteArguments { get; private set; } public int DeleteCalls { get; private set; } protected override object? Invoke(MethodInfo? targetMethod, object?[]? args) @@ -61,9 +62,7 @@ private class RecordingRepository : DispatchProxy if (targetMethod?.Name == nameof(IEventRepository.RemoveAllAsync)) { DeleteCalls++; - Query = args![0] is RepositoryQueryDescriptor descriptor - ? descriptor(new RepositoryQuery()) - : args[0] as IRepositoryQuery; + DeleteArguments = args; return Task.FromResult(1L); } diff --git a/tests/Exceptionless.Tests/Plugins/ThrottleBotsPluginTests.cs b/tests/Exceptionless.Tests/Plugins/ThrottleBotsPluginTests.cs index 586fe8ae4f..7baecf92c3 100644 --- a/tests/Exceptionless.Tests/Plugins/ThrottleBotsPluginTests.cs +++ b/tests/Exceptionless.Tests/Plugins/ThrottleBotsPluginTests.cs @@ -8,23 +8,24 @@ using Foundatio.Jobs; using Foundatio.Queues; using Foundatio.Serializer; +using Foundatio.Xunit; using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Time.Testing; using Xunit; namespace Exceptionless.Tests.Plugins; -public sealed class ThrottleBotsPluginTests : IDisposable +public sealed class ThrottleBotsPluginTests : TestWithLoggingBase { private readonly FakeTimeProvider _time = new(new DateTimeOffset(2026, 9, 21, 2, 25, 0, TimeSpan.Zero)); private readonly InMemoryCacheClient _cache; - private readonly InMemoryQueue _queue = new(); + private readonly InMemoryQueue _queue; private readonly ThrottleBotsPlugin _plugin; - public ThrottleBotsPluginTests() + public ThrottleBotsPluginTests(ITestOutputHelper output) : base(output) { - _cache = new InMemoryCacheClient(new InMemoryCacheClientOptions { TimeProvider = _time }); + _cache = new InMemoryCacheClient(new InMemoryCacheClientOptions { TimeProvider = _time, LoggerFactory = Log }); + _queue = new InMemoryQueue(new InMemoryQueueOptions { TimeProvider = _time, LoggerFactory = Log }); var configuration = new ConfigurationBuilder().AddInMemoryCollection(new Dictionary { [nameof(AppOptions.BaseURL)] = "http://localhost", @@ -33,7 +34,7 @@ public ThrottleBotsPluginTests() }).Build(); var serializer = new SystemTextJsonSerializer(new JsonSerializerOptions().ConfigureExceptionlessDefaults()); _plugin = new ThrottleBotsPlugin(_cache, _queue, serializer, _time, - AppOptions.ReadFromConfiguration(configuration), NullLoggerFactory.Instance); + AppOptions.ReadFromConfiguration(configuration), Log); } [Theory] @@ -87,9 +88,10 @@ private static EventContext CreateContext(string organizationId = "organization- new Project { Id = projectId, OrganizationId = organizationId, DeleteBotDataEnabled = enabled }); } - public void Dispose() + public override ValueTask DisposeAsync() { _queue.Dispose(); _cache.Dispose(); + return base.DisposeAsync(); } } diff --git a/tests/Exceptionless.Tests/Repositories/EventRepositoryTests.cs b/tests/Exceptionless.Tests/Repositories/EventRepositoryTests.cs index 95c5d2b628..c53ea89775 100644 --- a/tests/Exceptionless.Tests/Repositories/EventRepositoryTests.cs +++ b/tests/Exceptionless.Tests/Repositories/EventRepositoryTests.cs @@ -245,6 +245,47 @@ public async Task RemoveAllByClientIpAndDateAsync() Assert.Empty(events); } + [Theory] + [InlineData(true, true)] + [InlineData(true, false)] + [InlineData(false, true)] + [InlineData(false, false)] + public async Task RemoveAllAsync_WithProjectScope_PreservesOtherScopesAndFilters(bool hasStart, bool hasEnd) + { + const string clientIpAddress = "203.0.113.10"; + var start = DateTime.UtcNow.Date.AddHours(1); + var end = start.AddMinutes(5); + var inside = start.AddMinutes(1); + + PersistentEvent CreateEvent(string organizationId, string projectId, string ip, DateTime date) + { + var ev = _eventData.GenerateEvent(organizationId, projectId, TestConstants.StackId2, + occurrenceDate: date, generateData: false); + ev.AddRequestInfo(new RequestInfo { ClientIpAddress = ip }); + return ev; + } + + var matching = CreateEvent(TestConstants.OrganizationId, TestConstants.ProjectId, clientIpAddress, inside); + var otherProject = CreateEvent(TestConstants.OrganizationId, TestConstants.ProjectIdWithNoRoles, clientIpAddress, inside); + var otherOrganization = CreateEvent(TestConstants.OrganizationId2, TestConstants.ProjectId, clientIpAddress, inside); + var otherIp = CreateEvent(TestConstants.OrganizationId, TestConstants.ProjectId, "203.0.113.11", inside); + var before = CreateEvent(TestConstants.OrganizationId, TestConstants.ProjectId, clientIpAddress, start.AddMinutes(-1)); + var after = CreateEvent(TestConstants.OrganizationId, TestConstants.ProjectId, clientIpAddress, end.AddMinutes(1)); + await _repository.AddAsync([matching, otherProject, otherOrganization, otherIp, before, after], o => o.ImmediateConsistency()); + + long deleted = await _repository.RemoveAllAsync(TestConstants.OrganizationId, TestConstants.ProjectId, + clientIpAddress, hasStart ? start : null, hasEnd ? end : null, o => o.ImmediateConsistency()); + + Assert.Equal(1 + (hasStart ? 0 : 1) + (hasEnd ? 0 : 1), deleted); + Assert.Null(await _repository.GetByIdAsync(matching.Id)); + foreach (var preserved in new[] { otherProject, otherOrganization, otherIp }) + { + Assert.NotNull(await _repository.GetByIdAsync(preserved.Id)); + } + Assert.Equal(hasStart, await _repository.GetByIdAsync(before.Id) is not null); + Assert.Equal(hasEnd, await _repository.GetByIdAsync(after.Id) is not null); + } + private async Task CreateDataAsync() { var baseDate = DateTime.UtcNow.SubtractHours(1); From 3f173c0843494cbd003dc964d574fa58c20b3285 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E7=A8=8B=E6=A1=94?= Date: Thu, 24 Sep 2026 10:22:42 +0800 Subject: [PATCH 3/3] Make event deletion scope explicit in repository method names --- src/Exceptionless.Core/Jobs/CleanupDataJob.cs | 2 +- .../WorkItemHandlers/RemoveBotEventsWorkItemHandler.cs | 2 +- src/Exceptionless.Core/Repositories/EventRepository.cs | 4 ++-- .../Repositories/Interfaces/IEventRepository.cs | 4 ++-- .../RemoveBotEventsWorkItemHandlerTests.cs | 2 +- .../Repositories/EventRepositoryTests.cs | 8 ++++---- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/Exceptionless.Core/Jobs/CleanupDataJob.cs b/src/Exceptionless.Core/Jobs/CleanupDataJob.cs index 8282a7c6a1..0b043edaeb 100644 --- a/src/Exceptionless.Core/Jobs/CleanupDataJob.cs +++ b/src/Exceptionless.Core/Jobs/CleanupDataJob.cs @@ -579,7 +579,7 @@ private async Task EnforceEventRetentionDaysAsync(Organization organization, int var cutoff = _timeProvider.GetUtcNow().UtcDateTime.Date.SubtractDays(retentionDays); _logger.RetentionEnforcementEventStart(cutoff, organization.Name, organization.Id); - long removedEvents = await _eventRepository.RemoveAllAsync(organization.Id, null, null, cutoff); + long removedEvents = await _eventRepository.RemoveAllByOrganizationAndClientIpAsync(organization.Id, null, null, cutoff); _logger.RetentionEnforcementEventComplete(organization.Name, organization.Id, removedEvents); } diff --git a/src/Exceptionless.Core/Jobs/WorkItemHandlers/RemoveBotEventsWorkItemHandler.cs b/src/Exceptionless.Core/Jobs/WorkItemHandlers/RemoveBotEventsWorkItemHandler.cs index 6ea6855625..ea4cdf5ee4 100644 --- a/src/Exceptionless.Core/Jobs/WorkItemHandlers/RemoveBotEventsWorkItemHandler.cs +++ b/src/Exceptionless.Core/Jobs/WorkItemHandlers/RemoveBotEventsWorkItemHandler.cs @@ -35,7 +35,7 @@ public override async Task HandleItemAsync(WorkItemContext context) Log.LogInformation("Received remove bot events work item OrganizationId={OrganizationId} ProjectId={ProjectId}, ClientIpAddress={ClientIpAddress}, UtcStartDate={UtcStartDate}, UtcEndDate={UtcEndDate}", wi.OrganizationId, wi.ProjectId, wi.ClientIpAddress, wi.UtcStartDate, wi.UtcEndDate); await context.ReportProgressAsync(0, $"Starting deleting of bot events... OrganizationId={wi.OrganizationId}"); - long deleted = await _eventRepository.RemoveAllAsync(wi.OrganizationId, wi.ProjectId, wi.ClientIpAddress, wi.UtcStartDate, wi.UtcEndDate); + long deleted = await _eventRepository.RemoveAllByProjectAndClientIpAsync(wi.OrganizationId, wi.ProjectId, wi.ClientIpAddress, wi.UtcStartDate, wi.UtcEndDate); await context.ReportProgressAsync(100, $"Bot events deleted: {deleted} OrganizationId={wi.OrganizationId}"); Log.LogInformation("Removed {Deleted} bot events OrganizationId={OrganizationId} ProjectId={ProjectId}, ClientIpAddress={ClientIpAddress}, UtcStartDate={UtcStartDate}, UtcEndDate={UtcEndDate}", deleted, wi.OrganizationId, wi.ProjectId, wi.ClientIpAddress, wi.UtcStartDate, wi.UtcEndDate); } diff --git a/src/Exceptionless.Core/Repositories/EventRepository.cs b/src/Exceptionless.Core/Repositories/EventRepository.cs index c28d1907a9..0a128146d8 100644 --- a/src/Exceptionless.Core/Repositories/EventRepository.cs +++ b/src/Exceptionless.Core/Repositories/EventRepository.cs @@ -59,7 +59,7 @@ public async Task UpdateSessionStartLastActivityAsync(string id, DateTime return true; } - public Task RemoveAllAsync(string organizationId, string? clientIpAddress, DateTime? utcStart, DateTime? utcEnd, CommandOptionsDescriptor? options = null) + public Task RemoveAllByOrganizationAndClientIpAsync(string organizationId, string? clientIpAddress, DateTime? utcStart, DateTime? utcEnd, CommandOptionsDescriptor? options = null) { ArgumentException.ThrowIfNullOrEmpty(organizationId); @@ -77,7 +77,7 @@ public Task RemoveAllAsync(string organizationId, string? clientIpAddress, return RemoveAllAsync(q => query, options); } - public Task RemoveAllAsync(string organizationId, string projectId, string clientIpAddress, DateTime? utcStart, DateTime? utcEnd, CommandOptionsDescriptor? options = null) + public Task RemoveAllByProjectAndClientIpAsync(string organizationId, string projectId, string clientIpAddress, DateTime? utcStart, DateTime? utcEnd, CommandOptionsDescriptor? options = null) { ArgumentException.ThrowIfNullOrWhiteSpace(organizationId); ArgumentException.ThrowIfNullOrWhiteSpace(projectId); diff --git a/src/Exceptionless.Core/Repositories/Interfaces/IEventRepository.cs b/src/Exceptionless.Core/Repositories/Interfaces/IEventRepository.cs index 49ae487bf8..3d5f891a38 100644 --- a/src/Exceptionless.Core/Repositories/Interfaces/IEventRepository.cs +++ b/src/Exceptionless.Core/Repositories/Interfaces/IEventRepository.cs @@ -11,8 +11,8 @@ public interface IEventRepository : IRepositoryOwnedByOrganizationAndProject GetPreviousAndNextEventIdsAsync(PersistentEvent ev, AppFilter? systemFilter = null, DateTime? utcStart = null, DateTime? utcEnd = null); Task> GetOpenSessionsAsync(DateTime createdBeforeUtc, CommandOptionsDescriptor? options = null); Task UpdateSessionStartLastActivityAsync(string id, DateTime lastActivityUtc, bool isSessionEnd = false, bool hasError = false, bool sendNotifications = true); - Task RemoveAllAsync(string organizationId, string? clientIpAddress, DateTime? utcStart, DateTime? utcEnd, CommandOptionsDescriptor? options = null); - Task RemoveAllAsync(string organizationId, string projectId, string clientIpAddress, DateTime? utcStart, DateTime? utcEnd, CommandOptionsDescriptor? options = null); + Task RemoveAllByOrganizationAndClientIpAsync(string organizationId, string? clientIpAddress, DateTime? utcStart, DateTime? utcEnd, CommandOptionsDescriptor? options = null); + Task RemoveAllByProjectAndClientIpAsync(string organizationId, string projectId, string clientIpAddress, DateTime? utcStart, DateTime? utcEnd, CommandOptionsDescriptor? options = null); Task RemoveAllByStackIdsAsync(string[] stackIds); } diff --git a/tests/Exceptionless.Tests/Jobs/WorkItemHandlers/RemoveBotEventsWorkItemHandlerTests.cs b/tests/Exceptionless.Tests/Jobs/WorkItemHandlers/RemoveBotEventsWorkItemHandlerTests.cs index e3532cfcdf..77fd83b87d 100644 --- a/tests/Exceptionless.Tests/Jobs/WorkItemHandlers/RemoveBotEventsWorkItemHandlerTests.cs +++ b/tests/Exceptionless.Tests/Jobs/WorkItemHandlers/RemoveBotEventsWorkItemHandlerTests.cs @@ -59,7 +59,7 @@ private class RecordingRepository : DispatchProxy protected override object? Invoke(MethodInfo? targetMethod, object?[]? args) { - if (targetMethod?.Name == nameof(IEventRepository.RemoveAllAsync)) + if (targetMethod?.Name == nameof(IEventRepository.RemoveAllByProjectAndClientIpAsync)) { DeleteCalls++; DeleteArguments = args; diff --git a/tests/Exceptionless.Tests/Repositories/EventRepositoryTests.cs b/tests/Exceptionless.Tests/Repositories/EventRepositoryTests.cs index c53ea89775..ce5832ae6a 100644 --- a/tests/Exceptionless.Tests/Repositories/EventRepositoryTests.cs +++ b/tests/Exceptionless.Tests/Repositories/EventRepositoryTests.cs @@ -221,7 +221,7 @@ public async Task GetOpenSessionsAsync() } [Fact] - public async Task RemoveAllByClientIpAndDateAsync() + public async Task RemoveAllByOrganizationAndClientIpAsync_WithDateRange_RemovesMatchingEvents() { const string _clientIpAddress = "123.123.12.255"; const int NUMBER_OF_EVENTS_TO_CREATE = 50; @@ -239,7 +239,7 @@ public async Task RemoveAllByClientIpAndDateAsync() Assert.Equal(_clientIpAddress, ri.ClientIpAddress); }); - await _repository.RemoveAllAsync(TestConstants.OrganizationId, _clientIpAddress, DateTime.UtcNow.SubtractDays(3), DateTime.UtcNow.AddDays(2), o => o.ImmediateConsistency()); + await _repository.RemoveAllByOrganizationAndClientIpAsync(TestConstants.OrganizationId, _clientIpAddress, DateTime.UtcNow.SubtractDays(3), DateTime.UtcNow.AddDays(2), o => o.ImmediateConsistency()); events = (await _repository.GetByProjectIdAsync(TestConstants.ProjectId, o => o.PageLimit(NUMBER_OF_EVENTS_TO_CREATE))).Documents.ToList(); Assert.Empty(events); @@ -250,7 +250,7 @@ public async Task RemoveAllByClientIpAndDateAsync() [InlineData(true, false)] [InlineData(false, true)] [InlineData(false, false)] - public async Task RemoveAllAsync_WithProjectScope_PreservesOtherScopesAndFilters(bool hasStart, bool hasEnd) + public async Task RemoveAllByProjectAndClientIpAsync_WithProjectScope_PreservesOtherScopesAndFilters(bool hasStart, bool hasEnd) { const string clientIpAddress = "203.0.113.10"; var start = DateTime.UtcNow.Date.AddHours(1); @@ -273,7 +273,7 @@ PersistentEvent CreateEvent(string organizationId, string projectId, string ip, var after = CreateEvent(TestConstants.OrganizationId, TestConstants.ProjectId, clientIpAddress, end.AddMinutes(1)); await _repository.AddAsync([matching, otherProject, otherOrganization, otherIp, before, after], o => o.ImmediateConsistency()); - long deleted = await _repository.RemoveAllAsync(TestConstants.OrganizationId, TestConstants.ProjectId, + long deleted = await _repository.RemoveAllByProjectAndClientIpAsync(TestConstants.OrganizationId, TestConstants.ProjectId, clientIpAddress, hasStart ? start : null, hasEnd ? end : null, o => o.ImmediateConsistency()); Assert.Equal(1 + (hasStart ? 0 : 1) + (hasEnd ? 0 : 1), deleted);