diff --git a/src/Helldivers-2-Sync/Configuration/HelldiversSyncConfiguration.cs b/src/Helldivers-2-Sync/Configuration/HelldiversSyncConfiguration.cs index 61819c7..9310c35 100644 --- a/src/Helldivers-2-Sync/Configuration/HelldiversSyncConfiguration.cs +++ b/src/Helldivers-2-Sync/Configuration/HelldiversSyncConfiguration.cs @@ -32,12 +32,15 @@ public sealed class HelldiversSyncConfiguration public bool RunOnce { get; set; } = false; /// - /// Get the maximum number of entries returned by ArrowHead from the newsfeed API. + /// The maximum number of entries ArrowHead returns per newsfeed API call. The sync service pages through + /// the full history by repeatedly calling the API with an advancing + /// until a page comes back with fewer than this many entries. /// public uint NewsFeedMaxEntries { get; set; } = 1024; /// - /// Get all news feed entries that were published after this timestamp + /// The timestamp used for the first page of newsfeed entries fetched each sync. Every entry published on + /// or after this timestamp will eventually be fetched, one -sized page at a time. /// public uint NewsFeedFromTimestamp { get; set; } = 1000; } diff --git a/src/Helldivers-2-Sync/Services/ArrowHeadApiService.cs b/src/Helldivers-2-Sync/Services/ArrowHeadApiService.cs index 1ac5ae1..7403e7f 100644 --- a/src/Helldivers-2-Sync/Services/ArrowHeadApiService.cs +++ b/src/Helldivers-2-Sync/Services/ArrowHeadApiService.cs @@ -69,15 +69,18 @@ public async Task> GetWarStatus(string season, string language, Can } /// - /// Fetch the newsfeed of a given in . + /// Fetch a single page of the newsfeed of a given in , + /// starting at . ArrowHead returns at most + /// entries per call, oldest-first from onwards. /// - public async Task> LoadFeed(string season, string language, CancellationToken cancellationToken) + private async Task> LoadFeedPage(string season, string language, uint fromTimestamp, + CancellationToken cancellationToken) { // If the `NewsFeedMaxEntries` flag is not set to 0 we pass it in. // This parameter needs to be passed or a 400 status code will be returned occasionally. var request = options.Value.NewsFeedMaxEntries is 0 - ? BuildRequest($"/api/NewsFeed/{season}?fromTimestamp={options.Value.NewsFeedFromTimestamp}", language) - : BuildRequest($"/api/NewsFeed/{season}?maxEntries={options.Value.NewsFeedMaxEntries}&fromTimestamp={options.Value.NewsFeedFromTimestamp}", language); + ? BuildRequest($"/api/NewsFeed/{season}?fromTimestamp={fromTimestamp}", language) + : BuildRequest($"/api/NewsFeed/{season}?maxEntries={options.Value.NewsFeedMaxEntries}&fromTimestamp={fromTimestamp}", language); using var response = await http.SendAsync(request, cancellationToken); @@ -89,6 +92,43 @@ public async Task> LoadFeed(string season, string language, Cancell return await CollectStream(stream, cancellationToken); } + /// + /// Fetch the entire newsfeed of a given in , paging + /// through ArrowHead's API (which only ever returns up to + /// entries per call starting at a given timestamp) until every entry has been collected. + /// + public async Task> LoadFeed(string season, string language, CancellationToken cancellationToken) + { + var maxEntries = options.Value.NewsFeedMaxEntries; + var fromTimestamp = options.Value.NewsFeedFromTimestamp; + var items = new List(); + + // Safety net so a misbehaving upstream (eg. always returning a full page) can't loop forever. + for (var page = 0; page < 1_000; page++) + { + var raw = await LoadFeedPage(season, language, fromTimestamp, cancellationToken); + var pageItems = JsonSerializer.Deserialize( + raw.Span, + ArrowHeadSerializerContext.Default.ListNewsFeedItem + ) ?? []; + + if (pageItems.Count is 0) + break; + + items.AddRange(pageItems); + + // A page smaller than what we asked for means we've reached the end of the feed. + if (maxEntries is 0 || pageItems.Count < maxEntries) + break; + + // Advance the window to just after the newest entry we've seen so far, so the next page + // picks up where this one left off instead of re-fetching the same entries forever. + fromTimestamp = checked((uint)(pageItems.Max(item => item.Published) + 1)); + } + + return JsonSerializer.SerializeToUtf8Bytes(items, ArrowHeadSerializerContext.Default.ListNewsFeedItem); + } + /// /// Loads assignments of a given in . ///