fix: page through ArrowHead newsfeed instead of a fixed 1024-entry window

src/Helldivers-2-Sync/Configuration/HelldiversSyncConfiguration.cs

@@ -32,12 +32,15 @@ public sealed class HelldiversSyncConfiguration
     public bool RunOnce { get; set; } = false;
 
     /// <summary>
-    /// Get the maximum number of entries returned by ArrowHead from the newsfeed API.
+    /// The maximum number of entries ArrowHead returns per newsfeed API call. The sync service pages through
+    /// the full history by repeatedly calling the API with an advancing <see cref="NewsFeedFromTimestamp" />
+    /// until a page comes back with fewer than this many entries.
     /// </summary>
     public uint NewsFeedMaxEntries { get; set; } = 1024;
 
     /// <summary>
-    /// Get all news feed entries that were published after this timestamp
+    /// The timestamp used for the first page of newsfeed entries fetched each sync. Every entry published on
+    /// or after this timestamp will eventually be fetched, one <see cref="NewsFeedMaxEntries" />-sized page at a time.
     /// </summary>
     public uint NewsFeedFromTimestamp { get; set; } = 1000;
 }

src/Helldivers-2-Sync/Services/ArrowHeadApiService.cs

@@ -69,15 +69,18 @@ public async Task<Memory<byte>> GetWarStatus(string season, string language, Can
     }
 
     /// <summary>
-    /// Fetch the newsfeed of a given <paramref name="season" /> in <paramref name="language" />.
+    /// Fetch a single page of the newsfeed of a given <paramref name="season" /> in <paramref name="language" />,
+    /// starting at <paramref name="fromTimestamp" />. ArrowHead returns at most <see cref="HelldiversSyncConfiguration.NewsFeedMaxEntries" />
+    /// entries per call, oldest-first from <paramref name="fromTimestamp" /> onwards.
     /// </summary>
-    public async Task<Memory<byte>> LoadFeed(string season, string language, CancellationToken cancellationToken)
+    private async Task<Memory<byte>> LoadFeedPage(string season, string language, uint fromTimestamp,
+        CancellationToken cancellationToken)
     {
         // If the `NewsFeedMaxEntries` flag is not set to 0 we pass it in.
         // This parameter needs to be passed or a 400 status code will be returned occasionally.
         var request = options.Value.NewsFeedMaxEntries is 0
-            ? BuildRequest($"/api/NewsFeed/{season}?fromTimestamp={options.Value.NewsFeedFromTimestamp}", language)
-            : BuildRequest($"/api/NewsFeed/{season}?maxEntries={options.Value.NewsFeedMaxEntries}&fromTimestamp={options.Value.NewsFeedFromTimestamp}", language);
+            ? BuildRequest($"/api/NewsFeed/{season}?fromTimestamp={fromTimestamp}", language)
+            : BuildRequest($"/api/NewsFeed/{season}?maxEntries={options.Value.NewsFeedMaxEntries}&fromTimestamp={fromTimestamp}", language);
 
         using var response = await http.SendAsync(request, cancellationToken);
 
@@ -89,6 +92,43 @@ public async Task<Memory<byte>> LoadFeed(string season, string language, Cancell
         return await CollectStream(stream, cancellationToken);
     }
 
+    /// <summary>
+    /// Fetch the entire newsfeed of a given <paramref name="season" /> in <paramref name="language" />, paging
+    /// through ArrowHead's API (which only ever returns up to <see cref="HelldiversSyncConfiguration.NewsFeedMaxEntries" />
+    /// entries per call starting at a given timestamp) until every entry has been collected.
+    /// </summary>
+    public async Task<Memory<byte>> LoadFeed(string season, string language, CancellationToken cancellationToken)
+    {
+        var maxEntries = options.Value.NewsFeedMaxEntries;
+        var fromTimestamp = options.Value.NewsFeedFromTimestamp;
+        var items = new List<NewsFeedItem>();
+
+        // Safety net so a misbehaving upstream (eg. always returning a full page) can't loop forever.
+        for (var page = 0; page < 1_000; page++)
+        {
+            var raw = await LoadFeedPage(season, language, fromTimestamp, cancellationToken);
+            var pageItems = JsonSerializer.Deserialize(
+                raw.Span,
+                ArrowHeadSerializerContext.Default.ListNewsFeedItem
+            ) ?? [];
+
+            if (pageItems.Count is 0)
+                break;
+
+            items.AddRange(pageItems);
+
+            // A page smaller than what we asked for means we've reached the end of the feed.
+            if (maxEntries is 0 || pageItems.Count < maxEntries)
+                break;
+
+            // Advance the window to just after the newest entry we've seen so far, so the next page
+            // picks up where this one left off instead of re-fetching the same entries forever.
+            fromTimestamp = checked((uint)(pageItems.Max(item => item.Published) + 1));
+        }
+
+        return JsonSerializer.SerializeToUtf8Bytes(items, ArrowHeadSerializerContext.Default.ListNewsFeedItem);
+    }
+
     /// <summary>
     /// Loads assignments of a given <paramref name="season" /> in <paramref name="language" />.
     /// </summary>