diff --git a/src/Helldivers-2-Sync/Configuration/HelldiversSyncConfiguration.cs b/src/Helldivers-2-Sync/Configuration/HelldiversSyncConfiguration.cs
index 61819c7..9310c35 100644
--- a/src/Helldivers-2-Sync/Configuration/HelldiversSyncConfiguration.cs
+++ b/src/Helldivers-2-Sync/Configuration/HelldiversSyncConfiguration.cs
@@ -32,12 +32,15 @@ public sealed class HelldiversSyncConfiguration
public bool RunOnce { get; set; } = false;
///
- /// Get the maximum number of entries returned by ArrowHead from the newsfeed API.
+ /// The maximum number of entries ArrowHead returns per newsfeed API call. The sync service pages through
+ /// the full history by repeatedly calling the API with an advancing
+ /// until a page comes back with fewer than this many entries.
///
public uint NewsFeedMaxEntries { get; set; } = 1024;
///
- /// Get all news feed entries that were published after this timestamp
+ /// The timestamp used for the first page of newsfeed entries fetched each sync. Every entry published on
+ /// or after this timestamp will eventually be fetched, one -sized page at a time.
///
public uint NewsFeedFromTimestamp { get; set; } = 1000;
}
diff --git a/src/Helldivers-2-Sync/Services/ArrowHeadApiService.cs b/src/Helldivers-2-Sync/Services/ArrowHeadApiService.cs
index 1ac5ae1..7403e7f 100644
--- a/src/Helldivers-2-Sync/Services/ArrowHeadApiService.cs
+++ b/src/Helldivers-2-Sync/Services/ArrowHeadApiService.cs
@@ -69,15 +69,18 @@ public async Task> GetWarStatus(string season, string language, Can
}
///
- /// Fetch the newsfeed of a given in .
+ /// Fetch a single page of the newsfeed of a given in ,
+ /// starting at . ArrowHead returns at most
+ /// entries per call, oldest-first from onwards.
///
- public async Task> LoadFeed(string season, string language, CancellationToken cancellationToken)
+ private async Task> LoadFeedPage(string season, string language, uint fromTimestamp,
+ CancellationToken cancellationToken)
{
// If the `NewsFeedMaxEntries` flag is not set to 0 we pass it in.
// This parameter needs to be passed or a 400 status code will be returned occasionally.
var request = options.Value.NewsFeedMaxEntries is 0
- ? BuildRequest($"/api/NewsFeed/{season}?fromTimestamp={options.Value.NewsFeedFromTimestamp}", language)
- : BuildRequest($"/api/NewsFeed/{season}?maxEntries={options.Value.NewsFeedMaxEntries}&fromTimestamp={options.Value.NewsFeedFromTimestamp}", language);
+ ? BuildRequest($"/api/NewsFeed/{season}?fromTimestamp={fromTimestamp}", language)
+ : BuildRequest($"/api/NewsFeed/{season}?maxEntries={options.Value.NewsFeedMaxEntries}&fromTimestamp={fromTimestamp}", language);
using var response = await http.SendAsync(request, cancellationToken);
@@ -89,6 +92,43 @@ public async Task> LoadFeed(string season, string language, Cancell
return await CollectStream(stream, cancellationToken);
}
+ ///
+ /// Fetch the entire newsfeed of a given in , paging
+ /// through ArrowHead's API (which only ever returns up to
+ /// entries per call starting at a given timestamp) until every entry has been collected.
+ ///
+ public async Task> LoadFeed(string season, string language, CancellationToken cancellationToken)
+ {
+ var maxEntries = options.Value.NewsFeedMaxEntries;
+ var fromTimestamp = options.Value.NewsFeedFromTimestamp;
+ var items = new List();
+
+ // Safety net so a misbehaving upstream (eg. always returning a full page) can't loop forever.
+ for (var page = 0; page < 1_000; page++)
+ {
+ var raw = await LoadFeedPage(season, language, fromTimestamp, cancellationToken);
+ var pageItems = JsonSerializer.Deserialize(
+ raw.Span,
+ ArrowHeadSerializerContext.Default.ListNewsFeedItem
+ ) ?? [];
+
+ if (pageItems.Count is 0)
+ break;
+
+ items.AddRange(pageItems);
+
+ // A page smaller than what we asked for means we've reached the end of the feed.
+ if (maxEntries is 0 || pageItems.Count < maxEntries)
+ break;
+
+ // Advance the window to just after the newest entry we've seen so far, so the next page
+ // picks up where this one left off instead of re-fetching the same entries forever.
+ fromTimestamp = checked((uint)(pageItems.Max(item => item.Published) + 1));
+ }
+
+ return JsonSerializer.SerializeToUtf8Bytes(items, ArrowHeadSerializerContext.Default.ListNewsFeedItem);
+ }
+
///
/// Loads assignments of a given in .
///