diff --git a/README.md b/README.md index 3ef0e10c..da21f441 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,8 @@ docker run -it --rm -p 4000:4000 \ | `SEARCH_API_ADDR` | hivesearcher api endpoint | | `SEARCH_API_SECRET` | hivesearcher api auth token | | `STRIPE_INTERNAL_SECRET` | shared secret for the Stripe money endpoints (unset = they fail closed) | +| `DESK_INTERNAL_TOKEN` | shared secret sent on every curation desk call (`/private-api/curation-desk/*`), public reads included (unset = the desk routes answer 503) | +| `DESK_MEMO_BYTES` | byte budget of each curation desk memo store (fresh and last-good), LRU beyond it (default 64 MiB) | | `TURNSTILE_SECRET` | Cloudflare Turnstile secret for account-create captcha | | `CAPTCHA_MODE` | `hard` (default) or `off` (operator break-glass) | | `BLOCKSTREAM_CLIENT_ID` / `BLOCKSTREAM_CLIENT_SECRET` | optional Blockstream Enterprise esplora auth (BTC fallback) | diff --git a/dotnet/EcencyApi.Tests/CachePolicyTests.cs b/dotnet/EcencyApi.Tests/CachePolicyTests.cs index 0f591eb4..d1e5c392 100644 --- a/dotnet/EcencyApi.Tests/CachePolicyTests.cs +++ b/dotnet/EcencyApi.Tests/CachePolicyTests.cs @@ -10,34 +10,55 @@ namespace EcencyApi.Tests; /// public class CachePolicyTests { - public static TheoryData AllPolicies() => - new() { CachePolicy.ProMembers, CachePolicy.Announcements, CachePolicy.PostTips }; + // Keyed by route: several routes share a policy string, and a theory row + // that repeats its arguments is a duplicate test id that xUnit reports as a + // skip rather than running. + public static TheoryData AllPolicies() => + new() + { + { "pro-members", CachePolicy.ProMembers }, + { "announcements", CachePolicy.Announcements }, + { "post-tips", CachePolicy.PostTips }, + { "desk-feed", CachePolicy.CurationDeskFeed }, + { "desk-status", CachePolicy.CurationDeskStatus }, + { "desk-roster", CachePolicy.CurationDeskRoster }, + { "desk-recommendations", CachePolicy.CurationDeskRecommendations }, + { "desk-post", CachePolicy.CurationDeskPost }, + }; + + public static TheoryData DeskPolicies() => + new() + { + { "desk-feed", CachePolicy.CurationDeskFeed, 30 }, + { "desk-status", CachePolicy.CurationDeskStatus, 15 }, + { "desk-roster", CachePolicy.CurationDeskRoster, 600 }, + { "desk-recommendations", CachePolicy.CurationDeskRecommendations, 30 }, + { "desk-post", CachePolicy.CurationDeskPost, 15 }, + }; [Theory] [MemberData(nameof(AllPolicies))] - public void EveryPolicyIsPubliclyCacheableWithAMaxAge(string policy) + public void EveryPolicyIsPubliclyCacheableWithAMaxAge(string route, string policy) { - Assert.StartsWith("public, max-age=", policy); + Assert.True(policy.StartsWith("public, max-age=", StringComparison.Ordinal), route + ": " + policy); Assert.DoesNotContain("no-store", policy); Assert.DoesNotContain("private", policy); } [Theory] [MemberData(nameof(AllPolicies))] - public void APolicyOnlyAppliesToASuccessfulResponse(string policy) + public void APolicyOnlyAppliesToASuccessfulResponse(string route, string policy) { Assert.Equal(policy, CachePolicy.ForStatus(200, policy)); - // Pipe() turns upstream transport failures into these after the handler - // has already attached the policy. Caching one would keep a healthy + // Pipe() turns upstream transport failures into 504/500 after the + // handler has already attached the policy, and an upstream error + // passthrough keeps its own status. Caching either would keep a healthy // endpoint broken for the whole max-age. - Assert.Null(CachePolicy.ForStatus(504, policy)); - Assert.Null(CachePolicy.ForStatus(500, policy)); - - // Upstream error passthroughs must not be cached either. - Assert.Null(CachePolicy.ForStatus(404, policy)); - Assert.Null(CachePolicy.ForStatus(401, policy)); - Assert.Null(CachePolicy.ForStatus(429, policy)); + foreach (var status in new[] { 504, 500, 404, 401, 429 }) + { + Assert.True(CachePolicy.ForStatus(status, policy) == null, route + " " + status); + } } [Fact] @@ -49,6 +70,72 @@ public void AnnouncementsOutliveTheOtherPolicies() Assert.True(MaxAge(CachePolicy.ProMembers) > MaxAge(CachePolicy.PostTips)); } + [Theory] + [MemberData(nameof(DeskPolicies))] + public void DeskPoliciesRevalidateInTheBrowserAndAreSharedForTheirSMaxAge(string route, string policy, int sMaxAge) + { + // max-age=0 makes every browser poll revalidate; s-maxage is what shared + // caches and the in-process memo hold the body for. + Assert.True(MaxAge(policy) == 0, route); + Assert.True(CachePolicy.SharedMaxAge(policy) == sMaxAge, route); + Assert.DoesNotContain("stale-while-revalidate", policy); + } + + [Theory] + [MemberData(nameof(DeskPolicies))] + public void AnAgedPolicyOffersOnlyTheRestOfTheSharedWindow(string route, string policy, int sMaxAge) + { + // A body served from the in-process memo has already spent part of its + // life there; handing a shared cache a fresh window would let the two + // layers serve one answer for a lifetime each, in series. + Assert.Equal(policy, CachePolicy.Aged(policy, 0)); + Assert.Equal(sMaxAge - 1, CachePolicy.SharedMaxAge(CachePolicy.Aged(policy, 1))); + + foreach (var age in new[] { 1, sMaxAge - 1, sMaxAge, sMaxAge + 1, sMaxAge * 10 }) + { + var aged = CachePolicy.Aged(policy, age); + var remaining = CachePolicy.SharedMaxAge(aged); + + // Floored at a second, never longer than what is left, and still a + // policy of the same shape. + Assert.True(remaining >= 1, route + " " + age); + Assert.True(remaining <= Math.Max(1, sMaxAge - age), route + " " + age); + Assert.StartsWith("public, max-age=0, s-maxage=", aged); + Assert.Null(CachePolicy.ForStatus(504, aged)); + } + } + + [Theory] + [MemberData(nameof(DeskPolicies))] + public void TheStalePolicyIsShortAndNeverLongerThanTheRouteItself(string route, string policy, int sMaxAge) + { + // Served because the upstream call failed: still cacheable, so a burst + // does not all queue behind a struggling backend, but only for seconds. + var stale = CachePolicy.Stale(policy); + Assert.Equal("public, max-age=0, s-maxage=" + CachePolicy.StaleSharedMaxAge, stale); + Assert.True(CachePolicy.SharedMaxAge(stale) < sMaxAge, route); + Assert.Null(CachePolicy.ForStatus(500, stale)); + } + + [Fact] + public void ShorteningAPolicyLeavesItsOtherDirectivesAlone() + { + // A policy whose shared window is its max-age gains an s-maxage rather + // than having what browsers were told rewritten under them. + var aged = CachePolicy.Aged(CachePolicy.ProMembers, 100); + Assert.StartsWith(CachePolicy.ProMembers, aged); + Assert.Contains("stale-while-revalidate=3600", aged); + Assert.Equal(500, CachePolicy.SharedMaxAge(aged)); + } + + [Fact] + public void SharedMaxAgeFallsBackToMaxAgeWhenAPolicyHasNoSharedDirective() + { + Assert.Equal(600, CachePolicy.SharedMaxAge(CachePolicy.ProMembers)); + Assert.Equal(60, CachePolicy.SharedMaxAge(CachePolicy.PostTips)); + Assert.Throws(() => CachePolicy.SharedMaxAge("public")); + } + private static int MaxAge(string policy) { var token = policy.Split(',').Select(p => p.Trim()) diff --git a/dotnet/EcencyApi.Tests/CurationDeskAuthTests.cs b/dotnet/EcencyApi.Tests/CurationDeskAuthTests.cs new file mode 100644 index 00000000..3d3bf188 --- /dev/null +++ b/dotnet/EcencyApi.Tests/CurationDeskAuthTests.cs @@ -0,0 +1,733 @@ +using System.Text.Json.Nodes; +using EcencyApi.Handlers; +using EcencyApi.Infrastructure; +using Xunit; +using static EcencyApi.Tests.CurationDeskTestSupport; + +namespace EcencyApi.Tests; + +/// +/// Handler-level behaviour of the desk routes: the shared secret on every +/// upstream call, the fail-closed 503, the validation memo, the byte memo with +/// its single flight and last-good fallback, and the Cache-Control rules. +/// +[Collection("curation-desk")] +public class CurationDeskAuthTests +{ + // ---- the token ----------------------------------------------------------- + + [Fact] + public async Task EveryUpstreamCallCarriesTheDeskToken() + { + var upstream = Install(); + + foreach (var (name, handler, request, _) in PublicReads()) + { + await handler(request()); + var call = Assert.Single(upstream.Calls); + Assert.Equal(Token, call.Header(PrivateApi.DeskTokenHeader)); + Assert.Equal(HttpMethod.Get, call.Method); + Assert.StartsWith("curation/desk/", call.Endpoint); + Assert.Null(call.Payload); + upstream.Calls.Clear(); + CurationDeskMemo.ResetForTests(); + } + + foreach (var (name, handler, body) in SignedWrites()) + { + await handler(Post("/private-api/curation-desk/" + name, body)); + var call = Assert.Single(upstream.Calls); + Assert.Equal(Token, call.Header(PrivateApi.DeskTokenHeader)); + Assert.Equal(HttpMethod.Post, call.Method); + Assert.StartsWith("curation/desk/", call.Endpoint); + Assert.Equal("alice", call.Payload!["username"]!.GetValue()); + Assert.False(((JsonObject)call.Payload).ContainsKey("code")); + upstream.Calls.Clear(); + } + } + + [Fact] + public void TheTokenComesFromItsOwnEnvironmentVariable() + { + // Not set in the test environment: the desk is switched off by default. + Assert.Equal("", Config.DeskInternalToken); + } + + // ---- fail closed --------------------------------------------------------- + + [Fact] + public async Task WithoutTheTokenReadsAnswer503BeforeAnyUpstreamCall() + { + var upstream = Install(token: null); + + foreach (var (name, handler, request, _) in PublicReads()) + { + var ctx = request(); + await handler(ctx); + await Start(ctx); + Assert.Equal(503, ctx.Response.StatusCode); + Assert.Equal("curation desk not configured", Body(ctx)); + Assert.Null(CacheControl(ctx)); + } + Assert.Empty(upstream.Calls); + } + + [Fact] + public async Task WithoutTheTokenWritesAnswer503BeforeValidatingAnything() + { + var upstream = Install(token: null); + var validations = 0; + PrivateApi.DeskValidateCode = _ => { validations++; return Task.FromResult("alice"); }; + + foreach (var (name, handler, body) in SignedWrites()) + { + // A dark desk answers the same to a signed body and to an anonymous + // one, and neither costs a chain lookup: there is no work behind the + // route to authorize. + foreach (var payload in new[] { body, "{}" }) + { + var ctx = Post("/private-api/curation-desk/" + name, payload); + await handler(ctx); + await Start(ctx); + Assert.Equal(503, ctx.Response.StatusCode); + Assert.Equal("curation desk not configured", Body(ctx)); + Assert.Null(CacheControl(ctx)); + } + } + Assert.Equal(0, validations); + Assert.Empty(upstream.Calls); + } + + [Fact] + public async Task InvalidSignedCodesAre401WithTheRealValidator() + { + var upstream = Install(); + PrivateApi.DeskValidateCode = PrivateApi.ValidateCode; + + foreach (var (name, handler, _) in SignedWrites()) + { + var empty = Post("/private-api/curation-desk/" + name, "{}"); + await handler(empty); + Assert.Equal(401, empty.Response.StatusCode); + + // The parity probe: decodes to {"not":"valid"}, fails on structure + // before any account lookup. + var probe = Post("/private-api/curation-desk/" + name, "{\"code\":\"eyJub3QiOiJ2YWxpZCJ9\"}"); + await handler(probe); + Assert.Equal(401, probe.Response.StatusCode); + Assert.Equal("Unauthorized", Body(probe)); + } + Assert.Empty(upstream.Calls); + } + + [Fact] + public async Task ARejectedPayloadIs400AndNeverReachesUpstream() + { + var upstream = Install(); + + var mark = Post("/private-api/curation-desk/mark", "{\"code\":\"as:alice\",\"author\":\"bob\",\"permlink\":\"p\",\"state\":\"deleted\"}"); + await PrivateApi.CurationDeskMark(mark); + Assert.Equal(400, mark.Response.StatusCode); + Assert.Equal("invalid state", Body(mark)); + + var meta = Post("/private-api/curation-desk/recommend-meta", "{\"code\":\"as:alice\",\"author\":\"bob\",\"permlink\":\"p\",\"trx_id\":\"nope\"}"); + await PrivateApi.CurationDeskRecommendMeta(meta); + Assert.Equal(400, meta.Response.StatusCode); + Assert.Equal("invalid trx_id", Body(meta)); + + Assert.Empty(upstream.Calls); + } + + // ---- the validation memo ------------------------------------------------- + + [Fact] + public async Task ASuccessfulValidationIsRememberedWithinTheTtlAndForgottenAfter() + { + var upstream = Install(); + var validations = 0; + PrivateApi.DeskValidateCode = _ => { validations++; return Task.FromResult("alice"); }; + // Wide enough that two back-to-back in-process calls cannot straddle it + // on a loaded runner; the delay below is what expires it. + PrivateApi.DeskAuthMemoSeconds = 2; + var code = "memo-" + Guid.NewGuid().ToString("N"); + var body = "{\"code\":\"" + code + "\",\"author\":\"bob\",\"permlink\":\"p\"}"; + + await PrivateApi.CurationDeskMarkClear(Post("/private-api/curation-desk/mark-clear", body)); + await PrivateApi.CurationDeskMarkClear(Post("/private-api/curation-desk/mark-clear", body)); + Assert.Equal(1, validations); + Assert.Equal(2, upstream.Calls.Count); + Assert.All(upstream.Calls, c => Assert.Equal("alice", c.Payload!["username"]!.GetValue())); + + // A different code is a different memo entry. + await PrivateApi.CurationDeskMarkClear(Post("/private-api/curation-desk/mark-clear", body.Replace(code, code + "x"))); + Assert.Equal(2, validations); + + await Task.Delay(2300); + await PrivateApi.CurationDeskMarkClear(Post("/private-api/curation-desk/mark-clear", body)); + Assert.Equal(3, validations); + } + + [Fact] + public async Task AFailedValidationIsNeverRemembered() + { + var upstream = Install(); + var validations = 0; + string? answer = null; + PrivateApi.DeskValidateCode = _ => { validations++; return Task.FromResult(answer); }; + var code = "fail-" + Guid.NewGuid().ToString("N"); + var body = "{\"code\":\"" + code + "\",\"author\":\"bob\",\"permlink\":\"p\"}"; + + for (var i = 0; i < 3; i++) + { + var ctx = Post("/private-api/curation-desk/mark-clear", body); + await PrivateApi.CurationDeskMarkClear(ctx); + Assert.Equal(401, ctx.Response.StatusCode); + } + Assert.Equal(3, validations); + Assert.Empty(upstream.Calls); + + // Once the code validates it is remembered from that point, not before. + answer = "alice"; + await PrivateApi.CurationDeskMarkClear(Post("/private-api/curation-desk/mark-clear", body)); + await PrivateApi.CurationDeskMarkClear(Post("/private-api/curation-desk/mark-clear", body)); + Assert.Equal(4, validations); + Assert.Equal(2, upstream.Calls.Count); + } + + [Fact] + public async Task AMemoizedIdentityIsStillTheValidatedOneNotTheBodysUsername() + { + var upstream = Install(); + var body = "{\"code\":\"as:alice\",\"username\":\"victim\",\"author\":\"bob\",\"permlink\":\"p\"}"; + await PrivateApi.CurationDeskMarkClear(Post("/private-api/curation-desk/mark-clear", body)); + await PrivateApi.CurationDeskMarkClear(Post("/private-api/curation-desk/mark-clear", body)); + Assert.All(upstream.Calls, c => Assert.Equal("alice", c.Payload!["username"]!.GetValue())); + } + + // ---- client address ------------------------------------------------------ + + [Fact] + public async Task OnlyRecommendMetaForwardsTheProxySetClientAddress() + { + var upstream = Install(); + + foreach (var (name, handler, body) in SignedWrites()) + { + var ctx = Post("/private-api/curation-desk/" + name, body); + ctx.Request.Headers["X-Real-IP"] = "198.51.100.7"; + ctx.Request.Headers["X-Forwarded-For"] = "203.0.113.9, 198.51.100.7"; + await handler(ctx); + var call = Assert.Single(upstream.Calls); + Assert.Equal(name == "recommend-meta" ? "198.51.100.7" : null, call.Header("X-Real-IP-V")); + upstream.Calls.Clear(); + } + + // No proxy header: an empty value, never the forwarded-for chain. + var bare = Post("/private-api/curation-desk/recommend-meta", "{\"code\":\"as:alice\",\"author\":\"bob\",\"permlink\":\"p\"}"); + bare.Request.Headers["X-Forwarded-For"] = "203.0.113.9"; + await PrivateApi.CurationDeskRecommendMeta(bare); + Assert.Equal("", Assert.Single(upstream.Calls).Header("X-Real-IP-V")); + } + + [Fact] + public async Task RecommendMetaAcceptsABodyWithoutATrxId() + { + var upstream = Install(); + upstream.Answer = _ => Task.FromResult(JsonResponse(202, "{\"ok\":true}")); + var ctx = Post("/private-api/curation-desk/recommend-meta", "{\"code\":\"as:alice\",\"author\":\"bob\",\"permlink\":\"p\"}"); + await PrivateApi.CurationDeskRecommendMeta(ctx); + Assert.Equal(202, ctx.Response.StatusCode); + Assert.Equal("{\"ok\":true}", Body(ctx)); + Assert.Equal("curation/desk/recommendations/meta", Assert.Single(upstream.Calls).Endpoint); + } + + // ---- Cache-Control ------------------------------------------------------- + + [Fact] + public async Task ReadsCarryTheirPolicyOnlyOnA200() + { + var upstream = Install(); + + foreach (var (name, handler, request, policy) in PublicReads()) + { + var ok = request(); + await handler(ok); + await Start(ok); + Assert.Equal(200, ok.Response.StatusCode); + Assert.Equal(policy, CacheControl(ok)); + // Filled by this request, so the whole window and no age spent yet. + Assert.Null(Age(ok)); + Assert.StartsWith("application/json", ok.Response.ContentType); + + CurationDeskMemo.ResetForTests(); + upstream.Answer = _ => Task.FromResult(JsonResponse(404, "{\"error\":\"not found\"}")); + var missing = request(); + await handler(missing); + await Start(missing); + Assert.Equal(404, missing.Response.StatusCode); + Assert.Null(CacheControl(missing)); + Assert.Null(Age(missing)); + Assert.Equal("{\"error\":\"not found\"}", Body(missing)); + + upstream.Answer = _ => throw new UpstreamTimeoutException("u", new TimeoutException()); + CurationDeskMemo.ResetForTests(); + var timeout = request(); + await handler(timeout); + await Start(timeout); + Assert.Equal(504, timeout.Response.StatusCode); + Assert.Equal("Upstream Timeout", Body(timeout)); + Assert.Null(CacheControl(timeout)); + Assert.Null(Age(timeout)); + + upstream.Answer = _ => Task.FromResult(JsonResponse(200, "{}")); + CurationDeskMemo.ResetForTests(); + } + } + + [Fact] + public async Task WritesAreNeverCacheable() + { + Install(); + foreach (var (name, handler, body) in SignedWrites()) + { + var ctx = Post("/private-api/curation-desk/" + name, body); + await handler(ctx); + await Start(ctx); + Assert.Equal(200, ctx.Response.StatusCode); + Assert.Equal("no-store", CacheControl(ctx)); + } + } + + [Fact] + public async Task AMemoHitAdvertisesOnlyWhatIsLeftOfTheSharedWindow() + { + var upstream = Install(); + var clock = UseTestClock(); + const string body = "{\"curators\":[{\"username\":\"alice\"}]}"; + upstream.Answer = _ => Task.FromResult(JsonResponse(200, body)); + + var fill = Get("/private-api/curation-desk/roster"); + await PrivateApi.CurationDeskRoster(fill); + await Start(fill); + Assert.Equal(CachePolicy.CurationDeskRoster, CacheControl(fill)); + Assert.Null(Age(fill)); + + // 590 s into the roster's 600 s window. Sending the whole window again + // here would let a shared cache hold this body for another 600 s on top + // of the 590 it has already lived in the memo. + clock.Advance(TimeSpan.FromSeconds(590)); + var hit = Get("/private-api/curation-desk/roster"); + await PrivateApi.CurationDeskRoster(hit); + await Start(hit); + Assert.Equal(200, hit.Response.StatusCode); + Assert.Equal(body, Body(hit)); + Assert.Equal("public, max-age=0, s-maxage=10", CacheControl(hit)); + Assert.Null(Age(hit)); + + // Both readers were answered from one upstream call. + Assert.Single(upstream.Calls); + } + + [Fact] + public async Task AMemoHitAtTheEndOfItsWindowStaysCacheableForOneSecond() + { + var upstream = Install(); + var clock = UseTestClock(); + upstream.Answer = _ => Task.FromResult(JsonResponse(200, "{\"behind_seconds\":3}")); + await PrivateApi.CurationDeskStatus(Get("/private-api/curation-desk/status")); + + // Past the 15 s the status policy promises. The memo entry is about to + // lapse and refill, so the floor keeps this answer cacheable rather than + // sending s-maxage=0 to every reader in the last moment of a window. + clock.Advance(TimeSpan.FromSeconds(20)); + var hit = Get("/private-api/curation-desk/status"); + await PrivateApi.CurationDeskStatus(hit); + await Start(hit); + Assert.Equal("public, max-age=0, s-maxage=1", CacheControl(hit)); + Assert.Null(Age(hit)); + Assert.Single(upstream.Calls); + } + + [Fact] + public async Task TheLastGoodBodyCarriesTheShortWindowAndNoAgeHeader() + { + var upstream = Install(); + var clock = UseTestClock(); + const string body = "{\"curators\":[{\"username\":\"alice\"}]}"; + upstream.Answer = _ => Task.FromResult(JsonResponse(200, body)); + await PrivateApi.CurationDeskRoster(Get("/private-api/curation-desk/roster")); + + // The fresh entry lapses (simulated) and the backend stops answering, so + // the next read falls back to the last good body, now minutes old. + CurationDeskMemo.Fresh = new BytesCache(CurationDeskMemo.BudgetBytes); + clock.Advance(TimeSpan.FromSeconds(120)); + upstream.Answer = _ => throw new UpstreamTimeoutException("u", new TimeoutException()); + + var stale = Get("/private-api/curation-desk/roster"); + await PrivateApi.CurationDeskRoster(stale); + await Start(stale); + Assert.Equal(200, stale.Response.StatusCode); + Assert.Equal(body, Body(stale)); + + // Never the route's own window: a backend that recovers has to reach + // readers within a poll or two, not ten minutes later. + Assert.Equal("public, max-age=0, s-maxage=5", CacheControl(stale)); + Assert.Equal(CachePolicy.Stale(CachePolicy.CurationDeskRoster), CacheControl(stale)); + Assert.Null(Age(stale)); + } + + [Fact] + public void EachPolicySharedMaxAgeIsTheMemoTtl() + { + Assert.Equal(30, CachePolicy.SharedMaxAge(CachePolicy.CurationDeskFeed)); + Assert.Equal(15, CachePolicy.SharedMaxAge(CachePolicy.CurationDeskStatus)); + Assert.Equal(600, CachePolicy.SharedMaxAge(CachePolicy.CurationDeskRoster)); + Assert.Equal(30, CachePolicy.SharedMaxAge(CachePolicy.CurationDeskRecommendations)); + Assert.Equal(15, CachePolicy.SharedMaxAge(CachePolicy.CurationDeskPost)); + } + + // ---- the byte memo ------------------------------------------------------- + + [Fact] + public async Task ASecondReadWithinTheTtlIsServedFromTheMemoAsBytes() + { + var upstream = Install(); + const string body = "{\"items\":[{\"post_id\":1}],\"feed_version\":\"v1\"}"; + upstream.Answer = _ => Task.FromResult(JsonResponse(200, body)); + + var first = Get("/private-api/curation-desk/feed", "limit=10&sort=queue"); + await PrivateApi.CurationDeskFeed(first); + var second = Get("/private-api/curation-desk/feed", "sort=queue&limit=10&x=1"); + await PrivateApi.CurationDeskFeed(second); + + Assert.Single(upstream.Calls); + Assert.Equal("curation/desk/feed?limit=10&sort=queue", upstream.Calls[0].Endpoint); + Assert.Equal(body, Body(first)); + Assert.Equal(body, Body(second)); + + // Stored as the bytes that were served, keyed by the normalized endpoint. + Assert.True(CurationDeskMemo.TryGetFresh("curation/desk/feed?limit=10&sort=queue", + out var stored, out var storedType, out var storedAge)); + Assert.IsType(stored); + Assert.Equal(body, System.Text.Encoding.UTF8.GetString(stored)); + Assert.Equal("application/json; charset=utf-8", storedType); + Assert.Equal(0, storedAge); + Assert.False(CurationDeskMemo.Fresh.TryGet("curation/desk/feed", out _)); + } + + [Fact] + public async Task DifferentQuestionsAreDifferentMemoEntries() + { + var upstream = Install(); + await PrivateApi.CurationDeskFeed(Get("/private-api/curation-desk/feed", "sort=queue")); + await PrivateApi.CurationDeskFeed(Get("/private-api/curation-desk/feed", "sort=unique")); + await PrivateApi.CurationDeskFeed(Get("/private-api/curation-desk/feed", "window=full")); + Assert.Equal(3, upstream.Calls.Count); + Assert.Equal(3, CurationDeskMemo.Fresh.Count); + } + + [Fact] + public async Task ConcurrentReadsOfOneKeyMakeOneUpstreamCall() + { + var upstream = Install(); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + upstream.Answer = _ => release.Task; + + var requests = Enumerable.Range(0, 8).Select(_ => Get("/private-api/curation-desk/status")).ToArray(); + var pending = requests.Select(r => PrivateApi.CurationDeskStatus(r)).ToArray(); + + // Give every request time to reach the gate before the fill completes. + await Task.Delay(100); + Assert.Single(upstream.Calls); + release.SetResult(JsonResponse(200, "{\"behind_seconds\":3}")); + await Task.WhenAll(pending); + + Assert.Single(upstream.Calls); + Assert.All(requests, r => + { + Assert.Equal(200, r.Response.StatusCode); + Assert.Equal("{\"behind_seconds\":3}", Body(r)); + }); + } + + [Fact] + public async Task AKeysGateIsKeptWhileAnyReaderStillHoldsIt() + { + Install(); + const string key = "curation/desk/feed?limit=10"; + + // Two readers take the key's gate. The second stands for a request that + // has been handed the gate and has not reached its wait yet. + var first = CurationDeskMemo.GateFor(key); + var late = CurationDeskMemo.GateFor(key); + Assert.Same(first, late); + + // The first reader fills and hands the gate back. + Assert.True(await first.Semaphore.WaitAsync(TimeSpan.Zero)); + first.Semaphore.Release(); + CurationDeskMemo.ReleaseGate(key, first); + + // A third reader arrives after that. It must land on the gate the late + // reader is about to wait on, or the two of them fill one key at once + // and the fill that finishes second stores its answer over the first. + var newcomer = CurationDeskMemo.GateFor(key); + Assert.Same(late, newcomer); + Assert.True(await newcomer.Semaphore.WaitAsync(TimeSpan.Zero)); + Assert.False(await late.Semaphore.WaitAsync(TimeSpan.Zero)); + + newcomer.Semaphore.Release(); + CurationDeskMemo.ReleaseGate(key, newcomer); + Assert.Equal(1, CurationDeskMemo.GateCount); + + // With the last reader gone the entry is dropped, so a scan over many + // distinct keys leaves no semaphore per key behind. + CurationDeskMemo.ReleaseGate(key, late); + Assert.Equal(0, CurationDeskMemo.GateCount); + var afterwards = CurationDeskMemo.GateFor(key); + Assert.NotSame(late, afterwards); + CurationDeskMemo.ReleaseGate(key, afterwards); + Assert.Equal(0, CurationDeskMemo.GateCount); + } + + [Fact] + public async Task ALateWaiterNeverFillsAKeyBesideTheReaderFillingIt() + { + var upstream = Install(); + const string key = "curation/desk/status"; + + // A reader that has been handed the key's gate and has not waited on it + // yet: the fill below must not be able to drop the gate under it. + var late = CurationDeskMemo.GateFor(key); + + var firstFill = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + upstream.Answer = _ => firstFill.Task; + var first = Get("/private-api/curation-desk/status"); + var firstRequest = PrivateApi.CurationDeskStatus(first); + await Task.Delay(100); + Assert.Single(upstream.Calls); + firstFill.SetResult(JsonResponse(200, "{\"behind_seconds\":1}")); + await firstRequest.WaitAsync(TimeSpan.FromSeconds(3)); + Assert.Equal("{\"behind_seconds\":1}", Body(first)); + + // That entry lapses (simulated), so the next reader fills again. + CurationDeskMemo.Fresh = new BytesCache(CurationDeskMemo.BudgetBytes); + + var secondFill = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + upstream.Answer = _ => secondFill.Task; + var second = Get("/private-api/curation-desk/status"); + var secondRequest = PrivateApi.CurationDeskStatus(second); + await Task.Delay(100); + Assert.Equal(2, upstream.Calls.Count); + + // The late reader reaches its wait now and must queue behind the fill in + // flight instead of being admitted beside it. + Assert.False(await late.Semaphore.WaitAsync(TimeSpan.Zero)); + + secondFill.SetResult(JsonResponse(200, "{\"behind_seconds\":2}")); + await secondRequest.WaitAsync(TimeSpan.FromSeconds(3)); + Assert.Equal("{\"behind_seconds\":2}", Body(second)); + + // Once admitted it finds the answer memoized, so the key was filled once + // per reader that needed it and never twice at a time. + Assert.True(await late.Semaphore.WaitAsync(TimeSpan.FromSeconds(3))); + Assert.True(CurationDeskMemo.TryGetFresh(key, out var memoized, out _, out _)); + Assert.Equal("{\"behind_seconds\":2}", System.Text.Encoding.UTF8.GetString(memoized)); + late.Semaphore.Release(); + CurationDeskMemo.ReleaseGate(key, late); + Assert.Equal(2, upstream.Calls.Count); + Assert.Equal(0, CurationDeskMemo.GateCount); + } + + [Fact] + public async Task ASlowReaderDoesNotHoldTheGateOfItsKey() + { + var upstream = Install(); + var fill = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + upstream.Answer = _ => fill.Task; + + // A reader whose socket never drains takes the gate and starts the fill. + var slowBody = new BlockingBody(); + var slow = Get("/private-api/curation-desk/status"); + slow.Response.Body = slowBody; + var slowRequest = PrivateApi.CurationDeskStatus(slow); + await Task.Delay(100); + Assert.Single(upstream.Calls); + + // A second reader of the same key arrives during the fill, so it is + // queued on the gate rather than served from the memo. + var fast = Get("/private-api/curation-desk/status"); + var fastRequest = PrivateApi.CurationDeskStatus(fast); + await Task.Delay(100); + Assert.False(fastRequest.IsCompleted); + + fill.SetResult(JsonResponse(200, "{\"behind_seconds\":3}")); + await slowBody.WriteReached; + + // The fill is done and the slow reader is stuck in its write. The bound + // is far under CurationDeskMemo.FillWait on purpose: waiting for the + // gate to time out would also "answer", just seconds later. + await fastRequest.WaitAsync(TimeSpan.FromSeconds(3)); + Assert.Equal(200, fast.Response.StatusCode); + Assert.Equal("{\"behind_seconds\":3}", Body(fast)); + Assert.False(slowRequest.IsCompleted); + + slowBody.Release(); + await slowRequest.WaitAsync(TimeSpan.FromSeconds(3)); + Assert.Equal("{\"behind_seconds\":3}", slowBody.Text); + Assert.Single(upstream.Calls); + } + + [Fact] + public async Task A200ThatIsNotAJsonBodyIsNeitherCachedNorMemoized() + { + var upstream = Install(); + upstream.Answer = _ => Task.FromResult(TextResponse(200, "gateway login")); + + var page = Get("/private-api/curation-desk/status"); + await PrivateApi.CurationDeskStatus(page); + await Start(page); + Assert.Equal(200, page.Response.StatusCode); + Assert.Equal("gateway login", Body(page)); + Assert.StartsWith("text/html", page.Response.ContentType); + Assert.Null(CacheControl(page)); + Assert.Equal(0, CurationDeskMemo.Fresh.Count); + Assert.Equal(0, CurationDeskMemo.LastGood.Count); + + // A JSON body that is not an object or an array is the same case. + upstream.Answer = _ => Task.FromResult(JsonResponse(200, "\"maintenance\"")); + var scalar = Get("/private-api/curation-desk/status"); + await PrivateApi.CurationDeskStatus(scalar); + await Start(scalar); + Assert.Equal(200, scalar.Response.StatusCode); + Assert.Null(CacheControl(scalar)); + Assert.Equal(0, CurationDeskMemo.Fresh.Count); + + // And an answer the desk did give is preferred over that page, with the + // route's policy on it because it is a body this service holds. + CurationDeskMemo.ResetForTests(); + upstream.Answer = _ => Task.FromResult(JsonResponse(200, "{\"behind_seconds\":3}")); + await PrivateApi.CurationDeskStatus(Get("/private-api/curation-desk/status")); + CurationDeskMemo.Fresh = new BytesCache(CurationDeskMemo.BudgetBytes); + + upstream.Answer = _ => Task.FromResult(TextResponse(200, "gateway login")); + var stale = Get("/private-api/curation-desk/status"); + await PrivateApi.CurationDeskStatus(stale); + await Start(stale); + Assert.Equal(200, stale.Response.StatusCode); + Assert.Equal("{\"behind_seconds\":3}", Body(stale)); + Assert.StartsWith("application/json", stale.Response.ContentType); + + // A last-good body, so the short window rather than the route's own. + Assert.Equal(CachePolicy.Stale(CachePolicy.CurationDeskStatus), CacheControl(stale)); + } + + [Fact] + public async Task AJsonErrorBodyPassesThroughTheSameFenceAsAServedOne() + { + var upstream = Install(); + upstream.Answer = _ => Task.FromResult(JsonResponse(404, + "{\"error\":\"unknown post\",\"excluded_reason\":\"abuser\",\"detail\":{\"set_by\":\"alice\"}}")); + + var ctx = Get("/private-api/curation-desk/post/good-karma/nope", "", + new[] { ("author", "good-karma"), ("permlink", "nope") }); + await PrivateApi.CurationDeskPost(ctx); + await Start(ctx); + Assert.Equal(404, ctx.Response.StatusCode); + Assert.Null(CacheControl(ctx)); + var body = Body(ctx); + Assert.Contains("\"error\":\"unknown post\"", body); + Assert.DoesNotContain("excluded_reason", body); + Assert.DoesNotContain("set_by", body); + } + + [Fact] + public async Task AnUpstreamErrorAnswersWithTheLastGoodBody() + { + var upstream = Install(); + upstream.Answer = _ => Task.FromResult(JsonResponse(200, "{\"curators\":[{\"username\":\"alice\"}]}")); + await PrivateApi.CurationDeskRoster(Get("/private-api/curation-desk/roster")); + + // The fresh entry lapses (simulated), the last-good one is still there. + CurationDeskMemo.Fresh = new BytesCache(CurationDeskMemo.BudgetBytes); + + upstream.Answer = _ => throw new UpstreamTimeoutException("u", new TimeoutException()); + var stale = Get("/private-api/curation-desk/roster"); + await PrivateApi.CurationDeskRoster(stale); + Assert.Equal(200, stale.Response.StatusCode); + Assert.Equal("{\"curators\":[{\"username\":\"alice\"}]}", Body(stale)); + + upstream.Answer = _ => Task.FromResult(TextResponse(502, "bad gateway")); + var down = Get("/private-api/curation-desk/roster"); + await PrivateApi.CurationDeskRoster(down); + Assert.Equal(200, down.Response.StatusCode); + Assert.Equal("{\"curators\":[{\"username\":\"alice\"}]}", Body(down)); + + // With nothing good to fall back on, the backend's answer passes through. + CurationDeskMemo.ResetForTests(); + var bare = Get("/private-api/curation-desk/roster"); + await PrivateApi.CurationDeskRoster(bare); + Assert.Equal(502, bare.Response.StatusCode); + Assert.Equal("bad gateway", Body(bare)); + + // Errors are never memoized: the next read tries upstream again. + upstream.Answer = _ => Task.FromResult(JsonResponse(200, "{\"curators\":[]}")); + var recovered = Get("/private-api/curation-desk/roster"); + await PrivateApi.CurationDeskRoster(recovered); + Assert.Equal(200, recovered.Response.StatusCode); + Assert.Equal("{\"curators\":[]}", Body(recovered)); + } + + [Fact] + public async Task ANon200IsPipedThroughAndNotMemoized() + { + var upstream = Install(); + upstream.Answer = _ => Task.FromResult(JsonResponse(404, "{\"error\":\"unknown post\"}")); + var ctx = Get("/private-api/curation-desk/post/good-karma/nope", "", new[] { ("author", "good-karma"), ("permlink", "nope") }); + await PrivateApi.CurationDeskPost(ctx); + Assert.Equal(404, ctx.Response.StatusCode); + Assert.Equal("{\"error\":\"unknown post\"}", Body(ctx)); + Assert.Equal(0, CurationDeskMemo.Fresh.Count); + Assert.Equal(0, CurationDeskMemo.LastGood.Count); + Assert.Equal("curation/desk/post/good-karma/nope", Assert.Single(upstream.Calls).Endpoint); + } + + [Fact] + public async Task AnInvalidPostPathIs400BeforeAnyUpstreamCall() + { + var upstream = Install(); + foreach (var (author, permlink) in MalformedPostPaths) + { + var ctx = Get("/private-api/curation-desk/post/x/y", "", new[] { ("author", author), ("permlink", permlink) }); + await PrivateApi.CurationDeskPost(ctx); + await Start(ctx); + Assert.Equal(400, ctx.Response.StatusCode); + Assert.Equal("Invalid author or permlink", Body(ctx)); + Assert.Null(CacheControl(ctx)); + } + Assert.Empty(upstream.Calls); + } + + [Fact] + public async Task WithoutTheTokenAMalformedPostPathIs503LikeEveryOtherRoute() + { + var upstream = Install(token: null); + + // While the desk is dark every route answers the same, so this one does + // not single itself out by reporting on the path it was given. + foreach (var (author, permlink) in MalformedPostPaths) + { + var ctx = Get("/private-api/curation-desk/post/x/y", "", new[] { ("author", author), ("permlink", permlink) }); + await PrivateApi.CurationDeskPost(ctx); + await Start(ctx); + Assert.Equal(503, ctx.Response.StatusCode); + Assert.Equal("curation desk not configured", Body(ctx)); + Assert.Null(CacheControl(ctx)); + } + Assert.Empty(upstream.Calls); + } + + private static readonly (string Author, string Permlink)[] MalformedPostPaths = + { + ("..", "p"), ("good-karma", "a/b"), ("good-karma", "p?x=1"), ("x", "p"), + }; +} diff --git a/dotnet/EcencyApi.Tests/CurationDeskPayloadTests.cs b/dotnet/EcencyApi.Tests/CurationDeskPayloadTests.cs new file mode 100644 index 00000000..7c718d12 --- /dev/null +++ b/dotnet/EcencyApi.Tests/CurationDeskPayloadTests.cs @@ -0,0 +1,363 @@ +using System.Text.Json.Nodes; +using EcencyApi.Handlers; +using Xunit; + +namespace EcencyApi.Tests; + +/// +/// The desk writes forward a body built here, never the client's. What matters +/// is who the backend believes is acting (the validated username, always) and +/// that a value the backend would only reject is refused before it travels. +/// +public class CurationDeskPayloadTests +{ + private static readonly CurationDeskWrites.Route[] AllRoutes = + { + CurationDeskWrites.RosterFeed, CurationDeskWrites.Tick, CurationDeskWrites.Mark, + CurationDeskWrites.MarkClear, CurationDeskWrites.Marks, CurationDeskWrites.Cursor, + CurationDeskWrites.RecommendMeta, CurationDeskWrites.RecommendationDismiss, + }; + + private static JsonObject Body(string json) => (JsonObject)JsonNode.Parse(json)!; + + private static JsonObject Ok(CurationDeskWrites.Route route, string json) + { + var (payload, error) = CurationDeskWrites.Build(route, "alice", Body(json)); + Assert.Null(error); + Assert.NotNull(payload); + return payload!; + } + + private static string Rejected(CurationDeskWrites.Route route, string json) + { + var (payload, error) = CurationDeskWrites.Build(route, "alice", Body(json)); + Assert.Null(payload); + Assert.NotNull(error); + return error!; + } + + /// A body that passes each route's validation, with a forged identity attached. + private static string ValidBodyFor(CurationDeskWrites.Route route) + { + const string forged = "\"username\":\"victim\",\"code\":\"as:victim\","; + if (ReferenceEquals(route, CurationDeskWrites.Mark)) + return "{" + forged + "\"author\":\"bob\",\"permlink\":\"p\",\"state\":\"flagged\",\"reason\":\"farming\"}"; + if (ReferenceEquals(route, CurationDeskWrites.MarkClear) || ReferenceEquals(route, CurationDeskWrites.RecommendMeta)) + return "{" + forged + "\"author\":\"bob\",\"permlink\":\"p\"}"; + if (ReferenceEquals(route, CurationDeskWrites.Cursor)) + return "{" + forged + "\"post_id\":7,\"action\":\"advance\"}"; + if (ReferenceEquals(route, CurationDeskWrites.RecommendationDismiss)) + return "{" + forged + "\"author\":\"bob\",\"permlink\":\"p\",\"action\":\"restore\"}"; + return "{" + forged + "\"limit\":5}"; + } + + [Fact] + public void TheValidatedUsernameIsTheOnlyIdentityForwarded() + { + foreach (var route in AllRoutes) + { + var payload = Ok(route, ValidBodyFor(route)); + Assert.Equal("alice", payload["username"]!.GetValue()); + Assert.False(payload.ContainsKey("code"), route.UpstreamPath); + + // And no whitelist can ever be widened to include them. + Assert.DoesNotContain("username", route.Keys); + Assert.DoesNotContain("code", route.Keys); + } + } + + [Fact] + public void OnlyWhitelistedKeysTravel() + { + var payload = Ok(CurationDeskWrites.Mark, + "{\"author\":\"bob\",\"permlink\":\"p\",\"state\":\"noted\",\"note\":\"hi\",\"admin\":true,\"weight\":10000}"); + Assert.Equal(new[] { "username", "author", "permlink", "state", "note" }, payload.Select(kv => kv.Key).ToArray()); + + var tick = Ok(CurationDeskWrites.Tick, "{\"since\":\"t\",\"need\":[1],\"visible\":[2],\"curator\":\"x\"}"); + Assert.Equal(new[] { "username", "since", "need", "visible" }, tick.Select(kv => kv.Key).ToArray()); + } + + [Theory] + [InlineData("{\"permlink\":\"p\",\"state\":\"reviewed\"}", "author required")] + [InlineData("{\"author\":\"\",\"permlink\":\"p\",\"state\":\"reviewed\"}", "author required")] + [InlineData("{\"author\":\"bob\",\"state\":\"reviewed\"}", "permlink required")] + [InlineData("{\"author\":\"bob\",\"permlink\":\"\",\"state\":\"reviewed\"}", "permlink required")] + [InlineData("{\"author\":7,\"permlink\":\"p\",\"state\":\"reviewed\"}", "author required")] + [InlineData("{\"author\":null,\"permlink\":\"p\",\"state\":\"reviewed\"}", "author required")] + public void MarkRequiresANonEmptyAuthorAndPermlink(string body, string error) + { + Assert.Equal(error, Rejected(CurationDeskWrites.Mark, body)); + } + + [Fact] + public void EveryPostAddressedRouteRequiresAuthorAndPermlink() + { + Assert.Equal("author required", Rejected(CurationDeskWrites.MarkClear, "{\"permlink\":\"p\"}")); + Assert.Equal("permlink required", Rejected(CurationDeskWrites.MarkClear, "{\"author\":\"bob\",\"permlink\":\"\"}")); + Assert.Equal("author required", Rejected(CurationDeskWrites.RecommendMeta, "{\"permlink\":\"p\"}")); + Assert.Equal("permlink required", + Rejected(CurationDeskWrites.RecommendationDismiss, "{\"author\":\"bob\",\"action\":\"dismiss\"}")); + } + + [Theory] + [InlineData("reviewed")] + [InlineData("snoozed")] + [InlineData("flagged")] + [InlineData("noted")] + public void MarkAcceptsEachKnownState(string state) + { + var payload = Ok(CurationDeskWrites.Mark, $"{{\"author\":\"bob\",\"permlink\":\"p\",\"state\":\"{state}\"}}"); + Assert.Equal(state, payload["state"]!.GetValue()); + } + + [Theory] + [InlineData("{\"author\":\"bob\",\"permlink\":\"p\"}")] + [InlineData("{\"author\":\"bob\",\"permlink\":\"p\",\"state\":\"deleted\"}")] + [InlineData("{\"author\":\"bob\",\"permlink\":\"p\",\"state\":\"Reviewed\"}")] + [InlineData("{\"author\":\"bob\",\"permlink\":\"p\",\"state\":1}")] + public void MarkRefusesAnUnknownState(string body) + { + Assert.Equal("invalid state", Rejected(CurationDeskWrites.Mark, body)); + } + + [Fact] + public void MarksListStateIsOptionalButMustBeKnownWhenGiven() + { + Assert.Equal(new[] { "username" }, Ok(CurationDeskWrites.Marks, "{}").Select(kv => kv.Key).ToArray()); + Assert.Equal("snoozed", Ok(CurationDeskWrites.Marks, "{\"state\":\"snoozed\",\"limit\":10}")["state"]!.GetValue()); + Assert.Equal("invalid state", Rejected(CurationDeskWrites.Marks, "{\"state\":\"all\"}")); + } + + [Theory] + [InlineData("advance")] + [InlineData("rewind")] + public void CursorAcceptsEachKnownAction(string action) + { + var payload = Ok(CurationDeskWrites.Cursor, $"{{\"post_id\":42,\"action\":\"{action}\",\"reason\":\"oops\"}}"); + Assert.Equal(action, payload["action"]!.GetValue()); + Assert.Equal(42, payload["post_id"]!.GetValue()); + Assert.Equal("oops", payload["reason"]!.GetValue()); + } + + [Fact] + public void CursorRefusesAnUnknownActionOrAMissingPostId() + { + Assert.Equal("invalid action", Rejected(CurationDeskWrites.Cursor, "{\"post_id\":42,\"action\":\"jump\"}")); + Assert.Equal("invalid action", Rejected(CurationDeskWrites.Cursor, "{\"post_id\":42}")); + Assert.Equal("post_id required", Rejected(CurationDeskWrites.Cursor, "{\"action\":\"advance\"}")); + Assert.Equal("post_id required", Rejected(CurationDeskWrites.Cursor, "{\"post_id\":null,\"action\":\"advance\"}")); + Assert.Equal("post_id required", Rejected(CurationDeskWrites.Cursor, "{\"post_id\":{\"id\":1},\"action\":\"advance\"}")); + } + + [Fact] + public void DismissAcceptsDismissAndRestoreOnly() + { + Ok(CurationDeskWrites.RecommendationDismiss, "{\"author\":\"bob\",\"permlink\":\"p\",\"action\":\"dismiss\"}"); + Ok(CurationDeskWrites.RecommendationDismiss, "{\"author\":\"bob\",\"permlink\":\"p\",\"action\":\"restore\"}"); + Assert.Equal("invalid action", + Rejected(CurationDeskWrites.RecommendationDismiss, "{\"author\":\"bob\",\"permlink\":\"p\",\"action\":\"delete\"}")); + Assert.Equal("invalid action", + Rejected(CurationDeskWrites.RecommendationDismiss, "{\"author\":\"bob\",\"permlink\":\"p\"}")); + } + + [Fact] + public void RecommendMetaTrxIdIsOptionalAndStrictWhenPresent() + { + var without = Ok(CurationDeskWrites.RecommendMeta, "{\"author\":\"bob\",\"permlink\":\"p\"}"); + Assert.False(without.ContainsKey("trx_id")); + + var trx = new string('a', 40); + var with = Ok(CurationDeskWrites.RecommendMeta, $"{{\"author\":\"bob\",\"permlink\":\"p\",\"trx_id\":\"{trx}\"}}"); + Assert.Equal(trx, with["trx_id"]!.GetValue()); + + foreach (var bad in new[] { "\"abc\"", "\"" + new string('A', 40) + "\"", "\"" + new string('a', 39) + "\"", "null", "42" }) + { + Assert.Equal("invalid trx_id", + Rejected(CurationDeskWrites.RecommendMeta, $"{{\"author\":\"bob\",\"permlink\":\"p\",\"trx_id\":{bad}}}")); + } + } + + [Fact] + public void RecommendMetaForwardsOnlyAKnownUaClass() + { + Assert.Equal("mobile", + Ok(CurationDeskWrites.RecommendMeta, "{\"author\":\"bob\",\"permlink\":\"p\",\"ua_class\":\"mobile\"}")["ua_class"]!.GetValue()); + Assert.False( + Ok(CurationDeskWrites.RecommendMeta, "{\"author\":\"bob\",\"permlink\":\"p\",\"ua_class\":\"bot\"}").ContainsKey("ua_class")); + Assert.True(CurationDeskWrites.RecommendMeta.ForwardClientAddress); + Assert.All(AllRoutes.Where(r => !ReferenceEquals(r, CurationDeskWrites.RecommendMeta)), + r => Assert.False(r.ForwardClientAddress, r.UpstreamPath)); + } + + [Fact] + public void RosterFeedKeepsSeedOnlyForTheRandomOrderAndClampsLimit() + { + var random = Ok(CurationDeskWrites.RosterFeed, "{\"sort\":\"random\",\"seed\":\"abcd1234\",\"limit\":500}"); + Assert.Equal("abcd1234", random["seed"]!.GetValue()); + Assert.Equal(50, random["limit"]!.GetValue()); + + var queue = Ok(CurationDeskWrites.RosterFeed, "{\"sort\":\"queue\",\"seed\":\"abcd1234\",\"limit\":0}"); + Assert.False(queue.ContainsKey("seed")); + Assert.Equal(1, queue["limit"]!.GetValue()); + + var unknown = Ok(CurationDeskWrites.RosterFeed, "{\"sort\":\"payout\",\"seed\":\"abcd1234\",\"view\":\"excluded\",\"hide_reviewed\":false}"); + Assert.False(unknown.ContainsKey("sort")); + Assert.False(unknown.ContainsKey("seed")); + // The roster is the only feed that lists excluded rows, so its view + // allowlist is the public one plus that. + Assert.Equal("excluded", unknown["view"]!.GetValue()); + Assert.False(unknown["hide_reviewed"]!.GetValue()); + } + + [Fact] + public void RosterFeedTakesOnlyASeedTheBackendCanHashWith() + { + foreach (var seed in new[] { "\"abc\"", "\"" + new string('a', 17) + "\"", "\"ABCD1234\"", "\"abcd 1234\"", "\"abcd1234\\n\"", "42", "null" }) + { + var payload = Ok(CurationDeskWrites.RosterFeed, "{\"sort\":\"random\",\"seed\":" + seed + "}"); + Assert.False(payload.ContainsKey("seed"), seed); + } + Assert.Equal(new string('a', 16), + Ok(CurationDeskWrites.RosterFeed, "{\"sort\":\"random\",\"seed\":\"" + new string('a', 16) + "\"}")["seed"]!.GetValue()); + } + + [Fact] + public void RosterFeedFiltersFollowThePublicFeedsValueRules() + { + // Allowlists: a value the public feed drops is dropped here too, so the + // two feeds answer the same question for the same request. + var enums = Ok(CurationDeskWrites.RosterFeed, + "{\"view\":\"secret\",\"app\":\"hive\",\"window\":\"week\",\"community\":\"photography\",\"cursor\":\"a b\"}"); + Assert.Equal(new[] { "username" }, enums.Select(kv => kv.Key).ToArray()); + + var kept = Ok(CurationDeskWrites.RosterFeed, + "{\"view\":\"queue\",\"app\":\"ecency\",\"window\":\"full\",\"community\":\"hive-125125\",\"cursor\":\"s:abc.1:25\"}"); + Assert.Equal("queue", kept["view"]!.GetValue()); + Assert.Equal("ecency", kept["app"]!.GetValue()); + Assert.Equal("full", kept["window"]!.GetValue()); + Assert.Equal("hive-125125", kept["community"]!.GetValue()); + Assert.Equal("s:abc.1:25", kept["cursor"]!.GetValue()); + + // Trailing newlines and non-ASCII digits are not the value either. + var newline = Ok(CurationDeskWrites.RosterFeed, "{\"community\":\"hive-125125\\n\",\"cursor\":\"abc\\n\",\"view\":\"queue\\n\"}"); + Assert.Equal(new[] { "username" }, newline.Select(kv => kv.Key).ToArray()); + Assert.False(Ok(CurationDeskWrites.RosterFeed, "{\"community\":\"hive-\\u0661\\u0662\\u0663\\u0664\\u0665\"}").ContainsKey("community")); + + // Ranges clamp instead of travelling as sent, from a number or its + // string spelling; something that is neither is dropped. + var ranges = Ok(CurationDeskWrites.RosterFeed, + "{\"rep_min\":150,\"rep_max\":-1,\"min_words\":999999,\"max_words\":\"300\",\"limit\":\"99999999999\"}"); + Assert.Equal(100, ranges["rep_min"]!.GetValue()); + Assert.Equal(0, ranges["rep_max"]!.GetValue()); + Assert.Equal(50000, ranges["min_words"]!.GetValue()); + Assert.Equal(300, ranges["max_words"]!.GetValue()); + Assert.Equal(50, ranges["limit"]!.GetValue()); + + var unusable = Ok(CurationDeskWrites.RosterFeed, "{\"limit\":\"lots\",\"rep_min\":true,\"max_words\":null}"); + Assert.Equal(new[] { "username" }, unusable.Select(kv => kv.Key).ToArray()); + } + + [Fact] + public void RosterFeedNumbersAreWholeNumbersOrNothing() + { + // These names count rows, reputations and words. A fraction is none of + // them: truncating 1.9 to 1 would forward a filter nobody asked for, so + // it is dropped and the backend applies its default, exactly as the + // query string does with `limit=1.9`. + var fractions = Ok(CurationDeskWrites.RosterFeed, + "{\"limit\":1.9,\"rep_min\":10.5,\"rep_max\":99.9,\"min_words\":0.5,\"max_words\":300.25}"); + Assert.Equal(new[] { "username" }, fractions.Select(kv => kv.Key).ToArray()); + + // A whole number is kept, as a number or as its plain spelling. + Assert.Equal(12, Ok(CurationDeskWrites.RosterFeed, "{\"limit\":12}")["limit"]!.GetValue()); + Assert.Equal(12, Ok(CurationDeskWrites.RosterFeed, "{\"limit\":\"12\"}")["limit"]!.GetValue()); + Assert.Equal(40, Ok(CurationDeskWrites.RosterFeed, "{\"rep_min\":40}")["rep_min"]!.GetValue()); + + // JSON keeps no spelling of a number, so 1e6 is the number 1000000 and + // clamps to the bound the same way that value does in a query string. + Assert.Equal(50, Ok(CurationDeskWrites.RosterFeed, "{\"limit\":1e6}")["limit"]!.GetValue()); + Assert.Equal(50, Ok(CurationDeskWrites.RosterFeed, "{\"limit\":1000000}")["limit"]!.GetValue()); + Assert.Equal(1, Ok(CurationDeskWrites.RosterFeed, "{\"limit\":-5}")["limit"]!.GetValue()); + Assert.Equal(new[] { "username" }, Ok(CurationDeskWrites.RosterFeed, "{\"limit\":-1.5}").Select(kv => kv.Key).ToArray()); + + // A string is read by the query string's rule, so only a plain signed + // integer is a number there. + foreach (var spelling in new[] { "\"1e6\"", "\"1.9\"", "\"12.0\"", "\" 12\"", "\"0x0c\"" }) + { + Assert.False( + Ok(CurationDeskWrites.RosterFeed, "{\"limit\":" + spelling + "}").ContainsKey("limit"), spelling); + } + } + + [Fact] + public void TheTickNamesAtMost100IdsPerList() + { + var need = string.Join(",", Enumerable.Range(1, 150)); + var visible = string.Join(",", Enumerable.Range(1000, 101)); + var payload = Ok(CurationDeskWrites.Tick, + "{\"since\":\"t\",\"need\":[" + need + "],\"visible\":[" + visible + "]}"); + + Assert.Equal(CurationDeskWrites.MaxTickIds, ((JsonArray)payload["need"]!).Count); + Assert.Equal(CurationDeskWrites.MaxTickIds, ((JsonArray)payload["visible"]!).Count); + Assert.Equal(1, payload["need"]![0]!.GetValue()); + Assert.Equal(100, payload["need"]![99]!.GetValue()); + Assert.Equal(1000, payload["visible"]![0]!.GetValue()); + + // A list that already fits travels unchanged. + var short_ = Ok(CurationDeskWrites.Tick, "{\"since\":\"t\",\"need\":[1,2,3],\"visible\":[]}"); + Assert.Equal(3, ((JsonArray)short_["need"]!).Count); + Assert.Empty((JsonArray)short_["visible"]!); + } + + // ---- route 5 path -------------------------------------------------------- + + [Fact] + public void RealAuthorsAndPermlinksMapToTheUpstreamPathUnchanged() + { + Assert.Equal("curation/desk/post/good-karma/my-post-title-2026", + PrivateApi.CurationDeskPostPath("good-karma", "my-post-title-2026")); + Assert.Equal("curation/desk/post/user.name/re-a-b-c-20260905t101010z", + PrivateApi.CurationDeskPostPath("user.name", "re-a-b-c-20260905t101010z")); + } + + [Theory] + // Dot segments would resolve upward once the string becomes a Uri. + [InlineData("..", "p")] + [InlineData("good-karma", "..")] + [InlineData(".", "p")] + // Route values arrive percent-decoded, so a slash is a slash; and the + // still-encoded spelling is not a name character either. + [InlineData("a/b", "p")] + [InlineData("good-karma", "p/q")] + [InlineData("a%2Fb", "p")] + [InlineData("good-karma", "p%2Fq")] + // A question mark or hash would truncate the path. + [InlineData("a?x=1", "p")] + [InlineData("good-karma", "p?x=1")] + [InlineData("good-karma", "p#f")] + // Outside the Hive name grammar. + [InlineData("ab", "p")] + [InlineData("Good-Karma", "p")] + [InlineData("good_karma", "p")] + [InlineData("good-karma", "")] + [InlineData("good-karma", "P")] + [InlineData("good-karma", "a_b")] + [InlineData("", "p")] + [InlineData("undefined-undefined", "p")] + // `$` would match before a trailing newline; these anchor with \A and \z. + [InlineData("good-karma\n", "p")] + [InlineData("good-karma", "p\n")] + public void AnythingOutsideTheNameGrammarIsRejected(string author, string permlink) + { + Assert.Null(PrivateApi.CurationDeskPostPath(author, permlink)); + } + + [Fact] + public void ThePermlinkLengthBoundIsEnforced() + { + Assert.NotNull(PrivateApi.CurationDeskPostPath("good-karma", new string('a', 255))); + Assert.Null(PrivateApi.CurationDeskPostPath("good-karma", new string('a', 256))); + Assert.NotNull(PrivateApi.CurationDeskPostPath(new string('a', 16), "p")); + Assert.Null(PrivateApi.CurationDeskPostPath(new string('a', 17), "p")); + } +} diff --git a/dotnet/EcencyApi.Tests/CurationDeskPublicPayloadTests.cs b/dotnet/EcencyApi.Tests/CurationDeskPublicPayloadTests.cs new file mode 100644 index 00000000..653eedbf --- /dev/null +++ b/dotnet/EcencyApi.Tests/CurationDeskPublicPayloadTests.cs @@ -0,0 +1,147 @@ +using System.Text; +using System.Text.Json.Nodes; +using EcencyApi.Handlers; +using Xunit; +using static EcencyApi.Tests.CurationDeskTestSupport; + +namespace EcencyApi.Tests; + +/// +/// A public desk body is memoized and shared-cached for its s-maxage, so a key +/// that names a curator or carries a hashed address must never get in. The +/// backend is specified to omit them; this is the fence on this side, and it +/// has to hold for every public route and at any depth of the tree. +/// +[Collection("curation-desk")] +public class CurationDeskPublicPayloadTests +{ + private const string Leaky = + "{\"team_cursor\":{\"post_id\":1,\"created\":\"t\",\"set_by\":\"alice\",\"set_at\":\"t\"}," + + "\"active_curators\":[{\"username\":\"alice\"}],\"trail_alerts\":[]," + + "\"items\":[{\"post_id\":1,\"excluded_reason\":\"abuser\",\"marks\":[{\"curator\":\"alice\",\"note\":\"secret\"}]," + + "\"recommenders\":[{\"username\":\"bob\",\"ip_hash\":\"ab\",\"key_id\":3}]}],\"generated_at\":\"t\"}"; + + private static void AssertClean(JsonNode? node) + { + switch (node) + { + case JsonObject obj: + foreach (var kv in obj) + { + Assert.DoesNotContain(kv.Key, CurationDeskPublicPayload.PrivateKeys); + AssertClean(kv.Value); + } + break; + case JsonArray arr: + foreach (var item in arr) AssertClean(item); + break; + } + } + + [Fact] + public void TheFenceNamesEveryRosterOnlyKey() + { + Assert.Equal( + new[] { "active_curators", "excluded_reason", "ip_hash", "key_id", "note", "set_at", "set_by", "trail_alerts" }, + CurationDeskPublicPayload.PrivateKeys.OrderBy(k => k, StringComparer.Ordinal).ToArray()); + } + + [Fact] + public void StripRemovesPrivateKeysAtEveryDepthAndKeepsTheRest() + { + var node = JsonNode.Parse(Leaky); + Assert.True(CurationDeskPublicPayload.Strip(node)); + AssertClean(node); + + var obj = (JsonObject)node!; + Assert.Equal(1, obj["team_cursor"]!["post_id"]!.GetValue()); + Assert.Equal("t", obj["team_cursor"]!["created"]!.GetValue()); + Assert.Equal("t", obj["generated_at"]!.GetValue()); + Assert.Equal("alice", obj["items"]![0]!["marks"]![0]!["curator"]!.GetValue()); + Assert.Equal("bob", obj["items"]![0]!["recommenders"]![0]!["username"]!.GetValue()); + } + + [Fact] + public void ACleanBodyIsServedAsTheBytesItArrivedIn() + { + var r = JsonResponse(200, "{\"items\":[{\"post_id\":1,\"author\":\"bob\",\"trailed_by\":{\"curator\":\"alice\"}}],\"feed_version\":\"v\"}"); + Assert.False(CurationDeskPublicPayload.Strip(r.Json)); + Assert.Same(r.Bytes, CurationDeskPublicPayload.ToPublicBytes(r)); + } + + [Fact] + public void ALeakyBodyIsReserializedWithoutTheKeys() + { + var r = JsonResponse(200, Leaky); + var bytes = CurationDeskPublicPayload.ToPublicBytes(r); + Assert.NotSame(r.Bytes, bytes); + var text = Encoding.UTF8.GetString(bytes); + foreach (var key in CurationDeskPublicPayload.PrivateKeys) + { + Assert.DoesNotContain("\"" + key + "\"", text); + } + Assert.Contains("\"generated_at\":\"t\"", text); + } + + [Fact] + public void ALoneSurrogateSurvivesTheStrip() + { + // JavaScript strings are arbitrary UTF-16 and a title can carry half a + // surrogate pair. Stripping a key next to one re-serializes the tree, so + // that path has to go through JsJson.Stringify: System.Text.Json's writer + // throws on a lone surrogate and would turn one odd title into a 500 for + // the whole feed page. + var r = JsonResponse(200, + "{\"items\":[{\"title\":\"lone \\ud83d end\",\"note\":\"secret\"}],\"generated_at\":\"t\"}"); + + var bytes = CurationDeskPublicPayload.ToPublicBytes(r); + var text = Encoding.UTF8.GetString(bytes); + Assert.DoesNotContain("\"note\"", text); + Assert.Contains("\\ud83d", text); + Assert.Contains("\"generated_at\":\"t\"", text); + } + + [Fact] + public async Task ALoneSurrogateIsServedAndMemoizedTheSameWay() + { + var upstream = Install(); + upstream.Answer = _ => Task.FromResult(JsonResponse(200, + "{\"items\":[{\"title\":\"lone \\ud83d end\",\"note\":\"secret\"}],\"generated_at\":\"t\"}")); + + var ctx = Get("/private-api/curation-desk/status"); + await PrivateApi.CurationDeskStatus(ctx); + Assert.Equal(200, ctx.Response.StatusCode); + var served = Body(ctx); + Assert.DoesNotContain("\"note\"", served); + Assert.Contains("\\ud83d", served); + + var again = Get("/private-api/curation-desk/status"); + await PrivateApi.CurationDeskStatus(again); + Assert.Equal(served, Body(again)); + Assert.Single(upstream.Calls); + } + + [Fact] + public async Task EveryPublicRouteServesAndMemoizesTheStrippedBody() + { + var upstream = Install(); + upstream.Answer = _ => Task.FromResult(JsonResponse(200, Leaky)); + + foreach (var (name, handler, request, _) in PublicReads()) + { + var ctx = request(); + await handler(ctx); + Assert.Equal(200, ctx.Response.StatusCode); + var served = JsonNode.Parse(Body(ctx)); + AssertClean(served); + Assert.Equal("t", served!["generated_at"]!.GetValue()); + + // The memo holds the same clean bytes, so a hit cannot leak either. + var again = request(); + await handler(again); + AssertClean(JsonNode.Parse(Body(again))); + } + + Assert.Equal(5, upstream.Calls.Count); + } +} diff --git a/dotnet/EcencyApi.Tests/CurationDeskQueryTests.cs b/dotnet/EcencyApi.Tests/CurationDeskQueryTests.cs new file mode 100644 index 00000000..78bc8590 --- /dev/null +++ b/dotnet/EcencyApi.Tests/CurationDeskQueryTests.cs @@ -0,0 +1,212 @@ +using EcencyApi.Handlers; +using Xunit; + +namespace EcencyApi.Tests; + +/// +/// The public desk reads are memoized and shared-cached by their normalized +/// upstream URL. Whitelisting decides what a stranger can make the backend +/// compute; the fixed order and dropped defaults decide how many distinct keys +/// a burst of equivalent requests turns into. +/// +public class CurationDeskQueryTests +{ + private static KeyValuePair[] Q(params (string, string)[] pairs) => + pairs.Select(p => new KeyValuePair(p.Item1, p.Item2)).ToArray(); + + private static string Feed(params (string, string)[] pairs) => + CurationDeskQuery.Endpoint("curation/desk/feed", CurationDeskQuery.NormalizeFeed(Q(pairs))); + + private static string Recommendations(params (string, string)[] pairs) => + CurationDeskQuery.Endpoint("curation/desk/recommendations", CurationDeskQuery.NormalizeRecommendations(Q(pairs))); + + [Fact] + public void AnEmptyQueryIsTheBarePath() + { + Assert.Equal("curation/desk/feed", Feed()); + Assert.Equal("curation/desk/recommendations", Recommendations()); + } + + [Fact] + public void UnknownParametersAreDropped() + { + Assert.Equal("curation/desk/feed", + Feed(("x", "1"), ("order", "asc"), ("username", "alice"), ("code", "abc"), ("hide_reviewed", "0"), ("flagged", "1"))); + } + + [Fact] + public void DefaultsAreDroppedSoTheyCollapseOntoTheBarePath() + { + Assert.Equal("curation/desk/feed", + Feed(("limit", "25"), ("sort", "newest"), ("app", "all"), ("window", "all"), ("hide_curated", "1"), + ("rep_min", "0"), ("rep_max", "100"), ("min_words", "0"), ("max_words", "50000"), + ("has_images", "0"), ("new_authors", "0"), ("recommended", "0"))); + } + + [Theory] + [InlineData("0", "limit=1")] + [InlineData("-3", "limit=1")] + [InlineData("1", "limit=1")] + [InlineData("50", "limit=50")] + [InlineData("999", "limit=50")] + // Beyond int.MaxValue: "as many as you can", not "give me the default". + [InlineData("99999999999", "limit=50")] + [InlineData("-99999999999", "limit=1")] + // A plain whole number past the range clamps; the body path reads the same + // value out of JSON and answers the same. + [InlineData("1000000", "limit=50")] + [InlineData("25", "")] + [InlineData("abc", "")] + // Text is a number only when it spells a plain signed integer, here and for + // a string in a roster feed body: a fraction or an exponent is dropped + // rather than truncated. + [InlineData("1e1", "")] + [InlineData("1e6", "")] + [InlineData("1.9", "")] + [InlineData("12.0", "")] + [InlineData("", "")] + public void LimitIsClampedToItsRange(string given, string expected) + { + var url = Feed(("limit", given)); + Assert.Equal(expected.Length == 0 ? "curation/desk/feed" : "curation/desk/feed?" + expected, url); + } + + [Fact] + public void RangesAreClampedAndTheirNoOpBoundsDropped() + { + Assert.Equal("curation/desk/feed?rep_min=100", Feed(("rep_min", "150"))); + Assert.Equal("curation/desk/feed?rep_max=0", Feed(("rep_max", "-1"))); + Assert.Equal("curation/desk/feed?rep_min=40&rep_max=70", Feed(("rep_max", "70"), ("rep_min", "40"))); + Assert.Equal("curation/desk/feed", Feed(("min_words", "-5"))); + Assert.Equal("curation/desk/feed?min_words=50000", Feed(("min_words", "999999"))); + Assert.Equal("curation/desk/feed", Feed(("max_words", "999999"))); + Assert.Equal("curation/desk/feed?max_words=300", Feed(("max_words", "300"))); + Assert.Equal("curation/desk/feed", Feed(("rep_min", "high"))); + } + + [Fact] + public void FlagsAreZeroOrOneOnly() + { + Assert.Equal("curation/desk/feed?has_images=1&new_authors=1&recommended=1", + Feed(("has_images", "1"), ("new_authors", "1"), ("recommended", "1"))); + Assert.Equal("curation/desk/feed", + Feed(("has_images", "true"), ("new_authors", "yes"), ("recommended", "2"))); + Assert.Equal("curation/desk/feed?hide_curated=0", Feed(("hide_curated", "0"))); + Assert.Equal("curation/desk/feed", Feed(("hide_curated", "false"))); + } + + [Fact] + public void EnumsAcceptOnlyTheirAllowlist() + { + Assert.Equal("curation/desk/feed?sort=queue", Feed(("sort", "queue"))); + Assert.Equal("curation/desk/feed?sort=unique", Feed(("sort", "unique"))); + Assert.Equal("curation/desk/feed?view=new-authors", Feed(("view", "new-authors"))); + Assert.Equal("curation/desk/feed", Feed(("view", "excluded"))); + Assert.Equal("curation/desk/feed?app=peakd", Feed(("app", "peakd"))); + Assert.Equal("curation/desk/feed", Feed(("app", "hive"))); + Assert.Equal("curation/desk/feed?window=locked", Feed(("window", "locked"))); + Assert.Equal("curation/desk/feed", Feed(("window", "week"))); + Assert.Equal("curation/desk/feed", Feed(("sort", "Queue"))); + } + + [Fact] + public void APublicRandomSortFallsBackToNewestAndItsSeedIsDropped() + { + Assert.Equal("curation/desk/feed", Feed(("sort", "random"), ("seed", "abcd1234"))); + Assert.Equal("curation/desk/feed", Feed(("seed", "abcd1234"))); + Assert.Equal("curation/desk/feed", Feed(("sort", "payout"), ("order", "desc"))); + } + + [Fact] + public void UniqueImpliesRecommendedSoTheFlagIsRedundantThere() + { + Assert.Equal("curation/desk/feed?sort=unique", Feed(("sort", "unique"), ("recommended", "1"))); + Assert.Equal(Feed(("sort", "unique")), Feed(("sort", "unique"), ("recommended", "1"))); + } + + [Fact] + public void CursorMustMatchTheOpaqueCursorGrammar() + { + Assert.Equal("curation/desk/feed?cursor=2026-09-05T10%3A00%3A00Z%3A123", Feed(("cursor", "2026-09-05T10:00:00Z:123"))); + Assert.Equal("curation/desk/feed?cursor=s%3Aabc_def.1%3A25", Feed(("cursor", "s:abc_def.1:25"))); + Assert.Equal("curation/desk/feed", Feed(("cursor", "a b"))); + Assert.Equal("curation/desk/feed", Feed(("cursor", "a/b"))); + Assert.Equal("curation/desk/feed", Feed(("cursor", ""))); + Assert.Equal("curation/desk/feed", Feed(("cursor", new string('a', 81)))); + Assert.Equal("curation/desk/feed?cursor=" + new string('a', 80), Feed(("cursor", new string('a', 80)))); + } + + [Fact] + public void CommunityMustBeAHiveCommunityName() + { + Assert.Equal("curation/desk/feed?community=hive-125125", Feed(("community", "hive-125125"))); + Assert.Equal("curation/desk/feed?community=hive-12512", Feed(("community", "hive-12512"))); + Assert.Equal("curation/desk/feed", Feed(("community", "hive-1234"))); + Assert.Equal("curation/desk/feed", Feed(("community", "hive-1234567"))); + Assert.Equal("curation/desk/feed", Feed(("community", "photography"))); + Assert.Equal("curation/desk/feed", Feed(("community", "hive-125125/x"))); + } + + [Fact] + public void AValueWithATrailingNewlineIsNotTheValue() + { + // `$` in .NET also matches before a trailing newline, so every pattern + // here anchors with \A and \z instead. + Assert.Equal("curation/desk/feed", Feed(("cursor", "abc\n"))); + Assert.Equal("curation/desk/feed", Feed(("community", "hive-125125\n"))); + Assert.Equal("curation/desk/feed", Feed(("sort", "queue\n"))); + Assert.Equal("curation/desk/feed", Feed(("view", "queue\n"))); + Assert.Equal("curation/desk/recommendations", Recommendations(("cursor", "abc\n"))); + } + + [Fact] + public void CommunityDigitsAreAsciiDigitsOnly() + { + // .NET's `\d` matches every Unicode decimal digit; a community name is + // five or six ASCII digits and nothing else. + Assert.Equal("curation/desk/feed", Feed(("community", "hive-\u0661\u0662\u0663\u0664\u0665"))); + Assert.Equal("curation/desk/feed", Feed(("community", "hive-\u09E7\u09E8\u09E9\u09EA\u09EB\u09EC"))); + Assert.Equal("curation/desk/feed?community=hive-125125", Feed(("community", "hive-125125"))); + } + + [Fact] + public void ParametersAreEmittedInOneFixedOrderWhateverTheClientSent() + { + var a = Feed(("recommended", "1"), ("community", "hive-125125"), ("limit", "10"), ("view", "queue"), + ("cursor", "c1"), ("window", "full"), ("app", "ecency"), ("sort", "queue"), ("has_images", "1"), + ("rep_min", "30"), ("max_words", "800"), ("hide_curated", "0"), ("new_authors", "1"), ("rep_max", "75"), + ("min_words", "100")); + var b = Feed(("min_words", "100"), ("rep_max", "75"), ("new_authors", "1"), ("hide_curated", "0"), + ("max_words", "800"), ("rep_min", "30"), ("has_images", "1"), ("sort", "queue"), ("app", "ecency"), + ("window", "full"), ("cursor", "c1"), ("view", "queue"), ("limit", "10"), ("community", "hive-125125"), + ("recommended", "1")); + Assert.Equal(a, b); + Assert.Equal( + "curation/desk/feed?cursor=c1&limit=10&sort=queue&view=queue&app=ecency&community=hive-125125&window=full" + + "&rep_min=30&rep_max=75&min_words=100&max_words=800&has_images=1&new_authors=1&recommended=1&hide_curated=0", + a); + + // The order is the whitelist itself: nothing else can appear, and the + // shared-cache key upstream of this service lists the same names. + Assert.Equal(15, CurationDeskQuery.FeedOrder.Length); + Assert.Equal(CurationDeskQuery.FeedOrder.Length, CurationDeskQuery.FeedOrder.Distinct().Count()); + } + + [Fact] + public void ARepeatedKeyTakesItsFirstValue() + { + Assert.Equal("curation/desk/feed?limit=10", Feed(("limit", "10"), ("limit", "40"))); + } + + [Fact] + public void RecommendationsAcceptOnlyCursorLimitAndTheirTwoSorts() + { + Assert.Equal("curation/desk/recommendations?sort=unique", Recommendations(("sort", "unique"))); + Assert.Equal("curation/desk/recommendations?sort=newest", Recommendations(("sort", "newest"))); + Assert.Equal("curation/desk/recommendations", Recommendations(("sort", "queue"))); + Assert.Equal("curation/desk/recommendations?cursor=abc&limit=50", + Recommendations(("view", "all"), ("limit", "60"), ("cursor", "abc"), ("seed", "x"))); + Assert.Equal("curation/desk/recommendations?limit=10&sort=unique", + Recommendations(("sort", "unique"), ("limit", "10"))); + } +} diff --git a/dotnet/EcencyApi.Tests/CurationDeskTestSupport.cs b/dotnet/EcencyApi.Tests/CurationDeskTestSupport.cs new file mode 100644 index 00000000..6a7da1a4 --- /dev/null +++ b/dotnet/EcencyApi.Tests/CurationDeskTestSupport.cs @@ -0,0 +1,257 @@ +using System.Text; +using System.Text.Json.Nodes; +using EcencyApi.Handlers; +using EcencyApi.Infrastructure; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Http.Features; +using Xunit; + +namespace EcencyApi.Tests; + +/// +/// The desk handler tests replace static seams (the configured token, the +/// upstream call, code validation) and share the static memo, so they must not +/// run alongside each other. One collection serializes them. +/// +[CollectionDefinition("curation-desk", DisableParallelization = true)] +public class CurationDeskCollection { } + +/// +/// Handler-level scaffolding: a request context whose OnStarting callbacks can be +/// run (DefaultHttpContext drops them, and CacheWhenOk relies on them), a +/// recording upstream, and a reset of every seam between tests. +/// +internal static class CurationDeskTestSupport +{ + public const string Token = "test-desk-token"; + + /// HttpResponseFeature that keeps OnStarting callbacks so a test can fire them. + public sealed class StartingAwareResponseFeature : HttpResponseFeature + { + private readonly List<(Func Callback, object State)> _starting = new(); + + public override void OnStarting(Func callback, object state) => _starting.Add((callback, state)); + + public async Task RunStarting() + { + foreach (var (callback, state) in _starting) + { + await callback(state); + } + } + } + + /// + /// A response body whose writes block until the test releases them, so one + /// request can be held inside its response write while another runs. Stands + /// in for a reader on a slow connection. + /// + public sealed class BlockingBody : Stream + { + private readonly MemoryStream _inner = new(); + private readonly TaskCompletionSource _release = new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource _started = new(TaskCreationOptions.RunContinuationsAsynchronously); + + /// Completes once the handler has reached its first write. + public Task WriteReached => _started.Task; + + public void Release() => _release.TrySetResult(); + + public string Text => Encoding.UTF8.GetString(_inner.ToArray()); + + public override async ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cancellationToken = default) + { + _started.TrySetResult(); + await _release.Task; + await _inner.WriteAsync(buffer, cancellationToken); + } + + public override Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) => + WriteAsync(buffer.AsMemory(offset, count), cancellationToken).AsTask(); + + public override void Write(byte[] buffer, int offset, int count) => + WriteAsync(buffer, offset, count, CancellationToken.None).GetAwaiter().GetResult(); + + public override bool CanRead => false; + public override bool CanSeek => false; + public override bool CanWrite => true; + public override long Length => _inner.Length; + public override long Position { get => _inner.Position; set => throw new NotSupportedException(); } + public override void Flush() { } + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + public override void SetLength(long value) => throw new NotSupportedException(); + } + + public sealed record Call(string Endpoint, HttpMethod Method, List> Headers, JsonNode? Payload) + { + public string? Header(string name) => + Headers.FirstOrDefault(h => h.Key.Equals(name, StringComparison.OrdinalIgnoreCase)).Value; + } + + /// Records every upstream call and answers from a script. + public sealed class Recorder + { + public readonly List Calls = new(); + public Func> Answer = _ => Task.FromResult(JsonResponse(200, "{}")); + + public Task Handle(string endpoint, HttpMethod method, + IEnumerable> headers, JsonNode? payload) + { + var call = new Call(endpoint, method, headers.ToList(), payload?.DeepClone()); + lock (Calls) Calls.Add(call); + return Answer(call); + } + } + + /// + /// Fresh seams: token configured, validation accepts any non-empty code as + /// the account named in it (`code` "as:alice" -> "alice"), empty memo. + /// + public static Recorder Install(string? token = Token) + { + PrivateApi.DeskToken = token; + PrivateApi.DeskAuthMemoSeconds = 90; + PrivateApi.DeskValidateCode = body => + { + var code = body["code"]?.GetValue(); + return Task.FromResult(code != null && code.StartsWith("as:", StringComparison.Ordinal) ? code[3..] : null); + }; + CurationDeskMemo.ResetForTests(); + var recorder = new Recorder(); + PrivateApi.DeskUpstream = recorder.Handle; + return recorder; + } + + public static UpstreamResponse JsonResponse(int status, string json) + { + var bytes = Encoding.UTF8.GetBytes(json); + return new UpstreamResponse + { + Status = status, + Json = JsonNode.Parse(json), + Headers = new HttpResponseHeaders2(new HttpResponseMessage()), + Bytes = bytes, + }; + } + + public static UpstreamResponse TextResponse(int status, string text) + { + return new UpstreamResponse + { + Status = status, + RawText = text, + Headers = new HttpResponseHeaders2(new HttpResponseMessage()), + Bytes = Encoding.UTF8.GetBytes(text), + }; + } + + public static DefaultHttpContext Get(string path, string query = "", (string Name, string Value)[]? routeValues = null) + { + var ctx = NewContext(); + ctx.Request.Method = "GET"; + ctx.Request.Path = path; + if (query.Length > 0) + { + ctx.Request.QueryString = new QueryString(query.StartsWith('?') ? query : "?" + query); + } + if (routeValues != null) + { + foreach (var (name, value) in routeValues) + { + ctx.Request.RouteValues[name] = value; + } + } + return ctx; + } + + public static DefaultHttpContext Post(string path, string body) + { + var ctx = NewContext(); + ctx.Request.Method = "POST"; + ctx.Request.Path = path; + ctx.Request.ContentType = "application/json"; + var bytes = Encoding.UTF8.GetBytes(body); + ctx.Request.Body = new MemoryStream(bytes); + ctx.Request.ContentLength = bytes.Length; + return ctx; + } + + private static DefaultHttpContext NewContext() + { + var ctx = new DefaultHttpContext(); + ctx.Features.Set(new StartingAwareResponseFeature()); + ctx.Response.Body = new MemoryStream(); + return ctx; + } + + /// Fire the OnStarting callbacks, as the server would before the first byte. + public static Task Start(HttpContext ctx) => + ((StartingAwareResponseFeature)ctx.Features.Get()!).RunStarting(); + + public static string Body(HttpContext ctx) + { + ctx.Response.Body.Position = 0; + return new StreamReader(ctx.Response.Body, Encoding.UTF8).ReadToEnd(); + } + + public static string? CacheControl(HttpContext ctx) => + ctx.Response.Headers.TryGetValue("Cache-Control", out var v) ? v.ToString() : null; + + public static string? Age(HttpContext ctx) => + ctx.Response.Headers.TryGetValue("Age", out var v) ? v.ToString() : null; + + /// + /// A clock the test moves by hand. The memo reads its fill times from this, + /// so a test can put an entry near the end of its window without sleeping + /// through it (and without a wall-clock bound that turns into a flaky test + /// on a loaded machine). Reset by . + /// + public sealed class TestClock + { + // An arbitrary fixed epoch: only differences matter. + private long _ms = 1_700_000_000_000; + + public long NowMs() => _ms; + + public void Advance(TimeSpan by) => _ms += (long)by.TotalMilliseconds; + } + + /// Hand the memo a movable clock and return it. + public static TestClock UseTestClock() + { + var clock = new TestClock(); + CurationDeskMemo.NowMs = clock.NowMs; + return clock; + } + + /// Every public read, as (handler, request factory, policy). + public static IEnumerable<(string Name, Func Handler, Func Request, string Policy)> PublicReads() + { + yield return ("feed", PrivateApi.CurationDeskFeed, + () => Get("/private-api/curation-desk/feed", "limit=10"), CachePolicy.CurationDeskFeed); + yield return ("status", PrivateApi.CurationDeskStatus, + () => Get("/private-api/curation-desk/status"), CachePolicy.CurationDeskStatus); + yield return ("roster", PrivateApi.CurationDeskRoster, + () => Get("/private-api/curation-desk/roster"), CachePolicy.CurationDeskRoster); + yield return ("recommendations", PrivateApi.CurationDeskRecommendations, + () => Get("/private-api/curation-desk/recommendations", "sort=unique"), CachePolicy.CurationDeskRecommendations); + yield return ("post", PrivateApi.CurationDeskPost, + () => Get("/private-api/curation-desk/post/good-karma/hello-world", "", + new[] { ("author", "good-karma"), ("permlink", "hello-world") }), CachePolicy.CurationDeskPost); + } + + /// Every signed write, as (handler, a body that passes validation). + public static IEnumerable<(string Name, Func Handler, string Body)> SignedWrites() + { + const string code = "\"code\":\"as:alice\""; + yield return ("roster-feed", PrivateApi.CurationDeskRosterFeed, "{" + code + ",\"limit\":10}"); + yield return ("tick", PrivateApi.CurationDeskTick, "{" + code + ",\"since\":\"2026-09-05T00:00:00Z\",\"need\":[1],\"visible\":[1,2]}"); + yield return ("mark", PrivateApi.CurationDeskMark, "{" + code + ",\"author\":\"bob\",\"permlink\":\"p\",\"state\":\"reviewed\"}"); + yield return ("mark-clear", PrivateApi.CurationDeskMarkClear, "{" + code + ",\"author\":\"bob\",\"permlink\":\"p\"}"); + yield return ("marks", PrivateApi.CurationDeskMarks, "{" + code + ",\"state\":\"flagged\"}"); + yield return ("cursor", PrivateApi.CurationDeskCursor, "{" + code + ",\"post_id\":42,\"action\":\"advance\"}"); + yield return ("recommend-meta", PrivateApi.CurationDeskRecommendMeta, "{" + code + ",\"author\":\"bob\",\"permlink\":\"p\",\"ua_class\":\"web\"}"); + yield return ("recommendation-dismiss", PrivateApi.CurationDeskRecommendationDismiss, "{" + code + ",\"author\":\"bob\",\"permlink\":\"p\",\"action\":\"dismiss\"}"); + } +} diff --git a/dotnet/EcencyApi/Config.cs b/dotnet/EcencyApi/Config.cs index de3eddaf..b8e609d3 100644 --- a/dotnet/EcencyApi/Config.cs +++ b/dotnet/EcencyApi/Config.cs @@ -21,6 +21,27 @@ public static class Config public static string EnotifyInternalToken { get; } = Env("ENOTIFY_INTERNAL_TOKEN") ?? ""; + /// + /// Shared secret presented to the curation desk backend on every desk call, + /// public reads included: the desk answers only requests that carry it, so + /// the memo and rate limits in front of it cannot be bypassed by going + /// around this service. No default: when unset the desk routes fail closed + /// (503) rather than forward an empty secret. + /// + public static string DeskInternalToken { get; } = + Env("DESK_INTERNAL_TOKEN") ?? ""; + + /// + /// Byte budget of each curation desk memo store (the fresh one and the + /// last-good one), LRU beyond it. A feed page is tens of KB, so the default + /// holds thousands of distinct questions; it is exposed so a deployment can + /// shrink it without a rebuild if the process is short of memory. + /// + public static long DeskMemoBytes { get; } = + long.TryParse(Env("DESK_MEMO_BYTES"), out var deskBytes) && deskBytes >= 0 + ? deskBytes + : 64L * 1024 * 1024; + public static string HsClientSecret { get; } = Env("HIVESIGNER_SECRET") ?? "hivesignerclientsecret"; diff --git a/dotnet/EcencyApi/Handlers/PrivateApi.CurationDesk.cs b/dotnet/EcencyApi/Handlers/PrivateApi.CurationDesk.cs new file mode 100644 index 00000000..568171f8 --- /dev/null +++ b/dotnet/EcencyApi/Handlers/PrivateApi.CurationDesk.cs @@ -0,0 +1,1173 @@ +using System.Globalization; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.RegularExpressions; +using EcencyApi.Infrastructure; + +namespace EcencyApi.Handlers; + +/// +/// Curation desk gateway: /private-api/curation-desk/* -> curation/desk/* upstream. +/// +/// Five public reads and eight signed writes. This service does three things +/// for the desk that the generic pipe handlers do not: +/// +/// - every upstream call carries a shared secret header, reads included. The +/// desk backend answers nothing without it, so the memo, the cache policies +/// and the rate limits in front of this service cannot be skipped by calling +/// the backend directly. When the secret is not configured the routes fail +/// closed (503), the same way the payment routes do; +/// - public reads are whitelisted, clamped and emitted in one fixed order, so +/// every spelling of the same question collapses onto one memo entry and one +/// shared-cache key, and the answer is memoized as bytes for exactly the +/// s-maxage the response promises (single-flight per key, last-good on an +/// upstream error). A body served from that memo says how old it is and +/// offers shared caches only the rest of its window, so the two layers do +/// not each hold it for a full lifetime in series; +/// - writes resolve the caller from the signed code (memoized briefly, see +/// ) and forward only whitelisted +/// body fields under that username; a client-supplied username or code never +/// reaches the backend. +/// +public static partial class PrivateApi +{ + // ---- configuration seams ------------------------------------------------- + + /// Header carrying the shared secret to the desk backend. + internal const string DeskTokenHeader = "X-Desk-Internal-Token"; + + /// + /// The configured secret, or null when the desk is switched off. A static + /// field rather than a Config read so tests can flip it; production reads it + /// once at startup like every other setting. + /// + internal static string? DeskToken = string.IsNullOrWhiteSpace(Config.DeskInternalToken) + ? null + : Config.DeskInternalToken.Trim(); + + /// + /// The one upstream call every desk route goes through. Replaceable so tests + /// can observe the request (path, method, headers, payload) without a network. + /// + internal static Func>, JsonNode?, Task> + DeskUpstream = (endpoint, method, headers, payload) => ApiClient.ApiRequest(endpoint, method, headers, payload); + + /// Signed-code validation, replaceable for tests (no chain RPC). + internal static Func> DeskValidateCode = ValidateCode; + + /// + /// How long a successful code validation is remembered. A validation costs + /// one uncached account lookup per call, and a curator on the desk sends a + /// write every few seconds; 90 s keeps that to one lookup per curator per + /// window while a posting-key rotation lags by at most this much. Failures + /// are never remembered. + /// + internal static double DeskAuthMemoSeconds = 90; + + private const string DeskAuthMemoPrefix = "desk-auth:"; + + private const string DeskNotConfigured = "curation desk not configured"; + + // ---- public reads -------------------------------------------------------- + + // The one-line handlers here and under "signed writes" return the delegate's + // Task instead of awaiting it: they do nothing after the call, so an async + // state machine per request would be pure overhead and Routes.cs only needs + // a Task back. Handlers that do work of their own stay `async`. + + // GET /private-api/curation-desk/feed + public static Task CurationDeskFeed(HttpContext ctx) => + ServeDeskRead(ctx, + CurationDeskQuery.Endpoint("curation/desk/feed", CurationDeskQuery.NormalizeFeed(RawQuery(ctx))), + CachePolicy.CurationDeskFeed); + + // GET /private-api/curation-desk/status + public static Task CurationDeskStatus(HttpContext ctx) => + ServeDeskRead(ctx, "curation/desk/status", CachePolicy.CurationDeskStatus); + + // GET /private-api/curation-desk/roster + public static Task CurationDeskRoster(HttpContext ctx) => + ServeDeskRead(ctx, "curation/desk/roster", CachePolicy.CurationDeskRoster); + + // GET /private-api/curation-desk/recommendations + public static Task CurationDeskRecommendations(HttpContext ctx) => + ServeDeskRead(ctx, + CurationDeskQuery.Endpoint("curation/desk/recommendations", + CurationDeskQuery.NormalizeRecommendations(RawQuery(ctx))), + CachePolicy.CurationDeskRecommendations); + + // GET /private-api/curation-desk/post/{author}/{permlink} + public static async Task CurationDeskPost(HttpContext ctx) + { + // The unconfigured answer comes first, before this route looks at + // anything the caller sent. A dark desk answers 503 on every route the + // same way; answering 400 here instead would make this one route report + // on its own path grammar while the other four report nothing. + if (DeskToken == null) + { + await ctx.SendText(503, DeskNotConfigured); + return; + } + + var author = ctx.Request.RouteValues["author"]?.ToString() ?? ""; + var permlink = ctx.Request.RouteValues["permlink"]?.ToString() ?? ""; + var path = CurationDeskPostPath(author, permlink); + if (path == null) + { + await ctx.SendText(400, "Invalid author or permlink"); + return; + } + await ServeDeskRead(ctx, path, CachePolicy.CurationDeskPost); + } + + // \A and \z, not ^ and $: in .NET `$` also matches before a trailing + // newline, so "good-karma\n" would pass a `$`-anchored name check and + // travel into the upstream path. + private static readonly Regex DeskAuthorPattern = new(@"\A[a-z0-9.-]{3,16}\z", RegexOptions.Compiled); + private static readonly Regex DeskPermlinkPattern = new(@"\A[a-z0-9-]{1,255}\z", RegexOptions.Compiled); + + /// + /// Upstream path for a single post, or null when either value is not a plain + /// Hive name or permlink. Same reasoning as : route + /// values arrive percent-decoded, and a `/`, `?` or `#` left in place would be + /// re-parsed as URL structure once the string becomes a Uri, with the desk + /// secret attached to wherever it then points. The character classes already + /// exclude every structural character; the escaping stays as a second fence + /// and the dot-segment check as the one case escaping cannot fix. + /// + public static string? CurationDeskPostPath(string author, string permlink) + { + if (author is "." or ".." || permlink is "." or "..") + { + return null; + } + if (!DeskAuthorPattern.IsMatch(author) || !DeskPermlinkPattern.IsMatch(permlink)) + { + return null; + } + return $"curation/desk/post/{Uri.EscapeDataString(author)}/{Uri.EscapeDataString(permlink)}"; + } + + private static IEnumerable> RawQuery(HttpContext ctx) + { + foreach (var kv in ctx.Request.Query) + { + // A repeated key takes its first value, like Express `req.query` read + // as a scalar; the normalizer sees each key once. + var first = kv.Value.Count > 0 ? kv.Value[0] : null; + if (first != null) + { + yield return new KeyValuePair(kv.Key, first); + } + } + } + + /// + /// Serve one public desk read: memo hit, or a single-flight fill of the + /// normalized endpoint. + /// + /// Nothing is written to the client while the per-key gate is held. The gate + /// exists to collapse concurrent fills of one key onto one upstream call, so + /// a fill only computes the public bytes and stores them; what to send is + /// kept in locals, the gate is released in the finally, and the response is + /// written after it. Writing under the gate would make every reader of a key + /// wait for the slowest reader's socket to drain rather than for the upstream + /// call, which is the one thing the gate is meant to share. + /// + private static async Task ServeDeskRead(HttpContext ctx, string endpoint, string policy) + { + var token = DeskToken; + if (token == null) + { + await ctx.SendText(503, DeskNotConfigured); + return; + } + + if (CurationDeskMemo.TryGetFresh(endpoint, out var hit, out var hitType, out var hitAge)) + { + await SendPublicJson(ctx, policy, hitType, hit, hitAge); + return; + } + + // Filled under the gate, sent after it: bytes to serve, an error to + // answer with, or the upstream response to pass through. + byte[]? bytes = null; + string? bytesType = null; + // How long this service has held those bytes; a fill made here is new. + var bytesAge = 0; + var errorStatus = 0; + string? errorText = null; + UpstreamResponse? passthrough = null; + + // Held for the whole block, so the gate cannot be dropped and replaced + // between being handed out and being waited on. + var gate = CurationDeskMemo.GateFor(endpoint); + var entered = false; + var gateTimedOut = false; + + try + { + entered = await gate.Semaphore.WaitAsync(CurationDeskMemo.FillWait); + if (!entered) + { + // Someone else's fill is taking longer than a whole upstream + // timeout. Do not stack another one behind it; answer from what + // is known, once the gate has been handed back. + gateTimedOut = true; + } + // The fill that held the gate may have landed while this one queued. + else if (CurationDeskMemo.TryGetFresh(endpoint, out var fresh, out var freshType, out var freshAge)) + { + bytes = fresh; + bytesType = freshType; + bytesAge = freshAge; + } + else + { + UpstreamResponse? r = null; + try + { + r = await DeskUpstream(endpoint, HttpMethod.Get, DeskHeaders(token), null); + } + catch (UpstreamTimeoutException) + { + (errorStatus, errorText) = (504, "Upstream Timeout"); + } + catch (Exception) + { + (errorStatus, errorText) = (500, "Server Error"); + } + + if (r != null && r.Status == 200 && r.Json is JsonObject or JsonArray) + { + bytes = CurationDeskPublicPayload.ToPublicBytes(r); + bytesType = JsonContentType; + CurationDeskMemo.Store(endpoint, bytes, JsonContentType, CachePolicy.SharedMaxAge(policy)); + } + else + { + passthrough = r; + } + } + } + finally + { + if (entered) + { + gate.Semaphore.Release(); + } + CurationDeskMemo.ReleaseGate(endpoint, gate); + } + + if (gateTimedOut) + { + await ServeLastGoodOr(ctx, endpoint, policy, 504, "Upstream Timeout"); + return; + } + + if (bytes != null) + { + await SendPublicJson(ctx, policy, bytesType!, bytes, bytesAge); + return; + } + + if (errorText != null) + { + await ServeLastGoodOr(ctx, endpoint, policy, errorStatus, errorText); + return; + } + + var response = passthrough!; + + // A 5xx, or a 200 whose body is not a JSON object or array (an error + // page, a redirect body, a bare string): either way the backend is not + // answering the question this route asks, so a body it did answer with + // is better than passing that on, and neither is worth memoizing. + if ((response.Status >= 500 || response.Status == 200) + && CurationDeskMemo.TryGetLastGood(endpoint, out var stale, out var staleType, out var staleAge)) + { + await SendStaleJson(ctx, policy, staleType, stale, staleAge); + return; + } + + // 4xx (an unknown post, a rejected token), or nothing to fall back on: + // pass through the way Pipe would, unmemoized and with no Cache-Control + // of ours, so the client sees what the backend said. An error body is + // still a public body this service emits, so it goes through the same + // fence as a served one. + CurationDeskPublicPayload.Strip(response.Json); + await Upstream.SendLikeExpress(ctx, response.Status, response.Json, response.RawText); + } + + private const string JsonContentType = "application/json; charset=utf-8"; + + /// + /// Send a JSON body this service holds (a fresh fill, or a memo hit that is + /// already old) with the route's cache policy. + /// Only these bodies are publicly cacheable: an upstream passthrough carries + /// whatever the backend answered, which may be an error page or a body meant + /// for one caller, so it never gets a Cache-Control of ours. + /// + /// A hit goes out with the rest of its window rather than a new one, so the + /// memo and the caches downstream expire the same body at the same moment + /// instead of holding it for one lifetime each in series. The remaining + /// lifetime is the only freshness signal sent: an Age header on top of an + /// already shortened s-maxage would be subtracted a second time by a cache + /// that honours both, leaving the body stale on arrival. + /// + private static async Task SendPublicJson(HttpContext ctx, string policy, string contentType, byte[] bytes, int ageSeconds) + { + ctx.CacheWhenOk(CachePolicy.Aged(policy, ageSeconds)); + await WriteBytes(ctx, 200, contentType, bytes); + } + + /// + /// Send a last-good body: a real answer from the backend, but one kept + /// because the call that should have replaced it failed. It carries a short + /// window instead of the route's own, so an upstream that comes back is + /// picked up within a poll or two rather than at the end of a full one. + /// + private static async Task SendStaleJson(HttpContext ctx, string policy, string contentType, byte[] bytes, int ageSeconds) + { + // the body's real age is not advertised: the short window alone is the + // freshness, and an Age older than it would make the answer stale at once + _ = ageSeconds; + ctx.CacheWhenOk(CachePolicy.Stale(policy)); + await WriteBytes(ctx, 200, contentType, bytes); + } + + private static async Task ServeLastGoodOr(HttpContext ctx, string endpoint, string policy, int status, string text) + { + if (CurationDeskMemo.TryGetLastGood(endpoint, out var stale, out var staleType, out var staleAge)) + { + await SendStaleJson(ctx, policy, staleType, stale, staleAge); + return; + } + await ctx.SendText(status, text); + } + + private static async Task WriteBytes(HttpContext ctx, int status, string contentType, byte[] bytes) + { + ctx.Response.StatusCode = status; + ctx.Response.ContentType = contentType; + await ctx.Response.Body.WriteAsync(bytes); + } + + private static List> DeskHeaders(string token, string? clientIp = null) + { + var headers = new List> { new(DeskTokenHeader, token) }; + if (clientIp != null) + { + headers.Add(new KeyValuePair("X-Real-IP-V", clientIp)); + } + return headers; + } + + // ---- signed writes ------------------------------------------------------- + + // POST /private-api/curation-desk/roster-feed + public static Task CurationDeskRosterFeed(HttpContext ctx) => + ServeDeskWrite(ctx, CurationDeskWrites.RosterFeed); + + // POST /private-api/curation-desk/tick + public static Task CurationDeskTick(HttpContext ctx) => + ServeDeskWrite(ctx, CurationDeskWrites.Tick); + + // POST /private-api/curation-desk/mark + public static Task CurationDeskMark(HttpContext ctx) => + ServeDeskWrite(ctx, CurationDeskWrites.Mark); + + // POST /private-api/curation-desk/mark-clear + public static Task CurationDeskMarkClear(HttpContext ctx) => + ServeDeskWrite(ctx, CurationDeskWrites.MarkClear); + + // POST /private-api/curation-desk/marks + public static Task CurationDeskMarks(HttpContext ctx) => + ServeDeskWrite(ctx, CurationDeskWrites.Marks); + + // POST /private-api/curation-desk/cursor + public static Task CurationDeskCursor(HttpContext ctx) => + ServeDeskWrite(ctx, CurationDeskWrites.Cursor); + + // POST /private-api/curation-desk/recommend-meta + public static Task CurationDeskRecommendMeta(HttpContext ctx) => + ServeDeskWrite(ctx, CurationDeskWrites.RecommendMeta); + + // POST /private-api/curation-desk/recommendation-dismiss + public static Task CurationDeskRecommendationDismiss(HttpContext ctx) => + ServeDeskWrite(ctx, CurationDeskWrites.RecommendationDismiss); + + /// + /// Signed write: authenticate, fail closed when unconfigured, build the + /// whitelisted payload under the validated username, pipe. Never cacheable. + /// + private static async Task ServeDeskWrite(HttpContext ctx, CurationDeskWrites.Route route) + { + // Before anything else: a dark desk answers 503 whoever is asking, so + // validating first would spend one chain lookup per request on a route + // that cannot do any work. The answer reveals nothing a reader of the + // public routes cannot see, which answer 503 unauthenticated too. + var token = DeskToken; + if (token == null) + { + await ctx.SendText(503, DeskNotConfigured); + return; + } + + var body = await ctx.ReadBody(); + var username = await RequireAuthedUsernameCached(ctx, body); + if (username == null) + { + return; + } + + var (payload, error) = CurationDeskWrites.Build(route, username, body); + if (payload == null) + { + await ctx.SendText(400, error ?? "Invalid request"); + return; + } + + // The client address rides along only where the backend uses it (the + // recommendation meta ping). Same source as the signup path: the + // proxy-set header, never a forwarded-for chain a client can extend. + var headers = DeskHeaders(token, route.ForwardClientAddress ? SignupClientIp(ctx) : null); + + ctx.Response.Headers.CacheControl = "no-store"; + await Upstream.Pipe(DeskUpstream(route.UpstreamPath, HttpMethod.Post, headers, payload), ctx); + } + + /// + /// with a short memo of successful + /// validations, keyed by the SHA-256 of the code. The validation itself is + /// unchanged and still decides every miss; only its positive answer is + /// remembered, for . A failed validation is + /// never stored, so a probe costs the same as before and a code that stops + /// validating is refused on its next miss. Desk routes only. + /// + public static async Task RequireAuthedUsernameCached(HttpContext ctx, JsonObject body) + { + var code = body.Str("code"); + var memoKey = string.IsNullOrEmpty(code) ? null : DeskAuthMemoPrefix + Sha256Hex(code); + + if (memoKey != null && MemCache.Get(memoKey) is { Length: > 0 } remembered) + { + return remembered; + } + + var username = await DeskValidateCode(body); + if (string.IsNullOrEmpty(username)) + { + await ctx.SendText(401, "Unauthorized"); + return null; + } + + if (memoKey != null) + { + MemCache.Set(memoKey, username, DeskAuthMemoSeconds); + } + return username; + } + + private static string Sha256Hex(string value) => + Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(value))); +} + +/// +/// Whitelist, clamp and order the query of the public desk reads. +/// +/// Every accepted parameter has a fixed position and a default that is dropped, +/// so `?limit=25&sort=newest&x=1` and an empty query are the same +/// upstream URL, the same memo entry and the same shared-cache key. Unknown +/// names and unusable values are dropped, never errors: a public read should +/// answer the nearest sensible question rather than 400 on a typo. +/// +public static class CurationDeskQuery +{ + public const int DefaultLimit = 25; + public const int MaxLimit = 50; + public const int MaxWords = 50000; + + // Anchored with \A and \z (a `$` would also match before a trailing + // newline) and written with explicit digit classes: .NET's `\d` matches + // every Unicode decimal digit, so `hive-\d{5,6}` accepts Arabic-Indic or + // Devanagari digits that name no community here. + private static readonly Regex CursorPattern = new(@"\A[A-Za-z0-9_.:-]{1,80}\z", RegexOptions.Compiled); + private static readonly Regex CommunityPattern = new(@"\Ahive-[0-9]{5,6}\z", RegexOptions.Compiled); + + /// Random-order seed: one per browser session, roster feed only. + private static readonly Regex SeedPattern = new(@"\A[a-z0-9]{8,16}\z", RegexOptions.Compiled); + + public static readonly IReadOnlySet FeedSorts = new HashSet { "queue", "newest", "unique" }; + public static readonly IReadOnlySet RecommendationSorts = new HashSet { "unique", "newest" }; + public static readonly IReadOnlySet Views = + new HashSet { "queue", "latest", "new-authors", "recommended", "curated", "all" }; + public static readonly IReadOnlySet Apps = new HashSet { "all", "ecency", "peakd", "other" }; + public static readonly IReadOnlySet Windows = new HashSet { "full", "half", "eighth", "locked", "all" }; + + /// + /// Emission order of the feed parameters. Fixed so that the memo key and + /// the shared-cache key are stable regardless of how a client orders them. + /// + public static readonly string[] FeedOrder = + { + "cursor", "limit", "sort", "view", "app", "community", "window", "rep_min", "rep_max", + "min_words", "max_words", "has_images", "new_authors", "recommended", "hide_curated", + }; + + /// Route 1 (feed). + public static List> NormalizeFeed(IEnumerable> raw) + { + var q = First(raw); + var kept = new Dictionary(); + + if (q.TryGetValue("cursor", out var cursor) && CursorPattern.IsMatch(cursor)) + { + kept["cursor"] = cursor; + } + + var limit = ClampInt(q, "limit", 1, MaxLimit); + if (limit is { } l && l != DefaultLimit) + { + kept["limit"] = l.ToString(); + } + + // The public default is newest; an unknown sort (random and anything + // else the roster feed may accept) falls back to it and so disappears. + var sort = q.TryGetValue("sort", out var s) && FeedSorts.Contains(s) ? s : "newest"; + if (sort != "newest") + { + kept["sort"] = sort; + } + + if (q.TryGetValue("view", out var view) && Views.Contains(view)) + { + kept["view"] = view; + } + + if (q.TryGetValue("app", out var app) && Apps.Contains(app) && app != "all") + { + kept["app"] = app; + } + + if (q.TryGetValue("community", out var community) && CommunityPattern.IsMatch(community)) + { + kept["community"] = community; + } + + if (q.TryGetValue("window", out var window) && Windows.Contains(window) && window != "all") + { + kept["window"] = window; + } + + // Range floors at their minimum and ceilings at their maximum select + // everything, so they are the same question as leaving them out. + if (ClampInt(q, "rep_min", 0, 100) is { } repMin && repMin != 0) + { + kept["rep_min"] = repMin.ToString(); + } + if (ClampInt(q, "rep_max", 0, 100) is { } repMax && repMax != 100) + { + kept["rep_max"] = repMax.ToString(); + } + if (ClampInt(q, "min_words", 0, MaxWords) is { } minWords && minWords != 0) + { + kept["min_words"] = minWords.ToString(); + } + if (ClampInt(q, "max_words", 0, MaxWords) is { } maxWords && maxWords != MaxWords) + { + kept["max_words"] = maxWords.ToString(); + } + + if (Flag(q, "has_images") == true) kept["has_images"] = "1"; + if (Flag(q, "new_authors") == true) kept["new_authors"] = "1"; + + // sort=unique already means "recommended posts only", so the explicit + // flag adds nothing there and would only split the memo. + if (Flag(q, "recommended") == true && sort != "unique") kept["recommended"] = "1"; + + // hide_curated defaults to on; only switching it off says anything. + if (Flag(q, "hide_curated") == false) kept["hide_curated"] = "0"; + + return Ordered(kept, FeedOrder); + } + + /// Route 4 (recommendations): cursor, limit, sort in unique|newest. + public static List> NormalizeRecommendations(IEnumerable> raw) + { + var q = First(raw); + var kept = new Dictionary(); + + if (q.TryGetValue("cursor", out var cursor) && CursorPattern.IsMatch(cursor)) + { + kept["cursor"] = cursor; + } + + var limit = ClampInt(q, "limit", 1, MaxLimit); + if (limit is { } l && l != DefaultLimit) + { + kept["limit"] = l.ToString(); + } + + if (q.TryGetValue("sort", out var sort) && RecommendationSorts.Contains(sort)) + { + kept["sort"] = sort; + } + + return Ordered(kept, new[] { "cursor", "limit", "sort" }); + } + + /// The upstream endpoint with the normalized query appended. + public static string Endpoint(string path, IEnumerable> query) => + Upstream.AppendQuery(path, query); + + private static Dictionary First(IEnumerable> raw) + { + var q = new Dictionary(StringComparer.Ordinal); + foreach (var (key, value) in raw) + { + q.TryAdd(key, value); + } + return q; + } + + private static List> Ordered(Dictionary kept, string[] order) + { + var list = new List>(kept.Count); + foreach (var name in order) + { + if (kept.TryGetValue(name, out var value)) + { + list.Add(new KeyValuePair(name, value)); + } + } + return list; + } + + /// Integer within [min, max], clamped; null when absent or not a number. + private static int? ClampInt(Dictionary q, string key, int min, int max) => + q.TryGetValue(key, out var raw) ? ClampValue(raw, min, max) : null; + + /// + /// for a value that did not come from a query string: + /// the roster feed reads the same names out of a JSON body and must clamp + /// them the same way, or the two feeds answer different questions. + /// + /// Parsed as a double rather than an int so that a value outside the int + /// range still clamps into the range: `limit=99999999999` asks for as many + /// rows as there are, and 50 is the right answer to that, while dropping it + /// would silently serve the default page size instead. Only text that is not + /// a plain signed integer is refused, so `1e6`, `12.5` and `abc` are dropped + /// and fall back to the default the same way they did before. + /// + public static int? ClampValue(string? raw, int min, int max) + { + if (raw == null + || !double.TryParse(raw, System.Globalization.NumberStyles.AllowLeadingSign, + System.Globalization.CultureInfo.InvariantCulture, out var value) + || double.IsNaN(value)) + { + return null; + } + return (int)Math.Clamp(value, min, max); + } + + /// Opaque paging cursor grammar; shared with the roster feed body. + public static bool IsCursor(string? value) => value != null && CursorPattern.IsMatch(value); + + /// A `hive-NNNNN` community name; shared with the roster feed body. + public static bool IsCommunity(string? value) => value != null && CommunityPattern.IsMatch(value); + + /// A random-order seed; the roster feed is the only route that takes one. + public static bool IsSeed(string? value) => value != null && SeedPattern.IsMatch(value); + + /// "1" -> true, "0" -> false, anything else -> null (dropped). + private static bool? Flag(Dictionary q, string key) => + q.TryGetValue(key, out var raw) ? raw switch { "1" => true, "0" => false, _ => null } : null; +} + +/// +/// Payload rules for the signed desk writes: which body keys reach the backend +/// and the few values this service refuses outright rather than forward. +/// +public static class CurationDeskWrites +{ + public sealed record Route(string UpstreamPath, string[] Keys, bool ForwardClientAddress = false); + + public static readonly IReadOnlySet MarkStates = + new HashSet { "reviewed", "snoozed", "flagged", "noted" }; + public static readonly IReadOnlySet CursorActions = new HashSet { "advance", "rewind" }; + public static readonly IReadOnlySet DismissActions = new HashSet { "dismiss", "restore" }; + public static readonly IReadOnlySet UaClasses = new HashSet { "web", "mobile" }; + public static readonly IReadOnlySet RosterSorts = new HashSet { "queue", "newest", "unique", "random" }; + + /// + /// Views the roster feed takes: the public ones plus `excluded`, which is + /// the only place an excluded row is ever listed. + /// + public static readonly IReadOnlySet RosterViews = + new HashSet(CurationDeskQuery.Views, StringComparer.Ordinal) { "excluded" }; + + /// How many post ids one tick may name per list. + public const int MaxTickIds = 100; + + // \A and \z for the same reason as the name patterns above: `$` would let + // a trailing newline through. + private static readonly Regex TrxIdPattern = new(@"\A[0-9a-f]{40}\z", RegexOptions.Compiled); + + public static readonly Route RosterFeed = new("curation/desk/roster-feed", new[] + { + "cursor", "limit", "view", "app", "community", "min_words", "sort", "seed", "window", "rep_min", + "rep_max", "max_words", "has_images", "new_authors", "recommended", "flagged", "hide_curated", + "hide_reviewed", "hide_snoozed", + }); + + public static readonly Route Tick = new("curation/desk/tick", new[] { "since", "need", "visible" }); + + public static readonly Route Mark = new("curation/desk/marks", + new[] { "author", "permlink", "state", "reason", "note", "snooze_until" }); + + public static readonly Route MarkClear = new("curation/desk/marks/clear", new[] { "author", "permlink" }); + + public static readonly Route Marks = new("curation/desk/marks/list", new[] { "state", "cursor", "limit" }); + + public static readonly Route Cursor = new("curation/desk/cursors", new[] { "post_id", "action", "reason" }); + + public static readonly Route RecommendMeta = new("curation/desk/recommendations/meta", + new[] { "author", "permlink", "trx_id", "ua_class" }, ForwardClientAddress: true); + + public static readonly Route RecommendationDismiss = new("curation/desk/recommendations/dismiss", + new[] { "author", "permlink", "action" }); + + /// + /// The upstream body: the validated username plus the route's whitelisted + /// keys copied as the client sent them. `username` and `code` are never in + /// a whitelist, so a forged identity cannot ride along. Returns no payload + /// and a reason when a value the backend would only reject is caught here. + /// + public static (JsonObject? Payload, string? Error) Build(Route route, string username, JsonObject body) + { + var error = Validate(route, body); + if (error != null) + { + return (null, error); + } + + var payload = new JsonObject { ["username"] = username }; + foreach (var key in route.Keys) + { + if (key is "username" or "code") + { + continue; + } + CopyIfPresent(payload, body, key); + } + + if (ReferenceEquals(route, RosterFeed)) + { + NormalizeRosterFeed(payload, body); + } + + if (ReferenceEquals(route, Tick)) + { + // The backend caps both lists at this many ids; truncating here + // keeps a client bug from turning one tick into thousands of + // primary-key probes on the way to the same 400. + Truncate(payload, "need", MaxTickIds); + Truncate(payload, "visible", MaxTickIds); + } + + if (ReferenceEquals(route, RecommendMeta) && payload.ContainsKey("ua_class") + && !(body.Str("ua_class") is { } ua && UaClasses.Contains(ua))) + { + payload.Remove("ua_class"); + } + + return (payload, null); + } + + /// + /// The roster feed carries the filter names of the public feed, so it gets + /// the public feed's value rules: an out-of-range number is clamped rather + /// than forwarded, and a value outside an allowlist or a pattern is dropped + /// so the backend applies its default. Both feeds then answer the same + /// question for the same request, and a typo cannot ask for a query plan + /// nobody sized. + /// + private static void NormalizeRosterFeed(JsonObject payload, JsonObject body) + { + // An unknown sort is not an error for a read: drop it and let the + // backend apply its default, as the public feed does. + var sort = body.Str("sort"); + if (sort == null || !RosterSorts.Contains(sort)) + { + payload.Remove("sort"); + } + + // seed only means something to the random order; for any other sort it + // is noise that would make two identical feeds look different, and a + // seed outside the grammar is not a seed the backend can hash with. + if (sort != "random" || !CurationDeskQuery.IsSeed(body.Str("seed"))) + { + payload.Remove("seed"); + } + + KeepAllowed(payload, "view", RosterViews); + KeepAllowed(payload, "app", CurationDeskQuery.Apps); + KeepAllowed(payload, "window", CurationDeskQuery.Windows); + KeepMatching(payload, "cursor", CurationDeskQuery.IsCursor); + KeepMatching(payload, "community", CurationDeskQuery.IsCommunity); + + Clamp(payload, "limit", 1, CurationDeskQuery.MaxLimit); + Clamp(payload, "rep_min", 0, 100); + Clamp(payload, "rep_max", 0, 100); + Clamp(payload, "min_words", 0, CurationDeskQuery.MaxWords); + Clamp(payload, "max_words", 0, CurationDeskQuery.MaxWords); + } + + /// Drop a field whose value is not one of . + private static void KeepAllowed(JsonObject payload, string key, IReadOnlySet allowed) + { + if (payload.ContainsKey(key) && !(JsVal.AsString(payload[key]) is { } value && allowed.Contains(value))) + { + payload.Remove(key); + } + } + + /// Drop a field whose value does not match . + private static void KeepMatching(JsonObject payload, string key, Func matches) + { + if (payload.ContainsKey(key) && !matches(JsVal.AsString(payload[key]))) + { + payload.Remove(key); + } + } + + /// + /// Clamp a whole-number field into [min, max], accepting the number or its + /// string spelling; a value that is neither is dropped rather than forwarded. + /// + /// These names count rows, reputations and words, so only an integral value + /// is one of them: `1.9` is not "1", it is a client sending something else, + /// and truncating it would forward a filter nobody asked for. Strings go + /// through the query-string parser, so a body and a query string accept the + /// same spellings. A number is judged by its value, not its spelling, since + /// JSON parsing keeps no spelling: `1e6` and `1000000` are the same number + /// and clamp to the same bound, exactly as `1000000` does in a query string. + /// + private static void Clamp(JsonObject payload, string key, int min, int max) + { + if (!payload.ContainsKey(key)) + { + return; + } + var node = payload[key]; + int? value = JsVal.AsNumber(node) is { } number + ? (double.IsInteger(number) ? (int?)Math.Clamp(number, min, max) : null) + : CurationDeskQuery.ClampValue(JsVal.AsString(node), min, max); + if (value is { } clamped) + { + payload[key] = clamped; + } + else + { + payload.Remove(key); + } + } + + /// Keep at most elements of an array field. + private static void Truncate(JsonObject payload, string key, int max) + { + if (payload[key] is not JsonArray array || array.Count <= max) + { + return; + } + var kept = new JsonArray(); + for (var i = 0; i < max; i++) + { + kept.Add(array[i]?.DeepClone()); + } + payload[key] = kept; + } + + private static string? Validate(Route route, JsonObject body) + { + if (ReferenceEquals(route, Mark)) + { + return RequireAuthorPermlink(body) ?? RequireOneOf(body, "state", MarkStates); + } + if (ReferenceEquals(route, MarkClear)) + { + return RequireAuthorPermlink(body); + } + if (ReferenceEquals(route, Marks)) + { + return body.ContainsKey("state") ? RequireOneOf(body, "state", MarkStates) : null; + } + if (ReferenceEquals(route, Cursor)) + { + if (body.Field("post_id") is not JsonValue idValue + || idValue.GetValueKind() is not (JsonValueKind.Number or JsonValueKind.String)) + { + return "post_id required"; + } + return RequireOneOf(body, "action", CursorActions); + } + if (ReferenceEquals(route, RecommendMeta)) + { + var missing = RequireAuthorPermlink(body); + if (missing != null) return missing; + if (body.ContainsKey("trx_id")) + { + // Optional and informational, but a value that is not a + // transaction id is a client bug worth surfacing, not storing. + if (body.Str("trx_id") is not { } trx || !TrxIdPattern.IsMatch(trx)) + { + return "invalid trx_id"; + } + } + return null; + } + if (ReferenceEquals(route, RecommendationDismiss)) + { + return RequireAuthorPermlink(body) ?? RequireOneOf(body, "action", DismissActions); + } + return null; + } + + /// + /// Copy a body field only when the key is present (absent == undefined, which + /// JSON.stringify omits; a present null is kept), as the other passthroughs do. + /// + private static void CopyIfPresent(JsonObject target, JsonObject body, string key) + { + if (body.TryGetPropertyValue(key, out var value)) + { + target[key] = value?.DeepClone(); + } + } + + private static string? RequireAuthorPermlink(JsonObject body) => + RequireNonEmpty(body, "author") ?? RequireNonEmpty(body, "permlink"); + + private static string? RequireNonEmpty(JsonObject body, string key) => + body.Str(key) is { Length: > 0 } ? null : $"{key} required"; + + private static string? RequireOneOf(JsonObject body, string key, IReadOnlySet allowed) => + body.Str(key) is { } value && allowed.Contains(value) ? null : $"invalid {key}"; +} + +/// +/// What a public desk response may carry. The backend is specified to omit +/// these already; this is the fence on this side of the boundary, so a backend +/// change that starts leaking a curator's identity or a hashed address into a +/// publicly cached body is stopped here rather than served for its s-maxage. +/// +public static class CurationDeskPublicPayload +{ + public static readonly IReadOnlySet PrivateKeys = new HashSet(StringComparer.Ordinal) + { + "set_by", "set_at", "active_curators", "trail_alerts", "note", "excluded_reason", "ip_hash", "key_id", + }; + + /// + /// Remove every private key anywhere in the tree. Returns whether anything + /// was removed, so an untouched body can be served as the bytes it came in. + /// + public static bool Strip(JsonNode? node) + { + var removed = false; + switch (node) + { + case JsonObject obj: + foreach (var key in obj.Select(kv => kv.Key).ToArray()) + { + if (PrivateKeys.Contains(key)) + { + obj.Remove(key); + removed = true; + } + else + { + removed |= Strip(obj[key]); + } + } + break; + case JsonArray arr: + foreach (var item in arr) + { + removed |= Strip(item); + } + break; + } + return removed; + } + + /// + /// The body to memoize and serve: the upstream bytes as received when they + /// were already clean, otherwise the stripped tree re-serialized once. + /// + public static byte[] ToPublicBytes(UpstreamResponse r) + { + if (!Strip(r.Json)) + { + return r.Bytes.Length > 0 ? r.Bytes : Encoding.UTF8.GetBytes(JsJson.Stringify(r.Json)); + } + return Encoding.UTF8.GetBytes(JsJson.Stringify(r.Json)); + } +} + +/// +/// Byte memo for the public desk reads, keyed by the normalized upstream +/// endpoint. Two bounded stores: the fresh one holds a body for the s-maxage of +/// its route, the last-good one holds the most recent 200 for longer so an +/// upstream error answers with something recent rather than an error page. +/// Bytes, not trees: a hit is a lookup and a write, whatever the read rate. +/// +public static class CurationDeskMemo +{ + /// Budget of each store (DESK_MEMO_BYTES; 64 MiB by default). + internal static readonly long BudgetBytes = Config.DeskMemoBytes; + + /// + /// How long a last-good body stays eligible as a fallback. Long enough to + /// ride out a backend restart, short enough that a stale feed does not + /// outlive an outage by much. + /// + internal const int LastGoodTtlMs = 10 * 60 * 1000; + + /// + /// How long a request waits for another request's fill of the same key. A + /// fill is one upstream call, so this only bounds the case where that call + /// is itself timing out; the waiter then answers from last-good or 504. + /// + internal static readonly TimeSpan FillWait = TimeSpan.FromMilliseconds(Upstream.DefaultTimeoutMs + 1000); + + internal static BytesCache Fresh = new(BudgetBytes); + internal static BytesCache LastGood = new(BudgetBytes); + + /// Wall clock in milliseconds behind the fill times below. + internal static readonly Func SystemClock = () => DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); + + /// + /// The clock the fill times are read from. Replaceable so a test can age an + /// entry rather than wait for it; production never moves it. + /// + internal static Func NowMs = SystemClock; + + /// + /// One fill of one key at a time. A reader takes the key's gate, waits on + /// its semaphore, fills and hands the gate back. + /// + /// Users are counted rather than inferred from the semaphore: a reader that + /// has been handed the gate but has not reached its wait yet is a user, and + /// dropping the entry under it (the semaphore looks free, because that + /// reader has not taken it) would let the next reader create a replacement + /// and fill the same key beside it. Two fills of one key can then finish out + /// of order and store the older answer last. + /// + internal sealed class Gate + { + /// Admission to the fill; one holder at a time. + internal readonly SemaphoreSlim Semaphore = new(1, 1); + + /// Readers holding this gate. Guarded by . + internal int Users; + } + + private static readonly Dictionary Gates = new(StringComparer.Ordinal); + private static readonly object GateLock = new(); + + /// + /// A stored entry carries when it was filled as well as what it is, so a hit + /// can say how much of its shared window is left. Both travel in the one tag + /// the byte cache keeps beside the bytes, so they are evicted together and + /// no side table can outlive or contradict an entry. + /// + private const char TagSeparator = '|'; + + private const string DefaultContentType = "application/json; charset=utf-8"; + + public static bool TryGetFresh(string key, out byte[] bytes, out string contentType, out int ageSeconds) => + Read(Fresh, key, out bytes, out contentType, out ageSeconds); + + public static bool TryGetLastGood(string key, out byte[] bytes, out string contentType, out int ageSeconds) => + Read(LastGood, key, out bytes, out contentType, out ageSeconds); + + public static void Store(string key, byte[] bytes, string contentType, int ttlSeconds) + { + var tag = NowMs().ToString(CultureInfo.InvariantCulture) + TagSeparator + contentType; + Fresh.Set(key, bytes, ttlSeconds * 1000, tag); + LastGood.Set(key, bytes, LastGoodTtlMs, tag); + } + + private static bool Read(BytesCache cache, string key, out byte[] bytes, out string contentType, out int ageSeconds) + { + var hit = cache.TryGet(key, out bytes, out var tag); + var split = tag?.IndexOf(TagSeparator) ?? -1; + contentType = split >= 0 ? tag![(split + 1)..] : tag ?? DefaultContentType; + ageSeconds = split > 0 && long.TryParse(tag.AsSpan(0, split), NumberStyles.None, CultureInfo.InvariantCulture, out var filledAtMs) + ? (int)Math.Clamp((NowMs() - filledAtMs) / 1000, 0, int.MaxValue) + : 0; + return hit; + } + + /// + /// The gate of a key, counting this caller as one of its users. Every call + /// must be paired with a , whether or not the + /// caller went on to take the semaphore. + /// + internal static Gate GateFor(string key) + { + lock (GateLock) + { + if (!Gates.TryGetValue(key, out var gate)) + { + gate = new Gate(); + Gates[key] = gate; + } + gate.Users++; + return gate; + } + } + + /// + /// Give a gate back. The entry is dropped only when the last user leaves and + /// the key still maps to this same gate, so a scan over many distinct keys + /// leaves no semaphore per key behind while no in-flight reader ever loses + /// the gate it is about to wait on. + /// + internal static void ReleaseGate(string key, Gate gate) + { + lock (GateLock) + { + if (--gate.Users > 0) + { + return; + } + if (Gates.TryGetValue(key, out var current) && ReferenceEquals(current, gate)) + { + Gates.Remove(key); + } + } + } + + /// Gates currently held, for the tests that pin the cleanup. + internal static int GateCount + { + get { lock (GateLock) { return Gates.Count; } } + } + + internal static void ResetForTests() + { + Fresh = new BytesCache(BudgetBytes); + LastGood = new BytesCache(BudgetBytes); + NowMs = SystemClock; + lock (GateLock) + { + Gates.Clear(); + } + } +} diff --git a/dotnet/EcencyApi/Handlers/Routes.cs b/dotnet/EcencyApi/Handlers/Routes.cs index 9348c0a7..33fb4683 100644 --- a/dotnet/EcencyApi/Handlers/Routes.cs +++ b/dotnet/EcencyApi/Handlers/Routes.cs @@ -173,6 +173,23 @@ public static void Map(WebApplication app) app.MapPost("/private-api/chats-update", PrivateApi.ChatsUpdate); app.MapPost("/private-api/channel-add", PrivateApi.ChannelAdd); + // ---- Curation desk (see PrivateApi.CurationDesk.cs) ---- + // Literal segments under curation-desk/: a different first segment from + // the curation/{duration} route above, so the two can never collide. + app.MapGet("/private-api/curation-desk/feed", PrivateApi.CurationDeskFeed); + app.MapGet("/private-api/curation-desk/status", PrivateApi.CurationDeskStatus); + app.MapGet("/private-api/curation-desk/roster", PrivateApi.CurationDeskRoster); + app.MapGet("/private-api/curation-desk/recommendations", PrivateApi.CurationDeskRecommendations); + app.MapGet("/private-api/curation-desk/post/{author}/{permlink}", PrivateApi.CurationDeskPost); + app.MapPost("/private-api/curation-desk/roster-feed", PrivateApi.CurationDeskRosterFeed); + app.MapPost("/private-api/curation-desk/tick", PrivateApi.CurationDeskTick); + app.MapPost("/private-api/curation-desk/mark", PrivateApi.CurationDeskMark); + app.MapPost("/private-api/curation-desk/mark-clear", PrivateApi.CurationDeskMarkClear); + app.MapPost("/private-api/curation-desk/marks", PrivateApi.CurationDeskMarks); + app.MapPost("/private-api/curation-desk/cursor", PrivateApi.CurationDeskCursor); + app.MapPost("/private-api/curation-desk/recommend-meta", PrivateApi.CurationDeskRecommendMeta); + app.MapPost("/private-api/curation-desk/recommendation-dismiss", PrivateApi.CurationDeskRecommendationDismiss); + // ---- SSR RPC cache (internal, header-gated; see SsrRpc.cs) ---- app.MapPost("/private-api/ssr/rpc", SsrRpc.Rpc); app.MapGet("/private-api/ssr/stats", SsrRpc.Stats); diff --git a/dotnet/EcencyApi/Infrastructure/BytesCache.cs b/dotnet/EcencyApi/Infrastructure/BytesCache.cs index 26fce2df..3ecf3704 100644 --- a/dotnet/EcencyApi/Infrastructure/BytesCache.cs +++ b/dotnet/EcencyApi/Infrastructure/BytesCache.cs @@ -20,6 +20,9 @@ private sealed class Entry public required byte[] Bytes; public required long ExpiresAtMs; public required LinkedListNode Node; + // Small caller-defined label stored beside the bytes (a content type, + // say), so a value that is not self-describing can be served as it came. + public string? Tag; } private readonly Dictionary _map = new(); @@ -55,7 +58,13 @@ public BytesCache(long budgetBytes) private static long NowMs => Environment.TickCount64; /// Fresh entry or nothing; an expired entry is dropped on the way. - public bool TryGet(string key, out byte[] bytes) + public bool TryGet(string key, out byte[] bytes) => TryGet(key, out bytes, out _); + + /// + /// that also returns the tag the + /// entry was stored with (null when it had none). + /// + public bool TryGet(string key, out byte[] bytes, out string? tag) { lock (_lock) { @@ -66,20 +75,24 @@ public bool TryGet(string key, out byte[] bytes) _lru.Remove(entry.Node); _lru.AddLast(entry.Node); bytes = entry.Bytes; + tag = entry.Tag; return true; } RemoveLocked(key, entry); } } bytes = Array.Empty(); + tag = null; return false; } /// /// Store a value for . A value larger than the whole - /// budget is not stored (it would evict everything for one reader). + /// budget is not stored (it would evict everything for one reader). The + /// optional travels with the bytes and comes back from + /// . /// - public void Set(string key, byte[] bytes, int ttlMs) + public void Set(string key, byte[] bytes, int ttlMs, string? tag = null) { if (ttlMs <= 0 || bytes.Length > Budget) return; lock (_lock) @@ -89,7 +102,7 @@ public void Set(string key, byte[] bytes, int ttlMs) RemoveLocked(key, existing); } var node = _lru.AddLast(key); - var entry = new Entry { Bytes = bytes, ExpiresAtMs = NowMs + ttlMs, Node = node }; + var entry = new Entry { Bytes = bytes, ExpiresAtMs = NowMs + ttlMs, Node = node, Tag = tag }; _map[key] = entry; _byExpiry.Add((entry.ExpiresAtMs, key)); _bytes += bytes.Length; diff --git a/dotnet/EcencyApi/Infrastructure/CachePolicy.cs b/dotnet/EcencyApi/Infrastructure/CachePolicy.cs index b8057b70..5733adef 100644 --- a/dotnet/EcencyApi/Infrastructure/CachePolicy.cs +++ b/dotnet/EcencyApi/Infrastructure/CachePolicy.cs @@ -1,3 +1,5 @@ +using System.Globalization; + namespace EcencyApi.Infrastructure; /// @@ -36,6 +38,100 @@ public static class CachePolicy /// public const string PostTips = "public, max-age=60, stale-while-revalidate=600"; + /// + /// Curation desk public reads. `max-age=0` makes browsers revalidate on every + /// poll while `s-maxage` lets shared caches absorb the polling; the in-process + /// memo of each route uses the same s-maxage as its TTL (see + /// ), so the two layers never disagree on freshness. + /// A body served from that memo goes out through , which + /// hands the shared cache only what is left of the window rather than a fresh + /// one, and a last-good body through . + /// The feed and the recommendations list move with every new post; status is + /// the poll target and stays short; the roster changes when a curator is added + /// or promoted, so it can stay put for minutes; a single post's recommenders + /// are optimistic on the client and confirmed from here. + /// + public const string CurationDeskFeed = "public, max-age=0, s-maxage=30"; + public const string CurationDeskStatus = "public, max-age=0, s-maxage=15"; + public const string CurationDeskRoster = "public, max-age=0, s-maxage=600"; + public const string CurationDeskRecommendations = "public, max-age=0, s-maxage=30"; + public const string CurationDeskPost = "public, max-age=0, s-maxage=15"; + + /// + /// The `s-maxage` of a policy in seconds, or its `max-age` when it has no + /// shared-cache directive. Handlers that memoize a response derive their TTL + /// from this so the memo can never outlive what the policy promises. + /// + public static int SharedMaxAge(string policy) + { + var tokens = policy.Split(',', StringSplitOptions.TrimEntries); + foreach (var name in new[] { "s-maxage=", "max-age=" }) + { + foreach (var token in tokens) + { + if (token.StartsWith(name, StringComparison.Ordinal) + && int.TryParse(token.AsSpan(name.Length), out var seconds) && seconds >= 0) + { + return seconds; + } + } + } + throw new ArgumentException("policy carries no max-age", nameof(policy)); + } + + /// + /// Seconds a shared cache may keep a body that was served after the upstream + /// failed. It is still an answer the backend gave, so it stays publicly + /// cacheable rather than being refetched by every reader at once, but a + /// recovered backend has to reach readers within a poll or two rather than at + /// the end of a full window. + /// + public const int StaleSharedMaxAge = 5; + + /// + /// The same policy with its shared window reduced to what is left of it after + /// . + /// + /// A memoized body is served for the rest of its TTL, not for a fresh one: + /// without this a roster read from the memo a second before it lapses would + /// license a shared cache to hold that body for another whole window, so the + /// two layers together could serve one answer for nearly twice its TTL. The + /// floor of one second keeps the response cacheable at all, since the memo is + /// about to refill anyway. + /// + public static string Aged(string policy, int ageSeconds) => + ageSeconds <= 0 ? policy : WithSharedMaxAge(policy, Math.Max(1, SharedMaxAge(policy) - ageSeconds)); + + /// + /// The same policy cut down to , for a body + /// served because the upstream call failed. Never longer than the policy + /// itself, so a route with an even shorter window keeps its own. + /// + public static string Stale(string policy) => + WithSharedMaxAge(policy, Math.Min(StaleSharedMaxAge, SharedMaxAge(policy))); + + /// + /// Rewrite the `s-maxage` of a policy, leaving every other directive alone. + /// A policy that carries none gains one: its shared window was its `max-age` + /// (see ), and this caps that without touching what + /// browsers were told. + /// + private static string WithSharedMaxAge(string policy, int seconds) + { + var value = "s-maxage=" + seconds.ToString(CultureInfo.InvariantCulture); + var tokens = policy.Split(',', StringSplitOptions.TrimEntries); + var replaced = false; + for (var i = 0; i < tokens.Length; i++) + { + if (tokens[i].StartsWith("s-maxage=", StringComparison.Ordinal)) + { + tokens[i] = value; + replaced = true; + } + } + return string.Join(", ", replaced ? tokens : tokens.Append(value)); + } + /// /// A policy applies only to a successful response. Handlers attach it before /// the upstream call resolves, and can still turn diff --git a/dotnet/EcencyApi/Infrastructure/HttpContextExtensions.cs b/dotnet/EcencyApi/Infrastructure/HttpContextExtensions.cs index 4be22624..9919e1fb 100644 --- a/dotnet/EcencyApi/Infrastructure/HttpContextExtensions.cs +++ b/dotnet/EcencyApi/Infrastructure/HttpContextExtensions.cs @@ -1,3 +1,4 @@ +using System.Globalization; using System.Text.Json; using System.Text.Json.Nodes; diff --git a/dotnet/EcencyApi/Infrastructure/Upstream.cs b/dotnet/EcencyApi/Infrastructure/Upstream.cs index e5add5d0..4accea0e 100644 --- a/dotnet/EcencyApi/Infrastructure/Upstream.cs +++ b/dotnet/EcencyApi/Infrastructure/Upstream.cs @@ -21,6 +21,14 @@ public sealed class UpstreamResponse /// Set when the body wasn't parseable JSON (axios keeps the raw string). public string? RawText { get; init; } + /// + /// The body exactly as received, before parsing. Handlers that memoize a + /// response keep these bytes rather than the parsed tree, so a hit is a + /// dictionary lookup and a write instead of a clone and a re-serialization. + /// Empty when a caller built the response without a body. + /// + public byte[] Bytes { get; init; } = Array.Empty(); + public required HttpResponseHeaders2 Headers { get; init; } public bool BodyIsJson => RawText == null; @@ -204,11 +212,11 @@ private static async Task ReadUpstreamResponse(HttpResponseMes { AllowTrailingCommas = false, }); - return new UpstreamResponse { Status = status, Json = node, Headers = respHeaders }; + return new UpstreamResponse { Status = status, Json = node, Headers = respHeaders, Bytes = bytes }; } catch (JsonException) { - return new UpstreamResponse { Status = status, RawText = text, Headers = respHeaders }; + return new UpstreamResponse { Status = status, RawText = text, Headers = respHeaders, Bytes = bytes }; } } diff --git a/dotnet/parity/driver.py b/dotnet/parity/driver.py index 2ee86618..c03a52bb 100644 --- a/dotnet/parity/driver.py +++ b/dotnet/parity/driver.py @@ -241,6 +241,28 @@ def norm_body(text): "no behavior the reference ever had is changing." ) +CURATION_DESK_DIVERGENCE = ( + "Curation desk gateway route added after the port. The reference build has no such " + "route and answers 404 (POST) or the unmatched-GET template page; this one answers " + "503 while its shared secret is unconfigured (before it validates anything), 401 " + "for a missing or invalid signed code, 400 for a rejected path or body, and " + "otherwise proxies. Deterministic and " + "additive -- no behavior the reference ever had is changing." +) + +CURATION_DESK_ROUTES = [ + "/private-api/curation-desk/feed::get", + "/private-api/curation-desk/status::get", + "/private-api/curation-desk/roster::get", + "/private-api/curation-desk/recommendations::get", + "/private-api/curation-desk/post/x/x::get", +] + [ + f"/private-api/curation-desk/{route}::{case}" + for route in ("roster-feed", "tick", "mark", "mark-clear", "marks", "cursor", + "recommend-meta", "recommendation-dismiss") + for case in ("min", "pop", "badcode") +] + # Cases where the C# port intentionally differs from Node (Node bugs the port fixes). KNOWN_DIVERGENCES = { "/auth-api/hs-token-refresh::min": @@ -308,6 +330,9 @@ def norm_body(text): FAVORITE_TAGS_DIVERGENCE, "/private-api/favorite-tags-delete::badcode": FAVORITE_TAGS_DIVERGENCE, + # Curation desk routes, added after the port: five public reads and eight signed + # writes. Same shape as the entries above; one entry per generated case. + **{case: CURATION_DESK_DIVERGENCE for case in CURATION_DESK_ROUTES}, } # Deliberately NOT listed above: /wallet-api/portfolio-v2::pop, whose HP action list