From efe689a44fd9d1ef931d8b4a362a33c65e78328a Mon Sep 17 00:00:00 2001
From: Anthony Ettinger <anthony@chovy.com>
Date: Sun, 13 Sep 2026 05:34:00 +0000
Subject: [PATCH] A house crawler gets its own rung on the two profile routes

nichedb.dev pulls every author's OpenProfile.md into its people
collection and names itself in its user agent. On the free rung (120 a
minute, 600 an hour) a backfill of two hundred thousand authors takes
days. A request that names a house crawler AND asks for
/api/openprofiles or an author's openprofile.md is placed on a `house`
rung: 600 a minute, 36,000 an hour. Every other route and every other
caller is unchanged, and a session or a key still places the caller by
what it carries.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01S7yeJUHGxA4P5N74xnsRPQ
---
 apps/web/src/lib/tiers.js   | 42 +++++++++++++++++++++++++++++++++++++
 apps/web/test/tiers.test.js | 22 +++++++++++++++++++
 2 files changed, 64 insertions(+)

diff --git a/apps/web/src/lib/tiers.js b/apps/web/src/lib/tiers.js
index 1634ae7..00c64b4 100644
--- a/apps/web/src/lib/tiers.js
+++ b/apps/web/src/lib/tiers.js
@@ -117,8 +117,45 @@ export const TIERS = {
    * unchanged here.
    */
   pass: { name: 'pass', burst: envInt('TIER_SPONSOR_BURST', 2_000), hourly: SPONSOR_HOURLY },
+  /**
+   * A house crawler on the profile routes.
+   *
+   * nichedb.dev pulls every author's OpenProfile.md into its people collection
+   * (nichedb.dev/c/profiles). It names itself in its user agent and wants two
+   * things only: the listing at /api/openprofiles and the files the listing
+   * names. Under the free rung (120 a minute, 600 an hour) a backfill of two
+   * hundred thousand authors takes days; at 600 a minute it takes hours. The
+   * rung exists for those two routes and no other, so a stranger wearing the
+   * string gains a signed-in reader's pace on the two cheapest routes on the
+   * site and nothing anywhere else.
+   */
+  house: { name: 'house', burst: envInt('TIER_HOUSE_BURST', 600), hourly: envInt('TIER_HOUSE_HOURLY', 36_000) },
 };
 
+/** User agent prefixes of the house crawlers, as they identify themselves. */
+export const HOUSE_CRAWLERS = ['niche-db/'];
+
+/** The routes the house rung applies to, and nothing else. */
+const HOUSE_ROUTES = [/^\/api\/openprofiles$/, /^\/authors\/[^/]+\/openprofile\.md$/];
+
+/**
+ * Whether this request is a house crawler asking for a profile route.
+ *
+ * @param {Request} request
+ * @returns {boolean}
+ */
+export function isHouseCrawl(request) {
+  const ua = (request.headers.get('user-agent') ?? '').trim();
+  if (!HOUSE_CRAWLERS.some((prefix) => ua.startsWith(prefix))) return false;
+  let pathname;
+  try {
+    pathname = /** @type {any} */ (request).nextUrl?.pathname ?? new URL(request.url).pathname;
+  } catch {
+    return false;
+  }
+  return HOUSE_ROUTES.some((re) => re.test(pathname));
+}
+
 /**
  * Validated sponsor keys, by hash, with the time they were checked.
  *
@@ -204,6 +241,11 @@ export function tierFor(request) {
 
   if (/(^|;\s*)rsa_session=[^;]/.test(request.headers.get('cookie') ?? '')) return TIERS.session;
 
+  // Below the cookie and the key on purpose: a house crawler carrying either
+  // is placed by what it carries, and the string only matters when it is all
+  // the request has to say for itself.
+  if (isHouseCrawl(request)) return TIERS.house;
+
   return TIERS.anon;
 }
 
diff --git a/apps/web/test/tiers.test.js b/apps/web/test/tiers.test.js
index 10f2c7f..6c2b2a8 100644
--- a/apps/web/test/tiers.test.js
+++ b/apps/web/test/tiers.test.js
@@ -219,3 +219,25 @@ test('the tiers are separately metered, so one does not spend another', () => {
 function hashOf(token) {
   return hashToken(token);
 }
+
+test('a house crawler gets the house rung on the two profile routes and nowhere else', () => {
+  resetTierCache();
+  const ua = 'niche-db/0.1 (+https://nichedb.dev)';
+  const at = (path, agent = ua) =>
+    tierFor(new Request(`https://rssamplifier.com${path}`, { headers: agent ? { 'user-agent': agent } : {} })).name;
+  assert.equal(at('/api/openprofiles?limit=500'), 'house');
+  assert.equal(at('/authors/ada-lovelace/openprofile.md'), 'house');
+  // Same crawler, any other route: free, as before.
+  assert.equal(at('/authors/ada-lovelace'), 'anon');
+  assert.equal(at('/api/authors'), 'anon');
+  assert.equal(at('/topics'), 'anon');
+  // Same routes, any other caller: free, as before.
+  assert.equal(at('/api/openprofiles', 'curl/8.0'), 'anon');
+  assert.equal(at('/api/openprofiles', 'Mozilla/5.0 niche-db/0.1'), 'anon');
+  // A session or a key still places the caller by what it carries.
+  const r = new Request('https://rssamplifier.com/api/openprofiles', {
+    headers: { 'user-agent': ua, cookie: 'rsa_session=abc' },
+  });
+  assert.equal(tierFor(r).name, 'session');
+  assert.ok(TIERS.house.burst >= 600, 'the house rung is at least 600 a minute');
+});