fix: seed a ring from the topic's own keyword index, strongest first

packages/db/scripts/time-ring-queries.mjs

@@ -0,0 +1,61 @@
+// Read-only: time the statements the ring seed runs, against the database in
+// the environment, with their query plans. `node --env-file=<env> scripts/time-ring-queries.mjs one news`
+import { createClient } from '@libsql/client';
+
+const env = process.env;
+const db = createClient({ url: env['TURSO_DATABASE_URL'], authToken: env['TURSO_AUTH_TOKEN'] });
+const slugs = process.argv.slice(2).length ? process.argv.slice(2) : ['one'];
+
+const statements = (slug) => ({
+  label: {
+    sql: `select lk.keyword from feed_keywords lk where lk.slug = ? group by lk.keyword order by count(distinct lk.feed_id) desc, length(lk.keyword) asc, lk.keyword asc limit 1`,
+    args: [slug],
+  },
+  size_cap5: {
+    sql: `select count(*) as n from (select f.id from feeds f where f.status = 'active' and f.site_url is not null and f.site_url <> '' and exists (select 1 from feed_keywords k where k.feed_id = f.id and k.slug = ?) limit ?)`,
+    args: [slug, 5],
+  },
+  candidates_exists: {
+    sql: `select f.id from feeds f where f.status = 'active' and f.site_url is not null and f.site_url <> '' and exists (select 1 from feed_keywords k where k.feed_id = f.id and k.slug = ?) order by f.created_at asc, f.id asc limit ?`,
+    args: [slug, 100],
+  },
+  candidates_indexed: {
+    sql: `select f.id from feed_keywords k indexed by feed_keywords_slug_idx cross join feeds f on f.id = k.feed_id where k.slug = ? and f.status = 'active' and f.site_url is not null and f.site_url <> '' order by k.count desc limit ?`,
+    args: [slug, 100],
+  },
+  category_count_indexed: {
+    sql: `select count(*) as n from feed_keywords k indexed by feed_keywords_slug_idx cross join feeds f on f.id = k.feed_id where k.slug = ? and k.source = 'category' and f.status = 'active' and f.site_url is not null and f.site_url <> ''`,
+    args: [slug],
+  },
+  title_from_rollup: { sql: `select keyword from topics where slug = ?`, args: [slug] },
+  candidates_join: {
+    sql: `select f.id from feed_keywords k join feeds f on f.id = k.feed_id where k.slug = ? and f.status = 'active' and f.site_url is not null and f.site_url <> '' group by f.id order by f.created_at asc, f.id asc limit ?`,
+    args: [slug, 100],
+  },
+  category_count: {
+    sql: `select count(distinct k.feed_id) as n from feed_keywords k join feeds f on f.id = k.feed_id where k.slug = ? and k.source = 'category' and f.status = 'active' and f.site_url is not null and f.site_url <> ''`,
+    args: [slug],
+  },
+});
+
+const counts = await db.execute(`select (select count(*) from feeds) as feeds, (select count(*) from feed_keywords) as keywords`);
+console.log('feeds', counts.rows[0].feeds, 'keyword rows', counts.rows[0].keywords);
+
+for (const slug of slugs) {
+  const rows = await db.execute({ sql: `select count(*) as n from feed_keywords where slug = ?`, args: [slug] });
+  console.log(`\n== ${slug}: ${rows.rows[0].n} keyword rows`);
+  for (const [name, st] of Object.entries(statements(slug))) {
+    if (name === 'candidates_exists' || name === 'candidates_join') continue;
+    const plan = await db.execute({ sql: `explain query plan ${st.sql}`, args: st.args });
+    const started = Date.now();
+    let result = 'ok';
+    try {
+      const r = await db.execute({ sql: st.sql, args: st.args });
+      result = `${r.rows.length} rows`;
+    } catch (e) {
+      result = `FAILED ${String(e.message).slice(0, 80)}`;
+    }
+    console.log(`${name}: ${Date.now() - started} ms, ${result}`);
+    for (const p of plan.rows) console.log('   plan:', p.detail);
+  }
+}

packages/db/src/webrings.js

@@ -224,20 +224,24 @@ export async function memberBySlug(db, ringSlug, memberSlug) {
  * @returns {Promise<Array<{ id: string, slug: string, site_url: string, created_at: string }>>}
  */
 export async function topicRingCandidates(db, topicSlug, limit) {
-  // Walked from feeds in admission order (feeds_created_idx) with a primary
-  // key probe into feed_keywords per row, so the statement stops the moment
-  // `limit` members are found. The other way round, a range scan of the
-  // topic's keyword rows joined, grouped and sorted before the limit applies,
-  // is a full pass over a big topic: the first seed in production timed out
-  // on the largest topic before anything was written. A small topic walks
-  // the directory with a point lookup per feed, which is the cheap case.
+  // Driven from the topic's own keyword rows, strongest first. The index
+  // feed_keywords_slug_idx is (slug, count desc), so the range scan comes
+  // out already in this order and the statement stops at the `limit`th
+  // feed that can link; `cross join` pins that join order, because left to
+  // itself the planner drove every variant of this from feeds by status,
+  // 580,000 rows probed and sorted, and took 80 seconds on the biggest
+  // topic against a 30-second request deadline (measured 2026-09-13).
+  //
+  // So a ring's order is the topic's own: the sites most about it first.
+  // Stable all the same: a position is written once and new members append.
   const { rows } = await db.execute({
     sql: `select f.id, f.slug, f.site_url, f.created_at
-            from feeds f
-           where f.status = 'active'
+            from feed_keywords k indexed by feed_keywords_slug_idx
+            cross join feeds f on f.id = k.feed_id
+           where k.slug = ?
+             and f.status = 'active'
              and f.site_url is not null and f.site_url <> ''
-             and exists (select 1 from feed_keywords k where k.feed_id = f.id and k.slug = ?)
-           order by f.created_at asc, f.id asc
+           order by k.count desc
            limit ?`,
     args: [topicSlug, limit],
   });
@@ -273,11 +277,15 @@ export async function seedTopicRing(db, topicSlug, opts = {}) {
   const limit = Math.max(1, Number(opts.limit ?? DEFAULT_RING_LIMIT) || DEFAULT_RING_LIMIT);
   const now = nowIso();
 
-  const label = await db.execute({
-    sql: `select ${topicLabelSql('?')} as keyword`,
-    args: [topicSlug],
-  });
-  const title = String(label.rows[0]?.keyword ?? topicSlug);
+  // The rollup already holds the topic's label; recomputing it groups every
+  // keyword row of the slug (seven seconds on the biggest topic). The
+  // recomputation is the fallback for a topic the rollup has not seen.
+  const rolled = await db.execute({ sql: `select keyword from topics where slug = ?`, args: [topicSlug] });
+  let title = rolled.rows[0]?.keyword ? String(rolled.rows[0].keyword) : '';
+  if (!title) {
+    const label = await db.execute({ sql: `select ${topicLabelSql('?')} as keyword`, args: [topicSlug] });
+    title = String(label.rows[0]?.keyword ?? topicSlug);
+  }
 
   const existing = await ringBySlug(db, topicSlug);
   if (!existing) {
@@ -348,7 +356,9 @@ export const MIN_RING_TOPIC_LENGTH = 3;
 export async function topRingTopics(db, opts = {}) {
   const count = Math.max(1, Number(opts.count ?? 20) || 20);
   const minFeeds = Math.max(1, Number(opts.minFeeds ?? 5) || 5);
-  const pool = Math.max(count, Number(opts.pool ?? count * 15) || count * 15);
+  // Each candidate costs a range scan of its keyword rows (six seconds on
+  // the biggest topic), so the pool is kept to a few times the rings wanted.
+  const pool = Math.max(count, Number(opts.pool ?? count * 5) || count * 5);
   const { rows } = await db.execute({
     sql: `select slug, keyword from topics
            where feed_count >= ? and length(slug) >= ?
@@ -361,9 +371,9 @@ export async function topRingTopics(db, opts = {}) {
   for (const r of rows) {
     const slug = String(r.slug);
     const counted = await db.execute({
-      sql: `select count(distinct k.feed_id) as n
-              from feed_keywords k
-              join feeds f on f.id = k.feed_id
+      sql: `select count(*) as n
+              from feed_keywords k indexed by feed_keywords_slug_idx
+              cross join feeds f on f.id = k.feed_id
              where k.slug = ? and k.source = 'category'
                and f.status = 'active'
                and f.site_url is not null and f.site_url <> ''`,

packages/db/test/webrings.test.js

@@ -22,7 +22,7 @@ const feeds = {};
 
 /**
  * @param {string} slug
- * @param {{ site?: string|null, status?: string, topics?: string[], created?: string }} [opts]
+ * @param {{ site?: string|null, status?: string, topics?: string[], created?: string, count?: number }} [opts]
  */
 async function feed(slug, opts = {}) {
   const row = await q.insertFeed(db, {
@@ -40,7 +40,7 @@ async function feed(slug, opts = {}) {
   for (const topic of opts.topics ?? ['physics']) {
     await db.execute({
       sql: 'insert into feed_keywords (feed_id, slug, keyword, words, count, source) values (?, ?, ?, ?, ?, ?)',
-      args: [row.id, topic, topic === 'physics' ? 'Physics' : topic, 1, 3, 'category'],
+      args: [row.id, topic, topic === 'physics' ? 'Physics' : topic, 1, opts.count ?? 3, 'category'],
     });
   }
   feeds[slug] = row;
@@ -52,11 +52,12 @@ before(async () => {
   db = connect({ url: `file:${join(dir, 'test.db')}` });
   await migrate(db);
 
-  // Admission order is deliberately not alphabetical and not the order the
-  // topic strength would give, so the test can tell the three apart.
-  await feed('carol', { created: '2026-01-03T00:00:00.000Z' });
-  await feed('alice', { created: '2026-01-01T00:00:00.000Z' });
-  await feed('bob', { created: '2026-01-02T00:00:00.000Z' });
+  // Ring order is the topic's own strength, strongest first. Insertion order
+  // and the alphabet are both deliberately different from it, so the test
+  // can tell the three apart.
+  await feed('carol', { created: '2026-01-03T00:00:00.000Z', count: 3 });
+  await feed('alice', { created: '2026-01-01T00:00:00.000Z', count: 9 });
+  await feed('bob', { created: '2026-01-02T00:00:00.000Z', count: 6 });
   await feed('nosite', { created: '2026-01-01T12:00:00.000Z', site: null });
   await feed('dead', { created: '2026-01-01T13:00:00.000Z', status: 'dead' });
   await feed('other', { created: '2026-01-01T14:00:00.000Z', topics: ['chemistry'] });
@@ -67,7 +68,7 @@ after(async () => {
   await rm(dir, { recursive: true, force: true });
 });
 
-test('a topic ring is seeded in admission order, from the feeds that can link', async () => {
+test('a topic ring is seeded strongest first, from the feeds that can link', async () => {
   const result = await webrings.seedTopicRing(db, 'physics');
   assert.deepEqual(result, { slug: 'physics', created: true, added: 3, total: 3 });