fix: pick ring topics from what publishers file under, not the commonest word

apps/poller/src/index.js

@@ -806,7 +806,7 @@ async function ringTick() {
       lastRingSeed = Date.now();
       // Logged only when something changed: once the rings exist, a line
       // every six hours saying "0 added" is not a log.
-      if (seeded.created || seeded.added || seeded.failed) log('rings-seeded', seeded);
+      if (seeded.created || seeded.added || seeded.failed || seeded.dropped) log('rings-seeded', seeded);
     }
 
     const result = await verifyRingMembers(db, {

packages/db/src/webrings.js

@@ -322,28 +322,88 @@ export async function seedTopicRing(db, topicSlug, opts = {}) {
   return { slug: topicSlug, created: !existing, added: fresh.length, total: have.size + fresh.length };
 }
 
+/** A ring needs a subject; a slug this short is a stopword or a language code. */
+export const MIN_RING_TOPIC_LENGTH = 3;
+
 /**
- * The topics worth a ring: the most covered, by the rollup's own count.
+ * The topics worth a ring: the subjects publishers file themselves under.
+ *
+ * The rollup's own order is no use here. `topics` counts every phrase the
+ * crawler lifts out of prose, so its top of the table is "one", "de",
+ * "episode", "time": the most common words in a hundred thousand feeds, not
+ * their subjects. A ring named after one of those is a ring of everything.
+ * So the candidates are ranked by the feeds that carry the slug as their
+ * OWN category tag (feed_keywords.source = 'category'), which is what a
+ * publisher says the site is about, and only feeds a ring can use count: an
+ * active feed with a site to link from.
+ *
+ * Bounded the same way the seed is: the rollup (indexed by feed_count)
+ * names the pool, then each candidate costs one range scan of its own
+ * keyword rows, rather than one pass over every keyword row in the table.
  *
  * @param {Client} db
- * @param {{ count?: number, minFeeds?: number }} [opts]
+ * @param {{ count?: number, minFeeds?: number, pool?: number }} [opts]
  * @returns {Promise<Array<{ slug: string, keyword: string, feed_count: number }>>}
  */
 export async function topRingTopics(db, opts = {}) {
   const count = Math.max(1, Number(opts.count ?? 20) || 20);
   const minFeeds = Math.max(1, Number(opts.minFeeds ?? 5) || 5);
+  const pool = Math.max(count, Number(opts.pool ?? count * 15) || count * 15);
   const { rows } = await db.execute({
-    sql: `select slug, keyword, feed_count from topics
-           where feed_count >= ?
+    sql: `select slug, keyword from topics
+           where feed_count >= ? and length(slug) >= ?
            order by feed_count desc, slug asc
            limit ?`,
-    args: [minFeeds, count],
+    args: [minFeeds, MIN_RING_TOPIC_LENGTH, pool],
   });
-  return rows.map((r) => ({
-    slug: String(r.slug),
-    keyword: String(r.keyword),
-    feed_count: Number(r.feed_count ?? 0),
-  }));
+
+  const ranked = [];
+  for (const r of rows) {
+    const slug = String(r.slug);
+    const counted = await db.execute({
+      sql: `select count(distinct k.feed_id) as n
+              from feed_keywords k
+              join feeds f on f.id = k.feed_id
+             where k.slug = ? and k.source = 'category'
+               and f.status = 'active'
+               and f.site_url is not null and f.site_url <> ''`,
+      args: [slug],
+    });
+    const n = Number(counted.rows[0]?.n ?? 0);
+    if (n >= minFeeds) ranked.push({ slug, keyword: String(r.keyword), feed_count: n });
+  }
+  ranked.sort((a, b) => b.feed_count - a.feed_count || a.slug.localeCompare(b.slug));
+  return ranked.slice(0, count);
+}
+
+/**
+ * Remove topic rings whose topic no longer qualifies, when nobody would
+ * miss them: fewer than `keepActive` active members. A ring with members
+ * who put its links on their pages stays whatever the ranking says.
+ *
+ * @param {Client} db
+ * @param {string[]} keepSlugs the topics that qualify now
+ * @param {{ keepActive?: number }} [opts]
+ * @returns {Promise<string[]>} the slugs removed
+ */
+export async function dropStaleTopicRings(db, keepSlugs, opts = {}) {
+  const keepActive = Math.max(1, Number(opts.keepActive ?? 5) || 5);
+  const keep = new Set(keepSlugs.map(String));
+  const { rows } = await db.execute({
+    sql: `select r.slug,
+                 (select count(*) from ring_members m where m.ring_slug = r.slug and m.status = 'active') as active
+            from rings r
+           where r.kind = 'topic'`,
+  });
+  const gone = [];
+  for (const r of rows) {
+    const slug = String(r.slug);
+    if (keep.has(slug) || Number(r.active ?? 0) >= keepActive) continue;
+    await db.execute({ sql: `delete from ring_members where ring_slug = ?`, args: [slug] });
+    await db.execute({ sql: `delete from rings where slug = ?`, args: [slug] });
+    gone.push(slug);
+  }
+  return gone;
 }
 
 /**

packages/db/test/webrings.test.js

@@ -229,3 +229,36 @@ test('a ring goes when its topic does not, and a member goes with its feed', asy
   const { rows } = await db.execute({ sql: "select count(*) as n from ring_members where ring_slug = 'chemistry'", args: [] });
   assert.equal(Number(rows[0].n), 0);
 });
+
+test('ring topics come from what publishers file under, never the commonest word, and a stale ring goes', async () => {
+  // "one" is the commonest phrase in every feed's prose (source content); "de"
+  // is a two-letter tag. Neither is a subject.
+  for (const who of ['carol', 'alice', 'bob', 'other']) {
+    await db.execute({
+      sql: 'insert or ignore into feed_keywords (feed_id, slug, keyword, words, count, source) values (?, ?, ?, ?, ?, ?)',
+      args: [feeds[who].id, 'one', 'one', 1, 900, 'content'],
+    });
+    await db.execute({
+      sql: 'insert or ignore into feed_keywords (feed_id, slug, keyword, words, count, source) values (?, ?, ?, ?, ?, ?)',
+      args: [feeds[who].id, 'de', 'de', 1, 50, 'category'],
+    });
+  }
+  await q.refreshTopics(db, 1);
+
+  const top = await webrings.topRingTopics(db, { count: 5, minFeeds: 1 });
+  assert.deepEqual(
+    top.map((t) => t.slug),
+    ['physics', 'chemistry'],
+    'physics has three linkable feeds filed under it, chemistry one; one is prose, de is too short',
+  );
+  assert.equal(top[0].feed_count, await webrings.topicRingSize(db, 'physics'), 'counted on the feeds a ring can use, not the rollup');
+
+  // A ring made under the old ranking, with nobody active in it, is dropped
+  // when its topic no longer qualifies; a qualifying ring stays.
+  await webrings.seedTopicRing(db, 'one');
+  assert.ok(await webrings.ringBySlug(db, 'one'));
+  const gone = await webrings.dropStaleTopicRings(db, ['physics', 'chemistry'], { keepActive: 1 });
+  assert.deepEqual(gone, ['one']);
+  assert.equal(await webrings.ringBySlug(db, 'one'), null);
+  assert.ok(await webrings.ringBySlug(db, 'physics'), 'the qualifying ring is untouched');
+});

packages/ingest/src/webring.js

@@ -344,7 +344,7 @@ export async function verifyRingMembers(db, opts) {
  *
  * @param {import('@libsql/client').Client} db
  * @param {{ topics?: number, minMembers?: number, limit?: number, onError?: ((topic: string, err: unknown) => void)|null }} [opts]
- * @returns {Promise<{ rings: number, created: number, added: number, skipped: number, failed: number }>}
+ * @returns {Promise<{ rings: number, created: number, added: number, skipped: number, failed: number, dropped: number }>}
  */
 export async function seedTopRings(db, opts = {}) {
   const topics = Math.max(1, Number(opts.topics ?? 20) || 20);
@@ -355,7 +355,17 @@ export async function seedTopRings(db, opts = {}) {
   // on a topic, and a topic can be well covered by feeds that have no site
   // to link from or that the crawler has given up on.
   const candidates = await webrings.topRingTopics(db, { count: topics * 2, minFeeds: minMembers });
-  const tally = { rings: 0, created: 0, added: 0, skipped: 0, failed: 0 };
+  const tally = { rings: 0, created: 0, added: 0, skipped: 0, failed: 0, dropped: 0 };
+
+  // A topic ring made under an earlier ranking (the first seed took the
+  // rollup's top of the table, which is stopwords) goes when its topic no
+  // longer qualifies and nobody has linked to it yet.
+  try {
+    const gone = await webrings.dropStaleTopicRings(db, candidates.map((t) => t.slug), { keepActive: minMembers });
+    tally.dropped = gone.length;
+  } catch (err) {
+    opts.onError?.('(drop stale)', err);
+  }
 
   for (const topic of candidates) {
     if (tally.rings >= topics) break;