From 3cf68add4b782a4d47fcf3c815e716ae157ba0ef Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Sun, 13 Sep 2026 09:08:57 +0000 Subject: [PATCH] fix: pick ring topics from what publishers file under, not the commonest word The first seed took the topics rollup's top of the table, which is the most common phrases in a hundred thousand feeds' prose: "one", "de", "episode", "time". A ring named after one of those is a ring of everything. Ring topics are now ranked by the feeds that carry the slug as their own category tag (feed_keywords.source = 'category'), counted only over feeds a ring can use, with a three-character minimum so stopwords and language codes are out. Bounded like the seed: the rollup names a pool, each candidate costs one range scan of its own rows. A topic ring made under the old ranking, with fewer than five active members, is dropped when its topic no longer qualifies; a ring people have linked to stays. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01XYae2mH3khdwiXUVzcVMDw --- apps/poller/src/index.js | 2 +- packages/db/src/webrings.js | 80 +++++++++++++++++++++++++++---- packages/db/test/webrings.test.js | 33 +++++++++++++ packages/ingest/src/webring.js | 14 +++++- 4 files changed, 116 insertions(+), 13 deletions(-) diff --git a/apps/poller/src/index.js b/apps/poller/src/index.js index 9afca15..54ae241 100644 --- a/apps/poller/src/index.js +++ b/apps/poller/src/index.js @@ -806,7 +806,7 @@ async function ringTick() { lastRingSeed = Date.now(); // Logged only when something changed: once the rings exist, a line // every six hours saying "0 added" is not a log. - if (seeded.created || seeded.added || seeded.failed) log('rings-seeded', seeded); + if (seeded.created || seeded.added || seeded.failed || seeded.dropped) log('rings-seeded', seeded); } const result = await verifyRingMembers(db, { diff --git a/packages/db/src/webrings.js b/packages/db/src/webrings.js index 6a53e68..00a439b 100644 --- a/packages/db/src/webrings.js +++ b/packages/db/src/webrings.js @@ -322,28 +322,88 @@ export async function seedTopicRing(db, topicSlug, opts = {}) { return { slug: topicSlug, created: !existing, added: fresh.length, total: have.size + fresh.length }; } +/** A ring needs a subject; a slug this short is a stopword or a language code. */ +export const MIN_RING_TOPIC_LENGTH = 3; + /** - * The topics worth a ring: the most covered, by the rollup's own count. + * The topics worth a ring: the subjects publishers file themselves under. + * + * The rollup's own order is no use here. `topics` counts every phrase the + * crawler lifts out of prose, so its top of the table is "one", "de", + * "episode", "time": the most common words in a hundred thousand feeds, not + * their subjects. A ring named after one of those is a ring of everything. + * So the candidates are ranked by the feeds that carry the slug as their + * OWN category tag (feed_keywords.source = 'category'), which is what a + * publisher says the site is about, and only feeds a ring can use count: an + * active feed with a site to link from. + * + * Bounded the same way the seed is: the rollup (indexed by feed_count) + * names the pool, then each candidate costs one range scan of its own + * keyword rows, rather than one pass over every keyword row in the table. * * @param {Client} db - * @param {{ count?: number, minFeeds?: number }} [opts] + * @param {{ count?: number, minFeeds?: number, pool?: number }} [opts] * @returns {Promise>} */ export async function topRingTopics(db, opts = {}) { const count = Math.max(1, Number(opts.count ?? 20) || 20); const minFeeds = Math.max(1, Number(opts.minFeeds ?? 5) || 5); + const pool = Math.max(count, Number(opts.pool ?? count * 15) || count * 15); const { rows } = await db.execute({ - sql: `select slug, keyword, feed_count from topics - where feed_count >= ? + sql: `select slug, keyword from topics + where feed_count >= ? and length(slug) >= ? order by feed_count desc, slug asc limit ?`, - args: [minFeeds, count], + args: [minFeeds, MIN_RING_TOPIC_LENGTH, pool], }); - return rows.map((r) => ({ - slug: String(r.slug), - keyword: String(r.keyword), - feed_count: Number(r.feed_count ?? 0), - })); + + const ranked = []; + for (const r of rows) { + const slug = String(r.slug); + const counted = await db.execute({ + sql: `select count(distinct k.feed_id) as n + from feed_keywords k + join feeds f on f.id = k.feed_id + where k.slug = ? and k.source = 'category' + and f.status = 'active' + and f.site_url is not null and f.site_url <> ''`, + args: [slug], + }); + const n = Number(counted.rows[0]?.n ?? 0); + if (n >= minFeeds) ranked.push({ slug, keyword: String(r.keyword), feed_count: n }); + } + ranked.sort((a, b) => b.feed_count - a.feed_count || a.slug.localeCompare(b.slug)); + return ranked.slice(0, count); +} + +/** + * Remove topic rings whose topic no longer qualifies, when nobody would + * miss them: fewer than `keepActive` active members. A ring with members + * who put its links on their pages stays whatever the ranking says. + * + * @param {Client} db + * @param {string[]} keepSlugs the topics that qualify now + * @param {{ keepActive?: number }} [opts] + * @returns {Promise} the slugs removed + */ +export async function dropStaleTopicRings(db, keepSlugs, opts = {}) { + const keepActive = Math.max(1, Number(opts.keepActive ?? 5) || 5); + const keep = new Set(keepSlugs.map(String)); + const { rows } = await db.execute({ + sql: `select r.slug, + (select count(*) from ring_members m where m.ring_slug = r.slug and m.status = 'active') as active + from rings r + where r.kind = 'topic'`, + }); + const gone = []; + for (const r of rows) { + const slug = String(r.slug); + if (keep.has(slug) || Number(r.active ?? 0) >= keepActive) continue; + await db.execute({ sql: `delete from ring_members where ring_slug = ?`, args: [slug] }); + await db.execute({ sql: `delete from rings where slug = ?`, args: [slug] }); + gone.push(slug); + } + return gone; } /** diff --git a/packages/db/test/webrings.test.js b/packages/db/test/webrings.test.js index 804bca5..eb954ef 100644 --- a/packages/db/test/webrings.test.js +++ b/packages/db/test/webrings.test.js @@ -229,3 +229,36 @@ test('a ring goes when its topic does not, and a member goes with its feed', asy const { rows } = await db.execute({ sql: "select count(*) as n from ring_members where ring_slug = 'chemistry'", args: [] }); assert.equal(Number(rows[0].n), 0); }); + +test('ring topics come from what publishers file under, never the commonest word, and a stale ring goes', async () => { + // "one" is the commonest phrase in every feed's prose (source content); "de" + // is a two-letter tag. Neither is a subject. + for (const who of ['carol', 'alice', 'bob', 'other']) { + await db.execute({ + sql: 'insert or ignore into feed_keywords (feed_id, slug, keyword, words, count, source) values (?, ?, ?, ?, ?, ?)', + args: [feeds[who].id, 'one', 'one', 1, 900, 'content'], + }); + await db.execute({ + sql: 'insert or ignore into feed_keywords (feed_id, slug, keyword, words, count, source) values (?, ?, ?, ?, ?, ?)', + args: [feeds[who].id, 'de', 'de', 1, 50, 'category'], + }); + } + await q.refreshTopics(db, 1); + + const top = await webrings.topRingTopics(db, { count: 5, minFeeds: 1 }); + assert.deepEqual( + top.map((t) => t.slug), + ['physics', 'chemistry'], + 'physics has three linkable feeds filed under it, chemistry one; one is prose, de is too short', + ); + assert.equal(top[0].feed_count, await webrings.topicRingSize(db, 'physics'), 'counted on the feeds a ring can use, not the rollup'); + + // A ring made under the old ranking, with nobody active in it, is dropped + // when its topic no longer qualifies; a qualifying ring stays. + await webrings.seedTopicRing(db, 'one'); + assert.ok(await webrings.ringBySlug(db, 'one')); + const gone = await webrings.dropStaleTopicRings(db, ['physics', 'chemistry'], { keepActive: 1 }); + assert.deepEqual(gone, ['one']); + assert.equal(await webrings.ringBySlug(db, 'one'), null); + assert.ok(await webrings.ringBySlug(db, 'physics'), 'the qualifying ring is untouched'); +}); diff --git a/packages/ingest/src/webring.js b/packages/ingest/src/webring.js index 1f4b2f1..2469756 100644 --- a/packages/ingest/src/webring.js +++ b/packages/ingest/src/webring.js @@ -344,7 +344,7 @@ export async function verifyRingMembers(db, opts) { * * @param {import('@libsql/client').Client} db * @param {{ topics?: number, minMembers?: number, limit?: number, onError?: ((topic: string, err: unknown) => void)|null }} [opts] - * @returns {Promise<{ rings: number, created: number, added: number, skipped: number, failed: number }>} + * @returns {Promise<{ rings: number, created: number, added: number, skipped: number, failed: number, dropped: number }>} */ export async function seedTopRings(db, opts = {}) { const topics = Math.max(1, Number(opts.topics ?? 20) || 20); @@ -355,7 +355,17 @@ export async function seedTopRings(db, opts = {}) { // on a topic, and a topic can be well covered by feeds that have no site // to link from or that the crawler has given up on. const candidates = await webrings.topRingTopics(db, { count: topics * 2, minFeeds: minMembers }); - const tally = { rings: 0, created: 0, added: 0, skipped: 0, failed: 0 }; + const tally = { rings: 0, created: 0, added: 0, skipped: 0, failed: 0, dropped: 0 }; + + // A topic ring made under an earlier ranking (the first seed took the + // rollup's top of the table, which is stopwords) goes when its topic no + // longer qualifies and nobody has linked to it yet. + try { + const gone = await webrings.dropStaleTopicRings(db, candidates.map((t) => t.slug), { keepActive: minMembers }); + tally.dropped = gone.length; + } catch (err) { + opts.onError?.('(drop stale)', err); + } for (const topic of candidates) { if (tally.rings >= topics) break;