From 80454eb7f939d24cc4e10bc120bf8ffd19d0fc55 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Sun, 13 Sep 2026 09:16:51 +0000 Subject: [PATCH] fix: seed a ring from the topic's own keyword index, strongest first Measured against production (580,383 feeds, 3.5M keyword rows): every candidate query so far was driven by the planner from feeds by status, a probe and sort over every active feed, 80 seconds on the largest topic against a 30-second deadline; the title lookup grouped the slug's rows for seven more. Now the candidates come from feed_keywords_slug_idx (slug, count desc) with the join order pinned, so the statement stops at the hundredth feed that can link; the title is read from the topics rollup that already holds it; the category ranking pool is five times the rings wanted. Ring order is the topic's own, strongest first, still assigned once. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01XYae2mH3khdwiXUVzcVMDw --- packages/db/scripts/time-ring-queries.mjs | 61 +++++++++++++++++++++++ packages/db/src/webrings.js | 50 +++++++++++-------- packages/db/test/webrings.test.js | 17 ++++--- 3 files changed, 100 insertions(+), 28 deletions(-) create mode 100644 packages/db/scripts/time-ring-queries.mjs diff --git a/packages/db/scripts/time-ring-queries.mjs b/packages/db/scripts/time-ring-queries.mjs new file mode 100644 index 0000000..a286a9e --- /dev/null +++ b/packages/db/scripts/time-ring-queries.mjs @@ -0,0 +1,61 @@ +// Read-only: time the statements the ring seed runs, against the database in +// the environment, with their query plans. `node --env-file= scripts/time-ring-queries.mjs one news` +import { createClient } from '@libsql/client'; + +const env = process.env; +const db = createClient({ url: env['TURSO_DATABASE_URL'], authToken: env['TURSO_AUTH_TOKEN'] }); +const slugs = process.argv.slice(2).length ? process.argv.slice(2) : ['one']; + +const statements = (slug) => ({ + label: { + sql: `select lk.keyword from feed_keywords lk where lk.slug = ? group by lk.keyword order by count(distinct lk.feed_id) desc, length(lk.keyword) asc, lk.keyword asc limit 1`, + args: [slug], + }, + size_cap5: { + sql: `select count(*) as n from (select f.id from feeds f where f.status = 'active' and f.site_url is not null and f.site_url <> '' and exists (select 1 from feed_keywords k where k.feed_id = f.id and k.slug = ?) limit ?)`, + args: [slug, 5], + }, + candidates_exists: { + sql: `select f.id from feeds f where f.status = 'active' and f.site_url is not null and f.site_url <> '' and exists (select 1 from feed_keywords k where k.feed_id = f.id and k.slug = ?) order by f.created_at asc, f.id asc limit ?`, + args: [slug, 100], + }, + candidates_indexed: { + sql: `select f.id from feed_keywords k indexed by feed_keywords_slug_idx cross join feeds f on f.id = k.feed_id where k.slug = ? and f.status = 'active' and f.site_url is not null and f.site_url <> '' order by k.count desc limit ?`, + args: [slug, 100], + }, + category_count_indexed: { + sql: `select count(*) as n from feed_keywords k indexed by feed_keywords_slug_idx cross join feeds f on f.id = k.feed_id where k.slug = ? and k.source = 'category' and f.status = 'active' and f.site_url is not null and f.site_url <> ''`, + args: [slug], + }, + title_from_rollup: { sql: `select keyword from topics where slug = ?`, args: [slug] }, + candidates_join: { + sql: `select f.id from feed_keywords k join feeds f on f.id = k.feed_id where k.slug = ? and f.status = 'active' and f.site_url is not null and f.site_url <> '' group by f.id order by f.created_at asc, f.id asc limit ?`, + args: [slug, 100], + }, + category_count: { + sql: `select count(distinct k.feed_id) as n from feed_keywords k join feeds f on f.id = k.feed_id where k.slug = ? and k.source = 'category' and f.status = 'active' and f.site_url is not null and f.site_url <> ''`, + args: [slug], + }, +}); + +const counts = await db.execute(`select (select count(*) from feeds) as feeds, (select count(*) from feed_keywords) as keywords`); +console.log('feeds', counts.rows[0].feeds, 'keyword rows', counts.rows[0].keywords); + +for (const slug of slugs) { + const rows = await db.execute({ sql: `select count(*) as n from feed_keywords where slug = ?`, args: [slug] }); + console.log(`\n== ${slug}: ${rows.rows[0].n} keyword rows`); + for (const [name, st] of Object.entries(statements(slug))) { + if (name === 'candidates_exists' || name === 'candidates_join') continue; + const plan = await db.execute({ sql: `explain query plan ${st.sql}`, args: st.args }); + const started = Date.now(); + let result = 'ok'; + try { + const r = await db.execute({ sql: st.sql, args: st.args }); + result = `${r.rows.length} rows`; + } catch (e) { + result = `FAILED ${String(e.message).slice(0, 80)}`; + } + console.log(`${name}: ${Date.now() - started} ms, ${result}`); + for (const p of plan.rows) console.log(' plan:', p.detail); + } +} diff --git a/packages/db/src/webrings.js b/packages/db/src/webrings.js index 00a439b..5d4caa1 100644 --- a/packages/db/src/webrings.js +++ b/packages/db/src/webrings.js @@ -224,20 +224,24 @@ export async function memberBySlug(db, ringSlug, memberSlug) { * @returns {Promise>} */ export async function topicRingCandidates(db, topicSlug, limit) { - // Walked from feeds in admission order (feeds_created_idx) with a primary - // key probe into feed_keywords per row, so the statement stops the moment - // `limit` members are found. The other way round, a range scan of the - // topic's keyword rows joined, grouped and sorted before the limit applies, - // is a full pass over a big topic: the first seed in production timed out - // on the largest topic before anything was written. A small topic walks - // the directory with a point lookup per feed, which is the cheap case. + // Driven from the topic's own keyword rows, strongest first. The index + // feed_keywords_slug_idx is (slug, count desc), so the range scan comes + // out already in this order and the statement stops at the `limit`th + // feed that can link; `cross join` pins that join order, because left to + // itself the planner drove every variant of this from feeds by status, + // 580,000 rows probed and sorted, and took 80 seconds on the biggest + // topic against a 30-second request deadline (measured 2026-09-13). + // + // So a ring's order is the topic's own: the sites most about it first. + // Stable all the same: a position is written once and new members append. const { rows } = await db.execute({ sql: `select f.id, f.slug, f.site_url, f.created_at - from feeds f - where f.status = 'active' + from feed_keywords k indexed by feed_keywords_slug_idx + cross join feeds f on f.id = k.feed_id + where k.slug = ? + and f.status = 'active' and f.site_url is not null and f.site_url <> '' - and exists (select 1 from feed_keywords k where k.feed_id = f.id and k.slug = ?) - order by f.created_at asc, f.id asc + order by k.count desc limit ?`, args: [topicSlug, limit], }); @@ -273,11 +277,15 @@ export async function seedTopicRing(db, topicSlug, opts = {}) { const limit = Math.max(1, Number(opts.limit ?? DEFAULT_RING_LIMIT) || DEFAULT_RING_LIMIT); const now = nowIso(); - const label = await db.execute({ - sql: `select ${topicLabelSql('?')} as keyword`, - args: [topicSlug], - }); - const title = String(label.rows[0]?.keyword ?? topicSlug); + // The rollup already holds the topic's label; recomputing it groups every + // keyword row of the slug (seven seconds on the biggest topic). The + // recomputation is the fallback for a topic the rollup has not seen. + const rolled = await db.execute({ sql: `select keyword from topics where slug = ?`, args: [topicSlug] }); + let title = rolled.rows[0]?.keyword ? String(rolled.rows[0].keyword) : ''; + if (!title) { + const label = await db.execute({ sql: `select ${topicLabelSql('?')} as keyword`, args: [topicSlug] }); + title = String(label.rows[0]?.keyword ?? topicSlug); + } const existing = await ringBySlug(db, topicSlug); if (!existing) { @@ -348,7 +356,9 @@ export const MIN_RING_TOPIC_LENGTH = 3; export async function topRingTopics(db, opts = {}) { const count = Math.max(1, Number(opts.count ?? 20) || 20); const minFeeds = Math.max(1, Number(opts.minFeeds ?? 5) || 5); - const pool = Math.max(count, Number(opts.pool ?? count * 15) || count * 15); + // Each candidate costs a range scan of its keyword rows (six seconds on + // the biggest topic), so the pool is kept to a few times the rings wanted. + const pool = Math.max(count, Number(opts.pool ?? count * 5) || count * 5); const { rows } = await db.execute({ sql: `select slug, keyword from topics where feed_count >= ? and length(slug) >= ? @@ -361,9 +371,9 @@ export async function topRingTopics(db, opts = {}) { for (const r of rows) { const slug = String(r.slug); const counted = await db.execute({ - sql: `select count(distinct k.feed_id) as n - from feed_keywords k - join feeds f on f.id = k.feed_id + sql: `select count(*) as n + from feed_keywords k indexed by feed_keywords_slug_idx + cross join feeds f on f.id = k.feed_id where k.slug = ? and k.source = 'category' and f.status = 'active' and f.site_url is not null and f.site_url <> ''`, diff --git a/packages/db/test/webrings.test.js b/packages/db/test/webrings.test.js index eb954ef..b22802c 100644 --- a/packages/db/test/webrings.test.js +++ b/packages/db/test/webrings.test.js @@ -22,7 +22,7 @@ const feeds = {}; /** * @param {string} slug - * @param {{ site?: string|null, status?: string, topics?: string[], created?: string }} [opts] + * @param {{ site?: string|null, status?: string, topics?: string[], created?: string, count?: number }} [opts] */ async function feed(slug, opts = {}) { const row = await q.insertFeed(db, { @@ -40,7 +40,7 @@ async function feed(slug, opts = {}) { for (const topic of opts.topics ?? ['physics']) { await db.execute({ sql: 'insert into feed_keywords (feed_id, slug, keyword, words, count, source) values (?, ?, ?, ?, ?, ?)', - args: [row.id, topic, topic === 'physics' ? 'Physics' : topic, 1, 3, 'category'], + args: [row.id, topic, topic === 'physics' ? 'Physics' : topic, 1, opts.count ?? 3, 'category'], }); } feeds[slug] = row; @@ -52,11 +52,12 @@ before(async () => { db = connect({ url: `file:${join(dir, 'test.db')}` }); await migrate(db); - // Admission order is deliberately not alphabetical and not the order the - // topic strength would give, so the test can tell the three apart. - await feed('carol', { created: '2026-01-03T00:00:00.000Z' }); - await feed('alice', { created: '2026-01-01T00:00:00.000Z' }); - await feed('bob', { created: '2026-01-02T00:00:00.000Z' }); + // Ring order is the topic's own strength, strongest first. Insertion order + // and the alphabet are both deliberately different from it, so the test + // can tell the three apart. + await feed('carol', { created: '2026-01-03T00:00:00.000Z', count: 3 }); + await feed('alice', { created: '2026-01-01T00:00:00.000Z', count: 9 }); + await feed('bob', { created: '2026-01-02T00:00:00.000Z', count: 6 }); await feed('nosite', { created: '2026-01-01T12:00:00.000Z', site: null }); await feed('dead', { created: '2026-01-01T13:00:00.000Z', status: 'dead' }); await feed('other', { created: '2026-01-01T14:00:00.000Z', topics: ['chemistry'] }); @@ -67,7 +68,7 @@ after(async () => { await rm(dir, { recursive: true, force: true }); }); -test('a topic ring is seeded in admission order, from the feeds that can link', async () => { +test('a topic ring is seeded strongest first, from the feeds that can link', async () => { const result = await webrings.seedTopicRing(db, 'physics'); assert.deepEqual(result, { slug: 'physics', created: true, added: 3, total: 3 });