From 8ef8dba1d867189723b4ccc26a50d0831c2d545e Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Sun, 13 Sep 2026 08:59:34 +0000 Subject: [PATCH] fix: seed rings without timing out on the biggest topic, and retry a failed seed in minutes The first production seed died at the request deadline on the largest topic: the candidate query range-scanned every keyword row of the topic, joined, grouped and sorted them, and only then took a hundred; the size query counted the same join in full. Now both walk feeds in admission order with a primary-key probe into feed_keywords and stop at the rows they need (the size count stops at the minimum the caller asks about). A topic that still fails is counted and reported and the pass moves on, and the poller retries a failed seed ten minutes later instead of at the next six-hour mark, which is what the first deploy would have waited. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01XYae2mH3khdwiXUVzcVMDw --- apps/poller/src/index.js | 13 ++++++++-- packages/db/src/webrings.js | 41 ++++++++++++++++++------------- packages/db/test/webrings.test.js | 2 ++ packages/ingest/src/webring.js | 32 +++++++++++++++--------- 4 files changed, 58 insertions(+), 30 deletions(-) diff --git a/apps/poller/src/index.js b/apps/poller/src/index.js index 0c8c83b..9afca15 100644 --- a/apps/poller/src/index.js +++ b/apps/poller/src/index.js @@ -795,11 +795,18 @@ async function ringTick() { try { if (Date.now() - lastRingSeed >= ringSeedMs) { + // Stamped only once the pass has run: a pass that throws is retried + // ten minutes on, not six hours on, which is what the first deploy + // would have waited after the largest topic timed out. + const seeded = await seedTopRings(db, { + topics: ringTopics, + minMembers: 5, + onError: (topic, err) => log('ring-seed-error', { topic, message: String(err?.message ?? err) }), + }); lastRingSeed = Date.now(); - const seeded = await seedTopRings(db, { topics: ringTopics, minMembers: 5 }); // Logged only when something changed: once the rings exist, a line // every six hours saying "0 added" is not a log. - if (seeded.created || seeded.added) log('rings-seeded', seeded); + if (seeded.created || seeded.added || seeded.failed) log('rings-seeded', seeded); } const result = await verifyRingMembers(db, { @@ -814,6 +821,8 @@ async function ringTick() { if (result.checked) log('rings', result); } catch (err) { log('rings-error', { message: String(err?.message ?? err) }); + // Try the seed again soon rather than at the next six-hour mark. + lastRingSeed = Math.min(lastRingSeed, Date.now() - ringSeedMs + 10 * 60 * 1000); } finally { ringing = false; } diff --git a/packages/db/src/webrings.js b/packages/db/src/webrings.js index 014922c..6a53e68 100644 --- a/packages/db/src/webrings.js +++ b/packages/db/src/webrings.js @@ -224,18 +224,19 @@ export async function memberBySlug(db, ringSlug, memberSlug) { * @returns {Promise>} */ export async function topicRingCandidates(db, topicSlug, limit) { - // Driven from feed_keywords, whose (slug, count) index makes this a range - // scan of one topic's rows, rather than from feeds, where the same filter - // is a walk of the whole directory with a subquery per row. Grouped on the - // feed because a feed can carry several spellings of one slug. + // Walked from feeds in admission order (feeds_created_idx) with a primary + // key probe into feed_keywords per row, so the statement stops the moment + // `limit` members are found. The other way round, a range scan of the + // topic's keyword rows joined, grouped and sorted before the limit applies, + // is a full pass over a big topic: the first seed in production timed out + // on the largest topic before anything was written. A small topic walks + // the directory with a point lookup per feed, which is the cheap case. const { rows } = await db.execute({ sql: `select f.id, f.slug, f.site_url, f.created_at - from feed_keywords k - join feeds f on f.id = k.feed_id - where k.slug = ? - and f.status = 'active' + from feeds f + where f.status = 'active' and f.site_url is not null and f.site_url <> '' - group by f.id + and exists (select 1 from feed_keywords k where k.feed_id = f.id and k.slug = ?) order by f.created_at asc, f.id asc limit ?`, args: [topicSlug, limit], @@ -352,15 +353,21 @@ export async function topRingTopics(db, opts = {}) { * @param {string} topicSlug * @returns {Promise} */ -export async function topicRingSize(db, topicSlug) { +export async function topicRingSize(db, topicSlug, opts = {}) { + // `cap` stops the count once it is high enough to answer the caller's + // question ("at least five?"), so the biggest topics cost the same as the + // smallest. Without it the count is exact. + const cap = Number(opts.cap) > 0 ? Number(opts.cap) : null; const { rows } = await db.execute({ - sql: `select count(distinct f.id) as n - from feed_keywords k - join feeds f on f.id = k.feed_id - where k.slug = ? - and f.status = 'active' - and f.site_url is not null and f.site_url <> ''`, - args: [topicSlug], + sql: `select count(*) as n from ( + select f.id + from feeds f + where f.status = 'active' + and f.site_url is not null and f.site_url <> '' + and exists (select 1 from feed_keywords k where k.feed_id = f.id and k.slug = ?) + ${cap ? 'limit ?' : ''} + )`, + args: cap ? [topicSlug, cap] : [topicSlug], }); return Number(rows[0]?.n ?? 0); } diff --git a/packages/db/test/webrings.test.js b/packages/db/test/webrings.test.js index 82959ba..804bca5 100644 --- a/packages/db/test/webrings.test.js +++ b/packages/db/test/webrings.test.js @@ -128,6 +128,8 @@ test('the top topics are the most covered ones with enough feeds to ring', async assert.equal(await webrings.topicRingSize(db, 'physics'), 5, 'the feeds that can link, distinct'); assert.equal(await webrings.topicRingSize(db, 'chemistry'), 1); assert.equal(await webrings.topicRingSize(db, 'nothing'), 0); + assert.equal(await webrings.topicRingSize(db, 'physics', { cap: 3 }), 3, 'a capped count stops at the cap'); + assert.equal(await webrings.topicRingSize(db, 'chemistry', { cap: 3 }), 1, 'and is exact under it'); }); test('a check records status and stamp, and a descriptor sets made_by', async () => { diff --git a/packages/ingest/src/webring.js b/packages/ingest/src/webring.js index 371d224..1f4b2f1 100644 --- a/packages/ingest/src/webring.js +++ b/packages/ingest/src/webring.js @@ -337,9 +337,14 @@ export async function verifyRingMembers(db, opts) { * eligible feeds fall under `minMembers` is skipped rather than made into * a ring of two, since a ring needs somewhere to hop to. * + * A topic whose queries fail (the first production seed hit the request + * deadline on the largest topic) is counted as failed and reported through + * `onError`, and the pass goes on to the next one; one slow topic must not + * cost every other ring its seed. + * * @param {import('@libsql/client').Client} db - * @param {{ topics?: number, minMembers?: number, limit?: number }} [opts] - * @returns {Promise<{ rings: number, created: number, added: number, skipped: number }>} + * @param {{ topics?: number, minMembers?: number, limit?: number, onError?: ((topic: string, err: unknown) => void)|null }} [opts] + * @returns {Promise<{ rings: number, created: number, added: number, skipped: number, failed: number }>} */ export async function seedTopRings(db, opts = {}) { const topics = Math.max(1, Number(opts.topics ?? 20) || 20); @@ -350,19 +355,24 @@ export async function seedTopRings(db, opts = {}) { // on a topic, and a topic can be well covered by feeds that have no site // to link from or that the crawler has given up on. const candidates = await webrings.topRingTopics(db, { count: topics * 2, minFeeds: minMembers }); - const tally = { rings: 0, created: 0, added: 0, skipped: 0 }; + const tally = { rings: 0, created: 0, added: 0, skipped: 0, failed: 0 }; for (const topic of candidates) { if (tally.rings >= topics) break; - const size = await webrings.topicRingSize(db, topic.slug); - if (size < minMembers) { - tally.skipped += 1; - continue; + try { + const size = await webrings.topicRingSize(db, topic.slug, { cap: minMembers }); + if (size < minMembers) { + tally.skipped += 1; + continue; + } + const result = await webrings.seedTopicRing(db, topic.slug, { limit }); + tally.rings += 1; + if (result.created) tally.created += 1; + tally.added += result.added; + } catch (err) { + tally.failed += 1; + opts.onError?.(topic.slug, err); } - const result = await webrings.seedTopicRing(db, topic.slug, { limit }); - tally.rings += 1; - if (result.created) tally.created += 1; - tally.added += result.added; } return tally;