diff --git a/packages/db/scripts/curate-ring.mjs b/packages/db/scripts/curate-ring.mjs new file mode 100644 index 0000000..ec78a5c --- /dev/null +++ b/packages/db/scripts/curate-ring.mjs @@ -0,0 +1,38 @@ +// Make or extend a curated ring from feed slugs, idempotently. +// node --env-file= scripts/curate-ring.mjs "" "<description>" <feed-slug>... +// Existing members keep their position; new slugs append; a slug the directory +// does not have, or a feed with no site to link from, is reported and skipped. +import { connect, nowIso } from '../index.js'; + +const [slug, title, description, ...feedSlugs] = process.argv.slice(2); +if (!slug || !title || !feedSlugs.length) { + console.error('usage: curate-ring.mjs <ring-slug> "<title>" "<description>" <feed-slug>...'); + process.exit(2); +} +const db = connect(); +const now = nowIso(); +await db.execute({ + sql: `insert into rings (slug, title, description, kind, topic_slug, accepts, public, created_at, updated_at) + values (?, ?, ?, 'curated', null, null, 1, ?, ?) on conflict (slug) do nothing`, + args: [slug, title, description || null, now, now], +}); +const current = await db.execute({ sql: `select feed_id, position from ring_members where ring_slug = ?`, args: [slug] }); +const have = new Set(current.rows.map((r) => String(r.feed_id))); +let position = current.rows.reduce((max, r) => Math.max(max, Number(r.position)), -1); +for (const feedSlug of feedSlugs) { + const { rows } = await db.execute({ sql: `select id, slug, site_url, status from feeds where slug = ?`, args: [feedSlug] }); + const feed = rows[0]; + if (!feed) { console.log(`skip ${feedSlug}: not in the directory`); continue; } + if (!feed.site_url) { console.log(`skip ${feedSlug}: no site url`); continue; } + if (have.has(String(feed.id))) { console.log(`have ${feedSlug}`); continue; } + position += 1; + await db.execute({ + sql: `insert into ring_members (ring_slug, feed_id, member_slug, position, site_url, status, joined_at) + values (?, ?, ?, ?, ?, 'pending', ?) on conflict (ring_slug, feed_id) do nothing`, + args: [slug, feed.id, feed.slug, position, feed.site_url, now], + }); + console.log(`added ${feedSlug} at ${position}: ${feed.site_url} (${feed.status})`); +} +await db.execute({ sql: `update rings set updated_at = ? where slug = ?`, args: [now, slug] }); +const count = await db.execute({ sql: `select count(*) as n from ring_members where ring_slug = ?`, args: [slug] }); +console.log(`ring ${slug}: ${count.rows[0].n} members`); diff --git a/packages/db/src/webrings.js b/packages/db/src/webrings.js index 5d4caa1..ff8ff6a 100644 --- a/packages/db/src/webrings.js +++ b/packages/db/src/webrings.js @@ -333,6 +333,17 @@ export async function seedTopicRing(db, topicSlug, opts = {}) { /** A ring needs a subject; a slug this short is a stopword or a language code. */ export const MIN_RING_TOPIC_LENGTH = 3; +/** + * Category tags that are containers rather than subjects. Publishers file + * under these by default (WordPress's own is "uncategorized"), so they rank + * high and mean nothing; a ring of "uncategorized" is a ring of anything. + */ +export const RING_TOPIC_STOPLIST = new Set([ + 'uncategorized', 'uncategorised', 'general', 'misc', 'miscellaneous', 'other', 'others', + 'blog', 'blogs', 'blogging', 'post', 'posts', 'article', 'articles', 'feed', 'rss', 'default', + 'news-feed', 'updates', 'update', 'home', 'homepage', 'main', 'featured', 'all', 'various', 'random', +]); + /** * The topics worth a ring: the subjects publishers file themselves under. * @@ -370,6 +381,7 @@ export async function topRingTopics(db, opts = {}) { const ranked = []; for (const r of rows) { const slug = String(r.slug); + if (RING_TOPIC_STOPLIST.has(slug)) continue; const counted = await db.execute({ sql: `select count(*) as n from feed_keywords k indexed by feed_keywords_slug_idx diff --git a/packages/db/test/webrings.test.js b/packages/db/test/webrings.test.js index b22802c..74a25a9 100644 --- a/packages/db/test/webrings.test.js +++ b/packages/db/test/webrings.test.js @@ -246,11 +246,20 @@ test('ring topics come from what publishers file under, never the commonest word } await q.refreshTopics(db, 1); + for (const who of ['carol', 'alice', 'bob']) { + await db.execute({ + sql: 'insert or ignore into feed_keywords (feed_id, slug, keyword, words, count, source) values (?, ?, ?, ?, ?, ?)', + args: [feeds[who].id, 'uncategorized', 'Uncategorized', 1, 40, 'category'], + }); + } + await q.refreshTopics(db, 1); + assert.ok(webrings.RING_TOPIC_STOPLIST.has('uncategorized')); + const top = await webrings.topRingTopics(db, { count: 5, minFeeds: 1 }); assert.deepEqual( top.map((t) => t.slug), ['physics', 'chemistry'], - 'physics has three linkable feeds filed under it, chemistry one; one is prose, de is too short', + 'physics has three linkable feeds filed under it, chemistry one; one is prose, de is too short, uncategorized is a container', ); assert.equal(top[0].feed_count, await webrings.topicRingSize(db, 'physics'), 'counted on the feeds a ring can use, not the rollup');