From 548269f2f91b51718ef2ea0e56e6c0698bb99acc Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Sat, 5 Sep 2026 23:36:34 +0000 Subject: [PATCH 1/2] Removals: take a publisher down at their request, and keep them out Deleting a feed's rows honoured a removal request only until discovery or a resubmission found the same URL again; the directory had no memory of who had asked to leave. feed_removals is that memory, keyed by host as well as URL, and both insert paths consult it: insertFeed throws FeedRemovedError, which a submission surfaces and discovery records as rejected/removed-by-owner, and insertFeedsBulk drops the rows silently. removeFeed() deletes every feed on the host with its items, extracts, links and any author record left with nothing under it, then writes the row. packages/db/src/remove-feed.js runs it against TURSO_DATABASE_URL, and --list shows what is on record. Prompted by two removal requests by email (2026-08-23, 2026-09-04) that sat unanswered because honouring one was a hand-run SQL delete. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_013GX6xEcjGFxcvcMCFLhQng --- packages/db/index.js | 1 + .../20260905234500_feed_removals.sql | 30 +++ packages/db/src/queries.js | 11 + packages/db/src/removals.js | 209 ++++++++++++++++++ packages/db/src/remove-feed.js | 70 ++++++ packages/db/test/removals.test.js | 153 +++++++++++++ packages/ingest/src/keywords.js | 11 +- 7 files changed, 481 insertions(+), 4 deletions(-) create mode 100644 packages/db/migrations/20260905234500_feed_removals.sql create mode 100644 packages/db/src/removals.js create mode 100644 packages/db/src/remove-feed.js create mode 100644 packages/db/test/removals.test.js diff --git a/packages/db/index.js b/packages/db/index.js index fd7a5c6..3f9084b 100644 --- a/packages/db/index.js +++ b/packages/db/index.js @@ -21,3 +21,4 @@ export * as alerts from './src/alerts.js'; export * as social from './src/social.js'; export * as dataset from './src/dataset.js'; export * as traffic from './src/traffic.js'; +export * as removals from './src/removals.js'; diff --git a/packages/db/migrations/20260905234500_feed_removals.sql b/packages/db/migrations/20260905234500_feed_removals.sql new file mode 100644 index 0000000..ba262d1 --- /dev/null +++ b/packages/db/migrations/20260905234500_feed_removals.sql @@ -0,0 +1,30 @@ +-- Feeds removed at the owner's request, and kept out for good. +-- +-- Deleting a feed's rows honours a removal request for exactly as long as it +-- takes discovery or a resubmission to find the same URL again. This table is +-- the memory: every insert path checks it, so a publisher who asked to be +-- taken down stays down without anyone having to remember them. +-- +-- Matching is by host as well as by exact URL. A Substack, a Ghost site or a +-- personal domain is one publisher, and the request was about the publisher, +-- not about one of their several feed URLs. + +create table if not exists feed_removals ( + id text primary key, + -- The URL that was removed, as it appeared in feeds.feed_url. + feed_url text not null unique, + -- Lower-case hostname of that URL, without a leading "www.". Any feed on this + -- host is refused. + host text not null, + -- What the row looked like when it went, for the record. + slug text, + title text, + -- Why, and who asked: a name or address from the request email, or 'operator'. + reason text, + requested_by text, + -- How much went with it. + items_removed integer not null default 0, + created_at text not null +); + +create index if not exists feed_removals_host_idx on feed_removals (host); diff --git a/packages/db/src/queries.js b/packages/db/src/queries.js index a724ed6..ae9ec03 100644 --- a/packages/db/src/queries.js +++ b/packages/db/src/queries.js @@ -1,6 +1,7 @@ import { clusterKey, dedupeItems, topicSlug } from '@rssamplifier/feed'; import { newId, nowIso } from './client.js'; +import { FeedRemovedError, dropRemoved, isRemovedUrl, removalHost } from './removals.js'; import { topicLabelSql } from './topicLabel.js'; /** @@ -309,6 +310,13 @@ export async function takenSlugs(db, base) { * @returns {Promise<{ id: string, slug: string }>} */ export async function insertFeed(db, feed) { + // A publisher who asked to be taken down stays down, whichever path finds + // them again. Checked here rather than in each caller so no new caller can + // forget it. + if (await isRemovedUrl(db, feed.feed_url)) { + throw new FeedRemovedError(String(feed.feed_url), removalHost(feed.feed_url) ?? ''); + } + const id = newId(); const now = nowIso(); @@ -2626,6 +2634,9 @@ export async function markCrawlSuccess(db, id, feed, itemCount, intervalMinutes * @returns {Promise} rows actually inserted */ export async function insertFeedsBulk(db, feeds) { + // Removed publishers are dropped silently: a bulk import or a discovery run + // has nobody to tell, and "rows actually inserted" already says how many. + feeds = await dropRemoved(db, feeds); if (feeds.length === 0) return 0; const now = nowIso(); diff --git a/packages/db/src/removals.js b/packages/db/src/removals.js new file mode 100644 index 0000000..af25588 --- /dev/null +++ b/packages/db/src/removals.js @@ -0,0 +1,209 @@ +/** + * Removing a feed at its owner's request, and keeping it out. + * + * Two halves. `removeFeed` takes a publisher down: the feed row, its items, + * extracts, links and author association go with it (foreign keys cascade), + * an author record nothing else refers to goes too, and the request is + * written to `feed_removals`. `isRemovedUrl` / `dropRemoved` are the other + * half: the insert paths ask before adding a feed, so discovery, a bulk import + * or a fresh submission cannot bring a removed publisher back. + * + * Matching is by host, not only by exact URL. The request was about the + * publisher, and a publisher has as many feed URLs as their platform offers. + */ + +import { newId, nowIso } from './client.js'; + +/** @typedef {import('@libsql/client').Client} Client */ + +/** Thrown by an insert path when the URL belongs to a removed publisher. */ +export class FeedRemovedError extends Error { + /** + * @param {string} feedUrl + * @param {string} host + */ + constructor(feedUrl, host) { + super(`${host} was removed from the directory at its owner's request`); + this.name = 'FeedRemovedError'; + this.feedUrl = feedUrl; + this.host = host; + } +} + +/** + * The host a removal is keyed on: lower-case, no port, no leading "www.". + * + * @param {string} url + * @returns {string|null} null when the URL does not parse + */ +export function removalHost(url) { + try { + const host = new URL(String(url)).hostname.toLowerCase(); + return host.replace(/^www\./, '') || null; + } catch { + return null; + } +} + +/** + * Is this URL, or anything on its host, removed? + * + * @param {Client} db + * @param {string} feedUrl + * @returns {Promise} + */ +export async function isRemovedUrl(db, feedUrl) { + const host = removalHost(feedUrl); + const { rows } = await db.execute({ + sql: 'select 1 from feed_removals where feed_url = ? or host = ? limit 1', + args: [String(feedUrl), host ?? ''], + }); + return rows.length > 0; +} + +/** + * The subset of `feeds` whose URL is not removed, in one query. + * + * Bulk paths insert hundreds of rows at a time; asking per row would turn one + * round trip into hundreds. Hosts are looked up as a set instead. + * + * @template {{ feed_url: string }} T + * @param {Client} db + * @param {T[]} feeds + * @returns {Promise} + */ +export async function dropRemoved(db, feeds) { + if (feeds.length === 0) return feeds; + const hosts = [...new Set(feeds.map((f) => removalHost(f.feed_url)).filter(Boolean))]; + const urls = feeds.map((f) => String(f.feed_url)); + const removed = new Set(); + + // SQLite's parameter limit is comfortably above the 500-row chunks the bulk + // paths use, but chunk anyway so a larger caller cannot trip it. + const CHUNK = 400; + for (let i = 0; i < Math.max(hosts.length, urls.length); i += CHUNK) { + const h = hosts.slice(i, i + CHUNK); + const u = urls.slice(i, i + CHUNK); + const clauses = []; + const args = []; + if (h.length > 0) { + clauses.push(`host in (${h.map(() => '?').join(', ')})`); + args.push(...h); + } + if (u.length > 0) { + clauses.push(`feed_url in (${u.map(() => '?').join(', ')})`); + args.push(...u); + } + const { rows } = await db.execute({ + sql: `select feed_url, host from feed_removals where ${clauses.join(' or ')}`, + args, + }); + for (const row of rows) { + removed.add(String(row.host)); + removed.add(String(row.feed_url)); + } + } + if (removed.size === 0) return feeds; + return feeds.filter( + (f) => !removed.has(String(f.feed_url)) && !removed.has(removalHost(f.feed_url) ?? '') + ); +} + +/** + * Take a publisher down and remember it. + * + * Every feed on the URL's host is deleted, not just the URL given, because a + * removal request names a publisher. Authors left with no feed are deleted as + * well: an author page with nothing under it is still the person's name on the + * site. Returns what went, so the reply to the requester can say so. + * + * @param {Client} db + * @param {{ feed_url: string, reason?: string|null, requested_by?: string|null }} request + * @returns {Promise<{ host: string, feeds: { id: string, slug: string, feed_url: string, title: string|null, items: number }[], authors_removed: number, already_recorded: boolean }>} + */ +export async function removeFeed(db, request) { + const feedUrl = String(request.feed_url).trim(); + const host = removalHost(feedUrl); + if (!host) throw new Error(`not a URL: ${feedUrl}`); + + const { rows } = await db.execute({ + sql: `select id, slug, feed_url, title, + (select count(*) from feed_items where feed_id = feeds.id) as items + from feeds + where lower(feed_url) like ? or lower(feed_url) like ? or lower(site_url) like ? or lower(site_url) like ?`, + args: [`%://${host}/%`, `%://www.${host}/%`, `%://${host}/%`, `%://www.${host}/%`], + }); + const feeds = rows + .map((r) => ({ + id: String(r.id), + slug: String(r.slug), + feed_url: String(r.feed_url), + title: r.title == null ? null : String(r.title), + items: Number(r.items ?? 0), + })) + // `like` cannot anchor on the host boundary, so confirm each hit properly. + .filter((f) => removalHost(f.feed_url) === host || f.feed_url === feedUrl); + + let authorsRemoved = 0; + for (const feed of feeds) { + const { rows: authorRows } = await db.execute({ + sql: `select author_id from feed_authors where feed_id = ? + and author_id not in (select author_id from feed_authors where feed_id != ?)`, + args: [feed.id, feed.id], + }); + for (const row of authorRows) { + await db.execute({ sql: 'delete from authors where id = ?', args: [String(row.author_id)] }); + authorsRemoved += 1; + } + // Items, extracts, links and the author association cascade from here. + await db.execute({ sql: 'delete from feeds where id = ?', args: [feed.id] }); + } + + const { rows: existing } = await db.execute({ + sql: 'select 1 from feed_removals where feed_url = ? limit 1', + args: [feedUrl], + }); + const alreadyRecorded = existing.length > 0; + if (!alreadyRecorded) { + const first = feeds[0] ?? null; + await db.execute({ + sql: `insert into feed_removals + (id, feed_url, host, slug, title, reason, requested_by, items_removed, created_at) + values (?, ?, ?, ?, ?, ?, ?, ?, ?)`, + args: [ + newId(), + feedUrl, + host, + first?.slug ?? null, + first?.title ?? null, + request.reason ?? null, + request.requested_by ?? null, + feeds.reduce((sum, f) => sum + f.items, 0), + nowIso(), + ], + }); + } + + return { host, feeds, authors_removed: authorsRemoved, already_recorded: alreadyRecorded }; +} + +/** + * Every removal on record, newest first. + * + * @param {Client} db + * @returns {Promise<{ feed_url: string, host: string, slug: string|null, reason: string|null, requested_by: string|null, items_removed: number, created_at: string }[]>} + */ +export async function listRemovals(db) { + const { rows } = await db.execute( + 'select feed_url, host, slug, reason, requested_by, items_removed, created_at from feed_removals order by created_at desc' + ); + return rows.map((r) => ({ + feed_url: String(r.feed_url), + host: String(r.host), + slug: r.slug == null ? null : String(r.slug), + reason: r.reason == null ? null : String(r.reason), + requested_by: r.requested_by == null ? null : String(r.requested_by), + items_removed: Number(r.items_removed ?? 0), + created_at: String(r.created_at), + })); +} diff --git a/packages/db/src/remove-feed.js b/packages/db/src/remove-feed.js new file mode 100644 index 0000000..4dabfd5 --- /dev/null +++ b/packages/db/src/remove-feed.js @@ -0,0 +1,70 @@ +#!/usr/bin/env node +/** + * Take a publisher out of the directory at their request, for good. + * + * node packages/db/src/remove-feed.js https://someone.substack.com/feed \ + * --reason "removal request by email 2026-08-23" --by someone@example.com + * + * node packages/db/src/remove-feed.js --list + * + * Runs against TURSO_DATABASE_URL like migrate.js does. Deletes every feed on + * the URL's host with its items, extracts and orphaned author, and records the + * host in feed_removals so discovery and resubmission cannot bring it back. + */ + +import { connect } from './client.js'; +import { listRemovals, removeFeed } from './removals.js'; + +function parse(argv) { + const out = { url: null, reason: null, by: null, list: false }; + for (let i = 0; i < argv.length; i += 1) { + const arg = argv[i]; + if (arg === '--list') out.list = true; + else if (arg === '--reason') out.reason = argv[++i] ?? null; + else if (arg === '--by') out.by = argv[++i] ?? null; + else if (arg.startsWith('--')) throw new Error(`unknown option ${arg}`); + else out.url = arg; + } + return out; +} + +async function main() { + const opts = parse(process.argv.slice(2)); + const db = connect(); + + if (opts.list) { + const removals = await listRemovals(db); + if (removals.length === 0) console.log('no removals on record'); + for (const r of removals) { + console.log( + `${r.created_at} ${r.host} ${r.items_removed} items` + + `${r.requested_by ? ` by ${r.requested_by}` : ''}${r.reason ? ` (${r.reason})` : ''}` + ); + } + return; + } + + if (!opts.url) { + console.error('usage: remove-feed.js [--reason TEXT] [--by WHO] | --list'); + process.exit(2); + } + + const result = await removeFeed(db, { + feed_url: opts.url, + reason: opts.reason, + requested_by: opts.by, + }); + if (result.feeds.length === 0) { + console.log(`nothing listed on ${result.host}${result.already_recorded ? '; already on record' : ''}`); + } + for (const feed of result.feeds) { + console.log(`removed /${feed.slug} (${feed.feed_url}): ${feed.items} items`); + } + if (result.authors_removed > 0) console.log(`removed ${result.authors_removed} orphaned author record(s)`); + console.log(`${result.host} is now refused by every insert path`); +} + +main().catch((err) => { + console.error(err instanceof Error ? err.message : err); + process.exit(1); +}); diff --git a/packages/db/test/removals.test.js b/packages/db/test/removals.test.js new file mode 100644 index 0000000..0f3441b --- /dev/null +++ b/packages/db/test/removals.test.js @@ -0,0 +1,153 @@ +import assert from 'node:assert/strict'; +import { test, before, after } from 'node:test'; +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { connect, newId, nowIso } from '../src/client.js'; +import { migrate } from '../src/migrate.js'; +import * as q from '../src/queries.js'; +import { + FeedRemovedError, + dropRemoved, + isRemovedUrl, + listRemovals, + removalHost, + removeFeed, +} from '../src/removals.js'; + +let dir; +let db; + +before(async () => { + dir = await mkdtemp(join(tmpdir(), 'rssamp-removals-')); + db = connect({ url: `file:${join(dir, 'test.db')}` }); + await migrate(db); + // Cascades are the whole mechanism; a file database has them off by default. + await db.execute('pragma foreign_keys = on'); +}); + +after(async () => { + await rm(dir, { recursive: true, force: true }); +}); + +async function addItem(feedId, url) { + await db.execute({ + sql: `insert into feed_items (id, feed_id, guid, url, title, published_at, created_at) + values (?, ?, ?, ?, ?, ?, ?)`, + args: [newId(), feedId, url, url, 'post', nowIso(), nowIso()], + }); +} + +test('removalHost keys on the bare host', () => { + assert.equal(removalHost('https://WWW.Example.com:443/feed'), 'example.com'); + assert.equal(removalHost('https://judith.substack.com/feed'), 'judith.substack.com'); + assert.equal(removalHost('not a url'), null); +}); + +test('removeFeed deletes the feed, its items and an orphaned author, and records it', async () => { + const { id } = await q.insertFeed(db, { + slug: 'potter-substack-com', + feed_url: 'https://potter.substack.com/feed', + site_url: 'https://potter.substack.com', + title: 'Potter', + }); + await addItem(id, 'https://potter.substack.com/p/one'); + await addItem(id, 'https://potter.substack.com/p/two'); + + const authorId = newId(); + await db.execute({ + sql: 'insert into authors (id, slug, identity_key, name, norm_name, created_at, updated_at) values (?, ?, ?, ?, ?, ?, ?)', + args: [authorId, 'potter', 'potter', 'Potter', 'potter', nowIso(), nowIso()], + }); + await db.execute({ + sql: 'insert into feed_authors (feed_id, author_id, created_at) values (?, ?, ?)', + args: [id, authorId, nowIso()], + }); + + const result = await removeFeed(db, { + feed_url: 'https://potter.substack.com/feed', + reason: 'owner asked', + requested_by: 'potter@example.com', + }); + + assert.equal(result.host, 'potter.substack.com'); + assert.deepEqual( + result.feeds.map((f) => [f.slug, f.items]), + [['potter-substack-com', 2]] + ); + assert.equal(result.authors_removed, 1); + assert.equal(result.already_recorded, false); + + assert.equal(await q.feedBySlug(db, 'potter-substack-com'), null); + const { rows: items } = await db.execute({ + sql: 'select count(*) as n from feed_items where feed_id = ?', + args: [id], + }); + assert.equal(Number(items[0].n), 0); + const { rows: authors } = await db.execute({ + sql: 'select count(*) as n from authors where id = ?', + args: [authorId], + }); + assert.equal(Number(authors[0].n), 0); + + const removals = await listRemovals(db); + assert.equal(removals.length, 1); + assert.equal(removals[0].host, 'potter.substack.com'); + assert.equal(removals[0].items_removed, 2); + assert.equal(removals[0].requested_by, 'potter@example.com'); +}); + +test('a removed host stays out of every insert path', async () => { + assert.equal(await isRemovedUrl(db, 'https://potter.substack.com/feed'), true); + // Another URL on the same publisher's host is the same request. + assert.equal(await isRemovedUrl(db, 'https://www.potter.substack.com/feed?format=rss'), true); + assert.equal(await isRemovedUrl(db, 'https://someone-else.substack.com/feed'), false); + + await assert.rejects( + q.insertFeed(db, { + slug: 'potter-again', + feed_url: 'https://potter.substack.com/feed', + title: 'Potter again', + }), + FeedRemovedError + ); + + const inserted = await q.insertFeedsBulk(db, [ + { + slug: 'potter-bulk', + feed_url: 'https://potter.substack.com/feed?bulk=1', + title: 'Potter bulk', + next_fetch_at: nowIso(), + }, + { + slug: 'fine-blog', + feed_url: 'https://fine.example/feed', + title: 'Fine', + next_fetch_at: nowIso(), + }, + ]); + assert.equal(inserted, 1, 'only the feed that was not removed went in'); + assert.equal(await q.feedBySlug(db, 'potter-bulk'), null); + assert.ok(await q.feedBySlug(db, 'fine-blog')); + + const kept = await dropRemoved(db, [ + { feed_url: 'https://potter.substack.com/rss' }, + { feed_url: 'https://fine.example/feed' }, + ]); + assert.deepEqual( + kept.map((f) => f.feed_url), + ['https://fine.example/feed'] + ); +}); + +test('removing again is idempotent and reports nothing to delete', async () => { + const result = await removeFeed(db, { feed_url: 'https://potter.substack.com/feed' }); + assert.deepEqual(result.feeds, []); + assert.equal(result.already_recorded, true); + assert.equal((await listRemovals(db)).length, 1); +}); + +test('removeFeed refuses something that is not a URL', async () => { + await assert.rejects(removeFeed(db, { feed_url: 'nope' }), /not a URL/); +}); diff --git a/packages/ingest/src/keywords.js b/packages/ingest/src/keywords.js index baab052..4c9d809 100644 --- a/packages/ingest/src/keywords.js +++ b/packages/ingest/src/keywords.js @@ -151,14 +151,17 @@ export async function checkCandidate(db, candidate, opts = {}) { } catch (err) { // Another run inserting the same feed between the lookup and the insert is // a duplicate, not a failure. - const raced = await q.feedByUrl(db, feedUrl); + // A publisher who asked to be removed is a rejection with a reason, not a + // fault in the run: the candidate was fine, the answer is still no. + const removed = err?.name === 'FeedRemovedError'; + const raced = removed ? null : await q.feedByUrl(db, feedUrl); await discovery.markCandidate(db, String(candidate.id), { - status: raced ? 'rejected' : 'error', + status: raced || removed ? 'rejected' : 'error', feedUrl, slug: raced ? String(raced.slug) : null, - reason: raced ? 'already-indexed' : String(err?.message ?? err), + reason: raced ? 'already-indexed' : removed ? 'removed-by-owner' : String(err?.message ?? err), }); - return { status: raced ? 'rejected' : 'error' }; + return { status: raced || removed ? 'rejected' : 'error' }; } await q.upsertItems(db, inserted.id, feed.items); From 4ed912777105701d20e5a9cd873cf3d6bcdeefaf Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Sat, 5 Sep 2026 23:37:09 +0000 Subject: [PATCH 2/2] Migration timestamp in the past, as the test requires Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_013GX6xEcjGFxcvcMCFLhQng --- ...5234500_feed_removals.sql => 20260905233000_feed_removals.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename packages/db/migrations/{20260905234500_feed_removals.sql => 20260905233000_feed_removals.sql} (100%) diff --git a/packages/db/migrations/20260905234500_feed_removals.sql b/packages/db/migrations/20260905233000_feed_removals.sql similarity index 100% rename from packages/db/migrations/20260905234500_feed_removals.sql rename to packages/db/migrations/20260905233000_feed_removals.sql