Removals: take a publisher down at their request, and keep them out

packages/db/index.js

@@ -21,3 +21,4 @@ export * as alerts from './src/alerts.js';
 export * as social from './src/social.js';
 export * as dataset from './src/dataset.js';
 export * as traffic from './src/traffic.js';
+export * as removals from './src/removals.js';

packages/db/migrations/20260905233000_feed_removals.sql

@@ -0,0 +1,30 @@
+-- Feeds removed at the owner's request, and kept out for good.
+--
+-- Deleting a feed's rows honours a removal request for exactly as long as it
+-- takes discovery or a resubmission to find the same URL again. This table is
+-- the memory: every insert path checks it, so a publisher who asked to be
+-- taken down stays down without anyone having to remember them.
+--
+-- Matching is by host as well as by exact URL. A Substack, a Ghost site or a
+-- personal domain is one publisher, and the request was about the publisher,
+-- not about one of their several feed URLs.
+
+create table if not exists feed_removals (
+  id            text primary key,
+  -- The URL that was removed, as it appeared in feeds.feed_url.
+  feed_url      text not null unique,
+  -- Lower-case hostname of that URL, without a leading "www.". Any feed on this
+  -- host is refused.
+  host          text not null,
+  -- What the row looked like when it went, for the record.
+  slug          text,
+  title         text,
+  -- Why, and who asked: a name or address from the request email, or 'operator'.
+  reason        text,
+  requested_by  text,
+  -- How much went with it.
+  items_removed integer not null default 0,
+  created_at    text not null
+);
+
+create index if not exists feed_removals_host_idx on feed_removals (host);

packages/db/src/queries.js

@@ -1,6 +1,7 @@
 import { clusterKey, dedupeItems, topicSlug } from '@rssamplifier/feed';
 
 import { newId, nowIso } from './client.js';
+import { FeedRemovedError, dropRemoved, isRemovedUrl, removalHost } from './removals.js';
 import { topicLabelSql } from './topicLabel.js';
 
 /**
@@ -309,6 +310,13 @@ export async function takenSlugs(db, base) {
  * @returns {Promise<{ id: string, slug: string }>}
  */
 export async function insertFeed(db, feed) {
+  // A publisher who asked to be taken down stays down, whichever path finds
+  // them again. Checked here rather than in each caller so no new caller can
+  // forget it.
+  if (await isRemovedUrl(db, feed.feed_url)) {
+    throw new FeedRemovedError(String(feed.feed_url), removalHost(feed.feed_url) ?? '');
+  }
+
   const id = newId();
   const now = nowIso();
 
@@ -2626,6 +2634,9 @@ export async function markCrawlSuccess(db, id, feed, itemCount, intervalMinutes
  * @returns {Promise<number>} rows actually inserted
  */
 export async function insertFeedsBulk(db, feeds) {
+  // Removed publishers are dropped silently: a bulk import or a discovery run
+  // has nobody to tell, and "rows actually inserted" already says how many.
+  feeds = await dropRemoved(db, feeds);
   if (feeds.length === 0) return 0;
 
   const now = nowIso();

packages/db/src/removals.js

@@ -0,0 +1,209 @@
+/**
+ * Removing a feed at its owner's request, and keeping it out.
+ *
+ * Two halves. `removeFeed` takes a publisher down: the feed row, its items,
+ * extracts, links and author association go with it (foreign keys cascade),
+ * an author record nothing else refers to goes too, and the request is
+ * written to `feed_removals`. `isRemovedUrl` / `dropRemoved` are the other
+ * half: the insert paths ask before adding a feed, so discovery, a bulk import
+ * or a fresh submission cannot bring a removed publisher back.
+ *
+ * Matching is by host, not only by exact URL. The request was about the
+ * publisher, and a publisher has as many feed URLs as their platform offers.
+ */
+
+import { newId, nowIso } from './client.js';
+
+/** @typedef {import('@libsql/client').Client} Client */
+
+/** Thrown by an insert path when the URL belongs to a removed publisher. */
+export class FeedRemovedError extends Error {
+  /**
+   * @param {string} feedUrl
+   * @param {string} host
+   */
+  constructor(feedUrl, host) {
+    super(`${host} was removed from the directory at its owner's request`);
+    this.name = 'FeedRemovedError';
+    this.feedUrl = feedUrl;
+    this.host = host;
+  }
+}
+
+/**
+ * The host a removal is keyed on: lower-case, no port, no leading "www.".
+ *
+ * @param {string} url
+ * @returns {string|null} null when the URL does not parse
+ */
+export function removalHost(url) {
+  try {
+    const host = new URL(String(url)).hostname.toLowerCase();
+    return host.replace(/^www\./, '') || null;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Is this URL, or anything on its host, removed?
+ *
+ * @param {Client} db
+ * @param {string} feedUrl
+ * @returns {Promise<boolean>}
+ */
+export async function isRemovedUrl(db, feedUrl) {
+  const host = removalHost(feedUrl);
+  const { rows } = await db.execute({
+    sql: 'select 1 from feed_removals where feed_url = ? or host = ? limit 1',
+    args: [String(feedUrl), host ?? ''],
+  });
+  return rows.length > 0;
+}
+
+/**
+ * The subset of `feeds` whose URL is not removed, in one query.
+ *
+ * Bulk paths insert hundreds of rows at a time; asking per row would turn one
+ * round trip into hundreds. Hosts are looked up as a set instead.
+ *
+ * @template {{ feed_url: string }} T
+ * @param {Client} db
+ * @param {T[]} feeds
+ * @returns {Promise<T[]>}
+ */
+export async function dropRemoved(db, feeds) {
+  if (feeds.length === 0) return feeds;
+  const hosts = [...new Set(feeds.map((f) => removalHost(f.feed_url)).filter(Boolean))];
+  const urls = feeds.map((f) => String(f.feed_url));
+  const removed = new Set();
+
+  // SQLite's parameter limit is comfortably above the 500-row chunks the bulk
+  // paths use, but chunk anyway so a larger caller cannot trip it.
+  const CHUNK = 400;
+  for (let i = 0; i < Math.max(hosts.length, urls.length); i += CHUNK) {
+    const h = hosts.slice(i, i + CHUNK);
+    const u = urls.slice(i, i + CHUNK);
+    const clauses = [];
+    const args = [];
+    if (h.length > 0) {
+      clauses.push(`host in (${h.map(() => '?').join(', ')})`);
+      args.push(...h);
+    }
+    if (u.length > 0) {
+      clauses.push(`feed_url in (${u.map(() => '?').join(', ')})`);
+      args.push(...u);
+    }
+    const { rows } = await db.execute({
+      sql: `select feed_url, host from feed_removals where ${clauses.join(' or ')}`,
+      args,
+    });
+    for (const row of rows) {
+      removed.add(String(row.host));
+      removed.add(String(row.feed_url));
+    }
+  }
+  if (removed.size === 0) return feeds;
+  return feeds.filter(
+    (f) => !removed.has(String(f.feed_url)) && !removed.has(removalHost(f.feed_url) ?? '')
+  );
+}
+
+/**
+ * Take a publisher down and remember it.
+ *
+ * Every feed on the URL's host is deleted, not just the URL given, because a
+ * removal request names a publisher. Authors left with no feed are deleted as
+ * well: an author page with nothing under it is still the person's name on the
+ * site. Returns what went, so the reply to the requester can say so.
+ *
+ * @param {Client} db
+ * @param {{ feed_url: string, reason?: string|null, requested_by?: string|null }} request
+ * @returns {Promise<{ host: string, feeds: { id: string, slug: string, feed_url: string, title: string|null, items: number }[], authors_removed: number, already_recorded: boolean }>}
+ */
+export async function removeFeed(db, request) {
+  const feedUrl = String(request.feed_url).trim();
+  const host = removalHost(feedUrl);
+  if (!host) throw new Error(`not a URL: ${feedUrl}`);
+
+  const { rows } = await db.execute({
+    sql: `select id, slug, feed_url, title,
+            (select count(*) from feed_items where feed_id = feeds.id) as items
+          from feeds
+          where lower(feed_url) like ? or lower(feed_url) like ? or lower(site_url) like ? or lower(site_url) like ?`,
+    args: [`%://${host}/%`, `%://www.${host}/%`, `%://${host}/%`, `%://www.${host}/%`],
+  });
+  const feeds = rows
+    .map((r) => ({
+      id: String(r.id),
+      slug: String(r.slug),
+      feed_url: String(r.feed_url),
+      title: r.title == null ? null : String(r.title),
+      items: Number(r.items ?? 0),
+    }))
+    // `like` cannot anchor on the host boundary, so confirm each hit properly.
+    .filter((f) => removalHost(f.feed_url) === host || f.feed_url === feedUrl);
+
+  let authorsRemoved = 0;
+  for (const feed of feeds) {
+    const { rows: authorRows } = await db.execute({
+      sql: `select author_id from feed_authors where feed_id = ?
+              and author_id not in (select author_id from feed_authors where feed_id != ?)`,
+      args: [feed.id, feed.id],
+    });
+    for (const row of authorRows) {
+      await db.execute({ sql: 'delete from authors where id = ?', args: [String(row.author_id)] });
+      authorsRemoved += 1;
+    }
+    // Items, extracts, links and the author association cascade from here.
+    await db.execute({ sql: 'delete from feeds where id = ?', args: [feed.id] });
+  }
+
+  const { rows: existing } = await db.execute({
+    sql: 'select 1 from feed_removals where feed_url = ? limit 1',
+    args: [feedUrl],
+  });
+  const alreadyRecorded = existing.length > 0;
+  if (!alreadyRecorded) {
+    const first = feeds[0] ?? null;
+    await db.execute({
+      sql: `insert into feed_removals
+              (id, feed_url, host, slug, title, reason, requested_by, items_removed, created_at)
+            values (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+      args: [
+        newId(),
+        feedUrl,
+        host,
+        first?.slug ?? null,
+        first?.title ?? null,
+        request.reason ?? null,
+        request.requested_by ?? null,
+        feeds.reduce((sum, f) => sum + f.items, 0),
+        nowIso(),
+      ],
+    });
+  }
+
+  return { host, feeds, authors_removed: authorsRemoved, already_recorded: alreadyRecorded };
+}
+
+/**
+ * Every removal on record, newest first.
+ *
+ * @param {Client} db
+ * @returns {Promise<{ feed_url: string, host: string, slug: string|null, reason: string|null, requested_by: string|null, items_removed: number, created_at: string }[]>}
+ */
+export async function listRemovals(db) {
+  const { rows } = await db.execute(
+    'select feed_url, host, slug, reason, requested_by, items_removed, created_at from feed_removals order by created_at desc'
+  );
+  return rows.map((r) => ({
+    feed_url: String(r.feed_url),
+    host: String(r.host),
+    slug: r.slug == null ? null : String(r.slug),
+    reason: r.reason == null ? null : String(r.reason),
+    requested_by: r.requested_by == null ? null : String(r.requested_by),
+    items_removed: Number(r.items_removed ?? 0),
+    created_at: String(r.created_at),
+  }));
+}

packages/db/src/remove-feed.js

@@ -0,0 +1,70 @@
+#!/usr/bin/env node
+/**
+ * Take a publisher out of the directory at their request, for good.
+ *
+ *   node packages/db/src/remove-feed.js https://someone.substack.com/feed \
+ *     --reason "removal request by email 2026-08-23" --by someone@example.com
+ *
+ *   node packages/db/src/remove-feed.js --list
+ *
+ * Runs against TURSO_DATABASE_URL like migrate.js does. Deletes every feed on
+ * the URL's host with its items, extracts and orphaned author, and records the
+ * host in feed_removals so discovery and resubmission cannot bring it back.
+ */
+
+import { connect } from './client.js';
+import { listRemovals, removeFeed } from './removals.js';
+
+function parse(argv) {
+  const out = { url: null, reason: null, by: null, list: false };
+  for (let i = 0; i < argv.length; i += 1) {
+    const arg = argv[i];
+    if (arg === '--list') out.list = true;
+    else if (arg === '--reason') out.reason = argv[++i] ?? null;
+    else if (arg === '--by') out.by = argv[++i] ?? null;
+    else if (arg.startsWith('--')) throw new Error(`unknown option ${arg}`);
+    else out.url = arg;
+  }
+  return out;
+}
+
+async function main() {
+  const opts = parse(process.argv.slice(2));
+  const db = connect();
+
+  if (opts.list) {
+    const removals = await listRemovals(db);
+    if (removals.length === 0) console.log('no removals on record');
+    for (const r of removals) {
+      console.log(
+        `${r.created_at}  ${r.host}  ${r.items_removed} items` +
+          `${r.requested_by ? `  by ${r.requested_by}` : ''}${r.reason ? `  (${r.reason})` : ''}`
+      );
+    }
+    return;
+  }
+
+  if (!opts.url) {
+    console.error('usage: remove-feed.js <feed-url> [--reason TEXT] [--by WHO] | --list');
+    process.exit(2);
+  }
+
+  const result = await removeFeed(db, {
+    feed_url: opts.url,
+    reason: opts.reason,
+    requested_by: opts.by,
+  });
+  if (result.feeds.length === 0) {
+    console.log(`nothing listed on ${result.host}${result.already_recorded ? '; already on record' : ''}`);
+  }
+  for (const feed of result.feeds) {
+    console.log(`removed /${feed.slug} (${feed.feed_url}): ${feed.items} items`);
+  }
+  if (result.authors_removed > 0) console.log(`removed ${result.authors_removed} orphaned author record(s)`);
+  console.log(`${result.host} is now refused by every insert path`);
+}
+
+main().catch((err) => {
+  console.error(err instanceof Error ? err.message : err);
+  process.exit(1);
+});