Take feed discovery, OPML parsing and the submit_feed tool from @profullstack/submit-feed

apps/web/package.json

@@ -10,8 +10,11 @@
     "start": "node server.mjs"
   },
   "dependencies": {
+    "@profullstack/leaderboard": "^0.3.0",
+    "@profullstack/partners": "^0.2.0",
     "@profullstack/player": "^0.3.1",
     "@profullstack/rssamplifier": "workspace:*",
+    "@profullstack/submit-feed": "^0.1.1",
     "@profullstack/x402-gateway": "^0.6.0",
     "@rssamplifier/auth": "workspace:*",
     "@rssamplifier/db": "workspace:*",
@@ -26,8 +29,6 @@
     "@swc/helpers": "^0.5.23",
     "next": "^16.2.4",
     "react": "^19.2.0",
-    "react-dom": "^19.2.0",
-    "@profullstack/leaderboard": "^0.3.0",
-    "@profullstack/partners": "^0.2.0"
+    "react-dom": "^19.2.0"
   }
 }

apps/web/src/lib/mcp/tools.js

@@ -1,6 +1,7 @@
 import { q, newId, authors as people } from '@rssamplifier/db';
 import { topicSlug } from '@rssamplifier/feed';
 import { submitCatalogue, hashIp, EXPRESS_MAX } from '@rssamplifier/ingest';
+import { submitFeedTool } from '@profullstack/submit-feed/core';
 
 import { db, siteUrl } from '../db.js';
 import { readerView } from '../reader.js';
@@ -521,24 +522,13 @@ export const TOOLS = [
     },
   },
 
-  {
-    name: 'submit_feed',
-    title: 'Add a feed to the directory',
-    description:
-      'Submit one URL or a list of them. A site URL works as well as a feed URL — the feed is discovered from the page. Anyone may submit; there is no account and no review queue. Feeds resolve inline up to a handful and the rest are queued for the crawler, so the answer says which. Rate limited per caller.',
-    inputSchema: {
-      type: 'object',
-      properties: {
-        urls: {
-          type: 'array',
-          items: { type: 'string' },
-          description: 'Feed or site URLs. One is fine.',
-          maxItems: 200,
-        },
-      },
-      required: ['urls'],
-    },
-    annotations: { readOnlyHint: false, destructiveHint: false, openWorldHint: true },
+  // The name, description, schema and annotations are the shared contract's,
+  // so an agent that learned submit_feed on p0dcasters.com finds the same tool
+  // here. Only `run` is this directory's: the ledger, the express lane and
+  // the status page are its own.
+  submitFeedTool({
+    directory: 'RSS Amplifier',
+    maxUrls: 200,
     async run(args, ctx) {
       const urls = (Array.isArray(args?.urls) ? args.urls : [args?.urls])
         .map((u) => String(u ?? '').trim())
@@ -602,7 +592,7 @@ export const TOOLS = [
         statusUrl: `${siteUrl()}/submissions/${submissionId}`,
       };
     },
-  },
+  }),
 ];
 
 /** @type {Map<string, Tool>} */

packages/feed/package.json

@@ -16,6 +16,7 @@
   },
   "dependencies": {
     "@mozilla/readability": "^0.6.0",
+    "@profullstack/submit-feed": "^0.1.1",
     "@profullstack/x402-client": "0.2.0",
     "fast-xml-parser": "^5.2.5",
     "linkedom": "^0.18.13"

packages/feed/src/discover.js

@@ -1,182 +1,34 @@
 /**
  * Feed discovery: people submit "myblog.com", not "myblog.com/feed.xml".
  * Turning the former into the latter is most of what makes submission painless.
- */
-
-import { looksLikePlaylist } from './playlist.js';
-
-const COMMON_PATHS = [
-  '/feed',
-  '/feed.xml',
-  '/rss',
-  '/rss.xml',
-  '/atom.xml',
-  '/index.xml',
-  '/feed/',
-  '/feeds/posts/default',
-  '/?feed=rss2',
-  '/feed.json',
-];
-
-/** Content types that indicate the body is a feed rather than a web page. */
-const FEED_TYPES = [
-  'application/rss+xml',
-  'application/atom+xml',
-  'application/feed+json',
-  'application/xml',
-  'text/xml',
-  'application/json',
-];
-
-/**
- * Normalize user input into an absolute http(s) URL.
- *
- * Accepts "example.com", "//example.com" and "http://example.com", because all
- * three get pasted into submission boxes. Returns null for anything that is not
- * a usable web URL — including non-http schemes, which must never be fetched.
- *
- * @param {string} input
- * @returns {string|null}
- */
-export function normalizeUrl(input) {
-  if (typeof input !== 'string') return null;
-  let raw = input.trim();
-  if (!raw) return null;
-
-  if (raw.startsWith('//')) raw = `https:${raw}`;
-  if (!/^https?:\/\//i.test(raw)) {
-    if (/^[a-z][a-z0-9+.-]*:/i.test(raw)) return null; // mailto:, javascript:, file:
-    raw = `https://${raw}`;
-  }
-
-  try {
-    const u = new URL(raw);
-    if (u.protocol !== 'http:' && u.protocol !== 'https:') return null;
-    if (!plausibleHost(u.hostname)) return null;
-    u.hash = '';
-    return u.toString();
-  } catch {
-    return null;
-  }
-}
-
-/**
- * Is this hostname something a publisher could actually be at?
- *
- * `new URL()` is far more permissive about hosts than DNS is. Quotes, parens,
- * `=` and `&` are not forbidden host code points, so `new URL('https://x')`
- * happily parses `version="1.0"`, `zombies.)` and `z.` into a hostname — and
- * the check this replaced, `hostname.includes('.')`, waved all three through.
- *
- * That is not a theoretical gap. The bulk-upload scanner splits pasted text on
- * whitespace and offers every token as a URL, so pasting raw OPML instead of a
- * URL list turned the markup itself into feeds: `https://version="1.0"/`,
- * `https://text="gg.deals/`, and a sentence ending in "zombies." into
- * `https://zombies.)/`. Roughly 3,700 such rows reached the directory, and
- * every one of them is crawled forever on a cadence, failing `blocked-host`.
- *
- * Validating here rather than in the scanner is deliberate: `normalizeUrl` is
- * the single gate every entry path goes through -- web submit, OPML import,
- * the queue drain and discovery -- so one check covers all of them, and the
- * scanner cannot import this module anyway (it runs in the browser, and
- * pulling `@rssamplifier/feed` into client code fails the build on `node:dns`).
- *
- * Kept deliberately loose about what a real domain looks like: this rejects
- * things that cannot be hostnames, not things that are merely unusual. IDN is
- * already punycode by the time it arrives, and a bare IPv4 literal is allowed
- * through so that a feed genuinely served from one is not newly rejected --
- * private ranges are refused later, by `isPublicHost`.
- *
- * @param {string} hostname as parsed by `new URL`, so lowercased and punycoded
- * @returns {boolean}
- */
-function plausibleHost(hostname) {
-  const host = String(hostname ?? '');
-  if (!host || host.length > 253) return false;
-
-  // A dotted-quad is a legitimate, if unusual, place for a feed to live.
-  if (/^\d{1,3}(\.\d{1,3}){3}$/.test(host)) {
-    return host.split('.').every((n) => Number(n) <= 255);
-  }
-
-  // Anything outside the DNS alphabet is markup or prose, not a host.
-  if (!/^[a-z0-9.-]+$/.test(host)) return false;
-
-  const labels = host.split('.');
-  // Two labels at least, none empty -- which is what rules out `z.` and `you.`,
-  // whose trailing dot leaves an empty final label.
-  if (labels.length < 2) return false;
-  if (labels.some((l) => l === '' || l.length > 63 || l.startsWith('-') || l.endsWith('-'))) {
-    return false;
-  }
-
-  // A public suffix is letters. This is the part that rejects `1.0` while
-  // leaving `example.co.uk` and `xn--bcher-kva.de` alone.
-  return /^[a-z]{2,}$/.test(labels[labels.length - 1]);
-}
-
-/**
- * Extract feed URLs advertised by a page's <link rel="alternate"> tags.
- *
- * This is the correct, standards-based path — the guessed paths below are only
- * a fallback for sites that don't advertise.
  *
- * @param {string} html
- * @param {string} baseUrl for resolving relative hrefs
- * @returns {string[]} absolute feed URLs, in document order, deduped
+ * The mechanics live in @profullstack/submit-feed now, extracted from here so
+ * p0dcasters.com and any later directory take feeds the same way: the URL
+ * gate that refuses markup-as-hostname (the 3,700 rows of `https://version="1.0"/`
+ * that the browser scanner once pushed in), the <link rel="alternate"> walk,
+ * the conventional paths, and the body sniff. This module is the seam: the
+ * same names, so nothing in the workspace changed its imports, plus the one
+ * thing this directory does that the package does not, which is to admit a
+ * playlist as a feed.
  */
-export function findFeedLinks(html, baseUrl) {
-  if (typeof html !== 'string') return [];
-  const out = [];
-
-  // Match <link> tags, then pull attributes out individually: attribute order
-  // varies between generators and a single positional regex misses most of them.
-  const tags = html.match(/<link\b[^>]*>/gi) ?? [];
-
-  for (const tag of tags) {
-    const rel = /\brel\s*=\s*["']?([^"'>\s]+)/i.exec(tag)?.[1]?.toLowerCase();
-    if (rel !== 'alternate') continue;
-
-    const type = /\btype\s*=\s*["']?([^"'>\s]+)/i.exec(tag)?.[1]?.toLowerCase();
-    if (!type || !FEED_TYPES.includes(type)) continue;
 
-    const href = /\bhref\s*=\s*["']([^"']+)["']/i.exec(tag)?.[1];
-    if (!href) continue;
-
-    try {
-      const abs = new URL(href, baseUrl).toString();
-      if (!out.includes(abs)) out.push(abs);
-    } catch {
-      // skip unresolvable href
-    }
-  }
+import {
+  normalizeUrl,
+  findFeedLinks,
+  guessFeedUrls,
+  looksLikeFeed as looksLikeFeedDocument,
+} from '@profullstack/submit-feed/core';
+import { looksLikePlaylist } from './playlist.js';
 
-  return out;
-}
-
-/**
- * Candidate feed URLs to try for a site, in priority order.
- *
- * @param {string} siteUrl
- * @returns {string[]}
- */
-export function guessFeedUrls(siteUrl) {
-  const out = [];
-  for (const p of COMMON_PATHS) {
-    try {
-      out.push(new URL(p, siteUrl).toString());
-    } catch {
-      // skip
-    }
-  }
-  return out;
-}
+export { normalizeUrl, findFeedLinks, guessFeedUrls };
 
 /**
  * Decide whether a fetched response looks like a feed.
  *
- * Content-type alone is unreliable — plenty of feeds are served as text/plain
- * or text/html — so the body is sniffed too.
+ * A playlist is a feed here -- a list of media with titles -- so it is admitted
+ * on the same footing rather than sniffed for afterwards. Everything else is
+ * the shared sniff: content type first, then the head of the body, because
+ * plenty of feeds are served as text/plain or text/html.
  *
  * @param {string} contentType
  * @param {string} body
@@ -185,24 +37,6 @@ export function guessFeedUrls(siteUrl) {
  * @returns {boolean}
  */
 export function looksLikeFeed(contentType, body, url = '') {
-  // A playlist is a feed here — a list of media with titles — so it is admitted
-  // on the same footing rather than sniffed for afterwards.
   if (looksLikePlaylist(contentType, body, url)) return true;
-
-  const ct = (contentType || '').toLowerCase();
-  if (FEED_TYPES.some((t) => ct.includes(t))) {
-    // application/json is only a feed if it's actually JSON Feed.
-    if (ct.includes('json') && !ct.includes('feed+json')) {
-      return /"(?:items|version)"\s*:/.test(body ?? '');
-    }
-    return true;
-  }
-
-  const head = (body ?? '').slice(0, 2000).toLowerCase();
-  if (/<rss\b/.test(head)) return true;
-  if (/<feed\b[^>]*xmlns/.test(head)) return true;
-  if (/<rdf:rdf\b/.test(head)) return true;
-  if (/"version"\s*:\s*"https:\/\/jsonfeed\.org/.test(head)) return true;
-
-  return false;
+  return looksLikeFeedDocument(contentType, body);
 }

packages/feed/src/opml.js

@@ -1,75 +1,12 @@
-import { XMLParser } from 'fast-xml-parser';
-
-const parser = new XMLParser({
-  ignoreAttributes: false,
-  attributeNamePrefix: '@',
-  trimValues: true,
-});
-
-/**
- * Walk an OPML outline tree and collect every node that carries a feed URL.
- *
- * OPML nests arbitrarily — subscription lists are usually grouped into folders,
- * and folders can contain folders — so this recurses rather than reading only
- * the top level. Nodes without an xmlUrl are folders, not feeds.
- *
- * @param {unknown} node
- * @param {Array<{ url: string, title: string }>} out
- */
-function walk(node, out) {
-  if (!node) return;
-  const list = Array.isArray(node) ? node : [node];
-
-  for (const item of list) {
-    if (!item || typeof item !== 'object') continue;
-
-    const url = item['@xmlUrl'] || item['@xmlurl'];
-    if (typeof url === 'string' && url.trim()) {
-      const siteUrl = item['@htmlUrl'] || item['@htmlurl'];
-      out.push({
-        url: url.trim(),
-        title: (item['@title'] || item['@text'] || '').toString().trim(),
-        // Carried through for bulk imports, which trust the catalogue instead
-        // of fetching every feed to discover its site.
-        siteUrl: typeof siteUrl === 'string' && siteUrl.trim() ? siteUrl.trim() : null,
-      });
-    }
-
-    if (item.outline) walk(item.outline, out);
-  }
-}
-
 /**
- * Extract feed URLs from an OPML document.
- *
- * Deliberately lenient: a malformed OPML returns an empty list rather than
- * throwing, because this runs on user-submitted uploads and one bad file must
- * not take down the submit endpoint.
- *
- * @param {string} xml raw OPML
- * @returns {Array<{ url: string, title: string, siteUrl: string|null }>} deduped by URL, order preserved
+ * OPML. Reading it is shared with every other directory through
+ * @profullstack/submit-feed (the lenient walk that recurses into folders and
+ * returns an empty list for a malformed upload rather than throwing); writing
+ * it stays here, because the export uses this site's column names and is
+ * streamed in three parts for a document of hundreds of thousands of rows.
  */
-export function parseOpml(xml) {
-  if (typeof xml !== 'string' || !xml.trim()) return [];
-
-  let doc;
-  try {
-    doc = parser.parse(xml);
-  } catch {
-    return [];
-  }
 
-  const found = [];
-  walk(doc?.opml?.body?.outline, found);
-
-  const seen = new Set();
-  return found.filter((f) => {
-    const key = f.url.toLowerCase();
-    if (seen.has(key)) return false;
-    seen.add(key);
-    return true;
-  });
-}
+export { parseOpml } from '@profullstack/submit-feed/core';
 
 /**
  * Render a directory listing as an OPML subscription list.