diff --git a/apps/web/package.json b/apps/web/package.json index ce39a1a..ed957d3 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -10,8 +10,11 @@ "start": "node server.mjs" }, "dependencies": { + "@profullstack/leaderboard": "^0.3.0", + "@profullstack/partners": "^0.2.0", "@profullstack/player": "^0.3.1", "@profullstack/rssamplifier": "workspace:*", + "@profullstack/submit-feed": "^0.1.1", "@profullstack/x402-gateway": "^0.6.0", "@rssamplifier/auth": "workspace:*", "@rssamplifier/db": "workspace:*", @@ -26,8 +29,6 @@ "@swc/helpers": "^0.5.23", "next": "^16.2.4", "react": "^19.2.0", - "react-dom": "^19.2.0", - "@profullstack/leaderboard": "^0.3.0", - "@profullstack/partners": "^0.2.0" + "react-dom": "^19.2.0" } } diff --git a/apps/web/src/lib/mcp/tools.js b/apps/web/src/lib/mcp/tools.js index b076b17..58623d6 100644 --- a/apps/web/src/lib/mcp/tools.js +++ b/apps/web/src/lib/mcp/tools.js @@ -1,6 +1,7 @@ import { q, newId, authors as people } from '@rssamplifier/db'; import { topicSlug } from '@rssamplifier/feed'; import { submitCatalogue, hashIp, EXPRESS_MAX } from '@rssamplifier/ingest'; +import { submitFeedTool } from '@profullstack/submit-feed/core'; import { db, siteUrl } from '../db.js'; import { readerView } from '../reader.js'; @@ -521,24 +522,13 @@ export const TOOLS = [ }, }, - { - name: 'submit_feed', - title: 'Add a feed to the directory', - description: - 'Submit one URL or a list of them. A site URL works as well as a feed URL — the feed is discovered from the page. Anyone may submit; there is no account and no review queue. Feeds resolve inline up to a handful and the rest are queued for the crawler, so the answer says which. Rate limited per caller.', - inputSchema: { - type: 'object', - properties: { - urls: { - type: 'array', - items: { type: 'string' }, - description: 'Feed or site URLs. One is fine.', - maxItems: 200, - }, - }, - required: ['urls'], - }, - annotations: { readOnlyHint: false, destructiveHint: false, openWorldHint: true }, + // The name, description, schema and annotations are the shared contract's, + // so an agent that learned submit_feed on p0dcasters.com finds the same tool + // here. Only `run` is this directory's: the ledger, the express lane and + // the status page are its own. + submitFeedTool({ + directory: 'RSS Amplifier', + maxUrls: 200, async run(args, ctx) { const urls = (Array.isArray(args?.urls) ? args.urls : [args?.urls]) .map((u) => String(u ?? '').trim()) @@ -602,7 +592,7 @@ export const TOOLS = [ statusUrl: `${siteUrl()}/submissions/${submissionId}`, }; }, - }, + }), ]; /** @type {Map} */ diff --git a/packages/feed/package.json b/packages/feed/package.json index d4a0741..5782a4f 100644 --- a/packages/feed/package.json +++ b/packages/feed/package.json @@ -16,6 +16,7 @@ }, "dependencies": { "@mozilla/readability": "^0.6.0", + "@profullstack/submit-feed": "^0.1.1", "@profullstack/x402-client": "0.2.0", "fast-xml-parser": "^5.2.5", "linkedom": "^0.18.13" diff --git a/packages/feed/src/discover.js b/packages/feed/src/discover.js index 96a569d..60fd234 100644 --- a/packages/feed/src/discover.js +++ b/packages/feed/src/discover.js @@ -1,182 +1,34 @@ /** * Feed discovery: people submit "myblog.com", not "myblog.com/feed.xml". * Turning the former into the latter is most of what makes submission painless. - */ - -import { looksLikePlaylist } from './playlist.js'; - -const COMMON_PATHS = [ - '/feed', - '/feed.xml', - '/rss', - '/rss.xml', - '/atom.xml', - '/index.xml', - '/feed/', - '/feeds/posts/default', - '/?feed=rss2', - '/feed.json', -]; - -/** Content types that indicate the body is a feed rather than a web page. */ -const FEED_TYPES = [ - 'application/rss+xml', - 'application/atom+xml', - 'application/feed+json', - 'application/xml', - 'text/xml', - 'application/json', -]; - -/** - * Normalize user input into an absolute http(s) URL. - * - * Accepts "example.com", "//example.com" and "http://example.com", because all - * three get pasted into submission boxes. Returns null for anything that is not - * a usable web URL — including non-http schemes, which must never be fetched. - * - * @param {string} input - * @returns {string|null} - */ -export function normalizeUrl(input) { - if (typeof input !== 'string') return null; - let raw = input.trim(); - if (!raw) return null; - - if (raw.startsWith('//')) raw = `https:${raw}`; - if (!/^https?:\/\//i.test(raw)) { - if (/^[a-z][a-z0-9+.-]*:/i.test(raw)) return null; // mailto:, javascript:, file: - raw = `https://${raw}`; - } - - try { - const u = new URL(raw); - if (u.protocol !== 'http:' && u.protocol !== 'https:') return null; - if (!plausibleHost(u.hostname)) return null; - u.hash = ''; - return u.toString(); - } catch { - return null; - } -} - -/** - * Is this hostname something a publisher could actually be at? - * - * `new URL()` is far more permissive about hosts than DNS is. Quotes, parens, - * `=` and `&` are not forbidden host code points, so `new URL('https://x')` - * happily parses `version="1.0"`, `zombies.)` and `z.` into a hostname — and - * the check this replaced, `hostname.includes('.')`, waved all three through. - * - * That is not a theoretical gap. The bulk-upload scanner splits pasted text on - * whitespace and offers every token as a URL, so pasting raw OPML instead of a - * URL list turned the markup itself into feeds: `https://version="1.0"/`, - * `https://text="gg.deals/`, and a sentence ending in "zombies." into - * `https://zombies.)/`. Roughly 3,700 such rows reached the directory, and - * every one of them is crawled forever on a cadence, failing `blocked-host`. - * - * Validating here rather than in the scanner is deliberate: `normalizeUrl` is - * the single gate every entry path goes through -- web submit, OPML import, - * the queue drain and discovery -- so one check covers all of them, and the - * scanner cannot import this module anyway (it runs in the browser, and - * pulling `@rssamplifier/feed` into client code fails the build on `node:dns`). - * - * Kept deliberately loose about what a real domain looks like: this rejects - * things that cannot be hostnames, not things that are merely unusual. IDN is - * already punycode by the time it arrives, and a bare IPv4 literal is allowed - * through so that a feed genuinely served from one is not newly rejected -- - * private ranges are refused later, by `isPublicHost`. - * - * @param {string} hostname as parsed by `new URL`, so lowercased and punycoded - * @returns {boolean} - */ -function plausibleHost(hostname) { - const host = String(hostname ?? ''); - if (!host || host.length > 253) return false; - - // A dotted-quad is a legitimate, if unusual, place for a feed to live. - if (/^\d{1,3}(\.\d{1,3}){3}$/.test(host)) { - return host.split('.').every((n) => Number(n) <= 255); - } - - // Anything outside the DNS alphabet is markup or prose, not a host. - if (!/^[a-z0-9.-]+$/.test(host)) return false; - - const labels = host.split('.'); - // Two labels at least, none empty -- which is what rules out `z.` and `you.`, - // whose trailing dot leaves an empty final label. - if (labels.length < 2) return false; - if (labels.some((l) => l === '' || l.length > 63 || l.startsWith('-') || l.endsWith('-'))) { - return false; - } - - // A public suffix is letters. This is the part that rejects `1.0` while - // leaving `example.co.uk` and `xn--bcher-kva.de` alone. - return /^[a-z]{2,}$/.test(labels[labels.length - 1]); -} - -/** - * Extract feed URLs advertised by a page's tags. - * - * This is the correct, standards-based path — the guessed paths below are only - * a fallback for sites that don't advertise. * - * @param {string} html - * @param {string} baseUrl for resolving relative hrefs - * @returns {string[]} absolute feed URLs, in document order, deduped + * The mechanics live in @profullstack/submit-feed now, extracted from here so + * p0dcasters.com and any later directory take feeds the same way: the URL + * gate that refuses markup-as-hostname (the 3,700 rows of `https://version="1.0"/` + * that the browser scanner once pushed in), the walk, + * the conventional paths, and the body sniff. This module is the seam: the + * same names, so nothing in the workspace changed its imports, plus the one + * thing this directory does that the package does not, which is to admit a + * playlist as a feed. */ -export function findFeedLinks(html, baseUrl) { - if (typeof html !== 'string') return []; - const out = []; - - // Match tags, then pull attributes out individually: attribute order - // varies between generators and a single positional regex misses most of them. - const tags = html.match(/]*>/gi) ?? []; - - for (const tag of tags) { - const rel = /\brel\s*=\s*["']?([^"'>\s]+)/i.exec(tag)?.[1]?.toLowerCase(); - if (rel !== 'alternate') continue; - - const type = /\btype\s*=\s*["']?([^"'>\s]+)/i.exec(tag)?.[1]?.toLowerCase(); - if (!type || !FEED_TYPES.includes(type)) continue; - const href = /\bhref\s*=\s*["']([^"']+)["']/i.exec(tag)?.[1]; - if (!href) continue; - - try { - const abs = new URL(href, baseUrl).toString(); - if (!out.includes(abs)) out.push(abs); - } catch { - // skip unresolvable href - } - } +import { + normalizeUrl, + findFeedLinks, + guessFeedUrls, + looksLikeFeed as looksLikeFeedDocument, +} from '@profullstack/submit-feed/core'; +import { looksLikePlaylist } from './playlist.js'; - return out; -} - -/** - * Candidate feed URLs to try for a site, in priority order. - * - * @param {string} siteUrl - * @returns {string[]} - */ -export function guessFeedUrls(siteUrl) { - const out = []; - for (const p of COMMON_PATHS) { - try { - out.push(new URL(p, siteUrl).toString()); - } catch { - // skip - } - } - return out; -} +export { normalizeUrl, findFeedLinks, guessFeedUrls }; /** * Decide whether a fetched response looks like a feed. * - * Content-type alone is unreliable — plenty of feeds are served as text/plain - * or text/html — so the body is sniffed too. + * A playlist is a feed here -- a list of media with titles -- so it is admitted + * on the same footing rather than sniffed for afterwards. Everything else is + * the shared sniff: content type first, then the head of the body, because + * plenty of feeds are served as text/plain or text/html. * * @param {string} contentType * @param {string} body @@ -185,24 +37,6 @@ export function guessFeedUrls(siteUrl) { * @returns {boolean} */ export function looksLikeFeed(contentType, body, url = '') { - // A playlist is a feed here — a list of media with titles — so it is admitted - // on the same footing rather than sniffed for afterwards. if (looksLikePlaylist(contentType, body, url)) return true; - - const ct = (contentType || '').toLowerCase(); - if (FEED_TYPES.some((t) => ct.includes(t))) { - // application/json is only a feed if it's actually JSON Feed. - if (ct.includes('json') && !ct.includes('feed+json')) { - return /"(?:items|version)"\s*:/.test(body ?? ''); - } - return true; - } - - const head = (body ?? '').slice(0, 2000).toLowerCase(); - if (/]*xmlns/.test(head)) return true; - if (/} out - */ -function walk(node, out) { - if (!node) return; - const list = Array.isArray(node) ? node : [node]; - - for (const item of list) { - if (!item || typeof item !== 'object') continue; - - const url = item['@xmlUrl'] || item['@xmlurl']; - if (typeof url === 'string' && url.trim()) { - const siteUrl = item['@htmlUrl'] || item['@htmlurl']; - out.push({ - url: url.trim(), - title: (item['@title'] || item['@text'] || '').toString().trim(), - // Carried through for bulk imports, which trust the catalogue instead - // of fetching every feed to discover its site. - siteUrl: typeof siteUrl === 'string' && siteUrl.trim() ? siteUrl.trim() : null, - }); - } - - if (item.outline) walk(item.outline, out); - } -} - /** - * Extract feed URLs from an OPML document. - * - * Deliberately lenient: a malformed OPML returns an empty list rather than - * throwing, because this runs on user-submitted uploads and one bad file must - * not take down the submit endpoint. - * - * @param {string} xml raw OPML - * @returns {Array<{ url: string, title: string, siteUrl: string|null }>} deduped by URL, order preserved + * OPML. Reading it is shared with every other directory through + * @profullstack/submit-feed (the lenient walk that recurses into folders and + * returns an empty list for a malformed upload rather than throwing); writing + * it stays here, because the export uses this site's column names and is + * streamed in three parts for a document of hundreds of thousands of rows. */ -export function parseOpml(xml) { - if (typeof xml !== 'string' || !xml.trim()) return []; - - let doc; - try { - doc = parser.parse(xml); - } catch { - return []; - } - const found = []; - walk(doc?.opml?.body?.outline, found); - - const seen = new Set(); - return found.filter((f) => { - const key = f.url.toLowerCase(); - if (seen.has(key)) return false; - seen.add(key); - return true; - }); -} +export { parseOpml } from '@profullstack/submit-feed/core'; /** * Render a directory listing as an OPML subscription list. diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 811c330..9346329 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -55,6 +55,9 @@ importers: '@profullstack/rssamplifier': specifier: workspace:* version: link:../cli + '@profullstack/submit-feed': + specifier: ^0.1.1 + version: 0.1.1 '@profullstack/x402-gateway': specifier: ^0.6.0 version: 0.6.0 @@ -145,6 +148,9 @@ importers: '@mozilla/readability': specifier: ^0.6.0 version: 0.6.0 + '@profullstack/submit-feed': + specifier: ^0.1.1 + version: 0.1.1 '@profullstack/x402-client': specifier: 0.2.0 version: 0.2.0 @@ -621,6 +627,10 @@ packages: react: optional: true + '@profullstack/submit-feed@0.1.1': + resolution: {integrity: sha512-lnBeS1dHEnQS+1pHslwMrc8uhY8yWm4A9x1dTagPAGLVh3vvUf1xO//3bV2c8r1XYYPUnIJdEWyIP7jptbwmPg==} + engines: {node: '>=20.11'} + '@profullstack/x402-client@0.2.0': resolution: {integrity: sha512-kJGAomE9Tf/ruhsCStaKVXpjklCbOh2UFcMGTIfTqcWaSsW2NFWQRaTxJBLvc6r7Iy2+fpAWzfJnerolI6IrRQ==} engines: {node: '>=20.19'} @@ -1379,6 +1389,10 @@ snapshots: optionalDependencies: react: 19.2.8 + '@profullstack/submit-feed@0.1.1': + dependencies: + fast-xml-parser: 5.10.1 + '@profullstack/x402-client@0.2.0': dependencies: '@noble/curves': 2.4.0 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index a12d4f9..a5b4497 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -17,3 +17,4 @@ minimumReleaseAgeExclude: - '@profullstack/x402-client@0.2.0' - '@profullstack/leaderboard@0.3.0 || 0.3.1' - '@profullstack/partners@0.2.0' + - '@profullstack/submit-feed@0.1.1'