From e366df8287e5275e0c3bbc0fc0b4ac8f76bdac2d Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Sun, 13 Sep 2026 04:50:48 +0000 Subject: [PATCH] List every public profile, newest change first, for a directory to pull GET /api/openprofiles: each author's OpenProfile.md URL, page, accounts and site, ordered by the later of the author row's and the overlay's updated_at so a re-crawl and a correction both surface, with a keyset cursor over (updated_at, id), ?since= for what moved, ?limit= up to 500. Public profiles only, above the site's own confidence floor. This is what nichedb's profiles collection reads instead of walking author pages. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01S7yeJUHGxA4P5N74xnsRPQ --- README.md | 1 + apps/web/src/app/api/openprofiles/route.js | 74 ++++++++++++++++++++++ apps/web/src/app/skill.md/route.js | 3 + apps/web/src/lib/llms.js | 1 + apps/web/test/openprofile.test.js | 9 +++ packages/db/src/profiles.js | 62 ++++++++++++++++++ packages/db/test/profiles.test.js | 36 +++++++++++ 7 files changed, 186 insertions(+) create mode 100644 apps/web/src/app/api/openprofiles/route.js diff --git a/README.md b/README.md index d0d982c..9ddcc83 100644 --- a/README.md +++ b/README.md @@ -241,6 +241,7 @@ All send `access-control-allow-origin: *` and need no key. | `/api/authors` | The people behind the feeds; `?feed={url}` finds the people behind one feed | | `/authors/{slug}/openprofile.md` | One person as an [OpenProfile.md](https://logicsrc.com/openprofile), Broadcast section per show they publish | | `/api/authors/{slug}/openprofile` | The same file, `?format=json` for the parsed shape; `PUT` it to correct it (owner only) | +| `/api/openprofiles` | Every public profile, newest change first (`?since=`, `?cursor=`, `?limit=` up to 500); what a directory pulls | | `/api/authors/{slug}/claim` | `POST` to claim an author as yourself; verified by the address they published or by their site linking back | ```bash diff --git a/apps/web/src/app/api/openprofiles/route.js b/apps/web/src/app/api/openprofiles/route.js new file mode 100644 index 0000000..efb0a8e --- /dev/null +++ b/apps/web/src/app/api/openprofiles/route.js @@ -0,0 +1,74 @@ +import { profiles } from '@rssamplifier/db'; + +import { db, siteUrl } from '../../../lib/db.js'; +import { profileUrl } from '../../../lib/openprofile.js'; +import { json } from '../authors/route.js'; + +export const dynamic = 'force-dynamic'; + +/** Most profiles in one page. */ +const MAX = 500; + +/** + * Every public author profile, newest change first, for a directory that + * pulls them (nichedb's profiles collection reads this). + * + * `?since=` narrows to profiles changed at or after an ISO stamp, so a puller + * asks for what moved since its last pass; `?cursor=` continues a page; both + * compose. An owner who switched their file off is not listed. Keyless, like + * every read here. + * + * @param {Request} req + */ +export async function GET(req) { + const url = new URL(req.url); + const limit = Math.min(Math.max(Number(url.searchParams.get('limit') ?? 100) || 100, 1), MAX); + const since = (url.searchParams.get('since') ?? '').trim() || null; + const rawCursor = url.searchParams.get('cursor'); + const cursor = decodeCursor(rawCursor); + if (rawCursor && !cursor) return json({ error: 'bad-cursor' }, 400); + if (since && Number.isNaN(Date.parse(since))) return json({ error: 'bad-since' }, 400); + + const rows = await profiles.listOpenProfiles(db(), { since, limit, cursor }); + const base = siteUrl(); + const last = rows[rows.length - 1]; + + return json({ + openprofiles: rows.map((r) => ({ + id: r.slug, + name: r.name, + url: profileUrl(base, r.slug), + page: `${base}/authors/${encodeURIComponent(r.slug)}`, + updatedAt: r.updated_at, + accounts: r.urls, + web: r.site_url, + })), + next: rows.length === limit && last ? encodeCursor({ updatedAt: last.updated_at, id: last.id }) : null, + }); +} + +/** + * The cursor is the last row's (updated_at, id), which is exactly the keyset + * the query pages on; opaque to the caller, plain to us. + * + * @param {{ updatedAt: string, id: string }} at + * @returns {string} + */ +export function encodeCursor(at) { + return Buffer.from(JSON.stringify([at.updatedAt, at.id])).toString('base64url'); +} + +/** + * @param {string|null} raw + * @returns {{ updatedAt: string, id: string }|null} + */ +export function decodeCursor(raw) { + if (!raw) return null; + try { + const [updatedAt, id] = JSON.parse(Buffer.from(raw, 'base64url').toString('utf8')); + if (typeof updatedAt !== 'string' || typeof id !== 'string' || !updatedAt || !id) return null; + return { updatedAt, id }; + } catch { + return null; + } +} diff --git a/apps/web/src/app/skill.md/route.js b/apps/web/src/app/skill.md/route.js index beed234..6177885 100644 --- a/apps/web/src/app/skill.md/route.js +++ b/apps/web/src/app/skill.md/route.js @@ -55,6 +55,9 @@ is not a search engine for the whole web. person claims it (\`POST /api/authors/{slug}/claim\`) and corrects it with \`PUT /api/authors/{slug}/openprofile\` (text/markdown or a JSON patch; API key or an OpenAccess grant for \`openprofile:edit\`). +- \`GET ${base}/api/openprofiles?since=&limit=&cursor=\` — every public profile, + newest change first, with each file's URL, page, accounts and site; for a + directory that pulls them. - \`GET ${base}/api/feeds/{slug}\` also carries \`authors\` and \`links\`. \`links\` is the blog's own accounts — Mastodon, Bluesky, X, LinkedIn, GitHub and the rest — which is what a blog with no byline has instead of an author, and roughly a diff --git a/apps/web/src/lib/llms.js b/apps/web/src/lib/llms.js index 55aa625..800f41a 100644 --- a/apps/web/src/lib/llms.js +++ b/apps/web/src/lib/llms.js @@ -170,6 +170,7 @@ export async function llmsTxt(opts = {}) { `- [Authors, JSON](${base}/api/authors): the people behind the feeds and where else they publish; ?network=email|fediverse|bluesky|github|website|linktree, ?q= searches names, ?min= sets the confidence floor`, `- [One author, JSON](${base}/api/authors/{slug}): their links and everything they publish here; ?feed= on /api/authors finds the people behind one feed`, `- [One author, OpenProfile.md](${base}/authors/{slug}/openprofile.md): the same person as a portable profile file (logicsrc.com/openprofile) with a Broadcast section per show they publish; the person claims it at /authors/{slug} and edits it with PUT ${base}/api/authors/{slug}/openprofile`, + `- [All public profiles, JSON](${base}/api/openprofiles): every author's OpenProfile.md URL, newest change first; ?since= for what moved, ?cursor= to continue, ?limit= up to 500; what a directory pulls`, `- [OPML export](${base}/opml): the whole directory as a subscription list, one category with ?kind=, or one subject with ?topic=`, `- [Submit](${base}/api/submit): POST {"url":"..."} or {"urls":[...]} or {"opml":"..."}`, `- [Discover](${base}/api/discover): POST {"keywords":["..."]} — find blogs by subject`, diff --git a/apps/web/test/openprofile.test.js b/apps/web/test/openprofile.test.js index 9a1cb43..ba9fdeb 100644 --- a/apps/web/test/openprofile.test.js +++ b/apps/web/test/openprofile.test.js @@ -12,6 +12,7 @@ import { } from '../src/lib/openprofile.js'; import { claimVerdict, isOwner, linksBack } from '../src/lib/profileAuth.js'; import { principalFromToken } from '../src/lib/openaccess.js'; +import { decodeCursor, encodeCursor } from '../src/app/api/openprofiles/route.js'; const BASE = 'https://rssamplifier.test'; @@ -233,6 +234,14 @@ test('an OpenAccess token yields a principal with scopes and an email when the h assert.equal(await principalFromToken(null), null); }); +test('the listing cursor round-trips and rejects junk', () => { + const at = { updatedAt: '2026-09-13T04:00:00.000Z', id: 'a1' }; + assert.deepEqual(decodeCursor(encodeCursor(at)), at); + assert.equal(decodeCursor('not-base64-json'), null); + assert.equal(decodeCursor(Buffer.from('[1,2]').toString('base64url')), null); + assert.equal(decodeCursor(null), null); +}); + test('small helpers', () => { assert.equal(profileUrl(BASE, 'ada lovelace'), `${BASE}/authors/ada%20lovelace/openprofile.md`); }); diff --git a/packages/db/src/profiles.js b/packages/db/src/profiles.js index 659f1bd..41deaf7 100644 --- a/packages/db/src/profiles.js +++ b/packages/db/src/profiles.js @@ -173,3 +173,65 @@ export async function keywordsForFeeds(db, feedIds, perFeed = 8) { } return out; } + +/** + * Every author with a public profile, newest change first, for a directory + * that pulls profiles (nichedb): keyset paging over (updated_at, id), where + * updated_at is the later of the author row's and the overlay's, so an edit + * and a re-crawl both surface. `since` narrows to changes at or after a stamp. + * + * Only authors above the site's own confidence floor: a weak attribution + * the site itself does not list is not a person to publish elsewhere. + * + * @param {Client} db + * @param {{ since?: string|null, limit?: number, cursor?: { updatedAt: string, id: string }|null, minConfidence?: number }} [opts] + * @returns {Promise>} + */ +export async function listOpenProfiles(db, opts = {}) { + const limit = Math.min(Math.max(Number(opts.limit ?? 100) || 100, 1), 500); + const minConfidence = Number(opts.minConfidence ?? 0.6); + const stamp = 'max(a.updated_at, coalesce(p.updated_at, a.updated_at))'; + const where = ['a.confidence >= ?', 'coalesce(p.public, 1) = 1']; + const args = [minConfidence]; + if (opts.since) { + where.push(`${stamp} >= ?`); + args.push(opts.since); + } + if (opts.cursor) { + where.push(`(${stamp} < ? or (${stamp} = ? and a.id < ?))`); + args.push(opts.cursor.updatedAt, opts.cursor.updatedAt, opts.cursor.id); + } + const { rows } = await db.execute({ + sql: `select a.id, a.slug, a.name, a.site_url, ${stamp} as updated_at + from authors a + left join author_profiles p on p.author_id = a.id + where ${where.join(' and ')} + order by updated_at desc, a.id desc + limit ?`, + args: [...args, limit], + }); + const ids = rows.map((r) => String(r.id)); + /** @type {Map} */ + const urls = new Map(); + if (ids.length) { + const links = await db.execute({ + sql: `select author_id, url from author_links + where author_id in (${ids.map(() => '?').join(',')}) and network <> 'email' + order by verified desc, network asc`, + args: ids, + }); + for (const l of links.rows) { + const list = urls.get(String(l.author_id)) ?? []; + list.push(String(l.url)); + urls.set(String(l.author_id), list); + } + } + return rows.map((r) => ({ + id: String(r.id), + slug: String(r.slug), + name: String(r.name), + site_url: r.site_url == null ? null : String(r.site_url), + updated_at: String(r.updated_at), + urls: urls.get(String(r.id)) ?? [], + })); +} diff --git a/packages/db/test/profiles.test.js b/packages/db/test/profiles.test.js index f91dd34..e0dfd12 100644 --- a/packages/db/test/profiles.test.js +++ b/packages/db/test/profiles.test.js @@ -113,6 +113,42 @@ test('saving stores the overlay and the switch, and keeps the claim', async () = assert.deepEqual(mine.map((p) => p.slug), ['ada-lovelace']); }); +test('the listing is newest change first, pages by keyset, honours since and the public switch', async () => { + const bob = await authors.upsertAuthor(db, { + identityKey: 'https://bob.example', + slug: 'bob', + name: 'Bob', + normName: 'bob', + siteUrl: 'https://bob.example', + confidence: 0.9, + }); + await authors.addAuthorLinks(db, bob.id, [{ network: 'github', url: 'https://github.com/bob', source: 'rel-me' }]); + const weak = await authors.upsertAuthor(db, { identityKey: 'x@weak', slug: 'weak', name: 'Weak', normName: 'weak', confidence: 0.3 }); + assert.ok(weak.id); + + // Ada corrects her overlay after Bob's row landed, so Ada moved most + // recently: an edit surfaces the way a re-crawl does. + await new Promise((r) => setTimeout(r, 5)); + await profiles.saveProfile(db, ada.id, { overrides: { headline: 'Countess, programmer, host.' } }); + const all = await profiles.listOpenProfiles(db, { limit: 10 }); + assert.deepEqual(all.map((r) => r.slug), ['ada-lovelace', 'bob'], 'the weak attribution is not listed'); + assert.deepEqual(all[1].urls, ['https://github.com/bob']); + assert.ok(all[0].updated_at >= all[1].updated_at); + + const first = await profiles.listOpenProfiles(db, { limit: 1 }); + const second = await profiles.listOpenProfiles(db, { limit: 1, cursor: { updatedAt: first[0].updated_at, id: first[0].id } }); + assert.deepEqual([first[0].slug, second[0].slug], ['ada-lovelace', 'bob']); + const third = await profiles.listOpenProfiles(db, { limit: 1, cursor: { updatedAt: second[0].updated_at, id: second[0].id } }); + assert.deepEqual(third, []); + + const moved = await profiles.listOpenProfiles(db, { since: all[0].updated_at }); + assert.deepEqual(moved.map((r) => r.slug), ['ada-lovelace']); + + await profiles.saveProfile(db, ada.id, { public: false }); + assert.deepEqual((await profiles.listOpenProfiles(db, {})).map((r) => r.slug), ['bob'], 'public off hides the profile'); + await profiles.saveProfile(db, ada.id, { public: true }); +}); + test('a bad overrides column reads as an empty overlay rather than a crash', async () => { await db.execute({ sql: "update author_profiles set overrides = '{not json' where author_id = ?", args: [ada.id] }); const row = await profiles.profileForAuthor(db, ada.id);