diff --git a/Dockerfile b/Dockerfile index 7582d4b..940cf1d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,9 +25,11 @@ FROM base AS runtime ENV NODE_ENV=production # `ntsb-accidents` reads a 558 MB Microsoft Access database out of a zip, # because the NTSB publishes no working API. mdbtools turns that into NDJSON. -# Nothing else in the image needs either, and both are a few hundred kilobytes. +# xz-utils is for the dump adapters: MusicBrainz ships tar.xz, Bun.Archive +# cannot read xz, and the image's tar needs the xz binary to stream one member +# out (`tar -xJOf`). All three are a few hundred kilobytes. RUN apt-get update \ - && apt-get install -y --no-install-recommends mdbtools unzip \ + && apt-get install -y --no-install-recommends mdbtools unzip xz-utils \ && rm -rf /var/lib/apt/lists/* # Bun's isolated linker keeps each workspace's node_modules beside it, so the # whole deps stage comes across rather than only /app/node_modules. diff --git a/README.md b/README.md index 3876031..3f4520e 100644 --- a/README.md +++ b/README.md @@ -20,8 +20,8 @@ Adapters are one file each in `packages/adapters/src`. One hundred and twenty-ei | games | `steam`, `steam-news`, `igdb`, `igdb-catalog` (every game IGDB knows, walked by id then kept current from updated_at), `wikidata-games`, `steam-catalog` | IGDB only (Twitch client) | | packages | `npm`, `pypi`, `crates`, `go-modules`, `huggingface`, `github-releases` | no (GitHub token optional) | | filings | `edgar`, `federal-register`, `courtlistener` | CourtListener only | -| music | `musicbrainz` | no | -| books | `openlibrary`, `gutenberg-catalog`, `librivox-catalog` | no | +| music | `musicbrainz`, `musicbrainz-catalog`, `discogs-catalog` | no | +| books | `openlibrary`, `gutenberg-catalog`, `librivox-catalog`, `openlibrary-catalog` | no | | tabletop | `scryfall-sets`, `scryfall-cards` | no | | space | `launch-library` | no | | chess | `lichess-broadcasts` | no | @@ -41,7 +41,7 @@ Adapters are one file each in `packages/adapters/src`. One hundred and twenty-ei | ai-incidents | `rogue-ai-incidents`, `rogue-ai-research`, `aiid-reports` | no | | news | `newsfeed`, `gdelt`, `rssamplifier`, `brisk`, `news-channels` | no | | domains | `ntld-totals`, `ntld-tlds`, `ntld-launches`, `ntld-changes` | no | -| podcasts | `podcasts`, `p0dcasters` | no | +| podcasts | `podcasts`, `p0dcasters`, `podcastindex-catalog` | no | | aviation | `faa-nas-status`, `aviation-hazards`, `aviation-metar`, `ntsb-accidents`, `adsb-flights` | no (NTSB needs mdbtools + unzip, in the Dockerfile) | | water | `nwps-river-gauges`, `coops-water-levels`, `drought-monitor`, `ndbc-buoys`, `nws-surf-zone` | no | | consumer-finance | `cfpb-complaints`, `fdic-institutions`, `fdic-structure-changes` | no | @@ -228,6 +228,54 @@ export const example = defineAdapter({ Register it in `packages/adapters/src/index.js`. The core normalises items, hashes them so unchanged rows cost no write, writes in batches, records the run and reschedules. +### Streaming a dump + +A source that reads a multi-gigabyte file (MusicBrainz, Discogs, Open Library, Podcast Index) cannot return one array: the file does not fit in memory and the walk does not fit in one run. Instead `pull` yields batches and the core drains them one at a time, writing each and saving its cursor before reading the next. `pull` is an `async *` generator (or returns `{ items: }`, which is the same thing), each yielded value is `{ items, cursor }`, and the generator's `return` value is the run's `{ cursor, note, nextInMinutes }`. + +```js +import { dumpDir, gzipLines } from '@nichedb/core/dump'; + +export const example = defineAdapter({ + name: 'example-dump', + // ... + cadenceMinutes: 60, + budgetMs: 60 * 60_000, + async *pull({ cursor, http, log, deadline }) { + const version = await http.text('https://example.com/dumps/LATEST'); + const skip = cursor.version === version ? Number(cursor.skip) || 0 : 0; + if (cursor.version === version && cursor.done) return { cursor, note: 'unchanged' }; + + const dir = await dumpDir('example'); + const file = `${dir}/${version}.ndjson.gz`; + const dl = await http.download('https://example.com/dumps/latest.ndjson.gz', file); + if (!dl.complete) return { cursor: { version, skip }, note: 'download in progress', nextInMinutes: 1 }; + + let n = skip; + let batch = []; + for await (const line of gzipLines(file, { skip })) { + n += 1; + batch.push(toItem(JSON.parse(line))); + if (batch.length === 500) { + yield { items: batch, cursor: { version, skip: n } }; + batch = []; + if (Date.now() > deadline) return { cursor: { version, skip: n }, nextInMinutes: 1 }; + } + } + if (batch.length) yield { items: batch, cursor: { version, skip: n } }; + return { cursor: { version, skip: n, done: true }, note: 'complete' }; + }, +}); +``` + +The rules: + +- **`budgetMs`** is the run's wall-clock budget and replaces `INGEST_RUN_DEADLINE_MS` (4 minutes) for this adapter. The reaper window and the BullMQ lock widen to the largest budget declared, so an hour-long budget is safe to declare. The core hands it back as `deadline`; check it between batches and return. A batch written past the deadline ends the run from the core's side too, rescheduling in a minute, but the adapter checking first is what keeps a run inside its lock. +- **Each batch's `cursor`** is the position *after* that batch's items (a line count, a byte offset, a last id) plus whatever identifies the file (a dump date, a version, a `LATEST` value), so a new upstream file resets the walk. It is saved as soon as the batch is in the table. At-least-once delivery is fine: a batch re-written after a crash is an idempotent upsert on `(source, externalId)` and unchanged rows cost nothing. +- **`return`** the final `{ cursor, note, nextInMinutes }`. `nextInMinutes: 1` keeps an unfinished walk moving between cadences; `done: true` in the cursor lets the next run short-circuit when the upstream file has not changed. +- **`http.download(url, path, { timeoutMs, headers, onProgress })`** streams to disk and resumes with `Range` from whatever is already there (append on 206, restart on 200, 416 means whole). It returns `{ path, bytes, complete }`; on `complete: false`, return and call it again next run. The user agent is sent; Podcast Index refuses requests without one. Discogs ignores `Range` and rate-limits hard: expect a restart and keep requests to a handful an hour. +- **`dumpDir(name)`** is where the file goes: under `INGEST_DATA_DIR` when set (mount a volume there), else the OS temp dir, which a redeploy wipes. The cursor is the walk; the directory is a cache. +- **Readers**: `gzipLines(path, { skip })`, `xzLines(path, { member, skip })` (MusicBrainz: `member: 'mbdump/artist'`, needs `xz` on the host), `tsvJsonLines(path, { skip })` for Open Library's five-column rows (each record carries its `lineNo`), `lineOffsetReader(path, { offset })` for a plain file you want to seek in, `untar(path, dir)` and `sqliteRows(dbPath, sql)` for Podcast Index. Every reader yields every line, so `skip + lines read` is the file position. Skipping into a compressed file re-inflates from the top (seconds per gigabyte for gzip, slower for xz); a plain-file `offset` is a seek. + ## License MIT diff --git a/apps/cli/package.json b/apps/cli/package.json index 10552a8..9c0e979 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -1,6 +1,6 @@ { "name": "@profullstack/nichedb", - "version": "0.23.0", + "version": "0.24.0", "description": "CLI and MCP bridge for NicheDB: browse collections, manage sources and feeds, search items, from any deployment", "type": "module", "main": "src/index.js", diff --git a/apps/cli/src/index.js b/apps/cli/src/index.js index bcde674..6fdf282 100644 --- a/apps/cli/src/index.js +++ b/apps/cli/src/index.js @@ -15,7 +15,7 @@ import { homedir, tmpdir } from 'node:os'; import { join } from 'node:path'; import { createInterface } from 'node:readline'; -export const VERSION = '0.23.0'; +export const VERSION = '0.24.0'; const DEFAULT_API = process.env.NICHEDB_API ?? 'https://nichedb.dev'; const CONFIG_DIR = join(process.env.XDG_CONFIG_HOME ?? join(homedir(), '.config'), 'nichedb'); const CONFIG_FILE = join(CONFIG_DIR, 'config.json'); diff --git a/apps/web/package.json b/apps/web/package.json index 63dc470..7bdb014 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -1,6 +1,6 @@ { "name": "@nichedb/web", - "version": "0.23.0", + "version": "0.24.0", "private": true, "type": "module", "scripts": { diff --git a/apps/worker/package.json b/apps/worker/package.json index 846cb86..423a000 100644 --- a/apps/worker/package.json +++ b/apps/worker/package.json @@ -1,6 +1,6 @@ { "name": "@nichedb/worker", - "version": "0.23.0", + "version": "0.24.0", "private": true, "type": "module", "scripts": { diff --git a/package.json b/package.json index d21fc65..22a14eb 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "niche-db", - "version": "0.23.0", + "version": "0.24.0", "private": true, "type": "module", "description": "An open platform for large, ever-growing databases of real-time data: sources in, feeds out. Web, API, CLI and MCP on one Postgres.", diff --git a/packages/adapters/package.json b/packages/adapters/package.json index 4a06379..37a8656 100644 --- a/packages/adapters/package.json +++ b/packages/adapters/package.json @@ -1,6 +1,6 @@ { "name": "@nichedb/adapters", - "version": "0.23.0", + "version": "0.24.0", "private": true, "type": "module", "exports": { diff --git a/packages/adapters/src/discogs-catalog.js b/packages/adapters/src/discogs-catalog.js new file mode 100644 index 0000000..9758146 --- /dev/null +++ b/packages/adapters/src/discogs-catalog.js @@ -0,0 +1,884 @@ +import { createHash } from 'node:crypto'; +import { createReadStream } from 'node:fs'; +import { mkdir, open, readdir, rename, stat, unlink } from 'node:fs/promises'; +import { join } from 'node:path'; +import { pipeline } from 'node:stream'; +import { createGunzip } from 'node:zlib'; +import { defineAdapter, looseDate, slugify } from '@nichedb/core/adapter'; +import { dumpDir } from '@nichedb/core/dump'; + +/** + * Every artist and every master release on Discogs, from the monthly XML dumps. + * + * Discogs publishes its whole database on the first of each month as a handful + * of gzipped XML files under data.discogs.com, released to the public domain + * (CC0). This adapter reads two of them: `_artists.xml.gz` (about 474 MB, some + * nine million artists) and `_masters.xml.gz` (about 597 MB, the master + * releases that group an album's pressings). Releases and labels are left out: + * releases alone are 11 GB compressed and the music collection wants the + * catalogue of who and what, not every pressing. + * + * WHAT THE SERVER DOES, AND WHAT THAT DICTATES + * + * data.discogs.com answers every request with a 200 and the whole file. It + * ignores `Range`, so a download that drops is a download that starts over. + * It also rate-limits hard: a handful of requests in an hour bought a 429 with + * `retry-after: 3359`, and asking again inside that window re-arms it. So a + * run makes at most ONE request (a file, or the checksum list), never asks for + * a file whose complete copy is already on disk, and when it is told to wait + * it schedules the next run past the wait rather than sleeping into it. A + * month costs three requests spread over three runs ten minutes apart. + * The core's `http.request` retries a 429 after a minute, which is exactly the + * wrong move here, so the two requests go through `fetch` directly with the + * deployment's contact address in a descriptive user agent. + * + * A finished download is checked against `discogs__CHECKSUM.txt` + * (sha256) before it is walked; a mismatch discards the file. + * + * HOW THE FILE IS READ + * + * Records are not one per line: a profile carries literal newlines. The + * reader gunzips the file chunk by chunk and scans the text for the record + * tag (`` with `` as a child element; `` with + * the id as an attribute), holding one record at a time. Each record goes + * through a small tolerant element parser, no regex and no dependency, and a + * record that cannot become an item is counted and skipped, never thrown. + * + * The cursor is `{ month, entity, recordIndex }`: which dump, which of the two + * files, and how many records of it are already written. The core saves it + * after every batch of 500, so a crash costs one re-written batch. A new + * month resets the walk; skipping back into a file re-inflates from the top, + * which is well under a minute for either file. + */ + +export const BASE = 'https://data.discogs.com'; +export const ENTITIES = ['artists', 'masters']; +export const RECORD_TAG = { artists: 'artist', masters: 'master' }; +export const BATCH_SIZE = 500; +export const SUMMARY_MAX = 600; +export const BUDGET_MS = 55 * 60_000; +export const ATTRIBUTION = 'Discogs, CC0'; + +/** Minutes a run waits after stopping short of the end of a pass. */ +export const RESUME_MINUTES = 10; + +/** A month's file that is not there yet: look again in six hours. */ +export const NOT_PUBLISHED_MINUTES = 6 * 60; + +/** Below this much of the budget a run does not start a download. */ +export const MIN_DOWNLOAD_MS = 15 * 60_000; + +/** Wait when a 429 comes without a retry-after; the observed window was 3359 s. */ +export const DEFAULT_RETRY_AFTER_S = 3600; + +const PROJECT_URL = 'https://github.com/profullstack/niche-db'; + +/** `YYYYMM01` for the dump that covers the month `now` is in. */ +export function dumpMonth(now = new Date()) { + const d = new Date(now); + return `${d.getUTCFullYear()}${String(d.getUTCMonth() + 1).padStart(2, '0')}01`; +} + +export const fileName = (month, entity) => `discogs_${month}_${entity}.xml.gz`; +export const checksumName = (month) => `discogs_${month}_CHECKSUM.txt`; + +/** `https://data.discogs.com/?download=data%2F2026%2Fdiscogs_20260901_artists.xml.gz` */ +export function dumpUrl(month, name) { + const year = String(month).slice(0, 4); + return `${BASE}/?download=${encodeURIComponent(`data/${year}/${name}`)}`; +} + +/** Minutes until the next dump is due: the first of next month, six hours in. */ +export function nextDumpMinutes(now = new Date()) { + const d = new Date(now); + const next = Date.UTC(d.getUTCFullYear(), d.getUTCMonth() + 1, 1, 6, 0, 0); + return Math.max(60, Math.ceil((next - d.getTime()) / 60_000)); +} + +/** A `retry-after` header (seconds or an HTTP date) as seconds from `now`. */ +export function retryAfterSeconds(header, now = Date.now()) { + const s = String(header ?? '').trim(); + if (!s) return DEFAULT_RETRY_AFTER_S; + if (/^\d+$/.test(s)) return Math.max(0, Number(s)); + const at = Date.parse(s); + if (Number.isFinite(at)) return Math.max(0, Math.ceil((at - now) / 1000)); + return DEFAULT_RETRY_AFTER_S; +} + +/** Minutes to schedule past a retry-after, with a little slack so the window has closed. */ +export const retryAfterMinutes = (header, now) => + Math.ceil(retryAfterSeconds(header, now) / 60) + 2; + +/** Who is asking, for the two requests a run makes. */ +export function userAgent(env = {}) { + const contact = env?.contactEmail ? `; ${env.contactEmail}` : ''; + return `niche-db discogs-catalog (+${PROJECT_URL}${contact})`; +} + +/** + * `discogs__CHECKSUM.txt`: sha256sum's format, one ` ` per + * line. Read tolerantly: whichever token is 64 hex characters is the hash and + * the other is the file name, so a swapped or tab-separated line still reads. + */ +export function parseChecksums(text) { + const out = {}; + for (const raw of String(text ?? '').split(/\r?\n/)) { + const parts = raw.trim().split(/\s+/).filter(Boolean); + if (parts.length < 2) continue; + const hash = parts.find((p) => /^[0-9a-f]{64}$/i.test(p)); + const name = parts.find((p) => p !== hash && /\.\w+$/.test(p)); + if (hash && name) out[name.replace(/^\*/, '')] = hash.toLowerCase(); + } + return out; +} + +/** sha256 of a file on disk, streamed. */ +export async function sha256File(path) { + const hash = createHash('sha256'); + for await (const chunk of createReadStream(path)) hash.update(chunk); + return hash.digest('hex'); +} + +// ── XML ─────────────────────────────────────────────────────────────────────── + +const CODE_POINT = (n, original) => { + if (!Number.isFinite(n) || n <= 0 || n > 0x10ffff || (n >= 0xd800 && n <= 0xdfff)) + return original; + return String.fromCodePoint(n); +}; + +/** + * The five XML entities and numeric references, in one pass, so `&#13;` + * comes out as the literal ` ` its author wrote and never as a carriage + * return. Anything else is left as written. + */ +export function decodeXml(s) { + return String(s ?? '').replace( + /&(?:(amp)|(lt)|(gt)|(quot)|(apos)|#(\d{1,7})|#x([0-9a-fA-F]{1,6}));/g, + (m, amp, lt, gt, quot, apos, dec, hex) => { + if (amp) return '&'; + if (lt) return '<'; + if (gt) return '>'; + if (quot) return '"'; + if (apos) return "'"; + if (dec) return CODE_POINT(Number.parseInt(dec, 10), m); + return CODE_POINT(Number.parseInt(hex, 16), m); + }, + ); +} + +const isSpace = (c) => c === ' ' || c === '\n' || c === '\r' || c === '\t'; + +/** + * Where an opening ``, `/` or whitespace), so ``. With `opening` set, a self-closing `` does not + * count; that is what the depth counter needs. + */ +export function findOpenTag(xml, name, from = 0, { opening = false } = {}) { + const needle = `<${name}`; + let i = xml.indexOf(needle, from); + while (i !== -1) { + const c = xml[i + needle.length]; + if (c === '>' || c === '/' || isSpace(c)) { + if (!opening) return i; + const gt = xml.indexOf('>', i); + if (gt === -1 || xml[gt - 1] !== '/') return i; + } + i = xml.indexOf(needle, i + 1); + } + return -1; +} + +/** `a="1" b='two'` into `{ a: '1', b: 'two' }`, entities decoded; a bare attribute is `''`. */ +export function parseAttrs(s) { + const attrs = {}; + let i = 0; + const n = s.length; + while (i < n) { + while (i < n && isSpace(s[i])) i += 1; + if (i >= n) break; + let j = i; + while (j < n && !isSpace(s[j]) && s[j] !== '=' && s[j] !== '/') j += 1; + const key = s.slice(i, j); + if (!key) { + i = j + 1; + continue; + } + i = j; + while (i < n && isSpace(s[i])) i += 1; + if (s[i] !== '=') { + attrs[key] = ''; + continue; + } + i += 1; + while (i < n && isSpace(s[i])) i += 1; + const q = s[i]; + if (q === '"' || q === "'") { + const end = s.indexOf(q, i + 1); + const stop = end === -1 ? n : end; + attrs[key] = decodeXml(s.slice(i + 1, stop)); + i = stop + 1; + } else { + let k = i; + while (k < n && !isSpace(s[k])) k += 1; + attrs[key] = decodeXml(s.slice(i, k)); + i = k; + } + } + return attrs; +} + +/** + * The first element at or after `from`: `{ name, attrs, inner, start, end }`, + * or null when there is none. Comments, processing instructions, CDATA at the + * top level and stray close tags are stepped over. A nested element of the + * same name is counted so its close tag is not mistaken for the outer one; an + * element that never closes takes the rest of the text, so a truncated record + * still yields what it has rather than nothing. + */ +export function parseElement(xml, from = 0) { + const n = xml.length; + let i = xml.indexOf('<', from); + for (;;) { + if (i === -1 || i + 1 >= n) return null; + if (xml.startsWith('', i + 4); + if (e === -1) return null; + i = xml.indexOf('<', e + 3); + continue; + } + const c = xml[i + 1]; + if (c === '?' || c === '!' || c === '/' || isSpace(c) || c === '>') { + i = xml.indexOf('<', i + 1); + continue; + } + break; + } + let j = i + 1; + while (j < n && !isSpace(xml[j]) && xml[j] !== '>' && xml[j] !== '/') j += 1; + const name = xml.slice(i + 1, j); + const gt = xml.indexOf('>', j); + if (gt === -1) return { name, attrs: parseAttrs(xml.slice(j)), inner: '', start: i, end: n }; + const selfClosing = xml[gt - 1] === '/'; + const attrs = parseAttrs(xml.slice(j, selfClosing ? gt - 1 : gt)); + if (selfClosing) return { name, attrs, inner: '', start: i, end: gt + 1 }; + + const closeTag = ``; + let depth = 1; + let pos = gt + 1; + for (;;) { + const close = xml.indexOf(closeTag, pos); + if (close === -1) return { name, attrs, inner: xml.slice(gt + 1), start: i, end: n }; + const nested = findOpenTag(xml, name, pos, { opening: true }); + if (nested !== -1 && nested < close) { + depth += 1; + pos = nested + 1; + continue; + } + depth -= 1; + pos = close + closeTag.length; + if (depth === 0) return { name, attrs, inner: xml.slice(gt + 1, close), start: i, end: pos }; + } +} + +/** Every top-level element of a fragment, in order. */ +export function children(xml) { + const out = []; + let el = parseElement(xml, 0); + while (el) { + out.push(el); + el = parseElement(xml, el.end); + } + return out; +} + +/** An element's text: CDATA unwrapped, entities decoded, trimmed. */ +export function elementText(el) { + if (!el) return ''; + const inner = el.inner.replace(//g, '$1'); + return decodeXml(inner).trim(); +} + +const child = (els, name) => els.find((e) => e.name === name); +const childTexts = (els, name, childName) => { + const parent = child(els, name); + if (!parent) return []; + return children(parent.inner) + .filter((e) => e.name === childName) + .map(elementText) + .filter(Boolean); +}; + +/** + * Records of one tag out of a stream of text chunks, one at a time. + * + * Holds the text from the start of the record being assembled to the end of + * the chunk that completed it, never more, and counts every record so that + * `skip` plus records yielded is always the file position. A record is the + * text from ``) + * to the next ``; nothing in either file nests its own record tag. + */ +export async function* scanRecords(chunks, tag, { skip = 0 } = {}) { + const decoder = new TextDecoder('utf-8'); + const closeTag = ``; + let buf = ''; + let seen = 0; + for await (const chunk of chunks) { + buf += decoder.decode(chunk, { stream: true }); + let from = 0; + for (;;) { + const start = findOpenTag(buf, tag, from); + if (start === -1) break; + const close = buf.indexOf(closeTag, start); + if (close === -1) break; + const stop = close + closeTag.length; + seen += 1; + if (seen > skip) yield buf.slice(start, stop); + from = stop; + } + const pending = findOpenTag(buf, tag, from); + if (pending !== -1) buf = buf.slice(pending); + else buf = buf.slice(Math.max(from, buf.length - tag.length - 2)); + } + buf += decoder.decode(); + const start = findOpenTag(buf, tag, 0); + if (start !== -1) { + const close = buf.indexOf(closeTag, start); + if (close !== -1) { + seen += 1; + if (seen > skip) yield buf.slice(start, close + closeTag.length); + } + } +} + +/** `scanRecords` over a gzip file; the file is released when the reader stops early. */ +export async function* gzipRecords(path, tag, opts = {}) { + const rs = createReadStream(path); + const gz = createGunzip(); + pipeline(rs, gz, () => {}); + try { + yield* scanRecords(gz, tag, opts); + } finally { + rs.destroy(); + gz.destroy(); + } +} + +// ── Items ───────────────────────────────────────────────────────────────────── + +/** + * Discogs profile markup into plain text: `[a=Name]` and `[l=Label]` become + * the name, `[url=...]text[/url]` the text, `[b]`/`[i]`/`[u]` go away, and + * ` ` line breaks become spaces. + */ +export function plainProfile(s) { + return String(s ?? '') + .replace(/\[url=[^\]]*\]([\s\S]*?)\[\/url\]/gi, '$1') + .replace(/\[[almr]=([^\]]*)\]/gi, '$1') + .replace(/\[\/?[biu]\]/gi, '') + .replace(/\s+/g, ' ') + .trim(); +} + +/** At most `max` characters, cut at a word when it has to be cut. */ +export function trimTo(s, max = SUMMARY_MAX) { + const str = String(s ?? ''); + if (str.length <= max) return str; + const cut = str.lastIndexOf(' ', max - 3); + return `${str.slice(0, cut > max / 2 ? cut : max - 3).trimEnd()}...`; +} + +const discogsId = (s) => { + const n = Number(String(s ?? '').trim()); + return Number.isInteger(n) && n > 0 ? n : null; +}; + +/** One `` record as an item, or null when it has no id or no name. */ +export function artistItem(record) { + const el = typeof record === 'string' ? parseElement(record) : record; + if (el?.name !== 'artist') return null; + const els = children(el.inner); + const id = discogsId(elementText(child(els, 'id'))); + const name = elementText(child(els, 'name')); + if (!id || !name) return null; + const realName = elementText(child(els, 'realname')) || null; + const profile = plainProfile(elementText(child(els, 'profile'))); + const nameVariations = childTexts(els, 'namevariations', 'name'); + const aliases = childTexts(els, 'aliases', 'name'); + const members = childTexts(els, 'members', 'name'); + const groups = childTexts(els, 'groups', 'name'); + const urls = childTexts(els, 'urls', 'url'); + const dataQuality = elementText(child(els, 'data_quality')) || null; + return { + externalId: `discogs:artist:${id}`, + kind: 'artist', + title: name, + summary: profile ? trimTo(profile, SUMMARY_MAX) : null, + url: `https://www.discogs.com/artist/${id}`, + imageUrl: null, + publishedAt: null, + timeKnown: false, + tags: ['artist', 'discogs'], + data: { + discogsId: id, + name, + realName, + nameVariations, + aliases, + members, + groups, + urls, + dataQuality, + attribution: ATTRIBUTION, + }, + }; +} + +/** One `` record as an item, or null when it has no id or no title. */ +export function masterItem(record) { + const el = typeof record === 'string' ? parseElement(record) : record; + if (el?.name !== 'master') return null; + const id = discogsId(el.attrs.id); + const els = children(el.inner); + const title = elementText(child(els, 'title')); + if (!id || !title) return null; + const artists = []; + const artistsEl = child(els, 'artists'); + if (artistsEl) { + for (const a of children(artistsEl.inner)) { + if (a.name !== 'artist') continue; + const parts = children(a.inner); + const name = elementText(child(parts, 'name')); + if (!name) continue; + artists.push({ name, id: discogsId(elementText(child(parts, 'id'))) }); + } + } + const genres = childTexts(els, 'genres', 'genre'); + const styles = childTexts(els, 'styles', 'style'); + const yearNum = Number(elementText(child(els, 'year'))); + const year = Number.isInteger(yearNum) && yearNum > 0 ? yearNum : null; + const mainRelease = discogsId(elementText(child(els, 'main_release'))); + const dataQuality = elementText(child(els, 'data_quality')) || null; + const videosEl = child(els, 'videos'); + const videos = videosEl + ? children(videosEl.inner) + .filter((v) => v.name === 'video' && v.attrs.src) + .map((v) => v.attrs.src) + : []; + const names = artists.map((a) => a.name); + const when = year ? looseDate(String(year)) : null; + return { + externalId: `discogs:master:${id}`, + kind: 'master', + title: names.length ? `${names.join(', ')} - ${title}` : title, + summary: [year ? String(year) : null, ...genres, ...styles].filter(Boolean).join(' · ') || null, + url: `https://www.discogs.com/master/${id}`, + imageUrl: null, + publishedAt: when?.publishedAt ?? null, + timeKnown: false, + precision: when?.precision ?? 'day', + tags: [ + 'master', + 'discogs', + ...genres.map((g) => `genre:${slugify(g)}`), + ...styles.map((s) => `style:${slugify(s)}`), + ].filter((t) => !t.endsWith(':')), + data: { + discogsId: id, + title, + mainRelease, + year, + artists, + genres, + styles, + videos, + dataQuality, + attribution: ATTRIBUTION, + }, + }; +} + +/** A record of either file as an item; null (never a throw) for one that cannot be. */ +export function recordItem(entity, record) { + try { + return entity === 'masters' ? masterItem(record) : artistItem(record); + } catch { + return null; + } +} + +// ── Cursor ──────────────────────────────────────────────────────────────────── + +/** + * Where a run starts. A cursor from another month is a finished or abandoned + * walk of a file that no longer matters, so it resets to the first record of + * the first file; the checksum list and the verified files go with it. + */ +export function resumeFrom(prev, month) { + const same = prev?.month === month; + const entity = same && ENTITIES.includes(prev?.entity) ? prev.entity : ENTITIES[0]; + const idx = Math.floor(Number(prev?.recordIndex)); + return { + month, + entity, + recordIndex: same && idx > 0 ? idx : 0, + done: same && prev?.done === true, + verified: same && Array.isArray(prev?.verified) ? prev.verified.filter(Boolean) : [], + checksums: + same && prev?.checksums && typeof prev.checksums === 'object' ? prev.checksums : null, + }; +} + +// ── Download ────────────────────────────────────────────────────────────────── + +/** + * One request for a dump file, streamed to `.part` and renamed into + * place only when every byte the server announced has arrived. The server + * ignores `Range`, so nothing is resumed: a short body is deleted and the + * result says `complete: false`. A non-2xx status returns without a body + * (`status`, and `retryAfter` for a 429); a connection that drops throws. + */ +export async function fetchDump(fetchImpl, url, path, { userAgent: ua, signal, onProgress } = {}) { + const res = await fetchImpl(url, { + headers: { 'user-agent': ua, accept: '*/*' }, + signal, + redirect: 'follow', + }); + const out = { + status: res.status, + ok: res.ok, + bytes: 0, + total: null, + complete: false, + retryAfter: res.headers.get('retry-after'), + }; + if (!res.ok) { + await res.body?.cancel().catch(() => {}); + return out; + } + const len = Number(res.headers.get('content-length')); + out.total = res.headers.has('content-length') && Number.isFinite(len) && len >= 0 ? len : null; + const part = `${path}.part`; + const fh = await open(part, 'w'); + try { + for await (const chunk of res.body) { + await fh.write(chunk); + out.bytes += chunk.byteLength; + onProgress?.({ bytes: out.bytes, total: out.total }); + } + } catch (err) { + await fh.close(); + await unlink(part).catch(() => {}); + throw err; + } + await fh.close(); + out.complete = out.total === null ? true : out.bytes === out.total; + if (out.complete) await rename(part, path); + else await unlink(part).catch(() => {}); + return out; +} + +/** A small text file (the checksum list) with the same request discipline. */ +export async function fetchText(fetchImpl, url, { userAgent: ua, signal } = {}) { + const res = await fetchImpl(url, { + headers: { 'user-agent': ua, accept: 'text/plain, */*' }, + signal, + redirect: 'follow', + }); + const out = { + status: res.status, + ok: res.ok, + text: '', + retryAfter: res.headers.get('retry-after'), + }; + if (res.ok) out.text = await res.text(); + else await res.body?.cancel().catch(() => {}); + return out; +} + +const exists = async (path) => (await stat(path).catch(() => null))?.isFile() === true; + +/** Files of other months, and leftover partials, out of the cache directory. */ +export async function pruneDir(dir, month) { + const names = await readdir(dir).catch(() => []); + for (const name of names) { + if (!name.startsWith('discogs_')) continue; + if (name.includes(`_${month}_`) && !name.endsWith('.part')) continue; + await unlink(join(dir, name)).catch(() => {}); + } +} + +/** The configured directory, created, or the deployment's dump directory. */ +export async function cacheDir(configured) { + const dir = String(configured ?? '').trim(); + if (!dir) return dumpDir('discogs'); + await mkdir(dir, { recursive: true }); + return dir; +} + +const isAbort = (err) => err?.name === 'AbortError' || err?.name === 'TimeoutError'; + +// ── Adapter ─────────────────────────────────────────────────────────────────── + +export const discogsCatalog = defineAdapter({ + name: 'discogs-catalog', + title: 'Discogs: every artist and master release', + collection: 'music', + description: + 'Every artist and every master release on Discogs, from the monthly XML dumps: an artist row with real name, name variations, aliases, members, groups, links and data quality; a master row with its artists, year, genres, styles, main release and videos. Discogs releases the dumps under CC0, so the data is public domain and every row still credits Discogs. Two files a month, downloaded once each and walked in batches of 500 across runs; no images, which Discogs serves only with a key.', + docs: 'https://data.discogs.com/', + kinds: ['artist', 'master'], + cadenceMinutes: 30 * 24 * 60, + budgetMs: BUDGET_MS, + configFields: [ + { + key: 'cacheDir', + label: 'Dump directory', + help: 'Where the two monthly files are kept between runs. Empty means the deployment dump directory (INGEST_DATA_DIR, else the OS temp dir).', + placeholder: '/data/discogs', + }, + ], + defaults: { cacheDir: '' }, + defaultSources: [ + { slug: 'discogs-catalog', name: 'Music: every artist and master release on Discogs' }, + ], + async *pull({ config, cursor: prev, env, http, log, deadline }) { + const now = new Date(); + const month = dumpMonth(now); + const state = resumeFrom(prev, month); + const stopAt = Number.isFinite(deadline) ? deadline : Number.POSITIVE_INFINITY; + const fetchImpl = typeof http?.fetch === 'function' ? http.fetch : globalThis.fetch; + const ua = userAgent(env); + const cursorAt = (entity, recordIndex, extra = {}) => ({ + month, + entity, + recordIndex, + verified: state.verified, + checksums: state.checksums, + ...extra, + }); + + if (state.done) { + return { + cursor: cursorAt(state.entity, state.recordIndex, { done: true }), + note: `${month} already walked; unchanged`, + nextInMinutes: nextDumpMinutes(now), + }; + } + + const dir = await cacheDir(config?.cacheDir); + if (prev?.month !== month) await pruneDir(dir, month); + + let requests = 0; + let failures = 0; + let written = 0; + let bad = 0; + let batches = 0; + + const signalFor = () => + AbortSignal.timeout(Math.max(1000, Math.min(stopAt - Date.now(), 60 * 60_000))); + + for (let ei = ENTITIES.indexOf(state.entity); ei < ENTITIES.length; ei += 1) { + const entity = ENTITIES[ei]; + const name = fileName(month, entity); + const file = join(dir, name); + const at = ei === ENTITIES.indexOf(state.entity) ? state.recordIndex : 0; + const tally = () => + `${written} items written${bad ? `, ${bad} records skipped` : ''} this run`; + + // ── The file: on disk and whole, or one attempt to make it so ────────── + if (!(await exists(file))) { + if (requests >= 1) { + return { + cursor: cursorAt(entity, at), + note: `${name} needed next; one request a run, resuming in ${RESUME_MINUTES} min (${tally()})`, + nextInMinutes: RESUME_MINUTES, + }; + } + if (stopAt - Date.now() < MIN_DOWNLOAD_MS) { + return { + cursor: cursorAt(entity, at), + note: `not enough of the budget left to download ${name}; resuming in ${RESUME_MINUTES} min (${tally()})`, + nextInMinutes: RESUME_MINUTES, + }; + } + requests += 1; + let dl; + try { + log(`downloading ${name}`); + dl = await fetchDump(fetchImpl, dumpUrl(month, name), file, { + userAgent: ua, + signal: signalFor(), + }); + } catch (err) { + failures += 1; + if (isAbort(err)) { + return { + cursor: cursorAt(entity, at), + note: `download of ${name} cut at the deadline; trying again in ${RESUME_MINUTES} min`, + nextInMinutes: RESUME_MINUTES, + }; + } + throw new Error(`discogs: every request failed (${requests}): ${err?.message ?? err}`); + } + if (dl.status === 429) { + const wait = retryAfterMinutes(dl.retryAfter, now.getTime()); + log(`429 for ${name}; retry-after ${dl.retryAfter ?? 'unset'}, next run in ${wait} min`); + return { + cursor: cursorAt(entity, at), + note: `rate limited on ${name}; retry after ${wait} min`, + nextInMinutes: wait, + }; + } + if (dl.status === 404) { + // The place, not `prev`: batches of an earlier file in this run are + // already saved and the cursor must never step back behind them. + return { + cursor: cursorAt(entity, at), + note: `${name} is not published yet; looking again in ${NOT_PUBLISHED_MINUTES / 60} h`, + nextInMinutes: NOT_PUBLISHED_MINUTES, + }; + } + if (!dl.ok) { + failures += 1; + throw new Error(`discogs: every request failed (${requests}): ${dl.status} for ${name}`); + } + if (!dl.complete) { + log(`${name}: ${dl.bytes} of ${dl.total} bytes arrived; the server ignores Range`); + return { + cursor: cursorAt(entity, at), + note: `download of ${name} incomplete (${dl.bytes} of ${dl.total} bytes); trying again in ${RESUME_MINUTES} min`, + nextInMinutes: RESUME_MINUTES, + }; + } + log(`${name}: ${dl.bytes} bytes on disk`); + } + + // ── Verify once against the month's checksum list ──────────────────── + if (!state.verified.includes(entity)) { + if (!state.checksums) { + if (requests >= 1) { + return { + cursor: cursorAt(entity, at), + note: `${name} on disk; checksum list is the next run's one request, in ${RESUME_MINUTES} min (${tally()})`, + nextInMinutes: RESUME_MINUTES, + }; + } + requests += 1; + let got; + try { + got = await fetchText(fetchImpl, dumpUrl(month, checksumName(month)), { + userAgent: ua, + signal: signalFor(), + }); + } catch (err) { + failures += 1; + if (failures === requests) + throw new Error( + `discogs: every request failed (${requests}): ${err?.message ?? err}`, + ); + log(`checksum list unavailable (${err?.message ?? err}); trying next run`); + return { + cursor: cursorAt(entity, at), + note: `checksum list unavailable; trying again in ${RESUME_MINUTES} min`, + nextInMinutes: RESUME_MINUTES, + }; + } + if (got.status === 429) { + const wait = retryAfterMinutes(got.retryAfter, now.getTime()); + return { + cursor: cursorAt(entity, at), + note: `rate limited on the checksum list; retry after ${wait} min`, + nextInMinutes: wait, + }; + } + if (got.ok) state.checksums = parseChecksums(got.text); + else { + log(`${checksumName(month)} answered ${got.status}; walking unverified`); + state.checksums = {}; + } + } + const want = state.checksums[name]; + if (want) { + const have = await sha256File(file); + if (have !== want) { + await unlink(file).catch(() => {}); + log(`${name}: sha256 ${have} does not match ${want}; file discarded`); + return { + cursor: cursorAt(entity, at), + note: `${name} failed its checksum and was discarded; downloading again in ${RESUME_MINUTES} min`, + nextInMinutes: RESUME_MINUTES, + }; + } + log(`${name}: sha256 verified`); + } else log(`${name}: no checksum listed; walking unverified`); + state.verified = [...state.verified, entity]; + } + + // ── The walk ───────────────────────────────────────────────────────── + if (Date.now() > stopAt) { + return { + cursor: cursorAt(entity, at), + note: `deadline reached before walking ${name} (${tally()}); resuming in ${RESUME_MINUTES} min`, + nextInMinutes: RESUME_MINUTES, + }; + } + let n = at; + let batch = []; + const tag = RECORD_TAG[entity]; + try { + for await (const record of gzipRecords(file, tag, { skip: at })) { + n += 1; + const item = recordItem(entity, record); + if (item) batch.push(item); + else bad += 1; + if (batch.length >= BATCH_SIZE) { + written += batch.length; + batches += 1; + yield { items: batch, cursor: cursorAt(entity, n) }; + batch = []; + if (Date.now() > stopAt) { + return { + cursor: cursorAt(entity, n), + note: `stopped on the run deadline at ${entity} record ${n} after ${batches} batches (${tally()}); resuming in ${RESUME_MINUTES} min`, + nextInMinutes: RESUME_MINUTES, + }; + } + } + } + } catch (err) { + // A file that will not inflate is a bad copy: drop it so the next run + // fetches a fresh one; the cursor stays at the last batch written. + await unlink(file).catch(() => {}); + throw new Error( + `discogs: ${name} unreadable at record ${n} (${err?.message ?? err}); file discarded`, + ); + } + if (batch.length) { + written += batch.length; + batches += 1; + yield { items: batch, cursor: cursorAt(entity, n) }; + } + log(`${name}: walked to the end, ${n} records`); + state.recordIndex = 0; + if (ei + 1 < ENTITIES.length) { + state.entity = ENTITIES[ei + 1]; + if (Date.now() > stopAt) { + return { + cursor: cursorAt(state.entity, 0), + note: `${name} complete; stopped on the run deadline before ${ENTITIES[ei + 1]} (${tally()}); resuming in ${RESUME_MINUTES} min`, + nextInMinutes: RESUME_MINUTES, + }; + } + } else { + return { + cursor: cursorAt(entity, n, { done: true }), + note: `complete: ${month} walked (${tally()})`, + nextInMinutes: nextDumpMinutes(now), + }; + } + } + return { cursor: cursorAt(state.entity, state.recordIndex), note: 'nothing to do' }; + }, +}); diff --git a/packages/adapters/src/index.js b/packages/adapters/src/index.js index 8a5359e..9fffad0 100644 --- a/packages/adapters/src/index.js +++ b/packages/adapters/src/index.js @@ -24,6 +24,7 @@ import { d0rz } from './d0rz.js'; import { dealcatcher } from './dealcatcher.js'; import { dealnews } from './dealnews.js'; import { digitaloceanSizes } from './digitalocean.js'; +import { discogsCatalog } from './discogs-catalog.js'; import { droughtMonitor } from './droughtmonitor.js'; import { ecbFxRates } from './ecb.js'; import { edgar } from './edgar.js'; @@ -60,6 +61,7 @@ import { lowendbox } from './lowendbox.js'; import { mcpRegistry } from './mcpregistry.js'; import { isoMicExchanges } from './mic.js'; import { musicbrainz } from './musicbrainz.js'; +import { musicbrainzCatalog } from './musicbrainz-catalog.js'; import { nasdaqHalts } from './nasdaqhalts.js'; import { ndbcBuoys } from './ndbc.js'; import { newsChannels } from './newschannels.js'; @@ -74,6 +76,7 @@ import { nws } from './nws.js'; import { ocdsTenders } from './ocds.js'; import { openfda } from './openfda.js'; import { openlibrary } from './openlibrary.js'; +import { openlibraryCatalog } from './openlibrary-catalog.js'; import { openprofiles } from './openprofiles.js'; import { opensaas } from './opensaas.js'; import { openserver } from './openserver.js'; @@ -83,6 +86,7 @@ import { outreachgraph } from './outreachgraph.js'; import { ovhVps } from './ovh.js'; import { p0dcasters } from './p0dcasters.js'; import { buildingPermits } from './permits.js'; +import { podcastindexCatalog } from './podcastindex-catalog.js'; import { podcasts } from './podcasts.js'; import { pypi } from './pypi.js'; import { redditDeals } from './redditdeals.js'; @@ -235,6 +239,10 @@ export const ADAPTERS = [ tvmazeSchedule, tvmazeCatalog, thetvdbCatalog, + musicbrainzCatalog, + openlibraryCatalog, + podcastindexCatalog, + discogsCatalog, anilistAiring, imdbRatings, // Channels: the whole iptv-org directory, for matching a playlist by name. diff --git a/packages/adapters/src/musicbrainz-catalog.js b/packages/adapters/src/musicbrainz-catalog.js new file mode 100644 index 0000000..cff493c --- /dev/null +++ b/packages/adapters/src/musicbrainz-catalog.js @@ -0,0 +1,465 @@ +import { readdir, unlink } from 'node:fs/promises'; +import { join } from 'node:path'; +import { defineAdapter, looseDate, slugify } from '@nichedb/core/adapter'; +import { dumpDir, xzLines } from '@nichedb/core/dump'; + +/** + * MusicBrainz: every artist and every release group, for the `music` collection. + * + * musicbrainz-upcoming asks the web service for the releases dated in the next + * few months; this is the rest of the database. MetaBrainz publishes the whole + * thing twice a week as JSON dumps, one tar.xz per entity, and inside each the + * member `mbdump/` is NDJSON: one object per line, the last line + * without a newline. The core data is CC0, so it can be carried whole with a + * credit; the tags, genres, ratings and annotations in the same rows are CC + * BY-NC-SA and are dropped before anything is stored, as are the relation + * lists, which are most of the bytes and none of the catalogue. + * + * THE WALK + * + * `LATEST` names the current dump directory. Each run resolves it, downloads + * the current entity's archive into the dump directory with `http.download` + * (which resumes with Range; MetaBrainz honours it), and streams the member + * through `xzLines` from the line the cursor names, yielding a batch every + * few hundred rows with the cursor `{ dir, entity, line }` after it. The + * artist file is 1.7 GB and the walk is a couple of million lines, so a run + * is budgeted 55 minutes and stops itself short of that with a cursor and a + * ten-minute resume; a run that is still downloading yields nothing and asks + * for the same. Artists first, then release groups; when both are read the + * cursor is marked done and the weekly run only compares `LATEST`, restarting + * at line 0 of the artist file when a new directory appears. + * + * Memory is one batch of items: the archive is read by a spawned `tar`, each + * line is parsed, mapped and dropped, and the row's relations never survive + * the mapping. Resuming into an xz stream re-inflates from the top of the + * member (two to three minutes at the end of the artist file), which an hourly + * run absorbs. + * + * Every request carries a descriptive user agent. A failed `LATEST` is retried + * three times and a run in which every request failed throws; a download that + * fails three times in a row ends the run with the place kept (the partial + * file stays on disk for the next resume). A line that is not JSON, or not an + * entity, is counted and skipped, never thrown. An archive on disk that tar + * cannot read is removed and fetched once more in the same run, from the last + * batch yielded; a second unreadable copy throws. + */ + +export const BASE = 'https://data.metabrainz.org/pub/musicbrainz/data/json-dumps'; +export const LATEST_URL = `${BASE}/LATEST`; + +/** Entities walked, in this order. */ +export const ENTITIES = ['artist', 'release-group']; + +/** Who is asking. */ +export const USER_AGENT = 'nichedb (https://nichedb.dev; hello@nichedb.dev)'; + +export const PROVIDER = 'musicbrainz'; +export const ATTRIBUTION = 'MusicBrainz, CC0'; + +/** Rows per yielded batch: the memory a run holds at once. */ +export const BATCH_SIZE = 500; + +/** The dumps land twice a week; a week keeps one full walk between them. */ +export const CADENCE_MINUTES = 10_080; + +/** Wall-clock budget of one run. */ +export const BUDGET_MS = 55 * 60_000; + +/** When a run stops early (deadline, download, failures) it asks to continue in this many minutes. */ +export const RESUME_MINUTES = 10; + +/** A run stops yielding this close to its deadline, so the last batch lands inside the lock. */ +export const NEAR_MS = 60_000; + +/** Pause before retrying a failed request. */ +export const RETRY_PAUSE_MS = 5_000; + +/** Consecutive failures after which a run stops asking. */ +const FAILURE_STOP = 3; + +const UA_HEADERS = { 'user-agent': USER_AGENT }; + +/** The reader's own failure: tar could not read the archive on disk. Anything else is not the file's fault. */ +const unreadable = (err) => /^(?:tar|xz) exited \d+/.test(String(err?.message ?? '')); + +const sleep = (ms) => (ms > 0 ? new Promise((r) => setTimeout(r, ms)) : Promise.resolve()); + +export const dumpUrl = (dir, entity) => + `${BASE}/${encodeURIComponent(String(dir))}/${encodeURIComponent(String(entity))}.tar.xz`; + +/** The NDJSON member inside an entity's archive. */ +export const memberOf = (entity) => `mbdump/${entity}`; + +/** Where an entity's archive lives on disk; the dump directory is in the name so a new dump is a new file. */ +export const localFile = (dataDir, dir, entity) => join(dataDir, `${dir}-${entity}.tar.xz`); + +/** `LATEST` is one line, `20260912-001001`. Anything else is not a directory name. */ +export function parseLatest(text) { + const first = String(text ?? '') + .split('\n') + .map((l) => l.trim()) + .find(Boolean); + return first && /^\d{8}-\d{6}$/.test(first) ? first : null; +} + +/** + * Where a run starts. A cursor from another dump directory restarts the walk + * at the first entity's first line; `done` only holds for the same directory. + */ +export function resumeFrom(prev, dir) { + if (!prev || typeof prev !== 'object' || prev.dir !== dir) { + return { dir, entity: ENTITIES[0], line: 0, done: false }; + } + const entity = ENTITIES.includes(prev.entity) ? prev.entity : ENTITIES[0]; + const line = Math.max(0, Math.floor(Number(prev.line)) || 0); + return { dir, entity, line, done: prev.done === true }; +} + +/** One dump line as an object, or null: a bad line is the caller's count, not its crash. */ +export function parseRow(line) { + if (typeof line !== 'string' || !line.trim()) return null; + try { + const row = JSON.parse(line); + return row && typeof row === 'object' && !Array.isArray(row) ? row : null; + } catch { + return null; + } +} + +const str = (v) => (typeof v === 'string' && v.trim() ? v.trim() : null); +const strings = (v) => (Array.isArray(v) ? v.map(str).filter(Boolean) : []); +const uniq = (xs) => [...new Set(xs.filter(Boolean))]; +const names = (v) => uniq(Array.isArray(v) ? v.map((a) => str(a?.name)) : []); + +/** An artist row as an item. The CC BY-NC-SA fields and the relations never reach it. */ +export function artistItem(a) { + const id = str(a?.id); + const name = str(a?.name); + if (!id || !name) return null; + const type = str(a.type); + const country = str(a.country); + const life = a['life-span'] && typeof a['life-span'] === 'object' ? a['life-span'] : {}; + return { + externalId: `musicbrainz:artist:${id}`, + kind: 'artist', + title: name, + summary: str(a.disambiguation), + url: `https://musicbrainz.org/artist/${id}`, + imageUrl: null, + tags: uniq([ + 'artist', + PROVIDER, + type ? `type:${slugify(type)}` : null, + country ? `country:${country.toLowerCase()}` : null, + ]), + data: { + mbid: id, + sortName: str(a['sort-name']), + type, + gender: str(a.gender), + country, + area: str(a.area?.name), + beginArea: str(a['begin-area']?.name), + lifeSpan: { + begin: str(life.begin), + end: str(life.end), + ended: life.ended === true, + }, + aliases: names(a.aliases), + isnis: strings(a.isnis), + ipis: strings(a.ipis), + attribution: ATTRIBUTION, + }, + }; +} + +/** The credited artists of a release group: `[{ name, mbid }]`, in credit order. */ +export function artistCredit(credit) { + if (!Array.isArray(credit)) return []; + return credit + .map((c) => ({ name: str(c?.name) ?? str(c?.artist?.name), mbid: str(c?.artist?.id) })) + .filter((c) => c.name); +} + +/** The credit as it reads on a sleeve: names joined by their join phrases. */ +export function creditText(credit) { + if (!Array.isArray(credit)) return null; + const text = credit + .map((c) => `${str(c?.name) ?? str(c?.artist?.name) ?? ''}${c?.joinphrase ?? ''}`) + .join('') + .trim(); + return text || null; +} + +/** + * A release group row as an item. The cover is hot-linked from the Cover Art + * Archive by release-group id; it answers 404 for a group with no art. + */ +export function releaseGroupItem(g) { + const id = str(g?.id); + const title = str(g?.title); + if (!id || !title) return null; + const primary = str(g['primary-type']); + const secondary = strings(g['secondary-types']); + const credit = creditText(g['artist-credit']); + const firstReleaseDate = str(g['first-release-date']); + const when = looseDate(firstReleaseDate ?? ''); + const disambiguation = str(g.disambiguation); + return { + externalId: `musicbrainz:release-group:${id}`, + kind: 'release-group', + title, + summary: + [credit ? `by ${credit}` : null, disambiguation ? `(${disambiguation})` : null] + .filter(Boolean) + .join(' ') || null, + url: `https://musicbrainz.org/release-group/${id}`, + imageUrl: `https://coverartarchive.org/release-group/${id}/front-250`, + publishedAt: when.publishedAt, + timeKnown: false, + precision: when.precision, + tags: uniq([ + 'release-group', + PROVIDER, + primary ? `type:${slugify(primary)}` : null, + ...secondary.map((s) => `secondary:${slugify(s)}`), + ]), + data: { + mbid: id, + primaryType: primary, + secondaryTypes: secondary, + firstReleaseDate, + artistCredit: artistCredit(g['artist-credit']), + attribution: ATTRIBUTION, + }, + }; +} + +/** The item for one row of the named entity's member, or null when it is not one. */ +export function toItem(entity, row) { + if (!row) return null; + if (entity === 'artist') return artistItem(row); + if (entity === 'release-group') return releaseGroupItem(row); + return null; +} + +/** The entity after this one, or null at the end of the list. */ +export const nextEntity = (entity) => ENTITIES[ENTITIES.indexOf(entity) + 1] ?? null; + +/** + * Drop the archives of any other dump directory. The disk under the dump + * directory is a cache and this keeps it to one dump's worth. + */ +export async function pruneOthers(dataDir, dir) { + const keep = `${dir}-`; + let removed = 0; + for (const name of await readdir(dataDir)) { + if (!name.endsWith('.tar.xz') || name.startsWith(keep)) continue; + await unlink(join(dataDir, name)).catch(() => {}); + removed += 1; + } + return removed; +} + +/** + * The walk itself, as the async generator the core drains. + * + * `opts` is the test seam: `dataDir` in place of `dumpDir('musicbrainz')`, + * `pauseMs` in place of the retry pause and `now` in place of the clock. The + * adapter's `pull` passes none of them. + */ +export async function* walk( + { config, cursor: prev, http, log, deadline }, + { dataDir = null, pauseMs = RETRY_PAUSE_MS, now = Date.now } = {}, +) { + const stopAt = Number.isFinite(deadline) ? deadline : Number.POSITIVE_INFINITY; + const batchSize = Math.max(1, Math.floor(Number(config?.batchSize)) || BATCH_SIZE); + const near = () => now() > stopAt - NEAR_MS; + const stopIn = RESUME_MINUTES; + let requests = 0; + let failures = 0; + + // The current dump directory. Three tries; a run that never reached the server throws. + let dir = null; + for (let attempt = 0; attempt < FAILURE_STOP && !dir; attempt++) { + if (attempt > 0) await sleep(pauseMs); + requests += 1; + try { + const text = await http.text(LATEST_URL, { + headers: { ...UA_HEADERS, accept: 'text/plain, */*' }, + timeoutMs: 20_000, + }); + dir = parseLatest(text); + if (!dir) + throw new Error(`LATEST is not a dump directory name: ${String(text).slice(0, 40)}`); + } catch (err) { + failures += 1; + log(`LATEST unavailable (${err?.message ?? err})`); + } + } + if (!dir) throw new Error(`musicbrainz: every request failed (${requests}); see the log`); + + const state = resumeFrom(prev, dir); + if (state.done) { + log(`dump ${dir} already walked; nothing to do`); + return { cursor: prev, note: 'unchanged' }; + } + + const base = dataDir ?? (await dumpDir('musicbrainz')); + const pruned = await pruneOthers(base, dir).catch(() => 0); + if (pruned) log(`${pruned} archive${pruned === 1 ? '' : 's'} of an older dump removed`); + + let entity = state.entity; + let line = state.line; + let seen = 0; + let bad = 0; + const reread = new Set(); + const at = () => ({ dir, entity, line }); + const progress = () => `${seen} rows${bad ? `, ${bad} bad` : ''}`; + + while (entity) { + // ── Download, resuming whatever is on disk ──────────────────────────── + const file = localFile(base, dir, entity); + let dl = null; + let streak = 0; + while (!dl) { + if (near()) { + return { + cursor: at(), + note: `${progress()}; ${entity} download deferred by the run deadline at line ${line}, resuming in ${stopIn} min`, + nextInMinutes: stopIn, + }; + } + requests += 1; + let lastLogged = 0; + try { + dl = await http.download(dumpUrl(dir, entity), file, { + headers: UA_HEADERS, + timeoutMs: Number.isFinite(stopAt) + ? Math.max(60_000, stopAt - now() - NEAR_MS) + : 60 * 60_000, + onProgress: ({ bytes, total }) => { + if (bytes - lastLogged < 256 * 1024 * 1024) return; + lastLogged = bytes; + log( + `${entity}: ${Math.round(bytes / 1_048_576)} of ${total ? Math.round(total / 1_048_576) : '?'} MB`, + ); + }, + }); + } catch (err) { + failures += 1; + streak += 1; + log(`${entity} download failed (${err?.message ?? err})`); + if (streak >= FAILURE_STOP) { + if (failures === requests) + throw new Error(`musicbrainz: every request failed (${requests}); see the log`); + return { + cursor: at(), + note: `${progress()}; ${entity} download failed ${streak} times, resuming in ${stopIn} min from line ${line}`, + nextInMinutes: stopIn, + }; + } + await sleep(pauseMs); + } + } + if (!dl.complete) { + log(`${entity}: ${dl.bytes} bytes on disk, download in progress`); + return { + cursor: at(), + note: `${progress()}; ${entity} download in progress (${Math.round(dl.bytes / 1_048_576)} MB), resuming in ${stopIn} min`, + nextInMinutes: stopIn, + }; + } + + // ── Stream the member from the cursor's line ────────────────────────── + let batch = []; + let yielded = line; + try { + for await (const text of xzLines(file, { member: memberOf(entity), skip: line })) { + line += 1; + const item = toItem(entity, parseRow(text)); + if (!item) { + if (text.trim()) bad += 1; + continue; + } + batch.push(item); + if (batch.length >= batchSize) { + seen += batch.length; + yield { items: batch, cursor: at() }; + yielded = line; + batch = []; + if (near()) { + return { + cursor: at(), + note: `${progress()}; stopped on the run deadline at ${entity} line ${line} of ${dir}, resuming in ${stopIn} min`, + nextInMinutes: stopIn, + }; + } + } + } + } catch (err) { + if (!unreadable(err)) throw err; + // The file on disk is not one tar can read (a bad write, a volume that + // outlived a different build of the archive). Left there it would fail + // every run, since http.download sees a whole file and fetches nothing. + // Drop it and fetch it once more, resuming from the last batch yielded. + await unlink(file).catch(() => {}); + log(`${entity}: archive of ${dir} removed, ${err.message}`); + if (reread.has(entity)) { + throw new Error( + `musicbrainz: ${entity} archive of ${dir} unreadable twice (${err.message})`, + ); + } + reread.add(entity); + line = yielded; + continue; + } + if (batch.length) { + seen += batch.length; + yield { items: batch, cursor: at() }; + batch = []; + } + log(`${entity}: ${line} lines of ${dir} read`); + + const next = nextEntity(entity); + if (!next) { + return { + cursor: { ...at(), done: true }, + note: `${progress()}; dump ${dir} walked, ${ENTITIES.join(' and ')} complete`, + }; + } + entity = next; + line = 0; + } + return { cursor: at(), note: progress() }; +} + +export const musicbrainzCatalog = defineAdapter({ + name: 'musicbrainz-catalog', + title: 'MusicBrainz: every artist and release group', + collection: 'music', + description: + 'Every artist and every release group in MusicBrainz, from the twice-weekly JSON dumps: an artist row carries the name, sort name, disambiguation, type, gender, country, area, life span, aliases, ISNIs and IPIs; a release group row carries the title, artist credit, primary and secondary types, first release date and a Cover Art Archive front image where one exists. The core MusicBrainz data is CC0 and is carried whole with a credit on every row, while the tags, genres, ratings and annotations in the dumps are CC BY-NC-SA and are dropped. Downloads the 1.7 GB artist and 1.2 GB release-group archives with resume and streams them in 55-minute runs, resuming from its cursor until a dump is walked; a new dump restarts the walk.', + docs: 'https://musicbrainz.org/doc/MusicBrainz_Database/Download', + kinds: ['artist', 'release-group'], + cadenceMinutes: CADENCE_MINUTES, + budgetMs: BUDGET_MS, + configFields: [ + { + key: 'batchSize', + label: 'Rows per batch', + type: 'number', + placeholder: String(BATCH_SIZE), + help: 'Rows mapped and written together; the cursor is saved after each batch.', + }, + ], + defaults: { batchSize: BATCH_SIZE }, + defaultSources: [ + { + slug: 'musicbrainz-catalog', + name: 'Music: every artist and release group on MusicBrainz', + config: { batchSize: BATCH_SIZE }, + }, + ], + pull: (ctx) => walk(ctx), +}); diff --git a/packages/adapters/src/openlibrary-catalog.js b/packages/adapters/src/openlibrary-catalog.js new file mode 100644 index 0000000..8c7ed91 --- /dev/null +++ b/packages/adapters/src/openlibrary-catalog.js @@ -0,0 +1,692 @@ +import { readdir, unlink } from 'node:fs/promises'; +import { join } from 'node:path'; +import { defineAdapter, looseDate, slugify } from '@nichedb/core/adapter'; +import { dumpDir, gzipLines } from '@nichedb/core/dump'; + +/** + * Open Library: every work and every author, from the monthly dumps. + * + * openlibrary-new asks the search API for the books first published this + * year, a few hundred rows. This is the rest of the catalogue: Open Library + * publishes its whole database once a month as gzipped tab-separated files on + * archive.org (openlibrary.org/developers/dumps), one row per record with + * five columns: type, key, revision, last_modified and the record as JSON. + * The authors file is 780 MB compressed (about 15 million rows) and the works + * file 4 GB (about 40 million), so nothing here fits in a run or in memory: + * the file is downloaded with resume into the dump directory, then walked as + * lines, 500 rows a batch, with the cursor after every batch saying which file + * and which line to continue from. A run stops when its budget is spent and + * the next one, ten minutes later, re-inflates up to that line and carries on. + * + * The `_latest` URLs redirect twice (openlibrary.org, then archive.org, then + * an archive.org mirror) and land on a file named with the dump date. That + * date is the version: one HEAD request at the start of a run resolves it, the + * download then asks the DATED URL directly so a resume can never append the + * bytes of a newer dump to an older partial file, and a cursor that names a + * different date starts the walk over. Once both files are walked the cursor + * is `done` and every run until the date changes costs one HEAD request. + * + * A re-ingest of a new dump writes every row again in principle, but the + * table skips an unchanged row by hash and the walk skips a row before it + * reaches the table: the cursor of a complete pass carries the newest + * `last_modified` it saw, and the next pass drops rows modified before that + * mark, which on a monthly dump is nearly all of them. + * + * Works keep the `/works/OL...W` key as the external id, exactly as + * openlibrary-new does, so a work walked here and the same work found by the + * search source are one row. Authors are new here, `/authors/OL...A`. + * + * The data is Open Library's, published by the Internet Archive without a new + * copyright claim (the records are contributed and pooled from library + * catalogues); every row carries that attribution. + */ + +/** Who is asking; archive.org and openlibrary.org both want a readable agent. */ +export const USER_AGENT = 'nichedb (https://nichedb.dev; hello@nichedb.dev)'; + +export const PROVIDER = 'openlibrary'; +export const ATTRIBUTION = 'Open Library (Internet Archive); no new copyright asserted'; + +/** The subdirectory of the dump directory the two files live in. */ +export const DUMP_DIR = 'openlibrary-catalog'; + +/** The files of one dump, in the order they are walked. */ +export const FILES = ['authors', 'works']; + +/** Rows handed to the core at once; the memory a run holds. */ +export const BATCH_ROWS = 500; + +/** A run's wall-clock budget. The walk of the works file is several of these. */ +export const BUDGET_MS = 55 * 60_000; + +/** Open Library publishes a dump once a month. */ +export const CADENCE_MINUTES = 30 * 24 * 60; + +/** How soon an unfinished walk, or an unfinished download, picks up again. */ +export const RESUME_MINUTES = 10; + +/** The least a download is given; with less than this left a run does not start one. */ +export const DOWNLOAD_MIN_MS = 30_000; + +/** Consecutive failed requests after which a run stops asking. */ +export const FAILURE_STOP = 3; + +/** Pause before a retry; three requests in a second at archive.org looks like hammering. */ +export const PAUSE_MS = 2000; + +/** Summary and bio length. */ +export const SUMMARY_CHARS = 600; + +/** Subjects that become tags; the data keeps more. */ +export const SUBJECT_TAGS = 5; + +/** How many entries of a list field the data keeps, so one row stays a row. */ +export const LIST_KEPT = 100; + +const sleep = (ms) => (ms > 0 ? new Promise((r) => setTimeout(r, ms)) : Promise.resolve()); + +// ── URLs and files ─────────────────────────────────────────────────────────── + +/** The always-current URL of one file; a HEAD on it resolves the dump date. */ +export const latestUrl = (kind) => `https://openlibrary.org/data/ol_dump_${kind}_latest.txt.gz`; + +/** The dated file on archive.org, the same bytes for the life of the dump. */ +export const datedUrl = (kind, version) => + `https://archive.org/download/ol_dump_${version}/ol_dump_${kind}_${version}.txt.gz`; + +/** Where one file of one dump is kept locally. */ +export const localName = (kind, version) => `ol_dump_${kind}_${version}.txt.gz`; + +/** The dump date in a resolved URL, `2026-08-31`, or null. */ +export function versionFromUrl(url) { + const m = String(url ?? '').match( + /ol_dump_(?:authors|works|editions|all)_(\d{4}-\d{2}-\d{2})\.txt\.gz/, + ); + return m ? m[1] : null; +} + +// ── Rows ───────────────────────────────────────────────────────────────────── + +/** + * One line of a dump: `type \t key \t revision \t last_modified \t JSON`, the + * JSON being everything after the fourth tab. Null for a line that is not a + * row (blank, short, or cut mid-JSON, which the last line of a truncated + * download is); the caller counts it and moves on. + */ +export function parseRow(line) { + if (!line) return null; + const cols = splitN(line, '\t', 5); + if (cols.length < 5) return null; + let json; + try { + json = JSON.parse(cols[4]); + } catch { + return null; + } + if (!json || typeof json !== 'object') return null; + return { + type: cols[0], + key: cols[1], + revision: Number(cols[2]) || 0, + lastModified: cols[3], + json, + }; +} + +/** `s.split(sep)` limited to `n` fields, the last one keeping the rest. */ +export function splitN(s, sep, n) { + const out = []; + let from = 0; + while (out.length < n - 1) { + const at = s.indexOf(sep, from); + if (at === -1) break; + out.push(s.slice(from, at)); + from = at + 1; + } + out.push(s.slice(from)); + return out; +} + +/** A text field that is a string or `{ type: '/type/text', value }`. */ +export function textOf(v) { + if (typeof v === 'string') return v; + if (v && typeof v === 'object' && typeof v.value === 'string') return v.value; + return ''; +} + +/** Whitespace collapsed and cut to `n` characters on a word where it can. */ +export function trimTo(s, n = SUMMARY_CHARS) { + const text = String(s ?? '') + .replace(/\s+/g, ' ') + .trim(); + if (text.length <= n) return text || null; + const cut = text.slice(0, n); + const at = cut.lastIndexOf(' '); + return `${(at > n / 2 ? cut.slice(0, at) : cut).trim()}...`; +} + +/** Strings only, trimmed, non-empty, at most `n` of them. */ +export function strings(v, n = LIST_KEPT) { + if (!Array.isArray(v)) return []; + const out = []; + for (const s of v) { + if (typeof s !== 'string') continue; + const t = s.trim(); + if (t) out.push(t); + if (out.length >= n) break; + } + return out; +} + +/** Author keys of a work: `authors: [{ author: { key } }]`, or bare `{ key }`. */ +export function authorKeys(v) { + if (!Array.isArray(v)) return []; + const out = []; + for (const a of v) { + const key = a?.author?.key ?? a?.key; + if (typeof key === 'string' && key.startsWith('/authors/')) out.push(key); + } + return out; +} + +const MONTHS = { + january: 1, + february: 2, + march: 3, + april: 4, + may: 5, + june: 6, + july: 7, + august: 8, + september: 9, + october: 10, + november: 11, + december: 12, +}; + +const pad = (n) => String(n).padStart(2, '0'); + +/** + * `first_publish_date` is free text: `1964`, `June 1940`, `January 1, 1967`, + * `August 9, 2007`, `1907-02-16`, and worse. Read the shapes that occur, fall + * back to the first four-digit year in the string, and hand the result to + * looseDate so the precision matches what the text actually said. + */ +export function publishDate(s) { + const text = String(s ?? '').trim(); + if (!text) return looseDate(''); + if (/^\d{4}(-\d{2}){0,2}$/.test(text)) return looseDate(text); + let m = text.match(/^([A-Za-z]+)\.?\s+(\d{1,2}),?\s+(\d{4})$/); + if (m && MONTHS[m[1].toLowerCase()]) { + return looseDate(`${m[3]}-${pad(MONTHS[m[1].toLowerCase()])}-${pad(m[2])}`); + } + m = text.match(/^([A-Za-z]+)\.?,?\s+(\d{4})$/); + if (m && MONTHS[m[1].toLowerCase()]) { + return looseDate(`${m[2]}-${pad(MONTHS[m[1].toLowerCase()])}`); + } + m = text.match(/^(\d{1,2})\s+([A-Za-z]+)\.?\s+(\d{4})$/); + if (m && MONTHS[m[2].toLowerCase()]) { + return looseDate(`${m[3]}-${pad(MONTHS[m[2].toLowerCase()])}-${pad(m[1])}`); + } + m = text.match(/\b(\d{4})\b/); + return looseDate(m ? m[1] : ''); +} + +/** A cover or photo id becomes a URL; Open Library uses -1 for "none". */ +export function coverUrl(ids, kind) { + const id = Array.isArray(ids) ? Number(ids[0]) : Number.NaN; + if (!Number.isInteger(id) || id <= 0) return null; + return `https://covers.openlibrary.org/${kind}/id/${id}-M.jpg`; +} + +/** One work row as an item, or null for a row without a key or a title. */ +export function workItem(row) { + const d = row?.json; + const key = typeof d?.key === 'string' ? d.key : row?.key; + const title = typeof d?.title === 'string' ? d.title.trim() : ''; + if (!key || !/^\/works\/OL\d+W$/.test(key) || !title) return null; + const when = publishDate(d.first_publish_date); + const subjects = strings(d.subjects); + const subtitle = typeof d.subtitle === 'string' ? d.subtitle.trim() : ''; + return { + externalId: key, + kind: 'book', + title: subtitle ? `${title}: ${subtitle}` : title, + summary: trimTo(textOf(d.description)), + url: `https://openlibrary.org${key}`, + imageUrl: coverUrl(d.covers, 'b'), + publishedAt: when.publishedAt, + timeKnown: when.timeKnown, + precision: when.precision, + tags: [ + 'book', + PROVIDER, + ...subjects + .slice(0, SUBJECT_TAGS) + .map((s) => slugify(s)) + .filter(Boolean) + .map((s) => `subject:${s}`), + ], + data: { + provider: PROVIDER, + olKey: key, + title, + subtitle: subtitle || null, + authors: authorKeys(d.authors), + subjects, + subjectPlaces: strings(d.subject_places), + subjectPeople: strings(d.subject_people), + subjectTimes: strings(d.subject_times), + firstPublishDate: typeof d.first_publish_date === 'string' ? d.first_publish_date : null, + covers: Array.isArray(d.covers) + ? d.covers.filter((c) => Number.isInteger(c)).slice(0, 10) + : [], + revision: row.revision, + lastModified: row.lastModified, + attribution: ATTRIBUTION, + }, + }; +} + +/** The ids Open Library keeps for an author elsewhere; only the ones asked for. */ +export function remoteIds(v) { + if (!v || typeof v !== 'object') return {}; + const out = {}; + for (const k of ['wikidata', 'viaf', 'isni']) { + if (typeof v[k] === 'string' && v[k].trim()) out[k] = v[k].trim(); + } + return out; +} + +/** `links: [{ title, url }]`, kept as that. */ +export function authorLinks(v) { + if (!Array.isArray(v)) return []; + const out = []; + for (const l of v) { + if (typeof l?.url !== 'string' || !/^https?:\/\//.test(l.url)) continue; + out.push({ title: typeof l.title === 'string' ? l.title : null, url: l.url }); + if (out.length >= 20) break; + } + return out; +} + +/** One author row as an item, or null for a row without a key or a name. */ +export function authorItem(row) { + const d = row?.json; + const key = typeof d?.key === 'string' ? d.key : row?.key; + const name = typeof d?.name === 'string' ? d.name.trim() : ''; + if (!key || !/^\/authors\/OL\d+A$/.test(key) || !name) return null; + return { + externalId: key, + kind: 'author', + title: name, + summary: trimTo(textOf(d.bio)), + url: `https://openlibrary.org${key}`, + imageUrl: coverUrl(d.photos, 'a'), + publishedAt: null, + timeKnown: false, + precision: 'day', + tags: ['author', PROVIDER], + data: { + provider: PROVIDER, + olKey: key, + name, + personalName: typeof d.personal_name === 'string' ? d.personal_name : null, + birthDate: typeof d.birth_date === 'string' ? d.birth_date : null, + deathDate: typeof d.death_date === 'string' ? d.death_date : null, + alternateNames: strings(d.alternate_names, 20), + links: authorLinks(d.links), + remoteIds: remoteIds(d.remote_ids), + revision: row.revision, + lastModified: row.lastModified, + attribution: ATTRIBUTION, + }, + }; +} + +/** A parsed row as an item by its type; a redirect, a delete or a stray type is nothing. */ +export function rowItem(row) { + if (row?.type === '/type/work') return workItem(row); + if (row?.type === '/type/author') return authorItem(row); + return null; +} + +// ── Cursor ─────────────────────────────────────────────────────────────────── + +/** + * Where a run starts. + * + * `version` is the dump date the walk is over; `file` and `line` the position + * in it (the line count already read, so the reader skips that many); + * `lastModifiedWatermark` the newest last_modified a COMPLETE pass has seen, + * below which a later pass skips rows; `maxLastModified` the same for the pass + * in progress; `done` that both files of `version` are walked. + */ +export function resumeFrom(prev) { + const version = /^\d{4}-\d{2}-\d{2}$/.test(String(prev?.version ?? '')) ? prev.version : null; + const file = FILES.includes(prev?.file) ? prev.file : FILES[0]; + const line = Math.floor(Number(prev?.line)); + const watermark = + typeof prev?.lastModifiedWatermark === 'string' && prev.lastModifiedWatermark + ? prev.lastModifiedWatermark + : null; + const max = + typeof prev?.maxLastModified === 'string' && prev.maxLastModified ? prev.maxLastModified : null; + return { + version, + file, + line: line > 0 ? line : 0, + lastModifiedWatermark: watermark, + maxLastModified: max, + done: version !== null && prev?.done === true, + }; +} + +/** A row modified before the watermark was in the previous pass unchanged. */ +export const isStale = (lastModified, watermark) => + Boolean(watermark) && typeof lastModified === 'string' && lastModified < watermark; + +/** The local files of other dumps than `version`, which nothing will read again. */ +export function staleFiles(names, version) { + return names.filter( + (n) => + /^ol_dump_(?:authors|works)_\d{4}-\d{2}-\d{2}\.txt\.gz$/.test(n) && + !n.includes(`_${version}.`), + ); +} + +// ── The adapter ────────────────────────────────────────────────────────────── + +/** + * The dump date behind the `_latest` URL: one HEAD request through the two + * redirects, the date read off the final URL. Nothing else names the dump: + * Last-Modified is the upload, days after the date in the file name, and a + * dated URL built from it would ask archive.org for an item it does not have. + * A mirror that hands back something unnamed is a failed request, and a walk + * in progress carries on with the date its cursor already knows. + */ +export async function resolveVersion(http) { + const res = await http.request(latestUrl(FILES[0]), { + method: 'HEAD', + headers: { 'user-agent': USER_AGENT, accept: '*/*' }, + timeoutMs: 30_000, + }); + await res.body?.cancel?.().catch?.(() => {}); + if (!res.ok) throw new Error(`openlibrary answered ${res.status} for the latest dump`); + const version = versionFromUrl(res.url); + if (!version) { + throw new Error( + `openlibrary did not say which dump is latest (${String(res.url).slice(0, 120)})`, + ); + } + return version; +} + +export const openlibraryCatalog = defineAdapter({ + name: 'openlibrary-catalog', + title: 'Open Library: every work and author', + collection: 'books', + description: + 'Every work and every author in Open Library, from the monthly dumps: about 40 million works as book rows (title, description, cover, first publish date, subjects, author keys) and 15 million authors (name, bio, photo, dates, alternate names, links, Wikidata, VIAF and ISNI ids). Open Library data is published by the Internet Archive with no new copyright asserted and is attributed on every row. Downloads the two gzipped dumps with resume, walks them 500 rows a batch across as many runs as it takes, and after a complete pass skips the rows an earlier dump already carried. Works share the openlibrary-new external id, so the two sources merge.', + docs: 'https://openlibrary.org/developers/dumps', + kinds: ['book', 'author'], + cadenceMinutes: CADENCE_MINUTES, + budgetMs: BUDGET_MS, + configFields: [ + { + key: 'batchRows', + label: 'Rows per batch', + type: 'number', + placeholder: String(BATCH_ROWS), + help: 'Rows handed to the table at once. The cursor is saved after every batch.', + }, + { + key: 'pauseMs', + label: 'Pause before a retry (ms)', + type: 'number', + placeholder: String(PAUSE_MS), + help: 'After a failed request. Three failures in a row end the run; it resumes in ten minutes.', + }, + ], + defaults: { batchRows: BATCH_ROWS, pauseMs: PAUSE_MS }, + defaultSources: [ + { + slug: 'openlibrary-catalog', + name: 'Books: every work and author in Open Library', + config: { batchRows: BATCH_ROWS, pauseMs: PAUSE_MS }, + }, + ], + async *pull({ config, cursor: prev, http, log, deadline }) { + const batchRows = Math.max(1, Math.floor(Number(config?.batchRows)) || BATCH_ROWS); + const pause = + config?.pauseMs === 0 ? 0 : Math.max(0, Math.floor(Number(config?.pauseMs))) || PAUSE_MS; + const stopAt = Number.isFinite(deadline) ? deadline : Number.POSITIVE_INFINITY; + const state = resumeFrom(prev); + let requests = 0; + let failures = 0; + let streak = 0; + + const failed = (what, err) => { + failures += 1; + streak += 1; + log(`${what} failed (${err?.message ?? err})`); + return streak >= FAILURE_STOP; + }; + const allFailed = () => { + if (requests > 0 && failures === requests) + throw new Error(`openlibrary: every request failed (${requests}); see the log`); + }; + + // ── Which dump ─────────────────────────────────────────────────────── + let version = null; + while (version === null) { + requests += 1; + try { + version = await resolveVersion(http); + streak = 0; + } catch (err) { + if (failed('resolving the latest dump', err)) break; + await sleep(pause); + } + } + if (version === null) { + // The walk in progress does not need to know what is newest. + if (state.version && !state.done) version = state.version; + else { + allFailed(); + return { + cursor: prev ?? {}, + note: 'could not resolve the latest dump; resuming in 10 min', + nextInMinutes: RESUME_MINUTES, + }; + } + } + + if (state.version === version && state.done) { + log(`dump ${version} already walked; nothing to do`); + return { cursor: prev, note: `unchanged (${version})` }; + } + + const fresh = state.version !== version; + const watermark = state.lastModifiedWatermark; + let file = fresh ? FILES[0] : state.file; + let line = fresh ? 0 : state.line; + let maxLastModified = fresh ? null : state.maxLastModified; + const cursorAt = (f, n) => ({ + version, + file: f, + line: n, + lastModifiedWatermark: watermark, + maxLastModified, + done: false, + }); + if (fresh) { + log( + `dump ${version}${state.version ? ` replaces ${state.version}` : ''}` + + (watermark ? `; rows modified before ${watermark} are skipped` : ''), + ); + } + + const dir = await dumpDir(DUMP_DIR); + for (const name of staleFiles(await readdir(dir).catch(() => []), version)) { + await unlink(join(dir, name)).catch(() => {}); + } + + let batches = 0; + let rows = 0; + let bad = 0; + let stale = 0; + let written = 0; + + // ── The walk ───────────────────────────────────────────────────────── + for (let fi = FILES.indexOf(file); fi < FILES.length; fi++) { + const kind = FILES[fi]; + const path = join(dir, localName(kind, version)); + const summary = () => + `${written} rows written in ${batches} batches` + + (stale ? `, ${stale} unchanged skipped` : '') + + (bad ? `, ${bad} unreadable` : '') + + (failures ? `, ${failures} requests failed` : ''); + + // Download, resuming whatever is on disk, until the file is whole. + // With less than the floor of the transfer timeout left there is no + // point starting one: stop here and let the next run make the request. + // Past that the transfer is bounded by the run's own deadline through + // timeoutMs, and a cut past it is reported as in progress below, with + // the partial file kept for the resume. + if (stopAt - Date.now() < DOWNLOAD_MIN_MS) { + return { + cursor: cursorAt(kind, line), + note: `${summary()}; out of time before the ${kind} download, resuming in 10 min`, + nextInMinutes: RESUME_MINUTES, + }; + } + let dl = null; + while (dl === null) { + requests += 1; + try { + dl = await http.download(datedUrl(kind, version), path, { + headers: { 'user-agent': USER_AGENT }, + // The whole transfer, but never past the run's own deadline. + timeoutMs: Number.isFinite(stopAt) + ? Math.max(DOWNLOAD_MIN_MS, stopAt - Date.now()) + : BUDGET_MS, + }); + streak = 0; + } catch (err) { + if (Date.now() > stopAt) { + // The budget ran out mid-transfer; the partial file is on disk. + log(`${kind} download cut by the run deadline (${err?.message ?? err})`); + return { + cursor: cursorAt(kind, line), + note: `${summary()}; ${kind} download in progress, resuming in 10 min`, + nextInMinutes: RESUME_MINUTES, + }; + } + if (failed(`${kind} download`, err)) { + allFailed(); + return { + cursor: cursorAt(kind, line), + note: `${summary()}; stopped after repeated failures on the ${kind} download, resuming in 10 min`, + nextInMinutes: RESUME_MINUTES, + }; + } + await sleep(pause); + } + } + if (!dl.complete) { + log(`${kind} dump ${version}: ${dl.bytes} bytes so far`); + return { + cursor: cursorAt(kind, line), + note: `${summary()}; ${kind} download in progress (${dl.bytes} bytes), resuming in 10 min`, + nextInMinutes: RESUME_MINUTES, + }; + } + + // Walk the lines from where the cursor says, a batch at a time. + let batch = []; + let n = line; + let outOfTime = false; + try { + for await (const raw of gzipLines(path, { skip: line })) { + n += 1; + if (!raw) continue; + const row = parseRow(raw); + if (!row) { + bad += 1; + if (bad <= 5) log(`${kind} line ${n}: not a row`); + continue; + } + rows += 1; + if (maxLastModified === null || row.lastModified > maxLastModified) { + maxLastModified = row.lastModified; + } + if (isStale(row.lastModified, watermark)) { + stale += 1; + continue; + } + const item = rowItem(row); + if (!item) continue; + batch.push(item); + if (batch.length >= batchRows) { + written += batch.length; + batches += 1; + yield { items: batch, cursor: cursorAt(kind, n) }; + batch = []; + if (Date.now() > stopAt) { + outOfTime = true; + break; + } + } + } + } catch (err) { + // A file gzip cannot read is not a file worth keeping: a resume would + // hand the same bytes back forever. Drop it so the next run downloads + // it again; the batches already written keep their cursor. + await unlink(path).catch(() => {}); + throw new Error( + `${kind} dump ${version} unreadable at line ${n}, removed (${err?.message ?? err})`, + ); + } + if (batch.length) { + written += batch.length; + batches += 1; + yield { items: batch, cursor: cursorAt(kind, n) }; + batch = []; + } + if (outOfTime) { + return { + cursor: cursorAt(kind, n), + note: `${summary()}; out of time at ${kind} line ${n}, resuming in 10 min`, + nextInMinutes: RESUME_MINUTES, + }; + } + + // The file is walked. Drop it (the disk is shared) and move the cursor to + // the next file NOW, as an empty batch, so a crash before the next file's + // first batch does not send the next run back into this one. + log(`${kind} dump ${version} walked: ${n} lines`); + await unlink(path).catch(() => {}); + line = 0; + file = FILES[fi + 1] ?? null; + if (file) yield { items: [], cursor: cursorAt(file, 0) }; + } + + return { + cursor: { + version, + file: null, + line: 0, + lastModifiedWatermark: maxLastModified ?? watermark, + maxLastModified: null, + done: true, + completedAt: new Date().toISOString(), + }, + note: + `complete: dump ${version}, ${rows} rows read this run, ${written} written in ${batches} batches` + + (stale ? `, ${stale} unchanged skipped` : '') + + (bad ? `, ${bad} unreadable` : '') + + (failures ? `, ${failures} requests failed` : ''), + }; + }, +}); diff --git a/packages/adapters/src/podcastindex-catalog.js b/packages/adapters/src/podcastindex-catalog.js new file mode 100644 index 0000000..b99a751 --- /dev/null +++ b/packages/adapters/src/podcastindex-catalog.js @@ -0,0 +1,663 @@ +import { readdir, readFile, rm, stat, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { decodeEntities, defineAdapter, slugify, stripHtml } from '@nichedb/core/adapter'; +import { dumpDir, sqliteRows, untar } from '@nichedb/core/dump'; + +import { platformOf } from './podcastplatforms.js'; + +/** + * Every podcast in the Podcast Index, from the database it publishes. + * + * The two `podcasts` sources read rssamplifier, a live crawl of a large slice + * of this catalogue, because that is what a fifteen-minute poll can keep up + * with. This is the census behind it: the Podcast Index's own table of every + * feed it has ever indexed, published as one SQLite file inside a 1.8 GB tgz + * at public.podcastindex.org, keyless, rebuilt on its own schedule. A row here + * carries the same normalised feed URL the other two sources carry in + * `data.feedUrl`, which is the join. + * + * HOW A 1.8 GB FILE BECOMES ROWS + * + * One weekly run, budgeted at 55 minutes, in three parts that each resume: + * + * - A HEAD first. The dump's ETag (or its Last-Modified) is the version, and a + * version the cursor already marks `done` ends the run before a byte is + * fetched. That is what makes a weekly cadence cheap when nothing changed. + * - `http.download` with Range resume into `dumpDir('podcastindex')`. The + * server honours Range, so a run that stops mid-transfer leaves a partial + * file the next run appends to. It refuses a request without a descriptive + * user agent (403), so one is sent explicitly on every request. + * - `tar` extracts the 5.1 GB database beside the archive (Bun.Archive would + * hold the whole file in memory), the archive is deleted, and the walk is + * `select ... where id > ? order by id limit 500` with the last id in the + * cursor. Each batch of 500 is yielded and its cursor saved before the next + * is read, so a crash costs one re-written batch and never the walk. + * + * THE SCHEMA IS DISCOVERED, NOT ASSUMED + * + * The repository's `create_table_statement.sql` describes a MySQL table called + * `newsfeeds` with snake_case columns (`newest_item_pubdate`, `item_count`). + * The file actually published is a SQLite table called `podcasts` with + * camelCase columns (`newestItemPubdate`, `episodeCount`) and ten `categoryN` + * columns the MySQL DDL never mentions, confirmed from the first 4 MB of the + * real archive on 2026-09-13. So the table and its columns are read from + * `sqlite_master` and `pragma table_info` at run time, and every field this + * adapter wants is resolved from a list of the names it has been called under. + * A dump that renames a column again degrades to a null field, not a crash; + * only `id`, `url` and `title` are required. + * + * WHAT IS LEFT OUT + * + * Rows flagged `dead` (too many fetch errors, no longer checked), rows with no + * title and rows whose feed URL does not parse. About a third of the index is + * dead feeds, and a directory of podcasts nobody can fetch is not a directory. + */ + +export const DUMP_URL = 'https://public.podcastindex.org/podcastindex_feeds.db.tgz'; +export const DOCS_URL = 'https://github.com/Podcastindex-org/database'; + +/** Rows per batch: the memory a run holds at once, and the unit the cursor moves by. */ +export const BATCH = 500; +/** A weekly read. The HEAD makes an unchanged week cost one request. */ +export const CADENCE_MINUTES = 7 * 24 * 60; +/** Download, extract and a good part of the walk fit in one run. */ +export const BUDGET_MS = 55 * 60_000; +/** How soon an unfinished run picks up again. */ +export const RESUME_IN_MINUTES = 10; +/** A run stops yielding this far before its deadline, so the batch in flight lands inside it. */ +export const DEADLINE_MARGIN_MS = 30_000; +/** Consecutive request failures that end a run. */ +export const MAX_FAILURES = 3; +/** Pause between retries. */ +export const PAUSE_MS = 5_000; + +export const ATTRIBUTION = 'Podcast Index; dump under its terms, index data MIT'; + +/** The last epoch second this adapter believes: 2100-01-01. The dump has a few pubdates past it. */ +const EPOCH_CEILING = 4_102_444_800; + +/** + * The names each field has been published under, lowercased, in order of + * preference. The first list is the SQLite dump as it is; the second is the + * repository's MySQL DDL, in case a future dump follows it. + */ +export const FIELD_NAMES = { + id: ['id'], + url: ['url', 'feedurl', 'feed_url'], + title: ['title'], + link: ['link', 'website', 'siteurl', 'site_url'], + description: ['description'], + image: ['imageurl', 'image_url', 'artwork_url_600', 'artworkurl', 'artwork', 'image'], + dead: ['dead'], + itunesId: ['itunesid', 'itunes_id'], + language: ['language'], + episodeCount: ['episodecount', 'item_count', 'itemcount'], + newestItemPubdate: ['newestitempubdate', 'newest_item_pubdate'], + oldestItemPubdate: ['oldestitempubdate', 'oldest_item_pubdate'], + lastUpdate: ['lastupdate', 'last_update'], + explicit: ['explicit'], + generator: ['generator'], + host: ['host'], + author: ['itunesauthor', 'itunes_author'], + guid: ['podcastguid', 'podcast_guid'], + popularity: ['popularityscore', 'popularity_score', 'popularity'], + duplicateOf: ['duplicateof', 'duplicate_of'], + lastHttpStatus: ['lasthttpstatus', 'last_http_status'], +}; + +/** Without these there is no item to make. */ +const REQUIRED = ['id', 'url', 'title']; + +const sleep = (ms) => (ms > 0 ? Bun.sleep(ms) : Promise.resolve()); + +const message = (err) => String(err?.message ?? err).slice(0, 200); + +/** Trimmed, whitespace-collapsed text, or null when nothing is left. */ +export function clean(v) { + if (v === null || v === undefined) return null; + const s = String(v).replace(/\s+/g, ' ').trim(); + return s || null; +} + +/** A positive integer, or null. Zero is the dump's "unknown" for every numeric column. */ +export function positiveInt(v) { + const n = Number(v); + return Number.isFinite(n) && n > 0 ? Math.floor(n) : null; +} + +/** An epoch-seconds column as a Date, or null when zero, absent, or past the ceiling. */ +export function epochDate(v) { + const n = positiveInt(v); + return n !== null && n < EPOCH_CEILING ? new Date(n * 1000) : null; +} + +/** The dump's tinyint flags: 1 is set, 0 is not, and text arrives from a CSV import. */ +export function flag(v) { + if (v === null || v === undefined || v === '') return false; + const n = Number(v); + if (Number.isFinite(n)) return n > 0; + return /^(true|yes|y|t)$/i.test(String(v).trim()); +} + +/** + * The feed URL as the join key. + * + * Lowercase host (the URL class does that), no trailing slash, no fragment, + * the scheme the row declares and never upgraded: a feed that only answers on + * http is not made https by wishing. Anything that is not an http(s) URL is + * null, and a row with a null feed URL is skipped. + */ +export function normaliseFeedUrl(raw) { + const s = clean(raw); + if (!s) return null; + let u; + try { + u = new URL(s); + } catch { + return null; + } + if (u.protocol !== 'http:' && u.protocol !== 'https:') return null; + if (!u.hostname) return null; + const path = u.pathname.replace(/\/+$/, ''); + return `${u.protocol}//${u.host}${path}${u.search}`; +} + +/** The show's own site, if the row has a usable one. */ +export function siteUrl(raw) { + const s = clean(raw); + if (!s) return null; + try { + const u = new URL(s); + return u.protocol === 'http:' || u.protocol === 'https:' ? u.toString() : null; + } catch { + return null; + } +} + +/** + * A language tag reduced to its base, as the `podcasts` sources do: the feed + * says `en`, `en-us` and `EN-US` for one language, and a tag per spelling is + * a tag nobody can follow. + */ +export function langTag(language) { + const base = String(language ?? '') + .toLowerCase() + .split(/[-_]/)[0] + .trim(); + return /^[a-z]{2,3}$/.test(base) ? `lang:${base}` : null; +} + +/** The non-empty `categoryN` columns of a row, in column order, de-duplicated. */ +export function categoriesOf(row) { + const keys = Object.keys(row ?? {}) + .filter((k) => /^category\d+$/i.test(k)) + .sort((a, b) => Number(a.replace(/\D/g, '')) - Number(b.replace(/\D/g, ''))); + const out = []; + for (const k of keys) { + const c = clean(row[k]); + if (c && !out.includes(c)) out.push(c); + } + return out; +} + +/** Description text: tags out, entities resolved, one line, 600 characters. */ +export function summaryOf(description) { + const text = clean(decodeEntities(stripHtml(description))); + return text ? text.slice(0, 600) : null; +} + +/** + * One row of the dump as an item, or null when it is not a podcast a reader + * can reach: dead, untitled, or without a feed URL that parses. + * + * The row arrives with the logical field names (`selectSql` aliases the + * dump's columns onto them), so this never sees the dump's own spelling. + */ +export function toItem(row) { + const id = positiveInt(row?.id); + if (id === null) return null; + if (flag(row.dead)) return null; + + const title = clean(row.title); + const feedUrl = normaliseFeedUrl(row.url); + if (!title || !feedUrl) return null; + + const site = siteUrl(row.link); + const language = clean(row.language); + const categories = categoriesOf(row); + const newest = epochDate(row.newestItemPubdate); + const host = clean(row.host)?.toLowerCase() ?? null; + + return { + externalId: `podcastindex:feed:${id}`, + kind: 'show', + title: title.slice(0, 500), + summary: summaryOf(row.description), + url: site ?? feedUrl, + imageUrl: siteUrl(row.image), + /* + * The newest episode's date, as the `podcasts` sources use `lastPublishedAt`: + * "when did this show last speak" is the question a podcast directory is + * asked, and for most of this catalogue the answer is years ago. + */ + publishedAt: newest, + tags: [ + 'show', + 'podcast', + 'podcastindex', + langTag(language), + ...categories.map((c) => `category:${slugify(c)}`).filter((t) => t.length > 9), + ].filter(Boolean), + data: { + feedId: id, + feedUrl, + siteUrl: site, + itunesId: positiveInt(row.itunesId), + podcastGuid: clean(row.guid), + author: clean(row.author), + language, + categories, + episodeCount: positiveInt(row.episodeCount), + newestItemPubdate: newest?.toISOString() ?? null, + oldestItemPubdate: epochDate(row.oldestItemPubdate)?.toISOString() ?? null, + lastUpdate: epochDate(row.lastUpdate)?.toISOString() ?? null, + explicit: flag(row.explicit), + generator: clean(row.generator), + /* The dump's own `host` column (its registrable domain, occasionally a bare + * public suffix), beside the platform the house list files the feed under. */ + host, + platform: platformOf(feedUrl), + popularity: positiveInt(row.popularity), + duplicateOf: positiveInt(row.duplicateOf), + lastHttpStatus: positiveInt(row.lastHttpStatus), + attribution: ATTRIBUTION, + source: 'Podcast Index database dump', + dataset: DUMP_URL, + }, + }; +} + +/** + * A batch of rows as items, with the tally. One row that throws is one row + * lost and one line in the log; it never ends the walk. + */ +export function rowsToItems(rows, { log = () => {} } = {}) { + const items = []; + let kept = 0; + let skipped = 0; + let bad = 0; + for (const row of Array.isArray(rows) ? rows : []) { + try { + const item = toItem(row); + if (item) { + items.push(item); + kept += 1; + } else { + skipped += 1; + } + } catch (err) { + bad += 1; + if (bad <= 5) log(`row ${row?.id ?? '?'} dropped: ${message(err)}`); + } + } + return { items, kept, skipped, bad }; +} + +/** + * What identifies the file on the server, from its HEAD. The ETag is the + * version when there is one (S3-style, changes with every upload); the + * Last-Modified is the fallback and is kept beside it either way, because a + * date is what a person reading the cursor wants to see. + */ +export function dumpVersion(headers) { + const get = (k) => + typeof headers?.get === 'function' + ? headers.get(k) + : (headers?.[k] ?? headers?.[k.toLowerCase()]); + const etag = + String(get('etag') ?? '') + .replace(/^W\//i, '') + .replace(/"/g, '') + .trim() || null; + const lmRaw = clean(get('last-modified')); + const lm = lmRaw ? new Date(lmRaw) : null; + const lastModified = lm && !Number.isNaN(lm.getTime()) ? lm.toISOString() : null; + const bytes = positiveInt(get('content-length')); + return { version: etag ?? lastModified, etag, lastModified, bytes }; +} + +/** A version as a file name. */ +export function versionStamp(version) { + return slugify(version).slice(0, 80) || 'dump'; +} + +/** The user agent the Podcast Index asks for: who, why, and how to reach us. */ +export function userAgent(env = {}) { + const contact = clean(env?.contactEmail); + return `niche-db podcastindex-catalog/1 (+https://nichedb.dev; weekly read of the public dump${ + contact ? `; ${contact}` : '' + })`; +} + +/** + * Which table holds the feeds: `podcasts` as published, `newsfeeds` as + * documented, else the first table that has both a url and a title column. + * + * @param {{ name: string, columns: string[] }[]} tables + */ +export function pickTable(tables) { + const list = Array.isArray(tables) ? tables : []; + for (const want of ['podcasts', 'newsfeeds']) { + const hit = list.find((t) => String(t?.name).toLowerCase() === want); + if (hit) return hit; + } + return ( + list.find((t) => { + const cols = new Set((t?.columns ?? []).map((c) => String(c).toLowerCase())); + return cols.has('url') && cols.has('title'); + }) ?? null + ); +} + +/** + * The dump's column for each logical field, plus its category columns. + * + * @param {string[]} columnNames as `pragma table_info` reports them + * @returns {{ columns: Record, categories: string[] }} + */ +export function resolveColumns(columnNames) { + const byLower = new Map(); + for (const name of Array.isArray(columnNames) ? columnNames : []) { + const key = String(name).toLowerCase(); + if (!byLower.has(key)) byLower.set(key, String(name)); + } + const columns = {}; + for (const [field, names] of Object.entries(FIELD_NAMES)) { + columns[field] = names.map((n) => byLower.get(n)).find(Boolean) ?? null; + } + const missing = REQUIRED.filter((f) => !columns[f]); + if (missing.length) { + throw new Error( + `the feeds table has no ${missing.join(', ')} column; columns are ${[...byLower.values()].join(', ')}`, + ); + } + const categories = [...byLower.entries()] + .filter(([k]) => /^category_?\d+$/.test(k)) + .sort((a, b) => Number(a[0].replace(/\D/g, '')) - Number(b[0].replace(/\D/g, ''))) + .map(([, real]) => real); + return { columns, categories }; +} + +const quoteIdent = (name) => `"${String(name).replace(/"/g, '""')}"`; + +/** + * The walk's statement: every resolved column aliased onto its logical name, + * categories as `category1..N`, keyed and ordered on the id so `id > ?` is an + * index seek and the last id of a batch is the cursor. + */ +export function selectSql({ table, columns, categories }) { + const parts = []; + for (const [field, real] of Object.entries(columns)) { + if (real) parts.push(`${quoteIdent(real)} as ${quoteIdent(field)}`); + } + categories.forEach((real, i) => { + parts.push(`${quoteIdent(real)} as ${quoteIdent(`category${i + 1}`)}`); + }); + const id = quoteIdent(columns.id); + return `select ${parts.join(', ')} from ${quoteIdent(table)} where ${id} > ? order by ${id} limit ?`; +} + +/** The table and columns of a dump, read from the file rather than the docs. */ +export function discoverSchema(dbPath) { + const names = [ + ...sqliteRows( + dbPath, + "select name from sqlite_master where type = 'table' and name not like 'sqlite_%'", + ), + ].map((r) => r.name); + const tables = names.map((name) => ({ + name, + columns: [...sqliteRows(dbPath, `pragma table_info(${quoteIdent(name)})`)].map((c) => c.name), + })); + const table = pickTable(tables); + if (!table) { + throw new Error(`no feeds table in the dump; tables: ${names.join(', ') || 'none'}`); + } + return { table: table.name, ...resolveColumns(table.columns) }; +} + +async function exists(path) { + try { + await stat(path); + return true; + } catch { + return false; + } +} + +/** The extracted database, if the marker written after extraction says it is whole. */ +async function readyDatabase(marker) { + if (!(await exists(marker))) return null; + const path = (await readFile(marker, 'utf8')).trim(); + return path && (await exists(path)) ? path : null; +} + +/** The one `.db` file tar left in the directory, wherever in it the member landed. */ +async function findDatabase(dir, depth = 0) { + const entries = await readdir(dir, { withFileTypes: true }); + for (const e of entries) { + if (e.isFile() && e.name.endsWith('.db')) return join(dir, e.name); + } + if (depth < 2) { + for (const e of entries) { + if (e.isDirectory()) { + const hit = await findDatabase(join(dir, e.name), depth + 1); + if (hit) return hit; + } + } + } + return null; +} + +/** + * Everything in the dump directory that is not this version: the previous + * dump's 5 GB database, or a partial archive of a file the server has since + * replaced. Disk is the constraint here (7 GB per version), not history. + */ +async function pruneOthers(dir, stamp, log) { + const keep = new Set([stamp, `${stamp}.db.tgz`]); + for (const e of await readdir(dir, { withFileTypes: true })) { + if (keep.has(e.name)) continue; + await rm(join(dir, e.name), { recursive: true, force: true }); + log(`removed ${e.name} from the dump directory`); + } +} + +export const podcastindexCatalog = defineAdapter({ + name: 'podcastindex-catalog', + title: 'Podcast Index: every feed', + collection: 'podcasts', + description: + 'Every podcast the Podcast Index has catalogued, several million feeds, read weekly from the SQLite database it publishes as a 1.8 GB download rather than from its keyed API: title, description, website, artwork, language, categories, episode count, the newest episode date, the iTunes id, the generator and the hosting platform, with dead feeds left out. The dump is published under the Podcast Index terms and the index data itself is MIT licensed. Keyless; the schema is read from the file at run time and each row carries the same normalised feed URL the other podcast sources do.', + docs: DOCS_URL, + kinds: ['show'], + cadenceMinutes: CADENCE_MINUTES, + budgetMs: BUDGET_MS, + configFields: [ + { + key: 'batchSize', + label: 'Rows per batch', + type: 'number', + placeholder: String(BATCH), + help: 'Rows read and written at once; the cursor moves by this much. 50 to 2,000.', + }, + { + key: 'pauseMs', + label: 'Pause between retries (ms)', + type: 'number', + placeholder: String(PAUSE_MS), + help: `A failed request is retried up to ${MAX_FAILURES} times with this pause.`, + }, + ], + defaults: { batchSize: BATCH, pauseMs: PAUSE_MS }, + defaultSources: [ + { + slug: 'podcastindex-catalog', + name: 'Podcasts: every feed in the Podcast Index', + config: { batchSize: BATCH, pauseMs: PAUSE_MS }, + }, + ], + async *pull({ config, cursor: prev, env, http, log, deadline }) { + const batchSize = Math.max(50, Math.min(Math.floor(Number(config?.batchSize)) || BATCH, 2000)); + const pause = + config?.pauseMs === 0 ? 0 : Math.max(0, Math.floor(Number(config?.pauseMs))) || PAUSE_MS; + const stopAt = Number.isFinite(deadline) + ? deadline - DEADLINE_MARGIN_MS + : Number.POSITIVE_INFINITY; + const ua = userAgent(env); + const headers = { 'user-agent': ua }; + let succeeded = 0; + + /* One request, retried on consecutive failure. Returns null when the run + * should stop: either every request so far failed (thrown, so the run is + * an error) or this one failed three times after others succeeded (the + * place is kept and the run resumes in ten minutes). */ + const attempt = async (what, fn) => { + for (let failures = 0; ; ) { + try { + const out = await fn(); + succeeded += 1; + return out; + } catch (err) { + failures += 1; + log(`${what} failed (${failures}/${MAX_FAILURES}): ${message(err)}`); + if (failures >= MAX_FAILURES) { + if (succeeded === 0) throw new Error(`every request failed; last: ${message(err)}`); + return null; + } + if (Date.now() >= stopAt) return null; + await sleep(pause); + } + } + }; + + // 1. What the server has, and whether the cursor already walked it. + const head = await attempt('HEAD', async () => { + const res = await http.request(DUMP_URL, { method: 'HEAD', headers, timeoutMs: 60_000 }); + await res.body?.cancel().catch(() => {}); + if (!res.ok) throw new Error(`${res.status} from HEAD ${DUMP_URL}`); + return dumpVersion(res.headers); + }); + if (!head) return { cursor: prev ?? {}, note: 'HEAD failed', nextInMinutes: RESUME_IN_MINUTES }; + + let { version } = head; + const { lastModified, bytes } = head; + if (!version) { + version = new Date().toISOString().slice(0, 10); + log(`the dump has no etag or last-modified; using today (${version}) as its version`); + } + + const same = prev?.version === version; + if (same && prev?.done) { + log(`dump ${version} (${lastModified ?? 'no date'}) unchanged and fully read`); + return { cursor: prev, note: 'unchanged' }; + } + let afterId = same ? Math.max(0, Math.floor(Number(prev?.afterId)) || 0) : 0; + const at = (extra = {}) => ({ version, lastModified, afterId, ...extra }); + if (!same && prev?.version) log(`new dump ${version}; the walk starts over`); + /* A run that is at its deadline does no work: nothing is yielded, the place + * is kept, and the next run ten minutes on picks up exactly there. */ + const outOfTime = (note) => ({ cursor: at(), note, nextInMinutes: RESUME_IN_MINUTES }); + + // 2. The file, on disk and extracted. Resumable at every step. + const dir = await dumpDir('podcastindex'); + const stamp = versionStamp(version); + const tgz = join(dir, `${stamp}.db.tgz`); + const extractDir = join(dir, stamp); + const marker = join(extractDir, 'ready'); + let dbPath = await readyDatabase(marker); + + if (!dbPath) { + if (Date.now() >= stopAt) return outOfTime('out of time before the download'); + await pruneOthers(dir, stamp, log); + let lastLogged = 0; + const dl = await attempt('download', () => + http.download(DUMP_URL, tgz, { + headers, + timeoutMs: Math.max(60_000, stopAt - Date.now()), + onProgress: ({ bytes: got, total }) => { + if (got - lastLogged < 200 * 1024 * 1024) return; + lastLogged = got; + log( + `downloaded ${Math.round(got / 1e6)} of ${total ? Math.round(total / 1e6) : '?'} MB`, + ); + }, + }), + ); + if (!dl) { + // Either the deadline arrived mid-transfer or three attempts failed in a + // row; the partial file is on disk either way and the next run resumes it. + return outOfTime( + Date.now() >= stopAt + ? 'download in progress' + : 'download stopped after repeated failures', + ); + } + if (!dl.complete) { + log(`download in progress: ${dl.bytes} of ${bytes ?? '?'} bytes`); + return outOfTime('download in progress'); + } + + log(`extracting ${Math.round(dl.bytes / 1e6)} MB archive`); + await untar(tgz, extractDir); + dbPath = await findDatabase(extractDir); + if (!dbPath) throw new Error('the archive held no .db file'); + await writeFile(marker, `${dbPath}\n`); + // 1.8 GB the walk never reads again; the next version is a new download anyway. + await rm(tgz, { force: true }); + log(`extracted ${dbPath}`); + } + + // 3. The walk, 500 rows and one cursor at a time. + const schema = discoverSchema(dbPath); + const sql = selectSql(schema); + log( + `reading ${schema.table} (${Object.values(schema.columns).filter(Boolean).length} columns, ${schema.categories.length} category columns) from id ${afterId}`, + ); + + let batches = 0; + let kept = 0; + let skipped = 0; + let bad = 0; + for (;;) { + // Checked before every read, the first included: a run that arrives at its + // deadline reads nothing, and one that reaches it mid-walk stops after the + // batch already handed over. + if (Date.now() >= stopAt) { + log(`out of time at id ${afterId} after ${batches} batches: ${kept} shows this run`); + return outOfTime(`out of time at id ${afterId}: ${kept} shows, ${skipped} skipped`); + } + const rows = [...sqliteRows(dbPath, sql, [afterId, batchSize])]; + if (rows.length === 0) { + log(`walk complete at id ${afterId}: ${kept} shows, ${skipped} skipped, ${bad} bad rows`); + return { + cursor: at({ done: true }), + note: `complete: ${kept} shows, ${skipped} skipped, ${bad} bad rows`, + }; + } + + const out = rowsToItems(rows, { log }); + kept += out.kept; + skipped += out.skipped; + bad += out.bad; + + const lastId = positiveInt(rows[rows.length - 1]?.id); + if (lastId === null || lastId <= afterId) { + throw new Error(`the walk did not advance past id ${afterId}`); + } + afterId = lastId; + batches += 1; + yield { items: out.items, cursor: at() }; + } + }, +}); diff --git a/packages/adapters/test/fixtures/discogs-catalog-artists.xml b/packages/adapters/test/fixtures/discogs-catalog-artists.xml new file mode 100644 index 0000000..696dd85 --- /dev/null +++ b/packages/adapters/test/fixtures/discogs-catalog-artists.xml @@ -0,0 +1,82 @@ + +1The PersuaderJesper DahlbäckElectronic artist working out of Stockholm, active since 1994.Needs Votehttps://en.wikipedia.org/wiki/Jesper_Dahlbäckhttps://www.last.fm/music/Jesper+Dahlb%C3%A4ckPersuaderThe PresuaderJesper DahlbäckGroove MachineDick TrackLenkJanne Me' AmazonenFaxidThe Pinguin Man +2Mr. James Barth & A.D.CorrectMR JAMES BARTH & A. D.Mr Barth & A.D.Mr. Barth & A.D.Mr. James Barth & A. D.Puente LatinoYakari & DelanoCrushed Insect & The Sick PuppyADCLAlexi Delano & Cari LekebuschAlexi DelanoCari Lekebusch +3Josh WinkJosh WinkelmanElectronic music DJ, label owner, producer, and remixer, born April 20, 1970 in Philadelphia, Pennsylvania, USA. Formed [l=Ovum Recordings] as an independent label in October 1994 with former partner [a=King Britt].Needs Votehttp://www.joshwink.comhttp://www.ovumrecordings.com/artists/josh-winkhttp://joshwink.bandcamp.comhttps://soundcloud.com/joshwinkofficialhttp://www.dailymotion.com/JoshWink-vevohttp://www.facebook.com/JoshWinkOfficialhttp://instagram.com/joshwink1http://myspace.com/joshwinkhttp://myspace.com/ovumrecordingshttp://soundcloud.com/joshwinkofficialhttp://www.songkick.com/artists/250682-josh-winkhttp://twitter.com/joshwink1http://www.whosampled.com/Josh-Winkhttp://en.wikipedia.org/wiki/Josh_Winkhttp://www.youtube.com/user/JoshWinkVEVODJ Josh WinkDJ WinkDj WinkDosh WinkFFRR Josh WinkJ WinkJ. WInkelmanJ. WinkJ. Wink (DJ Wink)J. WinkelmanJ. WinkelmannJ. WnkJ.WinkJ.WinkelmanJoshJosh Wink "DJ Wink"Josh Wink (DJ WINK)Josh Wink (DJ Wink)Josh Wink (Dj Winx)Josh WinkelmanJosh WinkelmannJosh WinksJosh WinxJoshWinkWINKWiNKWin XWincWingWingsWinkWink, JoshWinkelmanWinksWinxWinxs佐瑟溫克Size 9The CrusherDinky DogAccent (3)J. DawgE-CultureAbundance Of CupsOff KeyJack Jones (4)Just King And Wink +4Johannes HeilJohannes HeilDj, producer, author. Born: February 3, 1978 in Hessen, Germany. Founder of [l=JH] and [l=Metatron Recordings].Needs Votehttps://www.instagram.com/johannes_heilhttps://de.ra.co/dj/johannesheilhttps://www.beatport.com/de/artist/johannes-heil/9308HeilHeil JohannesHeil, JohannesHellJ HeilJ. HeilJ. HeillJ.HeilJHJoh. HeilJohannas HeilJohannes HellThe TrinityUnity GainJim Henson ProjectAntonio Montana El ReyAge BeatsThe X ActThink Tank (2)The Hidden (2)CryptikItem OneC.R.S.Project 69State Of ChaosQuestion Authority (2) +5Heiko LauxHeiko LauxGerman DJ and producer based in Berlin. He is the founder of [l=Kanzleramt].Needs Votehttp://www.facebook.com/heikolauxofficialhttp://www.soundcloud.com/heikolauxhttp://www.myspace.com/heikolauxH. LauxH.LauxHeikoHeiko Laux ' Five'LauxGoldwaveApathismTokoX-Rated AngelsHLXOrange 25Total Planet RefreshmentItem OneRezzQMonobleedingLaux & OlssonDirect From The MachineSodiacThe Global SegmentsOffshore Funk4K (3) +6K.A.B.Needs Major Changes +7Sylk 130King James BrittNeeds Vote130Silk 130Silk130Sylk130シルク130思客130ScubaKing BrittFireflyOba FunkeDynamic (3)The Nova Dream SequenceFhloston ParadigmSoul LitchfieldSraddha +8Mood II SwingHouse music production team formed by John Ciafone and Lem Springsteen in New York City in the early 1990s.Needs Votehttp://moodiiswing.comhttp://mood2swing.bandcamp.comhttp://facebook.com/moodiiswinghttp://instagram.com/mood_ii_swinghttp://soundcloud.com/moodiiswinghttp://twitter.com/Mood_II_Swinghttp://en.wikipedia.org/wiki/Mood_II_Swinghttp://www.youtube.com/channel/UCMtX1dy2CHcA59LYcxkN3OwMood 11 SwingMood 2 SwingMood IIMood II Swing ProductionMood II Swing ProductionsMood Il SwingMood SwingMood To SwingMoog II SwingSwing II MoodThe House Of Mood II SwingUrbanizedWall Of SoundJohn CiafoneLem Springsteen +9Care CompanyCorrectMarkus ReinhardtCarsten KlatteJosé Alvarez-Brill +11DJ DoziaDozia BlakeyHouse producer from Philadelphia, Pennsylviania. +Runs [l=Nou Lion Recordings] and production company [l=Nou Lion Productions]. +Nephew of [a=Art Blakey].Needs VoteD.J. DoziaDoziaDozia SlimMusic From The TreesDozia BlakeyThe Nou LionSolaris (19) +13BlazeNew Jersey-based duo. +[a=Chris Herbert] was a member but left the group in the early '90s.Needs Votehttp://b-l-a-z-e.bandcamp.comhttp://funkypeople.bandcamp.comhttps://en.wikipedia.org/wiki/Blaze_(band)http://web.archive.org/web/20070611115508/http://www.mawrecords.com/bios2/pages/blazebio.htmlhttp://web.archive.org/web/20070610123503/http://www.westendrecords.com/artists/blaze.php3A Blaze ProjectBlaceBlaze 2001Blaze ProductionBlaze ProductionsBlaze ProjectBlazelBlazèThe BlazeThe Blaze TeamThe Mighty BlazeThe Original Blaze絢爛三人組Black RascalsExitKlubheadStardust (2)Project MSCJames Toney Jr. ProjectIn-Sync (2)Kevin HedgeJosh MilanChris Herbert +14Eight Miles HighRoman FlügelNeeds Votehttp://8mh.deRoman IVRoman Flügelro70Soylent GreenAcid TestList (2) +16Christian SmithChristian Smith-SolbakkenTech-House and Funky Techno DJ & producer. +Born in Sweden, Christian spent much of his youth and early adult life in Germany and America. +In 2009 Christian relocated to Sao Paulo, Brazil. +As of 2016, he was living in Mallorca, Spain. +Started his House & Techno label [l=Tronic] label in 1994, then restarted it in 2009, after a two-year hiatus. +Needs Votehttp://christiansmithmusic.comhttp://christiansmith.bandcamp.comhttp://www.facebook.com/officialchristiansmithhttp://www.mixcloud.com/christiansmithlive/http://www.myspace.com/christiansmithdjhttp://soundcloud.com/christiansmithhttp://twitter.com/CSmithLIVEhttp://en.wikipedia.org/wiki/Christian_Smith_(DJ)http://www.youtube.com/channel/UCl1NEpKMmZ7CuzQZpniD-uAC. SmithC.SmithChristian SChristian S.Christian Smith - SolbakkenChristian Smith-SolbakkenCristian SmithMousse TSmithSmith, ChristianSolbakkenNeuromancer (2)DestinationChristian Smith & John SelwayGalacticaForeign TexturesTimelineKings Of PunaniGoldrushInterzone (7) +17John SelwayJohn Brinton SelwayFrom the early 90s to the present, Selway has built a high quality catalog of productions, both on his own and as a collaborator in various artist and label projects. From his first electronic band Chaotic Sound Matrix, to his major contributions to the early productions of Deep Dish; from his first success in the techno world as part of the seminal New York duo Disintegrator to the most successful of his collaborations, Smith & Selway, and his deep and minimal techno label CSM; from the intelligent electro-funk of Synapse and [l=Serotonin], the electro and synth-pop of Memory Boy to the wild improvisational eclecticism of the Rancho Relaxo Allstars, Selway has created and helped to create one of the most stylistically wide ranging bodies of work in the world of electronic dance music.Needs Votehttp://www.selwaymusic.nethttp://www.facebook.com/pages/John-Selway/224601153031?ref=ts&fref=tshttp://www.myspace.com/selwaymusicJ SelwayJ. B. SelwayJ. SelwayJ.SelwayJohn Brinton SelwayJohn LewayJohn SelawaySelSellwaySelwaySelway, JohnSpy (2)Brinton McKayGalactic Spiral SoundSemblance FactorMemory BoyHighriseDr. TheopolisThree O'Clock HighZoid (12)SynapseExodus QuartetChristian Smith & John SelwayKoenig CylindersPsychedelic Research LabMoodsOctaves/TremelosDisintegratorResponsible Space PlayboysRancho Relaxo AllstarsPrana (2)SeltavCSMDharma (2)The FoundersNeurotic Drum BandEast Side ScientificMachines (8)I-Spy (2) +19Sound AssociatesNeeds VoteDaz Saund & Ben TisdallHousewerkBen TisdallDaz Saund +20Percy XAnthony Scott MacKinnonProducer and DJ from Glasgow, ScotlandNeeds Votehttp://www.myspace.com/percyxuk http://www.somarecords.com/artists/percyxPercy-XPercyXMionAbyssRecycle (2)ShortfictionTony Scott (4)The X-Man (3)Edit Select +21Faze ActionBritish band composed of brothers Simon and Robin Lee. Faze Action blended jazz-dance music with Western classical, pan-African, and Latin music. + +Contact: info [@] fazeaction.comNeeds Votehttp://www.fazeaction.comhttp://fazeactionrecords.bandcamp.comhttp://linktr.ee/fazeactionhttp://www.myspace.com/fazeactionhttp://www.soundcloud.com/fazeactionhttp://twitter.com/Faze_Action_http://www.youtube.com/c/FazeActionRecordsFAFace ActionSimon & Robin LeeOrtoSimon LeeRobin Lee +22DATacideDatacide began recording together in 1993, after Tetsu Inoue met Uwe Schmidt while vacationing near Frankfurt. They recorded a pair of dance tracks for 12-inch release on [l=Fax +49-69/450464], followed by a full-length which mixed more uptempo trance-oriented techno with beatless ambient and experimental soundscapes. A second Datacide release in a similar, though more ambient vein (entitled Datacide II) appeared the following year before Tetsu Inoue and Uwe Schmidt switched gears for their first Rather Interesting release, Flowerhead, an album of laidback ambient-jazz. Ondas, released in 1996, upped Flowerhead's weirdness factor significantly, dwelling for most of the album in channel separation experiments which fused the rhythmic abstraction of Atom Heart's recent solo work with the fizzier, more left-field of Tetsu Inoue's electronic treatments (Tetsu Inoue's girlfriend, Ingrid, even makes an extended cameo on "mouth trumpet").Needs VoteDatacide 2Masters Of Psychedelic AmbianceTetsu InoueUwe Schmidt +23Alex Hi-FiCorrectAlex HiFi +24Atom HeartUwe SchmidtAtom™ (also known as Atom Heart and often confused with Señor Coconut, real name Uwe Schmidt - born 27 August 1968) is a German composer, musician and record producer of electronic music. He is often regarded as the father of electrolatino, electrogospel and aciton (acid-reggaeton) music.Needs Votehttp://www.atom-tm.comhttps://www.instagram.com/atomtm_officialhttp://atomtm.bandcamp.comhttp://music.hyperreal.org/artists/atom_hearthttp://www.hyperreal.org/music/lists/motherhttps://en.wikipedia.org/wiki/Uwe_SchmidtA. HeartAtomAtom He@rtAtomHeartAtomheartHeartDropshadow DiseaseNaturalistFonosandwichSoundfieldsGeeez 'N' GoshSeñor CoconutAtom™Lassigue BendthausCoeur AtomiqueAlmost DigitalSemiacoustic NatureFlextoneVSVNInteractive MusicSilver SoundBASSReal IntelligenceDotsiD'AmmondAtomu ShinzoThe BitniksMono™Machine PaisleyBrownThe Roger Tubesound EnsembleSchnittstelleErik SatinLos Sampler'sThe Disk OrchestraSuperficial DepthUrban PrimitivismDOS TracksMidisportBund Deutscher ProgrammiererThe StereonerdsWeird ShitRuben RodriguezDr. MuellerSlotLisa CarbonReplicant Rumba RockersXOX CrewPaul VanderstukkenFumesPhresh PhantasyMicrosmiles21 BrothersDJ Marco FavatiTobias SelbermannTakeshi OndaLos NegritosMatt & SilverDon AtomUwe SchmidtLinger DecoreeJohn BryantTad DavisH. RothPlastique (6)Mike McCoy (4)Atomizer™MC Unknown (3)Phreak (5)DJ Roxy (2)HüllkurveMCH (2)Hugh & EyeLinear (6)Sagittarius A (2)Softcore (6)Pentatonic SurpriseNN (17) +25Tetsu Inoue井上徹Needs Votehttp://music.hyperreal.org/artists/tetsu/https://en.wikipedia.org/wiki/Tetsu_InoueInoueT. InoueTetsoTetsuTetsu InnoueTetsue InoueTetsuu InoueTrance MediaTrance Media NetworkOrganic CloudAmbiant OtakuDATacideOmMasters Of Psychedelic AmbianceHATDivinationShades Of OrionSecond Nature (2)Psychic MediaElectro Harmonix2350 BroadwayZenith (6)62 EulengasseCymatic ScanTime²Kelley Gabriel & Clocks Of Paradise +26Alexi DelanoTechno DJ and producer, born in Chile, raised in Sweden, and later based in New York. Nominated for the Swedish Music Award ‘P3 Guld’ in 2011 (an alternative to the Swedish Grammy).Needs Votehttps://www.facebook.com/alexidelanomusichttp://www.soundcloud.com/alexidelanohttp://twitter.com/AlexiDelanoA DelanoA. D.A. DelanoA.DA.D.A.DelanoADAlex DelanoAlexei DelanoAlexi "Adny" DelanoAlexi Delano (ADNY)Alexi Delano (N.Y.C.)Alexi DolanoAlexi V. DelanoDJ AlexiDJ Alexi DelanoDelanoDelano, AlexiLords Of SvekADNYG.O.L.LeivaBob BrewthbakerMr. CouscousA.D.1010Mr. James Barth & A.D.ADNY & The PersuaderPuente LatinoThe Pump PanelNu AllianceGiovani & MoslerYakari & DelanoStrong AccentsFurther ThoughtsCrushed Insect & The Sick PuppyADJDTracks And The CityADCLEast Side ScientificAlexi Delano & XpansulDriver & ManualAlexi Delano & Cari LekebuschBasic Need (2)El Campo (2) +27Cari LekebuschKari Pekka LekebuschSwedish electronic producer. + +Born: 1972. + +Based in Stockholm, Sweden. Performing vinyl DJ sets, creating audio/music productions, and graphic works. Collected music from the late 1970s, thru Hip Hop, Electro, House, Techno, and other similar styles. + +The early 1980s where filled with breakdancing and spray-painting everything that was possible around the Stockholm areas. + +Early musical influences came from USA & Germany. "Remember GrandMaster Flash, DJ Red Alert, African Bambaata and the Soul Sonic Force, BDP, Busy B, Soundmasters, Egyptian Lover, and how about Ice-T in "Reckless" by Chris The Glove Taylor (including one of the first great TR-808 and TB-303 programmings ever). That was some amazing stuff back then, and I will never forget when Kraftwerk released Man Machine and Computer world. Then we have Mantronix, Ultramagnetic MC's, Whoodini, them where great times!". + +All this got Cari to start collecting equipment for his own recording studio. International and national releases have been made on: Drumcode and Truesoul Records, Code Red, Corb, Proper NYC, Harthouse, FFrr, JakPot, Loop and Plumphouse Records, Missile, Jericho, Influence, Tortured, and Electrix Records UK, Primevil, Svek, Planet Rhythm, Tronic Music, Experimental NYC, Ohm Records / Telegram, Analog USA. Own label projects include: Djupt, Grundtakt, Trainspotters Nightmare, AudioMekanixc, Audio Pollution, KGB, Direkt, Kaun Trax, Spirit Fuel, together with Hybrid Productions as the main platform. + +In 1998 a legal twist started between Hybrid Productions and Awex INC owned pop group called "Hybrid", which resulted into a name change – thus the dot after the H in H. Productions. +Needs Votehttps://lekebusch.bandcamp.comhttp://www.carilekebusch.comhttp://www.lekebuschmusik.sehttp://blog.carilekebusch.comhttp://www.alivenotdead.com/CariLekebuschhttp://world.secondlife.com/resident/ce8435c8-f55c-4710-aab4-ac7156078221http://www.facebook.com/CariLekebuschOfficialhttp://www.flickr.com/photos/carilekebuschhttp://www.instagram.com/carilekebuschhttp://www.myspace.com/carilekebuschhttp://www.songkick.com/artists/307656-cari-lekebuschhttp://soundcloud.com/carilekebuschhttp://twitter.com/carilekebuschhttp://en.wikipedia.org/wiki/Cari_Lekebuschhttp://www.youtube.com/carilekebuschC LekebuschC-BlastC. LekeBuschC. LekebuschC. LekebushC.LekebuschC.LekebushCari Le KebuschCari LeekabuschCari Leke BuschCari Lekebusch den rykande BönsyrsanCari LekebushCari LelebuschCari [Den Rykande Bönsyrsan] LekebuschCarl LekebuschLekebuschLekebusch MusikLekebusch, CariMr. James BarthBraincellSzerementa ProgramsMagentaMystic Letter KShape ChangerVectorAgent OrangePhunkey Rhythm DoctorRotortypeFredRubberneckCerebusYakariCrushed InsectKari PekkaSir Jeremy Augustus Hutley Of Granith HallD-RangeThe Mantis (2)DJ Mystiska KC-BlastAgent C From KGBSwedish PeaceMr. James Barth & A.D.Puente LatinoThe Pump PanelAudio MekcanicksYakari & DelanoSteady MotionLotus EyeStrong AccentsKozmic Gurt BrodhasParallax (3)Duplex (4)Cre8orsTwobytesCrushed Insect & The Sick PuppyTwo Basement BoysADCLNFL KruDeform AntiAlexi Delano & Cari Lekebusch +28MoonchildrenChris GrayCorrectThe MoonchildrenChris GrayMoonjaazHiemmelDamaged SoleDa Jakka +29Sweet AbrahamJeffrey PeacockRhythm-forward, Sweet Abraham's music pulls deep house, soul, and jazz into structures built on Rhodes, analog systems, and live and programmed drums. Written, recorded, and cut as physical artifacts in a single room. + +Toronto-born, he debuted on London's Defender in 1997, releasing his first album on the label's Nitedance imprint the same year. The Spreading Outward EP for Chicago's Deep4Life followed before founding Diaspora Recordings with Asya Shein during their overlapping time in Detroit. + +Based at Goya Music's West London complex during the label's most prolific years and at the center of the broken beat scene, he built a catalog that drew Carl Craig and Kenny Dope as remixers and gave early solo platforms to Peven Everett and Lady Alma. A collaboration with Lars Behrenroth produced the piece that has opened Deeper Shades of House for more than twenty years. + +Present work is created at the studio base in New York. All works including the Diaspora archive collect at deepbasic.com.Needs Votehttps://sweetabraham.bandcamp.comhttps://www.diasporarecordings.comhttps://www.deepbasic.comJ PeacockJeff PeacockThe Lafayette Loft Project +30Groove CollectiveAmerican jazz-funk combo from New York City, formed in early 1990s. Needs Votehttp://www.facebook.com/groovecollectivehttp://www.instagram.com/groovecollectiveofficialhttp://en.wikipedia.org/wiki/Groove_CollectiveGroove CollectionMembers of the Groove CollectiveThe Groove CollectiveRichard WorthJay RodriguezItaal ShurJonathan MaronGenji SiraisiBill WareFabio MorgeraBarney McAllGordon Clay +31SwagSheffield-based electronic music production duo.Needs Votehttp://versionmusic.bandcamp.comS.W.A.G.Primitive1/2 Inch JackPoint Blank (2)Chris DuckenfieldRichard Brown +32Jay DenesProducer, songwriter and multi-instrumentalist from New York. +Co-founded the [l=Naked Music NYC] production company and [l=Naked Music Recordings] label with [a=David Boonshoft].Needs Votehttps://www.facebook.com/jay.denes/DenesJ DenesJ. DenesJ.D.J.DenesJDJayJay Denes (aka Blue Six)Jay DennesJayDenesJdBlue SixNaked Music NYCThump'N'JohnsonMotivation (11)LovetronicSummerland +33Blue SixJay DenesCorrectBlue 6Blue SixthBlue6BluesixJay DenesNaked Music NYCThump'N'JohnsonMotivation (11) +34Miguel MigsMiguel Sundance StewardBorn on December 6, 1972. + +Deep house DJ and producer from San Francisco, California, United States. + +Miguel Migs started his career at the age of 18 as songwriter and lead guitarist for Santa Cruz local dub band Zion Sounds. In 1998 he co-founded the label [l=Transport Recordings] and in 2004 he started his own label [l=Salted Music]. + +Miguel has produced over 100 remixes to date, including remixes for a number of mainstream artists such as Britney Spears, Macy Gray and Lionel Richie. Needs Votehttp://miguelmigs.comhttp://saltedmusic.comhttp://miguelmigs.bandcamp.comhttp://www.facebook.com/DJMiguelMigshttp://www.instagram.com/miguelmigsmusichttp://www.myspace.com/djmiguelmigshttp://soundcloud.com/miguelmigs1http://twitter.com/saltedmusichttp://en.wikipedia.org/wiki/Miguel_Migshttp://www.youtube.com/channel/UCKnCgr-eUxVcDhVmXppTMewDJ MigsDJ MiguelDJ Miguel MigsM. MigsMecuryMigMig'sMigsMiguelMiguel "Migs"Miguel 'Migs'Miguel ‘Migs’Miguels MigsMihael MigsMiquelPetalpusherDeluxe PusherMiguel Steward +35AttaboyNamed after Alec's Dad's hat company Attaboy is a joint collaboration between label founders Paul Ingall and Alec Greenhough and in house producer Si Bradshaw. They have had releases on Toko, Naked Music, Defected, Glasgow Underground and Wave.Needs VoteTh' AttaboyTh'attaboyAlec GreenhoughPaul IngallSimon Bradshaw +36LovetronicCorrectLove TronicJay DenesDave Warrin +38AquanoteGabriel RenéNeeds Votehttp://www.facebook.com/Aquanote-207612495944828Aqua NoteAquanote (G. Rene)Gabriel René +39Baby MammothBased in Hull, England.Needs VoteBaby MamouthThe MammothAndrew BurdallMark Blissenden +40Dave WarrinBrother of [a=John Warrin].Needs VoteD WarrinD. WarrinD.WarrinDave WarrenWarrinCentral Living2DBMindflightLovetronicJulius Papp & Dave WarrinSlide FiveBrothers WarrinLi'sha Project +41AutechreAn English electronic music duo formed in 1987 in Rochdale, Greater Manchester, UK by [a=Rob Brown (3)] and [a=Sean Booth]. Generally considered to be IDM in style, their music has incorporated a variety of genres and styles ranging from techno and hip hop to ambient, experimental electro, and musique concrète. Autechre use many different digital and analog synthesizers, samplers, and drum machines in their production. They are also heavily involved with the [a=Gescom] collective.Needs Votehttp://www.autechre.wshttp://autechre.warp.nethttp://warp.net/artists/91432-autechrehttp://www.allmusic.com/artist/autechre-mn0000759998http://archive.org/details/Autechrehttp://autechre.bandcamp.comhttp://www.facebook.com/Autechre-%C3%A6-106935292671631http://www.last.fm/music/Autechrehttp://myspace.com/myslbhttp://pitchfork.com/artists/168-autechrehttp://www.songkick.com/artists/49992-autechrehttp://www.twitch.tv/au7echrehttp://www.twitch.tv/4utechrehttp://en.wikipedia.org/wiki/Autechrehttp://www.youtube.com/channel/UCBUAlfIrcw1f0c4qGrYn3xAAEAeAutecherAuthechreaeæアウテカオウテカLego FeetBic?Sean BoothRob Brown (3) +42Sean BoothSean Anthony BoothBritish electronic musician, born 28 August 1972 in Wardle (Greater Manchester), England, UK.Needs Votehttps://post.lurk.org/@sean_aeBootBoothS. BoothS.BoothSeanSean Anthony BoothAutechreGescomLego FeetBic? +43Rob BrownRobert Michael BrownMontreal DJ. + + + +Needs Votehttp://www.djrobbrown.com/R. BrownRob "Beatdown" BrownRoberto BravoKoor Zonder Naam +45Aphex TwinRichard David JamesUK electronic musician, born 18 August 1971 in Limerick, Ireland. In 1991 he co-founded the [l=Rephlex] label with [a953996]. After having released a number of albums and EPs on Rephlex, [l=Warp Records], and other labels under many aliases, he gained more and more success from the mid-1990s with releases such as "[m=27457]" (1997), #36 on UK charts, and "[m=532]" (1999), #16 on UK charts. He won the 2015 Grammy Award for "Best Dance / Electronica Album". The two classic Aphex Twin logos were designed in 1991-1992 by [a2252378].Correcthttp://aphextwin.warp.nethttp://aphextwin.bandcamp.comhttp://www.cheetah-ep.comhttp://equipboard.com/pros/aphex-twinhttp://www.facebook.com/aphextwinafxhttp://foundation.app/@aphextwinhttp://www.instagram.com/aphextwin.afxhttp://soundcloud.com/richarddjameshttp://soundcloud.com/user18081971http://twitter.com/aphextwinhttp://vimeo.com/user68205945http://en.wikipedia.org/wiki/Aphex_Twinhttp://www.youtube.com/@aphextwin9341/afx\3 AFXA-F-X TwinA.F.X.A.TwinAFXAFX (Aka Aphex Twin)AfxApex TwinAphex TwinsAphexTwinA·F·XThe Aphex TwinTheAphexTwinafxエイフェックス・ツイン艾費克斯雙胞胎GAKCaustic WindowPower-PillThe Dice ManPolygon WindowBradley StriderQ-ChasticBlue Calx (2)SmojphaceRichard D. JamesThe TussBrian TregaskinKaren TregaskinSoit - P.P.Phonic Boy On Dopeuser48736353001user18081971 +46GAKRichard David JamesBorn: August 18, 1971, Limerick, IrelandNeeds Votehttp://warp.net/artists/91354-gak/infoAphex TwinCaustic WindowPower-PillThe Dice ManPolygon WindowBradley StriderQ-ChasticBlue Calx (2)SmojphaceRichard D. JamesThe TussBrian TregaskinKaren TregaskinSoit - P.P.Phonic Boy On Dopeuser48736353001user18081971Springs Wade + diff --git a/packages/adapters/test/fixtures/discogs-catalog-masters.xml b/packages/adapters/test/fixtures/discogs-catalog-masters.xml new file mode 100644 index 0000000..0526d13 --- /dev/null +++ b/packages/adapters/test/fixtures/discogs-catalog-masters.xml @@ -0,0 +1,6 @@ + +155102212070Samuel L SessionSamuel LElectronic2001New SoilCorrect +330981The Persuader&2Mr. James Barth & A.D.ElectronicHip Hop1999Stockholm <Sessions> "Vol. 1"Needs VoteRecorded at Nou Lion. +Second pressing. +1900003Josh WinkElectronic0Untitled AcidNeeds Vote + diff --git a/packages/adapters/test/fixtures/musicbrainz-catalog-artist-200.tar.xz b/packages/adapters/test/fixtures/musicbrainz-catalog-artist-200.tar.xz new file mode 100644 index 0000000..96794b1 Binary files /dev/null and b/packages/adapters/test/fixtures/musicbrainz-catalog-artist-200.tar.xz differ diff --git a/packages/adapters/test/fixtures/musicbrainz-catalog-instrument.tar.xz b/packages/adapters/test/fixtures/musicbrainz-catalog-instrument.tar.xz new file mode 100644 index 0000000..9dfeb58 Binary files /dev/null and b/packages/adapters/test/fixtures/musicbrainz-catalog-instrument.tar.xz differ diff --git a/packages/adapters/test/fixtures/musicbrainz-catalog-release-group-200.tar.xz b/packages/adapters/test/fixtures/musicbrainz-catalog-release-group-200.tar.xz new file mode 100644 index 0000000..89dfd93 Binary files /dev/null and b/packages/adapters/test/fixtures/musicbrainz-catalog-release-group-200.tar.xz differ diff --git a/packages/adapters/test/fixtures/openlibrary-catalog-authors.tsv b/packages/adapters/test/fixtures/openlibrary-catalog-authors.tsv new file mode 100644 index 0000000..121d645 --- /dev/null +++ b/packages/adapters/test/fixtures/openlibrary-catalog-authors.tsv @@ -0,0 +1,100 @@ +/type/author /authors/OL10000080A 1 2021-12-26T21:23:30.303089 {"type": {"key": "/type/author"}, "name": "Gunthard Heller", "key": "/authors/OL10000080A", "source_records": ["bwb:9788490157534"], "latest_revision": 1, "revision": 1, "created": {"type": "/type/datetime", "value": "2021-12-26T21:23:30.303089"}, "last_modified": {"type": "/type/datetime", "value": "2021-12-26T21:23:30.303089"}} +/type/author /authors/OL10000104A 1 2021-12-26T21:30:15.445616 {"type": {"key": "/type/author"}, "name": "Bob Rose", "key": "/authors/OL10000104A", "source_records": ["bwb:9781617800511"], "latest_revision": 1, "revision": 1, "created": {"type": "/type/datetime", "value": "2021-12-26T21:30:15.445616"}, "last_modified": {"type": "/type/datetime", "value": "2021-12-26T21:30:15.445616"}} +/type/author /authors/OL10000674A 1 2021-12-26T23:07:01.231950 {"type": {"key": "/type/author"}, "name": "Hannelore Berthold", "key": "/authors/OL10000674A", "source_records": ["bwb:9783850403481"], "latest_revision": 1, "revision": 1, "created": {"type": "/type/datetime", "value": "2021-12-26T23:07:01.231950"}, "last_modified": {"type": "/type/datetime", "value": "2021-12-26T23:07:01.231950"}} +/type/author /authors/OL10000702A 1 2021-12-26T23:07:39.687557 {"type": {"key": "/type/author"}, "name": "Brigitte Gabler", "key": "/authors/OL10000702A", "source_records": ["bwb:9788490156476"], "latest_revision": 1, "revision": 1, "created": {"type": "/type/datetime", "value": "2021-12-26T23:07:39.687557"}, "last_modified": {"type": "/type/datetime", "value": "2021-12-26T23:07:39.687557"}} +/type/author /authors/OL10000704A 1 2021-12-26T23:07:40.545377 {"type": {"key": "/type/author"}, "name": "Angelika Juritsch", "key": "/authors/OL10000704A", "source_records": ["bwb:9788490154434"], "latest_revision": 1, "revision": 1, "created": {"type": "/type/datetime", "value": "2021-12-26T23:07:40.545377"}, "last_modified": {"type": "/type/datetime", "value": "2021-12-26T23:07:40.545377"}} +/type/author /authors/OL10000942A 1 2021-12-26T23:36:51.351617 {"type": {"key": "/type/author"}, "name": "Chris Defilippis", "key": "/authors/OL10000942A", "source_records": ["bwb:9781440538179"], "latest_revision": 1, "revision": 1, "created": {"type": "/type/datetime", "value": "2021-12-26T23:36:51.351617"}, "last_modified": {"type": "/type/datetime", "value": "2021-12-26T23:36:51.351617"}} +/type/author /authors/OL10001074A 1 2021-12-27T00:06:27.822845 {"type": {"key": "/type/author"}, "name": "Chiara Fischer", "key": "/authors/OL10001074A", "source_records": ["bwb:9788490158265"], "latest_revision": 1, "revision": 1, "created": {"type": "/type/datetime", "value": "2021-12-27T00:06:27.822845"}, "last_modified": {"type": "/type/datetime", "value": "2021-12-27T00:06:27.822845"}} +/type/author /authors/OL10001164A 1 2021-12-27T00:12:04.103240 {"type": {"key": "/type/author"}, "name": "Vincent Padalino", "key": "/authors/OL10001164A", "source_records": ["bwb:9780989555401"], "latest_revision": 1, "revision": 1, "created": {"type": "/type/datetime", "value": "2021-12-27T00:12:04.103240"}, "last_modified": {"type": "/type/datetime", "value": "2021-12-27T00:12:04.103240"}} +/type/author /authors/OL10001200A 1 2021-12-27T00:16:19.340869 {"type": {"key": "/type/author"}, "name": "Steven T. Mann", "key": "/authors/OL10001200A", "source_records": ["bwb:9781575066950"], "latest_revision": 1, "revision": 1, "created": {"type": "/type/datetime", "value": "2021-12-27T00:16:19.340869"}, "last_modified": {"type": "/type/datetime", "value": "2021-12-27T00:16:19.340869"}} +/type/author /authors/OL10001338A 1 2021-12-27T00:50:08.345363 {"type": {"key": "/type/author"}, "name": "N. Pine", "key": "/authors/OL10001338A", "source_records": ["bwb:9781137037565"], "latest_revision": 1, "revision": 1, "created": {"type": "/type/datetime", "value": "2021-12-27T00:50:08.345363"}, "last_modified": {"type": "/type/datetime", "value": "2021-12-27T00:50:08.345363"}} +/type/author /authors/OL10001488A 1 2021-12-27T01:10:35.838074 {"type": {"key": "/type/author"}, "name": "Robert P. Sroufe", "key": "/authors/OL10001488A", "source_records": ["bwb:9781606493724"], "latest_revision": 1, "revision": 1, "created": {"type": "/type/datetime", "value": "2021-12-27T01:10:35.838074"}, "last_modified": {"type": "/type/datetime", "value": "2021-12-27T01:10:35.838074"}} +/type/author /authors/OL10001607A 1 2021-12-27T01:24:02.577747 {"type": {"key": "/type/author"}, "name": "B. M. Corrigan", "key": "/authors/OL10001607A", "source_records": ["bwb:9780802011466"], "latest_revision": 1, "revision": 1, "created": {"type": "/type/datetime", "value": "2021-12-27T01:24:02.577747"}, "last_modified": {"type": "/type/datetime", "value": "2021-12-27T01:24:02.577747"}} +/type/author /authors/OL10001802A 1 2021-12-27T01:44:34.945730 {"type": {"key": "/type/author"}, "name": "Volker Wiese", "key": "/authors/OL10001802A", "source_records": ["bwb:9783841604910"], "latest_revision": 1, "revision": 1, "created": {"type": "/type/datetime", "value": "2021-12-27T01:44:34.945730"}, "last_modified": {"type": "/type/datetime", "value": "2021-12-27T01:44:34.945730"}} +/type/author /authors/OL10002182A 1 2021-12-27T02:33:20.957118 {"type": {"key": "/type/author"}, "name": "S. J. Leinbach", "key": "/authors/OL10002182A", "source_records": ["bwb:9781466872998"], "latest_revision": 1, "revision": 1, "created": {"type": "/type/datetime", "value": "2021-12-27T02:33:20.957118"}, "last_modified": {"type": "/type/datetime", "value": "2021-12-27T02:33:20.957118"}} +/type/author /authors/OL1000222A 3 2009-05-21T20:13:18.258751 {"name": "Mu\u1e25ammad A\u1e25mad Gharab\u0101w\u012b", "created": {"type": "/type/datetime", "value": "2008-04-01T03:28:50.625462"}, "personal_name": "Mu\u1e25ammad A\u1e25mad Gharab\u0101w\u012b", "last_modified": {"type": "/type/datetime", "value": "2009-05-21T20:13:18.258751"}, "latest_revision": 3, "key": "/authors/OL1000222A", "birth_date": "1952", "type": {"key": "/type/author"}, "revision": 3} +/type/author /authors/OL1000253A 1 2008-04-01T03:28:50.625462 {"name": "\u02bbAbd Alla\u0304h ibn al-Muba\u0304rak al-Marwazi\u0304", "personal_name": "\u02bbAbd Alla\u0304h ibn al-Muba\u0304rak al-Marwazi\u0304", "death_date": "797?", "last_modified": {"type": "/type/datetime", "value": "2008-04-01T03:28:50.625462"}, "key": "/authors/OL1000253A", "birth_date": "736?", "type": {"key": "/type/author"}, "revision": 1} +/type/author /authors/OL10003193A 1 2021-12-27T04:51:46.995272 {"type": {"key": "/type/author"}, "name": "Henri Guerini", "key": "/authors/OL10003193A", "source_records": ["bwb:9782294729539"], "latest_revision": 1, "revision": 1, "created": {"type": "/type/datetime", "value": "2021-12-27T04:51:46.995272"}, "last_modified": {"type": "/type/datetime", "value": "2021-12-27T04:51:46.995272"}} +/type/author /authors/OL10003422A 1 2021-12-27T05:21:44.128473 {"type": {"key": "/type/author"}, "name": "Isabel Kainer", "key": "/authors/OL10003422A", "source_records": ["bwb:9783161533457"], "latest_revision": 1, "revision": 1, "created": {"type": "/type/datetime", "value": "2021-12-27T05:21:44.128473"}, "last_modified": {"type": "/type/datetime", "value": "2021-12-27T05:21:44.128473"}} +/type/author /authors/OL10003623A 1 2021-12-27T05:44:31.022964 {"type": {"key": "/type/author"}, "name": "Fairy Hill Publishing", "key": "/authors/OL10003623A", "source_records": ["bwb:9780997650617"], "latest_revision": 1, "revision": 1, "created": {"type": "/type/datetime", "value": "2021-12-27T05:44:31.022964"}, "last_modified": {"type": "/type/datetime", "value": "2021-12-27T05:44:31.022964"}} +/type/author /authors/OL10003948A 1 2021-12-27T06:08:49.523508 {"type": {"key": "/type/author"}, "name": "Paul Simeon Fingerote", "key": "/authors/OL10003948A", "source_records": ["bwb:9781476625478"], "latest_revision": 1, "revision": 1, "created": {"type": "/type/datetime", "value": "2021-12-27T06:08:49.523508"}, "last_modified": {"type": "/type/datetime", "value": "2021-12-27T06:08:49.523508"}} +/type/author /authors/OL10004326A 1 2021-12-27T06:51:21.848211 {"type": {"key": "/type/author"}, "name": "Magdalena Gercke", "key": "/authors/OL10004326A", "source_records": ["bwb:9783170251267"], "latest_revision": 1, "revision": 1, "created": {"type": "/type/datetime", "value": "2021-12-27T06:51:21.848211"}, "last_modified": {"type": "/type/datetime", "value": "2021-12-27T06:51:21.848211"}} +/type/author /authors/OL10004411A 1 2021-12-27T06:58:58.939245 {"type": {"key": "/type/author"}, "name": "Wanket", "key": "/authors/OL10004411A", "source_records": ["bwb:9781603812863"], "latest_revision": 1, "revision": 1, "created": {"type": "/type/datetime", "value": "2021-12-27T06:58:58.939245"}, "last_modified": {"type": "/type/datetime", "value": "2021-12-27T06:58:58.939245"}} +/type/author /authors/OL10004450A 1 2021-12-27T07:03:17.727507 {"type": {"key": "/type/author"}, "name": "Verena Roder", "key": "/authors/OL10004450A", "source_records": ["bwb:9783161542817"], "latest_revision": 1, "revision": 1, "created": {"type": "/type/datetime", "value": "2021-12-27T07:03:17.727507"}, "last_modified": {"type": "/type/datetime", "value": "2021-12-27T07:03:17.727507"}} +/type/author /authors/OL10004504A 1 2021-12-27T07:10:54.557410 {"type": {"key": "/type/author"}, "name": "Cornelia Meyer-Stoll", "key": "/authors/OL10004504A", "source_records": ["bwb:9783161530760"], "latest_revision": 1, "revision": 1, "created": {"type": "/type/datetime", "value": "2021-12-27T07:10:54.557410"}, "last_modified": {"type": "/type/datetime", "value": "2021-12-27T07:10:54.557410"}} +/type/author /authors/OL10004511A 1 2021-12-27T07:11:19.869728 {"type": {"key": "/type/author"}, "name": "William Mac Davis", "key": "/authors/OL10004511A", "source_records": ["bwb:9781581063639"], "latest_revision": 1, "revision": 1, "created": {"type": "/type/datetime", "value": "2021-12-27T07:11:19.869728"}, "last_modified": {"type": "/type/datetime", "value": "2021-12-27T07:11:19.869728"}} +/type/author /authors/OL10005003A 1 2021-12-27T08:12:58.350211 {"type": {"key": "/type/author"}, "name": "Dianna Renn", "key": "/authors/OL10005003A", "source_records": ["bwb:9781945293436"], "latest_revision": 1, "revision": 1, "created": {"type": "/type/datetime", "value": "2021-12-27T08:12:58.350211"}, "last_modified": {"type": "/type/datetime", "value": "2021-12-27T08:12:58.350211"}} +/type/author /authors/OL10005173A 1 2021-12-27T08:31:35.381507 {"type": {"key": "/type/author"}, "name": "Gerhard Wiesbeck", "key": "/authors/OL10005173A", "source_records": ["bwb:9783170239487"], "latest_revision": 1, "revision": 1, "created": {"type": "/type/datetime", "value": "2021-12-27T08:31:35.381507"}, "last_modified": {"type": "/type/datetime", "value": "2021-12-27T08:31:35.381507"}} +/type/author /authors/OL10005761A 1 2021-12-27T09:31:18.397285 {"type": {"key": "/type/author"}, "name": "Hannahe Buckmasco", "key": "/authors/OL10005761A", "source_records": ["bwb:9781093318784"], "latest_revision": 1, "revision": 1, "created": {"type": "/type/datetime", "value": "2021-12-27T09:31:18.397285"}, "last_modified": {"type": "/type/datetime", "value": "2021-12-27T09:31:18.397285"}} +/type/author /authors/OL10006166A 1 2021-12-27T10:04:27.228805 {"type": {"key": "/type/author"}, "name": "Katarina Kristinova", "key": "/authors/OL10006166A", "source_records": ["bwb:9783161558665"], "latest_revision": 1, "revision": 1, "created": {"type": "/type/datetime", "value": "2021-12-27T10:04:27.228805"}, "last_modified": {"type": "/type/datetime", "value": "2021-12-27T10:04:27.228805"}} +/type/author /authors/OL10006297A 1 2021-12-27T10:13:53.968483 {"type": {"key": "/type/author"}, "name": "Rrmoney Aeyers", "key": "/authors/OL10006297A", "source_records": ["bwb:9781097527120"], "latest_revision": 1, "revision": 1, "created": {"type": "/type/datetime", "value": "2021-12-27T10:13:53.968483"}, "last_modified": {"type": "/type/datetime", "value": "2021-12-27T10:13:53.968483"}} +/type/author /authors/OL10006601A 1 2021-12-27T10:41:12.157114 {"type": {"key": "/type/author"}, "name": "Viktor Press", "key": "/authors/OL10006601A", "source_records": ["bwb:9781952663048"], "latest_revision": 1, "revision": 1, "created": {"type": "/type/datetime", "value": "2021-12-27T10:41:12.157114"}, "last_modified": {"type": "/type/datetime", "value": "2021-12-27T10:41:12.157114"}} +/type/author /authors/OL10006738A 1 2021-12-27T10:52:30.675632 {"type": {"key": "/type/author"}, "name": "Karine Moreau-Guibert", "key": "/authors/OL10006738A", "source_records": ["bwb:9782503582931"], "latest_revision": 1, "revision": 1, "created": {"type": "/type/datetime", "value": "2021-12-27T10:52:30.675632"}, "last_modified": {"type": "/type/datetime", "value": "2021-12-27T10:52:30.675632"}} +/type/author /authors/OL10007200A 1 2021-12-27T11:38:09.616906 {"type": {"key": "/type/author"}, "name": "Jsephar Fannaei", "key": "/authors/OL10007200A", "source_records": ["bwb:9781097390700"], "latest_revision": 1, "revision": 1, "created": {"type": "/type/datetime", "value": "2021-12-27T11:38:09.616906"}, "last_modified": {"type": "/type/datetime", "value": "2021-12-27T11:38:09.616906"}} +/type/author /authors/OL10007390A 1 2021-12-27T11:58:15.759476 {"type": {"key": "/type/author"}, "name": "Urban Fruit Picking Publishing", "key": "/authors/OL10007390A", "source_records": ["bwb:9781650669533"], "latest_revision": 1, "revision": 1, "created": {"type": "/type/datetime", "value": "2021-12-27T11:58:15.759476"}, "last_modified": {"type": "/type/datetime", "value": "2021-12-27T11:58:15.759476"}} +/type/author /authors/OL10007488A 1 2021-12-27T12:06:22.056357 {"type": {"key": "/type/author"}, "name": "Silvia BL\u00c1ZQUEZ BAEZA", "key": "/authors/OL10007488A", "source_records": ["bwb:9798640157307"], "latest_revision": 1, "revision": 1, "created": {"type": "/type/datetime", "value": "2021-12-27T12:06:22.056357"}, "last_modified": {"type": "/type/datetime", "value": "2021-12-27T12:06:22.056357"}} +/type/author /authors/OL10008021A 1 2021-12-27T12:58:33.581164 {"type": {"key": "/type/author"}, "name": "Joachim Rung", "key": "/authors/OL10008021A", "source_records": ["bwb:9783161565601"], "latest_revision": 1, "revision": 1, "created": {"type": "/type/datetime", "value": "2021-12-27T12:58:33.581164"}, "last_modified": {"type": "/type/datetime", "value": "2021-12-27T12:58:33.581164"}} +/type/author /authors/OL10008028A 1 2021-12-27T12:59:14.979900 {"type": {"key": "/type/author"}, "name": "Richard L. T. Orth", "key": "/authors/OL10008028A", "source_records": ["bwb:9781620062746"], "latest_revision": 1, "revision": 1, "created": {"type": "/type/datetime", "value": "2021-12-27T12:59:14.979900"}, "last_modified": {"type": "/type/datetime", "value": "2021-12-27T12:59:14.979900"}} +/type/author /authors/OL10008453A 1 2021-12-27T13:45:24.103798 {"type": {"key": "/type/author"}, "name": "Neil Snyders", "key": "/authors/OL10008453A", "source_records": ["bwb:9780873354745"], "latest_revision": 1, "revision": 1, "created": {"type": "/type/datetime", "value": "2021-12-27T13:45:24.103798"}, "last_modified": {"type": "/type/datetime", "value": "2021-12-27T13:45:24.103798"}} +/type/author /authors/OL10008455A 1 2021-12-27T13:45:36.324837 {"type": {"key": "/type/author"}, "name": "Keith A. Mathison", "key": "/authors/OL10008455A", "source_records": ["bwb:9781642891355"], "latest_revision": 1, "revision": 1, "created": {"type": "/type/datetime", "value": "2021-12-27T13:45:36.324837"}, "last_modified": {"type": "/type/datetime", "value": "2021-12-27T13:45:36.324837"}} +/type/author /authors/OL10008821A 1 2021-12-27T14:27:36.405108 {"type": {"key": "/type/author"}, "name": "David Imperato", "key": "/authors/OL10008821A", "source_records": ["bwb:9781796320602"], "latest_revision": 1, "revision": 1, "created": {"type": "/type/datetime", "value": "2021-12-27T14:27:36.405108"}, "last_modified": {"type": "/type/datetime", "value": "2021-12-27T14:27:36.405108"}} +/type/author /authors/OL10008975A 1 2021-12-27T14:49:34.041676 {"type": {"key": "/type/author"}, "name": "Varunya Marchito", "key": "/authors/OL10008975A", "source_records": ["bwb:9781674242644"], "latest_revision": 1, "revision": 1, "created": {"type": "/type/datetime", "value": "2021-12-27T14:49:34.041676"}, "last_modified": {"type": "/type/datetime", "value": "2021-12-27T14:49:34.041676"}} +/type/author /authors/OL10009400A 1 2021-12-27T15:16:54.444272 {"type": {"key": "/type/author"}, "name": "Kodjovi M. Eklou", "key": "/authors/OL10009400A", "source_records": ["bwb:9781513559377"], "latest_revision": 1, "revision": 1, "created": {"type": "/type/datetime", "value": "2021-12-27T15:16:54.444272"}, "last_modified": {"type": "/type/datetime", "value": "2021-12-27T15:16:54.444272"}} +/type/author /authors/OL10009422A 1 2021-12-27T15:17:05.719063 {"type": {"key": "/type/author"}, "name": "Cian Ruane", "key": "/authors/OL10009422A", "source_records": ["bwb:9781513557724"], "latest_revision": 1, "revision": 1, "created": {"type": "/type/datetime", "value": "2021-12-27T15:17:05.719063"}, "last_modified": {"type": "/type/datetime", "value": "2021-12-27T15:17:05.719063"}} +/type/author /authors/OL10009491A 1 2021-12-27T15:19:26.935120 {"type": {"key": "/type/author"}, "name": "Sakai Ando", "key": "/authors/OL10009491A", "source_records": ["bwb:9781513561479"], "latest_revision": 1, "revision": 1, "created": {"type": "/type/datetime", "value": "2021-12-27T15:19:26.935120"}, "last_modified": {"type": "/type/datetime", "value": "2021-12-27T15:19:26.935120"}} +/type/author /authors/OL10009558A 3 2026-01-04T13:18:04.244649 {"key": "/authors/OL10009558A", "name": "Krishna Kumar Mohbey", "type": {"key": "/type/author"}, "source_records": ["bwb:9789811490514", "bwb:9789811490491"], "latest_revision": 3, "revision": 3, "created": {"type": "/type/datetime", "value": "2021-12-27T15:22:41.699235"}, "last_modified": {"type": "/type/datetime", "value": "2026-01-04T13:18:04.244649"}} +/type/author /authors/OL10009978A 1 2021-12-27T15:41:48.801241 {"type": {"key": "/type/author"}, "name": "Ernie Meyer", "key": "/authors/OL10009978A", "source_records": ["bwb:9781737953203"], "latest_revision": 1, "revision": 1, "created": {"type": "/type/datetime", "value": "2021-12-27T15:41:48.801241"}, "last_modified": {"type": "/type/datetime", "value": "2021-12-27T15:41:48.801241"}} +/type/author /authors/OL10010458A 1 2021-12-27T16:03:15.218542 {"type": {"key": "/type/author"}, "name": "Athina Theodoridis", "key": "/authors/OL10010458A", "source_records": ["bwb:9783828845831"], "latest_revision": 1, "revision": 1, "created": {"type": "/type/datetime", "value": "2021-12-27T16:03:15.218542"}, "last_modified": {"type": "/type/datetime", "value": "2021-12-27T16:03:15.218542"}} +/type/author /authors/OL10010466A 1 2021-12-27T16:03:27.003904 {"type": {"key": "/type/author"}, "name": "Cindy Idom", "key": "/authors/OL10010466A", "source_records": ["bwb:9781952005312"], "latest_revision": 1, "revision": 1, "created": {"type": "/type/datetime", "value": "2021-12-27T16:03:27.003904"}, "last_modified": {"type": "/type/datetime", "value": "2021-12-27T16:03:27.003904"}} +/type/author /authors/OL10010525A 1 2021-12-27T16:05:56.754290 {"type": {"key": "/type/author"}, "name": "Genevieve Verdier", "key": "/authors/OL10010525A", "source_records": ["bwb:9781513562834"], "latest_revision": 1, "revision": 1, "created": {"type": "/type/datetime", "value": "2021-12-27T16:05:56.754290"}, "last_modified": {"type": "/type/datetime", "value": "2021-12-27T16:05:56.754290"}} +/type/author /authors/OL10010669A 1 2021-12-27T16:13:50.340193 {"type": {"key": "/type/author"}, "name": "Alyassaa Perfect", "key": "/authors/OL10010669A", "source_records": ["bwb:9798708100085"], "latest_revision": 1, "revision": 1, "created": {"type": "/type/datetime", "value": "2021-12-27T16:13:50.340193"}, "last_modified": {"type": "/type/datetime", "value": "2021-12-27T16:13:50.340193"}} +/type/author /authors/OL10010838A 1 2021-12-27T16:22:57.869340 {"type": {"key": "/type/author"}, "name": "Page Publications", "key": "/authors/OL10010838A", "source_records": ["bwb:9781648331640"], "latest_revision": 1, "revision": 1, "created": {"type": "/type/datetime", "value": "2021-12-27T16:22:57.869340"}, "last_modified": {"type": "/type/datetime", "value": "2021-12-27T16:22:57.869340"}} +/type/author /authors/OL10011132A 1 2021-12-27T16:37:25.927223 {"type": {"key": "/type/author"}, "name": "Jin Way Lim", "key": "/authors/OL10011132A", "source_records": ["bwb:9781513526317"], "latest_revision": 1, "revision": 1, "created": {"type": "/type/datetime", "value": "2021-12-27T16:37:25.927223"}, "last_modified": {"type": "/type/datetime", "value": "2021-12-27T16:37:25.927223"}} +/type/author /authors/OL10011160A 1 2021-12-27T16:38:26.711854 {"type": {"key": "/type/author"}, "name": "The The Who", "key": "/authors/OL10011160A", "source_records": ["bwb:9781540029973"], "latest_revision": 1, "revision": 1, "created": {"type": "/type/datetime", "value": "2021-12-27T16:38:26.711854"}, "last_modified": {"type": "/type/datetime", "value": "2021-12-27T16:38:26.711854"}} +/type/author /authors/OL10011814A 1 2021-12-27T17:12:01.355071 {"type": {"key": "/type/author"}, "name": "James V. Mestaz", "key": "/authors/OL10011814A", "source_records": ["bwb:9781496228826"], "latest_revision": 1, "revision": 1, "created": {"type": "/type/datetime", "value": "2021-12-27T17:12:01.355071"}, "last_modified": {"type": "/type/datetime", "value": "2021-12-27T17:12:01.355071"}} +/type/author /authors/OL10012217A 1 2021-12-27T17:35:52.667142 {"type": {"key": "/type/author"}, "name": "Eric Wyckoff", "key": "/authors/OL10012217A", "source_records": ["bwb:9783161596148"], "latest_revision": 1, "revision": 1, "created": {"type": "/type/datetime", "value": "2021-12-27T17:35:52.667142"}, "last_modified": {"type": "/type/datetime", "value": "2021-12-27T17:35:52.667142"}} +/type/author /authors/OL10012292A 1 2021-12-27T17:40:15.429579 {"type": {"key": "/type/author"}, "name": "Marina Billinghurst", "key": "/authors/OL10012292A", "source_records": ["bwb:9781989819166"], "latest_revision": 1, "revision": 1, "created": {"type": "/type/datetime", "value": "2021-12-27T17:40:15.429579"}, "last_modified": {"type": "/type/datetime", "value": "2021-12-27T17:40:15.429579"}} +/type/author /authors/OL100131A 2 2008-09-08T16:19:54.382259 {"name": "Sindu Jotaryono", "personal_name": "Sindu Jotaryono", "last_modified": {"type": "/type/datetime", "value": "2008-09-08T16:19:54.382259"}, "key": "/authors/OL100131A", "type": {"key": "/type/author"}, "revision": 2} +/type/author /authors/OL1001663A 2 2008-08-20T18:02:14.485226 {"name": "Abigail Frost", "personal_name": "Abigail Frost", "last_modified": {"type": "/type/datetime", "value": "2008-08-20T18:02:14.485226"}, "key": "/authors/OL1001663A", "type": {"key": "/type/author"}, "revision": 2} +/type/author /authors/OL1001760A 2 2008-08-20T18:03:02.363531 {"name": "Peter Stupples", "personal_name": "Peter Stupples", "last_modified": {"type": "/type/datetime", "value": "2008-08-20T18:03:02.363531"}, "key": "/authors/OL1001760A", "type": {"key": "/type/author"}, "revision": 2} +/type/author /authors/OL10022776A 2 2023-07-14T15:22:43.068799 {"type": {"key": "/type/author"}, "name": "Jarrett M. Drake", "key": "/authors/OL10022776A", "source_records": ["bwb:9781634001380"], "remote_ids": {"goodreads": "22316986", "wikidata": "Q116384126"}, "alternate_names": ["Jarrett Drake", "Jarrett Martin Drake"], "latest_revision": 2, "revision": 2, "created": {"type": "/type/datetime", "value": "2021-12-28T01:27:59.160934"}, "last_modified": {"type": "/type/datetime", "value": "2023-07-14T15:22:43.068799"}} +/type/author /authors/OL1004923A 12 2025-06-12T17:04:37.435141 {"name": "Dorothea von Schlegel", "bio": {"type": "/type/text", "value": "German literary critic and writer"}, "links": [{"title": "Open Library subject", "url": "https://openlibrary.org/subjects/person:dorothea_von_schlegel_(1764-1839)", "type": {"key": "/type/link"}}], "personal_name": "Dorothea von Schlegel", "death_date": "3 August 1839", "alternate_names": ["Dorothea Schlegel", "Dorothea Von Schlegel", "Dorothea Mendelssohn Veit Schlegel"], "photos": [8445544], "birth_date": "24 October 1764", "type": {"key": "/type/author"}, "remote_ids": {"amazon": "B00JN9ELCW", "wikidata": "Q77271", "goodreads": "15907343", "librarything": "schlegeldorothea", "isni": "000000011827805X", "viaf": "95307649", "storygraph": "9c56d2d3-f242-427c-a07e-9e496935d461", "bookbrainz": "dc8be19f-32f6-4f85-9978-decaf59f5591", "gnd": "118607979", "lc_naf": "n86055038", "opac_sbn": "RAVV086752"}, "key": "/authors/OL1004923A", "latest_revision": 12, "revision": 12, "created": {"type": "/type/datetime", "value": "2008-04-01T03:28:50.625462"}, "last_modified": {"type": "/type/datetime", "value": "2025-06-12T17:04:37.435141"}} +/type/author /authors/OL100598A 3 2010-04-12T12:19:43.239002 {"name": "Syamsumar Dam", "personal_name": "Syamsumar Dam", "photos": [6286475], "last_modified": {"type": "/type/datetime", "value": "2010-04-12T12:19:43.239002"}, "latest_revision": 3, "key": "/authors/OL100598A", "type": {"key": "/type/author"}, "revision": 3} +/type/author /authors/OL10075763A 4 2023-09-28T16:14:41.000102 {"type": {"key": "/type/author"}, "name": "Tariro Ndoro", "key": "/authors/OL10075763A", "source_records": ["bwb:9781928215769"], "photos": [14375097], "remote_ids": {"viaf": "34156809515545121342", "storygraph": "fb7de527-38f9-4155-94e2-169be224e0d7", "librarything": "ndorotariro", "isni": "0000000500680162", "goodreads": "19411902"}, "latest_revision": 4, "revision": 4, "created": {"type": "/type/datetime", "value": "2021-12-30T10:43:27.697337"}, "last_modified": {"type": "/type/datetime", "value": "2023-09-28T16:14:41.000102"}} +/type/author /authors/OL101163A 3 2010-12-01T00:02:02.977151 {"name": "S. Akbar Zaidi", "personal_name": "S. Akbar Zaidi", "created": {"type": "/type/datetime", "value": "2008-04-01T03:28:50.625462"}, "photos": [6671475], "last_modified": {"type": "/type/datetime", "value": "2010-12-01T00:02:02.977151"}, "latest_revision": 3, "key": "/authors/OL101163A", "type": {"key": "/type/author"}, "revision": 3} +/type/author /authors/OL101261A 7 2025-08-03T21:06:47.732379 {"bio": {"type": "/type/text", "value": "Gulam Hassan Lobsang is a well known literary person in Baltistan region, northern part of Pakistan. He started his literary work during his government service and done remarkable contribution to the culture and language of Baltistan as well as Tibetian region.His great service for Baltistan is the \"Balti Gramer\" which had not been written before. There are many other books like \"Taarekh-e Bon falsafa\",\" Balti- English Gramer\", \" Yulstroong\" \"Aoot,\" \"Mimang Rgiastrit\" and other short fictions."}, "personal_name": "Ghulam Hassan Lobsang", "key": "/authors/OL101261A", "name": "Ghulam Hassan Lobsang", "birth_date": "1955", "type": {"key": "/type/author"}, "remote_ids": {"wikidata": "Q5557583"}, "latest_revision": 7, "revision": 7, "created": {"type": "/type/datetime", "value": "2008-04-01T03:28:50.625462"}, "last_modified": {"type": "/type/datetime", "value": "2025-08-03T21:06:47.732379"}} +/type/author /authors/OL101346A 3 2012-06-06T23:56:54.837573 {"created": {"type": "/type/datetime", "value": "2008-04-01T03:28:50.625462"}, "latest_revision": 3, "name": "Ibn-i \u1e24asan J\u0101rcav\u012b", "key": "/authors/OL101346A", "personal_name": "Ibn-i \u1e24asan J\u0101rcav\u012b", "birth_date": "1904", "death_date": "1973", "type": {"key": "/type/author"}, "last_modified": {"type": "/type/datetime", "value": "2012-06-06T23:56:54.837573"}, "revision": 3} +/type/author /authors/OL10142526A 3 2024-09-11T13:59:15.590863 {"source_records": ["amazon:1638147434", "bwb:9781638147459"], "key": "/authors/OL10142526A", "name": "Stephen C. Webb", "type": {"key": "/type/author"}, "alternate_names": ["Stephen C Webb"], "latest_revision": 3, "revision": 3, "created": {"type": "/type/datetime", "value": "2022-01-19T07:50:43.237774"}, "last_modified": {"type": "/type/datetime", "value": "2024-09-11T13:59:15.590863"}} +/type/author /authors/OL10145311A 1 2022-01-25T23:23:33.104345 {"type": {"key": "/type/author"}, "name": "Sylvain Ledda", "personal_name": "Sylvain Ledda", "birth_date": "(1971", "death_date": "...)", "key": "/authors/OL10145311A", "source_records": ["ia:vignyuneironiero0000unse"], "latest_revision": 1, "revision": 1, "created": {"type": "/type/datetime", "value": "2022-01-25T23:23:33.104345"}, "last_modified": {"type": "/type/datetime", "value": "2022-01-25T23:23:33.104345"}} +/type/author /authors/OL10295793A 3 2025-07-10T17:31:10.747821 {"type": {"key": "/type/author"}, "name": "David Marshall Hunt", "key": "/authors/OL10295793A", "source_records": ["bwb:9780989468701"], "photos": [15101920], "bio": {"type": "/type/text", "value": "David Marshall Hunt, PhD. My life of international adventures began when I served in the Air Force in the War in Viet Nam from 1964-65. Then came 40 years of business consulting, research, and teaching at universities around the world. I have experienced life in many cultures and learned about legends and myths, stories and events in exotic and sometimes dangerous places. It is my pleasure to share and discuss these exciting and fascinating tales in novel form with people from all nations and cultures.-Amazon"}, "birth_date": "1950", "alternate_names": ["Hunt, David (David Marshall)", "Hunt, David 1950-"], "remote_ids": {"viaf": "267354706", "amazon": "B00I3LOKWM"}, "latest_revision": 3, "revision": 3, "created": {"type": "/type/datetime", "value": "2022-03-02T14:55:20.130308"}, "last_modified": {"type": "/type/datetime", "value": "2025-07-10T17:31:10.747821"}} +/type/author /authors/OL1029882A 6 2025-06-12T17:07:34.832349 {"name": "Siegfried Bernfeld", "personal_name": "Siegfried Bernfeld", "remote_ids": {"viaf": "17341371", "wikidata": "Q23914", "isni": "0000000116006456", "lc_naf": "n83222657"}, "death_date": "1953", "birth_date": "1892", "type": {"key": "/type/author"}, "key": "/authors/OL1029882A", "latest_revision": 6, "revision": 6, "created": {"type": "/type/datetime", "value": "2008-04-01T03:28:50.625462"}, "last_modified": {"type": "/type/datetime", "value": "2025-06-12T17:07:34.832349"}} +/type/author /authors/OL10335663A 2 2022-03-07T11:07:44.941117 {"name": "Helen M. Urban", "key": "/authors/OL10335663A", "type": {"key": "/type/author"}, "death_date": "13 February 2003", "birth_date": "23 January 1915", "alternate_names": ["Helen Urban"], "latest_revision": 2, "revision": 2, "created": {"type": "/type/datetime", "value": "2022-03-07T11:06:39.499666"}, "last_modified": {"type": "/type/datetime", "value": "2022-03-07T11:07:44.941117"}} +/type/author /authors/OL10341344A 1 2022-03-22T04:55:48.162019 {"type": {"key": "/type/author"}, "name": "Jean-Pierre \u00c9nard", "personal_name": "Jean-Pierre \u00c9nard", "birth_date": "1943", "death_date": "1987", "key": "/authors/OL10341344A", "source_records": ["ia:envacancesalafer0000enar"], "latest_revision": 1, "revision": 1, "created": {"type": "/type/datetime", "value": "2022-03-22T04:55:48.162019"}, "last_modified": {"type": "/type/datetime", "value": "2022-03-22T04:55:48.162019"}} +/type/author /authors/OL10355109A 2 2025-01-23T17:05:45.758781 {"type": {"key": "/type/author"}, "name": "Franziska Davies", "key": "/authors/OL10355109A", "source_records": ["amazon:3525373171"], "birth_date": "1984", "remote_ids": {"wikidata": "Q113835702", "viaf": "55145067050666630364", "isni": "0000000459560731"}, "latest_revision": 2, "revision": 2, "created": {"type": "/type/datetime", "value": "2022-05-12T05:17:27.079904"}, "last_modified": {"type": "/type/datetime", "value": "2025-01-23T17:05:45.758781"}} +/type/author /authors/OL1043275A 6 2025-06-12T17:09:43.560571 {"name": "Ivan Alekseevich Kuratov", "personal_name": "Ivan Alekseevich Kuratov", "remote_ids": {"viaf": "13107815", "wikidata": "Q2366812", "isni": "000000005522765X", "lc_naf": "n81076432"}, "death_date": "1875", "birth_date": "1839", "type": {"key": "/type/author"}, "key": "/authors/OL1043275A", "latest_revision": 6, "revision": 6, "created": {"type": "/type/datetime", "value": "2008-04-01T03:28:50.625462"}, "last_modified": {"type": "/type/datetime", "value": "2025-06-12T17:09:43.560571"}} +/type/author /authors/OL1044450A 3 2010-07-29T10:36:12.669320 {"bio": {"type": "/type/text", "value": "Leiter der Wortdokumentation im Deutschen Rundfunkarchiv"}, "name": "Walter Roller", "personal_name": "Walter Roller", "created": {"type": "/type/datetime", "value": "2008-04-01T03:28:50.625462"}, "last_modified": {"type": "/type/datetime", "value": "2010-07-29T10:36:12.669320"}, "latest_revision": 3, "key": "/authors/OL1044450A", "type": {"key": "/type/author"}, "revision": 3} +/type/author /authors/OL10458357A 3 2022-06-19T09:39:42.620886 {"name": "Andrew Ford (Apache)", "key": "/authors/OL10458357A", "type": {"key": "/type/author"}, "photos": [12794804], "links": [{"title": "O'Reilly", "url": "https://www.oreilly.com/pub/au/571", "type": {"key": "/type/link"}}], "bio": {"type": "/type/text", "value": "Andrew Ford is an author and a software engineer employed by Symantec (formerly MessageLabs), having previously worked as an independent software consultant in both Great Britain and Germany. He has been involved with the Web since its early days, and his book Spinning the Web was the first on publishing with the Web. Andrew wrote the cronolog web server log file rotation program."}, "latest_revision": 3, "revision": 3, "created": {"type": "/type/datetime", "value": "2022-06-18T10:21:58.387129"}, "last_modified": {"type": "/type/datetime", "value": "2022-06-19T09:39:42.620886"}} +/type/author /authors/OL105178A 6 2025-08-01T04:36:55.215600 {"name": "Pramodakum\u0101ra Pa\u1e6dela", "personal_name": "Pramodakum\u0101ra Pa\u1e6dela", "birth_date": "1933", "type": {"key": "/type/author"}, "remote_ids": {"viaf": "38188184", "wikidata": "Q2764976", "isni": "000000002482112X", "lc_naf": "n79028983"}, "key": "/authors/OL105178A", "latest_revision": 6, "revision": 6, "created": {"type": "/type/datetime", "value": "2008-04-01T03:28:50.625462"}, "last_modified": {"type": "/type/datetime", "value": "2025-08-01T04:36:55.215600"}} +/type/author /authors/OL1053744A 6 2025-06-12T17:11:11.217531 {"name": "Alfredo Stroessner", "personal_name": "Alfredo Stroessner", "remote_ids": {"viaf": "107542263", "wikidata": "Q152534", "isni": "0000000114796212", "lc_naf": "n79148893"}, "death_date": "2006", "birth_date": "1912", "type": {"key": "/type/author"}, "key": "/authors/OL1053744A", "latest_revision": 6, "revision": 6, "created": {"type": "/type/datetime", "value": "2008-04-01T03:28:50.625462"}, "last_modified": {"type": "/type/datetime", "value": "2025-06-12T17:11:11.217531"}} +/type/author /authors/OL1056003A 6 2025-10-30T17:10:55.999786 {"name": "Franc Malavas\u030cic\u030c", "personal_name": "Franc Malavas\u030cic\u030c", "remote_ids": {"viaf": "40209828", "wikidata": "Q17379515", "isni": "0000000028876611", "lc_naf": "n85257891", "opac_sbn": "TSAV267481"}, "death_date": "1863", "birth_date": "1818", "type": {"key": "/type/author"}, "key": "/authors/OL1056003A", "latest_revision": 6, "revision": 6, "created": {"type": "/type/datetime", "value": "2008-04-01T03:28:50.625462"}, "last_modified": {"type": "/type/datetime", "value": "2025-10-30T17:10:55.999786"}} +/type/author /authors/OL10602438A 4 2024-11-14T20:26:48.740991 {"name": "Ingmar Hagemann", "key": "/authors/OL10602438A", "type": {"key": "/type/author"}, "photos": [12855130], "remote_ids": {"wikidata": "Q113457513", "viaf": "4338148632940730630008", "bookbrainz": "0c446d1e-f3ca-4208-bc8c-bae1c7d3a9ec"}, "bio": "German political scientist", "birth_date": "1981", "latest_revision": 4, "revision": 4, "created": {"type": "/type/datetime", "value": "2022-08-05T19:10:45.166110"}, "last_modified": {"type": "/type/datetime", "value": "2024-11-14T20:26:48.740991"}} +/type/author /authors/OL1060272A 6 2025-06-12T17:11:59.231881 {"name": "Hans Blum", "bio": {"type": "/type/text", "value": "German politician, Reichstag member, historian of 19th century Germany, cigar manufacturer"}, "links": [{"url": "https://viaf.org/viaf/56700841", "type": {"key": "/type/link"}, "title": "VIAF"}], "personal_name": "Blum, Hans", "death_date": "1910", "alternate_names": ["Hans Blum", "Johann Georg Max Hans Blum"], "birth_date": "1841", "type": {"key": "/type/author"}, "remote_ids": {"viaf": "56700841", "wikidata": "Q73975", "isni": "0000000116485856", "lc_naf": "n86832132", "project_gutenberg": "44416", "opac_sbn": "SBNV032200"}, "key": "/authors/OL1060272A", "latest_revision": 6, "revision": 6, "created": {"type": "/type/datetime", "value": "2008-04-01T03:28:50.625462"}, "last_modified": {"type": "/type/datetime", "value": "2025-06-12T17:11:59.231881"}} +/type/author /authors/OL10603660A 4 2022-08-15T21:04:42.375144 {"name": "Paul Langley", "key": "/authors/OL10603660A", "type": {"key": "/type/author"}, "birth_date": "1954", "alternate_names": ["Paul A. Langley"], "bio": "Mathematics teacher.", "remote_ids": {"isni": "0000000051613211", "viaf": "2404160668352403560000"}, "latest_revision": 4, "revision": 4, "created": {"type": "/type/datetime", "value": "2022-08-15T20:27:09.541101"}, "last_modified": {"type": "/type/datetime", "value": "2022-08-15T21:04:42.375144"}} +/type/author /authors/OL1063068A 4 2025-06-12T17:12:52.325589 {"name": "Desfontaines M.", "personal_name": "Desfontaines", "death_date": "1825", "birth_date": "1733", "title": "M.", "type": {"key": "/type/author"}, "remote_ids": {"viaf": "4928943", "wikidata": "Q3024383", "isni": "0000000120987728", "musicbrainz": "2e1cf841-1650-4fa9-a7a1-afe1f5be48b3", "lc_naf": "n90714226"}, "key": "/authors/OL1063068A", "latest_revision": 4, "revision": 4, "created": {"type": "/type/datetime", "value": "2008-04-01T03:28:50.625462"}, "last_modified": {"type": "/type/datetime", "value": "2025-06-12T17:12:52.325589"}} +/type/author /authors/OL106819A 3 2010-04-12T12:19:43.239002 {"name": "B. K. Keayla", "personal_name": "B. K. Keayla", "photos": [5722800], "last_modified": {"type": "/type/datetime", "value": "2010-04-12T12:19:43.239002"}, "latest_revision": 3, "key": "/authors/OL106819A", "type": {"key": "/type/author"}, "revision": 3} +/type/author /authors/OL1076363A 5 2011-03-29T14:26:52.266402 {"bio": {"type": "/type/text", "value": "Inhibitors of Cell Growth (Progress in Molecular and Subcellular Biology)\r\n1 edition - first published in 1998"}, "personal_name": "Alvaro Macieira-Coelho", "photos": [6716042, 6686930], "last_modified": {"type": "/type/datetime", "value": "2011-03-29T14:26:52.266402"}, "latest_revision": 5, "key": "/authors/OL1076363A", "name": "Alvaro Macieira-Coelho", "created": {"type": "/type/datetime", "value": "2008-04-01T03:28:50.625462"}, "birth_date": "1932", "type": {"key": "/type/author"}, "revision": 5} +/type/author /authors/OL10768430A 3 2024-06-18T04:03:10.603634 {"type": {"key": "/type/author"}, "name": "Drew B. David", "key": "/authors/OL10768430A", "source_records": ["bwb:9781387176045"], "photos": [14637036], "bio": {"type": "/type/text", "value": "Practitioner of Poesy, Spinner of Visual Yarns, Anarcho-Chameleon, Inveterate Loser, Primal Ruffian, Angry Old Man, Anti-Poet"}, "remote_ids": {"amazon": "B01MFG126A"}, "latest_revision": 3, "revision": 3, "created": {"type": "/type/datetime", "value": "2022-10-19T00:05:33.996376"}, "last_modified": {"type": "/type/datetime", "value": "2024-06-18T04:03:10.603634"}} +/type/author /authors/OL10977A 4 2022-09-28T17:19:00.185801 {"type": {"key": "/type/author"}, "name": "R. Rabindranath Menon", "title": "1927-.\u00b7", "personal_name": "R. Rabindranath Menon", "key": "/authors/OL10977A", "alternate_names": ["Menon, R. Rabindranath 1927-.\u00b7"], "birth_date": "1927", "latest_revision": 4, "revision": 4, "created": {"type": "/type/datetime", "value": "2008-04-01T03:28:50.625462"}, "last_modified": {"type": "/type/datetime", "value": "2022-09-28T17:19:00.185801"}} +/type/author /authors/OL1100412A 8 2025-10-30T17:15:26.631958 {"name": "Charles Harvard Gibbs-Smith", "personal_name": "Charles Harvard Gibbs-Smith", "death_date": "1981", "remote_ids": {"viaf": "107639634", "wikidata": "Q1064665", "isni": "0000000122142737", "lc_naf": "n50028207"}, "alternate_names": ["Charles H. Gibbs-Smith", "C. Gibbs-Smith", "C. H. Gibbs-Smith", "C.H Gibbs-Smith", "Gibbs-Smith, Charles Harvard", "Charles Gibbs-Smith", "Charles Harvard 1909- Gibbs-Smith", "Gibbs-Smith. Charles H", "C.H. GIBBS-SMITH", "Charles Harvard GIBBS-SMITH", "Charles Harvard 1909-1981 Gibbs-Smith", "Charles H Gibbs-Smith"], "type": {"key": "/type/author"}, "birth_date": "1909", "source_records": ["amazon:1014230055", "promise:bwb_daily_pallets_2022-03-17", "promise:bwb_daily_pallets_2021-08-13", "bwb:9781015228252", "promise:bwb_daily_pallets_2022-11-01:KR-467-113"], "key": "/authors/OL1100412A", "latest_revision": 8, "revision": 8, "created": {"type": "/type/datetime", "value": "2008-04-01T03:28:50.625462"}, "last_modified": {"type": "/type/datetime", "value": "2025-10-30T17:15:26.631958"}} +/type/author /authors/OL1130070A 8 2025-06-12T17:21:43.931543 {"name": "Eastman Kodak Company", "links": [{"url": "http://www.kodak.com/", "type": {"key": "/type/link"}, "title": "Kodak"}, {"url": "https://en.wikipedia.org/wiki/Eastman_Kodak", "type": {"key": "/type/link"}, "title": "Wikipedia"}], "personal_name": "Eastman Kodak.", "type": {"key": "/type/author"}, "alternate_names": ["Eastman Kodak", "EASTMAN KODAK COMPANY.", "Eastman Kodak.", "Eastman Kodak Company.", "The Eastman Kodak Company", "EASTMAN KODAK.", "Eastman Kodak Co.", "Eastman Kodak Co", "Kodak.", "Kodak", "Kodak Limited.", "KODAK", "Eastman Kodak Company staff", "Eastman Kodak Staff", "Eastman Kodak Company Staff", "Eastman Kodak Company. Professional and Finishing Markets Division.", "Canadian Kodak Company.", "Eastman Kodak Company. Radiography Markets Division.", "Eastman Kodak Company. Graphics Markets Division.", "Kodak Eastman", "Eastman Kodak Company. Medical Division.", "Kodak (Firm)", "Eastman Kodak Company. Advertising Information Service.", "KODAK.", "Eastman Kodak Company. Professional, Commercial and Industrial Markets Division.", "Eastman Kodak Company. X-ray Division", "Eastman Kodak Company. Research Laboratory.", "Kodak Microelectronics Seminar (1981 Dallas, Tex.)", "Eastman Kodak Company. Motion Picture and Audiovisual Markets Division.", "Eastman Kodak Company. Medical Division", "Eastman Kodak Company. X-ray Division.", "Kodak Unknown", "Eastman Kodak Editors", "Kodak Seminar : Aerial Photography as a Planning Tool (1973 Rochester and Ithaca, N.Y.)", "Eastman Kodak Company. Distillation Products Industries.", "Eastman Kodak Company. Professional, Commercial, and Industrial Markets Division.", "Kodak Microelectronics Seminar (1977 Monterey, Calif.)", "Kodak Limited. Medical Sales Division.", "Eastman Kodak Company. Motion Picture Film Dept.", "Eastman Kodak Company. U.S. Apparatus Division.", "Eastman Kodak Company. Motion Picture and Audiovisual Markets Division", "Eastman Kodak Company. Motion Picture and Education Markets Division", "Eastman Kodak Company. Graphic Arts Division.", "KODAK LIMITED.", "Kodak. Museum.", "Eastman Kodak Company. Consumer Markets Division.", "Eastman Kodak Company. X-ray Sales Division.", "Kodak Ekta AF-1", "Kodak EKTA FF", "Kodak Carousel 4600", "Eastman Kodak Company, Rochester, N.Y. Kodak Research Laboratories.", "Canadian Kodak Company", "Eastman Kodak Company. Professional Photography Division", "Eastman Kodak Company. professional and Finishing Markets Division.", "eastman kodak co.", "Eastman Kodak, Co."], "entity_type": "org", "remote_ids": {"wikidata": "Q486269", "isni": "0000000123648975", "viaf": "156056348", "goodreads": "194433", "lc_naf": "n80126274"}, "key": "/authors/OL1130070A", "latest_revision": 8, "revision": 8, "created": {"type": "/type/datetime", "value": "2008-04-01T03:28:50.625462"}, "last_modified": {"type": "/type/datetime", "value": "2025-06-12T17:21:43.931543"}} +/type/author /authors/OL1153625A 7 2025-10-30T17:22:16.175401 {"name": "Sam Aaronovitch", "bio": {"type": "/type/text", "value": "Economist, academic"}, "links": [{"url": "https://en.wikipedia.org/wiki/Sam_Aaronovitch", "type": {"key": "/type/link"}, "title": "Wikipedia"}], "personal_name": "Sam Aaronovitch", "death_date": "30 May 1998", "birth_date": "26 December 1919", "type": {"key": "/type/author"}, "remote_ids": {"viaf": "94508988", "wikidata": "Q3378851", "isni": "0000000109993801", "lc_naf": "n81068579", "goodreads": "1056858"}, "key": "/authors/OL1153625A", "latest_revision": 7, "revision": 7, "created": {"type": "/type/datetime", "value": "2008-04-01T03:28:50.625462"}, "last_modified": {"type": "/type/datetime", "value": "2025-10-30T17:22:16.175401"}} +/type/author /authors/OL119301A 11 2025-06-12T17:38:59.860282 {"name": "Charles P\u00e9guy", "bio": {"type": "/type/text", "value": "French poet, essayist, and editor"}, "links": [{"url": "https://en.wikipedia.org/wiki/Charles_P%C3%A9guy", "type": {"key": "/type/link"}, "title": "Wikipedia"}, {"url": "https://www.gutenberg.org/ebooks/author/49150", "type": {"key": "/type/link"}, "title": "Project Gutenberg"}], "personal_name": "Charles P\u00e9guy", "death_date": "5 September 1914", "alternate_names": ["Charles Pierre P\u00e9guy", "Charles-Pierre P\u00e9guy", "Charles Pierre Pe\u0301guy", "Charles Peguy", "Charles-Pierre Peguy", "Charles Pierre Peguy"], "photos": [8257125], "birth_date": "7 January 1873", "type": {"key": "/type/author"}, "remote_ids": {"viaf": "7395351", "wikidata": "Q334965", "isni": "000000012276072X", "musicbrainz": "a7bf17bd-8fbd-4b45-b40c-9f0abbefe899", "goodreads": "5449971", "imdb": "nm6772473", "lc_naf": "n80038479", "librarything": "pguycharles", "librivox": "8482", "project_gutenberg": "49150", "opac_sbn": "CFIV032413"}, "key": "/authors/OL119301A", "latest_revision": 11, "revision": 11, "created": {"type": "/type/datetime", "value": "2008-04-01T03:28:50.625462"}, "last_modified": {"type": "/type/datetime", "value": "2025-06-12T17:38:59.860282"}} +/type/author /authors/OL1206233A 9 2024-04-22T23:13:38.003272 {"name": "Camilo Taufic", "bio": {"type": "/type/text", "value": "\"Chile en la Hoguera 1973 - Instant\u00e1nea del Golpe Militar\", Ed. Cesoc-ChileAm\u00e9rica, 2003.\r\n\r\n\"Manual de \u00c9tica Period\u00edstica Comparada - La Autorregulaci\u00f3n del Periodismo\", Ed. Fucatel, 2005. \r\n\r\n\"Un extraterrestre en La Moneda\" - Y otras cr\u00f3nicas asombrosas. Editorial Planeta, Santiago de Chile, 2009."}, "personal_name": "Camilo Taufic", "photos": [6274294], "type": {"key": "/type/author"}, "links": [{"url": "https://www.editorialplaneta.cl", "title": "Website", "type": {"key": "/type/link"}}], "key": "/authors/OL1206233A", "latest_revision": 9, "revision": 9, "created": {"type": "/type/datetime", "value": "2008-04-01T03:28:50.625462"}, "last_modified": {"type": "/type/datetime", "value": "2024-04-22T23:13:38.003272"}} +/type/author /authors/OL12196704A 1 2023-01-14T04:25:20.356703 {"type": {"key": "/type/author"}, "key": "/authors/OL12196704A", "source_records": ["promise:bwb_daily_pallets_2022-08-13"], "latest_revision": 1, "revision": 1, "created": {"type": "/type/datetime", "value": "2023-01-14T04:25:20.356703"}, "last_modified": {"type": "/type/datetime", "value": "2023-01-14T04:25:20.356703"}} +/type/author /authors/OL12214973A 1 2023-01-14T08:59:44.827616 {"type": {"key": "/type/author"}, "key": "/authors/OL12214973A", "source_records": ["promise:bwb_daily_pallets_2022-03-17"], "latest_revision": 1, "revision": 1, "created": {"type": "/type/datetime", "value": "2023-01-14T08:59:44.827616"}, "last_modified": {"type": "/type/datetime", "value": "2023-01-14T08:59:44.827616"}} +/type/author /authors/OL12222578A 1 2023-01-14T11:11:47.458218 {"type": {"key": "/type/author"}, "key": "/authors/OL12222578A", "source_records": ["promise:bwb_daily_pallets_2022-03-17"], "latest_revision": 1, "revision": 1, "created": {"type": "/type/datetime", "value": "2023-01-14T11:11:47.458218"}, "last_modified": {"type": "/type/datetime", "value": "2023-01-14T11:11:47.458218"}} +/type/author /authors/OL1261312A 3 2026-05-22T15:22:09.075751 {"key": "/authors/OL1261312A", "type": {"key": "/type/author"}, "entity_type": "org", "source_records": ["marc:marc_columbia/Columbia-extract-20221130-008.mrc:552810359:1227", "marc:harvard_bibliographic_metadata/ab.bib.11.20150123.full.mrc:156314084:515", "marc:harvard_bibliographic_metadata/ab.bib.00.20150123.full.mrc:272580472:655", "marc:harvard_bibliographic_metadata/ab.bib.13.20150123.full.mrc:565083854:1951", "marc:harvard_bibliographic_metadata/ab.bib.01.20150123.full.mrc:665727382:901", "marc:harvard_bibliographic_metadata/ab.bib.01.20150123.full.mrc:852107146:572", "marc:harvard_bibliographic_metadata/20220215_001.bib.mrc:246154479:1440", "ia:b30592148", "marc:harvard_bibliographic_metadata/20220215_029.bib.mrc:307241378:3290", "marc:harvard_bibliographic_metadata/20220215_036.bib.mrc:147815030:2022", "marc:UniversityOfMichiganMarcCatalogRecords/umich_bib.mrc:350854477:1214"], "name": "Bayerische Julius-Maximilians-Universita\u0308t Wu\u0308rzburg.", "alternate_names": ["Bayerische Julius-Maximilians-Universitat Wurzburg", "Bayerische Julius-Maximilians-Universita\u0308t Wu\u0308rzburg. Geographisches Institut.", "Bayerische Julius Maximilians-Universitat Wurzburg", "Bayerische Julius-Maximilians-Universit\u00e4t W\u00fcrzburg.", "Bayerische Julius-Maximilians-Universit\u00e4t W\u00fcrzburg. Anatomisches Institut", "Bayerische Julius-Maximilians-Universit\u00e4t W\u00fcrzburg", "BAYERISCHE JULIUS-MAXIMILIANS-UNIVERSIT\u00c4T W\u00dcRZBURG", "Bayerische Julius-Maximilians-Universit\u00e4t W\u00fcrzburg. Institut f\u00fcr Notarrecht", "Bayerische Julius-Maximilians-Universit\u00e4t W\u00fcrzburg. Fakult\u00e4t f\u00fcr Rechts- und Staatswissenschaftlichen", "Bayerische Julius-Maximilians-Universit\u00e4t W\u00fcrzburg. Medizinische Fakult\u00e4t", "Bayerische Julius-Maximilians-Universit\u00e4t W\u00fcrzburg. Rechts- und Staatswissenschaftliche Fakult\u00e4t", "Bayerische Julius-Maximilians-Universit\u00e4t W\u00fcrzburg. Physikalisches Institut", "Bayerische Julius-Maximilians-Universit\u00e4t W\u00fcrzburg. Physiologisches Institut", "Bayerische Julius-Maximilians-Universit\u00e4t W\u00fcrzburg. Klinik f\u00fcr Haut- und Geschlechtskrankheiten"], "latest_revision": 3, "revision": 3, "created": {"type": "/type/datetime", "value": "2008-04-01T03:28:50.625462"}, "last_modified": {"type": "/type/datetime", "value": "2026-05-22T15:22:09.075751"}} +/type/author /authors/OL12652889A 5 2025-09-28T07:00:58.631337 {"type": {"key": "/type/author"}, "name": "Ana Mar\u00eda Mart\u00ednez Sagi", "key": "/authors/OL12652889A", "source_records": ["amazon:8416950180"], "photos": [15126880], "bio": "Ana Mar\u00eda Mart\u00ednez Sagi (Barcelona, 16 de febrero de 1907-Sampedor, 2 de enero de 2000) fue una poeta, sindicalista, periodista, feminista y atleta espa\u00f1ola, campeona de Espa\u00f1a en lanzamiento de jabalina, plusmarquista y pionera del deporte femenino espa\u00f1ol.\r\n\r\n\r\n----------\r\n\r\n\r\nAna Mar\u00eda Mart\u00ednez Sagi (Barcelona, \u200b\u200bFebruary 16, 1907 - Sampedor, January 2, 2000) was a Spanish poet, trade unionist, journalist, feminist and athlete, Spanish champion in javelin throw, record holder and pioneer of Spanish women's sport.", "birth_date": "1907-02-16", "death_date": "2000-01-02", "remote_ids": {"wikidata": "Q8198059", "viaf": "87168362", "isni": "0000000095418442", "lc_naf": "no00067264", "gnd": "123783178"}, "links": [{"title": "Biblioteca Nacional de Espa\u00f1a", "url": "https://datos.bne.es/resource/XX1409358", "type": {"key": "/type/link"}}], "latest_revision": 5, "revision": 5, "created": {"type": "/type/datetime", "value": "2023-04-14T08:29:18.105826"}, "last_modified": {"type": "/type/datetime", "value": "2025-09-28T07:00:58.631337"}} +/type/author /authors/OL14191950A 3 2025-09-14T17:53:44.620024 {"type": {"key": "/type/author"}, "name": "Liping He", "personal_name": "Liping He", "birth_date": "1958", "key": "/authors/OL14191950A", "source_records": ["marc:harvard_bibliographic_metadata/ab.bib.13.20150123.full.mrc:699113167:1376"], "bio": "\u8d3a\u529b\u5e73\uff0c\u5317\u4eac\u5e08\u8303\u5927\u5b66\u7ecf\u6d4e\u4e0e\u5de5\u5546\u7ba1\u7406\u5b66\u9662\u91d1\u878d\u7cfb\u6559\u6388\uff0c\u5b66\u672f\u517c\u804c\u5305\u62ec\u4e2d\u56fd\u4e16\u754c\u7ecf\u6d4e\u5b66\u4f1a\u526f\u4f1a\u957f\uff0c\u4e2d\u56fd\u56fd\u9645\u91d1\u878d\u5b66\u4f1a\u5e38\u52a1\u7406\u4e8b\u517c\u5b66\u672f\u59d4\u5458\u4f1a\u59d4\u5458\uff1b\u4e2d\u56fd\u793e\u4f1a\u79d1\u5b66\u9662\u4e16\u754c\u7ecf\u6d4e\u4e0e\u653f\u6cbb\u7814\u7a76\u6240\u5b66\u672f\u59d4\u5458\u4f1a\u59d4\u5458\u30021982\u5e74\u5f00\u542f\u9ad8\u6821\u6559\u5e08\u751f\u6daf\uff0c2001\u5e74\u81f3\u4eca\u5728\u5317\u5e08\u5927\u4efb\u6559\uff0c\u4e3a\u5317\u5e08\u5927\u5b66\u5b50\u4eec\u5f00\u8bbe\u4e86\u300a\u91d1\u878d\u5b66\u57fa\u7840\u300b\uff08\u53cc\u8bed\uff09\u8bfe\u7a0b\u548c\u300a\u56fd\u9645\u91d1\u878d\u4e0e\u4e2d\u56fd\u300b\uff08\u82f1\u8bed\uff09\u56fd\u9645\u7559\u5b66\u751f\uff08\u7814\u7a76\u751f\uff09\u8bfe\u7a0b\uff0c\u4e24\u95e8\u8bfe\u7a0b\u5148\u540e\u88ab\u8bc4\u9009\u4e3a\u6559\u80b2\u90e82007\u5e74\u9996\u6279\u53cc\u8bed\u6559\u5b66\u793a\u8303\u8bfe\u7a0b\u53ca2013\u5e74\u6765\u534e\u7559\u5b66\u82f1\u8bed\u6388\u8bfe\u54c1\u724c\u8bfe\u7a0b\u3002\u4e3b\u8981\u7814\u7a76\u65b9\u5411\u4e3a\u4e16\u754c\u7ecf\u6d4e\u548c\u56fd\u9645\u91d1\u878d\u3002\u8457\u6709\u300a\u4e16\u754c\u91d1\u878d\u53f2\uff1a\u4ece\u8d77\u6e90\u5230\u73b0\u4ee3\u4f53\u7cfb\u7684\u5f62\u6210\u300b\uff08\u4e2d\u56fd\u91d1\u878d\u51fa\u7248\u793e\uff0c2022\u5e74\uff09\u548cHyperinflation: A World History (Routledge, 2018)\u7b49\u3002", "links": [{"title": "\u3010\u6559\u5e08\u98ce\u91c7\u3011| \u8d3a\u529b\u5e73\uff1a\u865a\u6000\u82e5\u8c37\u5bb9\u5929\u4e0b\uff0c\u6c42\u77e5\u82e5\u6e34\u7814\u516b\u65b9", "url": "https://bs.bnu.edu.cn/ssfc/jgrw_20200903103607846601/248900.html", "type": {"key": "/type/link"}}, {"title": "\u5e7f\u4e1c\u4eba\u6587\u793e\u79d1\u7f51 | \u8d3a\u529b\u5e73", "url": "https://www.gdskl.com.cn/news_2067.shtml", "type": {"key": "/type/link"}}], "latest_revision": 3, "revision": 3, "created": {"type": "/type/datetime", "value": "2024-09-09T18:48:38.476534"}, "last_modified": {"type": "/type/datetime", "value": "2025-09-14T17:53:44.620024"}} +/type/author /authors/OL16210210A 1 2026-03-09T12:56:09.541018 {"type": {"key": "/type/author"}, "name": "Marie Louise Vert", "entity_type": "person", "key": "/authors/OL16210210A", "source_records": ["marc:harvard_bibliographic_metadata/20220215_032.bib.mrc:228102362:872"], "latest_revision": 1, "revision": 1, "created": {"type": "/type/datetime", "value": "2026-03-09T12:56:09.541018"}, "last_modified": {"type": "/type/datetime", "value": "2026-03-09T12:56:09.541018"}} +/type/author /authors/OL4244073A 5 2024-04-22T23:17:17.381447 {"name": "Sy Nguyen", "alternate_names": ["Sy Balthazar Nguyen", "Nguyen SH"], "photos": [-1], "title": "Docteur en m\u00e9decine", "type": {"key": "/type/author"}, "links": [{"url": "http://www.net-sante-environnement.fr", "title": "Website", "type": {"key": "/type/link"}}], "key": "/authors/OL4244073A", "latest_revision": 5, "revision": 5, "created": {"type": "/type/datetime", "value": "2008-04-30T20:50:18.033121"}, "last_modified": {"type": "/type/datetime", "value": "2024-04-22T23:17:17.381447"}} diff --git a/packages/adapters/test/fixtures/openlibrary-catalog-works.tsv b/packages/adapters/test/fixtures/openlibrary-catalog-works.tsv new file mode 100644 index 0000000..7798a6c --- /dev/null +++ b/packages/adapters/test/fixtures/openlibrary-catalog-works.tsv @@ -0,0 +1,100 @@ +/type/work /works/OL10000152W 3 2010-04-28T06:54:19.472104 {"title": "25 th\u00e8mes de travaux pratiques de chimie", "created": {"type": "/type/datetime", "value": "2009-12-11T01:57:19.964652"}, "covers": [3146558], "last_modified": {"type": "/type/datetime", "value": "2010-04-28T06:54:19.472104"}, "latest_revision": 3, "key": "/works/OL10000152W", "authors": [{"type": "/type/author_role", "author": {"key": "/authors/OL3965132A"}}], "type": {"key": "/type/work"}, "revision": 3} +/type/work /works/OL10000228W 3 2010-04-28T06:54:19.472104 {"title": "Prise en charge des traumatis\u00e9s cr\u00e2niens graves \u00e0 la phase pr\u00e9coce", "created": {"type": "/type/datetime", "value": "2009-12-11T01:57:19.964652"}, "covers": [3140456], "last_modified": {"type": "/type/datetime", "value": "2010-04-28T06:54:19.472104"}, "latest_revision": 3, "key": "/works/OL10000228W", "authors": [{"type": "/type/author_role", "author": {"key": "/authors/OL3965256A"}}], "type": {"key": "/type/work"}, "revision": 3} +/type/work /works/OL10000259W 3 2010-04-28T06:54:19.472104 {"title": "Cannibales", "created": {"type": "/type/datetime", "value": "2009-12-11T01:57:19.964652"}, "covers": [3140510], "last_modified": {"type": "/type/datetime", "value": "2010-04-28T06:54:19.472104"}, "latest_revision": 3, "key": "/works/OL10000259W", "authors": [{"type": "/type/author_role", "author": {"key": "/authors/OL3965289A"}}], "type": {"key": "/type/work"}, "revision": 3} +/type/work /works/OL10000494W 3 2010-04-28T06:54:19.472104 {"title": "Lorie de A \u00e0 Z", "created": {"type": "/type/datetime", "value": "2009-12-11T01:57:19.964652"}, "covers": [3140928], "last_modified": {"type": "/type/datetime", "value": "2010-04-28T06:54:19.472104"}, "latest_revision": 3, "key": "/works/OL10000494W", "authors": [{"type": "/type/author_role", "author": {"key": "/authors/OL3965522A"}}], "type": {"key": "/type/work"}, "revision": 3} +/type/work /works/OL10000499W 3 2010-04-28T06:54:19.472104 {"title": "Stairways to hell", "created": {"type": "/type/datetime", "value": "2009-12-11T01:57:19.964652"}, "covers": [3140941], "last_modified": {"type": "/type/datetime", "value": "2010-04-28T06:54:19.472104"}, "latest_revision": 3, "key": "/works/OL10000499W", "authors": [{"type": "/type/author_role", "author": {"key": "/authors/OL3965530A"}}], "type": {"key": "/type/work"}, "revision": 3} +/type/work /works/OL10000604W 4 2018-08-15T19:53:52.404341 {"title": "Lucia di Lammermoor", "created": {"type": "/type/datetime", "value": "2009-12-11T01:57:19.964652"}, "covers": [3141140], "last_modified": {"type": "/type/datetime", "value": "2018-08-15T19:53:52.404341"}, "latest_revision": 4, "key": "/works/OL10000604W", "authors": [{"type": {"key": "/type/author_role"}, "author": {"key": "/authors/OL284141A"}}], "type": {"key": "/type/work"}, "revision": 4} +/type/work /works/OL10000891W 4 2010-11-19T07:53:27.999546 {"title": "Rage dedans", "created": {"type": "/type/datetime", "value": "2009-12-11T01:57:19.964652"}, "covers": [3141965], "last_modified": {"type": "/type/datetime", "value": "2010-11-19T07:53:27.999546"}, "latest_revision": 4, "key": "/works/OL10000891W", "authors": [{"type": {"key": "/type/author_role"}, "author": {"key": "/authors/OL41560A"}}], "type": {"key": "/type/work"}, "revision": 4} +/type/work /works/OL10000903W 3 2010-04-28T06:54:19.472104 {"title": "Leon tete de con", "created": {"type": "/type/datetime", "value": "2009-12-11T01:57:19.964652"}, "covers": [3141971], "last_modified": {"type": "/type/datetime", "value": "2010-04-28T06:54:19.472104"}, "latest_revision": 3, "key": "/works/OL10000903W", "authors": [{"type": "/type/author_role", "author": {"key": "/authors/OL3966000A"}}], "type": {"key": "/type/work"}, "revision": 3} +/type/work /works/OL10000946W 3 2010-04-28T06:54:19.472104 {"title": "Karate-do kata", "created": {"type": "/type/datetime", "value": "2009-12-11T01:57:19.964652"}, "covers": [3142061], "last_modified": {"type": "/type/datetime", "value": "2010-04-28T06:54:19.472104"}, "latest_revision": 3, "key": "/works/OL10000946W", "authors": [{"type": "/type/author_role", "author": {"key": "/authors/OL3966048A"}}], "type": {"key": "/type/work"}, "revision": 3} +/type/work /works/OL10000971W 4 2026-01-29T00:31:09.958255 {"title": "Vie et m\u00e9ditation", "covers": [3142119], "key": "/works/OL10000971W", "authors": [{"type": {"key": "/type/author_role"}, "author": {"key": "/authors/OL83797A"}}], "type": {"key": "/type/work"}, "latest_revision": 4, "revision": 4, "created": {"type": "/type/datetime", "value": "2009-12-11T01:57:19.964652"}, "last_modified": {"type": "/type/datetime", "value": "2026-01-29T00:31:09.958255"}} +/type/work /works/OL10000986W 3 2010-07-22T05:59:53.479930 {"last_modified": {"type": "/type/datetime", "value": "2010-07-22T05:59:53.479930"}, "title": "Droits de l'homme et responsabilite\u0301", "created": {"type": "/type/datetime", "value": "2009-12-11T01:57:19.964652"}, "subjects": ["Congresses", "Human rights", "Globalization"], "latest_revision": 3, "key": "/works/OL10000986W", "authors": [{"type": {"key": "/type/author_role"}, "author": {"key": "/authors/OL3966096A"}}], "type": {"key": "/type/work"}, "revision": 3} +/type/work /works/OL10001466W 4 2019-03-12T19:05:27.298258 {"created": {"type": "/type/datetime", "value": "2009-12-11T01:57:29.804644"}, "subjects": ["Emigration and immigration", "Social aspects", "Economic aspects"], "latest_revision": 4, "key": "/works/OL10001466W", "title": "D'Un Voyage A L'Autre", "authors": [{"type": {"key": "/type/author_role"}, "author": {"key": "/authors/OL3966628A"}}], "type": {"key": "/type/work"}, "last_modified": {"type": "/type/datetime", "value": "2019-03-12T19:05:27.298258"}, "covers": [3143146], "revision": 4} +/type/work /works/OL10001503W 3 2010-04-28T06:54:19.472104 {"title": "Gym a toute heure", "created": {"type": "/type/datetime", "value": "2009-12-11T01:57:29.804644"}, "covers": [3143230], "last_modified": {"type": "/type/datetime", "value": "2010-04-28T06:54:19.472104"}, "latest_revision": 3, "key": "/works/OL10001503W", "authors": [{"type": "/type/author_role", "author": {"key": "/authors/OL3966674A"}}], "type": {"key": "/type/work"}, "revision": 3} +/type/work /works/OL1000176W 4 2012-07-09T00:02:05.517564 {"title": "Paying The Piper (Hammer's Slammers Series)", "created": {"type": "/type/datetime", "value": "2009-12-09T19:03:27.899466"}, "covers": [478394], "last_modified": {"type": "/type/datetime", "value": "2012-07-09T00:02:05.517564"}, "latest_revision": 4, "key": "/works/OL1000176W", "authors": [{"type": {"key": "/type/author_role"}, "author": {"key": "/authors/OL4740632A"}}], "type": {"key": "/type/work"}, "revision": 4} +/type/work /works/OL10001817W 3 2010-04-28T06:54:19.472104 {"title": "Response of Marine Ecosystems to Global Change - Ecological Impact of Appendicularians", "created": {"type": "/type/datetime", "value": "2009-12-11T01:57:29.804644"}, "covers": [3143980], "last_modified": {"type": "/type/datetime", "value": "2010-04-28T06:54:19.472104"}, "latest_revision": 3, "key": "/works/OL10001817W", "authors": [{"type": "/type/author_role", "author": {"key": "/authors/OL3967007A"}}], "type": {"key": "/type/work"}, "revision": 3} +/type/work /works/OL10001852W 3 2010-07-22T06:00:07.555136 {"last_modified": {"type": "/type/datetime", "value": "2010-07-22T06:00:07.555136"}, "latest_revision": 3, "key": "/works/OL10001852W", "authors": [{"type": {"key": "/type/author_role"}, "author": {"key": "/authors/OL3967061A"}}], "created": {"type": "/type/datetime", "value": "2009-12-11T01:57:29.804644"}, "title": "La Bourgogne de Lamartine \u00e0 nos jours", "subject_places": ["Burgundy (France)"], "subjects": ["History"], "subject_times": ["20th century", "19th century"], "type": {"key": "/type/work"}, "revision": 3} +/type/work /works/OL10001878W 5 2025-05-01T10:00:22.751856 {"subjects": ["Psychoanalysis", "Physicians", "History"], "key": "/works/OL10001878W", "title": "O\u00f9 en est la psychanalyse ?. Psychanalyse et figures de la modernit\u00e9", "authors": [{"type": {"key": "/type/author_role"}, "author": {"key": "/authors/OL3967093A"}}], "type": {"key": "/type/work"}, "covers": [3150495], "latest_revision": 5, "revision": 5, "created": {"type": "/type/datetime", "value": "2009-12-11T01:57:29.804644"}, "last_modified": {"type": "/type/datetime", "value": "2025-05-01T10:00:22.751856"}} +/type/work /works/OL10002011W 3 2010-04-28T06:54:19.472104 {"title": "Devenir efficace dans ses \u00e9tudes", "created": {"type": "/type/datetime", "value": "2009-12-11T01:57:38.254267"}, "covers": [3144472], "last_modified": {"type": "/type/datetime", "value": "2010-04-28T06:54:19.472104"}, "latest_revision": 3, "key": "/works/OL10002011W", "authors": [{"type": "/type/author_role", "author": {"key": "/authors/OL3967238A"}}], "type": {"key": "/type/work"}, "revision": 3} +/type/work /works/OL10002109W 1 2009-12-11T01:57:38.254267 {"title": "Degas", "created": {"type": "/type/datetime", "value": "2009-12-11T01:57:38.254267"}, "last_modified": {"type": "/type/datetime", "value": "2009-12-11T01:57:38.254267"}, "latest_revision": 1, "key": "/works/OL10002109W", "authors": [{"type": "/type/author_role", "author": {"key": "/authors/OL3967390A"}}], "type": {"key": "/type/work"}, "revision": 1} +/type/work /works/OL10002186W 3 2010-07-22T06:00:07.555136 {"created": {"type": "/type/datetime", "value": "2009-12-11T01:57:38.254267"}, "subject_places": ["France"], "subjects": ["Ecoles de sages-femmes", "Examens d'entr\u00e9e", "Guides", "Sages-femmes"], "latest_revision": 3, "key": "/works/OL10002186W", "title": "Pr\u00e9pa sage-femme", "authors": [{"type": {"key": "/type/author_role"}, "author": {"key": "/authors/OL3967471A"}}], "type": {"key": "/type/work"}, "last_modified": {"type": "/type/datetime", "value": "2010-07-22T06:00:07.555136"}, "revision": 3} +/type/work /works/OL10002250W 3 2010-04-28T06:54:19.472104 {"title": "Concours kin\u00e9 2002 - Biologie", "created": {"type": "/type/datetime", "value": "2009-12-11T01:57:38.254267"}, "covers": [3144918], "last_modified": {"type": "/type/datetime", "value": "2010-04-28T06:54:19.472104"}, "latest_revision": 3, "key": "/works/OL10002250W", "authors": [{"type": "/type/author_role", "author": {"key": "/authors/OL3967533A"}}], "type": {"key": "/type/work"}, "revision": 3} +/type/work /works/OL10002290W 1 2009-12-11T01:57:38.254267 {"title": "Suzuki Harunobu (1725-1770)", "created": {"type": "/type/datetime", "value": "2009-12-11T01:57:38.254267"}, "last_modified": {"type": "/type/datetime", "value": "2009-12-11T01:57:38.254267"}, "latest_revision": 1, "key": "/works/OL10002290W", "authors": [{"type": "/type/author_role", "author": {"key": "/authors/OL3967614A"}}], "type": {"key": "/type/work"}, "revision": 1} +/type/work /works/OL1000252W 11 2026-08-30T01:13:25.615043 {"title": "Elizabeth's Rake", "covers": [4636122], "key": "/works/OL1000252W", "authors": [{"author": {"key": "/authors/OL92943A"}, "type": {"key": "/type/author_role"}}], "type": {"key": "/type/work"}, "subjects": ["Fiction, Romance, Historical, Regency", "Man-woman relationships, fiction", "Relations entre hommes et femmes", "Romans, nouvelles"], "subject_times": ["Regency England"], "genres": ["/tags/OL177T"], "latest_revision": 11, "revision": 11, "created": {"type": "/type/datetime", "value": "2009-12-09T19:03:27.899466"}, "last_modified": {"type": "/type/datetime", "value": "2026-08-30T01:13:25.615043"}} +/type/work /works/OL10002593W 3 2010-04-28T06:54:19.472104 {"title": "Les 30 meilleurs groupes de rock anglais des ann\u00e9es 60", "created": {"type": "/type/datetime", "value": "2009-12-11T01:57:38.254267"}, "covers": [5203895], "last_modified": {"type": "/type/datetime", "value": "2010-04-28T06:54:19.472104"}, "latest_revision": 3, "key": "/works/OL10002593W", "authors": [{"type": "/type/author_role", "author": {"key": "/authors/OL3967985A"}}], "type": {"key": "/type/work"}, "revision": 3} +/type/work /works/OL10002777W 3 2010-04-28T06:54:19.472104 {"title": "Le livre des jeux de pions", "created": {"type": "/type/datetime", "value": "2009-12-11T01:57:38.254267"}, "covers": [3145908], "last_modified": {"type": "/type/datetime", "value": "2010-04-28T06:54:19.472104"}, "latest_revision": 3, "key": "/works/OL10002777W", "authors": [{"type": "/type/author_role", "author": {"key": "/authors/OL3968191A"}}], "type": {"key": "/type/work"}, "revision": 3} +/type/work /works/OL10003019W 1 2009-12-11T01:57:52.839053 {"title": "Maisons-Laffitte", "created": {"type": "/type/datetime", "value": "2009-12-11T01:57:52.839053"}, "last_modified": {"type": "/type/datetime", "value": "2009-12-11T01:57:52.839053"}, "latest_revision": 1, "key": "/works/OL10003019W", "authors": [{"type": "/type/author_role", "author": {"key": "/authors/OL3968399A"}}], "type": {"key": "/type/work"}, "revision": 1} +/type/work /works/OL10003563W 1 2009-12-11T01:57:52.839053 {"title": "Le Message cach\u00e9 de Pythagore", "created": {"type": "/type/datetime", "value": "2009-12-11T01:57:52.839053"}, "last_modified": {"type": "/type/datetime", "value": "2009-12-11T01:57:52.839053"}, "latest_revision": 1, "key": "/works/OL10003563W", "authors": [{"type": "/type/author_role", "author": {"key": "/authors/OL3969047A"}}], "type": {"key": "/type/work"}, "revision": 1} +/type/work /works/OL10003862W 3 2010-04-28T06:54:19.472104 {"title": "Le mobilier domestique, tome 2", "created": {"type": "/type/datetime", "value": "2009-12-11T01:57:52.839053"}, "covers": [3147802], "last_modified": {"type": "/type/datetime", "value": "2010-04-28T06:54:19.472104"}, "latest_revision": 3, "key": "/works/OL10003862W", "authors": [{"type": "/type/author_role", "author": {"key": "/authors/OL3969384A"}}], "type": {"key": "/type/work"}, "revision": 3} +/type/work /works/OL10003943W 3 2010-04-28T06:54:19.472104 {"title": "Automobilistes", "created": {"type": "/type/datetime", "value": "2009-12-11T01:57:52.839053"}, "covers": [3147991], "last_modified": {"type": "/type/datetime", "value": "2010-04-28T06:54:19.472104"}, "latest_revision": 3, "key": "/works/OL10003943W", "authors": [{"type": "/type/author_role", "author": {"key": "/authors/OL3969489A"}}], "type": {"key": "/type/work"}, "revision": 3} +/type/work /works/OL10004376W 2 2019-02-10T08:53:46.361687 {"last_modified": {"type": "/type/datetime", "value": "2019-02-10T08:53:46.361687"}, "title": "Boris Vian", "created": {"type": "/type/datetime", "value": "2009-12-11T01:58:02.183946"}, "subjects": ["French Authors", "Biography", "Jazz musicians"], "latest_revision": 2, "key": "/works/OL10004376W", "authors": [{"type": {"key": "/type/author_role"}, "author": {"key": "/authors/OL3969905A"}}], "type": {"key": "/type/work"}, "revision": 2} +/type/work /works/OL10004436W 3 2010-04-28T06:54:19.472104 {"title": "Cuisine de Corse de A \u00e0 Z", "created": {"type": "/type/datetime", "value": "2009-12-11T01:58:02.183946"}, "covers": [3148992], "last_modified": {"type": "/type/datetime", "value": "2010-04-28T06:54:19.472104"}, "latest_revision": 3, "key": "/works/OL10004436W", "authors": [{"type": "/type/author_role", "author": {"key": "/authors/OL3970000A"}}], "type": {"key": "/type/work"}, "revision": 3} +/type/work /works/OL10004471W 3 2010-04-28T06:54:19.472104 {"title": "Le Silence", "created": {"type": "/type/datetime", "value": "2009-12-11T01:58:02.183946"}, "covers": [3149310], "last_modified": {"type": "/type/datetime", "value": "2010-04-28T06:54:19.472104"}, "latest_revision": 3, "key": "/works/OL10004471W", "authors": [{"type": "/type/author_role", "author": {"key": "/authors/OL3970049A"}}], "type": {"key": "/type/work"}, "revision": 3} +/type/work /works/OL10004608W 4 2023-11-12T22:08:51.865274 {"title": "Le d\u00e9fi paysan", "covers": [3149556], "key": "/works/OL10004608W", "authors": [{"type": {"key": "/type/author_role"}, "author": {"key": "/authors/OL3970250A"}}], "type": {"key": "/type/work"}, "subjects": ["Farmers", "Agriculture"], "latest_revision": 4, "revision": 4, "created": {"type": "/type/datetime", "value": "2009-12-11T01:58:02.183946"}, "last_modified": {"type": "/type/datetime", "value": "2023-11-12T22:08:51.865274"}} +/type/work /works/OL10004728W 4 2025-12-09T17:51:52.215630 {"title": "Vers une France pa\u00efenne", "covers": [3149779], "key": "/works/OL10004728W", "authors": [{"type": {"key": "/type/author_role"}, "author": {"key": "/authors/OL3970413A"}}], "type": {"key": "/type/work"}, "subjects": ["Neopaganism", "Religion", "Secularization (Theology)", "Religious life and customs"], "latest_revision": 4, "revision": 4, "created": {"type": "/type/datetime", "value": "2009-12-11T01:58:02.183946"}, "last_modified": {"type": "/type/datetime", "value": "2025-12-09T17:51:52.215630"}} +/type/work /works/OL10005097W 3 2010-04-28T06:54:19.472104 {"title": "Autismes et psychoses infantiles", "created": {"type": "/type/datetime", "value": "2009-12-11T01:58:11.470855"}, "covers": [3150547], "last_modified": {"type": "/type/datetime", "value": "2010-04-28T06:54:19.472104"}, "latest_revision": 3, "key": "/works/OL10005097W", "authors": [{"type": "/type/author_role", "author": {"key": "/authors/OL3970849A"}}], "type": {"key": "/type/work"}, "revision": 3} +/type/work /works/OL10005167W 3 2010-04-28T06:54:19.472104 {"title": "Automatique", "created": {"type": "/type/datetime", "value": "2009-12-11T01:58:11.470855"}, "covers": [3150661], "last_modified": {"type": "/type/datetime", "value": "2010-04-28T06:54:19.472104"}, "latest_revision": 3, "key": "/works/OL10005167W", "authors": [{"type": "/type/author_role", "author": {"key": "/authors/OL3970928A"}}], "type": {"key": "/type/work"}, "revision": 3} +/type/work /works/OL10005262W 3 2010-04-28T06:54:19.472104 {"title": "Coffret civilisations imperiales 1 et 2", "created": {"type": "/type/datetime", "value": "2009-12-11T01:58:11.470855"}, "covers": [3150866], "last_modified": {"type": "/type/datetime", "value": "2010-04-28T06:54:19.472104"}, "latest_revision": 3, "key": "/works/OL10005262W", "authors": [{"type": "/type/author_role", "author": {"key": "/authors/OL3971031A"}}], "type": {"key": "/type/work"}, "revision": 3} +/type/work /works/OL10005366W 3 2010-04-28T06:54:19.472104 {"title": "B-A BA de Halloween", "created": {"type": "/type/datetime", "value": "2009-12-11T01:58:11.470855"}, "covers": [3151093], "last_modified": {"type": "/type/datetime", "value": "2010-04-28T06:54:19.472104"}, "latest_revision": 3, "key": "/works/OL10005366W", "authors": [{"type": "/type/author_role", "author": {"key": "/authors/OL3971173A"}}], "type": {"key": "/type/work"}, "revision": 3} +/type/work /works/OL10005398W 3 2010-04-28T06:54:19.472104 {"title": "EURL et EARL, num\u00e9ro 122, 3\u00e8me \u00e9dition", "created": {"type": "/type/datetime", "value": "2009-12-11T01:58:11.470855"}, "covers": [3151220], "last_modified": {"type": "/type/datetime", "value": "2010-04-28T06:54:19.472104"}, "latest_revision": 3, "key": "/works/OL10005398W", "authors": [{"type": "/type/author_role", "author": {"key": "/authors/OL3971215A"}}], "type": {"key": "/type/work"}, "revision": 3} +/type/work /works/OL10005708W 5 2023-04-04T09:23:12.215046 {"key": "/works/OL10005708W", "authors": [{"author": {"key": "/authors/OL3971558A"}, "type": {"key": "/type/author_role"}}], "title": "Les tentatrices du diable", "subject_places": ["Germany"], "subjects": ["Biography", "Friends and associates", "National socialism and women", "Women", "Heads of state", "Relations with women"], "subject_people": ["Adolf Hitler (1889-1945)"], "type": {"key": "/type/work"}, "covers": [13824038], "latest_revision": 5, "revision": 5, "created": {"type": "/type/datetime", "value": "2009-12-11T01:58:11.470855"}, "last_modified": {"type": "/type/datetime", "value": "2023-04-04T09:23:12.215046"}} +/type/work /works/OL10006856W 3 2010-04-28T06:54:19.472104 {"title": "Cours d'analyse math\u00e9matique", "created": {"type": "/type/datetime", "value": "2009-12-11T01:58:20.117391"}, "covers": [3154571], "last_modified": {"type": "/type/datetime", "value": "2010-04-28T06:54:19.472104"}, "latest_revision": 3, "key": "/works/OL10006856W", "authors": [{"type": "/type/author_role", "author": {"key": "/authors/OL3972865A"}}], "type": {"key": "/type/work"}, "revision": 3} +/type/work /works/OL10006900W 5 2024-08-27T02:26:29.121009 {"title": "C\u00e9zanne", "covers": [3154654], "key": "/works/OL10006900W", "authors": [{"type": {"key": "/type/author_role"}, "author": {"key": "/authors/OL3972920A"}}], "type": {"key": "/type/work"}, "subjects": ["Criticism and interpretation", "Nude in art", "Swimming in art", "Catalogs"], "latest_revision": 5, "revision": 5, "created": {"type": "/type/datetime", "value": "2009-12-11T01:58:20.117391"}, "last_modified": {"type": "/type/datetime", "value": "2024-08-27T02:26:29.121009"}} +/type/work /works/OL10007099W 3 2010-04-28T06:54:19.472104 {"title": "R\u00e9duisez votre stress", "created": {"type": "/type/datetime", "value": "2009-12-11T01:58:28.508302"}, "covers": [3155042], "last_modified": {"type": "/type/datetime", "value": "2010-04-28T06:54:19.472104"}, "latest_revision": 3, "key": "/works/OL10007099W", "authors": [{"type": "/type/author_role", "author": {"key": "/authors/OL3973167A"}}], "type": {"key": "/type/work"}, "revision": 3} +/type/work /works/OL1000806W 12 2026-08-29T16:27:20.122653 {"covers": [4982527, -1, 10117112], "key": "/works/OL1000806W", "authors": [{"author": {"key": "/authors/OL92956A"}, "type": {"key": "/type/author_role"}}], "title": "The Cold One", "subject_places": ["Los Angeles (Calif.)"], "subjects": ["Fiction", "Los angeles (calif.), fiction", "Fiction, horror", "Fiction, psychological", "Jacobs, peter (fictitious character), fiction", "Journalists, fiction"], "type": {"key": "/type/work"}, "description": {"type": "/type/text", "value": "Existing only to destroy humankind, a demonic force, disguised as a beautiful woman, ravages the lives of several people, including former street gang member Jerry Washington and journalist Peter James."}, "genres": ["/tags/OL171T"], "latest_revision": 12, "revision": 12, "created": {"type": "/type/datetime", "value": "2009-12-09T19:05:50.674576"}, "last_modified": {"type": "/type/datetime", "value": "2026-08-29T16:27:20.122653"}} +/type/work /works/OL1000826W 13 2026-08-30T05:40:05.176471 {"description": {"type": "/type/text", "value": "From Publishers Weekly\r\nIn this hokey, meandering novel, Julia is a girl gifted with a healing touch and the power to glimpse the future. When she sees a vision of her best friend's boyfriend, Jim, shot and bleeding to death, she does her best to keep him out of danger. But then another friend is shot while witnessing a gas station holdup, and Julia and Jim set out to wreak revenge on the gunman. Meanwhile, Julia's best friend discovers that the gunman just happens to be the deranged former boyfriend of Kary, the recently deceased half-sister Julia never knew. Julia's mother--also a healer--had died in an attempt to save Kary's life. In another part of town, a carload of good witches is hot on Julia's trail, determined to keep her from abusing her powers. Typically, Pike's writing is peppy enough to animate his most tangled plots; here, however, his style becomes choppy and unconvincing--unable to sustain the coincidence-riddled story. In addition, the text is littered with sexist one-liners which, along with a humorless running \"joke,\" are as irritating as they are offensive. Ages 13-up.\r\nCopyright 1990 Reed Business Information, Inc."}, "covers": [201808, 6905457, 3111301, -1], "key": "/works/OL1000826W", "authors": [{"type": {"key": "/type/author_role"}, "author": {"key": "/authors/OL92956A"}}], "title": "Witch", "subjects": ["Supernatural", "Adventure stories", "Witches", "Horror stories", "Extrasensory perception", "Fiction", "Children's fiction", "Witches, fiction"], "type": {"key": "/type/work"}, "genres": ["/tags/OL164T", "/tags/OL171T"], "latest_revision": 13, "revision": 13, "created": {"type": "/type/datetime", "value": "2009-12-09T19:05:50.674576"}, "last_modified": {"type": "/type/datetime", "value": "2026-08-30T05:40:05.176471"}} +/type/work /works/OL10009613W 1 2009-12-11T01:58:48.541720 {"title": "Je t'aime, je t'\u00e9cris", "created": {"type": "/type/datetime", "value": "2009-12-11T01:58:48.541720"}, "last_modified": {"type": "/type/datetime", "value": "2009-12-11T01:58:48.541720"}, "latest_revision": 1, "key": "/works/OL10009613W", "authors": [{"type": "/type/author_role", "author": {"key": "/authors/OL3976062A"}}], "type": {"key": "/type/work"}, "revision": 1} +/type/work /works/OL10009923W 2 2021-08-16T16:57:07.863263 {"title": "J'Observe Les Insectes", "key": "/works/OL10009923W", "authors": [{"type": {"key": "/type/author_role"}, "author": {"key": "/authors/OL3976369A"}}], "type": {"key": "/type/work"}, "subjects": ["Insects", "Juvenile literature", "Experiments", "Scientific recreations", "Insectes", "Ouvrages pour la jeunesse", "Exp\u00e9riences", "Jeux scientifiques", "Capture", "Technique", "Insecte", "Observation", "\u00c9levage"], "covers": [11658931], "latest_revision": 2, "revision": 2, "created": {"type": "/type/datetime", "value": "2009-12-11T01:58:48.541720"}, "last_modified": {"type": "/type/datetime", "value": "2021-08-16T16:57:07.863263"}} +/type/work /works/OL10009982W 3 2010-04-28T06:54:19.472104 {"title": "La Deesse Et La Panthere", "created": {"type": "/type/datetime", "value": "2009-12-11T01:58:48.541720"}, "covers": [3158550], "last_modified": {"type": "/type/datetime", "value": "2010-04-28T06:54:19.472104"}, "latest_revision": 3, "key": "/works/OL10009982W", "authors": [{"type": "/type/author_role", "author": {"key": "/authors/OL3976449A"}}], "type": {"key": "/type/work"}, "revision": 3} +/type/work /works/OL10010001W 3 2010-07-22T11:21:27.434339 {"created": {"type": "/type/datetime", "value": "2009-12-11T01:58:57.594996"}, "subject_places": ["Canada"], "subjects": ["Collective bargaining"], "latest_revision": 3, "key": "/works/OL10010001W", "title": "Shifting from traditional to mutual gains bargaining", "authors": [{"type": {"key": "/type/author_role"}, "author": {"key": "/authors/OL3976471A"}}], "type": {"key": "/type/work"}, "last_modified": {"type": "/type/datetime", "value": "2010-07-22T11:21:27.434339"}, "revision": 3} +/type/work /works/OL10010035W 3 2010-04-28T06:54:19.472104 {"title": "Sacred Journey", "created": {"type": "/type/datetime", "value": "2009-12-11T01:58:57.594996"}, "covers": [3158627], "last_modified": {"type": "/type/datetime", "value": "2010-04-28T06:54:19.472104"}, "latest_revision": 3, "key": "/works/OL10010035W", "authors": [{"type": "/type/author_role", "author": {"key": "/authors/OL3976497A"}}], "type": {"key": "/type/work"}, "revision": 3} +/type/work /works/OL10010192W 4 2024-08-11T22:34:40.725532 {"title": "Tahqiq al-qadiyah fi al-farq bayna al-rishwah wa-al-hadiyah", "subjects": ["Gifts (Islamic law)", "Bribery (Islamic law)", "Gifts", "Law and legislation"], "key": "/works/OL10010192W", "authors": [{"type": {"key": "/type/author_role"}, "author": {"key": "/authors/OL525554A"}}], "type": {"key": "/type/work"}, "latest_revision": 4, "revision": 4, "created": {"type": "/type/datetime", "value": "2009-12-11T01:58:57.594996"}, "last_modified": {"type": "/type/datetime", "value": "2024-08-11T22:34:40.725532"}} +/type/work /works/OL10010555W 2 2024-07-15T01:08:49.619279 {"title": "Grammaire fondamentale du latin. Le signifi\u00e9 du verbe", "key": "/works/OL10010555W", "authors": [{"type": {"key": "/type/author_role"}, "author": {"key": "/authors/OL3977047A"}}], "type": {"key": "/type/work"}, "subjects": ["Latin language", "Grammar", "Historical Grammar"], "latest_revision": 2, "revision": 2, "created": {"type": "/type/datetime", "value": "2009-12-11T01:58:57.594996"}, "last_modified": {"type": "/type/datetime", "value": "2024-07-15T01:08:49.619279"}} +/type/work /works/OL10010563W 1 2009-12-11T01:58:57.594996 {"title": "Les Terrains bois\u00e9s", "created": {"type": "/type/datetime", "value": "2009-12-11T01:58:57.594996"}, "last_modified": {"type": "/type/datetime", "value": "2009-12-11T01:58:57.594996"}, "latest_revision": 1, "key": "/works/OL10010563W", "authors": [{"type": "/type/author_role", "author": {"key": "/authors/OL3977057A"}}], "type": {"key": "/type/work"}, "revision": 1} +/type/work /works/OL10010595W 4 2022-12-27T17:25:51.581896 {"title": "Histoire des comptes de Poitou et ducs de Guyenne", "covers": [3159371], "key": "/works/OL10010595W", "authors": [{"type": {"key": "/type/author_role"}, "author": {"key": "/authors/OL3977098A"}}], "type": {"key": "/type/work"}, "subjects": ["History", "Nobility"], "latest_revision": 4, "revision": 4, "created": {"type": "/type/datetime", "value": "2009-12-11T01:58:57.594996"}, "last_modified": {"type": "/type/datetime", "value": "2022-12-27T17:25:51.581896"}} +/type/work /works/OL10010714W 3 2024-08-21T14:44:27.877816 {"title": "The Gutenberg Bible", "key": "/works/OL10010714W", "authors": [{"type": {"key": "/type/author_role"}, "author": {"key": "/authors/OL3977262A"}}], "type": {"key": "/type/work"}, "subjects": ["Gutenberg Bible", "Bibliography", "Printing", "History", "Origin and antecedents", "Incunabula", "Facsimiles"], "latest_revision": 3, "revision": 3, "created": {"type": "/type/datetime", "value": "2009-12-11T01:58:57.594996"}, "last_modified": {"type": "/type/datetime", "value": "2024-08-21T14:44:27.877816"}} +/type/work /works/OL10010744W 1 2009-12-11T01:58:57.594996 {"title": "La Pr\u00fdvoyance Dans L'Entreprise (Collection Manuel Pratique)", "created": {"type": "/type/datetime", "value": "2009-12-11T01:58:57.594996"}, "last_modified": {"type": "/type/datetime", "value": "2009-12-11T01:58:57.594996"}, "latest_revision": 1, "key": "/works/OL10010744W", "authors": [{"type": "/type/author_role", "author": {"key": "/authors/OL3977281A"}}], "type": {"key": "/type/work"}, "revision": 1} +/type/work /works/OL10010861W 1 2009-12-11T01:58:57.594996 {"title": "L'Escalier de cristal, tome 3", "created": {"type": "/type/datetime", "value": "2009-12-11T01:58:57.594996"}, "last_modified": {"type": "/type/datetime", "value": "2009-12-11T01:58:57.594996"}, "latest_revision": 1, "key": "/works/OL10010861W", "authors": [{"type": "/type/author_role", "author": {"key": "/authors/OL3977412A"}}], "type": {"key": "/type/work"}, "revision": 1} +/type/work /works/OL1001115W 1 2009-12-09T19:05:50.674576 {"title": "L' oeuvre grav\u00e9 de James Ensor [par] Jacques Damase", "created": {"type": "/type/datetime", "value": "2009-12-09T19:05:50.674576"}, "last_modified": {"type": "/type/datetime", "value": "2009-12-09T19:05:50.674576"}, "latest_revision": 1, "key": "/works/OL1001115W", "authors": [{"type": "/type/author_role", "author": {"key": "/authors/OL92998A"}}], "type": {"key": "/type/work"}, "revision": 1} +/type/work /works/OL1001139W 8 2021-12-24T20:23:32.371861 {"subjects": ["Portraits", "Interviews", "Rock musicians", "Rock music", "Pictorial works", "Rock music, pictorial works", "Mccartney, paul, 1942-"], "key": "/works/OL1001139W", "title": "Each One Believing", "authors": [{"type": {"key": "/type/author_role"}, "author": {"key": "/authors/OL92999A"}}], "type": {"key": "/type/work"}, "covers": [601244], "latest_revision": 8, "revision": 8, "created": {"type": "/type/datetime", "value": "2009-12-09T19:05:50.674576"}, "last_modified": {"type": "/type/datetime", "value": "2021-12-24T20:23:32.371861"}} +/type/work /works/OL1001327W 4 2023-04-08T16:20:31.601951 {"title": "Programa Especial de Educac\u0327a\u0303o", "subject_places": ["Brazil", "Rio de Janeiro"], "key": "/works/OL1001327W", "authors": [{"type": "/type/author_role", "author": {"key": "/authors/OL93023A"}}], "type": {"key": "/type/work"}, "subjects": ["School management and organization", "Education and state", "Programa Especial de Educa\u00e7\u00e3o (Rio de Janeiro, Brazil)"], "covers": [13864860], "latest_revision": 4, "revision": 4, "created": {"type": "/type/datetime", "value": "2009-12-09T19:07:21.849284"}, "last_modified": {"type": "/type/datetime", "value": "2023-04-08T16:20:31.601951"}} +/type/work /works/OL10014855W 4 2020-12-08T02:31:24.330491 {"title": "Eisenbahnrecht und Bahnreform. Hardcover-Ausgabe. 2. Auflage", "covers": [3165299], "key": "/works/OL10014855W", "authors": [{"type": {"key": "/type/author_role"}, "author": {"key": "/authors/OL3981688A"}}], "type": {"key": "/type/work"}, "subjects": ["Railroad law", "Privatization", "Law and legislation", "Magnetic levitation vehicles", "High speed ground transportation", "Deutsche Bundesbahn"], "latest_revision": 4, "revision": 4, "created": {"type": "/type/datetime", "value": "2009-12-11T01:59:38.336708"}, "last_modified": {"type": "/type/datetime", "value": "2020-12-08T02:31:24.330491"}} +/type/work /works/OL1001984W 3 2023-06-08T04:33:28.256287 {"title": "O homem que matou Getu\u0301lio Vargas", "subject_places": ["Brazil"], "subjects": ["Death and burial", "Politics and government", "Fiction"], "subject_people": ["Get\u00falio Vargas (1883-1954)"], "key": "/works/OL1001984W", "authors": [{"type": {"key": "/type/author_role"}, "author": {"key": "/authors/OL93105A"}}], "subject_times": ["20 century"], "type": {"key": "/type/work"}, "covers": [14344042], "latest_revision": 3, "revision": 3, "created": {"type": "/type/datetime", "value": "2009-12-09T19:07:21.849284"}, "last_modified": {"type": "/type/datetime", "value": "2023-06-08T04:33:28.256287"}} +/type/work /works/OL1002102W 3 2010-07-20T11:13:38.232174 {"created": {"type": "/type/datetime", "value": "2009-12-09T19:07:21.849284"}, "title": "O imagina\u0301rio na literatura de cordel", "last_modified": {"type": "/type/datetime", "value": "2010-07-20T11:13:38.232174"}, "latest_revision": 3, "key": "/works/OL1002102W", "authors": [{"type": "/type/author_role", "author": {"key": "/authors/OL93117A"}}], "type": {"key": "/type/work"}, "subjects": ["History and criticism", "Fantastic, The, in literature", "Fantasy poetry, Brazilian", "Chapbooks, Brazilian", "Brazilian poetry", "History", "Brazilian Fantasy poetry", "Brazilian Chapbooks"], "revision": 3} +/type/work /works/OL10023420W 5 2022-02-01T02:02:29.920090 {"title": "Besser ein Mann als gar kein \u00c4rger", "covers": [3182381], "key": "/works/OL10023420W", "authors": [{"author": {"key": "/authors/OL3168890A"}, "type": {"key": "/type/author_role"}}], "type": {"key": "/type/work"}, "subtitle": "drei Romane in einem Band", "latest_revision": 5, "revision": 5, "created": {"type": "/type/datetime", "value": "2009-12-11T02:00:50.445585"}, "last_modified": {"type": "/type/datetime", "value": "2022-02-01T02:02:29.920090"}} +/type/work /works/OL1002543W 3 2010-07-20T11:14:16.941048 {"created": {"type": "/type/datetime", "value": "2009-12-09T19:08:41.964839"}, "title": "A pinacoteca do MASP", "subject_places": ["Brazil", "S\u00e3o Paulo"], "last_modified": {"type": "/type/datetime", "value": "2010-07-20T11:14:16.941048"}, "latest_revision": 3, "key": "/works/OL1002543W", "authors": [{"type": "/type/author_role", "author": {"key": "/authors/OL93187A"}}], "type": {"key": "/type/work"}, "subjects": ["Catalogs", "Painting", "Museu de Arte de S\u00e3o Paulo Assis Chateaubriand"], "revision": 3} +/type/work /works/OL10026953W 5 2023-11-14T12:13:45.232895 {"title": "Ankommen in Deutschland", "key": "/works/OL10026953W", "authors": [{"type": {"key": "/type/author_role"}, "author": {"key": "/authors/OL3998974A"}}], "type": {"key": "/type/work"}, "subjects": ["Emigration and immigration", "Government policy", "Russian Jews", "Jews, russian", "Germany, emigration and immigration", "Soviet union, emigration and immigration", "History", "Jews", "Migrations"], "covers": [13759223], "latest_revision": 5, "revision": 5, "created": {"type": "/type/datetime", "value": "2009-12-11T02:01:19.014862"}, "last_modified": {"type": "/type/datetime", "value": "2023-11-14T12:13:45.232895"}} +/type/work /works/OL1003217W 3 2010-07-16T06:51:37.081465 {"subtitle": "confere\u0302ncia realizada, a 29 de Fevereiro de 1944 por iniciativa dos Ministe\u0301rios da Educac\u0327a\u0303o e da Guerra, no Pala\u0301cio Tiradentes, no Rio de Janeiro, quando do transcurso do terceiro centena\u0301rio da expulsa\u0303o dos Holandeses do Maranha\u0303o.", "title": "Os Holandeses no Maranha\u0303o", "created": {"type": "/type/datetime", "value": "2009-12-09T19:08:41.964839"}, "subject_places": ["Brazil", "Maranh\u00e3o (Brazil)"], "last_modified": {"type": "/type/datetime", "value": "2010-07-16T06:51:37.081465"}, "latest_revision": 3, "key": "/works/OL1003217W", "authors": [{"type": "/type/author_role", "author": {"key": "/authors/OL93289A"}}], "type": {"key": "/type/work"}, "subjects": ["History", "Dutch"], "revision": 3} +/type/work /works/OL1003220W 3 2010-07-16T06:51:38.283680 {"subtitle": "cenas e figuras da histo\u0301ria do Brasil", "title": "O tempo devolvido", "created": {"type": "/type/datetime", "value": "2009-12-09T19:08:41.964839"}, "subject_places": ["Brazil"], "last_modified": {"type": "/type/datetime", "value": "2010-07-16T06:51:38.283680"}, "latest_revision": 3, "key": "/works/OL1003220W", "authors": [{"type": "/type/author_role", "author": {"key": "/authors/OL93289A"}}], "subject_times": ["19th century", "20th century"], "type": {"key": "/type/work"}, "subjects": ["Civilization", "Biography", "History"], "revision": 3} +/type/work /works/OL1004067W 4 2022-12-29T23:59:18.622434 {"key": "/works/OL1004067W", "authors": [{"type": {"key": "/type/author_role"}, "author": {"key": "/authors/OL93421A"}}], "title": "Vale tudo", "subject_places": ["Brazil"], "subjects": ["Singers", "Biography"], "subject_people": ["Tim Maia"], "type": {"key": "/type/work"}, "covers": [13096745], "latest_revision": 4, "revision": 4, "created": {"type": "/type/datetime", "value": "2009-12-09T19:11:10.903132"}, "last_modified": {"type": "/type/datetime", "value": "2022-12-29T23:59:18.622434"}} +/type/work /works/OL1004595W 2 2020-11-22T23:30:30.184992 {"description": {"type": "/type/text", "value": "\"Work's three parts include poems of the author's first phase. Author belongs to the new generation of the Concretist movement that came after Paulo Leminski, the tropicalia, and pop culture. This movement expresses urban life with caustic irony\"--Handbook of Latin American Studies, v. 58."}, "title": "Primeiro tempo", "created": {"type": "/type/datetime", "value": "2009-12-09T19:12:24.848949"}, "last_modified": {"type": "/type/datetime", "value": "2020-11-22T23:30:30.184992"}, "latest_revision": 2, "key": "/works/OL1004595W", "authors": [{"type": {"key": "/type/author_role"}, "author": {"key": "/authors/OL93475A"}}], "type": {"key": "/type/work"}, "revision": 2} +/type/work /works/OL100486W 4 2022-12-17T04:46:56.786943 {"covers": [10275109], "key": "/works/OL100486W", "authors": [{"type": {"key": "/type/author_role"}, "author": {"key": "/authors/OL32293A"}}], "title": "Les \u0153uvres comple\u0300tes de Voltaire", "subjects": ["Translations into English", "French literature, history and criticism, 18th century", "Voltaire, 1694-1778"], "subject_people": ["Voltaire (1694-1778)"], "type": {"key": "/type/work"}, "latest_revision": 4, "revision": 4, "created": {"type": "/type/datetime", "value": "2009-10-17T17:59:12.548417"}, "last_modified": {"type": "/type/datetime", "value": "2022-12-17T04:46:56.786943"}} +/type/work /works/OL1006150W 3 2010-07-20T11:18:37.830539 {"created": {"type": "/type/datetime", "value": "2009-12-09T19:14:47.654413"}, "title": "Cartas perto do corac\u0327a\u0303o", "last_modified": {"type": "/type/datetime", "value": "2010-07-20T11:18:37.830539"}, "latest_revision": 3, "key": "/works/OL1006150W", "authors": [{"type": "/type/author_role", "author": {"key": "/authors/OL93692A"}}], "subject_people": ["Clarice Lispector", "Fernando Tavares Sabino"], "subject_times": ["20th century"], "type": {"key": "/type/work"}, "subjects": ["Correspondence", "Authors, Brazilian", "Brazilian Authors"], "revision": 3} +/type/work /works/OL101599W 3 2012-11-09T14:02:54.561466 {"created": {"type": "/type/datetime", "value": "2009-10-17T18:13:12.238883"}, "subjects": ["Machine knitting"], "latest_revision": 3, "description": {"type": "/type/text", "value": "This book is a modular Instruction Manual, teaching how to use the passap duo 80 knitting machine. I imagine it was published in the 1980's, but now is not available to buy."}, "key": "/works/OL101599W", "title": "The PASSAP handbook", "authors": [{"type": {"key": "/type/author_role"}, "author": {"key": "/authors/OL1209583A"}}], "type": {"key": "/type/work"}, "last_modified": {"type": "/type/datetime", "value": "2012-11-09T14:02:54.561466"}, "revision": 3} +/type/work /works/OL10176951W 3 2025-05-19T00:03:42.041334 {"title": "Jing ji xi tong di zi zu zhi li lun", "subjects": ["Works councils"], "key": "/works/OL10176951W", "authors": [{"type": {"key": "/type/author_role"}, "author": {"key": "/authors/OL4159595A"}}], "type": {"key": "/type/work"}, "description": {"type": "/type/text", "value": "Theory of self-organization in an economic system."}, "latest_revision": 3, "revision": 3, "created": {"type": "/type/datetime", "value": "2009-12-11T02:25:59.420834"}, "last_modified": {"type": "/type/datetime", "value": "2025-05-19T00:03:42.041334"}} +/type/work /works/OL1021908W 5 2023-11-13T00:22:38.062031 {"title": "Bekleyen adam\u0131n gerc\u0327ekles\u0327en du\u0308s\u0327u\u0308", "covers": [4349152], "subject_places": ["Turkey"], "subjects": ["Politics and government"], "subject_people": ["S\u00fcleyman Demirel (1924-)"], "key": "/works/OL1021908W", "authors": [{"type": {"key": "/type/author_role"}, "author": {"key": "/authors/OL97458A"}}], "subject_times": ["1980-"], "type": {"key": "/type/work"}, "description": {"type": "/type/text", "value": "Turkey; Demirel, S\u00fcleyman, 1924-; politics and government"}, "latest_revision": 5, "revision": 5, "created": {"type": "/type/datetime", "value": "2009-12-09T19:31:53.442512"}, "last_modified": {"type": "/type/datetime", "value": "2023-11-13T00:22:38.062031"}} +/type/work /works/OL1041137W 8 2025-09-28T06:46:52.194284 {"covers": [5697366, 2486832, 6235397, 11761412], "key": "/works/OL1041137W", "authors": [{"type": {"key": "/type/author_role"}, "author": {"key": "/authors/OL2690745A"}}], "title": "Angels' wings", "subjects": ["Art", "Arts"], "subject_people": ["Ludwig van Beethoven (1770-1827)"], "type": {"key": "/type/work"}, "first_publish_date": "1898", "latest_revision": 8, "revision": 8, "created": {"type": "/type/datetime", "value": "2009-12-09T20:12:00.802736"}, "last_modified": {"type": "/type/datetime", "value": "2025-09-28T06:46:52.194284"}} +/type/work /works/OL10419584W 8 2024-08-02T07:14:19.496814 {"title": "Right turn", "covers": [3851281], "subject_places": ["United States"], "subjects": ["Politics and government", "Afro-Americans", "Attorneys general", "Civil rights", "Biography", "History", "African Americans", "Civil rights, united states", "United states, politics and government, 1981-1989", "African americans, civil rights", "Noirs am\u00e9ricains", "Droits", "Histoire", "Procureurs g\u00e9n\u00e9raux", "Biographies", "Politique et gouvernement"], "subject_people": ["William Bradford Reynolds"], "key": "/works/OL10419584W", "authors": [{"type": "/type/author_role", "author": {"key": "/authors/OL4325227A"}}], "subject_times": ["1981-1989"], "type": {"key": "/type/work"}, "description": {"type": "/type/text", "value": "Raymond Wolters maintains that Ronald Reagan and William Bradford Reynolds made the \"right turn\" when they questioned and limited the use of racial considerations in drawing electoral boundaries. He also documents the Reagan administration's considerable success in reinforcing within the country, and reviving within the judiciary, the conviction that every person - black or white - should be considered an individual with unique talents and inalienable rights.\n\nThis book begins with a biographical chapter on William Bradford Reynolds, the Assistant Attorney General who was the principal architect of Reagan's civil rights policies. It then analyzes three main civil rights issues: voting rights, affirmative action, and school desegregation.\n\nWolters describes specific cases: at-large elections and minority vote dilutions; congressional districting in New Orleans; legislative districting in North Carolina; the debates over the Civil Rights Act of 1964; social science critiques of affirmative action; the question of quotas; and school desegregation and forced busing.\n\nBecause Ronald Reagan and William Bradford Reynolds were men of the right, and because most journalists and historians are on the left, Wolters feels the \"people of words\" have dealt harshly with the Reagan administration. In writing this book, he hopes to correct the record on a subject that has been badly represented."}, "latest_revision": 8, "revision": 8, "created": {"type": "/type/datetime", "value": "2009-12-11T02:57:21.217340"}, "last_modified": {"type": "/type/datetime", "value": "2024-08-02T07:14:19.496814"}} +/type/work /works/OL10474323W 13 2024-08-01T05:58:20.668632 {"title": "Lily White", "covers": [7903905, 3851352], "first_sentence": {"type": "/type/text", "value": "I was never a virgin."}, "subject_places": ["New York (State)"], "first_publish_date": "1996", "key": "/works/OL10474323W", "authors": [{"type": {"key": "/type/author_role"}, "author": {"key": "/authors/OL232913A"}}], "dewey_number": ["813/.54"], "type": {"key": "/type/work"}, "subjects": ["Fiction", "Married women", "Women lawyers", "Domestic fiction", "Wives", "Large type books", "Jews, fiction", "Women lawyers, fiction", "Fiction, thrillers, suspense", "Long island (n.y.), fiction", "White, lily (fictitious character), fiction", "Fiction, mystery & detective, women sleuths"], "description": {"type": "/type/text", "value": "In Susan Isaac's most ambitious and dazzling novel to date, we are introduced to Lee White, a criminal defense lawyer practicing on Long Island. Into her life drifts Norman Torkelson, a career con man charged with strangling to death his latest mark. At first, as Lee explains to us, the case seems routine, the evidence overwhelming.\n\nNorman - manly, magnetic, and morally reprehensible - is a man who crisscrosses America looking for patsies for his cruel marriage scam: Love 'em, liquidate their assets, leave 'em. Clearly, he murdered Bobette Frisch, the dumpy, sour fiftysomething bar owner who had fallen madly in love with him. But just as Lee is resigning herself to the inevitable \"Guilty!\" verdict, she begins to have doubts. What, after all, was Norman's motive? Why not do what he had done for the last twenty years: run, and leave behind a broke and brokenhearted victim?\n\nLee starts to wonder if her client is not merely not guilty but covering for the real killer and, in doing so, performing the first selfless act of his life.\n\nAs the Torkelson case unfolds, a second narrator chimes in to tell us the story behind the story: the tale of Lee's life. Born Lily White, Lee is a smart, pretty, and privileged child coming of age on Long Island. Her parents have little time for her or her younger sister, devoted as they are to the pursuit of shallowness. Her mother, Sylvia, who looks like Lauren Bacall's twin sister with a mild eating disorder, is busy with the exhausting work of keeping up her wardrobe.\n\nHer father, Leonard Weissberg - Weiss - and finally White, is consumed by his chi-chi Manhattan fur salon, his model-bookkeeper mistress, and his obsession with the family next door, the old-money, oh-so-social Taylors.\n\nWhen Lee marries Jazz Taylor, the scion of these blue-bloods, her life seems blessed. Suddenly she has her mother's approval, her father's love - and a sublime husband. No matter that she has to give up her dream job in the Manhattan D.A.'s Office to move back to Long Island with him; that's what marriage is, a series of compromises made in the name of love. Isn't it?"}, "latest_revision": 13, "revision": 13, "created": {"type": "/type/datetime", "value": "2009-12-11T03:01:45.497912"}, "last_modified": {"type": "/type/datetime", "value": "2024-08-01T05:58:20.668632"}} +/type/work /works/OL10610315W 8 2026-05-29T05:13:41.585797 {"title": "Mika\u00ebl", "key": "/works/OL10610315W", "authors": [{"author": {"key": "/authors/OL4414467A"}, "type": {"key": "/type/author_role"}}], "type": {"key": "/type/work"}, "subjects": ["Translations into German", "Danish literature", "LGBTQ fiction before Stonewall"], "description": "In Paris at the end of the 19th century, the painter Claude Zoret leads a life of celebrity and luxury with his prot\u00e9g\u00e9, the young Czech Mika\u00ebl. They are at the same time father and adopted son, teacher and student, painter and model, and lovers. Everything is going well until, unfortunately, the young man falls madly in love with the Russian princess, Lucia Zamikov.", "covers": [14139595], "latest_revision": 8, "revision": 8, "created": {"type": "/type/datetime", "value": "2009-12-11T03:16:16.902596"}, "last_modified": {"type": "/type/datetime", "value": "2026-05-29T05:13:41.585797"}} +/type/work /works/OL10616291W 2 2012-02-19T01:50:12.050812 {"title": "Malka-e-sehra", "created": {"type": "/type/datetime", "value": "2009-12-11T03:16:52.999032"}, "covers": [-1], "last_modified": {"type": "/type/datetime", "value": "2012-02-19T01:50:12.050812"}, "latest_revision": 2, "key": "/works/OL10616291W", "authors": [{"type": {"key": "/type/author_role"}, "author": {"key": "/authors/OL4417269A"}}], "type": {"key": "/type/work"}, "revision": 2} +/type/work /works/OL106217W 12 2026-08-29T23:22:17.827432 {"description": {"type": "/type/text", "value": "One of America's most mesmerizing storytellers, Luanne Rice enthralls readers with her moving tales of ordinary people in crisis--and how they are transformed by the enduring power of love and family. Now the author of Safe Harbor, True Blue, and other New York Times bestsellers presents the gripping story of a man fighting for his family, a woman searching for her sister--and the promise of a new life where both least expected it...The Secret HourBeneath his careful and controlled demeanor, attorney John O'Rourke is a man whose life is in turmoil. Since the death of his wife, he has been juggling the rigors of a controversial capital murder case and the demands of raising two children. Eleven-year-old Maggie's crooked bangs and rumpled clothes eloquently reproach John's earnest but haphazard attempts at mothering. Teddy, John's stalwart fourteen-year-old, has quietly assumed responsibilities far too weighty for his young shoulders, as he longs for the way things used to be and tries to ignore the hostility that has swirled around his family since his father took on the defense of a killer whose crimes have rocked Connecticut.A brick through the window one autumn morning signals a dangerous new level of hatred. But a quieter event also takes place that day. A woman arrives on the O'Rourke doorstep to find a household on the brink of chaos but brimming with love--and, she hopes, answers. Kate Harris is searching for the key to her own mystery. Six months ago her younger sister fled far from their beloved home following a devastating confrontation. After mailing a single postcard from the New England shore, Willa Harris vanished. With only a postmark to go on, Kate takes a leave of absence from her job as a marine biologist to come to the seaside Willa adored--and discovers the one man who may be able to help her.Compelling and evocative, at once suspenseful, heartbreaking, and triumphant, The Secret Hour is an unforgettable novel that explores the power of sisterly love, the gift of second chances--and the way magic can sometimes be the most real thing in the whole world.From the Hardcover edition."}, "title": "The secret hour", "covers": [374065], "subject_places": ["Connecticut"], "subjects": ["Single-parent families in fiction", "Single-parent families", "Sisters in fiction", "Fiction", "Connecticut in fiction", "Sisters", "Serial murderers", "Legal stories", "Man-woman relationships", "Large type books", "Serial murders, fiction", "Lawyers, fiction", "Widowers, fiction", "Fiction, family life, general", "Fiction, thrillers, suspense", "Connecticut, fiction", "Sisters, fiction"], "key": "/works/OL106217W", "authors": [{"type": {"key": "/type/author_role"}, "author": {"key": "/authors/OL20252A"}}], "type": {"key": "/type/work"}, "genres": ["/tags/OL180T"], "latest_revision": 12, "revision": 12, "created": {"type": "/type/datetime", "value": "2009-10-17T21:25:07.728178"}, "last_modified": {"type": "/type/datetime", "value": "2026-08-29T23:22:17.827432"}} +/type/work /works/OL10703806W 9 2022-02-10T22:48:51.173836 {"title": "Annuaire de la Soci\u00e9t\u00e9 d'Arch\u00e9ologie de Bruxelles, 1 - 1890", "covers": [5778420], "key": "/works/OL10703806W", "authors": [{"author": {"key": "/authors/OL4473083A"}, "type": {"key": "/type/author_role"}}], "type": {"key": "/type/work"}, "description": "L'Annuaire de la SRAB constitue le rapport d'activit\u00e9 de la soci\u00e9t\u00e9. Il a \u00e9t\u00e9 publi\u00e9 jusqu'en 1925. Il ne doit pas \u00eatre confondu avec les Annales.", "latest_revision": 9, "revision": 9, "created": {"type": "/type/datetime", "value": "2009-12-11T03:28:23.366415"}, "last_modified": {"type": "/type/datetime", "value": "2022-02-10T22:48:51.173836"}} +/type/work /works/OL107195W 83 2025-07-09T06:50:10.802609 {"description": {"type": "/type/text", "value": "Bunyan's allegory uses the everyday world of common experience as a metaphor for the spiritual journey of the soul toward God. The hero, Christian, encounters many obstacles in his quest: the Valley of the Shadow of Death, Vanity Fair, Doubting Castle, the Wicket Gate, as well as those who tempt him from his path (e.g., Talkative, Mr. Worldly Wiseman, the Giant Despair). But in the end he reaches Beulah Land, where he awaits the crossing of the river of death and his entry into the heavenly city. \"Pilgrim's Progress\" was enormously influential not only as a best-selling inspirational tract in the late 17th century, but as an ancestor of the 18th-century English novel, and many of its themes and ideas have entered permanently into Western culture."}, "title": "The Pilgrim's Progress", "covers": [8229224, 6401719, 5627463, 5908886, 899978, 680273, 709354, 536725, 6070009, 229586, 295725, 6270659, 5912733, 6070277, 5574488, 4813143, 2822183, 569712, 562303, 6030384, 8816781, 9153070, 10670474, 10835487, 11417801, 2294016, 3332074, 1568784, 5018847, 6851797, 7231236, 7714216, 7807848, 12091605, 2479536, 2656919, 7271072, 8102716, 8426827, 11031336, 528907, 570455, 10719133, 10881125, 11051020, 8016792, 8021539, 7269262, 8038866, 8177977, 914246, 7243632, 7439469, 10514998, 6783387, 6787294, 6814347, 6848694, 6855329, 6857938, 7001915, 7023825, 7021534, 3044406, 2402217, 536738, 2804886, 780320, 10983138, 732428, 902392, 3055104, 832884, 7713865, 7890171, 10983118, 11423513, 6783375, 6783376, 6785075, 6814032, 6848688, 6848886, 7433776, 7519762, 8587261, 12017753, 7174383, 7616052, 7703894, 10226491, 10516196, 6814344, 6814345, 6848689, 6921642, 6942210, 6948431, 6949468, 8527646, 8529054, 8154200, 8186761, 8197580, 8208406, 8609199, 876709, 11515582], "subject_places": ["England"], "subjects": ["Bibliography", "Readers", "Fiction", "Spiritual life", "Church of England", "Christian biography", "German language", "Aneityumese language", "Cree (Langue)", "Christian life", "Aneityumese Catechisms", "English Authors", "Benga", "Ephrata Cloister", "English Poets", "Classic Literature", "Germans", "Aneityumese Hymns", "Open Library Staff Picks", "Pilgrims and pilgrimages", "Textes", "Clergy", "Salvation", "Allegories", "Portraits", "Juvenile fiction", "Puritan movements", "Puritans in literature", "Cree language", "Texts", "Christian pilgrims and pilgrimages", "Puritans", "African languages", "Xhosa language", "Biography", "Railroads", "Canadian National railways", "Christian pilgrims and pilgrimages in literature", "Pilgrim's progress (Bunyan, John)", "British and irish fiction (fictional works by one author)", "Christian life, fiction", "God, worship and love", "Early works to 1800", "Fiction, christian, historical", "Juvenile literature", "Fiction, general", "Fiction, christian, classic & allegory", "Bible, study", "Bible, juvenile literature", "Children's fiction", "Fiction, religious", "Fiction, christian, general", "English fiction", "Christian fiction", "Reindeer", "Christian pilgrims and pilgrimages in fiction", "Fiction, historical, general", "Romans", "Vie chr\u00e9tienne", "English literature", "Syriac language", "Repentance", "Christianity", "Translations into Kafir (Bantu)", "Kafir language (Bantu)", "English Christian fiction", "Male authors", "Puritan authors", "Limited editions", "Specimens", "Religion", "Miniature books", "Bunyan, john , 1628-1688", "Christian pilgrims and pilgrimages--fiction", "Authors, english", "Authors, english--early modern, 1500-1700--biography", "Puritans--england--biography", "Christian biography--england", "Pr3330.a2 t48 2004", "823/.4", "Cri (Langue)", "Kankanay language", "Botolan Sambal dialect", "Library", "Personal copy", "Tagbanua language", "Health", "Diarrhea", "Treatment", "Gonadal Steroid Hormones", "Embryology", "Fetus", "Hormones", "Pregnancy", "Dreams", "Poverty", "Adventure and adventurers", "Giants", "Christians", "Conduct of life", "Methodist authors"], "subject_people": ["John Bunyan (1628-1688)", "George Herbert (1593-1633)", "John Donne (1572-1631)", "Walter Paget"], "key": "/works/OL107195W", "authors": [{"author": {"key": "/authors/OL18112A"}, "type": {"key": "/type/author_role"}}], "subject_times": ["Early modern, 1500-1700", "1800-1870"], "type": {"key": "/type/work"}, "first_publish_date": "1774", "latest_revision": 83, "revision": 83, "created": {"type": "/type/datetime", "value": "2009-10-17T21:38:03.501896"}, "last_modified": {"type": "/type/datetime", "value": "2025-07-09T06:50:10.802609"}} +/type/work /works/OL109166W 8 2026-02-09T12:03:39.838343 {"subtitle": "Chapters On The Sex-Discord", "covers": [5889970, 6163903], "key": "/works/OL109166W", "authors": [{"type": {"key": "/type/author_role"}, "author": {"key": "/authors/OL29586A"}}], "title": "Our Women", "subjects": ["Women", "Sex role", "Social conditions", "Man-woman relationships"], "type": {"key": "/type/work"}, "first_publish_date": "1920", "latest_revision": 8, "revision": 8, "created": {"type": "/type/datetime", "value": "2009-10-17T23:16:23.155408"}, "last_modified": {"type": "/type/datetime", "value": "2026-02-09T12:03:39.838343"}} +/type/work /works/OL11314941W 7 2022-11-17T10:23:35.158462 {"description": "About the book... Easy Connections.\r\n\r\n'No more exams. No more boring holiday work. Just two blissful weeks painting in the country - then, art college at last. Just paint, paint, paint!'\r\n\r\nCathy Harlow is a gifted painter. She is seventeen and three glorious years at art college stretch blissfully ahead of her.\r\n\r\nBut when she meets Paul Devlin, lead guitarist of the rock group Easy Connection and a millionaire superstar her dreams are shattered. Dev is beautiful, brilliant, and explosively violent. Cathy is attracted and repelled in equal measure, but Dev is determined to have her, and Dev usually gets what he wants...\r\n\r\nEasy Connections is a powerful and compelling novel, a love story with a difference, set against a vivid background of art school and the larger-than-life world of successful rock stars.'", "title": "Easy connections", "covers": [7216324], "subject_places": ["London", "America"], "subjects": ["Rock music", "Sex", "Pregnancy", "Love triangle", "Rape", "Art", "Children's fiction", "Friendship, fiction"], "key": "/works/OL11314941W", "authors": [{"type": {"key": "/type/author_role"}, "author": {"key": "/authors/OL656038A"}}], "subject_times": ["1980's"], "type": {"key": "/type/work"}, "latest_revision": 7, "revision": 7, "created": {"type": "/type/datetime", "value": "2009-12-11T04:26:30.476274"}, "last_modified": {"type": "/type/datetime", "value": "2022-11-17T10:23:35.158462"}} +/type/work /works/OL11675949W 7 2026-08-31T04:15:19.397201 {"title": "Hong lou meng", "key": "/works/OL11675949W", "authors": [{"author": {"key": "/authors/OL683896A"}, "type": {"key": "/type/author_role"}}, {"author": {"key": "/authors/OL7553925A"}, "type": {"key": "/type/author_role"}}, {"author": {"key": "/authors/OL7625969A"}, "type": {"key": "/type/author_role"}}], "type": {"key": "/type/work"}, "subjects": ["Chinese fiction", "Translations into English", "China, fiction", "Fiction, general", "Near and far eastern fiction (fictional works by one author)", "Fiction", "History", "Baoyu Jia (Fictitious character)", "Upper class", "Social life and customs", "Families", "Women", "Popular Print Disabled Books", "Qing Dynasty (China)", "Fiction, historical", "Chia, pao-yu (fictitious character), fiction", "Fiction, historical, general", "Chinese Classical fiction", "Chu zhong", "Ke wai du wu", "Chang pian xiao shuo", "Zhong xue yu wen ke", "Zhang hui xiao shuo", "Gu dian xiao shuo", "Illustrations", "Translation into English"], "covers": [6655966, 238206, 6492648, 8751037, 8759216, 8773466, 10750061], "subject_places": ["China", "Zhong guo"], "subject_people": ["Xueqin Cao (ca. 1717-1763)", "Gao, E (ca. 1738-ca. 1815)", "Xueqin Cao (approximately 1717-1763)"], "subject_times": ["Qing dynasty, 1644-1912", "Ch\u02bbing dynasty, 1644-1912", "18th century", "Qing dai"], "description": "Ben shu shi wo guo si da gu dian ming zhu zhi yi, yi jia bao yu, lin dai yu, xue bao chai de ai qing jiu ge wei xian suo, yi jia, shi, wang, xue si da jia zu wei zhong xin, yi qing chao feng jian she hui wei bei jing, xie chu le feng jian da jia zu de xing shuai, tong shi ye zhe she chu wo guo feng jian she hui xing shuai de li shi.", "genres": ["/tags/OL170T"], "latest_revision": 7, "revision": 7, "created": {"type": "/type/datetime", "value": "2009-12-11T05:06:24.468834"}, "last_modified": {"type": "/type/datetime", "value": "2026-08-31T04:15:19.397201"}} +/type/work /works/OL11688780W 9 2026-08-28T13:02:59.474466 {"first_publish_date": "1985", "title": "Human error", "covers": [6699544], "lc_classifications": ["PS3566.R416 H86 1985"], "key": "/works/OL11688780W", "authors": [{"type": {"key": "/type/author_role"}, "author": {"key": "/authors/OL384687A"}}], "dewey_number": ["813/.54"], "type": {"key": "/type/work"}, "subjects": ["Science fiction", "Fiction, science fiction, hard science fiction"], "genres": ["/tags/OL179T"], "latest_revision": 9, "revision": 9, "created": {"type": "/type/datetime", "value": "2009-12-11T05:08:08.675825"}, "last_modified": {"type": "/type/datetime", "value": "2026-08-28T13:02:59.474466"}} +/type/work /works/OL1194091W 3 2010-04-28T07:25:45.460388 {"last_modified": {"type": "/type/datetime", "value": "2010-04-28T07:25:45.460388"}, "title": "The History of Rome, V2", "created": {"type": "/type/datetime", "value": "2009-12-09T20:42:01.245695"}, "covers": [2899503], "first_publish_date": "August 9, 2007", "latest_revision": 3, "key": "/works/OL1194091W", "authors": [{"type": "/type/author_role", "author": {"key": "/authors/OL120597A"}}], "type": {"key": "/type/work"}, "revision": 3} +/type/work /works/OL1195665W 4 2023-10-17T02:16:22.781770 {"key": "/works/OL1195665W", "title": "The Universal Church", "first_publish_date": "January 1, 1967", "authors": [{"type": {"key": "/type/author_role"}, "author": {"key": "/authors/OL120702A"}}], "type": {"key": "/type/work"}, "covers": [13794200], "subjects": ["Religion and sociology"], "latest_revision": 4, "revision": 4, "created": {"type": "/type/datetime", "value": "2009-12-09T20:42:11.538566"}, "last_modified": {"type": "/type/datetime", "value": "2023-10-17T02:16:22.781770"}} +/type/work /works/OL1197509W 2 2010-01-21T23:50:04.403274 {"key": "/works/OL1197509W", "created": {"type": "/type/datetime", "value": "2009-12-09T20:42:30.295689"}, "title": "Fighting for Freedom", "first_publish_date": "January 2005", "latest_revision": 2, "last_modified": {"type": "/type/datetime", "value": "2010-01-21T23:50:04.403274"}, "authors": [{"type": "/type/author_role", "author": {"key": "/authors/OL120890A"}}], "type": {"key": "/type/work"}, "revision": 2} +/type/work /works/OL1201380W 2 2010-01-22T02:56:57.029231 {"key": "/works/OL1201380W", "created": {"type": "/type/datetime", "value": "2009-12-09T20:43:07.045478"}, "title": "Electronic Dsm-IV Version 2.O Plus", "first_publish_date": "October 1997", "latest_revision": 2, "last_modified": {"type": "/type/datetime", "value": "2010-01-22T02:56:57.029231"}, "authors": [{"type": "/type/author_role", "author": {"key": "/authors/OL121260A"}}], "type": {"key": "/type/work"}, "revision": 2} +/type/work /works/OL1205831W 4 2022-06-17T03:02:39.215178 {"title": "Asia in the Making of Europe, Volume II: A Century of Wonder. Book 1", "covers": [1148115], "first_publish_date": "September 1, 1970", "key": "/works/OL1205831W", "authors": [{"type": {"key": "/type/author_role"}, "author": {"key": "/authors/OL121772A"}}], "type": {"key": "/type/work"}, "subjects": ["Asia, history", "East and west"], "latest_revision": 4, "revision": 4, "created": {"type": "/type/datetime", "value": "2009-12-09T20:43:50.169177"}, "last_modified": {"type": "/type/datetime", "value": "2022-06-17T03:02:39.215178"}} +/type/work /works/OL120655W 3 2010-04-28T07:28:04.087299 {"last_modified": {"type": "/type/datetime", "value": "2010-04-28T07:28:04.087299"}, "created": {"type": "/type/datetime", "value": "2009-10-18T08:31:40.736268"}, "title": "The Experiences Of A Planter In The Jungles Of Mysore V1", "covers": [2874221], "first_publish_date": "July 25, 2007", "latest_revision": 3, "key": "/works/OL120655W", "authors": [{"type": {"key": "/type/author_role"}, "author": {"key": "/authors/OL1237574A"}}], "type": {"key": "/type/work"}, "revision": 3} +/type/work /works/OL12081419W 4 2024-08-16T12:49:54.965469 {"title": "National Exhibition Children's Art 1964", "first_publish_date": "1964", "key": "/works/OL12081419W", "type": {"key": "/type/work"}, "latest_revision": 4, "revision": 4, "created": {"type": "/type/datetime", "value": "2009-12-11T06:03:46.789867"}, "last_modified": {"type": "/type/datetime", "value": "2024-08-16T12:49:54.965469"}} +/type/work /works/OL1220129W 3 2021-08-15T14:57:17.481869 {"key": "/works/OL1220129W", "title": "Historic Decisions of the Supreme Court", "first_publish_date": "June 1940", "authors": [{"type": {"key": "/type/author_role"}, "author": {"key": "/authors/OL123337A"}}], "type": {"key": "/type/work"}, "subjects": ["Law reports, digests", "Cases", "Constitutional law", "United States. Supreme Court"], "covers": [11644768], "latest_revision": 3, "revision": 3, "created": {"type": "/type/datetime", "value": "2009-12-09T20:46:36.446984"}, "last_modified": {"type": "/type/datetime", "value": "2021-08-15T14:57:17.481869"}} +/type/work /works/OL1220346W 2 2010-01-22T17:43:03.868408 {"key": "/works/OL1220346W", "created": {"type": "/type/datetime", "value": "2009-12-09T20:46:36.446984"}, "title": "Commission on Human Rights (United Nations Economic & Social Council Official Records, 1996)", "first_publish_date": "April 1997", "latest_revision": 2, "last_modified": {"type": "/type/datetime", "value": "2010-01-22T17:43:03.868408"}, "authors": [{"type": "/type/author_role", "author": {"key": "/authors/OL123345A"}}], "type": {"key": "/type/work"}, "revision": 2} +/type/work /works/OL13272970W 9 2026-01-27T14:49:01.555836 {"key": "/works/OL13272970W", "title": "Murder, London - New York", "first_publish_date": "1976", "authors": [{"author": {"key": "/authors/OL397914A"}, "type": {"key": "/type/author_role"}}], "dewey_number": ["823/.9/1J"], "type": {"key": "/type/work"}, "subjects": ["Children's stories", "Readers", "English language, juvenile literature"], "covers": [-1], "latest_revision": 9, "revision": 9, "created": {"type": "/type/datetime", "value": "2009-12-11T09:04:57.900578"}, "last_modified": {"type": "/type/datetime", "value": "2026-01-27T14:49:01.555836"}} +/type/work /works/OL14910309W 7 2020-04-09T07:36:44.305041 {"created": {"type": "/type/datetime", "value": "2010-03-12T07:37:55.760061"}, "subjects": ["Religious life", "Teenagers", "Children: Grades 1-2"], "latest_revision": 7, "key": "/works/OL14910309W", "title": "Chicken Soup for the Soul Teens Talkin Faith (Chicken Soup for the Soul)", "authors": [{"type": {"key": "/type/author_role"}, "author": {"key": "/authors/OL1658765A"}}], "type": {"key": "/type/work"}, "last_modified": {"type": "/type/datetime", "value": "2020-04-09T07:36:44.305041"}, "covers": [-1], "revision": 7} +/type/work /works/OL15017584W 2 2024-08-16T17:49:15.248209 {"title": "Antologia Comentada De La Literatura Espa\u00f1ola S. Xvi", "key": "/works/OL15017584W", "type": {"key": "/type/work"}, "latest_revision": 2, "revision": 2, "created": {"type": "/type/datetime", "value": "2010-03-19T04:42:23.376010"}, "last_modified": {"type": "/type/datetime", "value": "2024-08-16T17:49:15.248209"}} +/type/work /works/OL15017736W 2 2024-08-16T17:42:59.231055 {"title": "\u00a1Cada Vez Mas Simple!", "key": "/works/OL15017736W", "type": {"key": "/type/work"}, "latest_revision": 2, "revision": 2, "created": {"type": "/type/datetime", "value": "2010-03-19T04:43:47.791409"}, "last_modified": {"type": "/type/datetime", "value": "2024-08-16T17:42:59.231055"}} diff --git a/packages/adapters/test/fixtures/podcastindex-catalog-head.txt b/packages/adapters/test/fixtures/podcastindex-catalog-head.txt new file mode 100644 index 0000000..bfdd508 --- /dev/null +++ b/packages/adapters/test/fixtures/podcastindex-catalog-head.txt @@ -0,0 +1,12 @@ +HTTP/2 200 +date: Sun, 13 Sep 2026 10:34:49 GMT +content-type: application/gzip +content-length: 1826623856 +accept-ranges: bytes +last-modified: Sat, 12 Sep 2026 23:24:31 GMT +x-rgw-object-type: Normal +etag: "b60fa45859813600fc0320e710b1a73f-117" +x-amz-tagging-count: 0 +cf-cache-status: DYNAMIC +server: cloudflare + diff --git a/packages/adapters/test/fixtures/podcastindex-catalog-listing.txt b/packages/adapters/test/fixtures/podcastindex-catalog-listing.txt new file mode 100644 index 0000000..334a22f --- /dev/null +++ b/packages/adapters/test/fixtures/podcastindex-catalog-listing.txt @@ -0,0 +1 @@ +-rw-r--r-- dave/dave 5094928384 2026-09-12 23:19 ./podcastindex_feeds.db diff --git a/packages/adapters/test/fixtures/podcastindex-catalog-newsfeeds.sql b/packages/adapters/test/fixtures/podcastindex-catalog-newsfeeds.sql new file mode 100644 index 0000000..cc6eb13 --- /dev/null +++ b/packages/adapters/test/fixtures/podcastindex-catalog-newsfeeds.sql @@ -0,0 +1,73 @@ +CREATE TABLE `newsfeeds` ( + `id` bigint NOT NULL AUTO_INCREMENT, + `title` varchar(768) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL DEFAULT '', + `url` varchar(768) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL DEFAULT '', + `lastcheck` int NOT NULL DEFAULT '0', + `lastupdate` int NOT NULL DEFAULT '0', + `lastmod` int NOT NULL DEFAULT '0', + `createdon` int NOT NULL DEFAULT '0', + `content` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + `link` varchar(768) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL DEFAULT '', + `errors` int NOT NULL DEFAULT '0', + `updated` tinyint NOT NULL DEFAULT '0', + `lastitemid` varchar(768) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT '', + `pubdate` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL DEFAULT '', + `contenthash` varchar(40) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT '', + `lasthttpstatus` int NOT NULL DEFAULT '0', + `lastgoodhttpstatus` int NOT NULL DEFAULT '0', + `dead` tinyint NOT NULL DEFAULT '0', + `contenttype` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL DEFAULT '', + `itunes_id` bigint DEFAULT NULL, + `duplicateof` bigint DEFAULT NULL, + `original_url` varchar(768) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT '', + `artwork_url_600` varchar(768) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT '', + `description` mediumtext CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL, + `itunes_author` varchar(2048) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL, + `itunes_owner_email` varchar(1024) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL, + `itunes_owner_name` varchar(1024) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL, + `itunes_new_feed_url` varchar(768) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, + `explicit` tinyint NOT NULL DEFAULT '0', + `image` varchar(768) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL DEFAULT '', + `itunes_type` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL, + `type` tinyint NOT NULL DEFAULT '0', + `generator` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL, + `parse_errors` int NOT NULL DEFAULT '0', + `lastparse` int NOT NULL DEFAULT '0', + `pullnow` tinyint NOT NULL DEFAULT '0' COMMENT 'Scan this feed immediately.', + `parsenow` tinyint NOT NULL DEFAULT '0' COMMENT 'Scan this feed immediately.', + `newest_item_pubdate` int NOT NULL DEFAULT '0', + `update_frequency` tinyint NOT NULL DEFAULT '0', + `priority` tinyint NOT NULL DEFAULT '0', + `language` varchar(8) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL DEFAULT '' COMMENT 'Language in the feed.', + `detected_language` varchar(8) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL DEFAULT '' COMMENT 'Language we detected.', + `chash` varchar(40) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT '', + `oldest_item_pubdate` int NOT NULL DEFAULT '0', + `item_count` int NOT NULL DEFAULT '0', + `popularity` int NOT NULL DEFAULT '0', + `podcast_chapters` varchar(768) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT '', + `podcast_locked` tinyint NOT NULL DEFAULT '0', + `podcast_owner` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL DEFAULT '', + PRIMARY KEY (`id`), + UNIQUE KEY `url` (`url`), + KEY `title` (`title`), + KEY `itunes_id` (`itunes_id`), + KEY `updated` (`updated`), + KEY `errors` (`errors`), + KEY `lasthttpstatus` (`lasthttpstatus`), + KEY `lastgoodhttpstatus` (`lastgoodhttpstatus`), + KEY `dead` (`dead`), + KEY `original_url` (`original_url`), + KEY `lastcheck` (`lastcheck`), + KEY `lastupdate` (`lastupdate`), + KEY `pullnow` (`pullnow`), + KEY `parsenow` (`parsenow`), + KEY `newest_item_pubdate` (`newest_item_pubdate`), + KEY `update_frequency` (`update_frequency`), + KEY `language` (`language`), + KEY `priority` (`priority`), + KEY `chash` (`chash`), + KEY `item_count` (`item_count`), + KEY `podcast_locked` (`podcast_locked`), + KEY `podcast_owner` (`podcast_owner`), + CONSTRAINT `newsfeeds_ibfk_3` FOREIGN KEY (`itunes_id`) REFERENCES `directory_apple` (`itunes_id`) ON DELETE SET NULL ON UPDATE CASCADE +) ENGINE=InnoDB AUTO_INCREMENT=1336054 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='Main podcasts table' \ No newline at end of file diff --git a/packages/adapters/test/fixtures/podcastindex-catalog-range-head.txt b/packages/adapters/test/fixtures/podcastindex-catalog-range-head.txt new file mode 100644 index 0000000..16da18d --- /dev/null +++ b/packages/adapters/test/fixtures/podcastindex-catalog-range-head.txt @@ -0,0 +1,13 @@ +HTTP/2 206 +date: Sun, 13 Sep 2026 10:30:06 GMT +content-type: application/gzip +content-length: 4194304 +content-range: bytes 0-4194303/1826623856 +accept-ranges: bytes +last-modified: Sat, 12 Sep 2026 23:24:31 GMT +x-rgw-object-type: Normal +etag: "b60fa45859813600fc0320e710b1a73f-117" +x-amz-tagging-count: 0 +cf-cache-status: DYNAMIC +server: cloudflare + diff --git a/packages/adapters/test/fixtures/podcastindex-catalog-schema.sql b/packages/adapters/test/fixtures/podcastindex-catalog-schema.sql new file mode 100644 index 0000000..ff3e9dc --- /dev/null +++ b/packages/adapters/test/fixtures/podcastindex-catalog-schema.sql @@ -0,0 +1,44 @@ +CREATE TABLE podcasts ( + id INTEGER PRIMARY KEY, + url TEXT NOT NULL UNIQUE, + title TEXT NOT NULL, + lastUpdate INTEGER, + link TEXT NOT NULL, + lastHttpStatus INTEGER, + dead INTEGER, + contentType TEXT NOT NULL, + itunesId INTEGER, + originalUrl TEXT NOT NULL, + itunesAuthor TEXT NOT NULL, + itunesOwnerName TEXT NOT NULL, + explicit INTEGER, + imageUrl TEXT NOT NULL, + itunesType TEXT NOT NULL, + generator TEXT NOT NULL, + newestItemPubdate INTEGER, + language TEXT NOT NULL, + oldestItemPubdate INTEGER, + episodeCount INTEGER, + popularityScore INTEGER, + priority INTEGER, + createdOn INTEGER, + updateFrequency INTEGER, + chash TEXT NOT NULL, + host TEXT NOT NULL, + newestEnclosureUrl TEXT NOT NULL, + podcastGuid TEXT NOT NULL, + description TEXT NOT NULL, + category1 TEXT NOT NULL, + category2 TEXT NOT NULL, + category3 TEXT NOT NULL, + category4 TEXT NOT NULL, + category5 TEXT NOT NULL, + category6 TEXT NOT NULL, + category7 TEXT NOT NULL, + category8 TEXT NOT NULL, + category9 TEXT NOT NULL, + category10 TEXT NOT NULL, + newestEnclosureDuration INTEGER, + podcastId INTEGER, + duplicateOf INTEGER +); diff --git a/packages/auth/package.json b/packages/auth/package.json index e461271..8c74166 100644 --- a/packages/auth/package.json +++ b/packages/auth/package.json @@ -1,6 +1,6 @@ { "name": "@nichedb/auth", - "version": "0.23.0", + "version": "0.24.0", "private": true, "type": "module", "exports": { diff --git a/packages/config/package.json b/packages/config/package.json index 0532796..a222044 100644 --- a/packages/config/package.json +++ b/packages/config/package.json @@ -1,6 +1,6 @@ { "name": "@nichedb/config", - "version": "0.23.0", + "version": "0.24.0", "private": true, "type": "module", "exports": { diff --git a/packages/config/src/index.js b/packages/config/src/index.js index fd939d7..be461c7 100644 --- a/packages/config/src/index.js +++ b/packages/config/src/index.js @@ -75,6 +75,12 @@ export const config = { budget: num('INGEST_DETAIL_BUDGET', 150), /** Sweep every enabled source on boot regardless of next_run_at. */ onBoot: bool('INGEST_ON_BOOT', false), + /** + * Where bulk dumps land on disk (a multi-gigabyte tarball, the file it + * inflates to). Empty means the OS temp dir, which is ephemeral; point it at + * a mounted volume so a redeploy mid-walk does not re-download everything. + */ + dataDir: opt('INGEST_DATA_DIR', ''), }, enrich: { diff --git a/packages/core/package.json b/packages/core/package.json index 1e7b0c1..57cd919 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,6 +1,6 @@ { "name": "@nichedb/core", - "version": "0.23.0", + "version": "0.24.0", "private": true, "type": "module", "exports": { @@ -9,7 +9,8 @@ "./http": "./src/http.js", "./names": "./src/names.js", "./opensite": "./src/opensite.js", - "./profiles": "./src/profiles.js" + "./profiles": "./src/profiles.js", + "./dump": "./src/dump.js" }, "dependencies": { "@nichedb/adapters": "workspace:*", diff --git a/packages/core/src/adapter.js b/packages/core/src/adapter.js index f427b2e..cb02ef8 100644 --- a/packages/core/src/adapter.js +++ b/packages/core/src/adapter.js @@ -17,8 +17,14 @@ import { canonicalUrl } from './canonical.js'; * @property {string} [docs] upstream documentation URL * @property {string[]} kinds item kinds this adapter emits * @property {number} [cadenceMinutes=60] + * @property {number} [budgetMs] wall-clock budget for one run, replacing the + * deployment's INGEST_RUN_DEADLINE_MS. Declare it on a + * walk that takes longer than a few minutes (a dump); + * the reaper window and the job lock widen to the + * largest budget any adapter declares. The core hands + * it back as `deadline` and expects pull to stop there. * @property {ConfigField[]} [configFields] - * @property {(ctx: PullContext) => Promise} pull + * @property {(ctx: PullContext) => Promise | AsyncIterable} pull * * @typedef {object} ConfigField * @property {string} key @@ -33,7 +39,7 @@ import { canonicalUrl } from './canonical.js'; * @property {object} config the source's config, merged over the adapter's defaults * @property {object} cursor whatever pull returned as cursor last time, or {} * @property {object} env deployment secrets the adapter may need (tokens) - * @property {object} http fetchJson / fetchText helpers with UA and timeout + * @property {object} http json / text / request / download, with UA and timeout * @property {(msg: string) => void} log * @property {number} budget detail lookups this run may spend * @property {number} deadline Date.now() past which the adapter should return @@ -41,10 +47,25 @@ import { canonicalUrl } from './canonical.js'; * the `data` this source last wrote for those ids * * @typedef {object} PullResult - * @property {Item[]} items + * @property {Item[] | AsyncIterable} items + * an array is written in one go; an async iterable of + * batches is drained one batch at a time, its cursor + * saved after every batch that carries one, and the + * iterator's RETURN value is read as the final + * { cursor, note, nextInMinutes }. `pull` may also be + * an `async *` generator, which is the same thing. * @property {object} [cursor] resume state for next time * @property {string} [note] one line for the run log * @property {number} [nextInMinutes] override the cadence for the next run only + * + * @typedef {object} Batch + * @property {Item[]} items a few hundred items, the memory a run holds at once + * @property {object} [cursor] where to resume if the process dies after this batch + * is written: the position AFTER these items (a line + * count, a byte offset, an id) plus what identifies the + * file (a dump date, a version), so a new upstream file + * resets the walk. At-least-once is safe: upserts are + * idempotent per (source, externalId). */ const KINDS = new Set(['minute', 'day', 'month', 'year']); diff --git a/packages/core/src/dump.js b/packages/core/src/dump.js new file mode 100644 index 0000000..b0fbdf0 --- /dev/null +++ b/packages/core/src/dump.js @@ -0,0 +1,290 @@ +import { Database } from 'bun:sqlite'; +import { createReadStream } from 'node:fs'; +import { mkdir } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { pipeline } from 'node:stream'; +import { createGunzip } from 'node:zlib'; +import { config } from '@nichedb/config'; + +/** + * Reading bulk dumps without holding them. + * + * A dump adapter downloads a file of gigabytes (`http.download`), then walks it + * as lines, a batch at a time, and yields those batches to the core with a + * cursor that says how far it got. Everything here is an async generator over + * a byte stream: at any moment the memory held is one decompressor chunk, the + * tail of the line being assembled and the line being handed over. The whole + * inflated file never exists in memory and, for the compressed readers, never + * on disk either. + * + * Resuming has a cost and it differs by reader, so each one says what it is. + */ + +/** + * The directory a dump lives in, created. Under `INGEST_DATA_DIR` when the + * deployment sets one (a mounted volume), else the OS temp dir, which a + * redeploy wipes: the adapter's cursor, not the directory, is the walk. + */ +export async function dumpDir(name) { + if (!/^[a-z0-9][a-z0-9._-]*$/i.test(String(name))) { + throw new Error(`dump dir name ${name} must be a slug`); + } + const dir = join(config.ingest.dataDir || join(tmpdir(), 'nichedb-dumps'), name); + await mkdir(dir, { recursive: true }); + return dir; +} + +/** + * Lines of a gzip file, streamed through node:zlib. + * + * Every line is yielded, empty ones included, and the final line is flushed + * whether or not the file ends in a newline (MusicBrainz's does not), so + * `skip + lines yielded` is always the file position: keep that in the cursor. + * + * Resume cost: `skip` re-inflates from the start of the file and counts + * newlines without decoding, at roughly the inflater's speed (hundreds of MB/s + * of inflated text on one core). Open Library's 20 GB works file is a few + * tens of seconds per gigabyte to walk past, which an hourly run absorbs. + * + * @param {string} path + * @param {{ skip?: number }} [opts] + * @returns {AsyncGenerator} + */ +export async function* gzipLines(path, opts = {}) { + const rs = createReadStream(path); + const gz = createGunzip(); + // pipeline, not pipe: a reader that stops early (every run that hits its + // deadline mid-file) destroys the gunzip side, and pipeline takes the file + // stream down with it. `pipe` leaves that stream, and its descriptor, open. + // Errors on either side reach the consumer through gz's own 'error'. + pipeline(rs, gz, () => {}); + try { + yield* splitLines(gz, opts); + } finally { + rs.destroy(); + gz.destroy(); + } +} + +/** + * Lines of an xz file, or of one member of an xz-compressed tar. + * + * Bun.Archive cannot read xz and buffers a whole archive anyway, so this + * spawns the system tools and streams their stdout: `tar --to-stdout` for a + * member (tar handles the base-256 sizes of members past 8 GB, which + * MusicBrainz's artist file is), `xz -dc` for a plain .xz. Chosen over + * extracting the member to disk because the inflated member is ten times the + * archive (17.5 GB for artist, 374 GB for release) and the disk is ephemeral + * and shared with the web role; an adapter that wants byte-seek resume on a + * member that fits can `untar` it once and read it with `lineOffsetReader`. + * Needs `xz` on the host (xz-utils in the image). + * + * Resume cost: as `gzipLines`, but xz inflates at a fraction of gzip's speed + * (100-200 MB/s), so skipping to the end of the artist member is two to three + * minutes of one core. Fine for an hourly walk that moves hundreds of + * thousands of lines a run; if it is not, extract once and seek. + * + * @param {string} path + * @param {{ member?: string, skip?: number }} [opts] + * @returns {AsyncGenerator} + */ +export function xzLines(path, { member, skip = 0 } = {}) { + const cmd = member + ? ['tar', '--extract', '--xz', '--to-stdout', '--file', path, member] + : ['xz', '-dc', path]; + return processLines(cmd, { skip }); +} + +/** + * Open Library's dump rows: `type \t key \t revision \t last_modified \t JSON`. + * + * The JSON column is the fifth field and the rest of the line, split on the + * first four tabs only. Each record carries its 1-based `lineNo` in the file + * so the cursor can be `{ skip: lineNo }` even though a blank or malformed + * line yields nothing. Reads .gz through `gzipLines`, anything else as plain + * text, with the same resume cost as the reader underneath. + * + * @param {string} path + * @param {{ skip?: number }} [opts] + * @returns {AsyncGenerator<{ type: string, key: string, revision: number, lastModified: string, json: object, lineNo: number }>} + */ +export async function* tsvJsonLines(path, { skip = 0 } = {}) { + const lines = path.endsWith('.gz') ? gzipLines(path, { skip }) : plainLines(path, { skip }); + let lineNo = skip; + for await (const line of lines) { + lineNo += 1; + if (!line) continue; + const cols = splitN(line, '\t', 5); + if (cols.length < 5) continue; + let json; + try { + json = JSON.parse(cols[4]); + } catch (err) { + throw new Error(`${path} line ${lineNo}: ${err.message}`); + } + yield { + type: cols[0], + key: cols[1], + revision: Number(cols[2]), + lastModified: cols[3], + json, + lineNo, + }; + } +} + +/** + * Lines of a plain text file with the position to resume from. + * + * Yields `{ line, lineNo, offset }` where `offset` is the byte just after this + * line's newline: store it and pass it back as `offset` to continue in O(1), + * a seek. `skip` is the other way in, counting lines from `offset` at disk + * speed without decoding, O(bytes skipped). `lineNo` counts from `offset`, + * not from the top of the file, so a cursor is `{ offset }` or `{ skip }`, + * not both. + * + * @param {string} path + * @param {{ offset?: number, skip?: number }} [opts] + * @returns {AsyncGenerator<{ line: string, lineNo: number, offset: number }>} + */ +export async function* lineOffsetReader(path, { offset = 0, skip = 0 } = {}) { + let pos = offset; + let lineNo = 0; + let rest = null; + for await (const chunk of createReadStream(path, { start: offset })) { + let buf = rest ? Buffer.concat([rest, asBuffer(chunk)]) : asBuffer(chunk); + rest = null; + let start = 0; + let idx = buf.indexOf(10, start); + while (idx !== -1) { + lineNo += 1; + pos += idx + 1 - start; + if (lineNo > skip) yield { line: lineText(buf, start, idx), lineNo, offset: pos }; + start = idx + 1; + idx = buf.indexOf(10, start); + } + if (start < buf.length) rest = Buffer.from(buf.subarray(start)); + buf = null; + } + if (rest?.length) { + lineNo += 1; + pos += rest.length; + if (lineNo > skip) yield { line: lineText(rest, 0, rest.length), lineNo, offset: pos }; + } +} + +/** + * Rows of a SQLite file, one at a time (Podcast Index ships one inside a + * tgz). `bun:sqlite` walks the statement lazily, so a five-gigabyte table + * costs one row of memory. Sync, because the driver is. + */ +export function* sqliteRows(dbPath, sqlText, params = []) { + const db = new Database(dbPath, { readonly: true }); + try { + yield* db.query(sqlText).iterate(...params); + } finally { + db.close(); + } +} + +/** + * Extract an archive (tar, tar.gz, tar.xz; tar sniffs the compression) into + * `dir`, optionally only the named members. For the case where the inflated + * file is what you need on disk: a SQLite database, or a member you would + * rather seek through than re-inflate. + */ +export async function untar(path, dir, { members = [] } = {}) { + await mkdir(dir, { recursive: true }); + const proc = Bun.spawn(['tar', '--extract', '--file', path, '--directory', dir, ...members], { + stdout: 'ignore', + stderr: 'pipe', + stdin: 'ignore', + }); + const code = await proc.exited; + if (code !== 0) { + const err = await new Response(proc.stderr).text(); + throw new Error(`tar exited ${code}${err ? `: ${err.trim().slice(0, 200)}` : ''}`); + } +} + +/** + * Byte chunks in, lines out. + * + * The newline is one byte in UTF-8 and never inside a multibyte sequence, so + * the split happens on bytes and each complete line is decoded whole; a + * character straddling two chunks is reassembled before anyone reads it. The + * remainder after the last newline is copied out of the chunk so the chunk + * itself can be freed. A trailing CR is dropped. Lines under `skip` are + * counted, not decoded. + * + * @param {AsyncIterable} chunks + * @param {{ skip?: number }} [opts] + * @returns {AsyncGenerator} + */ +export async function* splitLines(chunks, { skip = 0 } = {}) { + let rest = null; + let seen = 0; + for await (const chunk of chunks) { + let buf = rest ? Buffer.concat([rest, asBuffer(chunk)]) : asBuffer(chunk); + rest = null; + let start = 0; + let idx = buf.indexOf(10, start); + while (idx !== -1) { + seen += 1; + if (seen > skip) yield lineText(buf, start, idx); + start = idx + 1; + idx = buf.indexOf(10, start); + } + if (start < buf.length) rest = Buffer.from(buf.subarray(start)); + buf = null; + } + if (rest?.length) { + seen += 1; + if (seen > skip) yield lineText(rest, 0, rest.length); + } +} + +/** Plain-text lines, the string-only counterpart of `lineOffsetReader`. */ +function plainLines(path, opts) { + return splitLines(createReadStream(path), opts); +} + +/** Lines of a process's stdout; the process is killed if the reader stops early. */ +async function* processLines(cmd, opts) { + const proc = Bun.spawn(cmd, { stdout: 'pipe', stderr: 'pipe', stdin: 'ignore' }); + let finished = false; + try { + yield* splitLines(proc.stdout, opts); + const code = await proc.exited; + finished = true; + if (code !== 0) { + const err = await new Response(proc.stderr).text(); + throw new Error(`${cmd[0]} exited ${code}${err ? `: ${err.trim().slice(0, 200)}` : ''}`); + } + } finally { + if (!finished && proc.exitCode === null) proc.kill(); + } +} + +const asBuffer = (c) => + Buffer.isBuffer(c) ? c : Buffer.from(c.buffer, c.byteOffset, c.byteLength); + +function lineText(buf, start, end) { + const stop = end > start && buf[end - 1] === 13 ? end - 1 : end; + return buf.toString('utf8', start, stop); +} + +/** `s.split(sep)` limited to `n` fields, the last one keeping the rest. */ +function splitN(s, sep, n) { + const out = []; + let from = 0; + while (out.length < n - 1) { + const at = s.indexOf(sep, from); + if (at === -1) break; + out.push(s.slice(from, at)); + from = at + 1; + } + out.push(s.slice(from)); + return out; +} diff --git a/packages/core/src/http.js b/packages/core/src/http.js index 459b542..814d309 100644 --- a/packages/core/src/http.js +++ b/packages/core/src/http.js @@ -7,6 +7,9 @@ * one polite wait rather than a retry storm. */ +import { mkdir, open, stat, unlink } from 'node:fs/promises'; +import { dirname } from 'node:path'; + export function makeHttp({ userAgent, log = () => {} }) { async function request(url, { headers = {}, timeoutMs = 30_000, method = 'GET', body } = {}) { const doFetch = () => @@ -27,8 +30,98 @@ export function makeHttp({ userAgent, log = () => {} }) { return res; } + /** + * Fetch a large file to disk, resuming whatever is already there. + * + * The dumps this serves are 1.7 to 11 GB, so the body is streamed chunk by + * chunk onto an open file handle and never held in memory; the timeout covers + * the whole transfer and defaults to an hour, because `request`'s 30 s is a + * ceiling on an API call, not on a download. The user agent goes with it: + * Podcast Index answers 403 without one. + * + * Resume: a partial file on disk becomes `Range: bytes=-`. A 206 is + * appended to it; a 200 means the server ignored the range (Discogs does) and + * the file starts over; a 416 means the file is already whole. The total from + * Content-Range or Content-Length, when the server gives one, is checked at + * the end: short is `complete: false` (call again to resume), long is an + * error. A connection that drops mid-body throws and leaves the partial file + * where the next call picks it up. + * + * @param {string} url + * @param {string} filePath + * @param {{ timeoutMs?: number, headers?: object, onProgress?: (p: { bytes: number, total: number|null }) => void }} [opts] + * @returns {Promise<{ path: string, bytes: number, complete: boolean }>} + */ + async function download( + url, + filePath, + { timeoutMs = 60 * 60_000, headers = {}, onProgress } = {}, + ) { + await mkdir(dirname(filePath), { recursive: true }); + const have = (await stat(filePath).catch(() => null))?.size ?? 0; + const req = { accept: '*/*', ...headers }; + if (have > 0) req.range = `bytes=${have}-`; + + const res = await request(url, { headers: req, timeoutMs }); + + if (res.status === 416) { + // Nothing left to send. The server's idea of the whole, if it says. + await res.body?.cancel().catch(() => {}); + const total = rangeTotal(res.headers.get('content-range')); + if (total !== null && total !== have) { + // Our file is longer than theirs: a different file under the same name. + await unlink(filePath).catch(() => {}); + throw new Error(`have ${have} bytes of ${url.slice(0, 120)} but the server has ${total}`); + } + return { path: filePath, bytes: have, complete: true }; + } + if (!res.ok) { + await res.body?.cancel().catch(() => {}); + throw new Error(`${res.status} from ${url.slice(0, 120)}`); + } + + let bytes = 0; + let total = null; + let append = false; + if (res.status === 206) { + const cr = res.headers.get('content-range') ?? ''; + const start = Number(cr.match(/^bytes\s+(\d+)-/)?.[1]); + if (start !== have) { + await res.body?.cancel().catch(() => {}); + throw new Error(`asked for bytes=${have}- of ${url.slice(0, 120)}, got ${cr || '206'}`); + } + append = true; + bytes = have; + total = rangeTotal(cr); + } else { + // 200: the whole file, whatever we asked for. + if (have > 0) log(`${new URL(url).host} ignored the range; restarting the download`); + const len = Number(res.headers.get('content-length')); + total = Number.isFinite(len) && len >= 0 && res.headers.has('content-length') ? len : null; + } + + const fh = await open(filePath, append ? 'a' : 'w'); + try { + for await (const chunk of res.body) { + await fh.write(chunk); + bytes += chunk.byteLength; + if (total !== null && bytes > total) { + throw new Error(`${url.slice(0, 120)} sent more than its ${total} bytes`); + } + onProgress?.({ bytes, total }); + } + } finally { + await fh.close(); + } + + const complete = total === null ? true : bytes === total; + if (!complete) log(`${url.slice(0, 120)}: ${bytes} of ${total} bytes so far`); + return { path: filePath, bytes, complete }; + } + return { request, + download, async json(url, opts) { const res = await request(url, opts); if (!res.ok) throw new Error(`${res.status} from ${url.slice(0, 120)}`); @@ -48,3 +141,9 @@ export function makeHttp({ userAgent, log = () => {} }) { }, }; } + +/** The total out of `Content-Range: bytes 0-99/1000` (or the `bytes` star form of a 416), or null. */ +function rangeTotal(header) { + const m = String(header ?? '').match(/\/(\d+)\s*$/); + return m ? Number(m[1]) : null; +} diff --git a/packages/core/src/index.js b/packages/core/src/index.js index 6218ed0..b37c3c2 100644 --- a/packages/core/src/index.js +++ b/packages/core/src/index.js @@ -1,4 +1,5 @@ export * from './adapter.js'; +export * from './dump.js'; export { describeEnrichers, enrichPending } from './enrich.js'; export { scanFeeds } from './feedscan.js'; export { makeHttp } from './http.js'; diff --git a/packages/core/src/ingest.js b/packages/core/src/ingest.js index e529066..e9bcb14 100644 --- a/packages/core/src/ingest.js +++ b/packages/core/src/ingest.js @@ -44,12 +44,18 @@ export function envFor() { * per write. A failure records the error on the source and leaves the cursor * where it was, so the next run resumes rather than skips. */ -export async function runSource(sourceId, { log = console.log } = {}) { +export async function runSource( + sourceId, + { log = console.log, resolveAdapter = adapterByName } = {}, +) { const source = await q.getSourceById(sourceId); if (!source) return { skipped: 'no such source' }; if (!source.enabled) return { skipped: 'disabled' }; - const adapter = adapterByName(source.adapter); + // The registry, unless a caller brings its own: a test runs a fake adapter + // through the real loop without mocking the registry, which is in an import + // cycle with this package and hangs Bun's loader when mocked. + const adapter = resolveAdapter(source.adapter); if (!adapter) { /* * Almost always a deploy in flight rather than a broken source: the new @@ -80,6 +86,10 @@ export async function runSource(sourceId, { log = console.log } = {}) { typeof source.config === 'string' ? JSON.parse(source.config) : source.config; const cursor = typeof source.cursor === 'string' ? JSON.parse(source.cursor) : source.cursor; + // An adapter that declares a budget (a walk of a hundred thousand documents, + // an hour through a dump) gets it; everything else gets the deployment's. + const deadline = started + (adapter.budgetMs ?? config.ingest.runDeadlineMs); + try { const result = await adapter.pull({ config: { ...adapter.defaults, ...(storedConfig ?? {}) }, @@ -88,112 +98,55 @@ export async function runSource(sourceId, { log = console.log } = {}) { http: makeHttp({ userAgent: UA(), log: l }), log: l, budget: config.ingest.budget, - // An adapter that declares a budget (a walk of a hundred thousand - // documents) gets it; everything else gets the deployment's deadline. - deadline: started + (adapter.budgetMs ?? config.ingest.runDeadlineMs), + deadline, // What this source wrote last time for these ids, so an adapter that // keeps a rolling window per item can extend it rather than restate it. previous: (externalIds) => q.previousItemData({ sourceId: source.id, externalIds }), }); - let pulled = (result?.items ?? []).map(normaliseItem).filter(Boolean); - /* - * People are not rows an adapter can write on its own. A document an - * adapter fetched or compiled (openprofiles reads the apps' own files, - * sportarr-persons writes one per athlete from Wikidata) is matched to a - * profile by its identity - * keys, stored as one of that profile's sources, and the profile is - * re-rendered under the owner's overrides; what reaches the collection is - * one row per person, not one per document. Done here because it needs - * the database, which an adapter never sees. - */ - if (Array.isArray(adapter.kinds) && adapter.kinds.includes('openprofile')) { - pulled = await absorbProfiles({ source, pulled, log: l }); - } - - /* - * Drop what another source in this collection already carries. - * - * `(source_id, external_id)` cannot see across sources, so without this a - * collection that aggregates aggregators hands a reader the same story once - * per source that indexed it -- a BBC piece arriving from the newsroom's own - * feed, from GDELT and from two directories that both index the BBC. - * - * Only for a collection that has opted in (the query checks), and never - * against this source's own rows, or a source's second run would discard - * everything its first run wrote. First writer keeps the story, which makes - * the winner a property of source order rather than of luck -- so seed the - * source you would rather read from before the ones that echo it. + * Two shapes come back. A paged API hands over one array and is written in + * one go. A dump hands over an async iterable of `{ items, cursor }` batches + * (or `pull` is itself an async generator) and is drained one batch at a + * time, so a multi-gigabyte file never has more than one batch in memory + * and the cursor each batch carried is on the source before the next one is + * read. The iterator's return value is the run's `{ cursor, note, + * nextInMinutes }`, exactly where the array form puts them. */ - const dedupes = await q.collectionDedupesUrls(source.collection_id); - let items = pulled; - - if (dedupes) { - const claimed = await q.claimedDedupeKeys({ - collectionId: source.collection_id, - sourceId: source.id, - keys: pulled.map((it) => it.dedupeKey), - }); - - /* - * Two folds, because a story arrives twice in two different ways. - * - * Across sources: another source in this collection already carries it. - * Never against this source's own rows, or a second run would discard - * everything the first one wrote. - * - * Within one pull: one publisher can expose the same article through two - * feeds, and an adapter that keys an item on (feed, url) has no way to - * see that -- the keys genuinely differ. Measured on the news collection: - * aiornot.vote publishes `latest-media` and `photorealistic` carrying the - * same posts, which is 5 duplicate URLs in 1,195. The batch fold is what - * the cross-source filter cannot do, because it deliberately ignores this - * source. - * - * First one wins in both, so the winner is a property of order rather - * than of luck. - */ - const seen = new Set(); - items = pulled.filter((it) => { - if (!it.dedupeKey) return true; - if (claimed.has(it.dedupeKey) || seen.has(it.dedupeKey)) return false; - seen.add(it.dedupeKey); - return true; - }); + const batches = asBatches(result); + const totals = { seen: 0, added: 0, updated: 0 }; + let outcome = result; - const dropped = pulled.length - items.length; - if (dropped > 0) l(`${dropped} duplicate ${dropped === 1 ? 'story' : 'stories'} dropped`); + if (batches) { + const walk = await drainBatches({ batches, source, adapter, deadline, totals, log: l }); + // `{ items: walk(), note }` may say something up front; the walk's own + // return value, being later and better informed, wins where both speak. + outcome = { ...(batches === result ? {} : result), ...walk }; + } else { + const w = await writeBatch({ source, adapter, items: result?.items ?? [], log: l }); + totals.seen += w.seen; + totals.added += w.added; + totals.updated += w.updated; } - let added = 0; - let updated = 0; - for (let i = 0; i < items.length; i += 200) { - const r = await q.upsertItems({ - collectionId: source.collection_id, - sourceId: source.id, - items: items.slice(i, i + 200), - }); - added += r.added; - updated += r.updated; - } - - const nextRunAt = result?.nextInMinutes - ? new Date(Date.now() + result.nextInMinutes * 60_000) + const nextRunAt = outcome?.nextInMinutes + ? new Date(Date.now() + outcome.nextInMinutes * 60_000) : null; await q.finishRun({ runId, sourceId: source.id, status: 'ok', - seen: items.length, - added, - updated, - note: result?.note ?? null, - cursor: result?.cursor, + seen: totals.seen, + added: totals.added, + updated: totals.updated, + note: outcome?.note ?? null, + cursor: outcome?.cursor, nextRunAt, }); - l(`seen ${items.length}, added ${added}, updated ${updated} in ${Date.now() - started}ms`); - return { seen: items.length, added, updated }; + l( + `seen ${totals.seen}, added ${totals.added}, updated ${totals.updated} in ${Date.now() - started}ms`, + ); + return { ...totals }; } catch (err) { const message = String(err?.message ?? err).slice(0, 1000); await q.finishRun({ runId, sourceId: source.id, status: 'error', error: message }); @@ -202,6 +155,154 @@ export async function runSource(sourceId, { log = console.log } = {}) { } } +/** + * The async iterable of batches a pull handed back, or null for the array form. + * + * Either `{ items: }` or the pull's own return value being one + * (an `async *pull`). An array is deliberately not one: arrays are sync + * iterables only, so the check is exact. + */ +function asBatches(result) { + if (result && typeof result[Symbol.asyncIterator] === 'function') return result; + const items = result?.items; + if (items && typeof items[Symbol.asyncIterator] === 'function') return items; + return null; +} + +/** + * Walk the batches. Manual iteration rather than `for await`, because the + * generator's return value is the run's outcome and `for await` throws it away. + * + * Two safety nets an adapter should not need but a dump makes cheap to have: + * the cursor from each batch is saved as soon as the batch is written, so the + * worst a crash costs is one batch re-written (upsertItems is idempotent on + * `(source_id, external_id)` and skips unchanged content); and a batch that + * lands past the deadline ends the run, closing the generator so its files and + * processes are released, and asks for the next run in a minute. An adapter + * that stops itself at the deadline (which it should) returns its own outcome + * and never hits the second one. + */ +async function drainBatches({ batches, source, adapter, deadline, totals, log }) { + const it = batches[Symbol.asyncIterator](); + let count = 0; + let lastCursor; + try { + for (;;) { + const { value, done } = await it.next(); + if (done) return value ?? { cursor: lastCursor }; + const w = await writeBatch({ source, adapter, items: value?.items ?? [], log }); + totals.seen += w.seen; + totals.added += w.added; + totals.updated += w.updated; + count += 1; + if (value?.cursor !== undefined) { + lastCursor = value.cursor; + await q.saveCursor(source.id, value.cursor); + } + if (Date.now() > deadline) { + log(`out of time after ${count} batch${count === 1 ? '' : 'es'}; resuming in a minute`); + await it.return?.(); + return { cursor: lastCursor, note: `out of time after ${count} batches`, nextInMinutes: 1 }; + } + } + } catch (err) { + // Let the generator's own cleanup run (a spawned xz, an open file) before + // the error reaches finishRun. Its cursor stays where the last batch left it. + await it.return?.().catch(() => {}); + throw err; + } +} + +/** + * One batch of an adapter's items into the table: normalise, fold duplicates, + * upsert in slices of 200. The whole of a paged pull is one batch; a dump is + * many. Returns the counts finishRun records. + */ +async function writeBatch({ source, adapter, items: raw, log }) { + let pulled = (raw ?? []).map(normaliseItem).filter(Boolean); + + /* + * People are not rows an adapter can write on its own. A document an + * adapter fetched or compiled (openprofiles reads the apps' own files, + * sportarr-persons writes one per athlete from Wikidata) is matched to a + * profile by its identity + * keys, stored as one of that profile's sources, and the profile is + * re-rendered under the owner's overrides; what reaches the collection is + * one row per person, not one per document. Done here because it needs + * the database, which an adapter never sees. + */ + if (Array.isArray(adapter.kinds) && adapter.kinds.includes('openprofile')) { + pulled = await absorbProfiles({ source, pulled, log }); + } + + /* + * Drop what another source in this collection already carries. + * + * `(source_id, external_id)` cannot see across sources, so without this a + * collection that aggregates aggregators hands a reader the same story once + * per source that indexed it -- a BBC piece arriving from the newsroom's own + * feed, from GDELT and from two directories that both index the BBC. + * + * Only for a collection that has opted in (the query checks), and never + * against this source's own rows, or a source's second run would discard + * everything its first run wrote. First writer keeps the story, which makes + * the winner a property of source order rather than of luck -- so seed the + * source you would rather read from before the ones that echo it. + */ + const dedupes = await q.collectionDedupesUrls(source.collection_id); + let items = pulled; + + if (dedupes) { + const claimed = await q.claimedDedupeKeys({ + collectionId: source.collection_id, + sourceId: source.id, + keys: pulled.map((it) => it.dedupeKey), + }); + + /* + * Two folds, because a story arrives twice in two different ways. + * + * Across sources: another source in this collection already carries it. + * Never against this source's own rows, or a second run would discard + * everything the first one wrote. + * + * Within one pull: one publisher can expose the same article through two + * feeds, and an adapter that keys an item on (feed, url) has no way to + * see that -- the keys genuinely differ. Measured on the news collection: + * aiornot.vote publishes `latest-media` and `photorealistic` carrying the + * same posts, which is 5 duplicate URLs in 1,195. The batch fold is what + * the cross-source filter cannot do, because it deliberately ignores this + * source. + * + * First one wins in both, so the winner is a property of order rather + * than of luck. + */ + const seen = new Set(); + items = pulled.filter((it) => { + if (!it.dedupeKey) return true; + if (claimed.has(it.dedupeKey) || seen.has(it.dedupeKey)) return false; + seen.add(it.dedupeKey); + return true; + }); + + const dropped = pulled.length - items.length; + if (dropped > 0) log(`${dropped} duplicate ${dropped === 1 ? 'story' : 'stories'} dropped`); + } + + let added = 0; + let updated = 0; + for (let i = 0; i < items.length; i += 200) { + const r = await q.upsertItems({ + collectionId: source.collection_id, + sourceId: source.id, + items: items.slice(i, i + 200), + }); + added += r.added; + updated += r.updated; + } + return { seen: items.length, added, updated }; +} + /** The openprofiles adapter's documents into the profiles tables; back come the people. */ async function absorbProfiles({ source, pulled, log }) { const people = new Map(); diff --git a/packages/db/package.json b/packages/db/package.json index 8486a83..7dc9c60 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -1,6 +1,6 @@ { "name": "@nichedb/db", - "version": "0.23.0", + "version": "0.24.0", "private": true, "type": "module", "exports": { diff --git a/packages/db/src/queries.js b/packages/db/src/queries.js index 539c35c..574fdef 100644 --- a/packages/db/src/queries.js +++ b/packages/db/src/queries.js @@ -388,10 +388,28 @@ export async function requestRun(id) { * The sources whose turn it is. Ordered by how overdue, so a starved one is * served first after an outage. */ -export async function dueSources({ limit = 20, force = false } = {}) { +export async function dueSources({ limit = 20, force = false, runningMinutes = 30 } = {}) { + /* + * A source whose run is still in flight is not due, whatever its clock says. + * + * `startRun` pushes `next_run_at` one cadence out, which is enough for a run + * shorter than its cadence and for nothing else: an hour-long dump walk on an + * hourly source, a `nextInMinutes: 1` it asked for last time, or a reaper that + * reset the clock, all bring the source back while the first run is still + * writing, and the tick enqueues a second one beside it. The bound keeps this + * from ever parking a source for good: a run older than `runningMinutes` is + * what the reaper marks abandoned on the same tick, so past that it no longer + * counts, and `force` (the boot sweep) gets the same guard because the old + * container may still be draining exactly those runs. + */ return sql` - select id, slug, adapter, next_run_at from sources + select id, slug, adapter, next_run_at from sources s where enabled and (${force} or next_run_at <= now()) + and not exists ( + select 1 from runs r + where r.source_id = s.id and r.status = 'running' + and r.started_at > now() - (${`${runningMinutes} minutes`})::interval + ) order by next_run_at limit ${limit} `; } @@ -475,6 +493,20 @@ export async function finishRun({ `; } +/** + * Checkpoint a run in progress. A dump adapter hands the core its items a batch + * at a time, and the cursor it carried with each batch is written here as soon + * as that batch is in the table, so a container that dies an hour into a walk + * resumes at the last batch written rather than at the start of the file. + * `finishRun` still writes the final cursor when the run ends. + */ +export async function saveCursor(sourceId, cursor) { + await sql` + update sources set cursor = ${JSON.stringify(cursor ?? {})}::jsonb, updated_at = now() + where id = ${sourceId} + `; +} + export async function listRuns(sourceId, { limit = 20 } = {}) { return sql` select * from runs where source_id = ${sourceId} order by started_at desc limit ${limit} diff --git a/packages/enrichers/package.json b/packages/enrichers/package.json index ce45cb6..a36d0e4 100644 --- a/packages/enrichers/package.json +++ b/packages/enrichers/package.json @@ -1,6 +1,6 @@ { "name": "@nichedb/enrichers", - "version": "0.23.0", + "version": "0.24.0", "private": true, "type": "module", "exports": { diff --git a/packages/knowledge/package.json b/packages/knowledge/package.json index c655292..fd9e755 100644 --- a/packages/knowledge/package.json +++ b/packages/knowledge/package.json @@ -1,6 +1,6 @@ { "name": "@nichedb/knowledge", - "version": "0.23.0", + "version": "0.24.0", "private": true, "type": "module", "exports": { diff --git a/packages/notify/package.json b/packages/notify/package.json index a9bd761..1cb9414 100644 --- a/packages/notify/package.json +++ b/packages/notify/package.json @@ -1,6 +1,6 @@ { "name": "@nichedb/notify", - "version": "0.23.0", + "version": "0.24.0", "private": true, "type": "module", "exports": { diff --git a/packages/payments/package.json b/packages/payments/package.json index a26a6b8..d5ca2dd 100644 --- a/packages/payments/package.json +++ b/packages/payments/package.json @@ -1,6 +1,6 @@ { "name": "@nichedb/payments", - "version": "0.23.0", + "version": "0.24.0", "private": true, "type": "module", "exports": { diff --git a/packages/premium/package.json b/packages/premium/package.json index 2649acd..7e51cdc 100644 --- a/packages/premium/package.json +++ b/packages/premium/package.json @@ -1,6 +1,6 @@ { "name": "@nichedb/premium", - "version": "0.23.0", + "version": "0.24.0", "private": true, "type": "module", "exports": { diff --git a/packages/queue/package.json b/packages/queue/package.json index ca9ab13..f1c8fff 100644 --- a/packages/queue/package.json +++ b/packages/queue/package.json @@ -1,6 +1,6 @@ { "name": "@nichedb/queue", - "version": "0.23.0", + "version": "0.24.0", "private": true, "type": "module", "exports": { diff --git a/packages/queue/src/workers.js b/packages/queue/src/workers.js index 014f9b1..b9d4b52 100644 --- a/packages/queue/src/workers.js +++ b/packages/queue/src/workers.js @@ -20,8 +20,11 @@ const longestRunMs = () => * forward as the job begins, so a long run is not re-enqueued by the next tick. */ async function runTick(job) { - await q.reapStaleRuns({ minutes: Math.ceil(longestRunMs() / 60_000) + 10 }); - const due = await q.dueSources({ limit: 50, force: Boolean(job.data?.force) }); + // One window for both: a run younger than this is in flight and must not be + // enqueued again; one older than it was just marked abandoned and may be. + const runningMinutes = Math.ceil(longestRunMs() / 60_000) + 10; + await q.reapStaleRuns({ minutes: runningMinutes }); + const due = await q.dueSources({ limit: 50, force: Boolean(job.data?.force), runningMinutes }); for (const s of due) { await queues.run.add( 'run', diff --git a/test/books-openlibrary-catalog.test.js b/test/books-openlibrary-catalog.test.js new file mode 100644 index 0000000..952f5c5 --- /dev/null +++ b/test/books-openlibrary-catalog.test.js @@ -0,0 +1,760 @@ +import { afterAll, beforeAll, describe, expect, spyOn, test } from 'bun:test'; +import { mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { gzipSync } from 'node:zlib'; + +/** + * Open Library's monthly dumps, walked as batches. + * + * The fixtures are 100 real rows of each dump, lifted from a Range GET of the + * first 4 MB of `ol_dump_authors_latest` and `ol_dump_works_latest` on + * 2026-09-13 (dump 2026-08-31), chosen so the parser meets what the files + * carry: descriptions and bios as strings and as `{ value }`, publish dates + * in every shape, `-1` covers, authors without a name, links, remote ids. + * The network is a fake that writes those rows, gzipped, where the resume + * helper would; the dump directory is a temp dir the tests own. + */ +process.env.DATABASE_URL ??= 'postgres://test:test@localhost:5432/test'; +process.env.SITE_URL ??= 'https://nichedb.test'; + +const { config } = await import('../packages/config/src/index.js'); +const { normaliseItem } = await import('../packages/core/src/adapter.js'); +const { toItem: searchItem } = await import('../packages/adapters/src/openlibrary.js'); +const { + ATTRIBUTION, + authorItem, + BUDGET_MS, + CADENCE_MINUTES, + coverUrl, + datedUrl, + FILES, + isStale, + latestUrl, + localName, + openlibraryCatalog, + parseRow, + publishDate, + resolveVersion, + resumeFrom, + rowItem, + splitN, + staleFiles, + textOf, + trimTo, + USER_AGENT, + versionFromUrl, + workItem, +} = await import('../packages/adapters/src/openlibrary-catalog.js'); + +const fixture = (name) => + readFile(new URL(`../packages/adapters/test/fixtures/${name}`, import.meta.url), 'utf8'); + +const bodies = { + authors: await fixture('openlibrary-catalog-authors.tsv'), + works: await fixture('openlibrary-catalog-works.tsv'), +}; + +/** Built at runtime so the character itself never appears in this file. */ +const EM_DASH = String.fromCharCode(0x2014); + +const VERSION = '2026-08-31'; +const rowsOf = (kind) => bodies[kind].split('\n').filter(Boolean).map(parseRow); +const itemsOf = (kind) => rowsOf(kind).map(rowItem).filter(Boolean); +const rowByKey = (kind, key) => rowsOf(kind).find((r) => r.key === key); + +/** + * A fake openlibrary.org + archive.org. `request` answers the HEAD with the + * mirror URL of the dated file; `download` writes the fixture, gzipped, to + * the path the adapter asked for, or half of it when `complete` says no. + * `fail(n)` decides per request, counting HEADs and downloads together. + */ +function provider({ + version = VERSION, + fail = () => false, + complete = () => true, + files = bodies, + raw = null, +} = {}) { + const calls = []; + const http = { + async request(url, opts) { + calls.push({ kind: 'head', url, method: opts?.method, headers: opts?.headers ?? {} }); + if (fail(calls.length)) throw new Error('connection reset'); + return { + ok: true, + status: 200, + url: `https://ia800909.us.archive.org/1/items/ol_dump_${version}/ol_dump_authors_${version}.txt.gz`, + headers: new Headers({ 'last-modified': 'Wed, 02 Sep 2026 16:01:56 GMT' }), + body: null, + }; + }, + async download(url, path, opts) { + calls.push({ kind: 'download', url, path, headers: opts?.headers ?? {} }); + if (fail(calls.length)) throw new Error('connection reset'); + const kind = url.includes('_authors_') ? 'authors' : 'works'; + const bytes = raw?.[kind] ?? gzipSync(files[kind]); + if (!complete(calls.length, kind)) { + const half = Math.floor(bytes.length / 2); + await writeFile(path, bytes.subarray(0, half)); + return { path, bytes: half, complete: false }; + } + await writeFile(path, bytes); + return { path, bytes: bytes.length, complete: true }; + }, + }; + return { http, calls }; +} + +/** + * Drive the generator by hand: every batch, then the return value. + * + * `stopAfterBatches` is the clock: the deadline is an hour away until that + * many batches have come out, then `Date.now` jumps past it, so the adapter + * meets its deadline exactly where a long walk would, between two batches. + */ +async function run({ + config: cfg = {}, + cursor = {}, + p = provider(), + deadline, + stopAfterBatches = Number.POSITIVE_INFINITY, + log, +} = {}) { + const realNow = Date.now; + let past = false; + const clock = spyOn(Date, 'now').mockImplementation(() => (past ? 1e15 : realNow())); + try { + const gen = openlibraryCatalog.pull({ + config: { batchRows: 40, pauseMs: 0, ...cfg }, + cursor, + env: {}, + http: p.http, + log: log ?? (() => {}), + deadline: + deadline ?? + (Number.isFinite(stopAfterBatches) ? realNow() + 3_600_000 : Number.POSITIVE_INFINITY), + }); + const batches = []; + for (;;) { + const { value, done } = await gen.next(); + if (done) return { batches, outcome: value, calls: p.calls }; + batches.push(value); + if (batches.length >= stopAfterBatches) past = true; + } + } finally { + clock.mockRestore(); + } +} + +const idsOf = (batches) => batches.flatMap((b) => b.items.map((i) => i.externalId)); + +let dir; +let savedDataDir; +beforeAll(async () => { + dir = await mkdtemp(join(tmpdir(), 'nichedb-openlibrary-catalog-')); + savedDataDir = config.ingest.dataDir; + config.ingest.dataDir = dir; +}); +afterAll(async () => { + config.ingest.dataDir = savedDataDir; + await rm(dir, { recursive: true, force: true }); +}); + +const exists = (p) => + stat(p).then( + () => true, + () => false, + ); +const localPath = (kind, version = VERSION) => + join(dir, 'openlibrary-catalog', localName(kind, version)); + +describe('the rows', () => { + test('every fixture line is a five-column row with the JSON whole', () => { + for (const kind of FILES) { + const rows = rowsOf(kind); + expect(rows).toHaveLength(100); + for (const r of rows) { + expect(r).not.toBeNull(); + expect(r.type).toBe(kind === 'authors' ? '/type/author' : '/type/work'); + expect(r.key).toBe(r.json.key); + expect(r.revision).toBeGreaterThan(0); + expect(r.lastModified).toMatch(/^\d{4}-\d{2}-\d{2}T/); + } + } + }); + + test('a cut line, a short line, a blank and garbage are not rows', () => { + const line = bodies.works.split('\n')[0]; + expect(parseRow(line.slice(0, line.length - 20))).toBeNull(); + expect(parseRow('/type/work\t/works/OL1W\t3')).toBeNull(); + expect(parseRow('')).toBeNull(); + expect(parseRow(null)).toBeNull(); + expect(parseRow('')).toBeNull(); + expect(parseRow('/type/work\t/works/OL1W\t3\t2020-01-01T00:00:00\t"just a string"')).toBeNull(); + }); + + test('the JSON column keeps its own tabs', () => { + expect(splitN('a\tb\tc\td\t{"x":"1\t2"}', '\t', 5)).toEqual([ + 'a', + 'b', + 'c', + 'd', + '{"x":"1\t2"}', + ]); + expect(splitN('a\tb', '\t', 5)).toEqual(['a', 'b']); + }); + + test('text fields come as strings or as {value}', () => { + expect(textOf('plain')).toBe('plain'); + expect(textOf({ type: '/type/text', value: 'boxed' })).toBe('boxed'); + expect(textOf(null)).toBe(''); + expect(textOf(42)).toBe(''); + expect(trimTo(' a b ')).toBe('a b'); + expect(trimTo('')).toBeNull(); + const long = trimTo('word '.repeat(200), 600); + expect(long.length).toBeLessThanOrEqual(603); + expect(long.endsWith('...')).toBe(true); + }); + + test('first_publish_date in every shape the dump uses', () => { + const day = (s) => publishDate(s).publishedAt?.toISOString().slice(0, 10); + expect(publishDate('1964')).toMatchObject({ precision: 'year', timeKnown: false }); + expect(day('1964')).toBe('1964-07-01'); + expect(publishDate('June 1940')).toMatchObject({ precision: 'month' }); + expect(day('June 1940')).toBe('1940-06-15'); + expect(publishDate('January 1, 1967')).toMatchObject({ precision: 'day' }); + expect(day('January 1, 1967')).toBe('1967-01-01'); + expect(day('August 9, 2007')).toBe('2007-08-09'); + expect(day('9 August 2007')).toBe('2007-08-09'); + expect(day('1907-02-16')).toBe('1907-02-16'); + expect(day('2005-03')).toBe('2005-03-15'); + expect(publishDate('c. 1850').precision).toBe('year'); + expect(day('c. 1850')).toBe('1850-07-01'); + expect(publishDate('unknown').publishedAt).toBeNull(); + expect(publishDate('').publishedAt).toBeNull(); + expect(publishDate(null).publishedAt).toBeNull(); + }); + + test('covers and photos: an id becomes a URL, -1 and rubbish become nothing', () => { + expect(coverUrl([3146558], 'b')).toBe('https://covers.openlibrary.org/b/id/3146558-M.jpg'); + expect(coverUrl([8445544], 'a')).toBe('https://covers.openlibrary.org/a/id/8445544-M.jpg'); + expect(coverUrl([-1], 'b')).toBeNull(); + expect(coverUrl([-1, 12], 'b')).toBeNull(); + expect(coverUrl([], 'b')).toBeNull(); + expect(coverUrl(null, 'b')).toBeNull(); + expect(coverUrl(['x'], 'b')).toBeNull(); + }); +}); + +describe('the work item', () => { + test('the shape, on a row with a boxed description, a cover and many subjects', () => { + const row = rowByKey('works', '/works/OL1000806W'); + const item = workItem(row); + expect(item.externalId).toBe('/works/OL1000806W'); + expect(item.kind).toBe('book'); + expect(item.title).toBe(row.json.title); + expect(item.summary).toBe(trimTo(row.json.description.value)); + expect(item.summary.length).toBeLessThanOrEqual(603); + expect(item.url).toBe('https://openlibrary.org/works/OL1000806W'); + expect(item.imageUrl).toBe(`https://covers.openlibrary.org/b/id/${row.json.covers[0]}-M.jpg`); + expect(item.tags.slice(0, 2)).toEqual(['book', 'openlibrary']); + const subjectTags = item.tags.filter((t) => t.startsWith('subject:')); + expect(subjectTags).toHaveLength(5); + expect(subjectTags[0]).toBe('subject:fiction'); + expect(item.data).toMatchObject({ + provider: 'openlibrary', + olKey: '/works/OL1000806W', + subjects: row.json.subjects, + authors: row.json.authors.map((a) => a.author.key), + lastModified: row.lastModified, + attribution: ATTRIBUTION, + }); + expect(item.data.authors[0]).toMatch(/^\/authors\/OL\d+A$/); + expect(normaliseItem(item)).not.toBeNull(); + }); + + test('the external id is the one openlibrary-new writes, so the rows merge', () => { + const row = rowByKey('works', '/works/OL1041137W'); + const viaSearch = searchItem({ key: row.json.key, title: row.json.title }); + expect(workItem(row).externalId).toBe(viaSearch.externalId); + expect(workItem(row).url).toBe(viaSearch.url); + }); + + test('publish dates land with the precision the text had', () => { + expect(workItem(rowByKey('works', '/works/OL1041137W'))).toMatchObject({ + precision: 'year', + timeKnown: false, + }); + expect(workItem(rowByKey('works', '/works/OL1041137W')).data.firstPublishDate).toBe('1898'); + const day = workItem(rowByKey('works', '/works/OL1194091W')); + expect(day.precision).toBe('day'); + expect(day.publishedAt.toISOString().slice(0, 10)).toBe('2007-08-09'); + const month = workItem(rowByKey('works', '/works/OL1197509W')); + expect(month.precision).toBe('month'); + expect(month.publishedAt.toISOString().slice(0, 7)).toBe('2005-01'); + const none = workItem(rowByKey('works', '/works/OL10000152W')); + expect(none.publishedAt).toBeNull(); + expect(none.data.firstPublishDate).toBeNull(); + }); + + test('a -1 cover is no image, no authors is an empty list, no description is no summary', () => { + const item = workItem(rowByKey('works', '/works/OL10616291W')); + expect(item.imageUrl).toBeNull(); + expect(workItem(rowByKey('works', '/works/OL12081419W')).data.authors).toEqual([]); + expect(workItem(rowByKey('works', '/works/OL10000152W')).summary).toBeNull(); + }); + + test('a subject list and the places, people and times ride in the data', () => { + const row = rowsOf('works').find((r) => r.json.subject_people?.length); + const item = workItem(row); + expect(item.data.subjectPeople).toEqual(row.json.subject_people); + expect(Array.isArray(item.data.subjectPlaces)).toBe(true); + expect(Array.isArray(item.data.subjectTimes)).toBe(true); + }); + + test('every fixture work is an item the core accepts', () => { + const items = itemsOf('works'); + expect(items).toHaveLength(100); + for (const it of items) { + expect(normaliseItem(it)).not.toBeNull(); + expect(it.externalId).toMatch(/^\/works\/OL\d+W$/); + expect(it.data.attribution).toBe(ATTRIBUTION); + } + }); + + test('no key, no title, or the wrong type is nothing', () => { + const row = rowByKey('works', '/works/OL1041137W'); + expect(workItem({ ...row, json: { ...row.json, title: '' } })).toBeNull(); + expect(workItem({ ...row, json: { ...row.json, key: '/books/OL1M' } })).toBeNull(); + expect(workItem(null)).toBeNull(); + expect(rowItem({ ...row, type: '/type/redirect' })).toBeNull(); + expect(rowItem({ ...row, type: '/type/delete' })).toBeNull(); + }); +}); + +describe('the author item', () => { + test('the shape, on a row with a bio, a photo, links, ids and alternate names', () => { + const row = rowByKey('authors', '/authors/OL1004923A'); + const item = authorItem(row); + expect(item).toMatchObject({ + externalId: '/authors/OL1004923A', + kind: 'author', + title: row.json.name, + url: 'https://openlibrary.org/authors/OL1004923A', + imageUrl: 'https://covers.openlibrary.org/a/id/8445544-M.jpg', + publishedAt: null, + tags: ['author', 'openlibrary'], + }); + expect(item.summary).toBe(trimTo(row.json.bio.value)); + expect(item.data).toEqual({ + provider: 'openlibrary', + olKey: '/authors/OL1004923A', + name: row.json.name, + personalName: row.json.personal_name ?? null, + birthDate: row.json.birth_date, + deathDate: row.json.death_date, + alternateNames: row.json.alternate_names, + links: [ + { + title: 'Open Library subject', + url: 'https://openlibrary.org/subjects/person:dorothea_von_schlegel_(1764-1839)', + }, + ], + remoteIds: { wikidata: 'Q77271', viaf: '95307649', isni: '000000011827805X' }, + revision: row.revision, + lastModified: row.lastModified, + attribution: ATTRIBUTION, + }); + expect(normaliseItem(item)).not.toBeNull(); + }); + + test('only wikidata, viaf and isni are kept of the remote ids', () => { + const item = authorItem(rowByKey('authors', '/authors/OL10075763A')); + expect(item.data.remoteIds).toEqual({ + viaf: '34156809515545121342', + isni: '0000000500680162', + }); + }); + + test('an author without a name is skipped, not thrown', () => { + const nameless = rowsOf('authors').filter((r) => r.json.name === undefined); + expect(nameless.length).toBeGreaterThanOrEqual(3); + for (const r of nameless) expect(authorItem(r)).toBeNull(); + expect(itemsOf('authors')).toHaveLength(100 - nameless.length); + }); + + test('every fixture author with a name is an item the core accepts', () => { + for (const it of itemsOf('authors')) { + expect(normaliseItem(it)).not.toBeNull(); + expect(it.externalId).toMatch(/^\/authors\/OL\d+A$/); + expect(it.summary === null || it.summary.length <= 603).toBe(true); + } + }); +}); + +describe('the cursor', () => { + test('resumeFrom reads what it wrote and starts over on rubbish', () => { + expect(resumeFrom({})).toEqual({ + version: null, + file: 'authors', + line: 0, + lastModifiedWatermark: null, + maxLastModified: null, + done: false, + }); + expect( + resumeFrom({ + version: VERSION, + file: 'works', + line: 12500, + lastModifiedWatermark: '2026-07-30T00:00:00', + maxLastModified: '2026-08-31T01:00:00', + done: false, + }), + ).toEqual({ + version: VERSION, + file: 'works', + line: 12500, + lastModifiedWatermark: '2026-07-30T00:00:00', + maxLastModified: '2026-08-31T01:00:00', + done: false, + }); + expect(resumeFrom({ version: 'latest', file: 'editions', line: -3, done: true })).toMatchObject( + { + version: null, + file: 'authors', + line: 0, + done: false, + }, + ); + }); + + test('stale means modified before the watermark; no watermark means nothing is stale', () => { + expect(isStale('2021-01-01T00:00:00', '2022-01-01T00:00:00')).toBe(true); + expect(isStale('2022-01-01T00:00:00', '2022-01-01T00:00:00')).toBe(false); + expect(isStale('2023-01-01T00:00:00', '2022-01-01T00:00:00')).toBe(false); + expect(isStale('2021-01-01T00:00:00', null)).toBe(false); + expect(isStale(undefined, '2022-01-01T00:00:00')).toBe(false); + }); + + test('the version comes off the mirror URL, never off Last-Modified, and other dumps are stale files', async () => { + expect( + versionFromUrl( + 'https://ia800909.us.archive.org/1/items/ol_dump_2026-08-31/ol_dump_authors_2026-08-31.txt.gz', + ), + ).toBe('2026-08-31'); + expect(versionFromUrl('https://openlibrary.org/data/ol_dump_works_latest.txt.gz')).toBeNull(); + expect(versionFromUrl(null)).toBeNull(); + // Last-Modified is the upload (Sep 2 for the Aug 31 dump): a date taken + // from it would name an archive.org item that does not exist. + const unnamed = provider({}).http; + unnamed.request = async () => ({ + ok: true, + status: 200, + url: 'https://ia800909.us.archive.org/1/items/ol_dump_latest/ol_dump_authors_latest.txt.gz', + headers: new Headers({ 'last-modified': 'Wed, 02 Sep 2026 16:01:56 GMT' }), + body: null, + }); + await expect(resolveVersion(unnamed)).rejects.toThrow(/did not say which dump is latest/); + expect( + staleFiles( + [ + 'ol_dump_authors_2026-07-31.txt.gz', + 'ol_dump_works_2026-08-31.txt.gz', + 'ol_dump_works_2026-08-31.txt.gz.part', + 'notes.txt', + ], + '2026-08-31', + ), + ).toEqual(['ol_dump_authors_2026-07-31.txt.gz']); + expect(latestUrl('works')).toBe('https://openlibrary.org/data/ol_dump_works_latest.txt.gz'); + expect(datedUrl('works', '2026-08-31')).toBe( + 'https://archive.org/download/ol_dump_2026-08-31/ol_dump_works_2026-08-31.txt.gz', + ); + }); +}); + +describe('the walk', () => { + test('a run cut by its deadline stops after one batch; the next resumes from the cursor and completes', async () => { + const authorIds = itemsOf('authors').map((i) => i.externalId); + const workIds = itemsOf('works').map((i) => i.externalId); + + // Run 1: the deadline passes after the first batch, so one batch and out. + const first = await run({ stopAfterBatches: 1 }); + expect(first.batches).toHaveLength(1); + expect(first.batches[0].items).toHaveLength(40); + expect(first.batches[0].items[0].kind).toBe('author'); + expect(first.batches[0].cursor).toMatchObject({ + version: VERSION, + file: 'authors', + done: false, + }); + const line = first.batches[0].cursor.line; + expect(line).toBeGreaterThanOrEqual(40); + expect(first.outcome).toMatchObject({ nextInMinutes: 10 }); + expect(first.outcome.cursor).toEqual(first.batches[0].cursor); + expect(first.outcome.note).toMatch(/out of time at authors line \d+/); + expect(first.calls.map((c) => c.kind)).toEqual(['head', 'download']); + expect(first.calls[0].method).toBe('HEAD'); + for (const c of first.calls) expect(c.headers['user-agent']).toBe(USER_AGENT); + expect(first.calls[1].url).toBe(datedUrl('authors', VERSION)); + expect(first.calls[1].path).toBe(localPath('authors')); + expect(await exists(localPath('authors'))).toBe(true); + + // Run 2: from that cursor, with all the time in the world. + const second = await run({ cursor: first.outcome.cursor }); + const kinds = second.batches.map((b) => [b.items.length, b.cursor.file]); + expect(kinds).toEqual([ + [40, 'authors'], + [17, 'authors'], + [0, 'works'], + [40, 'works'], + [40, 'works'], + [20, 'works'], + ]); + expect(second.batches[2].cursor).toMatchObject({ file: 'works', line: 0, version: VERSION }); + expect(second.batches[3].cursor.line).toBeGreaterThanOrEqual(40); + for (const b of second.batches) expect(b.cursor.version).toBe(VERSION); + expect(second.outcome.nextInMinutes).toBeUndefined(); + expect(second.outcome.note).toMatch(/^complete: dump 2026-08-31/); + expect(second.outcome.cursor).toMatchObject({ + version: VERSION, + file: null, + line: 0, + done: true, + maxLastModified: null, + lastModifiedWatermark: '2026-08-31T04:15:19.397201', + }); + expect(second.calls.map((c) => c.kind)).toEqual(['head', 'download', 'download']); + expect(second.calls[2].url).toBe(datedUrl('works', VERSION)); + + // Across the two runs: every row once, nothing twice. + const ids = [...idsOf(first.batches), ...idsOf(second.batches)]; + expect(new Set(ids).size).toBe(ids.length); + expect(ids.sort()).toEqual([...authorIds, ...workIds].sort()); + // The walked files are gone; the disk is shared. + expect(await exists(localPath('authors'))).toBe(false); + expect(await exists(localPath('works'))).toBe(false); + + // Run 3: same dump, already walked: one HEAD, nothing else. + const third = await run({ cursor: second.outcome.cursor }); + expect(third.batches).toEqual([]); + expect(third.outcome).toMatchObject({ note: `unchanged (${VERSION})` }); + expect(third.outcome.nextInMinutes).toBeUndefined(); + expect(third.outcome.cursor).toEqual(second.outcome.cursor); + expect(third.calls.map((c) => c.kind)).toEqual(['head']); + }); + + test('a deadline already near yields nothing, makes no download and asks for ten minutes', async () => { + for (const deadline of [0, Date.now() - 1, Date.now() + 5_000]) { + const p = provider(); + const got = await run({ + p, + deadline, + cursor: { version: VERSION, file: 'works', line: 700 }, + }); + expect(got.batches).toEqual([]); + expect(got.outcome).toMatchObject({ + cursor: { version: VERSION, file: 'works', line: 700, done: false }, + nextInMinutes: 10, + }); + expect(got.outcome.note).toMatch(/out of time before the works download/); + expect(p.calls.map((c) => c.kind)).toEqual(['head']); + } + expect(await exists(localPath('works'))).toBe(false); + }); + + test('a new dump after a complete pass skips the rows modified before the watermark', async () => { + const watermark = '2021-12-27T14:49:34.041676'; + const expected = FILES.flatMap((kind) => + rowsOf(kind) + .filter((r) => r.lastModified >= watermark) + .map(rowItem) + .filter(Boolean) + .map((i) => i.externalId), + ); + expect(expected.length).toBeGreaterThan(10); + expect(expected.length).toBeLessThan(190); + const logs = []; + const got = await run({ + cursor: { + version: '2026-07-31', + file: null, + line: 0, + lastModifiedWatermark: watermark, + done: true, + }, + log: (m) => logs.push(m), + }); + expect(idsOf(got.batches).sort()).toEqual(expected.sort()); + expect(got.outcome.cursor).toMatchObject({ + version: VERSION, + done: true, + lastModifiedWatermark: '2026-08-31T04:15:19.397201', + }); + expect(got.outcome.note).toMatch(/unchanged skipped/); + expect(logs.some((m) => m.includes('replaces 2026-07-31'))).toBe(true); + for (const b of got.batches) expect(b.cursor.lastModifiedWatermark).toBe(watermark); + }); + + test('a new dump while a walk is in progress starts the walk over, keeping the old pass watermark', async () => { + const got = await run({ + stopAfterBatches: 1, + cursor: { + version: '2026-07-31', + file: 'works', + line: 9000, + lastModifiedWatermark: '2000-01-01T00:00:00', + maxLastModified: '2026-08-01T00:00:00', + done: false, + }, + }); + expect(got.batches[0].cursor).toMatchObject({ version: VERSION, file: 'authors' }); + expect(got.batches[0].cursor.lastModifiedWatermark).toBe('2000-01-01T00:00:00'); + expect(got.batches[0].items[0].kind).toBe('author'); + expect(got.calls[1].url).toBe(datedUrl('authors', VERSION)); + await rm(localPath('authors'), { force: true }); + }); + + test('a stale file of another dump is removed before the walk', async () => { + const old = localPath('works', '2026-06-30'); + await writeFile(old, 'old'); + await run({ stopAfterBatches: 1 }); + expect(await exists(old)).toBe(false); + await rm(localPath('authors'), { force: true }); + }); + + test('a download still in progress yields nothing and comes back in ten minutes', async () => { + const p = provider({ complete: () => false }); + const got = await run({ p }); + expect(got.batches).toEqual([]); + expect(got.outcome).toMatchObject({ + cursor: { version: VERSION, file: 'authors', line: 0, done: false }, + nextInMinutes: 10, + }); + expect(got.outcome.note).toMatch(/authors download in progress/); + expect(p.calls.map((c) => c.kind)).toEqual(['head', 'download']); + // The partial file stays where the resume helper will pick it up. + expect((await stat(localPath('authors'))).size).toBeGreaterThan(0); + await rm(localPath('authors'), { force: true }); + }); + + test('the works download in progress keeps the cursor at works line 0, after the authors', async () => { + const p = provider({ complete: (_n, kind) => kind !== 'works' }); + const got = await run({ p }); + expect(got.batches.map((b) => [b.items.length, b.cursor.file])).toEqual([ + [40, 'authors'], + [40, 'authors'], + [17, 'authors'], + [0, 'works'], + ]); + expect(got.outcome).toMatchObject({ + cursor: { version: VERSION, file: 'works', line: 0 }, + nextInMinutes: 10, + }); + expect(got.outcome.note).toMatch(/works download in progress/); + await rm(localPath('works'), { force: true }); + }); + + test('a bad row is counted and skipped; the line count still names the file position', async () => { + const lines = bodies.authors.split('\n').filter(Boolean); + const broken = [ + lines[0], + lines[1].slice(0, lines[1].length - 30), + '', + 'this is not a row at all', + ...lines.slice(2), + lines[3].slice(0, 40), + ].join('\n'); + const logs = []; + const p = provider({ files: { authors: broken, works: bodies.works } }); + const got = await run({ p, log: (m) => logs.push(m) }); + const authorBatches = got.batches.filter((b) => b.cursor.file === 'authors' && b.items.length); + expect(idsOf(authorBatches)).toHaveLength(itemsOf('authors').length - 1); + expect(authorBatches.at(-1).cursor.line).toBe(lines.length + 3); + expect(logs.filter((m) => /not a row/.test(m))).toHaveLength(3); + expect(got.outcome.note).toMatch(/3 unreadable/); + expect(got.outcome.cursor.done).toBe(true); + }); + + test('a file gzip cannot read is removed and the run fails; the place is kept by the batches', async () => { + const p = provider({ raw: { authors: Buffer.from('this is not gzip') } }); + await expect(run({ p })).rejects.toThrow(/authors dump 2026-08-31 unreadable/); + expect(await exists(localPath('authors'))).toBe(false); + }); +}); + +describe('failures', () => { + test('three failed downloads stop the run without losing the place; the HEAD counts as a request that worked', async () => { + const p = provider({ fail: (n) => n > 1 }); + const got = await run({ + p, + cursor: { version: VERSION, file: 'works', line: 500, done: false }, + }); + expect(got.batches).toEqual([]); + expect(got.outcome).toMatchObject({ + cursor: { version: VERSION, file: 'works', line: 500 }, + nextInMinutes: 10, + }); + expect(got.outcome.note).toMatch(/repeated failures on the works download/); + expect(p.calls.map((c) => c.kind)).toEqual(['head', 'download', 'download', 'download']); + }); + + test('a run in which every request failed throws', async () => { + const p = provider({ fail: () => true }); + await expect(run({ p })).rejects.toThrow(/every request failed \(3\)/); + expect(p.calls.map((c) => c.kind)).toEqual(['head', 'head', 'head']); + }); + + test('a walk in progress carries on when the latest dump cannot be resolved', async () => { + const p = provider({ fail: (n) => n <= 3 }); + const got = await run({ + p, + stopAfterBatches: 1, + cursor: { version: '2026-07-31', file: 'works', line: 0, done: false }, + }); + expect(p.calls.map((c) => c.kind)).toEqual(['head', 'head', 'head', 'download']); + expect(p.calls[3].url).toBe(datedUrl('works', '2026-07-31')); + expect(got.batches[0].cursor).toMatchObject({ version: '2026-07-31', file: 'works' }); + expect(got.batches[0].items[0].kind).toBe('book'); + await rm(localPath('works', '2026-07-31'), { force: true }); + }); + + test('nothing in progress and no version is a stop, and with every request failed a throw', async () => { + const p = provider({ fail: () => true }); + await expect( + run({ p, cursor: { version: VERSION, file: null, line: 0, done: true } }), + ).rejects.toThrow(/every request failed/); + }); +}); + +describe('the adapter', () => { + test('declares the walk: books collection, both kinds, a 55 minute budget, a monthly cadence', () => { + expect(openlibraryCatalog.name).toBe('openlibrary-catalog'); + expect(openlibraryCatalog.collection).toBe('books'); + expect(openlibraryCatalog.kinds).toEqual(['book', 'author']); + expect(openlibraryCatalog.budgetMs).toBe(BUDGET_MS); + expect(BUDGET_MS).toBe(55 * 60_000); + expect(openlibraryCatalog.cadenceMinutes).toBe(CADENCE_MINUTES); + expect(CADENCE_MINUTES).toBe(30 * 24 * 60); + expect(openlibraryCatalog.defaultSources[0].slug).toBe('openlibrary-catalog'); + expect(openlibraryCatalog.defaults).toEqual({ batchRows: 500, pauseMs: 2000 }); + }); + + test('the description states the licence and nothing carries an em dash', async () => { + expect(openlibraryCatalog.description).toMatch(/no new copyright asserted/); + expect(ATTRIBUTION).toMatch(/Open Library/); + const src = await readFile( + new URL('../packages/adapters/src/openlibrary-catalog.js', import.meta.url), + 'utf8', + ); + expect(src).not.toContain(EM_DASH); + expect(openlibraryCatalog.description).not.toContain(EM_DASH); + for (const kind of FILES) { + for (const it of itemsOf(kind)) { + expect(JSON.stringify(it.data)).toContain(ATTRIBUTION); + } + } + }); +}); diff --git a/test/dump-helpers-edges.test.js b/test/dump-helpers-edges.test.js new file mode 100644 index 0000000..c47f993 --- /dev/null +++ b/test/dump-helpers-edges.test.js @@ -0,0 +1,362 @@ +import { afterAll, beforeAll, describe, expect, test } from 'bun:test'; +import { mkdtemp, readdir, readFile, rm, stat, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { gzipSync } from 'node:zlib'; + +process.env.DATABASE_URL ??= 'postgres://test:test@localhost:5432/test'; +process.env.SITE_URL ??= 'https://nichedb.test'; + +const { makeHttp } = await import('../packages/core/src/http.js'); +const { gzipLines, tsvJsonLines, xzLines, sqliteRows, lineOffsetReader } = await import( + '../packages/core/src/dump.js' +); + +const http = () => makeHttp({ userAgent: 'review/1' }); +const collect = async (it) => { + const out = []; + for await (const x of it) out.push(x); + return out; +}; + +let dir; +beforeAll(async () => { + dir = await mkdtemp(join(tmpdir(), 'nichedb-review-')); +}); +afterAll(async () => { + if (dir) await rm(dir, { recursive: true, force: true }); +}); + +const fdCount = async () => (await readdir('/proc/self/fd')).length; + +describe('gzipLines', () => { + test('a 1 MB line with no trailing newline arrives whole', async () => { + const big = 'é'.repeat(600_000); // 1.2 MB of UTF-8 + const path = join(dir, 'big.gz'); + await writeFile(path, gzipSync(`first\n${big}`)); + const got = await collect(gzipLines(path)); + expect(got.length).toBe(2); + expect(got[0]).toBe('first'); + expect(got[1].length).toBe(600_000); + expect(got[1]).toBe(big); + expect(await collect(gzipLines(path, { skip: 1 }))).toEqual([big]); + expect(await collect(gzipLines(path, { skip: 2 }))).toEqual([]); + }); + + test('a corrupt gzip file rejects rather than yielding nothing', async () => { + const path = join(dir, 'bad.gz'); + await writeFile(path, Buffer.from('this is not gzip at all, not even close')); + await expect(collect(gzipLines(path))).rejects.toThrow(); + }); + + test('a missing file rejects', async () => { + await expect(collect(gzipLines(join(dir, 'nope.gz')))).rejects.toThrow(); + }); + + test('returning early from many readers does not leak file descriptors', async () => { + const path = join(dir, 'many.gz'); + await writeFile(path, gzipSync(Array.from({ length: 20000 }, (_, i) => `l${i}`).join('\n'))); + const before = await fdCount(); + for (let i = 0; i < 100; i += 1) { + const it = gzipLines(path); + expect((await it.next()).value).toBe('l0'); + await it.return(); + } + await Bun.sleep(50); + const after = await fdCount(); + expect(after - before).toBeLessThan(10); + }); +}); + +describe('tsvJsonLines', () => { + test('escaped tabs and quotes inside the JSON column survive; short lines are skipped', async () => { + const json = JSON.stringify({ + title: 'A "quoted"\ttitle', + key: '/works/OL1W', + notes: { value: 'line\nbreak\\slash' }, + }); + const rows = [ + `/type/work\t/works/OL1W\t3\t2020-01-01T00:00:00.000000\t${json}`, + '', + 'junk\tline', + `/type/author\t/authors/OL1A\t1\t2019-01-01\t{"name":"x"}`, + ]; + const path = join(dir, 'ol.txt'); + await writeFile(path, `${rows.join('\n')}\n`); + const got = await collect(tsvJsonLines(path)); + expect(got.length).toBe(2); + expect(got[0].json.title).toBe('A "quoted"\ttitle'); + expect(got[0].json.notes.value).toBe('line\nbreak\\slash'); + expect(got[0].lineNo).toBe(1); + expect(got[1].lineNo).toBe(4); + expect(got[1].type).toBe('/type/author'); + // resume from lineNo of the first record yields only the fourth + expect((await collect(tsvJsonLines(path, { skip: 1 }))).map((r) => r.lineNo)).toEqual([4]); + // gz path identical + const gz = join(dir, 'ol.txt.gz'); + await writeFile(gz, gzipSync(`${rows.join('\n')}\n`)); + expect((await collect(tsvJsonLines(gz))).map((r) => r.lineNo)).toEqual([1, 4]); + }); + + test('bad JSON names the line', async () => { + const path = join(dir, 'olbad.txt'); + await writeFile(path, `/type/work\t/w\t1\t2020\t{"ok":1}\n/type/work\t/w2\t1\t2020\t{nope\n`); + await expect(collect(tsvJsonLines(path))).rejects.toThrow(/line 2/); + }); +}); + +const haveXz = Bun.which('xz') && Bun.which('tar'); +describe.skipIf(!haveXz)('review: xzLines', () => { + test('a corrupt xz rejects instead of ending as an empty file', async () => { + const path = join(dir, 'corrupt.xz'); + await writeFile(path, Buffer.from('definitely not xz')); + await expect(collect(xzLines(path))).rejects.toThrow(/xz exited/); + }); + + test('a truncated xz stream yields what it can then rejects', async () => { + const text = Array.from({ length: 5000 }, (_, i) => `row ${i} ${'y'.repeat(50)}`).join('\n'); + const xz = Bun.spawn(['xz', '-zc'], { stdin: new Response(text).body, stdout: 'pipe' }); + const whole = Buffer.from(await new Response(xz.stdout).arrayBuffer()); + const path = join(dir, 'trunc.xz'); + await writeFile(path, whole.subarray(0, Math.floor(whole.length / 2))); + let n = 0; + let err = null; + try { + for await (const _ of xzLines(path)) n += 1; + } catch (e) { + err = e; + } + expect(err).not.toBeNull(); + expect(String(err.message)).toMatch(/xz exited/); + expect(n).toBeLessThan(5000); + }); + + test('no orphan process after an early return', async () => { + const text = Array.from({ length: 200000 }, (_, i) => `row ${i}`).join('\n'); + const xz = Bun.spawn(['xz', '-zc'], { stdin: new Response(text).body, stdout: 'pipe' }); + const path = join(dir, 'long.xz'); + await writeFile(path, Buffer.from(await new Response(xz.stdout).arrayBuffer())); + const before = await fdCount(); + for (let i = 0; i < 20; i += 1) { + const it = xzLines(path); + await it.next(); + await it.return(); + } + await Bun.sleep(100); + expect((await fdCount()) - before).toBeLessThan(10); + }); +}); + +describe('sqliteRows', () => { + test('iterates rows lazily with params', async () => { + const { Database } = await import('bun:sqlite'); + const path = join(dir, 'p.db'); + const db = new Database(path); + db.exec('create table podcasts (id integer primary key, title text)'); + for (let i = 1; i <= 10; i += 1) db.query('insert into podcasts values (?, ?)').run(i, `p${i}`); + db.close(); + const rows = [...sqliteRows(path, 'select * from podcasts where id > ? order by id', [7])]; + expect(rows.map((r) => r.id)).toEqual([8, 9, 10]); + }); +}); + +describe('lineOffsetReader', () => { + test('offset after a CRLF line and a multibyte line lands on the next line', async () => { + const path = join(dir, 'crlf.txt'); + await writeFile(path, 'ab\r\nçd\r\nlast'); + const got = await collect(lineOffsetReader(path)); + expect(got.map((r) => r.line)).toEqual(['ab', 'çd', 'last']); + const resumed = await collect(lineOffsetReader(path, { offset: got[0].offset })); + expect(resumed.map((r) => r.line)).toEqual(['çd', 'last']); + expect(resumed[1].offset).toBe(Buffer.byteLength('ab\r\nçd\r\nlast')); + }); +}); + +/* ------------------------------------------------------------ download -- */ + +const BODY = Buffer.alloc(300_000); +for (let i = 0; i < BODY.length; i += 1) BODY[i] = i % 251; + +let server; +let base; +const hits = []; +beforeAll(() => { + server = Bun.serve({ + port: 0, + fetch(req) { + const url = new URL(req.url); + const range = req.headers.get('range'); + hits.push({ path: url.pathname, range }); + const total = BODY.length; + if (url.pathname === '/small') { + // pretends to be smaller than our partial file + if (range) { + return new Response(null, { status: 416, headers: { 'content-range': 'bytes */1000' } }); + } + return new Response(BODY.subarray(0, 1000), { headers: { 'content-length': '1000' } }); + } + if (url.pathname === '/nolen') { + return new Response( + new ReadableStream({ + start(c) { + c.enqueue(BODY.subarray(0, 100_000)); + c.close(); + }, + }), + ); + } + if (url.pathname === '/huge') { + const size = 200 * 1024 * 1024; + const chunk = new Uint8Array(1024 * 1024); + let sent = 0; + return new Response( + new ReadableStream({ + pull(c) { + if (sent >= size) return c.close(); + c.enqueue(chunk); + sent += chunk.length; + }, + }), + { headers: { 'content-length': String(size) } }, + ); + } + if (url.pathname === '/ignores') { + return new Response(BODY, { headers: { 'content-length': String(total) } }); + } + if (url.pathname === '/wrongstart') { + // a broken server that honours a range but says a different start + return new Response(BODY.subarray(10), { + status: 206, + headers: { 'content-range': `bytes 10-${total - 1}/${total}` }, + }); + } + if (!range) { + return new Response(BODY, { headers: { 'content-length': String(total) } }); + } + const start = Number(range.match(/bytes=(\d+)-/)?.[1]); + if (start >= total) { + return new Response(null, { + status: 416, + headers: { 'content-range': `bytes */${total}` }, + }); + } + return new Response(BODY.subarray(start), { + status: 206, + headers: { 'content-range': `bytes ${start}-${total - 1}/${total}` }, + }); + }, + }); + base = `http://127.0.0.1:${server.port}`; +}); + +/** Raw TCP server that sends MORE than its Content-Length. */ +let over; +let overBase; +beforeAll(() => { + over = Bun.listen({ + hostname: '127.0.0.1', + port: 0, + socket: { + data(socket) { + const head = ['HTTP/1.1 200 OK', 'Content-Length: 1000', 'Connection: close', '', ''].join( + '\r\n', + ); + socket.write(head); + socket.write(BODY.subarray(0, 5000)); + socket.flush(); + setTimeout(() => socket.end(), 30); + }, + }, + }); + overBase = `http://127.0.0.1:${over.port}`; +}); + +afterAll(() => { + server?.stop(true); + over?.stop(true); +}); + +describe('http.download', () => { + test('206 appends exactly the missing tail', async () => { + const path = join(dir, 'a.bin'); + await writeFile(path, BODY.subarray(0, 123_456)); + hits.length = 0; + const r = await http().download(`${base}/file`, path); + expect(r).toEqual({ path, bytes: BODY.length, complete: true }); + expect(hits[0].range).toBe('bytes=123456-'); + expect(await readFile(path)).toEqual(BODY); + }); + + test('200 to a ranged request restarts the file, no duplicate prefix', async () => { + const path = join(dir, 'b.bin'); + await writeFile(path, BODY.subarray(0, 123_456)); + const r = await http().download(`${base}/ignores`, path); + expect(r.complete).toBe(true); + expect((await stat(path)).size).toBe(BODY.length); + expect(await readFile(path)).toEqual(BODY); + }); + + test('416 on a whole file is complete with no write', async () => { + const path = join(dir, 'c.bin'); + await writeFile(path, BODY); + const m1 = (await stat(path)).mtimeMs; + await Bun.sleep(20); + const r = await http().download(`${base}/file`, path); + expect(r).toEqual({ path, bytes: BODY.length, complete: true }); + expect((await stat(path)).mtimeMs).toBe(m1); + }); + + test('416 where our file is longer than theirs unlinks and throws', async () => { + const path = join(dir, 'd.bin'); + await writeFile(path, BODY.subarray(0, 2000)); + await expect(http().download(`${base}/small`, path)).rejects.toThrow(/server has 1000/); + expect(await stat(path).catch(() => null)).toBeNull(); + // and the next call downloads the right file from scratch + const r = await http().download(`${base}/small`, path); + expect(r).toEqual({ path, bytes: 1000, complete: true }); + }); + + test('a 206 whose Content-Range start is not what we asked for is refused', async () => { + const path = join(dir, 'e.bin'); + await writeFile(path, BODY.subarray(0, 500)); + await expect(http().download(`${base}/wrongstart`, path)).rejects.toThrow( + /asked for bytes=500-/, + ); + expect((await stat(path)).size).toBe(500); + }); + + test('a body longer than Content-Length is cut at the declared length by fetch itself', async () => { + const path = join(dir, 'f.bin'); + const r = await http().download(`${overBase}/x`, path); + expect(r).toEqual({ path, bytes: 1000, complete: true }); + expect((await stat(path)).size).toBe(1000); + }); + + test('no Content-Length: whatever arrived counts as complete', async () => { + const path = join(dir, 'g.bin'); + const r = await http().download(`${base}/nolen`, path); + expect(r.complete).toBe(true); + expect(r.bytes).toBe(100_000); + }); + + test('200 MB streams to disk with flat memory', async () => { + const path = join(dir, 'huge.bin'); + Bun.gc(true); + const before = process.memoryUsage().rss; + let peak = before; + const r = await http().download(`${base}/huge`, path, { + onProgress: ({ bytes }) => { + if (bytes % (32 * 1024 * 1024) === 0) peak = Math.max(peak, process.memoryUsage().rss); + }, + }); + Bun.gc(true); + const after = process.memoryUsage().rss; + expect(r).toEqual({ path, bytes: 200 * 1024 * 1024, complete: true }); + expect((await stat(path)).size).toBe(200 * 1024 * 1024); + console.log( + `rss before ${(before / 1e6).toFixed(0)} MB, peak ${(peak / 1e6).toFixed(0)} MB, after ${(after / 1e6).toFixed(0)} MB`, + ); + expect(peak - before).toBeLessThan(120 * 1024 * 1024); + await rm(path); + }, 120_000); +}); diff --git a/test/dump-helpers.test.js b/test/dump-helpers.test.js new file mode 100644 index 0000000..aa8892a --- /dev/null +++ b/test/dump-helpers.test.js @@ -0,0 +1,366 @@ +import { afterAll, beforeAll, describe, expect, test } from 'bun:test'; +import { mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { gzipSync } from 'node:zlib'; + +/** + * The pieces a dump adapter is built from: a download that resumes, and line + * readers that never hold the file. The server here is a local Bun.serve that + * speaks Range the way archive.org, MusicBrainz and Podcast Index do, plus one + * route that ignores it the way Discogs does. + */ +process.env.DATABASE_URL ??= 'postgres://test:test@localhost:5432/test'; +process.env.SITE_URL ??= 'https://nichedb.test'; + +const { makeHttp } = await import('../packages/core/src/http.js'); +const { dumpDir, gzipLines, lineOffsetReader, splitLines, tsvJsonLines, xzLines, untar } = + await import('../packages/core/src/dump.js'); + +let dir; +let server; +let base; +const seenUA = []; +const BODY = Buffer.from( + Array.from({ length: 4000 }, (_, i) => `line ${i} ${'x'.repeat(i % 37)}`).join('\n'), +); + +/** How many bytes each request served, so a resume can be shown to be one. */ +const served = []; + +beforeAll(async () => { + dir = await mkdtemp(join(tmpdir(), 'nichedb-dump-test-')); + server = Bun.serve({ + port: 0, + fetch(req) { + const url = new URL(req.url); + seenUA.push(req.headers.get('user-agent')); + const range = req.headers.get('range'); + + if (url.pathname === '/redirect') return Response.redirect(`${base}/file`, 302); + if (url.pathname === '/noranges') { + // Discogs: a 200 with the whole body whatever was asked for. + served.push(BODY.length); + return new Response(BODY, { headers: { 'content-length': String(BODY.length) } }); + } + if (url.pathname !== '/file') return new Response('no', { status: 404 }); + + const total = BODY.length; + if (!range) { + served.push(total); + return new Response(BODY, { + headers: { 'content-length': String(total), 'accept-ranges': 'bytes' }, + }); + } + const start = Number(range.match(/bytes=(\d+)-/)?.[1]); + if (start >= total) { + served.push(0); + return new Response(null, { + status: 416, + headers: { 'content-range': `bytes */${total}` }, + }); + } + const slice = BODY.subarray(start); + served.push(slice.length); + return new Response(slice, { + status: 206, + headers: { + 'content-length': String(slice.length), + 'content-range': `bytes ${start}-${total - 1}/${total}`, + }, + }); + }, + }); + base = `http://127.0.0.1:${server.port}`; +}); + +/** + * A server that promises the rest of the file and hangs up after CUT bytes. + * Raw TCP, because Bun.serve corrects a Content-Length that the body does not + * live up to, and the point is a body that stops short of its header. + */ +const CUT = 40_000; +let cutter; +let cutBase; +beforeAll(() => { + cutter = Bun.listen({ + hostname: '127.0.0.1', + port: 0, + socket: { + data(socket, data) { + const start = Number(String(data).match(/range: bytes=(\d+)-/i)?.[1] ?? 0); + const total = BODY.length; + const slice = BODY.subarray(start, Math.min(start + CUT, total)); + served.push(slice.length); + const head = [ + `HTTP/1.1 ${start ? '206 Partial Content' : '200 OK'}`, + `Content-Length: ${total - start}`, + ...(start ? [`Content-Range: bytes ${start}-${total - 1}/${total}`] : []), + 'Connection: close', + '', + '', + ].join('\r\n'); + socket.write(head); + socket.write(slice); + socket.flush(); + // A beat before hanging up, so the bytes are read before the close is; + // a close in the same packet makes Bun's fetch drop the lot. + setTimeout(() => socket.end(), 30); + }, + }, + }); + cutBase = `http://127.0.0.1:${cutter.port}`; +}); + +afterAll(async () => { + server?.stop(true); + cutter?.stop(true); + if (dir) await rm(dir, { recursive: true, force: true }); +}); + +const http = () => makeHttp({ userAgent: 'niche-db-test/1 (+https://nichedb.test)' }); + +describe('http.download', () => { + test('streams a whole file to disk with the caller user agent', async () => { + const path = join(dir, 'whole.bin'); + served.length = 0; + const progress = []; + const r = await http().download(`${base}/file`, path, { + onProgress: (p) => progress.push(p), + }); + expect(r).toEqual({ path, bytes: BODY.length, complete: true }); + expect(await readFile(path)).toEqual(BODY); + expect(served).toEqual([BODY.length]); + expect(seenUA.at(-1)).toBe('niche-db-test/1 (+https://nichedb.test)'); + expect(progress.at(-1)).toEqual({ bytes: BODY.length, total: BODY.length }); + }); + + test('resumes a partial file with Range and appends the 206', async () => { + const path = join(dir, 'partial.bin'); + await writeFile(path, BODY.subarray(0, 1500)); + served.length = 0; + const r = await http().download(`${base}/file`, path); + expect(r).toEqual({ path, bytes: BODY.length, complete: true }); + expect(await readFile(path)).toEqual(BODY); + // Only the missing tail crossed the wire. + expect(served).toEqual([BODY.length - 1500]); + }); + + test('a file already whole is a 416 and no transfer', async () => { + const path = join(dir, 'done.bin'); + await writeFile(path, BODY); + served.length = 0; + const r = await http().download(`${base}/file`, path); + expect(r).toEqual({ path, bytes: BODY.length, complete: true }); + expect(served).toEqual([0]); + }); + + test('a server that ignores Range restarts the file rather than appending', async () => { + const path = join(dir, 'noranges.bin'); + await writeFile(path, BODY.subarray(0, 1500)); + const r = await http().download(`${base}/noranges`, path); + expect(r.complete).toBe(true); + expect((await stat(path)).size).toBe(BODY.length); + expect(await readFile(path)).toEqual(BODY); + }); + + test('a connection that drops mid-body leaves a partial file the next call resumes', async () => { + const path = join(dir, 'cut.bin'); + served.length = 0; + // Each attempt either throws (the socket closed under the body) or reports + // `complete: false`; either way the bytes that arrived are on disk and the + // next call asks for the rest: three calls for this file. + const attempt = () => + http() + .download(`${cutBase}/file`, path) + .catch(() => null); + let r = null; + for (let i = 0; i < 20 && !r?.complete; i += 1) { + r = await attempt(); + if (r) expect(r.complete).toBe((await stat(path)).size === BODY.length); + } + expect(r).toEqual({ path, bytes: BODY.length, complete: true }); + expect(await readFile(path)).toEqual(BODY); + expect(served).toEqual([CUT, CUT, BODY.length - 2 * CUT]); + }); + + test('follows redirects, as Open Library needs, and lands the right bytes', async () => { + const path = join(dir, 'redirected.bin'); + const r = await http().download(`${base}/redirect`, path); + expect(r.complete).toBe(true); + expect(await readFile(path)).toEqual(BODY); + }); + + test('a 404 is an error and writes nothing', async () => { + const path = join(dir, 'missing.bin'); + await expect(http().download(`${base}/nope`, path)).rejects.toThrow(/404/); + expect(await stat(path).catch(() => null)).toBeNull(); + }); +}); + +describe('dumpDir', () => { + test('is created under the data dir and refuses a path that is not a slug', async () => { + const d = await dumpDir('musicbrainz'); + expect((await stat(d)).isDirectory()).toBe(true); + expect(d.endsWith(join('nichedb-dumps', 'musicbrainz'))).toBe(true); + await expect(dumpDir('../etc')).rejects.toThrow(/slug/); + }); +}); + +const collect = async (it) => { + const out = []; + for await (const x of it) out.push(x); + return out; +}; + +describe('gzipLines', () => { + const lines = Array.from({ length: 2500 }, (_, i) => JSON.stringify({ i, name: `né ${i}` })); + + test('yields every line and flushes a final line with no trailing newline', async () => { + const path = join(dir, 'a.ndjson.gz'); + await writeFile(path, gzipSync(lines.join('\n'))); + const got = await collect(gzipLines(path)); + expect(got.length).toBe(2500); + expect(got[0]).toBe(lines[0]); + expect(got.at(-1)).toBe(lines.at(-1)); + expect(JSON.parse(got[1234]).name).toBe('né 1234'); + }); + + test('a trailing newline does not add a phantom empty line, CRLF is stripped', async () => { + const path = join(dir, 'b.gz'); + await writeFile(path, gzipSync(`one\r\ntwo\r\n`)); + expect(await collect(gzipLines(path))).toEqual(['one', 'two']); + }); + + test('skip counts lines from the top so a cursor of { skip } resumes exactly', async () => { + const path = join(dir, 'a.ndjson.gz'); + const got = await collect(gzipLines(path, { skip: 2497 })); + expect(got).toEqual(lines.slice(2497)); + }); + + test('a line larger than one chunk arrives whole, multibyte and all', async () => { + const big = `{"pad":"${'é'.repeat(200_000)}"}`; + const path = join(dir, 'c.gz'); + await writeFile(path, gzipSync(`${big}\nshort`)); + const got = await collect(gzipLines(path)); + expect(got.length).toBe(2); + expect(got[0]).toBe(big); + expect(JSON.parse(got[0]).pad.length).toBe(200_000); + }); +}); + +describe('tsvJsonLines', () => { + test('splits the five Open Library columns and parses the JSON', async () => { + const rows = [ + `/type/author\t/authors/OL1A\t1\t2021-12-26T21:23:30.303089\t{"key":"/authors/OL1A","name":"A\\u00e9"}`, + '', + `/type/work\t/works/OL2W\t3\t2010-04-28T06:54:19.472104\t{"key":"/works/OL2W","title":"T\\twith tab"}`, + ]; + const path = join(dir, 'ol.txt.gz'); + await writeFile(path, gzipSync(rows.join('\n'))); + const got = await collect(tsvJsonLines(path)); + expect(got).toEqual([ + { + type: '/type/author', + key: '/authors/OL1A', + revision: 1, + lastModified: '2021-12-26T21:23:30.303089', + json: { key: '/authors/OL1A', name: 'Aé' }, + lineNo: 1, + }, + { + type: '/type/work', + key: '/works/OL2W', + revision: 3, + lastModified: '2010-04-28T06:54:19.472104', + json: { key: '/works/OL2W', title: 'T\twith tab' }, + lineNo: 3, + }, + ]); + // Resume from the record's own lineNo. + expect((await collect(tsvJsonLines(path, { skip: 1 }))).map((r) => r.key)).toEqual([ + '/works/OL2W', + ]); + }); +}); + +describe('lineOffsetReader', () => { + test('offsets resume in O(1) and skip resumes by count', async () => { + const path = join(dir, 'plain.txt'); + await writeFile(path, 'alpha\nbéta\ngamma\ndelta'); + const all = await collect(lineOffsetReader(path)); + expect(all.map((r) => r.line)).toEqual(['alpha', 'béta', 'gamma', 'delta']); + expect(all.map((r) => r.lineNo)).toEqual([1, 2, 3, 4]); + // 'alpha\n' is 6 bytes, 'béta\n' is 6 bytes (two-byte e-acute), 'gamma\n' 6, 'delta' 5. + expect(all.map((r) => r.offset)).toEqual([6, 12, 18, 23]); + const rest = await collect(lineOffsetReader(path, { offset: 12 })); + expect(rest.map((r) => r.line)).toEqual(['gamma', 'delta']); + expect(rest.map((r) => r.offset)).toEqual([18, 23]); + const skipped = await collect(lineOffsetReader(path, { skip: 3 })); + expect(skipped).toEqual([{ line: 'delta', lineNo: 4, offset: 23 }]); + }); +}); + +describe('splitLines', () => { + test('reassembles a line split across chunks and flushes the last one', async () => { + async function* chunks() { + yield Buffer.from('ab'); + yield Buffer.from('c\nd'); + yield Buffer.from('e'); + } + expect(await collect(splitLines(chunks()))).toEqual(['abc', 'de']); + }); +}); + +const haveXz = Bun.which('xz') && Bun.which('tar'); +describe.skipIf(!haveXz)('xzLines', () => { + test('streams one member out of a tar.xz through the system tar', async () => { + const src = join(dir, 'mb'); + await Bun.write(join(src, 'TIMESTAMP'), '2026-09-12 01:28:38+00'); + await Bun.write(join(src, 'mbdump', 'instrument'), '{"id":1}\n{"id":2}\n{"id":3}'); + await Bun.write(join(src, 'JSON_DUMPS_SCHEMA_NUMBER'), '1'); + const archive = join(dir, 'instrument.tar.xz'); + const tar = Bun.spawn( + [ + 'tar', + '-cJf', + archive, + '-C', + src, + 'TIMESTAMP', + 'mbdump/instrument', + 'JSON_DUMPS_SCHEMA_NUMBER', + ], + { stderr: 'pipe' }, + ); + expect(await tar.exited).toBe(0); + + const got = await collect(xzLines(archive, { member: 'mbdump/instrument' })); + expect(got).toEqual(['{"id":1}', '{"id":2}', '{"id":3}']); + expect(await collect(xzLines(archive, { member: 'mbdump/instrument', skip: 2 }))).toEqual([ + '{"id":3}', + ]); + + // Stopping early kills the process rather than leaving it to fill a pipe. + const it = xzLines(archive, { member: 'mbdump/instrument' }); + expect((await it.next()).value).toBe('{"id":1}'); + await it.return(); + + // A missing member is an error, not an empty file. + await expect(collect(xzLines(archive, { member: 'mbdump/nope' }))).rejects.toThrow( + /tar exited/, + ); + + // untar puts the member on disk for the byte-seek path. + const out = join(dir, 'untarred'); + await untar(archive, out, { members: ['mbdump/instrument'] }); + expect((await stat(join(out, 'mbdump', 'instrument'))).size).toBe(26); + }); + + test('reads a plain .xz file too', async () => { + const plain = join(dir, 'plain.txt.xz'); + const xz = Bun.spawn(['xz', '-zc'], { stdin: new Response('p\nq\n').body, stdout: 'pipe' }); + await Bun.write(plain, await new Response(xz.stdout).arrayBuffer()); + expect(await collect(xzLines(plain))).toEqual(['p', 'q']); + }); +}); diff --git a/test/dump-ingest-edges.test.js b/test/dump-ingest-edges.test.js new file mode 100644 index 0000000..a46260a --- /dev/null +++ b/test/dump-ingest-edges.test.js @@ -0,0 +1,195 @@ +import { describe, expect, mock, test } from 'bun:test'; + +process.env.DATABASE_URL ??= 'postgres://test:test@localhost:5432/test'; +process.env.SITE_URL ??= 'https://nichedb.test'; + +const calls = []; +const state = { source: null, dedupes: false, failUpsertOn: null }; +const realQueries = await import('../packages/db/src/queries.js'); +mock.module('../packages/db/src/queries.js', () => ({ + ...realQueries, + getSourceById: async () => state.source, + startRun: async () => 7, + finishRun: async (args) => { + calls.push(['finishRun', args]); + }, + saveCursor: async (sourceId, cursor) => { + calls.push(['saveCursor', { sourceId, cursor }]); + }, + collectionDedupesUrls: async () => state.dedupes, + claimedDedupeKeys: async () => new Set(), + upsertItems: async ({ items }) => { + const ids = items.map((i) => i.externalId); + if (state.failUpsertOn && ids.includes(state.failUpsertOn)) throw new Error('pg went away'); + calls.push(['upsert', ids]); + return { added: items.length, updated: 0 }; + }, + previousItemData: async () => new Map(), +})); + +const { runSource: run } = await import('../packages/core/src/ingest.js'); +const fakes = new Map(); +const runSource = (id, opts) => run(id, { ...opts, resolveAdapter: (name) => fakes.get(name) }); +const item = (n) => ({ externalId: `x${n}`, title: `Item ${n}`, url: `https://ex.test/${n}` }); +const batch = (from, to) => Array.from({ length: to - from + 1 }, (_, i) => item(from + i)); + +function useAdapter(spec) { + fakes.set('fake-dump', { name: 'fake-dump', kinds: ['thing'], defaults: {}, ...spec }); + state.source = { + id: 42, + slug: 'fake', + adapter: 'fake-dump', + enabled: true, + collection_id: 1, + config: '{}', + cursor: '{"version":"v1","skip":0}', + }; + state.failUpsertOn = null; + calls.length = 0; +} +const only = (kind) => calls.filter((c) => c[0] === kind).map((c) => c[1]); + +describe('streaming ingest', () => { + test('a batch is pulled only after the previous one is upserted and its cursor saved', async () => { + useAdapter({ + async *pull() { + for (let i = 0; i < 50; i += 1) { + calls.push(['pull', i]); + yield { items: batch(i * 2 + 1, i * 2 + 2), cursor: { skip: (i + 1) * 2 } }; + } + return { cursor: { skip: 100, done: true } }; + }, + }); + await runSource(42, { log: () => {} }); + const seq = calls.map((c) => c[0]).filter((k) => k !== 'finishRun'); + const expected = []; + for (let i = 0; i < 50; i += 1) expected.push('pull', 'upsert', 'saveCursor'); + expect(seq).toEqual(expected); + const [finish] = only('finishRun'); + expect(finish.cursor).toEqual({ skip: 100, done: true }); + expect(finish.seen).toBe(100); + }); + + test('an upsert that throws on batch 2 errors the run with the cursor at batch 1 and closes the generator', async () => { + let closed = false; + let pulled = 0; + useAdapter({ + async *pull() { + try { + pulled += 1; + yield { items: batch(1, 2), cursor: { skip: 2 } }; + pulled += 1; + yield { items: batch(3, 4), cursor: { skip: 4 } }; + pulled += 1; + yield { items: batch(5, 6), cursor: { skip: 6 } }; + return { cursor: { skip: 6, done: true } }; + } finally { + closed = true; + } + }, + }); + state.failUpsertOn = 'x3'; + const out = await runSource(42, { log: () => {} }); + expect(out).toEqual({ error: 'pg went away' }); + expect(closed).toBe(true); + expect(pulled).toBe(2); + expect(only('upsert')).toEqual([['x1', 'x2']]); + expect(only('saveCursor')).toEqual([{ sourceId: 42, cursor: { skip: 2 } }]); + const [finish] = only('finishRun'); + expect(finish.status).toBe('error'); + expect(finish.cursor).toBeUndefined(); + expect(finish.error).toBe('pg went away'); + }); + + test('a return value with no cursor leaves finishRun with none, so coalesce keeps the last saved', async () => { + useAdapter({ + async *pull() { + yield { items: batch(1, 2), cursor: { skip: 2 } }; + return { note: 'complete' }; + }, + }); + await runSource(42, { log: () => {} }); + const [finish] = only('finishRun'); + expect(finish.note).toBe('complete'); + expect(finish.cursor).toBeUndefined(); + }); + + test('a generator with no return value ends with the last batch cursor', async () => { + useAdapter({ + async *pull() { + yield { items: batch(1, 2), cursor: { skip: 2 } }; + yield { items: batch(3, 3), cursor: { skip: 3 } }; + }, + }); + await runSource(42, { log: () => {} }); + const [finish] = only('finishRun'); + expect(finish.cursor).toEqual({ skip: 3 }); + expect(finish.status).toBe('ok'); + }); + + test('totals sum updated and added across batches and dedupe applies per batch', async () => { + state.dedupes = true; + useAdapter({ + async *pull() { + yield { + items: [ + { ...item(1), url: 'https://ex.test/same' }, + { ...item(2), url: 'https://ex.test/same' }, + ], + cursor: { skip: 2 }, + }; + yield { items: [{ ...item(3), url: 'https://ex.test/same' }], cursor: { skip: 3 } }; + return { cursor: { skip: 3, done: true } }; + }, + }); + const out = await runSource(42, { log: () => {} }); + state.dedupes = false; + // Within-batch fold drops x2; batch 2 cannot see batch 1 (claimed is + // re-queried per batch against the table, which the mock says is empty). + expect(only('upsert')).toEqual([['x1'], ['x3']]); + expect(out).toEqual({ seen: 2, added: 2, updated: 0 }); + }); + + test('the array path: a pull that returns a plain array of items still works', async () => { + useAdapter({ pull: async () => ({ items: batch(1, 3) }) }); + const out = await runSource(42, { log: () => {} }); + expect(only('upsert')).toEqual([['x1', 'x2', 'x3']]); + expect(only('saveCursor')).toEqual([]); + const [finish] = only('finishRun'); + expect(finish.cursor).toBeUndefined(); + expect(finish.nextRunAt).toBeNull(); + expect(out).toEqual({ seen: 3, added: 3, updated: 0 }); + }); + + test('a pull that returns nothing at all', async () => { + useAdapter({ pull: async () => undefined }); + const out = await runSource(42, { log: () => {} }); + expect(out).toEqual({ seen: 0, added: 0, updated: 0 }); + expect(only('finishRun')[0].status).toBe('ok'); + }); + + test('a pull whose next() rejects (reader error) errors the run and keeps the saved cursor', async () => { + useAdapter({ + pull: async () => ({ + items: { + [Symbol.asyncIterator]() { + let n = 0; + return { + async next() { + n += 1; + if (n === 1) + return { value: { items: batch(1, 2), cursor: { skip: 2 } }, done: false }; + throw new Error('xz exited 1: corrupt'); + }, + // no return(): the core must cope with an iterator that has none + }; + }, + }, + }), + }); + const out = await runSource(42, { log: () => {} }); + expect(out).toEqual({ error: 'xz exited 1: corrupt' }); + expect(only('saveCursor')).toEqual([{ sourceId: 42, cursor: { skip: 2 } }]); + expect(only('finishRun')[0].status).toBe('error'); + }); +}); diff --git a/test/dump-ingest.test.js b/test/dump-ingest.test.js new file mode 100644 index 0000000..1ae006a --- /dev/null +++ b/test/dump-ingest.test.js @@ -0,0 +1,219 @@ +import { describe, expect, mock, test } from 'bun:test'; + +/** + * A dump adapter hands the core its items a batch at a time. + * + * The array form of `pull` holds every item in memory until the run ends and + * writes the cursor once, at the finish. A file of gigabytes cannot do either, + * so `pull` may yield `{ items, cursor }` batches and the core drains them one + * by one: each batch through the same normalise/dedupe/upsert path, each + * batch's cursor saved before the next is read, the generator's return value + * taken as the run's outcome. These tests run `runSource` for real against a + * recording stand-in for the queries module, because the property under test + * is the ORDER of writes and saves, which nothing short of the real loop shows. + */ +process.env.DATABASE_URL ??= 'postgres://test:test@localhost:5432/test'; +process.env.SITE_URL ??= 'https://nichedb.test'; + +const calls = []; +const state = { source: null, dedupes: false }; +const realQueries = await import('../packages/db/src/queries.js'); +mock.module('../packages/db/src/queries.js', () => ({ + ...realQueries, + getSourceById: async () => state.source, + startRun: async () => 7, + finishRun: async (args) => { + calls.push(['finishRun', args]); + }, + saveCursor: async (sourceId, cursor) => { + calls.push(['saveCursor', { sourceId, cursor }]); + }, + collectionDedupesUrls: async () => state.dedupes, + claimedDedupeKeys: async () => new Set(), + upsertItems: async ({ items }) => { + calls.push(['upsert', items.map((i) => i.externalId)]); + return { added: items.length, updated: 0 }; + }, + previousItemData: async () => new Map(), +})); + +const { runSource: run } = await import('../packages/core/src/ingest.js'); + +// The fake adapter goes in through `resolveAdapter`, not a mock of the registry: +// `@nichedb/adapters` and `@nichedb/core` import each other, and mocking one +// end of that cycle hangs Bun's loader for every file that loads after this one. +const fakes = new Map(); +const runSource = (id, opts) => run(id, { ...opts, resolveAdapter: (name) => fakes.get(name) }); + +const item = (n) => ({ externalId: `x${n}`, title: `Item ${n}`, url: `https://ex.test/${n}` }); +const batch = (from, to) => Array.from({ length: to - from + 1 }, (_, i) => item(from + i)); + +function useAdapter(spec) { + fakes.set('fake-dump', { name: 'fake-dump', kinds: ['thing'], defaults: {}, ...spec }); + state.source = { + id: 42, + slug: 'fake', + adapter: 'fake-dump', + enabled: true, + collection_id: 1, + config: '{}', + cursor: '{"version":"v1","skip":0}', + }; + calls.length = 0; +} + +const only = (kind) => calls.filter((c) => c[0] === kind).map((c) => c[1]); + +describe('a pull that yields batches', () => { + test('three batches are three upserts, three cursor saves, and the return value ends the run', async () => { + const seenCursor = []; + useAdapter({ + async *pull({ cursor }) { + seenCursor.push(cursor); + yield { items: batch(1, 3), cursor: { version: 'v1', skip: 3 } }; + yield { items: batch(4, 6), cursor: { version: 'v1', skip: 6 } }; + yield { items: batch(7, 8), cursor: { version: 'v1', skip: 8 } }; + return { + cursor: { version: 'v1', skip: 8, done: true }, + note: 'complete', + nextInMinutes: 30, + }; + }, + }); + + const out = await runSource(42, { log: () => {} }); + + expect(seenCursor).toEqual([{ version: 'v1', skip: 0 }]); + expect(only('upsert')).toEqual([ + ['x1', 'x2', 'x3'], + ['x4', 'x5', 'x6'], + ['x7', 'x8'], + ]); + expect(only('saveCursor')).toEqual([ + { sourceId: 42, cursor: { version: 'v1', skip: 3 } }, + { sourceId: 42, cursor: { version: 'v1', skip: 6 } }, + { sourceId: 42, cursor: { version: 'v1', skip: 8 } }, + ]); + + // Each save lands AFTER its batch is in the table, never before. + const order = calls.map((c) => c[0]).filter((k) => k !== 'finishRun'); + expect(order).toEqual(['upsert', 'saveCursor', 'upsert', 'saveCursor', 'upsert', 'saveCursor']); + + const [finish] = only('finishRun'); + expect(finish.status).toBe('ok'); + expect(finish.cursor).toEqual({ version: 'v1', skip: 8, done: true }); + expect(finish.note).toBe('complete'); + expect(finish.seen).toBe(8); + expect(finish.added).toBe(8); + expect(finish.nextRunAt).toBeInstanceOf(Date); + expect(out).toEqual({ seen: 8, added: 8, updated: 0 }); + }); + + test('`{ items: }` is the same contract', async () => { + async function* walk() { + yield { items: batch(1, 2), cursor: { skip: 2 } }; + yield { items: batch(3, 3) }; + return { cursor: { skip: 3, done: true } }; + } + useAdapter({ pull: async () => ({ items: walk(), note: 'from the wrapper' }) }); + + await runSource(42, { log: () => {} }); + + expect(only('upsert')).toEqual([['x1', 'x2'], ['x3']]); + // A batch with no cursor saves nothing; only the ones that carry one do. + expect(only('saveCursor')).toEqual([{ sourceId: 42, cursor: { skip: 2 } }]); + const [finish] = only('finishRun'); + expect(finish.cursor).toEqual({ skip: 3, done: true }); + expect(finish.note).toBe('from the wrapper'); + expect(finish.seen).toBe(3); + }); + + test('a batch that lands past the deadline ends the run at the last saved cursor', async () => { + let closed = false; + useAdapter({ + budgetMs: 1, + async *pull() { + try { + yield { items: batch(1, 2), cursor: { skip: 2 } }; + await Bun.sleep(5); + yield { items: batch(3, 4), cursor: { skip: 4 } }; + yield { items: batch(5, 6), cursor: { skip: 6 } }; + return { note: 'never reached' }; + } finally { + closed = true; + } + }, + }); + + await runSource(42, { log: () => {} }); + + // The first batch is inside the budget; the second crosses it, is still + // written (it was already read), and then the generator is closed so its + // files and processes go with it. The third is never asked for. + expect(only('upsert').length).toBeLessThanOrEqual(2); + expect(closed).toBe(true); + const [finish] = only('finishRun'); + expect(finish.status).toBe('ok'); + expect(finish.nextRunAt).toBeInstanceOf(Date); + expect(finish.note).toMatch(/out of time/); + expect(finish.cursor).toEqual(only('saveCursor').at(-1).cursor); + }); + + test('a failure mid-walk keeps the cursors already saved and closes the generator', async () => { + let closed = false; + useAdapter({ + async *pull() { + try { + yield { items: batch(1, 2), cursor: { skip: 2 } }; + throw new Error('disk fell off'); + } finally { + closed = true; + } + }, + }); + + const out = await runSource(42, { log: () => {} }); + + expect(out).toEqual({ error: 'disk fell off' }); + expect(closed).toBe(true); + expect(only('saveCursor')).toEqual([{ sourceId: 42, cursor: { skip: 2 } }]); + const [finish] = only('finishRun'); + expect(finish.status).toBe('error'); + // No cursor on the error finish: `coalesce` keeps what saveCursor wrote. + expect(finish.cursor).toBeUndefined(); + }); + + test('a batch of three hundred is two upserts, and the totals add up across batches', async () => { + useAdapter({ + async *pull() { + yield { items: batch(1, 300), cursor: { skip: 300 } }; + yield { items: batch(301, 350), cursor: { skip: 350 } }; + return { cursor: { skip: 350, done: true } }; + }, + }); + + const out = await runSource(42, { log: () => {} }); + + expect(only('upsert').map((ids) => ids.length)).toEqual([200, 100, 50]); + expect(only('saveCursor').length).toBe(2); + expect(out).toEqual({ seen: 350, added: 350, updated: 0 }); + }); +}); + +describe('the array form', () => { + test('is written in one go with no interim cursor save', async () => { + useAdapter({ + pull: async () => ({ items: batch(1, 5), cursor: { since: 'abc' }, note: '5 things' }), + }); + + const out = await runSource(42, { log: () => {} }); + + expect(only('upsert')).toEqual([['x1', 'x2', 'x3', 'x4', 'x5']]); + expect(only('saveCursor')).toEqual([]); + const [finish] = only('finishRun'); + expect(finish.cursor).toEqual({ since: 'abc' }); + expect(finish.note).toBe('5 things'); + expect(finish.seen).toBe(5); + expect(out).toEqual({ seen: 5, added: 5, updated: 0 }); + }); +}); diff --git a/test/dump-reaper-guard.test.js b/test/dump-reaper-guard.test.js new file mode 100644 index 0000000..f064a1d --- /dev/null +++ b/test/dump-reaper-guard.test.js @@ -0,0 +1,115 @@ +import { beforeAll, describe, expect, test } from 'bun:test'; +import { readdir, readFile } from 'node:fs/promises'; +import { PGlite } from '@electric-sql/pglite'; +import { citext } from '@electric-sql/pglite/contrib/citext'; +import { pg_trgm } from '@electric-sql/pglite/contrib/pg_trgm'; + +/** + * A source with a run in flight must not be enqueued again. + * + * `startRun` pushes `next_run_at` one cadence forward, which covers a run + * shorter than its cadence. A dump adapter with an hour's budget on an hourly + * source, a source that asked for `nextInMinutes: 1` last time, or the reaper + * resetting a clock, all bring the source back while the first run is still + * writing -- and without a guard the tick enqueues a second run of the same + * source beside it. The guard is bounded by the same window the reaper uses + * on the same tick, so a run older than it (already marked abandoned) can + * never park a source for good. + * + * Against a real Postgres in process, because the behaviour is the `not + * exists` and nothing about it survives being mocked. + */ +let db; +beforeAll(async () => { + db = await new PGlite({ extensions: { citext, pg_trgm } }); + const dir = new URL('../packages/db/migrations/', import.meta.url).pathname; + for (const f of (await readdir(dir)).filter((n) => n.endsWith('.sql')).sort()) { + await db.exec(await readFile(dir + f, 'utf8')); + } +}, 60_000); + +const rows = async (sql, params) => (await db.query(sql, params)).rows; +const one = async (sql, params) => (await rows(sql, params))[0]; + +let n = 0; +async function source({ minutesOut = -1, enabled = true } = {}) { + n += 1; + const c = await one(`insert into collections (slug, name) values ($1, 'T') returning id`, [ + `reap${n}-${Math.random()}`, + ]); + const s = await one( + `insert into sources (collection_id, adapter, slug, name, enabled, next_run_at) + values ($1, 'x', $2, 'S', $3, now() + make_interval(mins => $4)) returning id`, + [c.id, `src${n}-${Math.random()}`, enabled, minutesOut], + ); + return s.id; +} + +async function run(sourceId, { status = 'running', minutesAgo = 0 } = {}) { + await db.query( + `insert into runs (source_id, status, started_at) + values ($1, $2, now() - make_interval(mins => $3))`, + [sourceId, status, minutesAgo], + ); +} + +/** The statement `dueSources` runs, kept identical to the query. */ +const DUE = ` + select id, slug, adapter, next_run_at from sources s + where enabled and ($1 or next_run_at <= now()) + and not exists ( + select 1 from runs r + where r.source_id = s.id and r.status = 'running' + and r.started_at > now() - ($2)::interval + ) + order by next_run_at limit $3`; + +const due = async ({ force = false, runningMinutes = 70, limit = 50 } = {}) => + (await rows(DUE, [force, `${runningMinutes} minutes`, limit])).map((r) => r.id); + +describe('dueSources and a run in flight', () => { + test('an overdue source with a run inside the window is not due', async () => { + const id = await source(); + await run(id, { minutesAgo: 30 }); + expect(await due()).not.toContain(id); + }); + + test('the same source with no running run is due', async () => { + const id = await source(); + await run(id, { status: 'ok', minutesAgo: 30 }); + await run(id, { status: 'error', minutesAgo: 5 }); + expect(await due()).toContain(id); + }); + + test('a run older than the window no longer blocks: the reaper owns it now', async () => { + const id = await source(); + await run(id, { minutesAgo: 71 }); + expect(await due({ runningMinutes: 70 })).toContain(id); + // Widen the window (a larger budgetMs was declared) and the same run blocks again. + expect(await due({ runningMinutes: 90 })).not.toContain(id); + }); + + test('the boot sweep (force) gets the same guard', async () => { + const running = await source({ minutesOut: 600 }); + const idle = await source({ minutesOut: 600 }); + await run(running, { minutesAgo: 10 }); + const ids = await due({ force: true }); + expect(ids).toContain(idle); + expect(ids).not.toContain(running); + }); + + test('reaping marks the old run abandoned and the source becomes due on the next tick', async () => { + const id = await source(); + await run(id, { minutesAgo: 100 }); + expect(await due({ runningMinutes: 70 })).toContain(id); + // The reaper's statement, as in queries.js: a run past the window is an error. + await db.query( + `update runs set status = 'error', finished_at = now(), error = 'abandoned (process exited)' + where status = 'running' and started_at < now() - ($1)::interval`, + ['70 minutes'], + ); + const left = await rows(`select status from runs where source_id = $1`, [id]); + expect(left.map((r) => r.status)).toEqual(['error']); + expect(await due({ runningMinutes: 70 })).toContain(id); + }); +}); diff --git a/test/dump-reaper-window.test.js b/test/dump-reaper-window.test.js new file mode 100644 index 0000000..1f78d87 --- /dev/null +++ b/test/dump-reaper-window.test.js @@ -0,0 +1,79 @@ +import { beforeAll, describe, expect, test } from 'bun:test'; +import { readdir, readFile } from 'node:fs/promises'; +import { PGlite } from '@electric-sql/pglite'; +import { citext } from '@electric-sql/pglite/contrib/citext'; +import { pg_trgm } from '@electric-sql/pglite/contrib/pg_trgm'; + +let db; +beforeAll(async () => { + db = await new PGlite({ extensions: { citext, pg_trgm } }); + const dir = new URL('../packages/db/migrations/', import.meta.url).pathname; + for (const f of (await readdir(dir)).filter((n) => n.endsWith('.sql')).sort()) { + await db.exec(await readFile(dir + f, 'utf8')); + } +}, 60_000); +const rows = async (sql, params) => (await db.query(sql, params)).rows; +const one = async (sql, params) => (await rows(sql, params))[0]; + +const DUE = ` + select id from sources s + where enabled and ($1 or next_run_at <= now()) + and not exists ( + select 1 from runs r + where r.source_id = s.id and r.status = 'running' + and r.started_at > now() - ($2)::interval + ) + order by next_run_at limit $3`; +const REAP = `update runs set status = 'error', finished_at = now(), error = 'abandoned (process exited)' + where status = 'running' and started_at < now() - ($1)::interval returning id, source_id`; + +// budgetMs = 60 min -> runTick window = ceil(60) + 10 = 70 +const WINDOW = `${Math.ceil((60 * 60_000) / 60_000) + 10} minutes`; + +async function scenario(minutesAgo) { + const c = await one(`insert into collections (slug, name) values ($1, 'T') returning id`, [ + `r-${Math.random()}`, + ]); + // hourly cadence: startRun pushed next_run_at 60 min out from start, so with + // the run 50 min in it is 10 min away, and 71 min in it is 11 min overdue. + const s = await one( + `insert into sources (collection_id, adapter, slug, name, enabled, cadence_minutes, next_run_at) + values ($1, 'x', $2, 'S', true, 60, now() - make_interval(mins => $3) + interval '60 minutes') returning id`, + [c.id, `s-${Math.random()}`, minutesAgo], + ); + const r = await one( + `insert into runs (source_id, status, started_at) values ($1, 'running', now() - make_interval(mins => $2)) returning id`, + [s.id, minutesAgo], + ); + return { sourceId: s.id, runId: r.id }; +} + +describe('reaper + guard at a 60 minute budget', () => { + test('50 minutes in: not reaped, not due', async () => { + const { sourceId, runId } = await scenario(50); + const reaped = await rows(REAP, [WINDOW]); + expect(reaped.map((r) => r.id)).not.toContain(runId); + expect((await rows(DUE, [false, WINDOW, 50])).map((r) => r.id)).not.toContain(sourceId); + // even the boot sweep leaves it alone + expect((await rows(DUE, [true, WINDOW, 50])).map((r) => r.id)).not.toContain(sourceId); + expect((await one('select status from runs where id = $1', [runId])).status).toBe('running'); + }); + + test('69 minutes in (budget passed by 9): still inside the window, still not due', async () => { + const { sourceId, runId } = await scenario(69); + expect((await rows(REAP, [WINDOW])).map((r) => r.id)).not.toContain(runId); + expect((await rows(DUE, [false, WINDOW, 50])).map((r) => r.id)).not.toContain(sourceId); + }); + + test('71 minutes in (budget passed by 11): reaped, then due on the same tick', async () => { + const { sourceId, runId } = await scenario(71); + // The reaper runs first in runTick, and would set next_run_at = now() for it. + const reaped = await rows(REAP, [WINDOW]); + expect(reaped.map((r) => r.id)).toContain(runId); + await db.query(`update sources set next_run_at = now() where id = $1`, [sourceId]); + expect((await rows(DUE, [false, WINDOW, 50])).map((r) => r.id)).toContain(sourceId); + expect((await one('select status, error from runs where id = $1', [runId])).error).toMatch( + /abandoned/, + ); + }); +}); diff --git a/test/music-discogs-catalog.test.js b/test/music-discogs-catalog.test.js new file mode 100644 index 0000000..e9bd007 --- /dev/null +++ b/test/music-discogs-catalog.test.js @@ -0,0 +1,719 @@ +import { afterAll, beforeAll, describe, expect, test } from 'bun:test'; +import { createHash } from 'node:crypto'; +import { mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { gzipSync } from 'node:zlib'; + +/** + * The Discogs dump adapter: two gzipped XML files a month, one download + * attempt a run, a record-tag scanner that never holds the file, and a cursor + * of `{ month, entity, recordIndex }` the next run picks up exactly. + * + * The artists fixture is the first forty records of the real September 2026 + * artists dump, carved from the head of the file; the masters fixture is the + * documented `` shape. The server here is a fake `fetch` + * that behaves like data.discogs.com: a 200 with the whole body whatever was + * asked, a 429 with a retry-after, a 404 for a month that is not there yet. + */ +process.env.DATABASE_URL ??= 'postgres://test:test@localhost:5432/test'; +process.env.SITE_URL ??= 'https://nichedb.test'; + +const { + artistItem, + checksumName, + children, + decodeXml, + discogsCatalog, + dumpMonth, + dumpUrl, + fileName, + findOpenTag, + masterItem, + nextDumpMinutes, + parseAttrs, + parseChecksums, + parseElement, + plainProfile, + recordItem, + resumeFrom, + retryAfterMinutes, + retryAfterSeconds, + scanRecords, + trimTo, + userAgent, +} = await import('../packages/adapters/src/discogs-catalog.js'); +const { normaliseItem } = await import('../packages/core/src/adapter.js'); + +const FIXTURES = join(import.meta.dir, '../packages/adapters/test/fixtures'); +const artistsXml = await readFile(join(FIXTURES, 'discogs-catalog-artists.xml'), 'utf8'); +const mastersXml = await readFile(join(FIXTURES, 'discogs-catalog-masters.xml'), 'utf8'); +const MONTH = dumpMonth(); + +const sha256 = (buf) => createHash('sha256').update(buf).digest('hex'); +const ARTIST_IDS = [...artistsXml.matchAll(/(\d+)<\/id>/g)].map((m) => Number(m[1])); + +/** The forty real records, `copies` times over with ids moved up a block each copy. */ +function bigArtists(copies) { + const body = artistsXml.replace(/^\n|<\/artists>\n$/g, ''); + const parts = []; + for (let k = 0; k < copies; k += 1) { + parts.push( + body.replace( + /(\d+)<\/id>/g, + (_, id) => `${Number(id) + k * 100000}`, + ), + ); + } + return `\n${parts.join('')}\n`; +} + +/** + * A fake data.discogs.com. `modes` maps a file name to how it answers: + * 'ok' (default), 'short' (announces more bytes than it sends), '429', + * '404', 'throw' (the socket drops), or a function returning a Response. + */ +function provider({ artists = artistsXml, masters = mastersXml, modes = {}, checksums } = {}) { + const bodies = { + [fileName(MONTH, 'artists')]: gzipSync(Buffer.from(artists)), + [fileName(MONTH, 'masters')]: gzipSync(Buffer.from(masters)), + }; + const sums = + checksums ?? + Object.entries(bodies) + .map(([name, buf]) => `${sha256(buf)} ${name}`) + .join('\n'); + const requests = []; + const uas = []; + const fetch = async (url, init = {}) => { + const name = decodeURIComponent(new URL(url).searchParams.get('download') ?? '') + .split('/') + .pop(); + requests.push(name); + uas.push(init.headers?.['user-agent']); + const mode = modes[name] ?? 'ok'; + if (typeof mode === 'function') return mode(); + if (mode === 'throw') throw new Error('ECONNRESET'); + if (mode === '404') return new Response('no', { status: 404 }); + if (mode === '429') + return new Response('slow down', { status: 429, headers: { 'retry-after': '3359' } }); + if (name === checksumName(MONTH)) return new Response(sums, { status: 200 }); + const body = bodies[name]; + if (!body) return new Response('no', { status: 404 }); + if (mode === 'short') { + return new Response(body.subarray(0, body.length - 100), { + status: 200, + headers: { 'content-length': String(body.length) }, + }); + } + return new Response(body, { status: 200, headers: { 'content-length': String(body.length) } }); + }; + return { fetch, requests, uas, bodies }; +} + +let root; +let dirs = 0; +beforeAll(async () => { + root = await mkdtemp(join(tmpdir(), 'nichedb-discogs-test-')); +}); +afterAll(async () => { + await rm(root, { recursive: true, force: true }); +}); +function freshDir() { + dirs += 1; + return join(root, `run-${dirs}`); +} + +/** Drive `pull` the way the core does: collect every batch, keep the return value. */ +async function run(p, { cursor = {}, dir, deadline = Number.POSITIVE_INFINITY, log } = {}) { + const gen = discogsCatalog.pull({ + config: { cacheDir: dir }, + cursor, + env: { contactEmail: 'ops@nichedb.test' }, + http: { fetch: p.fetch }, + log: log ?? (() => {}), + deadline, + }); + const batches = []; + for (;;) { + const { value, done } = await gen.next(); + if (done) return { batches, items: batches.flatMap((b) => b.items), ...value }; + batches.push(value); + } +} + +const exists = async (path) => (await stat(path).catch(() => null)) !== null; + +describe('the XML pieces', () => { + test('the five entities and numeric references decode once, nothing else changes', () => { + expect(decodeXml('Mr. James Barth & A.D. <x> "q" 'a'')).toBe( + 'Mr. James Barth & A.D. "q" \'a\'', + ); + expect(decodeXml('a bA')).toBe('a\rbA'); + expect(decodeXml('&#13;')).toBe(' '); + expect(decodeXml(' �')).toBe(' �'); + }); + + test('an open tag must end at the name, so ', () => { + const xml = '1'; + expect(findOpenTag(xml, 'artist')).toBe(9); + expect(findOpenTag(xml, 'name')).toBe(-1); + expect(findOpenTag('x', 'name', 0, { opening: true })).toBe(14); + }); + + test('attributes in either quote, bare, or unquoted', () => { + expect(parseAttrs(' id="18500" type=\'primary\' embed=true checked')).toEqual({ + id: '18500', + type: 'primary', + embed: 'true', + checked: '', + }); + expect(parseAttrs(' uri="a&b"')).toEqual({ uri: 'a&b' }); + }); + + test('elements nest by depth, self-close, and survive truncation', () => { + const el = parseElement('intail'); + expect(el.name).toBe('a'); + expect(el.attrs).toEqual({ x: '1' }); + expect(el.inner).toBe('intail'); + expect(parseElement('', 0)).toMatchObject({ name: 'c', inner: '', end: 4 }); + expect(parseElement('never closed', 0)).toMatchObject({ + name: 'a', + inner: 'never closed', + }); + expect(parseElement('no tags here')).toBeNull(); + expect(children('1 3').map((e) => e.name)).toEqual(['a', 'b', 'c']); + }); + + test('records are found across chunk boundaries and skip counts them', async () => { + const bytes = Buffer.from(artistsXml); + async function* chunks(size) { + for (let i = 0; i < bytes.length; i += size) yield bytes.subarray(i, i + size); + } + const all = []; + for await (const r of scanRecords(chunks(7), 'artist')) all.push(r); + expect(all).toHaveLength(40); + expect(all[0].startsWith('1')).toBe(true); + expect(all[39].endsWith('')).toBe(true); + const tail = []; + for await (const r of scanRecords(chunks(4096), 'artist', { skip: 38 })) tail.push(r); + expect(tail.map((r) => r.match(/(\d+) parseElement(r).attrs.id)).toEqual([]); + const mb = Buffer.from(mastersXml); + async function* mchunks() { + for (let i = 0; i < mb.length; i += 50) yield mb.subarray(i, i + 50); + } + for await (const r of scanRecords(mchunks(), 'master')) masters.push(r); + expect(masters.map((r) => parseElement(r).attrs.id)).toEqual(['18500', '18512', '18520']); + }); + + test('profile markup becomes plain text and summaries cut at a word', () => { + expect( + plainProfile( + 'Formed [l=Ovum Recordings] with [a=King Britt]. \nSee [url=http://x]site[/url] [b]now[/b].', + ), + ).toBe('Formed Ovum Recordings with King Britt. See site now.'); + expect(trimTo('short', 10)).toBe('short'); + const long = trimTo('word '.repeat(200), 600); + expect(long.length).toBeLessThanOrEqual(600); + expect(long.endsWith('...')).toBe(true); + }); +}); + +describe('the items', () => { + const records = (xml, tag) => children(parseElement(xml).inner).filter((e) => e.name === tag); + + test('an artist row, from the real first record of the dump', () => { + const [persuader, barth] = records(artistsXml, 'artist'); + const item = normaliseItem(artistItem(persuader)); + expect(item.externalId).toBe('discogs:artist:1'); + expect(item.kind).toBe('artist'); + expect(item.title).toBe('The Persuader'); + expect(item.url).toBe('https://www.discogs.com/artist/1'); + expect(item.summary).toBe('Electronic artist working out of Stockholm, active since 1994.'); + expect(item.imageUrl).toBeNull(); + expect(item.tags).toEqual(['artist', 'discogs']); + expect(item.data).toMatchObject({ + discogsId: 1, + name: 'The Persuader', + realName: 'Jesper Dahlbäck', + nameVariations: ['Persuader', 'The Presuader'], + dataQuality: 'Needs Vote', + attribution: 'Discogs, CC0', + }); + expect(item.data.aliases).toContain('Groove Machine'); + expect(item.data.urls).toEqual([ + 'https://en.wikipedia.org/wiki/Jesper_Dahlbäck', + 'https://www.last.fm/music/Jesper+Dahlb%C3%A4ck', + ]); + expect(item.data.members).toEqual([]); + + const group = artistItem(barth); + expect(group.title).toBe('Mr. James Barth & A.D.'); + expect(group.data.members).toEqual(['Alexi Delano', 'Cari Lekebusch']); + expect(group.data.aliases).toContain('Yakari & Delano'); + expect(group.summary).toBeNull(); + }); + + test('every real record in the fixture becomes an item with a positive id', () => { + const items = records(artistsXml, 'artist').map(artistItem); + expect(items).toHaveLength(40); + expect(items.every((i) => i && i.data.discogsId > 0 && i.title)).toBe(true); + expect(items.map((i) => i.data.discogsId)).toEqual(ARTIST_IDS); + const withGroups = items.filter((i) => i.data.groups.length); + expect(withGroups.length).toBeGreaterThan(0); + }); + + test('a master row: id from the attribute, artists, genres, styles, year, videos', () => { + const [soil, stockholm, undated] = records(mastersXml, 'master'); + const item = normaliseItem(masterItem(soil)); + expect(item.externalId).toBe('discogs:master:18500'); + expect(item.kind).toBe('master'); + expect(item.title).toBe('Samuel L Session - New Soil'); + expect(item.url).toBe('https://www.discogs.com/master/18500'); + expect(item.summary).toBe('2001 · Electronic · Techno'); + expect(item.publishedAt.toISOString().slice(0, 4)).toBe('2001'); + expect(item.precision).toBe('year'); + expect(item.tags).toEqual(['master', 'discogs', 'genre:electronic', 'style:techno']); + expect(item.data).toMatchObject({ + discogsId: 18500, + title: 'New Soil', + mainRelease: 155102, + year: 2001, + artists: [{ name: 'Samuel L Session', id: 212070 }], + genres: ['Electronic'], + styles: ['Techno'], + dataQuality: 'Correct', + attribution: 'Discogs, CC0', + }); + expect(item.data.videos).toEqual([ + 'https://www.youtube.com/watch?v=f05Ai921itM', + 'https://www.youtube.com/watch?v=v23rSPG_StA', + ]); + + const two = masterItem(stockholm); + expect(two.title).toBe('The Persuader, Mr. James Barth & A.D. - Stockholm "Vol. 1"'); + expect(two.data.title).toBe('Stockholm "Vol. 1"'); + expect(two.tags).toEqual([ + 'master', + 'discogs', + 'genre:electronic', + 'genre:hip-hop', + 'style:deep-house', + 'style:tech-house', + ]); + expect(two.data.videos).toEqual([]); + + const none = masterItem(undated); + expect(none.data.year).toBeNull(); + expect(none.publishedAt).toBeNull(); + expect(none.summary).toBe('Electronic · Acid'); + }); + + test('a record without its id or its name is null, never a throw', () => { + expect(recordItem('artists', 'Nobody')).toBeNull(); + expect(recordItem('artists', '7')).toBeNull(); + expect(recordItem('artists', 'xN')).toBeNull(); + expect(recordItem('masters', 'T')).toBeNull(); + expect(recordItem('masters', '')).toBeNull(); + expect(recordItem('masters', null)).toBeNull(); + expect(recordItem('artists', ' { + test('month, urls, names', () => { + expect(dumpMonth(new Date('2026-09-13T10:00:00Z'))).toBe('20260901'); + expect(dumpMonth(new Date('2026-12-31T23:59:59Z'))).toBe('20261201'); + expect(dumpUrl('20260901', fileName('20260901', 'artists'))).toBe( + 'https://data.discogs.com/?download=data%2F2026%2Fdiscogs_20260901_artists.xml.gz', + ); + expect(checksumName('20260901')).toBe('discogs_20260901_CHECKSUM.txt'); + expect(userAgent({ contactEmail: 'ops@nichedb.test' })).toBe( + 'niche-db discogs-catalog (+https://github.com/profullstack/niche-db; ops@nichedb.test)', + ); + expect(userAgent({})).toContain('discogs-catalog'); + }); + + test('retry-after in seconds or as a date, and the minutes scheduled past it', () => { + const now = Date.parse('2026-09-13T10:00:00Z'); + expect(retryAfterSeconds('3359', now)).toBe(3359); + expect(retryAfterSeconds('Sun, 13 Sep 2026 10:10:00 GMT', now)).toBe(600); + expect(retryAfterSeconds(null, now)).toBe(3600); + expect(retryAfterSeconds('soon', now)).toBe(3600); + expect(retryAfterMinutes('3359', now)).toBe(58); + expect(retryAfterMinutes(undefined, now)).toBe(62); + }); + + test('the next dump is the first of next month, six hours in', () => { + const m = nextDumpMinutes(new Date('2026-09-30T06:00:00Z')); + expect(m).toBe(24 * 60); + expect(nextDumpMinutes(new Date('2026-10-01T05:59:00Z'))).toBeGreaterThan(30 * 24 * 60); + }); + + test('checksum lines in sha256sum form, tolerant of order and tabs', () => { + const h = 'a'.repeat(64); + expect( + parseChecksums( + `${h} discogs_20260901_artists.xml.gz\r\ndiscogs_x.xml.gz\t${'B'.repeat(64)}\n\nnoise`, + ), + ).toEqual({ 'discogs_20260901_artists.xml.gz': h, 'discogs_x.xml.gz': 'b'.repeat(64) }); + }); + + test('a cursor from another month starts over; this month resumes where it was', () => { + expect(resumeFrom({}, '20260901')).toEqual({ + month: '20260901', + entity: 'artists', + recordIndex: 0, + done: false, + verified: [], + checksums: null, + }); + expect( + resumeFrom( + { + month: '20260901', + entity: 'masters', + recordIndex: 1500, + verified: ['artists'], + checksums: { a: 'b' }, + }, + '20260901', + ), + ).toMatchObject({ + entity: 'masters', + recordIndex: 1500, + verified: ['artists'], + checksums: { a: 'b' }, + }); + expect( + resumeFrom( + { month: '20260801', entity: 'masters', recordIndex: 1500, done: true }, + '20260901', + ), + ).toMatchObject({ entity: 'artists', recordIndex: 0, done: false }); + expect( + resumeFrom({ month: '20260901', entity: 'labels', recordIndex: -4 }, '20260901'), + ).toMatchObject({ + entity: 'artists', + recordIndex: 0, + }); + }); + + test('the adapter declares its budget, kinds and licence', () => { + expect(discogsCatalog.name).toBe('discogs-catalog'); + expect(discogsCatalog.collection).toBe('music'); + expect(discogsCatalog.kinds).toEqual(['artist', 'master']); + expect(discogsCatalog.budgetMs).toBe(55 * 60_000); + expect(discogsCatalog.description).toContain('CC0'); + expect(discogsCatalog.defaultSources[0].slug).toBe('discogs-catalog'); + }); +}); + +describe('a walk', () => { + /** Drive `pull` until the cursor says done, the way the scheduler would across runs. */ + async function runs(p, dir, cursor = {}, max = 8) { + const out = []; + for (let i = 0; i < max; i += 1) { + const r = await run(p, { dir, cursor }); + out.push(r); + cursor = r.cursor; + if (cursor?.done) break; + } + return out; + } + + test('one request a run: download, checksum list, walk; the second file the same way; then the pass is complete', async () => { + const dir = freshDir(); + const p = provider(); + const [first, second, third, fourth] = await runs(p, dir); + // Three requests for the month, one a run, in this order; runs 2 and 3 walk. + expect(p.requests).toEqual([ + fileName(MONTH, 'artists'), + checksumName(MONTH), + fileName(MONTH, 'masters'), + ]); + // Run 1 spends its one request on the artists file and stops there. + expect(p.uas.every((ua) => ua?.includes('discogs-catalog'))).toBe(true); + expect(first.items).toEqual([]); + expect(first.cursor).toMatchObject({ month: MONTH, entity: 'artists', recordIndex: 0 }); + expect(first.nextInMinutes).toBe(10); + expect(first.note).toContain('checksum list'); + expect(await exists(join(dir, fileName(MONTH, 'artists')))).toBe(true); + + // Run 2: the checksum list, the file verified, walked to the end; masters wait. + expect(p.requests[1]).toBe(checksumName(MONTH)); + expect(second.batches).toHaveLength(1); + expect(second.items.map((i) => i.externalId)).toEqual( + ARTIST_IDS.map((id) => `discogs:artist:${id}`), + ); + expect(second.batches[0].cursor).toMatchObject({ + month: MONTH, + entity: 'artists', + recordIndex: 40, + }); + expect(second.cursor).toMatchObject({ + month: MONTH, + entity: 'masters', + recordIndex: 0, + verified: ['artists'], + }); + expect(second.cursor.checksums[fileName(MONTH, 'artists')]).toBe( + sha256(p.bodies[fileName(MONTH, 'artists')]), + ); + expect(second.nextInMinutes).toBe(10); + expect(second.note).toContain('one request a run'); + + // Run 3: the masters file is the one request; the list is already in the + // cursor, so the same run verifies it, walks it and completes the pass. + expect(p.requests[2]).toBe(fileName(MONTH, 'masters')); + expect(p.requests).toHaveLength(3); + expect(third.items.map((i) => i.externalId)).toEqual([ + 'discogs:master:18500', + 'discogs:master:18512', + 'discogs:master:18520', + ]); + expect(third.cursor).toMatchObject({ + month: MONTH, + entity: 'masters', + recordIndex: 3, + done: true, + }); + expect(third.cursor.verified).toEqual(['artists', 'masters']); + expect(third.note).toContain('complete'); + expect(third.nextInMinutes).toBeGreaterThanOrEqual(60); + expect(fourth).toBeUndefined(); + + // Another run this month asks nothing and waits for the next dump. + const again = await run(p, { dir, cursor: third.cursor }); + expect(p.requests).toHaveLength(3); + expect(again.items).toEqual([]); + expect(again.note).toContain('unchanged'); + expect(again.cursor.done).toBe(true); + }); + + /** Files on disk and verified, so nothing stands between the deadline and the walk. */ + async function onDisk(dir, p) { + await Bun.write(join(dir, fileName(MONTH, 'artists')), p.bodies[fileName(MONTH, 'artists')]); + await Bun.write(join(dir, fileName(MONTH, 'masters')), p.bodies[fileName(MONTH, 'masters')]); + return { + month: MONTH, + entity: 'artists', + recordIndex: 0, + verified: ['artists', 'masters'], + checksums: {}, + }; + } + + test('a deadline already passed yields nothing and asks for ten minutes', async () => { + const dir = freshDir(); + const p = provider({ artists: bigArtists(30) }); + const verified = await onDisk(dir, p); + const out = await run(p, { dir, cursor: verified, deadline: Date.now() - 1 }); + expect(p.requests).toEqual([]); + expect(out.batches).toEqual([]); + expect(out.cursor).toMatchObject({ entity: 'artists', recordIndex: 0 }); + expect(out.nextInMinutes).toBe(10); + expect(out.note).toContain('deadline'); + }); + + test('the deadline stops a run after a batch and the next run resumes at that record', async () => { + const dir = freshDir(); + const big = bigArtists(30); // 1,200 records: two full batches and a tail + const p = provider({ artists: big }); + const verified = await onDisk(dir, p); + + // A clock that jumps a minute per reading: the walk starts inside the + // deadline and the first batch lands past it. + const realNow = Date.now; + const base = realNow(); + let ticks = 0; + Date.now = () => { + ticks += 1; + return base + ticks * 60_000; + }; + let first; + try { + first = await run(p, { dir, cursor: verified, deadline: base + 90_000 }); + } finally { + Date.now = realNow; + } + expect(p.requests).toEqual([]); + expect(first.batches).toHaveLength(1); + expect(first.items).toHaveLength(500); + expect(first.cursor).toMatchObject({ entity: 'artists', recordIndex: 500 }); + expect(first.nextInMinutes).toBe(10); + expect(first.note).toContain('deadline'); + + const second = await run(p, { dir, cursor: first.cursor }); + expect(p.requests).toEqual([]); + // Record 501 is the 21st record of the 13th copy. + const expected = ARTIST_IDS[20] + 12 * 100000; + expect(second.items[0].externalId).toBe(`discogs:artist:${expected}`); + expect(second.items.filter((i) => i.kind === 'artist')).toHaveLength(700); + expect(second.items.filter((i) => i.kind === 'master')).toHaveLength(3); + expect(second.batches.map((b) => b.cursor.recordIndex)).toEqual([1000, 1200, 3]); + expect(second.cursor).toMatchObject({ entity: 'masters', recordIndex: 3, done: true }); + }); + + test('a download the server cut short yields nothing, leaves no file, and tries again in ten minutes', async () => { + const dir = freshDir(); + const p = provider({ modes: { [fileName(MONTH, 'artists')]: 'short' } }); + const out = await run(p, { dir }); + expect(out.items).toEqual([]); + expect(out.cursor).toMatchObject({ month: MONTH, entity: 'artists', recordIndex: 0 }); + expect(out.nextInMinutes).toBe(10); + expect(out.note).toContain('incomplete'); + expect(await exists(join(dir, fileName(MONTH, 'artists')))).toBe(false); + expect(await exists(join(dir, `${fileName(MONTH, 'artists')}.part`))).toBe(false); + expect(p.requests).toHaveLength(1); + }); + + test('a 429 schedules the next run past retry-after and keeps the place', async () => { + const dir = freshDir(); + const p = provider({ modes: { [fileName(MONTH, 'masters')]: '429' } }); + const cursor = { + month: MONTH, + entity: 'masters', + recordIndex: 0, + verified: ['artists'], + checksums: {}, + }; + const out = await run(p, { dir, cursor }); + expect(out.items).toEqual([]); + expect(out.nextInMinutes).toBe(58); + expect(out.cursor).toMatchObject({ + month: MONTH, + entity: 'masters', + recordIndex: 0, + verified: ['artists'], + }); + expect(out.note).toContain('rate limited'); + expect(p.requests).toHaveLength(1); + + // The checksum list rate limited the same way, with no retry-after: an hour and two. + const p2 = provider({ + modes: { + [checksumName(MONTH)]: () => new Response('slow down', { status: 429 }), + }, + }); + await Bun.write(join(dir, fileName(MONTH, 'masters')), p2.bodies[fileName(MONTH, 'masters')]); + const listed = await run(p2, { dir, cursor: { ...cursor, checksums: null } }); + expect(p2.requests).toEqual([checksumName(MONTH)]); + expect(listed.items).toEqual([]); + expect(listed.nextInMinutes).toBe(62); + }); + + test('a month whose file is not published yet starts the new month at record 0 and looks again in six hours', async () => { + const dir = freshDir(); + const p = provider({ modes: { [fileName(MONTH, 'artists')]: '404' } }); + const old = { month: '20250101', entity: 'masters', recordIndex: 9, done: true }; + const out = await run(p, { dir, cursor: old }); + expect(out.items).toEqual([]); + expect(out.cursor).toMatchObject({ month: MONTH, entity: 'artists', recordIndex: 0 }); + expect(out.cursor.done).toBeFalsy(); + expect(out.nextInMinutes).toBe(360); + expect(out.note).toContain('not published'); + }); + + test('a 404 on the second file never steps the cursor back behind batches already saved', async () => { + const dir = freshDir(); + const p = provider({ + artists: bigArtists(3), + modes: { [fileName(MONTH, 'masters')]: '404' }, + }); + await Bun.write(join(dir, fileName(MONTH, 'artists')), p.bodies[fileName(MONTH, 'artists')]); + const prev = { + month: MONTH, + entity: 'artists', + recordIndex: 40, + verified: ['artists'], + checksums: {}, + }; + const out = await run(p, { dir, cursor: prev }); + expect(out.items).toHaveLength(80); + expect(out.batches[0].cursor).toMatchObject({ entity: 'artists', recordIndex: 120 }); + expect(p.requests).toEqual([fileName(MONTH, 'masters')]); + expect(out.cursor).toMatchObject({ month: MONTH, entity: 'masters', recordIndex: 0 }); + expect(out.nextInMinutes).toBe(360); + // And the next run walks nothing of artists again. + const next = await run(p, { dir, cursor: out.cursor }); + expect(next.items).toEqual([]); + expect(next.cursor).toMatchObject({ entity: 'masters', recordIndex: 0 }); + }); + + test('a bad record is skipped and counted; the rest of the file is written', async () => { + const dir = freshDir(); + const broken = artistsXml.replace( + '', + 'No Id\n77\n78After The Bad Ones\n', + ); + const p = provider({ artists: broken }); + const notes = []; + const [, out] = await runs(p, dir, {}, 2); + expect(out.items).toHaveLength(41); + expect(out.items.at(-1).externalId).toBe('discogs:artist:78'); + expect(out.batches[0].cursor.recordIndex).toBe(43); + expect(out.note).toContain('2 records skipped'); + expect(notes).toEqual([]); + }); + + test('a checksum that does not match discards the file; the next run downloads it again', async () => { + const dir = freshDir(); + const p = provider({ checksums: `${'0'.repeat(64)} ${fileName(MONTH, 'artists')}` }); + const [, out] = await runs(p, dir, {}, 2); + expect(out.items).toEqual([]); + expect(out.nextInMinutes).toBe(10); + expect(out.note).toContain('checksum'); + expect(await exists(join(dir, fileName(MONTH, 'artists')))).toBe(false); + expect(out.cursor.checksums[fileName(MONTH, 'artists')]).toBe('0'.repeat(64)); + expect(out.cursor.verified).toEqual([]); + + // The list is remembered; the next run spends its one request on the file again. + const again = await run(p, { dir, cursor: out.cursor }); + expect(p.requests).toEqual([ + fileName(MONTH, 'artists'), + checksumName(MONTH), + fileName(MONTH, 'artists'), + ]); + expect(again.items).toEqual([]); + expect(again.note).toContain('checksum'); + }); + + test('a run whose only request fails throws and leaves the cursor alone', async () => { + const dir = freshDir(); + const p = provider({ modes: { [fileName(MONTH, 'artists')]: 'throw' } }); + await expect(run(p, { dir })).rejects.toThrow(/every request failed \(1\)/); + const p5 = provider({ + modes: { [fileName(MONTH, 'artists')]: () => new Response('down', { status: 503 }) }, + }); + await expect(run(p5, { dir })).rejects.toThrow(/503/); + expect(await exists(join(dir, fileName(MONTH, 'artists')))).toBe(false); + }); + + test('too little budget left to download waits rather than starting a transfer it cannot finish', async () => { + const dir = freshDir(); + const p = provider(); + const out = await run(p, { dir, deadline: Date.now() + 60_000 }); + expect(p.requests).toEqual([]); + expect(out.items).toEqual([]); + expect(out.nextInMinutes).toBe(10); + expect(out.note).toContain('budget'); + }); + + test('a new month clears last month files and any partial from the cache', async () => { + const dir = freshDir(); + const p = provider(); + await Bun.write(join(dir, 'discogs_20250101_artists.xml.gz'), 'old'); + await Bun.write(join(dir, `${fileName(MONTH, 'masters')}.part`), 'half'); + await writeFile(join(dir, 'unrelated.txt'), 'keep'); + const [first, second] = await runs(p, dir, { month: '20250101', done: true }, 2); + expect(first.cursor).toMatchObject({ month: MONTH, entity: 'artists', recordIndex: 0 }); + expect(second.items).toHaveLength(40); + expect(await exists(join(dir, 'discogs_20250101_artists.xml.gz'))).toBe(false); + expect(await exists(join(dir, `${fileName(MONTH, 'masters')}.part`))).toBe(false); + expect(await exists(join(dir, 'unrelated.txt'))).toBe(true); + }); +}); diff --git a/test/music-musicbrainz-catalog.test.js b/test/music-musicbrainz-catalog.test.js new file mode 100644 index 0000000..bfacb16 --- /dev/null +++ b/test/music-musicbrainz-catalog.test.js @@ -0,0 +1,651 @@ +import { afterAll, beforeAll, describe, expect, test } from 'bun:test'; +import { copyFile, mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +/** + * MusicBrainz's JSON dumps, walked a batch at a time. + * + * The fixtures are real bytes: `instrument.tar.xz` exactly as MetaBrainz + * serves it (the smallest dump, 468 KB), and the first 200 rows of the real + * artist and release-group members, cut out of a Range GET of each archive's + * first 2 MB and re-packed as tar.xz with the same member names and the same + * missing final newline. The walk runs against a fake `http` whose download + * copies a fixture into a temp dump directory, so what is under test is the + * adapter's cursor, its deadline, its resume and its mapping, not the network. + */ +process.env.DATABASE_URL ??= 'postgres://test:test@localhost:5432/test'; +process.env.SITE_URL ??= 'https://nichedb.test'; + +const { + ATTRIBUTION, + BATCH_SIZE, + BUDGET_MS, + ENTITIES, + LATEST_URL, + NEAR_MS, + USER_AGENT, + artistCredit, + artistItem, + creditText, + dumpUrl, + localFile, + memberOf, + musicbrainzCatalog, + nextEntity, + parseLatest, + parseRow, + pruneOthers, + releaseGroupItem, + resumeFrom, + toItem, + walk, +} = await import('../packages/adapters/src/musicbrainz-catalog.js'); +const { normaliseItem } = await import('../packages/core/src/adapter.js'); +const { xzLines } = await import('../packages/core/src/dump.js'); + +const FIXTURES = new URL('../packages/adapters/test/fixtures/', import.meta.url).pathname; +const fixture = (entity) => join(FIXTURES, `musicbrainz-catalog-${entity}-200.tar.xz`); +const INSTRUMENT = join(FIXTURES, 'musicbrainz-catalog-instrument.tar.xz'); + +const DIR = '20260912-001001'; +const NEWER = '20260915-001001'; + +let tmp; +let rows; + +const collect = async (it) => { + const out = []; + for await (const x of it) out.push(x); + return out; +}; + +/** The first few real rows of each member, parsed. */ +async function firstRows(entity, n) { + const out = []; + for await (const line of xzLines(fixture(entity), { member: memberOf(entity) })) { + out.push(JSON.parse(line)); + if (out.length >= n) break; + } + return out; +} + +beforeAll(async () => { + tmp = await mkdtemp(join(tmpdir(), 'nichedb-mb-catalog-')); + rows = { + artist: await firstRows('artist', 3), + 'release-group': await firstRows('release-group', 3), + }; +}); + +afterAll(async () => { + await rm(tmp, { recursive: true, force: true }); +}); + +/** + * A stand-in for the core's http. `latest` is what LATEST answers (a function + * may throw); `files` maps an entity to the archive a download copies into + * place, or to `{ partial }` for a download that is not done, or to a + * function that throws. + */ +function provider({ latest = DIR, files = {} } = {}) { + const calls = { text: [], download: [] }; + const archives = { + artist: fixture('artist'), + 'release-group': fixture('release-group'), + ...files, + }; + return { + calls, + http: { + async text(url, opts) { + calls.text.push({ url, headers: opts?.headers ?? {} }); + return typeof latest === 'function' ? latest() : `${latest}\n`; + }, + async download(url, filePath, opts) { + calls.download.push({ url, filePath, headers: opts?.headers ?? {} }); + const entity = ENTITIES.find((e) => url.endsWith(`/${e}.tar.xz`)); + const src = archives[entity]; + if (typeof src === 'function') return src(filePath); + if (src && typeof src === 'object' && src.partial) { + await writeFile(filePath, Buffer.alloc(src.partial)); + return { path: filePath, bytes: src.partial, complete: false }; + } + await copyFile(src, filePath); + return { path: filePath, bytes: (await readFile(filePath)).length, complete: true }; + }, + }, + }; +} + +const ctx = ( + p, + { cursor = {}, deadline = Number.POSITIVE_INFINITY, batchSize = BATCH_SIZE } = {}, +) => ({ + config: { batchSize }, + cursor, + env: {}, + http: p.http, + log: () => {}, + deadline, +}); + +/** Drain a walk by hand, keeping its return value the way the core does. */ +async function drain(gen) { + const batches = []; + const it = gen[Symbol.asyncIterator](); + for (;;) { + const { value, done } = await it.next(); + if (done) return { batches, outcome: value }; + batches.push(value); + } +} + +const run = (p, opts = {}, seam = {}) => + drain(walk(ctx(p, opts), { dataDir: seam.dataDir ?? tmp, pauseMs: 0, ...seam })); + +describe('the fixtures are real dump bytes', () => { + test('instrument.tar.xz as served: 1058 JSON rows, the last one without a newline, none of them an item here', async () => { + const lines = await collect(xzLines(INSTRUMENT, { member: memberOf('instrument') })); + expect(lines).toHaveLength(1058); + for (const l of lines) expect(typeof JSON.parse(l).id).toBe('string'); + expect(lines.at(-1)).toMatch(/\}$/); + expect(toItem('instrument', parseRow(lines[0]))).toBeNull(); + }); + + test('the artist and release-group members hold 200 rows each with the fields the dump carries', async () => { + for (const entity of ENTITIES) { + const lines = await collect(xzLines(fixture(entity), { member: memberOf(entity) })); + expect(lines).toHaveLength(200); + } + const a = rows.artist[0]; + for (const k of [ + 'id', + 'name', + 'sort-name', + 'life-span', + 'aliases', + 'relations', + 'tags', + 'genres', + 'rating', + 'annotation', + ]) { + expect(a).toHaveProperty(k); + } + const g = rows['release-group'][0]; + for (const k of [ + 'id', + 'title', + 'artist-credit', + 'primary-type', + 'secondary-types', + 'first-release-date', + 'tags', + 'genres', + 'rating', + ]) { + expect(g).toHaveProperty(k); + } + }); +}); + +describe('items', () => { + test('an artist row: MusicBrainz id in the external id, the CC0 fields kept, the CC BY-NC-SA fields and relations dropped', () => { + const a = rows.artist[0]; + const item = artistItem(a); + expect(item.externalId).toBe(`musicbrainz:artist:${a.id}`); + expect(item.kind).toBe('artist'); + expect(item.title).toBe(a.name); + expect(item.summary).toBe(a.disambiguation || null); + expect(item.url).toBe(`https://musicbrainz.org/artist/${a.id}`); + expect(item.imageUrl).toBeNull(); + expect(item.tags).toContain('artist'); + expect(item.tags).toContain('musicbrainz'); + expect(item.tags).toContain(`type:${a.type.toLowerCase()}`); + expect(item.tags).toContain(`country:${a.country.toLowerCase()}`); + expect(item.data).toMatchObject({ + mbid: a.id, + sortName: a['sort-name'], + type: a.type, + gender: a.gender, + country: a.country, + area: a.area.name, + beginArea: a['begin-area'].name, + lifeSpan: { begin: a['life-span'].begin, end: a['life-span'].end, ended: false }, + attribution: ATTRIBUTION, + }); + expect(item.data.aliases).toEqual([...new Set(a.aliases.map((x) => x.name))]); + expect(item.data.isnis).toEqual(a.isnis); + expect(item.data.ipis).toEqual(a.ipis); + for (const k of ['tags', 'genres', 'rating', 'annotation', 'relations']) { + expect(item.data).not.toHaveProperty(k); + } + expect(JSON.stringify(item)).not.toContain('"relations"'); + const stored = normaliseItem(item); + expect(stored.externalId).toBe(item.externalId); + expect(stored.data.attribution).toBe('MusicBrainz, CC0'); + }); + + test('a release group row: title, credit, types, first release date as a loose date, Cover Art Archive front', () => { + const g = rows['release-group'][0]; + const item = releaseGroupItem(g); + expect(item.externalId).toBe(`musicbrainz:release-group:${g.id}`); + expect(item.kind).toBe('release-group'); + expect(item.title).toBe(g.title); + expect(item.summary).toBe(`by ${creditText(g['artist-credit'])}`); + expect(item.url).toBe(`https://musicbrainz.org/release-group/${g.id}`); + expect(item.imageUrl).toBe(`https://coverartarchive.org/release-group/${g.id}/front-250`); + expect(item.tags).toEqual( + expect.arrayContaining([ + 'release-group', + 'musicbrainz', + `type:${g['primary-type'].toLowerCase()}`, + ...g['secondary-types'].map((s) => `secondary:${s.toLowerCase()}`), + ]), + ); + expect(item.data).toEqual({ + mbid: g.id, + primaryType: g['primary-type'], + secondaryTypes: g['secondary-types'], + firstReleaseDate: g['first-release-date'], + artistCredit: g['artist-credit'].map((c) => ({ name: c.name, mbid: c.artist.id })), + attribution: ATTRIBUTION, + }); + // "1979" is a year: the date lands mid-year at year precision. + expect(g['first-release-date']).toBe('1979'); + expect(item.precision).toBe('year'); + expect(item.publishedAt.toISOString()).toBe('1979-07-01T12:00:00.000Z'); + expect(item.timeKnown).toBe(false); + expect(normaliseItem(item)).not.toBeNull(); + }); + + test('credits: join phrases read on the sleeve, ids ride along, rubbish is ignored', () => { + const credit = [ + { name: 'Simon', joinphrase: ' & ', artist: { id: 'a1', name: 'Paul Simon' } }, + { name: 'Garfunkel', joinphrase: '', artist: { id: 'a2', name: 'Art Garfunkel' } }, + { junk: true }, + ]; + expect(creditText(credit)).toBe('Simon & Garfunkel'); + expect(artistCredit(credit)).toEqual([ + { name: 'Simon', mbid: 'a1' }, + { name: 'Garfunkel', mbid: 'a2' }, + ]); + expect(creditText(null)).toBeNull(); + expect(artistCredit('no')).toEqual([]); + }); + + test('rows that are not entities map to nothing', () => { + expect(artistItem({ id: 'x' })).toBeNull(); + expect(artistItem({ name: 'x' })).toBeNull(); + expect(releaseGroupItem({ id: 'x', title: '' })).toBeNull(); + expect(toItem('artist', null)).toBeNull(); + expect(toItem('label', { id: 'x', name: 'y' })).toBeNull(); + expect(parseRow('not json')).toBeNull(); + expect(parseRow('[1,2]')).toBeNull(); + expect(parseRow('')).toBeNull(); + expect(parseRow('{"id":"x"}')).toEqual({ id: 'x' }); + }); + + test('an artist with no type or country carries neither tag and empty lists', () => { + const item = artistItem({ id: 'abc', name: 'Nobody', type: null, country: null }); + expect(item.tags).toEqual(['artist', 'musicbrainz']); + expect(item.data.aliases).toEqual([]); + expect(item.data.lifeSpan).toEqual({ begin: null, end: null, ended: false }); + }); +}); + +describe('pure pieces', () => { + test('LATEST', () => { + expect(parseLatest('20260912-001001\n')).toBe('20260912-001001'); + expect(parseLatest(' \n20260912-001001')).toBe('20260912-001001'); + expect(parseLatest('')).toBeNull(); + expect(parseLatest('')).toBeNull(); + expect(parseLatest(null)).toBeNull(); + }); + + test('urls and files', () => { + expect(dumpUrl(DIR, 'artist')).toBe( + 'https://data.metabrainz.org/pub/musicbrainz/data/json-dumps/20260912-001001/artist.tar.xz', + ); + expect(memberOf('release-group')).toBe('mbdump/release-group'); + expect(localFile('/d', DIR, 'artist')).toBe('/d/20260912-001001-artist.tar.xz'); + expect(nextEntity('artist')).toBe('release-group'); + expect(nextEntity('release-group')).toBeNull(); + }); + + test('resume: same dump keeps the place, another dump restarts, done holds only for the same dump', () => { + expect(resumeFrom({}, DIR)).toEqual({ dir: DIR, entity: 'artist', line: 0, done: false }); + expect(resumeFrom({ dir: DIR, entity: 'release-group', line: 1200 }, DIR)).toEqual({ + dir: DIR, + entity: 'release-group', + line: 1200, + done: false, + }); + expect(resumeFrom({ dir: DIR, entity: 'release-group', line: 99, done: true }, DIR).done).toBe( + true, + ); + expect(resumeFrom({ dir: DIR, entity: 'release-group', line: 99, done: true }, NEWER)).toEqual({ + dir: NEWER, + entity: 'artist', + line: 0, + done: false, + }); + expect(resumeFrom({ dir: DIR, entity: 'label', line: -4 }, DIR)).toEqual({ + dir: DIR, + entity: 'artist', + line: 0, + done: false, + }); + }); + + test('pruning keeps the current dump and removes the others', async () => { + const d = await mkdtemp(join(tmp, 'prune-')); + await writeFile(join(d, `${DIR}-artist.tar.xz`), 'a'); + await writeFile(join(d, `${NEWER}-artist.tar.xz`), 'b'); + await writeFile(join(d, `${NEWER}-release-group.tar.xz`), 'c'); + await writeFile(join(d, 'notes.txt'), 'd'); + expect(await pruneOthers(d, DIR)).toBe(2); + expect(await readFile(join(d, `${DIR}-artist.tar.xz`), 'utf8')).toBe('a'); + expect(await readFile(join(d, 'notes.txt'), 'utf8')).toBe('d'); + }); +}); + +describe('the walk', () => { + test('batches carry the cursor after their rows, a run stops near its deadline, the next run resumes there and finishes both entities', async () => { + const d = await mkdtemp(join(tmp, 'walk-')); + const p = provider(); + let t = 0; + const now = () => t; + + // Run 1: the clock is far from the deadline until the first batch is out. + const it = walk(ctx(p, { batchSize: 80, deadline: NEAR_MS + 1 }), { + dataDir: d, + pauseMs: 0, + now, + })[Symbol.asyncIterator](); + const first = await it.next(); + expect(first.done).toBe(false); + expect(first.value.items).toHaveLength(80); + expect(first.value.items[0].externalId).toBe(`musicbrainz:artist:${rows.artist[0].id}`); + expect(first.value.cursor).toEqual({ dir: DIR, entity: 'artist', line: 80 }); + t = NEAR_MS + 2; + const stop = await it.next(); + expect(stop.done).toBe(true); + expect(stop.value.cursor).toEqual({ dir: DIR, entity: 'artist', line: 80 }); + expect(stop.value.nextInMinutes).toBe(10); + expect(stop.value.note).toContain('deadline'); + expect(p.calls.download).toHaveLength(1); + expect(p.calls.download[0].url).toBe(dumpUrl(DIR, 'artist')); + expect(p.calls.download[0].filePath).toBe(localFile(d, DIR, 'artist')); + + // Run 2: from line 80, through the rest of the artists and all the release groups. + const second = await run(p, { batchSize: 80, cursor: stop.value.cursor }, { dataDir: d }); + expect(second.batches.map((b) => [b.cursor.entity, b.items.length, b.cursor.line])).toEqual([ + ['artist', 80, 160], + ['artist', 40, 200], + ['release-group', 80, 80], + ['release-group', 80, 160], + ['release-group', 40, 200], + ]); + expect(second.batches[0].items[0].externalId).not.toBe(first.value.items[0].externalId); + expect(second.batches[2].items[0].externalId).toBe( + `musicbrainz:release-group:${rows['release-group'][0].id}`, + ); + expect(second.outcome.cursor).toEqual({ + dir: DIR, + entity: 'release-group', + line: 200, + done: true, + }); + expect(second.outcome.nextInMinutes).toBeUndefined(); + expect(second.outcome.note).toContain('walked'); + // The artist archive is asked for again (a whole file on disk is one Range + // request answered 416, which http.download reports complete), then the + // release-group one. + expect(p.calls.download.slice(1).map((c) => c.url)).toEqual([ + dumpUrl(DIR, 'artist'), + dumpUrl(DIR, 'release-group'), + ]); + + // Run 3: same LATEST, done cursor: nothing read, nothing fetched. + const third = await run(p, { cursor: second.outcome.cursor }, { dataDir: d }); + expect(third.batches).toEqual([]); + expect(third.outcome).toEqual({ cursor: second.outcome.cursor, note: 'unchanged' }); + expect(p.calls.download).toHaveLength(3); + + // Run 4: a new dump restarts at the first artist line and the old archives go. + const p2 = provider({ latest: NEWER }); + const fourth = await run(p2, { cursor: second.outcome.cursor }, { dataDir: d }); + expect(fourth.batches[0].cursor).toEqual({ dir: NEWER, entity: 'artist', line: 200 }); + expect(fourth.batches[0].items).toHaveLength(200); + expect(p2.calls.download[0].url).toBe(dumpUrl(NEWER, 'artist')); + await expect(readFile(localFile(d, DIR, 'artist'))).rejects.toThrow(); + }); + + test('every item of a whole run is one the table would store, and no row holds the CC BY-NC-SA fields', async () => { + const d = await mkdtemp(join(tmp, 'store-')); + const { batches } = await run(provider(), {}, { dataDir: d }); + const items = batches.flatMap((b) => b.items); + expect(items).toHaveLength(400); + expect(new Set(items.map((i) => i.externalId)).size).toBe(400); + for (const it of items) { + const stored = normaliseItem(it); + expect(stored).not.toBeNull(); + expect(stored.data.attribution).toBe(ATTRIBUTION); + expect(stored.data).not.toHaveProperty('tags'); + expect(stored.data).not.toHaveProperty('genres'); + expect(stored.data).not.toHaveProperty('rating'); + expect(stored.data).not.toHaveProperty('annotation'); + } + expect(items.filter((i) => i.kind === 'artist')).toHaveLength(200); + expect(items.filter((i) => i.kind === 'release-group')).toHaveLength(200); + }); + + test('a download still in progress yields nothing and asks to resume in ten minutes', async () => { + const d = await mkdtemp(join(tmp, 'partial-')); + const p = provider({ files: { artist: { partial: 4096 } } }); + const { batches, outcome } = await run(p, {}, { dataDir: d }); + expect(batches).toEqual([]); + expect(outcome.cursor).toEqual({ dir: DIR, entity: 'artist', line: 0 }); + expect(outcome.nextInMinutes).toBe(10); + expect(outcome.note).toContain('download in progress'); + expect((await readFile(localFile(d, DIR, 'artist'))).length).toBe(4096); + }); + + test('a run that reaches its deadline while a second entity is still to be fetched keeps that place', async () => { + const d = await mkdtemp(join(tmp, 'switch-')); + const p = provider(); + let t = 0; + const it = walk(ctx(p, { batchSize: 500, deadline: NEAR_MS + 1 }), { + dataDir: d, + pauseMs: 0, + now: () => t, + })[Symbol.asyncIterator](); + const first = await it.next(); + expect(first.value.cursor).toEqual({ dir: DIR, entity: 'artist', line: 200 }); + t = NEAR_MS + 2; + const stop = await it.next(); + expect(stop.done).toBe(true); + expect(stop.value.cursor).toEqual({ dir: DIR, entity: 'release-group', line: 0 }); + expect(stop.value.nextInMinutes).toBe(10); + expect(p.calls.download.map((c) => c.url)).toEqual([dumpUrl(DIR, 'artist')]); + }); + + test('a bad row is counted and skipped, never thrown; the line count still moves past it', async () => { + const d = await mkdtemp(join(tmp, 'bad-')); + const good = rows.artist.map((r) => JSON.stringify(r)); + const member = [good[0], 'this is not json', '', '[1,2,3]', '{"name":"no id"}', good[1]].join( + '\n', + ); + await mkdir(join(d, 'src', 'mbdump'), { recursive: true }); + await writeFile(join(d, 'src', 'mbdump', 'artist'), member); + // The archive lives outside the dump directory, which the walk prunes. + const archive = join(d, 'src', 'bad-artist.tar.xz'); + const proc = Bun.spawn(['tar', '-cJf', archive, '-C', join(d, 'src'), 'mbdump/artist']); + expect(await proc.exited).toBe(0); + await mkdir(join(d, 'data')); + const p = provider({ files: { artist: archive } }); + const { batches, outcome } = await run(p, { batchSize: 500 }, { dataDir: join(d, 'data') }); + expect(batches[0].items.map((i) => i.data.mbid)).toEqual([ + rows.artist[0].id, + rows.artist[1].id, + ]); + expect(batches[0].cursor).toEqual({ dir: DIR, entity: 'artist', line: 6 }); + expect(outcome.note).toContain('3 bad'); + expect(outcome.cursor.done).toBe(true); + }); + + test('every request carries the user agent', async () => { + const d = await mkdtemp(join(tmp, 'ua-')); + const p = provider(); + await run(p, {}, { dataDir: d }); + expect(p.calls.text[0].url).toBe(LATEST_URL); + for (const c of [...p.calls.text, ...p.calls.download]) { + expect(c.headers['user-agent']).toBe(USER_AGENT); + } + }); +}); + +describe('failures', () => { + test('LATEST failing three times is a run in which every request failed, and it throws', async () => { + const d = await mkdtemp(join(tmp, 'latest-')); + const p = provider({ + latest: () => { + throw new Error('503 from data.metabrainz.org'); + }, + }); + await expect(run(p, {}, { dataDir: d })).rejects.toThrow(/every request failed \(3\)/); + expect(p.calls.text).toHaveLength(3); + expect(p.calls.download).toHaveLength(0); + }); + + test('a LATEST that is not a directory name counts as a failure too', async () => { + const d = await mkdtemp(join(tmp, 'latest2-')); + const p = provider({ latest: 'maintenance' }); + await expect(run(p, {}, { dataDir: d })).rejects.toThrow(/every request failed/); + }); + + test('a download failing three times in a row ends the run with the place kept and no throw', async () => { + const d = await mkdtemp(join(tmp, 'dl-')); + const p = provider({ + files: { + artist: () => { + throw new Error('socket hang up'); + }, + }, + }); + const cursor = { dir: DIR, entity: 'artist', line: 12_345 }; + const { batches, outcome } = await run(p, { cursor }, { dataDir: d }); + expect(batches).toEqual([]); + expect(outcome.cursor).toEqual(cursor); + expect(outcome.nextInMinutes).toBe(10); + expect(outcome.note).toContain('failed 3 times'); + expect(p.calls.download).toHaveLength(3); + }); + + test('an archive tar cannot read is removed and fetched once more, from the last batch yielded, with no row lost or repeated', async () => { + const d = await mkdtemp(join(tmp, 'unreadable-')); + // The real artist archive cut at 400 KB: xz hands tar some rows, then fails. + const cut = join(d, 'cut.tar.xz'); + await writeFile(cut, (await readFile(fixture('artist'))).subarray(0, 400_000)); + // The archive lives outside the dump directory, which the walk prunes. + await mkdir(join(d, 'data')); + let tries = 0; + const p = provider({ + files: { + artist: async (filePath) => { + tries += 1; + await copyFile(tries === 1 ? cut : fixture('artist'), filePath); + return { path: filePath, bytes: 1, complete: true }; + }, + }, + }); + const logs = []; + const { batches, outcome } = await drain( + walk( + { ...ctx(p, { batchSize: 37 }), log: (m) => logs.push(m) }, + { dataDir: join(d, 'data'), pauseMs: 0 }, + ), + ); + expect(tries).toBe(2); + expect(logs.some((m) => /artist: archive of .* removed, tar exited/.test(m))).toBe(true); + // Two batches came out of the cut archive before tar failed; the fetch repeated + // from line 74, so the rows the cut stopped inside are read once from the whole file. + expect(batches.slice(0, 3).map((b) => [b.cursor.entity, b.cursor.line])).toEqual([ + ['artist', 37], + ['artist', 74], + ['artist', 111], + ]); + const artists = batches.flatMap((b) => b.items).filter((i) => i.kind === 'artist'); + const all = await collect(xzLines(fixture('artist'), { member: memberOf('artist') })); + expect(artists.map((i) => i.data.mbid)).toEqual(all.map((l) => JSON.parse(l).id)); + expect(outcome.cursor).toEqual({ dir: DIR, entity: 'release-group', line: 200, done: true }); + }); + + test('an archive unreadable twice throws, with the file gone so the next run fetches it fresh', async () => { + const d = await mkdtemp(join(tmp, 'unreadable2-')); + const p = provider({ + files: { + artist: async (filePath) => { + await writeFile(filePath, Buffer.alloc(50_000, 0x41)); + return { path: filePath, bytes: 50_000, complete: true }; + }, + }, + }); + await expect(run(p, {}, { dataDir: d })).rejects.toThrow( + /artist archive of .* unreadable twice/, + ); + expect(p.calls.download).toHaveLength(2); + await expect(readFile(localFile(d, DIR, 'artist'))).rejects.toThrow(); + }); + + test('a download that fails twice and then lands resets the streak and the walk goes on', async () => { + const d = await mkdtemp(join(tmp, 'dl2-')); + let tries = 0; + const p = provider({ + files: { + artist: async (filePath) => { + tries += 1; + if (tries < 3) throw new Error('reset by peer'); + await copyFile(fixture('artist'), filePath); + return { path: filePath, bytes: 1, complete: true }; + }, + }, + }); + const { batches, outcome } = await run(p, {}, { dataDir: d }); + expect(tries).toBe(3); + expect(batches[0].items).toHaveLength(200); + expect(outcome.cursor.done).toBe(true); + }); +}); + +describe('the adapter', () => { + test('declares what the core needs and says the licence', async () => { + expect(musicbrainzCatalog.name).toBe('musicbrainz-catalog'); + expect(musicbrainzCatalog.collection).toBe('music'); + expect(musicbrainzCatalog.kinds).toEqual(['artist', 'release-group']); + expect(musicbrainzCatalog.budgetMs).toBe(BUDGET_MS); + expect(BUDGET_MS).toBe(55 * 60_000); + expect(musicbrainzCatalog.cadenceMinutes).toBe(10_080); + expect(musicbrainzCatalog.defaultSources[0].slug).toBe('musicbrainz-catalog'); + expect(musicbrainzCatalog.description).toContain('CC0'); + expect(musicbrainzCatalog.description).toContain('CC BY-NC-SA'); + expect(typeof musicbrainzCatalog.pull).toBe('function'); + const src = await readFile( + new URL('../packages/adapters/src/musicbrainz-catalog.js', import.meta.url), + 'utf8', + ); + expect(src).not.toContain(String.fromCharCode(0x2014)); + }); + + test('pull is the walk: an async iterable the core can drain, and close before it starts', async () => { + const p = provider(); + const out = musicbrainzCatalog.pull(ctx(p)); + expect(typeof out[Symbol.asyncIterator]).toBe('function'); + expect(typeof out.next).toBe('function'); + // Closed before its first step: nothing was asked of the network. + expect(await out.return()).toEqual({ value: undefined, done: true }); + expect(p.calls.text).toHaveLength(0); + }); +}); diff --git a/test/podcasts-podcastindex-catalog.test.js b/test/podcasts-podcastindex-catalog.test.js new file mode 100644 index 0000000..4bae4d1 --- /dev/null +++ b/test/podcasts-podcastindex-catalog.test.js @@ -0,0 +1,731 @@ +import { Database } from 'bun:sqlite'; +import { afterAll, beforeAll, describe, expect, test } from 'bun:test'; +import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +/** + * The Podcast Index dump, walked without the dump. + * + * The real file is a 1.8 GB tgz holding a 5.1 GB SQLite database, so nothing + * here downloads it. What is real is the schema: the CREATE TABLE in + * `podcastindex-catalog-schema.sql` was cut from the first 4 MB of the archive + * on 2026-09-13, and every test database here is created from it, so the + * column names the adapter resolves are the ones the dump actually has. The + * HEAD fixture is the real server's answer the same day, and the listing is + * the real tar member. + */ +process.env.DATABASE_URL ??= 'postgres://test:test@localhost:5432/test'; +process.env.SITE_URL ??= 'https://nichedb.test'; + +const { config } = await import('../packages/config/src/index.js'); +const { normaliseItem } = await import('../packages/core/src/adapter.js'); +const { + ATTRIBUTION, + BUDGET_MS, + CADENCE_MINUTES, + categoriesOf, + discoverSchema, + dumpVersion, + epochDate, + FIELD_NAMES, + flag, + langTag, + MAX_FAILURES, + normaliseFeedUrl, + pickTable, + podcastindexCatalog, + resolveColumns, + RESUME_IN_MINUTES, + rowsToItems, + selectSql, + summaryOf, + toItem, + userAgent, + versionStamp, +} = await import('../packages/adapters/src/podcastindex-catalog.js'); + +const FIXTURES = join(import.meta.dir, '../packages/adapters/test/fixtures'); +const SCHEMA = await readFile(join(FIXTURES, 'podcastindex-catalog-schema.sql'), 'utf8'); +const HEAD = await readFile(join(FIXTURES, 'podcastindex-catalog-head.txt'), 'utf8'); +const NEWSFEEDS = await readFile(join(FIXTURES, 'podcastindex-catalog-newsfeeds.sql'), 'utf8'); +const LISTING = await readFile(join(FIXTURES, 'podcastindex-catalog-listing.txt'), 'utf8'); + +/** The real HEAD response as a header map. */ +function headersOf(text) { + const out = {}; + for (const line of text.split('\n')) { + const m = line.match(/^([a-z0-9-]+):\s*(.*)$/i); + if (m) out[m[1].toLowerCase()] = m[2].trim(); + } + return out; +} +const REAL_HEAD = headersOf(HEAD); + +/** The 42 real column names, from the fixture DDL. */ +const REAL_COLUMNS = [...SCHEMA.matchAll(/^\s{4}(\w+)\s+(?:INTEGER|TEXT)/gm)].map((m) => m[1]); + +/** The repository's MySQL DDL column names, in case a dump ever follows it. */ +const NEWSFEEDS_COLUMNS = [ + ...NEWSFEEDS.matchAll(/^\s{2}`(\w+)`\s+(?:bigint|varchar|int|tinyint|longtext|mediumtext)/gm), +].map((m) => m[1]); + +/** + * Rows in the dump's own column names. The first two carry the values of the + * first two rows of the repository's sample CSV (Libsyn and Anchor shows); the + * rest are the edge cases a real dump has by the thousand. + */ +const ROWS = [ + { + id: 1, + url: 'https://markalanwilliams.libsyn.com/rss', + title: 'Christianity Questions and Answers', + lastUpdate: 1599840661, + link: 'http://markalanwilliams.libsyn.com/webpage', + dead: 0, + itunesId: 1000000618, + itunesAuthor: 'Dr. Mark Alan Williams', + explicit: 0, + imageUrl: 'https://ssl-static.libsyn.com/p/assets/e/7/5/d/e75de19145e2153b/thumb.jpg', + generator: 'Libsyn WebEngine 2.0', + newestItemPubdate: 1592074395, + language: '', + episodeCount: 12, + host: 'libsyn.com', + description: + 'Dr. Mark Alan Williams and friends answer questions about the Christian faith: questions about the God, Jesus, the Bible, eternity, belief, religion the reasonableness of faith and others.', + category1: 'Religion', + category2: 'Spirituality', + category3: 'Christianity', + }, + { + id: 2, + url: 'https://anchor.fm/s/19ccb320/podcast/rss', + title: 'Rahdo Talks Through', + lastUpdate: 1600183477, + link: 'https://patreon.com/rahdo', + dead: 0, + itunesId: 1000016089, + itunesAuthor: 'Richard Ham', + explicit: 0, + imageUrl: + 'https://d3t3ozftmdmh3i.cloudfront.net/staging/podcast_uploaded_nologo/4228456/81316933823cb437.jpeg', + generator: 'Anchor Podcasts', + newestItemPubdate: 1599770045, + language: 'en', + episodeCount: 340, + host: 'anchor.fm', + description: 'A podcast all about boardgames, hosted by Richard "Rahdo" Ham', + category1: 'Leisure', + category2: 'Games', + category3: 'Hobbies', + category4: 'Leisure', + }, + // Dead: too many errors, no longer checked. Not a podcast anyone can fetch. + { id: 3, url: 'https://dead.example/feed.xml', title: 'Gone', dead: 1, language: 'en' }, + // No title. + { id: 4, url: 'https://untitled.example/feed.xml', title: ' ', dead: 0 }, + // A feed url that does not parse. + { id: 5, url: 'not a url', title: 'Broken Feed', dead: 0 }, + // Upper-case host, trailing slash, fragment, no site: url falls back to the feed. + { + id: 6, + url: 'HTTPS://Example.COM/Feeds/Show/#top', + title: 'Self Hosted & Proud', + link: '', + dead: 0, + language: 'DE-de', + description: '

Hello & welcome to the show’s feed.


Second line.', + newestItemPubdate: 0, + explicit: 1, + imageUrl: '', + }, + // Text where numbers should be, from a CSV import gone wrong. + { + id: 7, + url: 'http://plain.example/rss', + title: 'Odd Numbers', + dead: 'no', + newestItemPubdate: 'soon', + explicit: 'yes', + itunesId: null, + language: 'en-US', + episodeCount: -3, + }, +]; + +/** + * A row as the walk hands it to `toItem`: `selectSql` aliases the dump's + * columns onto the logical field names, so a direct call maps them the same way. + */ +const { columns: REAL_MAP } = resolveColumns(REAL_COLUMNS); +function logical(row) { + const out = {}; + for (const [field, real] of Object.entries(REAL_MAP)) { + if (real && row[real] !== undefined) out[field] = row[real]; + } + for (const k of Object.keys(row)) if (/^category\d+$/.test(k)) out[k] = row[k]; + return out; +} + +const listing = LISTING.trim().split(/\s+/); +/** The member name inside the real archive, from the real listing. */ +const MEMBER = listing[listing.length - 1]; + +let work; +let tgz; +let dataDirBefore; + +/** A SQLite database created from the real DDL, filled with `rows`, as a tgz laid out like the real one. */ +async function makeArchive(rows, name = 'dump') { + const src = join(work, `${name}-src`); + await mkdir(src, { recursive: true }); + const dbPath = join(src, MEMBER.replace(/^\.\//, '')); + const db = new Database(dbPath); + db.run(SCHEMA); + const cols = REAL_COLUMNS; + const insert = db.prepare( + `insert into podcasts (${cols.map((c) => `"${c}"`).join(', ')}) values (${cols.map(() => '?').join(', ')})`, + ); + for (const row of rows) { + insert.run( + ...cols.map((c) => { + if (row[c] !== undefined) return row[c]; + // The real table is NOT NULL on its text columns and defaults nothing. + return /TEXT/.test(SCHEMA.match(new RegExp(`\\n\\s+${c}\\s+(\\w+)`))?.[1] ?? '') + ? '' + : null; + }), + ); + } + db.close(); + const out = join(work, `${name}.tgz`); + const proc = Bun.spawn(['tar', '-czf', out, '-C', src, MEMBER], { + stdout: 'ignore', + stderr: 'pipe', + }); + if ((await proc.exited) !== 0) throw new Error(await new Response(proc.stderr).text()); + return out; +} + +beforeAll(async () => { + work = await mkdtemp(join(tmpdir(), 'nichedb-podcastindex-test-')); + dataDirBefore = config.ingest.dataDir; + // dumpDir reads this at call time; the module is shared with every other test file. + config.ingest.dataDir = join(work, 'data'); + tgz = await makeArchive(ROWS); +}); + +afterAll(async () => { + config.ingest.dataDir = dataDirBefore; + await rm(work, { recursive: true, force: true }); +}); + +/** + * A stand-in for ctx.http: HEAD answers with the fixture headers, download + * copies the archive (or a prefix of it) to where the adapter asked. + */ +function fakeHttp({ + headers = REAL_HEAD, + archive = () => tgz, + partial = false, + headFails = 0, + downloadFails = 0, +} = {}) { + const seen = { heads: [], downloads: [], agents: [] }; + let hf = headFails; + let df = downloadFails; + const http = { + async request(url, opts = {}) { + seen.heads.push(url); + seen.agents.push(opts.headers?.['user-agent']); + expect(opts.method).toBe('HEAD'); + if (hf > 0) { + hf -= 1; + throw new Error('socket hang up'); + } + return new Response(null, { status: 200, headers }); + }, + async download(_url, filePath, opts = {}) { + seen.downloads.push(filePath); + seen.agents.push(opts.headers?.['user-agent']); + if (df > 0) { + df -= 1; + throw new Error('ECONNRESET'); + } + const bytes = await readFile(archive()); + const part = partial ? bytes.subarray(0, 64) : bytes; + await writeFile(filePath, part); + return { path: filePath, bytes: part.length, complete: !partial }; + }, + }; + return { http, seen }; +} + +/** Run pull to the end, collecting the batches and the return value. */ +async function drain(gen) { + const batches = []; + for (;;) { + const { value, done } = await gen.next(); + if (done) return { batches, outcome: value }; + batches.push(value); + } +} + +const pull = (http, { cursor = {}, deadline = Number.POSITIVE_INFINITY, batchSize = 50 } = {}) => + podcastindexCatalog.pull({ + config: { batchSize, pauseMs: 0 }, + cursor, + env: { contactEmail: 'ops@nichedb.test' }, + http, + log: () => {}, + deadline, + }); + +const ids = (batches) => batches.flatMap((b) => b.items.map((i) => i.externalId)); + +describe('registration', () => { + test('podcasts collection, show kind, a weekly cadence and a 55 minute budget', () => { + expect(podcastindexCatalog.name).toBe('podcastindex-catalog'); + expect(podcastindexCatalog.collection).toBe('podcasts'); + expect(podcastindexCatalog.kinds).toEqual(['show']); + expect(podcastindexCatalog.cadenceMinutes).toBe(CADENCE_MINUTES); + expect(CADENCE_MINUTES).toBe(7 * 24 * 60); + expect(podcastindexCatalog.budgetMs).toBe(BUDGET_MS); + expect(BUDGET_MS).toBe(55 * 60_000); + expect(podcastindexCatalog.defaultSources.map((s) => s.slug)).toEqual(['podcastindex-catalog']); + }); + + test('the description states the licence and nothing here carries an em dash', async () => { + expect(podcastindexCatalog.description).toMatch(/MIT/); + const src = await readFile( + join(import.meta.dir, '../packages/adapters/src/podcastindex-catalog.js'), + 'utf8', + ); + const emDash = String.fromCharCode(0x2014); + expect(src.includes(emDash)).toBe(false); + expect(podcastindexCatalog.description.includes(emDash)).toBe(false); + }); + + test('the user agent says who is asking and how to reach them', () => { + expect(userAgent({ contactEmail: 'ops@nichedb.test' })).toBe( + 'niche-db podcastindex-catalog/1 (+https://nichedb.dev; weekly read of the public dump; ops@nichedb.test)', + ); + expect(userAgent({})).toMatch(/^niche-db podcastindex-catalog\/1 \(\+https:\/\/nichedb\.dev/); + }); +}); + +describe('the real schema', () => { + test('the fixture is the dump as published: a podcasts table with camelCase columns', () => { + expect(SCHEMA.startsWith('CREATE TABLE podcasts (')).toBe(true); + expect(REAL_COLUMNS).toHaveLength(42); + expect(REAL_COLUMNS).toContain('newestItemPubdate'); + expect(REAL_COLUMNS).toContain('episodeCount'); + expect(REAL_COLUMNS).toContain('category10'); + expect(MEMBER).toBe('./podcastindex_feeds.db'); + }); + + test('every field resolves against the real columns, and the categories are in order', () => { + const { columns, categories } = resolveColumns(REAL_COLUMNS); + expect(columns.id).toBe('id'); + expect(columns.url).toBe('url'); + expect(columns.image).toBe('imageUrl'); + expect(columns.newestItemPubdate).toBe('newestItemPubdate'); + expect(columns.episodeCount).toBe('episodeCount'); + expect(columns.guid).toBe('podcastGuid'); + expect(columns.host).toBe('host'); + expect(categories).toEqual(Array.from({ length: 10 }, (_, i) => `category${i + 1}`)); + for (const field of Object.keys(FIELD_NAMES)) expect(columns).toHaveProperty(field); + }); + + test('the repository DDL (newsfeeds, snake_case) resolves too, with nulls where it has nothing', () => { + expect(NEWSFEEDS_COLUMNS.length).toBeGreaterThan(30); + const { columns, categories } = resolveColumns(NEWSFEEDS_COLUMNS); + expect(columns.image).toBe('artwork_url_600'); + expect(columns.newestItemPubdate).toBe('newest_item_pubdate'); + expect(columns.episodeCount).toBe('item_count'); + expect(columns.itunesId).toBe('itunes_id'); + expect(columns.host).toBeNull(); + expect(categories).toEqual([]); + }); + + test('a table without id, url and title is refused by name', () => { + expect(() => resolveColumns(['id', 'feedTitle'])).toThrow(/no url, title column/); + }); + + test('the table is picked by name first, then by shape', () => { + expect( + pickTable([ + { name: 'other', columns: ['url', 'title'] }, + { name: 'podcasts', columns: [] }, + ]).name, + ).toBe('podcasts'); + expect(pickTable([{ name: 'newsfeeds', columns: [] }]).name).toBe('newsfeeds'); + expect(pickTable([{ name: 'feeds', columns: ['URL', 'Title'] }]).name).toBe('feeds'); + expect(pickTable([{ name: 'episodes', columns: ['id'] }])).toBeNull(); + expect(pickTable(null)).toBeNull(); + }); + + test('discoverSchema reads it from the file and selectSql keys the walk on the id', async () => { + const src = join(work, 'dump-src', 'podcastindex_feeds.db'); + const schema = discoverSchema(src); + expect(schema.table).toBe('podcasts'); + const sql = selectSql(schema); + expect(sql).toMatch(/^select "id" as "id", "url" as "url", "title" as "title"/); + expect(sql).toMatch(/"imageUrl" as "image"/); + expect(sql).toMatch(/"category10" as "category10"/); + expect(sql).toMatch(/from "podcasts" where "id" > \? order by "id" limit \?$/); + }); +}); + +describe('the version', () => { + test('is the real ETag, with the Last-Modified and size beside it', () => { + const v = dumpVersion(REAL_HEAD); + expect(v.version).toBe('b60fa45859813600fc0320e710b1a73f-117'); + expect(v.etag).toBe('b60fa45859813600fc0320e710b1a73f-117'); + expect(v.lastModified).toBe('2026-09-12T23:24:31.000Z'); + expect(v.bytes).toBe(1826623856); + expect(dumpVersion(new Headers(REAL_HEAD)).version).toBe(v.version); + }); + + test('falls back to Last-Modified, then to nothing', () => { + expect(dumpVersion({ 'last-modified': 'Sat, 12 Sep 2026 23:24:31 GMT' }).version).toBe( + '2026-09-12T23:24:31.000Z', + ); + expect(dumpVersion({}).version).toBeNull(); + expect(dumpVersion(null).version).toBeNull(); + }); + + test('is a file name', () => { + expect(versionStamp('b60fa45859813600fc0320e710b1a73f-117')).toBe( + 'b60fa45859813600fc0320e710b1a73f-117', + ); + expect(versionStamp('2026-09-12T23:24:31.000Z')).toMatch(/^2026-09-12t/); + expect(versionStamp('')).toBe('dump'); + }); +}); + +describe('one row as an item', () => { + test('the show, in the podcasts collection shape', () => { + const item = toItem(logical(ROWS[1])); + expect(item.externalId).toBe('podcastindex:feed:2'); + expect(item.kind).toBe('show'); + expect(item.title).toBe('Rahdo Talks Through'); + expect(item.url).toBe('https://patreon.com/rahdo'); + expect(item.imageUrl).toMatch(/^https:\/\/d3t3ozftmdmh3i\.cloudfront\.net/); + expect(item.publishedAt).toEqual(new Date(1599770045 * 1000)); + expect(item.summary).toBe('A podcast all about boardgames, hosted by Richard "Rahdo" Ham'); + expect(item.tags).toEqual([ + 'show', + 'podcast', + 'podcastindex', + 'lang:en', + 'category:leisure', + 'category:games', + 'category:hobbies', + ]); + expect(item.data.feedUrl).toBe('https://anchor.fm/s/19ccb320/podcast/rss'); + expect(item.data.feedId).toBe(2); + expect(item.data.itunesId).toBe(1000016089); + expect(item.data.language).toBe('en'); + expect(item.data.categories).toEqual(['Leisure', 'Games', 'Hobbies']); + expect(item.data.episodeCount).toBe(340); + expect(item.data.newestItemPubdate).toBe('2020-09-10T20:34:05.000Z'); + expect(item.data.lastUpdate).toBe('2020-09-15T15:24:37.000Z'); + expect(item.data.explicit).toBe(false); + expect(item.data.generator).toBe('Anchor Podcasts'); + expect(item.data.host).toBe('anchor.fm'); + expect(item.data.platform).toBe('anchor.fm'); + expect(item.data.author).toBe('Richard Ham'); + expect(item.data.attribution).toBe(ATTRIBUTION); + expect(ATTRIBUTION).toBe('Podcast Index; dump under its terms, index data MIT'); + + const stored = normaliseItem(item); + expect(stored).not.toBeNull(); + expect(stored.dedupeKey).toBeTruthy(); + expect(stored.data.feedUrl).toBe(item.data.feedUrl); + }); + + test('the feed url is the join key: lowercase host, no trailing slash, no fragment, scheme kept', () => { + const item = toItem(logical(ROWS[5])); + expect(item.data.feedUrl).toBe('https://example.com/Feeds/Show'); + expect(item.url).toBe('https://example.com/Feeds/Show'); + expect(item.data.siteUrl).toBeNull(); + expect(item.title).toBe('Self Hosted & Proud'); + expect(item.summary).toBe('Hello & welcome to the show’s feed. Second line.'); + expect(item.tags).toContain('lang:de'); + expect(item.publishedAt).toBeNull(); + expect(item.imageUrl).toBeNull(); + expect(item.data.explicit).toBe(true); + expect(item.data.platform).toBeNull(); + + expect(normaliseFeedUrl('http://Host.Example:8080/a/b/?x=1#f')).toBe( + 'http://host.example:8080/a/b?x=1', + ); + expect(normaliseFeedUrl('https://host.example/')).toBe('https://host.example'); + expect(normaliseFeedUrl('ftp://host.example/feed')).toBeNull(); + expect(normaliseFeedUrl('feed.xml')).toBeNull(); + expect(normaliseFeedUrl(null)).toBeNull(); + }); + + test('a language tag is its base; categories keep column order and drop repeats', () => { + expect(langTag('en-US')).toBe('lang:en'); + expect(langTag('DE-de')).toBe('lang:de'); + expect(langTag('')).toBeNull(); + expect(langTag('english')).toBeNull(); + expect( + categoriesOf({ category2: 'B', category1: 'A', category10: 'A', category3: '' }), + ).toEqual(['A', 'B']); + }); + + test('dead, untitled and unparseable rows are nothing', () => { + expect(toItem(logical(ROWS[2]))).toBeNull(); + expect(toItem(logical(ROWS[3]))).toBeNull(); + expect(toItem(logical(ROWS[4]))).toBeNull(); + expect(toItem({ id: 'x', url: 'https://a.example/f', title: 'A' })).toBeNull(); + expect(toItem(null)).toBeNull(); + }); + + test('text in numeric columns becomes null, never NaN or a throw', () => { + const item = toItem(logical(ROWS[6])); + expect(item).not.toBeNull(); + expect(item.publishedAt).toBeNull(); + expect(item.data.newestItemPubdate).toBeNull(); + expect(item.data.explicit).toBe(true); + expect(item.data.itunesId).toBeNull(); + expect(item.data.episodeCount).toBeNull(); + expect(item.tags).toContain('lang:en'); + expect(flag('no')).toBe(false); + expect(flag(1)).toBe(true); + expect(flag('0')).toBe(false); + expect(epochDate(0)).toBeNull(); + expect(epochDate(4_200_000_000)).toBeNull(); + expect(summaryOf(`${'x'.repeat(700)}`)).toHaveLength(600); + }); + + test('a row that throws inside the mapping is one bad row, not a failed batch', () => { + const cursed = { + id: 9, + url: 'https://a.example/f', + get title() { + throw new Error('corrupt cell'); + }, + }; + const logged = []; + const out = rowsToItems([logical(ROWS[0]), cursed, logical(ROWS[2])], { + log: (m) => logged.push(m), + }); + expect(out.items.map((i) => i.externalId)).toEqual(['podcastindex:feed:1']); + expect(out).toMatchObject({ kept: 1, bad: 1, skipped: 1 }); + expect(logged[0]).toMatch(/row 9 dropped: corrupt cell/); + expect(rowsToItems(null)).toEqual({ items: [], kept: 0, skipped: 0, bad: 0 }); + }); +}); + +describe('the walk', () => { + test('one run: HEAD, download, extract, then every batch carries the id to resume from', async () => { + const { http, seen } = fakeHttp(); + const { batches, outcome } = await drain(pull(http, { batchSize: 50 })); + + expect(seen.heads).toEqual(['https://public.podcastindex.org/podcastindex_feeds.db.tgz']); + expect(seen.downloads).toHaveLength(1); + expect(seen.downloads[0]).toBe( + join(work, 'data', 'podcastindex', 'b60fa45859813600fc0320e710b1a73f-117.db.tgz'), + ); + // A descriptive user agent on every request. + for (const ua of seen.agents) + expect(ua).toMatch(/^niche-db podcastindex-catalog\/1 \(.*ops@nichedb\.test\)$/); + + // 7 rows in one batch of 50: 4 shows, 3 skipped. + expect(batches).toHaveLength(1); + expect(ids(batches)).toEqual([ + 'podcastindex:feed:1', + 'podcastindex:feed:2', + 'podcastindex:feed:6', + 'podcastindex:feed:7', + ]); + expect(batches[0].cursor).toEqual({ + version: 'b60fa45859813600fc0320e710b1a73f-117', + lastModified: '2026-09-12T23:24:31.000Z', + afterId: 7, + }); + expect(outcome.cursor).toEqual({ ...batches[0].cursor, done: true }); + expect(outcome.note).toMatch(/^complete: 4 shows, 3 skipped, 0 bad rows$/); + expect(outcome.nextInMinutes).toBeUndefined(); + + // The archive was deleted after extraction; the database and its marker stay. + const dir = join(work, 'data', 'podcastindex', 'b60fa45859813600fc0320e710b1a73f-117'); + await expect(stat(seen.downloads[0])).rejects.toThrow(); + expect((await stat(join(dir, 'ready'))).isFile()).toBe(true); + expect((await stat(join(dir, 'podcastindex_feeds.db'))).isFile()).toBe(true); + }); + + test('a run at its deadline yields nothing; one that runs out mid-walk stops after the batch in hand, and the next resumes exactly there', async () => { + // Batches of 50 are the floor, so a two-run walk needs a database with more rows. + const many = Array.from({ length: 120 }, (_, i) => ({ + id: 100 + i, + url: `https://many.example/${i}/feed.xml`, + title: `Show ${i}`, + dead: i % 10 === 9 ? 1 : 0, + language: 'en', + })); + const archive = await makeArchive(many, 'many'); + const headers = { ...REAL_HEAD, etag: '"many-1"' }; + const p = fakeHttp({ headers, archive: () => archive }); + + // Deadline already past before anything is on disk: no download, nothing yielded, back in ten. + const late = await drain(pull(p.http, { batchSize: 50, deadline: Date.now() - 1 })); + expect(late.batches).toEqual([]); + expect(late.outcome).toEqual({ + cursor: { version: 'many-1', lastModified: '2026-09-12T23:24:31.000Z', afterId: 0 }, + note: 'out of time before the download', + nextInMinutes: RESUME_IN_MINUTES, + }); + expect(p.seen.downloads).toHaveLength(0); + + // The clock runs out after the first batch: that batch is yielded, then the run returns. + const realNow = Date.now; + let elapsed = 0; + Date.now = () => realNow() + elapsed; + let first; + try { + const gen = pull(p.http, { batchSize: 50, deadline: realNow() + 60_000 }); + const one = await gen.next(); + expect(one.done).toBe(false); + elapsed = 120_000; + const end = await gen.next(); + expect(end.done).toBe(true); + first = { batches: [one.value], outcome: end.value }; + } finally { + Date.now = realNow; + } + expect(p.seen.downloads).toHaveLength(1); + expect(first.batches[0].items).toHaveLength(45); + expect(first.batches[0].cursor).toEqual({ + version: 'many-1', + lastModified: '2026-09-12T23:24:31.000Z', + afterId: 149, + }); + expect(first.outcome.cursor).toEqual(first.batches[0].cursor); + expect(first.outcome.nextInMinutes).toBe(RESUME_IN_MINUTES); + expect(first.outcome.note).toMatch(/out of time at id 149/); + + // At the deadline with the database already extracted: nothing read, the place kept. + const idle = await drain( + pull(p.http, { batchSize: 50, cursor: first.outcome.cursor, deadline: Date.now() - 1 }), + ); + expect(idle.batches).toEqual([]); + expect(idle.outcome).toEqual({ + cursor: first.outcome.cursor, + note: 'out of time at id 149: 0 shows, 0 skipped', + nextInMinutes: RESUME_IN_MINUTES, + }); + expect(p.seen.downloads).toHaveLength(1); + + // Next run, from that cursor: no second download (the extract is cached), rows after 149 only. + const second = await drain(pull(p.http, { batchSize: 50, cursor: first.outcome.cursor })); + expect(p.seen.downloads).toHaveLength(1); + expect(second.batches.map((b) => b.cursor.afterId)).toEqual([199, 219]); + expect(ids(second.batches)[0]).toBe('podcastindex:feed:150'); + expect(ids(second.batches)).toHaveLength(63); + expect(ids(second.batches)).not.toContain('podcastindex:feed:149'); + expect(second.outcome.cursor).toEqual({ + version: 'many-1', + lastModified: '2026-09-12T23:24:31.000Z', + afterId: 219, + done: true, + }); + expect(second.outcome.nextInMinutes).toBeUndefined(); + + // Same file, walk done: one HEAD and nothing else. + const third = await drain(pull(p.http, { cursor: second.outcome.cursor })); + expect(third.batches).toEqual([]); + expect(third.outcome).toEqual({ cursor: second.outcome.cursor, note: 'unchanged' }); + expect(p.seen.downloads).toHaveLength(1); + + // A new file on the server: the walk starts over and the old extract is pruned. + const fresh = fakeHttp({ headers: { ...REAL_HEAD, etag: '"many-2"' }, archive: () => archive }); + const fourth = await drain(pull(fresh.http, { cursor: second.outcome.cursor })); + expect(fresh.seen.downloads).toHaveLength(1); + expect(fourth.batches[0].cursor).toMatchObject({ version: 'many-2', afterId: 149 }); + expect(fourth.outcome.cursor.done).toBe(true); + await expect(stat(join(work, 'data', 'podcastindex', 'many-1'))).rejects.toThrow(); + expect((await stat(join(work, 'data', 'podcastindex', 'many-2'))).isDirectory()).toBe(true); + + // The core closes the generator at its own backstop after writing a batch: the + // cursor of that batch is exactly where the next run starts. + const gen = pull(fresh.http, { + batchSize: 50, + cursor: { version: 'many-2', lastModified: '2026-09-12T23:24:31.000Z', afterId: 0 }, + }); + const one = await gen.next(); + await gen.return(); + expect(one.value.cursor.afterId).toBe(149); + const resumed = await drain(pull(fresh.http, { batchSize: 50, cursor: one.value.cursor })); + expect(ids(resumed.batches)[0]).toBe('podcastindex:feed:150'); + expect(ids(resumed.batches)).toHaveLength(63); + expect(fresh.seen.downloads).toHaveLength(1); + }); + + test('a download still in progress yields nothing and comes back in ten minutes', async () => { + const headers = { ...REAL_HEAD, etag: '"partial-1"' }; + const partial = fakeHttp({ headers, partial: true }); + const cursor = { version: 'partial-1', lastModified: '2026-09-12T23:24:31.000Z', afterId: 0 }; + const first = await drain(pull(partial.http, { cursor })); + expect(first.batches).toEqual([]); + expect(first.outcome).toEqual({ + cursor, + note: 'download in progress', + nextInMinutes: RESUME_IN_MINUTES, + }); + const file = join(work, 'data', 'podcastindex', 'partial-1.db.tgz'); + expect((await stat(file)).size).toBe(64); + + // The next run finishes the download (the real http.download resumes with Range) and walks. + const whole = fakeHttp({ headers }); + const second = await drain(pull(whole.http, { cursor: first.outcome.cursor })); + expect(whole.seen.downloads).toEqual([file]); + expect(ids(second.batches)).toHaveLength(4); + expect(second.outcome.cursor.done).toBe(true); + }); + + test('three consecutive download failures stop the run and keep the place', async () => { + const headers = { ...REAL_HEAD, etag: '"flaky-1"' }; + const cursor = { version: 'flaky-1', lastModified: '2026-09-12T23:24:31.000Z', afterId: 4 }; + const flaky = fakeHttp({ headers, downloadFails: MAX_FAILURES }); + const out = await drain(pull(flaky.http, { cursor })); + expect(out.batches).toEqual([]); + expect(flaky.seen.downloads).toHaveLength(MAX_FAILURES); + expect(out.outcome.cursor).toEqual(cursor); + expect(out.outcome.nextInMinutes).toBe(RESUME_IN_MINUTES); + expect(out.outcome.note).toMatch(/repeated failures/); + + // Two failures and a third success is a normal run. + const recovering = fakeHttp({ headers, downloadFails: MAX_FAILURES - 1 }); + const ok = await drain(pull(recovering.http, { cursor })); + expect(recovering.seen.downloads).toHaveLength(MAX_FAILURES); + expect(ids(ok.batches)).toEqual(['podcastindex:feed:6', 'podcastindex:feed:7']); + expect(ok.outcome.cursor.done).toBe(true); + }); + + test('a run in which every request failed throws', async () => { + const dead = fakeHttp({ headFails: MAX_FAILURES }); + await expect(drain(pull(dead.http))).rejects.toThrow( + /every request failed; last: socket hang up/, + ); + expect(dead.seen.heads).toHaveLength(MAX_FAILURES); + expect(dead.seen.downloads).toHaveLength(0); + }); + + test('a 403 (no user agent) is a failed request too', async () => { + const http = { + async request() { + return new Response('forbidden', { status: 403 }); + }, + async download() { + throw new Error('never reached'); + }, + }; + await expect(drain(pull(http))).rejects.toThrow(/403 from HEAD/); + }); +});