diff --git a/apps/web/README.md b/apps/web/README.md index cf641b7df..05a29651a 100644 --- a/apps/web/README.md +++ b/apps/web/README.md @@ -149,7 +149,7 @@ The app is a space syncer. For each watched space it asks the space host (the au ### Directory -Published projects, the DJ stats, sets, press kits, shows, calendar events and follows the site reads from other repos, and the other apps' records it reads are served by [HappyView](https://happyview.dev) 2.16, not the app. Each `lua/.lua` script is collected: a record script has HappyView follow Jetstream for that collection and decides what it keeps, and a query script answers that `computer.localnetwork.directory.*` query ([`lexicons/computer/localnetwork/directory/`](../../lexicons/computer/localnetwork/directory/)). Indexing another collection is one record script there, with its lexicon in `src/lib/server/atproto/lexicons.ts` (another authority's goes in `lexicons/` with `tools/lexicons.sh pull `, and in `REFERENCED_DOCS`). The scripts today: +Published projects, the DJ stats, sets, press kits, shows, calendar events and follows the site reads from other repos, and the other apps' records it reads are served by [HappyView](https://happyview.dev) 2.16, not the app. Each `lua/.lua` script is collected: a record script has HappyView follow Jetstream for that collection and decides what it keeps, and a query script answers that `computer.localnetwork.directory.*` query ([`lexicons/computer/localnetwork/directory/`](../../lexicons/computer/localnetwork/directory/)). Indexing another collection is one record script there, with its lexicon in the repo's [`lexicons/`](../../lexicons/) (another authority's copied in with `tools/lexicons.sh pull `); the app picks up every file there. The scripts today: - `project.lua` runs on every incoming project record and keeps only the root account's, since Jetstream filters by collection and anyone can write one. - `dj/set.lua` keeps every account's sets (anyone can come to manage a project), and stores what the site shows (title, date, kind, venue, address, who it was played with, links, and the tracklist as written, with tempo, keys and transitions) plus each named track's title, artists, IDs and start, which the support queries count. Its address, sources and `createdAt` are kept as written, since locallibrary reads its own sets back before replacing them; set views leave out the sources and `createdAt`. The queries only read sets by accounts that manage one of the root's projects. MusicBrainz IDs are stored as bare lowercase UUIDs whether a set writes `mbid:` or the UUID; placeholder credits (ID, Unknown Artist, Various Artists) aren't, and a set whose date isn't YYYY-MM-DD isn't kept. diff --git a/apps/web/knip.json b/apps/web/knip.json index d42306855..bb658d4aa 100644 --- a/apps/web/knip.json +++ b/apps/web/knip.json @@ -1,6 +1,12 @@ { "$schema": "https://unpkg.com/knip@6/schema.json", - "entry": ["src/routes/**/+*.{ts,svelte}", "scripts/*.ts", "tests/**/*.ts", "src/**/*.test.ts"], + "entry": [ + "src/routes/**/+*.{ts,svelte}", + "scripts/*.ts", + "tests/**/*.ts", + "src/**/*.test.ts", + "src/lib/server/atproto/lexicon-files.ts" + ], "project": ["src/**/*.{ts,svelte}", "scripts/**/*.ts", "tests/**/*.ts"], "ignoreDependencies": [ "@tailwindcss/forms", diff --git a/apps/web/package.json b/apps/web/package.json index 3927fdd6e..e11ba9438 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -6,7 +6,8 @@ "license": "MIT", "type": "module", "imports": { - "#lib/*": "./src/lib/*" + "#lib/*": "./src/lib/*", + "#lexicon-files": "./scripts/lib/lexicon-files.ts" }, "scripts": { "dev": "tsx scripts/dev-network.ts --app", diff --git a/apps/web/scripts/lib/lexicon-files.ts b/apps/web/scripts/lib/lexicon-files.ts new file mode 100644 index 000000000..f7b153bcb --- /dev/null +++ b/apps/web/scripts/lib/lexicon-files.ts @@ -0,0 +1,12 @@ +import { readdirSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; + +const root = join(import.meta.dirname, '../../../../lexicons'); + +/** + * Every JSON document in the repo's lexicons/, read from disk. This is what `#lexicon-files` resolves to outside Vite + * (the dev network and the other tsx scripts); under Vite it's src/lib/server/atproto/lexicon-files.ts. + */ +export const LEXICON_FILES: unknown[] = readdirSync(root, { recursive: true, encoding: 'utf8' }) + .filter((file) => file.endsWith('.json')) + .map((file) => JSON.parse(readFileSync(join(root, file), 'utf8'))); diff --git a/apps/web/scripts/publish-lexicons.ts b/apps/web/scripts/publish-lexicons.ts index 203fd9d00..ac69a7422 100644 --- a/apps/web/scripts/publish-lexicons.ts +++ b/apps/web/scripts/publish-lexicons.ts @@ -18,12 +18,11 @@ * points at the temporary dev network, and `pnpm dev` publishes the * lexicons there itself. */ -import { existsSync, readdirSync, readFileSync } from 'node:fs'; +import { existsSync } from 'node:fs'; import { resolveTxt } from 'node:dns/promises'; -import { join } from 'node:path'; import { AtpAgent } from '@atproto/api'; import Cloudflare from 'cloudflare'; -import { parseLexiconDoc } from '@atproto/lexicon'; +import { LEXICON_DOCS } from '../src/lib/server/atproto/lexicons.ts'; import { canonicalJson } from '../src/lib/server/audit/chain.ts'; import { IdentityResolver } from '../src/lib/server/atproto/identity.ts'; import { AgentRootClient } from '../src/lib/server/atproto/repo-client.ts'; @@ -32,13 +31,6 @@ import { syncLexiconRecords } from './lib/lexicon-dns.ts'; if (existsSync('.env')) process.loadEnvFile('.env'); -// Only lexicons/computer/ is ours: the other folders hold other authorities' lexicons, copied in as they publish them. -const ours = join(import.meta.dirname, '../../../lexicons/computer'); - -const docs = readdirSync(ours, { recursive: true, encoding: 'utf8' }) - .filter((file) => file.endsWith('.json')) - .map((file) => parseLexiconDoc(JSON.parse(readFileSync(join(ours, file), 'utf8')))); - const identifier = process.env.ROOT_IDENTIFIER; const password = process.env.ROOT_PASSWORD; @@ -65,7 +57,7 @@ await agent.login({ identifier: did, password }); const root = new AgentRootClient(did, agent); -for (const doc of docs) { +for (const doc of LEXICON_DOCS) { const record = { $type: 'com.atproto.lexicon.schema', ...JSON.parse(JSON.stringify(doc)) }; const existing = await root.getRecord({ collection: 'com.atproto.lexicon.schema', rkey: doc.id }); @@ -79,7 +71,7 @@ for (const doc of docs) { console.log(`published ${doc.id}`); } -const authorities = new Set(docs.map((doc) => doc.id.split('.').slice(0, -1).toReversed().join('.'))); +const authorities = new Set(LEXICON_DOCS.map((doc) => doc.id.split('.').slice(0, -1).toReversed().join('.'))); const names = [...authorities].map((authority) => `_lexicon.${authority}`).toSorted((a, b) => a.localeCompare(b)); diff --git a/apps/web/src/lib/server/atproto/lexicon-files.ts b/apps/web/src/lib/server/atproto/lexicon-files.ts new file mode 100644 index 000000000..0f2de644d --- /dev/null +++ b/apps/web/src/lib/server/atproto/lexicon-files.ts @@ -0,0 +1,8 @@ +/** + * Every JSON document in the repo's lexicons/, bundled in when Vite builds or serves the app (the production image + * doesn't ship that folder). Vite resolves `#lexicon-files` here (vite.config.ts). Scripts run with tsx, which has no + * `import.meta.glob`, so Node resolves it to scripts/lib/lexicon-files.ts, which reads the same files from disk. + */ +export const LEXICON_FILES: unknown[] = Object.values( + import.meta.glob('../../../../../../lexicons/**/*.json', { import: 'default', eager: true }) +); diff --git a/apps/web/src/lib/server/atproto/lexicons.test.ts b/apps/web/src/lib/server/atproto/lexicons.test.ts index 871fca343..88705adf3 100644 --- a/apps/web/src/lib/server/atproto/lexicons.test.ts +++ b/apps/web/src/lib/server/atproto/lexicons.test.ts @@ -1,9 +1,28 @@ import type { JsonObject } from '@atproto/lex-json'; +import type { LexiconDoc } from '@atproto/lexicon'; import { describe, expect, it } from 'vitest'; +import { z } from 'zod'; import { COLLECTION } from '#lib/nsid.ts'; +import { LEXICON_FILES } from '../../../../scripts/lib/lexicon-files.ts'; import { DEMO_DJ_SETS } from '../../../../scripts/seed-data.ts'; import { LEXICON_DOCS, RecordValidationError, REFERENCED_DOCS, validateRecord } from './lexicons.ts'; +describe('the bundled lexicons', () => { + // The scripts' loader reads lexicons/ from disk, so this also checks the two loaders agree. + const onDisk = LEXICON_FILES.map((file) => z.object({ id: z.string() }).parse(file).id).toSorted(); + + it("are every file in lexicons/, ours apart from other authorities'", () => { + const ids = (docs: LexiconDoc[]) => docs.map((doc) => doc.id); + + expect([...ids(LEXICON_DOCS), ...ids(REFERENCED_DOCS)].toSorted()).toEqual(onDisk); + expect(ids(LEXICON_DOCS).toSorted()).toEqual(onDisk.filter((id) => id.startsWith('computer.localnetwork.'))); + expect(ids(LEXICON_DOCS)).toContain('computer.localnetwork.locallibraryPublishing'); + expect(ids(REFERENCED_DOCS)).toEqual( + expect.arrayContaining(['app.userinput.discussion', 'com.atproto.repo.strongRef']) + ); + }); +}); + describe('LEXICON_DOCS', () => { it('keeps the refs as written, so published schemas match lexicons/', () => { const refs = JSON.stringify(LEXICON_DOCS).match(/"#[^"]*"|"lex:[^"]*"/g) ?? []; diff --git a/apps/web/src/lib/server/atproto/lexicons.ts b/apps/web/src/lib/server/atproto/lexicons.ts index ccd871635..6f1eb2e97 100644 --- a/apps/web/src/lib/server/atproto/lexicons.ts +++ b/apps/web/src/lib/server/atproto/lexicons.ts @@ -1,153 +1,34 @@ import type { Agent } from '@atproto/api'; import { isObject } from '@atproto/lex-data'; import type { JsonObject, JsonValue } from '@atproto/lex-json'; -import { jsonToLex, lexPermissionSet, Lexicons, parseLexiconDoc } from '@atproto/lexicon'; -import auditEntry from '../../../../../../lexicons/computer/localnetwork/audit/entry.json' with { type: 'json' }; -import bookingInquiry from '../../../../../../lexicons/computer/localnetwork/booking/inquiry.json' with { type: 'json' }; -import bookingMessage from '../../../../../../lexicons/computer/localnetwork/booking/message.json' with { type: 'json' }; -import bookingStatus from '../../../../../../lexicons/computer/localnetwork/booking/status.json' with { type: 'json' }; -import calendarEvent from '../../../../../../lexicons/community/lexicon/calendar/event.json' with { type: 'json' }; -import address from '../../../../../../lexicons/community/lexicon/location/address.json' with { type: 'json' }; -import fsq from '../../../../../../lexicons/community/lexicon/location/fsq.json' with { type: 'json' }; -import geo from '../../../../../../lexicons/community/lexicon/location/geo.json' with { type: 'json' }; -import hthree from '../../../../../../lexicons/community/lexicon/location/hthree.json' with { type: 'json' }; -import strongRef from '../../../../../../lexicons/com/atproto/repo/strongRef.json' with { type: 'json' }; -import discussion from '../../../../../../lexicons/app/userinput/discussion.json' with { type: 'json' }; -import userinputSpace from '../../../../../../lexicons/app/userinput/space.json' with { type: 'json' }; -import bskyProfile from '../../../../../../lexicons/app/bsky/actor/profile.json' with { type: 'json' }; -import bskyFollow from '../../../../../../lexicons/app/bsky/graph/follow.json' with { type: 'json' }; -import verification from '../../../../../../lexicons/app/bsky/graph/verification.json' with { type: 'json' }; -import labelDefs from '../../../../../../lexicons/com/atproto/label/defs.json' with { type: 'json' }; -import defs from '../../../../../../lexicons/computer/localnetwork/defs.json' with { type: 'json' }; -import directoryDefs from '../../../../../../lexicons/computer/localnetwork/directory/defs.json' with { type: 'json' }; -import epk from '../../../../../../lexicons/computer/localnetwork/epk.json' with { type: 'json' }; -import epkContent from '../../../../../../lexicons/computer/localnetwork/epk/content.json' with { type: 'json' }; -import epkDefs from '../../../../../../lexicons/computer/localnetwork/epk/defs.json' with { type: 'json' }; -import epkDraft from '../../../../../../lexicons/computer/localnetwork/epk/draft.json' with { type: 'json' }; -import epkGrant from '../../../../../../lexicons/computer/localnetwork/epk/grant.json' with { type: 'json' }; -import epkRequest from '../../../../../../lexicons/computer/localnetwork/epk/request.json' with { type: 'json' }; -import follow from '../../../../../../lexicons/computer/localnetwork/follow.json' with { type: 'json' }; -import locallibrarySettings from '../../../../../../lexicons/computer/localnetwork/locallibrary/settings.json' with { type: 'json' }; -import getArtist from '../../../../../../lexicons/computer/localnetwork/directory/getArtist.json' with { type: 'json' }; -import getCharts from '../../../../../../lexicons/computer/localnetwork/directory/getCharts.json' with { type: 'json' }; -import getFollowStatus from '../../../../../../lexicons/computer/localnetwork/directory/getFollowStatus.json' with { type: 'json' }; -import getProject from '../../../../../../lexicons/computer/localnetwork/directory/getProject.json' with { type: 'json' }; -import getProjects from '../../../../../../lexicons/computer/localnetwork/directory/getProjects.json' with { type: 'json' }; -import getRecords from '../../../../../../lexicons/computer/localnetwork/directory/getRecords.json' with { type: 'json' }; -import getSets from '../../../../../../lexicons/computer/localnetwork/directory/getSets.json' with { type: 'json' }; -import getShows from '../../../../../../lexicons/computer/localnetwork/directory/getShows.json' with { type: 'json' }; -import getStats from '../../../../../../lexicons/computer/localnetwork/directory/getStats.json' with { type: 'json' }; -import listSets from '../../../../../../lexicons/computer/localnetwork/directory/listSets.json' with { type: 'json' }; -import listRecords from '../../../../../../lexicons/computer/localnetwork/directory/listRecords.json' with { type: 'json' }; -import listShows from '../../../../../../lexicons/computer/localnetwork/directory/listShows.json' with { type: 'json' }; -import requestIndex from '../../../../../../lexicons/computer/localnetwork/directory/requestIndex.json' with { type: 'json' }; -import djDefs from '../../../../../../lexicons/computer/localnetwork/dj/defs.json' with { type: 'json' }; -import djRecording from '../../../../../../lexicons/computer/localnetwork/dj/recording.json' with { type: 'json' }; -import djProfile from '../../../../../../lexicons/computer/localnetwork/dj/profile.json' with { type: 'json' }; -import djSet from '../../../../../../lexicons/computer/localnetwork/dj/set.json' with { type: 'json' }; -import matchProjects from '../../../../../../lexicons/computer/localnetwork/directory/matchProjects.json' with { type: 'json' }; -import searchArtists from '../../../../../../lexicons/computer/localnetwork/directory/searchArtists.json' with { type: 'json' }; -import searchProjects from '../../../../../../lexicons/computer/localnetwork/directory/searchProjects.json' with { type: 'json' }; -import decision from '../../../../../../lexicons/computer/localnetwork/moderation/decision.json' with { type: 'json' }; -import request from '../../../../../../lexicons/computer/localnetwork/moderation/request.json' with { type: 'json' }; -import show from '../../../../../../lexicons/computer/localnetwork/show.json' with { type: 'json' }; -import permissions from '../../../../../../lexicons/computer/localnetwork/permissions.json' with { type: 'json' }; -import plannerSave from '../../../../../../lexicons/computer/localnetwork/planner/save.json' with { type: 'json' }; -import plannerShow from '../../../../../../lexicons/computer/localnetwork/planner/show.json' with { type: 'json' }; -import project from '../../../../../../lexicons/computer/localnetwork/project.json' with { type: 'json' }; -import projectDetails from '../../../../../../lexicons/computer/localnetwork/project/details.json' with { type: 'json' }; -import spaceAudit from '../../../../../../lexicons/computer/localnetwork/space/audit.json' with { type: 'json' }; -import spaceBackstage from '../../../../../../lexicons/computer/localnetwork/space/backstage.json' with { type: 'json' }; -import spaceBooking from '../../../../../../lexicons/computer/localnetwork/space/booking.json' with { type: 'json' }; -import spaceEpk from '../../../../../../lexicons/computer/localnetwork/space/epk.json' with { type: 'json' }; -import spaceIndustry from '../../../../../../lexicons/computer/localnetwork/space/industry.json' with { type: 'json' }; -import spaceModeration from '../../../../../../lexicons/computer/localnetwork/space/moderation.json' with { type: 'json' }; -import spacePlanner from '../../../../../../lexicons/computer/localnetwork/space/planner.json' with { type: 'json' }; -import spaceStaff from '../../../../../../lexicons/computer/localnetwork/space/staff.json' with { type: 'json' }; -import staffRole from '../../../../../../lexicons/computer/localnetwork/staff/role.json' with { type: 'json' }; +import { jsonToLex, type LexiconDoc, lexPermissionSet, Lexicons, parseLexiconDoc } from '@atproto/lexicon'; +import { LEXICON_FILES } from '#lexicon-files'; +import { NS, PERMISSIONS } from '#lib/nsid.ts'; -/** Every Local Network lexicon this app uses, for validation and publishing. */ -export const LEXICON_DOCS = [ - defs, - project, - projectDetails, - auditEntry, - request, - decision, - staffRole, - bookingInquiry, - bookingMessage, - bookingStatus, - plannerSave, - plannerShow, - epkDefs, - epk, - epkContent, - epkRequest, - epkGrant, - epkDraft, - spaceAudit, - spaceModeration, - spaceStaff, - spaceIndustry, - spaceBooking, - spacePlanner, - spaceEpk, - spaceBackstage, - permissions, - directoryDefs, - searchProjects, - matchProjects, - getProject, - getProjects, - getStats, - getSets, - listSets, - listShows, - getShows, - getFollowStatus, - getCharts, - getArtist, - searchArtists, - getRecords, - listRecords, - requestIndex, - djDefs, - djProfile, - djSet, - djRecording, - show, - follow, - locallibrarySettings -].map((doc) => parseLexiconDoc(doc)); +/** Every document in lexicons/, by NSID, so the order doesn't depend on which loader read them. */ +const DOCS: LexiconDoc[] = LEXICON_FILES.map((file) => parseLexiconDoc(file)).toSorted((a, b) => + a.id.localeCompare(b.id) +); + +/** Ours are every Local Network tool's, all of lexicons/computer/; the other folders are other authorities'. */ +const isOurs = (doc: LexiconDoc) => doc.id.startsWith(`${NS}.`); + +/** Every Local Network lexicon, for validation and publishing. */ +export const LEXICON_DOCS = DOCS.filter(isOurs); /** The permission set as PDSes read it to expand `include:computer.localnetwork.permissions` at sign-in. */ -export const PERMISSION_SET = lexPermissionSet.parse(permissions.defs.main); +export const PERMISSION_SET = lexPermissionSet.parse(LEXICON_DOCS.find((doc) => doc.id === PERMISSIONS)?.defs.main); /** - * Other authorities' lexicons that ours reference, this app writes, or HappyView keeps for it (calendar events, the - * locations they can hold, strongRef, Bluesky's verifications, profiles and follows and the label defs profiles refer - * to, and userinput.app's space), copied into lexicons/ as they publish them. + * Other authorities' lexicons that ours reference, this app writes, or HappyView keeps for it (calendar events and + * the locations they can hold, strongRef, Bluesky's verifications, profiles and follows and the label defs profiles + * refer to, and userinput.app's spaces and the bug reports posted to them), copied into lexicons/ as they publish + * them. They're validated against and offered to HappyView, never published. */ -export const REFERENCED_DOCS = [ - address, - geo, - fsq, - hthree, - calendarEvent, - strongRef, - labelDefs, - verification, - bskyProfile, - bskyFollow, - userinputSpace -].map((doc) => parseLexiconDoc(doc)); - -/** Other apps' records this app writes but nothing indexes: bug reports on userinput.app. Only validated, never published. */ -const WRITTEN_DOCS = [parseLexiconDoc(discussion)]; +export const REFERENCED_DOCS = DOCS.filter((doc) => !isOurs(doc)); // Lexicons rewrites refs in place (`#claim` becomes `lex:…#claim`), so it gets a copy and the exported docs stay publishable. -export const lexicons = new Lexicons(structuredClone([...LEXICON_DOCS, ...REFERENCED_DOCS, ...WRITTEN_DOCS])); +export const lexicons = new Lexicons(structuredClone(DOCS)); /** Publishes every lexicon as a com.atproto.lexicon.schema record in the agent's own repo. */ export async function publishLexicons(agent: Agent): Promise { diff --git a/apps/web/vite.config.ts b/apps/web/vite.config.ts index 4b0283c04..dfdb1ee9d 100644 --- a/apps/web/vite.config.ts +++ b/apps/web/vite.config.ts @@ -61,6 +61,9 @@ export default defineConfig({ } } }, + // Under Vite, `#lexicon-files` bundles the lexicons in. Node (the tsx scripts) resolves it through package.json to + // a loader that reads them from disk, since tsx has no `import.meta.glob`. + resolve: { alias: { '#lexicon-files': `${import.meta.dirname}/src/lib/server/atproto/lexicon-files.ts` } }, server: { host: '127.0.0.1', port: Number(process.env.PORT ?? 5173), strictPort: true }, preview: { host: '127.0.0.1', port: 4173 }, test: {