diff --git a/bundles/campaigns/manifest.json b/bundles/campaigns/manifest.json index a9e379f0e..9aa81ace0 100644 --- a/bundles/campaigns/manifest.json +++ b/bundles/campaigns/manifest.json @@ -1,7 +1,7 @@ { "id": "campaigns", "name": "Campaigns", - "version": "1.0.2", + "version": "1.0.3", "description": "Social media campaign manager — draft, schedule, and publish posts across platforms (starting with Reddit)", "type": "mcp-server", "author": "Crow", @@ -22,9 +22,12 @@ "env_vars": [ { "name": "CROW_CAMPAIGNS_ENCRYPTION_KEY", - "description": "AES-256-GCM encryption key for Reddit credentials (generate with: openssl rand -base64 32)", + "description": "Key that encrypts your saved Reddit credentials (AES-256-GCM). Crow generates it at install and saves it in Settings → Passwords (this machine only); a reinstall takes it back from the keychain. Without it, re-enter the Reddit credentials.", "required": true, - "secret": true + "secret": true, + "generate": "secret", + "keychain": true, + "keychain_label": "data encryption key" } ], "notes": "12 MCP tools for social media campaign management. Reddit support via snoowrap. Credentials encrypted at rest with AES-256-GCM. Dashboard panel with approval workflow." diff --git a/bundles/campaigns/panel/campaigns.js b/bundles/campaigns/panel/campaigns.js index 7484aef2d..acbca67a3 100644 --- a/bundles/campaigns/panel/campaigns.js +++ b/bundles/campaigns/panel/campaigns.js @@ -67,7 +67,7 @@ export default { } if (action === "save_creds") { - const ENCRYPTION_KEY = process.env.CROW_CAMPAIGNS_ENCRYPTION_KEY; + const ENCRYPTION_KEY = (await import(pathToFileURL(join(serverDir, "key.js")).href)).campaignsKey(); if (!ENCRYPTION_KEY) { return res.redirectAfterPost("/dashboard/campaigns?view=setup&error=CROW_CAMPAIGNS_ENCRYPTION_KEY+not+set"); } diff --git a/bundles/campaigns/panel/routes.js b/bundles/campaigns/panel/routes.js index 7d6bddac1..39b0a4bda 100644 --- a/bundles/campaigns/panel/routes.js +++ b/bundles/campaigns/panel/routes.js @@ -24,7 +24,7 @@ const { createDbClient } = await import(pathToFileURL(join(serverDir, "db.js")). const { encrypt, decrypt } = await import(pathToFileURL(join(serverDir, "crypto.js")).href); const { createRedditClient, testAuth } = await import(pathToFileURL(join(serverDir, "reddit-client.js")).href); -const ENCRYPTION_KEY = process.env.CROW_CAMPAIGNS_ENCRYPTION_KEY; +const { campaignsKey } = await import(pathToFileURL(join(serverDir, "key.js")).href); /** * @param {Function} authMiddleware - Dashboard auth middleware @@ -228,6 +228,7 @@ export default function campaignsRouter(authMiddleware) { // --- Credentials: save (encrypt + validate) --- router.post("/api/credentials", async (req, res) => { try { + const ENCRYPTION_KEY = campaignsKey(); if (!ENCRYPTION_KEY) return res.status(500).json({ error: "CROW_CAMPAIGNS_ENCRYPTION_KEY not set" }); const { platform, username, client_id, client_secret, password } = req.body; diff --git a/bundles/campaigns/server/key.js b/bundles/campaigns/server/key.js new file mode 100644 index 000000000..678443e0c --- /dev/null +++ b/bundles/campaigns/server/key.js @@ -0,0 +1,20 @@ +/** + * The campaigns credentials key (CROW_CAMPAIGNS_ENCRYPTION_KEY), read when needed. + * + * The MCP server gets it in its own env: the gateway env overlaid by this instance's + * campaigns entry in mcp-addons.json (proxy.js). The dashboard panel runs in the gateway, + * so it reads them in that SAME order — the entry first — and both always use one key. A + * generated key reaches only the entry (mode 600). Never logged. + */ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { homedir } from "node:os"; + +export function campaignsKey(env = process.env) { + let fromEntry = ""; + try { + const file = join(env.CROW_HOME || join(homedir(), ".crow"), "mcp-addons.json"); + fromEntry = JSON.parse(readFileSync(file, "utf8"))?.campaigns?.env?.CROW_CAMPAIGNS_ENCRYPTION_KEY || ""; + } catch { /* no entry */ } + return fromEntry || env.CROW_CAMPAIGNS_ENCRYPTION_KEY || ""; +} diff --git a/bundles/mastodon/manifest.json b/bundles/mastodon/manifest.json index 4ac00dc38..1e51cefad 100644 --- a/bundles/mastodon/manifest.json +++ b/bundles/mastodon/manifest.json @@ -1,7 +1,7 @@ { "id": "mastodon", "name": "Mastodon", - "version": "1.0.3", + "version": "1.0.4", "description": "The flagship ActivityPub server — federated microblogging at scale. Heaviest of the small-AP bundles. Hosts a public (or invite-only) Mastodon instance with the full v1/v2 API surface, web UI, and Sidekiq background job processing. Media is kept on local disk.", "type": "bundle", "author": "Crow", @@ -33,14 +33,14 @@ "env_vars": [ { "name": "MASTODON_LOCAL_DOMAIN", "description": "Public domain that appears in handles (@user@). IMMUTABLE after first boot — changing breaks all federation identities.", "required": true }, { "name": "MASTODON_WEB_DOMAIN", "description": "Domain where the web UI lives (if using apex-delegation this differs from LOCAL_DOMAIN; otherwise same).", "required": false }, - { "name": "MASTODON_DB_PASSWORD", "description": "Password for the bundled Postgres role. Generated by Crow at install; never shown.", "required": true, "secret": true, "generate": "secret" }, - { "name": "MASTODON_SECRET_KEY_BASE", "description": "Rails secret_key_base. Generated by Crow at install; never shown.", "required": true, "secret": true, "generate": "secret" }, - { "name": "MASTODON_OTP_SECRET", "description": "2FA OTP secret. Generated by Crow at install; never shown.", "required": true, "secret": true, "generate": "secret" }, + { "name": "MASTODON_DB_PASSWORD", "description": "Password for the bundled Postgres role. Generated by Crow at install; saved in Settings → Passwords (this machine only), so a database restored from a backup still opens after a reinstall.", "required": true, "secret": true, "generate": "secret", "keychain": true, "keychain_label": "database password" }, + { "name": "MASTODON_SECRET_KEY_BASE", "description": "Rails secret_key_base. Generated by Crow at install; saved in Settings → Passwords (this machine only): export the keychain to keep a copy, or data encrypted with it cannot be recovered on another machine.", "required": true, "secret": true, "generate": "secret", "keychain": true, "keychain_label": "session signing key (secret_key_base)" }, + { "name": "MASTODON_OTP_SECRET", "description": "2FA OTP secret. Generated by Crow at install; saved in Settings → Passwords (this machine only): export the keychain to keep a copy, or data encrypted with it cannot be recovered on another machine.", "required": true, "secret": true, "generate": "secret", "keychain": true, "keychain_label": "two-factor secret" }, { "name": "MASTODON_VAPID_PRIVATE_KEY", "description": "Web Push VAPID private key (P-256). Generated by Crow at install; never shown.", "required": true, "secret": true, "generate": "vapid_private_key" }, { "name": "MASTODON_VAPID_PUBLIC_KEY", "description": "Web Push VAPID public key, derived from the private key. Generated by Crow at install.", "required": true, "generate": "vapid_public_key", "pair": "MASTODON_VAPID_PRIVATE_KEY" }, - { "name": "MASTODON_AR_ENCRYPTION_DETERMINISTIC_KEY", "description": "Active Record encryption deterministic key (Mastodon 4.3+ refuses to boot without it). Generated by Crow at install; never shown.", "required": true, "secret": true, "generate": "secret" }, - { "name": "MASTODON_AR_ENCRYPTION_KEY_DERIVATION_SALT", "description": "Active Record encryption key-derivation salt (Mastodon 4.3+). Generated by Crow at install; never shown.", "required": true, "secret": true, "generate": "secret" }, - { "name": "MASTODON_AR_ENCRYPTION_PRIMARY_KEY", "description": "Active Record encryption primary key (Mastodon 4.3+). Generated by Crow at install; never shown.", "required": true, "secret": true, "generate": "secret" }, + { "name": "MASTODON_AR_ENCRYPTION_DETERMINISTIC_KEY", "description": "Active Record encryption deterministic key (Mastodon 4.3+ refuses to boot without it). Generated by Crow at install; saved in Settings → Passwords (this machine only): export the keychain to keep a copy, or data encrypted with it cannot be recovered on another machine.", "required": true, "secret": true, "generate": "secret", "keychain": true, "keychain_label": "data encryption key (deterministic)" }, + { "name": "MASTODON_AR_ENCRYPTION_KEY_DERIVATION_SALT", "description": "Active Record encryption key-derivation salt (Mastodon 4.3+). Generated by Crow at install; saved in Settings → Passwords (this machine only): export the keychain to keep a copy, or data encrypted with it cannot be recovered on another machine.", "required": true, "secret": true, "generate": "secret", "keychain": true, "keychain_label": "data encryption key-derivation salt" }, + { "name": "MASTODON_AR_ENCRYPTION_PRIMARY_KEY", "description": "Active Record encryption primary key (Mastodon 4.3+). Generated by Crow at install; saved in Settings → Passwords (this machine only): export the keychain to keep a copy, or data encrypted with it cannot be recovered on another machine.", "required": true, "secret": true, "generate": "secret", "keychain": true, "keychain_label": "data encryption key (primary)" }, { "name": "MASTODON_ACCESS_TOKEN", "description": "Admin access token (Settings → Development → New Application, grant read/write/admin:read/admin:write).", "required": false, "secret": true }, { "name": "MASTODON_URL", "description": "Internal URL the MCP server uses to reach Mastodon's web container (default http://mastodon-web:3000 over crow-federation).", "default": "http://mastodon-web:3000", "required": false }, { "name": "MASTODON_MEDIA_RETENTION_DAYS", "description": "Remote media cache retention in days (default 14; 7 on Pi-class hosts).", "default": "14", "required": false }, diff --git a/bundles/peertube/manifest.json b/bundles/peertube/manifest.json index 102c5e2c0..f62e57e45 100644 --- a/bundles/peertube/manifest.json +++ b/bundles/peertube/manifest.json @@ -1,7 +1,7 @@ { "id": "peertube", "name": "PeerTube", - "version": "1.0.3", + "version": "1.0.4", "description": "Federated video platform over ActivityPub — YouTube-alternative on the fediverse. Upload and transcode video, federate channels, stream via WebTorrent or HLS. Heaviest bundle in the federated line — video is kept on local disk, so plan for disk space and keep the transcoding/retention policy tight.", "type": "bundle", "author": "Crow", @@ -32,8 +32,8 @@ }, "env_vars": [ { "name": "PEERTUBE_WEBSERVER_HOSTNAME", "description": "Public domain (subdomain; IMMUTABLE after first federation). Appears in video URLs, channel URLs, every ActivityPub actor.", "required": true }, - { "name": "PEERTUBE_DB_PASSWORD", "description": "Password for the bundled Postgres role. Generated by Crow at install; never shown.", "required": true, "secret": true, "generate": "secret" }, - { "name": "PEERTUBE_SECRET", "description": "Server secret that signs federation requests. IMMUTABLE: generated once by Crow at install, kept across reinstalls, never shown.", "required": true, "secret": true, "generate": "secret" }, + { "name": "PEERTUBE_DB_PASSWORD", "description": "Password for the bundled Postgres role. Generated by Crow at install; saved in Settings → Passwords (this machine only), so a database restored from a backup still opens after a reinstall.", "required": true, "secret": true, "generate": "secret", "keychain": true, "keychain_label": "database password" }, + { "name": "PEERTUBE_SECRET", "description": "Server secret that signs federation requests. IMMUTABLE: generated once by Crow at install, kept across reinstalls, saved in Settings → Passwords (this machine only): export the keychain to keep a copy, or data encrypted with it cannot be recovered on another machine.", "required": true, "secret": true, "generate": "secret", "keychain": true, "keychain_label": "server secret" }, { "name": "PEERTUBE_ACCESS_TOKEN", "description": "OAuth bearer token (Administration → Users → your account → Personal Access Tokens, OR POST /api/v1/users/token with username+password + client_id+client_secret from /api/v1/oauth-clients/local).", "required": false, "secret": true }, { "name": "PEERTUBE_URL", "description": "Internal URL the MCP server uses to reach PeerTube (default http://peertube:9000 over crow-federation).", "default": "http://peertube:9000", "required": false }, { "name": "PEERTUBE_ADMIN_EMAIL", "description": "Admin email — where takedown notices, abuse reports, and federation alerts go.", "required": false }, diff --git a/bundles/pixelfed/manifest.json b/bundles/pixelfed/manifest.json index dff761ff3..853912cd9 100644 --- a/bundles/pixelfed/manifest.json +++ b/bundles/pixelfed/manifest.json @@ -1,7 +1,7 @@ { "id": "pixelfed", "name": "Pixelfed", - "version": "1.0.3", + "version": "1.0.4", "description": "Federated photo-sharing server over ActivityPub — Instagram-alternative on the fediverse. Publish photos/stories/collections; remote Mastodon/GoToSocial/Funkwhale followers see your posts in their timelines. Media is kept on local disk.", "type": "bundle", "author": "Crow", @@ -32,8 +32,8 @@ }, "env_vars": [ { "name": "PIXELFED_HOSTNAME", "description": "Public domain (subdomain; path-mounts break ActivityPub).", "required": true }, - { "name": "PIXELFED_DB_PASSWORD", "description": "Password for the bundled Postgres role. Generated by Crow at install; never shown.", "required": true, "secret": true, "generate": "secret" }, - { "name": "PIXELFED_APP_KEY", "description": "Laravel application key (base64: + 32 random bytes). Generated by Crow at install; never shown.", "required": true, "secret": true, "generate": "laravel_key" }, + { "name": "PIXELFED_DB_PASSWORD", "description": "Password for the bundled Postgres role. Generated by Crow at install; saved in Settings → Passwords (this machine only), so a database restored from a backup still opens after a reinstall.", "required": true, "secret": true, "generate": "secret", "keychain": true, "keychain_label": "database password" }, + { "name": "PIXELFED_APP_KEY", "description": "Laravel application key (base64: + 32 random bytes). Generated by Crow at install; saved in Settings → Passwords (this machine only): export the keychain to keep a copy, or data encrypted with it cannot be recovered on another machine.", "required": true, "secret": true, "generate": "laravel_key", "keychain": true, "keychain_label": "data encryption key (APP_KEY)" }, { "name": "PIXELFED_ACCESS_TOKEN", "description": "OAuth2 Personal Access Token (Settings → Development → New Application).", "required": false, "secret": true }, { "name": "PIXELFED_URL", "description": "Internal URL the MCP server uses to reach Pixelfed (default http://pixelfed:80 over crow-federation).", "default": "http://pixelfed:80", "required": false }, { "name": "PIXELFED_OPEN_REGISTRATION", "description": "Allow new user signups (true/false). Default false — opening registration without moderation tooling invites abuse.", "default": "false", "required": false }, diff --git a/bundles/tax/.env.example b/bundles/tax/.env.example index 9088cffad..2a123a0d1 100644 --- a/bundles/tax/.env.example +++ b/bundles/tax/.env.example @@ -1,8 +1,7 @@ # Crow Tax — Environment Variables -# Required: Passphrase for encrypting PII (SSN, names) at rest -# Set a passphrase of your own (keep a copy: without it saved returns cannot be read). -CROW_TAX_ENCRYPTION_KEY= +# CROW_TAX_ENCRYPTION_KEY is generated by Crow at install and saved in +# Settings → Passwords; it is not set here. # Optional: Path to tax database (defaults to Crow's main DB) # CROW_TAX_DB_PATH=/path/to/tax.db diff --git a/bundles/tax/engine/ingest/index.js b/bundles/tax/engine/ingest/index.js index a2b0ec811..17a5fccfd 100644 --- a/bundles/tax/engine/ingest/index.js +++ b/bundles/tax/engine/ingest/index.js @@ -6,7 +6,7 @@ * Returns extracted data with confidence scores. */ -import { readFileSync } from "node:fs"; +import { readFileSync, statSync } from "node:fs"; import { extname } from "node:path"; import { extractW2 } from "./w2-reader.js"; import { extract1099 } from "./1099-reader.js"; @@ -37,19 +37,39 @@ const EXTRACTORS = { * @param {string} documentType - Type of document (w2, 1099-sa, 1098-t, etc.) * @returns {Promise<{ data: object, confidence: object, warnings: string[], raw_text: string }>} */ -export async function ingestDocument(filePath, documentType) { +const MAX_DOCUMENT_BYTES = 25 * 1024 * 1024; + +export async function ingestDocument(filePath, documentType, { keys = null } = {}) { + // Checked BEFORE reading: the path comes from the AI (crow_tax_ingest_document), so a + // device, FIFO or huge file must never be read. + const ext = extname(filePath).toLowerCase(); + if (ext !== ".pdf" && ext !== ".enc") throw new Error(`Only PDF files are supported. Got: ${ext}`); + const st = statSync(filePath); + if (!st.isFile()) throw new Error("Not a regular file"); + if (st.size > MAX_DOCUMENT_BYTES) throw new Error("Document is larger than 25 MB"); + const buf = readFileSync(filePath); + // A document the dashboard stored encrypted at rest (.enc): open it with the tax key. + const { isEncryptedDocument, decryptDocument } = await import("../../server/documents-crypto.js"); + if (isEncryptedDocument(buf)) { + const k = keys || [process.env.CROW_TAX_ENCRYPTION_KEY, process.env.CROW_TAX_ENCRYPTION_KEY_PREVIOUS]; + return ingestPdfBuffer((await decryptDocument(buf, k)).plain, documentType, filePath); + } + if (ext !== ".pdf") throw new Error(`Only PDF files are supported. Got: ${ext}`); + return ingestPdfBuffer(buf, documentType, filePath); +} + +/** + * Ingest a PDF already in memory (the dashboard upload: the file on disk is stored + * encrypted, so the plaintext is never written out). `label` names it in errors. + */ +export async function ingestPdfBuffer(pdfBuffer, documentType, label = "the uploaded document") { + const filePath = label; const type = documentType.toLowerCase(); const extractor = EXTRACTORS[type]; if (!extractor) { throw new Error(`Unsupported document type: ${documentType}. Supported: ${Object.keys(EXTRACTORS).join(", ")}`); } - const ext = extname(filePath).toLowerCase(); - if (ext !== ".pdf") { - throw new Error(`Only PDF files are supported. Got: ${ext}`); - } - - const pdfBuffer = readFileSync(filePath); // Pipeline: try form fields first, then text extraction, then OCR let result; diff --git a/bundles/tax/manifest.json b/bundles/tax/manifest.json index 5b579e806..25b0f4072 100644 --- a/bundles/tax/manifest.json +++ b/bundles/tax/manifest.json @@ -1,7 +1,7 @@ { "id": "tax", "name": "Tax Filing Assistant", - "version": "2.0.3", + "version": "2.1.0", "description": "Prepare federal income taxes — PDF document ingestion, calculation, form generation, and guided filing. Federal returns only (no state). Supports 1040, Schedules 1/C/D/SE, Form 8889, Form 8863, and common credits. 2024 + 2025 tax years.", "type": "mcp-server", "author": "Crow", @@ -28,10 +28,14 @@ }, { "name": "CROW_TAX_ENCRYPTION_KEY", - "description": "Passphrase that encrypts personal data at rest (SSN, names). Choose your own and keep a copy: without it saved returns can't be read. Changing it keeps the key it replaces in CROW_TAX_ENCRYPTION_KEY_PREVIOUS, and the tax server moves saved returns to the new key when it next starts. Restart it between two changes.", + "description": "Key that encrypts your saved returns and uploaded documents at rest. Crow generates it at install and saves it in Settings → Passwords (this machine only). Export the keychain to keep a copy: without the key, saved returns and documents can't be read on another machine. A reinstall takes it back from the keychain.", "required": true, "secret": true, - "previous_key": "CROW_TAX_ENCRYPTION_KEY_PREVIOUS" + "generate": "secret", + "keychain": true, + "keychain_label": "data encryption key", + "previous_key": "CROW_TAX_ENCRYPTION_KEY_PREVIOUS", + "replace_values_sha256": ["113e3beef46e2a2ab77b0c3ea62ef37d823b82eab4f713a44af32bcdd2655365"] }, { "name": "CROW_TAX_ENCRYPTION_KEY_PREVIOUS", @@ -41,5 +45,5 @@ } ], "webUI": null, - "notes": "Federal returns only — no state income tax support. Supports MFJ, Single, HoH filing. 2025 tax year. Generates filled IRS PDFs ready to mail and step-by-step guided filing instructions. PII is encrypted at rest. Not a substitute for professional tax advice." + "notes": "Federal returns only — no state income tax support. Supports MFJ, Single, HoH filing. 2025 tax year. Generates filled IRS PDFs ready to mail and step-by-step guided filing instructions. Saved returns and uploaded documents are encrypted at rest with a key Crow generates and keeps in Settings → Passwords; the figures read from documents (shown on the Documents tab) are stored unencrypted in Crow's database. Documents uploaded before tax 2.1.0 are encrypted when the dashboard next starts; plain copies may remain in earlier backups or on the disk until overwritten. Not a substitute for professional tax advice." } diff --git a/bundles/tax/panel/routes.js b/bundles/tax/panel/routes.js index 4b058f259..785ff750b 100644 --- a/bundles/tax/panel/routes.js +++ b/bundles/tax/panel/routes.js @@ -9,12 +9,32 @@ import { Router } from "express"; import multer from "multer"; -import { readFileSync, writeFileSync, mkdirSync, existsSync } from "node:fs"; +import { readFileSync, mkdirSync, existsSync } from "node:fs"; import { join } from "node:path"; import { homedir } from "node:os"; import { pathToFileURL } from "node:url"; +import { randomBytes } from "node:crypto"; -const TAX_DOCS_DIR = join(homedir(), ".crow", "tax-documents"); +// This instance's documents folder (CROW_HOME). Files stored before 2026-10 went to +// ~/.crow/tax-documents for every instance; their rows keep absolute paths, so the +// encryption pass below also looks there. +const TAX_DOCS_DIR = join(process.env.CROW_HOME || join(homedir(), ".crow"), "tax-documents"); +const LEGACY_TAX_DOCS_DIR = join(homedir(), ".crow", "tax-documents"); + +/** The tax key, read when needed, in the SAME order the tax MCP server gets it (its env is + * the gateway env overlaid by this instance's tax entry in mcp-addons.json, proxy.js): the + * entry first, then the gateway env. A generated key reaches only that entry. Never logged. */ +const ADDONS_FILE = join(process.env.CROW_HOME || join(homedir(), ".crow"), "mcp-addons.json"); +function addonEnv() { + try { const env = JSON.parse(readFileSync(ADDONS_FILE, "utf8"))?.tax?.env; return env && typeof env === "object" ? env : {}; } catch { return {}; } +} +export function taxKeys() { + const a = addonEnv(); + return { + key: a.CROW_TAX_ENCRYPTION_KEY || process.env.CROW_TAX_ENCRYPTION_KEY || "", + previous: a.CROW_TAX_ENCRYPTION_KEY_PREVIOUS || process.env.CROW_TAX_ENCRYPTION_KEY_PREVIOUS || "", + }; +} // Resolve bundle directory (installed vs repo) function resolveBundleDir() { @@ -59,9 +79,10 @@ const upload = multer({ export default function taxRouter(authMiddleware) { const router = Router(); const bundleDir = resolveBundleDir(); + const docsCrypto = import(pathToFileURL(join(bundleDir, "server", "documents-crypto.js")).href); // Ensure directories and tables exist - mkdirSync(TAX_DOCS_DIR, { recursive: true }); + mkdirSync(TAX_DOCS_DIR, { recursive: true, mode: 0o700 }); (async () => { try { const db = await getDb(); @@ -86,6 +107,17 @@ export default function taxRouter(authMiddleware) { try { await db.execute({ sql: "ALTER TABLE tax_documents ADD COLUMN owner TEXT DEFAULT 'taxpayer'", args: [] }); } catch {} // Column already exists + // Documents stored before encryption at rest: move each to its encrypted form + // (verified before the plain file is removed; documents-crypto.js). Idempotent. + const { key, previous } = taxKeys(); + // CROW_TAX_KEEP_PLAIN_DOCUMENTS=1: after `scripts/documents.mjs decrypt --in-place` (a downgrade). + if (key && process.env.CROW_TAX_KEEP_PLAIN_DOCUMENTS !== "1") { + const { encryptStoredDocuments } = await docsCrypto; + const r = await encryptStoredDocuments(db, { key, previous, docsDirs: [TAX_DOCS_DIR, LEGACY_TAX_DOCS_DIR] }); + if (r.encrypted || r.reencrypted) console.log(`[tax] encrypted ${r.encrypted} stored document(s), re-encrypted ${r.reencrypted}`); + if (r.errors.length) console.warn(`[tax] ${r.errors.length} stored document(s) were left as they are: ${r.errors.join("; ")}`); + if (r.locked) console.warn("[tax] another run holds the documents lock; plain documents (if any) are encrypted at the next start"); + } } catch {} })(); @@ -175,11 +207,22 @@ export default function taxRouter(authMiddleware) { return res.status(400).json({ error: "No PDF file uploaded" }); } + // Stored encrypted with the tax key; the plaintext is never written to disk. + const { key } = taxKeys(); + if (!key) { + return res.status(409).json({ error: "The tax extension has no encryption key yet. Reinstall it in Extensions (your returns and documents are kept), then upload again." }); + } + const { encryptDocument, writePrivateAtomic } = await docsCrypto; + let sealed; + try { sealed = await encryptDocument(req.file.buffer, key); } catch (err) { + // The example key once published in the repo: a reinstall moves this install to a new key. + return res.status(409).json({ error: `${err.message} Reinstall the tax extension in Extensions (your returns and documents are kept): it moves to a new key.` }); + } const db = await getDb(); const filename = req.file.originalname; const safeName = filename.replace(/[^a-zA-Z0-9._-]/g, "_"); - const filePath = join(TAX_DOCS_DIR, `${Date.now()}-${safeName}`); - writeFileSync(filePath, req.file.buffer); + const filePath = join(TAX_DOCS_DIR, `${Date.now()}-${randomBytes(3).toString("hex")}-${safeName}.enc`); + writePrivateAtomic(filePath, sealed); // Record upload with owner tag await db.execute({ @@ -194,8 +237,8 @@ export default function taxRouter(authMiddleware) { // Try ingestion let ingestionResult = null; try { - const { ingestDocument } = await import(pathToFileURL(join(bundleDir, "engine", "ingest", "index.js")).href); - ingestionResult = await ingestDocument(filePath, doc_type); + const { ingestPdfBuffer } = await import(pathToFileURL(join(bundleDir, "engine", "ingest", "index.js")).href); + ingestionResult = await ingestPdfBuffer(req.file.buffer, doc_type, filename); await db.execute({ sql: `UPDATE tax_documents SET status = 'ingested', @@ -313,8 +356,11 @@ export default function taxRouter(authMiddleware) { const db = await getDb(); // Get file path to delete the uploaded file const result = await db.execute({ sql: "SELECT file_path FROM tax_documents WHERE id = ?", args: [req.params.id] }); - if (result.rows[0]?.file_path) { - try { (await import("node:fs")).unlinkSync(result.rows[0].file_path); } catch {} + // Only a regular file inside a documents folder is ever removed (the path comes from the DB). + const { storedDocumentPath } = await docsCrypto; + const fp = storedDocumentPath(result.rows[0]?.file_path, [TAX_DOCS_DIR, LEGACY_TAX_DOCS_DIR]); + if (fp) { + try { (await import("node:fs")).unlinkSync(fp); } catch {} } await db.execute({ sql: "DELETE FROM tax_documents WHERE id = ?", args: [req.params.id] }); if (req.headers.accept?.includes("text/html")) { diff --git a/bundles/tax/scripts/documents.mjs b/bundles/tax/scripts/documents.mjs new file mode 100644 index 000000000..96b492af8 --- /dev/null +++ b/bundles/tax/scripts/documents.mjs @@ -0,0 +1,51 @@ +#!/usr/bin/env node +/** + * Crow Tax — stored documents, by hand. + * + * node scripts/documents.mjs encrypt encrypt every plain stored document + * (what the dashboard does at start-up) + * node scripts/documents.mjs decrypt --out write plain copies there (0600); + * the database is not changed + * node scripts/documents.mjs decrypt --in-place undo the encryption (a downgrade): + * set CROW_TAX_KEEP_PLAIN_DOCUMENTS=1 + * first, or the next start encrypts again + * + * The key comes from this instance's tax entry in mcp-addons.json, else + * CROW_TAX_ENCRYPTION_KEY (and _PREVIOUS) — the MCP server's order. It is never printed. + */ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { homedir } from "node:os"; +import { createDbClient } from "../server/db.js"; +import { encryptStoredDocuments, decryptStoredDocuments } from "../server/documents-crypto.js"; + +function keys() { + let entry = {}; + try { entry = JSON.parse(readFileSync(join(process.env.CROW_HOME || join(homedir(), ".crow"), "mcp-addons.json"), "utf8"))?.tax?.env || {}; } catch { /* none */ } + return { + key: entry.CROW_TAX_ENCRYPTION_KEY || process.env.CROW_TAX_ENCRYPTION_KEY || "", + previous: entry.CROW_TAX_ENCRYPTION_KEY_PREVIOUS || process.env.CROW_TAX_ENCRYPTION_KEY_PREVIOUS || "", + }; +} + +const [cmd, ...rest] = process.argv.slice(2); +const { key, previous } = keys(); +if (!key) { console.error("No tax key: install the tax extension, or set CROW_TAX_ENCRYPTION_KEY."); process.exit(2); } +// The database of the SAME instance as the key and the folders (CROW_HOME), never another one's. +if (process.env.CROW_HOME && !process.env.CROW_DB_PATH && !process.env.CROW_DATA_DIR && !process.env.CROW_TAX_DB_PATH) { + process.env.CROW_DB_PATH = join(process.env.CROW_HOME, "data", "crow.db"); +} +const db = createDbClient(); +let out; +if (cmd === "encrypt") { + out = await encryptStoredDocuments(db, { key, previous }); +} else if (cmd === "decrypt" && rest[0] === "--in-place") { + out = await decryptStoredDocuments(db, { keys: [key, previous], inPlace: true }); +} else if (cmd === "decrypt" && rest[0] === "--out" && rest[1]) { + out = await decryptStoredDocuments(db, { keys: [key, previous], outDir: rest[1] }); +} else { + console.error("usage: documents.mjs encrypt | decrypt --out | decrypt --in-place"); + process.exit(2); +} +console.log(JSON.stringify(out, null, 2)); +process.exit(out.errors?.length || out.locked ? 1 : 0); diff --git a/bundles/tax/server/documents-crypto.js b/bundles/tax/server/documents-crypto.js new file mode 100644 index 000000000..e9884585f --- /dev/null +++ b/bundles/tax/server/documents-crypto.js @@ -0,0 +1,275 @@ +/** + * Crow Tax — uploaded documents encrypted at rest (recovery keys, queue item 30, 2026-10-16). + * + * A W-2 or 1099 holds SSNs and income. Uploads are stored encrypted with the tax key + * (CROW_TAX_ENCRYPTION_KEY, generated by Crow and saved in the keychain): + * "CROWTAXDOC1\n" + keyId(8) + salt(16) + iv(12) + tag(16) + AES-256-GCM ciphertext + * key = PBKDF2-SHA256(passphrase, salt, 100k) — the same derivation as crypto.js + * keyId = HMAC-SHA256(that derived key, "crow-tax-document-key-id")[0..8] — says WHICH + * key sealed a file, so a wrong key is detected before anything is replaced. It + * comes from the salted PBKDF2 output, so it offers no cheaper offline guess + * check than the ciphertext itself, and it differs per file. + * + * Files stored before this change are plain PDFs. encryptStoredDocuments() moves them one + * row at a time and never deletes a byte it has not verified: + * read → encrypt → write a private temp file + fsync → rename to a name unique to this + * run (..enc) → read back + decrypt + compare → compare-and-swap the row's + * file_path → only then unlink the plain file (and fsync the folder). + * A lost swap removes only this run's own .enc. A crash leaves the plain file plus at most a + * stray .enc; a crash after the swap leaves a plain sibling, which the next run removes once + * the .enc it points to is verified. Runs (the panel at boot, the CLI) hold an exclusive lock + * file in the documents folder, so two migrators never interleave. + * decryptStoredDocuments() is the inverse (in place), or writes plain copies to a folder + * without touching the database. Only regular files inside the documents folder(s) are ever + * read, written or removed: a file_path from the database pointing anywhere else (or a + * symlink) is skipped and reported by row id. + */ +import { createCipheriv, createDecipheriv, createHmac, randomBytes, pbkdf2 } from "node:crypto"; +import { promisify } from "node:util"; +import { openSync, writeSync, fsyncSync, closeSync, renameSync, readFileSync, unlinkSync, existsSync, lstatSync, realpathSync, mkdirSync, statSync, readdirSync, utimesSync, chmodSync } from "node:fs"; +import { join, dirname, basename, resolve, sep } from "node:path"; +import { homedir } from "node:os"; +import { isExampleKey } from "./crypto.js"; + +const pbkdf2Async = promisify(pbkdf2); +export const MAGIC = Buffer.from("CROWTAXDOC1\n", "utf8"); +const KID = 8, SALT = 16, IV = 12, TAG = 16, KEYLEN = 32, ITER = 100000; +const HEADER = MAGIC.length + KID + SALT + IV + TAG; +const LOCK_STALE_MS = 30 * 60 * 1000; +const ENC_NAME = /\.[0-9a-f]{8}\.enc$/; +const ENC_SUFFIX_LEN = ".00000000.enc".length; + +/** This instance's documents folder (CROW_HOME: co-hosted instances each have their own). */ +export function taxDocsDir(env = process.env) { + return join(env.CROW_HOME || join(homedir(), ".crow"), "tax-documents"); +} +/** Where every instance stored uploads before 2026-10 (it ignored CROW_HOME). */ +export function legacyTaxDocsDir() { + return join(homedir(), ".crow", "tax-documents"); +} + +const kidOf = (derived) => createHmac("sha256", derived).update("crow-tax-document-key-id").digest().subarray(0, KID); + +export function isEncryptedDocument(buf) { + return Buffer.isBuffer(buf) && buf.length >= HEADER && buf.subarray(0, MAGIC.length).equals(MAGIC); +} + +function usable(passphrase) { + if (!passphrase) throw new Error("CROW_TAX_ENCRYPTION_KEY is not set: documents cannot be stored encrypted."); + if (isExampleKey(passphrase)) throw new Error("CROW_TAX_ENCRYPTION_KEY is the example value published in Crow's repository; documents are never stored with it."); +} + +export async function encryptDocument(plain, passphrase) { + usable(passphrase); + const salt = randomBytes(SALT); + const iv = randomBytes(IV); + const derived = await pbkdf2Async(passphrase, salt, ITER, KEYLEN, "sha256"); + const c = createCipheriv("aes-256-gcm", derived, iv); + const ct = Buffer.concat([c.update(plain), c.final()]); + return Buffer.concat([MAGIC, kidOf(derived), salt, iv, c.getAuthTag(), ct]); +} + +function parts(buf) { + let o = MAGIC.length; + const kid = buf.subarray(o, o += KID), salt = buf.subarray(o, o += SALT), iv = buf.subarray(o, o += IV), tag = buf.subarray(o, o += TAG); + return { kid, salt, iv, tag, ct: buf.subarray(o) }; +} + +/** The passphrase (of `passphrases`) that sealed this file, and its derived key; or null. */ +async function sealing(buf, passphrases) { + if (!isEncryptedDocument(buf)) return null; + const { kid, salt } = parts(buf); + for (const p of (passphrases || []).filter(Boolean)) { + const derived = await pbkdf2Async(p, salt, ITER, KEYLEN, "sha256"); + if (kidOf(derived).equals(kid)) return { passphrase: p, derived }; + } + return null; +} +export async function sealedWith(buf, passphrases) { + return (await sealing(buf, passphrases))?.passphrase || null; +} + +export async function decryptDocument(buf, passphrases) { + if (!isEncryptedDocument(buf)) throw new Error("not an encrypted tax document"); + const k = await sealing(buf, Array.isArray(passphrases) ? passphrases : [passphrases]); + if (!k) throw new Error("this document was stored with a different tax key (not the current or previous one)"); + const { iv, tag, ct } = parts(buf); + const d = createDecipheriv("aes-256-gcm", k.derived, iv); + d.setAuthTag(tag); + return { plain: Buffer.concat([d.update(ct), d.final()]), passphrase: k.passphrase }; +} + +function fsyncDir(dir) { + try { const fd = openSync(dir, "r"); try { fsyncSync(fd); } finally { closeSync(fd); } } catch { /* best-effort */ } +} + +/** Crash-safe private write: temp (0600, exclusive) → fsync → rename → fsync the folder. */ +export function writePrivateAtomic(path, buf) { + mkdirSync(dirname(path), { recursive: true, mode: 0o700 }); + const tmp = join(dirname(path), `.${basename(path)}.tmp-${process.pid}-${randomBytes(4).toString("hex")}`); + const fd = openSync(tmp, "wx", 0o600); + try { writeSync(fd, buf); fsyncSync(fd); } finally { closeSync(fd); } + renameSync(tmp, path); + fsyncDir(dirname(path)); +} + +/** The path, when it is a regular file (not a symlink) whose real location is inside one of `dirs`. */ +export function storedDocumentPath(p, dirs) { + if (typeof p !== "string" || !p) return null; + const abs = resolve(p); + for (const d of dirs) { + let root; + try { root = realpathSync(d); } catch { continue; } + if (!abs.startsWith(resolve(d) + sep) && !abs.startsWith(root + sep)) continue; + try { + if (!lstatSync(abs).isFile()) return null; + return realpathSync(abs).startsWith(root + sep) ? abs : null; + } catch { return null; } + } + return null; +} + +/** + * A row's file. Fallbacks, for a crow.db restored from a backup: the same name in this + * instance's folder (restored under another CROW_HOME); and, when the row still names the + * plain file a later run migrated, that file's single encrypted successor .<8hex>.enc + * (in its own folder or this one). The caller swaps the row to whatever is found. + */ +function resolveRowFile(p, dirs) { + const direct = storedDocumentPath(p, dirs); + if (direct) return direct; + if (typeof p !== "string" || !p) return null; + const name = basename(p); + const here = storedDocumentPath(join(dirs[0], name), dirs); + if (here) return here; + if (ENC_NAME.test(name)) return null; + for (const d of [dirname(resolve(p)), dirs[0]]) { + let list = []; + try { list = readdirSync(d).filter((f) => f.startsWith(name + ".") && ENC_NAME.test(f) && f.slice(0, -ENC_SUFFIX_LEN) === name); } catch { continue; } + if (list.length === 1) { const hit = storedDocumentPath(join(d, list[0]), dirs); if (hit) return hit; } + } + return null; +} + +/** + * Exclusive lock in the documents folder (also tightens the folder to 0700). A lock not + * touched for 30 minutes is stale: it is taken over by an atomic rename, which only one + * contender can win. The holder touches it as it goes (`touch`). + */ +function withLock(dir) { + mkdirSync(dir, { recursive: true, mode: 0o700 }); + try { chmodSync(dir, 0o700); } catch { /* not ours to change */ } + const lock = join(dir, ".documents.lock"); + const take = () => { const fd = openSync(lock, "wx", 0o600); closeSync(fd); }; + try { + take(); + } catch { + try { + if (Date.now() - statSync(lock).mtimeMs < LOCK_STALE_MS) return null; + const aside = `${lock}.stale-${process.pid}-${randomBytes(4).toString("hex")}`; + renameSync(lock, aside); // ENOENT for every contender but one + unlinkSync(aside); + take(); + } catch { return null; } + } + const release = () => { try { unlinkSync(lock); } catch {} }; + release.touch = () => { try { const t = new Date(); utimesSync(lock, t, t); } catch {} }; + return release; +} + +async function swapPath(db, id, from, to) { + const r = await db.execute({ sql: "UPDATE tax_documents SET file_path = ? WHERE id = ? AND file_path = ?", args: [to, id, from] }); + return Number(r.rowsAffected ?? r.changes ?? 0) > 0; +} + + + +/** + * Move every plain stored document to its encrypted form (and re-encrypt one only the + * previous key opens). Idempotent. Returns { encrypted, reencrypted, cleaned, skipped, errors, locked }. + */ +export async function encryptStoredDocuments(db, { key, previous = null, docsDirs = [taxDocsDir(), legacyTaxDocsDir()] } = {}) { + const out = { encrypted: 0, reencrypted: 0, cleaned: 0, skipped: 0, errors: [], locked: false }; + if (!key) return { ...out, errors: ["CROW_TAX_ENCRYPTION_KEY is not set"] }; + try { usable(key); } catch (err) { return { ...out, errors: [err.message] }; } + let rows; + try { rows = (await db.execute("SELECT id, file_path FROM tax_documents")).rows; } catch { return out; } + const release = withLock(docsDirs[0]); + if (!release) return { ...out, locked: true }; + try { + for (const r of rows) { + release.touch(); + const p = resolveRowFile(r.file_path, docsDirs); + if (!p) { out.skipped++; continue; } + try { + const buf = readFileSync(p); + if (isEncryptedDocument(buf)) { + // Found by a fallback (a restored crow.db): point the row at it. + if (p !== resolve(r.file_path) && (await swapPath(db, r.id, r.file_path, p))) r.file_path = p; + const { plain, passphrase } = await decryptDocument(buf, [key, previous]); + // A crash after the swap left the plain original next to it: remove it once the + // .enc is verified to hold exactly those bytes. + const sibling = ENC_NAME.test(p) ? storedDocumentPath(p.replace(ENC_NAME, ""), docsDirs) : null; + if (sibling && readFileSync(sibling).equals(plain)) { unlinkSync(sibling); fsyncDir(dirname(sibling)); out.cleaned++; } + if (passphrase === key) { out.skipped++; continue; } + const next = await encryptDocument(plain, key); + if (!(await decryptDocument(next, [key])).plain.equals(plain)) throw new Error("verification failed"); + writePrivateAtomic(p, next); + out.reencrypted++; + continue; + } + const target = `${p.replace(ENC_NAME, "")}.${randomBytes(4).toString("hex")}.enc`; + writePrivateAtomic(target, await encryptDocument(buf, key)); + if (!(await decryptDocument(readFileSync(target), [key])).plain.equals(buf)) { unlinkSync(target); throw new Error("verification failed"); } + if (!(await swapPath(db, r.id, r.file_path, target))) { unlinkSync(target); out.skipped++; continue; } + unlinkSync(p); + fsyncDir(dirname(p)); + out.encrypted++; + } catch (err) { + out.errors.push(`document ${r.id}: ${err.message}`); + } + } + } finally { release(); } + return out; +} + +/** + * The inverse. inPlace: plain file written + verified, row swapped back, .enc removed + * (for a downgrade: set CROW_TAX_KEEP_PLAIN_DOCUMENTS=1 first, or the next start encrypts + * them again). Otherwise plain copies go to outDir (0700, files 0600), database untouched. + */ +export async function decryptStoredDocuments(db, { keys, inPlace = false, outDir = null, docsDirs = [taxDocsDir(), legacyTaxDocsDir()] } = {}) { + const out = { decrypted: 0, skipped: 0, errors: [], locked: false }; + if (!inPlace && !outDir) throw new Error("choose inPlace or outDir"); + const rows = (await db.execute("SELECT id, filename, file_path FROM tax_documents")).rows; + const release = inPlace ? withLock(docsDirs[0]) : () => {}; + if (!release) return { ...out, locked: true }; + try { + for (const r of rows) { + const p = resolveRowFile(r.file_path, docsDirs); + if (!p) { out.skipped++; continue; } + try { + const buf = readFileSync(p); + if (!isEncryptedDocument(buf)) { out.skipped++; continue; } + const { plain } = await decryptDocument(buf, keys); + if (inPlace) { + const target = ENC_NAME.test(p) ? p.replace(ENC_NAME, "") : p.endsWith(".enc") ? p.slice(0, -4) : `${p}.pdf`; + if (existsSync(target)) throw new Error(`${basename(target)} already exists`); + writePrivateAtomic(target, plain); + if (!readFileSync(target).equals(plain)) { unlinkSync(target); throw new Error("verification failed"); } + if (!(await swapPath(db, r.id, r.file_path, target))) { unlinkSync(target); out.skipped++; continue; } + unlinkSync(p); + } else { + const name = `${r.id}-${String(r.filename || "document.pdf").replace(/[^a-zA-Z0-9._-]/g, "_")}`; + mkdirSync(outDir, { recursive: true, mode: 0o700 }); + try { chmodSync(outDir, 0o700); } catch { /* not ours */ } + writePrivateAtomic(join(outDir, name), plain); + } + out.decrypted++; + } catch (err) { + out.errors.push(`document ${r.id}: ${err.message}`); + } + } + } finally { release(); } + return out; +} diff --git a/registry/add-ons.json b/registry/add-ons.json index 7b33aebfb..a43b8a02b 100644 --- a/registry/add-ons.json +++ b/registry/add-ons.json @@ -745,7 +745,7 @@ { "id": "campaigns", "name": "Campaigns", - "version": "1.0.2", + "version": "1.0.3", "description": "Social media campaign manager — draft, schedule, and publish posts across platforms (starting with Reddit)", "type": "mcp-server", "author": "Crow", @@ -780,9 +780,12 @@ "env_vars": [ { "name": "CROW_CAMPAIGNS_ENCRYPTION_KEY", - "description": "AES-256-GCM encryption key for Reddit credentials (generate with: openssl rand -base64 32)", + "description": "Key that encrypts your saved Reddit credentials (AES-256-GCM). Crow generates it at install and saves it in Settings → Passwords (this machine only); a reinstall takes it back from the keychain. Without it, re-enter the Reddit credentials.", "required": true, - "secret": true + "secret": true, + "generate": "secret", + "keychain": true, + "keychain_label": "data encryption key" } ], "notes": "12 MCP tools for social media campaign management. Reddit support via snoowrap. Credentials encrypted at rest with AES-256-GCM. Dashboard panel with approval workflow.", @@ -3038,7 +3041,7 @@ { "id": "mastodon", "name": "Mastodon", - "version": "1.0.3", + "version": "1.0.4", "description": "The flagship ActivityPub server — federated microblogging at scale. Heaviest of the small-AP bundles. Hosts a public (or invite-only) Mastodon instance with the full v1/v2 API surface, web UI, and Sidekiq background job processing. Media is kept on local disk.", "type": "bundle", "author": "Crow", @@ -3106,24 +3109,30 @@ }, { "name": "MASTODON_DB_PASSWORD", - "description": "Password for the bundled Postgres role. Generated by Crow at install; never shown.", + "description": "Password for the bundled Postgres role. Generated by Crow at install; saved in Settings → Passwords (this machine only), so a database restored from a backup still opens after a reinstall.", "required": true, "secret": true, - "generate": "secret" + "generate": "secret", + "keychain": true, + "keychain_label": "database password" }, { "name": "MASTODON_SECRET_KEY_BASE", - "description": "Rails secret_key_base. Generated by Crow at install; never shown.", + "description": "Rails secret_key_base. Generated by Crow at install; saved in Settings → Passwords (this machine only): export the keychain to keep a copy, or data encrypted with it cannot be recovered on another machine.", "required": true, "secret": true, - "generate": "secret" + "generate": "secret", + "keychain": true, + "keychain_label": "session signing key (secret_key_base)" }, { "name": "MASTODON_OTP_SECRET", - "description": "2FA OTP secret. Generated by Crow at install; never shown.", + "description": "2FA OTP secret. Generated by Crow at install; saved in Settings → Passwords (this machine only): export the keychain to keep a copy, or data encrypted with it cannot be recovered on another machine.", "required": true, "secret": true, - "generate": "secret" + "generate": "secret", + "keychain": true, + "keychain_label": "two-factor secret" }, { "name": "MASTODON_VAPID_PRIVATE_KEY", @@ -3141,24 +3150,30 @@ }, { "name": "MASTODON_AR_ENCRYPTION_DETERMINISTIC_KEY", - "description": "Active Record encryption deterministic key (Mastodon 4.3+ refuses to boot without it). Generated by Crow at install; never shown.", + "description": "Active Record encryption deterministic key (Mastodon 4.3+ refuses to boot without it). Generated by Crow at install; saved in Settings → Passwords (this machine only): export the keychain to keep a copy, or data encrypted with it cannot be recovered on another machine.", "required": true, "secret": true, - "generate": "secret" + "generate": "secret", + "keychain": true, + "keychain_label": "data encryption key (deterministic)" }, { "name": "MASTODON_AR_ENCRYPTION_KEY_DERIVATION_SALT", - "description": "Active Record encryption key-derivation salt (Mastodon 4.3+). Generated by Crow at install; never shown.", + "description": "Active Record encryption key-derivation salt (Mastodon 4.3+). Generated by Crow at install; saved in Settings → Passwords (this machine only): export the keychain to keep a copy, or data encrypted with it cannot be recovered on another machine.", "required": true, "secret": true, - "generate": "secret" + "generate": "secret", + "keychain": true, + "keychain_label": "data encryption key-derivation salt" }, { "name": "MASTODON_AR_ENCRYPTION_PRIMARY_KEY", - "description": "Active Record encryption primary key (Mastodon 4.3+). Generated by Crow at install; never shown.", + "description": "Active Record encryption primary key (Mastodon 4.3+). Generated by Crow at install; saved in Settings → Passwords (this machine only): export the keychain to keep a copy, or data encrypted with it cannot be recovered on another machine.", "required": true, "secret": true, - "generate": "secret" + "generate": "secret", + "keychain": true, + "keychain_label": "data encryption key (primary)" }, { "name": "MASTODON_ACCESS_TOKEN", @@ -4530,7 +4545,7 @@ { "id": "peertube", "name": "PeerTube", - "version": "1.0.3", + "version": "1.0.4", "description": "Federated video platform over ActivityPub — YouTube-alternative on the fediverse. Upload and transcode video, federate channels, stream via WebTorrent or HLS. Heaviest bundle in the federated line — video is kept on local disk, so plan for disk space and keep the transcoding/retention policy tight.", "type": "bundle", "author": "Crow", @@ -4591,17 +4606,21 @@ }, { "name": "PEERTUBE_DB_PASSWORD", - "description": "Password for the bundled Postgres role. Generated by Crow at install; never shown.", + "description": "Password for the bundled Postgres role. Generated by Crow at install; saved in Settings → Passwords (this machine only), so a database restored from a backup still opens after a reinstall.", "required": true, "secret": true, - "generate": "secret" + "generate": "secret", + "keychain": true, + "keychain_label": "database password" }, { "name": "PEERTUBE_SECRET", - "description": "Server secret that signs federation requests. IMMUTABLE: generated once by Crow at install, kept across reinstalls, never shown.", + "description": "Server secret that signs federation requests. IMMUTABLE: generated once by Crow at install, kept across reinstalls, saved in Settings → Passwords (this machine only): export the keychain to keep a copy, or data encrypted with it cannot be recovered on another machine.", "required": true, "secret": true, - "generate": "secret" + "generate": "secret", + "keychain": true, + "keychain_label": "server secret" }, { "name": "PEERTUBE_ACCESS_TOKEN", @@ -4734,7 +4753,7 @@ { "id": "pixelfed", "name": "Pixelfed", - "version": "1.0.3", + "version": "1.0.4", "description": "Federated photo-sharing server over ActivityPub — Instagram-alternative on the fediverse. Publish photos/stories/collections; remote Mastodon/GoToSocial/Funkwhale followers see your posts in their timelines. Media is kept on local disk.", "type": "bundle", "author": "Crow", @@ -4794,17 +4813,21 @@ }, { "name": "PIXELFED_DB_PASSWORD", - "description": "Password for the bundled Postgres role. Generated by Crow at install; never shown.", + "description": "Password for the bundled Postgres role. Generated by Crow at install; saved in Settings → Passwords (this machine only), so a database restored from a backup still opens after a reinstall.", "required": true, "secret": true, - "generate": "secret" + "generate": "secret", + "keychain": true, + "keychain_label": "database password" }, { "name": "PIXELFED_APP_KEY", - "description": "Laravel application key (base64: + 32 random bytes). Generated by Crow at install; never shown.", + "description": "Laravel application key (base64: + 32 random bytes). Generated by Crow at install; saved in Settings → Passwords (this machine only): export the keychain to keep a copy, or data encrypted with it cannot be recovered on another machine.", "required": true, "secret": true, - "generate": "laravel_key" + "generate": "laravel_key", + "keychain": true, + "keychain_label": "data encryption key (APP_KEY)" }, { "name": "PIXELFED_ACCESS_TOKEN", @@ -5885,7 +5908,7 @@ { "id": "tax", "name": "Tax Filing Assistant", - "version": "2.0.3", + "version": "2.1.0", "description": "Prepare federal income taxes — PDF document ingestion, calculation, form generation, and guided filing. Federal returns only (no state). Supports 1040, Schedules 1/C/D/SE, Form 8889, Form 8863, and common credits. 2024 + 2025 tax years.", "type": "mcp-server", "author": "Crow", @@ -5926,10 +5949,16 @@ }, { "name": "CROW_TAX_ENCRYPTION_KEY", - "description": "Passphrase that encrypts personal data at rest (SSN, names). Choose your own and keep a copy: without it saved returns can't be read. Changing it keeps the key it replaces in CROW_TAX_ENCRYPTION_KEY_PREVIOUS, and the tax server moves saved returns to the new key when it next starts. Restart it between two changes.", + "description": "Key that encrypts your saved returns and uploaded documents at rest. Crow generates it at install and saves it in Settings → Passwords (this machine only). Export the keychain to keep a copy: without the key, saved returns and documents can't be read on another machine. A reinstall takes it back from the keychain.", "required": true, "secret": true, - "previous_key": "CROW_TAX_ENCRYPTION_KEY_PREVIOUS" + "generate": "secret", + "keychain": true, + "keychain_label": "data encryption key", + "previous_key": "CROW_TAX_ENCRYPTION_KEY_PREVIOUS", + "replace_values_sha256": [ + "113e3beef46e2a2ab77b0c3ea62ef37d823b82eab4f713a44af32bcdd2655365" + ] }, { "name": "CROW_TAX_ENCRYPTION_KEY_PREVIOUS", @@ -5939,7 +5968,7 @@ } ], "webUI": null, - "notes": "Federal returns only — no state income tax support. Supports MFJ, Single, HoH filing. 2025 tax year. Generates filled IRS PDFs ready to mail and step-by-step guided filing instructions. PII is encrypted at rest. Not a substitute for professional tax advice.", + "notes": "Federal returns only — no state income tax support. Supports MFJ, Single, HoH filing. 2025 tax year. Generates filled IRS PDFs ready to mail and step-by-step guided filing instructions. Saved returns and uploaded documents are encrypted at rest with a key Crow generates and keeps in Settings → Passwords; the figures read from documents (shown on the Documents tab) are stored unencrypted in Crow's database. Documents uploaded before tax 2.1.0 are encrypted when the dashboard next starts; plain copies may remain in earlier backups or on the disk until overwritten. Not a substitute for professional tax advice.", "official": true }, { diff --git a/scripts/lib/bundle-contract.mjs b/scripts/lib/bundle-contract.mjs index 187cb0874..549e068e7 100644 --- a/scripts/lib/bundle-contract.mjs +++ b/scripts/lib/bundle-contract.mjs @@ -205,6 +205,13 @@ export function validateManifest(manifest, bundleDir, opts = {}) { const other = (manifest.env_vars || []).find((o) => o && o.name === v.previous_key); if (typeof v.previous_key !== "string" || !other || other.name === v.name) errors.push(`env_vars ${v.name}: previous_key must name another declared env var`); } + if (v.replace_values_sha256 !== undefined) { + if (!Array.isArray(v.replace_values_sha256) || v.replace_values_sha256.length === 0 || !v.replace_values_sha256.every((h) => typeof h === "string" && /^[0-9a-f]{64}$/.test(h))) { + errors.push(`env_vars ${v.name}: replace_values_sha256 must be a non-empty list of lowercase sha256 hex digests`); + } else if (v.generate === undefined || typeof v.previous_key !== "string") { + errors.push(`env_vars ${v.name}: replace_values_sha256 needs generate and previous_key (the replaced value moves there)`); + } + } if (v.default_in_crow_home !== undefined) { const rel = v.default_in_crow_home; if (typeof rel !== "string" || rel === "" || rel.startsWith("/") || rel.includes("\\") || rel.includes("$") || rel.split("/").includes("..")) { diff --git a/servers/gateway/boot/mcp-mounts.js b/servers/gateway/boot/mcp-mounts.js index 6ceb3bdea..6e91cb26c 100644 --- a/servers/gateway/boot/mcp-mounts.js +++ b/servers/gateway/boot/mcp-mounts.js @@ -235,6 +235,13 @@ export async function mountMcpServers(app, deps) { if (errors.length > 0) { console.warn(`[bundles] Asset repair errors:`, errors); } + // Recovery keys: keys an older install kept only in its .env go into the keychain once. + try { + const { backfillBundleKeychain } = await import("../routes/bundles.js"); + await backfillBundleKeychain(); + } catch (err) { + console.warn(`[keychain] backfill skipped: ${err.message}`); + } // Skill sync: one notification per skill the owner EDITED, once per new // shipped version (the record's notifiedFor). Unverified copies are never // notified — we cannot tell the owner they edited something we cannot verify. diff --git a/servers/gateway/bundle-env-secrets.js b/servers/gateway/bundle-env-secrets.js index 6b5ef4ff2..ebd085857 100644 --- a/servers/gateway/bundle-env-secrets.js +++ b/servers/gateway/bundle-env-secrets.js @@ -70,6 +70,13 @@ export function generatedEnvKeys(manifest) { .map((v) => v.name); } +/** Whether `value` is one the manifest lists as publicly known (sha256 of the trimmed value). */ +export function isReplacedValue(spec, value) { + const list = spec?.replace_values_sha256; + if (!Array.isArray(list) || typeof value !== "string" || value === "") return false; + return list.includes(createHash("sha256").update(value.trim()).digest("hex")); +} + export function newSecretValue(kind = "secret") { if (kind === "laravel_key") return `base64:${randomBytes(32).toString("base64")}`; return randomBytes(32).toString("base64url"); @@ -135,11 +142,15 @@ export function keychainEligibleKeys(manifest) { * minted plaintext of keychain:true keys created by THIS call (empty on reinstall) * persist writes the retained copy; call it only after `minted` is safely in the * keychain — otherwise a lost plaintext would leave an unusable hash behind. - * Order per key: installed .env → retained copy → new value (never regenerated). + * Order per key: installed .env → retained copy → compose fallback → `keychain` (plaintext of + * a readable keychain entry, keychain:true keys only — install-hooks keychainRestoreValues) → + * new value (never regenerated). The keychain rung is the rebuilt-machine path: crow.db + * restored and the keychain Export imported, but no .env and no retained copy. A restored + * value is REUSED (listed in `restored`), never minted, so its entry is never overwritten. */ -export function planGeneratedEnv(bundleId, manifest, { destDir, crowHome, composeText = null, priorInstall = false }) { +export function planGeneratedEnv(bundleId, manifest, { destDir, crowHome, composeText = null, priorInstall = false, keychain = null }) { const keys = generatedEnvKeys(manifest); - if (keys.length === 0) return { env: {}, minted: {}, reusedPlain: {}, persist() {} }; + if (keys.length === 0) return { env: {}, minted: {}, reusedPlain: {}, restored: [], replaced: [], replaceSkipped: [], persist() {} }; if (typeof bundleId !== "string" || !/^[a-z0-9][a-z0-9-]*$/.test(bundleId) || bundleId.length > 64) { throw new Error(`Invalid bundle ID: ${JSON.stringify(bundleId)}`); } @@ -157,13 +168,16 @@ export function planGeneratedEnv(bundleId, manifest, { destDir, crowHome, compos // keychain:true values kept from an earlier install (typed, or generated before): the // keychain may not hold them, and the form no longer shows the field — see install-hooks. const reusedPlain = {}; + const restored = []; + const replaced = []; + const replaceSkipped = []; const existingFor = (k) => installed[k] || retained[k] || legacy[k] || ""; // Private halves first, so a public half can be derived from (or minted with) its pair. const order = [...keys].sort((a, b) => (byName.get(a)?.generate === "vapid_public_key") - (byName.get(b)?.generate === "vapid_public_key")); const pairs = {}; for (const k of order) { const spec = byName.get(k) || {}; - const existing = existingFor(k); + let existing = existingFor(k); if (spec.generate === "vapid_public_key") { const priv = env[spec.pair]; if (!priv || byName.get(spec.pair)?.generate !== "vapid_private_key") { @@ -174,6 +188,27 @@ export function planGeneratedEnv(bundleId, manifest, { destDir, crowHome, compos env[k] = pairs[spec.pair] || vapidPublicFromPrivate(priv) || existing; continue; } + // A value once published in a repo example (env_vars[].replace_values_sha256, e.g. tax's + // old .env.example key) is never kept: a new key is generated and the old one moves to + // the previous_key slot, so data saved with it still opens and is re-encrypted. + if (existing && isReplacedValue(spec, existing)) { + const prevSlot = spec.previous_key; + if (prevSlot && !installed[prevSlot] && !retained[prevSlot]) { + env[prevSlot] = existing; + replaced.push(k); + existing = ""; + } else { + replaceSkipped.push(k); + } + } + const fromKeychain = !existing && spec.keychain === true && !isReplacedValue(spec, keychain?.[k]) && keychain && typeof keychain[k] === "string" && keychain[k] !== "" ? keychain[k] : ""; + if (fromKeychain) { + // The keychain holds the plaintext; a store_as key gets a fresh hash of it (any hash + // of the same plaintext verifies). Its entry exists, so nothing is saved again. + env[k] = spec.store_as === "argon2id" ? argon2idPhc(fromKeychain) : fromKeychain; + restored.push(k); + continue; + } if (existing) { env[k] = existing; if (spec.keychain === true && !spec.store_as) { @@ -203,7 +238,7 @@ export function planGeneratedEnv(bundleId, manifest, { destDir, crowHome, compos formatEnvLines({ ...retained, ...env }), ); }; - return { env, minted, reusedPlain, persist }; + return { env, minted, reusedPlain, restored, replaced, replaceSkipped, persist }; } /** diff --git a/servers/gateway/dashboard/panels/nest/health-signals.js b/servers/gateway/dashboard/panels/nest/health-signals.js index 108259a40..fe8509ca8 100644 --- a/servers/gateway/dashboard/panels/nest/health-signals.js +++ b/servers/gateway/dashboard/panels/nest/health-signals.js @@ -15,6 +15,8 @@ * updates — auto_update_* version comparison (info if update available) * backup — newest of THIS instance's files in CROW_BACKUP_DIR (none → info; * >7d → warn; owned by another instance → warn) + * keychainExport — extension passwords that are in no keychain Export yet (info; no + * card when there are none) * providers — alwaysResident provider residency (unreachable ≥threshold → warn) * externalEngines — engines another machine runs (spec 2026-09-23): own id, * info at most — never warn, never a push; no card when none @@ -56,6 +58,8 @@ import { readReservation } from "../../../box-reservation.js"; import { localBackupContext, newestBackupOwnership } from "../../../../shared/backup-naming.js"; import { getStats as getOutboxStats } from "../../../../sharing/sync-outbox-drain.js"; import { createSidecarHealthReader } from "../../../artifact-origin/sidecar-health.js"; +import { unexportedCount } from "../../../keychain/reminder.js"; +import { loadKeychainKey } from "../../../keychain/key.js"; // ─── Module-level 30s cache ─────────────────────────────────────────────────── @@ -672,6 +676,19 @@ async function backupSignal(db, nowFn, lang) { }; } +// Keychain Export reminder (recovery keys, queue item 30): a password an extension saved to the +// keychain that is in no keychain Export lives only on this machine. Info only (strip, never a push); no card +// when every generated key is exported or there are none. +async function keychainExportSignal(db, lang) { + const n = await unexportedCount(db, loadKeychainKey()?.id); + if (n <= 0) return null; + return { + id: "keychainExport", severity: "info", state: "info", label: t("signals.keychainExport.label", lang), + value: String(n), issueLabel: fill(t("signals.keychainExport.issue", lang), { n }), + actionLabel: t("signals.keychainExport.action", lang), actionHref: "/dashboard/settings?section=passwords", + }; +} + // ─── Security-maintenance signals (W2) ──────────────────────────────────────── async function loginsSignal(db, lang) { @@ -1181,6 +1198,7 @@ export async function collectHealthSignals(db, opts = {}) { syncDialSignal(db, lang, nowFn), updatesSignal(db), backupSignal(db, nowFn, lang), + keychainExportSignal(db, lang), syncConflictsSignal(db), outboxSignal(db), loginsSignal(db, lang), diff --git a/servers/gateway/dashboard/shared/i18n.js b/servers/gateway/dashboard/shared/i18n.js index 40b5fa838..3db4c6c00 100644 --- a/servers/gateway/dashboard/shared/i18n.js +++ b/servers/gateway/dashboard/shared/i18n.js @@ -229,6 +229,9 @@ export const translations = { "signals.integrations.action": { en: "Check connections", es: "Revisar conexiones" }, // backup "signals.backup.label": { en: "Backup", es: "Copia de seguridad" }, + "signals.keychainExport.label": { en: "Keychain export", es: "Exportación del llavero" }, + "signals.keychainExport.issue": { en: "{n} saved extension password(s) are not in a keychain Export yet. If this machine is lost, they cannot be recovered without one.", es: "{n} contraseña(s) guardadas de extensiones aún no están en una exportación del llavero. Si pierdes esta máquina, no se pueden recuperar sin una." }, + "signals.keychainExport.action": { en: "Export keychain", es: "Exportar llavero" }, "signals.backup.verified": { en: "checked ✓", es: "verificada ✓" }, "signals.backup.damaged": { en: "may be damaged", es: "puede estar dañada" }, "signals.backup.damagedIssue": { en: "Your latest backup may be damaged — make a new backup now", es: "Tu última copia de seguridad puede estar dañada — haz una copia nueva ahora" }, @@ -2355,7 +2358,7 @@ export const translations = { "passwords.statusRemoved": { en: "Extension removed", es: "Extensión eliminada" }, "passwords.statusUnreadable": { en: "Unreadable — key missing", es: "Ilegible: falta la clave" }, "passwords.noReauth": { en: "Revealing, copying, deleting and exporting passwords need Crow to confirm it's you, but this Crow has no dashboard password and no two-factor authentication. Set one in Settings → Change Password or Two-Factor Auth first.", es: "Para mostrar, copiar, eliminar o exportar contraseñas, Crow tiene que confirmar que eres tú, pero esta instancia no tiene contraseña del panel ni autenticación de dos factores. Configura una primero en Ajustes → Cambiar contraseña o Autenticación 2FA." }, - "passwords.deleteGeneratedConfirm": { en: "Crow generated this token and keeps no other copy. Without it you cannot sign in to the extension's admin page. Delete it anyway?", es: "Crow generó este token y no guarda otra copia. Sin él no podrás entrar en la página de administración de la extensión. ¿Eliminarlo de todos modos?" }, + "passwords.deleteGeneratedConfirm": { en: "Crow generated this and keeps no other copy. Without it you may not be able to sign in to the extension, or read data it encrypted. Delete it anyway?", es: "Crow generó esto y no guarda otra copia. Sin ello quizá no puedas entrar en la extensión ni leer los datos que cifró. ¿Eliminarlo de todos modos?" }, "passwords.exportTitle": { en: "Export and import", es: "Exportar e importar" }, "passwords.exportHint": { en: "Export writes every readable password into one file encrypted with a passphrase you choose. Keep it offline. Import it here, or on a new machine, to get the passwords back.", es: "Exportar guarda todas las contraseñas legibles en un archivo cifrado con una frase de paso que eliges tú. Guárdalo fuera de línea. Impórtalo aquí, o en una máquina nueva, para recuperar las contraseñas." }, "passwords.exportPassphrase": { en: "Passphrase (at least 12 characters)", es: "Frase de paso (al menos 12 caracteres)" }, diff --git a/servers/gateway/keychain/api.js b/servers/gateway/keychain/api.js index b72533bcf..03b2a50e2 100644 --- a/servers/gateway/keychain/api.js +++ b/servers/gateway/keychain/api.js @@ -18,6 +18,8 @@ import { exportableEntries, importEntries, ensureWriteKey, } from "./store.js"; import { vaultwardenStatus, saveToVault } from "./vault-save.js"; +import { recordKeychainExport, recordKeychainImport, exportSnapshot } from "./reminder.js"; +import { invalidateHealthCache } from "../dashboard/panels/nest/health-signals.js"; const BASE = "/dashboard/keychain/api"; @@ -165,7 +167,7 @@ export function keychainApiRouter({ if (!entry) return res.status(404).json({ error: "No such password." }); // C5: a generated token may be the ONLY plaintext of a hash the extension still uses. if (entry.origin === "generated" && entry.status === "active" && req.body?.confirm_generated !== true) { - return res.status(409).json({ code: "generated_in_use", error: `Crow generated this token and keeps no other copy: without it you cannot sign in to ${entry.bundle_id || "the extension"}'s admin page. Export or save it elsewhere first.` }); + return res.status(409).json({ code: "generated_in_use", error: `Crow generated this and keeps no other copy: without it you may not be able to sign in to ${entry.bundle_id || "the extension"}, or read data it encrypted. Export or save it elsewhere first.` }); } await deleteEntry(db, entry.id); await audit(db, "keychain_delete", { ip: req.ip, details: auditDetails(entry) }); @@ -178,8 +180,11 @@ export function keychainApiRouter({ if (passphrase.length < MIN_PASSPHRASE) return res.status(400).json({ error: `Choose a passphrase of at least ${MIN_PASSPHRASE} characters.` }); const key = readKey(); if (!key) return res.status(409).json(KEY_MISSING); + const snapshot = await exportSnapshot(db, key.id).catch(() => null); const entries = await exportableEntries(db, key); const file = await sealExport(entries, passphrase); + if (snapshot) await recordKeychainExport(db, key.id, snapshot).catch(() => {}); // the Nest reminder; never blocks the download + invalidateHealthCache(); await audit(db, "keychain_export", { ip: req.ip, details: { count: entries.length } }); res.set("Content-Disposition", `attachment; filename="crow-keychain-${new Date().toISOString().slice(0, 10)}.json"`); res.json(file); @@ -189,7 +194,10 @@ export function keychainApiRouter({ if (!(await requireGrant(req, res))) return; const entries = await openExport(req.body?.file, typeof req.body?.passphrase === "string" ? req.body.passphrase : ""); if (!entries) return res.status(400).json({ error: "That passphrase does not open this file, or it is not a Crow keychain export." }); - const out = await importEntries(db, await writeKey(db, req.ip), entries); + const ikey = await writeKey(db, req.ip); + const { ids, ...out } = await importEntries(db, ikey, entries); + await recordKeychainImport(db, ikey.id, ids).catch(() => {}); + invalidateHealthCache(); await audit(db, "keychain_import", { ip: req.ip, details: out }); res.json({ ok: true, ...out }); })); diff --git a/servers/gateway/keychain/install-hooks.js b/servers/gateway/keychain/install-hooks.js index 0c6dd0684..5cd3acb8e 100644 --- a/servers/gateway/keychain/install-hooks.js +++ b/servers/gateway/keychain/install-hooks.js @@ -10,9 +10,12 @@ * - the optional vault copy uses the typed vault credentials once, then drops them. */ import { createDbClient, auditLog } from "../../db.js"; -import { keychainGeneratedKeys, keychainEligibleKeys, expandKeychainTemplate } from "../bundle-env-secrets.js"; +import { keychainGeneratedKeys, keychainEligibleKeys, expandKeychainTemplate, isReplacedValue } from "../bundle-env-secrets.js"; import { createNotification } from "../../shared/notifications.js"; -import { saveExtensionSecret, reactivateBundleEntries, hasExtensionEntry, markBundleRemoved, ensureWriteKey } from "./store.js"; +import { saveExtensionSecret, reactivateBundleEntries, hasExtensionEntry, markBundleRemoved, ensureWriteKey, listEntries, openEntrySecret } from "./store.js"; +import { loadKeychainKey } from "./key.js"; +import { unexportedCount } from "./reminder.js"; +import { readSetting, writeSetting } from "../dashboard/settings/registry.js"; import { vaultwardenStatus, saveToVault, VAULT_REASONS } from "./vault-save.js"; let _override = null; @@ -20,6 +23,8 @@ export function _setKeychainDepsForTest(deps) { _override = deps || null; } function deps() { return { openDb: () => createDbClient(), + // The key to READ with; null when missing or damaged (restore is best-effort). + readKey: () => { try { return loadKeychainKey(); } catch { return null; } }, // The key to save with (created on first use; refuses a damaged key while entries exist). writeKey: (db) => ensureWriteKey(db, { onNewKey: async ({ orphaned }) => { @@ -119,6 +124,11 @@ export async function recordKeychainForInstall({ bundleId, manifest, env, minted try { await saveOne(key, s); } catch (err) { log(`Could not save ${s.k} to Crow keychain (${err?.code || err?.name || "error"}); the install continues`); } } if (out.saved) log(`Saved ${out.saved} password(s) to Crow keychain (Settings → Passwords)`); + if (mintedList.length && out.saved) { + try { + if (key && (await unexportedCount(db, key.id)) > 0) log("This key exists only on this machine: export your keychain (Settings → Passwords → Export) so it can be recovered if the machine is lost."); + } catch { /* the reminder never blocks an install */ } + } if (keychainReq?.vault && saved.length) { const st = d.vault.status(); @@ -153,6 +163,96 @@ export async function recordKeychainForInstall({ bundleId, manifest, env, minted return out; } +/** + * For planGeneratedEnv's keychain rung: `values` = plaintext of this bundle's keychain:true + * GENERATED keys that a readable entry holds (any status: an uninstall marks entries + * extension_removed and keeps them); `unreadable` = such keys whose entry exists but is + * sealed with a keychain key this machine does not have (or has no key file at all). This + * is how a rebuilt machine gets its keys back: restore crow.db, import the keychain Export, + * reinstall. Read-only and best-effort: a failure returns nothing and the install mints as + * before. Values are never logged and never synced (crow_keychain is LOCAL_ONLY). + */ +export async function keychainRestoreValues(bundleId, manifest) { + const keys = keychainGeneratedKeys(manifest); + const out = { values: {}, unreadable: [], failed: false }; + if (keys.length === 0) return out; + let db; + try { + const key = deps().readKey(); + db = deps().openDb(); + for (const e of await listEntries(db, { keyId: key?.id ?? null })) { + if (e.kind !== "extension" || e.bundle_id !== bundleId || !keys.includes(e.env_key)) continue; + if (!e.readable || !key) { out.unreadable.push(e.env_key); continue; } + // A damaged entry counts as unreadable (it is never minted over), not as "none". + try { + const plain = await openEntrySecret(db, key, e.id); + if (typeof plain === "string" && plain !== "") out.values[e.env_key] = plain; + else out.unreadable.push(e.env_key); + } catch { out.unreadable.push(e.env_key); } + } + } catch { + // FAIL CLOSED: the keychain could not be read, so the caller must not mint a + // keychain:true key (a mint would replace an entry this read could not see). + return { values: {}, unreadable: [], failed: true }; + } finally { + try { db?.close(); } catch {} + } + return out; +} + +/** + * Boot backfill (recovery keys, queue item 30): an installed extension whose manifest now + * saves a generated key to the keychain (campaigns/tax, mastodon/pixelfed/peertube data + * keys) but was installed before that has the key only in its .env / retained copy / MCP + * entry, and the field is no longer shown in Configure. Save each such value once, so it is + * visible in Settings → Passwords and goes into the user's Export. Never overwrites an + * entry (any status, readable or not); skips store_as keys (only a hash is on disk). No + * first view (nothing new was minted). Best-effort; names only in `log`. + * `values` = { envKey: { plain, origin } } gathered by the caller. + */ +export const BACKFILL_SETTING = "keychain_backfilled_keys"; + +export async function backfillKeychainEntries(bundleId, manifest, values, { log = () => {} } = {}) { + const specs = new Map((manifest?.env_vars || []).map((v) => [v.name, v])); + // Never a publicly known value (replace_values_sha256): the next install replaces it. + const todo = keychainGeneratedKeys(manifest).filter((k) => !specs.get(k)?.store_as && values?.[k]?.plain && !isReplacedValue(specs.get(k), values[k].plain)); + if (todo.length === 0) return 0; + let db; + let saved = 0; + try { + db = deps().openDb(); + // Once per bundle+key: a key the user later DELETED in Settings → Passwords is never + // brought back by a later boot (local setting; never synced). + let done = {}; + try { done = JSON.parse((await readSetting(db, BACKFILL_SETTING)) || "{}") || {}; } catch { done = {}; } + const mark = async (ks) => { for (const k of ks) done[`${bundleId}:${k}`] = true; await writeSetting(db, BACKFILL_SETTING, JSON.stringify(done), { scope: "local" }); }; + const missing = []; + const present = []; + for (const k of todo) { + if (done[`${bundleId}:${k}`]) continue; + if (await hasExtensionEntry(db, bundleId, k)) present.push(k); else missing.push(k); + } + if (present.length) await mark(present); + if (missing.length === 0) return 0; + const key = await deps().writeKey(db); + const tEnv = templateEnv(manifest, Object.fromEntries(Object.entries(values).map(([k, v]) => [k, v.plain]))); + for (const k of missing) { + const spec = specs.get(k) || {}; + const label = `${manifest?.name || bundleId} — ${spec.keychain_label || k}`; + const r = await saveExtensionSecret(db, key, { bundleId, envKey: k, label, username: expandKeychainTemplate(spec.keychain_username, tEnv), url: expandKeychainTemplate(spec.keychain_url, tEnv), secret: values[k].plain, origin: values[k].origin === "generated" ? "generated" : "typed", firstView: false }); + await deps().audit(db, "keychain_save", { details: { entry_id: r.id, bundle_id: bundleId, env_key: k, origin: "kept", backfill: true } }); + saved++; + } + await mark(missing); + log(`${bundleId}: saved ${missing.join(", ")} to Crow keychain (kept from the existing install)`); + } catch (err) { + log(`${bundleId}: could not save kept keys to Crow keychain (${err?.code || err?.name || "error"})`); + } finally { + try { db?.close(); } catch {} + } + return saved; +} + export async function markBundleKeychainRemoved(bundleId) { let db; try { diff --git a/servers/gateway/keychain/reminder.js b/servers/gateway/keychain/reminder.js new file mode 100644 index 000000000..66d58fed2 --- /dev/null +++ b/servers/gateway/keychain/reminder.js @@ -0,0 +1,75 @@ +/** + * Keychain Export reminder (recovery keys, queue item 30, decision 2026-10-16). + * + * A key an extension saved to the keychain (Crow generated it, or it was kept from an older + * install) exists only on this machine: the keychain never syncs, and backups never copy + * secrets/. The user's passphrase-encrypted Export is the one off-machine copy. So while an + * extension entry is in no Export, the Nest shows an info issue (health signal + * "keychainExport"), and the install log says so. + * + * "In an Export" = the entry's sealed value was fingerprinted by the last export, or it came + * in through an Import (it is in that file). A re-sealed value (a re-mint, a re-save) has a + * new fingerprint, so it counts again. The fingerprints are a LOCAL setting (not in the sync + * allowlist): another instance's export says nothing about this machine's keys. audit_log is + * not used: it is pruned after 90 days. Manual entries are the user's own and never counted. + */ +import { createHash } from "node:crypto"; +import { readSetting, writeSetting } from "../dashboard/settings/registry.js"; +import { ensureKeychainTable } from "./store.js"; + +export const LAST_EXPORT_SETTING = "keychain_export_fingerprints"; + +const fp = (sealed) => createHash("sha256").update(String(sealed)).digest("hex").slice(0, 32); + +async function readState(db) { + try { + const v = JSON.parse((await readSetting(db, LAST_EXPORT_SETTING)) || "null"); + if (v && typeof v === "object" && v.fp && typeof v.fp === "object") return v; + } catch { /* none or damaged: nothing counts as exported */ } + return { at: null, fp: {} }; +} + +async function rows(db, keyId, { activeOnly = true, ids = null } = {}) { + await ensureKeychainTable(db); + const { rows: r } = await db.execute({ sql: `SELECT id, secret_sealed FROM crow_keychain WHERE kind = 'extension' AND key_id = ?${activeOnly ? " AND status = 'active'" : ""}`, args: [String(keyId || "")] }); + return ids ? r.filter((x) => ids.has(Number(x.id))) : r; +} +const activeRows = (db, keyId, ids = null) => rows(db, keyId, { ids }); + +/** + * Fingerprints of every readable extension entry (any status: the Export carries removed + * extensions' entries too). Take it BEFORE the Export is built, so an entry saved while it + * is sealed counts as not exported (a false reminder, never a false all-clear). + */ +export async function exportSnapshot(db, keyId) { + const fps = {}; + for (const r of await rows(db, keyId, { activeOnly: false })) fps[Number(r.id)] = fp(r.secret_sealed); + return fps; +} + +/** After an Export built from `snapshot` was sealed. */ +export async function recordKeychainExport(db, keyId, snapshot = null, at = new Date()) { + const fps = snapshot || (await exportSnapshot(db, keyId)); + await writeSetting(db, LAST_EXPORT_SETTING, JSON.stringify({ at: at.toISOString(), fp: fps }), { scope: "local" }); +} + +/** After an Import: the imported entries are in the file they came from. */ +export async function recordKeychainImport(db, keyId, ids) { + if (!Array.isArray(ids) || ids.length === 0) return; + const st = await readState(db); + for (const r of await activeRows(db, keyId, new Set(ids.map(Number)))) st.fp[Number(r.id)] = fp(r.secret_sealed); + await writeSetting(db, LAST_EXPORT_SETTING, JSON.stringify(st), { scope: "local" }); +} + +export async function lastKeychainExport(db) { + return (await readState(db)).at; +} + +/** Active, readable extension entries whose current sealed value is in no Export. */ +export async function unexportedCount(db, keyId) { + if (!keyId) return 0; + const st = await readState(db); + let n = 0; + for (const r of await activeRows(db, keyId)) if (st.fp[Number(r.id)] !== fp(r.secret_sealed)) n++; + return n; +} diff --git a/servers/gateway/keychain/store.js b/servers/gateway/keychain/store.js index 2914e73b1..e67197e91 100644 --- a/servers/gateway/keychain/store.js +++ b/servers/gateway/keychain/store.js @@ -199,6 +199,7 @@ export async function importEntries(db, key, entries) { const current = await listEntries(db, { keyId: key.id }); let imported = 0; let skipped = 0; + const ids = []; for (let e of Array.isArray(entries) ? entries : []) { // A non-http(s) url is dropped (not the entry) so an otherwise-good export still imports. if (e && typeof e.url === "string" && e.url && !/^https?:\/\//i.test(e.url)) e = { ...e, url: null }; @@ -209,6 +210,7 @@ export async function importEntries(db, key, entries) { const origin = e.origin === "generated" ? "generated" : "typed"; const { id } = await saveExtensionSecret(db, key, { bundleId: e.bundle_id, envKey: e.env_key, label: e.label, username: e.username, url: e.url, secret: e.secret, origin }); if (e.status === "extension_removed") await db.execute({ sql: "UPDATE crow_keychain SET status = 'extension_removed' WHERE id = ?", args: [id] }); + ids.push(id); imported++; } else { const dup = current.some((c) => c.kind === "manual" && c.readable && c.label === cleanText(e.label) && (c.username || null) === cleanText(e.username) && (c.url || null) === cleanText(e.url, 2048)); @@ -217,5 +219,5 @@ export async function importEntries(db, key, entries) { imported++; } } - return { imported, skipped }; + return { imported, skipped, ids }; } diff --git a/servers/gateway/routes/bundles.js b/servers/gateway/routes/bundles.js index 940b3bd64..0ed8dadf1 100644 --- a/servers/gateway/routes/bundles.js +++ b/servers/gateway/routes/bundles.js @@ -55,14 +55,14 @@ import { import { isModelOrchestrationDisabled, isModelBundleManifest } from "../../shared/model-orchestration.js"; import { syncBundleSkills, removeBundleSkills, readSkillLedger, skillSyncMode } from "../bundle-skill-sync.js"; import { envValueProblem, encodeEnvValue, formatEnvLines, updateEnvText, pathEnvKeys } from "../bundle-env-codec.js"; -import { sanitizeKeychainRequest, recordKeychainForInstall, markBundleKeychainRemoved } from "../keychain/install-hooks.js"; +import { sanitizeKeychainRequest, recordKeychainForInstall, markBundleKeychainRemoved, keychainRestoreValues, backfillKeychainEntries } from "../keychain/install-hooks.js"; import { precreateDirs, runPostInstall, hookEnv, spawnGroup, pullTimeoutMs, resolveComposeProject, classifyProjectOwners } from "../bundle-lifecycle.js"; import { provisionNtfy, deprovisionNtfy, AUTOWIRE_KIND } from "../push/ntfy-provision.js"; import { checkComposeModel, loadComposeModel, usesHostNetwork } from "../compose-guard.js"; import { resolveDataDir } from "../../db.js"; import { readStoredNtfyConfig } from "../push/ntfy-config.js"; import { isPlaceholderValue } from "../bundle-env-placeholders.js"; -import { planGeneratedEnv, retainedEnvPath, stripGeneratedKeys, parseEnvText, writePrivateFile, gatewayExcludedKeys, gatewayGeneratedKeys, retainGeneratedForReinstall, envPatternViolation, breachedValueViolation } from "../bundle-env-secrets.js"; +import { keychainGeneratedKeys, planGeneratedEnv, retainedEnvPath, stripGeneratedKeys, parseEnvText, writePrivateFile, gatewayExcludedKeys, gatewayGeneratedKeys, retainGeneratedForReinstall, envPatternViolation, breachedValueViolation } from "../bundle-env-secrets.js"; /** * Seed an STT/TTS profile from a bundle manifest's {stt,tts}ProfileSeed into the @@ -780,9 +780,11 @@ async function refreshVersionedBundle({ id, appSrc, destDir, runner }) { const topFiles = isDocker ? ["manifest.json", "settings-section.js", "package.json", "package-lock.json"] : ["manifest.json", "package.json", "package-lock.json", "pyproject.toml", "uv.lock", "settings-section.js", "main.py", "run.sh", "config.py"]; + // engine/ + pdf/: tax's calculator, ingest and form filler (its panel imports the + // installed engine/, so a version bump must ship it with the panel). const dirs = isDocker ? ["server", "panel", "skills"] - : ["server", "panel", "skills", "curriculum", "public", "scripts", "templates", "routes", "config", "src"]; + : ["server", "panel", "skills", "curriculum", "public", "scripts", "templates", "routes", "config", "src", "engine", "pdf"]; // Never copied for ANY type: docker-compose.yml, Dockerfile, entrypoint.sh, // .env*, node_modules/, data/ — simply absent from both lists above. The ONE // exception is a docker bundle's explicit opt-in, manifest.refreshFiles @@ -984,6 +986,35 @@ function installBundleSkills(bundleId, destDir, manifest, job) { * defaults to the module's `run`). * @returns {Promise<{repaired: string[], errors: Array<{id, error}>}>} */ +/** + * Boot: save the keychain:true generated keys of extensions installed before their manifest + * saved them to the keychain (install-hooks backfillKeychainEntries). Values come from the + * installed .env, else the retained copy, else the MCP entry; never logged. + */ +export async function backfillBundleKeychain({ log = (m) => console.log(`[keychain] ${m}`) } = {}) { + let n = 0; + const addons = readJsonSafe(MCP_ADDONS_PATH, {}) || {}; + for (const rec of getInstalled()) { + const id = rec?.id; + if (typeof id !== "string" || !/^[a-z0-9][a-z0-9-]*$/.test(id)) continue; + const manifest = getInstalledFirstManifest(id); + const keys = keychainGeneratedKeys(manifest); + if (keys.length === 0) continue; + let installed = {}, retained = {}; + try { installed = parseEnvText(readFileSync(join(BUNDLES_DIR, id, ".env"), "utf8")); } catch { /* none */ } + try { retained = parseEnvText(readFileSync(retainedEnvPath(CROW_HOME, id), "utf8")); } catch { /* none */ } + const entryEnv = addons?.[id]?.env || {}; + const values = {}; + for (const k of keys) { + const plain = installed[k] || retained[k] || entryEnv[k] || ""; + // The retained copy only ever holds values Crow generated; anything else was typed. + if (plain) values[k] = { plain, origin: retained[k] && retained[k] === plain ? "generated" : "typed" }; + } + n += await backfillKeychainEntries(id, manifest, values, { log }); + } + return n; +} + export async function repairInstalledBundleAssets({ appBundles = APP_BUNDLES, run: runner = run, skillMode = skillSyncMode(), skillLedger = null, log = (m) => console.log(m) } = {}) { const repaired = []; const errors = []; @@ -1237,6 +1268,9 @@ function readJsonSafe(path, fallback) { /** Write JSON file (creates parent dirs) */ function writeJsonSafe(path, data) { mkdirSync(dirname(path), { recursive: true }); + // mcp-addons.json carries MCP children's secrets (generated keys reach their MCP entry + // here): always written atomically at mode 600, whatever the umask or the old file's mode. + if (path === MCP_ADDONS_PATH) { writePrivateFile(path, JSON.stringify(data, null, 2)); return; } writeFileSync(path, JSON.stringify(data, null, 2)); } @@ -2509,7 +2543,29 @@ export async function runInstallJob(bundleId, envVars, { job, installedSnapshot, const reqEnv = stripGeneratedKeys(manifest, envVars); let composeTextForEnv = null; try { const cp = join(destDir, "docker-compose.yml"); if (existsSync(cp)) composeTextForEnv = readFileSync(cp, "utf8"); } catch { /* no fallbacks */ } - const plan = planGeneratedEnv(bundleId, manifest, { destDir, crowHome: CROW_HOME, composeText: composeTextForEnv, priorInstall: priorInstallEnv }); + // A rebuilt machine (crow.db restored + keychain Export imported) has no .env and no + // retained copy: its keychain:true generated keys come back from the keychain instead of + // being minted again (recovery keys, queue item 30). Best-effort; values never logged. + const kcRestore = await keychainRestoreValues(bundleId, manifest); + const plan = planGeneratedEnv(bundleId, manifest, { destDir, crowHome: CROW_HOME, composeText: composeTextForEnv, priorInstall: priorInstallEnv, keychain: kcRestore.values }); + // A key the keychain holds but cannot read (sealed with a keychain key this machine does + // not have: crow.db restored, Export not imported yet) is never minted over: the new + // value would replace that entry, and a later Import skips a bundle+key that has a + // readable entry, so the old key (and the data saved with it) would be lost for good. + const mintingKc = Object.keys(plan.minted).filter((k) => manifest.env_vars?.find((v) => v.name === k)?.keychain === true); + const blocked = kcRestore.failed ? mintingKc : mintingKc.filter((k) => kcRestore.unreadable.includes(k)); + if (blocked.length > 0) { + // A prior install's folder (its .env holds other generated secrets) is kept as it is. + if (!priorInstallEnv) rmSync(destDir, { recursive: true, force: true }); + const reason = kcRestore.failed + ? `Crow keychain could not be read, so ${blocked.join(", ")} was not generated (a new value could replace one saved there). Try the install again; nothing was written.` + : `Crow keychain holds ${blocked.join(", ")} for this extension, sealed with a keychain key this machine does not have. Import your keychain Export first (Settings → Passwords → Import), then install again. To start fresh instead, delete the old saved password(s) there (this needs a dashboard password or two-factor sign-in).`; + appendLog(job, reason); + return { ok: false, reason }; + } + if (plan.restored.length > 0) appendLog(job, `Restored ${plan.restored.length} key(s) from Crow keychain (Settings → Passwords): ${plan.restored.join(", ")}`); + if (plan.replaced.length > 0) appendLog(job, `${plan.replaced.join(", ")} was the example value once published in Crow's repository: a new key was generated, and the old one kept in its previous-key slot so data saved with it still opens`); + if (plan.replaceSkipped?.length > 0) appendLog(job, `${plan.replaceSkipped.join(", ")} is still the example value once published in Crow's repository, but its previous-key slot already holds another key, so it was left as it is. Open that data, clear the previous-key slot in Configure, then reinstall.`); const generated = plan.env; const writeEnv = { ...(reqEnv || {}), ...generated }; let installEnv = writeEnv; diff --git a/tests/bundle-env-placeholders.test.js b/tests/bundle-env-placeholders.test.js index 585e3e581..fea09770f 100644 --- a/tests/bundle-env-placeholders.test.js +++ b/tests/bundle-env-placeholders.test.js @@ -242,12 +242,16 @@ test("tax: at start, returns only the previous key opens are re-encrypted to the } finally { db.close?.(); } }); -test("Configure: a tax key change always keeps the key just replaced in _PREVIOUS (and warns when an older one is dropped)", async () => { +test("Configure: a typed key change always keeps the key just replaced in _PREVIOUS (and warns when an older one is dropped)", async () => { const express = (await import("express")).default; const id = "tax"; const dir = join(CROW_HOME, "bundles", id); mkdirSync(dir, { recursive: true }); - copyFileSync(join(REPO, "bundles", id, "manifest.json"), join(dir, "manifest.json")); + // tax's key is generated since 2.1.0 (Configure never takes it). The previous_key mechanism + // still serves any typed key: test it on the tax manifest as a typed field. + const typed = JSON.parse(readFileSync(join(REPO, "bundles", id, "manifest.json"), "utf8")); + for (const v of typed.env_vars) if (v.name === "CROW_TAX_ENCRYPTION_KEY") { delete v.generate; delete v.keychain; delete v.keychain_label; delete v.replace_values_sha256; } + writeFileSync(join(dir, "manifest.json"), JSON.stringify(typed)); writeFileSync(join(dir, ".env"), "CROW_TAX_ENCRYPTION_KEY=old-pass-1\n"); const app = express(); app.use(express.json()); app.use(B.default()); const server = app.listen(0, "127.0.0.1"); await new Promise((r) => server.once("listening", r)); @@ -269,3 +273,18 @@ test("Configure: a tax key change always keeps the key just replaced in _PREVIOU assert.equal(parseEnvText(readFileSync(join(dir, ".env"), "utf8")).CROW_TAX_ENCRYPTION_KEY_PREVIOUS, "newer-pass-3"); } finally { server.close(); } }); + +test("Configure never overwrites tax's generated key (it is not a form field)", async () => { + const express = (await import("express")).default; + const id = "tax"; + const dir = join(CROW_HOME, "bundles", id); + mkdirSync(dir, { recursive: true }); + copyFileSync(join(REPO, "bundles", id, "manifest.json"), join(dir, "manifest.json")); + writeFileSync(join(dir, ".env"), "CROW_TAX_ENCRYPTION_KEY=generated-key-0123456789\n"); + const app = express(); app.use(express.json()); app.use(B.default()); + const server = app.listen(0, "127.0.0.1"); await new Promise((r) => server.once("listening", r)); + try { + await fetch(`http://127.0.0.1:${server.address().port}/bundles/api/env`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ bundle_id: id, env_vars: { CROW_TAX_ENCRYPTION_KEY: "typed-over-it" } }) }); + assert.equal(parseEnvText(readFileSync(join(dir, ".env"), "utf8")).CROW_TAX_ENCRYPTION_KEY, "generated-key-0123456789"); + } finally { server.close(); } +}); diff --git a/tests/bundle-form-friction.test.js b/tests/bundle-form-friction.test.js index 2ab3258ae..600d91738 100644 --- a/tests/bundle-form-friction.test.js +++ b/tests/bundle-form-friction.test.js @@ -6,7 +6,7 @@ * - internal machine secrets (DB/Redis passwords, app signing keys, JWT/OTP/VAPID, shared * runner secrets) carry `generate` — never a typed field * - every survey F1 key is generated, or exempt with a written reason - * - every generated key of a docker bundle is consumed by its compose file + * - every generated key is consumed by its compose file or its MCP server * - every required human field has a description * - optional fields with a default fold under "Advanced" (bundle-env-form.js) * - the fediverse bundles carry no dead per-bundle S3 fields @@ -81,8 +81,6 @@ const INTERNAL_SECRET_PATTERNS = [ * entry needs a reason a reviewer would accept. */ const EXEMPT = { - "campaigns:CROW_CAMPAIGNS_ENCRYPTION_KEY": "data-at-rest key for an MCP-server bundle (no compose); losing it loses the data, so generation waits for the recovery-key decision (survey §5 Q5)", - "tax:CROW_TAX_ENCRYPTION_KEY": "data-at-rest key for an MCP-server bundle (no compose); losing it loses the data, so generation waits for the recovery-key decision (survey §5 Q5)", "crowdsec:CROWDSEC_API_KEY": "minted by `cscli bouncers add` after first boot and read by the MCP/panel from process.env, not by compose — a post-install hook, not pre-generation (survey §1.3)", "frigate:FRIGATE_RTSP_PASSWORD": "the operator's camera RTSP password, substituted into config.yml camera URLs ({FRIGATE_RTSP_PASSWORD}) — human input, not an internal secret", "lemmy:LEMMY_JWT": "an admin login JWT obtained from the running app after the admin registers (survey F2), not a value Crow can invent", @@ -140,13 +138,14 @@ test("the internal-secret patterns do not swallow human credentials", () => { } }); -test("every generated key of a docker bundle is consumed by its compose file", () => { +test("every generated key is consumed: by its compose file, or by its MCP server (server.envKeys)", () => { for (const { dir, m } of MANIFESTS) { const keys = S.generatedEnvKeys(m); if (keys.length === 0) continue; const text = composeText(dir); - assert.ok(text, `${dir}: generated keys only make sense for a compose bundle`); - const consumed = B.composeConsumedKeys(text); + // An MCP-server bundle (campaigns, tax: their data keys) has no compose file: its MCP + // entry is the only consumer, so every generated key must be in server.envKeys. + const consumed = text ? B.composeConsumedKeys(text) : { keys: new Set() }; // Consumed = interpolated by the compose file, or listed in server.envKeys (the // installer puts those into this instance's MCP entry). Explicit interpolation only: // an env_file .env would make `consumed.all` pass vacuously. diff --git a/tests/bundle-generated-secrets.test.js b/tests/bundle-generated-secrets.test.js index 958677660..d38fc3e1c 100644 --- a/tests/bundle-generated-secrets.test.js +++ b/tests/bundle-generated-secrets.test.js @@ -59,6 +59,15 @@ const EXPECT = [ { id: "maker-lab-advanced", key: "MLA_ADMIN_PASSWORD", propagate: false, username: "${MLA_ADMIN_USER}" }, { id: "actual-budget", key: "ACTUAL_SERVER_PASSWORD", propagate: false }, { id: "lemmy", key: "LEMMY_ADMIN_PASSWORD", propagate: false, username: "admin_pending" }, + { id: "mastodon", key: "MASTODON_SECRET_KEY_BASE", propagate: false }, + { id: "mastodon", key: "MASTODON_OTP_SECRET", propagate: false }, + { id: "mastodon", key: "MASTODON_AR_ENCRYPTION_DETERMINISTIC_KEY", propagate: false }, + { id: "mastodon", key: "MASTODON_AR_ENCRYPTION_KEY_DERIVATION_SALT", propagate: false }, + { id: "mastodon", key: "MASTODON_AR_ENCRYPTION_PRIMARY_KEY", propagate: false }, + { id: "peertube", key: "PEERTUBE_SECRET", propagate: false }, + { id: "mastodon", key: "MASTODON_DB_PASSWORD", propagate: false }, + { id: "pixelfed", key: "PIXELFED_DB_PASSWORD", propagate: false }, + { id: "peertube", key: "PEERTUBE_DB_PASSWORD", propagate: false }, ]; test("each key is generated, keychain-saved, and propagated only where an MCP child reads it", () => { @@ -307,3 +316,13 @@ test("actual-budget: a stale typed password never becomes the new server's passw const server = readFileSync(join(REPO, "bundles/actual-budget/server/server.js"), "utf8"); assert.match(server, /ensureBootstrapped\(ACTUAL_URL, process\.env\.ACTUAL_SERVER_PASSWORD/, "only the generated password bootstraps"); }); + +test("recovery keys (c): the Pixelfed APP_KEY (laravel_key) is saved to the keychain too, and stays out of the gateway .env", () => { + const v = specOf("pixelfed", "PIXELFED_APP_KEY"); + assert.equal(v.generate, "laravel_key"); + assert.equal(v.keychain, true); + assert.ok(v.keychain_label); + assert.equal(S.gatewayGeneratedKeys(manifestOf("pixelfed")).includes("PIXELFED_APP_KEY"), false); + const r = validateManifest(manifestOf("pixelfed"), join(REPO, "bundles", "pixelfed")); + assert.deepEqual(r.errors || [], []); +}); diff --git a/tests/campaigns-tax-generated-keys.test.js b/tests/campaigns-tax-generated-keys.test.js new file mode 100644 index 000000000..571aa4f11 --- /dev/null +++ b/tests/campaigns-tax-generated-keys.test.js @@ -0,0 +1,198 @@ +/** + * Campaigns + tax get generated data keys (recovery keys, queue item 30, decided 2026-10-16): + * generated at install, saved to the keychain, reaching the MCP child through its + * mcp-addons entry and the dashboard panels in the same order. Tax uploads are stored + * encrypted; an install still on the example key published in the repo moves to a new key. + * Scratch CROW_HOME / CROW_DATA_DIR; no docker, no network. + */ +import { test, after } from "node:test"; +import assert from "node:assert/strict"; +import express from "express"; +import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, readdirSync, rmSync, existsSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, dirname } from "node:path"; +import { execFileSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; + +process.env.CROW_HOME = mkdtempSync(join(tmpdir(), "crow-ctkeys-home-")); +process.env.CROW_DATA_DIR = mkdtempSync(join(tmpdir(), "crow-ctkeys-data-")); +process.env.CROW_DISABLE_HEALTH_MONITOR = "1"; +process.env.CROW_DISABLE_INSTANCE_SYNC = "1"; +process.env.CROW_DISABLE_NOSTR = "1"; +delete process.env.CROW_TAX_ENCRYPTION_KEY; +delete process.env.CROW_CAMPAIGNS_ENCRYPTION_KEY; +execFileSync(process.execPath, ["scripts/init-db.js"], { env: process.env, stdio: "pipe" }); +const HOME = process.env.CROW_HOME; +after(() => { for (const d of [HOME, process.env.CROW_DATA_DIR]) rmSync(d, { recursive: true, force: true }); }); + +const REPO = fileURLToPath(new URL("..", import.meta.url)); +const manifestOf = (id) => JSON.parse(readFileSync(join(REPO, "bundles", id, "manifest.json"), "utf8")); +const specOf = (id, k) => manifestOf(id).env_vars.find((v) => v.name === k); +const S = await import("../servers/gateway/bundle-env-secrets.js"); +const { validateManifest } = await import("../scripts/lib/bundle-contract.mjs"); +const D = await import("../bundles/tax/server/documents-crypto.js"); +const EXAMPLE = "change-this-to-a-strong-passphrase"; + +test("both data keys are generated, saved to the keychain, and reach only their own MCP entry", () => { + for (const [id, k] of [["campaigns", "CROW_CAMPAIGNS_ENCRYPTION_KEY"], ["tax", "CROW_TAX_ENCRYPTION_KEY"]]) { + const v = specOf(id, k); + assert.equal(v.generate, "secret", `${id} generate`); + assert.equal(v.keychain, true, `${id} keychain`); + assert.equal(v.keychain_label, "data encryption key"); + assert.ok(manifestOf(id).server.envKeys.includes(k), `${id}: in server.envKeys, so the MCP entry gets it`); + assert.equal(S.gatewayGeneratedKeys(manifestOf(id)).includes(k), false, `${id}: never the shared gateway .env`); + assert.deepEqual(validateManifest(manifestOf(id), join(REPO, "bundles", id)).errors || [], [], `${id} manifest contract`); + } + assert.doesNotMatch(readFileSync(join(REPO, "bundles", "tax", ".env.example"), "utf8"), /^CROW_TAX_ENCRYPTION_KEY=/m); +}); + +test("the tax manifest says honestly what is and is not encrypted", () => { + const notes = manifestOf("tax").notes; + assert.match(notes, /returns and uploaded documents are encrypted at rest/); + assert.match(notes, /stored unencrypted/); + assert.doesNotMatch(notes, /PII is encrypted at rest/); +}); + +test("an install still on the example key published in the repo gets a new key; the old one moves to the previous slot", () => { + const m = manifestOf("tax"); + const destDir = mkdtempSync(join(tmpdir(), "crow-ctkeys-dest-")); + writeFileSync(join(destDir, ".env"), `CROW_TAX_ENCRYPTION_KEY=${EXAMPLE}\n`); + const plan = S.planGeneratedEnv("tax", m, { destDir, crowHome: HOME }); + assert.notEqual(plan.env.CROW_TAX_ENCRYPTION_KEY, EXAMPLE); + assert.ok(plan.minted.CROW_TAX_ENCRYPTION_KEY, "the new key is minted, so it goes to the keychain"); + assert.equal(plan.env.CROW_TAX_ENCRYPTION_KEY_PREVIOUS, EXAMPLE, "returns saved with it still open and move to the new key"); + assert.deepEqual(plan.replaced, ["CROW_TAX_ENCRYPTION_KEY"]); + // A previous slot already in use is never overwritten (it may hold the only copy of an older key). + writeFileSync(join(destDir, ".env"), `CROW_TAX_ENCRYPTION_KEY=${EXAMPLE}\nCROW_TAX_ENCRYPTION_KEY_PREVIOUS=older-key-1234567890\n`); + const kept = S.planGeneratedEnv("tax", m, { destDir, crowHome: HOME }); + assert.equal(kept.env.CROW_TAX_ENCRYPTION_KEY, EXAMPLE); + assert.equal(kept.env.CROW_TAX_ENCRYPTION_KEY_PREVIOUS, undefined); + assert.deepEqual(kept.replaceSkipped, ["CROW_TAX_ENCRYPTION_KEY"], "the install log says why it was left"); + // A typed key of the user's own is kept as it is. + writeFileSync(join(destDir, ".env"), "CROW_TAX_ENCRYPTION_KEY=my-own-key-0987654321\n"); + assert.equal(S.planGeneratedEnv("tax", m, { destDir, crowHome: HOME }).env.CROW_TAX_ENCRYPTION_KEY, "my-own-key-0987654321"); + rmSync(destDir, { recursive: true, force: true }); +}); + +test("replace_values_sha256 is validated: hex digests, and it needs generate + previous_key", () => { + const base = { id: "x", name: "X", version: "1.0.0", type: "mcp-server", category: "misc", description: "d", server: { command: "node", args: ["s.js"] } }; + const bad = validateManifest({ ...base, env_vars: [{ name: "K", secret: true, replace_values_sha256: ["nothex"] }] }, "/nonexistent").errors.join("\n"); + assert.match(bad, /replace_values_sha256 must be/); + const lone = validateManifest({ ...base, env_vars: [{ name: "K", secret: true, generate: "secret", replace_values_sha256: ["a".repeat(64)] }] }, "/nonexistent").errors.join("\n"); + assert.match(lone, /needs generate and previous_key/); +}); + +function writeAddons(obj) { + writeFileSync(join(HOME, "mcp-addons.json"), JSON.stringify(obj), { mode: 0o600 }); +} + +test("the campaigns panel reads its key in the MCP child's order: the instance's MCP entry first, then the gateway env", async () => { + const { campaignsKey } = await import("../bundles/campaigns/server/key.js"); + writeAddons({ campaigns: { env: { CROW_CAMPAIGNS_ENCRYPTION_KEY: "from-entry" } } }); + assert.equal(campaignsKey({ CROW_HOME: HOME }), "from-entry"); + assert.equal(campaignsKey({ CROW_HOME: HOME, CROW_CAMPAIGNS_ENCRYPTION_KEY: "from-gateway-env" }), "from-entry", "the entry wins, as in proxy.js"); + writeAddons({}); + assert.equal(campaignsKey({ CROW_HOME: HOME, CROW_CAMPAIGNS_ENCRYPTION_KEY: "from-gateway-env" }), "from-gateway-env"); +}); + +// --- the tax panel, mounted as the gateway mounts it --- +const TAX_KEY = "tax-key-" + "t".repeat(40); +const PDF = Buffer.from("%PDF-1.4\n% fake 1099 with SSN 000-00-0000\n%%EOF\n"); +const { createDbClient } = await import("../servers/db.js"); + +async function mountTax() { + const { default: taxRouter, taxKeys } = await import("../bundles/tax/panel/routes.js"); + const app = express(); + app.use((req, res, next) => { res.redirectAfterPost = (u) => res.redirect(303, u); next(); }); + app.use(taxRouter((req, res, next) => next())); + const server = await new Promise((r) => { const s = app.listen(0, "127.0.0.1", () => r(s)); }); + after(() => server.close()); + return { base: `http://127.0.0.1:${server.address().port}`, taxKeys }; +} + +test("tax panel: an upload is stored ONLY encrypted (key from the MCP entry, none in the gateway env); delete removes it", async () => { + writeAddons({ tax: { env: { CROW_TAX_ENCRYPTION_KEY: TAX_KEY } } }); + const docs = join(HOME, "tax-documents"); + // A document stored before this change: plain, in the documents folder. + mkdirSync(docs, { recursive: true }); + const old = join(docs, "100-old-w2.pdf"); + writeFileSync(old, PDF); + const db0 = createDbClient(); + await db0.execute(`CREATE TABLE IF NOT EXISTS tax_documents (id INTEGER PRIMARY KEY AUTOINCREMENT, filename TEXT NOT NULL, doc_type TEXT NOT NULL, owner TEXT DEFAULT 'taxpayer', file_path TEXT NOT NULL, status TEXT DEFAULT 'uploaded', return_id TEXT, extracted_data TEXT, confidence TEXT, warnings TEXT, uploaded_at TEXT DEFAULT (datetime('now')), ingested_at TEXT)`); + await db0.execute({ sql: "INSERT INTO tax_documents (filename, doc_type, file_path) VALUES ('old-w2.pdf', 'w2', ?)", args: [old] }); + db0.close(); + + const { base, taxKeys } = await mountTax(); + assert.equal(taxKeys().key, TAX_KEY); + + // The panel's start-up pass encrypts the old document (verified, then the plain file goes). + for (let i = 0; i < 100 && existsSync(old); i++) await new Promise((r) => setTimeout(r, 50)); + assert.equal(existsSync(old), false, "the plain document was encrypted at start-up"); + + const fd = new FormData(); + fd.append("doc_type", "1099-int"); + fd.append("document", new Blob([PDF], { type: "application/pdf" }), "my 1099.pdf"); + const res = await fetch(`${base}/api/tax/upload`, { method: "POST", body: fd }); + assert.equal(res.status, 201, await res.clone().text()); + const body = await res.json(); + const stored = readdirSync(docs).filter((f) => !f.startsWith(".")); + assert.equal(stored.length, 2); + for (const f of stored) { + assert.match(f, /\.enc$/); + const buf = readFileSync(join(docs, f)); + assert.equal(D.isEncryptedDocument(buf), true); + assert.equal(buf.includes(Buffer.from("000-00-0000")), false, "no plaintext on disk"); + assert.ok((await D.decryptDocument(buf, [TAX_KEY])).plain.equals(PDF)); + } + + const del = await fetch(`${base}/api/tax/documents/${body.id}/delete`, { method: "POST", headers: { accept: "application/json" } }); + assert.equal(del.status, 200); + assert.equal(readdirSync(docs).filter((f) => !f.startsWith(".")).length, 1); +}); + +test("tax panel: delete never removes a file outside the documents folder, whatever the row says", async () => { + const { base } = await mountTax(); + const outside = join(HOME, "not-a-tax-document.txt"); + writeFileSync(outside, "keep me"); + const db = createDbClient(); + await db.execute({ sql: "INSERT INTO tax_documents (filename, doc_type, file_path) VALUES ('x', 'w2', ?)", args: [outside] }); + const id = Number((await db.execute("SELECT max(id) AS id FROM tax_documents")).rows[0].id); + db.close(); + await fetch(`${base}/api/tax/documents/${id}/delete`, { method: "POST", headers: { accept: "application/json" } }); + assert.equal(readFileSync(outside, "utf8"), "keep me"); +}); + +test("tax panel: no key yet → an upload is refused, nothing is written", async () => { + writeAddons({}); + const { base } = await mountTax(); + const count = () => readdirSync(join(HOME, "tax-documents")).filter((f) => !f.startsWith(".")).length; + const before = count(); + const fd = new FormData(); + fd.append("doc_type", "w2"); + fd.append("document", new Blob([PDF], { type: "application/pdf" }), "w2.pdf"); + const res = await fetch(`${base}/api/tax/upload`, { method: "POST", body: fd }); + assert.equal(res.status, 409); + assert.equal(count(), before); +}); + +test("the MCP ingest tool opens a stored encrypted document with the tax key", async () => { + const { ingestDocument } = await import("../bundles/tax/engine/ingest/index.js"); + const p = join(HOME, "tax-documents", "x.pdf.0123abcd.enc"); + D.writePrivateAtomic(p, await D.encryptDocument(PDF, TAX_KEY)); + // The fake PDF has no fields, so extraction fails — but only AFTER it was decrypted. + await assert.rejects(ingestDocument(p, "w2", { keys: [TAX_KEY] }), /All extraction methods failed/); + await assert.rejects(ingestDocument(p, "w2", { keys: ["wrong-" + "w".repeat(40)] }), /different tax key/); + // The path comes from the AI: nothing but a bounded regular .pdf/.enc file is ever read. + await assert.rejects(ingestDocument("/dev/zero", "w2"), /Only PDF files/); + const { symlinkSync } = await import("node:fs"); + const dev = join(HOME, "zero.pdf"); + symlinkSync("/dev/zero", dev); + await assert.rejects(ingestDocument(dev, "w2"), /Not a regular file/); +}); + +test("a version bump ships tax's engine/ and pdf/ to the installed copy (the panel imports the installed engine)", () => { + const src = readFileSync(join(REPO, "servers", "gateway", "routes", "bundles.js"), "utf8"); + const nonDocker = src.match(/: \["server", "panel", "skills", "curriculum"[^\]]*\]/)[0]; + assert.match(nonDocker, /"engine"/); + assert.match(nonDocker, /"pdf"/); +}); diff --git a/tests/keychain-export-reminder.test.js b/tests/keychain-export-reminder.test.js new file mode 100644 index 000000000..39daea219 --- /dev/null +++ b/tests/keychain-export-reminder.test.js @@ -0,0 +1,118 @@ +/** + * Export reminder (recovery keys, queue item 30): while a key Crow GENERATED is in no keychain + * Export, the Nest shows an info issue (strip only, never a push). It goes away when the user + * exports, and comes back for a key saved after that export. The last-export time is a local + * setting: it never syncs to a paired instance. Real init-db in a scratch home. + */ +import { test, after } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { execFileSync } from "node:child_process"; + +process.env.CROW_HOME = mkdtempSync(join(tmpdir(), "crow-kcrem-home-")); +process.env.CROW_DATA_DIR = mkdtempSync(join(tmpdir(), "crow-kcrem-data-")); +process.env.CROW_DISABLE_HEALTH_MONITOR = "1"; +process.env.CROW_DISABLE_INSTANCE_SYNC = "1"; +process.env.CROW_DISABLE_NOSTR = "1"; +execFileSync(process.execPath, ["scripts/init-db.js"], { env: process.env, stdio: "pipe" }); + +const K = await import("../servers/gateway/keychain/store.js"); +const { createKeychainKey } = await import("../servers/gateway/keychain/key.js"); +const { createDbClient } = await import("../servers/db.js"); +const HS = await import("../servers/gateway/dashboard/panels/nest/health-signals.js"); +const { isSyncable } = await import("../servers/gateway/dashboard/settings/sync-allowlist.js"); +const R = await import("../servers/gateway/keychain/reminder.js"); +after(() => { for (const d of [process.env.CROW_HOME, process.env.CROW_DATA_DIR]) rmSync(d, { recursive: true, force: true }); }); + +const key = createKeychainKey(); +const db = createDbClient(); +after(() => db.close()); + +async function signal() { + HS.invalidateHealthCache(); + const r = await HS.collectHealthSignals(db, { lang: "en" }); + return { issue: r.issues.find((i) => i.id === "keychainExport") || null, card: r.details.find((d) => d.id === "keychainExport") || null }; +} + +test("no generated keys → no reminder at all (a manual entry never counts)", async () => { + await K.addManualSecret(db, key, { label: "my bank", secret: "hunter2-hunter2" }); + const s = await signal(); + assert.equal(s.issue, null); + assert.equal(s.card, null); +}); + +test("an extension key that was never exported → an info issue that links to Settings → Passwords", async () => { + await K.saveExtensionSecret(db, key, { bundleId: "tax", envKey: "CROW_TAX_ENCRYPTION_KEY", label: "Tax — data encryption key", secret: "k1-" + "x".repeat(40), origin: "generated" }); + const s = await signal(); + assert.ok(s.issue, "the reminder is shown"); + assert.equal(s.issue.severity, "info", "info only: shown on the Nest strip, never pushed"); + assert.match(s.issue.label, /^1 .*not in a keychain Export/i); + assert.equal(s.issue.actionHref, "/dashboard/settings?section=passwords"); +}); + +test("a key KEPT from an older install (origin typed) counts too", async () => { + await K.saveExtensionSecret(db, key, { bundleId: "mastodon", envKey: "MASTODON_OTP_SECRET", label: "Mastodon — two-factor secret", secret: "kept-" + "z".repeat(40), origin: "typed" }); + assert.equal(await R.unexportedCount(db, key.id), 2); +}); + +test("after an export the reminder is gone; a key saved or re-saved after the export brings it back", async () => { + await R.recordKeychainExport(db, key.id); + assert.equal((await signal()).issue, null, "everything is in an Export"); + await K.saveExtensionSecret(db, key, { bundleId: "campaigns", envKey: "CROW_CAMPAIGNS_ENCRYPTION_KEY", label: "Campaigns — data encryption key", secret: "k2-" + "y".repeat(40), origin: "generated" }); + assert.match((await signal()).issue.label, /^1 /, "a new key is in no Export"); + await R.recordKeychainExport(db, key.id); + // A re-mint over an existing row keeps its created_at but is a new secret: it counts again. + await K.saveExtensionSecret(db, key, { bundleId: "tax", envKey: "CROW_TAX_ENCRYPTION_KEY", label: "Tax — data encryption key", secret: "k3-" + "w".repeat(40), origin: "generated" }); + assert.equal(await R.unexportedCount(db, key.id), 1); +}); + +test("entries that came in through an Import are in that file: they do not count", async () => { + await R.recordKeychainExport(db, key.id); + const { ids } = await K.importEntries(db, key, [{ kind: "extension", bundle_id: "pixelfed", env_key: "PIXELFED_APP_KEY", label: "Pixelfed — data encryption key", secret: "base64:" + "q".repeat(44), origin: "generated" }]); + assert.equal(ids.length, 1); + assert.equal(await R.unexportedCount(db, key.id), 1, "before recording the import it would count"); + await R.recordKeychainImport(db, key.id, ids); + assert.equal(await R.unexportedCount(db, key.id), 0); +}); + +test("the export fingerprints are local: not in the sync allowlist", () => { + assert.equal(isSyncable(R.LAST_EXPORT_SETTING), false); +}); + +test("an entry of a removed extension is in the Export too: reinstalling it does not raise the reminder", async () => { + await K.saveExtensionSecret(db, key, { bundleId: "peertube", envKey: "PEERTUBE_SECRET", label: "PeerTube — server secret", secret: "pt-" + "p".repeat(40), origin: "generated" }); + await K.markBundleRemoved(db, "peertube"); + await R.recordKeychainExport(db, key.id); + await K.reactivateBundleEntries(db, "peertube", ["PEERTUBE_SECRET"]); + assert.equal(await R.unexportedCount(db, key.id), 0); +}); + +test("the real export + import routes record the reminder state (HTTP, injected re-auth gate)", async () => { + const express = (await import("express")).default; + const { keychainApiRouter } = await import("../servers/gateway/keychain/api.js"); + const gate = { method: async () => "password", isGranted: () => true, expiresAt: () => null, verify: async () => ({ ok: true }) }; + const app = express(); + app.use(express.json({ limit: "5mb" })); + app.use((req, res, next) => { req.dashboardSession = "test-session"; next(); }); + app.use(keychainApiRouter({ gate, openDb: () => createDbClient() })); + const server = await new Promise((r) => { const s = app.listen(0, "127.0.0.1", () => r(s)); }); + try { + const url = (p) => `http://127.0.0.1:${server.address().port}/dashboard/keychain/api${p}`; + await K.saveExtensionSecret(db, key, { bundleId: "romm", envKey: "ROMM_AUTH_SECRET_KEY", label: "RomM — secret", secret: "rm-" + "r".repeat(40), origin: "generated" }); + assert.ok((await R.unexportedCount(db, key.id)) > 0); + const ex = await fetch(url("/export"), { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ passphrase: "a long export passphrase" }) }); + assert.equal(ex.status, 200); + const file = await ex.json(); + assert.equal(await R.unexportedCount(db, key.id), 0, "everything readable is in that Export"); + HS.invalidateHealthCache(); + assert.equal((await signal()).issue, null); + // Import into a fresh keychain row set: what comes in is in the file, so it never counts. + await db.execute("DELETE FROM crow_keychain WHERE bundle_id = 'romm'"); + const im = await fetch(url("/import"), { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ file, passphrase: "a long export passphrase" }) }); + assert.equal(im.status, 200); + assert.equal((await im.json()).imported, 1); + assert.equal(await R.unexportedCount(db, key.id), 0); + } finally { server.close(); } +}); diff --git a/tests/keychain-restore.test.js b/tests/keychain-restore.test.js new file mode 100644 index 000000000..ac67ddd86 --- /dev/null +++ b/tests/keychain-restore.test.js @@ -0,0 +1,250 @@ +/** + * A reinstall restores generated keys from the Crow keychain (queue item 30, recovery keys + * option 1, decision 2026-10-16). Before this, a rebuilt machine (crow.db restored, keychain + * Export imported, extension reinstalled) MINTED a new key and then overwrote the imported + * keychain entry with it: data saved under the old key became unreadable. Real init-db in a + * scratch home; no docker. + */ +import { test, after } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, rmSync, existsSync, statSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { execFileSync } from "node:child_process"; + +process.env.CROW_HOME = mkdtempSync(join(tmpdir(), "crow-kcrestore-home-")); +process.env.CROW_DATA_DIR = mkdtempSync(join(tmpdir(), "crow-kcrestore-data-")); +process.env.CROW_AUTO_UPDATE = "0"; +process.env.CROW_DISABLE_HEALTH_MONITOR = "1"; +process.env.CROW_DISABLE_INSTANCE_SYNC = "1"; +process.env.CROW_DISABLE_NOSTR = "1"; +execFileSync(process.execPath, ["scripts/init-db.js"], { env: process.env, stdio: "pipe" }); + +const CROW_HOME = process.env.CROW_HOME; +const B = await import("../servers/gateway/routes/bundles.js"); +const H = await import("../servers/gateway/keychain/install-hooks.js"); +const K = await import("../servers/gateway/keychain/store.js"); +const P = await import("../servers/gateway/keychain/argon2-phc.js"); +const S = await import("../servers/gateway/bundle-env-secrets.js"); +const { parseEnvText } = await import("../servers/gateway/bundle-env-codec.js"); +const { createDbClient } = await import("../servers/db.js"); +const { loadKeychainKey } = await import("../servers/gateway/keychain/key.js"); + +const FIXTURES = mkdtempSync(join(tmpdir(), "crow-kcrestore-app-")); +B._setAppBundlesForTest(FIXTURES); +B._setAppEnvPathForTest(join(mkdtempSync(join(tmpdir(), "crow-kcrestore-gw-")), ".env")); +after(() => { + B._setAppEnvPathForTest(null); + for (const d of [CROW_HOME, process.env.CROW_DATA_DIR, FIXTURES]) rmSync(d, { recursive: true, force: true }); +}); + +const MANIFEST = (id) => ({ + id, name: "Demo Data", description: "d", type: "bundle", category: "infrastructure", version: "0.1.0", + env_vars: [ + { name: "DEMO_DATA_KEY", secret: true, generate: "secret", keychain: true, keychain_label: "data encryption key" }, + { name: "DEMO_ADMIN_TOKEN", secret: true, generate: "secret", keychain: true, store_as: "argon2id", keychain_label: "admin token" }, + { name: "DEMO_DB_PASSWORD", secret: true, generate: "secret" }, + ], +}); +function fixture(id) { + mkdirSync(join(FIXTURES, id), { recursive: true }); + writeFileSync(join(FIXTURES, id, "manifest.json"), JSON.stringify(MANIFEST(id))); + return MANIFEST(id); +} +const db = () => createDbClient(); +async function install(id, manifest) { + const job = B._createJobForTest(id, "install"); + const out = await B.runInstallJob(id, {}, { job, installedSnapshot: [], consentVerified: false, manifest }); + assert.equal(out.ok, true, out.reason); + return { job, env: parseEnvText(readFileSync(join(CROW_HOME, "bundles", id, ".env"), "utf8")) }; +} +// "This machine is lost": the installed copy and the retained copy are gone, the keychain rows +// (restored crow.db + imported Export) remain. +function loseFiles(id) { + rmSync(join(CROW_HOME, "bundles", id), { recursive: true, force: true }); + rmSync(S.retainedEnvPath(CROW_HOME, id), { force: true }); +} +async function entry(id, envKey) { + const d = db(); + try { + const e = (await K.listEntries(d, { keyId: loadKeychainKey().id })).find((x) => x.bundle_id === id && x.env_key === envKey); + return e ? { ...e, plain: e.readable ? await K.openEntrySecret(d, loadKeychainKey(), e.id) : null } : null; + } finally { d.close(); } +} + +test("planGeneratedEnv: a keychain value is used when .env, the retained copy and fallbacks have none, and it is reused, never minted", () => { + const m = MANIFEST("plan-a"); + const destDir = mkdtempSync(join(tmpdir(), "crow-kcrestore-dest-")); + const plan = S.planGeneratedEnv("plan-a", m, { destDir, crowHome: CROW_HOME, keychain: { DEMO_DATA_KEY: "kept-data-key", DEMO_ADMIN_TOKEN: "kept-token" } }); + assert.equal(plan.env.DEMO_DATA_KEY, "kept-data-key"); + assert.equal(P.verifyArgon2idPhc("kept-token", plan.env.DEMO_ADMIN_TOKEN), true, "a hashed key gets a hash of the restored plaintext"); + assert.deepEqual(Object.keys(plan.minted), [], "nothing restored is minted"); + assert.deepEqual(plan.restored.sort(), ["DEMO_ADMIN_TOKEN", "DEMO_DATA_KEY"]); + assert.ok(plan.env.DEMO_DB_PASSWORD && plan.env.DEMO_DB_PASSWORD.length >= 40, "a non-keychain key is still minted"); + rmSync(destDir, { recursive: true, force: true }); +}); + +test("planGeneratedEnv: the installed .env and the retained copy still win over the keychain (they are what the data uses)", () => { + const m = MANIFEST("plan-b"); + const destDir = mkdtempSync(join(tmpdir(), "crow-kcrestore-dest-")); + writeFileSync(join(destDir, ".env"), "DEMO_DATA_KEY=from-env\n"); + const plan = S.planGeneratedEnv("plan-b", m, { destDir, crowHome: CROW_HOME, keychain: { DEMO_DATA_KEY: "from-keychain" } }); + assert.equal(plan.env.DEMO_DATA_KEY, "from-env"); + assert.equal(plan.restored.includes("DEMO_DATA_KEY"), false); + rmSync(destDir, { recursive: true, force: true }); +}); + +test("keychainRestoreValues: only keychain:true generated keys, only readable entries", async () => { + const id = "kcr-values"; const m = fixture(id); + await install(id, m); + const { values: v, unreadable } = await H.keychainRestoreValues(id, m); + assert.deepEqual(Object.keys(v).sort(), ["DEMO_ADMIN_TOKEN", "DEMO_DATA_KEY"], "DEMO_DB_PASSWORD is never in the keychain, so never looked up"); + assert.deepEqual(unreadable, []); + assert.equal((await H.keychainRestoreValues(id, m)).failed, false); + assert.equal(v.DEMO_DATA_KEY, (await entry(id, "DEMO_DATA_KEY")).plain); + assert.deepEqual(await H.keychainRestoreValues(id, { ...m, env_vars: [] }), { values: {}, unreadable: [], failed: false }, "a manifest with no keychain keys reads nothing"); +}); + +test("machine lost → reinstall restores the SAME data key from the keychain (and does not overwrite the entry)", async () => { + const id = "kcr-e2e"; const m = fixture(id); + const first = await install(id, m); + assert.match(first.job.log.join("\n"), /export your keychain/, "a first generated key reminds the user to export"); + const before = await entry(id, "DEMO_DATA_KEY"); + assert.equal(first.env.DEMO_DATA_KEY, before.plain); + const tokenBefore = (await entry(id, "DEMO_ADMIN_TOKEN")).plain; + + assert.equal(await H.markBundleKeychainRemoved(id), 2, "uninstall marks the entries removed (they are kept)"); + loseFiles(id); + + const { job, env } = await install(id, m); + assert.equal(env.DEMO_DATA_KEY, before.plain, "the data key is the one the data was saved with"); + assert.equal(P.verifyArgon2idPhc(tokenBefore, env.DEMO_ADMIN_TOKEN), true, "the admin token still signs in"); + const after = await entry(id, "DEMO_DATA_KEY"); + assert.equal(after.plain, before.plain, "the keychain entry was not overwritten"); + assert.equal(after.status, "active", "the entry is active again"); + assert.equal(after.first_view_pending, false, "nothing new to show"); + const retained = parseEnvText(readFileSync(S.retainedEnvPath(CROW_HOME, id), "utf8")); + assert.equal(retained.DEMO_DATA_KEY, before.plain, "the restored key is retained for the next reinstall"); + const log = job.log.join("\n"); + assert.match(log, /Restored 2 key\(s\) from Crow keychain/); + assert.ok(!log.includes(before.plain) && !log.includes(tokenBefore), "no value in the job log"); +}); + +test("an entry sealed with another machine's keychain key (Export not imported yet): the install is REFUSED, nothing is minted over it", async () => { + const id = "kcr-unreadable"; const m = fixture(id); + await install(id, m); + const sealedBefore = async () => { const d = db(); try { return (await d.execute({ sql: "SELECT secret_sealed FROM crow_keychain WHERE bundle_id = ? AND env_key = 'DEMO_DATA_KEY'", args: [id] })).rows[0].secret_sealed; } finally { d.close(); } }; + const d = db(); + try { await d.execute({ sql: "UPDATE crow_keychain SET key_id = 'ffffffffffffffff' WHERE bundle_id = ? AND env_key = 'DEMO_DATA_KEY'", args: [id] }); } finally { d.close(); } + const sealed = await sealedBefore(); + loseFiles(id); + const job = B._createJobForTest(id, "install"); + const out = await B.runInstallJob(id, {}, { job, installedSnapshot: [], consentVerified: false, manifest: m }); + assert.equal(out.ok, false); + assert.match(out.reason, /DEMO_DATA_KEY.*Import your keychain Export first/); + assert.equal(await sealedBefore(), sealed, "the old sealed key is untouched, so a later Import can still bring it back"); + assert.equal(existsSync(join(CROW_HOME, "bundles", id)), false, "nothing was written"); + // Starting fresh is the user's explicit choice: delete the old entry, then the install mints. + const d2 = db(); + try { await d2.execute({ sql: "DELETE FROM crow_keychain WHERE bundle_id = ? AND env_key = 'DEMO_DATA_KEY'", args: [id] }); } finally { d2.close(); } + const again = await install(id, m); + assert.ok(again.env.DEMO_DATA_KEY); +}); + +test("a keychain read failure FAILS CLOSED: no keychain:true key is minted (it could replace an entry the read missed)", async () => { + const id = "kcr-fail"; const m = fixture(id); + H._setKeychainDepsForTest({ openDb: () => { throw new Error("db down"); } }); + try { + assert.deepEqual(await H.keychainRestoreValues(id, m), { values: {}, unreadable: [], failed: true }); + const job = B._createJobForTest(id, "install"); + const out = await B.runInstallJob(id, {}, { job, installedSnapshot: [], consentVerified: false, manifest: m }); + assert.equal(out.ok, false); + assert.match(out.reason, /could not be read.*Try the install again/); + } finally { H._setKeychainDepsForTest(null); } +}); + +test("one damaged entry counts as unreadable (refused), never as 'none' (minted over)", async () => { + const id = "kcr-damaged"; const m = fixture(id); + await install(id, m); + const d = db(); + try { await d.execute({ sql: "UPDATE crow_keychain SET secret_sealed = 'not-a-sealed-value' WHERE bundle_id = ? AND env_key = 'DEMO_DATA_KEY'", args: [id] }); } finally { d.close(); } + const r = await H.keychainRestoreValues(id, m); + assert.deepEqual(r.unreadable, ["DEMO_DATA_KEY"]); + assert.ok(r.values.DEMO_ADMIN_TOKEN, "the other entry is still read"); + loseFiles(id); + const job = B._createJobForTest(id, "install"); + const out = await B.runInstallJob(id, {}, { job, installedSnapshot: [], consentVerified: false, manifest: m }); + assert.equal(out.ok, false); + assert.match(out.reason, /DEMO_DATA_KEY/); +}); + +test("crow_keychain stays local-only (never synced to a paired instance)", async () => { + const sync = await import("../servers/sharing/instance-sync.js"); + assert.ok((sync.LOCAL_ONLY_TABLES || []).includes("crow_keychain")); + assert.equal((sync.SYNCED_TABLES || []).includes("crow_keychain"), false); +}); + +test("mcp-addons.json (it carries MCP children's generated keys) is written at mode 600", async () => { + const id = "kcr-mcp"; + mkdirSync(join(FIXTURES, id), { recursive: true }); + const m = { id, name: "Demo MCP", description: "d", type: "mcp-server", category: "productivity", version: "0.1.0", + server: { command: "node", args: ["server/index.js"], envKeys: ["DEMO_MCP_KEY"] }, + env_vars: [{ name: "DEMO_MCP_KEY", secret: true, generate: "secret", keychain: true, keychain_label: "data encryption key" }] }; + writeFileSync(join(FIXTURES, id, "manifest.json"), JSON.stringify(m)); + mkdirSync(join(FIXTURES, id, "server"), { recursive: true }); + writeFileSync(join(FIXTURES, id, "server", "index.js"), ""); + const job = B._createJobForTest(id, "install"); + const out = await B.runInstallJob(id, {}, { job, installedSnapshot: [], consentVerified: false, manifest: m }); + assert.equal(out.ok, true, out.reason); + const path = join(CROW_HOME, "mcp-addons.json"); + assert.equal(statSync(path).mode & 0o777, 0o600); + const entryEnv = JSON.parse(readFileSync(path, "utf8"))[id].env; + assert.equal(entryEnv.DEMO_MCP_KEY, (await entry(id, "DEMO_MCP_KEY")).plain, "the MCP child gets the keychain's plaintext"); +}); + +test("crow.db restored on a machine with no keychain key file at all: the install is refused, not minted over", async () => { + const id = "kcr-nokey"; const m = fixture(id); + await install(id, m); + loseFiles(id); + const { keychainKeyPath } = await import("../servers/gateway/keychain/key.js"); + const { renameSync } = await import("node:fs"); + const kp = keychainKeyPath(); + renameSync(kp, kp + ".away"); + try { + const job = B._createJobForTest(id, "install"); + const out = await B.runInstallJob(id, {}, { job, installedSnapshot: [], consentVerified: false, manifest: m }); + assert.equal(out.ok, false); + assert.match(out.reason, /Import your keychain Export first/); + assert.equal(existsSync(kp), false, "no new keychain key was created by the refused install"); + } finally { renameSync(kp + ".away", kp); } +}); + +test("boot backfill: a key kept only in an older install's .env is saved to the keychain once, never over an existing entry", async () => { + const id = "kcr-backfill"; const m = fixture(id); + // An install from before the manifest saved this key to the keychain: .env only. + mkdirSync(join(CROW_HOME, "bundles", id), { recursive: true }); + writeFileSync(join(CROW_HOME, "bundles", id, "manifest.json"), JSON.stringify(m)); + writeFileSync(join(CROW_HOME, "bundles", id, ".env"), "DEMO_DATA_KEY=typed-long-ago-123456\nDEMO_ADMIN_TOKEN=$argon2id$v=19$m=65540,t=3,p=4$AAAA$BBBB\n"); + const instPath = join(CROW_HOME, "installed.json"); + const prev = existsSync(instPath) ? JSON.parse(readFileSync(instPath, "utf8")) : []; + const list = Array.isArray(prev) ? prev : Object.entries(prev).map(([i, v]) => ({ id: i, ...v })); + writeFileSync(instPath, JSON.stringify([...list.filter((r) => r.id !== id), { id, version: "0.1.0" }])); + const logs = []; + const n = await B.backfillBundleKeychain({ log: (l) => logs.push(l) }); + assert.ok(n >= 1); + const e = await entry(id, "DEMO_DATA_KEY"); + assert.equal(e.plain, "typed-long-ago-123456"); + assert.equal(e.origin, "typed"); + assert.equal(e.first_view_pending, false); + assert.equal(await entry(id, "DEMO_ADMIN_TOKEN"), null, "a store_as key has only a hash on disk: never backfilled"); + assert.ok(!logs.join("\n").includes("typed-long-ago"), "names only in the log"); + // A second boot changes nothing, even if the .env now differs. + writeFileSync(join(CROW_HOME, "bundles", id, ".env"), "DEMO_DATA_KEY=something-else-999999\n"); + await B.backfillBundleKeychain({ log: () => {} }); + assert.equal((await entry(id, "DEMO_DATA_KEY")).plain, "typed-long-ago-123456"); + // The user deletes it in Settings → Passwords: a later boot never brings it back. + const d = db(); + try { await d.execute({ sql: "DELETE FROM crow_keychain WHERE bundle_id = ? AND env_key = 'DEMO_DATA_KEY'", args: [id] }); } finally { d.close(); } + await B.backfillBundleKeychain({ log: () => {} }); + assert.equal(await entry(id, "DEMO_DATA_KEY"), null, "a deleted entry stays deleted"); +}); diff --git a/tests/keychain-store.test.js b/tests/keychain-store.test.js index c265bf920..4148e299e 100644 --- a/tests/keychain-store.test.js +++ b/tests/keychain-store.test.js @@ -173,13 +173,13 @@ test("Export → Import round-trips through a passphrase file; wrong passphrase const db2 = freshDb(); const key2 = KEY.createKeychainKey({ crowHome: home() }); await K.saveExtensionSecret(db2, key, { bundleId: "workspace", envKey: "WORKSPACE_ADMIN_PASSWORD", label: "WS admin", secret: "stale", origin: "typed" }); - const out = await K.importEntries(db2, key2, await X.openExport(file, "correct horse battery")); + const { ids: _ids, ...out } = await K.importEntries(db2, key2, await X.openExport(file, "correct horse battery")); assert.deepEqual(out, { imported: 2, skipped: 0 }, "the unreadable restored row is replaced"); const list = await K.listEntries(db2, { keyId: key2.id }); assert.ok(list.every((e) => e.readable)); const ws = list.find((e) => e.env_key === "WORKSPACE_ADMIN_PASSWORD"); assert.equal(await K.openEntrySecret(db2, key2, ws.id), "ws-pass 1"); - assert.deepEqual(await K.importEntries(db2, key2, await X.openExport(file, "correct horse battery")), { imported: 0, skipped: 2 }, "re-import overwrites nothing"); + assert.deepEqual(await K.importEntries(db2, key2, await X.openExport(file, "correct horse battery")), { imported: 0, skipped: 2, ids: [] }, "re-import overwrites nothing"); }); test("init-db creates crow_keychain with the same columns as the store's lazy ensure (no CHECK constraints)", async () => { @@ -246,7 +246,7 @@ test("import drops a non-http(s) url but still imports the entry", async () => { { kind: "manual", label: "Evil link", username: null, url: "javascript:alert(1)", secret: "s1" }, { kind: "manual", label: "Good link", username: null, url: "HTTPS://ok.example", secret: "s2" }, ]); - assert.deepEqual(out, { imported: 2, skipped: 0 }); + assert.deepEqual(out, { imported: 2, skipped: 0, ids: [] }, "manual entries are not extension ids"); const list = await K.listEntries(db, { keyId: key.id }); assert.equal(list.find((e) => e.label === "Evil link").url, null); assert.equal(list.find((e) => e.label === "Good link").url, "HTTPS://ok.example"); diff --git a/tests/tax-documents-crypto.test.js b/tests/tax-documents-crypto.test.js new file mode 100644 index 000000000..59cd335fb --- /dev/null +++ b/tests/tax-documents-crypto.test.js @@ -0,0 +1,213 @@ +// tests/tax-documents-crypto.test.js — uploaded tax documents encrypted at rest, and the +// safe, reversible migration of files stored before (recovery keys, queue item 30). +import { test, after } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, existsSync, rmSync, readdirSync, statSync, symlinkSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, basename } from "node:path"; +import { createDbClient } from "../servers/db.js"; +import * as D from "../bundles/tax/server/documents-crypto.js"; + +const ROOT = mkdtempSync(join(tmpdir(), "crow-taxdocs-")); +after(() => rmSync(ROOT, { recursive: true, force: true })); +const KEY = "k-" + "a".repeat(41); +const OLD = "o-" + "b".repeat(41); +const PDF = Buffer.from("%PDF-1.4\n% fake W-2 with SSN 000-00-0000\n%%EOF\n"); +const EXAMPLE = "change-this-to-a-strong-passphrase"; + +async function fresh(name) { + const docs = join(ROOT, name, "tax-documents"); + mkdirSync(docs, { recursive: true }); + const db = createDbClient(join(ROOT, name, "t.db")); + after(() => { try { db.close(); } catch {} }); + await db.execute("CREATE TABLE tax_documents (id INTEGER PRIMARY KEY, filename TEXT, file_path TEXT NOT NULL)"); + return { docs, db }; +} +async function addPlain(db, docs, id, bytes = PDF) { + const p = join(docs, `${id}-w2.pdf`); + writeFileSync(p, bytes); + await db.execute({ sql: "INSERT INTO tax_documents (id, filename, file_path) VALUES (?, 'w2.pdf', ?)", args: [id, p] }); + return p; +} +const pathOf = async (db, id) => (await db.execute({ sql: "SELECT file_path FROM tax_documents WHERE id = ?", args: [id] })).rows[0].file_path; +const files = (dir) => readdirSync(dir).filter((f) => !f.startsWith(".")); + +test("round trip; the key id names the sealing key; a wrong key is refused; the previous key still opens", async () => { + const enc = await D.encryptDocument(PDF, KEY); + assert.equal(D.isEncryptedDocument(enc), true); + assert.equal(enc.includes(Buffer.from("000-00-0000")), false, "no plaintext in the stored bytes"); + assert.ok((await D.decryptDocument(enc, [KEY])).plain.equals(PDF)); + assert.equal(await D.sealedWith(enc, [OLD, KEY]), KEY); + // The key id comes from the salted PBKDF2 output: it differs per file for the same key, + // so it is no cheaper offline guess check than the ciphertext and links nothing. + const enc2 = await D.encryptDocument(PDF, KEY); + assert.equal(enc.subarray(D.MAGIC.length, D.MAGIC.length + 8).equals(enc2.subarray(D.MAGIC.length, D.MAGIC.length + 8)), false); + await assert.rejects(D.decryptDocument(enc, ["wrong-" + "c".repeat(40)]), /different tax key/); + assert.ok((await D.decryptDocument(await D.encryptDocument(PDF, OLD), [KEY, OLD])).plain.equals(PDF)); + await assert.rejects(D.encryptDocument(PDF, ""), /not set/); +}); + +test("the example key published in the repo is never used to store a document", async () => { + await assert.rejects(D.encryptDocument(PDF, EXAMPLE), /example value/); + const { docs, db } = await fresh("ex"); + const p = await addPlain(db, docs, 1); + const r = await D.encryptStoredDocuments(db, { key: EXAMPLE, docsDirs: [docs] }); + assert.equal(r.encrypted, 0); + assert.ok(readFileSync(p).equals(PDF), "the plain file is left alone"); +}); + +test("migration: plain → ..enc (0600), row updated, plain file gone; a re-run changes nothing", async () => { + const { docs, db } = await fresh("m1"); + const p = await addPlain(db, docs, 1); + const r = await D.encryptStoredDocuments(db, { key: KEY, docsDirs: [docs] }); + assert.deepEqual([r.encrypted, r.errors], [1, []]); + const np = await pathOf(db, 1); + assert.match(basename(np), /^1-w2\.pdf\.[0-9a-f]{8}\.enc$/); + assert.equal(existsSync(p), false); + assert.equal(statSync(np).mode & 0o777, 0o600); + assert.ok((await D.decryptDocument(readFileSync(np), [KEY])).plain.equals(PDF)); + const again = await D.encryptStoredDocuments(db, { key: KEY, docsDirs: [docs] }); + assert.deepEqual([again.encrypted, again.reencrypted, again.skipped], [0, 0, 1]); + assert.deepEqual(files(docs), [basename(np)], "no temp or lock file is left"); +}); + +test("a crash after the swap but before the unlink leaves a plain sibling: the next run removes it once the .enc is verified", async () => { + const { docs, db } = await fresh("m2"); + const p = await addPlain(db, docs, 1); + const enc = `${p}.0123abcd.enc`; + D.writePrivateAtomic(enc, await D.encryptDocument(PDF, KEY)); + await db.execute({ sql: "UPDATE tax_documents SET file_path = ? WHERE id = 1", args: [enc] }); + const r = await D.encryptStoredDocuments(db, { key: KEY, docsDirs: [docs] }); + assert.equal(r.cleaned, 1); + assert.equal(existsSync(p), false); + assert.deepEqual(files(docs), [basename(enc)]); +}); + +test("a sibling that does NOT match the .enc's contents is never removed", async () => { + const { docs, db } = await fresh("m2b"); + const p = await addPlain(db, docs, 1, Buffer.from("%PDF-1.4 a different file")); + const enc = `${p}.0123abcd.enc`; + D.writePrivateAtomic(enc, await D.encryptDocument(PDF, KEY)); + await db.execute({ sql: "UPDATE tax_documents SET file_path = ? WHERE id = 1", args: [enc] }); + const r = await D.encryptStoredDocuments(db, { key: KEY, docsDirs: [docs] }); + assert.equal(r.cleaned, 0); + assert.equal(existsSync(p), true); +}); + +test("a lost compare-and-swap (the row changed meanwhile) keeps the plain file and removes only this run's .enc", async () => { + const { docs, db } = await fresh("m3"); + const p = await addPlain(db, docs, 1); + const racing = { execute: async (q) => { + if (typeof q === "object" && /^UPDATE tax_documents SET file_path/.test(q.sql)) await db.execute({ sql: "UPDATE tax_documents SET file_path = ? WHERE id = 1", args: [p + ".moved"] }); + return db.execute(q); + } }; + const r = await D.encryptStoredDocuments(racing, { key: KEY, docsDirs: [docs] }); + assert.equal(r.encrypted, 0); + assert.equal(existsSync(p), true, "nothing unverified was deleted"); + assert.deepEqual(files(docs), [basename(p)]); +}); + +test("two migrators at once: the second finds the lock and touches nothing; the document is encrypted exactly once", async () => { + const { docs, db } = await fresh("m3b"); + await addPlain(db, docs, 1); + const [a, b] = await Promise.all([ + D.encryptStoredDocuments(db, { key: KEY, docsDirs: [docs] }), + D.encryptStoredDocuments(db, { key: KEY, docsDirs: [docs] }), + ]); + assert.equal(a.encrypted + b.encrypted, 1); + assert.equal(a.locked || b.locked, true); + const np = await pathOf(db, 1); + assert.ok((await D.decryptDocument(readFileSync(np), [KEY])).plain.equals(PDF)); + assert.deepEqual(files(docs), [basename(np)]); +}); + +test("no key: nothing is touched", async () => { + const { docs, db } = await fresh("m4"); + const p = await addPlain(db, docs, 1); + const r = await D.encryptStoredDocuments(db, { key: "", docsDirs: [docs] }); + assert.equal(r.encrypted, 0); + assert.ok(readFileSync(p).equals(PDF)); +}); + +test("a file_path outside the documents folder (or a symlink out of it) is never read, rewritten or deleted", async () => { + const { docs, db } = await fresh("m5"); + const outside = join(ROOT, "m5", "secret.txt"); + writeFileSync(outside, "not a tax document"); + await db.execute({ sql: "INSERT INTO tax_documents (id, filename, file_path) VALUES (1, 'x', ?)", args: [outside] }); + const link = join(docs, "link.pdf"); + symlinkSync(outside, link); + await db.execute({ sql: "INSERT INTO tax_documents (id, filename, file_path) VALUES (2, 'y', ?)", args: [link] }); + const r = await D.encryptStoredDocuments(db, { key: KEY, docsDirs: [docs] }); + assert.deepEqual([r.encrypted, r.skipped], [0, 2]); + assert.equal(readFileSync(outside, "utf8"), "not a tax document"); + assert.equal(D.storedDocumentPath(outside, [docs]), null); + assert.equal(D.storedDocumentPath(link, [docs]), null); +}); + +test("re-encrypts a document only the previous key opens (after a key change)", async () => { + const { docs, db } = await fresh("m6"); + const p = join(docs, "1-w2.pdf.0123abcd.enc"); + D.writePrivateAtomic(p, await D.encryptDocument(PDF, OLD)); + await db.execute({ sql: "INSERT INTO tax_documents (id, filename, file_path) VALUES (1, 'w2.pdf', ?)", args: [p] }); + const r = await D.encryptStoredDocuments(db, { key: KEY, previous: OLD, docsDirs: [docs] }); + assert.equal(r.reencrypted, 1); + assert.equal(await D.sealedWith(readFileSync(p), [OLD, KEY]), KEY); +}); + +test("a crow.db restored under another home: a row pointing at the old home resolves to the same name in this folder", async () => { + const { docs, db } = await fresh("m8"); + const enc = join(docs, "1-w2.pdf.0123abcd.enc"); + D.writePrivateAtomic(enc, await D.encryptDocument(PDF, KEY)); + await db.execute({ sql: "INSERT INTO tax_documents (id, filename, file_path) VALUES (1, 'w2.pdf', ?)", args: ["/home/someone-else/.crow/tax-documents/1-w2.pdf.0123abcd.enc"] }); + const out = join(ROOT, "m8-export"); + const ex = await D.decryptStoredDocuments(db, { keys: [KEY], outDir: out, docsDirs: [docs] }); + assert.equal(ex.decrypted, 1); + assert.ok(readFileSync(join(out, files(out)[0])).equals(PDF)); +}); + +test("reverse in place restores the identical bytes and the row; export copies leave the database alone", async () => { + const { docs, db } = await fresh("m7"); + const p = await addPlain(db, docs, 1); + await D.encryptStoredDocuments(db, { key: KEY, docsDirs: [docs] }); + const enc = await pathOf(db, 1); + const out = join(ROOT, "m7-export"); + const ex = await D.decryptStoredDocuments(db, { keys: [KEY], outDir: out, docsDirs: [docs] }); + assert.equal(ex.decrypted, 1); + assert.ok(readFileSync(join(out, files(out)[0])).equals(PDF)); + assert.equal(statSync(join(out, files(out)[0])).mode & 0o777, 0o600); + assert.equal(await pathOf(db, 1), enc, "export does not move the row"); + const back = await D.decryptStoredDocuments(db, { keys: [KEY], inPlace: true, docsDirs: [docs] }); + assert.equal(back.decrypted, 1); + assert.equal(await pathOf(db, 1), p); + assert.ok(readFileSync(p).equals(PDF)); + assert.equal(existsSync(enc), false); +}); + +test("a crow.db restored from a backup taken BEFORE the migration: the row (still naming the plain file) finds its .enc and is repointed", async () => { + const { docs, db } = await fresh("m9"); + const p = await addPlain(db, docs, 1); + await D.encryptStoredDocuments(db, { key: KEY, docsDirs: [docs] }); + const enc = await pathOf(db, 1); + await db.execute({ sql: "UPDATE tax_documents SET file_path = ? WHERE id = 1", args: [p] }); // the old backup's row + const r = await D.encryptStoredDocuments(db, { key: KEY, docsDirs: [docs] }); + assert.deepEqual(r.errors, []); + assert.equal(await pathOf(db, 1), enc, "the row points at the encrypted file again"); + assert.ok((await D.decryptDocument(readFileSync(enc), [KEY])).plain.equals(PDF)); +}); + +test("a stale lock (no run touched it for 30 minutes) is taken over; a fresh one is respected", async () => { + const { docs, db } = await fresh("m10"); + await addPlain(db, docs, 1); + const lock = join(docs, ".documents.lock"); + writeFileSync(lock, ""); + const held = await D.encryptStoredDocuments(db, { key: KEY, docsDirs: [docs] }); + assert.equal(held.locked, true, "a live lock is respected"); + const { utimesSync } = await import("node:fs"); + const old = new Date(Date.now() - 31 * 60 * 1000); + utimesSync(lock, old, old); + const r = await D.encryptStoredDocuments(db, { key: KEY, docsDirs: [docs] }); + assert.equal(r.locked, false); + assert.equal(r.encrypted, 1); + assert.equal(existsSync(lock), false, "released after the run"); + assert.equal(statSync(docs).mode & 0o777, 0o700, "the folder is private"); +});