From fbca92b8986bf8e61fd6fd2f86e50dc11e65c6c4 Mon Sep 17 00:00:00 2001 From: kh0pper <93635715+kh0pper@users.noreply.github.com> Date: Fri, 16 Oct 2026 18:28:53 -0500 Subject: [PATCH 1/4] feat(keychain): a reinstall restores generated keys from the keychain; a reminder to export; federated data keys saved too Recovery keys, option 1 (queue item 30, decided 2026-10-16): the keychain plus its passphrase-encrypted Export is how a generated key survives a lost machine. - A reinstall that finds no .env and no retained copy now takes a keychain:true generated key from a readable keychain entry (restored crow.db + imported Export) instead of minting a new one, which also overwrote the imported entry and stranded data saved with the old key. A store_as key gets a fresh hash of the restored plaintext. Best-effort: no key file or an unreadable entry mints, as before, and says so by key name. Values are never logged; crow_keychain stays local-only. - Export reminder: a Nest info issue while a key Crow generated is in no keychain Export (created after the last export, a local setting); the install log says so too. - Mastodon (secret_key_base, OTP secret, three Active Record encryption keys), Pixelfed (APP_KEY) and PeerTube (server secret) save their generated keys to the keychain. mastodon/pixelfed/peertube 1.0.4. Claude-Session: https://claude.ai/code/session_01N9fXzvNy8rCEArj1y5p6fp --- bundles/mastodon/manifest.json | 12 +- bundles/peertube/manifest.json | 4 +- bundles/pixelfed/manifest.json | 4 +- registry/add-ons.json | 48 ++++-- servers/gateway/bundle-env-secrets.js | 21 ++- .../dashboard/panels/nest/health-signals.js | 17 ++ servers/gateway/dashboard/shared/i18n.js | 3 + servers/gateway/keychain/api.js | 2 + servers/gateway/keychain/install-hooks.js | 45 ++++- servers/gateway/keychain/reminder.js | 39 +++++ servers/gateway/routes/bundles.js | 9 +- tests/bundle-generated-secrets.test.js | 16 ++ tests/keychain-export-reminder.test.js | 75 +++++++++ tests/keychain-restore.test.js | 154 ++++++++++++++++++ 14 files changed, 415 insertions(+), 34 deletions(-) create mode 100644 servers/gateway/keychain/reminder.js create mode 100644 tests/keychain-export-reminder.test.js create mode 100644 tests/keychain-restore.test.js diff --git a/bundles/mastodon/manifest.json b/bundles/mastodon/manifest.json index 4ac00dc38..d0e17af05 100644 --- a/bundles/mastodon/manifest.json +++ b/bundles/mastodon/manifest.json @@ -1,7 +1,7 @@ { "id": "mastodon", "name": "Mastodon", - "version": "1.0.3", + "version": "1.0.4", "description": "The flagship ActivityPub server — federated microblogging at scale. Heaviest of the small-AP bundles. Hosts a public (or invite-only) Mastodon instance with the full v1/v2 API surface, web UI, and Sidekiq background job processing. Media is kept on local disk.", "type": "bundle", "author": "Crow", @@ -34,13 +34,13 @@ { "name": "MASTODON_LOCAL_DOMAIN", "description": "Public domain that appears in handles (@user@). IMMUTABLE after first boot — changing breaks all federation identities.", "required": true }, { "name": "MASTODON_WEB_DOMAIN", "description": "Domain where the web UI lives (if using apex-delegation this differs from LOCAL_DOMAIN; otherwise same).", "required": false }, { "name": "MASTODON_DB_PASSWORD", "description": "Password for the bundled Postgres role. Generated by Crow at install; never shown.", "required": true, "secret": true, "generate": "secret" }, - { "name": "MASTODON_SECRET_KEY_BASE", "description": "Rails secret_key_base. Generated by Crow at install; never shown.", "required": true, "secret": true, "generate": "secret" }, - { "name": "MASTODON_OTP_SECRET", "description": "2FA OTP secret. Generated by Crow at install; never shown.", "required": true, "secret": true, "generate": "secret" }, + { "name": "MASTODON_SECRET_KEY_BASE", "description": "Rails secret_key_base. Generated by Crow at install; saved in Settings → Passwords (this machine only): export the keychain to keep a copy, or data encrypted with it cannot be recovered on another machine.", "required": true, "secret": true, "generate": "secret", "keychain": true, "keychain_label": "session signing key (secret_key_base)" }, + { "name": "MASTODON_OTP_SECRET", "description": "2FA OTP secret. Generated by Crow at install; saved in Settings → Passwords (this machine only): export the keychain to keep a copy, or data encrypted with it cannot be recovered on another machine.", "required": true, "secret": true, "generate": "secret", "keychain": true, "keychain_label": "two-factor secret" }, { "name": "MASTODON_VAPID_PRIVATE_KEY", "description": "Web Push VAPID private key (P-256). Generated by Crow at install; never shown.", "required": true, "secret": true, "generate": "vapid_private_key" }, { "name": "MASTODON_VAPID_PUBLIC_KEY", "description": "Web Push VAPID public key, derived from the private key. Generated by Crow at install.", "required": true, "generate": "vapid_public_key", "pair": "MASTODON_VAPID_PRIVATE_KEY" }, - { "name": "MASTODON_AR_ENCRYPTION_DETERMINISTIC_KEY", "description": "Active Record encryption deterministic key (Mastodon 4.3+ refuses to boot without it). Generated by Crow at install; never shown.", "required": true, "secret": true, "generate": "secret" }, - { "name": "MASTODON_AR_ENCRYPTION_KEY_DERIVATION_SALT", "description": "Active Record encryption key-derivation salt (Mastodon 4.3+). Generated by Crow at install; never shown.", "required": true, "secret": true, "generate": "secret" }, - { "name": "MASTODON_AR_ENCRYPTION_PRIMARY_KEY", "description": "Active Record encryption primary key (Mastodon 4.3+). Generated by Crow at install; never shown.", "required": true, "secret": true, "generate": "secret" }, + { "name": "MASTODON_AR_ENCRYPTION_DETERMINISTIC_KEY", "description": "Active Record encryption deterministic key (Mastodon 4.3+ refuses to boot without it). Generated by Crow at install; saved in Settings → Passwords (this machine only): export the keychain to keep a copy, or data encrypted with it cannot be recovered on another machine.", "required": true, "secret": true, "generate": "secret", "keychain": true, "keychain_label": "data encryption key (deterministic)" }, + { "name": "MASTODON_AR_ENCRYPTION_KEY_DERIVATION_SALT", "description": "Active Record encryption key-derivation salt (Mastodon 4.3+). Generated by Crow at install; saved in Settings → Passwords (this machine only): export the keychain to keep a copy, or data encrypted with it cannot be recovered on another machine.", "required": true, "secret": true, "generate": "secret", "keychain": true, "keychain_label": "data encryption key-derivation salt" }, + { "name": "MASTODON_AR_ENCRYPTION_PRIMARY_KEY", "description": "Active Record encryption primary key (Mastodon 4.3+). Generated by Crow at install; saved in Settings → Passwords (this machine only): export the keychain to keep a copy, or data encrypted with it cannot be recovered on another machine.", "required": true, "secret": true, "generate": "secret", "keychain": true, "keychain_label": "data encryption key (primary)" }, { "name": "MASTODON_ACCESS_TOKEN", "description": "Admin access token (Settings → Development → New Application, grant read/write/admin:read/admin:write).", "required": false, "secret": true }, { "name": "MASTODON_URL", "description": "Internal URL the MCP server uses to reach Mastodon's web container (default http://mastodon-web:3000 over crow-federation).", "default": "http://mastodon-web:3000", "required": false }, { "name": "MASTODON_MEDIA_RETENTION_DAYS", "description": "Remote media cache retention in days (default 14; 7 on Pi-class hosts).", "default": "14", "required": false }, diff --git a/bundles/peertube/manifest.json b/bundles/peertube/manifest.json index 102c5e2c0..bacf54e66 100644 --- a/bundles/peertube/manifest.json +++ b/bundles/peertube/manifest.json @@ -1,7 +1,7 @@ { "id": "peertube", "name": "PeerTube", - "version": "1.0.3", + "version": "1.0.4", "description": "Federated video platform over ActivityPub — YouTube-alternative on the fediverse. Upload and transcode video, federate channels, stream via WebTorrent or HLS. Heaviest bundle in the federated line — video is kept on local disk, so plan for disk space and keep the transcoding/retention policy tight.", "type": "bundle", "author": "Crow", @@ -33,7 +33,7 @@ "env_vars": [ { "name": "PEERTUBE_WEBSERVER_HOSTNAME", "description": "Public domain (subdomain; IMMUTABLE after first federation). Appears in video URLs, channel URLs, every ActivityPub actor.", "required": true }, { "name": "PEERTUBE_DB_PASSWORD", "description": "Password for the bundled Postgres role. Generated by Crow at install; never shown.", "required": true, "secret": true, "generate": "secret" }, - { "name": "PEERTUBE_SECRET", "description": "Server secret that signs federation requests. IMMUTABLE: generated once by Crow at install, kept across reinstalls, never shown.", "required": true, "secret": true, "generate": "secret" }, + { "name": "PEERTUBE_SECRET", "description": "Server secret that signs federation requests. IMMUTABLE: generated once by Crow at install, kept across reinstalls, saved in Settings → Passwords (this machine only): export the keychain to keep a copy, or data encrypted with it cannot be recovered on another machine.", "required": true, "secret": true, "generate": "secret", "keychain": true, "keychain_label": "server secret" }, { "name": "PEERTUBE_ACCESS_TOKEN", "description": "OAuth bearer token (Administration → Users → your account → Personal Access Tokens, OR POST /api/v1/users/token with username+password + client_id+client_secret from /api/v1/oauth-clients/local).", "required": false, "secret": true }, { "name": "PEERTUBE_URL", "description": "Internal URL the MCP server uses to reach PeerTube (default http://peertube:9000 over crow-federation).", "default": "http://peertube:9000", "required": false }, { "name": "PEERTUBE_ADMIN_EMAIL", "description": "Admin email — where takedown notices, abuse reports, and federation alerts go.", "required": false }, diff --git a/bundles/pixelfed/manifest.json b/bundles/pixelfed/manifest.json index dff761ff3..8bce6dda3 100644 --- a/bundles/pixelfed/manifest.json +++ b/bundles/pixelfed/manifest.json @@ -1,7 +1,7 @@ { "id": "pixelfed", "name": "Pixelfed", - "version": "1.0.3", + "version": "1.0.4", "description": "Federated photo-sharing server over ActivityPub — Instagram-alternative on the fediverse. Publish photos/stories/collections; remote Mastodon/GoToSocial/Funkwhale followers see your posts in their timelines. Media is kept on local disk.", "type": "bundle", "author": "Crow", @@ -33,7 +33,7 @@ "env_vars": [ { "name": "PIXELFED_HOSTNAME", "description": "Public domain (subdomain; path-mounts break ActivityPub).", "required": true }, { "name": "PIXELFED_DB_PASSWORD", "description": "Password for the bundled Postgres role. Generated by Crow at install; never shown.", "required": true, "secret": true, "generate": "secret" }, - { "name": "PIXELFED_APP_KEY", "description": "Laravel application key (base64: + 32 random bytes). Generated by Crow at install; never shown.", "required": true, "secret": true, "generate": "laravel_key" }, + { "name": "PIXELFED_APP_KEY", "description": "Laravel application key (base64: + 32 random bytes). Generated by Crow at install; saved in Settings → Passwords (this machine only): export the keychain to keep a copy, or data encrypted with it cannot be recovered on another machine.", "required": true, "secret": true, "generate": "laravel_key", "keychain": true, "keychain_label": "data encryption key (APP_KEY)" }, { "name": "PIXELFED_ACCESS_TOKEN", "description": "OAuth2 Personal Access Token (Settings → Development → New Application).", "required": false, "secret": true }, { "name": "PIXELFED_URL", "description": "Internal URL the MCP server uses to reach Pixelfed (default http://pixelfed:80 over crow-federation).", "default": "http://pixelfed:80", "required": false }, { "name": "PIXELFED_OPEN_REGISTRATION", "description": "Allow new user signups (true/false). Default false — opening registration without moderation tooling invites abuse.", "default": "false", "required": false }, diff --git a/registry/add-ons.json b/registry/add-ons.json index 7b33aebfb..630496464 100644 --- a/registry/add-ons.json +++ b/registry/add-ons.json @@ -3038,7 +3038,7 @@ { "id": "mastodon", "name": "Mastodon", - "version": "1.0.3", + "version": "1.0.4", "description": "The flagship ActivityPub server — federated microblogging at scale. Heaviest of the small-AP bundles. Hosts a public (or invite-only) Mastodon instance with the full v1/v2 API surface, web UI, and Sidekiq background job processing. Media is kept on local disk.", "type": "bundle", "author": "Crow", @@ -3113,17 +3113,21 @@ }, { "name": "MASTODON_SECRET_KEY_BASE", - "description": "Rails secret_key_base. Generated by Crow at install; never shown.", + "description": "Rails secret_key_base. Generated by Crow at install; saved in Settings → Passwords (this machine only): export the keychain to keep a copy, or data encrypted with it cannot be recovered on another machine.", "required": true, "secret": true, - "generate": "secret" + "generate": "secret", + "keychain": true, + "keychain_label": "session signing key (secret_key_base)" }, { "name": "MASTODON_OTP_SECRET", - "description": "2FA OTP secret. Generated by Crow at install; never shown.", + "description": "2FA OTP secret. Generated by Crow at install; saved in Settings → Passwords (this machine only): export the keychain to keep a copy, or data encrypted with it cannot be recovered on another machine.", "required": true, "secret": true, - "generate": "secret" + "generate": "secret", + "keychain": true, + "keychain_label": "two-factor secret" }, { "name": "MASTODON_VAPID_PRIVATE_KEY", @@ -3141,24 +3145,30 @@ }, { "name": "MASTODON_AR_ENCRYPTION_DETERMINISTIC_KEY", - "description": "Active Record encryption deterministic key (Mastodon 4.3+ refuses to boot without it). Generated by Crow at install; never shown.", + "description": "Active Record encryption deterministic key (Mastodon 4.3+ refuses to boot without it). Generated by Crow at install; saved in Settings → Passwords (this machine only): export the keychain to keep a copy, or data encrypted with it cannot be recovered on another machine.", "required": true, "secret": true, - "generate": "secret" + "generate": "secret", + "keychain": true, + "keychain_label": "data encryption key (deterministic)" }, { "name": "MASTODON_AR_ENCRYPTION_KEY_DERIVATION_SALT", - "description": "Active Record encryption key-derivation salt (Mastodon 4.3+). Generated by Crow at install; never shown.", + "description": "Active Record encryption key-derivation salt (Mastodon 4.3+). Generated by Crow at install; saved in Settings → Passwords (this machine only): export the keychain to keep a copy, or data encrypted with it cannot be recovered on another machine.", "required": true, "secret": true, - "generate": "secret" + "generate": "secret", + "keychain": true, + "keychain_label": "data encryption key-derivation salt" }, { "name": "MASTODON_AR_ENCRYPTION_PRIMARY_KEY", - "description": "Active Record encryption primary key (Mastodon 4.3+). Generated by Crow at install; never shown.", + "description": "Active Record encryption primary key (Mastodon 4.3+). Generated by Crow at install; saved in Settings → Passwords (this machine only): export the keychain to keep a copy, or data encrypted with it cannot be recovered on another machine.", "required": true, "secret": true, - "generate": "secret" + "generate": "secret", + "keychain": true, + "keychain_label": "data encryption key (primary)" }, { "name": "MASTODON_ACCESS_TOKEN", @@ -4530,7 +4540,7 @@ { "id": "peertube", "name": "PeerTube", - "version": "1.0.3", + "version": "1.0.4", "description": "Federated video platform over ActivityPub — YouTube-alternative on the fediverse. Upload and transcode video, federate channels, stream via WebTorrent or HLS. Heaviest bundle in the federated line — video is kept on local disk, so plan for disk space and keep the transcoding/retention policy tight.", "type": "bundle", "author": "Crow", @@ -4598,10 +4608,12 @@ }, { "name": "PEERTUBE_SECRET", - "description": "Server secret that signs federation requests. IMMUTABLE: generated once by Crow at install, kept across reinstalls, never shown.", + "description": "Server secret that signs federation requests. IMMUTABLE: generated once by Crow at install, kept across reinstalls, saved in Settings → Passwords (this machine only): export the keychain to keep a copy, or data encrypted with it cannot be recovered on another machine.", "required": true, "secret": true, - "generate": "secret" + "generate": "secret", + "keychain": true, + "keychain_label": "server secret" }, { "name": "PEERTUBE_ACCESS_TOKEN", @@ -4734,7 +4746,7 @@ { "id": "pixelfed", "name": "Pixelfed", - "version": "1.0.3", + "version": "1.0.4", "description": "Federated photo-sharing server over ActivityPub — Instagram-alternative on the fediverse. Publish photos/stories/collections; remote Mastodon/GoToSocial/Funkwhale followers see your posts in their timelines. Media is kept on local disk.", "type": "bundle", "author": "Crow", @@ -4801,10 +4813,12 @@ }, { "name": "PIXELFED_APP_KEY", - "description": "Laravel application key (base64: + 32 random bytes). Generated by Crow at install; never shown.", + "description": "Laravel application key (base64: + 32 random bytes). Generated by Crow at install; saved in Settings → Passwords (this machine only): export the keychain to keep a copy, or data encrypted with it cannot be recovered on another machine.", "required": true, "secret": true, - "generate": "laravel_key" + "generate": "laravel_key", + "keychain": true, + "keychain_label": "data encryption key (APP_KEY)" }, { "name": "PIXELFED_ACCESS_TOKEN", diff --git a/servers/gateway/bundle-env-secrets.js b/servers/gateway/bundle-env-secrets.js index 6b5ef4ff2..603f14f5a 100644 --- a/servers/gateway/bundle-env-secrets.js +++ b/servers/gateway/bundle-env-secrets.js @@ -135,11 +135,15 @@ export function keychainEligibleKeys(manifest) { * minted plaintext of keychain:true keys created by THIS call (empty on reinstall) * persist writes the retained copy; call it only after `minted` is safely in the * keychain — otherwise a lost plaintext would leave an unusable hash behind. - * Order per key: installed .env → retained copy → new value (never regenerated). + * Order per key: installed .env → retained copy → compose fallback → `keychain` (plaintext of + * a readable keychain entry, keychain:true keys only — install-hooks keychainRestoreValues) → + * new value (never regenerated). The keychain rung is the rebuilt-machine path: crow.db + * restored and the keychain Export imported, but no .env and no retained copy. A restored + * value is REUSED (listed in `restored`), never minted, so its entry is never overwritten. */ -export function planGeneratedEnv(bundleId, manifest, { destDir, crowHome, composeText = null, priorInstall = false }) { +export function planGeneratedEnv(bundleId, manifest, { destDir, crowHome, composeText = null, priorInstall = false, keychain = null }) { const keys = generatedEnvKeys(manifest); - if (keys.length === 0) return { env: {}, minted: {}, reusedPlain: {}, persist() {} }; + if (keys.length === 0) return { env: {}, minted: {}, reusedPlain: {}, restored: [], persist() {} }; if (typeof bundleId !== "string" || !/^[a-z0-9][a-z0-9-]*$/.test(bundleId) || bundleId.length > 64) { throw new Error(`Invalid bundle ID: ${JSON.stringify(bundleId)}`); } @@ -157,6 +161,7 @@ export function planGeneratedEnv(bundleId, manifest, { destDir, crowHome, compos // keychain:true values kept from an earlier install (typed, or generated before): the // keychain may not hold them, and the form no longer shows the field — see install-hooks. const reusedPlain = {}; + const restored = []; const existingFor = (k) => installed[k] || retained[k] || legacy[k] || ""; // Private halves first, so a public half can be derived from (or minted with) its pair. const order = [...keys].sort((a, b) => (byName.get(a)?.generate === "vapid_public_key") - (byName.get(b)?.generate === "vapid_public_key")); @@ -174,6 +179,14 @@ export function planGeneratedEnv(bundleId, manifest, { destDir, crowHome, compos env[k] = pairs[spec.pair] || vapidPublicFromPrivate(priv) || existing; continue; } + const fromKeychain = !existing && spec.keychain === true && keychain && typeof keychain[k] === "string" && keychain[k] !== "" ? keychain[k] : ""; + if (fromKeychain) { + // The keychain holds the plaintext; a store_as key gets a fresh hash of it (any hash + // of the same plaintext verifies). Its entry exists, so nothing is saved again. + env[k] = spec.store_as === "argon2id" ? argon2idPhc(fromKeychain) : fromKeychain; + restored.push(k); + continue; + } if (existing) { env[k] = existing; if (spec.keychain === true && !spec.store_as) { @@ -203,7 +216,7 @@ export function planGeneratedEnv(bundleId, manifest, { destDir, crowHome, compos formatEnvLines({ ...retained, ...env }), ); }; - return { env, minted, reusedPlain, persist }; + return { env, minted, reusedPlain, restored, persist }; } /** diff --git a/servers/gateway/dashboard/panels/nest/health-signals.js b/servers/gateway/dashboard/panels/nest/health-signals.js index 108259a40..2730955b1 100644 --- a/servers/gateway/dashboard/panels/nest/health-signals.js +++ b/servers/gateway/dashboard/panels/nest/health-signals.js @@ -15,6 +15,8 @@ * updates — auto_update_* version comparison (info if update available) * backup — newest of THIS instance's files in CROW_BACKUP_DIR (none → info; * >7d → warn; owned by another instance → warn) + * keychainExport — keys Crow generated that are in no keychain Export yet (info; no + * card when there are none) * providers — alwaysResident provider residency (unreachable ≥threshold → warn) * externalEngines — engines another machine runs (spec 2026-09-23): own id, * info at most — never warn, never a push; no card when none @@ -56,6 +58,7 @@ import { readReservation } from "../../../box-reservation.js"; import { localBackupContext, newestBackupOwnership } from "../../../../shared/backup-naming.js"; import { getStats as getOutboxStats } from "../../../../sharing/sync-outbox-drain.js"; import { createSidecarHealthReader } from "../../../artifact-origin/sidecar-health.js"; +import { unexportedGeneratedCount } from "../../../keychain/reminder.js"; // ─── Module-level 30s cache ─────────────────────────────────────────────────── @@ -672,6 +675,19 @@ async function backupSignal(db, nowFn, lang) { }; } +// Keychain Export reminder (recovery keys, queue item 30): a key Crow generated that is in no +// keychain Export lives only on this machine. Info only (strip, never a push); no card +// when every generated key is exported or there are none. +async function keychainExportSignal(db, lang) { + const n = await unexportedGeneratedCount(db); + if (n <= 0) return null; + return { + id: "keychainExport", severity: "info", state: "info", label: t("signals.keychainExport.label", lang), + value: String(n), issueLabel: fill(t("signals.keychainExport.issue", lang), { n }), + actionLabel: t("signals.keychainExport.action", lang), actionHref: "/dashboard/settings?section=passwords", + }; +} + // ─── Security-maintenance signals (W2) ──────────────────────────────────────── async function loginsSignal(db, lang) { @@ -1181,6 +1197,7 @@ export async function collectHealthSignals(db, opts = {}) { syncDialSignal(db, lang, nowFn), updatesSignal(db), backupSignal(db, nowFn, lang), + keychainExportSignal(db, lang), syncConflictsSignal(db), outboxSignal(db), loginsSignal(db, lang), diff --git a/servers/gateway/dashboard/shared/i18n.js b/servers/gateway/dashboard/shared/i18n.js index 40b5fa838..1aaacec34 100644 --- a/servers/gateway/dashboard/shared/i18n.js +++ b/servers/gateway/dashboard/shared/i18n.js @@ -229,6 +229,9 @@ export const translations = { "signals.integrations.action": { en: "Check connections", es: "Revisar conexiones" }, // backup "signals.backup.label": { en: "Backup", es: "Copia de seguridad" }, + "signals.keychainExport.label": { en: "Keychain export", es: "Exportación del llavero" }, + "signals.keychainExport.issue": { en: "{n} password(s) Crow generated are not in a keychain Export yet. If this machine is lost, they cannot be recovered without one.", es: "{n} contraseña(s) que Crow generó aún no están en una exportación del llavero. Si pierdes esta máquina, no se pueden recuperar sin una." }, + "signals.keychainExport.action": { en: "Export keychain", es: "Exportar llavero" }, "signals.backup.verified": { en: "checked ✓", es: "verificada ✓" }, "signals.backup.damaged": { en: "may be damaged", es: "puede estar dañada" }, "signals.backup.damagedIssue": { en: "Your latest backup may be damaged — make a new backup now", es: "Tu última copia de seguridad puede estar dañada — haz una copia nueva ahora" }, diff --git a/servers/gateway/keychain/api.js b/servers/gateway/keychain/api.js index b72533bcf..6a5b56e60 100644 --- a/servers/gateway/keychain/api.js +++ b/servers/gateway/keychain/api.js @@ -18,6 +18,7 @@ import { exportableEntries, importEntries, ensureWriteKey, } from "./store.js"; import { vaultwardenStatus, saveToVault } from "./vault-save.js"; +import { recordKeychainExport } from "./reminder.js"; const BASE = "/dashboard/keychain/api"; @@ -180,6 +181,7 @@ export function keychainApiRouter({ if (!key) return res.status(409).json(KEY_MISSING); const entries = await exportableEntries(db, key); const file = await sealExport(entries, passphrase); + await recordKeychainExport(db).catch(() => {}); // the Nest reminder; never blocks the download await audit(db, "keychain_export", { ip: req.ip, details: { count: entries.length } }); res.set("Content-Disposition", `attachment; filename="crow-keychain-${new Date().toISOString().slice(0, 10)}.json"`); res.json(file); diff --git a/servers/gateway/keychain/install-hooks.js b/servers/gateway/keychain/install-hooks.js index 0c6dd0684..57d1677a2 100644 --- a/servers/gateway/keychain/install-hooks.js +++ b/servers/gateway/keychain/install-hooks.js @@ -12,7 +12,9 @@ import { createDbClient, auditLog } from "../../db.js"; import { keychainGeneratedKeys, keychainEligibleKeys, expandKeychainTemplate } from "../bundle-env-secrets.js"; import { createNotification } from "../../shared/notifications.js"; -import { saveExtensionSecret, reactivateBundleEntries, hasExtensionEntry, markBundleRemoved, ensureWriteKey } from "./store.js"; +import { saveExtensionSecret, reactivateBundleEntries, hasExtensionEntry, markBundleRemoved, ensureWriteKey, listEntries, openEntrySecret } from "./store.js"; +import { loadKeychainKey } from "./key.js"; +import { lastKeychainExport } from "./reminder.js"; import { vaultwardenStatus, saveToVault, VAULT_REASONS } from "./vault-save.js"; let _override = null; @@ -20,6 +22,8 @@ export function _setKeychainDepsForTest(deps) { _override = deps || null; } function deps() { return { openDb: () => createDbClient(), + // The key to READ with; null when missing or damaged (restore is best-effort). + readKey: () => { try { return loadKeychainKey(); } catch { return null; } }, // The key to save with (created on first use; refuses a damaged key while entries exist). writeKey: (db) => ensureWriteKey(db, { onNewKey: async ({ orphaned }) => { @@ -119,6 +123,11 @@ export async function recordKeychainForInstall({ bundleId, manifest, env, minted try { await saveOne(key, s); } catch (err) { log(`Could not save ${s.k} to Crow keychain (${err?.code || err?.name || "error"}); the install continues`); } } if (out.saved) log(`Saved ${out.saved} password(s) to Crow keychain (Settings → Passwords)`); + if (mintedList.length && out.saved) { + try { + if (!(await lastKeychainExport(db))) log("This key exists only on this machine: export your keychain (Settings → Passwords → Export) so it can be recovered if the machine is lost."); + } catch { /* the reminder never blocks an install */ } + } if (keychainReq?.vault && saved.length) { const st = d.vault.status(); @@ -153,6 +162,40 @@ export async function recordKeychainForInstall({ bundleId, manifest, env, minted return out; } +/** + * Plaintext of this bundle's keychain:true GENERATED keys that a readable keychain entry + * holds (any status: an uninstall marks entries extension_removed and keeps them), for + * planGeneratedEnv's keychain rung. This is how a rebuilt machine gets its keys back: restore + * crow.db, import the keychain Export, reinstall. Best-effort and read-only: no key file, a + * damaged one, or no table → {} and the install mints as before. An entry sealed under + * another keychain key is reported by NAME through `log`, never read. Values never leave + * this process: they are not logged and never synced (crow_keychain is LOCAL_ONLY). + */ +export async function keychainRestoreValues(bundleId, manifest, { log = () => {} } = {}) { + const keys = keychainGeneratedKeys(manifest); + if (keys.length === 0) return {}; + const out = {}; + let db; + try { + const key = deps().readKey(); + if (!key) return {}; + db = deps().openDb(); + const unreadable = []; + for (const e of await listEntries(db, { keyId: key.id })) { + if (e.kind !== "extension" || e.bundle_id !== bundleId || !keys.includes(e.env_key)) continue; + if (!e.readable) { unreadable.push(e.env_key); continue; } + const plain = await openEntrySecret(db, key, e.id); + if (typeof plain === "string" && plain !== "") out[e.env_key] = plain; + } + if (unreadable.length) log(`Crow keychain holds ${unreadable.join(", ")} sealed with another keychain key, so it could not be read; a new one is generated. Import the keychain Export from the old machine first to keep the old one.`); + } catch { + return {}; + } finally { + try { db?.close(); } catch {} + } + return out; +} + export async function markBundleKeychainRemoved(bundleId) { let db; try { diff --git a/servers/gateway/keychain/reminder.js b/servers/gateway/keychain/reminder.js new file mode 100644 index 000000000..6a21e8a66 --- /dev/null +++ b/servers/gateway/keychain/reminder.js @@ -0,0 +1,39 @@ +/** + * Keychain Export reminder (recovery keys, queue item 30, decision 2026-10-16). + * + * A key Crow GENERATES for an extension (an admin password, a data encryption key) exists + * only on this machine: the keychain never syncs, and backups never copy secrets/. The + * user's passphrase-encrypted Export is the one off-machine copy. So while a generated + * key is in no Export, the Nest shows an info issue (health signal "keychainExport"). + * + * "In no Export" = created after the last successful export (or there was never one). + * The time is a LOCAL setting (not in the sync allowlist): another instance's export says + * nothing about this machine's keys. audit_log is not used: it is pruned after 90 days. + */ +import { readSetting, writeSetting } from "../dashboard/settings/registry.js"; +import { ensureKeychainTable } from "./store.js"; + +export const LAST_EXPORT_SETTING = "keychain_last_export_at"; + +/** Call after an Export file was sealed and is about to be sent. */ +export async function recordKeychainExport(db, at = new Date()) { + await writeSetting(db, LAST_EXPORT_SETTING, at.toISOString(), { scope: "local" }); +} + +export async function lastKeychainExport(db) { + const v = await readSetting(db, LAST_EXPORT_SETTING); + return typeof v === "string" && !Number.isNaN(Date.parse(v)) ? v : null; +} + +/** Active generated extension entries created after the last export (all when none). */ +export async function unexportedGeneratedCount(db) { + await ensureKeychainTable(db); + const since = await lastKeychainExport(db); + const { rows } = await db.execute({ + sql: `SELECT COUNT(*) AS n FROM crow_keychain + WHERE kind = 'extension' AND origin = 'generated' AND status = 'active' + AND (? IS NULL OR created_at > ?)`, + args: [since, since], + }); + return Number(rows[0]?.n ?? 0); +} diff --git a/servers/gateway/routes/bundles.js b/servers/gateway/routes/bundles.js index 940b3bd64..ab662a18e 100644 --- a/servers/gateway/routes/bundles.js +++ b/servers/gateway/routes/bundles.js @@ -55,7 +55,7 @@ import { import { isModelOrchestrationDisabled, isModelBundleManifest } from "../../shared/model-orchestration.js"; import { syncBundleSkills, removeBundleSkills, readSkillLedger, skillSyncMode } from "../bundle-skill-sync.js"; import { envValueProblem, encodeEnvValue, formatEnvLines, updateEnvText, pathEnvKeys } from "../bundle-env-codec.js"; -import { sanitizeKeychainRequest, recordKeychainForInstall, markBundleKeychainRemoved } from "../keychain/install-hooks.js"; +import { sanitizeKeychainRequest, recordKeychainForInstall, markBundleKeychainRemoved, keychainRestoreValues } from "../keychain/install-hooks.js"; import { precreateDirs, runPostInstall, hookEnv, spawnGroup, pullTimeoutMs, resolveComposeProject, classifyProjectOwners } from "../bundle-lifecycle.js"; import { provisionNtfy, deprovisionNtfy, AUTOWIRE_KIND } from "../push/ntfy-provision.js"; import { checkComposeModel, loadComposeModel, usesHostNetwork } from "../compose-guard.js"; @@ -2509,7 +2509,12 @@ export async function runInstallJob(bundleId, envVars, { job, installedSnapshot, const reqEnv = stripGeneratedKeys(manifest, envVars); let composeTextForEnv = null; try { const cp = join(destDir, "docker-compose.yml"); if (existsSync(cp)) composeTextForEnv = readFileSync(cp, "utf8"); } catch { /* no fallbacks */ } - const plan = planGeneratedEnv(bundleId, manifest, { destDir, crowHome: CROW_HOME, composeText: composeTextForEnv, priorInstall: priorInstallEnv }); + // A rebuilt machine (crow.db restored + keychain Export imported) has no .env and no + // retained copy: its keychain:true generated keys come back from the keychain instead of + // being minted again (recovery keys, queue item 30). Best-effort; values never logged. + const fromKeychain = await keychainRestoreValues(bundleId, manifest, { log: (m) => appendLog(job, m) }); + const plan = planGeneratedEnv(bundleId, manifest, { destDir, crowHome: CROW_HOME, composeText: composeTextForEnv, priorInstall: priorInstallEnv, keychain: fromKeychain }); + if (plan.restored.length > 0) appendLog(job, `Restored ${plan.restored.length} key(s) from Crow keychain (Settings → Passwords): ${plan.restored.join(", ")}`); const generated = plan.env; const writeEnv = { ...(reqEnv || {}), ...generated }; let installEnv = writeEnv; diff --git a/tests/bundle-generated-secrets.test.js b/tests/bundle-generated-secrets.test.js index 958677660..20404e88c 100644 --- a/tests/bundle-generated-secrets.test.js +++ b/tests/bundle-generated-secrets.test.js @@ -59,6 +59,12 @@ const EXPECT = [ { id: "maker-lab-advanced", key: "MLA_ADMIN_PASSWORD", propagate: false, username: "${MLA_ADMIN_USER}" }, { id: "actual-budget", key: "ACTUAL_SERVER_PASSWORD", propagate: false }, { id: "lemmy", key: "LEMMY_ADMIN_PASSWORD", propagate: false, username: "admin_pending" }, + { id: "mastodon", key: "MASTODON_SECRET_KEY_BASE", propagate: false }, + { id: "mastodon", key: "MASTODON_OTP_SECRET", propagate: false }, + { id: "mastodon", key: "MASTODON_AR_ENCRYPTION_DETERMINISTIC_KEY", propagate: false }, + { id: "mastodon", key: "MASTODON_AR_ENCRYPTION_KEY_DERIVATION_SALT", propagate: false }, + { id: "mastodon", key: "MASTODON_AR_ENCRYPTION_PRIMARY_KEY", propagate: false }, + { id: "peertube", key: "PEERTUBE_SECRET", propagate: false }, ]; test("each key is generated, keychain-saved, and propagated only where an MCP child reads it", () => { @@ -307,3 +313,13 @@ test("actual-budget: a stale typed password never becomes the new server's passw const server = readFileSync(join(REPO, "bundles/actual-budget/server/server.js"), "utf8"); assert.match(server, /ensureBootstrapped\(ACTUAL_URL, process\.env\.ACTUAL_SERVER_PASSWORD/, "only the generated password bootstraps"); }); + +test("recovery keys (c): the Pixelfed APP_KEY (laravel_key) is saved to the keychain too, and stays out of the gateway .env", () => { + const v = specOf("pixelfed", "PIXELFED_APP_KEY"); + assert.equal(v.generate, "laravel_key"); + assert.equal(v.keychain, true); + assert.ok(v.keychain_label); + assert.equal(S.gatewayGeneratedKeys(manifestOf("pixelfed")).includes("PIXELFED_APP_KEY"), false); + const r = validateManifest(manifestOf("pixelfed"), join(REPO, "bundles", "pixelfed")); + assert.deepEqual(r.errors || [], []); +}); diff --git a/tests/keychain-export-reminder.test.js b/tests/keychain-export-reminder.test.js new file mode 100644 index 000000000..cd37a119c --- /dev/null +++ b/tests/keychain-export-reminder.test.js @@ -0,0 +1,75 @@ +/** + * Export reminder (recovery keys, queue item 30): while a key Crow GENERATED is in no keychain + * Export, the Nest shows an info issue (strip only, never a push). It goes away when the user + * exports, and comes back for a key saved after that export. The last-export time is a local + * setting: it never syncs to a paired instance. Real init-db in a scratch home. + */ +import { test, after } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { execFileSync } from "node:child_process"; + +process.env.CROW_HOME = mkdtempSync(join(tmpdir(), "crow-kcrem-home-")); +process.env.CROW_DATA_DIR = mkdtempSync(join(tmpdir(), "crow-kcrem-data-")); +process.env.CROW_DISABLE_HEALTH_MONITOR = "1"; +process.env.CROW_DISABLE_INSTANCE_SYNC = "1"; +process.env.CROW_DISABLE_NOSTR = "1"; +execFileSync(process.execPath, ["scripts/init-db.js"], { env: process.env, stdio: "pipe" }); + +const K = await import("../servers/gateway/keychain/store.js"); +const { createKeychainKey } = await import("../servers/gateway/keychain/key.js"); +const { createDbClient } = await import("../servers/db.js"); +const HS = await import("../servers/gateway/dashboard/panels/nest/health-signals.js"); +const { isSyncable } = await import("../servers/gateway/dashboard/settings/sync-allowlist.js"); +const R = await import("../servers/gateway/keychain/reminder.js"); +after(() => { for (const d of [process.env.CROW_HOME, process.env.CROW_DATA_DIR]) rmSync(d, { recursive: true, force: true }); }); + +const key = createKeychainKey(); +const db = createDbClient(); +after(() => db.close()); + +async function signal() { + HS.invalidateHealthCache(); + const r = await HS.collectHealthSignals(db, { lang: "en" }); + return { issue: r.issues.find((i) => i.id === "keychainExport") || null, card: r.details.find((d) => d.id === "keychainExport") || null }; +} + +test("no generated keys → no reminder at all (a manual entry never counts)", async () => { + await K.addManualSecret(db, key, { label: "my bank", secret: "hunter2-hunter2" }); + const s = await signal(); + assert.equal(s.issue, null); + assert.equal(s.card, null); +}); + +test("a generated key that was never exported → an info issue that links to Settings → Passwords", async () => { + await K.saveExtensionSecret(db, key, { bundleId: "tax", envKey: "CROW_TAX_ENCRYPTION_KEY", label: "Tax — data encryption key", secret: "k1-" + "x".repeat(40), origin: "generated" }); + const s = await signal(); + assert.ok(s.issue, "the reminder is shown"); + assert.equal(s.issue.severity, "info", "info only: shown on the Nest strip, never pushed"); + assert.match(s.issue.label, /1 .*not in a keychain Export/i); + assert.equal(s.issue.actionHref, "/dashboard/settings?section=passwords"); + assert.deepEqual(await R.unexportedGeneratedCount(db), 1); +}); + +test("after an export the reminder is gone; a key generated after the export brings it back", async () => { + await R.recordKeychainExport(db, new Date(Date.now() + 1000)); + assert.equal((await signal()).issue, null, "everything generated is in an Export"); + await new Promise((r) => setTimeout(r, 1100)); + await K.saveExtensionSecret(db, key, { bundleId: "campaigns", envKey: "CROW_CAMPAIGNS_ENCRYPTION_KEY", label: "Campaigns — data encryption key", secret: "k2-" + "y".repeat(40), origin: "generated" }); + const s = await signal(); + assert.ok(s.issue, "the newer key is in no Export"); + assert.match(s.issue.label, /^1 /); +}); + +test("the last-export time is local: it is not in the sync allowlist", () => { + assert.equal(isSyncable(R.LAST_EXPORT_SETTING), false); +}); + +test("POST /api/keychain/export records the export time (route wiring)", async () => { + const src = (await import("node:fs")).readFileSync(new URL("../servers/gateway/keychain/api.js", import.meta.url), "utf8"); + const exportRoute = src.slice(src.indexOf("/export`"), src.indexOf("/import`")); + assert.match(exportRoute, /recordKeychainExport\(db/); + assert.ok(exportRoute.indexOf("recordKeychainExport") > exportRoute.indexOf("sealExport"), "only after the file is sealed"); +}); diff --git a/tests/keychain-restore.test.js b/tests/keychain-restore.test.js new file mode 100644 index 000000000..564c5ed3c --- /dev/null +++ b/tests/keychain-restore.test.js @@ -0,0 +1,154 @@ +/** + * A reinstall restores generated keys from the Crow keychain (queue item 30, recovery keys + * option 1, decision 2026-10-16). Before this, a rebuilt machine (crow.db restored, keychain + * Export imported, extension reinstalled) MINTED a new key and then overwrote the imported + * keychain entry with it: data saved under the old key became unreadable. Real init-db in a + * scratch home; no docker. + */ +import { test, after } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, rmSync, existsSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { execFileSync } from "node:child_process"; + +process.env.CROW_HOME = mkdtempSync(join(tmpdir(), "crow-kcrestore-home-")); +process.env.CROW_DATA_DIR = mkdtempSync(join(tmpdir(), "crow-kcrestore-data-")); +process.env.CROW_AUTO_UPDATE = "0"; +process.env.CROW_DISABLE_HEALTH_MONITOR = "1"; +process.env.CROW_DISABLE_INSTANCE_SYNC = "1"; +process.env.CROW_DISABLE_NOSTR = "1"; +execFileSync(process.execPath, ["scripts/init-db.js"], { env: process.env, stdio: "pipe" }); + +const CROW_HOME = process.env.CROW_HOME; +const B = await import("../servers/gateway/routes/bundles.js"); +const H = await import("../servers/gateway/keychain/install-hooks.js"); +const K = await import("../servers/gateway/keychain/store.js"); +const P = await import("../servers/gateway/keychain/argon2-phc.js"); +const S = await import("../servers/gateway/bundle-env-secrets.js"); +const { parseEnvText } = await import("../servers/gateway/bundle-env-codec.js"); +const { createDbClient } = await import("../servers/db.js"); +const { loadKeychainKey } = await import("../servers/gateway/keychain/key.js"); + +const FIXTURES = mkdtempSync(join(tmpdir(), "crow-kcrestore-app-")); +B._setAppBundlesForTest(FIXTURES); +B._setAppEnvPathForTest(join(mkdtempSync(join(tmpdir(), "crow-kcrestore-gw-")), ".env")); +after(() => { + B._setAppEnvPathForTest(null); + for (const d of [CROW_HOME, process.env.CROW_DATA_DIR, FIXTURES]) rmSync(d, { recursive: true, force: true }); +}); + +const MANIFEST = (id) => ({ + id, name: "Demo Data", description: "d", type: "bundle", category: "infrastructure", version: "0.1.0", + env_vars: [ + { name: "DEMO_DATA_KEY", secret: true, generate: "secret", keychain: true, keychain_label: "data encryption key" }, + { name: "DEMO_ADMIN_TOKEN", secret: true, generate: "secret", keychain: true, store_as: "argon2id", keychain_label: "admin token" }, + { name: "DEMO_DB_PASSWORD", secret: true, generate: "secret" }, + ], +}); +function fixture(id) { + mkdirSync(join(FIXTURES, id), { recursive: true }); + writeFileSync(join(FIXTURES, id, "manifest.json"), JSON.stringify(MANIFEST(id))); + return MANIFEST(id); +} +const db = () => createDbClient(); +async function install(id, manifest) { + const job = B._createJobForTest(id, "install"); + const out = await B.runInstallJob(id, {}, { job, installedSnapshot: [], consentVerified: false, manifest }); + assert.equal(out.ok, true, out.reason); + return { job, env: parseEnvText(readFileSync(join(CROW_HOME, "bundles", id, ".env"), "utf8")) }; +} +// "This machine is lost": the installed copy and the retained copy are gone, the keychain rows +// (restored crow.db + imported Export) remain. +function loseFiles(id) { + rmSync(join(CROW_HOME, "bundles", id), { recursive: true, force: true }); + rmSync(S.retainedEnvPath(CROW_HOME, id), { force: true }); +} +async function entry(id, envKey) { + const d = db(); + try { + const e = (await K.listEntries(d, { keyId: loadKeychainKey().id })).find((x) => x.bundle_id === id && x.env_key === envKey); + return e ? { ...e, plain: e.readable ? await K.openEntrySecret(d, loadKeychainKey(), e.id) : null } : null; + } finally { d.close(); } +} + +test("planGeneratedEnv: a keychain value is used when .env, the retained copy and fallbacks have none, and it is reused, never minted", () => { + const m = MANIFEST("plan-a"); + const destDir = mkdtempSync(join(tmpdir(), "crow-kcrestore-dest-")); + const plan = S.planGeneratedEnv("plan-a", m, { destDir, crowHome: CROW_HOME, keychain: { DEMO_DATA_KEY: "kept-data-key", DEMO_ADMIN_TOKEN: "kept-token" } }); + assert.equal(plan.env.DEMO_DATA_KEY, "kept-data-key"); + assert.equal(P.verifyArgon2idPhc("kept-token", plan.env.DEMO_ADMIN_TOKEN), true, "a hashed key gets a hash of the restored plaintext"); + assert.deepEqual(Object.keys(plan.minted), [], "nothing restored is minted"); + assert.deepEqual(plan.restored.sort(), ["DEMO_ADMIN_TOKEN", "DEMO_DATA_KEY"]); + assert.ok(plan.env.DEMO_DB_PASSWORD && plan.env.DEMO_DB_PASSWORD.length >= 40, "a non-keychain key is still minted"); + rmSync(destDir, { recursive: true, force: true }); +}); + +test("planGeneratedEnv: the installed .env and the retained copy still win over the keychain (they are what the data uses)", () => { + const m = MANIFEST("plan-b"); + const destDir = mkdtempSync(join(tmpdir(), "crow-kcrestore-dest-")); + writeFileSync(join(destDir, ".env"), "DEMO_DATA_KEY=from-env\n"); + const plan = S.planGeneratedEnv("plan-b", m, { destDir, crowHome: CROW_HOME, keychain: { DEMO_DATA_KEY: "from-keychain" } }); + assert.equal(plan.env.DEMO_DATA_KEY, "from-env"); + assert.equal(plan.restored.includes("DEMO_DATA_KEY"), false); + rmSync(destDir, { recursive: true, force: true }); +}); + +test("keychainRestoreValues: only keychain:true generated keys, only readable entries", async () => { + const id = "kcr-values"; const m = fixture(id); + await install(id, m); + const v = await H.keychainRestoreValues(id, m); + assert.deepEqual(Object.keys(v).sort(), ["DEMO_ADMIN_TOKEN", "DEMO_DATA_KEY"], "DEMO_DB_PASSWORD is never in the keychain, so never looked up"); + assert.equal(v.DEMO_DATA_KEY, (await entry(id, "DEMO_DATA_KEY")).plain); + assert.deepEqual(await H.keychainRestoreValues(id, { ...m, env_vars: [] }), {}, "a manifest with no keychain keys reads nothing"); +}); + +test("machine lost → reinstall restores the SAME data key from the keychain (and does not overwrite the entry)", async () => { + const id = "kcr-e2e"; const m = fixture(id); + const first = await install(id, m); + assert.match(first.job.log.join("\n"), /export your keychain/, "a first generated key reminds the user to export"); + const before = await entry(id, "DEMO_DATA_KEY"); + assert.equal(first.env.DEMO_DATA_KEY, before.plain); + const tokenBefore = (await entry(id, "DEMO_ADMIN_TOKEN")).plain; + + assert.equal(await H.markBundleKeychainRemoved(id), 2, "uninstall marks the entries removed (they are kept)"); + loseFiles(id); + + const { job, env } = await install(id, m); + assert.equal(env.DEMO_DATA_KEY, before.plain, "the data key is the one the data was saved with"); + assert.equal(P.verifyArgon2idPhc(tokenBefore, env.DEMO_ADMIN_TOKEN), true, "the admin token still signs in"); + const after = await entry(id, "DEMO_DATA_KEY"); + assert.equal(after.plain, before.plain, "the keychain entry was not overwritten"); + assert.equal(after.status, "active", "the entry is active again"); + assert.equal(after.first_view_pending, false, "nothing new to show"); + const retained = parseEnvText(readFileSync(S.retainedEnvPath(CROW_HOME, id), "utf8")); + assert.equal(retained.DEMO_DATA_KEY, before.plain, "the restored key is retained for the next reinstall"); + const log = job.log.join("\n"); + assert.match(log, /Restored 2 key\(s\) from Crow keychain/); + assert.ok(!log.includes(before.plain) && !log.includes(tokenBefore), "no value in the job log"); +}); + +test("an entry sealed with another machine's keychain key is not readable: the install mints, as before, and says so", async () => { + const id = "kcr-unreadable"; const m = fixture(id); + await install(id, m); + const d = db(); + try { await d.execute({ sql: "UPDATE crow_keychain SET key_id = 'ffffffffffffffff' WHERE bundle_id = ? AND env_key = 'DEMO_DATA_KEY'", args: [id] }); } finally { d.close(); } + loseFiles(id); + const { job, env } = await install(id, m); + assert.ok(env.DEMO_DATA_KEY, "a new key was minted"); + assert.match(job.log.join("\n"), /DEMO_DATA_KEY.*another keychain key|could not be read/i); +}); + +test("a keychain read failure never fails the install (best-effort: it mints, as before)", async () => { + const id = "kcr-fail"; const m = fixture(id); + H._setKeychainDepsForTest({ openDb: () => { throw new Error("db down"); } }); + try { + assert.deepEqual(await H.keychainRestoreValues(id, m), {}); + } finally { H._setKeychainDepsForTest(null); } +}); + +test("crow_keychain stays local-only (never synced to a paired instance)", async () => { + const sync = await import("../servers/sharing/instance-sync.js"); + assert.ok((sync.LOCAL_ONLY_TABLES || []).includes("crow_keychain")); + assert.equal((sync.SYNCED_TABLES || []).includes("crow_keychain"), false); +}); From 220e60a35e93e0c887a1a728e04b87f8d5f30832 Mon Sep 17 00:00:00 2001 From: kh0pper <93635715+kh0pper@users.noreply.github.com> Date: Fri, 16 Oct 2026 18:36:32 -0500 Subject: [PATCH 2/4] =?UTF-8?q?fix(keychain):=20review=20=E2=80=94=20never?= =?UTF-8?q?=20mint=20over=20an=20unreadable=20entry;=20boot=20backfill=20o?= =?UTF-8?q?f=20kept=20keys;=20reminder=20counts=20what=20no=20Export=20hol?= =?UTF-8?q?ds;=20mcp-addons.json=20at=20600?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Plan review findings: - An install that would mint a keychain:true key whose entry exists but is sealed with a keychain key this machine lacks (crow.db restored, Export not imported) is refused: the new value would replace that entry and a later Import would skip it, losing the old key for good. Deleting the old entry is the explicit way to start fresh. - Boot backfill: an extension installed before its manifest saved a generated key to the keychain gets that key (from .env, retained copy or MCP entry) saved once, so it is visible in Passwords and goes into the Export (the boot refresh never runs the install job). Never over an existing entry; never a store_as hash. - The Export reminder fingerprints the sealed values the last Export held (and those an Import brought in), so a re-minted or kept key counts and an imported one does not; the health cache is dropped on export and import. - mcp-addons.json (MCP children's generated keys) is written atomically at mode 600. - The delete warning for a generated entry also covers data keys. Claude-Session: https://claude.ai/code/session_01N9fXzvNy8rCEArj1y5p6fp --- servers/gateway/boot/mcp-mounts.js | 7 ++ .../dashboard/panels/nest/health-signals.js | 11 +-- servers/gateway/dashboard/shared/i18n.js | 4 +- servers/gateway/keychain/api.js | 13 ++- servers/gateway/keychain/install-hooks.js | 70 +++++++++++---- servers/gateway/keychain/reminder.js | 73 +++++++++++----- servers/gateway/keychain/store.js | 4 +- servers/gateway/routes/bundles.js | 51 ++++++++++- tests/keychain-export-reminder.test.js | 43 ++++++--- tests/keychain-restore.test.js | 87 +++++++++++++++++-- tests/keychain-store.test.js | 6 +- 11 files changed, 288 insertions(+), 81 deletions(-) diff --git a/servers/gateway/boot/mcp-mounts.js b/servers/gateway/boot/mcp-mounts.js index 6ceb3bdea..6e91cb26c 100644 --- a/servers/gateway/boot/mcp-mounts.js +++ b/servers/gateway/boot/mcp-mounts.js @@ -235,6 +235,13 @@ export async function mountMcpServers(app, deps) { if (errors.length > 0) { console.warn(`[bundles] Asset repair errors:`, errors); } + // Recovery keys: keys an older install kept only in its .env go into the keychain once. + try { + const { backfillBundleKeychain } = await import("../routes/bundles.js"); + await backfillBundleKeychain(); + } catch (err) { + console.warn(`[keychain] backfill skipped: ${err.message}`); + } // Skill sync: one notification per skill the owner EDITED, once per new // shipped version (the record's notifiedFor). Unverified copies are never // notified — we cannot tell the owner they edited something we cannot verify. diff --git a/servers/gateway/dashboard/panels/nest/health-signals.js b/servers/gateway/dashboard/panels/nest/health-signals.js index 2730955b1..fe8509ca8 100644 --- a/servers/gateway/dashboard/panels/nest/health-signals.js +++ b/servers/gateway/dashboard/panels/nest/health-signals.js @@ -15,7 +15,7 @@ * updates — auto_update_* version comparison (info if update available) * backup — newest of THIS instance's files in CROW_BACKUP_DIR (none → info; * >7d → warn; owned by another instance → warn) - * keychainExport — keys Crow generated that are in no keychain Export yet (info; no + * keychainExport — extension passwords that are in no keychain Export yet (info; no * card when there are none) * providers — alwaysResident provider residency (unreachable ≥threshold → warn) * externalEngines — engines another machine runs (spec 2026-09-23): own id, @@ -58,7 +58,8 @@ import { readReservation } from "../../../box-reservation.js"; import { localBackupContext, newestBackupOwnership } from "../../../../shared/backup-naming.js"; import { getStats as getOutboxStats } from "../../../../sharing/sync-outbox-drain.js"; import { createSidecarHealthReader } from "../../../artifact-origin/sidecar-health.js"; -import { unexportedGeneratedCount } from "../../../keychain/reminder.js"; +import { unexportedCount } from "../../../keychain/reminder.js"; +import { loadKeychainKey } from "../../../keychain/key.js"; // ─── Module-level 30s cache ─────────────────────────────────────────────────── @@ -675,11 +676,11 @@ async function backupSignal(db, nowFn, lang) { }; } -// Keychain Export reminder (recovery keys, queue item 30): a key Crow generated that is in no -// keychain Export lives only on this machine. Info only (strip, never a push); no card +// Keychain Export reminder (recovery keys, queue item 30): a password an extension saved to the +// keychain that is in no keychain Export lives only on this machine. Info only (strip, never a push); no card // when every generated key is exported or there are none. async function keychainExportSignal(db, lang) { - const n = await unexportedGeneratedCount(db); + const n = await unexportedCount(db, loadKeychainKey()?.id); if (n <= 0) return null; return { id: "keychainExport", severity: "info", state: "info", label: t("signals.keychainExport.label", lang), diff --git a/servers/gateway/dashboard/shared/i18n.js b/servers/gateway/dashboard/shared/i18n.js index 1aaacec34..3db4c6c00 100644 --- a/servers/gateway/dashboard/shared/i18n.js +++ b/servers/gateway/dashboard/shared/i18n.js @@ -230,7 +230,7 @@ export const translations = { // backup "signals.backup.label": { en: "Backup", es: "Copia de seguridad" }, "signals.keychainExport.label": { en: "Keychain export", es: "Exportación del llavero" }, - "signals.keychainExport.issue": { en: "{n} password(s) Crow generated are not in a keychain Export yet. If this machine is lost, they cannot be recovered without one.", es: "{n} contraseña(s) que Crow generó aún no están en una exportación del llavero. Si pierdes esta máquina, no se pueden recuperar sin una." }, + "signals.keychainExport.issue": { en: "{n} saved extension password(s) are not in a keychain Export yet. If this machine is lost, they cannot be recovered without one.", es: "{n} contraseña(s) guardadas de extensiones aún no están en una exportación del llavero. Si pierdes esta máquina, no se pueden recuperar sin una." }, "signals.keychainExport.action": { en: "Export keychain", es: "Exportar llavero" }, "signals.backup.verified": { en: "checked ✓", es: "verificada ✓" }, "signals.backup.damaged": { en: "may be damaged", es: "puede estar dañada" }, @@ -2358,7 +2358,7 @@ export const translations = { "passwords.statusRemoved": { en: "Extension removed", es: "Extensión eliminada" }, "passwords.statusUnreadable": { en: "Unreadable — key missing", es: "Ilegible: falta la clave" }, "passwords.noReauth": { en: "Revealing, copying, deleting and exporting passwords need Crow to confirm it's you, but this Crow has no dashboard password and no two-factor authentication. Set one in Settings → Change Password or Two-Factor Auth first.", es: "Para mostrar, copiar, eliminar o exportar contraseñas, Crow tiene que confirmar que eres tú, pero esta instancia no tiene contraseña del panel ni autenticación de dos factores. Configura una primero en Ajustes → Cambiar contraseña o Autenticación 2FA." }, - "passwords.deleteGeneratedConfirm": { en: "Crow generated this token and keeps no other copy. Without it you cannot sign in to the extension's admin page. Delete it anyway?", es: "Crow generó este token y no guarda otra copia. Sin él no podrás entrar en la página de administración de la extensión. ¿Eliminarlo de todos modos?" }, + "passwords.deleteGeneratedConfirm": { en: "Crow generated this and keeps no other copy. Without it you may not be able to sign in to the extension, or read data it encrypted. Delete it anyway?", es: "Crow generó esto y no guarda otra copia. Sin ello quizá no puedas entrar en la extensión ni leer los datos que cifró. ¿Eliminarlo de todos modos?" }, "passwords.exportTitle": { en: "Export and import", es: "Exportar e importar" }, "passwords.exportHint": { en: "Export writes every readable password into one file encrypted with a passphrase you choose. Keep it offline. Import it here, or on a new machine, to get the passwords back.", es: "Exportar guarda todas las contraseñas legibles en un archivo cifrado con una frase de paso que eliges tú. Guárdalo fuera de línea. Impórtalo aquí, o en una máquina nueva, para recuperar las contraseñas." }, "passwords.exportPassphrase": { en: "Passphrase (at least 12 characters)", es: "Frase de paso (al menos 12 caracteres)" }, diff --git a/servers/gateway/keychain/api.js b/servers/gateway/keychain/api.js index 6a5b56e60..0238012c9 100644 --- a/servers/gateway/keychain/api.js +++ b/servers/gateway/keychain/api.js @@ -18,7 +18,8 @@ import { exportableEntries, importEntries, ensureWriteKey, } from "./store.js"; import { vaultwardenStatus, saveToVault } from "./vault-save.js"; -import { recordKeychainExport } from "./reminder.js"; +import { recordKeychainExport, recordKeychainImport } from "./reminder.js"; +import { invalidateHealthCache } from "../dashboard/panels/nest/health-signals.js"; const BASE = "/dashboard/keychain/api"; @@ -166,7 +167,7 @@ export function keychainApiRouter({ if (!entry) return res.status(404).json({ error: "No such password." }); // C5: a generated token may be the ONLY plaintext of a hash the extension still uses. if (entry.origin === "generated" && entry.status === "active" && req.body?.confirm_generated !== true) { - return res.status(409).json({ code: "generated_in_use", error: `Crow generated this token and keeps no other copy: without it you cannot sign in to ${entry.bundle_id || "the extension"}'s admin page. Export or save it elsewhere first.` }); + return res.status(409).json({ code: "generated_in_use", error: `Crow generated this and keeps no other copy: without it you may not be able to sign in to ${entry.bundle_id || "the extension"}, or read data it encrypted. Export or save it elsewhere first.` }); } await deleteEntry(db, entry.id); await audit(db, "keychain_delete", { ip: req.ip, details: auditDetails(entry) }); @@ -181,7 +182,8 @@ export function keychainApiRouter({ if (!key) return res.status(409).json(KEY_MISSING); const entries = await exportableEntries(db, key); const file = await sealExport(entries, passphrase); - await recordKeychainExport(db).catch(() => {}); // the Nest reminder; never blocks the download + await recordKeychainExport(db, key.id).catch(() => {}); // the Nest reminder; never blocks the download + invalidateHealthCache(); await audit(db, "keychain_export", { ip: req.ip, details: { count: entries.length } }); res.set("Content-Disposition", `attachment; filename="crow-keychain-${new Date().toISOString().slice(0, 10)}.json"`); res.json(file); @@ -191,7 +193,10 @@ export function keychainApiRouter({ if (!(await requireGrant(req, res))) return; const entries = await openExport(req.body?.file, typeof req.body?.passphrase === "string" ? req.body.passphrase : ""); if (!entries) return res.status(400).json({ error: "That passphrase does not open this file, or it is not a Crow keychain export." }); - const out = await importEntries(db, await writeKey(db, req.ip), entries); + const ikey = await writeKey(db, req.ip); + const { ids, ...out } = await importEntries(db, ikey, entries); + await recordKeychainImport(db, ikey.id, ids).catch(() => {}); + invalidateHealthCache(); await audit(db, "keychain_import", { ip: req.ip, details: out }); res.json({ ok: true, ...out }); })); diff --git a/servers/gateway/keychain/install-hooks.js b/servers/gateway/keychain/install-hooks.js index 57d1677a2..c949e21b1 100644 --- a/servers/gateway/keychain/install-hooks.js +++ b/servers/gateway/keychain/install-hooks.js @@ -163,39 +163,75 @@ export async function recordKeychainForInstall({ bundleId, manifest, env, minted } /** - * Plaintext of this bundle's keychain:true GENERATED keys that a readable keychain entry - * holds (any status: an uninstall marks entries extension_removed and keeps them), for - * planGeneratedEnv's keychain rung. This is how a rebuilt machine gets its keys back: restore - * crow.db, import the keychain Export, reinstall. Best-effort and read-only: no key file, a - * damaged one, or no table → {} and the install mints as before. An entry sealed under - * another keychain key is reported by NAME through `log`, never read. Values never leave - * this process: they are not logged and never synced (crow_keychain is LOCAL_ONLY). + * For planGeneratedEnv's keychain rung: `values` = plaintext of this bundle's keychain:true + * GENERATED keys that a readable entry holds (any status: an uninstall marks entries + * extension_removed and keeps them); `unreadable` = such keys whose entry exists but is + * sealed with a keychain key this machine does not have (or has no key file at all). This + * is how a rebuilt machine gets its keys back: restore crow.db, import the keychain Export, + * reinstall. Read-only and best-effort: a failure returns nothing and the install mints as + * before. Values are never logged and never synced (crow_keychain is LOCAL_ONLY). */ -export async function keychainRestoreValues(bundleId, manifest, { log = () => {} } = {}) { +export async function keychainRestoreValues(bundleId, manifest) { const keys = keychainGeneratedKeys(manifest); - if (keys.length === 0) return {}; - const out = {}; + const out = { values: {}, unreadable: [] }; + if (keys.length === 0) return out; let db; try { const key = deps().readKey(); - if (!key) return {}; db = deps().openDb(); - const unreadable = []; - for (const e of await listEntries(db, { keyId: key.id })) { + for (const e of await listEntries(db, { keyId: key?.id ?? null })) { if (e.kind !== "extension" || e.bundle_id !== bundleId || !keys.includes(e.env_key)) continue; - if (!e.readable) { unreadable.push(e.env_key); continue; } + if (!e.readable || !key) { out.unreadable.push(e.env_key); continue; } const plain = await openEntrySecret(db, key, e.id); - if (typeof plain === "string" && plain !== "") out[e.env_key] = plain; + if (typeof plain === "string" && plain !== "") out.values[e.env_key] = plain; } - if (unreadable.length) log(`Crow keychain holds ${unreadable.join(", ")} sealed with another keychain key, so it could not be read; a new one is generated. Import the keychain Export from the old machine first to keep the old one.`); } catch { - return {}; + return { values: {}, unreadable: [] }; } finally { try { db?.close(); } catch {} } return out; } +/** + * Boot backfill (recovery keys, queue item 30): an installed extension whose manifest now + * saves a generated key to the keychain (campaigns/tax, mastodon/pixelfed/peertube data + * keys) but was installed before that has the key only in its .env / retained copy / MCP + * entry, and the field is no longer shown in Configure. Save each such value once, so it is + * visible in Settings → Passwords and goes into the user's Export. Never overwrites an + * entry (any status, readable or not); skips store_as keys (only a hash is on disk). No + * first view (nothing new was minted). Best-effort; names only in `log`. + * `values` = { envKey: { plain, origin } } gathered by the caller. + */ +export async function backfillKeychainEntries(bundleId, manifest, values, { log = () => {} } = {}) { + const specs = new Map((manifest?.env_vars || []).map((v) => [v.name, v])); + const todo = keychainGeneratedKeys(manifest).filter((k) => !specs.get(k)?.store_as && values?.[k]?.plain); + if (todo.length === 0) return 0; + let db; + let saved = 0; + try { + db = deps().openDb(); + const missing = []; + for (const k of todo) if (!(await hasExtensionEntry(db, bundleId, k))) missing.push(k); + if (missing.length === 0) return 0; + const key = await deps().writeKey(db); + const tEnv = templateEnv(manifest, Object.fromEntries(Object.entries(values).map(([k, v]) => [k, v.plain]))); + for (const k of missing) { + const spec = specs.get(k) || {}; + const label = `${manifest?.name || bundleId} — ${spec.keychain_label || k}`; + const r = await saveExtensionSecret(db, key, { bundleId, envKey: k, label, username: expandKeychainTemplate(spec.keychain_username, tEnv), url: expandKeychainTemplate(spec.keychain_url, tEnv), secret: values[k].plain, origin: values[k].origin === "generated" ? "generated" : "typed", firstView: false }); + await deps().audit(db, "keychain_save", { details: { entry_id: r.id, bundle_id: bundleId, env_key: k, origin: "kept", backfill: true } }); + saved++; + } + log(`${bundleId}: saved ${missing.join(", ")} to Crow keychain (kept from the existing install)`); + } catch (err) { + log(`${bundleId}: could not save kept keys to Crow keychain (${err?.code || err?.name || "error"})`); + } finally { + try { db?.close(); } catch {} + } + return saved; +} + export async function markBundleKeychainRemoved(bundleId) { let db; try { diff --git a/servers/gateway/keychain/reminder.js b/servers/gateway/keychain/reminder.js index 6a21e8a66..6074494d0 100644 --- a/servers/gateway/keychain/reminder.js +++ b/servers/gateway/keychain/reminder.js @@ -1,39 +1,64 @@ /** * Keychain Export reminder (recovery keys, queue item 30, decision 2026-10-16). * - * A key Crow GENERATES for an extension (an admin password, a data encryption key) exists - * only on this machine: the keychain never syncs, and backups never copy secrets/. The - * user's passphrase-encrypted Export is the one off-machine copy. So while a generated - * key is in no Export, the Nest shows an info issue (health signal "keychainExport"). + * A key an extension saved to the keychain (Crow generated it, or it was kept from an older + * install) exists only on this machine: the keychain never syncs, and backups never copy + * secrets/. The user's passphrase-encrypted Export is the one off-machine copy. So while an + * extension entry is in no Export, the Nest shows an info issue (health signal + * "keychainExport"), and the install log says so. * - * "In no Export" = created after the last successful export (or there was never one). - * The time is a LOCAL setting (not in the sync allowlist): another instance's export says - * nothing about this machine's keys. audit_log is not used: it is pruned after 90 days. + * "In an Export" = the entry's sealed value was fingerprinted by the last export, or it came + * in through an Import (it is in that file). A re-sealed value (a re-mint, a re-save) has a + * new fingerprint, so it counts again. The fingerprints are a LOCAL setting (not in the sync + * allowlist): another instance's export says nothing about this machine's keys. audit_log is + * not used: it is pruned after 90 days. Manual entries are the user's own and never counted. */ +import { createHash } from "node:crypto"; import { readSetting, writeSetting } from "../dashboard/settings/registry.js"; import { ensureKeychainTable } from "./store.js"; -export const LAST_EXPORT_SETTING = "keychain_last_export_at"; +export const LAST_EXPORT_SETTING = "keychain_export_fingerprints"; -/** Call after an Export file was sealed and is about to be sent. */ -export async function recordKeychainExport(db, at = new Date()) { - await writeSetting(db, LAST_EXPORT_SETTING, at.toISOString(), { scope: "local" }); +const fp = (sealed) => createHash("sha256").update(String(sealed)).digest("hex").slice(0, 32); + +async function readState(db) { + try { + const v = JSON.parse((await readSetting(db, LAST_EXPORT_SETTING)) || "null"); + if (v && typeof v === "object" && v.fp && typeof v.fp === "object") return v; + } catch { /* none or damaged: nothing counts as exported */ } + return { at: null, fp: {} }; +} + +async function activeRows(db, keyId, ids = null) { + await ensureKeychainTable(db); + const { rows } = await db.execute({ sql: "SELECT id, secret_sealed FROM crow_keychain WHERE kind = 'extension' AND status = 'active' AND key_id = ?", args: [String(keyId || "")] }); + return ids ? rows.filter((r) => ids.has(Number(r.id))) : rows; +} + +/** After an Export was sealed: every readable extension entry is in that file. */ +export async function recordKeychainExport(db, keyId, at = new Date()) { + const fps = {}; + for (const r of await activeRows(db, keyId)) fps[Number(r.id)] = fp(r.secret_sealed); + await writeSetting(db, LAST_EXPORT_SETTING, JSON.stringify({ at: at.toISOString(), fp: fps }), { scope: "local" }); +} + +/** After an Import: the imported entries are in the file they came from. */ +export async function recordKeychainImport(db, keyId, ids) { + if (!Array.isArray(ids) || ids.length === 0) return; + const st = await readState(db); + for (const r of await activeRows(db, keyId, new Set(ids.map(Number)))) st.fp[Number(r.id)] = fp(r.secret_sealed); + await writeSetting(db, LAST_EXPORT_SETTING, JSON.stringify(st), { scope: "local" }); } export async function lastKeychainExport(db) { - const v = await readSetting(db, LAST_EXPORT_SETTING); - return typeof v === "string" && !Number.isNaN(Date.parse(v)) ? v : null; + return (await readState(db)).at; } -/** Active generated extension entries created after the last export (all when none). */ -export async function unexportedGeneratedCount(db) { - await ensureKeychainTable(db); - const since = await lastKeychainExport(db); - const { rows } = await db.execute({ - sql: `SELECT COUNT(*) AS n FROM crow_keychain - WHERE kind = 'extension' AND origin = 'generated' AND status = 'active' - AND (? IS NULL OR created_at > ?)`, - args: [since, since], - }); - return Number(rows[0]?.n ?? 0); +/** Active, readable extension entries whose current sealed value is in no Export. */ +export async function unexportedCount(db, keyId) { + if (!keyId) return 0; + const st = await readState(db); + let n = 0; + for (const r of await activeRows(db, keyId)) if (st.fp[Number(r.id)] !== fp(r.secret_sealed)) n++; + return n; } diff --git a/servers/gateway/keychain/store.js b/servers/gateway/keychain/store.js index 2914e73b1..e67197e91 100644 --- a/servers/gateway/keychain/store.js +++ b/servers/gateway/keychain/store.js @@ -199,6 +199,7 @@ export async function importEntries(db, key, entries) { const current = await listEntries(db, { keyId: key.id }); let imported = 0; let skipped = 0; + const ids = []; for (let e of Array.isArray(entries) ? entries : []) { // A non-http(s) url is dropped (not the entry) so an otherwise-good export still imports. if (e && typeof e.url === "string" && e.url && !/^https?:\/\//i.test(e.url)) e = { ...e, url: null }; @@ -209,6 +210,7 @@ export async function importEntries(db, key, entries) { const origin = e.origin === "generated" ? "generated" : "typed"; const { id } = await saveExtensionSecret(db, key, { bundleId: e.bundle_id, envKey: e.env_key, label: e.label, username: e.username, url: e.url, secret: e.secret, origin }); if (e.status === "extension_removed") await db.execute({ sql: "UPDATE crow_keychain SET status = 'extension_removed' WHERE id = ?", args: [id] }); + ids.push(id); imported++; } else { const dup = current.some((c) => c.kind === "manual" && c.readable && c.label === cleanText(e.label) && (c.username || null) === cleanText(e.username) && (c.url || null) === cleanText(e.url, 2048)); @@ -217,5 +219,5 @@ export async function importEntries(db, key, entries) { imported++; } } - return { imported, skipped }; + return { imported, skipped, ids }; } diff --git a/servers/gateway/routes/bundles.js b/servers/gateway/routes/bundles.js index ab662a18e..8025fa09d 100644 --- a/servers/gateway/routes/bundles.js +++ b/servers/gateway/routes/bundles.js @@ -55,14 +55,14 @@ import { import { isModelOrchestrationDisabled, isModelBundleManifest } from "../../shared/model-orchestration.js"; import { syncBundleSkills, removeBundleSkills, readSkillLedger, skillSyncMode } from "../bundle-skill-sync.js"; import { envValueProblem, encodeEnvValue, formatEnvLines, updateEnvText, pathEnvKeys } from "../bundle-env-codec.js"; -import { sanitizeKeychainRequest, recordKeychainForInstall, markBundleKeychainRemoved, keychainRestoreValues } from "../keychain/install-hooks.js"; +import { sanitizeKeychainRequest, recordKeychainForInstall, markBundleKeychainRemoved, keychainRestoreValues, backfillKeychainEntries } from "../keychain/install-hooks.js"; import { precreateDirs, runPostInstall, hookEnv, spawnGroup, pullTimeoutMs, resolveComposeProject, classifyProjectOwners } from "../bundle-lifecycle.js"; import { provisionNtfy, deprovisionNtfy, AUTOWIRE_KIND } from "../push/ntfy-provision.js"; import { checkComposeModel, loadComposeModel, usesHostNetwork } from "../compose-guard.js"; import { resolveDataDir } from "../../db.js"; import { readStoredNtfyConfig } from "../push/ntfy-config.js"; import { isPlaceholderValue } from "../bundle-env-placeholders.js"; -import { planGeneratedEnv, retainedEnvPath, stripGeneratedKeys, parseEnvText, writePrivateFile, gatewayExcludedKeys, gatewayGeneratedKeys, retainGeneratedForReinstall, envPatternViolation, breachedValueViolation } from "../bundle-env-secrets.js"; +import { keychainGeneratedKeys, planGeneratedEnv, retainedEnvPath, stripGeneratedKeys, parseEnvText, writePrivateFile, gatewayExcludedKeys, gatewayGeneratedKeys, retainGeneratedForReinstall, envPatternViolation, breachedValueViolation } from "../bundle-env-secrets.js"; /** * Seed an STT/TTS profile from a bundle manifest's {stt,tts}ProfileSeed into the @@ -984,6 +984,35 @@ function installBundleSkills(bundleId, destDir, manifest, job) { * defaults to the module's `run`). * @returns {Promise<{repaired: string[], errors: Array<{id, error}>}>} */ +/** + * Boot: save the keychain:true generated keys of extensions installed before their manifest + * saved them to the keychain (install-hooks backfillKeychainEntries). Values come from the + * installed .env, else the retained copy, else the MCP entry; never logged. + */ +export async function backfillBundleKeychain({ log = (m) => console.log(`[keychain] ${m}`) } = {}) { + let n = 0; + const addons = readJsonSafe(MCP_ADDONS_PATH, {}) || {}; + for (const rec of getInstalled()) { + const id = rec?.id; + if (typeof id !== "string" || !/^[a-z0-9][a-z0-9-]*$/.test(id)) continue; + const manifest = getInstalledFirstManifest(id); + const keys = keychainGeneratedKeys(manifest); + if (keys.length === 0) continue; + let installed = {}, retained = {}; + try { installed = parseEnvText(readFileSync(join(BUNDLES_DIR, id, ".env"), "utf8")); } catch { /* none */ } + try { retained = parseEnvText(readFileSync(retainedEnvPath(CROW_HOME, id), "utf8")); } catch { /* none */ } + const entryEnv = addons?.[id]?.env || {}; + const values = {}; + for (const k of keys) { + const plain = installed[k] || retained[k] || entryEnv[k] || ""; + // The retained copy only ever holds values Crow generated; anything else was typed. + if (plain) values[k] = { plain, origin: retained[k] && retained[k] === plain ? "generated" : "typed" }; + } + n += await backfillKeychainEntries(id, manifest, values, { log }); + } + return n; +} + export async function repairInstalledBundleAssets({ appBundles = APP_BUNDLES, run: runner = run, skillMode = skillSyncMode(), skillLedger = null, log = (m) => console.log(m) } = {}) { const repaired = []; const errors = []; @@ -1237,6 +1266,9 @@ function readJsonSafe(path, fallback) { /** Write JSON file (creates parent dirs) */ function writeJsonSafe(path, data) { mkdirSync(dirname(path), { recursive: true }); + // mcp-addons.json carries MCP children's secrets (generated keys reach their MCP entry + // here): always written atomically at mode 600, whatever the umask or the old file's mode. + if (path === MCP_ADDONS_PATH) { writePrivateFile(path, JSON.stringify(data, null, 2)); return; } writeFileSync(path, JSON.stringify(data, null, 2)); } @@ -2512,8 +2544,19 @@ export async function runInstallJob(bundleId, envVars, { job, installedSnapshot, // A rebuilt machine (crow.db restored + keychain Export imported) has no .env and no // retained copy: its keychain:true generated keys come back from the keychain instead of // being minted again (recovery keys, queue item 30). Best-effort; values never logged. - const fromKeychain = await keychainRestoreValues(bundleId, manifest, { log: (m) => appendLog(job, m) }); - const plan = planGeneratedEnv(bundleId, manifest, { destDir, crowHome: CROW_HOME, composeText: composeTextForEnv, priorInstall: priorInstallEnv, keychain: fromKeychain }); + const kcRestore = await keychainRestoreValues(bundleId, manifest); + const plan = planGeneratedEnv(bundleId, manifest, { destDir, crowHome: CROW_HOME, composeText: composeTextForEnv, priorInstall: priorInstallEnv, keychain: kcRestore.values }); + // A key the keychain holds but cannot read (sealed with a keychain key this machine does + // not have: crow.db restored, Export not imported yet) is never minted over: the new + // value would replace that entry, and a later Import skips a bundle+key that has a + // readable entry, so the old key (and the data saved with it) would be lost for good. + const blocked = Object.keys(plan.minted).filter((k) => kcRestore.unreadable.includes(k)); + if (blocked.length > 0) { + rmSync(destDir, { recursive: true, force: true }); + const reason = `Crow keychain holds ${blocked.join(", ")} for this extension, sealed with a keychain key this machine does not have. Import your keychain Export first (Settings → Passwords → Import), then install again. To start fresh instead, delete the old saved password(s) there.`; + appendLog(job, reason); + return { ok: false, reason }; + } if (plan.restored.length > 0) appendLog(job, `Restored ${plan.restored.length} key(s) from Crow keychain (Settings → Passwords): ${plan.restored.join(", ")}`); const generated = plan.env; const writeEnv = { ...(reqEnv || {}), ...generated }; diff --git a/tests/keychain-export-reminder.test.js b/tests/keychain-export-reminder.test.js index cd37a119c..4e95838b3 100644 --- a/tests/keychain-export-reminder.test.js +++ b/tests/keychain-export-reminder.test.js @@ -43,33 +43,50 @@ test("no generated keys → no reminder at all (a manual entry never counts)", a assert.equal(s.card, null); }); -test("a generated key that was never exported → an info issue that links to Settings → Passwords", async () => { +test("an extension key that was never exported → an info issue that links to Settings → Passwords", async () => { await K.saveExtensionSecret(db, key, { bundleId: "tax", envKey: "CROW_TAX_ENCRYPTION_KEY", label: "Tax — data encryption key", secret: "k1-" + "x".repeat(40), origin: "generated" }); const s = await signal(); assert.ok(s.issue, "the reminder is shown"); assert.equal(s.issue.severity, "info", "info only: shown on the Nest strip, never pushed"); - assert.match(s.issue.label, /1 .*not in a keychain Export/i); + assert.match(s.issue.label, /^1 .*not in a keychain Export/i); assert.equal(s.issue.actionHref, "/dashboard/settings?section=passwords"); - assert.deepEqual(await R.unexportedGeneratedCount(db), 1); }); -test("after an export the reminder is gone; a key generated after the export brings it back", async () => { - await R.recordKeychainExport(db, new Date(Date.now() + 1000)); - assert.equal((await signal()).issue, null, "everything generated is in an Export"); - await new Promise((r) => setTimeout(r, 1100)); +test("a key KEPT from an older install (origin typed) counts too", async () => { + await K.saveExtensionSecret(db, key, { bundleId: "mastodon", envKey: "MASTODON_OTP_SECRET", label: "Mastodon — two-factor secret", secret: "kept-" + "z".repeat(40), origin: "typed" }); + assert.equal(await R.unexportedCount(db, key.id), 2); +}); + +test("after an export the reminder is gone; a key saved or re-saved after the export brings it back", async () => { + await R.recordKeychainExport(db, key.id); + assert.equal((await signal()).issue, null, "everything is in an Export"); await K.saveExtensionSecret(db, key, { bundleId: "campaigns", envKey: "CROW_CAMPAIGNS_ENCRYPTION_KEY", label: "Campaigns — data encryption key", secret: "k2-" + "y".repeat(40), origin: "generated" }); - const s = await signal(); - assert.ok(s.issue, "the newer key is in no Export"); - assert.match(s.issue.label, /^1 /); + assert.match((await signal()).issue.label, /^1 /, "a new key is in no Export"); + await R.recordKeychainExport(db, key.id); + // A re-mint over an existing row keeps its created_at but is a new secret: it counts again. + await K.saveExtensionSecret(db, key, { bundleId: "tax", envKey: "CROW_TAX_ENCRYPTION_KEY", label: "Tax — data encryption key", secret: "k3-" + "w".repeat(40), origin: "generated" }); + assert.equal(await R.unexportedCount(db, key.id), 1); +}); + +test("entries that came in through an Import are in that file: they do not count", async () => { + await R.recordKeychainExport(db, key.id); + const { ids } = await K.importEntries(db, key, [{ kind: "extension", bundle_id: "pixelfed", env_key: "PIXELFED_APP_KEY", label: "Pixelfed — data encryption key", secret: "base64:" + "q".repeat(44), origin: "generated" }]); + assert.equal(ids.length, 1); + assert.equal(await R.unexportedCount(db, key.id), 1, "before recording the import it would count"); + await R.recordKeychainImport(db, key.id, ids); + assert.equal(await R.unexportedCount(db, key.id), 0); }); -test("the last-export time is local: it is not in the sync allowlist", () => { +test("the export fingerprints are local: not in the sync allowlist", () => { assert.equal(isSyncable(R.LAST_EXPORT_SETTING), false); }); -test("POST /api/keychain/export records the export time (route wiring)", async () => { +test("the export and import routes record the reminder state and drop the health cache", async () => { const src = (await import("node:fs")).readFileSync(new URL("../servers/gateway/keychain/api.js", import.meta.url), "utf8"); const exportRoute = src.slice(src.indexOf("/export`"), src.indexOf("/import`")); - assert.match(exportRoute, /recordKeychainExport\(db/); + assert.match(exportRoute, /recordKeychainExport\(db, key\.id\)/); assert.ok(exportRoute.indexOf("recordKeychainExport") > exportRoute.indexOf("sealExport"), "only after the file is sealed"); + assert.match(exportRoute, /invalidateHealthCache\(\)/); + const importRoute = src.slice(src.indexOf("/import`"), src.indexOf("/vault-save`")); + assert.match(importRoute, /recordKeychainImport\(db, ikey\.id, ids\)/); }); diff --git a/tests/keychain-restore.test.js b/tests/keychain-restore.test.js index 564c5ed3c..789350202 100644 --- a/tests/keychain-restore.test.js +++ b/tests/keychain-restore.test.js @@ -7,7 +7,7 @@ */ import { test, after } from "node:test"; import assert from "node:assert/strict"; -import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, rmSync, existsSync } from "node:fs"; +import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, rmSync, existsSync, statSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { execFileSync } from "node:child_process"; @@ -97,10 +97,11 @@ test("planGeneratedEnv: the installed .env and the retained copy still win over test("keychainRestoreValues: only keychain:true generated keys, only readable entries", async () => { const id = "kcr-values"; const m = fixture(id); await install(id, m); - const v = await H.keychainRestoreValues(id, m); + const { values: v, unreadable } = await H.keychainRestoreValues(id, m); assert.deepEqual(Object.keys(v).sort(), ["DEMO_ADMIN_TOKEN", "DEMO_DATA_KEY"], "DEMO_DB_PASSWORD is never in the keychain, so never looked up"); + assert.deepEqual(unreadable, []); assert.equal(v.DEMO_DATA_KEY, (await entry(id, "DEMO_DATA_KEY")).plain); - assert.deepEqual(await H.keychainRestoreValues(id, { ...m, env_vars: [] }), {}, "a manifest with no keychain keys reads nothing"); + assert.deepEqual(await H.keychainRestoreValues(id, { ...m, env_vars: [] }), { values: {}, unreadable: [] }, "a manifest with no keychain keys reads nothing"); }); test("machine lost → reinstall restores the SAME data key from the keychain (and does not overwrite the entry)", async () => { @@ -128,22 +129,32 @@ test("machine lost → reinstall restores the SAME data key from the keychain (a assert.ok(!log.includes(before.plain) && !log.includes(tokenBefore), "no value in the job log"); }); -test("an entry sealed with another machine's keychain key is not readable: the install mints, as before, and says so", async () => { +test("an entry sealed with another machine's keychain key (Export not imported yet): the install is REFUSED, nothing is minted over it", async () => { const id = "kcr-unreadable"; const m = fixture(id); await install(id, m); + const sealedBefore = async () => { const d = db(); try { return (await d.execute({ sql: "SELECT secret_sealed FROM crow_keychain WHERE bundle_id = ? AND env_key = 'DEMO_DATA_KEY'", args: [id] })).rows[0].secret_sealed; } finally { d.close(); } }; const d = db(); try { await d.execute({ sql: "UPDATE crow_keychain SET key_id = 'ffffffffffffffff' WHERE bundle_id = ? AND env_key = 'DEMO_DATA_KEY'", args: [id] }); } finally { d.close(); } + const sealed = await sealedBefore(); loseFiles(id); - const { job, env } = await install(id, m); - assert.ok(env.DEMO_DATA_KEY, "a new key was minted"); - assert.match(job.log.join("\n"), /DEMO_DATA_KEY.*another keychain key|could not be read/i); + const job = B._createJobForTest(id, "install"); + const out = await B.runInstallJob(id, {}, { job, installedSnapshot: [], consentVerified: false, manifest: m }); + assert.equal(out.ok, false); + assert.match(out.reason, /DEMO_DATA_KEY.*Import your keychain Export first/); + assert.equal(await sealedBefore(), sealed, "the old sealed key is untouched, so a later Import can still bring it back"); + assert.equal(existsSync(join(CROW_HOME, "bundles", id)), false, "nothing was written"); + // Starting fresh is the user's explicit choice: delete the old entry, then the install mints. + const d2 = db(); + try { await d2.execute({ sql: "DELETE FROM crow_keychain WHERE bundle_id = ? AND env_key = 'DEMO_DATA_KEY'", args: [id] }); } finally { d2.close(); } + const again = await install(id, m); + assert.ok(again.env.DEMO_DATA_KEY); }); test("a keychain read failure never fails the install (best-effort: it mints, as before)", async () => { const id = "kcr-fail"; const m = fixture(id); H._setKeychainDepsForTest({ openDb: () => { throw new Error("db down"); } }); try { - assert.deepEqual(await H.keychainRestoreValues(id, m), {}); + assert.deepEqual(await H.keychainRestoreValues(id, m), { values: {}, unreadable: [] }); } finally { H._setKeychainDepsForTest(null); } }); @@ -152,3 +163,63 @@ test("crow_keychain stays local-only (never synced to a paired instance)", async assert.ok((sync.LOCAL_ONLY_TABLES || []).includes("crow_keychain")); assert.equal((sync.SYNCED_TABLES || []).includes("crow_keychain"), false); }); + +test("mcp-addons.json (it carries MCP children's generated keys) is written at mode 600", async () => { + const id = "kcr-mcp"; + mkdirSync(join(FIXTURES, id), { recursive: true }); + const m = { id, name: "Demo MCP", description: "d", type: "mcp-server", category: "productivity", version: "0.1.0", + server: { command: "node", args: ["server/index.js"], envKeys: ["DEMO_MCP_KEY"] }, + env_vars: [{ name: "DEMO_MCP_KEY", secret: true, generate: "secret", keychain: true, keychain_label: "data encryption key" }] }; + writeFileSync(join(FIXTURES, id, "manifest.json"), JSON.stringify(m)); + mkdirSync(join(FIXTURES, id, "server"), { recursive: true }); + writeFileSync(join(FIXTURES, id, "server", "index.js"), ""); + const job = B._createJobForTest(id, "install"); + const out = await B.runInstallJob(id, {}, { job, installedSnapshot: [], consentVerified: false, manifest: m }); + assert.equal(out.ok, true, out.reason); + const path = join(CROW_HOME, "mcp-addons.json"); + assert.equal(statSync(path).mode & 0o777, 0o600); + const entryEnv = JSON.parse(readFileSync(path, "utf8"))[id].env; + assert.equal(entryEnv.DEMO_MCP_KEY, (await entry(id, "DEMO_MCP_KEY")).plain, "the MCP child gets the keychain's plaintext"); +}); + +test("crow.db restored on a machine with no keychain key file at all: the install is refused, not minted over", async () => { + const id = "kcr-nokey"; const m = fixture(id); + await install(id, m); + loseFiles(id); + const { keychainKeyPath } = await import("../servers/gateway/keychain/key.js"); + const { renameSync } = await import("node:fs"); + const kp = keychainKeyPath(); + renameSync(kp, kp + ".away"); + try { + const job = B._createJobForTest(id, "install"); + const out = await B.runInstallJob(id, {}, { job, installedSnapshot: [], consentVerified: false, manifest: m }); + assert.equal(out.ok, false); + assert.match(out.reason, /Import your keychain Export first/); + assert.equal(existsSync(kp), false, "no new keychain key was created by the refused install"); + } finally { renameSync(kp + ".away", kp); } +}); + +test("boot backfill: a key kept only in an older install's .env is saved to the keychain once, never over an existing entry", async () => { + const id = "kcr-backfill"; const m = fixture(id); + // An install from before the manifest saved this key to the keychain: .env only. + mkdirSync(join(CROW_HOME, "bundles", id), { recursive: true }); + writeFileSync(join(CROW_HOME, "bundles", id, "manifest.json"), JSON.stringify(m)); + writeFileSync(join(CROW_HOME, "bundles", id, ".env"), "DEMO_DATA_KEY=typed-long-ago-123456\nDEMO_ADMIN_TOKEN=$argon2id$v=19$m=65540,t=3,p=4$AAAA$BBBB\n"); + const instPath = join(CROW_HOME, "installed.json"); + const prev = existsSync(instPath) ? JSON.parse(readFileSync(instPath, "utf8")) : []; + const list = Array.isArray(prev) ? prev : Object.entries(prev).map(([i, v]) => ({ id: i, ...v })); + writeFileSync(instPath, JSON.stringify([...list.filter((r) => r.id !== id), { id, version: "0.1.0" }])); + const logs = []; + const n = await B.backfillBundleKeychain({ log: (l) => logs.push(l) }); + assert.ok(n >= 1); + const e = await entry(id, "DEMO_DATA_KEY"); + assert.equal(e.plain, "typed-long-ago-123456"); + assert.equal(e.origin, "typed"); + assert.equal(e.first_view_pending, false); + assert.equal(await entry(id, "DEMO_ADMIN_TOKEN"), null, "a store_as key has only a hash on disk: never backfilled"); + assert.ok(!logs.join("\n").includes("typed-long-ago"), "names only in the log"); + // A second boot changes nothing, even if the .env now differs. + writeFileSync(join(CROW_HOME, "bundles", id, ".env"), "DEMO_DATA_KEY=something-else-999999\n"); + await B.backfillBundleKeychain({ log: () => {} }); + assert.equal((await entry(id, "DEMO_DATA_KEY")).plain, "typed-long-ago-123456"); +}); diff --git a/tests/keychain-store.test.js b/tests/keychain-store.test.js index c265bf920..4148e299e 100644 --- a/tests/keychain-store.test.js +++ b/tests/keychain-store.test.js @@ -173,13 +173,13 @@ test("Export → Import round-trips through a passphrase file; wrong passphrase const db2 = freshDb(); const key2 = KEY.createKeychainKey({ crowHome: home() }); await K.saveExtensionSecret(db2, key, { bundleId: "workspace", envKey: "WORKSPACE_ADMIN_PASSWORD", label: "WS admin", secret: "stale", origin: "typed" }); - const out = await K.importEntries(db2, key2, await X.openExport(file, "correct horse battery")); + const { ids: _ids, ...out } = await K.importEntries(db2, key2, await X.openExport(file, "correct horse battery")); assert.deepEqual(out, { imported: 2, skipped: 0 }, "the unreadable restored row is replaced"); const list = await K.listEntries(db2, { keyId: key2.id }); assert.ok(list.every((e) => e.readable)); const ws = list.find((e) => e.env_key === "WORKSPACE_ADMIN_PASSWORD"); assert.equal(await K.openEntrySecret(db2, key2, ws.id), "ws-pass 1"); - assert.deepEqual(await K.importEntries(db2, key2, await X.openExport(file, "correct horse battery")), { imported: 0, skipped: 2 }, "re-import overwrites nothing"); + assert.deepEqual(await K.importEntries(db2, key2, await X.openExport(file, "correct horse battery")), { imported: 0, skipped: 2, ids: [] }, "re-import overwrites nothing"); }); test("init-db creates crow_keychain with the same columns as the store's lazy ensure (no CHECK constraints)", async () => { @@ -246,7 +246,7 @@ test("import drops a non-http(s) url but still imports the entry", async () => { { kind: "manual", label: "Evil link", username: null, url: "javascript:alert(1)", secret: "s1" }, { kind: "manual", label: "Good link", username: null, url: "HTTPS://ok.example", secret: "s2" }, ]); - assert.deepEqual(out, { imported: 2, skipped: 0 }); + assert.deepEqual(out, { imported: 2, skipped: 0, ids: [] }, "manual entries are not extension ids"); const list = await K.listEntries(db, { keyId: key.id }); assert.equal(list.find((e) => e.label === "Evil link").url, null); assert.equal(list.find((e) => e.label === "Good link").url, "HTTPS://ok.example"); From a6fba7cdf6041e80078cd426e9703ab0fbb2aec1 Mon Sep 17 00:00:00 2001 From: kh0pper <93635715+kh0pper@users.noreply.github.com> Date: Fri, 16 Oct 2026 19:04:51 -0500 Subject: [PATCH 3/4] =?UTF-8?q?fix(keychain):=20code=20review=20=E2=80=94?= =?UTF-8?q?=20fail=20closed=20on=20a=20keychain=20read=20error;=20a=20dele?= =?UTF-8?q?ted=20entry=20stays=20deleted;=20the=20reminder=20snapshot=20co?= =?UTF-8?q?vers=20removed=20extensions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - keychainRestoreValues fails CLOSED: a read error refuses any keychain:true mint (it could replace an entry the read missed), and one damaged entry counts as unreadable instead of clearing the result. - The refusal keeps a prior install's folder (its .env holds other secrets) and says that deleting needs a dashboard password or two-factor sign-in. - Boot backfill runs once per bundle+key (local marker): a key the user deletes in Passwords is never brought back by the next boot. - Export fingerprints are taken before the Export is built and cover every status, so a reactivated removed extension is not nagged about and a key saved mid-export is. - The install-log reminder follows the same rule as the Nest (anything unexported). - Mastodon/Pixelfed/PeerTube database passwords go to the keychain too: a database restored from a backup still opens after a reinstall. - Tests: fail-closed, damaged entry, deleted-stays-deleted, removed extension, and the real export/import HTTP routes. Claude-Session: https://claude.ai/code/session_01N9fXzvNy8rCEArj1y5p6fp --- bundles/mastodon/manifest.json | 2 +- bundles/peertube/manifest.json | 2 +- bundles/pixelfed/manifest.json | 2 +- registry/add-ons.json | 18 ++++++---- servers/gateway/keychain/api.js | 5 +-- servers/gateway/keychain/install-hooks.js | 34 ++++++++++++++---- servers/gateway/keychain/reminder.js | 23 +++++++++---- servers/gateway/routes/bundles.js | 10 ++++-- tests/bundle-generated-secrets.test.js | 3 ++ tests/keychain-export-reminder.test.js | 42 ++++++++++++++++++----- tests/keychain-restore.test.js | 31 +++++++++++++++-- 11 files changed, 134 insertions(+), 38 deletions(-) diff --git a/bundles/mastodon/manifest.json b/bundles/mastodon/manifest.json index d0e17af05..1e51cefad 100644 --- a/bundles/mastodon/manifest.json +++ b/bundles/mastodon/manifest.json @@ -33,7 +33,7 @@ "env_vars": [ { "name": "MASTODON_LOCAL_DOMAIN", "description": "Public domain that appears in handles (@user@). IMMUTABLE after first boot — changing breaks all federation identities.", "required": true }, { "name": "MASTODON_WEB_DOMAIN", "description": "Domain where the web UI lives (if using apex-delegation this differs from LOCAL_DOMAIN; otherwise same).", "required": false }, - { "name": "MASTODON_DB_PASSWORD", "description": "Password for the bundled Postgres role. Generated by Crow at install; never shown.", "required": true, "secret": true, "generate": "secret" }, + { "name": "MASTODON_DB_PASSWORD", "description": "Password for the bundled Postgres role. Generated by Crow at install; saved in Settings → Passwords (this machine only), so a database restored from a backup still opens after a reinstall.", "required": true, "secret": true, "generate": "secret", "keychain": true, "keychain_label": "database password" }, { "name": "MASTODON_SECRET_KEY_BASE", "description": "Rails secret_key_base. Generated by Crow at install; saved in Settings → Passwords (this machine only): export the keychain to keep a copy, or data encrypted with it cannot be recovered on another machine.", "required": true, "secret": true, "generate": "secret", "keychain": true, "keychain_label": "session signing key (secret_key_base)" }, { "name": "MASTODON_OTP_SECRET", "description": "2FA OTP secret. Generated by Crow at install; saved in Settings → Passwords (this machine only): export the keychain to keep a copy, or data encrypted with it cannot be recovered on another machine.", "required": true, "secret": true, "generate": "secret", "keychain": true, "keychain_label": "two-factor secret" }, { "name": "MASTODON_VAPID_PRIVATE_KEY", "description": "Web Push VAPID private key (P-256). Generated by Crow at install; never shown.", "required": true, "secret": true, "generate": "vapid_private_key" }, diff --git a/bundles/peertube/manifest.json b/bundles/peertube/manifest.json index bacf54e66..f62e57e45 100644 --- a/bundles/peertube/manifest.json +++ b/bundles/peertube/manifest.json @@ -32,7 +32,7 @@ }, "env_vars": [ { "name": "PEERTUBE_WEBSERVER_HOSTNAME", "description": "Public domain (subdomain; IMMUTABLE after first federation). Appears in video URLs, channel URLs, every ActivityPub actor.", "required": true }, - { "name": "PEERTUBE_DB_PASSWORD", "description": "Password for the bundled Postgres role. Generated by Crow at install; never shown.", "required": true, "secret": true, "generate": "secret" }, + { "name": "PEERTUBE_DB_PASSWORD", "description": "Password for the bundled Postgres role. Generated by Crow at install; saved in Settings → Passwords (this machine only), so a database restored from a backup still opens after a reinstall.", "required": true, "secret": true, "generate": "secret", "keychain": true, "keychain_label": "database password" }, { "name": "PEERTUBE_SECRET", "description": "Server secret that signs federation requests. IMMUTABLE: generated once by Crow at install, kept across reinstalls, saved in Settings → Passwords (this machine only): export the keychain to keep a copy, or data encrypted with it cannot be recovered on another machine.", "required": true, "secret": true, "generate": "secret", "keychain": true, "keychain_label": "server secret" }, { "name": "PEERTUBE_ACCESS_TOKEN", "description": "OAuth bearer token (Administration → Users → your account → Personal Access Tokens, OR POST /api/v1/users/token with username+password + client_id+client_secret from /api/v1/oauth-clients/local).", "required": false, "secret": true }, { "name": "PEERTUBE_URL", "description": "Internal URL the MCP server uses to reach PeerTube (default http://peertube:9000 over crow-federation).", "default": "http://peertube:9000", "required": false }, diff --git a/bundles/pixelfed/manifest.json b/bundles/pixelfed/manifest.json index 8bce6dda3..853912cd9 100644 --- a/bundles/pixelfed/manifest.json +++ b/bundles/pixelfed/manifest.json @@ -32,7 +32,7 @@ }, "env_vars": [ { "name": "PIXELFED_HOSTNAME", "description": "Public domain (subdomain; path-mounts break ActivityPub).", "required": true }, - { "name": "PIXELFED_DB_PASSWORD", "description": "Password for the bundled Postgres role. Generated by Crow at install; never shown.", "required": true, "secret": true, "generate": "secret" }, + { "name": "PIXELFED_DB_PASSWORD", "description": "Password for the bundled Postgres role. Generated by Crow at install; saved in Settings → Passwords (this machine only), so a database restored from a backup still opens after a reinstall.", "required": true, "secret": true, "generate": "secret", "keychain": true, "keychain_label": "database password" }, { "name": "PIXELFED_APP_KEY", "description": "Laravel application key (base64: + 32 random bytes). Generated by Crow at install; saved in Settings → Passwords (this machine only): export the keychain to keep a copy, or data encrypted with it cannot be recovered on another machine.", "required": true, "secret": true, "generate": "laravel_key", "keychain": true, "keychain_label": "data encryption key (APP_KEY)" }, { "name": "PIXELFED_ACCESS_TOKEN", "description": "OAuth2 Personal Access Token (Settings → Development → New Application).", "required": false, "secret": true }, { "name": "PIXELFED_URL", "description": "Internal URL the MCP server uses to reach Pixelfed (default http://pixelfed:80 over crow-federation).", "default": "http://pixelfed:80", "required": false }, diff --git a/registry/add-ons.json b/registry/add-ons.json index 630496464..6b5f3b48d 100644 --- a/registry/add-ons.json +++ b/registry/add-ons.json @@ -3106,10 +3106,12 @@ }, { "name": "MASTODON_DB_PASSWORD", - "description": "Password for the bundled Postgres role. Generated by Crow at install; never shown.", + "description": "Password for the bundled Postgres role. Generated by Crow at install; saved in Settings → Passwords (this machine only), so a database restored from a backup still opens after a reinstall.", "required": true, "secret": true, - "generate": "secret" + "generate": "secret", + "keychain": true, + "keychain_label": "database password" }, { "name": "MASTODON_SECRET_KEY_BASE", @@ -4601,10 +4603,12 @@ }, { "name": "PEERTUBE_DB_PASSWORD", - "description": "Password for the bundled Postgres role. Generated by Crow at install; never shown.", + "description": "Password for the bundled Postgres role. Generated by Crow at install; saved in Settings → Passwords (this machine only), so a database restored from a backup still opens after a reinstall.", "required": true, "secret": true, - "generate": "secret" + "generate": "secret", + "keychain": true, + "keychain_label": "database password" }, { "name": "PEERTUBE_SECRET", @@ -4806,10 +4810,12 @@ }, { "name": "PIXELFED_DB_PASSWORD", - "description": "Password for the bundled Postgres role. Generated by Crow at install; never shown.", + "description": "Password for the bundled Postgres role. Generated by Crow at install; saved in Settings → Passwords (this machine only), so a database restored from a backup still opens after a reinstall.", "required": true, "secret": true, - "generate": "secret" + "generate": "secret", + "keychain": true, + "keychain_label": "database password" }, { "name": "PIXELFED_APP_KEY", diff --git a/servers/gateway/keychain/api.js b/servers/gateway/keychain/api.js index 0238012c9..03b2a50e2 100644 --- a/servers/gateway/keychain/api.js +++ b/servers/gateway/keychain/api.js @@ -18,7 +18,7 @@ import { exportableEntries, importEntries, ensureWriteKey, } from "./store.js"; import { vaultwardenStatus, saveToVault } from "./vault-save.js"; -import { recordKeychainExport, recordKeychainImport } from "./reminder.js"; +import { recordKeychainExport, recordKeychainImport, exportSnapshot } from "./reminder.js"; import { invalidateHealthCache } from "../dashboard/panels/nest/health-signals.js"; const BASE = "/dashboard/keychain/api"; @@ -180,9 +180,10 @@ export function keychainApiRouter({ if (passphrase.length < MIN_PASSPHRASE) return res.status(400).json({ error: `Choose a passphrase of at least ${MIN_PASSPHRASE} characters.` }); const key = readKey(); if (!key) return res.status(409).json(KEY_MISSING); + const snapshot = await exportSnapshot(db, key.id).catch(() => null); const entries = await exportableEntries(db, key); const file = await sealExport(entries, passphrase); - await recordKeychainExport(db, key.id).catch(() => {}); // the Nest reminder; never blocks the download + if (snapshot) await recordKeychainExport(db, key.id, snapshot).catch(() => {}); // the Nest reminder; never blocks the download invalidateHealthCache(); await audit(db, "keychain_export", { ip: req.ip, details: { count: entries.length } }); res.set("Content-Disposition", `attachment; filename="crow-keychain-${new Date().toISOString().slice(0, 10)}.json"`); diff --git a/servers/gateway/keychain/install-hooks.js b/servers/gateway/keychain/install-hooks.js index c949e21b1..f62e935be 100644 --- a/servers/gateway/keychain/install-hooks.js +++ b/servers/gateway/keychain/install-hooks.js @@ -14,7 +14,8 @@ import { keychainGeneratedKeys, keychainEligibleKeys, expandKeychainTemplate } f import { createNotification } from "../../shared/notifications.js"; import { saveExtensionSecret, reactivateBundleEntries, hasExtensionEntry, markBundleRemoved, ensureWriteKey, listEntries, openEntrySecret } from "./store.js"; import { loadKeychainKey } from "./key.js"; -import { lastKeychainExport } from "./reminder.js"; +import { unexportedCount } from "./reminder.js"; +import { readSetting, writeSetting } from "../dashboard/settings/registry.js"; import { vaultwardenStatus, saveToVault, VAULT_REASONS } from "./vault-save.js"; let _override = null; @@ -125,7 +126,7 @@ export async function recordKeychainForInstall({ bundleId, manifest, env, minted if (out.saved) log(`Saved ${out.saved} password(s) to Crow keychain (Settings → Passwords)`); if (mintedList.length && out.saved) { try { - if (!(await lastKeychainExport(db))) log("This key exists only on this machine: export your keychain (Settings → Passwords → Export) so it can be recovered if the machine is lost."); + if (key && (await unexportedCount(db, key.id)) > 0) log("This key exists only on this machine: export your keychain (Settings → Passwords → Export) so it can be recovered if the machine is lost."); } catch { /* the reminder never blocks an install */ } } @@ -173,7 +174,7 @@ export async function recordKeychainForInstall({ bundleId, manifest, env, minted */ export async function keychainRestoreValues(bundleId, manifest) { const keys = keychainGeneratedKeys(manifest); - const out = { values: {}, unreadable: [] }; + const out = { values: {}, unreadable: [], failed: false }; if (keys.length === 0) return out; let db; try { @@ -182,11 +183,17 @@ export async function keychainRestoreValues(bundleId, manifest) { for (const e of await listEntries(db, { keyId: key?.id ?? null })) { if (e.kind !== "extension" || e.bundle_id !== bundleId || !keys.includes(e.env_key)) continue; if (!e.readable || !key) { out.unreadable.push(e.env_key); continue; } - const plain = await openEntrySecret(db, key, e.id); - if (typeof plain === "string" && plain !== "") out.values[e.env_key] = plain; + // A damaged entry counts as unreadable (it is never minted over), not as "none". + try { + const plain = await openEntrySecret(db, key, e.id); + if (typeof plain === "string" && plain !== "") out.values[e.env_key] = plain; + else out.unreadable.push(e.env_key); + } catch { out.unreadable.push(e.env_key); } } } catch { - return { values: {}, unreadable: [] }; + // FAIL CLOSED: the keychain could not be read, so the caller must not mint a + // keychain:true key (a mint would replace an entry this read could not see). + return { values: {}, unreadable: [], failed: true }; } finally { try { db?.close(); } catch {} } @@ -203,6 +210,8 @@ export async function keychainRestoreValues(bundleId, manifest) { * first view (nothing new was minted). Best-effort; names only in `log`. * `values` = { envKey: { plain, origin } } gathered by the caller. */ +export const BACKFILL_SETTING = "keychain_backfilled_keys"; + export async function backfillKeychainEntries(bundleId, manifest, values, { log = () => {} } = {}) { const specs = new Map((manifest?.env_vars || []).map((v) => [v.name, v])); const todo = keychainGeneratedKeys(manifest).filter((k) => !specs.get(k)?.store_as && values?.[k]?.plain); @@ -211,8 +220,18 @@ export async function backfillKeychainEntries(bundleId, manifest, values, { log let saved = 0; try { db = deps().openDb(); + // Once per bundle+key: a key the user later DELETED in Settings → Passwords is never + // brought back by a later boot (local setting; never synced). + let done = {}; + try { done = JSON.parse((await readSetting(db, BACKFILL_SETTING)) || "{}") || {}; } catch { done = {}; } + const mark = async (ks) => { for (const k of ks) done[`${bundleId}:${k}`] = true; await writeSetting(db, BACKFILL_SETTING, JSON.stringify(done), { scope: "local" }); }; const missing = []; - for (const k of todo) if (!(await hasExtensionEntry(db, bundleId, k))) missing.push(k); + const present = []; + for (const k of todo) { + if (done[`${bundleId}:${k}`]) continue; + if (await hasExtensionEntry(db, bundleId, k)) present.push(k); else missing.push(k); + } + if (present.length) await mark(present); if (missing.length === 0) return 0; const key = await deps().writeKey(db); const tEnv = templateEnv(manifest, Object.fromEntries(Object.entries(values).map(([k, v]) => [k, v.plain]))); @@ -223,6 +242,7 @@ export async function backfillKeychainEntries(bundleId, manifest, values, { log await deps().audit(db, "keychain_save", { details: { entry_id: r.id, bundle_id: bundleId, env_key: k, origin: "kept", backfill: true } }); saved++; } + await mark(missing); log(`${bundleId}: saved ${missing.join(", ")} to Crow keychain (kept from the existing install)`); } catch (err) { log(`${bundleId}: could not save kept keys to Crow keychain (${err?.code || err?.name || "error"})`); diff --git a/servers/gateway/keychain/reminder.js b/servers/gateway/keychain/reminder.js index 6074494d0..66d58fed2 100644 --- a/servers/gateway/keychain/reminder.js +++ b/servers/gateway/keychain/reminder.js @@ -29,16 +29,27 @@ async function readState(db) { return { at: null, fp: {} }; } -async function activeRows(db, keyId, ids = null) { +async function rows(db, keyId, { activeOnly = true, ids = null } = {}) { await ensureKeychainTable(db); - const { rows } = await db.execute({ sql: "SELECT id, secret_sealed FROM crow_keychain WHERE kind = 'extension' AND status = 'active' AND key_id = ?", args: [String(keyId || "")] }); - return ids ? rows.filter((r) => ids.has(Number(r.id))) : rows; + const { rows: r } = await db.execute({ sql: `SELECT id, secret_sealed FROM crow_keychain WHERE kind = 'extension' AND key_id = ?${activeOnly ? " AND status = 'active'" : ""}`, args: [String(keyId || "")] }); + return ids ? r.filter((x) => ids.has(Number(x.id))) : r; } +const activeRows = (db, keyId, ids = null) => rows(db, keyId, { ids }); -/** After an Export was sealed: every readable extension entry is in that file. */ -export async function recordKeychainExport(db, keyId, at = new Date()) { +/** + * Fingerprints of every readable extension entry (any status: the Export carries removed + * extensions' entries too). Take it BEFORE the Export is built, so an entry saved while it + * is sealed counts as not exported (a false reminder, never a false all-clear). + */ +export async function exportSnapshot(db, keyId) { const fps = {}; - for (const r of await activeRows(db, keyId)) fps[Number(r.id)] = fp(r.secret_sealed); + for (const r of await rows(db, keyId, { activeOnly: false })) fps[Number(r.id)] = fp(r.secret_sealed); + return fps; +} + +/** After an Export built from `snapshot` was sealed. */ +export async function recordKeychainExport(db, keyId, snapshot = null, at = new Date()) { + const fps = snapshot || (await exportSnapshot(db, keyId)); await writeSetting(db, LAST_EXPORT_SETTING, JSON.stringify({ at: at.toISOString(), fp: fps }), { scope: "local" }); } diff --git a/servers/gateway/routes/bundles.js b/servers/gateway/routes/bundles.js index 8025fa09d..93fdcd6fd 100644 --- a/servers/gateway/routes/bundles.js +++ b/servers/gateway/routes/bundles.js @@ -2550,10 +2550,14 @@ export async function runInstallJob(bundleId, envVars, { job, installedSnapshot, // not have: crow.db restored, Export not imported yet) is never minted over: the new // value would replace that entry, and a later Import skips a bundle+key that has a // readable entry, so the old key (and the data saved with it) would be lost for good. - const blocked = Object.keys(plan.minted).filter((k) => kcRestore.unreadable.includes(k)); + const mintingKc = Object.keys(plan.minted).filter((k) => manifest.env_vars?.find((v) => v.name === k)?.keychain === true); + const blocked = kcRestore.failed ? mintingKc : mintingKc.filter((k) => kcRestore.unreadable.includes(k)); if (blocked.length > 0) { - rmSync(destDir, { recursive: true, force: true }); - const reason = `Crow keychain holds ${blocked.join(", ")} for this extension, sealed with a keychain key this machine does not have. Import your keychain Export first (Settings → Passwords → Import), then install again. To start fresh instead, delete the old saved password(s) there.`; + // A prior install's folder (its .env holds other generated secrets) is kept as it is. + if (!priorInstallEnv) rmSync(destDir, { recursive: true, force: true }); + const reason = kcRestore.failed + ? `Crow keychain could not be read, so ${blocked.join(", ")} was not generated (a new value could replace one saved there). Try the install again; nothing was written.` + : `Crow keychain holds ${blocked.join(", ")} for this extension, sealed with a keychain key this machine does not have. Import your keychain Export first (Settings → Passwords → Import), then install again. To start fresh instead, delete the old saved password(s) there (this needs a dashboard password or two-factor sign-in).`; appendLog(job, reason); return { ok: false, reason }; } diff --git a/tests/bundle-generated-secrets.test.js b/tests/bundle-generated-secrets.test.js index 20404e88c..d38fc3e1c 100644 --- a/tests/bundle-generated-secrets.test.js +++ b/tests/bundle-generated-secrets.test.js @@ -65,6 +65,9 @@ const EXPECT = [ { id: "mastodon", key: "MASTODON_AR_ENCRYPTION_KEY_DERIVATION_SALT", propagate: false }, { id: "mastodon", key: "MASTODON_AR_ENCRYPTION_PRIMARY_KEY", propagate: false }, { id: "peertube", key: "PEERTUBE_SECRET", propagate: false }, + { id: "mastodon", key: "MASTODON_DB_PASSWORD", propagate: false }, + { id: "pixelfed", key: "PIXELFED_DB_PASSWORD", propagate: false }, + { id: "peertube", key: "PEERTUBE_DB_PASSWORD", propagate: false }, ]; test("each key is generated, keychain-saved, and propagated only where an MCP child reads it", () => { diff --git a/tests/keychain-export-reminder.test.js b/tests/keychain-export-reminder.test.js index 4e95838b3..39daea219 100644 --- a/tests/keychain-export-reminder.test.js +++ b/tests/keychain-export-reminder.test.js @@ -81,12 +81,38 @@ test("the export fingerprints are local: not in the sync allowlist", () => { assert.equal(isSyncable(R.LAST_EXPORT_SETTING), false); }); -test("the export and import routes record the reminder state and drop the health cache", async () => { - const src = (await import("node:fs")).readFileSync(new URL("../servers/gateway/keychain/api.js", import.meta.url), "utf8"); - const exportRoute = src.slice(src.indexOf("/export`"), src.indexOf("/import`")); - assert.match(exportRoute, /recordKeychainExport\(db, key\.id\)/); - assert.ok(exportRoute.indexOf("recordKeychainExport") > exportRoute.indexOf("sealExport"), "only after the file is sealed"); - assert.match(exportRoute, /invalidateHealthCache\(\)/); - const importRoute = src.slice(src.indexOf("/import`"), src.indexOf("/vault-save`")); - assert.match(importRoute, /recordKeychainImport\(db, ikey\.id, ids\)/); +test("an entry of a removed extension is in the Export too: reinstalling it does not raise the reminder", async () => { + await K.saveExtensionSecret(db, key, { bundleId: "peertube", envKey: "PEERTUBE_SECRET", label: "PeerTube — server secret", secret: "pt-" + "p".repeat(40), origin: "generated" }); + await K.markBundleRemoved(db, "peertube"); + await R.recordKeychainExport(db, key.id); + await K.reactivateBundleEntries(db, "peertube", ["PEERTUBE_SECRET"]); + assert.equal(await R.unexportedCount(db, key.id), 0); +}); + +test("the real export + import routes record the reminder state (HTTP, injected re-auth gate)", async () => { + const express = (await import("express")).default; + const { keychainApiRouter } = await import("../servers/gateway/keychain/api.js"); + const gate = { method: async () => "password", isGranted: () => true, expiresAt: () => null, verify: async () => ({ ok: true }) }; + const app = express(); + app.use(express.json({ limit: "5mb" })); + app.use((req, res, next) => { req.dashboardSession = "test-session"; next(); }); + app.use(keychainApiRouter({ gate, openDb: () => createDbClient() })); + const server = await new Promise((r) => { const s = app.listen(0, "127.0.0.1", () => r(s)); }); + try { + const url = (p) => `http://127.0.0.1:${server.address().port}/dashboard/keychain/api${p}`; + await K.saveExtensionSecret(db, key, { bundleId: "romm", envKey: "ROMM_AUTH_SECRET_KEY", label: "RomM — secret", secret: "rm-" + "r".repeat(40), origin: "generated" }); + assert.ok((await R.unexportedCount(db, key.id)) > 0); + const ex = await fetch(url("/export"), { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ passphrase: "a long export passphrase" }) }); + assert.equal(ex.status, 200); + const file = await ex.json(); + assert.equal(await R.unexportedCount(db, key.id), 0, "everything readable is in that Export"); + HS.invalidateHealthCache(); + assert.equal((await signal()).issue, null); + // Import into a fresh keychain row set: what comes in is in the file, so it never counts. + await db.execute("DELETE FROM crow_keychain WHERE bundle_id = 'romm'"); + const im = await fetch(url("/import"), { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ file, passphrase: "a long export passphrase" }) }); + assert.equal(im.status, 200); + assert.equal((await im.json()).imported, 1); + assert.equal(await R.unexportedCount(db, key.id), 0); + } finally { server.close(); } }); diff --git a/tests/keychain-restore.test.js b/tests/keychain-restore.test.js index 789350202..ac67ddd86 100644 --- a/tests/keychain-restore.test.js +++ b/tests/keychain-restore.test.js @@ -100,8 +100,9 @@ test("keychainRestoreValues: only keychain:true generated keys, only readable en const { values: v, unreadable } = await H.keychainRestoreValues(id, m); assert.deepEqual(Object.keys(v).sort(), ["DEMO_ADMIN_TOKEN", "DEMO_DATA_KEY"], "DEMO_DB_PASSWORD is never in the keychain, so never looked up"); assert.deepEqual(unreadable, []); + assert.equal((await H.keychainRestoreValues(id, m)).failed, false); assert.equal(v.DEMO_DATA_KEY, (await entry(id, "DEMO_DATA_KEY")).plain); - assert.deepEqual(await H.keychainRestoreValues(id, { ...m, env_vars: [] }), { values: {}, unreadable: [] }, "a manifest with no keychain keys reads nothing"); + assert.deepEqual(await H.keychainRestoreValues(id, { ...m, env_vars: [] }), { values: {}, unreadable: [], failed: false }, "a manifest with no keychain keys reads nothing"); }); test("machine lost → reinstall restores the SAME data key from the keychain (and does not overwrite the entry)", async () => { @@ -150,14 +151,33 @@ test("an entry sealed with another machine's keychain key (Export not imported y assert.ok(again.env.DEMO_DATA_KEY); }); -test("a keychain read failure never fails the install (best-effort: it mints, as before)", async () => { +test("a keychain read failure FAILS CLOSED: no keychain:true key is minted (it could replace an entry the read missed)", async () => { const id = "kcr-fail"; const m = fixture(id); H._setKeychainDepsForTest({ openDb: () => { throw new Error("db down"); } }); try { - assert.deepEqual(await H.keychainRestoreValues(id, m), { values: {}, unreadable: [] }); + assert.deepEqual(await H.keychainRestoreValues(id, m), { values: {}, unreadable: [], failed: true }); + const job = B._createJobForTest(id, "install"); + const out = await B.runInstallJob(id, {}, { job, installedSnapshot: [], consentVerified: false, manifest: m }); + assert.equal(out.ok, false); + assert.match(out.reason, /could not be read.*Try the install again/); } finally { H._setKeychainDepsForTest(null); } }); +test("one damaged entry counts as unreadable (refused), never as 'none' (minted over)", async () => { + const id = "kcr-damaged"; const m = fixture(id); + await install(id, m); + const d = db(); + try { await d.execute({ sql: "UPDATE crow_keychain SET secret_sealed = 'not-a-sealed-value' WHERE bundle_id = ? AND env_key = 'DEMO_DATA_KEY'", args: [id] }); } finally { d.close(); } + const r = await H.keychainRestoreValues(id, m); + assert.deepEqual(r.unreadable, ["DEMO_DATA_KEY"]); + assert.ok(r.values.DEMO_ADMIN_TOKEN, "the other entry is still read"); + loseFiles(id); + const job = B._createJobForTest(id, "install"); + const out = await B.runInstallJob(id, {}, { job, installedSnapshot: [], consentVerified: false, manifest: m }); + assert.equal(out.ok, false); + assert.match(out.reason, /DEMO_DATA_KEY/); +}); + test("crow_keychain stays local-only (never synced to a paired instance)", async () => { const sync = await import("../servers/sharing/instance-sync.js"); assert.ok((sync.LOCAL_ONLY_TABLES || []).includes("crow_keychain")); @@ -222,4 +242,9 @@ test("boot backfill: a key kept only in an older install's .env is saved to the writeFileSync(join(CROW_HOME, "bundles", id, ".env"), "DEMO_DATA_KEY=something-else-999999\n"); await B.backfillBundleKeychain({ log: () => {} }); assert.equal((await entry(id, "DEMO_DATA_KEY")).plain, "typed-long-ago-123456"); + // The user deletes it in Settings → Passwords: a later boot never brings it back. + const d = db(); + try { await d.execute({ sql: "DELETE FROM crow_keychain WHERE bundle_id = ? AND env_key = 'DEMO_DATA_KEY'", args: [id] }); } finally { d.close(); } + await B.backfillBundleKeychain({ log: () => {} }); + assert.equal(await entry(id, "DEMO_DATA_KEY"), null, "a deleted entry stays deleted"); }); From 4a0711049c96fbe987bc7fa70c363bbcdce1cd86 Mon Sep 17 00:00:00 2001 From: Kevin Hopper <93635715+kh0pper@users.noreply.github.com> Date: Fri, 16 Oct 2026 19:40:10 -0500 Subject: [PATCH 4/4] feat(campaigns,tax): generated data keys in the keychain; tax documents encrypted at rest (campaigns 1.0.3, tax 2.1.0) (#513) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(campaigns,tax): generated data keys in the keychain; tax documents encrypted at rest (campaigns 1.0.3, tax 2.1.0) Recovery keys, option 1 (queue item 30, decided 2026-10-16). Stacked on the keychain restore + Export reminder change. - CROW_CAMPAIGNS_ENCRYPTION_KEY and CROW_TAX_ENCRYPTION_KEY are generated at install and saved to the keychain (a reinstall takes them back from it). They reach the MCP child through its mcp-addons entry; both dashboard panels read them in the child's own order (the entry first, then the gateway env), so panel and server always use one key. - An install still on the tax example key once published in the repo gets a new key and the old one moves to CROW_TAX_ENCRYPTION_KEY_PREVIOUS (generic manifest field replace_values_sha256, validated by the bundle contract), so its returns still open and move to the new key. - Tax uploads are stored only encrypted (/tax-documents, AES-256-GCM with a key id in the header); ingestion reads the upload from memory. Documents stored before are encrypted at dashboard start, one row at a time and verified before the plain file is removed: unique per-run names, compare-and-swap on the row, a lock file against two migrators, crash leftovers cleaned once verified, and only regular files inside the documents folder are ever touched. scripts/documents.mjs reverses it (in place, or as plain copies to a folder). The MCP ingest tool opens a stored .enc. - A version bump now ships a non-docker bundle's engine/ and pdf/ (tax's panel imports the installed engine). - The tax manifest says exactly what is encrypted (returns, documents) and what is not (the figures read from documents). Claude-Session: https://claude.ai/code/session_01N9fXzvNy8rCEArj1y5p6fp * test(tax): the previous_key Configure test runs on a typed field; Configure never overwrites the generated tax key Claude-Session: https://claude.ai/code/session_01N9fXzvNy8rCEArj1y5p6fp * fix(tax): code review — key id from the salted derived key; ingest checks the path before reading; restored rows find their .enc; safer lock - The document key id is now an HMAC under the salted PBKDF2 output (per file), not under the passphrase: no cheap offline guess check, nothing linkable across files. - The MCP ingest tool checks the extension (.pdf/.enc), that the path is a regular file, and a 25 MB cap BEFORE reading (the AI chooses the path). - A crow.db restored from a backup taken before the migration (rows still naming the plain file) finds the single .<8hex>.enc successor and repoints the row. - The documents lock: stale takeover by atomic rename (one winner), touched per row, folder tightened to 0700; the panel logs a held lock. --out folders are 0700. - Upload names get a random part (two same-millisecond uploads never share a file). - An install on the published example key: the upload refusal and the install log say to reinstall, and say why when the previous-key slot is already in use. - The CLI uses the database of the same CROW_HOME as its key and folders. Claude-Session: https://claude.ai/code/session_01N9fXzvNy8rCEArj1y5p6fp --- bundles/campaigns/manifest.json | 9 +- bundles/campaigns/panel/campaigns.js | 2 +- bundles/campaigns/panel/routes.js | 3 +- bundles/campaigns/server/key.js | 20 ++ bundles/tax/.env.example | 5 +- bundles/tax/engine/ingest/index.js | 36 ++- bundles/tax/manifest.json | 12 +- bundles/tax/panel/routes.js | 64 ++++- bundles/tax/scripts/documents.mjs | 51 ++++ bundles/tax/server/documents-crypto.js | 275 +++++++++++++++++++++ registry/add-ons.json | 23 +- scripts/lib/bundle-contract.mjs | 7 + servers/gateway/bundle-env-secrets.js | 30 ++- servers/gateway/keychain/install-hooks.js | 5 +- servers/gateway/routes/bundles.js | 6 +- tests/bundle-env-placeholders.test.js | 23 +- tests/bundle-form-friction.test.js | 11 +- tests/campaigns-tax-generated-keys.test.js | 198 +++++++++++++++ tests/tax-documents-crypto.test.js | 213 ++++++++++++++++ 19 files changed, 942 insertions(+), 51 deletions(-) create mode 100644 bundles/campaigns/server/key.js create mode 100644 bundles/tax/scripts/documents.mjs create mode 100644 bundles/tax/server/documents-crypto.js create mode 100644 tests/campaigns-tax-generated-keys.test.js create mode 100644 tests/tax-documents-crypto.test.js diff --git a/bundles/campaigns/manifest.json b/bundles/campaigns/manifest.json index a9e379f0e..9aa81ace0 100644 --- a/bundles/campaigns/manifest.json +++ b/bundles/campaigns/manifest.json @@ -1,7 +1,7 @@ { "id": "campaigns", "name": "Campaigns", - "version": "1.0.2", + "version": "1.0.3", "description": "Social media campaign manager — draft, schedule, and publish posts across platforms (starting with Reddit)", "type": "mcp-server", "author": "Crow", @@ -22,9 +22,12 @@ "env_vars": [ { "name": "CROW_CAMPAIGNS_ENCRYPTION_KEY", - "description": "AES-256-GCM encryption key for Reddit credentials (generate with: openssl rand -base64 32)", + "description": "Key that encrypts your saved Reddit credentials (AES-256-GCM). Crow generates it at install and saves it in Settings → Passwords (this machine only); a reinstall takes it back from the keychain. Without it, re-enter the Reddit credentials.", "required": true, - "secret": true + "secret": true, + "generate": "secret", + "keychain": true, + "keychain_label": "data encryption key" } ], "notes": "12 MCP tools for social media campaign management. Reddit support via snoowrap. Credentials encrypted at rest with AES-256-GCM. Dashboard panel with approval workflow." diff --git a/bundles/campaigns/panel/campaigns.js b/bundles/campaigns/panel/campaigns.js index 7484aef2d..acbca67a3 100644 --- a/bundles/campaigns/panel/campaigns.js +++ b/bundles/campaigns/panel/campaigns.js @@ -67,7 +67,7 @@ export default { } if (action === "save_creds") { - const ENCRYPTION_KEY = process.env.CROW_CAMPAIGNS_ENCRYPTION_KEY; + const ENCRYPTION_KEY = (await import(pathToFileURL(join(serverDir, "key.js")).href)).campaignsKey(); if (!ENCRYPTION_KEY) { return res.redirectAfterPost("/dashboard/campaigns?view=setup&error=CROW_CAMPAIGNS_ENCRYPTION_KEY+not+set"); } diff --git a/bundles/campaigns/panel/routes.js b/bundles/campaigns/panel/routes.js index 7d6bddac1..39b0a4bda 100644 --- a/bundles/campaigns/panel/routes.js +++ b/bundles/campaigns/panel/routes.js @@ -24,7 +24,7 @@ const { createDbClient } = await import(pathToFileURL(join(serverDir, "db.js")). const { encrypt, decrypt } = await import(pathToFileURL(join(serverDir, "crypto.js")).href); const { createRedditClient, testAuth } = await import(pathToFileURL(join(serverDir, "reddit-client.js")).href); -const ENCRYPTION_KEY = process.env.CROW_CAMPAIGNS_ENCRYPTION_KEY; +const { campaignsKey } = await import(pathToFileURL(join(serverDir, "key.js")).href); /** * @param {Function} authMiddleware - Dashboard auth middleware @@ -228,6 +228,7 @@ export default function campaignsRouter(authMiddleware) { // --- Credentials: save (encrypt + validate) --- router.post("/api/credentials", async (req, res) => { try { + const ENCRYPTION_KEY = campaignsKey(); if (!ENCRYPTION_KEY) return res.status(500).json({ error: "CROW_CAMPAIGNS_ENCRYPTION_KEY not set" }); const { platform, username, client_id, client_secret, password } = req.body; diff --git a/bundles/campaigns/server/key.js b/bundles/campaigns/server/key.js new file mode 100644 index 000000000..678443e0c --- /dev/null +++ b/bundles/campaigns/server/key.js @@ -0,0 +1,20 @@ +/** + * The campaigns credentials key (CROW_CAMPAIGNS_ENCRYPTION_KEY), read when needed. + * + * The MCP server gets it in its own env: the gateway env overlaid by this instance's + * campaigns entry in mcp-addons.json (proxy.js). The dashboard panel runs in the gateway, + * so it reads them in that SAME order — the entry first — and both always use one key. A + * generated key reaches only the entry (mode 600). Never logged. + */ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { homedir } from "node:os"; + +export function campaignsKey(env = process.env) { + let fromEntry = ""; + try { + const file = join(env.CROW_HOME || join(homedir(), ".crow"), "mcp-addons.json"); + fromEntry = JSON.parse(readFileSync(file, "utf8"))?.campaigns?.env?.CROW_CAMPAIGNS_ENCRYPTION_KEY || ""; + } catch { /* no entry */ } + return fromEntry || env.CROW_CAMPAIGNS_ENCRYPTION_KEY || ""; +} diff --git a/bundles/tax/.env.example b/bundles/tax/.env.example index 9088cffad..2a123a0d1 100644 --- a/bundles/tax/.env.example +++ b/bundles/tax/.env.example @@ -1,8 +1,7 @@ # Crow Tax — Environment Variables -# Required: Passphrase for encrypting PII (SSN, names) at rest -# Set a passphrase of your own (keep a copy: without it saved returns cannot be read). -CROW_TAX_ENCRYPTION_KEY= +# CROW_TAX_ENCRYPTION_KEY is generated by Crow at install and saved in +# Settings → Passwords; it is not set here. # Optional: Path to tax database (defaults to Crow's main DB) # CROW_TAX_DB_PATH=/path/to/tax.db diff --git a/bundles/tax/engine/ingest/index.js b/bundles/tax/engine/ingest/index.js index a2b0ec811..17a5fccfd 100644 --- a/bundles/tax/engine/ingest/index.js +++ b/bundles/tax/engine/ingest/index.js @@ -6,7 +6,7 @@ * Returns extracted data with confidence scores. */ -import { readFileSync } from "node:fs"; +import { readFileSync, statSync } from "node:fs"; import { extname } from "node:path"; import { extractW2 } from "./w2-reader.js"; import { extract1099 } from "./1099-reader.js"; @@ -37,19 +37,39 @@ const EXTRACTORS = { * @param {string} documentType - Type of document (w2, 1099-sa, 1098-t, etc.) * @returns {Promise<{ data: object, confidence: object, warnings: string[], raw_text: string }>} */ -export async function ingestDocument(filePath, documentType) { +const MAX_DOCUMENT_BYTES = 25 * 1024 * 1024; + +export async function ingestDocument(filePath, documentType, { keys = null } = {}) { + // Checked BEFORE reading: the path comes from the AI (crow_tax_ingest_document), so a + // device, FIFO or huge file must never be read. + const ext = extname(filePath).toLowerCase(); + if (ext !== ".pdf" && ext !== ".enc") throw new Error(`Only PDF files are supported. Got: ${ext}`); + const st = statSync(filePath); + if (!st.isFile()) throw new Error("Not a regular file"); + if (st.size > MAX_DOCUMENT_BYTES) throw new Error("Document is larger than 25 MB"); + const buf = readFileSync(filePath); + // A document the dashboard stored encrypted at rest (.enc): open it with the tax key. + const { isEncryptedDocument, decryptDocument } = await import("../../server/documents-crypto.js"); + if (isEncryptedDocument(buf)) { + const k = keys || [process.env.CROW_TAX_ENCRYPTION_KEY, process.env.CROW_TAX_ENCRYPTION_KEY_PREVIOUS]; + return ingestPdfBuffer((await decryptDocument(buf, k)).plain, documentType, filePath); + } + if (ext !== ".pdf") throw new Error(`Only PDF files are supported. Got: ${ext}`); + return ingestPdfBuffer(buf, documentType, filePath); +} + +/** + * Ingest a PDF already in memory (the dashboard upload: the file on disk is stored + * encrypted, so the plaintext is never written out). `label` names it in errors. + */ +export async function ingestPdfBuffer(pdfBuffer, documentType, label = "the uploaded document") { + const filePath = label; const type = documentType.toLowerCase(); const extractor = EXTRACTORS[type]; if (!extractor) { throw new Error(`Unsupported document type: ${documentType}. Supported: ${Object.keys(EXTRACTORS).join(", ")}`); } - const ext = extname(filePath).toLowerCase(); - if (ext !== ".pdf") { - throw new Error(`Only PDF files are supported. Got: ${ext}`); - } - - const pdfBuffer = readFileSync(filePath); // Pipeline: try form fields first, then text extraction, then OCR let result; diff --git a/bundles/tax/manifest.json b/bundles/tax/manifest.json index 5b579e806..25b0f4072 100644 --- a/bundles/tax/manifest.json +++ b/bundles/tax/manifest.json @@ -1,7 +1,7 @@ { "id": "tax", "name": "Tax Filing Assistant", - "version": "2.0.3", + "version": "2.1.0", "description": "Prepare federal income taxes — PDF document ingestion, calculation, form generation, and guided filing. Federal returns only (no state). Supports 1040, Schedules 1/C/D/SE, Form 8889, Form 8863, and common credits. 2024 + 2025 tax years.", "type": "mcp-server", "author": "Crow", @@ -28,10 +28,14 @@ }, { "name": "CROW_TAX_ENCRYPTION_KEY", - "description": "Passphrase that encrypts personal data at rest (SSN, names). Choose your own and keep a copy: without it saved returns can't be read. Changing it keeps the key it replaces in CROW_TAX_ENCRYPTION_KEY_PREVIOUS, and the tax server moves saved returns to the new key when it next starts. Restart it between two changes.", + "description": "Key that encrypts your saved returns and uploaded documents at rest. Crow generates it at install and saves it in Settings → Passwords (this machine only). Export the keychain to keep a copy: without the key, saved returns and documents can't be read on another machine. A reinstall takes it back from the keychain.", "required": true, "secret": true, - "previous_key": "CROW_TAX_ENCRYPTION_KEY_PREVIOUS" + "generate": "secret", + "keychain": true, + "keychain_label": "data encryption key", + "previous_key": "CROW_TAX_ENCRYPTION_KEY_PREVIOUS", + "replace_values_sha256": ["113e3beef46e2a2ab77b0c3ea62ef37d823b82eab4f713a44af32bcdd2655365"] }, { "name": "CROW_TAX_ENCRYPTION_KEY_PREVIOUS", @@ -41,5 +45,5 @@ } ], "webUI": null, - "notes": "Federal returns only — no state income tax support. Supports MFJ, Single, HoH filing. 2025 tax year. Generates filled IRS PDFs ready to mail and step-by-step guided filing instructions. PII is encrypted at rest. Not a substitute for professional tax advice." + "notes": "Federal returns only — no state income tax support. Supports MFJ, Single, HoH filing. 2025 tax year. Generates filled IRS PDFs ready to mail and step-by-step guided filing instructions. Saved returns and uploaded documents are encrypted at rest with a key Crow generates and keeps in Settings → Passwords; the figures read from documents (shown on the Documents tab) are stored unencrypted in Crow's database. Documents uploaded before tax 2.1.0 are encrypted when the dashboard next starts; plain copies may remain in earlier backups or on the disk until overwritten. Not a substitute for professional tax advice." } diff --git a/bundles/tax/panel/routes.js b/bundles/tax/panel/routes.js index 4b058f259..785ff750b 100644 --- a/bundles/tax/panel/routes.js +++ b/bundles/tax/panel/routes.js @@ -9,12 +9,32 @@ import { Router } from "express"; import multer from "multer"; -import { readFileSync, writeFileSync, mkdirSync, existsSync } from "node:fs"; +import { readFileSync, mkdirSync, existsSync } from "node:fs"; import { join } from "node:path"; import { homedir } from "node:os"; import { pathToFileURL } from "node:url"; +import { randomBytes } from "node:crypto"; -const TAX_DOCS_DIR = join(homedir(), ".crow", "tax-documents"); +// This instance's documents folder (CROW_HOME). Files stored before 2026-10 went to +// ~/.crow/tax-documents for every instance; their rows keep absolute paths, so the +// encryption pass below also looks there. +const TAX_DOCS_DIR = join(process.env.CROW_HOME || join(homedir(), ".crow"), "tax-documents"); +const LEGACY_TAX_DOCS_DIR = join(homedir(), ".crow", "tax-documents"); + +/** The tax key, read when needed, in the SAME order the tax MCP server gets it (its env is + * the gateway env overlaid by this instance's tax entry in mcp-addons.json, proxy.js): the + * entry first, then the gateway env. A generated key reaches only that entry. Never logged. */ +const ADDONS_FILE = join(process.env.CROW_HOME || join(homedir(), ".crow"), "mcp-addons.json"); +function addonEnv() { + try { const env = JSON.parse(readFileSync(ADDONS_FILE, "utf8"))?.tax?.env; return env && typeof env === "object" ? env : {}; } catch { return {}; } +} +export function taxKeys() { + const a = addonEnv(); + return { + key: a.CROW_TAX_ENCRYPTION_KEY || process.env.CROW_TAX_ENCRYPTION_KEY || "", + previous: a.CROW_TAX_ENCRYPTION_KEY_PREVIOUS || process.env.CROW_TAX_ENCRYPTION_KEY_PREVIOUS || "", + }; +} // Resolve bundle directory (installed vs repo) function resolveBundleDir() { @@ -59,9 +79,10 @@ const upload = multer({ export default function taxRouter(authMiddleware) { const router = Router(); const bundleDir = resolveBundleDir(); + const docsCrypto = import(pathToFileURL(join(bundleDir, "server", "documents-crypto.js")).href); // Ensure directories and tables exist - mkdirSync(TAX_DOCS_DIR, { recursive: true }); + mkdirSync(TAX_DOCS_DIR, { recursive: true, mode: 0o700 }); (async () => { try { const db = await getDb(); @@ -86,6 +107,17 @@ export default function taxRouter(authMiddleware) { try { await db.execute({ sql: "ALTER TABLE tax_documents ADD COLUMN owner TEXT DEFAULT 'taxpayer'", args: [] }); } catch {} // Column already exists + // Documents stored before encryption at rest: move each to its encrypted form + // (verified before the plain file is removed; documents-crypto.js). Idempotent. + const { key, previous } = taxKeys(); + // CROW_TAX_KEEP_PLAIN_DOCUMENTS=1: after `scripts/documents.mjs decrypt --in-place` (a downgrade). + if (key && process.env.CROW_TAX_KEEP_PLAIN_DOCUMENTS !== "1") { + const { encryptStoredDocuments } = await docsCrypto; + const r = await encryptStoredDocuments(db, { key, previous, docsDirs: [TAX_DOCS_DIR, LEGACY_TAX_DOCS_DIR] }); + if (r.encrypted || r.reencrypted) console.log(`[tax] encrypted ${r.encrypted} stored document(s), re-encrypted ${r.reencrypted}`); + if (r.errors.length) console.warn(`[tax] ${r.errors.length} stored document(s) were left as they are: ${r.errors.join("; ")}`); + if (r.locked) console.warn("[tax] another run holds the documents lock; plain documents (if any) are encrypted at the next start"); + } } catch {} })(); @@ -175,11 +207,22 @@ export default function taxRouter(authMiddleware) { return res.status(400).json({ error: "No PDF file uploaded" }); } + // Stored encrypted with the tax key; the plaintext is never written to disk. + const { key } = taxKeys(); + if (!key) { + return res.status(409).json({ error: "The tax extension has no encryption key yet. Reinstall it in Extensions (your returns and documents are kept), then upload again." }); + } + const { encryptDocument, writePrivateAtomic } = await docsCrypto; + let sealed; + try { sealed = await encryptDocument(req.file.buffer, key); } catch (err) { + // The example key once published in the repo: a reinstall moves this install to a new key. + return res.status(409).json({ error: `${err.message} Reinstall the tax extension in Extensions (your returns and documents are kept): it moves to a new key.` }); + } const db = await getDb(); const filename = req.file.originalname; const safeName = filename.replace(/[^a-zA-Z0-9._-]/g, "_"); - const filePath = join(TAX_DOCS_DIR, `${Date.now()}-${safeName}`); - writeFileSync(filePath, req.file.buffer); + const filePath = join(TAX_DOCS_DIR, `${Date.now()}-${randomBytes(3).toString("hex")}-${safeName}.enc`); + writePrivateAtomic(filePath, sealed); // Record upload with owner tag await db.execute({ @@ -194,8 +237,8 @@ export default function taxRouter(authMiddleware) { // Try ingestion let ingestionResult = null; try { - const { ingestDocument } = await import(pathToFileURL(join(bundleDir, "engine", "ingest", "index.js")).href); - ingestionResult = await ingestDocument(filePath, doc_type); + const { ingestPdfBuffer } = await import(pathToFileURL(join(bundleDir, "engine", "ingest", "index.js")).href); + ingestionResult = await ingestPdfBuffer(req.file.buffer, doc_type, filename); await db.execute({ sql: `UPDATE tax_documents SET status = 'ingested', @@ -313,8 +356,11 @@ export default function taxRouter(authMiddleware) { const db = await getDb(); // Get file path to delete the uploaded file const result = await db.execute({ sql: "SELECT file_path FROM tax_documents WHERE id = ?", args: [req.params.id] }); - if (result.rows[0]?.file_path) { - try { (await import("node:fs")).unlinkSync(result.rows[0].file_path); } catch {} + // Only a regular file inside a documents folder is ever removed (the path comes from the DB). + const { storedDocumentPath } = await docsCrypto; + const fp = storedDocumentPath(result.rows[0]?.file_path, [TAX_DOCS_DIR, LEGACY_TAX_DOCS_DIR]); + if (fp) { + try { (await import("node:fs")).unlinkSync(fp); } catch {} } await db.execute({ sql: "DELETE FROM tax_documents WHERE id = ?", args: [req.params.id] }); if (req.headers.accept?.includes("text/html")) { diff --git a/bundles/tax/scripts/documents.mjs b/bundles/tax/scripts/documents.mjs new file mode 100644 index 000000000..96b492af8 --- /dev/null +++ b/bundles/tax/scripts/documents.mjs @@ -0,0 +1,51 @@ +#!/usr/bin/env node +/** + * Crow Tax — stored documents, by hand. + * + * node scripts/documents.mjs encrypt encrypt every plain stored document + * (what the dashboard does at start-up) + * node scripts/documents.mjs decrypt --out write plain copies there (0600); + * the database is not changed + * node scripts/documents.mjs decrypt --in-place undo the encryption (a downgrade): + * set CROW_TAX_KEEP_PLAIN_DOCUMENTS=1 + * first, or the next start encrypts again + * + * The key comes from this instance's tax entry in mcp-addons.json, else + * CROW_TAX_ENCRYPTION_KEY (and _PREVIOUS) — the MCP server's order. It is never printed. + */ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { homedir } from "node:os"; +import { createDbClient } from "../server/db.js"; +import { encryptStoredDocuments, decryptStoredDocuments } from "../server/documents-crypto.js"; + +function keys() { + let entry = {}; + try { entry = JSON.parse(readFileSync(join(process.env.CROW_HOME || join(homedir(), ".crow"), "mcp-addons.json"), "utf8"))?.tax?.env || {}; } catch { /* none */ } + return { + key: entry.CROW_TAX_ENCRYPTION_KEY || process.env.CROW_TAX_ENCRYPTION_KEY || "", + previous: entry.CROW_TAX_ENCRYPTION_KEY_PREVIOUS || process.env.CROW_TAX_ENCRYPTION_KEY_PREVIOUS || "", + }; +} + +const [cmd, ...rest] = process.argv.slice(2); +const { key, previous } = keys(); +if (!key) { console.error("No tax key: install the tax extension, or set CROW_TAX_ENCRYPTION_KEY."); process.exit(2); } +// The database of the SAME instance as the key and the folders (CROW_HOME), never another one's. +if (process.env.CROW_HOME && !process.env.CROW_DB_PATH && !process.env.CROW_DATA_DIR && !process.env.CROW_TAX_DB_PATH) { + process.env.CROW_DB_PATH = join(process.env.CROW_HOME, "data", "crow.db"); +} +const db = createDbClient(); +let out; +if (cmd === "encrypt") { + out = await encryptStoredDocuments(db, { key, previous }); +} else if (cmd === "decrypt" && rest[0] === "--in-place") { + out = await decryptStoredDocuments(db, { keys: [key, previous], inPlace: true }); +} else if (cmd === "decrypt" && rest[0] === "--out" && rest[1]) { + out = await decryptStoredDocuments(db, { keys: [key, previous], outDir: rest[1] }); +} else { + console.error("usage: documents.mjs encrypt | decrypt --out | decrypt --in-place"); + process.exit(2); +} +console.log(JSON.stringify(out, null, 2)); +process.exit(out.errors?.length || out.locked ? 1 : 0); diff --git a/bundles/tax/server/documents-crypto.js b/bundles/tax/server/documents-crypto.js new file mode 100644 index 000000000..e9884585f --- /dev/null +++ b/bundles/tax/server/documents-crypto.js @@ -0,0 +1,275 @@ +/** + * Crow Tax — uploaded documents encrypted at rest (recovery keys, queue item 30, 2026-10-16). + * + * A W-2 or 1099 holds SSNs and income. Uploads are stored encrypted with the tax key + * (CROW_TAX_ENCRYPTION_KEY, generated by Crow and saved in the keychain): + * "CROWTAXDOC1\n" + keyId(8) + salt(16) + iv(12) + tag(16) + AES-256-GCM ciphertext + * key = PBKDF2-SHA256(passphrase, salt, 100k) — the same derivation as crypto.js + * keyId = HMAC-SHA256(that derived key, "crow-tax-document-key-id")[0..8] — says WHICH + * key sealed a file, so a wrong key is detected before anything is replaced. It + * comes from the salted PBKDF2 output, so it offers no cheaper offline guess + * check than the ciphertext itself, and it differs per file. + * + * Files stored before this change are plain PDFs. encryptStoredDocuments() moves them one + * row at a time and never deletes a byte it has not verified: + * read → encrypt → write a private temp file + fsync → rename to a name unique to this + * run (..enc) → read back + decrypt + compare → compare-and-swap the row's + * file_path → only then unlink the plain file (and fsync the folder). + * A lost swap removes only this run's own .enc. A crash leaves the plain file plus at most a + * stray .enc; a crash after the swap leaves a plain sibling, which the next run removes once + * the .enc it points to is verified. Runs (the panel at boot, the CLI) hold an exclusive lock + * file in the documents folder, so two migrators never interleave. + * decryptStoredDocuments() is the inverse (in place), or writes plain copies to a folder + * without touching the database. Only regular files inside the documents folder(s) are ever + * read, written or removed: a file_path from the database pointing anywhere else (or a + * symlink) is skipped and reported by row id. + */ +import { createCipheriv, createDecipheriv, createHmac, randomBytes, pbkdf2 } from "node:crypto"; +import { promisify } from "node:util"; +import { openSync, writeSync, fsyncSync, closeSync, renameSync, readFileSync, unlinkSync, existsSync, lstatSync, realpathSync, mkdirSync, statSync, readdirSync, utimesSync, chmodSync } from "node:fs"; +import { join, dirname, basename, resolve, sep } from "node:path"; +import { homedir } from "node:os"; +import { isExampleKey } from "./crypto.js"; + +const pbkdf2Async = promisify(pbkdf2); +export const MAGIC = Buffer.from("CROWTAXDOC1\n", "utf8"); +const KID = 8, SALT = 16, IV = 12, TAG = 16, KEYLEN = 32, ITER = 100000; +const HEADER = MAGIC.length + KID + SALT + IV + TAG; +const LOCK_STALE_MS = 30 * 60 * 1000; +const ENC_NAME = /\.[0-9a-f]{8}\.enc$/; +const ENC_SUFFIX_LEN = ".00000000.enc".length; + +/** This instance's documents folder (CROW_HOME: co-hosted instances each have their own). */ +export function taxDocsDir(env = process.env) { + return join(env.CROW_HOME || join(homedir(), ".crow"), "tax-documents"); +} +/** Where every instance stored uploads before 2026-10 (it ignored CROW_HOME). */ +export function legacyTaxDocsDir() { + return join(homedir(), ".crow", "tax-documents"); +} + +const kidOf = (derived) => createHmac("sha256", derived).update("crow-tax-document-key-id").digest().subarray(0, KID); + +export function isEncryptedDocument(buf) { + return Buffer.isBuffer(buf) && buf.length >= HEADER && buf.subarray(0, MAGIC.length).equals(MAGIC); +} + +function usable(passphrase) { + if (!passphrase) throw new Error("CROW_TAX_ENCRYPTION_KEY is not set: documents cannot be stored encrypted."); + if (isExampleKey(passphrase)) throw new Error("CROW_TAX_ENCRYPTION_KEY is the example value published in Crow's repository; documents are never stored with it."); +} + +export async function encryptDocument(plain, passphrase) { + usable(passphrase); + const salt = randomBytes(SALT); + const iv = randomBytes(IV); + const derived = await pbkdf2Async(passphrase, salt, ITER, KEYLEN, "sha256"); + const c = createCipheriv("aes-256-gcm", derived, iv); + const ct = Buffer.concat([c.update(plain), c.final()]); + return Buffer.concat([MAGIC, kidOf(derived), salt, iv, c.getAuthTag(), ct]); +} + +function parts(buf) { + let o = MAGIC.length; + const kid = buf.subarray(o, o += KID), salt = buf.subarray(o, o += SALT), iv = buf.subarray(o, o += IV), tag = buf.subarray(o, o += TAG); + return { kid, salt, iv, tag, ct: buf.subarray(o) }; +} + +/** The passphrase (of `passphrases`) that sealed this file, and its derived key; or null. */ +async function sealing(buf, passphrases) { + if (!isEncryptedDocument(buf)) return null; + const { kid, salt } = parts(buf); + for (const p of (passphrases || []).filter(Boolean)) { + const derived = await pbkdf2Async(p, salt, ITER, KEYLEN, "sha256"); + if (kidOf(derived).equals(kid)) return { passphrase: p, derived }; + } + return null; +} +export async function sealedWith(buf, passphrases) { + return (await sealing(buf, passphrases))?.passphrase || null; +} + +export async function decryptDocument(buf, passphrases) { + if (!isEncryptedDocument(buf)) throw new Error("not an encrypted tax document"); + const k = await sealing(buf, Array.isArray(passphrases) ? passphrases : [passphrases]); + if (!k) throw new Error("this document was stored with a different tax key (not the current or previous one)"); + const { iv, tag, ct } = parts(buf); + const d = createDecipheriv("aes-256-gcm", k.derived, iv); + d.setAuthTag(tag); + return { plain: Buffer.concat([d.update(ct), d.final()]), passphrase: k.passphrase }; +} + +function fsyncDir(dir) { + try { const fd = openSync(dir, "r"); try { fsyncSync(fd); } finally { closeSync(fd); } } catch { /* best-effort */ } +} + +/** Crash-safe private write: temp (0600, exclusive) → fsync → rename → fsync the folder. */ +export function writePrivateAtomic(path, buf) { + mkdirSync(dirname(path), { recursive: true, mode: 0o700 }); + const tmp = join(dirname(path), `.${basename(path)}.tmp-${process.pid}-${randomBytes(4).toString("hex")}`); + const fd = openSync(tmp, "wx", 0o600); + try { writeSync(fd, buf); fsyncSync(fd); } finally { closeSync(fd); } + renameSync(tmp, path); + fsyncDir(dirname(path)); +} + +/** The path, when it is a regular file (not a symlink) whose real location is inside one of `dirs`. */ +export function storedDocumentPath(p, dirs) { + if (typeof p !== "string" || !p) return null; + const abs = resolve(p); + for (const d of dirs) { + let root; + try { root = realpathSync(d); } catch { continue; } + if (!abs.startsWith(resolve(d) + sep) && !abs.startsWith(root + sep)) continue; + try { + if (!lstatSync(abs).isFile()) return null; + return realpathSync(abs).startsWith(root + sep) ? abs : null; + } catch { return null; } + } + return null; +} + +/** + * A row's file. Fallbacks, for a crow.db restored from a backup: the same name in this + * instance's folder (restored under another CROW_HOME); and, when the row still names the + * plain file a later run migrated, that file's single encrypted successor .<8hex>.enc + * (in its own folder or this one). The caller swaps the row to whatever is found. + */ +function resolveRowFile(p, dirs) { + const direct = storedDocumentPath(p, dirs); + if (direct) return direct; + if (typeof p !== "string" || !p) return null; + const name = basename(p); + const here = storedDocumentPath(join(dirs[0], name), dirs); + if (here) return here; + if (ENC_NAME.test(name)) return null; + for (const d of [dirname(resolve(p)), dirs[0]]) { + let list = []; + try { list = readdirSync(d).filter((f) => f.startsWith(name + ".") && ENC_NAME.test(f) && f.slice(0, -ENC_SUFFIX_LEN) === name); } catch { continue; } + if (list.length === 1) { const hit = storedDocumentPath(join(d, list[0]), dirs); if (hit) return hit; } + } + return null; +} + +/** + * Exclusive lock in the documents folder (also tightens the folder to 0700). A lock not + * touched for 30 minutes is stale: it is taken over by an atomic rename, which only one + * contender can win. The holder touches it as it goes (`touch`). + */ +function withLock(dir) { + mkdirSync(dir, { recursive: true, mode: 0o700 }); + try { chmodSync(dir, 0o700); } catch { /* not ours to change */ } + const lock = join(dir, ".documents.lock"); + const take = () => { const fd = openSync(lock, "wx", 0o600); closeSync(fd); }; + try { + take(); + } catch { + try { + if (Date.now() - statSync(lock).mtimeMs < LOCK_STALE_MS) return null; + const aside = `${lock}.stale-${process.pid}-${randomBytes(4).toString("hex")}`; + renameSync(lock, aside); // ENOENT for every contender but one + unlinkSync(aside); + take(); + } catch { return null; } + } + const release = () => { try { unlinkSync(lock); } catch {} }; + release.touch = () => { try { const t = new Date(); utimesSync(lock, t, t); } catch {} }; + return release; +} + +async function swapPath(db, id, from, to) { + const r = await db.execute({ sql: "UPDATE tax_documents SET file_path = ? WHERE id = ? AND file_path = ?", args: [to, id, from] }); + return Number(r.rowsAffected ?? r.changes ?? 0) > 0; +} + + + +/** + * Move every plain stored document to its encrypted form (and re-encrypt one only the + * previous key opens). Idempotent. Returns { encrypted, reencrypted, cleaned, skipped, errors, locked }. + */ +export async function encryptStoredDocuments(db, { key, previous = null, docsDirs = [taxDocsDir(), legacyTaxDocsDir()] } = {}) { + const out = { encrypted: 0, reencrypted: 0, cleaned: 0, skipped: 0, errors: [], locked: false }; + if (!key) return { ...out, errors: ["CROW_TAX_ENCRYPTION_KEY is not set"] }; + try { usable(key); } catch (err) { return { ...out, errors: [err.message] }; } + let rows; + try { rows = (await db.execute("SELECT id, file_path FROM tax_documents")).rows; } catch { return out; } + const release = withLock(docsDirs[0]); + if (!release) return { ...out, locked: true }; + try { + for (const r of rows) { + release.touch(); + const p = resolveRowFile(r.file_path, docsDirs); + if (!p) { out.skipped++; continue; } + try { + const buf = readFileSync(p); + if (isEncryptedDocument(buf)) { + // Found by a fallback (a restored crow.db): point the row at it. + if (p !== resolve(r.file_path) && (await swapPath(db, r.id, r.file_path, p))) r.file_path = p; + const { plain, passphrase } = await decryptDocument(buf, [key, previous]); + // A crash after the swap left the plain original next to it: remove it once the + // .enc is verified to hold exactly those bytes. + const sibling = ENC_NAME.test(p) ? storedDocumentPath(p.replace(ENC_NAME, ""), docsDirs) : null; + if (sibling && readFileSync(sibling).equals(plain)) { unlinkSync(sibling); fsyncDir(dirname(sibling)); out.cleaned++; } + if (passphrase === key) { out.skipped++; continue; } + const next = await encryptDocument(plain, key); + if (!(await decryptDocument(next, [key])).plain.equals(plain)) throw new Error("verification failed"); + writePrivateAtomic(p, next); + out.reencrypted++; + continue; + } + const target = `${p.replace(ENC_NAME, "")}.${randomBytes(4).toString("hex")}.enc`; + writePrivateAtomic(target, await encryptDocument(buf, key)); + if (!(await decryptDocument(readFileSync(target), [key])).plain.equals(buf)) { unlinkSync(target); throw new Error("verification failed"); } + if (!(await swapPath(db, r.id, r.file_path, target))) { unlinkSync(target); out.skipped++; continue; } + unlinkSync(p); + fsyncDir(dirname(p)); + out.encrypted++; + } catch (err) { + out.errors.push(`document ${r.id}: ${err.message}`); + } + } + } finally { release(); } + return out; +} + +/** + * The inverse. inPlace: plain file written + verified, row swapped back, .enc removed + * (for a downgrade: set CROW_TAX_KEEP_PLAIN_DOCUMENTS=1 first, or the next start encrypts + * them again). Otherwise plain copies go to outDir (0700, files 0600), database untouched. + */ +export async function decryptStoredDocuments(db, { keys, inPlace = false, outDir = null, docsDirs = [taxDocsDir(), legacyTaxDocsDir()] } = {}) { + const out = { decrypted: 0, skipped: 0, errors: [], locked: false }; + if (!inPlace && !outDir) throw new Error("choose inPlace or outDir"); + const rows = (await db.execute("SELECT id, filename, file_path FROM tax_documents")).rows; + const release = inPlace ? withLock(docsDirs[0]) : () => {}; + if (!release) return { ...out, locked: true }; + try { + for (const r of rows) { + const p = resolveRowFile(r.file_path, docsDirs); + if (!p) { out.skipped++; continue; } + try { + const buf = readFileSync(p); + if (!isEncryptedDocument(buf)) { out.skipped++; continue; } + const { plain } = await decryptDocument(buf, keys); + if (inPlace) { + const target = ENC_NAME.test(p) ? p.replace(ENC_NAME, "") : p.endsWith(".enc") ? p.slice(0, -4) : `${p}.pdf`; + if (existsSync(target)) throw new Error(`${basename(target)} already exists`); + writePrivateAtomic(target, plain); + if (!readFileSync(target).equals(plain)) { unlinkSync(target); throw new Error("verification failed"); } + if (!(await swapPath(db, r.id, r.file_path, target))) { unlinkSync(target); out.skipped++; continue; } + unlinkSync(p); + } else { + const name = `${r.id}-${String(r.filename || "document.pdf").replace(/[^a-zA-Z0-9._-]/g, "_")}`; + mkdirSync(outDir, { recursive: true, mode: 0o700 }); + try { chmodSync(outDir, 0o700); } catch { /* not ours */ } + writePrivateAtomic(join(outDir, name), plain); + } + out.decrypted++; + } catch (err) { + out.errors.push(`document ${r.id}: ${err.message}`); + } + } + } finally { release(); } + return out; +} diff --git a/registry/add-ons.json b/registry/add-ons.json index 6b5f3b48d..a43b8a02b 100644 --- a/registry/add-ons.json +++ b/registry/add-ons.json @@ -745,7 +745,7 @@ { "id": "campaigns", "name": "Campaigns", - "version": "1.0.2", + "version": "1.0.3", "description": "Social media campaign manager — draft, schedule, and publish posts across platforms (starting with Reddit)", "type": "mcp-server", "author": "Crow", @@ -780,9 +780,12 @@ "env_vars": [ { "name": "CROW_CAMPAIGNS_ENCRYPTION_KEY", - "description": "AES-256-GCM encryption key for Reddit credentials (generate with: openssl rand -base64 32)", + "description": "Key that encrypts your saved Reddit credentials (AES-256-GCM). Crow generates it at install and saves it in Settings → Passwords (this machine only); a reinstall takes it back from the keychain. Without it, re-enter the Reddit credentials.", "required": true, - "secret": true + "secret": true, + "generate": "secret", + "keychain": true, + "keychain_label": "data encryption key" } ], "notes": "12 MCP tools for social media campaign management. Reddit support via snoowrap. Credentials encrypted at rest with AES-256-GCM. Dashboard panel with approval workflow.", @@ -5905,7 +5908,7 @@ { "id": "tax", "name": "Tax Filing Assistant", - "version": "2.0.3", + "version": "2.1.0", "description": "Prepare federal income taxes — PDF document ingestion, calculation, form generation, and guided filing. Federal returns only (no state). Supports 1040, Schedules 1/C/D/SE, Form 8889, Form 8863, and common credits. 2024 + 2025 tax years.", "type": "mcp-server", "author": "Crow", @@ -5946,10 +5949,16 @@ }, { "name": "CROW_TAX_ENCRYPTION_KEY", - "description": "Passphrase that encrypts personal data at rest (SSN, names). Choose your own and keep a copy: without it saved returns can't be read. Changing it keeps the key it replaces in CROW_TAX_ENCRYPTION_KEY_PREVIOUS, and the tax server moves saved returns to the new key when it next starts. Restart it between two changes.", + "description": "Key that encrypts your saved returns and uploaded documents at rest. Crow generates it at install and saves it in Settings → Passwords (this machine only). Export the keychain to keep a copy: without the key, saved returns and documents can't be read on another machine. A reinstall takes it back from the keychain.", "required": true, "secret": true, - "previous_key": "CROW_TAX_ENCRYPTION_KEY_PREVIOUS" + "generate": "secret", + "keychain": true, + "keychain_label": "data encryption key", + "previous_key": "CROW_TAX_ENCRYPTION_KEY_PREVIOUS", + "replace_values_sha256": [ + "113e3beef46e2a2ab77b0c3ea62ef37d823b82eab4f713a44af32bcdd2655365" + ] }, { "name": "CROW_TAX_ENCRYPTION_KEY_PREVIOUS", @@ -5959,7 +5968,7 @@ } ], "webUI": null, - "notes": "Federal returns only — no state income tax support. Supports MFJ, Single, HoH filing. 2025 tax year. Generates filled IRS PDFs ready to mail and step-by-step guided filing instructions. PII is encrypted at rest. Not a substitute for professional tax advice.", + "notes": "Federal returns only — no state income tax support. Supports MFJ, Single, HoH filing. 2025 tax year. Generates filled IRS PDFs ready to mail and step-by-step guided filing instructions. Saved returns and uploaded documents are encrypted at rest with a key Crow generates and keeps in Settings → Passwords; the figures read from documents (shown on the Documents tab) are stored unencrypted in Crow's database. Documents uploaded before tax 2.1.0 are encrypted when the dashboard next starts; plain copies may remain in earlier backups or on the disk until overwritten. Not a substitute for professional tax advice.", "official": true }, { diff --git a/scripts/lib/bundle-contract.mjs b/scripts/lib/bundle-contract.mjs index 187cb0874..549e068e7 100644 --- a/scripts/lib/bundle-contract.mjs +++ b/scripts/lib/bundle-contract.mjs @@ -205,6 +205,13 @@ export function validateManifest(manifest, bundleDir, opts = {}) { const other = (manifest.env_vars || []).find((o) => o && o.name === v.previous_key); if (typeof v.previous_key !== "string" || !other || other.name === v.name) errors.push(`env_vars ${v.name}: previous_key must name another declared env var`); } + if (v.replace_values_sha256 !== undefined) { + if (!Array.isArray(v.replace_values_sha256) || v.replace_values_sha256.length === 0 || !v.replace_values_sha256.every((h) => typeof h === "string" && /^[0-9a-f]{64}$/.test(h))) { + errors.push(`env_vars ${v.name}: replace_values_sha256 must be a non-empty list of lowercase sha256 hex digests`); + } else if (v.generate === undefined || typeof v.previous_key !== "string") { + errors.push(`env_vars ${v.name}: replace_values_sha256 needs generate and previous_key (the replaced value moves there)`); + } + } if (v.default_in_crow_home !== undefined) { const rel = v.default_in_crow_home; if (typeof rel !== "string" || rel === "" || rel.startsWith("/") || rel.includes("\\") || rel.includes("$") || rel.split("/").includes("..")) { diff --git a/servers/gateway/bundle-env-secrets.js b/servers/gateway/bundle-env-secrets.js index 603f14f5a..ebd085857 100644 --- a/servers/gateway/bundle-env-secrets.js +++ b/servers/gateway/bundle-env-secrets.js @@ -70,6 +70,13 @@ export function generatedEnvKeys(manifest) { .map((v) => v.name); } +/** Whether `value` is one the manifest lists as publicly known (sha256 of the trimmed value). */ +export function isReplacedValue(spec, value) { + const list = spec?.replace_values_sha256; + if (!Array.isArray(list) || typeof value !== "string" || value === "") return false; + return list.includes(createHash("sha256").update(value.trim()).digest("hex")); +} + export function newSecretValue(kind = "secret") { if (kind === "laravel_key") return `base64:${randomBytes(32).toString("base64")}`; return randomBytes(32).toString("base64url"); @@ -143,7 +150,7 @@ export function keychainEligibleKeys(manifest) { */ export function planGeneratedEnv(bundleId, manifest, { destDir, crowHome, composeText = null, priorInstall = false, keychain = null }) { const keys = generatedEnvKeys(manifest); - if (keys.length === 0) return { env: {}, minted: {}, reusedPlain: {}, restored: [], persist() {} }; + if (keys.length === 0) return { env: {}, minted: {}, reusedPlain: {}, restored: [], replaced: [], replaceSkipped: [], persist() {} }; if (typeof bundleId !== "string" || !/^[a-z0-9][a-z0-9-]*$/.test(bundleId) || bundleId.length > 64) { throw new Error(`Invalid bundle ID: ${JSON.stringify(bundleId)}`); } @@ -162,13 +169,15 @@ export function planGeneratedEnv(bundleId, manifest, { destDir, crowHome, compos // keychain may not hold them, and the form no longer shows the field — see install-hooks. const reusedPlain = {}; const restored = []; + const replaced = []; + const replaceSkipped = []; const existingFor = (k) => installed[k] || retained[k] || legacy[k] || ""; // Private halves first, so a public half can be derived from (or minted with) its pair. const order = [...keys].sort((a, b) => (byName.get(a)?.generate === "vapid_public_key") - (byName.get(b)?.generate === "vapid_public_key")); const pairs = {}; for (const k of order) { const spec = byName.get(k) || {}; - const existing = existingFor(k); + let existing = existingFor(k); if (spec.generate === "vapid_public_key") { const priv = env[spec.pair]; if (!priv || byName.get(spec.pair)?.generate !== "vapid_private_key") { @@ -179,7 +188,20 @@ export function planGeneratedEnv(bundleId, manifest, { destDir, crowHome, compos env[k] = pairs[spec.pair] || vapidPublicFromPrivate(priv) || existing; continue; } - const fromKeychain = !existing && spec.keychain === true && keychain && typeof keychain[k] === "string" && keychain[k] !== "" ? keychain[k] : ""; + // A value once published in a repo example (env_vars[].replace_values_sha256, e.g. tax's + // old .env.example key) is never kept: a new key is generated and the old one moves to + // the previous_key slot, so data saved with it still opens and is re-encrypted. + if (existing && isReplacedValue(spec, existing)) { + const prevSlot = spec.previous_key; + if (prevSlot && !installed[prevSlot] && !retained[prevSlot]) { + env[prevSlot] = existing; + replaced.push(k); + existing = ""; + } else { + replaceSkipped.push(k); + } + } + const fromKeychain = !existing && spec.keychain === true && !isReplacedValue(spec, keychain?.[k]) && keychain && typeof keychain[k] === "string" && keychain[k] !== "" ? keychain[k] : ""; if (fromKeychain) { // The keychain holds the plaintext; a store_as key gets a fresh hash of it (any hash // of the same plaintext verifies). Its entry exists, so nothing is saved again. @@ -216,7 +238,7 @@ export function planGeneratedEnv(bundleId, manifest, { destDir, crowHome, compos formatEnvLines({ ...retained, ...env }), ); }; - return { env, minted, reusedPlain, restored, persist }; + return { env, minted, reusedPlain, restored, replaced, replaceSkipped, persist }; } /** diff --git a/servers/gateway/keychain/install-hooks.js b/servers/gateway/keychain/install-hooks.js index f62e935be..5cd3acb8e 100644 --- a/servers/gateway/keychain/install-hooks.js +++ b/servers/gateway/keychain/install-hooks.js @@ -10,7 +10,7 @@ * - the optional vault copy uses the typed vault credentials once, then drops them. */ import { createDbClient, auditLog } from "../../db.js"; -import { keychainGeneratedKeys, keychainEligibleKeys, expandKeychainTemplate } from "../bundle-env-secrets.js"; +import { keychainGeneratedKeys, keychainEligibleKeys, expandKeychainTemplate, isReplacedValue } from "../bundle-env-secrets.js"; import { createNotification } from "../../shared/notifications.js"; import { saveExtensionSecret, reactivateBundleEntries, hasExtensionEntry, markBundleRemoved, ensureWriteKey, listEntries, openEntrySecret } from "./store.js"; import { loadKeychainKey } from "./key.js"; @@ -214,7 +214,8 @@ export const BACKFILL_SETTING = "keychain_backfilled_keys"; export async function backfillKeychainEntries(bundleId, manifest, values, { log = () => {} } = {}) { const specs = new Map((manifest?.env_vars || []).map((v) => [v.name, v])); - const todo = keychainGeneratedKeys(manifest).filter((k) => !specs.get(k)?.store_as && values?.[k]?.plain); + // Never a publicly known value (replace_values_sha256): the next install replaces it. + const todo = keychainGeneratedKeys(manifest).filter((k) => !specs.get(k)?.store_as && values?.[k]?.plain && !isReplacedValue(specs.get(k), values[k].plain)); if (todo.length === 0) return 0; let db; let saved = 0; diff --git a/servers/gateway/routes/bundles.js b/servers/gateway/routes/bundles.js index 93fdcd6fd..0ed8dadf1 100644 --- a/servers/gateway/routes/bundles.js +++ b/servers/gateway/routes/bundles.js @@ -780,9 +780,11 @@ async function refreshVersionedBundle({ id, appSrc, destDir, runner }) { const topFiles = isDocker ? ["manifest.json", "settings-section.js", "package.json", "package-lock.json"] : ["manifest.json", "package.json", "package-lock.json", "pyproject.toml", "uv.lock", "settings-section.js", "main.py", "run.sh", "config.py"]; + // engine/ + pdf/: tax's calculator, ingest and form filler (its panel imports the + // installed engine/, so a version bump must ship it with the panel). const dirs = isDocker ? ["server", "panel", "skills"] - : ["server", "panel", "skills", "curriculum", "public", "scripts", "templates", "routes", "config", "src"]; + : ["server", "panel", "skills", "curriculum", "public", "scripts", "templates", "routes", "config", "src", "engine", "pdf"]; // Never copied for ANY type: docker-compose.yml, Dockerfile, entrypoint.sh, // .env*, node_modules/, data/ — simply absent from both lists above. The ONE // exception is a docker bundle's explicit opt-in, manifest.refreshFiles @@ -2562,6 +2564,8 @@ export async function runInstallJob(bundleId, envVars, { job, installedSnapshot, return { ok: false, reason }; } if (plan.restored.length > 0) appendLog(job, `Restored ${plan.restored.length} key(s) from Crow keychain (Settings → Passwords): ${plan.restored.join(", ")}`); + if (plan.replaced.length > 0) appendLog(job, `${plan.replaced.join(", ")} was the example value once published in Crow's repository: a new key was generated, and the old one kept in its previous-key slot so data saved with it still opens`); + if (plan.replaceSkipped?.length > 0) appendLog(job, `${plan.replaceSkipped.join(", ")} is still the example value once published in Crow's repository, but its previous-key slot already holds another key, so it was left as it is. Open that data, clear the previous-key slot in Configure, then reinstall.`); const generated = plan.env; const writeEnv = { ...(reqEnv || {}), ...generated }; let installEnv = writeEnv; diff --git a/tests/bundle-env-placeholders.test.js b/tests/bundle-env-placeholders.test.js index 585e3e581..fea09770f 100644 --- a/tests/bundle-env-placeholders.test.js +++ b/tests/bundle-env-placeholders.test.js @@ -242,12 +242,16 @@ test("tax: at start, returns only the previous key opens are re-encrypted to the } finally { db.close?.(); } }); -test("Configure: a tax key change always keeps the key just replaced in _PREVIOUS (and warns when an older one is dropped)", async () => { +test("Configure: a typed key change always keeps the key just replaced in _PREVIOUS (and warns when an older one is dropped)", async () => { const express = (await import("express")).default; const id = "tax"; const dir = join(CROW_HOME, "bundles", id); mkdirSync(dir, { recursive: true }); - copyFileSync(join(REPO, "bundles", id, "manifest.json"), join(dir, "manifest.json")); + // tax's key is generated since 2.1.0 (Configure never takes it). The previous_key mechanism + // still serves any typed key: test it on the tax manifest as a typed field. + const typed = JSON.parse(readFileSync(join(REPO, "bundles", id, "manifest.json"), "utf8")); + for (const v of typed.env_vars) if (v.name === "CROW_TAX_ENCRYPTION_KEY") { delete v.generate; delete v.keychain; delete v.keychain_label; delete v.replace_values_sha256; } + writeFileSync(join(dir, "manifest.json"), JSON.stringify(typed)); writeFileSync(join(dir, ".env"), "CROW_TAX_ENCRYPTION_KEY=old-pass-1\n"); const app = express(); app.use(express.json()); app.use(B.default()); const server = app.listen(0, "127.0.0.1"); await new Promise((r) => server.once("listening", r)); @@ -269,3 +273,18 @@ test("Configure: a tax key change always keeps the key just replaced in _PREVIOU assert.equal(parseEnvText(readFileSync(join(dir, ".env"), "utf8")).CROW_TAX_ENCRYPTION_KEY_PREVIOUS, "newer-pass-3"); } finally { server.close(); } }); + +test("Configure never overwrites tax's generated key (it is not a form field)", async () => { + const express = (await import("express")).default; + const id = "tax"; + const dir = join(CROW_HOME, "bundles", id); + mkdirSync(dir, { recursive: true }); + copyFileSync(join(REPO, "bundles", id, "manifest.json"), join(dir, "manifest.json")); + writeFileSync(join(dir, ".env"), "CROW_TAX_ENCRYPTION_KEY=generated-key-0123456789\n"); + const app = express(); app.use(express.json()); app.use(B.default()); + const server = app.listen(0, "127.0.0.1"); await new Promise((r) => server.once("listening", r)); + try { + await fetch(`http://127.0.0.1:${server.address().port}/bundles/api/env`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ bundle_id: id, env_vars: { CROW_TAX_ENCRYPTION_KEY: "typed-over-it" } }) }); + assert.equal(parseEnvText(readFileSync(join(dir, ".env"), "utf8")).CROW_TAX_ENCRYPTION_KEY, "generated-key-0123456789"); + } finally { server.close(); } +}); diff --git a/tests/bundle-form-friction.test.js b/tests/bundle-form-friction.test.js index 2ab3258ae..600d91738 100644 --- a/tests/bundle-form-friction.test.js +++ b/tests/bundle-form-friction.test.js @@ -6,7 +6,7 @@ * - internal machine secrets (DB/Redis passwords, app signing keys, JWT/OTP/VAPID, shared * runner secrets) carry `generate` — never a typed field * - every survey F1 key is generated, or exempt with a written reason - * - every generated key of a docker bundle is consumed by its compose file + * - every generated key is consumed by its compose file or its MCP server * - every required human field has a description * - optional fields with a default fold under "Advanced" (bundle-env-form.js) * - the fediverse bundles carry no dead per-bundle S3 fields @@ -81,8 +81,6 @@ const INTERNAL_SECRET_PATTERNS = [ * entry needs a reason a reviewer would accept. */ const EXEMPT = { - "campaigns:CROW_CAMPAIGNS_ENCRYPTION_KEY": "data-at-rest key for an MCP-server bundle (no compose); losing it loses the data, so generation waits for the recovery-key decision (survey §5 Q5)", - "tax:CROW_TAX_ENCRYPTION_KEY": "data-at-rest key for an MCP-server bundle (no compose); losing it loses the data, so generation waits for the recovery-key decision (survey §5 Q5)", "crowdsec:CROWDSEC_API_KEY": "minted by `cscli bouncers add` after first boot and read by the MCP/panel from process.env, not by compose — a post-install hook, not pre-generation (survey §1.3)", "frigate:FRIGATE_RTSP_PASSWORD": "the operator's camera RTSP password, substituted into config.yml camera URLs ({FRIGATE_RTSP_PASSWORD}) — human input, not an internal secret", "lemmy:LEMMY_JWT": "an admin login JWT obtained from the running app after the admin registers (survey F2), not a value Crow can invent", @@ -140,13 +138,14 @@ test("the internal-secret patterns do not swallow human credentials", () => { } }); -test("every generated key of a docker bundle is consumed by its compose file", () => { +test("every generated key is consumed: by its compose file, or by its MCP server (server.envKeys)", () => { for (const { dir, m } of MANIFESTS) { const keys = S.generatedEnvKeys(m); if (keys.length === 0) continue; const text = composeText(dir); - assert.ok(text, `${dir}: generated keys only make sense for a compose bundle`); - const consumed = B.composeConsumedKeys(text); + // An MCP-server bundle (campaigns, tax: their data keys) has no compose file: its MCP + // entry is the only consumer, so every generated key must be in server.envKeys. + const consumed = text ? B.composeConsumedKeys(text) : { keys: new Set() }; // Consumed = interpolated by the compose file, or listed in server.envKeys (the // installer puts those into this instance's MCP entry). Explicit interpolation only: // an env_file .env would make `consumed.all` pass vacuously. diff --git a/tests/campaigns-tax-generated-keys.test.js b/tests/campaigns-tax-generated-keys.test.js new file mode 100644 index 000000000..571aa4f11 --- /dev/null +++ b/tests/campaigns-tax-generated-keys.test.js @@ -0,0 +1,198 @@ +/** + * Campaigns + tax get generated data keys (recovery keys, queue item 30, decided 2026-10-16): + * generated at install, saved to the keychain, reaching the MCP child through its + * mcp-addons entry and the dashboard panels in the same order. Tax uploads are stored + * encrypted; an install still on the example key published in the repo moves to a new key. + * Scratch CROW_HOME / CROW_DATA_DIR; no docker, no network. + */ +import { test, after } from "node:test"; +import assert from "node:assert/strict"; +import express from "express"; +import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, readdirSync, rmSync, existsSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, dirname } from "node:path"; +import { execFileSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; + +process.env.CROW_HOME = mkdtempSync(join(tmpdir(), "crow-ctkeys-home-")); +process.env.CROW_DATA_DIR = mkdtempSync(join(tmpdir(), "crow-ctkeys-data-")); +process.env.CROW_DISABLE_HEALTH_MONITOR = "1"; +process.env.CROW_DISABLE_INSTANCE_SYNC = "1"; +process.env.CROW_DISABLE_NOSTR = "1"; +delete process.env.CROW_TAX_ENCRYPTION_KEY; +delete process.env.CROW_CAMPAIGNS_ENCRYPTION_KEY; +execFileSync(process.execPath, ["scripts/init-db.js"], { env: process.env, stdio: "pipe" }); +const HOME = process.env.CROW_HOME; +after(() => { for (const d of [HOME, process.env.CROW_DATA_DIR]) rmSync(d, { recursive: true, force: true }); }); + +const REPO = fileURLToPath(new URL("..", import.meta.url)); +const manifestOf = (id) => JSON.parse(readFileSync(join(REPO, "bundles", id, "manifest.json"), "utf8")); +const specOf = (id, k) => manifestOf(id).env_vars.find((v) => v.name === k); +const S = await import("../servers/gateway/bundle-env-secrets.js"); +const { validateManifest } = await import("../scripts/lib/bundle-contract.mjs"); +const D = await import("../bundles/tax/server/documents-crypto.js"); +const EXAMPLE = "change-this-to-a-strong-passphrase"; + +test("both data keys are generated, saved to the keychain, and reach only their own MCP entry", () => { + for (const [id, k] of [["campaigns", "CROW_CAMPAIGNS_ENCRYPTION_KEY"], ["tax", "CROW_TAX_ENCRYPTION_KEY"]]) { + const v = specOf(id, k); + assert.equal(v.generate, "secret", `${id} generate`); + assert.equal(v.keychain, true, `${id} keychain`); + assert.equal(v.keychain_label, "data encryption key"); + assert.ok(manifestOf(id).server.envKeys.includes(k), `${id}: in server.envKeys, so the MCP entry gets it`); + assert.equal(S.gatewayGeneratedKeys(manifestOf(id)).includes(k), false, `${id}: never the shared gateway .env`); + assert.deepEqual(validateManifest(manifestOf(id), join(REPO, "bundles", id)).errors || [], [], `${id} manifest contract`); + } + assert.doesNotMatch(readFileSync(join(REPO, "bundles", "tax", ".env.example"), "utf8"), /^CROW_TAX_ENCRYPTION_KEY=/m); +}); + +test("the tax manifest says honestly what is and is not encrypted", () => { + const notes = manifestOf("tax").notes; + assert.match(notes, /returns and uploaded documents are encrypted at rest/); + assert.match(notes, /stored unencrypted/); + assert.doesNotMatch(notes, /PII is encrypted at rest/); +}); + +test("an install still on the example key published in the repo gets a new key; the old one moves to the previous slot", () => { + const m = manifestOf("tax"); + const destDir = mkdtempSync(join(tmpdir(), "crow-ctkeys-dest-")); + writeFileSync(join(destDir, ".env"), `CROW_TAX_ENCRYPTION_KEY=${EXAMPLE}\n`); + const plan = S.planGeneratedEnv("tax", m, { destDir, crowHome: HOME }); + assert.notEqual(plan.env.CROW_TAX_ENCRYPTION_KEY, EXAMPLE); + assert.ok(plan.minted.CROW_TAX_ENCRYPTION_KEY, "the new key is minted, so it goes to the keychain"); + assert.equal(plan.env.CROW_TAX_ENCRYPTION_KEY_PREVIOUS, EXAMPLE, "returns saved with it still open and move to the new key"); + assert.deepEqual(plan.replaced, ["CROW_TAX_ENCRYPTION_KEY"]); + // A previous slot already in use is never overwritten (it may hold the only copy of an older key). + writeFileSync(join(destDir, ".env"), `CROW_TAX_ENCRYPTION_KEY=${EXAMPLE}\nCROW_TAX_ENCRYPTION_KEY_PREVIOUS=older-key-1234567890\n`); + const kept = S.planGeneratedEnv("tax", m, { destDir, crowHome: HOME }); + assert.equal(kept.env.CROW_TAX_ENCRYPTION_KEY, EXAMPLE); + assert.equal(kept.env.CROW_TAX_ENCRYPTION_KEY_PREVIOUS, undefined); + assert.deepEqual(kept.replaceSkipped, ["CROW_TAX_ENCRYPTION_KEY"], "the install log says why it was left"); + // A typed key of the user's own is kept as it is. + writeFileSync(join(destDir, ".env"), "CROW_TAX_ENCRYPTION_KEY=my-own-key-0987654321\n"); + assert.equal(S.planGeneratedEnv("tax", m, { destDir, crowHome: HOME }).env.CROW_TAX_ENCRYPTION_KEY, "my-own-key-0987654321"); + rmSync(destDir, { recursive: true, force: true }); +}); + +test("replace_values_sha256 is validated: hex digests, and it needs generate + previous_key", () => { + const base = { id: "x", name: "X", version: "1.0.0", type: "mcp-server", category: "misc", description: "d", server: { command: "node", args: ["s.js"] } }; + const bad = validateManifest({ ...base, env_vars: [{ name: "K", secret: true, replace_values_sha256: ["nothex"] }] }, "/nonexistent").errors.join("\n"); + assert.match(bad, /replace_values_sha256 must be/); + const lone = validateManifest({ ...base, env_vars: [{ name: "K", secret: true, generate: "secret", replace_values_sha256: ["a".repeat(64)] }] }, "/nonexistent").errors.join("\n"); + assert.match(lone, /needs generate and previous_key/); +}); + +function writeAddons(obj) { + writeFileSync(join(HOME, "mcp-addons.json"), JSON.stringify(obj), { mode: 0o600 }); +} + +test("the campaigns panel reads its key in the MCP child's order: the instance's MCP entry first, then the gateway env", async () => { + const { campaignsKey } = await import("../bundles/campaigns/server/key.js"); + writeAddons({ campaigns: { env: { CROW_CAMPAIGNS_ENCRYPTION_KEY: "from-entry" } } }); + assert.equal(campaignsKey({ CROW_HOME: HOME }), "from-entry"); + assert.equal(campaignsKey({ CROW_HOME: HOME, CROW_CAMPAIGNS_ENCRYPTION_KEY: "from-gateway-env" }), "from-entry", "the entry wins, as in proxy.js"); + writeAddons({}); + assert.equal(campaignsKey({ CROW_HOME: HOME, CROW_CAMPAIGNS_ENCRYPTION_KEY: "from-gateway-env" }), "from-gateway-env"); +}); + +// --- the tax panel, mounted as the gateway mounts it --- +const TAX_KEY = "tax-key-" + "t".repeat(40); +const PDF = Buffer.from("%PDF-1.4\n% fake 1099 with SSN 000-00-0000\n%%EOF\n"); +const { createDbClient } = await import("../servers/db.js"); + +async function mountTax() { + const { default: taxRouter, taxKeys } = await import("../bundles/tax/panel/routes.js"); + const app = express(); + app.use((req, res, next) => { res.redirectAfterPost = (u) => res.redirect(303, u); next(); }); + app.use(taxRouter((req, res, next) => next())); + const server = await new Promise((r) => { const s = app.listen(0, "127.0.0.1", () => r(s)); }); + after(() => server.close()); + return { base: `http://127.0.0.1:${server.address().port}`, taxKeys }; +} + +test("tax panel: an upload is stored ONLY encrypted (key from the MCP entry, none in the gateway env); delete removes it", async () => { + writeAddons({ tax: { env: { CROW_TAX_ENCRYPTION_KEY: TAX_KEY } } }); + const docs = join(HOME, "tax-documents"); + // A document stored before this change: plain, in the documents folder. + mkdirSync(docs, { recursive: true }); + const old = join(docs, "100-old-w2.pdf"); + writeFileSync(old, PDF); + const db0 = createDbClient(); + await db0.execute(`CREATE TABLE IF NOT EXISTS tax_documents (id INTEGER PRIMARY KEY AUTOINCREMENT, filename TEXT NOT NULL, doc_type TEXT NOT NULL, owner TEXT DEFAULT 'taxpayer', file_path TEXT NOT NULL, status TEXT DEFAULT 'uploaded', return_id TEXT, extracted_data TEXT, confidence TEXT, warnings TEXT, uploaded_at TEXT DEFAULT (datetime('now')), ingested_at TEXT)`); + await db0.execute({ sql: "INSERT INTO tax_documents (filename, doc_type, file_path) VALUES ('old-w2.pdf', 'w2', ?)", args: [old] }); + db0.close(); + + const { base, taxKeys } = await mountTax(); + assert.equal(taxKeys().key, TAX_KEY); + + // The panel's start-up pass encrypts the old document (verified, then the plain file goes). + for (let i = 0; i < 100 && existsSync(old); i++) await new Promise((r) => setTimeout(r, 50)); + assert.equal(existsSync(old), false, "the plain document was encrypted at start-up"); + + const fd = new FormData(); + fd.append("doc_type", "1099-int"); + fd.append("document", new Blob([PDF], { type: "application/pdf" }), "my 1099.pdf"); + const res = await fetch(`${base}/api/tax/upload`, { method: "POST", body: fd }); + assert.equal(res.status, 201, await res.clone().text()); + const body = await res.json(); + const stored = readdirSync(docs).filter((f) => !f.startsWith(".")); + assert.equal(stored.length, 2); + for (const f of stored) { + assert.match(f, /\.enc$/); + const buf = readFileSync(join(docs, f)); + assert.equal(D.isEncryptedDocument(buf), true); + assert.equal(buf.includes(Buffer.from("000-00-0000")), false, "no plaintext on disk"); + assert.ok((await D.decryptDocument(buf, [TAX_KEY])).plain.equals(PDF)); + } + + const del = await fetch(`${base}/api/tax/documents/${body.id}/delete`, { method: "POST", headers: { accept: "application/json" } }); + assert.equal(del.status, 200); + assert.equal(readdirSync(docs).filter((f) => !f.startsWith(".")).length, 1); +}); + +test("tax panel: delete never removes a file outside the documents folder, whatever the row says", async () => { + const { base } = await mountTax(); + const outside = join(HOME, "not-a-tax-document.txt"); + writeFileSync(outside, "keep me"); + const db = createDbClient(); + await db.execute({ sql: "INSERT INTO tax_documents (filename, doc_type, file_path) VALUES ('x', 'w2', ?)", args: [outside] }); + const id = Number((await db.execute("SELECT max(id) AS id FROM tax_documents")).rows[0].id); + db.close(); + await fetch(`${base}/api/tax/documents/${id}/delete`, { method: "POST", headers: { accept: "application/json" } }); + assert.equal(readFileSync(outside, "utf8"), "keep me"); +}); + +test("tax panel: no key yet → an upload is refused, nothing is written", async () => { + writeAddons({}); + const { base } = await mountTax(); + const count = () => readdirSync(join(HOME, "tax-documents")).filter((f) => !f.startsWith(".")).length; + const before = count(); + const fd = new FormData(); + fd.append("doc_type", "w2"); + fd.append("document", new Blob([PDF], { type: "application/pdf" }), "w2.pdf"); + const res = await fetch(`${base}/api/tax/upload`, { method: "POST", body: fd }); + assert.equal(res.status, 409); + assert.equal(count(), before); +}); + +test("the MCP ingest tool opens a stored encrypted document with the tax key", async () => { + const { ingestDocument } = await import("../bundles/tax/engine/ingest/index.js"); + const p = join(HOME, "tax-documents", "x.pdf.0123abcd.enc"); + D.writePrivateAtomic(p, await D.encryptDocument(PDF, TAX_KEY)); + // The fake PDF has no fields, so extraction fails — but only AFTER it was decrypted. + await assert.rejects(ingestDocument(p, "w2", { keys: [TAX_KEY] }), /All extraction methods failed/); + await assert.rejects(ingestDocument(p, "w2", { keys: ["wrong-" + "w".repeat(40)] }), /different tax key/); + // The path comes from the AI: nothing but a bounded regular .pdf/.enc file is ever read. + await assert.rejects(ingestDocument("/dev/zero", "w2"), /Only PDF files/); + const { symlinkSync } = await import("node:fs"); + const dev = join(HOME, "zero.pdf"); + symlinkSync("/dev/zero", dev); + await assert.rejects(ingestDocument(dev, "w2"), /Not a regular file/); +}); + +test("a version bump ships tax's engine/ and pdf/ to the installed copy (the panel imports the installed engine)", () => { + const src = readFileSync(join(REPO, "servers", "gateway", "routes", "bundles.js"), "utf8"); + const nonDocker = src.match(/: \["server", "panel", "skills", "curriculum"[^\]]*\]/)[0]; + assert.match(nonDocker, /"engine"/); + assert.match(nonDocker, /"pdf"/); +}); diff --git a/tests/tax-documents-crypto.test.js b/tests/tax-documents-crypto.test.js new file mode 100644 index 000000000..59cd335fb --- /dev/null +++ b/tests/tax-documents-crypto.test.js @@ -0,0 +1,213 @@ +// tests/tax-documents-crypto.test.js — uploaded tax documents encrypted at rest, and the +// safe, reversible migration of files stored before (recovery keys, queue item 30). +import { test, after } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, existsSync, rmSync, readdirSync, statSync, symlinkSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, basename } from "node:path"; +import { createDbClient } from "../servers/db.js"; +import * as D from "../bundles/tax/server/documents-crypto.js"; + +const ROOT = mkdtempSync(join(tmpdir(), "crow-taxdocs-")); +after(() => rmSync(ROOT, { recursive: true, force: true })); +const KEY = "k-" + "a".repeat(41); +const OLD = "o-" + "b".repeat(41); +const PDF = Buffer.from("%PDF-1.4\n% fake W-2 with SSN 000-00-0000\n%%EOF\n"); +const EXAMPLE = "change-this-to-a-strong-passphrase"; + +async function fresh(name) { + const docs = join(ROOT, name, "tax-documents"); + mkdirSync(docs, { recursive: true }); + const db = createDbClient(join(ROOT, name, "t.db")); + after(() => { try { db.close(); } catch {} }); + await db.execute("CREATE TABLE tax_documents (id INTEGER PRIMARY KEY, filename TEXT, file_path TEXT NOT NULL)"); + return { docs, db }; +} +async function addPlain(db, docs, id, bytes = PDF) { + const p = join(docs, `${id}-w2.pdf`); + writeFileSync(p, bytes); + await db.execute({ sql: "INSERT INTO tax_documents (id, filename, file_path) VALUES (?, 'w2.pdf', ?)", args: [id, p] }); + return p; +} +const pathOf = async (db, id) => (await db.execute({ sql: "SELECT file_path FROM tax_documents WHERE id = ?", args: [id] })).rows[0].file_path; +const files = (dir) => readdirSync(dir).filter((f) => !f.startsWith(".")); + +test("round trip; the key id names the sealing key; a wrong key is refused; the previous key still opens", async () => { + const enc = await D.encryptDocument(PDF, KEY); + assert.equal(D.isEncryptedDocument(enc), true); + assert.equal(enc.includes(Buffer.from("000-00-0000")), false, "no plaintext in the stored bytes"); + assert.ok((await D.decryptDocument(enc, [KEY])).plain.equals(PDF)); + assert.equal(await D.sealedWith(enc, [OLD, KEY]), KEY); + // The key id comes from the salted PBKDF2 output: it differs per file for the same key, + // so it is no cheaper offline guess check than the ciphertext and links nothing. + const enc2 = await D.encryptDocument(PDF, KEY); + assert.equal(enc.subarray(D.MAGIC.length, D.MAGIC.length + 8).equals(enc2.subarray(D.MAGIC.length, D.MAGIC.length + 8)), false); + await assert.rejects(D.decryptDocument(enc, ["wrong-" + "c".repeat(40)]), /different tax key/); + assert.ok((await D.decryptDocument(await D.encryptDocument(PDF, OLD), [KEY, OLD])).plain.equals(PDF)); + await assert.rejects(D.encryptDocument(PDF, ""), /not set/); +}); + +test("the example key published in the repo is never used to store a document", async () => { + await assert.rejects(D.encryptDocument(PDF, EXAMPLE), /example value/); + const { docs, db } = await fresh("ex"); + const p = await addPlain(db, docs, 1); + const r = await D.encryptStoredDocuments(db, { key: EXAMPLE, docsDirs: [docs] }); + assert.equal(r.encrypted, 0); + assert.ok(readFileSync(p).equals(PDF), "the plain file is left alone"); +}); + +test("migration: plain → ..enc (0600), row updated, plain file gone; a re-run changes nothing", async () => { + const { docs, db } = await fresh("m1"); + const p = await addPlain(db, docs, 1); + const r = await D.encryptStoredDocuments(db, { key: KEY, docsDirs: [docs] }); + assert.deepEqual([r.encrypted, r.errors], [1, []]); + const np = await pathOf(db, 1); + assert.match(basename(np), /^1-w2\.pdf\.[0-9a-f]{8}\.enc$/); + assert.equal(existsSync(p), false); + assert.equal(statSync(np).mode & 0o777, 0o600); + assert.ok((await D.decryptDocument(readFileSync(np), [KEY])).plain.equals(PDF)); + const again = await D.encryptStoredDocuments(db, { key: KEY, docsDirs: [docs] }); + assert.deepEqual([again.encrypted, again.reencrypted, again.skipped], [0, 0, 1]); + assert.deepEqual(files(docs), [basename(np)], "no temp or lock file is left"); +}); + +test("a crash after the swap but before the unlink leaves a plain sibling: the next run removes it once the .enc is verified", async () => { + const { docs, db } = await fresh("m2"); + const p = await addPlain(db, docs, 1); + const enc = `${p}.0123abcd.enc`; + D.writePrivateAtomic(enc, await D.encryptDocument(PDF, KEY)); + await db.execute({ sql: "UPDATE tax_documents SET file_path = ? WHERE id = 1", args: [enc] }); + const r = await D.encryptStoredDocuments(db, { key: KEY, docsDirs: [docs] }); + assert.equal(r.cleaned, 1); + assert.equal(existsSync(p), false); + assert.deepEqual(files(docs), [basename(enc)]); +}); + +test("a sibling that does NOT match the .enc's contents is never removed", async () => { + const { docs, db } = await fresh("m2b"); + const p = await addPlain(db, docs, 1, Buffer.from("%PDF-1.4 a different file")); + const enc = `${p}.0123abcd.enc`; + D.writePrivateAtomic(enc, await D.encryptDocument(PDF, KEY)); + await db.execute({ sql: "UPDATE tax_documents SET file_path = ? WHERE id = 1", args: [enc] }); + const r = await D.encryptStoredDocuments(db, { key: KEY, docsDirs: [docs] }); + assert.equal(r.cleaned, 0); + assert.equal(existsSync(p), true); +}); + +test("a lost compare-and-swap (the row changed meanwhile) keeps the plain file and removes only this run's .enc", async () => { + const { docs, db } = await fresh("m3"); + const p = await addPlain(db, docs, 1); + const racing = { execute: async (q) => { + if (typeof q === "object" && /^UPDATE tax_documents SET file_path/.test(q.sql)) await db.execute({ sql: "UPDATE tax_documents SET file_path = ? WHERE id = 1", args: [p + ".moved"] }); + return db.execute(q); + } }; + const r = await D.encryptStoredDocuments(racing, { key: KEY, docsDirs: [docs] }); + assert.equal(r.encrypted, 0); + assert.equal(existsSync(p), true, "nothing unverified was deleted"); + assert.deepEqual(files(docs), [basename(p)]); +}); + +test("two migrators at once: the second finds the lock and touches nothing; the document is encrypted exactly once", async () => { + const { docs, db } = await fresh("m3b"); + await addPlain(db, docs, 1); + const [a, b] = await Promise.all([ + D.encryptStoredDocuments(db, { key: KEY, docsDirs: [docs] }), + D.encryptStoredDocuments(db, { key: KEY, docsDirs: [docs] }), + ]); + assert.equal(a.encrypted + b.encrypted, 1); + assert.equal(a.locked || b.locked, true); + const np = await pathOf(db, 1); + assert.ok((await D.decryptDocument(readFileSync(np), [KEY])).plain.equals(PDF)); + assert.deepEqual(files(docs), [basename(np)]); +}); + +test("no key: nothing is touched", async () => { + const { docs, db } = await fresh("m4"); + const p = await addPlain(db, docs, 1); + const r = await D.encryptStoredDocuments(db, { key: "", docsDirs: [docs] }); + assert.equal(r.encrypted, 0); + assert.ok(readFileSync(p).equals(PDF)); +}); + +test("a file_path outside the documents folder (or a symlink out of it) is never read, rewritten or deleted", async () => { + const { docs, db } = await fresh("m5"); + const outside = join(ROOT, "m5", "secret.txt"); + writeFileSync(outside, "not a tax document"); + await db.execute({ sql: "INSERT INTO tax_documents (id, filename, file_path) VALUES (1, 'x', ?)", args: [outside] }); + const link = join(docs, "link.pdf"); + symlinkSync(outside, link); + await db.execute({ sql: "INSERT INTO tax_documents (id, filename, file_path) VALUES (2, 'y', ?)", args: [link] }); + const r = await D.encryptStoredDocuments(db, { key: KEY, docsDirs: [docs] }); + assert.deepEqual([r.encrypted, r.skipped], [0, 2]); + assert.equal(readFileSync(outside, "utf8"), "not a tax document"); + assert.equal(D.storedDocumentPath(outside, [docs]), null); + assert.equal(D.storedDocumentPath(link, [docs]), null); +}); + +test("re-encrypts a document only the previous key opens (after a key change)", async () => { + const { docs, db } = await fresh("m6"); + const p = join(docs, "1-w2.pdf.0123abcd.enc"); + D.writePrivateAtomic(p, await D.encryptDocument(PDF, OLD)); + await db.execute({ sql: "INSERT INTO tax_documents (id, filename, file_path) VALUES (1, 'w2.pdf', ?)", args: [p] }); + const r = await D.encryptStoredDocuments(db, { key: KEY, previous: OLD, docsDirs: [docs] }); + assert.equal(r.reencrypted, 1); + assert.equal(await D.sealedWith(readFileSync(p), [OLD, KEY]), KEY); +}); + +test("a crow.db restored under another home: a row pointing at the old home resolves to the same name in this folder", async () => { + const { docs, db } = await fresh("m8"); + const enc = join(docs, "1-w2.pdf.0123abcd.enc"); + D.writePrivateAtomic(enc, await D.encryptDocument(PDF, KEY)); + await db.execute({ sql: "INSERT INTO tax_documents (id, filename, file_path) VALUES (1, 'w2.pdf', ?)", args: ["/home/someone-else/.crow/tax-documents/1-w2.pdf.0123abcd.enc"] }); + const out = join(ROOT, "m8-export"); + const ex = await D.decryptStoredDocuments(db, { keys: [KEY], outDir: out, docsDirs: [docs] }); + assert.equal(ex.decrypted, 1); + assert.ok(readFileSync(join(out, files(out)[0])).equals(PDF)); +}); + +test("reverse in place restores the identical bytes and the row; export copies leave the database alone", async () => { + const { docs, db } = await fresh("m7"); + const p = await addPlain(db, docs, 1); + await D.encryptStoredDocuments(db, { key: KEY, docsDirs: [docs] }); + const enc = await pathOf(db, 1); + const out = join(ROOT, "m7-export"); + const ex = await D.decryptStoredDocuments(db, { keys: [KEY], outDir: out, docsDirs: [docs] }); + assert.equal(ex.decrypted, 1); + assert.ok(readFileSync(join(out, files(out)[0])).equals(PDF)); + assert.equal(statSync(join(out, files(out)[0])).mode & 0o777, 0o600); + assert.equal(await pathOf(db, 1), enc, "export does not move the row"); + const back = await D.decryptStoredDocuments(db, { keys: [KEY], inPlace: true, docsDirs: [docs] }); + assert.equal(back.decrypted, 1); + assert.equal(await pathOf(db, 1), p); + assert.ok(readFileSync(p).equals(PDF)); + assert.equal(existsSync(enc), false); +}); + +test("a crow.db restored from a backup taken BEFORE the migration: the row (still naming the plain file) finds its .enc and is repointed", async () => { + const { docs, db } = await fresh("m9"); + const p = await addPlain(db, docs, 1); + await D.encryptStoredDocuments(db, { key: KEY, docsDirs: [docs] }); + const enc = await pathOf(db, 1); + await db.execute({ sql: "UPDATE tax_documents SET file_path = ? WHERE id = 1", args: [p] }); // the old backup's row + const r = await D.encryptStoredDocuments(db, { key: KEY, docsDirs: [docs] }); + assert.deepEqual(r.errors, []); + assert.equal(await pathOf(db, 1), enc, "the row points at the encrypted file again"); + assert.ok((await D.decryptDocument(readFileSync(enc), [KEY])).plain.equals(PDF)); +}); + +test("a stale lock (no run touched it for 30 minutes) is taken over; a fresh one is respected", async () => { + const { docs, db } = await fresh("m10"); + await addPlain(db, docs, 1); + const lock = join(docs, ".documents.lock"); + writeFileSync(lock, ""); + const held = await D.encryptStoredDocuments(db, { key: KEY, docsDirs: [docs] }); + assert.equal(held.locked, true, "a live lock is respected"); + const { utimesSync } = await import("node:fs"); + const old = new Date(Date.now() - 31 * 60 * 1000); + utimesSync(lock, old, old); + const r = await D.encryptStoredDocuments(db, { key: KEY, docsDirs: [docs] }); + assert.equal(r.locked, false); + assert.equal(r.encrypted, 1); + assert.equal(existsSync(lock), false, "released after the run"); + assert.equal(statSync(docs).mode & 0o777, 0o700, "the folder is private"); +});