From 69a66e5d071244f2cc3fd3168a8d796006123e3d Mon Sep 17 00:00:00 2001 From: NB <208086304+chitcommit@users.noreply.github.com> Date: Sat, 3 Oct 2026 07:00:16 +0000 Subject: [PATCH] feat(mercury): keep the seven API tokens alive and report which survived MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mercury deletes an API token after any 45-day period with no API call (docs.mercury.com/docs/api-token-security-policies). ChittyFinance had no outbound Mercury client at all — not one reference to api.mercury.com — so the seven per-entity tokens have been idle since they were provisioned and some are probably already gone. The keepalive is therefore also the liveness probe: its first run is the inventory. One read-only GET https://api.mercury.com/api/v1/accounts?limit=1 per entity, on the existing 09:00 UTC cron. No new scheduled trigger, so no new recurring spend. No write endpoint is ever called. Three states, not two. A 401 is a successful probe with a negative result, so alive (2xx), dead (401/403) and indeterminate (timeout, 429, 5xx, unexpected status, absent binding) are classified separately by a pure function — conflating "the token is dead" with "we could not reach Mercury" would make the probe useless. Mercury's errors.errorCode is carried alongside the status so a 403 from an IP-allowlist rejection stays distinguishable from a revoked token. The seven bindings are declared as secrets_store_secrets against the account-level store e914522471964c3c8cf1e601770edcc3, the same store and binding names CHITTYOS/chittysecrets/wrangler.json uses. Because the store is account-level this needs no ChittySecrets broker in the path. Bindings do not inherit into env blocks, so top level and all three envs carry them in both configs; deploy/system-wrangler.jsonc is the live deploy path and the root config is the Workers Builds one (#111), mirrored to stop them drifting further. Credential handling: bindings are referenced by name, .get() happens at the call site, and the value exists only inside the Authorization header. Nothing logged, returned or persisted holds a token. The 2xx body is cancelled unread because /accounts carries account and routing numbers; only non-2xx bodies are parsed, and only for errorCode — never Mercury's prose message. Recorded in FINANCE_KV rather than a new Neon table: the only vaguely related existing table, integrations, is tenant-FK'd and these bindings have no tenant mapping; a new table means a destructive drizzle-kit push for seven rows a day; and KV is already this repo's operational-state store (sessions, inbound-email index, Wave webhook secrets). Token liveness is rotation state, which the operator KV policy allows. Per-token and latest-run keys carry no TTL. GET /api/v1/mercury-tokens reads the last run; POST .../probe runs it now, so liveness is readable without waiting for 09:00 UTC. Both are behind serviceAuth: the /api/v1/* neighbours are all public, but an open endpoint enumerating which banking tokens are alive is an information leak. The two cron jobs are isolated with allSettled — processLeaseExpirations throws outright on an unbound DATABASE_URL and must not be able to skip the keepalive. Co-Authored-By: Claude Opus 5 --- deploy/system-wrangler.jsonc | 46 +++ .../__tests__/mercury-token-keepalive.test.ts | 289 ++++++++++++++ .../__tests__/routes-mercury-tokens.test.ts | 101 +++++ server/app.ts | 13 + server/env.ts | 17 + server/lib/mercury-token-keepalive.ts | 354 ++++++++++++++++++ server/routes/mercury-tokens.ts | 56 +++ server/worker.ts | 29 +- wrangler.jsonc | 46 +++ 9 files changed, 947 insertions(+), 4 deletions(-) create mode 100644 server/__tests__/mercury-token-keepalive.test.ts create mode 100644 server/__tests__/routes-mercury-tokens.test.ts create mode 100644 server/lib/mercury-token-keepalive.ts create mode 100644 server/routes/mercury-tokens.ts diff --git a/deploy/system-wrangler.jsonc b/deploy/system-wrangler.jsonc index 9ccc1c1..206a31a 100644 --- a/deploy/system-wrangler.jsonc +++ b/deploy/system-wrangler.jsonc @@ -38,6 +38,22 @@ { "name": "EMAIL", "allowed_sender_addresses": ["finance@chitty.cc", "noreply@chitty.cc"] } ], + // Mercury API tokens — one per entity — from the ACCOUNT-LEVEL Cloudflare Secrets + // Store. Same store and same binding names as CHITTYOS/chittysecrets/wrangler.json; + // because the store is account-level, chittyfinance binds them directly and does not + // need the ChittySecrets broker in the path. Consumed by the daily keepalive probe in + // server/lib/mercury-token-keepalive.ts — Mercury deletes a token after 45 days with + // no API call. Bindings do NOT inherit into env blocks, so each env repeats them. + "secrets_store_secrets": [ + { "binding": "MERCURY_TOKEN_ARIBIA_LLC", "store_id": "e914522471964c3c8cf1e601770edcc3", "secret_name": "MERCURY_TOKEN_ARIBIA_LLC" }, + { "binding": "MERCURY_TOKEN_ARIBIA_LLC_CITY_STUDIO", "store_id": "e914522471964c3c8cf1e601770edcc3", "secret_name": "MERCURY_TOKEN_ARIBIA_LLC_CITY_STUDIO" }, + { "binding": "MERCURY_TOKEN_ARIBIA_LLC_APT_ARLENE", "store_id": "e914522471964c3c8cf1e601770edcc3", "secret_name": "MERCURY_TOKEN_ARIBIA_LLC_APT_ARLENE" }, + { "binding": "MERCURY_TOKEN_CHICAGO_FURNISHED_CONDOS", "store_id": "e914522471964c3c8cf1e601770edcc3", "secret_name": "MERCURY_TOKEN_CHICAGO_FURNISHED_CONDOS" }, + { "binding": "MERCURY_TOKEN_IT_CAN_BE_LLC", "store_id": "e914522471964c3c8cf1e601770edcc3", "secret_name": "MERCURY_TOKEN_IT_CAN_BE_LLC" }, + { "binding": "MERCURY_TOKEN_CHITTY_SERVICES", "store_id": "e914522471964c3c8cf1e601770edcc3", "secret_name": "MERCURY_TOKEN_CHITTY_SERVICES" }, + { "binding": "MERCURY_TOKEN_JEAN_ARLENE_VENTURING", "store_id": "e914522471964c3c8cf1e601770edcc3", "secret_name": "MERCURY_TOKEN_JEAN_ARLENE_VENTURING" } + ], + "vars": { "MODE": "system", "NODE_ENV": "production", @@ -112,6 +128,16 @@ "send_email": [ { "name": "EMAIL" } ], + "secrets_store_secrets": [ + { "binding": "MERCURY_TOKEN_ARIBIA_LLC", "store_id": "e914522471964c3c8cf1e601770edcc3", "secret_name": "MERCURY_TOKEN_ARIBIA_LLC" }, + { "binding": "MERCURY_TOKEN_ARIBIA_LLC_CITY_STUDIO", "store_id": "e914522471964c3c8cf1e601770edcc3", "secret_name": "MERCURY_TOKEN_ARIBIA_LLC_CITY_STUDIO" }, + { "binding": "MERCURY_TOKEN_ARIBIA_LLC_APT_ARLENE", "store_id": "e914522471964c3c8cf1e601770edcc3", "secret_name": "MERCURY_TOKEN_ARIBIA_LLC_APT_ARLENE" }, + { "binding": "MERCURY_TOKEN_CHICAGO_FURNISHED_CONDOS", "store_id": "e914522471964c3c8cf1e601770edcc3", "secret_name": "MERCURY_TOKEN_CHICAGO_FURNISHED_CONDOS" }, + { "binding": "MERCURY_TOKEN_IT_CAN_BE_LLC", "store_id": "e914522471964c3c8cf1e601770edcc3", "secret_name": "MERCURY_TOKEN_IT_CAN_BE_LLC" }, + { "binding": "MERCURY_TOKEN_CHITTY_SERVICES", "store_id": "e914522471964c3c8cf1e601770edcc3", "secret_name": "MERCURY_TOKEN_CHITTY_SERVICES" }, + { "binding": "MERCURY_TOKEN_JEAN_ARLENE_VENTURING", "store_id": "e914522471964c3c8cf1e601770edcc3", "secret_name": "MERCURY_TOKEN_JEAN_ARLENE_VENTURING" } + ], + "vars": { "MODE": "system", "NODE_ENV": "development", @@ -137,6 +163,16 @@ "staging": { "name": "chittyfinance-staging", "workers_dev": true, + "secrets_store_secrets": [ + { "binding": "MERCURY_TOKEN_ARIBIA_LLC", "store_id": "e914522471964c3c8cf1e601770edcc3", "secret_name": "MERCURY_TOKEN_ARIBIA_LLC" }, + { "binding": "MERCURY_TOKEN_ARIBIA_LLC_CITY_STUDIO", "store_id": "e914522471964c3c8cf1e601770edcc3", "secret_name": "MERCURY_TOKEN_ARIBIA_LLC_CITY_STUDIO" }, + { "binding": "MERCURY_TOKEN_ARIBIA_LLC_APT_ARLENE", "store_id": "e914522471964c3c8cf1e601770edcc3", "secret_name": "MERCURY_TOKEN_ARIBIA_LLC_APT_ARLENE" }, + { "binding": "MERCURY_TOKEN_CHICAGO_FURNISHED_CONDOS", "store_id": "e914522471964c3c8cf1e601770edcc3", "secret_name": "MERCURY_TOKEN_CHICAGO_FURNISHED_CONDOS" }, + { "binding": "MERCURY_TOKEN_IT_CAN_BE_LLC", "store_id": "e914522471964c3c8cf1e601770edcc3", "secret_name": "MERCURY_TOKEN_IT_CAN_BE_LLC" }, + { "binding": "MERCURY_TOKEN_CHITTY_SERVICES", "store_id": "e914522471964c3c8cf1e601770edcc3", "secret_name": "MERCURY_TOKEN_CHITTY_SERVICES" }, + { "binding": "MERCURY_TOKEN_JEAN_ARLENE_VENTURING", "store_id": "e914522471964c3c8cf1e601770edcc3", "secret_name": "MERCURY_TOKEN_JEAN_ARLENE_VENTURING" } + ], + "vars": { "MODE": "system", "NODE_ENV": "staging", @@ -168,6 +204,16 @@ "send_email": [ { "name": "EMAIL", "allowed_sender_addresses": ["finance@chitty.cc", "noreply@chitty.cc"] } ], + "secrets_store_secrets": [ + { "binding": "MERCURY_TOKEN_ARIBIA_LLC", "store_id": "e914522471964c3c8cf1e601770edcc3", "secret_name": "MERCURY_TOKEN_ARIBIA_LLC" }, + { "binding": "MERCURY_TOKEN_ARIBIA_LLC_CITY_STUDIO", "store_id": "e914522471964c3c8cf1e601770edcc3", "secret_name": "MERCURY_TOKEN_ARIBIA_LLC_CITY_STUDIO" }, + { "binding": "MERCURY_TOKEN_ARIBIA_LLC_APT_ARLENE", "store_id": "e914522471964c3c8cf1e601770edcc3", "secret_name": "MERCURY_TOKEN_ARIBIA_LLC_APT_ARLENE" }, + { "binding": "MERCURY_TOKEN_CHICAGO_FURNISHED_CONDOS", "store_id": "e914522471964c3c8cf1e601770edcc3", "secret_name": "MERCURY_TOKEN_CHICAGO_FURNISHED_CONDOS" }, + { "binding": "MERCURY_TOKEN_IT_CAN_BE_LLC", "store_id": "e914522471964c3c8cf1e601770edcc3", "secret_name": "MERCURY_TOKEN_IT_CAN_BE_LLC" }, + { "binding": "MERCURY_TOKEN_CHITTY_SERVICES", "store_id": "e914522471964c3c8cf1e601770edcc3", "secret_name": "MERCURY_TOKEN_CHITTY_SERVICES" }, + { "binding": "MERCURY_TOKEN_JEAN_ARLENE_VENTURING", "store_id": "e914522471964c3c8cf1e601770edcc3", "secret_name": "MERCURY_TOKEN_JEAN_ARLENE_VENTURING" } + ], + "vars": { "MODE": "system", "NODE_ENV": "production", diff --git a/server/__tests__/mercury-token-keepalive.test.ts b/server/__tests__/mercury-token-keepalive.test.ts new file mode 100644 index 0000000..64a9ddb --- /dev/null +++ b/server/__tests__/mercury-token-keepalive.test.ts @@ -0,0 +1,289 @@ +import { describe, it, expect, vi } from 'vitest'; +import { + MERCURY_TOKEN_BINDINGS, + MERCURY_PROBE_URL, + classifyProbe, + extractErrorCode, + probeToken, + runMercuryTokenKeepalive, + persistKeepaliveReport, + readLatestKeepaliveReport, + KV_RUN_LATEST, + kvTokenKey, + type MercuryKeepaliveEnv, +} from '../lib/mercury-token-keepalive'; + +/** + * No DB module is mocked here. The classifier is a pure function and is tested + * against real `Response` objects; the fetch seam is an injected `fetchImpl` + * parameter, not a module mock; KV is a real in-memory Map behind the KVNamespace + * surface the module actually uses. + */ +function makeKv() { + const store = new Map(); + return { + store, + get: async (k: string) => store.get(k) ?? null, + put: async (k: string, v: string) => { store.set(k, v); }, + delete: async (k: string) => { store.delete(k); }, + } as unknown as KVNamespace & { store: Map }; +} + +const secret = (value: string) => ({ get: async () => value }); + +describe('classifyProbe — the three states are distinguished', () => { + // The whole point of the probe: a 401 is a successful probe with a negative + // result, and must never be conflated with "we could not reach Mercury". + it('2xx is alive', () => { + for (const status of [200, 204, 299]) { + expect(classifyProbe({ status }).liveness).toBe('alive'); + } + }); + + it('401 is dead, not alive and not indeterminate', () => { + const c = classifyProbe({ status: 401, errorCode: 'noTokenInDB' }); + expect(c.liveness).toBe('dead'); + expect(c.liveness).not.toBe('alive'); + expect(c.liveness).not.toBe('indeterminate'); + expect(c.reason).toContain('401'); + expect(c.reason).toContain('noTokenInDB'); + }); + + it('403 is dead', () => { + expect(classifyProbe({ status: 403 }).liveness).toBe('dead'); + }); + + it('timeout / network failure is indeterminate, not dead', () => { + const c = classifyProbe({ transportError: 'TimeoutError' }); + expect(c.liveness).toBe('indeterminate'); + expect(c.liveness).not.toBe('dead'); + expect(c.reason).toContain('transport_error'); + }); + + it('5xx is indeterminate, not dead', () => { + for (const status of [500, 502, 503]) { + expect(classifyProbe({ status }).liveness).toBe('indeterminate'); + } + }); + + it('429 is indeterminate', () => { + expect(classifyProbe({ status: 429 }).liveness).toBe('indeterminate'); + }); + + it('an absent Secrets Store binding is indeterminate, never dead', () => { + const c = classifyProbe({ bindingMissing: true }); + expect(c.liveness).toBe('indeterminate'); + expect(c.reason).toContain('binding_missing'); + }); + + it('404 and other unexpected statuses are indeterminate', () => { + expect(classifyProbe({ status: 404 }).liveness).toBe('indeterminate'); + expect(classifyProbe({ status: 418 }).liveness).toBe('indeterminate'); + }); + + // Drive the classifier off real Response statuses rather than hand-written numbers, + // so a change to how status is read is caught too. + it('classifies real Response objects', async () => { + expect(classifyProbe({ status: new Response('', { status: 200 }).status }).liveness).toBe('alive'); + expect(classifyProbe({ status: new Response('', { status: 401 }).status }).liveness).toBe('dead'); + expect(classifyProbe({ status: new Response('', { status: 503 }).status }).liveness).toBe('indeterminate'); + }); +}); + +describe('extractErrorCode', () => { + it('pulls errors.errorCode from a Mercury error body', () => { + expect(extractErrorCode({ errors: { errorCode: 'noTokenInDB', message: 'No matching token found' } })) + .toBe('noTokenInDB'); + }); + + it('handles an array of errors', () => { + expect(extractErrorCode({ errors: [{ errorCode: 'noAuthTokenHeader' }] })).toBe('noAuthTokenHeader'); + }); + + it('returns null for shapes it does not recognise', () => { + expect(extractErrorCode(null)).toBeNull(); + expect(extractErrorCode('nope')).toBeNull(); + expect(extractErrorCode({})).toBeNull(); + expect(extractErrorCode({ errors: {} })).toBeNull(); + }); + + it('sanitizes and caps the code, and never returns the message', () => { + const out = extractErrorCode({ errors: { errorCode: 'bad code\n