From 75ad14d7ad15a67db4273e663f4eba0ffe842a2d Mon Sep 17 00:00:00 2001 From: thegreatfeez Date: Sat, 17 Oct 2026 10:12:46 +0100 Subject: [PATCH] feat: add seeded snapshots, backups, secret rotation and CD - deterministic seed dataset with local IPFS, raw-event export and anonymised snapshots - checksummed backups, restore script and post-restore chain reconciliation - env/file/Vault secret loading with hot rotation and JWT grace - digest-pinned CD with migration lint and post-deploy checks closes #147 closes #146 closes #145 closes #144 --- .env.example | 29 ++ .github/workflows/cd.yml | 135 ++++++++ .github/workflows/ci.yml | 27 ++ .github/workflows/deploy-env.yml | 80 +++++ .github/workflows/restore-drill.yml | 62 ++++ .gitignore | 1 + README.md | 31 +- docker-compose.yml | 22 ++ docs/api.md | 4 +- docs/architecture.md | 42 +++ docs/deployment.md | 51 +++ docs/runbooks/backup-restore.md | 65 ++++ package-lock.json | 506 +++++++++++++++++++++++++++- package.json | 9 +- prisma.config.ts | 22 +- prisma/seed.ts | 278 +++++++++++++++ scripts/backup.sh | 118 +++++++ scripts/check-migration-compat.mjs | 78 +++++ scripts/post-deploy-checks.sh | 80 +++++ scripts/post-restore-reconcile.ts | 62 ++++ scripts/restore.sh | 38 +++ scripts/snapshot-export.ts | 74 ++++ src/app.ts | 12 + src/config/index.ts | 73 +++- src/config/secrets.ts | 411 ++++++++++++++++++++++ src/lib/gemini.ts | 22 +- src/lib/llm.ts | 15 +- src/lib/snapshot.ts | 192 +++++++++++ src/middlewares/auth.middleware.ts | 18 +- src/seed/raw-events.ts | 96 ++++++ src/seed/rng.ts | 35 ++ src/seed/scenario.ts | 489 +++++++++++++++++++++++++++ src/server.ts | 12 + src/services/auth.service.ts | 3 +- src/services/indexer.service.ts | 40 ++- src/services/ingestion.service.ts | 24 +- src/services/ipfs.service.ts | 64 +++- src/services/reconcile.service.ts | 220 ++++++++++++ tests/fixtures/restore-seed.sql | 16 + tests/helpers/env.ts | 14 +- tests/jwt-rotation.test.ts | 73 ++++ tests/migration-compat.test.ts | 44 +++ tests/post-deploy-checks.test.ts | 58 ++++ tests/reconcile.test.ts | 129 +++++++ tests/secrets.test.ts | 136 ++++++++ tests/seed.test.ts | 117 +++++++ 46 files changed, 4077 insertions(+), 50 deletions(-) create mode 100644 .github/workflows/cd.yml create mode 100644 .github/workflows/deploy-env.yml create mode 100644 .github/workflows/restore-drill.yml create mode 100644 docs/deployment.md create mode 100644 docs/runbooks/backup-restore.md create mode 100644 prisma/seed.ts create mode 100755 scripts/backup.sh create mode 100755 scripts/check-migration-compat.mjs create mode 100755 scripts/post-deploy-checks.sh create mode 100644 scripts/post-restore-reconcile.ts create mode 100755 scripts/restore.sh create mode 100644 scripts/snapshot-export.ts create mode 100644 src/config/secrets.ts create mode 100644 src/lib/snapshot.ts create mode 100644 src/seed/raw-events.ts create mode 100644 src/seed/rng.ts create mode 100644 src/seed/scenario.ts create mode 100644 src/services/reconcile.service.ts create mode 100644 tests/fixtures/restore-seed.sql create mode 100644 tests/jwt-rotation.test.ts create mode 100644 tests/migration-compat.test.ts create mode 100644 tests/post-deploy-checks.test.ts create mode 100644 tests/reconcile.test.ts create mode 100644 tests/secrets.test.ts create mode 100644 tests/seed.test.ts diff --git a/.env.example b/.env.example index bed94e6..8549e2d 100644 --- a/.env.example +++ b/.env.example @@ -9,6 +9,20 @@ PORT=8000 # Postgres and Redis. These defaults match docker-compose.yml (local only, # not secrets). Point them at your own servers for anything shared. DATABASE_URL=postgresql://oracledesk:oracledesk@127.0.0.1:5433/oracledesk +# Alternatively, mount the URL from a file: DATABASE_URL_FILE=/run/secrets/database_url +# Every secret below also accepts _FILE (env/file provider) — see #145. + +# ── Secret loading (#145) ────────────────────────────────────────────────── +# env: process environment (default); file: _FILE mounts; vault: Vault KV v2. +SECRETS_PROVIDER=env +# Refresh interval for file/vault providers (minimum 30000 ms). +SECRETS_REFRESH_MS=300000 +# How long a rotated JWT_SECRET remains valid for already-issued tokens. +JWT_PREVIOUS_SECRET_GRACE_SECS=86400 +# Required only when SECRETS_PROVIDER=vault. +VAULT_ADDR= +VAULT_TOKEN_FILE= +VAULT_KV_PATH= REDIS_URL=redis://127.0.0.1:6379 # Comma-separated browser origins allowed to call the API. CORS_ORIGINS=http://localhost:3000 @@ -72,7 +86,22 @@ GEMINI_API_KEY= PINATA_API_KEY= PINATA_SECRET_API_KEY= IPFS_GATEWAY_URL=https://gateway.pinata.cloud/ipfs +# pinata (default) or local (development only; stores bytes in .ipfs-local/). +IPFS_TRANSPORT=pinata +IPFS_LOCAL_DIR=.ipfs-local # ── Data sources ─────────────────────────────────────────────────────────── NEWSAPI_KEY= FRED_API_KEY= + +# ── Seed / snapshot (#147) ───────────────────────────────────────────────── +# The seed refuses production and non-local databases. Set to 1 only when you +# intentionally seed a remote development database. +SEED_ALLOW_REMOTE=0 +# Base id for seeded on-chain market/trace rows, and the starting ledger. +SEED_ONCHAIN_BASE=1000000 +SEED_BASE_LEDGER=6000000 +# Optional comma-separated real testnet ids to link (read-only). +SEED_TESTNET_MARKET_IDS= +# Required by scripts/snapshot-export.ts; never commit the value. +SNAPSHOT_SALT= diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml new file mode 100644 index 0000000..db08159 --- /dev/null +++ b/.github/workflows/cd.yml @@ -0,0 +1,135 @@ +name: CD + +on: + push: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + packages: write + id-token: write + pull-requests: read + +concurrency: + group: cd-main + cancel-in-progress: false + +jobs: + verify: + name: Verify release commit + uses: ./.github/workflows/ci.yml + + image: + name: Build, sign and attest image + needs: verify + runs-on: ubuntu-latest + outputs: + digest: ${{ steps.build.outputs.digest }} + tag: ${{ steps.meta.outputs.tag }} + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - uses: docker/setup-buildx-action@v3 + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - id: meta + run: echo "tag=ghcr.io/${GITHUB_REPOSITORY,,}:${GITHUB_SHA::12}" >> "$GITHUB_OUTPUT" + - id: build + uses: docker/build-push-action@v6 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tag }} + provenance: true + sbom: true + - name: Install cosign + uses: sigstore/cosign-installer@v3 + - name: Sign image + run: cosign sign --yes "${{ steps.meta.outputs.tag }}@${{ steps.build.outputs.digest }}" + - name: SBOM (SPDX) + uses: anchore/sbom-action@v0 + with: + image: ${{ steps.meta.outputs.tag }}@${{ steps.build.outputs.digest }} + format: spdx-json + output-file: sbom.spdx.json + - uses: actions/upload-artifact@v4 + with: + name: sbom + path: sbom.spdx.json + + staging: + name: Deploy staging + needs: image + uses: ./.github/workflows/deploy-env.yml + with: + environment: staging + namespace: oracledesk-staging + image: ghcr.io/${{ github.repository }}@${{ needs.image.outputs.digest }} + digest: ${{ needs.image.outputs.digest }} + values_file: values-staging.yaml + secrets: inherit + + staging-checks: + name: Staging post-deploy checks + needs: [image, staging] + runs-on: ubuntu-latest + environment: staging + steps: + - uses: actions/checkout@v4 + - uses: azure/setup-kubectl@v4 + - name: Checks + env: + API_URL: ${{ vars.API_URL }} + READY_CHECKS: "3" + INDEXER_STATUS_URL: ${{ vars.INDEXER_STATUS_URL }} + MAX_INDEXER_LAG_LEDGERS: "120" + PROMETHEUS_URL: ${{ vars.PROMETHEUS_URL }} + KUBE_NAMESPACE: oracledesk-staging + DEPLOYMENTS: api indexer worker + DIGEST: ${{ needs.image.outputs.digest }} + KUBECONFIG_DATA: ${{ secrets.KUBECONFIG }} + run: | + echo "$KUBECONFIG_DATA" > "$RUNNER_TEMP/kubeconfig" + export KUBECONFIG="$RUNNER_TEMP/kubeconfig" + ./scripts/post-deploy-checks.sh + + production: + name: Deploy production + needs: [image, staging-checks] + uses: ./.github/workflows/deploy-env.yml + with: + environment: production + namespace: oracledesk + image: ghcr.io/${{ github.repository }}@${{ needs.image.outputs.digest }} + digest: ${{ needs.image.outputs.digest }} + values_file: values-production.yaml + secrets: inherit + + production-checks: + name: Production post-deploy checks + needs: [image, production] + runs-on: ubuntu-latest + environment: production + steps: + - uses: actions/checkout@v4 + - uses: azure/setup-kubectl@v4 + - name: Checks + env: + API_URL: ${{ vars.API_URL }} + READY_CHECKS: "5" + INDEXER_STATUS_URL: ${{ vars.INDEXER_STATUS_URL }} + MAX_INDEXER_LAG_LEDGERS: "120" + PROMETHEUS_URL: ${{ vars.PROMETHEUS_URL }} + KUBE_NAMESPACE: oracledesk + DEPLOYMENTS: api indexer worker + DIGEST: ${{ needs.image.outputs.digest }} + KUBECONFIG_DATA: ${{ secrets.KUBECONFIG }} + run: | + echo "$KUBECONFIG_DATA" > "$RUNNER_TEMP/kubeconfig" + export KUBECONFIG="$RUNNER_TEMP/kubeconfig" + ./scripts/post-deploy-checks.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 59ab0c1..5fc1384 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,6 +4,7 @@ on: push: branches: [main] pull_request: + workflow_call: jobs: backend: @@ -55,3 +56,29 @@ jobs: - run: npx prisma migrate deploy - run: npm run build - run: npm test + + - name: Migration compatibility (new migrations only) + if: github.event_name == 'pull_request' + run: | + git fetch origin "${{ github.base_ref }}" --depth=1 + changed="$(git diff --name-only "origin/${{ github.base_ref }}...HEAD" -- prisma/migrations || true)" + if [[ -n "$changed" ]]; then + printf '%s\n' "$changed" | xargs node scripts/check-migration-compat.mjs + else + echo "No migration files changed" + fi + + lint-workflows: + name: Workflow and script lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: actionlint + uses: docker://rhysd/actionlint:latest + with: + args: -color + - name: shellcheck + uses: ludeeus/action-shellcheck@master + with: + scandir: ./scripts + additional_files: ./scripts/backup.sh ./scripts/restore.sh diff --git a/.github/workflows/deploy-env.yml b/.github/workflows/deploy-env.yml new file mode 100644 index 0000000..fee9acf --- /dev/null +++ b/.github/workflows/deploy-env.yml @@ -0,0 +1,80 @@ +name: Deploy environment + +on: + workflow_call: + inputs: + environment: + required: true + type: string + namespace: + required: true + type: string + image: + required: true + type: string + digest: + required: true + type: string + values_file: + required: true + type: string + secrets: + KUBECONFIG: + required: true + DATABASE_URL: + required: true + HELM_VALUES: + required: false + +permissions: + contents: read + id-token: write + +jobs: + deploy: + runs-on: ubuntu-latest + environment: ${{ inputs.environment }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/setup-node@v4 + with: + node-version-file: .nvmrc + cache: npm + - run: npm ci + - uses: azure/setup-helm@v4 + - name: kubeconfig + run: | + echo "${{ secrets.KUBECONFIG }}" > "$RUNNER_TEMP/kubeconfig" + echo "KUBECONFIG=$RUNNER_TEMP/kubeconfig" >> "$GITHUB_ENV" + - name: Verify image signature + uses: sigstore/cosign-installer@v3 + - run: cosign verify "${{ inputs.image }}" --certificate-identity-regexp "https://github.com/${GITHUB_REPOSITORY}/.github/workflows/cd.yml@refs/heads/main" --certificate-oidc-issuer https://token.actions.githubusercontent.com + - name: Migration lint (changed migrations only) + run: | + changed="$(git diff --name-only HEAD^ HEAD -- prisma/migrations || true)" + if [[ -n "$changed" ]]; then + printf '%s\n' "$changed" | xargs node scripts/check-migration-compat.mjs + else + echo "No migration files changed in this release" + fi + - name: Apply migrations + env: + DATABASE_URL: ${{ secrets.DATABASE_URL }} + run: npx prisma migrate deploy + - name: Helm upgrade + env: + HELM_VALUES: ${{ secrets.HELM_VALUES }} + NAMESPACE: ${{ inputs.namespace }} + IMAGE: ${{ inputs.image }} + VALUES_FILE: ${{ inputs.values_file }} + run: | + test -d deploy/helm/oracledesk-backend || { echo "Helm chart missing (deploy/helm/oracledesk-backend)"; exit 1; } + extra=() + if [[ -n "$HELM_VALUES" ]]; then printf '%s' "$HELM_VALUES" > "$RUNNER_TEMP/values-secret.yaml"; extra=(-f "$RUNNER_TEMP/values-secret.yaml"); fi + helm upgrade --install oracledesk deploy/helm/oracledesk-backend \ + --namespace "$NAMESPACE" --create-namespace \ + -f "deploy/helm/oracledesk-backend/$VALUES_FILE" "${extra[@]}" \ + --set image.repository="${IMAGE%@*}" --set image.digest="${IMAGE#*@}" \ + --atomic --timeout 10m diff --git a/.github/workflows/restore-drill.yml b/.github/workflows/restore-drill.yml new file mode 100644 index 0000000..a140046 --- /dev/null +++ b/.github/workflows/restore-drill.yml @@ -0,0 +1,62 @@ +name: Restore drill + +on: + schedule: + - cron: "0 3 1 * *" + workflow_dispatch: + +permissions: + contents: read + +jobs: + restore: + runs-on: ubuntu-latest + services: + postgres: + image: postgres:16-alpine + env: + POSTGRES_USER: oracledesk + POSTGRES_PASSWORD: oracledesk + POSTGRES_DB: oracledesk + ports: ["5432:5432"] + options: >- + --health-cmd "pg_isready -U oracledesk" + --health-interval 5s --health-timeout 5s --health-retries 10 + env: + DATABASE_URL: postgresql://oracledesk:oracledesk@localhost:5432/oracledesk + NODE_ENV: test + CHAIN_EXECUTION_MODE: dry-run + IPFS_TRANSPORT: local + SEED_ALLOW_REMOTE: "0" + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version-file: .nvmrc + cache: npm + - run: npm ci + - run: npx prisma generate + - run: npx prisma migrate deploy + - name: Seed fixture data + run: psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -f tests/fixtures/restore-seed.sql + - name: Backup + run: BACKUP_DIR="$RUNNER_TEMP/backups" ./scripts/backup.sh + - name: Wipe and restore + run: | + psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -c 'DROP SCHEMA public CASCADE; CREATE SCHEMA public;' + DUMP="$(find "$RUNNER_TEMP/backups" -name "oracledesk-*.dump" -type f | head -1)" + ./scripts/restore.sh "$DUMP" --force + - name: Reconcile against recorded RPC events + env: + RECONCILE_FAKE_RPC: tests/fixtures/events.json + run: | + npx tsx scripts/post-restore-reconcile.ts \ + --manifest "$RUNNER_TEMP"/backups/oracledesk-*-manifest.json \ + --rewind-ledgers 720 --scan-transfers \ + --out "$RUNNER_TEMP/reconcile-report.json" + - uses: actions/upload-artifact@v4 + if: always() + with: + name: reconcile-report + path: ${{ runner.temp }}/reconcile-report.json + if-no-files-found: ignore diff --git a/.gitignore b/.gitignore index 423e09b..d50f4d4 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ dist-tests recovery.dat # Build output and local artefacts +.ipfs-local/ dist/ *.log .DS_Store diff --git a/README.md b/README.md index e623a44..54671ce 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,23 @@ Key switches: - `CHAIN_EXECUTION_MODE=dry-run` (default): every contract write is built and simulated, logged, and never signed. `live` signs with `AGENT_SECRET_KEY` and is refused on any network but testnet. - `PAYMENTS_RECIPIENT`: where daily-pass payments must go. Defaults to the treasury contract; see [docs/STATUS.md](docs/STATUS.md) before relying on that. - `ADMIN_ADDRESSES`: G-addresses allowed to trigger market generation. +- `SECRETS_PROVIDER`: `env` (default), `file` (`_FILE` mounts) or + `vault`. See [docs/architecture.md](docs/architecture.md#secrets-145). +- `IPFS_TRANSPORT=local` stores pinned bytes in `.ipfs-local/` for offline + development; production refuses it. + +### Offline seeded environment (#147) + +```bash +IPFS_TRANSPORT=local npm run db:seed # deterministic; writes .ipfs-local/ +IPFS_TRANSPORT=local npm run dev +``` + +Every seeded question and trace hash commits to bytes stored under +`.ipfs-local/`, so `POST /traces/verify` and the trace-hash checks pass +without Pinata or network access. `npm run db:seed -- --reset` truncates first; +`--now` shifts timestamps; `--from-snapshot ` loads an anonymised NDJSON +snapshot; `--with-raw-events ` writes a `getEvents` fixture. ## API at a glance @@ -74,7 +91,7 @@ The full contract with the frontend, including request and response shapes, erro | Area | Endpoints | |---|---| -| Health | `GET /health` | +| Health | `GET /health`, `GET /readyz` | | Auth | `POST /auth/challenge`, `POST /auth/verify` (signed challenge transaction → JWT) | | Markets | `GET /markets`, `GET /markets/:id`, `GET /markets/on-chain/:onChainMarketId`, `GET /markets/on-chain/:onChainMarketId/state` (live from the chain), `POST /markets/generate` (admin), `GET /markets/generation-status/:jobId` | | Traces | `GET /traces`, `GET /traces/:id`, `POST /traces/verify`, `POST /traces/:id/unlock` (daily pass, verified on-chain), `GET`/`PUT /traces/access/allowance`, `GET /traces/payments` | @@ -93,8 +110,15 @@ The full contract with the frontend, including request and response shapes, erro | `npm run sync:contracts` | Copy generated contract code from `contracts/` into `src/generated/` | | `npm run check:contracts` | Fail if `src/generated/` is stale (CI runs this) | | `npm run prisma:migrate` | `prisma migrate dev` | +| `npm run db:seed` | Deterministic local dataset (#147); `--reset`, `--now`, `--from-snapshot`, `--link-testnet` | +| `npm run snapshot:export -- --out ` | Anonymised NDJSON snapshot (`SNAPSHOT_SALT` required) | +| `npm run reconcile:post-restore -- --manifest … --out …` | Replay the chain gap after a restore (#146) | +| `scripts/backup.sh` / `scripts/restore.sh` | Dump with manifest + checksum / restore (#146) | +| `scripts/check-migration-compat.mjs` | Expand/contract migration lint (#144) | +| `scripts/post-deploy-checks.sh` | Readiness, digest, indexer lag and burn-rate checks | | `scripts/smoke.sh` | Smoke test a running server (health, login, markets, live chain read) | | `docker compose up -d` / `down` | Start / stop local Postgres and Redis | +| `docker compose --profile backup up -d backup` | Optional backup loop | ## Project structure @@ -107,10 +131,11 @@ src/ IPFS, trace publishing, market creation, LLM-backed logic services/stellar/ Binding clients and contract error names lib/ Prisma, Redis, logger, categories, amounts, resolution specs + seed/ Deterministic dataset + raw-event encoder (#147) generated/ Copied from the contracts submodule. Do not edit. -prisma/ Schema and forward-only migrations +prisma/ Schema, forward-only migrations, seed runner tests/ node:test suites and fixtures -scripts/ sync-contracts.mjs, smoke.sh +scripts/ Contract sync, seed/snapshot/reconcile, backup/restore, deploy checks docker-compose.yml Local Postgres and Redis contracts/ OracleDesk-SmartContract submodule (pinned) docs/ API contract, architecture, porting notes, status, backlog diff --git a/docker-compose.yml b/docker-compose.yml index 86c5acd..51700e3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,5 +20,27 @@ services: test: ["CMD", "redis-cli", "ping"] interval: 5s retries: 10 + # Optional nightly backup loop (#146). Start with: + # docker compose --profile backup up -d backup + backup: + image: postgres:16-alpine + profiles: ["backup"] + depends_on: + postgres: + condition: service_healthy + environment: + DATABASE_URL: postgresql://oracledesk:oracledesk@postgres:5432/oracledesk + BACKUP_DIR: /backups + BACKUP_KEEP_DAILY: "7" + BACKUP_KEEP_WEEKLY: "4" + volumes: + - ./backups:/backups + - .:/repo:ro + entrypoint: > + sh -c "apk add --no-cache nodejs git >/dev/null && + while true; do + cd /repo && DATABASE_URL=$$DATABASE_URL BACKUP_DIR=$$BACKUP_DIR ./scripts/backup.sh || true; + sleep 21600; + done" volumes: pgdata: diff --git a/docs/api.md b/docs/api.md index d915afb..df4f011 100644 --- a/docs/api.md +++ b/docs/api.md @@ -4,7 +4,7 @@ This is the contract between `OracleDesk-Frontend` and this backend. The Soroban contracts in `OracleDesk-SmartContract/oracledesk-stellar` are the source of truth for anything on-chain; this API never overrides them. -Base URL: `/api/v1` (health is at `/health`). +Base URL: `/api/v1` (liveness is `/health`, readiness is `/readyz`). ## Conventions @@ -78,6 +78,8 @@ removes the enum value. `settlementCurrency` stays in responses (always `GET /health` → `{ status: "healthy", uptime: number }` +`GET /readyz` → `200` when the database is reachable, `503 NOT_READY` otherwise. + ### Auth (signed challenge, SEP-10 style) The old `POST /auth/connect` issued a JWT for any address it was handed, so diff --git a/docs/architecture.md b/docs/architecture.md index 24b23b9..dce2f95 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -63,3 +63,45 @@ Outcomes are decided on-chain by the resolver contract. The backend only reads ` ## Generated code `src/generated/` is copied from the pinned `contracts` submodule by `scripts/sync-contracts.mjs`, with a do-not-edit header and `// @ts-nocheck`. CI fails if it's stale. + + +## Secrets (#145) + +`src/config/secrets.ts` is the only runtime source of secret values. + +- `SECRETS_PROVIDER=env` (default) reads the environment; `file` reads + `_FILE` mounts and re-reads them on refresh; `vault` reads a Vault KV v2 + path using a token file. AWS Secrets Manager can be added through the same + `SecretsProvider` interface. +- Boot resolves files in `loadConfig`, then `initSecrets` validates and stores + raw values as Buffers. Empty or invalid refresh values keep the previous + value; a provider failure keeps everything. +- JWT rotation keeps the previous key for + `JWT_PREVIOUS_SECRET_GRACE_SECS` (default 24h), then zeroes it. Expired tokens + are never retried against the previous key. +- SDK clients (Gemini, Anthropic) rebuild when the key version changes. + Values already copied into an SDK are immutable strings; zeroing the store + only guarantees the long-lived copy is gone. +- `AGENT_SECRET_KEY` is deliberately not in the runtime secret store: rotating + the agent identity is an on-chain registration change, so it stays env-only + and requires a restart. `DATABASE_URL` must be available at boot (env or + `DATABASE_URL_FILE`); rotating it through Vault would not rebuild Prisma. + +## Seed and snapshots (#147) + +`npm run db:seed` builds a deterministic dataset through `src/seed/scenario.ts`. +Every question/trace hash is computed with the production helpers and the exact +bytes are written under `.ipfs-local/` (CIDv1 raw/sha2-256). It refuses +production, live-chain mode and non-local databases unless +`SEED_ALLOW_REMOTE=1`. + +`npm run snapshot:export -- --out ` streams anonymised NDJSON. The column +map is fail-closed: a new column without an anonymisation rule stops the export. +Hashed identifiers use `SNAPSHOT_SALT`; Stellar addresses are mapped to +deterministic throwaway G-addresses. + +## Backups (#146) + +Custom-format dumps plus a manifest (migrations, cursor, max ledger, counts) +and checksums. Restore then replays the chain gap with +`scripts/post-restore-reconcile.ts`; see the runbook. diff --git a/docs/deployment.md b/docs/deployment.md new file mode 100644 index 0000000..4e2bba8 --- /dev/null +++ b/docs/deployment.md @@ -0,0 +1,51 @@ +# Deployment (#144) + +## Pipeline + +1. **Verify** — CI jobs (contract sync, migrations, build, tests, actionlint, + shellcheck). +2. **Image** — buildx pushes `ghcr.io/@sha256:`, cosign signs the + digest, Syft emits an SPDX SBOM. +3. **Staging** — `deploy-env.yml` verifies the cosign identity, lints migrations + and runs `helm upgrade --install --atomic`. +4. **Staging checks** — readiness must hold 3 consecutive probes, indexer lag + must be ≤ 120 ledgers and the cursor must advance. +5. **Production** — same reusable workflow with the production GitHub + Environment (required reviewers). +6. **Production checks** — 5 consecutive readiness probes plus the same lag, + cursor and Prometheus checks. + +Secrets (kubeconfig, Helm values) live in GitHub Environments. The image is +referenced by digest, never by a mutable tag. + +## Migration rule: expand / contract + +`scripts/check-migration-compat.mjs` (run on the migrations changed in a PR/release) blocks: + +- `DROP TABLE`, `DROP COLUMN`, `RENAME`; +- `ALTER COLUMN … TYPE`; +- `SET NOT NULL`; +- `CREATE INDEX` without `CONCURRENTLY`. + +Ship additive migrations first, deploy both versions, then remove old columns +in a later release. `--allow-breaking` exists only for an explicitly approved +maintenance window. + +## Rollback + +Roll back the Helm release to the previous revision. If the new release added +only expand-phase changes, the old code keeps working. Do **not** roll back the +database; forward-fix instead. + +## Post-deploy checks + +```bash +API_URL=https://api.example \ +INDEXER_STATUS_URL=https://api.example/internal/indexer-status \ +MAX_INDEXER_LAG_LEDGERS=120 \ +PROMETHEUS_URL=https://prometheus.example \ +KUBE_NAMESPACE=oracledesk DEPLOYMENTS="api indexer worker" \ +DIGEST= ./scripts/post-deploy-checks.sh +``` + +Checks are read-only. `SKIP_KUBE=1` omits the digest check in local tests. diff --git a/docs/runbooks/backup-restore.md b/docs/runbooks/backup-restore.md new file mode 100644 index 0000000..b8707ab --- /dev/null +++ b/docs/runbooks/backup-restore.md @@ -0,0 +1,65 @@ +# Backup and restore runbook + +## Take a backup + +```bash +DATABASE_URL=… BACKUP_DIR=./backups ./scripts/backup.sh +# encrypted with age: +DATABASE_URL=… BACKUP_DIR=./backups ./scripts/backup.sh --encrypt-to age1… +# only the critical NDJSON tables: +DATABASE_URL=… BACKUP_DIR=./backups ./scripts/backup.sh --critical-only +``` + +The script writes a custom-format `pg_dump`, its `.sha256`, and a +`-manifest.json` containing migrations, indexer cursor(s), the highest +`chain_events.ledger`, the newest payment timestamp and row counts. Keep +`BACKUP_KEEP_DAILY` / `BACKUP_KEEP_WEEKLY` dumps (default 7 / 4). + +The optional Compose profile runs the same script every 6 hours: + +```bash +docker compose --profile backup up -d backup +``` + +## Restore + +Restore into an empty database: + +```bash +DATABASE_URL=… ./scripts/restore.sh backups/oracledesk-….dump +``` + +The script verifies the checksum, refuses a non-empty target unless `--force` +is passed, runs `pg_restore`, then `prisma migrate deploy`. Print the manifest +to see which indexer cursor the snapshot expects. + +## Reconcile after restore + +Postgres is now behind the chain. Replay the ledger gap and rebuild +projections: + +```bash +DATABASE_URL=… INDEXER_CURSOR_ID='stellar:Test SDF Network ; September 2015' \ + npx tsx scripts/post-restore-reconcile.ts \ + --manifest backups/oracledesk-…-manifest.json \ + --rewind-ledgers 720 --scan-transfers \ + --out reconcile-report.json +``` + +The report lists: + +- the RPC retention gap (events that can never be replayed); +- events re-projected and newly indexed; +- CONFIRMED payments that could not be re-verified (expected once RPC + retention drops them); +- markets with a missing `resolutionSpec` or a hash mismatch (exit 1); +- recipient transfers with no `payment_events` row (`--scan-transfers`). + +A finality rewind of 720 ledgers (about an hour) is the default. Increase it +when restoring an old snapshot. The process is read-only against Stellar. + +## Drill + +`.github/workflows/restore-drill.yml` seeds a scratch database, backs it up, +drops the schema, restores, and reconciles against `tests/fixtures/events.json` +(`RECONCILE_FAKE_RPC`). It runs monthly and on demand. diff --git a/package-lock.json b/package-lock.json index 11b8f12..3929a43 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7,7 +7,7 @@ "": { "name": "oracledesk-backend", "version": "1.0.0", - "license": "ISC", + "license": "MIT", "dependencies": { "@anthropic-ai/sdk": "~0.96.0", "@google/generative-ai": "~0.24.1", @@ -51,6 +51,7 @@ "@types/uuid": "~10.0.0", "nodemon": "~3.1.14", "ts-node-dev": "~2.0.0", + "tsx": "^4.23.15", "typescript": "~6.0.3" } }, @@ -131,6 +132,448 @@ "node": ">=6" } }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.2.tgz", + "integrity": "sha512-XExcO+dvLKvVtNTibSTBej1NCAbaGhWn9Ww1ZPx80qsahhPFe/8jgWP0IchNe0F3HwkU7n8ejhH8bjonqht8mQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.28.2.tgz", + "integrity": "sha512-kXXoiPVVGQcnIYGOeaovwOURpniDBpSq4A03qkQ+BMQqtGG6HYap3xne9C1O1yo4TR3qxlCX5IqqmX6fFo2Lqg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.28.2.tgz", + "integrity": "sha512-5YfKeeI8qWfBZIX+u2xZC3Zlb3Os/gLS2sbEKM+I4ZOcsWmHS2WLysCcQZDAFRslDUU5Oiq44gf6PYN1vGwG5A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.28.2.tgz", + "integrity": "sha512-O387ite7SzUyCcy3JQX4P4bLtEA7bLLkx+esve5JHnyYfNTxcVpXZo9jhdB0lTKN44gztELTdU7nS8Nr16Fs1Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.28.2.tgz", + "integrity": "sha512-n4KqkOQrraxHJcgjM1RvwbigfQKIKJVpM7xp+KsxiyUSrRdIXnt73VhrPAx0fV44hgfmIVKjxMN9J1t5jySVkw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.28.2.tgz", + "integrity": "sha512-uq6suIWYP37qzGddBKPw5QEQPi6HiLGsO7UmkpfyaYNQ3D+rN6w6WfwH+nuqcGXWvawGwxOEroO4YGnFh95azw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.28.2.tgz", + "integrity": "sha512-n+I0BTSRIoy+d6RPKnEVwql5UwBJolytvY4mAOIEJorKlqgPII8ix6slVVrfZ5Tnj7glIZvloylbB/EJPMWEXw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.28.2.tgz", + "integrity": "sha512-78XJTJkvPs0kz2w61301PJjXl4g7q3JqiYMZ/M/yVI73EHBrCRTgkhu9oqG7vPqq+a/yadEW8aD+agKlk5xrmg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.28.2.tgz", + "integrity": "sha512-XlDnu2q5yoqems+xay6wSAcg9DDD7K9RLKZEBOMZm3ckNpJBvOX20tSfby8KfrrhINDyv9V2YVZKY/SpoGJI8w==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.2.tgz", + "integrity": "sha512-pW4AC0P3it8c7do9MVM4p51FzHzdM/TZrerurgRcHJ2WTa1VQ1CIq18xncfpBJw4ojkiZZrKW2yIBWBP92j6Ug==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.28.2.tgz", + "integrity": "sha512-CYbnj78HsIeA+DhgUKgFCfvNsTHFhMMrinUrMZpDXJXKN8T3XViTZ/+wtHeVxEWY8ewSzTFN+nRmSwO2tZaLUQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.28.2.tgz", + "integrity": "sha512-buwkd8nsph4R+ajRvw0qM5Hja/TXQow3ptzWO2EbG/cqcIkHloRrdlBtQlshyYGTNFvfkfJ5tpPLVkY4DtsPfQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.28.2.tgz", + "integrity": "sha512-ZVykbDyk7519VwiNb9Lcj9m8XM6v5V9uKPvrEMkkEedVewf+0itkhahp4HDpgERXhwLRpWFypsGbG/J8s0QjJA==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.28.2.tgz", + "integrity": "sha512-CAXl+Dtd9UUuJd8pKKdwh6MLm3MUMiqMPmhZ3tTSXPqfyQ3vDl6R5hZdZ/kYojK4ofXtdfSv1tFq8XzWx3heNQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.28.2.tgz", + "integrity": "sha512-GeXCej4IQtU1B+QlDV8W/RRvbzI3O/Stss+/bCXv4lZls5WGRtu2a+3JkA3i4qIUlMXpcHebWpF8AkJhATowuA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.28.2.tgz", + "integrity": "sha512-3H1weTYZPxt/WOhByszQZybS9w5lKzUn1FDMsgEChbHWQwHYQQRfBxgCcZvPhjHfKyJjIievvMmEUawJrdY9Dg==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.2.tgz", + "integrity": "sha512-4xTZr1FUmSoQW4XIWmit3tzQrUTZM+N3P0XV8xROKYF50XfI7xeO90+1bZvNwxIufQ9hDQVRJH5YhgPVF8A/HQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.28.2.tgz", + "integrity": "sha512-sSATRjPeDBg3pdgHoQfoYBob11Kk1FGa9lui5RIHZCoCkJa9QKlvl3/vKz2usCmYYjs7ymJR/2Nnsqe+Hjt5nw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.28.2.tgz", + "integrity": "sha512-lqnzCV+mM0gIADaKihiCg6ifgfU2L3h5E33rNQBN1Y4MaVGnzryzmvvf7UHxprpQdE8hpqLolJ9Rl+SkIRDpyw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.28.2.tgz", + "integrity": "sha512-AL2qJILH7lNjrDmCQDvdxMfAUIv8KMNZOvrwAQ8i8//ntL9FflhOyMJ8OZSMBb8/AWXe3/5v5S20y3zCoZWKoQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.28.2.tgz", + "integrity": "sha512-QtiuPytchRyC4rwUKhexJdQKvDuZ6hWloi3igqPQNUJCS1/v9EiO3UTOXR6A3FoMo4fnAKbWJdqaIwhOzh8qEw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.28.2.tgz", + "integrity": "sha512-WkhYDmpTjLvGlScA1rwjRUmhl4k8oXR3cIbtqWmELgU/dFeHHlEllxDvdWcNJV9rbzCexB5vz8gtNewWLgCT7Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.28.2.tgz", + "integrity": "sha512-GPMSkTOtMnv2U2F8gxe4Io6qmVs+YKyp832Etqqxr0hFngmXQ3rzwytelm3GIn7T4VviRUlf3sOgBOiTdvaf7g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.28.2.tgz", + "integrity": "sha512-PIhhEkE9uPBleRBrQEJpUn7MBnibZzbGzYWPmY3x+YoVg/95zbjB4CxPPOQ8l5tYYM4mMaCthF8/1DIfBQQyWQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.28.2.tgz", + "integrity": "sha512-YmJbfTlvU7Sdn9BB+4PRES4oB6pxgS37MAONj+hBr/cpXS1aBPKXxNnDbu+QCWPj0o9dgyxeq79g6c5P8KeuYA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.28.2.tgz", + "integrity": "sha512-5ebpxr3nWMzrL/rnUI755Jkuee0bHL/Gq0WTF9lvcpv73wAp5eu8MfBUgWK9bhWvZjj7yX8etf/8tI8Ney695g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, "node_modules/@google/generative-ai": { "version": "0.24.1", "resolved": "https://registry.npmjs.org/@google/generative-ai/-/generative-ai-0.24.1.tgz", @@ -1963,6 +2406,48 @@ "node": ">= 0.4" } }, + "node_modules/esbuild": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.2.tgz", + "integrity": "sha512-HKVLS8dvII+xoKW9kmqxbRKrnWEXfJJr/FZhhJmiqIB0e053QNYFqOBouTMO/k5sID4MvCiUCvv8b9M4h32wIA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.28.2", + "@esbuild/android-arm": "0.28.2", + "@esbuild/android-arm64": "0.28.2", + "@esbuild/android-x64": "0.28.2", + "@esbuild/darwin-arm64": "0.28.2", + "@esbuild/darwin-x64": "0.28.2", + "@esbuild/freebsd-arm64": "0.28.2", + "@esbuild/freebsd-x64": "0.28.2", + "@esbuild/linux-arm": "0.28.2", + "@esbuild/linux-arm64": "0.28.2", + "@esbuild/linux-ia32": "0.28.2", + "@esbuild/linux-loong64": "0.28.2", + "@esbuild/linux-mips64el": "0.28.2", + "@esbuild/linux-ppc64": "0.28.2", + "@esbuild/linux-riscv64": "0.28.2", + "@esbuild/linux-s390x": "0.28.2", + "@esbuild/linux-x64": "0.28.2", + "@esbuild/netbsd-arm64": "0.28.2", + "@esbuild/netbsd-x64": "0.28.2", + "@esbuild/openbsd-arm64": "0.28.2", + "@esbuild/openbsd-x64": "0.28.2", + "@esbuild/openharmony-arm64": "0.28.2", + "@esbuild/sunos-x64": "0.28.2", + "@esbuild/win32-arm64": "0.28.2", + "@esbuild/win32-ia32": "0.28.2", + "@esbuild/win32-x64": "0.28.2" + } + }, "node_modules/escape-html": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", @@ -4732,6 +5217,25 @@ "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", "license": "0BSD" }, + "node_modules/tsx": { + "version": "4.23.15", + "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.23.15.tgz", + "integrity": "sha512-Yiex1Ovn8z2xPpOWckIiysV1SSyRMY9BkLF++q0yKiDxCqRhosKfMg3janKkiLBwZ5c/YryloKwGZcrEmtwxKw==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "~0.28.0" + }, + "bin": { + "tsx": "dist/cli.mjs" + }, + "engines": { + "node": ">=18.0.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + } + }, "node_modules/type-is": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.1.0.tgz", diff --git a/package.json b/package.json index ed7eb80..8725856 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,11 @@ "prisma:generate": "prisma generate", "prisma:studio": "prisma studio", "sync:contracts": "node scripts/sync-contracts.mjs", - "check:contracts": "node scripts/sync-contracts.mjs --check" + "check:contracts": "node scripts/sync-contracts.mjs --check", + "db:seed": "tsx prisma/seed.ts", + "db:reset": "prisma migrate reset --force --skip-seed && npm run db:seed", + "snapshot:export": "tsx scripts/snapshot-export.ts", + "reconcile:post-restore": "tsx scripts/post-restore-reconcile.ts" }, "keywords": [], "author": "", @@ -61,6 +65,7 @@ "@types/uuid": "~10.0.0", "nodemon": "~3.1.14", "ts-node-dev": "~2.0.0", - "typescript": "~6.0.3" + "typescript": "~6.0.3", + "tsx": "^4.23.15" } } diff --git a/prisma.config.ts b/prisma.config.ts index 831a20f..ccadfb8 100644 --- a/prisma.config.ts +++ b/prisma.config.ts @@ -1,14 +1,34 @@ // This file was generated by Prisma, and assumes you have installed the following: // npm install --save-dev prisma dotenv import "dotenv/config"; +import fs from "node:fs"; import { defineConfig } from "prisma/config"; +/** + * Resolve DATABASE_URL_FILE for the Prisma CLI (#145), so + * `npx prisma migrate deploy` works with a mounted secret. The app itself + * resolves the same file in src/config/index.ts. + */ +function databaseUrl(): string | undefined { + const direct = process.env["DATABASE_URL"]; + const file = process.env["DATABASE_URL_FILE"]; + if (direct && file) throw new Error("DATABASE_URL and DATABASE_URL_FILE cannot both be set"); + if (file) { + const value = fs.readFileSync(file, "utf8").replace(/[\r\n]+$/, ""); + if (!value.trim()) throw new Error("DATABASE_URL_FILE is empty"); + return value; + } + return direct; +} + export default defineConfig({ schema: "prisma/schema.prisma", migrations: { path: "prisma/migrations", + // #147 — `npx prisma migrate reset` and `npx prisma db seed` use this. + seed: "tsx prisma/seed.ts", }, datasource: { - url: process.env["DATABASE_URL"], + url: databaseUrl(), }, }); diff --git a/prisma/seed.ts b/prisma/seed.ts new file mode 100644 index 0000000..199f11b --- /dev/null +++ b/prisma/seed.ts @@ -0,0 +1,278 @@ +/** + * `npm run db:seed` / `npx prisma db seed` (#147). + * + * Deterministic dataset. The same seed always produces the same rows, every + * hash is computed with the production helpers, and the pinned bytes are + * written to the local IPFS stub so `POST /traces/verify` can pass offline. + */ +import 'dotenv/config'; +import fs from 'node:fs'; +import path from 'node:path'; +import { PrismaClient } from '@prisma/client'; +import { canonicalBytes } from '../src/utils/hash.util'; +import { questionDocument } from '../src/services/market-chain.service'; +import { cidForBytes } from '../src/services/ipfs.service'; +import { buildSeedData, SEED_EPOCH } from '../src/seed/scenario'; +import { encodeSeedEvents } from '../src/seed/raw-events'; + +const prisma = new PrismaClient(); + +const TABLE_ORDER = [ + 'copyTrade', + 'position', + 'subscription', + 'paymentEvent', + 'spendingAllowance', + 'agentLog', + 'signalCache', + 'reasoningTrace', + 'trade', + 'chainEvent', + 'indexerCursor', + 'marketGenerationJob', + 'market', + 'user', +] as const; + +interface Args { + reset: boolean; + now: boolean; + linkTestnet: boolean; + withRawEvents?: string; + fromSnapshot?: string; +} + +function parseArgs(argv: string[]): Args { + const args: Args = { reset: false, now: false, linkTestnet: false }; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === '--reset') args.reset = true; + else if (arg === '--now') args.now = true; + else if (arg === '--link-testnet') args.linkTestnet = true; + else if (arg === '--with-raw-events') { + const next = argv[i + 1]; + if (!next || next.startsWith('--')) throw new Error('--with-raw-events needs a file path'); + args.withRawEvents = next; + i++; + } else if (arg === '--from-snapshot') { + const next = argv[i + 1]; + if (!next || next.startsWith('--')) throw new Error('--from-snapshot needs a directory'); + args.fromSnapshot = next; + i++; + } + } + return args; +} + +function assertSafeDatabase(): void { + if (process.env.NODE_ENV === 'production') { + throw new Error('Refusing to seed: NODE_ENV=production'); + } + if (process.env.CHAIN_EXECUTION_MODE === 'live') { + throw new Error('Refusing to seed with CHAIN_EXECUTION_MODE=live; the seed is read-only'); + } + const url = process.env.DATABASE_URL; + if (!url) throw new Error('DATABASE_URL is required'); + let host: string; + try { + host = new URL(url).hostname; + } catch { + throw new Error('DATABASE_URL is not a valid URL'); + } + const allowed = new Set(['localhost', '127.0.0.1', '::1', 'postgres', 'oracledesk']); + if (!allowed.has(host) && process.env.SEED_ALLOW_REMOTE !== '1') { + throw new Error(`Refusing to seed remote database host "${host}"; set SEED_ALLOW_REMOTE=1 to override`); + } +} + +async function reset(): Promise { + for (const table of TABLE_ORDER) { + await (prisma as any)[table].deleteMany({}); + } +} + +async function insertData(data: ReturnType): Promise { + const sets: Array<[string, any[]]> = [ + ['user', data.users], + ['market', data.markets], + ['reasoningTrace', data.traces], + ['trade', data.trades], + ['position', data.positions], + ['copyTrade', data.copyTrades], + ['subscription', data.subscriptions], + ['paymentEvent', data.paymentEvents], + ['spendingAllowance', data.spendingAllowances], + ['agentLog', data.agentLogs], + ['signalCache', data.signalCache], + ['chainEvent', data.chainEvents], + ['indexerCursor', data.indexerCursors], + ['marketGenerationJob', data.marketGenerationJobs], + ]; + for (const [table, rows] of sets) { + if (rows.length === 0) continue; + await (prisma as any)[table].createMany({ data: rows, skipDuplicates: true }); + } +} + +/** Writes the exact bytes each seeded hash commits to, under `.ipfs-local/`. */ +function writeLocalIpfs(data: ReturnType): string { + const dir = process.env.IPFS_LOCAL_DIR || '.ipfs-local'; + fs.mkdirSync(dir, { recursive: true }); + for (const market of data.markets) { + const bytes = canonicalBytes(questionDocument(market as any, market.resolutionHash)); + fs.writeFileSync(path.join(dir, cidForBytes(bytes)), bytes); + } + for (const trace of data.traces) { + const payload: Record = { + agentType: trace.agentType, + decisionType: trace.decisionType, + marketId: trace.marketId, + sourcesUsed: trace.sourcesUsed, + probabilityEstimate: trace.probabilityEstimate, + marketProbability: trace.marketProbability, + edge: trace.edge, + confidenceInterval: trace.confidenceInterval, + ...(trace.agentType === 'TRADER' + ? { betFraction: trace.betFraction ?? undefined, betSizeUsdc: trace.betSizeUsdc ?? undefined } + : {}), + hedgeConditions: trace.hedgeConditions, + agentWallet: trace.agentWallet, + timestamp: trace.createdAt.toISOString(), + traceId: trace.id, + }; + const bytes = canonicalBytes(payload); + fs.writeFileSync(path.join(dir, cidForBytes(bytes)), bytes); + } + return dir; +} + +function readNdjson(dir: string): Record { + const tables: Record = {}; + for (const entry of fs.readdirSync(dir)) { + if (!entry.endsWith('.ndjson')) continue; + const table = entry.replace(/\.ndjson$/, ''); + tables[table] = fs + .readFileSync(path.join(dir, entry), 'utf8') + .split('\n') + .filter(Boolean) + .map((line) => JSON.parse(line)); + } + return tables; +} + +async function insertSnapshot(dir: string): Promise { + const tables = readNdjson(dir); + const order = [ + 'user', + 'market', + 'reasoning_trace', + 'trade', + 'position', + 'copy_trade', + 'subscription', + 'payment_event', + 'spending_allowance', + 'agent_log', + 'signal_cache', + 'chain_event', + 'indexer_cursor', + ]; + const map: Record = { + user: 'user', + market: 'market', + reasoning_trace: 'reasoningTrace', + trade: 'trade', + position: 'position', + copy_trade: 'copyTrade', + subscription: 'subscription', + payment_event: 'paymentEvent', + spending_allowance: 'spendingAllowance', + agent_log: 'agentLog', + signal_cache: 'signalCache', + chain_event: 'chainEvent', + indexer_cursor: 'indexerCursor', + }; + for (const key of order) { + const rows = tables[key]; + if (!rows?.length) continue; + await (prisma as any)[map[key]].createMany({ data: rows, skipDuplicates: true }); + } +} + +async function linkTestnet(): Promise { + // Reads only, never writes to chain. A real on-chain id may only be linked + // when `get_market` succeeds; the hash is copied from chain, never invented. + const ids = (process.env.SEED_TESTNET_MARKET_IDS ?? '0,1') + .split(',') + .map((v) => v.trim()) + .filter(Boolean); + if (ids.length === 0) return; + const { getOnChainMarket } = await import('../src/services/chain.service'); + for (const id of ids) { + try { + const market = await getOnChainMarket(BigInt(id)); + const tags = (market.category as any)?.tag ? [(market.category as any).tag] : []; + const backendCategory = + tags[0] === 'Macro' ? 'MACRO' + : tags[0] === 'Geopolitics' ? 'GEOPOLITICAL' + : tags[0] === 'Crypto' ? 'CRYPTO' + : tags[0] === 'Sports' ? 'SPORTS' + : tags[0] === 'Culture' ? 'ENTERTAINMENT' + : 'MACRO'; + await (prisma as any).market.createMany({ + data: [{ + id: `seed-testnet-market-${id}`, + question: `[testnet market ${id}] (question text unavailable)`, + category: backendCategory, + status: 'ACTIVE', + initialYesProb: 0.5, + currentYesProb: 0.5, + confidenceInterval: { lower: 0.4, upper: 0.6 }, + expiryTimestamp: new Date(SEED_EPOCH + 30 * 86_400_000), + onChainMarketId: BigInt(id), + questionHash: Buffer.from((market as any).question_hash).toString('hex'), + metaUri: (market as any).meta_uri, + resolutionHash: Buffer.from((market as any).resolution_hash).toString('hex'), + seedAmountRaw: (market as any).seed_amount?.toString?.() ?? null, + }], + skipDuplicates: true, + }); + console.log(`Linked testnet market ${id}`); + } catch (err) { + console.warn(`Skipping testnet market ${id}: ${err instanceof Error ? err.message : String(err)}`); + } + } +} + +async function main(): Promise { + assertSafeDatabase(); + const args = parseArgs(process.argv.slice(2)); + if (args.reset) await reset(); + + if (args.fromSnapshot) { + await insertSnapshot(args.fromSnapshot); + console.log(`Loaded snapshot from ${args.fromSnapshot}`); + } else { + const data = buildSeedData({ now: args.now }); + await insertData(data); + const dir = writeLocalIpfs(data); + console.log( + `Seeded ${data.markets.length} markets, ${data.traces.length} traces, ${data.chainEvents.length} chain events (IPFS stub: ${dir})`, + ); + if (args.withRawEvents) { + fs.writeFileSync(args.withRawEvents, JSON.stringify(encodeSeedEvents(data.chainEvents), null, 2) + '\n'); + console.log(`Wrote raw getEvents fixture to ${args.withRawEvents}`); + } + } + + if (args.linkTestnet) await linkTestnet(); +} + +main() + .catch((err) => { + console.error(err instanceof Error ? err.message : err); + process.exitCode = 1; + }) + .finally(async () => { + await prisma.$disconnect(); + }); diff --git a/scripts/backup.sh b/scripts/backup.sh new file mode 100755 index 0000000..9836348 --- /dev/null +++ b/scripts/backup.sh @@ -0,0 +1,118 @@ +#!/usr/bin/env bash +# Postgres backup with a consistency manifest (#146). +# +# DATABASE_URL=… BACKUP_DIR=./backups scripts/backup.sh [--encrypt-to ] +# DATABASE_URL=… BACKUP_DIR=./backups scripts/backup.sh --critical-only +set -euo pipefail + +command -v pg_dump >/dev/null 2>&1 || { echo "pg_dump not found" >&2; exit 1; } +command -v psql >/dev/null 2>&1 || { echo "psql not found" >&2; exit 1; } +DATABASE_URL="${DATABASE_URL:?DATABASE_URL is required}" +BACKUP_DIR="${BACKUP_DIR:-./backups}" +BACKUP_KEEP_DAILY="${BACKUP_KEEP_DAILY:-7}" +BACKUP_KEEP_WEEKLY="${BACKUP_KEEP_WEEKLY:-4}" +CRITICAL_ONLY=0 +ENCRYPT_TO="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --critical-only) CRITICAL_ONLY=1; shift ;; + --encrypt-to) ENCRYPT_TO="${2:?--encrypt-to needs a recipient}"; shift 2 ;; + *) echo "Unknown argument: $1" >&2; exit 2 ;; + esac +done + +mkdir -p "$BACKUP_DIR" +STAMP="$(date -u +%Y%m%dT%H%M%SZ)" +GIT_SHA="$(git rev-parse --short HEAD 2>/dev/null || echo unknown)" +BASE="oracledesk-${STAMP}-${GIT_SHA}" + +write_critical() { + local out="$1" + psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -c "\copy (SELECT row_to_json(t) FROM (SELECT id, \"onChainMarketId\", \"resolutionSpec\", \"resolutionHash\", \"questionHash\", \"metaUri\" FROM markets) t) TO '${out}/markets.ndjson' WITH (FORMAT text)" + psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -c "\copy (SELECT row_to_json(t) FROM (SELECT * FROM payment_events) t) TO '${out}/payment_events.ndjson' WITH (FORMAT text)" + psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -c "\copy (SELECT row_to_json(t) FROM (SELECT * FROM subscriptions) t) TO '${out}/subscriptions.ndjson' WITH (FORMAT text)" +} + +write_manifest() { + local out="$1" + local tmp + tmp="$(mktemp)" + { + echo "MIGRATIONS" + psql "$DATABASE_URL" -At -v ON_ERROR_STOP=1 -c "SELECT migration_name FROM _prisma_migrations ORDER BY finished_at" || true + echo "CURSORS" + psql "$DATABASE_URL" -At -F $'\t' -v ON_ERROR_STOP=1 -c 'SELECT id, "lastLedger" FROM indexer_cursors' || true + echo "MAXLEDGER" + psql "$DATABASE_URL" -At -v ON_ERROR_STOP=1 -c "SELECT COALESCE(MAX(ledger),0) FROM chain_events" || echo 0 + echo "MAXPAYMENT" + psql "$DATABASE_URL" -At -v ON_ERROR_STOP=1 -c "SELECT COALESCE(MAX(\"createdAt\")::text,'') FROM payment_events" || echo "" + echo "COUNTS" + psql "$DATABASE_URL" -At -F $'\t' -v ON_ERROR_STOP=1 -c " + SELECT 'users', count(*) FROM users UNION ALL + SELECT 'markets', count(*) FROM markets UNION ALL + SELECT 'reasoning_traces', count(*) FROM reasoning_traces UNION ALL + SELECT 'trades', count(*) FROM trades UNION ALL + SELECT 'positions', count(*) FROM positions UNION ALL + SELECT 'copy_trades', count(*) FROM copy_trades UNION ALL + SELECT 'subscriptions', count(*) FROM subscriptions UNION ALL + SELECT 'payment_events', count(*) FROM payment_events UNION ALL + SELECT 'chain_events', count(*) FROM chain_events" || true + } > "$tmp" + + node - "$tmp" "$out" "$STAMP" "$GIT_SHA" <<'NODE' +const fs = require('fs'); +const [,, tmp, out, createdAt, gitSha] = process.argv; +const lines = fs.readFileSync(tmp, 'utf8').split('\n'); +let section = ''; +const manifest = { createdAt, gitSha, migrations: [], cursors: [], maxChainEventLedger: 0, maxPaymentEventCreatedAt: null, rowCounts: {} }; +for (const line of lines) { + if (['MIGRATIONS','CURSORS','MAXLEDGER','MAXPAYMENT','COUNTS'].includes(line)) { section = line; continue; } + if (!line) continue; + if (section === 'MIGRATIONS') manifest.migrations.push(line); + else if (section === 'CURSORS') { const [id, ledger] = line.split('\t'); manifest.cursors.push({ id, lastLedger: Number(ledger) }); } + else if (section === 'MAXLEDGER') manifest.maxChainEventLedger = Number(line); + else if (section === 'MAXPAYMENT') manifest.maxPaymentEventCreatedAt = line || null; + else if (section === 'COUNTS') { const [table, count] = line.split('\t'); manifest.rowCounts[table] = Number(count); } +} +fs.writeFileSync(out, JSON.stringify(manifest, null, 2) + '\n'); +NODE + rm -f "$tmp" +} + +if [[ "$CRITICAL_ONLY" == "1" ]]; then + OUT="${BACKUP_DIR}/${BASE}-critical" + mkdir -p "$OUT" + write_critical "$OUT" + echo "$OUT" + exit 0 +fi + +DUMP="${BACKUP_DIR}/${BASE}.dump" +pg_dump "$DATABASE_URL" --format=custom --no-owner --serializable-deferrable --file "$DUMP" +if [[ -n "$ENCRYPT_TO" ]]; then + command -v age >/dev/null 2>&1 || { echo "age is required for --encrypt-to" >&2; rm -f "$DUMP"; exit 1; } + age -r "$ENCRYPT_TO" -o "${DUMP}.age" "$DUMP" + rm -f "$DUMP" + DUMP="${DUMP}.age" +fi + +DUMP_DIR="$(dirname "$DUMP")" +DUMP_NAME="$(basename "$DUMP")" +if command -v sha256sum >/dev/null 2>&1; then + (cd "$DUMP_DIR" && sha256sum "$DUMP_NAME" > "${DUMP_NAME}.sha256") +else + (cd "$DUMP_DIR" && shasum -a 256 "$DUMP_NAME" > "${DUMP_NAME}.sha256") +fi +write_manifest "${BACKUP_DIR}/${BASE}-manifest.json" + +# Retention: keep the newest daily/weekly counts. A simple, deterministic rule +# is enough here; the runbook documents how to tune it. +KEEP=$(( BACKUP_KEEP_DAILY > BACKUP_KEEP_WEEKLY ? BACKUP_KEEP_DAILY : BACKUP_KEEP_WEEKLY )) +i=0 +while IFS= read -r old; do + i=$((i + 1)) + if (( i > KEEP )); then rm -f "$old" "${old}.sha256"; fi +done < <(find "$BACKUP_DIR" -maxdepth 1 -name 'oracledesk-*.dump*' -type f ! -name '*.sha256' | sort -r) + +echo "$DUMP" diff --git a/scripts/check-migration-compat.mjs b/scripts/check-migration-compat.mjs new file mode 100755 index 0000000..7ef1b7a --- /dev/null +++ b/scripts/check-migration-compat.mjs @@ -0,0 +1,78 @@ +#!/usr/bin/env node +/** + * Expand/contract migration linter (#144). + * + * Flags statements that break old code during a rolling deploy: + * DROP TABLE/COLUMN, ALTER COLUMN ... TYPE, RENAME, SET NOT NULL without a + * default, and CREATE INDEX without CONCURRENTLY. + */ +import fs from 'node:fs'; +import path from 'node:path'; + +const RULES = [ + { name: 'drop-table', re: /\bDROP\s+TABLE\b/i, message: 'DROP TABLE breaks running code' }, + { name: 'drop-column', re: /\bDROP\s+COLUMN\b/i, message: 'DROP COLUMN breaks running code' }, + { name: 'alter-type', re: /\bALTER\s+COLUMN\b[\s\S]*?\bTYPE\b/i, message: 'ALTER COLUMN TYPE needs a backfill or a maintenance window' }, + { name: 'rename', re: /\bRENAME\s+(?:COLUMN|TO)\b/i, message: 'RENAME breaks old queries' }, + { name: 'set-not-null', re: /\bSET\s+NOT\s+NULL\b/i, message: 'SET NOT NULL without a default can fail on existing rows' }, + { name: 'non-concurrent-index', re: /\bCREATE\s+(?:UNIQUE\s+)?INDEX\b(?![\s\S]*?\bCONCURRENTLY\b)/i, message: 'CREATE INDEX should use CONCURRENTLY' }, +]; + +export function checkSql(sql) { + const withoutComments = sql + .replace(/--[^\n]*/g, '') + .replace(/\/\*[\s\S]*?\*\//g, '') + .trim(); + const statements = withoutComments.split(';').map((s) => s.trim()).filter(Boolean); + const findings = []; + for (const statement of statements) { + for (const rule of RULES) { + const match = rule.re.exec(statement); + if (!match) continue; + // SET NOT NULL is safe when the same statement supplies a default. + if (rule.name === 'set-not-null' && /\bDEFAULT\b/i.test(statement)) continue; + findings.push({ + rule: rule.name, + message: rule.message, + snippet: match[0].replace(/\s+/g, ' ').slice(0, 160), + }); + } + } + return findings; +} + +function filesAt(target) { + const stat = fs.statSync(target); + if (stat.isFile()) return [target]; + return fs.readdirSync(target, { withFileTypes: true }).flatMap((entry) => { + const full = path.join(target, entry.name); + if (entry.isDirectory()) return filesAt(full); + return entry.name.endsWith('.sql') ? [full] : []; + }); +} + +function main() { + const args = process.argv.slice(2); + const targets = args.filter((arg) => !arg.startsWith('--')); + const allowBreaking = args.includes('--allow-breaking'); + if (targets.length === 0) { + console.error('usage: check-migration-compat.mjs [--allow-breaking]'); + process.exit(2); + } + let failed = false; + for (const target of targets) { + for (const file of filesAt(target)) { + for (const finding of checkSql(fs.readFileSync(file, 'utf8'))) { + console.log(`${file}: ${finding.rule}: ${finding.message}`); + console.log(` ${finding.snippet}`); + failed = true; + } + } + } + if (failed && !allowBreaking) { + console.log('Mark the PR with `migration:breaking` and get approval before deploying.'); + process.exit(1); + } +} + +if (process.argv[1] && import.meta.url === `file://${path.resolve(process.argv[1])}`) main(); diff --git a/scripts/post-deploy-checks.sh b/scripts/post-deploy-checks.sh new file mode 100755 index 0000000..5b4e5a7 --- /dev/null +++ b/scripts/post-deploy-checks.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# Post-deploy verification (#144). Read-only and idempotent. +set -uo pipefail + +API_URL="${API_URL:-http://localhost:3000}" +READY_PATH="${READY_PATH:-/readyz}" +READY_CHECKS="${READY_CHECKS:-3}" +READY_INTERVAL="${READY_INTERVAL:-2}" +MAX_INDEXER_LAG_LEDGERS="${MAX_INDEXER_LAG_LEDGERS:-120}" +INDEXER_STATUS_URL="${INDEXER_STATUS_URL:-}" +CURSOR_ADVANCE_CHECKS="${CURSOR_ADVANCE_CHECKS:-2}" +PROMETHEUS_URL="${PROMETHEUS_URL:-}" +KUBE_NAMESPACE="${KUBE_NAMESPACE:-}" +DEPLOYMENTS="${DEPLOYMENTS:-api indexer worker}" +DIGEST="${DIGEST:-}" +SKIP_KUBE="${SKIP_KUBE:-0}" + +fail=0 +ok() { echo "OK: $*"; } +bad() { echo "FAIL: $*" >&2; fail=1; } + +ready=0 +for ((i=1; i<=READY_CHECKS; i++)); do + code="$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 "${API_URL}${READY_PATH}" || true)" + if [[ "$code" == "200" ]]; then ready=$((ready+1)); else break; fi + sleep "$READY_INTERVAL" +done +if [[ "$ready" -eq "$READY_CHECKS" ]]; then ok "readiness held ${READY_CHECKS}/${READY_CHECKS}"; else bad "readiness did not hold (${ready}/${READY_CHECKS})"; fi + +if [[ "$SKIP_KUBE" != "1" && -n "$KUBE_NAMESPACE" && -n "$DIGEST" ]]; then + for deployment in $DEPLOYMENTS; do + image="$(kubectl -n "$KUBE_NAMESPACE" get deployment "$deployment" -o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null || true)" + if [[ "$image" == *"@sha256:${DIGEST}" ]]; then ok "$deployment runs digest ${DIGEST:0:12}"; else bad "$deployment image '$image' does not match digest ${DIGEST:0:12}"; fi + done +fi + +if [[ -n "$INDEXER_STATUS_URL" ]]; then + first="$(curl -s --max-time 5 "$INDEXER_STATUS_URL" || true)" + latest="$(echo "$first" | jq -r '.latestLedger // empty' 2>/dev/null)" + cursor="$(echo "$first" | jq -r '.lastLedger // empty' 2>/dev/null)" + if [[ -z "$latest" || -z "$cursor" ]]; then + bad "indexer status missing latestLedger/lastLedger" + else + lag=$(( latest - cursor )) + if (( lag <= MAX_INDEXER_LAG_LEDGERS )); then ok "indexer lag ${lag} <= ${MAX_INDEXER_LAG_LEDGERS}"; else bad "indexer lag ${lag} > ${MAX_INDEXER_LAG_LEDGERS}"; fi + sleep "$READY_INTERVAL" + second="$(curl -s --max-time 5 "$INDEXER_STATUS_URL" || true)" + nextCursor="$(echo "$second" | jq -r '.lastLedger // empty' 2>/dev/null)" + if [[ -n "$nextCursor" ]] && (( nextCursor > cursor )); then ok "indexer cursor advanced"; else bad "indexer cursor did not advance"; fi + fi +elif [[ -n "${DATABASE_URL:-}" && -n "${INDEXER_CURSOR_ID:-}" ]]; then + cursor="$(psql "$DATABASE_URL" -At -c "SELECT \"lastLedger\" FROM indexer_cursors WHERE id='${INDEXER_CURSOR_ID}'" 2>/dev/null || true)" + if [[ -z "$cursor" ]]; then + bad "indexer cursor not found" + else + rpc_url="${STELLAR_RPC_URL:-https://soroban-testnet.stellar.org}" + latest="$(curl -s --max-time 5 -H 'content-type: application/json' \ + -d '{"jsonrpc":"2.0","id":1,"method":"getHealth"}' "$rpc_url" | jq -r '.result.latestLedger // empty' 2>/dev/null || true)" + if [[ -n "$latest" ]]; then + lag=$(( latest - cursor )) + if (( lag <= MAX_INDEXER_LAG_LEDGERS )); then ok "indexer lag ${lag} <= ${MAX_INDEXER_LAG_LEDGERS}"; else bad "indexer lag ${lag} > ${MAX_INDEXER_LAG_LEDGERS}"; fi + else + echo "SKIP: could not read latestLedger from ${rpc_url}" + fi + sleep "$READY_INTERVAL" + nextCursor="$(psql "$DATABASE_URL" -At -c "SELECT \"lastLedger\" FROM indexer_cursors WHERE id='${INDEXER_CURSOR_ID}'" 2>/dev/null || true)" + if [[ -n "$nextCursor" ]] && (( nextCursor > cursor )); then ok "indexer cursor advanced"; else bad "indexer cursor did not advance"; fi + fi +else + echo "SKIP: no INDEXER_STATUS_URL or DATABASE_URL/INDEXER_CURSOR_ID for indexer lag" +fi + +if [[ -n "$PROMETHEUS_URL" ]]; then + firing="$(curl -sG --max-time 5 "$PROMETHEUS_URL/api/v1/query" --data-urlencode 'query=ALERTS{alertstate="firing",severity="critical"}' | jq -r '.data.result | length' 2>/dev/null || echo unknown)" + if [[ "$firing" == "0" ]]; then ok "no critical burn-rate alerts"; else bad "critical alerts firing (count=${firing})"; fi + ratio="$(curl -sG --max-time 5 "$PROMETHEUS_URL/api/v1/query" --data-urlencode 'query=sum(rate(http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(http_requests_total[5m])), 0.001)' | jq -r '.data.result[0].value[1] // empty' 2>/dev/null || true)" + if [[ -n "$ratio" ]] && awk "BEGIN{exit !($ratio < 0.05)}"; then ok "5xx ratio ${ratio} < 0.05"; else bad "5xx ratio ${ratio:-unknown} >= 0.05"; fi +fi + +exit "$fail" diff --git a/scripts/post-restore-reconcile.ts b/scripts/post-restore-reconcile.ts new file mode 100644 index 0000000..a160662 --- /dev/null +++ b/scripts/post-restore-reconcile.ts @@ -0,0 +1,62 @@ +/** + * CLI wrapper for post-restore reconciliation (#146). + */ +import 'dotenv/config'; +import fs from 'node:fs'; +import { PrismaClient } from '@prisma/client'; +import { rpcServer } from '../src/services/stellar/clients'; +import { verifyUsdcTransfer } from '../src/services/payment-verification.service'; +import { + REWIND_LEDGERS_DEFAULT, + fixtureSource, + reconcile, + type BackupManifest, +} from '../src/services/reconcile.service'; + +interface Args { + manifest?: string; + out?: string; + rewindLedgers: number; + scanTransfers: boolean; +} + +function parseArgs(argv: string[]): Args { + const args: Args = { rewindLedgers: REWIND_LEDGERS_DEFAULT, scanTransfers: false }; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === '--manifest') args.manifest = argv[++i]; + else if (arg === '--out') args.out = argv[++i]; + else if (arg === '--rewind-ledgers') args.rewindLedgers = Number(argv[++i]); + else if (arg === '--scan-transfers') args.scanTransfers = true; + } + return args; +} + +async function main(): Promise { + const args = parseArgs(process.argv.slice(2)); + if (!args.out) throw new Error('--out is required'); + const manifest: BackupManifest = args.manifest + ? JSON.parse(fs.readFileSync(args.manifest, 'utf8')) + : {}; + const source = process.env.RECONCILE_FAKE_RPC + ? fixtureSource(process.env.RECONCILE_FAKE_RPC) + : rpcServer; + const prisma = new PrismaClient(); + try { + const report = await reconcile(args, { prisma, source, verifyPayment: verifyUsdcTransfer }, manifest); + fs.writeFileSync(args.out, JSON.stringify(report, null, 2) + '\n'); + console.log(`Wrote ${args.out}`); + if (report.specs.missing.length || report.specs.hashMismatch.length) { + process.exitCode = 1; + } + } finally { + await prisma.$disconnect(); + } +} + +if (require.main === module) { + main().catch((err) => { + console.error(err instanceof Error ? err.message : err); + process.exitCode = 1; + }); +} diff --git a/scripts/restore.sh b/scripts/restore.sh new file mode 100755 index 0000000..211c63d --- /dev/null +++ b/scripts/restore.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Restore a dump produced by scripts/backup.sh (#146). +# DATABASE_URL=… scripts/restore.sh [--force] +set -euo pipefail + +command -v pg_restore >/dev/null 2>&1 || { echo "pg_restore not found" >&2; exit 1; } +command -v psql >/dev/null 2>&1 || { echo "psql not found" >&2; exit 1; } +DATABASE_URL="${DATABASE_URL:?DATABASE_URL is required}" +DUMP="${1:?usage: scripts/restore.sh [--force]}" +FORCE=0 +[[ "${2:-}" == "--force" ]] && FORCE=1 + +if [[ "$DUMP" == *.age ]]; then + echo "Encrypted dump: decrypt it first (age -d -i -o $DUMP)" >&2 + exit 1 +fi + +if [[ -f "${DUMP}.sha256" ]]; then + (cd "$(dirname "$DUMP")" && { sha256sum -c "$(basename "$DUMP").sha256" 2>/dev/null || shasum -a 256 -c "$(basename "$DUMP").sha256"; }) +else + echo "Warning: no checksum next to $DUMP" >&2 +fi + +COUNT="$(psql "$DATABASE_URL" -At -v ON_ERROR_STOP=1 -c "SELECT count(*) FROM information_schema.tables WHERE table_schema='public'")" +if [[ "$COUNT" != "0" && "$FORCE" != "1" ]]; then + echo "Target database is not empty ($COUNT public tables). Use --force to restore anyway." >&2 + exit 1 +fi + +pg_restore "$DATABASE_URL" --no-owner --exit-on-error --clean --if-exists "$DUMP" +npx prisma migrate deploy + +MANIFEST="${DUMP%.dump}-manifest.json" +[[ "$MANIFEST" == *.age-manifest.json ]] && MANIFEST="${DUMP%.age}-manifest.json" +if [[ -f "$MANIFEST" ]]; then + echo "--- backup manifest ---" + cat "$MANIFEST" +fi diff --git a/scripts/snapshot-export.ts b/scripts/snapshot-export.ts new file mode 100644 index 0000000..329fffa --- /dev/null +++ b/scripts/snapshot-export.ts @@ -0,0 +1,74 @@ +/** + * CLI wrapper for the anonymised NDJSON snapshot exporter (#147). + * Pure rules live in src/lib/snapshot.ts so they can be unit tested. + */ +import fs from 'node:fs'; +import path from 'node:path'; +import { PrismaClient } from '@prisma/client'; +import { EXCLUDED_TABLES, RULES, assertNoSecrets, anonymizeRow, buildWalletMap, jsonSafe, SnapshotExportError } from '../src/lib/snapshot'; + +interface Args { + out: string; + tables: string[]; + limitPerTable?: number; +} + +function parseArgs(argv: string[]): Args { + let out = ''; + const tables: string[] = []; + let limitPerTable: number | undefined; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === '--out') out = argv[++i] ?? ''; + else if (arg === '--tables') tables.push(...(argv[++i] ?? '').split(',').map((t) => t.trim()).filter(Boolean)); + else if (arg === '--limit-per-table') limitPerTable = Number(argv[++i]); + } + if (!out) throw new SnapshotExportError('--out is required'); + return { out, tables, limitPerTable }; +} + +export async function exportSnapshot(prisma: PrismaClient, args: Args, salt: string): Promise { + const excluded = args.tables.filter((table) => (EXCLUDED_TABLES as readonly string[]).includes(table)); + if (excluded.length > 0) { + throw new SnapshotExportError(`Tables are excluded from snapshots: ${excluded.join(', ')}`); + } + const users = await prisma.user.findMany({ take: args.limitPerTable }); + const wallets = buildWalletMap(salt, users); + const selected = args.tables.length + ? Object.keys(RULES).filter((model) => args.tables.includes(RULES[model].output)) + : Object.keys(RULES); + fs.mkdirSync(args.out, { recursive: true }); + const written: string[] = []; + + for (const model of selected) { + const rows = await (prisma as any)[model].findMany({ take: args.limitPerTable }); + const rule = RULES[model]; + const file = path.join(args.out, `${rule.output}.ndjson`); + const lines = rows.map((row: Record) => jsonSafe(anonymizeRow(model, row, wallets, salt))); + const text = lines.length ? `${lines.join('\n')}\n` : ''; + assertNoSecrets(text, salt); + fs.writeFileSync(file, text); + written.push(file); + } + return written; +} + +async function main(): Promise { + const salt = process.env.SNAPSHOT_SALT; + if (!salt) throw new SnapshotExportError('SNAPSHOT_SALT is required'); + const args = parseArgs(process.argv.slice(2)); + const prisma = new PrismaClient(); + try { + const files = await exportSnapshot(prisma, args, salt); + console.log(`Wrote ${files.length} NDJSON files to ${args.out}`); + } finally { + await prisma.$disconnect(); + } +} + +if (require.main === module) { + main().catch((err) => { + console.error(err instanceof Error ? err.message : err); + process.exitCode = 1; + }); +} diff --git a/src/app.ts b/src/app.ts index cca72c1..c86a7da 100644 --- a/src/app.ts +++ b/src/app.ts @@ -6,6 +6,7 @@ import morgan from 'morgan'; import hpp from 'hpp'; import { config } from './config'; +import { prisma } from './lib/prisma'; import { logger } from './lib/logger'; import { errorMiddleware } from './middlewares/error.middleware'; import { globalLimiter } from './middlewares/rate-limit.middleware'; @@ -54,6 +55,17 @@ app.get('/health', (_req, res) => { res.json({ ok: true, data: { status: 'healthy', uptime: process.uptime() }, error: null }); }); +// Readiness for rolling deploys (#144): the process is only ready when it can +// reach the database. Keep this cheap — it runs every few seconds. +app.get('/readyz', async (_req, res) => { + try { + await prisma.$queryRaw`SELECT 1`; + res.json({ ok: true, data: { status: 'ready' }, error: null }); + } catch { + res.status(503).json({ ok: false, data: null, error: { code: 'NOT_READY', message: 'Database is not reachable' } }); + } +}); + // ─── API Routes ─── app.use('/api/v1', router); diff --git a/src/config/index.ts b/src/config/index.ts index 8ef478a..8798b81 100644 --- a/src/config/index.ts +++ b/src/config/index.ts @@ -2,6 +2,7 @@ import crypto from 'crypto'; import dotenv from 'dotenv'; import { Keypair, Networks, StrKey } from '@stellar/stellar-sdk'; import { z } from 'zod'; +import { resolveFileVars, validateSecrets, SECRET_NAMES } from './secrets'; dotenv.config(); @@ -54,6 +55,24 @@ const envSchema = z // G… addresses allowed to call admin endpoints. ADMIN_ADDRESSES: commaList(stellarAccount), + // ── Secrets (#145) ──────────────────────────────────────────────────── + // env: process environment (default); file: _FILE mounts; vault: + // Vault KV v2, refreshed every SECRETS_REFRESH_MS. + SECRETS_PROVIDER: z.enum(['env', 'file', 'vault']).default('env'), + SECRETS_REFRESH_MS: z + .string() + .default('300000') + .transform(Number) + .refine((v) => Number.isFinite(v) && v >= 30000, 'must be at least 30000 ms'), + JWT_PREVIOUS_SECRET_GRACE_SECS: z + .string() + .default('86400') + .transform(Number) + .refine((v) => Number.isFinite(v) && v >= 0, 'must be a non-negative integer'), + VAULT_ADDR: optionalString, + VAULT_TOKEN_FILE: optionalString, + VAULT_KV_PATH: optionalString, + // ── Stellar ─────────────────────────────────────────────────────────── STELLAR_RPC_URL: z.string().url().default('https://soroban-testnet.stellar.org'), STELLAR_NETWORK_PASSPHRASE: z.string().default(Networks.TESTNET), @@ -90,24 +109,23 @@ const envSchema = z PINATA_API_KEY: optionalString, PINATA_SECRET_API_KEY: optionalString, IPFS_GATEWAY_URL: z.string().url().default('https://gateway.pinata.cloud/ipfs'), + // #147 — development-only local IPFS stub selected explicitly. + IPFS_TRANSPORT: z.enum(['pinata', 'local']).default('pinata'), + IPFS_LOCAL_DIR: z.string().default('.ipfs-local'), // ── Data sources ────────────────────────────────────────────────────── NEWSAPI_KEY: optionalString, FRED_API_KEY: optionalString, }) .superRefine((env, ctx) => { - if (env.JWT_SECRET && env.JWT_SECRET.length < 32) { - ctx.addIssue({ code: 'custom', path: ['JWT_SECRET'], message: 'must be at least 32 characters' }); - } - if (env.NODE_ENV === 'production') { - for (const key of ['JWT_SECRET', 'AUTH_SIGNING_SECRET'] as const) { - if (!env[key]) ctx.addIssue({ code: 'custom', path: [key], message: 'is required in production' }); - } + if (env.SECRETS_PROVIDER === 'vault') { + if (!env.VAULT_ADDR) ctx.addIssue({ code: 'custom', path: ['VAULT_ADDR'], message: 'is required when SECRETS_PROVIDER=vault' }); + else if (!URL.canParse(env.VAULT_ADDR)) ctx.addIssue({ code: 'custom', path: ['VAULT_ADDR'], message: 'must be a URL' }); + if (!env.VAULT_TOKEN_FILE) ctx.addIssue({ code: 'custom', path: ['VAULT_TOKEN_FILE'], message: 'is required when SECRETS_PROVIDER=vault' }); + if (!env.VAULT_KV_PATH) ctx.addIssue({ code: 'custom', path: ['VAULT_KV_PATH'], message: 'is required when SECRETS_PROVIDER=vault' }); } - for (const key of ['AUTH_SIGNING_SECRET', 'AGENT_SECRET_KEY'] as const) { - if (env[key] && !StrKey.isValidEd25519SecretSeed(env[key])) { - ctx.addIssue({ code: 'custom', path: [key], message: 'must be a Stellar secret seed (S…)' }); - } + if (env.NODE_ENV === 'production' && env.IPFS_TRANSPORT === 'local') { + ctx.addIssue({ code: 'custom', path: ['IPFS_TRANSPORT'], message: 'local IPFS is development-only' }); } if (env.CHAIN_EXECUTION_MODE === 'live') { if (!env.AGENT_SECRET_KEY) { @@ -132,22 +150,49 @@ export class ConfigError extends Error {} /** Parses an env object. Throws ConfigError naming every bad variable. */ export function loadConfig(env: NodeJS.ProcessEnv): Config { - const parsed = envSchema.safeParse(env); + let resolvedEnv: NodeJS.ProcessEnv; + try { + resolvedEnv = resolveFileVars(env); + } catch (err) { + throw new ConfigError(`Invalid environment configuration:\n ${err instanceof Error ? err.message : String(err)}`); + } + + const parsed = envSchema.safeParse(resolvedEnv); if (!parsed.success) { const lines = parsed.error.issues.map((i) => ` ${i.path.join('.') || '(root)'} ${i.message}`); throw new ConfigError(`Invalid environment configuration:\n${lines.join('\n')}`); } const data = parsed.data; + // Using _FILE implies the file provider, so refresh re-reads the + // mounts instead of the (possibly empty) plain environment variable. + if ( + data.SECRETS_PROVIDER === 'env' && + SECRET_NAMES.some((name) => Boolean(resolvedEnv[`${name}_FILE`])) + ) { + data.SECRETS_PROVIDER = 'file'; + } + + const problems = validateSecrets(data, data.NODE_ENV, { + // A remote provider supplies secret values after loadConfig; boot-time + // presence checks move to initSecrets(). + requireProductionSecrets: data.SECRETS_PROVIDER !== 'vault', + }); + if (problems.length > 0) { + throw new ConfigError(`Invalid environment configuration:\n${problems.map((p) => ` ${p}`).join('\n')}`); + } + const ephemeralSecrets: string[] = []; - if (!data.JWT_SECRET) { + if (!data.JWT_SECRET && data.NODE_ENV !== 'production') { data.JWT_SECRET = crypto.randomBytes(32).toString('hex'); ephemeralSecrets.push('JWT_SECRET'); } - if (!data.AUTH_SIGNING_SECRET) { + if (!data.AUTH_SIGNING_SECRET && data.NODE_ENV !== 'production') { data.AUTH_SIGNING_SECRET = Keypair.random().secret(); ephemeralSecrets.push('AUTH_SIGNING_SECRET'); } if (!data.AUTH_WEB_AUTH_DOMAIN) data.AUTH_WEB_AUTH_DOMAIN = data.AUTH_HOME_DOMAIN; + // Prisma's env("DATABASE_URL") reads this process-wide value at client build. + if (data.DATABASE_URL) process.env.DATABASE_URL = data.DATABASE_URL; return { ...data, ephemeralSecrets }; } diff --git a/src/config/secrets.ts b/src/config/secrets.ts new file mode 100644 index 0000000..119c09f --- /dev/null +++ b/src/config/secrets.ts @@ -0,0 +1,411 @@ +/** + * Secret loading, rotation and accessors (#145). + * + * Boot-time values are resolved synchronously by `loadConfig` (including + * `_FILE`). `initSecrets` moves them into this store; at runtime every + * consumer reads through `secret()` so a refresh can swap a value without a + * restart. + * + * Raw values are kept as Buffers so a retired value can be zeroed. Strings + * handed to an SDK are immutable copies and remain until GC — documented in + * docs/architecture.md. + */ +import fs from 'fs'; +import { StrKey } from '@stellar/stellar-sdk'; + +export const SECRET_NAMES = [ + 'JWT_SECRET', + 'AUTH_SIGNING_SECRET', + 'PINATA_API_KEY', + 'PINATA_SECRET_API_KEY', + 'ANTHROPIC_API_KEY', + 'GEMINI_API_KEY', + 'NEWSAPI_KEY', + 'FRED_API_KEY', + 'DATABASE_URL', +] as const; + +export type SecretName = (typeof SECRET_NAMES)[number]; +export type SecretsProviderKind = 'env' | 'file' | 'vault'; + +export interface SecretsProvider { + readonly kind: SecretsProviderKind; + load(names: readonly SecretName[]): Promise>>; +} + +export interface SecretEntry { + current: string; + previous?: { value: string; retiredAt: number }; + version: number; +} + +export class SecretsNotReadyError extends Error { + constructor() { + super('Secrets have not been initialised; call initSecrets() during bootstrap'); + this.name = 'SecretsNotReadyError'; + } +} + +interface InternalEntry { + current: Buffer; + previous?: { value: Buffer; retiredAt: number }; + version: number; +} + +interface ResolvedSecrets { + JWT_SECRET: string; + AUTH_SIGNING_SECRET: string; + PINATA_API_KEY: string; + PINATA_SECRET_API_KEY: string; + ANTHROPIC_API_KEY: string; + GEMINI_API_KEY: string; + NEWSAPI_KEY: string; + FRED_API_KEY: string; + DATABASE_URL: string; + AGENT_SECRET_KEY?: string; + NODE_ENV?: string; + JWT_PREVIOUS_SECRET_GRACE_SECS?: number; + SECRETS_PROVIDER?: SecretsProviderKind; + SECRETS_REFRESH_MS?: number; + VAULT_ADDR?: string; + VAULT_TOKEN_FILE?: string; + VAULT_KV_PATH?: string; +} + +const store = new Map(); +const listeners = new Map void>>(); +let ready = false; +let graceMs = 86_400_000; +let activeProvider: SecretsProvider | null = null; + +function wraps(value: string): Buffer { + return Buffer.from(value, 'utf8'); +} + +function publicEntry(entry: InternalEntry): SecretEntry { + const current = entry.current.toString('utf8'); + const previous = entry.previous + ? { value: entry.previous.value.toString('utf8'), retiredAt: entry.previous.retiredAt } + : undefined; + return { current, previous, version: entry.version }; +} + +function assertReady(): void { + if (!ready) throw new SecretsNotReadyError(); +} + +export function secret(name: SecretName): string { + assertReady(); + return store.get(name)?.current.toString('utf8') ?? ''; +} + +export function secretEntry(name: SecretName): SecretEntry { + assertReady(); + const entry = store.get(name); + if (!entry) return { current: '', version: 0 }; + return publicEntry(entry); +} + +/** The previous value while it is inside the JWT rotation grace window. */ +export function previousSecret(name: SecretName): string | null { + assertReady(); + const previous = store.get(name)?.previous; + if (!previous) return null; + if (Date.now() - previous.retiredAt > graceMs) { + retirePrevious(name); + return null; + } + return previous.value.toString('utf8'); +} + +export function onSecretChange(name: SecretName, cb: (entry: SecretEntry) => void): () => void { + const set = listeners.get(name) ?? new Set(); + set.add(cb); + listeners.set(name, set); + return () => set.delete(cb); +} + +function emit(name: SecretName, entry: InternalEntry): void { + const value = publicEntry(entry); + for (const cb of listeners.get(name) ?? []) { + try { + cb(value); + } catch { + // A listener must never break the refresh loop. + } + } +} + +function retirePrevious(name: SecretName): void { + const entry = store.get(name); + if (!entry?.previous) return; + entry.previous.value.fill(0); + delete entry.previous; +} + +/** + * Validation shared by boot and refresh. Returns human-readable problems; + * callers decide whether to throw (boot) or keep the old value (refresh). + */ +export function validateSecrets( + values: Partial>, + nodeEnv: string, + options: { requireProductionSecrets?: boolean } = {}, +): string[] { + const problems: string[] = []; + const requireProd = options.requireProductionSecrets ?? true; + if (values.JWT_SECRET && values.JWT_SECRET.length < 32) { + problems.push('JWT_SECRET must be at least 32 characters'); + } + if (values.AUTH_SIGNING_SECRET && !StrKey.isValidEd25519SecretSeed(values.AUTH_SIGNING_SECRET)) { + problems.push('AUTH_SIGNING_SECRET must be a Stellar secret seed (S…)'); + } + if (values.AGENT_SECRET_KEY && !StrKey.isValidEd25519SecretSeed(values.AGENT_SECRET_KEY)) { + problems.push('AGENT_SECRET_KEY must be a Stellar secret seed (S…)'); + } + if (nodeEnv === 'production' && requireProd) { + if (!values.JWT_SECRET) problems.push('JWT_SECRET is required in production'); + if (!values.AUTH_SIGNING_SECRET) problems.push('AUTH_SIGNING_SECRET is required in production'); + } + return problems; +} + +function setEntry(name: SecretName, value: string, options: { rotate?: boolean } = {}): boolean { + const existing = store.get(name); + const next = wraps(value); + if (!existing) { + store.set(name, { current: next, version: 1 }); + emit(name, store.get(name)!); + return true; + } + if (existing.current.equals(next)) { + next.fill(0); + return false; + } + if (options.rotate && existing.current.length > 0) { + if (existing.previous) existing.previous.value.fill(0); + existing.previous = { value: existing.current, retiredAt: Date.now() }; + } else { + existing.current.fill(0); + } + existing.current = next; + existing.version += 1; + emit(name, existing); + return true; +} + +/** Populates the store from already-resolved config. */ +export async function initSecrets(cfg: ResolvedSecrets, provider?: SecretsProvider): Promise { + const values: Partial> = {}; + for (const name of SECRET_NAMES) values[name] = (cfg as unknown as Record)[name] ?? ''; + + const kind = cfg.SECRETS_PROVIDER ?? 'env'; + const remote = provider ?? (kind === 'vault' ? vaultProvider(cfg) : undefined); + if (remote) { + const loaded = await remote.load(SECRET_NAMES); + for (const name of SECRET_NAMES) { + if (loaded[name] !== undefined && loaded[name] !== '') values[name] = loaded[name]; + } + } + + const problems = validateSecrets( + { ...values, AGENT_SECRET_KEY: cfg.AGENT_SECRET_KEY }, + cfg.NODE_ENV ?? process.env.NODE_ENV ?? 'development', + ); + if (problems.length > 0) throw new Error(`Invalid secrets:\n${problems.map((p) => ` ${p}`).join('\n')}`); + + graceMs = (cfg.JWT_PREVIOUS_SECRET_GRACE_SECS ?? 86_400) * 1000; + for (const name of SECRET_NAMES) setEntry(name, values[name] ?? ''); + activeProvider = remote ?? (kind === 'file' ? fileProvider() : envProvider()); + ready = true; + + // Prisma reads DATABASE_URL from the environment when the client is built. + if (values.DATABASE_URL) process.env.DATABASE_URL = values.DATABASE_URL; +} + +export function setSecretsForTest(values: Partial>): void { + for (const name of SECRET_NAMES) setEntry(name, values[name] ?? ''); + ready = true; + activeProvider = null; +} + + +/** + * One refresh pass. Exported so tests can exercise rotation without waiting + * for the production minimum interval. + */ +export async function refreshSecretsOnce( + cfg: ResolvedSecrets, + provider: SecretsProvider | null = activeProvider, +): Promise<{ updated: SecretName[]; problems: string[] }> { + if (!provider) return { updated: [], problems: [] }; + graceMs = (cfg.JWT_PREVIOUS_SECRET_GRACE_SECS ?? 86_400) * 1000; + const updated: SecretName[] = []; + const problems: string[] = []; + let loaded: Partial>; + try { + loaded = await provider.load(SECRET_NAMES); + } catch { + return { updated, problems: ['provider refresh failed; keeping current values'] }; + } + for (const name of SECRET_NAMES) { + const value = loaded[name]; + if (value === undefined) continue; + const current = secret(name); + if (value === '' && current !== '') { + problems.push(`${name} refresh returned an empty value; keeping the previous value`); + continue; + } + if (name === 'JWT_SECRET' && value && value.length < 32) { + problems.push('JWT_SECRET refresh failed validation; keeping the previous value'); + continue; + } + if (name === 'AUTH_SIGNING_SECRET' && value && !StrKey.isValidEd25519SecretSeed(value)) { + problems.push('AUTH_SIGNING_SECRET refresh failed validation; keeping the previous value'); + continue; + } + if (setEntry(name, value, { rotate: true })) updated.push(name); + } + for (const name of SECRET_NAMES) { + const previous = store.get(name)?.previous; + if (previous && Date.now() - previous.retiredAt > graceMs) retirePrevious(name); + } + return { updated, problems }; +} + +export function startSecretRefresh(cfg: ResolvedSecrets): () => void { + const provider = activeProvider; + if (!provider) return () => undefined; + + let stopped = false; + let timer: NodeJS.Timeout | null = null; + graceMs = (cfg.JWT_PREVIOUS_SECRET_GRACE_SECS ?? 86_400) * 1000; + + const tick = async () => { + const result = await refreshSecretsOnce(cfg, provider); + for (const problem of result.problems) console.warn(`[secrets] ${problem}`); + + if (!stopped) { + const base = Math.max(30_000, cfg.SECRETS_REFRESH_MS ?? 300_000); + const jitter = Math.floor(base * 0.1 * (Math.random() * 2 - 1)); + timer = setTimeout(tick, Math.max(1_000, base + jitter)); + timer.unref?.(); + } + }; + + const base = Math.max(30_000, cfg.SECRETS_REFRESH_MS ?? 300_000); + timer = setTimeout(tick, base); + timer.unref?.(); + return () => { + stopped = true; + if (timer) clearTimeout(timer); + activeProvider = null; + }; +} + +class EnvSecretsProvider implements SecretsProvider { + readonly kind = 'env' as const; + async load(names: readonly SecretName[]): Promise>> { + const values: Partial> = {}; + for (const name of names) { + if (process.env[name] !== undefined) values[name] = process.env[name]!; + } + return values; + } +} + +class FileSecretsProvider implements SecretsProvider { + readonly kind = 'file' as const; + async load(names: readonly SecretName[]): Promise>> { + const values: Partial> = {}; + for (const name of names) { + const file = process.env[`${name}_FILE`]; + if (file) values[name] = readSecretFile(file, name); + else if (process.env[name] !== undefined) values[name] = process.env[name]!; + } + return values; + } +} + +/** Vault KV v2. Reads its token from a file, never from the environment. */ +class VaultSecretsProvider implements SecretsProvider { + readonly kind = 'vault' as const; + constructor( + private readonly address: string, + private readonly tokenFile: string, + private readonly kvPath: string, + private readonly fetchImpl: typeof fetch = fetch, + ) {} + + async load(names: readonly SecretName[]): Promise>> { + const token = fs.readFileSync(this.tokenFile, 'utf8').trim(); + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), 5_000); + try { + const response = await this.fetchImpl( + `${this.address.replace(/\/$/, '')}/v1/${this.kvPath.replace(/^\//, '')}`, + { headers: { 'X-Vault-Token': token }, signal: controller.signal }, + ); + if (!response.ok) throw new Error(`Vault responded ${response.status}`); + const body = (await response.json()) as { data?: { data?: Record } }; + const data = body.data?.data ?? {}; + const values: Partial> = {}; + for (const name of names) { + if (typeof data[name] === 'string') values[name] = data[name] as string; + } + return values; + } finally { + clearTimeout(timer); + } + } +} + +export function vaultProvider(cfg: ResolvedSecrets, fetchImpl?: typeof fetch): SecretsProvider { + if (!cfg.VAULT_ADDR || !cfg.VAULT_TOKEN_FILE || !cfg.VAULT_KV_PATH) { + throw new Error('vault provider requires VAULT_ADDR, VAULT_TOKEN_FILE and VAULT_KV_PATH'); + } + return new VaultSecretsProvider(cfg.VAULT_ADDR, cfg.VAULT_TOKEN_FILE, cfg.VAULT_KV_PATH, fetchImpl); +} + +export function envProvider(): SecretsProvider { + return new EnvSecretsProvider(); +} + +export function fileProvider(): SecretsProvider { + return new FileSecretsProvider(); +} + +/** Reads a mounted secret file with a size cap; never includes contents in errors. */ +export function readSecretFile(path: string, name: string): string { + let stat: fs.Stats; + try { + stat = fs.statSync(path); + } catch { + throw new Error(`${name}_FILE could not be read`); + } + if (!stat.isFile()) throw new Error(`${name}_FILE is not a regular file`); + if (stat.size > 64 * 1024) throw new Error(`${name}_FILE is larger than 64 KiB`); + const value = fs.readFileSync(path, 'utf8').replace(/[\r\n]+$/, ''); + if (value.trim() === '') throw new Error(`${name}_FILE is empty`); + return value; +} + +/** + * Resolves `_FILE` for every secret before Zod parses the environment. + * Synchronous so `loadConfig` stays unit-testable. + */ +export function resolveFileVars(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv { + const next: NodeJS.ProcessEnv = { ...env }; + for (const name of SECRET_NAMES) { + const fileVar = `${name}_FILE`; + if (!next[fileVar]) continue; + if (next[name]) throw new Error(`${name} and ${fileVar} cannot both be set`); + try { + next[name] = readSecretFile(next[fileVar]!, name); + } catch (err) { + // Never echo file contents, only the variable name. + throw new Error(err instanceof Error ? err.message : `${fileVar} could not be read`); + } + } + return next; +} diff --git a/src/lib/gemini.ts b/src/lib/gemini.ts index 5039149..dd84db5 100644 --- a/src/lib/gemini.ts +++ b/src/lib/gemini.ts @@ -1,9 +1,19 @@ import { GoogleGenerativeAI } from '@google/generative-ai'; -import { config } from '../config'; +import { secret, secretEntry } from '../config/secrets'; import { logger } from './logger'; -// ─── Singleton client ───────────────────────────────────────────────────────── -const genAI = new GoogleGenerativeAI(config.GEMINI_API_KEY); +// ─── Lazy client, rebuilt after key rotation (#145) ────────────────────────── +let cachedClient: GoogleGenerativeAI | null = null; +let cachedVersion = -1; + +export function getGenAI(): GoogleGenerativeAI { + const entry = secretEntry('GEMINI_API_KEY'); + if (!cachedClient || entry.version !== cachedVersion) { + cachedClient = new GoogleGenerativeAI(entry.current); + cachedVersion = entry.version; + } + return cachedClient; +} export const GEMINI_MODEL = 'gemini-2.0-flash'; @@ -96,7 +106,7 @@ export async function callGemini( await waitForCallSlot(); try { - const model = genAI.getGenerativeModel({ + const model = getGenAI().getGenerativeModel({ model: GEMINI_MODEL, systemInstruction: systemPrompt, generationConfig: { @@ -150,7 +160,7 @@ export async function callGeminiJSON( for (let attempt = 1; attempt <= 3; attempt++) { try { - const model = genAI.getGenerativeModel({ + const model = getGenAI().getGenerativeModel({ model: GEMINI_MODEL, systemInstruction: systemPrompt, generationConfig: { @@ -217,4 +227,4 @@ export async function callGeminiJSON( throw lastError; } -export default genAI; \ No newline at end of file +export default getGenAI; \ No newline at end of file diff --git a/src/lib/llm.ts b/src/lib/llm.ts index 9aaabbb..a0bc27a 100644 --- a/src/lib/llm.ts +++ b/src/lib/llm.ts @@ -19,7 +19,7 @@ */ import Anthropic from '@anthropic-ai/sdk'; -import { config } from '../config'; +import { secretEntry } from '../config/secrets'; import { logger } from './logger'; import { callGeminiJSON } from './gemini'; @@ -27,11 +27,14 @@ const CLAUDE_MODEL = 'claude-sonnet-4-6'; // ─── Lazy Anthropic client ──────────────────────────────────────────────────── let _anthropic: Anthropic | null = null; - -function getAnthropicClient(): Anthropic | null { - if (!config.ANTHROPIC_API_KEY) return null; - if (!_anthropic) { - _anthropic = new Anthropic({ apiKey: config.ANTHROPIC_API_KEY }); +let _anthropicVersion = -1; + +export function getAnthropicClient(): Anthropic | null { + const entry = secretEntry('ANTHROPIC_API_KEY'); + if (!entry.current) return null; + if (!_anthropic || entry.version !== _anthropicVersion) { + _anthropic = new Anthropic({ apiKey: entry.current }); + _anthropicVersion = entry.version; } return _anthropic; } diff --git a/src/lib/snapshot.ts b/src/lib/snapshot.ts new file mode 100644 index 0000000..c7ec294 --- /dev/null +++ b/src/lib/snapshot.ts @@ -0,0 +1,192 @@ +import crypto from 'node:crypto'; +import { Keypair } from '@stellar/stellar-sdk'; +import { DEPLOYED_AGENT } from '../config/contracts'; + +type Rule = 'keep' | 'drop' | 'null' | 'wallet' | 'hash' | 'address-map'; + +interface TableRule { + output: string; + columns: Record; +} + +export const RULES: Record = { + user: { + output: 'users', + columns: { id: 'keep', walletAddress: 'wallet', email: 'null', createdAt: 'keep', updatedAt: 'keep' }, + }, + market: { + output: 'markets', + columns: { + id: 'keep', question: 'keep', category: 'keep', status: 'keep', settlementCurrency: 'keep', + initialYesProb: 'keep', currentYesProb: 'keep', confidenceInterval: 'keep', expiryTimestamp: 'keep', + resolutionOracle: 'keep', resolvedOutcome: 'keep', resolvedAt: 'keep', minimumLiquidity: 'keep', + totalLiquidity: 'keep', onChainMarketId: 'keep', creationTxHash: 'keep', seedAmountRaw: 'keep', + questionHash: 'keep', metaUri: 'keep', resolutionHash: 'keep', resolutionSpec: 'keep', + createdAt: 'keep', updatedAt: 'keep', + }, + }, + reasoningTrace: { + output: 'reasoning_traces', + columns: { + id: 'keep', marketId: 'keep', agentType: 'keep', decisionType: 'keep', sourcesUsed: 'keep', + probabilityEstimate: 'keep', marketProbability: 'keep', edge: 'keep', confidenceInterval: 'keep', + betFraction: 'keep', betSizeUsdc: 'keep', hedgeConditions: 'keep', agentWallet: 'wallet', + signature: 'keep', ipfsCid: 'keep', traceHash: 'keep', onChainTraceId: 'keep', publishTxHash: 'keep', + verified: 'keep', isPublic: 'keep', previewSources: 'keep', createdAt: 'keep', updatedAt: 'keep', + }, + }, + trade: { + output: 'trades', + columns: { + id: 'keep', marketId: 'keep', direction: 'keep', status: 'keep', amount: 'keep', price: 'keep', + edgeDetected: 'keep', kellyFraction: 'keep', txHash: 'hash', builderFee: 'keep', errorMessage: 'keep', + executedAt: 'keep', createdAt: 'keep', updatedAt: 'keep', + }, + }, + position: { + output: 'positions', + columns: { + id: 'keep', marketId: 'keep', tradeId: 'keep', direction: 'keep', status: 'keep', entryPrice: 'keep', + currentPrice: 'keep', size: 'keep', pnl: 'keep', hedgeMarketId: 'keep', closedAt: 'keep', + closeReason: 'keep', createdAt: 'keep', updatedAt: 'keep', + }, + }, + copyTrade: { + output: 'copy_trades', + columns: { + id: 'keep', userId: 'keep', traceId: 'keep', marketId: 'keep', tradeId: 'keep', direction: 'keep', + amount: 'keep', amountRaw: 'keep', txHash: 'hash', builderFee: 'keep', status: 'keep', pnl: 'keep', + createdAt: 'keep', updatedAt: 'keep', + }, + }, + subscription: { + output: 'subscriptions', + columns: { + id: 'keep', userId: 'keep', traceId: 'keep', type: 'keep', status: 'keep', amountPaid: 'keep', + currency: 'keep', txHash: 'hash', expiresAt: 'keep', createdAt: 'keep', updatedAt: 'keep', + }, + }, + paymentEvent: { + output: 'payment_events', + columns: { + id: 'keep', userId: 'keep', txHash: 'hash', type: 'keep', amount: 'keep', amountRaw: 'keep', + fromAddress: 'wallet', currency: 'keep', status: 'keep', traceId: 'keep', metadata: 'address-map', + createdAt: 'keep', confirmedAt: 'keep', + }, + }, + spendingAllowance: { + output: 'spending_allowances', + columns: { + id: 'keep', userId: 'keep', dailyLimit: 'keep', perTraceLimit: 'keep', spentToday: 'keep', + currency: 'keep', lastResetAt: 'keep', isActive: 'keep', createdAt: 'keep', updatedAt: 'keep', + }, + }, + agentLog: { + output: 'agent_logs', + columns: { + id: 'keep', agentType: 'keep', level: 'keep', action: 'keep', marketId: 'keep', data: 'drop', + error: 'drop', createdAt: 'keep', + }, + }, + marketGenerationJob: { + output: 'market_generation_jobs', + columns: { + id: 'keep', status: 'keep', marketId: 'keep', traceId: 'keep', error: 'drop', + startedAt: 'keep', completedAt: 'keep', + }, + }, + chainEvent: { + output: 'chain_events', + columns: { + id: 'keep', contractId: 'keep', contractName: 'keep', eventName: 'keep', ledger: 'keep', + ledgerClosedAt: 'keep', txHash: 'keep', params: 'address-map', createdAt: 'keep', + }, + }, + indexerCursor: { + output: 'indexer_cursors', + columns: { id: 'keep', lastLedger: 'keep', updatedAt: 'keep' }, + }, +}; + +/** Tables intentionally not exported: raw ingestion payloads may contain source text. */ +export const EXCLUDED_TABLES = ['signal_cache'] as const; + +export class SnapshotExportError extends Error {} + +export function jsonSafe(value: unknown): string { + return JSON.stringify(value, (_key, item) => (typeof item === 'bigint' ? item.toString() : item)); +} + +export function hmac(salt: string, value: string): Buffer { + return crypto.createHmac('sha256', salt).update(value).digest(); +} + +export function pseudonymWallet(salt: string, address: string): string { + return Keypair.fromRawEd25519Seed(hmac(salt, address)).publicKey(); +} + +export function pseudonymHash(salt: string, hash: string): string { + return hmac(salt, hash).toString('hex'); +} + +export function buildWalletMap(salt: string, users: Array<{ walletAddress: string }>): Map { + return new Map(users.map((user) => [user.walletAddress, pseudonymWallet(salt, user.walletAddress)])); +} + +function mapAddresses(value: unknown, wallets: Map, salt: string): unknown { + if (typeof value === 'string') { + if (wallets.has(value)) return wallets.get(value)!; + if (/^[0-9a-f]{64}$/i.test(value)) return pseudonymHash(salt, value); + return value; + } + if (Array.isArray(value)) return value.map((item) => mapAddresses(item, wallets, salt)); + if (value && typeof value === 'object') { + return Object.fromEntries( + Object.entries(value as Record).map(([key, item]) => [ + key, + /^(from|to|trader|holder|agent|creator|wallet|address|walletAddress)$/i.test(key) && typeof item === 'string' + ? mapAddresses(item, wallets, salt) + : mapAddresses(item, wallets, salt), + ]), + ); + } + return value; +} + +export function anonymizeRow( + table: string, + row: Record, + wallets: Map, + salt: string, +): Record { + const rule = RULES[table]; + if (!rule) throw new SnapshotExportError(`No snapshot rule for table ${table}`); + const out: Record = {}; + for (const [column, value] of Object.entries(row)) { + const action = rule.columns[column]; + if (!action) { + throw new SnapshotExportError(`Unmapped column ${table}.${column}; add it to scripts/snapshot-export.ts`); + } + if (action === 'drop') continue; + if (action === 'null') out[column] = null; + else if (action === 'wallet') out[column] = wallets.get(value) ?? (value === DEPLOYED_AGENT ? value : pseudonymWallet(salt, value)); + else if (action === 'hash') out[column] = typeof value === 'string' ? pseudonymHash(salt, value) : value; + else if (action === 'address-map') out[column] = mapAddresses(value, wallets, salt); + else out[column] = value; + } + return out; +} + +const SECRET_PATTERNS = [ + /S[A-Z2-7]{55}/, + /sk-ant-/, + /AIza[0-9A-Za-z_-]{20,}/, +]; + +export function assertNoSecrets(text: string, salt: string): void { + const found = [...SECRET_PATTERNS, new RegExp(salt.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'))].some((pattern) => + pattern.test(text), + ); + if (found) throw new SnapshotExportError('Snapshot output contains a secret-looking value; refusing to keep it'); +} + diff --git a/src/middlewares/auth.middleware.ts b/src/middlewares/auth.middleware.ts index d64aad3..1750dbc 100644 --- a/src/middlewares/auth.middleware.ts +++ b/src/middlewares/auth.middleware.ts @@ -1,6 +1,7 @@ import type { Request, Response, NextFunction } from 'express'; import jwt from 'jsonwebtoken'; import { config } from '../config'; +import { previousSecret, secret, secretEntry } from '../config/secrets'; import { sendError } from '../utils/response.util'; export interface JwtPayload { @@ -22,10 +23,21 @@ declare global { function decode(req: Request): { user?: JwtPayload; error?: 'missing' | 'expired' | 'invalid' } { const authHeader = req.headers.authorization; if (!authHeader?.startsWith('Bearer ')) return { error: 'missing' }; + const token = authHeader.slice(7); try { - return { user: jwt.verify(authHeader.slice(7), config.JWT_SECRET) as JwtPayload }; + return { user: jwt.verify(token, secret('JWT_SECRET')) as JwtPayload }; } catch (err) { - return { error: err instanceof jwt.TokenExpiredError ? 'expired' : 'invalid' }; + // Expiry is a property of the token, not the key. Never retry it with the + // previous secret: that would turn an expired token into a valid one. + if (err instanceof jwt.TokenExpiredError) return { error: 'expired' }; + if (!(err instanceof jwt.JsonWebTokenError)) return { error: 'invalid' }; + const previous = previousSecret('JWT_SECRET'); + if (!previous) return { error: 'invalid' }; + try { + return { user: jwt.verify(token, previous) as JwtPayload }; + } catch { + return { error: 'invalid' }; + } } } @@ -67,5 +79,5 @@ export function requireAdmin(req: Request, res: Response, next: NextFunction): v * Generate a signed JWT for a user */ export function signToken(userId: string, walletAddress: string): string { - return jwt.sign({ userId, walletAddress }, config.JWT_SECRET, { expiresIn: '7d' }); + return jwt.sign({ userId, walletAddress }, secret('JWT_SECRET'), { expiresIn: '7d' }); } diff --git a/src/seed/raw-events.ts b/src/seed/raw-events.ts new file mode 100644 index 0000000..7a7edf4 --- /dev/null +++ b/src/seed/raw-events.ts @@ -0,0 +1,96 @@ +/** + * Encodes seeded chain_events rows as the `getEvents` JSON shape used by + * tests/fixtures/events.json, so `backfillEvents()` can rebuild the same + * projections from bytes rather than trusting the database (#147). + */ +import { Address, nativeToScVal, xdr } from '@stellar/stellar-sdk'; + +const sym = (value: string) => xdr.ScVal.scvSymbol(value); +const u64 = (value: bigint | string) => nativeToScVal(BigInt(value), { type: 'u64' }); +const u32 = (value: number) => nativeToScVal(value, { type: 'u32' }); +const i128 = (value: bigint | string) => nativeToScVal(BigInt(value), { type: 'i128' }); +const address = (value: string) => Address.fromString(value).toScVal(); +const str = (value: string) => nativeToScVal(value, { type: 'string' }); +const bytes = (hex: string) => nativeToScVal(Buffer.from(hex, 'hex'), { type: 'bytes' }); + +function map(entries: Record): xdr.ScVal { + return xdr.ScVal.scvMap( + Object.entries(entries).map(([key, val]) => new xdr.ScMapEntry({ key: sym(key), val })), + ); +} + +/** Contract enum values (`Outcome`, `Category`) encode as vec[symbol]. */ +const enumValue = (tag: string) => xdr.ScVal.scvVec([sym(tag)]); + +export function encodeSeedEvent(event: any): any { + const p = event.params ?? {}; + let topics: xdr.ScVal[]; + let value: xdr.ScVal; + + switch (event.eventName) { + case 'Trade': + case 'trade': + topics = [sym('trade'), u64(p.market_id), address(p.trader)]; + value = map({ + outcome: enumValue(p.outcome?.tag ?? 'Yes'), + is_buy: xdr.ScVal.scvBool(Boolean(p.is_buy)), + collateral: i128(p.collateral), + shares: i128(p.shares), + fee: i128(p.fee), + reserve_yes: i128(p.reserve_yes), + reserve_no: i128(p.reserve_no), + }); + break; + case 'MarketCreated': + case 'market_created': + topics = [sym('market_created'), u64(p.market_id)]; + value = map({ + creator: address(p.creator), + category: enumValue(p.category?.tag ?? 'Macro'), + close_time: u64(p.close_time), + seed_amount: i128(p.seed_amount), + initial_yes_bps: u32(Number(p.initial_yes_bps)), + }); + break; + case 'MarketResolved': + case 'market_resolved': + topics = [sym('market_resolved'), u64(p.market_id)]; + value = map({ + outcome: p.outcome?.tag ? enumValue(p.outcome.tag) : xdr.ScVal.scvVoid(), + }); + break; + case 'TracePublished': + case 'trace_published': + topics = [sym('trace_published'), u64(p.trace_id), u64(p.market_id), address(p.agent)]; + value = map({ + action: str(String(p.action)), + trace_hash: bytes(String(p.trace_hash)), + ipfs_cid: str(String(p.ipfs_cid)), + published_at: u64(p.published_at), + }); + break; + default: + throw new Error(`No raw-event encoder for ${event.eventName}`); + } + + return { + type: 'contract', + ledgerClosedAt: new Date(event.ledgerClosedAt).toISOString(), + transactionIndex: 1, + operationIndex: 0, + inSuccessfulContractCall: true, + id: event.id, + ledger: event.ledger, + txHash: event.txHash, + contractId: event.contractId, + topic: topics.map((topic) => topic.toXDR('base64')), + value: value.toXDR('base64'), + }; +} + +export function encodeSeedEvents(events: any[]): { _comment: string; events: any[] } { + return { + _comment: 'Generated by `npm run db:seed -- --with-raw-events`. Deterministic; do not commit secrets.', + events: events.map(encodeSeedEvent), + }; +} diff --git a/src/seed/rng.ts b/src/seed/rng.ts new file mode 100644 index 0000000..bfe0e79 --- /dev/null +++ b/src/seed/rng.ts @@ -0,0 +1,35 @@ +import crypto from 'crypto'; + +/** mulberry32: tiny deterministic PRNG for reproducible seed data. */ +export function mulberry32(seed: number): () => number { + let state = seed >>> 0; + return () => { + state = (state + 0x6d2b79f5) >>> 0; + let t = state; + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4_294_967_296; + }; +} + +export function int(rng: () => number, min: number, max: number): number { + return min + Math.floor(rng() * (max - min + 1)); +} + +export function pick(rng: () => number, values: readonly T[]): T { + return values[Math.floor(rng() * values.length)]; +} + +/** v4-shaped UUID from the deterministic stream. */ +export function uuidFrom(rng: () => number): string { + const bytes = Buffer.alloc(16); + for (let i = 0; i < 16; i++) bytes[i] = int(rng, 0, 255); + bytes[6] = (bytes[6] & 0x0f) | 0x40; + bytes[8] = (bytes[8] & 0x3f) | 0x80; + const hex = bytes.toString('hex'); + return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`; +} + +export function seedHash(label: string): string { + return crypto.createHash('sha256').update(label).digest('hex'); +} diff --git a/src/seed/scenario.ts b/src/seed/scenario.ts new file mode 100644 index 0000000..3fe2a9a --- /dev/null +++ b/src/seed/scenario.ts @@ -0,0 +1,489 @@ +/** + * Deterministic, internally consistent seed dataset (#147). + * + * Every generated hash is computed with the production helpers: + * questionHash = sha256(canonical questionDocument) + * resolutionHash = sha256(XDR(ResolutionSpec)) + * traceHash = sha256(canonical trace payload) + * and the pinned bytes are available from the local IPFS stub because the + * CID is CIDv1(raw, sha2-256) over exactly those bytes. + */ +import crypto from 'crypto'; +import { Keypair } from '@stellar/stellar-sdk'; +import { canonicalBytes, sha256Hex } from '../utils/hash.util'; +import { resolutionHash, type ResolutionSpecJson } from '../lib/resolution-spec'; +import { questionDocument } from '../services/market-chain.service'; +import { cidForBytes } from '../services/ipfs.service'; +import { calcBuy, priceYesBps, seedReserves } from '../generated/fpmm'; +import { CONTRACTS, DEPLOYED_AGENT } from '../config/contracts'; +import { config } from '../config'; +import { CURSOR_ID } from '../services/indexer.service'; +import { int, mulberry32, pick, seedHash, uuidFrom } from './rng'; + +export const SEED_EPOCH = Date.UTC(2026, 0, 1, 0, 0, 0); +const USDC_UNIT = 10_000_000n; +const ONCHAIN_BASE = BigInt(process.env.SEED_ONCHAIN_BASE ?? '1000000'); +const SEED_BASE_LEDGER = Number(process.env.SEED_BASE_LEDGER ?? '6000000'); + +const CATEGORIES = [ + 'FED', + 'ECB', + 'ELECTION', + 'GEOPOLITICAL', + 'CRYPTO', + 'MACRO', + 'SPORTS', + 'ENTERTAINMENT', + 'POLITICS', +] as const; + +const STATUSES = ['PENDING', 'ACTIVE', 'RESOLVING', 'RESOLVED', 'CANCELLED'] as const; + +const SOURCES = ['NEWSAPI', 'FRED', 'GDELT', 'ANTHROPIC']; + +export interface SeedOptions { + seed?: number; + now?: boolean; + marketCount?: number; +} + +export interface SeedData { + users: any[]; + markets: any[]; + traces: any[]; + trades: any[]; + positions: any[]; + copyTrades: any[]; + subscriptions: any[]; + paymentEvents: any[]; + spendingAllowances: any[]; + chainEvents: any[]; + indexerCursors: any[]; + agentLogs: any[]; + signalCache: any[]; + marketGenerationJobs: any[]; +} + +function walletFor(label: string): Keypair { + return Keypair.fromRawEd25519Seed(crypto.createHash('sha256').update(label).digest()); +} + +export function buildSeedData(options: SeedOptions = {}): SeedData { + const rng = mulberry32(options.seed ?? 20260101); + const base = options.now ? Date.now() : SEED_EPOCH; + const marketCount = options.marketCount ?? 45; + const agent = walletFor('oracledesk-seed-agent'); + const agentWallet = agent.publicKey(); + + const users = Array.from({ length: 12 }, (_, i) => { + const wallet = walletFor(`oracledesk-seed-user-${i}`); + return { + id: `seed-user-${i}`, + walletAddress: wallet.publicKey(), + email: i % 3 === 0 ? `seed-user-${i}@example.test` : null, + createdAt: new Date(base - 90 * 86_400_000 + i * 3_600_000), + updatedAt: new Date(base - 90 * 86_400_000 + i * 3_600_000), + }; + }); + + const spec: ResolutionSpecJson = { + kind: 'signers', + signers: [agentWallet], + threshold: 1, + disputeWindow: 86_400, + }; + const specHash = resolutionHash(spec).toString('hex'); + + const markets: any[] = []; + const traces: any[] = []; + const trades: any[] = []; + const positions: any[] = []; + const chainEvents: any[] = []; + let ledger = SEED_BASE_LEDGER; + let tradeCounter = 0; + let traceOnchainCounter = 0; + + const eventId = () => `${String(ledger).padStart(10, '0')}-${String(chainEvents.length + 1).padStart(10, '0')}`; + + for (let i = 0; i < marketCount; i++) { + const category = CATEGORIES[i % CATEGORIES.length]; + const status = STATUSES[i % STATUSES.length]; + const createdAt = new Date(base - (marketCount - i) * 86_400_000 - int(rng, 0, 20) * 3_600_000); + const expiry = ['RESOLVED', 'CANCELLED'].includes(status) + ? new Date(base - (i % 10 + 1) * 86_400_000) + : new Date(base + (i % 25 + 5) * 86_400_000); + const initialYesProb = Number((0.25 + rng() * 0.5).toFixed(4)); + const resolved = status === 'RESOLVED'; + const onChainMarketId = status === 'PENDING' ? null : ONCHAIN_BASE + BigInt(i); + const seedRaw = 250n * USDC_UNIT + BigInt(int(rng, 0, 20)) * USDC_UNIT; + const marketId = `seed-market-${i}`; + + const market: any = { + id: marketId, + question: `Will ${category.toLowerCase()} indicator ${i + 1} move above its ${initialYesProb > 0.5 ? 'upper' : 'lower'} band by expiry?`, + category, + status, + settlementCurrency: 'USDC', + initialYesProb, + currentYesProb: status === 'PENDING' ? null : initialYesProb, + confidenceInterval: { + lower: Number(Math.max(0.01, initialYesProb - 0.12).toFixed(4)), + upper: Number(Math.min(0.99, initialYesProb + 0.12).toFixed(4)), + }, + expiryTimestamp: expiry, + resolutionOracle: 'oracledesk-resolver', + resolvedOutcome: resolved ? rng() > 0.5 : null, + resolvedAt: resolved ? new Date(expiry.getTime() + 3_600_000) : null, + minimumLiquidity: 100, + totalLiquidity: Number(seedRaw) / 1e7, + onChainMarketId, + creationTxHash: sha256Hex(`seed-market-created-${i}`), + seedAmountRaw: seedRaw.toString(), + questionHash: '', + metaUri: '', + resolutionHash: specHash, + resolutionSpec: spec, + createdAt, + updatedAt: new Date(base - 60_000), + }; + + const doc = questionDocument(market as any, specHash); + const docBytes = canonicalBytes(doc); + market.questionHash = sha256Hex(docBytes); + market.metaUri = `ipfs://${cidForBytes(docBytes)}`; + markets.push(market); + + if (onChainMarketId !== null) { + ledger += 2; + chainEvents.push({ + id: eventId(), + contractId: CONTRACTS.marketCore, + contractName: 'marketCore', + eventName: 'MarketCreated', + ledger, + ledgerClosedAt: new Date(createdAt.getTime() + 60_000), + txHash: market.creationTxHash, + params: { + market_id: onChainMarketId.toString(), + creator: agentWallet, + category: { tag: doc.contractCategory, values: undefined }, + close_time: Math.floor(expiry.getTime() / 1000).toString(), + seed_amount: seedRaw.toString(), + initial_yes_bps: Math.min(9900, Math.max(100, Math.round(initialYesProb * 10_000))), + }, + }); + } + + // Trades and positions for markets that are (or were) open. + let reserveYes = 0n; + let reserveNo = 0n; + if (['ACTIVE', 'RESOLVING', 'RESOLVED', 'CANCELLED'].includes(status)) { + const seeded = seedReserves(seedRaw, BigInt(Math.min(9900, Math.max(100, Math.round(initialYesProb * 10_000))))); + reserveYes = seeded.reserveYes; + reserveNo = seeded.reserveNo; + const tradeCount = status === 'ACTIVE' ? int(rng, 1, 3) : 1; + for (let t = 0; t < tradeCount; t++) { + const yes = rng() >= 0.5; + const collateral = (2n + BigInt(int(rng, 0, 6))) * USDC_UNIT; + const quote = yes + ? calcBuy(reserveYes, reserveNo, collateral, 100n) + : calcBuy(reserveNo, reserveYes, collateral, 100n); + if (yes) { + reserveYes = quote.newReserveBought; + reserveNo = quote.newReserveOther; + } else { + reserveNo = quote.newReserveBought; + reserveYes = quote.newReserveOther; + } + const tradeId = `seed-trade-${tradeCounter++}`; + const txHash = sha256Hex(`seed-trade-tx-${tradeId}`); + const executedAt = new Date(createdAt.getTime() + (t + 1) * 3_600_000); + trades.push({ + id: tradeId, + marketId, + direction: yes ? 'YES' : 'NO', + status: 'EXECUTED', + amount: Number(collateral) / 1e7, + price: Number(priceYesBps(reserveYes, reserveNo)) / 10_000, + edgeDetected: Number((rng() * 0.12 - 0.02).toFixed(4)), + kellyFraction: Number((rng() * 0.2).toFixed(4)), + txHash, + builderFee: 0, + executedAt, + createdAt: executedAt, + updatedAt: executedAt, + }); + positions.push({ + id: `seed-position-${tradeId}`, + marketId, + tradeId, + direction: yes ? 'YES' : 'NO', + status: status === 'ACTIVE' || status === 'RESOLVING' ? 'OPEN' : 'CLOSED', + entryPrice: Number(priceYesBps(reserveYes, reserveNo)) / 10_000, + currentPrice: Number(priceYesBps(reserveYes, reserveNo)) / 10_000, + size: Number(quote.sharesOut) / 1e7, + pnl: 0, + closedAt: status === 'ACTIVE' || status === 'RESOLVING' ? null : new Date(expiry.getTime() + 3_600_000), + closeReason: status === 'CANCELLED' ? 'VOIDED' : resolved ? 'YES_RESOLVED' : null, + createdAt: executedAt, + updatedAt: executedAt, + }); + + ledger += 1; + chainEvents.push({ + id: eventId(), + contractId: CONTRACTS.marketCore, + contractName: 'marketCore', + eventName: 'Trade', + ledger, + ledgerClosedAt: executedAt, + txHash, + params: { + market_id: onChainMarketId!.toString(), + trader: agentWallet, + outcome: { tag: yes ? 'Yes' : 'No' }, + is_buy: true, + collateral: collateral.toString(), + shares: quote.sharesOut.toString(), + fee: quote.fee.toString(), + reserve_yes: reserveYes.toString(), + reserve_no: reserveNo.toString(), + }, + }); + } + market.currentYesProb = Number(priceYesBps(reserveYes, reserveNo)) / 10_000; + market.totalLiquidity = Number(reserveYes + reserveNo) / 1e7; + } + + const traceCount = int(rng, 1, 4); + for (let j = 0; j < traceCount; j++) { + const agentType = j % 2 === 0 ? 'MARKET_MAKER' : 'TRADER'; + const marketProbability = market.currentYesProb ?? initialYesProb; + const probabilityEstimate = Number(Math.min(0.98, Math.max(0.02, marketProbability + (rng() * 0.16 - 0.08))).toFixed(4)); + const traceId = `seed-trace-${i}-${j}`; + const published = j === 0 && onChainMarketId !== null; + const createdAtTrace = new Date(createdAt.getTime() + (j + 1) * 600_000); + const sourcesUsed = Array.from({ length: 3 }, (_, s) => ({ + source: SOURCES[(i + j + s) % SOURCES.length], + weight: Number((0.2 + rng() * 0.3).toFixed(3)), + signal: `${SOURCES[(i + j + s) % SOURCES.length].toLowerCase()} signal ${i}-${j}-${s}`, + })); + const payload: Record = { + agentType, + decisionType: j === 0 ? 'CREATE_MARKET' : 'TRADE', + marketId, + sourcesUsed, + probabilityEstimate, + marketProbability, + edge: Number((probabilityEstimate - marketProbability).toFixed(4)), + confidenceInterval: { + lower: Number(Math.max(0.01, probabilityEstimate - 0.1).toFixed(4)), + upper: Number(Math.min(0.99, probabilityEstimate + 0.1).toFixed(4)), + }, + ...(agentType === 'TRADER' ? { betFraction: Number((rng() * 0.2).toFixed(4)), betSizeUsdc: Number((rng() * 50).toFixed(2)) } : {}), + hedgeConditions: ['resolution source disagrees with market price'], + agentWallet, + timestamp: createdAtTrace.toISOString(), + traceId, + }; + const bytes = canonicalBytes(payload); + traces.push({ + id: traceId, + marketId, + agentType, + decisionType: payload.decisionType, + sourcesUsed, + probabilityEstimate, + marketProbability, + edge: payload.edge, + confidenceInterval: payload.confidenceInterval, + betFraction: payload.betFraction ?? null, + betSizeUsdc: payload.betSizeUsdc ?? null, + hedgeConditions: payload.hedgeConditions, + agentWallet, + ipfsCid: cidForBytes(bytes), + traceHash: sha256Hex(bytes), + onChainTraceId: published ? ONCHAIN_BASE + 10_000n + BigInt(traceOnchainCounter++) : null, + publishTxHash: published ? sha256Hex(`seed-trace-publish-${traceId}`) : null, + verified: published, + isPublic: published, + previewSources: sourcesUsed.slice(0, 2), + createdAt: createdAtTrace, + updatedAt: createdAtTrace, + }); + + if (published) { + ledger += 1; + chainEvents.push({ + id: eventId(), + contractId: CONTRACTS.reasoningRegistry, + contractName: 'reasoningRegistry', + eventName: 'TracePublished', + ledger, + ledgerClosedAt: createdAtTrace, + txHash: sha256Hex(`seed-trace-publish-${traceId}`), + params: { + trace_id: (ONCHAIN_BASE + 10_000n + BigInt(traceOnchainCounter - 1)).toString(), + market_id: onChainMarketId!.toString(), + agent: agentWallet, + action: payload.decisionType, + trace_hash: sha256Hex(bytes), + ipfs_cid: cidForBytes(bytes), + published_at: Math.floor(createdAtTrace.getTime() / 1000).toString(), + }, + }); + } + } + + if (status === 'RESOLVED' || status === 'CANCELLED') { + ledger += 1; + chainEvents.push({ + id: eventId(), + contractId: CONTRACTS.marketCore, + contractName: 'marketCore', + eventName: 'MarketResolved', + ledger, + ledgerClosedAt: market.resolvedAt ?? expiry, + txHash: sha256Hex(`seed-market-resolved-${i}`), + params: { + market_id: onChainMarketId!.toString(), + outcome: status === 'CANCELLED' ? null : { tag: market.resolvedOutcome ? 'Yes' : 'No' }, + }, + }); + } + } + + // Subscriptions and payments across every access path. + const subscriptions: any[] = []; + const paymentEvents: any[] = []; + const spendingAllowances: any[] = []; + const statuses = ['ACTIVE', 'EXPIRED', 'LIMIT_REACHED'] as const; + users.forEach((user, i) => { + const status = statuses[i % statuses.length]; + const txHash = sha256Hex(`seed-subscription-tx-${i}`); + const createdAt = new Date(base - (20 - i) * 86_400_000); + const expiresAt = status === 'ACTIVE' ? new Date(base + 12 * 3_600_000) : new Date(base - 86_400_000); + const trace = traces[i % traces.length]; + subscriptions.push({ + id: `seed-subscription-${i}`, + userId: user.id, + traceId: i % 4 === 0 ? null : trace.id, + type: 'DAILY_PASS', + status, + amountPaid: 0.5, + currency: 'USDC', + txHash, + expiresAt, + createdAt, + updatedAt: createdAt, + }); + paymentEvents.push({ + id: `seed-payment-${i}`, + userId: user.id, + txHash, + type: 'DAILY_PASS', + amount: 0.5, + amountRaw: '5000000', + fromAddress: user.walletAddress, + currency: 'USDC', + status: 'CONFIRMED', + traceId: null, + metadata: { subscriptionId: `seed-subscription-${i}`, ledger: SEED_BASE_LEDGER + i }, + createdAt, + confirmedAt: createdAt, + }); + spendingAllowances.push({ + id: `seed-allowance-${i}`, + userId: user.id, + dailyLimit: 5, + perTraceLimit: 0.5, + spentToday: 0.5, + currency: 'USDC', + lastResetAt: new Date(base - 3_600_000), + isActive: true, + createdAt, + updatedAt: createdAt, + }); + }); + + const copyTrades = traces.slice(0, 8).map((trace, i) => ({ + id: `seed-copy-${i}`, + userId: users[i % users.length].id, + traceId: trace.id, + marketId: trace.marketId, + tradeId: trades[i % Math.max(1, trades.length)]?.id ?? null, + direction: i % 2 === 0 ? 'YES' : 'NO', + amount: 5, + amountRaw: '50000000', + txHash: sha256Hex(`seed-copy-tx-${i}`), + builderFee: 0.05, + status: 'EXECUTED', + pnl: 0, + createdAt: new Date(base - i * 3_600_000), + updatedAt: new Date(base - i * 3_600_000), + })); + + const marketGenerationJobs = markets.slice(0, 5).map((market, i) => ({ + id: `seed-generation-job-${i}`, + status: 'COMPLETED', + marketId: market.id, + traceId: traces.find((trace) => trace.marketId === market.id)?.id ?? null, + error: null, + startedAt: new Date(market.createdAt.getTime() - 600_000), + completedAt: market.createdAt, + })); + + const agentLogs = Array.from({ length: 12 }, (_, i) => ({ + id: `seed-log-${i}`, + agentType: i % 2 === 0 ? 'MARKET_MAKER' : 'TRADER', + level: pick(rng, ['INFO', 'WARN', 'DEBUG', 'ERROR'] as const), + action: i % 3 === 0 ? 'market_generation' : 'trace_generation', + marketId: markets[i % markets.length].id, + data: { seed: true, index: i }, + error: null, + createdAt: new Date(base - i * 1_800_000), + })); + + const signalCache = [ + { + id: 'seed-signal-0', + source: 'NEWSAPI', + payload: [{ source: 'NEWSAPI', title: 'Seed signal', content: 'Deterministic fixture', publishedAt: new Date(base).toISOString(), relevanceScore: 0.7, category: 'FED' }], + fetchedAt: new Date(base - 3_600_000), + expiresAt: new Date(base + 3_600_000), + }, + { + id: 'seed-signal-1', + source: 'FRED', + payload: [{ source: 'FRED', title: 'FEDFUNDS: 4.25', content: 'Seed macro observation', publishedAt: new Date(base).toISOString(), relevanceScore: 0.9, category: 'FED' }], + fetchedAt: new Date(base - 7_200_000), + expiresAt: new Date(base + 3_600_000), + }, + ]; + + const maxLedger = chainEvents.reduce((max, event) => Math.max(max, event.ledger), SEED_BASE_LEDGER); + + return { + users, + markets, + traces, + trades, + positions, + copyTrades, + subscriptions, + paymentEvents, + spendingAllowances, + chainEvents, + indexerCursors: [{ id: CURSOR_ID, lastLedger: maxLedger }], + agentLogs, + signalCache, + marketGenerationJobs, + }; +} + +/** CIDv1 raw leaves for exactly the bytes the seed hashed. */ +export function seedCid(bytes: Buffer): string { + return cidForBytes(bytes); +} + +export { ONCHAIN_BASE, SEED_BASE_LEDGER, USDC_UNIT, walletFor, DEPLOYED_AGENT, config }; diff --git a/src/server.ts b/src/server.ts index 2a73c5d..997f147 100644 --- a/src/server.ts +++ b/src/server.ts @@ -2,6 +2,7 @@ import http from 'http'; import { Server } from 'socket.io'; import app from './app'; import { config } from './config'; +import { initSecrets, startSecretRefresh } from './config/secrets'; import { logger } from './lib/logger'; import { prisma } from './lib/prisma'; import { startIngestionCron } from './cron/ingestion.cron'; @@ -10,8 +11,18 @@ import { startEventListener } from './services/indexer.service'; import { closeRedis } from './lib/redis'; let stopEventListener: (() => void) | null = null; +let stopSecretRefresh: (() => void) | null = null; async function bootstrap(): Promise { + // 0. Resolve secrets (mounted files or the configured remote provider). + try { + await initSecrets(config); + stopSecretRefresh = startSecretRefresh(config); + } catch (err) { + logger.fatal({ err: err instanceof Error ? err.message : err }, 'Failed to initialise secrets'); + process.exit(1); + } + // 1. Connect to database try { await prisma.$connect(); @@ -69,6 +80,7 @@ async function bootstrap(): Promise { server.close(async () => { if (stopEventListener) stopEventListener(); + if (stopSecretRefresh) stopSecretRefresh(); await prisma.$disconnect(); await closeRedis(); diff --git a/src/services/auth.service.ts b/src/services/auth.service.ts index 7ba6117..606a11c 100644 --- a/src/services/auth.service.ts +++ b/src/services/auth.service.ts @@ -8,6 +8,7 @@ */ import { Keypair, StrKey, TransactionBuilder, WebAuth } from '@stellar/stellar-sdk'; import { config } from '../config'; +import { secret } from '../config/secrets'; import { prisma } from '../lib/prisma'; import { getRedis } from '../lib/redis'; import { AppError } from '../middlewares/error.middleware'; @@ -50,7 +51,7 @@ export function setChallengeStore(next: ChallengeStore): ChallengeStore { return prev; } -const serverKeypair = () => Keypair.fromSecret(config.AUTH_SIGNING_SECRET); +const serverKeypair = () => Keypair.fromSecret(secret('AUTH_SIGNING_SECRET')); export function assertStellarAccount(address: unknown): asserts address is string { if (typeof address !== 'string' || !StrKey.isValidEd25519PublicKey(address)) { diff --git a/src/services/indexer.service.ts b/src/services/indexer.service.ts index 674a393..167ad2d 100644 --- a/src/services/indexer.service.ts +++ b/src/services/indexer.service.ts @@ -151,7 +151,10 @@ async function applySideEffects(event: DecodedEvent, emit: IndexerDeps['emit']): } if (event.contractName === 'reasoningRegistry' && event.eventName === 'TracePublished') { - const traceHash = Buffer.from(p.trace_hash).toString('hex'); + const traceHash = + typeof p.trace_hash === 'string' + ? p.trace_hash + : Buffer.from(p.trace_hash).toString('hex'); const onChainTraceId = BigInt(p.trace_id); const trace = await prisma.reasoningTrace.findFirst({ where: { traceHash }, select: { id: true, onChainTraceId: true } }); if (trace && trace.onChainTraceId === null) { @@ -239,6 +242,41 @@ export async function backfillEvents( return { startLedger, lastLedger: latestLedger, seen, stored }; } +/** + * Re-applies one stored event's projections (#146). Unlike the normal path it + * does not insert the event first, so a restore can rebuild projections for + * rows that survived the backup. Callers replay in (ledger, id) order. + */ +export async function reprojectEvent( + event: DecodedEvent, + emit: IndexerDeps['emit'] = () => undefined, +): Promise { + await applySideEffects(event, emit); +} + +/** Converts a stored row back into the decoded shape used by applySideEffects. */ +export function decodedFromStored(row: { + id: string; + contractId: string; + contractName: string; + eventName: string; + ledger: number; + ledgerClosedAt: Date; + txHash: string; + params: unknown; +}): DecodedEvent { + return { + id: row.id, + contractId: row.contractId, + contractName: row.contractName as ContractName, + eventName: row.eventName, + ledger: row.ledger, + ledgerClosedAt: row.ledgerClosedAt.toISOString(), + txHash: row.txHash, + params: row.params as Record, + }; +} + /** Polls every INDEXER_POLL_MS. Returns a stop function. */ export function startEventListener(emit: IndexerDeps['emit']): () => void { if (!config.INDEXER_ENABLED) { diff --git a/src/services/ingestion.service.ts b/src/services/ingestion.service.ts index a34d7fe..c3fba16 100644 --- a/src/services/ingestion.service.ts +++ b/src/services/ingestion.service.ts @@ -2,6 +2,7 @@ import axios from 'axios'; import { prisma } from '../lib/prisma'; import { logger } from '../lib/logger'; import { config } from '../config'; +import { secret } from '../config/secrets'; import type { RawSignal, AggregatedSignals } from '../types'; import dayjs from 'dayjs'; @@ -47,7 +48,8 @@ export async function fetchNewsSignals(): Promise { return cached; } - if (!config.NEWSAPI_KEY) { + const newsApiKey = secret('NEWSAPI_KEY'); + if (!newsApiKey) { logger.warn('NEWSAPI_KEY not set — skipping news ingestion'); return []; } @@ -68,7 +70,7 @@ export async function fetchNewsSignals(): Promise { language: 'en', sortBy: 'publishedAt', pageSize: 30, - apiKey: config.NEWSAPI_KEY, + apiKey: newsApiKey, }, timeout: 10_000, }, @@ -107,10 +109,20 @@ export async function fetchFedData(): Promise { for (const seriesId of series) { try { - const apiKeyParam = config.FRED_API_KEY ? `&api_key=${config.FRED_API_KEY}` : ''; - const url = `https://api.stlouisfed.org/fred/series/observations?series_id=${seriesId}&file_type=json&sort_order=desc&limit=2${apiKeyParam}`; - - const { data } = await axios.get<{ observations: any[] }>(url, { timeout: 10_000 }); + const fredApiKey = secret('FRED_API_KEY'); + const { data } = await axios.get<{ observations: any[] }>( + 'https://api.stlouisfed.org/fred/series/observations', + { + params: { + series_id: seriesId, + file_type: 'json', + sort_order: 'desc', + limit: 2, + ...(fredApiKey ? { api_key: fredApiKey } : {}), + }, + timeout: 10_000, + }, + ); const observations = data.observations ?? []; if (observations.length > 0) { diff --git a/src/services/ipfs.service.ts b/src/services/ipfs.service.ts index cef5860..51d6b95 100644 --- a/src/services/ipfs.service.ts +++ b/src/services/ipfs.service.ts @@ -1,11 +1,45 @@ import axios from 'axios'; +import crypto from 'crypto'; +import fs from 'fs'; +import path from 'path'; import { config } from '../config'; +import { secret } from '../config/secrets'; import { logger } from '../lib/logger'; import { AppError } from '../middlewares/error.middleware'; import { canonicalBytes, sha256Hex } from '../utils/hash.util'; const PINATA_BASE = 'https://api.pinata.cloud'; +/** RFC 4648 base32 (lowercase, no padding) used by CIDv1 multibase 'b'. */ +const BASE32 = 'abcdefghijklmnopqrstuvwxyz234567'; + +export function base32Encode(bytes: Uint8Array): string { + let bits = 0; + let value = 0; + let output = ''; + for (const byte of bytes) { + value = (value << 8) | byte; + bits += 8; + while (bits >= 5) { + output += BASE32[(value >>> (bits - 5)) & 31]; + bits -= 5; + } + } + if (bits > 0) output += BASE32[(value << (5 - bits)) & 31]; + return output; +} + +/** + * CIDv1(raw, sha2-256). This is the real value Pinata returns for a file + * pinned with `cidVersion: 1`; no dependency needed. + */ +export function cidForBytes(bytes: Buffer): string { + const digest = crypto.createHash('sha256').update(bytes).digest(); + const multihash = Buffer.concat([Buffer.from([0x12, 0x20]), digest]); + const cidBytes = Buffer.concat([Buffer.from([0x01, 0x55]), multihash]); + return `b${base32Encode(cidBytes)}`; +} + /** * Where JSON documents are pinned. The default talks to Pinata; tests inject * an in-memory implementation with `setIpfsTransport`. @@ -19,7 +53,9 @@ export interface IpfsTransport { export const pinataTransport: IpfsTransport = { async pin(bytes, name) { - if (!config.PINATA_API_KEY || !config.PINATA_SECRET_API_KEY) { + const apiKey = secret('PINATA_API_KEY'); + const secretKey = secret('PINATA_SECRET_API_KEY'); + if (!apiKey || !secretKey) { throw new AppError(503, 'IPFS_NOT_CONFIGURED', 'PINATA_API_KEY and PINATA_SECRET_API_KEY are not set'); } // pinFileToIPFS stores the file's bytes verbatim. pinJSONToIPFS would @@ -30,8 +66,8 @@ export const pinataTransport: IpfsTransport = { form.append('pinataOptions', JSON.stringify({ cidVersion: 1 })); const { data } = await axios.post<{ IpfsHash: string }>(`${PINATA_BASE}/pinning/pinFileToIPFS`, form, { headers: { - pinata_api_key: config.PINATA_API_KEY, - pinata_secret_api_key: config.PINATA_SECRET_API_KEY, + pinata_api_key: apiKey, + pinata_secret_api_key: secretKey, }, timeout: 20_000, }); @@ -54,7 +90,27 @@ export const pinataTransport: IpfsTransport = { }, }; -let transport: IpfsTransport = pinataTransport; +/** Development-only transport: stores bytes on disk under `/`. */ +export function localIpfsTransport(dir: string): IpfsTransport { + return { + async pin(bytes) { + const cid = cidForBytes(bytes); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(path.join(dir, cid), bytes); + return cid; + }, + async fetch(cid) { + try { + return fs.readFileSync(path.join(dir, cid)); + } catch { + throw new AppError(404, 'IPFS_NOT_FOUND', 'Content is not in the local IPFS store', { cid }); + } + }, + }; +} + +let transport: IpfsTransport = + config.IPFS_TRANSPORT === 'local' ? localIpfsTransport(config.IPFS_LOCAL_DIR) : pinataTransport; /** Test seam. Returns the previous transport. */ export function setIpfsTransport(next: IpfsTransport): IpfsTransport { diff --git a/src/services/reconcile.service.ts b/src/services/reconcile.service.ts new file mode 100644 index 0000000..3301731 --- /dev/null +++ b/src/services/reconcile.service.ts @@ -0,0 +1,220 @@ +/** + * Post-restore reconciliation core (#146). The CLI wrapper is + * scripts/post-restore-reconcile.ts; keeping the logic here lets tests import + * it without spawning a process. + */ +import fs from 'node:fs'; +import type { PrismaClient } from '@prisma/client'; +import { Contract, scValToNative, xdr } from '@stellar/stellar-sdk'; +import { CONTRACTS, PAYMENTS_RECIPIENT } from '../config/contracts'; +import { config } from '../config'; +import { resolutionHash, type ResolutionSpecJson } from '../lib/resolution-spec'; +import { + CURSOR_ID, + backfillEvents, + decodedFromStored, + reprojectEvent, + type EventSource, +} from './indexer.service'; +import type { verifyUsdcTransfer } from './payment-verification.service'; + +export const REWIND_LEDGERS_DEFAULT = 720; + +export interface BackupManifest { + createdAt?: string; + gitSha?: string; + migrations?: string[]; + cursors?: Array<{ id: string; lastLedger: number }>; + maxChainEventLedger?: number | null; + maxPaymentEventCreatedAt?: string | null; + rowCounts?: Record; +} + +export interface ReconcileReport { + manifest: BackupManifest; + rpcOldestLedger: number; + rpcLatestLedger: number; + cursorBefore: number | null; + cursorAfter: number; + eventGap: { from: number; to: number } | null; + reprojected: number; + newEvents: number; + payments: { + reverified: number; + failed: Array<{ txHash: string; code: string }>; + unverifiable: number; + }; + specs: { checked: number; missing: string[]; hashMismatch: string[] }; + unclaimedTransfers: Array<{ txHash: string; from: string; amountRaw: string; ledger: number }>; +} + +export interface ReconcileDeps { + prisma: PrismaClient; + source: EventSource; + verifyPayment: typeof verifyUsdcTransfer; + emit?: (event: string, payload: unknown) => void; +} + +interface Args { + manifest?: string; + out?: string; + rewindLedgers: number; + scanTransfers: boolean; +} + +function parseArgs(argv: string[]): Args { + const args: Args = { rewindLedgers: REWIND_LEDGERS_DEFAULT, scanTransfers: false }; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === '--manifest') args.manifest = argv[++i]; + else if (arg === '--out') args.out = argv[++i]; + else if (arg === '--rewind-ledgers') args.rewindLedgers = Number(argv[++i]); + else if (arg === '--scan-transfers') args.scanTransfers = true; + } + return args; +} + +/** A fake getEvents source used by the restore drill and tests. */ +export function fixtureSource(file: string): EventSource { + const fixture = JSON.parse(fs.readFileSync(file, 'utf8')); + const events = (fixture.events ?? fixture).map((event: any) => ({ + ...event, + contractId: new Contract(event.contractId), + topic: event.topic.map((t: string) => xdr.ScVal.fromXDR(t, 'base64')), + value: xdr.ScVal.fromXDR(event.value, 'base64'), + })); + return { + async getHealth() { + const ledgers = events.map((e: any) => e.ledger); + return { + status: 'healthy', + oldestLedger: Math.min(...ledgers), + latestLedger: Math.max(...ledgers), + ledgerRetentionWindow: 120960, + } as any; + }, + async getEvents() { + return { + events, + cursor: 'end', + latestLedger: Math.max(...events.map((e: any) => e.ledger)), + oldestLedger: Math.min(...events.map((e: any) => e.ledger)), + } as any; + }, + }; +} + +export async function reconcile( + args: Args, + deps: ReconcileDeps, + manifest: BackupManifest, +): Promise { + const { prisma, source, verifyPayment } = deps; + const health = await source.getHealth(); + const cursorRow = await prisma.indexerCursor.findUnique({ where: { id: CURSOR_ID } }); + if (!cursorRow) { + throw new Error( + `No indexer cursor for ${CURSOR_ID}. The backup belongs to a different network passphrase.`, + ); + } + + const cursorBefore = cursorRow.lastLedger; + const boundary = manifest.maxChainEventLedger ?? cursorBefore; + const requestedSafe = Math.min(boundary, cursorBefore) - args.rewindLedgers; + const safeLedger = Math.max(health.oldestLedger, requestedSafe); + const eventGap = + requestedSafe < health.oldestLedger + ? { from: requestedSafe, to: health.oldestLedger - 1 } + : null; + + const backfill = await backfillEvents(safeLedger, { + source, + emit: deps.emit ?? (() => undefined), + }); + + const stored = await prisma.chainEvent.findMany({ + where: { ledger: { gte: safeLedger } }, + orderBy: [{ ledger: 'asc' }, { id: 'asc' }], + }); + for (const row of stored) { + await reprojectEvent(decodedFromStored(row as any), deps.emit ?? (() => undefined)); + } + + const payments = { reverified: 0, failed: [] as Array<{ txHash: string; code: string }>, unverifiable: 0 }; + const confirmed = await prisma.paymentEvent.findMany({ where: { status: 'CONFIRMED' } }); + for (const payment of confirmed) { + try { + await verifyPayment({ + txHash: payment.txHash, + token: CONTRACTS.usdc, + from: payment.fromAddress ?? '', + to: PAYMENTS_RECIPIENT, + minAmount: BigInt(config.DAILY_PASS_PRICE_RAW), + }); + payments.reverified++; + } catch (err) { + const code = (err as { code?: string }).code ?? 'PAYMENT_VERIFY_FAILED'; + if (code === 'PAYMENT_NOT_FOUND') payments.unverifiable++; + else payments.failed.push({ txHash: payment.txHash, code }); + } + } + + const specs = { checked: 0, missing: [] as string[], hashMismatch: [] as string[] }; + const markets = await prisma.market.findMany({ + where: { resolutionHash: { not: null } }, + select: { id: true, resolutionHash: true, resolutionSpec: true }, + }); + for (const market of markets) { + specs.checked++; + if (!market.resolutionSpec) { + specs.missing.push(market.id); + continue; + } + const computed = resolutionHash(market.resolutionSpec as unknown as ResolutionSpecJson).toString('hex'); + if (computed !== market.resolutionHash) specs.hashMismatch.push(market.id); + } + + const unclaimedTransfers: ReconcileReport['unclaimedTransfers'] = []; + if (args.scanTransfers) { + try { + const page = await source.getEvents({ + startLedger: safeLedger, + filters: [{ type: 'contract', contractIds: [CONTRACTS.usdc] }], + limit: 200, + }); + for (const event of page.events) { + const topics = event.topic ?? []; + if (topics.length < 3) continue; + const name = scValToNative(topics[0]); + if (name !== 'transfer') continue; + const from = String(scValToNative(topics[1])); + const to = String(scValToNative(topics[2])); + if (to !== PAYMENTS_RECIPIENT) continue; + const amount = scValToNative(event.value); + const txHash = event.txHash; + const existing = await prisma.paymentEvent.findUnique({ where: { txHash } }); + if (!existing) { + unclaimedTransfers.push({ txHash, from, amountRaw: String(amount), ledger: event.ledger }); + } + } + } catch { + // Scanning is best-effort; a retention gap is reported separately. + } + } + + const after = await prisma.indexerCursor.findUnique({ where: { id: CURSOR_ID } }); + return { + manifest, + rpcOldestLedger: health.oldestLedger, + rpcLatestLedger: health.latestLedger, + cursorBefore, + cursorAfter: after?.lastLedger ?? backfill.lastLedger, + eventGap, + reprojected: stored.length, + newEvents: backfill.stored, + payments, + specs, + unclaimedTransfers, + }; +} + diff --git a/tests/fixtures/restore-seed.sql b/tests/fixtures/restore-seed.sql new file mode 100644 index 0000000..53a2e0c --- /dev/null +++ b/tests/fixtures/restore-seed.sql @@ -0,0 +1,16 @@ +-- Minimal pre-backup data for the restore drill (#146). +INSERT INTO users (id, "walletAddress", email, "createdAt", "updatedAt") +VALUES ('restore-user', 'GCM7WU3RADBCIKBLGPPAEYV5WHASWMEQX6BU524QR3QFWSE7BIWCXAIF', NULL, now(), now()) +ON CONFLICT (id) DO NOTHING; + +INSERT INTO markets (id, question, category, status, "settlementCurrency", "initialYesProb", "confidenceInterval", "expiryTimestamp", "minimumLiquidity", "totalLiquidity", "createdAt", "updatedAt") +VALUES ('restore-market', 'Restore drill market?', 'MACRO', 'ACTIVE', 'USDC', 0.5, '{"lower":0.4,"upper":0.6}'::jsonb, now() + interval '30 days', 100, 100, now(), now()) +ON CONFLICT (id) DO NOTHING; + +INSERT INTO chain_events (id, "contractId", "contractName", "eventName", ledger, "ledgerClosedAt", "txHash", params, "createdAt") +VALUES ('restore-event', 'CC4MMHWZ6ZRYAOQRR42KIIWNEZNFM4CWQ5Y4NNWTWNRUYH2O7E3SRK2O', 'marketCore', 'trade', 4951600, now(), repeat('a', 64), '{}'::jsonb, now()) +ON CONFLICT (id) DO NOTHING; + +INSERT INTO indexer_cursors (id, "lastLedger", "updatedAt") +VALUES ('stellar:Test SDF Network ; September 2015', 4951600, now()) +ON CONFLICT (id) DO UPDATE SET "lastLedger" = EXCLUDED."lastLedger"; diff --git a/tests/helpers/env.ts b/tests/helpers/env.ts index 6b432ba..ca79cf1 100644 --- a/tests/helpers/env.ts +++ b/tests/helpers/env.ts @@ -1,13 +1,25 @@ +import { Keypair } from '@stellar/stellar-sdk'; +import { SECRET_NAMES, setSecretsForTest } from '../../src/config/secrets'; + export function setupTestEnv(): void { process.env.NODE_ENV = 'test'; process.env.DATABASE_URL = process.env.DATABASE_URL || 'postgresql://test:test@localhost:5432/test'; process.env.JWT_SECRET = process.env.JWT_SECRET || '01234567890123456789012345678901'; // Throwaway key generated for tests only; never funded, never used on-chain. - process.env.AUTH_SIGNING_SECRET = ''; + process.env.AUTH_SIGNING_SECRET = process.env.AUTH_SIGNING_SECRET || Keypair.random().secret(); process.env.AUTH_HOME_DOMAIN = 'localhost'; process.env.CHAIN_EXECUTION_MODE = 'dry-run'; process.env.AGENT_SECRET_KEY = ''; process.env.STELLAR_RPC_URL = 'http://127.0.0.1:1'; // tests must never reach a real RPC process.env.INDEXER_ENABLED = 'false'; process.env.ADMIN_ADDRESSES = ''; + + // #145 — consumers read through the secret store; tests must not require a + // running bootstrap() to seed it. + setSecretsForTest( + Object.fromEntries(SECRET_NAMES.map((name) => [name, process.env[name] ?? ''])) as Record< + (typeof SECRET_NAMES)[number], + string + >, + ); } diff --git a/tests/jwt-rotation.test.ts b/tests/jwt-rotation.test.ts new file mode 100644 index 0000000..15076f5 --- /dev/null +++ b/tests/jwt-rotation.test.ts @@ -0,0 +1,73 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import jwt from 'jsonwebtoken'; +import { setupTestEnv } from './helpers/env'; + +setupTestEnv(); + +function fakeProvider(values: Record) { + return { kind: 'file' as const, async load() { return values; } }; +} + +function requestWith(token: string): any { + return { headers: { authorization: `Bearer ${token}` } }; +} + +test('a token signed with the previous key works during the grace window, then fails', async () => { + const { initSecrets, refreshSecretsOnce } = await import('../src/config/secrets'); + const { optionalAuth } = await import('../src/middlewares/auth.middleware'); + + const oldSecret = 'o'.repeat(32); + const newSecret = 'n'.repeat(32); + const cfg: any = { + DATABASE_URL: 'postgresql://u:p@localhost:5432/db', + JWT_SECRET: oldSecret, + AUTH_SIGNING_SECRET: (await import('@stellar/stellar-sdk')).Keypair.random().secret(), + NODE_ENV: 'test', + JWT_PREVIOUS_SECRET_GRACE_SECS: 1, + SECRETS_PROVIDER: 'file', + }; + await initSecrets(cfg, fakeProvider({ JWT_SECRET: oldSecret })); + const token = jwt.sign({ userId: 'u1', walletAddress: 'G' }, oldSecret, { expiresIn: '7d' }); + + await refreshSecretsOnce(cfg, fakeProvider({ JWT_SECRET: newSecret })); + + const during = requestWith(token); + let nextCalled = false; + optionalAuth(during, {} as any, () => { nextCalled = true; }); + assert.equal(nextCalled, true); + assert.equal(during.user?.userId, 'u1', 'old token accepted during grace'); + + const realNow = Date.now; + Date.now = () => realNow() + 2_000; + try { + const after = requestWith(token); + let afterNext = false; + optionalAuth(after, {} as any, () => { afterNext = true; }); + assert.equal(afterNext, true, 'optionalAuth always calls next'); + assert.equal(after.user, undefined, 'old token rejected after grace'); + } finally { + Date.now = realNow; + } +}); + +test('an expired token is never retried against the previous key', async () => { + const { initSecrets, refreshSecretsOnce } = await import('../src/config/secrets'); + const { optionalAuth } = await import('../src/middlewares/auth.middleware'); + const oldSecret = 'p'.repeat(32); + const cfg: any = { + DATABASE_URL: 'postgresql://u:p@localhost:5432/db', + JWT_SECRET: oldSecret, + AUTH_SIGNING_SECRET: (await import('@stellar/stellar-sdk')).Keypair.random().secret(), + NODE_ENV: 'test', + JWT_PREVIOUS_SECRET_GRACE_SECS: 600, + SECRETS_PROVIDER: 'file', + }; + await initSecrets(cfg, fakeProvider({ JWT_SECRET: oldSecret })); + const expired = jwt.sign({ userId: 'u1', walletAddress: 'G' }, oldSecret, { expiresIn: -1 }); + await refreshSecretsOnce(cfg, fakeProvider({ JWT_SECRET: 'q'.repeat(32) })); + + const req = requestWith(expired); + optionalAuth(req, {} as any, () => undefined); + assert.equal(req.user, undefined, 'expired stays expired even with a previous key'); +}); diff --git a/tests/migration-compat.test.ts b/tests/migration-compat.test.ts new file mode 100644 index 0000000..6d3883e --- /dev/null +++ b/tests/migration-compat.test.ts @@ -0,0 +1,44 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { spawnSync } from 'node:child_process'; + +const SCRIPT = path.resolve(__dirname, '../../scripts/check-migration-compat.mjs'); + +function check(sql: string): { status: number | null; output: string } { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'oracledesk-migration-')); + const file = path.join(dir, 'migration.sql'); + fs.writeFileSync(file, sql); + const result = spawnSync(process.execPath, [SCRIPT, file], { encoding: 'utf8' }); + fs.rmSync(dir, { recursive: true, force: true }); + return { status: result.status, output: `${result.stdout}${result.stderr}` }; +} + +test('migration linter flags breaking and non-concurrent statements', () => { + const result = check(` + -- DROP TABLE users; (commented out, ignored) + DROP TABLE old_payments; + ALTER TABLE users DROP COLUMN email; + ALTER TABLE users ALTER COLUMN balance TYPE numeric; + ALTER TABLE users RENAME COLUMN wallet TO address; + ALTER TABLE users ALTER COLUMN name SET NOT NULL; + CREATE INDEX trades_market_idx ON trades (market_id); + CREATE UNIQUE INDEX CONCURRENTLY users_wallet_idx ON users (wallet_address); + `); + assert.equal(result.status, 1); + for (const rule of ['drop-table', 'drop-column', 'alter-type', 'rename', 'set-not-null', 'non-concurrent-index']) { + assert.match(result.output, new RegExp(rule)); + } +}); + +test('migration linter accepts expand/contract-safe SQL', () => { + const result = check(` + ALTER TABLE users ADD COLUMN display_name text; + CREATE INDEX CONCURRENTLY users_display_name_idx ON users (display_name); + ALTER TABLE users ADD CONSTRAINT users_display_name_not_null CHECK (display_name IS NOT NULL) NOT VALID; + `); + assert.equal(result.status, 0, result.output); + assert.equal(result.output.trim(), ''); +}); diff --git a/tests/post-deploy-checks.test.ts b/tests/post-deploy-checks.test.ts new file mode 100644 index 0000000..8979947 --- /dev/null +++ b/tests/post-deploy-checks.test.ts @@ -0,0 +1,58 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import http from 'node:http'; +import path from 'node:path'; +import { spawn } from 'node:child_process'; + +const SCRIPT = path.resolve(__dirname, '../../scripts/post-deploy-checks.sh'); + +function run(env: Record): Promise<{ code: number | null; stdout: string; stderr: string }> { + return new Promise((resolve) => { + const child = spawn('bash', [SCRIPT], { + env: { ...process.env, SKIP_KUBE: '1', READY_CHECKS: '2', READY_INTERVAL: '0', ...env }, + }); + let stdout = ''; + let stderr = ''; + child.stdout.on('data', (d) => (stdout += d)); + child.stderr.on('data', (d) => (stderr += d)); + child.on('close', (code) => resolve({ code, stdout, stderr })); + }); +} + +test('post-deploy checks pass when readiness, lag and cursor are healthy', async () => { + let cursor = 1000; + const server = http.createServer((req, res) => { + if (req.url === '/readyz') { + res.writeHead(200).end('ok'); + return; + } + if (req.url === '/status') { + res.writeHead(200, { 'content-type': 'application/json' }).end( + JSON.stringify({ latestLedger: 1050, lastLedger: cursor++ }), + ); + return; + } + res.writeHead(404).end(); + }); + await new Promise((resolve) => server.listen(0, resolve)); + const port = (server.address() as any).port; + const result = await run({ + API_URL: `http://127.0.0.1:${port}`, + INDEXER_STATUS_URL: `http://127.0.0.1:${port}/status`, + MAX_INDEXER_LAG_LEDGERS: '120', + }); + await new Promise((resolve) => server.close(resolve)); + assert.equal(result.code, 0, result.stderr); + assert.match(result.stdout, /readiness held/); + assert.match(result.stdout, /indexer cursor advanced/); +}); + +test('post-deploy checks fail when readiness does not hold', async () => { + const server = http.createServer((_req, res) => res.writeHead(500).end('no')); + await new Promise((resolve) => server.listen(0, resolve)); + const port = (server.address() as any).port; + const result = await run({ API_URL: `http://127.0.0.1:${port}` }); + await new Promise((resolve) => server.close(resolve)); + assert.equal(result.code, 1); + assert.match(result.stderr, /readiness did not hold/); +}); diff --git a/tests/reconcile.test.ts b/tests/reconcile.test.ts new file mode 100644 index 0000000..710ffb5 --- /dev/null +++ b/tests/reconcile.test.ts @@ -0,0 +1,129 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { Keypair } from '@stellar/stellar-sdk'; +import { setupTestEnv } from './helpers/env'; + +setupTestEnv(); + +async function setup() { + const { prisma } = await import('../src/lib/prisma'); + const { fixtureSource, reconcile } = await import('../src/services/reconcile.service'); + const { resolutionHash } = await import('../src/lib/resolution-spec'); + const { CURSOR_ID } = await import('../src/services/indexer.service'); + + const db = { + cursor: 4_951_615 as number | null, + events: new Map(), + markets: new Map(), + traces: [] as any[], + payments: [] as any[], + }; + + const signer = Keypair.random().publicKey(); + const spec = { kind: 'signers', signers: [signer], threshold: 1, disputeWindow: 3600 } as const; + db.markets.set('m0', { + id: 'm0', + onChainMarketId: 0n, + question: 'Market zero?', + status: 'ACTIVE', + resolutionSpec: spec, + resolutionHash: resolutionHash(spec as any).toString('hex'), + }); + db.traces.push({ id: 't0', traceHash: 'f4b58d4ac4068163f34bd9486f0563c70873afa5f64e915ab7e85af7dee15e4e', onChainTraceId: null }); + + (prisma.indexerCursor.findUnique as any) = async ({ where }: any) => + where.id === CURSOR_ID && db.cursor !== null ? { id: CURSOR_ID, lastLedger: db.cursor } : null; + (prisma.indexerCursor.upsert as any) = async ({ update }: any) => { + db.cursor = update.lastLedger; + return { id: CURSOR_ID, lastLedger: update.lastLedger }; + }; + (prisma.chainEvent.create as any) = async ({ data }: any) => { + db.events.set(data.id, data); + return data; + }; + (prisma.chainEvent.findMany as any) = async ({ where }: any) => + [...db.events.values()] + .filter((event) => event.ledger >= where.ledger.gte) + .sort((a, b) => a.ledger - b.ledger || a.id.localeCompare(b.id)); + (prisma.market.findUnique as any) = async ({ where }: any) => + [...db.markets.values()].find((market) => market.onChainMarketId === where.onChainMarketId) ?? null; + (prisma.market.findMany as any) = async () => [...db.markets.values()]; + (prisma.market.update as any) = async ({ where, data }: any) => Object.assign(db.markets.get(where.id), data); + (prisma.position.updateMany as any) = async () => ({ count: 0 }); + (prisma.reasoningTrace.findFirst as any) = async ({ where }: any) => + db.traces.find((trace) => trace.traceHash === where.traceHash) ?? null; + (prisma.reasoningTrace.update as any) = async ({ where, data }: any) => + Object.assign(db.traces.find((trace) => trace.id === where.id)!, data); + (prisma.paymentEvent.findMany as any) = async () => db.payments; + (prisma.paymentEvent.findUnique as any) = async ({ where }: any) => + db.payments.find((payment) => payment.txHash === where.txHash) ?? null; + (prisma.$transaction as any) = async (ops: Promise[]) => Promise.all(ops); + + const source = fixtureSource('tests/fixtures/events.json'); + return { prisma, db, source, reconcile, CURSOR_ID }; +} + +test('reconcile rewinds, reprojects and verifies payments and specs', async () => { + const { prisma, db, source, reconcile } = await setup(); + db.payments.push({ + txHash: 'a'.repeat(64), + fromAddress: 'GCM7WU3RADBCIKBLGPPAEYV5WHASWMEQX6BU524QR3QFWSE7BIWCXAIF', + status: 'CONFIRMED', + }); + + let verified = 0; + const report = await reconcile( + { rewindLedgers: 720, scanTransfers: false }, + { + prisma: prisma as any, + source, + verifyPayment: (async () => { + verified++; + return { txHash: 'a'.repeat(64), from: 'G', to: 'G', amount: 5_000_000n, ledger: 1 }; + }) as any, + }, + { maxChainEventLedger: 4_951_615, cursors: [{ id: 'x', lastLedger: 4_951_615 }] }, + ); + + assert.equal(verified, 1, 'confirmed payment was re-verified'); + assert.equal(report.payments.reverified, 1); + assert.deepEqual(report.specs, { checked: 1, missing: [], hashMismatch: [] }); + assert.ok(report.reprojected >= 1, 'stored events were reprojected'); + assert.equal(report.eventGap?.from, 4_950_895); + assert.equal(db.markets.get('m0').status, 'RESOLVED', 'projection applied resolution'); +}); + +test('reconcile refuses a cursor from a different network', async () => { + const { prisma, db, source, reconcile } = await setup(); + db.cursor = null; + await assert.rejects( + reconcile( + { rewindLedgers: 720, scanTransfers: false }, + { prisma: prisma as any, source, verifyPayment: (async () => ({})) as any }, + {}, + ), + /No indexer cursor/, + ); +}); + +test('reconcile reports a resolution hash mismatch', async () => { + const { prisma, db, source, reconcile } = await setup(); + db.markets.get('m0').resolutionHash = 'ff'.repeat(32); + const report = await reconcile( + { rewindLedgers: 720, scanTransfers: false }, + { prisma: prisma as any, source, verifyPayment: (async () => ({})) as any }, + {}, + ); + assert.deepEqual(report.specs.hashMismatch, ['m0']); +}); + +test('reconcile reports a missing resolution preimage', async () => { + const { prisma, db, source, reconcile } = await setup(); + db.markets.get('m0').resolutionSpec = null; + const report = await reconcile( + { rewindLedgers: 720, scanTransfers: false }, + { prisma: prisma as any, source, verifyPayment: (async () => ({})) as any }, + {}, + ); + assert.deepEqual(report.specs.missing, ['m0']); +}); diff --git a/tests/secrets.test.ts b/tests/secrets.test.ts new file mode 100644 index 0000000..290e3a3 --- /dev/null +++ b/tests/secrets.test.ts @@ -0,0 +1,136 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { Keypair } from '@stellar/stellar-sdk'; +import { setupTestEnv } from './helpers/env'; + +setupTestEnv(); + +const BASE = { + DATABASE_URL: 'postgresql://u:p@localhost:5432/db', + JWT_SECRET: '01234567890123456789012345678901', + AUTH_SIGNING_SECRET: Keypair.random().secret(), + NODE_ENV: 'test', +}; + +test('loadConfig resolves NAME_FILE and refuses both forms', async () => { + const { loadConfig } = await import('../src/config'); + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'oracledesk-secret-')); + const file = path.join(dir, 'jwt'); + fs.writeFileSync(file, `${BASE.JWT_SECRET}\n`); + + const config = loadConfig({ + DATABASE_URL: BASE.DATABASE_URL, + JWT_SECRET_FILE: file, + AUTH_SIGNING_SECRET: BASE.AUTH_SIGNING_SECRET, + }); + assert.equal(config.JWT_SECRET, BASE.JWT_SECRET, 'trailing newline is stripped'); + + assert.throws( + () => loadConfig({ DATABASE_URL: BASE.DATABASE_URL, JWT_SECRET: BASE.JWT_SECRET, JWT_SECRET_FILE: file }), + /cannot both be set/, + ); + + const empty = path.join(dir, 'empty'); + fs.writeFileSync(empty, '\n'); + assert.throws( + () => loadConfig({ DATABASE_URL: BASE.DATABASE_URL, JWT_SECRET_FILE: empty }), + /JWT_SECRET_FILE is empty/, + ); + fs.rmSync(dir, { recursive: true, force: true }); +}); + +test('loadConfig rejects a short JWT at boot', async () => { + const { loadConfig } = await import('../src/config'); + assert.throws( + () => loadConfig({ ...BASE, JWT_SECRET: 'short' }), + /JWT_SECRET must be at least 32 characters/, + ); +}); + +test('file provider refresh rotates JWT, keeps the previous value inside the grace window, then zeroes it', async () => { + const { initSecrets, previousSecret, refreshSecretsOnce, secret, secretEntry } = await import('../src/config/secrets'); + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'oracledesk-refresh-')); + const file = path.join(dir, 'jwt'); + fs.writeFileSync(file, 'a'.repeat(32)); + const cfg: any = { ...BASE, JWT_SECRET: 'a'.repeat(32), SECRETS_PROVIDER: 'file', JWT_PREVIOUS_SECRET_GRACE_SECS: 60 }; + + // Point the file provider at a temporary NAME_FILE, not a real mount. + const previousEnv = process.env.JWT_SECRET_FILE; + process.env.JWT_SECRET_FILE = file; + try { + const { fileProvider } = await import('../src/config/secrets'); + await initSecrets(cfg, fileProvider()); + fs.writeFileSync(file, 'b'.repeat(32)); + const result = await refreshSecretsOnce(cfg); + assert.deepEqual(result.updated, ['JWT_SECRET']); + assert.equal(secret('JWT_SECRET'), 'b'.repeat(32)); + assert.equal(previousSecret('JWT_SECRET'), 'a'.repeat(32), 'previous value survives the grace window'); + assert.ok(secretEntry('JWT_SECRET').version >= 2); + + const realNow = Date.now; + Date.now = () => realNow() + 61_000; + try { + assert.equal(previousSecret('JWT_SECRET'), null, 'previous value is dropped after the grace window'); + assert.equal(secretEntry('JWT_SECRET').previous, undefined); + } finally { + Date.now = realNow; + } + } finally { + if (previousEnv === undefined) delete process.env.JWT_SECRET_FILE; + else process.env.JWT_SECRET_FILE = previousEnv; + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +test('an empty or invalid refresh does not wipe a working key', async () => { + const { initSecrets, refreshSecretsOnce, secret } = await import('../src/config/secrets'); + const cfg: any = { ...BASE, JWT_SECRET: 'c'.repeat(32), SECRETS_PROVIDER: 'file' }; + let next = ''; + const provider = { kind: 'file' as const, async load() { return { JWT_SECRET: next }; } }; + + await initSecrets(cfg, provider); + next = ''; + let result = await refreshSecretsOnce(cfg, provider); + assert.equal(secret('JWT_SECRET'), 'c'.repeat(32)); + assert.match(result.problems.join('\n'), /empty value/); + + next = 'short'; + result = await refreshSecretsOnce(cfg, provider); + assert.equal(secret('JWT_SECRET'), 'c'.repeat(32)); + assert.match(result.problems.join('\n'), /failed validation/); + + next = 'd'.repeat(32); + result = await refreshSecretsOnce(cfg, provider); + assert.equal(secret('JWT_SECRET'), 'd'.repeat(32)); + assert.deepEqual(result.updated, ['JWT_SECRET']); +}); + +test('a provider failure keeps current values and never throws', async () => { + const { initSecrets, refreshSecretsOnce, secret } = await import('../src/config/secrets'); + const cfg: any = { ...BASE, JWT_SECRET: 'e'.repeat(32), SECRETS_PROVIDER: 'file' }; + const healthy = { kind: 'file' as const, async load() { return { JWT_SECRET: 'e'.repeat(32) }; } }; + await initSecrets(cfg, healthy); + const provider = { kind: 'file' as const, async load(): Promise { throw new Error('vault down'); } }; + const result = await refreshSecretsOnce(cfg, provider); + assert.equal(secret('JWT_SECRET'), 'e'.repeat(32)); + assert.match(result.problems.join('\n'), /provider refresh failed/); +}); + +test('LLM clients are rebuilt only after their key rotates', async () => { + const { setSecretsForTest } = await import('../src/config/secrets'); + const { getGenAI } = await import('../src/lib/gemini'); + const { getAnthropicClient } = await import('../src/lib/llm'); + + setSecretsForTest({ GEMINI_API_KEY: 'key-one', ANTHROPIC_API_KEY: 'anthropic-one' }); + const geminiOne = getGenAI(); + const anthropicOne = getAnthropicClient(); + assert.equal(getGenAI(), geminiOne, 'same version reuses the client'); + assert.equal(getAnthropicClient(), anthropicOne); + + setSecretsForTest({ GEMINI_API_KEY: 'key-two', ANTHROPIC_API_KEY: 'anthropic-two' }); + assert.notEqual(getGenAI(), geminiOne, 'rotation rebuilds Gemini'); + assert.notEqual(getAnthropicClient(), anthropicOne, 'rotation rebuilds Anthropic'); +}); diff --git a/tests/seed.test.ts b/tests/seed.test.ts new file mode 100644 index 0000000..4414524 --- /dev/null +++ b/tests/seed.test.ts @@ -0,0 +1,117 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { Contract, xdr } from '@stellar/stellar-sdk'; +import { setupTestEnv } from './helpers/env'; + +setupTestEnv(); + +test('seed scenario is deterministic and internally consistent', async () => { + const { buildSeedData } = await import('../src/seed/scenario'); + const { canonicalBytes, sha256Hex } = await import('../src/utils/hash.util'); + const { questionDocument } = await import('../src/services/market-chain.service'); + const { resolutionHash } = await import('../src/lib/resolution-spec'); + const { cidForBytes } = await import('../src/services/ipfs.service'); + + const a = buildSeedData({ seed: 42, marketCount: 20 }); + const b = buildSeedData({ seed: 42, marketCount: 20 }); + assert.deepEqual(a, b, 'same seed produces the same bytes'); + assert.equal(a.markets.length, 20); + assert.equal(a.users.length, 12); + assert.ok(a.trades.length > 0, 'active markets have trades'); + assert.ok(a.chainEvents.length > 0); + + for (const market of a.markets) { + const docBytes = canonicalBytes(questionDocument(market as any, market.resolutionHash)); + assert.equal(market.questionHash, sha256Hex(docBytes), `${market.id} question hash matches its bytes`); + assert.equal(market.metaUri, `ipfs://${cidForBytes(docBytes)}`); + assert.equal(resolutionHash(market.resolutionSpec).toString('hex'), market.resolutionHash); + } + + const trace = a.traces[0]; + const payload: Record = { + agentType: trace.agentType, + decisionType: trace.decisionType, + marketId: trace.marketId, + sourcesUsed: trace.sourcesUsed, + probabilityEstimate: trace.probabilityEstimate, + marketProbability: trace.marketProbability, + edge: trace.edge, + confidenceInterval: trace.confidenceInterval, + ...(trace.agentType === 'TRADER' + ? { betFraction: trace.betFraction ?? undefined, betSizeUsdc: trace.betSizeUsdc ?? undefined } + : {}), + hedgeConditions: trace.hedgeConditions, + agentWallet: trace.agentWallet, + timestamp: trace.createdAt.toISOString(), + traceId: trace.id, + }; + const bytes = canonicalBytes(payload); + assert.equal(trace.traceHash, sha256Hex(bytes), 'trace hash commits to the pinned payload'); + assert.equal(trace.ipfsCid, cidForBytes(bytes)); +}); + +test('transcripts encode into raw getEvents responses that the indexer decodes', async () => { + const { buildSeedData } = await import('../src/seed/scenario'); + const { encodeSeedEvents } = await import('../src/seed/raw-events'); + const { decodeEvent } = await import('../src/services/indexer.service'); + + const data = buildSeedData({ seed: 7, marketCount: 6 }); + const fixture = encodeSeedEvents(data.chainEvents); + const decoded = fixture.events.map((event: any) => + decodeEvent({ + ...event, + contractId: new Contract(event.contractId), + topic: event.topic.map((topic: string) => xdr.ScVal.fromXDR(topic, 'base64')), + value: xdr.ScVal.fromXDR(event.value, 'base64'), + } as any), + ); + assert.equal(decoded.length, data.chainEvents.length); + for (const event of decoded) { + assert.ok(event, 'every encoded event decodes with the contract spec'); + } + const names = decoded.map((event) => event!.eventName); + assert.ok(names.includes('Trade')); + assert.ok(names.includes('MarketCreated')); + assert.ok(names.includes('MarketResolved')); + assert.ok(names.includes('TracePublished')); +}); + +test('local IPFS stub stores and fetches the exact bytes under the CID', async () => { + const { localIpfsTransport, cidForBytes } = await import('../src/services/ipfs.service'); + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'oracledesk-ipfs-')); + const transport = localIpfsTransport(dir); + const bytes = Buffer.from('{"hello":"world"}'); + const cid = await transport.pin(bytes, 'test'); + assert.equal(cid, cidForBytes(bytes)); + assert.deepEqual(await transport.fetch(cid), bytes); + fs.rmSync(dir, { recursive: true, force: true }); +}); + +test('snapshot anonymisation is fail-closed and deterministic', async () => { + const { RULES, anonymizeRow, assertNoSecrets, buildWalletMap, SnapshotExportError } = await import('../src/lib/snapshot'); + const { buildSeedData } = await import('../src/seed/scenario'); + const data = buildSeedData({ seed: 11, marketCount: 5 }); + const salt = 'test-salt'; + const wallets = buildWalletMap(salt, data.users); + const user = anonymizeRow('user', data.users[0], wallets, salt); + const again = anonymizeRow('user', data.users[0], wallets, salt); + assert.equal(user.walletAddress, again.walletAddress, 'pseudonym is stable'); + assert.notEqual(user.walletAddress, data.users[0].walletAddress); + assert.equal(user.email, null); + + const payment = anonymizeRow('paymentEvent', data.paymentEvents[0], wallets, salt); + assert.notEqual(payment.txHash, data.paymentEvents[0].txHash, 'hashes are keyed'); + assert.equal(payment.fromAddress, user.walletAddress); + + assert.throws( + () => anonymizeRow('user', { ...data.users[0], unexpectedColumn: 'leak' }, wallets, salt), + SnapshotExportError, + 'an unmapped column stops the export', + ); + assert.ok(RULES.user); + assert.throws(() => assertNoSecrets('S' + 'A'.repeat(55), salt), /secret-looking/); + assert.throws(() => assertNoSecrets(`salt:${salt}`, salt), /secret-looking/); +});