diff --git a/.env.example b/.env.example index bed94e6..ae81bd8 100644 --- a/.env.example +++ b/.env.example @@ -9,6 +9,20 @@ PORT=8000 # Postgres and Redis. These defaults match docker-compose.yml (local only, # not secrets). Point them at your own servers for anything shared. DATABASE_URL=postgresql://oracledesk:oracledesk@127.0.0.1:5433/oracledesk +# Alternatively, mount the URL from a file: DATABASE_URL_FILE=/run/secrets/database_url +# Every secret below also accepts _FILE (env/file provider) — see #145. + +# ── Secret loading (#145) ────────────────────────────────────────────────── +# env: process environment (default); file: _FILE mounts; vault: Vault KV v2. +SECRETS_PROVIDER=env +# Refresh interval for file/vault providers (minimum 30000 ms). +SECRETS_REFRESH_MS=300000 +# How long a rotated JWT_SECRET remains valid for already-issued tokens. +JWT_PREVIOUS_SECRET_GRACE_SECS=86400 +# Required only when SECRETS_PROVIDER=vault. +VAULT_ADDR= +VAULT_TOKEN_FILE= +VAULT_KV_PATH= REDIS_URL=redis://127.0.0.1:6379 # Comma-separated browser origins allowed to call the API. CORS_ORIGINS=http://localhost:3000 @@ -60,10 +74,24 @@ RESOLUTION_SIGNERS= RESOLUTION_THRESHOLD=1 RESOLUTION_DISPUTE_WINDOW_SECS=86400 +# Accept a well-formed inbound X-Request-Id; set false without a trusted proxy. +TRUST_INBOUND_REQUEST_ID=true + # Event indexer (Stellar RPC getEvents poller). INDEXER_ENABLED=true INDEXER_POLL_MS=5000 +# Health probes (#111): /livez (process only) and /readyz (dependencies). +# HEALTH_CRITICAL: comma list of database,redis,rpc,indexer that fail readiness. +HEALTH_CRITICAL=database,redis +HEALTH_PROBE_TIMEOUT_MS=1500 +HEALTH_CACHE_TTL_MS=2000 +INDEXER_MAX_LAG_LEDGERS=120 +INDEXER_MAX_STALENESS_MS=60000 +LIVENESS_MAX_LOOP_DELAY_MS=2000 +# Drain + close must fit the 15 s forced-exit window. +SHUTDOWN_DRAIN_MS=5000 + # ── LLM providers (market and trace generation) ──────────────────────────── ANTHROPIC_API_KEY= GEMINI_API_KEY= @@ -72,7 +100,64 @@ GEMINI_API_KEY= PINATA_API_KEY= PINATA_SECRET_API_KEY= IPFS_GATEWAY_URL=https://gateway.pinata.cloud/ipfs +# pinata (default) or local (development only; stores bytes in .ipfs-local/). +IPFS_TRANSPORT=pinata +IPFS_LOCAL_DIR=.ipfs-local # ── Data sources ─────────────────────────────────────────────────────────── NEWSAPI_KEY= FRED_API_KEY= + +# ── Resilience & Timeouts ────────────────────────────────────────────────── +TIMEOUT_RPC_READ_MS=8000 +TIMEOUT_RPC_WRITE_MS=45000 +TIMEOUT_RPC_EVENTS_MS=15000 +TIMEOUT_LLM_MS=60000 +TIMEOUT_IPFS_PIN_MS=20000 +TIMEOUT_IPFS_FETCH_MS=15000 +TIMEOUT_DATASOURCE_MS=10000 + +# Concurrency bulkheads +BULKHEAD_LLM=2 +BULKHEAD_IPFS=4 +BULKHEAD_RPC_READ=16 +BULKHEAD_RPC_WRITE=4 +BULKHEAD_RPC_EVENTS=4 +BULKHEAD_DATASOURCE=8 +BULKHEAD_QUEUE=32 + +# Deadlines & caps +HTTP_REQUEST_TIMEOUT_MS=30000 +CYCLE_DEADLINE_MS=600000 +IPFS_MAX_BYTES=2097152 + +# ── Trace Audit (Registry Verification) ──────────────────────────────────── +TRACE_AUDIT_ENABLED=true +TRACE_AUDIT_CRON=40 */2 * * * +TRACE_AUDIT_RECHECK_HOURS=168 +TRACE_AUDIT_MAX_RPC=500 +TRACE_AUDIT_MAX_FETCHES=100 +TRACE_AUDIT_MAX_BYTES=2097152 + +# Trace publication queue (#131): pins traces, waits for the market to be +# on-chain, publishes once, and confirms via the indexer's TracePublished event. +TRACE_PUBLISH_ENABLED=true +TRACE_PUBLISH_CRON=*/30 * * * * * +TRACE_PUBLISH_MAX_ATTEMPTS=8 +TRACE_CONFIRM_TIMEOUT_MS=600000 + +# ── Copy Trading ─────────────────────────────────────────────────────────── +COPY_TRADE_TTL_MINUTES=30 +COPY_TRADE_MAX_PENDING=5 + +# ── Seed / snapshot (#147) ───────────────────────────────────────────────── +# The seed refuses production and non-local databases. Set to 1 only when you +# intentionally seed a remote development database. +SEED_ALLOW_REMOTE=0 +# Base id for seeded on-chain market/trace rows, and the starting ledger. +SEED_ONCHAIN_BASE=1000000 +SEED_BASE_LEDGER=6000000 +# Optional comma-separated real testnet ids to link (read-only). +SEED_TESTNET_MARKET_IDS= +# Required by scripts/snapshot-export.ts; never commit the value. +SNAPSHOT_SALT= diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml new file mode 100644 index 0000000..db08159 --- /dev/null +++ b/.github/workflows/cd.yml @@ -0,0 +1,135 @@ +name: CD + +on: + push: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + packages: write + id-token: write + pull-requests: read + +concurrency: + group: cd-main + cancel-in-progress: false + +jobs: + verify: + name: Verify release commit + uses: ./.github/workflows/ci.yml + + image: + name: Build, sign and attest image + needs: verify + runs-on: ubuntu-latest + outputs: + digest: ${{ steps.build.outputs.digest }} + tag: ${{ steps.meta.outputs.tag }} + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - uses: docker/setup-buildx-action@v3 + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - id: meta + run: echo "tag=ghcr.io/${GITHUB_REPOSITORY,,}:${GITHUB_SHA::12}" >> "$GITHUB_OUTPUT" + - id: build + uses: docker/build-push-action@v6 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tag }} + provenance: true + sbom: true + - name: Install cosign + uses: sigstore/cosign-installer@v3 + - name: Sign image + run: cosign sign --yes "${{ steps.meta.outputs.tag }}@${{ steps.build.outputs.digest }}" + - name: SBOM (SPDX) + uses: anchore/sbom-action@v0 + with: + image: ${{ steps.meta.outputs.tag }}@${{ steps.build.outputs.digest }} + format: spdx-json + output-file: sbom.spdx.json + - uses: actions/upload-artifact@v4 + with: + name: sbom + path: sbom.spdx.json + + staging: + name: Deploy staging + needs: image + uses: ./.github/workflows/deploy-env.yml + with: + environment: staging + namespace: oracledesk-staging + image: ghcr.io/${{ github.repository }}@${{ needs.image.outputs.digest }} + digest: ${{ needs.image.outputs.digest }} + values_file: values-staging.yaml + secrets: inherit + + staging-checks: + name: Staging post-deploy checks + needs: [image, staging] + runs-on: ubuntu-latest + environment: staging + steps: + - uses: actions/checkout@v4 + - uses: azure/setup-kubectl@v4 + - name: Checks + env: + API_URL: ${{ vars.API_URL }} + READY_CHECKS: "3" + INDEXER_STATUS_URL: ${{ vars.INDEXER_STATUS_URL }} + MAX_INDEXER_LAG_LEDGERS: "120" + PROMETHEUS_URL: ${{ vars.PROMETHEUS_URL }} + KUBE_NAMESPACE: oracledesk-staging + DEPLOYMENTS: api indexer worker + DIGEST: ${{ needs.image.outputs.digest }} + KUBECONFIG_DATA: ${{ secrets.KUBECONFIG }} + run: | + echo "$KUBECONFIG_DATA" > "$RUNNER_TEMP/kubeconfig" + export KUBECONFIG="$RUNNER_TEMP/kubeconfig" + ./scripts/post-deploy-checks.sh + + production: + name: Deploy production + needs: [image, staging-checks] + uses: ./.github/workflows/deploy-env.yml + with: + environment: production + namespace: oracledesk + image: ghcr.io/${{ github.repository }}@${{ needs.image.outputs.digest }} + digest: ${{ needs.image.outputs.digest }} + values_file: values-production.yaml + secrets: inherit + + production-checks: + name: Production post-deploy checks + needs: [image, production] + runs-on: ubuntu-latest + environment: production + steps: + - uses: actions/checkout@v4 + - uses: azure/setup-kubectl@v4 + - name: Checks + env: + API_URL: ${{ vars.API_URL }} + READY_CHECKS: "5" + INDEXER_STATUS_URL: ${{ vars.INDEXER_STATUS_URL }} + MAX_INDEXER_LAG_LEDGERS: "120" + PROMETHEUS_URL: ${{ vars.PROMETHEUS_URL }} + KUBE_NAMESPACE: oracledesk + DEPLOYMENTS: api indexer worker + DIGEST: ${{ needs.image.outputs.digest }} + KUBECONFIG_DATA: ${{ secrets.KUBECONFIG }} + run: | + echo "$KUBECONFIG_DATA" > "$RUNNER_TEMP/kubeconfig" + export KUBECONFIG="$RUNNER_TEMP/kubeconfig" + ./scripts/post-deploy-checks.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 59ab0c1..14487f3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,8 +2,9 @@ name: CI on: push: - branches: [main] + branches: [main, develop] pull_request: + workflow_call: jobs: backend: @@ -53,5 +54,33 @@ jobs: - run: npx prisma generate - run: npx prisma migrate deploy + - name: Verify schema datamodel matches migrations + run: npx prisma migrate diff --from-migrations prisma/migrations --to-schema-datamodel prisma/schema.prisma --exit-code + - name: Audit database constraints + run: npx tsx scripts/audit-constraints.ts --json audit.json - run: npm run build - run: npm test + # Integration tier (#135): real Postgres 16 + Redis 7 from the service + # containers above. TEST_DATABASE_URL is the admin DB, used to clone a + # migrated template database per test file. + - name: Integration tests + run: npm run test:integration + env: + TEST_DATABASE_URL: postgresql://oracledesk:oracledesk@localhost:5432/postgres + TEST_REDIS_URL: redis://localhost:6379 + + migrations: + name: Migration lint + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: actions/setup-node@v4 + with: + node-version-file: .nvmrc + + - name: Lint new migrations + run: npm run lint:migrations diff --git a/.github/workflows/deploy-env.yml b/.github/workflows/deploy-env.yml new file mode 100644 index 0000000..fee9acf --- /dev/null +++ b/.github/workflows/deploy-env.yml @@ -0,0 +1,80 @@ +name: Deploy environment + +on: + workflow_call: + inputs: + environment: + required: true + type: string + namespace: + required: true + type: string + image: + required: true + type: string + digest: + required: true + type: string + values_file: + required: true + type: string + secrets: + KUBECONFIG: + required: true + DATABASE_URL: + required: true + HELM_VALUES: + required: false + +permissions: + contents: read + id-token: write + +jobs: + deploy: + runs-on: ubuntu-latest + environment: ${{ inputs.environment }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/setup-node@v4 + with: + node-version-file: .nvmrc + cache: npm + - run: npm ci + - uses: azure/setup-helm@v4 + - name: kubeconfig + run: | + echo "${{ secrets.KUBECONFIG }}" > "$RUNNER_TEMP/kubeconfig" + echo "KUBECONFIG=$RUNNER_TEMP/kubeconfig" >> "$GITHUB_ENV" + - name: Verify image signature + uses: sigstore/cosign-installer@v3 + - run: cosign verify "${{ inputs.image }}" --certificate-identity-regexp "https://github.com/${GITHUB_REPOSITORY}/.github/workflows/cd.yml@refs/heads/main" --certificate-oidc-issuer https://token.actions.githubusercontent.com + - name: Migration lint (changed migrations only) + run: | + changed="$(git diff --name-only HEAD^ HEAD -- prisma/migrations || true)" + if [[ -n "$changed" ]]; then + printf '%s\n' "$changed" | xargs node scripts/check-migration-compat.mjs + else + echo "No migration files changed in this release" + fi + - name: Apply migrations + env: + DATABASE_URL: ${{ secrets.DATABASE_URL }} + run: npx prisma migrate deploy + - name: Helm upgrade + env: + HELM_VALUES: ${{ secrets.HELM_VALUES }} + NAMESPACE: ${{ inputs.namespace }} + IMAGE: ${{ inputs.image }} + VALUES_FILE: ${{ inputs.values_file }} + run: | + test -d deploy/helm/oracledesk-backend || { echo "Helm chart missing (deploy/helm/oracledesk-backend)"; exit 1; } + extra=() + if [[ -n "$HELM_VALUES" ]]; then printf '%s' "$HELM_VALUES" > "$RUNNER_TEMP/values-secret.yaml"; extra=(-f "$RUNNER_TEMP/values-secret.yaml"); fi + helm upgrade --install oracledesk deploy/helm/oracledesk-backend \ + --namespace "$NAMESPACE" --create-namespace \ + -f "deploy/helm/oracledesk-backend/$VALUES_FILE" "${extra[@]}" \ + --set image.repository="${IMAGE%@*}" --set image.digest="${IMAGE#*@}" \ + --atomic --timeout 10m diff --git a/.github/workflows/restore-drill.yml b/.github/workflows/restore-drill.yml new file mode 100644 index 0000000..a140046 --- /dev/null +++ b/.github/workflows/restore-drill.yml @@ -0,0 +1,62 @@ +name: Restore drill + +on: + schedule: + - cron: "0 3 1 * *" + workflow_dispatch: + +permissions: + contents: read + +jobs: + restore: + runs-on: ubuntu-latest + services: + postgres: + image: postgres:16-alpine + env: + POSTGRES_USER: oracledesk + POSTGRES_PASSWORD: oracledesk + POSTGRES_DB: oracledesk + ports: ["5432:5432"] + options: >- + --health-cmd "pg_isready -U oracledesk" + --health-interval 5s --health-timeout 5s --health-retries 10 + env: + DATABASE_URL: postgresql://oracledesk:oracledesk@localhost:5432/oracledesk + NODE_ENV: test + CHAIN_EXECUTION_MODE: dry-run + IPFS_TRANSPORT: local + SEED_ALLOW_REMOTE: "0" + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version-file: .nvmrc + cache: npm + - run: npm ci + - run: npx prisma generate + - run: npx prisma migrate deploy + - name: Seed fixture data + run: psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -f tests/fixtures/restore-seed.sql + - name: Backup + run: BACKUP_DIR="$RUNNER_TEMP/backups" ./scripts/backup.sh + - name: Wipe and restore + run: | + psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -c 'DROP SCHEMA public CASCADE; CREATE SCHEMA public;' + DUMP="$(find "$RUNNER_TEMP/backups" -name "oracledesk-*.dump" -type f | head -1)" + ./scripts/restore.sh "$DUMP" --force + - name: Reconcile against recorded RPC events + env: + RECONCILE_FAKE_RPC: tests/fixtures/events.json + run: | + npx tsx scripts/post-restore-reconcile.ts \ + --manifest "$RUNNER_TEMP"/backups/oracledesk-*-manifest.json \ + --rewind-ledgers 720 --scan-transfers \ + --out "$RUNNER_TEMP/reconcile-report.json" + - uses: actions/upload-artifact@v4 + if: always() + with: + name: reconcile-report + path: ${{ runner.temp }}/reconcile-report.json + if-no-files-found: ignore diff --git a/.gitignore b/.gitignore index 423e09b..d50f4d4 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ dist-tests recovery.dat # Build output and local artefacts +.ipfs-local/ dist/ *.log .DS_Store diff --git a/.squawk.toml b/.squawk.toml new file mode 100644 index 0000000..f5d5e41 --- /dev/null +++ b/.squawk.toml @@ -0,0 +1,4 @@ +# Squawk configuration for scripts/lint-migrations.sh (#104). +# Rules are suppressed per migration through prisma/migration-allowlist.json, +# never here. Keep this list empty. +excluded_rules = [] diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index aeb5e85..f3ca209 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -31,6 +31,8 @@ Forgot `--recurse-submodules`? Run `git submodule update --init`. ```bash npm run check:contracts # generated contract code matches the submodule npx prisma migrate deploy # against a scratch database +npx prisma migrate diff --from-migrations prisma/migrations --to-schema-datamodel prisma/schema.prisma --exit-code +npx tsx scripts/audit-constraints.ts --json audit.json npm run build npm test ``` @@ -45,6 +47,32 @@ Migrations are forward-only: never edit one that has been merged. If a migration changes data (not just shape), say so in the PR and in a comment at the top of the SQL. +When adding `CHECK` constraints or foreign keys to existing tables, follow the +`NOT VALID` -> audit -> `VALIDATE` pattern to avoid long-running exclusive locks: +1. Add the constraint or foreign key with `NOT VALID` (only acquires a fast lock). +2. Run `scripts/audit-constraints.ts [--fix]` to detect and repair or quarantine any invalid rows. +3. Validate the constraint in a subsequent migration using `ALTER TABLE ... VALIDATE CONSTRAINT ...` (takes `SHARE UPDATE EXCLUSIVE`, keeping tables available for reads and writes). + +### Migration lint + +CI (`migrations` job) runs `npm run lint:migrations` against the migrations a +PR adds. Locally, run it after committing so it can diff against the base +(`MIGRATIONS_BASE_REF=upstream/develop npm run lint:migrations`). + +- Editing, deleting or renaming a merged migration fails the lint. +- Every new migration must start with `SET lock_timeout = '5s';` so a + migration blocked by a busy table fails fast instead of queuing queries. +- New migrations are checked with a pinned [squawk](https://squawkhq.com/docs/) + (lock-heavy changes such as non-concurrent indexes, column type changes, + drops and renames are rejected). +- To suppress a rule, add an entry to `prisma/migration-allowlist.json` scoped + to one migration, listing the rules, a `reason` and an `approvedBy` reviewer: + `{ "migration": "", "rules": ["ban-drop-column"], "reason": "...", "approvedBy": "@reviewer" }`. +- Enum values: add with `ALTER TYPE ... ADD VALUE` (the new value cannot be used + in the same transaction). Removing or renaming values needs expand/contract + (stop writing the value, migrate rows, recreate the type in a later release) + and an allowlist entry. + ## Chain writes Anything that signs must go through `src/services/chain.service.ts`, which @@ -113,3 +141,27 @@ in code, tests, fixtures or logs. Tests generate throwaway keys with `Keypair.random()`. New config goes in `.env.example` with an empty value. If you commit a secret by accident, tell a maintainer privately (see [SECURITY.md](SECURITY.md)) so it can be rotated; removing it in a later commit is not enough. + +## Tests + +There are two tiers: + +| Command | What runs | Needs | +|---|---|---| +| `npm test` | Unit tests (`tests/*.test.ts`), Prisma and Redis mocked | nothing | +| `npm run test:integration` | `tests/integration/*.int.test.ts` against real Postgres 16 + Redis 7 | Docker, or `TEST_DATABASE_URL` + `TEST_REDIS_URL` | +| `npm run test:all` | both | as above | + +Integration tests get services from Testcontainers locally, or from +`TEST_DATABASE_URL` (an admin connection, used for `CREATE DATABASE`) and +`TEST_REDIS_URL` when set, as in CI. A migrated template database is created +once; each test file gets its own clone, and tests truncate all tables in +`beforeEach`. Without Docker or the `TEST_*` URLs, the tier skips with a message. + +Writing an integration test: + +- Call `await integrationEnv('