feat(health): add livez, dependency-aware readyz and admin status

.env.example

@@ -78,6 +78,17 @@ RESOLUTION_DISPUTE_WINDOW_SECS=86400
 INDEXER_ENABLED=true
 INDEXER_POLL_MS=5000
 
+# Health probes (#111): /livez (process only) and /readyz (dependencies).
+# HEALTH_CRITICAL: comma list of database,redis,rpc,indexer that fail readiness.
+HEALTH_CRITICAL=database,redis
+HEALTH_PROBE_TIMEOUT_MS=1500
+HEALTH_CACHE_TTL_MS=2000
+INDEXER_MAX_LAG_LEDGERS=120
+INDEXER_MAX_STALENESS_MS=60000
+LIVENESS_MAX_LOOP_DELAY_MS=2000
+# Drain + close must fit the 15 s forced-exit window.
+SHUTDOWN_DRAIN_MS=5000
+
 # ── LLM providers (market and trace generation) ────────────────────────────
 ANTHROPIC_API_KEY=
 GEMINI_API_KEY=

docs/api.md

@@ -4,7 +4,7 @@ This is the contract between `OracleDesk-Frontend` and this backend. The
 Soroban contracts in `OracleDesk-SmartContract/oracledesk-stellar` are the
 source of truth for anything on-chain; this API never overrides them.
 
-Base URL: `/api/v1` (liveness is `/health`, readiness is `/readyz`).
+Base URL: `/api/v1` (liveness is `/livez`, readiness is `/readyz`).
 
 ## Conventions
 
@@ -127,9 +127,13 @@ useMutation({
 
 ### Health
 
-`GET /health` → `{ status: "healthy", uptime: number }`
+`GET /livez` → `{ status: "alive" }`. Process health only (event-loop delay, fatal flag); never touches a dependency, so an outage cannot restart every instance. `503 NOT_ALIVE` when the loop is stalled.
 
-`GET /readyz` → `200` when the database is reachable, `503 NOT_READY` otherwise.
+`GET /readyz` → `200 { status: "ready", checks }` when every critical dependency is up, `503 NOT_READY` with `status: "not_ready"` otherwise, or `status: "draining"` once shutdown has begun. Each check is `{ name, ok, critical, latencyMs, error? }` for `database`, `redis`, `rpc` and `indexer`; `error` is one of `timeout`, `connection_refused`, `rpc_unhealthy`, `lagging`, `unavailable`. Probes time out after `HEALTH_PROBE_TIMEOUT_MS`, results are cached for `HEALTH_CACHE_TTL_MS` (failures for 500 ms) and concurrent calls share one run. `HEALTH_CRITICAL` (default `database,redis`) picks which dependencies fail readiness. Both endpoints bypass the rate limiter.
+
+`GET /health` → `{ status: "healthy", uptime: number }`. Deprecated: it sends `Deprecation: true` and `Link: </readyz>; rel="successor-version"`.
+
+`GET /api/v1/admin/status` (admin JWT) → probe details with latencies, indexer cursor vs RPC latest ledger and lag, last success/failure per cron job (in memory, reset on restart), chain execution mode, agent public key, LLM provider configuration and Gemini breaker state, and process info.
 
 ### Auth (signed challenge, SEP-10 style)
 

docs/architecture.md

@@ -73,6 +73,10 @@ Outcomes are decided on-chain by the resolver contract. The backend only reads `
 - The ledger cursor lives in `indexer_cursors`. If it is older than RPC retention (about 7 days on testnet), the indexer warns and resumes from the oldest retained ledger; anything in between is lost to it.
 - New `Trade` and `TracePublished` events are pushed to clients over socket.io as `TRADE_EXECUTED` and `REASONING_PUBLISHED`.
 
+## Health and degraded mode (#111)
+
+`/livez` checks only the process; `/readyz` probes Postgres, Redis, RPC and the indexer lag. `HEALTH_CRITICAL` decides which failures return 503. By default Redis and Postgres are critical (login and every data endpoint need them) and RPC is not: an instance without RPC stays ready and keeps serving database reads, while on-chain state, resolution, trace and payment endpoints return 502. The indexer fails when it is more than `INDEXER_MAX_LAG_LEDGERS` behind or its cursor is older than `INDEXER_MAX_STALENESS_MS`; with no cursor yet it reports `initialising`. On SIGTERM `/readyz` returns 503 `draining` for `SHUTDOWN_DRAIN_MS` before the server stops accepting connections.
+
 ## Generated code
 
 `src/generated/` is copied from the pinned `contracts` submodule by `scripts/sync-contracts.mjs`, with a do-not-edit header and `// @ts-nocheck`. CI fails if it's stale.

scripts/smoke.sh

@@ -17,6 +17,9 @@ need() { node -e "const j=JSON.parse(require('fs').readFileSync(0));if(!j.ok){co
 step "GET /health"
 curl -fsS "$API/health" | need 'console.log(j.data.status)'
 
+step "GET /readyz"
+curl -fsS "$API/readyz" | need 'console.log(j.data.status)'
+
 step "throwaway keypair"
 KEYS=$(node -e 'const {Keypair}=require("@stellar/stellar-sdk");const k=Keypair.random();console.log(k.publicKey()+" "+k.secret())')
 ADDRESS=${KEYS% *}; SECRET=${KEYS#* }

src/app.ts

@@ -6,18 +6,23 @@ import morgan from 'morgan';
 import hpp from 'hpp';
 
 import { config } from './config';
-import { prisma } from './lib/prisma';
 import { logger } from './lib/logger';
 import { errorMiddleware } from './middlewares/error.middleware';
 import { globalLimiter } from './middlewares/rate-limit.middleware';
 import { abortMiddleware } from './middlewares/abort.middleware';
 import router from './routes';
+import { createHealthRouter } from './routes/health.routes';
+import { health } from './services/health.service';
 
 const app = express();
 
 // BigInt (u64 market ids, i128 amounts) serialises as a decimal string.
 app.set('json replacer', (_key: string, value: unknown) => (typeof value === 'bigint' ? value.toString() : value));
 
+// ─── Liveness / Readiness (#111) ───
+// Before the rate limiter, compression and CORS: orchestrators poll often from one IP.
+app.use(createHealthRouter(health));
+
 // ─── Security ───
 app.use(helmet());
 app.use(hpp());
@@ -58,21 +63,13 @@ app.use(
 app.use(abortMiddleware);
 
 // ─── Health Check ───
+// Deprecated in favour of /livez and /readyz; response shape is unchanged.
 app.get('/health', (_req, res) => {
+  res.setHeader('Deprecation', 'true');
+  res.setHeader('Link', '</readyz>; rel="successor-version"');
   res.json({ ok: true, data: { status: 'healthy', uptime: process.uptime() }, error: null });
 });
 
-// Readiness for rolling deploys (#144): the process is only ready when it can
-// reach the database. Keep this cheap — it runs every few seconds.
-app.get('/readyz', async (_req, res) => {
-  try {
-    await prisma.$queryRaw`SELECT 1`;
-    res.json({ ok: true, data: { status: 'ready' }, error: null });
-  } catch {
-    res.status(503).json({ ok: false, data: null, error: { code: 'NOT_READY', message: 'Database is not reachable' } });
-  }
-});
-
 // ─── API Routes ───
 app.use('/api/v1', router);
 

src/config/index.ts

@@ -25,6 +25,12 @@ const optionalAddress = z
     'must be a G… account or C… contract address',
   );
 
+export const PROBE_NAMES = ['database', 'redis', 'rpc', 'indexer'] as const;
+
+const probeName = z
+  .string()
+  .refine((v) => (PROBE_NAMES as readonly string[]).includes(v), `must be one of ${PROBE_NAMES.join(', ')}`);
+
 const commaList = (item: z.ZodType<string, string>, fallback = "") =>
   z
     .string()
@@ -101,6 +107,18 @@ const envSchema = z
     INDEXER_ENABLED: z.enum(['true', 'false']).default('true').transform((v) => v === 'true'),
     INDEXER_POLL_MS: z.string().default('5000').transform(Number),
 
+    // ── Health probes (#111) ──────────────────────────────────────────────
+    // Dependencies that make /readyz fail. RPC is non-critical by default so
+    // cached DB reads can still be served while chain reads return 502.
+    HEALTH_CRITICAL: commaList(probeName, 'database,redis'),
+    HEALTH_PROBE_TIMEOUT_MS: z.string().default('1500').transform(Number),
+    HEALTH_CACHE_TTL_MS: z.string().default('2000').transform(Number),
+    INDEXER_MAX_LAG_LEDGERS: z.string().default('120').transform(Number),
+    INDEXER_MAX_STALENESS_MS: z.string().default('60000').transform(Number),
+    LIVENESS_MAX_LOOP_DELAY_MS: z.string().default('2000').transform(Number),
+    // Time /readyz reports "draining" before the HTTP server stops accepting.
+    SHUTDOWN_DRAIN_MS: z.string().default('5000').transform(Number),
+
     // ── LLM providers (optional; generation fails at runtime without one) ─
     ANTHROPIC_API_KEY: optionalString,
     GEMINI_API_KEY: optionalString,

src/controllers/admin.controller.ts

@@ -5,6 +5,13 @@ import { TraceAuditStatus } from '@prisma/client';
 import { runTraceAudit } from '../services/trace-audit.service';
 import { sendSuccess, sendError, parsePagination, buildPaginationMeta } from '../utils/response.util';
 import { logger } from '../lib/logger';
+import { config } from '../config';
+import { secret } from '../config/secrets';
+import { health } from '../services/health.service';
+import { indexerStatus } from '../services/indexer.service';
+import { agentSigner } from '../services/stellar/clients';
+import { geminiBreakerState } from '../lib/gemini';
+import { getCronStatus } from '../cron/registry';
 
 /**
  * GET /admin/trace-audit
@@ -83,3 +90,40 @@ export async function triggerTraceAuditRun(req: Request, res: Response): Promise
 
   sendSuccess(res, { runId, message: 'Trace audit run scheduled' }, 202);
 }
+
+/**
+ * GET /admin/status (#111)
+ * Probe details, indexer lag, cron runs, chain mode and LLM state. Unlike
+ * /readyz this includes ledger numbers, so it is admin-only.
+ */
+export async function getSystemStatus(_req: Request, res: Response): Promise<void> {
+  const { ready, checks } = await health.readiness();
+  const latestLedger = checks.find((c) => c.name === 'rpc')?.detail?.latestLedger;
+  const indexer = indexerStatus();
+  sendSuccess(res, {
+    ready,
+    draining: health.isDraining(),
+    checks,
+    indexer: {
+      ...indexer,
+      rpcLatestLedger: typeof latestLedger === 'number' ? latestLedger : null,
+      lagLedgers: typeof latestLedger === 'number' && indexer.lastLedger !== null ? latestLedger - indexer.lastLedger : null,
+    },
+    crons: getCronStatus(),
+    chain: {
+      mode: config.CHAIN_EXECUTION_MODE,
+      agentPublicKey: agentSigner().publicKey ?? null,
+      preflight: 'not_implemented',
+    },
+    llm: {
+      anthropicConfigured: secret('ANTHROPIC_API_KEY') !== '',
+      geminiConfigured: secret('GEMINI_API_KEY') !== '',
+      geminiBreaker: geminiBreakerState(),
+    },
+    process: {
+      uptimeSeconds: process.uptime(),
+      memory: process.memoryUsage(),
+      version: process.env.npm_package_version ?? null,
+    },
+  });
+}

src/cron/ingestion.cron.ts

@@ -1,6 +1,7 @@
 import cron from 'node-cron';
 import { runMarketMakerCycle } from '../agents/market-maker.agent';
 import { logger } from '../lib/logger';
+import { recordCronRun } from './registry';
 
 let isRunning = false;
 
@@ -22,9 +23,12 @@ export function startIngestionCron(): void {
     }
 
     isRunning = true;
+    const startedAt = new Date();
     try {
       await runMarketMakerCycle();
+      recordCronRun('ingestion', 'success', startedAt, new Date());
     } catch (err) {
+      recordCronRun('ingestion', 'failure', startedAt, new Date(), err);
       logger.error({ err }, 'Ingestion cron: cycle failed');
     } finally {
       isRunning = false;

src/cron/monitor.cron.ts

@@ -3,6 +3,7 @@ import { runTraderCycle } from '../agents/trader.agent';
 import { expireStaleSubscriptions } from '../services/subscription.service';
 import { logger } from '../lib/logger';
 import { purgeExpiredIdempotencyKeys } from '../middlewares/idempotency.middleware';
+import { recordCronRun } from './registry';
 
 let traderRunning = false;
 
@@ -24,9 +25,12 @@ export function startMonitorCron(): void {
     }
 
     traderRunning = true;
+    const startedAt = new Date();
     try {
       await runTraderCycle();
+      recordCronRun('trader', 'success', startedAt, new Date());
     } catch (err) {
+      recordCronRun('trader', 'failure', startedAt, new Date(), err);
       logger.error({ err }, 'Monitor cron: trader cycle failed');
     } finally {
       traderRunning = false;
@@ -35,19 +39,25 @@ export function startMonitorCron(): void {
 
   // Subscription expiry — every hour
   cron.schedule('0 * * * *', async () => {
+    const startedAt = new Date();
     try {
       await expireStaleSubscriptions();
+      recordCronRun('subscription-expiry', 'success', startedAt, new Date());
     } catch (err) {
+      recordCronRun('subscription-expiry', 'failure', startedAt, new Date(), err);
       logger.error({ err }, 'Monitor cron: subscription expiry failed');
     }
   });
 
   // Expired Idempotency-Key rows (24h TTL, #123) — hourly at :10, in batches of 5 000
   cron.schedule('10 * * * *', async () => {
+    const startedAt = new Date();
     try {
       const deleted = await purgeExpiredIdempotencyKeys();
+      recordCronRun('idempotency-purge', 'success', startedAt, new Date());
       if (deleted > 0) logger.info({ deleted }, 'Monitor cron: purged expired idempotency keys');
     } catch (err) {
+      recordCronRun('idempotency-purge', 'failure', startedAt, new Date(), err);
       logger.error({ err }, 'Monitor cron: idempotency key purge failed');
     }
   });

src/cron/registry.ts

@@ -0,0 +1,26 @@
+export type CronOutcome = 'success' | 'failure';
+
+interface CronRun {
+  startedAt: string;
+  finishedAt: string;
+  error?: string;
+}
+
+const lastRuns = new Map<string, { lastSuccess?: CronRun; lastFailure?: CronRun }>();
+
+/** In-memory last success/failure per cron job (#111); not persisted. */
+export function recordCronRun(job: string, outcome: CronOutcome, startedAt: Date, finishedAt: Date, error?: unknown): void {
+  const entry = lastRuns.get(job) ?? {};
+  const run: CronRun = { startedAt: startedAt.toISOString(), finishedAt: finishedAt.toISOString() };
+  if (outcome === 'success') {
+    entry.lastSuccess = run;
+  } else {
+    run.error = error instanceof Error ? error.message : String(error);
+    entry.lastFailure = run;
+  }
+  lastRuns.set(job, entry);
+}
+
+export function getCronStatus(): Record<string, { lastSuccess?: CronRun; lastFailure?: CronRun }> {
+  return Object.fromEntries(lastRuns);
+}

src/lib/gemini.ts

@@ -56,6 +56,13 @@ function isQuotaExhausted(): boolean {
   return Date.now() < _quotaExhaustedUntil;
 }
 
+/** Read-only breaker state for the admin status endpoint. */
+export function geminiBreakerState(): { open: boolean; until: string | null } {
+  return isQuotaExhausted()
+    ? { open: true, until: new Date(_quotaExhaustedUntil).toISOString() }
+    : { open: false, until: null };
+}
+
 function markQuotaExhausted(): void {
   _quotaExhaustedUntil = Date.now() + QUOTA_COOLDOWN_MS;
   logger.error(

src/routes/admin.routes.ts

@@ -1,11 +1,12 @@
 import { Router } from 'express';
 import { requireAuth, requireAdmin } from '../middlewares/auth.middleware';
-import { getTraceAuditSummary, triggerTraceAuditRun } from '../controllers/admin.controller';
+import { getTraceAuditSummary, triggerTraceAuditRun, getSystemStatus } from '../controllers/admin.controller';
 
 const router = Router();
 
 router.use(requireAuth, requireAdmin);
 
+router.get('/status', getSystemStatus);
 router.get('/trace-audit', getTraceAuditSummary);
 router.post('/trace-audit/run', triggerTraceAuditRun);
 

src/routes/health.routes.ts

@@ -0,0 +1,41 @@
+import { Router } from 'express';
+import type { HealthService } from '../services/health.service';
+
+/**
+ * /livez and /readyz (#111). Mounted before the global rate limiter so
+ * orchestrators polling from one IP are never throttled. Responses carry only
+ * name, ok, critical, latencyMs and a sanitised error class; probe detail
+ * (ledger numbers, lag) is admin-only.
+ */
+export function createHealthRouter(health: HealthService): Router {
+  const router = Router();
+
+  router.get('/livez', (_req, res) => {
+    const live = health.liveness();
+    if (live.alive) {
+      res.json({ ok: true, data: { status: 'alive' }, error: null });
+      return;
+    }
+    res.status(503).json({ ok: false, data: { status: 'unhealthy' }, error: { code: 'NOT_ALIVE', message: 'Process is not healthy' } });
+  });
+
+  router.get('/readyz', async (_req, res) => {
+    if (health.isDraining()) {
+      res.status(503).json({ ok: false, data: { status: 'draining', checks: [] }, error: { code: 'NOT_READY', message: 'Server is shutting down' } });
+      return;
+    }
+    const { ready, checks } = await health.readiness();
+    const publicChecks = checks.map(({ name, ok, critical, latencyMs, error }) => ({ name, ok, critical, latencyMs, ...(error ? { error } : {}) }));
+    if (ready) {
+      res.json({ ok: true, data: { status: 'ready', checks: publicChecks }, error: null });
+      return;
+    }
+    res.status(503).json({
+      ok: false,
+      data: { status: 'not_ready', checks: publicChecks },
+      error: { code: 'NOT_READY', message: 'A critical dependency is unavailable' },
+    });
+  });
+
+  return router;
+}