diff --git a/README.md b/README.md index 86430e78..692b7881 100644 --- a/README.md +++ b/README.md @@ -347,24 +347,61 @@ any other. See [`docs/websocket-backpressure.md`](docs/websocket-backpressure.md Lens gates `/price`, `/pools`, and `/candles` behind x402 micropayments on Stellar (testnet by default). The `/status` endpoint is free. -### 1. Free health check (no payment) +### 1. Free health check & status monitoring (`/status`) + +`/status` provides a structured, comprehensive health contract separating **process health** (runtime container, memory, uptime) from **data health** (ingestion currency, last ledger, ingest lag). ```bash curl http://localhost:3002/status -# { -# "ok": true, -# "network": "testnet", -# "watchedPairs": ["XLM:native/USDC:GA5ZSEJYB37JRC5AVCIA5MOP4RHTM335X2KGX3IHOJAPP5RE34K4KZVN"], -# "lastIndexedLedger": 53842917, -# "lastProcessedAt": "2026-05-07T18:45:11.220Z", -# "ingestLagSeconds": 4 -# } ``` -`/status` answers for one network — pass `?network=mainnet` (or the -`x-network` header) to check the mainnet indexer instead. `ingestLagSeconds` is -`null` until that network has ingested, and `lastIndexedLedger` is `null` until -its SDEX ingester has recorded a trade. +```json +{ + "ok": true, + "status": "healthy", + "network": "testnet", + "health": { + "process": "healthy", + "data": "healthy" + }, + "process": { + "uptimeSeconds": 86420, + "memoryUsageMb": 18.42, + "status": "healthy" + }, + "data": { + "lastIndexedLedger": 53842917, + "lastProcessedAt": "2026-05-07T18:45:11.220Z", + "ingestLagSeconds": 4, + "isStale": false + }, + "watchedPairs": ["XLM:native/USDC:GA5ZSEJYB37JRC5AVCIA5MOP4RHTM335X2KGX3IHOJAPP5RE34K4KZVN"], + "lastIndexedLedger": 53842917, + "lastProcessedAt": "2026-05-07T18:45:11.220Z", + "ingestLagSeconds": 4 +} +``` + +#### Health Differentiation & Paging Conditions +- **Process Health vs Data Health:** The HTTP status code distinguishes service availability from data staleness. + - `HTTP 200 OK` (`status: "healthy"`): Both process runtime and data feeds are functioning normally. + - `HTTP 200 OK` (`status: "degraded"`, `health.data: "stale"`): The API server is healthy and responding, but ingestion lag exceeds 300 seconds. **Do not page engineers for intermittent upstream ledger delays.** + - `HTTP 503 Service Unavailable` (`pagingCondition: "DATABASE_DOWN"`): Critical infrastructure failure (e.g. Postgres is down or unreachable). **Uptime monitors should page on HTTP 503.** + +#### Monitoring Configuration Example (Datadog / Prometheus / BetterStack) +Configure your synthetic monitor / HTTP probe: +```yaml +# Prometheus Blackbox / Uptime monitor probe example +probe: + target: "http://lens-api.internal:3002/status?network=mainnet" + method: "GET" + expect_status: 200 + alert_conditions: + - severity: critical # Page on-call + condition: "status_code == 503 || body.pagingCondition == 'DATABASE_DOWN'" + - severity: warning # Slack notification only + condition: "body.health.data == 'stale' || body.data.ingestLagSeconds > 300" +``` ### 2. Paid request without `X-PAYMENT` → `402` with payment requirements diff --git a/openapi.json b/openapi.json index 1afa5785..96703c47 100644 --- a/openapi.json +++ b/openapi.json @@ -192,7 +192,7 @@ ], "responses": { "200": { - "description": "API status", + "description": "API status and health diagnostics", "headers": { "Cache-Control": { "$ref": "#/components/headers/CacheControl" @@ -206,6 +206,14 @@ "ok": { "type": "boolean" }, + "status": { + "type": "string", + "enum": [ + "healthy", + "degraded", + "unhealthy" + ] + }, "network": { "type": "string", "enum": [ @@ -213,6 +221,52 @@ "mainnet" ] }, + "health": { + "type": "object", + "properties": { + "process": { + "type": "string" + }, + "data": { + "type": "string" + } + } + }, + "process": { + "type": "object", + "properties": { + "uptimeSeconds": { + "type": "number" + }, + "memoryUsageMb": { + "type": "number" + }, + "status": { + "type": "string" + } + } + }, + "data": { + "type": "object", + "properties": { + "lastIndexedLedger": { + "type": "integer", + "nullable": true + }, + "lastProcessedAt": { + "type": "string", + "format": "date-time", + "nullable": true + }, + "ingestLagSeconds": { + "type": "integer", + "nullable": true + }, + "isStale": { + "type": "boolean" + } + } + }, "watchedPairs": { "type": "array", "items": { @@ -239,6 +293,30 @@ }, "429": { "$ref": "#/components/responses/TooManyRequests" + }, + "503": { + "description": "Service unhealthy or database unreachable", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "ok": { + "type": "boolean" + }, + "status": { + "type": "string" + }, + "error": { + "type": "string" + }, + "pagingCondition": { + "type": "string" + } + } + } + } + } } } } diff --git a/openapi.yaml b/openapi.yaml index 50c63ee1..06e5644d 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -157,7 +157,7 @@ paths: - $ref: '#/components/parameters/NetworkQuery' responses: '200': - description: API status + description: API status and health diagnostics headers: Cache-Control: $ref: '#/components/headers/CacheControl' @@ -168,9 +168,43 @@ paths: properties: ok: type: boolean + status: + type: string + enum: [healthy, degraded, unhealthy] network: type: string enum: [testnet, mainnet] + health: + type: object + properties: + process: + type: string + data: + type: string + process: + type: object + properties: + uptimeSeconds: + type: number + memoryUsageMb: + type: number + status: + type: string + data: + type: object + properties: + lastIndexedLedger: + type: integer + nullable: true + lastProcessedAt: + type: string + format: date-time + nullable: true + ingestLagSeconds: + type: integer + nullable: true + isStale: + type: boolean watchedPairs: type: array items: @@ -185,6 +219,21 @@ paths: ingestLagSeconds: type: integer nullable: true + '503': + description: Service unhealthy or database unreachable + content: + application/json: + schema: + type: object + properties: + ok: + type: boolean + status: + type: string + error: + type: string + pagingCondition: + type: string '429': $ref: '#/components/responses/TooManyRequests' diff --git a/src/__tests__/statusContract.test.ts b/src/__tests__/statusContract.test.ts new file mode 100644 index 00000000..f39913b7 --- /dev/null +++ b/src/__tests__/statusContract.test.ts @@ -0,0 +1,102 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest' +import type { FastifyInstance } from 'fastify' + +const mockQuery = vi.fn() +vi.mock('../db', () => ({ + pgPool: { + query: (...args: any[]) => mockQuery(...args), + }, + prisma: { + apiKey: { + findUnique: vi.fn(), + }, + }, +})) + +import { buildApp } from '../app' + +describe('GET /status contract & health separation (#273)', () => { + let app: FastifyInstance + + beforeEach(async () => { + mockQuery.mockReset() + app = await buildApp() + await app.ready() + }) + + afterEach(async () => { + await app.close() + }) + + it('returns 200 with separate process and data health fields when healthy', async () => { + const recentDate = new Date(Date.now() - 30 * 1000) // 30s ago + mockQuery.mockResolvedValueOnce({ + rows: [ + { + last_ledger: 123456, + last_processed_at: recentDate, + }, + ], + }) + + const res = await app.inject({ + method: 'GET', + url: '/status', + }) + + expect(res.statusCode).toBe(200) + const body = res.json() + expect(body.ok).toBe(true) + expect(body.status).toBe('healthy') + expect(body.health).toBeDefined() + expect(body.health.process).toBe('healthy') + expect(body.health.data).toBe('healthy') + expect(body.process.status).toBe('healthy') + expect(typeof body.process.uptimeSeconds).toBe('number') + expect(typeof body.process.memoryUsageMb).toBe('number') + expect(body.data.isStale).toBe(false) + expect(body.data.lastIndexedLedger).toBe(123456) + }) + + it('distinguishes process health from data health when data is stale', async () => { + const staleDate = new Date(Date.now() - 600 * 1000) // 10 minutes ago (> 300s) + mockQuery.mockResolvedValueOnce({ + rows: [ + { + last_ledger: 100000, + last_processed_at: staleDate, + }, + ], + }) + + const res = await app.inject({ + method: 'GET', + url: '/status', + }) + + expect(res.statusCode).toBe(200) + const body = res.json() + expect(body.ok).toBe(true) + expect(body.status).toBe('degraded') + expect(body.health.process).toBe('healthy') // Process is still running fine + expect(body.health.data).toBe('stale') // Data ingestion is stale + expect(body.data.isStale).toBe(true) + expect(body.data.ingestLagSeconds).toBeGreaterThan(300) + }) + + it('returns 503 with DATABASE_DOWN paging condition when database fails', async () => { + mockQuery.mockRejectedValueOnce(new Error('Connection terminated unexpectedly')) + + const res = await app.inject({ + method: 'GET', + url: '/status', + }) + + expect(res.statusCode).toBe(503) + const body = res.json() + expect(body.ok).toBe(false) + expect(body.status).toBe('unhealthy') + expect(body.pagingCondition).toBe('DATABASE_DOWN') + expect(body.error).toContain('Connection terminated') + }) +}) diff --git a/src/api/rest.ts b/src/api/rest.ts index e6d9905e..ae557ecf 100644 --- a/src/api/rest.ts +++ b/src/api/rest.ts @@ -8,6 +8,7 @@ import { config, getNetworkConfig, activeNetwork, type NetworkName } from '../co import '../middleware/network' // declares req.network on the FastifyRequest type import { statusResponseSchema, + statusErrorResponseSchema, priceResponseSchema, routeResponseSchema, historyResponseSchema, @@ -73,43 +74,85 @@ export async function registerRESTRoutes(app: FastifyInstance) { // up the validating serializer. installResponseValidation(app) - // GET /status — public health/monitoring endpoint (no API key required) - app.get('/status', { config: { public: true }, schema: { response: { 200: statusResponseSchema } } }, async (req) => { - const network = req.network ?? activeNetwork - const result = await pgPool.query( - `SELECT last_ledger, last_processed_at - FROM indexer_state - WHERE network = $1 - ORDER BY updated_at DESC - LIMIT 1`, - [network] - ) - const lastProcessedAt = result.rows[0]?.last_processed_at ?? null - return { - ok: true, - network, - watchedPairs: getNetworkConfig(network).pairs.map(p => p.pairKey), - lastIndexedLedger: result.rows[0]?.last_ledger ?? null, - lastProcessedAt, - // Seconds since the last write for this network — a stalled ingester is - // visible to `/status` polling without needing Prometheus. Null until the - // network has ingested at least once. - ingestLagSeconds: lastProcessedAt + // GET /status — public health/monitoring endpoint (no API key required) (#273) + app.get( + '/status', + { + config: { public: true }, + schema: { + response: { + 200: statusResponseSchema, + 503: statusErrorResponseSchema, + }, + }, + }, + async (req, reply) => { + const network = req.network ?? activeNetwork + const mem = process.memoryUsage() + const uptimeSeconds = Math.round(process.uptime()) + const memoryUsageMb = Math.round((mem.heapUsed / 1024 / 1024) * 100) / 100 + + let result + try { + result = await pgPool.query( + `SELECT last_ledger, last_processed_at + FROM indexer_state + WHERE network = $1 + ORDER BY updated_at DESC + LIMIT 1`, + [network] + ) + } catch (err: any) { + return reply.status(503).send({ + ok: false, + status: 'unhealthy', + error: err?.message || 'Database query failed', + pagingCondition: 'DATABASE_DOWN', + }) + } + + const lastProcessedAt = result.rows[0]?.last_processed_at ?? null + const lastIndexedLedger = result.rows[0]?.last_ledger ?? null + const ingestLagSeconds = lastProcessedAt ? Math.max(0, Math.round((Date.now() - new Date(lastProcessedAt).getTime()) / 1000)) - : null, - // Ingesters that have crashed at least once on this network, newest - // first. An ingester that crash-loops but restarts successfully is - // otherwise invisible here: the process is up and the port answers. - // Empty when nothing has crashed. Messages only — never a stack, since - // this is served over HTTP. - crashes: listCrashes(network).map(c => ({ - name: c.name, - network: c.network, - message: c.message, - at: c.at.toISOString(), - })), + : null + const isStale = ingestLagSeconds !== null && ingestLagSeconds > 300 + const processHealth = 'healthy' as const + const dataHealth = isStale ? ('stale' as const) : ('healthy' as const) + const overallStatus = isStale ? ('degraded' as const) : ('healthy' as const) + + return { + ok: true, + status: overallStatus, + network, + health: { + process: processHealth, + data: dataHealth, + }, + process: { + uptimeSeconds, + memoryUsageMb, + status: 'healthy', + }, + data: { + lastIndexedLedger, + lastProcessedAt, + ingestLagSeconds, + isStale, + }, + watchedPairs: getNetworkConfig(network).pairs.map(p => p.pairKey), + lastIndexedLedger, + lastProcessedAt, + ingestLagSeconds, + crashes: listCrashes(network).map(c => ({ + name: c.name, + network: c.network, + message: c.message, + at: c.at.toISOString(), + })), + } } - }) + ) // GET /price/:assetA/:assetB diff --git a/src/api/schemas.ts b/src/api/schemas.ts index ddbaff53..126ff2af 100644 --- a/src/api/schemas.ts +++ b/src/api/schemas.ts @@ -65,22 +65,56 @@ export const readyResponseSchema = { }, } as const -/** GET /status */ +/** GET /status (#273) */ export const statusResponseSchema = { type: 'object', - required: ['ok', 'network', 'watchedPairs', 'lastIndexedLedger', 'lastProcessedAt', 'ingestLagSeconds'], + required: [ + 'ok', + 'status', + 'network', + 'health', + 'process', + 'data', + 'watchedPairs', + 'lastIndexedLedger', + 'lastProcessedAt', + 'ingestLagSeconds', + ], additionalProperties: false, properties: { ok: { type: 'boolean' }, + status: { type: 'string', enum: ['healthy', 'degraded', 'unhealthy'] }, network: { type: 'string', enum: ['testnet', 'mainnet'] }, + health: { + type: 'object', + required: ['process', 'data'], + properties: { + process: { type: 'string', enum: ['healthy', 'degraded'] }, + data: { type: 'string', enum: ['healthy', 'stale'] }, + }, + }, + process: { + type: 'object', + required: ['uptimeSeconds', 'memoryUsageMb', 'status'], + properties: { + uptimeSeconds: { type: 'number' }, + memoryUsageMb: { type: 'number' }, + status: { type: 'string' }, + }, + }, + data: { + type: 'object', + required: ['lastIndexedLedger', 'lastProcessedAt', 'ingestLagSeconds', 'isStale'], + properties: { + lastIndexedLedger: { type: ['integer', 'null'] }, + lastProcessedAt: {}, + ingestLagSeconds: { type: ['integer', 'null'] }, + isStale: { type: 'boolean' }, + }, + }, watchedPairs: { type: 'array', items: { type: 'string' } }, lastIndexedLedger: { type: ['integer', 'null'] }, - // The pg driver returns a timestamp column as a JS Date; the validating - // serializer sees that pre-serialization object, so accept Date | string | - // null here (a Date stringifies to an ISO string in the response body). lastProcessedAt: {}, - // Seconds since the last ingester write for this network, null before the - // first one. Lets `/status` reveal a stalled ingester on its own. ingestLagSeconds: { type: ['integer', 'null'] }, // Ingesters that have crashed at least once on this network, newest first, // and empty when none have. Present so a crash-looping ingester that keeps @@ -103,6 +137,17 @@ export const statusResponseSchema = { }, } as const +export const statusErrorResponseSchema = { + type: 'object', + required: ['ok', 'status', 'error', 'pagingCondition'], + properties: { + ok: { type: 'boolean', const: false }, + status: { type: 'string', const: 'unhealthy' }, + error: { type: 'string' }, + pagingCondition: { type: 'string' }, + }, +} as const + /** GET /price/:assetA/:assetB */ export const priceResponseSchema = { type: 'object',