docs(api): document exact /status schema separating process vs data health (#273)

README.md

@@ -270,24 +270,61 @@ any other. See [`docs/websocket-backpressure.md`](docs/websocket-backpressure.md
 
 Lens gates `/price`, `/pools`, and `/candles` behind x402 micropayments on Stellar (testnet by default). The `/status` endpoint is free.
 
-### 1. Free health check (no payment)
+### 1. Free health check & status monitoring (`/status`)
+
+`/status` provides a structured, comprehensive health contract separating **process health** (runtime container, memory, uptime) from **data health** (ingestion currency, last ledger, ingest lag).
 
 ```bash
 curl http://localhost:3002/status
-# {
-#   "ok": true,
-#   "network": "testnet",
-#   "watchedPairs": ["XLM:native/USDC:GA5ZSEJYB37JRC5AVCIA5MOP4RHTM335X2KGX3IHOJAPP5RE34K4KZVN"],
-#   "lastIndexedLedger": 53842917,
-#   "lastProcessedAt": "2026-05-07T18:45:11.220Z",
-#   "ingestLagSeconds": 4
-# }
 ```
 
-`/status` answers for one network — pass `?network=mainnet` (or the
-`x-network` header) to check the mainnet indexer instead. `ingestLagSeconds` is
-`null` until that network has ingested, and `lastIndexedLedger` is `null` until
-its SDEX ingester has recorded a trade.
+```json
+{
+  "ok": true,
+  "status": "healthy",
+  "network": "testnet",
+  "health": {
+    "process": "healthy",
+    "data": "healthy"
+  },
+  "process": {
+    "uptimeSeconds": 86420,
+    "memoryUsageMb": 18.42,
+    "status": "healthy"
+  },
+  "data": {
+    "lastIndexedLedger": 53842917,
+    "lastProcessedAt": "2026-05-07T18:45:11.220Z",
+    "ingestLagSeconds": 4,
+    "isStale": false
+  },
+  "watchedPairs": ["XLM:native/USDC:GA5ZSEJYB37JRC5AVCIA5MOP4RHTM335X2KGX3IHOJAPP5RE34K4KZVN"],
+  "lastIndexedLedger": 53842917,
+  "lastProcessedAt": "2026-05-07T18:45:11.220Z",
+  "ingestLagSeconds": 4
+}
+```
+
+#### Health Differentiation & Paging Conditions
+- **Process Health vs Data Health:** The HTTP status code distinguishes service availability from data staleness.
+  - `HTTP 200 OK` (`status: "healthy"`): Both process runtime and data feeds are functioning normally.
+  - `HTTP 200 OK` (`status: "degraded"`, `health.data: "stale"`): The API server is healthy and responding, but ingestion lag exceeds 300 seconds. **Do not page engineers for intermittent upstream ledger delays.**
+  - `HTTP 503 Service Unavailable` (`pagingCondition: "DATABASE_DOWN"`): Critical infrastructure failure (e.g. Postgres is down or unreachable). **Uptime monitors should page on HTTP 503.**
+
+#### Monitoring Configuration Example (Datadog / Prometheus / BetterStack)
+Configure your synthetic monitor / HTTP probe:
+```yaml
+# Prometheus Blackbox / Uptime monitor probe example
+probe:
+  target: "http://lens-api.internal:3002/status?network=mainnet"
+  method: "GET"
+  expect_status: 200
+  alert_conditions:
+    - severity: critical # Page on-call
+      condition: "status_code == 503 || body.pagingCondition == 'DATABASE_DOWN'"
+    - severity: warning  # Slack notification only
+      condition: "body.health.data == 'stale' || body.data.ingestLagSeconds > 300"
+```
 
 ### 2. Paid request without `X-PAYMENT` → `402` with payment requirements
 

openapi.json

@@ -192,7 +192,7 @@
         ],
         "responses": {
           "200": {
-            "description": "API status",
+            "description": "API status and health diagnostics",
             "headers": {
               "Cache-Control": {
                 "$ref": "#/components/headers/CacheControl"
@@ -206,13 +206,67 @@
                     "ok": {
                       "type": "boolean"
                     },
+                    "status": {
+                      "type": "string",
+                      "enum": [
+                        "healthy",
+                        "degraded",
+                        "unhealthy"
+                      ]
+                    },
                     "network": {
                       "type": "string",
                       "enum": [
                         "testnet",
                         "mainnet"
                       ]
                     },
+                    "health": {
+                      "type": "object",
+                      "properties": {
+                        "process": {
+                          "type": "string"
+                        },
+                        "data": {
+                          "type": "string"
+                        }
+                      }
+                    },
+                    "process": {
+                      "type": "object",
+                      "properties": {
+                        "uptimeSeconds": {
+                          "type": "number"
+                        },
+                        "memoryUsageMb": {
+                          "type": "number"
+                        },
+                        "status": {
+                          "type": "string"
+                        }
+                      }
+                    },
+                    "data": {
+                      "type": "object",
+                      "properties": {
+                        "lastIndexedLedger": {
+                          "type": "integer",
+                          "nullable": true
+                        },
+                        "lastProcessedAt": {
+                          "type": "string",
+                          "format": "date-time",
+                          "nullable": true
+                        },
+                        "ingestLagSeconds": {
+                          "type": "integer",
+                          "nullable": true
+                        },
+                        "isStale": {
+                          "type": "boolean"
+                        }
+                      }
+                    },
                     "watchedPairs": {
                       "type": "array",
                       "items": {
@@ -239,6 +293,30 @@
           },
           "429": {
             "$ref": "#/components/responses/TooManyRequests"
+          },
+          "503": {
+            "description": "Service unhealthy or database unreachable",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "type": "object",
+                  "properties": {
+                    "ok": {
+                      "type": "boolean"
+                    },
+                    "status": {
+                      "type": "string"
+                    },
+                    "error": {
+                      "type": "string"
+                    },
+                    "pagingCondition": {
+                      "type": "string"
+                    }
+                  }
+                }
+              }
+            }
           }
         }
       }

openapi.yaml

@@ -152,7 +152,7 @@ paths:
         - $ref: '#/components/parameters/NetworkQuery'
       responses:
         '200':
-          description: API status
+          description: API status and health diagnostics
           headers:
             Cache-Control:
               $ref: '#/components/headers/CacheControl'
@@ -163,9 +163,43 @@ paths:
                 properties:
                   ok:
                     type: boolean
+                  status:
+                    type: string
+                    enum: [healthy, degraded, unhealthy]
                   network:
                     type: string
                     enum: [testnet, mainnet]
+                  health:
+                    type: object
+                    properties:
+                      process:
+                        type: string
+                      data:
+                        type: string
+                  process:
+                    type: object
+                    properties:
+                      uptimeSeconds:
+                        type: number
+                      memoryUsageMb:
+                        type: number
+                      status:
+                        type: string
+                  data:
+                    type: object
+                    properties:
+                      lastIndexedLedger:
+                        type: integer
+                        nullable: true
+                      lastProcessedAt:
+                        type: string
+                        format: date-time
+                        nullable: true
+                      ingestLagSeconds:
+                        type: integer
+                        nullable: true
+                      isStale:
+                        type: boolean
                   watchedPairs:
                     type: array
                     items:
@@ -180,6 +214,21 @@ paths:
                   ingestLagSeconds:
                     type: integer
                     nullable: true
+        '503':
+          description: Service unhealthy or database unreachable
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  ok:
+                    type: boolean
+                  status:
+                    type: string
+                  error:
+                    type: string
+                  pagingCondition:
+                    type: string
         '429':
           $ref: '#/components/responses/TooManyRequests'
 

src/__tests__/statusContract.test.ts

@@ -0,0 +1,102 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'
+import type { FastifyInstance } from 'fastify'
+
+const mockQuery = vi.fn()
+vi.mock('../db', () => ({
+  pgPool: {
+    query: (...args: any[]) => mockQuery(...args),
+  },
+  prisma: {
+    apiKey: {
+      findUnique: vi.fn(),
+    },
+  },
+}))
+
+import { buildApp } from '../app'
+
+describe('GET /status contract & health separation (#273)', () => {
+  let app: FastifyInstance
+
+  beforeEach(async () => {
+    mockQuery.mockReset()
+    app = await buildApp()
+    await app.ready()
+  })
+
+  afterEach(async () => {
+    await app.close()
+  })
+
+  it('returns 200 with separate process and data health fields when healthy', async () => {
+    const recentDate = new Date(Date.now() - 30 * 1000) // 30s ago
+    mockQuery.mockResolvedValueOnce({
+      rows: [
+        {
+          last_ledger: 123456,
+          last_processed_at: recentDate,
+        },
+      ],
+    })
+
+    const res = await app.inject({
+      method: 'GET',
+      url: '/status',
+    })
+
+    expect(res.statusCode).toBe(200)
+    const body = res.json()
+    expect(body.ok).toBe(true)
+    expect(body.status).toBe('healthy')
+    expect(body.health).toBeDefined()
+    expect(body.health.process).toBe('healthy')
+    expect(body.health.data).toBe('healthy')
+    expect(body.process.status).toBe('healthy')
+    expect(typeof body.process.uptimeSeconds).toBe('number')
+    expect(typeof body.process.memoryUsageMb).toBe('number')
+    expect(body.data.isStale).toBe(false)
+    expect(body.data.lastIndexedLedger).toBe(123456)
+  })
+
+  it('distinguishes process health from data health when data is stale', async () => {
+    const staleDate = new Date(Date.now() - 600 * 1000) // 10 minutes ago (> 300s)
+    mockQuery.mockResolvedValueOnce({
+      rows: [
+        {
+          last_ledger: 100000,
+          last_processed_at: staleDate,
+        },
+      ],
+    })
+
+    const res = await app.inject({
+      method: 'GET',
+      url: '/status',
+    })
+
+    expect(res.statusCode).toBe(200)
+    const body = res.json()
+    expect(body.ok).toBe(true)
+    expect(body.status).toBe('degraded')
+    expect(body.health.process).toBe('healthy') // Process is still running fine
+    expect(body.health.data).toBe('stale') // Data ingestion is stale
+    expect(body.data.isStale).toBe(true)
+    expect(body.data.ingestLagSeconds).toBeGreaterThan(300)
+  })
+
+  it('returns 503 with DATABASE_DOWN paging condition when database fails', async () => {
+    mockQuery.mockRejectedValueOnce(new Error('Connection terminated unexpectedly'))
+
+    const res = await app.inject({
+      method: 'GET',
+      url: '/status',
+    })
+
+    expect(res.statusCode).toBe(503)
+    const body = res.json()
+    expect(body.ok).toBe(false)
+    expect(body.status).toBe('unhealthy')
+    expect(body.pagingCondition).toBe('DATABASE_DOWN')
+    expect(body.error).toContain('Connection terminated')
+  })
+})