From dd38633c013c4129b602151528abc0ce12013c9b Mon Sep 17 00:00:00 2001 From: gorrettivibes-coder Date: Sun, 18 Oct 2026 15:46:47 +0100 Subject: [PATCH] Add auth metrics and global limits Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- README.md | 27 +++++++++- docs/http-metrics.md | 55 ++++++++++++++++++-- prisma/schema.prisma | 1 + src/__tests__/auth.test.ts | 72 +++++++++++++++++++++++++++ src/__tests__/globalRateLimit.test.ts | 68 +++++++++++++++++++++++++ src/api/admin.ts | 17 ++++++- src/api/auth.ts | 64 ++++++++++++++++++++---- src/api/usage.ts | 7 +++ src/app.ts | 23 ++++++--- src/metrics.ts | 26 ++++++++++ src/middleware/globalRateLimit.ts | 68 +++++++++++++++++++++++++ src/routes/pairs.ts | 8 ++- 12 files changed, 411 insertions(+), 25 deletions(-) create mode 100644 src/__tests__/globalRateLimit.test.ts create mode 100644 src/middleware/globalRateLimit.ts diff --git a/README.md b/README.md index 13334c3b..6640d5d8 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,24 @@ disagree, the spec is the thing to fix. Still reading across both networks, and ignoring the parameter: `/candles/:assetA/:assetB`, `/price/twap/*`, and `/price/vwap/*`. +### API key authentication errors + +API-key authentication failures return HTTP 401 with a machine-readable `code`: + +| Code | Meaning | +|---|---| +| `missing` | No Authorization header was supplied. | +| `malformed` | The Authorization header is not a valid Bearer header. | +| `unknown` | The key was not recognized; never-issued and unknown keys have the same response. | +| `revoked` | The recognized key has been revoked. | +| `expired` | The recognized key has passed its expiry time. | + +Unknown keys always receive the same response, regardless of whether the key +ever existed. Admin-token failures use `admin_unauthorized`. + +Admin key creation accepts an optional future ISO date-time in `expiresAt`. +Omitting it creates a key without an expiry. + ```bash curl "https://api.example.com/price/XLM/USDC?network=mainnet" ``` @@ -108,13 +126,17 @@ The GraphQL schema is exported as a canonical Schema Definition Language (SDL) a Prometheus metrics are exposed on `GET /metrics` (public, no API key required). -Alongside the ingestion and database metrics, Lens exports the three HTTP -signals needed to answer "is the API healthy": +Alongside the ingestion and database metrics, Lens exports HTTP request, +latency, authentication and rate-limit signals: | Metric | Type | Labels | |---|---|---| | `http_requests_total` | Counter | `method`, `route`, `status_class` | | `http_request_duration_seconds` | Histogram | `method`, `route` | +| `api_key_auth_failures_total` | Counter | `reason` | +| `admin_auth_failures_total` | Counter | — | +| `rate_limit_rejections_total` | Counter | `scope` | +| `global_shed_requests_total` | Counter | — | `route` is the matched route **template** (`/price/:assetA/:assetB`), never the resolved URL, so the number of time series stays bounded no matter how many @@ -515,6 +537,7 @@ curl -s http://localhost:3002/price/XLM/USDC | jq '{price, ammPrice, lastUpdated | `NODE_ENV` | Environment mode (`development`, `test`, `production`) | `development` | No | | `PORT` | API server port | `3002` | No | | `HOST` | API server host | `0.0.0.0` | No | +| `GLOBAL_RATE_LIMIT_MAX` | Maximum accepted non-health requests per rolling minute per instance | `1000` | No | | `DATABASE_URL` | PostgreSQL connection string | - | **Yes** | | `REDIS_URL` | Redis connection string | - | **Yes** | | `PRICE_CACHE_TTL` | Cache duration for price data (seconds) | `10` | No | diff --git a/docs/http-metrics.md b/docs/http-metrics.md index 4965f6f4..f7d9ce4b 100644 --- a/docs/http-metrics.md +++ b/docs/http-metrics.md @@ -7,10 +7,10 @@ Lens exports Prometheus metrics for the HTTP layer on the existing public is different: they are labelled by `network` so the per-network ingester loops cannot overwrite each other's series. -These three signals — request rate, error rate and p95 latency — are what answer -"is the API healthy". The pre-existing `price_requests_total` counts price calls -specifically and cannot answer that question; it is unchanged and still means -what it always meant. +Request rate, error rate, latency, authentication failures and load shedding +help answer "is the API healthy". The pre-existing `price_requests_total` +counts price calls specifically and cannot answer that question; it is +unchanged and still means what it always meant. The same cardinality rule applies to every metric family: labels must come from fixed buckets or server-defined values. Never put raw request paths, asset @@ -37,6 +37,53 @@ Buckets: `0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10`. Observed for every request that produces a response, successful or not. +### `api_key_auth_failures_total` + +Counter. Label: `reason`, with the bounded values `missing`, `malformed`, +`unknown`, `revoked` and `expired`. The `reason` value matches the +machine-readable `code` in the corresponding API-key 401 response. + +### `admin_auth_failures_total` + +Counter. No labels. This tracks rejected authentication attempts on admin +routes independently of API-key failures. + +### Rate-limit counters + +`rate_limit_rejections_total` is a counter labelled by the bounded `scope` +values `key` and `ip`; it counts requests rejected by the per-key or fallback +IP limiter. Those requests return HTTP 429 and code `PER_KEY_RATE_LIMITED` or +`IP_RATE_LIMITED`. + +`global_shed_requests_total` is an unlabeled counter for requests shed by the +per-instance global ceiling. Those requests return HTTP 503 with code +`GLOBAL_RATE_LIMITED`. + +Authentication metrics never include keys, hashes, tokens, key prefixes or +caller addresses as labels. + +## Global request ceiling + +`GLOBAL_RATE_LIMIT_MAX` configures the maximum accepted non-health requests +per rolling 60-second window on each instance; it defaults to `1000` and must +be a positive integer. The global check runs after the per-key/IP limiter, so +per-key 429 responses remain distinct and are not counted against the global +ceiling. + +At 80% of the ceiling, rounded up, the service starts shedding expensive +route templates (GraphQL, price, route, pools, candles, history, screener, +benchmark, oracle, compare, basket, venues, spreads, volumes, book, verify +and settle) to preserve room for other routes. Once the ceiling is full, it +sheds all non-health routes. +`/status`, `/live` and `/ready` are always exempt. Shed requests include a +`Retry-After` header. + +The ceiling is **per instance**, not deployment-wide. With `N` equally +configured instances the approximate aggregate ceiling is +`GLOBAL_RATE_LIMIT_MAX x N`; choose the per-instance value based on instance +capacity and expected autoscaling so scaling out does not imply a higher +deployment-wide traffic budget than intended. + ## Why the labels look like this Cardinality is the whole design constraint. A histogram labelled by `req.url` diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 29189728..6f8838c4 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -263,6 +263,7 @@ model ApiKey { ratePerMin Int @default(60) @map("rate_per_min") ratePerDay Int @default(10000) @map("rate_per_day") revokedAt DateTime? @map("revoked_at") + expiresAt DateTime? @map("expires_at") createdAt DateTime @default(now()) @map("created_at") monthlyQuotaCents Int? @default(10000) @map("monthly_quota_cents") dailyQuotaCents Int? @default(500) @map("daily_quota_cents") diff --git a/src/__tests__/auth.test.ts b/src/__tests__/auth.test.ts index ebd5c951..77915164 100644 --- a/src/__tests__/auth.test.ts +++ b/src/__tests__/auth.test.ts @@ -22,6 +22,7 @@ import { lookupApiKey, } from '../api/auth' import { registerAdminRoutes, generateApiKey } from '../api/admin' +import { register, api_key_auth_failures_total, admin_auth_failures_total } from '../metrics' const mockFindUnique = prisma.apiKey.findUnique as unknown as ReturnType const mockCreate = prisma.apiKey.create as unknown as ReturnType @@ -31,6 +32,11 @@ function sha256(s: string): string { return createHash('sha256').update(s, 'utf8').digest('hex') } +async function apiKeyAuthFailureCount(reason: string): Promise { + const metric = await api_key_auth_failures_total.get() + return metric.values.find((value) => value.labels.reason === reason)?.value ?? 0 +} + /** Builds an app with auth enabled + one protected and one public route. */ async function buildAuthedApp() { const app = Fastify() @@ -90,18 +96,47 @@ describe('lookupApiKey', () => { }) expect(await lookupApiKey('lens_plain')).toBeNull() }) + + it('returns null for an expired key', async () => { + mockFindUnique.mockResolvedValue({ + id: 'key-1', + label: 'acme', + ratePerMin: 60, + ratePerDay: 1000, + revokedAt: null, + expiresAt: new Date(Date.now() - 1000), + }) + expect(await lookupApiKey('lens_plain')).toBeNull() + }) }) describe('API key auth hook', () => { it('returns 401 when no key is provided', async () => { + const before = await apiKeyAuthFailureCount('missing') const app = await buildAuthedApp() const res = await app.inject({ method: 'GET', url: '/price/test' }) expect(res.statusCode).toBe(401) expect(res.json()).toMatchObject({ error: 'Unauthorized' }) + expect(res.json().code).toBe('missing') + expect(await apiKeyAuthFailureCount('missing')).toBe(before + 1) expect(mockFindUnique).not.toHaveBeenCalled() }) + it('counts malformed authorization separately from missing credentials', async () => { + const before = await apiKeyAuthFailureCount('malformed') + const app = await buildAuthedApp() + const res = await app.inject({ + method: 'GET', + url: '/price/test', + headers: { authorization: 'Basic value' }, + }) + expect(res.statusCode).toBe(401) + expect(res.json()).toMatchObject({ code: 'malformed' }) + expect(await apiKeyAuthFailureCount('malformed')).toBe(before + 1) + }) + it('returns 401 for an invalid key', async () => { + const before = await apiKeyAuthFailureCount('unknown') mockFindUnique.mockResolvedValue(null) const app = await buildAuthedApp() const res = await app.inject({ @@ -110,9 +145,13 @@ describe('API key auth hook', () => { headers: { authorization: 'Bearer wrong' }, }) expect(res.statusCode).toBe(401) + expect(res.json().code).toBe('unknown') + expect(res.json().message).toBe('Unknown API key.') + expect(await apiKeyAuthFailureCount('unknown')).toBe(before + 1) }) it('returns 401 for a revoked key', async () => { + const before = await apiKeyAuthFailureCount('revoked') mockFindUnique.mockResolvedValue({ id: 'key-1', label: 'acme', ratePerMin: 60, ratePerDay: 1000, revokedAt: new Date(), }) @@ -123,6 +162,31 @@ describe('API key auth hook', () => { headers: { authorization: 'Bearer revoked' }, }) expect(res.statusCode).toBe(401) + expect(res.json().code).toBe('revoked') + expect(res.json().message).toBe('Revoked API key. Request a replacement key.') + expect(await apiKeyAuthFailureCount('revoked')).toBe(before + 1) + }) + + it('returns a distinct code and message for an expired key', async () => { + const before = await apiKeyAuthFailureCount('expired') + mockFindUnique.mockResolvedValue({ + id: 'key-1', + label: 'acme', + ratePerMin: 60, + ratePerDay: 1000, + revokedAt: null, + expiresAt: new Date(Date.now() - 1000), + }) + const app = await buildAuthedApp() + const res = await app.inject({ + method: 'GET', + url: '/price/test', + headers: { authorization: 'Bearer test-expired-api-key' }, + }) + expect(res.statusCode).toBe(401) + expect(res.json().code).toBe('expired') + expect(res.json().message).toBe('Expired API key. Request a replacement key.') + expect(await apiKeyAuthFailureCount('expired')).toBe(before + 1) }) it('allows a valid key through and attaches metadata', async () => { @@ -207,12 +271,14 @@ describe('admin endpoints', () => { }) it('rejects minting without the admin token', async () => { + const before = (await admin_auth_failures_total.get()).values[0]?.value ?? 0 const app = await buildAdminApp() const res = await app.inject({ method: 'POST', url: '/admin/keys', payload: { label: 'x' }, }) expect(res.statusCode).toBe(401) expect(mockCreate).not.toHaveBeenCalled() + expect((await admin_auth_failures_total.get()).values[0]?.value).toBe(before + 1) }) it('mints a key and returns the plaintext once (stored as hash only)', async () => { @@ -280,4 +346,10 @@ describe('admin endpoints', () => { }) expect(res.statusCode).toBe(404) }) + + it('registers authentication counters in the Prometheus registry', async () => { + const output = await register.metrics() + expect(output).toContain('api_key_auth_failures_total') + expect(output).toContain('admin_auth_failures_total') + }) }) diff --git a/src/__tests__/globalRateLimit.test.ts b/src/__tests__/globalRateLimit.test.ts new file mode 100644 index 00000000..6e6d1cc5 --- /dev/null +++ b/src/__tests__/globalRateLimit.test.ts @@ -0,0 +1,68 @@ +import { afterEach, describe, expect, it } from 'vitest' +import Fastify from 'fastify' +import rateLimit from '@fastify/rate-limit' +import { registerGlobalRateLimit } from '../middleware/globalRateLimit' +import { global_shed_requests_total } from '../metrics' + +async function buildApp(globalMax: number, perKeyMax = 10) { + const app = Fastify() + await app.register(rateLimit, { + max: perKeyMax, + timeWindow: '1 minute', + keyGenerator: () => 'test-key', + allowList: (req) => ['/status', '/live', '/ready'].includes(req.url), + }) + registerGlobalRateLimit(app, globalMax) + app.get('/work', async () => ({ ok: true })) + app.get('/cheap', async () => ({ ok: true })) + app.get('/price/test', async () => ({ ok: true })) + app.get('/status', async () => ({ ok: true })) + app.get('/live', async () => ({ ok: true })) + app.get('/ready', async () => ({ ok: true })) + await app.ready() + return app +} + +describe('global request ceiling', () => { + let app: Awaited> | undefined + + afterEach(async () => { + await app?.close() + app = undefined + }) + + it('returns a distinct global response while the caller remains within its per-key quota', async () => { + app = await buildApp(1, 10) + const before = (await global_shed_requests_total.get()).values[0]?.value ?? 0 + + expect((await app.inject({ method: 'GET', url: '/work' })).statusCode).toBe(200) + const shed = await app.inject({ method: 'GET', url: '/work' }) + + expect(shed.statusCode).toBe(503) + expect(shed.json()).toMatchObject({ code: 'GLOBAL_RATE_LIMITED' }) + expect(shed.headers['retry-after']).toBeDefined() + expect((await global_shed_requests_total.get()).values[0]?.value).toBe(before + 1) + }) + + it('sheds expensive routes before cheap routes as the ceiling approaches', async () => { + app = await buildApp(5) + + for (let i = 0; i < 4; i++) { + expect((await app.inject({ method: 'GET', url: '/cheap' })).statusCode).toBe(200) + } + expect((await app.inject({ method: 'GET', url: '/price/test' })).json()).toMatchObject({ + code: 'GLOBAL_RATE_LIMITED', + }) + expect((await app.inject({ method: 'GET', url: '/cheap' })).statusCode).toBe(200) + expect((await app.inject({ method: 'GET', url: '/cheap' })).statusCode).toBe(503) + }) + + it('never sheds health endpoints', async () => { + app = await buildApp(1) + expect((await app.inject({ method: 'GET', url: '/work' })).statusCode).toBe(200) + + for (const path of ['/status', '/live', '/ready']) { + expect((await app.inject({ method: 'GET', url: path })).statusCode).toBe(200) + } + }) +}) diff --git a/src/api/admin.ts b/src/api/admin.ts index b493081e..0dec4d22 100644 --- a/src/api/admin.ts +++ b/src/api/admin.ts @@ -1,6 +1,7 @@ import type { FastifyInstance, FastifyRequest, FastifyReply } from 'fastify' import { randomBytes, timingSafeEqual } from 'crypto' import { prisma } from '../db' +import { admin_auth_failures_total } from '../metrics' import { hashApiKey } from './auth' /** Generates a new opaque API key with a recognizable `lens_` prefix. */ @@ -28,6 +29,7 @@ function isAdminAuthorized(req: FastifyRequest): boolean { interface CreateKeyBody { label?: string + expiresAt?: string ratePerMin?: number ratePerDay?: number monthlyQuotaCents?: number @@ -50,16 +52,25 @@ export async function registerAdminRoutes(app: FastifyInstance) { { config: { public: true } }, async (req: FastifyRequest<{ Body: CreateKeyBody }>, reply: FastifyReply) => { if (!isAdminAuthorized(req)) { + admin_auth_failures_total.inc() return reply.status(401).send({ error: 'Unauthorized', + code: 'admin_unauthorized', message: 'Provide a valid X-Admin-Token header.', }) } - const { label, ratePerMin, ratePerDay, monthlyQuotaCents, dailyQuotaCents, overagePolicy } = req.body ?? {} + const { label, expiresAt, ratePerMin, ratePerDay, monthlyQuotaCents, dailyQuotaCents, overagePolicy } = req.body ?? {} if (!label || typeof label !== 'string' || label.trim().length === 0) { return reply.status(400).send({ error: 'label is required' }) } + if (expiresAt !== undefined && typeof expiresAt !== 'string') { + return reply.status(400).send({ error: 'expiresAt must be a valid future date' }) + } + const expiration = expiresAt === undefined ? undefined : new Date(expiresAt) + if (expiration && (!Number.isFinite(expiration.getTime()) || expiration <= new Date())) { + return reply.status(400).send({ error: 'expiresAt must be a valid future date' }) + } // Generate the key, store only its hash, and return the plaintext once. const plaintext = generateApiKey() @@ -67,6 +78,7 @@ export async function registerAdminRoutes(app: FastifyInstance) { data: { hash: hashApiKey(plaintext), label: label.trim(), + ...(expiration ? { expiresAt: expiration } : {}), ...(ratePerMin !== undefined ? { ratePerMin } : {}), ...(ratePerDay !== undefined ? { ratePerDay } : {}), ...(monthlyQuotaCents !== undefined ? { monthlyQuotaCents } : {}), @@ -78,6 +90,7 @@ export async function registerAdminRoutes(app: FastifyInstance) { return reply.status(201).send({ id: record.id, label: record.label, + expiresAt: record.expiresAt, ratePerMin: record.ratePerMin, ratePerDay: record.ratePerDay, monthlyQuotaCents: record.monthlyQuotaCents, @@ -94,8 +107,10 @@ export async function registerAdminRoutes(app: FastifyInstance) { { config: { public: true } }, async (req: FastifyRequest<{ Params: { id: string } }>, reply: FastifyReply) => { if (!isAdminAuthorized(req)) { + admin_auth_failures_total.inc() return reply.status(401).send({ error: 'Unauthorized', + code: 'admin_unauthorized', message: 'Provide a valid X-Admin-Token header.', }) } diff --git a/src/api/auth.ts b/src/api/auth.ts index 44cf0d12..65ae3ca8 100644 --- a/src/api/auth.ts +++ b/src/api/auth.ts @@ -2,6 +2,9 @@ import type { FastifyInstance, FastifyRequest, FastifyReply } from 'fastify' import { createHash } from 'crypto' import fp from 'fastify-plugin' import { prisma } from '../db' +import { api_key_auth_failures_total } from '../metrics' + +type ApiKeyAuthFailureReason = 'missing' | 'malformed' | 'unknown' | 'revoked' | 'expired' /** * Metadata for an authenticated API key, attached to each request as @@ -38,24 +41,36 @@ export function extractBearer(header: string | undefined): string | null { /** * Looks up an API key by its plaintext value, returning the key context only - * when the key exists and has not been revoked. + * when the key exists and has not been revoked or expired. */ export async function lookupApiKey(plaintext: string): Promise { + return (await lookupApiKeyResult(plaintext)).context +} + +async function lookupApiKeyResult( + plaintext: string, +): Promise<{ context: ApiKeyContext | null; failureReason?: ApiKeyAuthFailureReason }> { const hash = hashApiKey(plaintext) const record = await prisma.apiKey.findUnique({ where: { hash } }) - if (!record || record.revokedAt) return null + if (!record) return { context: null, failureReason: 'unknown' } + if (record.revokedAt) return { context: null, failureReason: 'revoked' } + if (record.expiresAt && record.expiresAt <= new Date()) { + return { context: null, failureReason: 'expired' } + } return { - id: record.id, - label: record.label, - ratePerMin: record.ratePerMin, - ratePerDay: record.ratePerDay, + context: { + id: record.id, + label: record.label, + ratePerMin: record.ratePerMin, + ratePerDay: record.ratePerDay, + }, } } /** * Fastify plugin that authenticates requests via `Authorization: Bearer `. * - * A request without a valid, non-revoked key gets 401. Valid requests have + * A request without a valid, non-revoked, unexpired key gets 401. Valid requests have * their key metadata attached to `req.apiKey` so downstream rate limiting can * read per-key quotas. * @@ -74,19 +89,48 @@ async function apiKeyAuthPlugin(app: FastifyInstance) { const routeConfig = (req.routeOptions?.config ?? {}) as { public?: boolean } if (routeConfig.public) return - const token = extractBearer(req.headers['authorization']) + const authorization = req.headers['authorization'] + const token = extractBearer(authorization) + if (authorization !== undefined && !token) { + api_key_auth_failures_total.inc({ reason: 'malformed' }) + return reply.status(401).send({ + error: 'Unauthorized', + code: 'malformed', + message: 'Malformed Authorization header.', + }) + } + + if (authorization === undefined) { + api_key_auth_failures_total.inc({ reason: 'missing' }) + return reply.status(401).send({ + error: 'Unauthorized', + code: 'missing', + message: 'Missing API key. Provide an Authorization header.', + }) + } + if (!token) { + api_key_auth_failures_total.inc({ reason: 'malformed' }) return reply.status(401).send({ error: 'Unauthorized', + code: 'malformed', message: 'Missing API key. Provide an Authorization: Bearer header.', }) } - const keyContext = await lookupApiKey(token) + const result = await lookupApiKeyResult(token) + const keyContext = result.context if (!keyContext) { + const reason = result.failureReason ?? 'unknown' + api_key_auth_failures_total.inc({ reason }) return reply.status(401).send({ error: 'Unauthorized', - message: 'Invalid or revoked API key.', + code: reason, + message: reason === 'revoked' + ? 'Revoked API key. Request a replacement key.' + : reason === 'expired' + ? 'Expired API key. Request a replacement key.' + : 'Unknown API key.', }) } diff --git a/src/api/usage.ts b/src/api/usage.ts index f6cb9413..12315153 100644 --- a/src/api/usage.ts +++ b/src/api/usage.ts @@ -1,6 +1,7 @@ import type { FastifyInstance, FastifyRequest, FastifyReply } from 'fastify' import fp from 'fastify-plugin' import { getUsageSummary, getAllUsageSummaries } from '../x402/metering' +import { admin_auth_failures_total, api_key_auth_failures_total } from '../metrics' function isAdminAuthorized(req: FastifyRequest): boolean { const adminToken = process.env.ADMIN_TOKEN @@ -21,8 +22,10 @@ export async function registerUsageRoutes(app: FastifyInstance) { { config: { public: true } }, async (req, reply) => { if (!isAdminAuthorized(req)) { + admin_auth_failures_total.inc() return reply.status(401).send({ error: 'Unauthorized', + code: 'admin_unauthorized', message: 'Provide a valid X-Admin-Token header.', }) } @@ -37,8 +40,10 @@ export async function registerUsageRoutes(app: FastifyInstance) { { config: { public: true } }, async (req, reply) => { if (!isAdminAuthorized(req)) { + admin_auth_failures_total.inc() return reply.status(401).send({ error: 'Unauthorized', + code: 'admin_unauthorized', message: 'Provide a valid X-Admin-Token header.', }) } @@ -52,8 +57,10 @@ export async function registerUsageRoutes(app: FastifyInstance) { async (req, reply) => { const apiKey = (req as any).apiKey as { id: string } | undefined if (!apiKey) { + api_key_auth_failures_total.inc({ reason: 'missing' }) return reply.status(401).send({ error: 'Unauthorized', + code: 'missing', message: 'Missing API key. Provide an Authorization: Bearer header.', }) } diff --git a/src/app.ts b/src/app.ts index 671dac37..3cb9e615 100644 --- a/src/app.ts +++ b/src/app.ts @@ -28,7 +28,8 @@ import { registerDiscoveryRoutes } from './routes/discovery' import { registerVenueRoutes } from './routes/venues' import { registerSettleRoute } from './routes/facilitator' import { registerCachePolicy } from './middleware/cachePolicy' -import { getMetrics } from './metrics' +import { registerGlobalRateLimit, parseGlobalRateLimit } from './middleware/globalRateLimit' +import { getMetrics, rate_limit_rejections_total } from './metrics' import { loggerOptions, createDestination } from './logger' export interface BuildAppOptions { @@ -103,14 +104,22 @@ export async function buildApp(options: BuildAppOptions = {}): Promise req.apiKey?.id ?? req.ip, allowList: (req) => req.url === '/status', - errorResponseBuilder: (req, context) => ({ - statusCode: 429, - error: 'Too Many Requests', - message: `Rate limit exceeded, retry in ${context.after}`, - retryAfter: context.after - }) + errorResponseBuilder: (req, context) => { + rate_limit_rejections_total.inc({ scope: req.apiKey ? 'key' : 'ip' }) + return { + statusCode: 429, + error: 'Too Many Requests', + code: req.apiKey ? 'PER_KEY_RATE_LIMITED' : 'IP_RATE_LIMITED', + message: `Rate limit exceeded, retry in ${context.after}`, + retryAfter: context.after + } + } }) + // The per-instance ceiling runs after the per-key/IP limiter. Health checks + // bypass it, and expensive route templates are shed before other routes. + registerGlobalRateLimit(app, parseGlobalRateLimit(process.env.GLOBAL_RATE_LIMIT_MAX)) + // Specific limit for /status (higher for monitoring) app.addHook('onRoute', (routeOptions) => { if (routeOptions.url === '/status') { diff --git a/src/metrics.ts b/src/metrics.ts index 39a3a505..ac8ac646 100644 --- a/src/metrics.ts +++ b/src/metrics.ts @@ -46,6 +46,32 @@ export const price_requests_total = new Counter({ registers: [register] }) +export const api_key_auth_failures_total = new Counter({ + name: 'api_key_auth_failures_total', + help: 'Total number of API key authentication failures, by bounded reason', + labelNames: ['reason'], + registers: [register] +}) + +export const admin_auth_failures_total = new Counter({ + name: 'admin_auth_failures_total', + help: 'Total number of admin authentication failures', + registers: [register] +}) + +export const rate_limit_rejections_total = new Counter({ + name: 'rate_limit_rejections_total', + help: 'Total number of requests rejected by the per-key or fallback IP rate limiter', + labelNames: ['scope'], + registers: [register] +}) + +export const global_shed_requests_total = new Counter({ + name: 'global_shed_requests_total', + help: 'Total number of requests shed by the per-instance global request ceiling', + registers: [register] +}) + export const x402_payments_received_total = new Counter({ name: 'x402_payments_received_total', help: 'Total number of valid x402 payments received', diff --git a/src/middleware/globalRateLimit.ts b/src/middleware/globalRateLimit.ts new file mode 100644 index 00000000..5406f573 --- /dev/null +++ b/src/middleware/globalRateLimit.ts @@ -0,0 +1,68 @@ +import type { FastifyInstance, FastifyRequest, FastifyReply } from 'fastify' +import { global_shed_requests_total } from '../metrics' + +const WINDOW_MS = 60_000 +const EXPENSIVE_ROUTE_PREFIXES = new Set([ + 'graphql', + 'price', + 'prices', + 'route', + 'pools', + 'candles', + 'history', + 'screener', + 'benchmark', + 'oracle', + 'compare', + 'basket', + 'venues', + 'spreads', + 'volumes', + 'book', + 'verify', + 'settle', +]) +const HEALTH_PATHS = new Set(['/status', '/live', '/ready']) + +export function parseGlobalRateLimit(value: string | undefined): number { + if (value === undefined) return 1000 + if (!/^[1-9]\d*$/.test(value)) { + throw new Error('GLOBAL_RATE_LIMIT_MAX must be a positive integer') + } + const max = Number(value) + if (!Number.isSafeInteger(max)) { + throw new Error('GLOBAL_RATE_LIMIT_MAX must be a positive safe integer') + } + return max +} + +export function registerGlobalRateLimit(app: FastifyInstance, max: number): void { + const acceptedAt: number[] = [] + const expensiveThreshold = Math.max(1, Math.ceil(max * 0.8)) + + app.addHook('onRequest', async (req: FastifyRequest, reply: FastifyReply) => { + const route = req.routeOptions?.url ?? req.url.split('?')[0] + if (HEALTH_PATHS.has(route)) return + + const now = Date.now() + while (acceptedAt.length > 0 && acceptedAt[0] <= now - WINDOW_MS) { + acceptedAt.shift() + } + + const expensive = EXPENSIVE_ROUTE_PREFIXES.has(route.split('/')[1]) + if (acceptedAt.length >= max || (expensive && acceptedAt.length >= expensiveThreshold)) { + global_shed_requests_total.inc() + const retryAfter = Math.max(1, Math.ceil((acceptedAt[0] + WINDOW_MS - now) / 1000)) + reply.header('Retry-After', String(retryAfter)) + return reply.status(503).send({ + statusCode: 503, + error: 'Service Unavailable', + code: 'GLOBAL_RATE_LIMITED', + message: 'The service request ceiling has been reached. Retry later.', + retryAfter, + }) + } + + acceptedAt.push(now) + }) +} diff --git a/src/routes/pairs.ts b/src/routes/pairs.ts index 199ada5a..cf95e555 100644 --- a/src/routes/pairs.ts +++ b/src/routes/pairs.ts @@ -3,6 +3,7 @@ import { getActivePairs, parseAssetStr, makePairKey, registerPair, persistPair, import { pgPool } from '../db' import { activeNetwork } from '../config' import '../middleware/network' // declares req.network on the FastifyRequest type +import { admin_auth_failures_total } from '../metrics' export async function registerPairsRoutes(app: FastifyInstance) { // GET /pairs — list all active pairs with latest price metadata @@ -65,7 +66,12 @@ export async function registerPairsRoutes(app: FastifyInstance) { const ADMIN_API_KEY = process.env.ADMIN_API_KEY const key = req.headers['x-admin-key'] ?? req.headers['authorization']?.replace(/^Bearer /, '') if (!ADMIN_API_KEY || key !== ADMIN_API_KEY) { - return reply.status(401).send({ error: 'Unauthorized — provide a valid X-Admin-Key header' }) + admin_auth_failures_total.inc() + return reply.status(401).send({ + error: 'Unauthorized', + code: 'admin_unauthorized', + message: 'Provide valid admin authentication.', + }) } const { assetA: assetAStr, assetB: assetBStr } = req.body ?? {}