From ac27a8673d6e008a4e352d866c7d0b6e1106baff Mon Sep 17 00:00:00 2001 From: Harshil Date: Sat, 12 Sep 2026 23:13:40 +0530 Subject: [PATCH 01/20] test: add integration tests, new test compose and env --- Makefile | 19 +- backend/src/controllers/order.ts | 8 +- backend/src/middleware/rateLimit.ts | 4 +- benchmarks/api.bench.ts | 50 +++- benchmarks/run.sh | 47 ++++ compose.test.yml | 86 +++++++ tests/.gitignore | 34 +++ tests/bun.lock | 73 ++++++ tests/integration/api/http.test.ts | 88 +++++++ tests/integration/api/orders.test.ts | 43 ++++ tests/integration/flows/trading.test.ts | 235 +++++++++++++++++++ tests/integration/helpers/api.ts | 102 ++++++++ tests/integration/helpers/engine.ts | 100 ++++++++ tests/integration/helpers/users.ts | 31 +++ tests/integration/helpers/wait-for.ts | 25 ++ tests/integration/persistence/worker.test.ts | 56 +++++ tests/integration/redis/contracts.test.ts | 79 +++++++ tests/integration/setup/database.ts | 62 +++++ tests/integration/setup/env.ts | 28 +++ tests/package.json | 19 ++ tests/run-integration.sh | 45 ++++ tests/tsconfig.json | 29 +++ 22 files changed, 1248 insertions(+), 15 deletions(-) create mode 100644 benchmarks/run.sh create mode 100644 compose.test.yml create mode 100644 tests/.gitignore create mode 100644 tests/bun.lock create mode 100644 tests/integration/api/http.test.ts create mode 100644 tests/integration/api/orders.test.ts create mode 100644 tests/integration/flows/trading.test.ts create mode 100644 tests/integration/helpers/api.ts create mode 100644 tests/integration/helpers/engine.ts create mode 100644 tests/integration/helpers/users.ts create mode 100644 tests/integration/helpers/wait-for.ts create mode 100644 tests/integration/persistence/worker.test.ts create mode 100644 tests/integration/redis/contracts.test.ts create mode 100644 tests/integration/setup/database.ts create mode 100644 tests/integration/setup/env.ts create mode 100644 tests/package.json create mode 100644 tests/run-integration.sh create mode 100644 tests/tsconfig.json diff --git a/Makefile b/Makefile index 67b5fb4..d8ddbd5 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,7 @@ ENV ?= dev DEV_FILE := compose.dev.yml +TEST_FILE := compose.test.yml PROD_FILE := compose.prod.yml BOT_FILE := compose.bot.yml @@ -12,11 +13,15 @@ ifeq ($(ENV),prod) COMPOSE := docker compose -f $(PROD_FILE) endif +ifeq ($(ENV),test) +COMPOSE := docker compose -p paperdrill-test -f $(TEST_FILE) +endif + ifeq ($(ENV),bot) COMPOSE := docker compose -f $(BOT_FILE) endif -.PHONY: up build down deploy start restart stop status logs migrate-dev migrate-reset migrate-deploy seed clean +.PHONY: up build down deploy start restart stop status logs migrate-dev migrate-reset migrate-deploy seed test-unit test-integration test-e2e bench clean up: $(COMPOSE) up -d @@ -57,5 +62,17 @@ migrate-deploy: db-seed: $(COMPOSE) exec backend bun prisma db seed +test-unit: + cd backend && set -a && . ../.env.test && set +a && bun test + cd engine && set -a && . ../.env.test && set +a && bun test + +test-integration: + sh tests/run-integration.sh + +test-e2e: + +bench: + sh benchmarks/run.sh + clean: docker system prune -f diff --git a/backend/src/controllers/order.ts b/backend/src/controllers/order.ts index f31f57d..8d7976e 100644 --- a/backend/src/controllers/order.ts +++ b/backend/src/controllers/order.ts @@ -146,5 +146,11 @@ export async function cancelOrder(req: Request, res: Response) { return; } - res.status(200).json(formatCancel(engineResponse.data as Record)); + const result = engineResponse.data as Record; + if (typeof result.message === "string") { + res.status(200).json({ message: result.message }); + return; + } + + res.status(200).json(formatCancel(result)); } diff --git a/backend/src/middleware/rateLimit.ts b/backend/src/middleware/rateLimit.ts index e03d949..5aa8e2a 100644 --- a/backend/src/middleware/rateLimit.ts +++ b/backend/src/middleware/rateLimit.ts @@ -26,7 +26,9 @@ function createLimiter(prefix: string, windowMs: number, limit: number) { windowMs, limit, keyGenerator: bucketKey, - store: new RedisStore({ prefix, sendCommand }), + ...(config.rateLimit.enabled && { + store: new RedisStore({ prefix, sendCommand }), + }), standardHeaders: true, legacyHeaders: false, skip: (req) => !config.rateLimit.enabled || req.principal?.type === "service", diff --git a/benchmarks/api.bench.ts b/benchmarks/api.bench.ts index 70330ea..62e6bb0 100644 --- a/benchmarks/api.bench.ts +++ b/benchmarks/api.bench.ts @@ -5,6 +5,7 @@ const CONCURRENCY = parseInt(process.env.CONCURRENCY ?? "50", 10); const DURATION_SEC = parseInt(process.env.DURATION_SEC ?? "10", 10); const SERVICE_TOKEN = process.env.SERVICE_TOKEN; const SERVICE_EMAIL = process.env.SERVICE_EMAIL; +const BENCHMARK_BALANCE = "1000000000000"; // --- Helpers --- @@ -19,13 +20,13 @@ function stats(times: number[]) { return { avg, p50, p95, p99, total: sum, count: times.length }; } -function report(name: string, times: number[], errors: number) { +function report(name: string, times: number[], errors: number, elapsedMs: number) { const s = stats(times); - const opsPerSec = s.total === 0 ? 0 : (s.count / s.total) * 1000; + const opsPerSec = elapsedMs === 0 ? 0 : (s.count / elapsedMs) * 1000; const errRate = ((errors / (s.count + errors)) * 100).toFixed(1); console.log(` ${name}`); console.log( - ` ${s.count} ok, ${errors} err (${errRate}% err) | ${(s.total / 1000).toFixed(1)}s`, + ` ${s.count} ok, ${errors} err (${errRate}% err) | ${(elapsedMs / 1000).toFixed(1)}s`, ); console.log( ` ${opsPerSec.toFixed(0)} req/sec | avg ${s.avg.toFixed(1)}ms | p50 ${s.p50?.toFixed(1)}ms | p95 ${s.p95?.toFixed(1)}ms | p99 ${s.p99?.toFixed(1)}ms`, @@ -38,6 +39,7 @@ async function runLoadTest(name: string, fn: RequestFn, durationMs: number) { const times: number[] = []; let errors = 0; let running = true; + const startedAt = performance.now(); const worker = async () => { while (running) { @@ -62,13 +64,14 @@ async function runLoadTest(name: string, fn: RequestFn, durationMs: number) { await Promise.all(workers); clearTimeout(deadline); - report(name, times, errors); + report(name, times, errors, performance.now() - startedAt); + return errors; } // --- Load test scenarios --- async function benchMarkets() { - await runLoadTest( + return runLoadTest( "GET /v1/markets (public)", async () => { try { @@ -83,7 +86,7 @@ async function benchMarkets() { } async function benchOrderbook() { - await runLoadTest( + return runLoadTest( "GET /v1/markets/SOL_USD/orderbook (public)", async () => { try { @@ -98,7 +101,7 @@ async function benchOrderbook() { } async function benchCreateOrder() { - await runLoadTest( + return runLoadTest( "POST /v1/orders (LIMIT, no match)", async () => { try { @@ -127,8 +130,26 @@ async function benchCreateOrder() { ); } +async function seedServiceBalance() { + if (!SERVICE_TOKEN || !SERVICE_EMAIL) return; + + const res = await fetch(`${API_URL}/v1/deposits`, { + method: "POST", + headers: { + "Content-Type": "application/json", + "X-Api-Key": SERVICE_TOKEN, + "X-Service-Email": SERVICE_EMAIL, + }, + body: JSON.stringify({ asset: "USD", amount: BENCHMARK_BALANCE }), + }); + + if (!res.ok) { + throw new Error(`Failed to fund benchmark service account (${res.status}): ${await res.text()}`); + } +} + async function benchTrades() { - await runLoadTest( + return runLoadTest( "GET /v1/markets/SOL_USD/trades (public)", async () => { try { @@ -162,15 +183,20 @@ try { process.exit(1); } -await benchMarkets(); +let totalErrors = 0; + +totalErrors += await benchMarkets(); console.log(); -await benchOrderbook(); +totalErrors += await benchOrderbook(); console.log(); if (SERVICE_TOKEN && SERVICE_EMAIL) { - await benchCreateOrder(); + await seedServiceBalance(); + totalErrors += await benchCreateOrder(); console.log(); } else { console.log(" POST /v1/orders skipped (no service token or email)\n"); } -await benchTrades(); +totalErrors += await benchTrades(); console.log(); + +if (totalErrors > 0) process.exitCode = 1; diff --git a/benchmarks/run.sh b/benchmarks/run.sh new file mode 100644 index 0000000..ff60a59 --- /dev/null +++ b/benchmarks/run.sh @@ -0,0 +1,47 @@ +#!/bin/sh + +set -u + +compose() { + docker compose -p paperdrill-benchmark -f compose.test.yml "$@" +} + +cleanup() { + compose down --volumes --remove-orphans +} + +trap cleanup EXIT INT TERM + +status=0 + +compose down --volumes --remove-orphans || true +compose build || status=$? + +if [ "$status" -eq 0 ]; then + compose up -d --wait backend engine worker || status=$? +fi + +if [ "$status" -eq 0 ]; then + ( + set -a + . ./.env.test + set +a + + if [ "$NODE_ENV" != "test" ] || [ "$TEST_ENVIRONMENT" != "integration" ]; then + echo "Refusing to benchmark outside the test environment" >&2 + exit 1 + fi + + export API_URL="http://127.0.0.1:8001" + + cd benchmarks + bun run bench:api + ) || status=$? +fi + +if [ "$status" -ne 0 ]; then + compose ps + compose logs --no-color backend engine worker postgres redis setup +fi + +exit "$status" diff --git a/compose.test.yml b/compose.test.yml new file mode 100644 index 0000000..91800d0 --- /dev/null +++ b/compose.test.yml @@ -0,0 +1,86 @@ +services: + postgres: + image: timescale/timescaledb:2.28.2-pg17 + ports: + - "5434:5432" + environment: + POSTGRES_DB: paperdrill_test + POSTGRES_USER: postgres + POSTGRES_PASSWORD: password + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres -d paperdrill_test"] + interval: 2s + timeout: 5s + retries: 15 + tmpfs: + - /var/lib/postgresql/data + + redis: + image: redis:7.2 + ports: + - "6381:6379" + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 2s + timeout: 5s + retries: 15 + tmpfs: + - /data + + setup: + build: + context: ./backend + target: dev + command: sh -c "bun prisma migrate deploy && bun prisma db seed" + env_file: ./.env.test + depends_on: + postgres: + condition: service_healthy + + backend: + build: + context: ./backend + target: runner + ports: + - "8001:8000" + env_file: ./.env.test + depends_on: + redis: + condition: service_healthy + setup: + condition: service_completed_successfully + healthcheck: + test: + [ + "CMD", + "bun", + "-e", + "const r = await fetch('http://localhost:8000/health'); if (!r.ok) process.exit(1)", + ] + interval: 2s + timeout: 5s + retries: 20 + + engine: + build: + context: ./engine + target: runner + env_file: ./.env.test + depends_on: + redis: + condition: service_healthy + setup: + condition: service_completed_successfully + tmpfs: + - /usr/src/app/snapshots + + worker: + build: + context: ./worker + target: runner + env_file: ./.env.test + depends_on: + redis: + condition: service_healthy + setup: + condition: service_completed_successfully diff --git a/tests/.gitignore b/tests/.gitignore new file mode 100644 index 0000000..a14702c --- /dev/null +++ b/tests/.gitignore @@ -0,0 +1,34 @@ +# dependencies (bun install) +node_modules + +# output +out +dist +*.tgz + +# code coverage +coverage +*.lcov + +# logs +logs +_.log +report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json + +# dotenv environment variable files +.env +.env.development.local +.env.test.local +.env.production.local +.env.local + +# caches +.eslintcache +.cache +*.tsbuildinfo + +# IntelliJ based IDEs +.idea + +# Finder (MacOS) folder config +.DS_Store diff --git a/tests/bun.lock b/tests/bun.lock new file mode 100644 index 0000000..7ae91e4 --- /dev/null +++ b/tests/bun.lock @@ -0,0 +1,73 @@ +{ + "lockfileVersion": 1, + "configVersion": 1, + "workspaces": { + "": { + "name": "tests", + "dependencies": { + "pg": "^8.20.0", + "redis": "^6.0.0", + }, + "devDependencies": { + "@types/bun": "latest", + "@types/pg": "^8.20.0", + "typescript": "^5", + }, + }, + }, + "packages": { + "@redis/bloom": ["@redis/bloom@6.2.1", "", { "peerDependencies": { "@redis/client": "^6.2.1" } }, "sha512-huQgNLaCIZfQ9SeLn4q9124uOUd8HbZDYHwwUzNcRgHqCHiHKl2dDxMqJCeWh8cMqZAoWuHR8XnWbDMIf+o7ag=="], + + "@redis/client": ["@redis/client@6.2.1", "", { "dependencies": { "cluster-key-slot": "1.1.2" }, "peerDependencies": { "@node-rs/xxhash": "^1.1.0", "@opentelemetry/api": ">=1 <2" }, "optionalPeers": ["@node-rs/xxhash", "@opentelemetry/api"] }, "sha512-LzxBY7SIBvvJiyCgcaJZZakE3fJrZZ++i24+EDW9fKpCl68D35uJcKFpZZwCfOoG9WZTbyZlMzMeM0gtOAMU9Q=="], + + "@redis/json": ["@redis/json@6.2.1", "", { "peerDependencies": { "@redis/client": "^6.2.1" } }, "sha512-AFIUJ8Gj0DaaSBHYuSt8+O0oYWM+50OK1c0OmodB7XERIA8+BbyV3O4v76f9iccWasd1/7qjfZTpuzexUaZtrQ=="], + + "@redis/search": ["@redis/search@6.2.1", "", { "peerDependencies": { "@redis/client": "^6.2.1" } }, "sha512-2vfOAOyYFE7UUw3sBBlkqqruBtOUS4HRY5MtW4hp83llrwvtrTE4r22CEqXddlV+54zkLxBE4nmsIJ/dpezQrQ=="], + + "@redis/time-series": ["@redis/time-series@6.2.1", "", { "peerDependencies": { "@redis/client": "^6.2.1" } }, "sha512-kiYniph04dJOole+L359B6C9E+jYS2uDP7hca6Onj0xF38ZIpyxARO0Iq0W4ZRn1e8Q6vqW00QFZVSMRA/2Ijw=="], + + "@types/bun": ["@types/bun@1.4.2", "", { "dependencies": { "bun-types": "1.4.2" } }, "sha512-GimotNn7+ZV0uVArItBbriZsR1oNf0+WTzPkdcFrzShI7k2norL0uzEaJT8T33dWr7O/c9ZDuAFQrctKCi72oQ=="], + + "@types/node": ["@types/node@22.20.2", "", { "dependencies": { "undici-types": "~6.21.0" } }, "sha512-xlvWf4Vs9n1PEVYwP1n4vvG07M6y8WgvJ2t0vbrWTmijsIHp1cS+uJ2kMIRdY3nHZK0nCYKrPeD171+SzF4/zw=="], + + "@types/pg": ["@types/pg@8.23.1", "", { "dependencies": { "@types/node": "*", "pg-protocol": "*", "pg-types": "^2.2.0" } }, "sha512-fKVHpikPdg4GKks3JuLEhvwSyvwzF23hnabPy6DD8ljVbC7+6J5dQzdv4arV6jqq57djnMgs1HKBxX4P8aBI3A=="], + + "bun-types": ["bun-types@1.4.2", "", { "dependencies": { "@types/node": "*" } }, "sha512-bxV1FgK7yBIzjRe5zBozIM4Bem11ZJcCXSrjWRG3YWLt8yFDePu4cLjpebO8OvPeIE9trbyPF4fuj3Cia4Fj3w=="], + + "cluster-key-slot": ["cluster-key-slot@1.1.2", "", {}, "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA=="], + + "pg": ["pg@8.23.0", "", { "dependencies": { "pg-connection-string": "^2.14.0", "pg-pool": "^3.14.0", "pg-protocol": "^1.16.0", "pg-types": "2.2.0", "pgpass": "1.0.5" }, "optionalDependencies": { "pg-cloudflare": "^1.4.0" }, "peerDependencies": { "pg-native": ">=3.0.1" }, "optionalPeers": ["pg-native"] }, "sha512-Ip2EQCngowJLGOfCwkFhPXU7/ljlhn6Rxlmy4XYfL2Y+vyRM59+8uR2xqRWKdYmbXmxCFOAmKxBuSUCdF34qLg=="], + + "pg-cloudflare": ["pg-cloudflare@1.4.0", "", {}, "sha512-Vo7z/6rrQYxpNRylp4Tlob2elzbh+N/MOQbxFVWCxS7oEx6jF53GTJFxK2WWpKuBRkmiin4Mt+xofFDjx09R0A=="], + + "pg-connection-string": ["pg-connection-string@2.14.0", "", {}, "sha512-XwWDGcLRGCXAR8F/AM5bG7Q+A3Wm2s6QeEjlOKZLlH3UYcguiqCWKyWXVag5TLTIjR7oOJUY8kcADaZgWPyLeg=="], + + "pg-int8": ["pg-int8@1.0.1", "", {}, "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw=="], + + "pg-pool": ["pg-pool@3.14.0", "", { "peerDependencies": { "pg": ">=8.0" } }, "sha512-gKtPkFdQPU3DksooVLi9LsjZxrsBUZIpa+7aVx+LV5pNh0KzP4Zleud2po+ConrxbuXGBJ6Hfer6hdgpIBpBaw=="], + + "pg-protocol": ["pg-protocol@1.16.0", "", {}, "sha512-sILXutLVjCLjcDuOmvhX5e2Z4cS5qG/6Bu3VkpFwdf/633ElGLpEh9bgmuI5I4sqKqkifQiGyiCcx1HdtrK7tg=="], + + "pg-types": ["pg-types@2.2.0", "", { "dependencies": { "pg-int8": "1.0.1", "postgres-array": "~2.0.0", "postgres-bytea": "~1.0.0", "postgres-date": "~1.0.4", "postgres-interval": "^1.1.0" } }, "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA=="], + + "pgpass": ["pgpass@1.0.5", "", { "dependencies": { "split2": "^4.1.0" } }, "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug=="], + + "postgres-array": ["postgres-array@2.0.0", "", {}, "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA=="], + + "postgres-bytea": ["postgres-bytea@1.0.1", "", {}, "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ=="], + + "postgres-date": ["postgres-date@1.0.7", "", {}, "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q=="], + + "postgres-interval": ["postgres-interval@1.2.0", "", { "dependencies": { "xtend": "^4.0.0" } }, "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ=="], + + "redis": ["redis@6.2.1", "", { "dependencies": { "@redis/bloom": "6.2.1", "@redis/client": "6.2.1", "@redis/json": "6.2.1", "@redis/search": "6.2.1", "@redis/time-series": "6.2.1" } }, "sha512-Z9VHtgYs48PiQC77X9O2Er8Hj4T+5BtFjT91/vi5Is1D04N72cA946ZslM1ImJw8ZctFBZWAVjM7S5wJNeHMpg=="], + + "split2": ["split2@4.2.0", "", {}, "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg=="], + + "typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="], + + "undici-types": ["undici-types@6.21.0", "", {}, "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ=="], + + "xtend": ["xtend@4.0.2", "", {}, "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ=="], + } +} diff --git a/tests/integration/api/http.test.ts b/tests/integration/api/http.test.ts new file mode 100644 index 0000000..c885aa7 --- /dev/null +++ b/tests/integration/api/http.test.ts @@ -0,0 +1,88 @@ +import { beforeAll, expect, test } from "bun:test"; +import { publicApi, waitForSystemReady } from "../helpers/api"; + +beforeAll(waitForSystemReady); + +test("root exposes the public API contract", async () => { + const response = await publicApi.request>("/"); + + expect(response).toMatchObject({ + status: 200, + data: { + message: "Welcome to PaperDrill", + status: "running", + success: true, + }, + }); +}); + +test("validation failures use the public JSON error contract", async () => { + const response = await publicApi.request<{ + error: string; + issues: Array<{ path: string }>; + }>("/v1/auth/login", { + method: "POST", + body: JSON.stringify({ email: "not-an-email", password: "" }), + }); + + expect(response.status).toBe(400); + expect(response.data.error).toBe("validation_error"); + expect(response.data.issues).toEqual( + expect.arrayContaining([ + expect.objectContaining({ path: "email" }), + expect.objectContaining({ path: "password" }), + ]), + ); +}); + +test("anonymous users cannot access account or order operations", async () => { + const balance = await publicApi.request<{ error: string }>("/v1/balances"); + const order = await publicApi.request<{ error: string }>("/v1/orders", { + method: "POST", + body: JSON.stringify({ + type: "LIMIT", + side: "BUY", + symbol: "BTC_USD", + price: "100.00", + qty: "0.1000", + }), + }); + + for (const response of [balance, order]) { + expect(response).toMatchObject({ + status: 401, + data: { error: "Authentication required" }, + }); + } +}); + +test("malformed API keys are rejected before protected handlers run", async () => { + const response = await publicApi.request<{ error: string }>("/v1/balances", { + headers: { "x-api-key": "not-a-paperdrill-key" }, + }); + + expect(response).toMatchObject({ status: 401, data: { error: "Malformed API key" } }); +}); + +test("invalid session cookies are cleared and cannot authorize a request", async () => { + const response = await publicApi.request<{ error: string }>("/v1/balances", { + headers: { cookie: "token=invalid-token" }, + }); + + expect(response).toMatchObject({ status: 401, data: { error: "Authentication required" } }); + expect(response.headers.get("set-cookie")).toContain("token="); +}); + +test("browser clients receive credentialed CORS preflight headers", async () => { + const response = await fetch(`${process.env.API_BASE_URL}/v1/auth/login`, { + method: "OPTIONS", + headers: { + origin: "http://localhost:3000", + "access-control-request-method": "POST", + }, + }); + + expect(response.status).toBe(200); + expect(response.headers.get("access-control-allow-origin")).toBe("http://localhost:3000"); + expect(response.headers.get("access-control-allow-credentials")).toBe("true"); +}); diff --git a/tests/integration/api/orders.test.ts b/tests/integration/api/orders.test.ts new file mode 100644 index 0000000..501850c --- /dev/null +++ b/tests/integration/api/orders.test.ts @@ -0,0 +1,43 @@ +import { beforeAll, expect, test } from "bun:test"; +import { waitForSystemReady } from "../helpers/api"; +import { createTestUser } from "../helpers/users"; + +beforeAll(waitForSystemReady); + +test("invalid orders are rejected without creating open state", async () => { + const { client } = await createTestUser("validation"); + const invalidOrders = [ + { type: "LIMIT", side: "BUY", price: "100.00", qty: "0.1000" }, + { type: "LIMIT", side: "HOLD", symbol: "BTC_USD", price: "100.00", qty: "0.1000" }, + { type: "STOP", side: "BUY", symbol: "BTC_USD", price: "100.00", qty: "0.1000" }, + { type: "LIMIT", side: "BUY", symbol: "BTC_USD", qty: "0.1000" }, + { type: "LIMIT", side: "BUY", symbol: "BTC_USD", price: "100.00", qty: "0" }, + { type: "LIMIT", side: "BUY", symbol: "BTC_USD", price: "0", qty: "0.1000" }, + { type: "LIMIT", side: "BUY", symbol: "DOGE_USD", price: "1.00", qty: "1" }, + ]; + + for (const order of invalidOrders) { + const response = await client.request>("/v1/orders", { + method: "POST", + body: JSON.stringify(order), + }); + expect(response.status).toBeGreaterThanOrEqual(400); + expect(response.status).toBeLessThan(500); + } + + expect(await client.getOpenOrders()).toMatchObject({ status: 200, data: [] }); +}); + +test("an account without base inventory cannot place a sell order", async () => { + const { client } = await createTestUser("no-inventory"); + const response = await client.createOrder({ + type: "LIMIT", + side: "SELL", + symbol: "BTC_USD", + price: "100.00", + qty: "0.1000", + }); + + expect(response).toMatchObject({ status: 400, data: { error: "Insufficient balance" } }); + expect(await client.getOpenOrders()).toMatchObject({ status: 200, data: [] }); +}); diff --git a/tests/integration/flows/trading.test.ts b/tests/integration/flows/trading.test.ts new file mode 100644 index 0000000..3c9bec4 --- /dev/null +++ b/tests/integration/flows/trading.test.ts @@ -0,0 +1,235 @@ +import { afterAll, beforeAll, expect, test } from "bun:test"; +import { publicApi, waitForSystemReady } from "../helpers/api"; +import { EngineClient } from "../helpers/engine"; +import { createTestUser, seedBalance } from "../helpers/users"; +import { waitFor } from "../helpers/wait-for"; +import { TestDatabase, type PersistedOrder } from "../setup/database"; + +const engine = new EngineClient(); +const database = new TestDatabase(); + +beforeAll(async () => { + await engine.connect(); + await waitForSystemReady(); +}); + +afterAll(async () => { + await Promise.all([engine.close(), database.close()]); +}); + +function waitForOrder(id: string, status: PersistedOrder["status"]) { + return waitFor( + () => database.order(id), + (order) => order?.status === status, + { description: `${id} to persist as ${status}` }, + ); +} + +test("a complete trade settles live balances and persists user-visible history", async () => { + const buyer = await createTestUser("trade-buyer"); + const seller = await createTestUser("trade-seller"); + await seedBalance(engine, seller.id, "BTC", "10000"); + + const sell = await seller.client.createOrder({ + type: "LIMIT", + side: "SELL", + symbol: "BTC_USD", + price: "100.00", + qty: "0.5000", + }); + const buy = await buyer.client.createOrder({ + type: "LIMIT", + side: "BUY", + symbol: "BTC_USD", + price: "100.00", + qty: "0.5000", + }); + + expect(sell).toMatchObject({ status: 200, data: { status: "OPEN", filledQty: "0.0000" } }); + expect(buy).toMatchObject({ + status: 200, + data: { status: "FILLED", filledQty: "0.5000", averagePrice: "100.00" }, + }); + expect(await buyer.client.getBalances()).toMatchObject({ + status: 200, + data: { + USD: { available: "9950.00", locked: "0.00" }, + BTC: { available: "0.5000", locked: "0.0000" }, + }, + }); + expect(await seller.client.getBalances()).toMatchObject({ + status: 200, + data: { + USD: { available: "10050.00", locked: "0.00" }, + BTC: { available: "0.5000", locked: "0.0000" }, + }, + }); + + const [buyOrder, sellOrder] = await Promise.all([ + waitForOrder(buy.data.id, "FILLED"), + waitForOrder(sell.data.id, "FILLED"), + ]); + expect(buyOrder).toMatchObject({ userId: buyer.id, filledQty: "5000", spentAmount: "5000" }); + expect(sellOrder).toMatchObject({ userId: seller.id, filledQty: "5000" }); + + const buyerHistory = await waitFor( + () => buyer.client.getOrders(), + (response) => response.status === 200 && response.data.some((order) => order.id === buy.data.id), + { description: "buyer order history" }, + ); + expect(buyerHistory.data).toEqual( + expect.arrayContaining([ + expect.objectContaining({ id: buy.data.id, status: "FILLED", filledQty: "0.5000" }), + ]), + ); + expect(buyerHistory.data.some((order) => order.id === sell.data.id)).toBe(false); + + const buyerTrades = await waitFor( + () => buyer.client.getTrades(), + (response) => response.status === 200 && response.data.length === 1, + { description: "buyer fill history" }, + ); + const fillId = buyerTrades.data[0]?.id as string; + expect(buyerTrades.data[0]).toMatchObject({ + symbol: "BTC_USD", + price: "100.00", + qty: "0.5000", + side: "BUY", + isMaker: false, + orderId: buy.data.id, + }); + expect(await database.fill(fillId)).toMatchObject({ + buyOrderId: buy.data.id, + sellOrderId: sell.data.id, + buyerId: buyer.id, + sellerId: seller.id, + }); + + const ticker = await waitFor( + () => publicApi.getTicker("BTC_USD"), + (response) => response.status === 200 && response.data.lastPrice === "100.00", + { description: "worker-derived BTC ticker" }, + ); + expect(ticker.data).toMatchObject({ + symbol: "BTC_USD", + lastPrice: "100.00", + openPrice: "100.00", + high: "100.00", + low: "100.00", + volume: "0.5000", + }); + + expect(await buyer.client.cancelOrder(buy.data.id)).toMatchObject({ + status: 400, + data: { error: "Filled orders cannot be cancelled" }, + }); +}); + +test("a partial fill propagates through balances, open orders, fills, and persistence", async () => { + const buyer = await createTestUser("partial-buyer"); + const seller = await createTestUser("partial-seller"); + await seedBalance(engine, seller.id, "SOL", "1000"); + + const sell = await seller.client.createOrder({ + type: "LIMIT", + side: "SELL", + symbol: "SOL_USD", + price: "100.00", + qty: "10.00", + }); + const buy = await buyer.client.createOrder({ + type: "LIMIT", + side: "BUY", + symbol: "SOL_USD", + price: "100.00", + qty: "4.00", + }); + + expect(buy).toMatchObject({ status: 200, data: { status: "FILLED", filledQty: "4.00" } }); + expect(await seller.client.getOpenOrders()).toMatchObject({ + status: 200, + data: [ + expect.objectContaining({ + id: sell.data.id, + status: "PARTIALLY_FILLED", + qty: "10.00", + filledQty: "4.00", + }), + ], + }); + expect(await buyer.client.getBalances()).toMatchObject({ + data: { + USD: { available: "9600.00", locked: "0.00" }, + SOL: { available: "4.00", locked: "0.00" }, + }, + }); + expect(await seller.client.getBalances()).toMatchObject({ + data: { + USD: { available: "10400.00", locked: "0.00" }, + SOL: { available: "0.00", locked: "6.00" }, + }, + }); + + const [persistedSell, persistedBuy] = await Promise.all([ + waitForOrder(sell.data.id, "PARTIALLY_FILLED"), + waitForOrder(buy.data.id, "FILLED"), + ]); + expect(persistedSell).toMatchObject({ qty: "1000", filledQty: "400", status: "PARTIALLY_FILLED" }); + expect(persistedBuy).toMatchObject({ qty: "400", filledQty: "400", status: "FILLED" }); + + const trades = await waitFor( + () => seller.client.getTrades(), + (response) => response.status === 200 && response.data.length === 1, + { description: "partial-fill trade persistence" }, + ); + expect(trades.data[0]).toMatchObject({ qty: "4.00", side: "SELL", orderId: sell.data.id }); + + await seller.client.cancelOrder(sell.data.id); +}); + +test("cancelling an open order releases funds and persists ownership-safe state", async () => { + const owner = await createTestUser("cancel-owner"); + const other = await createTestUser("cancel-other"); + const created = await owner.client.createOrder({ + type: "LIMIT", + side: "BUY", + symbol: "ETH_USD", + price: "100.00", + qty: "1.000", + }); + + expect(created).toMatchObject({ status: 200, data: { status: "OPEN" } }); + expect(await owner.client.getBalances()).toMatchObject({ + data: { USD: { available: "9900.00", locked: "100.00" } }, + }); + + await waitForOrder(created.data.id, "OPEN"); + expect(await other.client.getOrder(created.data.id)).toMatchObject({ + status: 404, + data: { error: "Order not found" }, + }); + expect(await other.client.cancelOrder(created.data.id)).toMatchObject({ + status: 400, + data: { error: "Order not found" }, + }); + + expect(await owner.client.cancelOrder(created.data.id)).toMatchObject({ + status: 200, + data: { id: created.data.id, symbol: "ETH_USD", qty: "1.000", filledQty: "0.000" }, + }); + expect(await owner.client.getBalances()).toMatchObject({ + data: { USD: { available: "10000.00", locked: "0.00" } }, + }); + expect((await owner.client.getOpenOrders()).data.some((order) => order.id === created.data.id)).toBe( + false, + ); + expect(await waitForOrder(created.data.id, "CANCELLED")).toMatchObject({ + filledQty: "0", + status: "CANCELLED", + }); + + expect(await owner.client.cancelOrder(created.data.id)).toMatchObject({ + status: 200, + data: { message: "Order already cancelled" }, + }); +}); diff --git a/tests/integration/helpers/api.ts b/tests/integration/helpers/api.ts new file mode 100644 index 0000000..1a7af1c --- /dev/null +++ b/tests/integration/helpers/api.ts @@ -0,0 +1,102 @@ +import { env } from "../setup/env"; +import { waitFor } from "./wait-for"; + +export interface ApiResponse { + status: number; + data: T; + headers: Headers; +} + +export interface OrderInput { + type: "LIMIT" | "MARKET"; + side: "BUY" | "SELL"; + symbol: string; + price?: string; + qty: string; +} + +export interface OrderResponse { + id: string; + userId?: string; + symbol: string; + type?: "LIMIT" | "MARKET"; + side?: "BUY" | "SELL"; + status?: "OPEN" | "PARTIALLY_FILLED" | "FILLED" | "CANCELLED"; + price?: string; + qty?: string; + filledQty?: string; + averagePrice?: string; +} + +export type Balances = Record; + +export class ApiClient { + private cookie: string | undefined; + + async request(path: string, init: RequestInit = {}): Promise> { + const headers = new Headers(init.headers); + if (init.body && !headers.has("content-type")) { + headers.set("content-type", "application/json"); + } + if (this.cookie) headers.set("cookie", this.cookie); + + const response = await fetch(`${env.apiBaseUrl}${path}`, { ...init, headers }); + const setCookie = response.headers.get("set-cookie"); + if (setCookie) this.cookie = setCookie.split(";", 1)[0]; + + const data = (await response.json()) as T; + return { status: response.status, data, headers: response.headers }; + } + + signup(email: string, name: string, password: string) { + return this.request<{ id: string; email: string; emailVerified: boolean }>("/v1/auth/signup", { + method: "POST", + body: JSON.stringify({ email, name, password }), + }); + } + + getBalances() { + return this.request("/v1/balances"); + } + + createOrder(input: OrderInput) { + return this.request("/v1/orders", { + method: "POST", + body: JSON.stringify(input), + }); + } + + cancelOrder(id: string) { + return this.request>(`/v1/orders/${id}`, { method: "DELETE" }); + } + + getOrders(query = "") { + return this.request(`/v1/orders${query}`); + } + + getOpenOrders() { + return this.request("/v1/orders/open"); + } + + getOrder(id: string) { + return this.request(`/v1/orders/${id}`); + } + + getTrades() { + return this.request[]>("/v1/trades"); + } + + getTicker(symbol: string) { + return this.request>(`/v1/markets/${symbol}/ticker`); + } +} + +export const publicApi = new ApiClient(); + +export async function waitForSystemReady() { + await waitFor( + () => publicApi.request>("/v1/markets/BTC_USD/orderbook"), + (response) => response.status === 200, + { description: "backend-to-engine readiness", timeoutMs: 15000, intervalMs: 100 }, + ); +} diff --git a/tests/integration/helpers/engine.ts b/tests/integration/helpers/engine.ts new file mode 100644 index 0000000..4680bc9 --- /dev/null +++ b/tests/integration/helpers/engine.ts @@ -0,0 +1,100 @@ +import { createClient, type RedisClientType } from "redis"; +import { env } from "../setup/env"; +import { waitFor } from "./wait-for"; + +export interface EngineResponse { + correlationId: string; + success: boolean; + data?: T; + error?: string; +} + +export interface BrokerCommand { + correlationId: string; + responseQueue: string; + type: string; + payload: Record; +} + +export class EngineClient { + readonly redis: RedisClientType; + + constructor() { + this.redis = createClient({ url: env.redisUrl }); + } + + connect() { + return this.redis.connect(); + } + + async close() { + if (this.redis.isOpen) await this.redis.quit(); + } + + async request(type: string, payload: Record) { + const correlationId = crypto.randomUUID(); + const responseQueue = `integration-response-${correlationId}`; + + await this.redis.xAdd(env.incomingStream, "*", { + correlationId, + responseQueue, + type, + payload: JSON.stringify(payload), + }); + + const result = await this.redis.brPop(responseQueue, 5); + await this.redis.del(responseQueue); + if (!result) throw new Error(`Engine did not respond to ${type}`); + + const response = JSON.parse(result.element) as EngineResponse; + return { correlationId, responseQueue, response }; + } + + async command(type: string, payload: Record): Promise { + const { response } = await this.request(type, payload); + if (!response.success) throw new Error(response.error ?? `Engine command ${type} failed`); + return response.data as T; + } + + async findBrokerCommand( + type: string, + acceptPayload: (payload: Record) => boolean, + ): Promise { + const command = await waitFor( + async () => { + const entries = await this.redis.xRange(env.incomingStream, "-", "+"); + for (let index = entries.length - 1; index >= 0; index--) { + const message = entries[index]?.message; + if ( + !message || + message.type !== type || + !message.payload || + !message.correlationId || + !message.responseQueue + ) { + continue; + } + const payload = JSON.parse(message.payload) as Record; + if (acceptPayload(payload)) { + return { + correlationId: message.correlationId, + responseQueue: message.responseQueue, + type: message.type, + payload, + }; + } + } + return undefined; + }, + (value) => value !== undefined, + { description: `${type} command in broker stream` }, + ); + + if (!command) throw new Error(`Missing ${type} broker command`); + return command; + } + + addStreamEvent(stream: "stream:order" | "stream:fill", event: Record) { + return this.redis.xAdd(stream, "*", { data: JSON.stringify(event) }); + } +} diff --git a/tests/integration/helpers/users.ts b/tests/integration/helpers/users.ts new file mode 100644 index 0000000..3e5f3dd --- /dev/null +++ b/tests/integration/helpers/users.ts @@ -0,0 +1,31 @@ +import { ApiClient } from "./api"; +import type { EngineClient } from "./engine"; + +const password = "integration-password"; + +export interface TestUser { + id: string; + email: string; + client: ApiClient; +} + +export async function createTestUser(label: string): Promise { + const client = new ApiClient(); + const email = `${label}-${crypto.randomUUID()}@test.paperdrill`; + const response = await client.signup(email, label, password); + + if (response.status !== 201 || !response.data.emailVerified) { + throw new Error(`Failed to create test user ${email}: ${JSON.stringify(response.data)}`); + } + + return { id: response.data.id, email, client }; +} + +export async function seedBalance( + engine: EngineClient, + userId: string, + asset: string, + amount: string, +) { + await engine.command("create_deposit", { userId, asset, amount }); +} diff --git a/tests/integration/helpers/wait-for.ts b/tests/integration/helpers/wait-for.ts new file mode 100644 index 0000000..618f06d --- /dev/null +++ b/tests/integration/helpers/wait-for.ts @@ -0,0 +1,25 @@ +export async function waitFor( + probe: () => Promise, + accept: (value: T) => boolean, + options: { description: string; timeoutMs?: number; intervalMs?: number }, +): Promise { + const timeoutMs = options.timeoutMs ?? 8000; + const intervalMs = options.intervalMs ?? 50; + const deadline = Date.now() + timeoutMs; + let lastValue: T | undefined; + let lastError: unknown; + + while (Date.now() < deadline) { + try { + lastValue = await probe(); + if (accept(lastValue)) return lastValue; + } catch (error) { + lastError = error; + } + + await Bun.sleep(intervalMs); + } + + const detail = lastError instanceof Error ? lastError.message : JSON.stringify(lastValue); + throw new Error(`Timed out waiting for ${options.description}; last result: ${detail}`); +} diff --git a/tests/integration/persistence/worker.test.ts b/tests/integration/persistence/worker.test.ts new file mode 100644 index 0000000..9e24616 --- /dev/null +++ b/tests/integration/persistence/worker.test.ts @@ -0,0 +1,56 @@ +import { afterAll, beforeAll, expect, test } from "bun:test"; +import { waitForSystemReady } from "../helpers/api"; +import { EngineClient } from "../helpers/engine"; +import { createTestUser } from "../helpers/users"; +import { waitFor } from "../helpers/wait-for"; +import { TestDatabase } from "../setup/database"; + +const engine = new EngineClient(); +const database = new TestDatabase(); + +beforeAll(async () => { + await engine.connect(); + await waitForSystemReady(); +}); + +afterAll(async () => { + await Promise.all([engine.close(), database.close()]); +}); + +test("worker persists a duplicate fill event only once", async () => { + const buyer = await createTestUser("duplicate-fill-buyer"); + const seller = await createTestUser("duplicate-fill-seller"); + const fill = { + id: crypto.randomUUID(), + symbol: "ETH_USD", + price: "12345", + qty: "100", + buyOrderId: crypto.randomUUID(), + sellOrderId: crypto.randomUUID(), + buyerId: buyer.id, + sellerId: seller.id, + isBuyerMaker: false, + createdAt: Date.now(), + }; + + await engine.addStreamEvent("stream:fill", { event: "fill", fill }); + const secondEventId = await engine.addStreamEvent("stream:fill", { event: "fill", fill }); + + await waitFor( + () => engine.redis.get("worker:fill:last_id"), + (cursor) => cursor === secondEventId, + { description: "worker duplicate-fill checkpoint" }, + ); + + expect(await database.fillCount(fill.id)).toBe(1); + expect(await database.fill(fill.id)).toEqual({ + id: fill.id, + symbol: "ETH_USD", + price: "12345", + qty: "100", + buyOrderId: fill.buyOrderId, + sellOrderId: fill.sellOrderId, + buyerId: buyer.id, + sellerId: seller.id, + }); +}); diff --git a/tests/integration/redis/contracts.test.ts b/tests/integration/redis/contracts.test.ts new file mode 100644 index 0000000..7a48ee4 --- /dev/null +++ b/tests/integration/redis/contracts.test.ts @@ -0,0 +1,79 @@ +import { afterAll, beforeAll, expect, test } from "bun:test"; +import { waitForSystemReady } from "../helpers/api"; +import { EngineClient } from "../helpers/engine"; +import { createTestUser } from "../helpers/users"; + +const engine = new EngineClient(); + +beforeAll(async () => { + await engine.connect(); + await waitForSystemReady(); +}); + +afterAll(() => engine.close()); + +test("backend writes complete place and cancel commands to the Redis broker", async () => { + const user = await createTestUser("backend-redis"); + const created = await user.client.createOrder({ + type: "LIMIT", + side: "BUY", + symbol: "BTC_USD", + price: "123.45", + qty: "0.2500", + }); + + expect(created).toMatchObject({ status: 200, data: { status: "OPEN" } }); + + const placeCommand = await engine.findBrokerCommand( + "create_order", + (payload) => payload.id === created.data.id, + ); + expect(typeof placeCommand.correlationId).toBe("string"); + expect(placeCommand.responseQueue).toMatch(/^response-queue-/); + expect(placeCommand.payload).toEqual({ + id: created.data.id, + userId: user.id, + type: "LIMIT", + side: "BUY", + symbol: "BTC_USD", + price: "12345", + qty: "2500", + }); + + const cancelled = await user.client.cancelOrder(created.data.id); + expect(cancelled).toMatchObject({ status: 200, data: { id: created.data.id } }); + + const cancelCommand = await engine.findBrokerCommand( + "cancel_order", + (payload) => payload.id === created.data.id, + ); + expect(typeof cancelCommand.correlationId).toBe("string"); + expect(cancelCommand.responseQueue).toMatch(/^response-queue-/); + expect(cancelCommand.payload).toEqual({ userId: user.id, id: created.data.id }); +}); + +test("engine consumes a real Redis order command and returns a correlated response", async () => { + const user = await createTestUser("redis-engine"); + const orderId = crypto.randomUUID(); + const result = await engine.request>("create_order", { + id: orderId, + userId: user.id, + type: "LIMIT", + side: "BUY", + symbol: "SOL_USD", + price: "100", + qty: "100", + }); + + expect(result.response).toMatchObject({ + correlationId: result.correlationId, + success: true, + data: { id: orderId, symbol: "SOL_USD", status: "OPEN", filledQty: "0" }, + }); + expect(await user.client.getBalances()).toMatchObject({ + status: 200, + data: { USD: { available: "9999.00", locked: "1.00" } }, + }); + + await engine.command("cancel_order", { userId: user.id, id: orderId }); +}); diff --git a/tests/integration/setup/database.ts b/tests/integration/setup/database.ts new file mode 100644 index 0000000..b140796 --- /dev/null +++ b/tests/integration/setup/database.ts @@ -0,0 +1,62 @@ +import { Pool } from "pg"; +import { env } from "./env"; + +export interface PersistedOrder { + id: string; + userId: string; + symbol: string; + price: string | null; + qty: string; + filledQty: string; + status: "OPEN" | "PARTIALLY_FILLED" | "FILLED" | "CANCELLED"; + lockedAmount: string | null; + spentAmount: string; + averagePrice: string | null; +} + +export interface PersistedFill { + id: string; + symbol: string; + price: string; + qty: string; + buyOrderId: string; + sellOrderId: string; + buyerId: string; + sellerId: string; +} + +export class TestDatabase { + private readonly pool = new Pool({ connectionString: env.databaseUrl }); + + async close() { + await this.pool.end(); + } + + async order(id: string): Promise { + const result = await this.pool.query( + `SELECT id, "userId", symbol, price::text, qty::text, "filledQty"::text, + status, "lockedAmount"::text, "spentAmount"::text, "averagePrice"::text + FROM "Order" WHERE id = $1`, + [id], + ); + return result.rows[0]; + } + + async fill(id: string): Promise { + const result = await this.pool.query( + `SELECT id, symbol, price::text, qty::text, "buyOrderId", "sellOrderId", + "buyerId", "sellerId" + FROM "Fill" WHERE id = $1`, + [id], + ); + return result.rows[0]; + } + + async fillCount(id: string): Promise { + const result = await this.pool.query<{ count: string }>( + `SELECT COUNT(*)::text AS count FROM "Fill" WHERE id = $1`, + [id], + ); + return Number(result.rows[0]?.count ?? 0); + } +} diff --git a/tests/integration/setup/env.ts b/tests/integration/setup/env.ts new file mode 100644 index 0000000..0159708 --- /dev/null +++ b/tests/integration/setup/env.ts @@ -0,0 +1,28 @@ +function required(name: string): string { + const value = process.env[name]?.trim(); + if (!value) throw new Error(`${name} is required for integration tests`); + return value; +} + +const nodeEnv = required("NODE_ENV"); +const databaseUrl = new URL(required("DATABASE_URL")); +const redisUrl = new URL(required("REDIS_URL")); + +if (nodeEnv !== "test") { + throw new Error("Refusing to run integration tests outside the explicit test environment"); +} + +if (!databaseUrl.pathname.toLowerCase().includes("test")) { + throw new Error("Refusing to use a database whose name does not contain 'test'"); +} + +if (redisUrl.protocol !== "redis:" && redisUrl.protocol !== "rediss:") { + throw new Error("REDIS_URL must use the redis protocol"); +} + +export const env = { + apiBaseUrl: required("API_BASE_URL").replace(/\/$/, ""), + databaseUrl: databaseUrl.toString(), + redisUrl: redisUrl.toString(), + incomingStream: required("INCOMING_STREAM"), +}; diff --git a/tests/package.json b/tests/package.json new file mode 100644 index 0000000..0d3fe49 --- /dev/null +++ b/tests/package.json @@ -0,0 +1,19 @@ +{ + "name": "paperdrill-tests", + "type": "module", + "private": true, + "scripts": { + "test": "bun test integration --timeout 20000 --max-concurrency 1", + "test:unit": "cd ../backend && set -a && . ../.env.test && set +a && bun test && cd ../engine && set -a && . ../.env.test && set +a && bun test", + "test:integration": "cd .. && sh tests/run-integration.sh" + }, + "dependencies": { + "pg": "^8.20.0", + "redis": "^6.0.0" + }, + "devDependencies": { + "@types/bun": "latest", + "@types/pg": "^8.20.0", + "typescript": "^5" + } +} diff --git a/tests/run-integration.sh b/tests/run-integration.sh new file mode 100644 index 0000000..f3068d7 --- /dev/null +++ b/tests/run-integration.sh @@ -0,0 +1,45 @@ +#!/bin/sh + +set -u + +compose() { + docker compose -p paperdrill-test -f compose.test.yml "$@" +} + +cleanup() { + compose down --volumes --remove-orphans +} + +trap cleanup EXIT INT TERM + +status=0 + +compose down --volumes --remove-orphans || true + +compose build || status=$? + +if [ "$status" -eq 0 ]; then + compose up -d --wait backend engine worker || status=$? +fi + +if [ "$status" -eq 0 ]; then + ( + set -a + . ./.env.test + set +a + + export API_BASE_URL="http://127.0.0.1:8001" + export DATABASE_URL="postgresql://postgres:password@127.0.0.1:5434/paperdrill_test" + export REDIS_URL="redis://127.0.0.1:6381" + + cd tests + bun run test + ) || status=$? +fi + +if [ "$status" -ne 0 ]; then + compose ps + compose logs --no-color backend engine worker postgres redis setup +fi + +exit "$status" \ No newline at end of file diff --git a/tests/tsconfig.json b/tests/tsconfig.json new file mode 100644 index 0000000..bfa0fea --- /dev/null +++ b/tests/tsconfig.json @@ -0,0 +1,29 @@ +{ + "compilerOptions": { + // Environment setup & latest features + "lib": ["ESNext"], + "target": "ESNext", + "module": "Preserve", + "moduleDetection": "force", + "jsx": "react-jsx", + "allowJs": true, + + // Bundler mode + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "verbatimModuleSyntax": true, + "noEmit": true, + + // Best practices + "strict": true, + "skipLibCheck": true, + "noFallthroughCasesInSwitch": true, + "noUncheckedIndexedAccess": true, + "noImplicitOverride": true, + + // Some stricter flags (disabled by default) + "noUnusedLocals": false, + "noUnusedParameters": false, + "noPropertyAccessFromIndexSignature": false + } +} From 805a125d1395b8d54c37af0398dfb53bbbef6f68 Mon Sep 17 00:00:00 2001 From: Harshil Date: Sun, 13 Sep 2026 19:59:52 +0530 Subject: [PATCH 02/20] refactor: standardize API error responses across controllers --- backend/src/app.ts | 20 +++---- backend/src/controllers/auth.ts | 33 +++++++----- backend/src/controllers/key.ts | 18 ++++--- backend/src/controllers/leaderboard.ts | 13 +++-- backend/src/controllers/market.ts | 27 +++++++--- backend/src/controllers/order.ts | 41 +++++++++------ backend/src/controllers/user.ts | 25 +++++---- backend/src/middleware/auth.ts | 22 ++++---- backend/src/middleware/errors.ts | 57 ++++++++++++++++++++ backend/src/middleware/rateLimit.ts | 3 +- backend/src/store/pendingResponses.ts | 9 ++++ backend/src/types/express.d.ts | 1 + backend/src/utils/apiError.ts | 64 +++++++++++++++++++++++ backend/src/utils/engineClient.ts | 52 +++++++++++------- backend/src/utils/formatter.ts | 5 +- backend/src/utils/validation.ts | 16 +++--- backend/tests/access.test.ts | 10 ++-- backend/tests/auth.test.ts | 5 +- bot/src/client.ts | 6 ++- frontend/src/lib/api.ts | 31 ++++++++--- tests/integration/api/http.test.ts | 56 +++++++++++++++----- tests/integration/api/orders.test.ts | 5 +- tests/integration/flows/trading.test.ts | 23 ++++---- tests/integration/helpers/api.ts | 11 +++- tests/integration/redis/contracts.test.ts | 2 +- 25 files changed, 403 insertions(+), 152 deletions(-) create mode 100644 backend/src/middleware/errors.ts create mode 100644 backend/src/utils/apiError.ts diff --git a/backend/src/app.ts b/backend/src/app.ts index 9213d13..a1388d1 100644 --- a/backend/src/app.ts +++ b/backend/src/app.ts @@ -1,4 +1,4 @@ -import express, { type NextFunction, type Request, type Response } from "express"; +import express, { type Request, type Response } from "express"; import cors from "cors"; import { pingRedis } from "./redis"; import { prisma } from "./db"; @@ -7,11 +7,14 @@ import { config } from "./config"; import { logger } from "./utils/logger"; import { apiLimiter } from "./middleware/rateLimit"; import { authenticate } from "./middleware/auth"; +import { errorHandler, notFoundHandler, requestContext } from "./middleware/errors"; +import { sendApiError } from "./utils/apiError"; export const app = express(); app.set("trust proxy", 1); +app.use(requestContext); app.use(cors(config.cors)); app.use(express.json()); app.use(express.urlencoded({ extended: true })); @@ -35,19 +38,10 @@ app.get("/health", async (_req: Request, res: Response) => { res.status(200).json({ success: true, status: "ok" }); } catch (err) { logger.error("Health check failed", err); - res.status(503).json({ success: false, status: "error" }); + sendApiError(res, 503, "SERVICE_UNAVAILABLE", "A required service is unavailable"); } }); app.use("/v1", appRouter); - -app.use((err: unknown, _req: Request, res: Response, _next: NextFunction) => { - logger.error("Unhandled error", err); - const message = - config.app.env === "production" - ? "Internal server error" - : err instanceof Error - ? err.message - : "Internal server error"; - res.status(500).json({ error: message }); -}); +app.use(notFoundHandler); +app.use(errorHandler); diff --git a/backend/src/controllers/auth.ts b/backend/src/controllers/auth.ts index 8125b09..26257e8 100644 --- a/backend/src/controllers/auth.ts +++ b/backend/src/controllers/auth.ts @@ -16,6 +16,7 @@ import { isEmailDeliveryEnabled, sendVerificationEmail } from "../utils/emailCli import { sendToEngine } from "../utils/engineClient"; import { assetPrecision } from "../store/market"; import { toBigInt } from "../utils/convert"; +import { ApiError, sendApiError } from "../utils/apiError"; const INITIAL_BALANCE_ASSET = "USD"; const INITIAL_BALANCE_AMOUNT = "10000"; @@ -55,7 +56,7 @@ export async function signup(req: Request, res: Response) { const bypassEmailVerification = config.app.env !== "production" && !emailDeliveryEnabled; if (config.app.env === "production" && !emailDeliveryEnabled) { - res.status(503).json({ error: "Account registration is currently unavailable" }); + sendApiError(res, 503, "SERVICE_UNAVAILABLE", "Account registration is currently unavailable"); return; } @@ -63,7 +64,7 @@ export async function signup(req: Request, res: Response) { const existingUser = await prisma.user.findUnique({ where: { email } }); if (existingUser) { - res.status(400).json({ error: "User already exists" }); + sendApiError(res, 409, "CONFLICT", "An account with this email already exists"); return; } @@ -116,7 +117,11 @@ export async function signup(req: Request, res: Response) { }); } catch (e) { logger.error("Signup failed", e); - res.status(500).json({ error: "Internal server error" }); + if (e instanceof ApiError) { + sendApiError(res, e.status, e.code, e.message, e.details); + return; + } + sendApiError(res, 500, "INTERNAL_ERROR", "Account creation failed unexpectedly"); } } @@ -136,13 +141,13 @@ export async function signin(req: Request, res: Response) { }); if (!user) { - res.status(400).json({ error: "Invalid email or password" }); + sendApiError(res, 401, "INVALID_CREDENTIALS", "Invalid email or password"); return; } let match = await bcrypt.compare(password, user.password); if (!match) { - res.status(400).json({ error: "Invalid email or password" }); + sendApiError(res, 401, "INVALID_CREDENTIALS", "Invalid email or password"); return; } @@ -157,7 +162,7 @@ export async function signin(req: Request, res: Response) { }); } catch (e) { logger.error("Signin failed", e); - res.status(500).json({ error: "Internal server error" }); + sendApiError(res, 500, "INTERNAL_ERROR", "Sign in failed unexpectedly"); } } @@ -186,17 +191,17 @@ export async function verifyEmail(req: Request, res: Response) { }); if (!verification) { - res.status(404).json({ error: "Invalid verification token" }); + sendApiError(res, 404, "INVALID_VERIFICATION_TOKEN", "Verification token was not found"); return; } if (verification.expiresAt < new Date()) { - res.status(400).json({ error: "Verification token has expired" }); + sendApiError(res, 410, "VERIFICATION_TOKEN_EXPIRED", "Verification token has expired"); return; } if (verification.usedAt) { - res.status(400).json({ error: "Verification token has already been used" }); + sendApiError(res, 409, "VERIFICATION_TOKEN_USED", "Verification token has already been used"); return; } @@ -213,7 +218,7 @@ export async function verifyEmail(req: Request, res: Response) { res.status(200).json({ message: "Email verified successfully" }); } catch (e) { logger.error("Email verification failed", e); - res.status(500).json({ error: "Internal server error" }); + sendApiError(res, 500, "INTERNAL_ERROR", "Email verification failed unexpectedly"); } } @@ -227,7 +232,7 @@ export async function resendVerificationEmail(req: Request, res: Response) { const { email } = parsedBody.data; if (!isEmailDeliveryEnabled()) { - res.status(503).json({ error: "Email delivery is currently unavailable" }); + sendApiError(res, 503, "SERVICE_UNAVAILABLE", "Email delivery is currently unavailable"); return; } @@ -235,12 +240,12 @@ export async function resendVerificationEmail(req: Request, res: Response) { const user = await prisma.user.findUnique({ where: { email } }); if (!user) { - res.status(404).json({ error: "User not found" }); + sendApiError(res, 404, "USER_NOT_FOUND", "User not found"); return; } if (user.emailVerified) { - res.status(400).json({ error: "Email is already verified" }); + sendApiError(res, 409, "CONFLICT", "Email is already verified"); return; } @@ -262,6 +267,6 @@ export async function resendVerificationEmail(req: Request, res: Response) { }); } catch (e) { logger.error("Resend verification email failed", e); - res.status(500).json({ error: "Internal server error" }); + sendApiError(res, 500, "INTERNAL_ERROR", "Verification email could not be sent"); } } diff --git a/backend/src/controllers/key.ts b/backend/src/controllers/key.ts index 0df6178..eea7ad7 100644 --- a/backend/src/controllers/key.ts +++ b/backend/src/controllers/key.ts @@ -7,6 +7,7 @@ import { hashSecret } from "../middleware/auth"; import { API_KEY_PREFIX } from "../types/principal"; import { getUserId } from "./user"; import { logger } from "../utils/logger"; +import { sendApiError } from "../utils/apiError"; const MAX_ACTIVE_KEYS_PER_USER = 10; @@ -27,9 +28,12 @@ export async function createKey(req: Request, res: Response) { }); if (activeKeys >= MAX_ACTIVE_KEYS_PER_USER) { - res - .status(400) - .json({ error: `Maximum of ${MAX_ACTIVE_KEYS_PER_USER} active API keys reached` }); + sendApiError( + res, + 409, + "API_KEY_LIMIT_REACHED", + `Maximum of ${MAX_ACTIVE_KEYS_PER_USER} active API keys reached`, + ); return; } @@ -51,7 +55,7 @@ export async function createKey(req: Request, res: Response) { }); } catch (e) { logger.error("createKey failed", e); - res.status(500).json({ error: "Internal server error" }); + sendApiError(res, 500, "INTERNAL_ERROR", "API key could not be created"); } } @@ -75,7 +79,7 @@ export async function listKeys(req: Request, res: Response) { res.status(200).json({ keys }); } catch (e) { logger.error("listKeys failed", e); - res.status(500).json({ error: "Internal server error" }); + sendApiError(res, 500, "INTERNAL_ERROR", "API keys could not be loaded"); } } @@ -90,13 +94,13 @@ export async function revokeKey(req: Request, res: Response) { }); if (result.count === 0) { - res.status(404).json({ error: "API key not found or already revoked" }); + sendApiError(res, 404, "API_KEY_NOT_FOUND", "API key not found or already revoked"); return; } res.status(200).json({ success: true, message: "API key revoked" }); } catch (e) { logger.error("revokeKey failed", e); - res.status(500).json({ error: "Internal server error" }); + sendApiError(res, 500, "INTERNAL_ERROR", "API key could not be revoked"); } } diff --git a/backend/src/controllers/leaderboard.ts b/backend/src/controllers/leaderboard.ts index 24f941f..527a8d3 100644 --- a/backend/src/controllers/leaderboard.ts +++ b/backend/src/controllers/leaderboard.ts @@ -9,6 +9,7 @@ import { logger } from "../utils/logger"; import { sendValidationError } from "../utils/validation"; import { config } from "../config"; import { isLeaderboardStale } from "../utils/leaderboard"; +import { sendApiError } from "../utils/apiError"; export async function getLeaderboard(req: Request, res: Response) { const parsedQuery = leaderboardQuerySchema.safeParse(req.query); @@ -42,7 +43,7 @@ export async function getLeaderboard(req: Request, res: Response) { ); if (!snapshot) { - res.status(503).json({ error: "Leaderboard is being calculated" }); + sendApiError(res, 503, "LEADERBOARD_UNAVAILABLE", "Leaderboard is being calculated"); return; } @@ -62,7 +63,7 @@ export async function getLeaderboard(req: Request, res: Response) { }); } catch (error) { logger.error("Failed to fetch leaderboard", error); - res.status(500).json({ error: "Internal server error" }); + sendApiError(res, 500, "INTERNAL_ERROR", "Leaderboard could not be loaded"); } } @@ -75,10 +76,8 @@ export async function getMyLeaderboardEntry(req: Request, res: Response) { select: { emailVerified: true }, }); if (!user) { - res - .clearCookie("token", config.cookie) - .status(401) - .json({ error: "Authentication required" }); + res.clearCookie("token", config.cookie); + sendApiError(res, 401, "AUTHENTICATION_REQUIRED", "Authentication required"); return; } if (!user.emailVerified) { @@ -120,6 +119,6 @@ export async function getMyLeaderboardEntry(req: Request, res: Response) { }); } catch (error) { logger.error("Failed to fetch personal leaderboard entry", error); - res.status(500).json({ error: "Internal server error" }); + sendApiError(res, 500, "INTERNAL_ERROR", "Leaderboard entry could not be loaded"); } } diff --git a/backend/src/controllers/market.ts b/backend/src/controllers/market.ts index d5bf418..8b7a303 100644 --- a/backend/src/controllers/market.ts +++ b/backend/src/controllers/market.ts @@ -13,6 +13,7 @@ import { } from "../utils/formatter"; import { logger } from "../utils/logger"; import { marketStore } from "../store/market"; +import { sendApiError, sendEngineError } from "../utils/apiError"; const intervalMap = { "15M": "15 minutes", @@ -28,7 +29,7 @@ export async function getMarkets(_req: Request, res: Response) { res.status(200).json({ data: markets }); } catch (e) { logger.error("Failed to fetch markets", e); - res.status(500).json({ error: "Internal server error" }); + sendApiError(res, 500, "INTERNAL_ERROR", "Markets could not be loaded"); } } @@ -48,11 +49,15 @@ export async function getTrades(req: Request, res: Response) { const { symbol } = parsedParams.data; const { limit = 50 } = parsedQueries.data; + if (!marketStore.has(symbol)) { + sendApiError(res, 404, "MARKET_NOT_FOUND", `Market ${symbol} was not found`); + return; + } const engineResponse = await sendToEngine("get_trades", { symbol, limit }); if (!engineResponse.success) { - res.status(400).json({ error: engineResponse.error }); + sendEngineError(res, engineResponse.error); return; } @@ -68,11 +73,15 @@ export async function getDepth(req: Request, res: Response) { } const { symbol } = parsedParams.data; + if (!marketStore.has(symbol)) { + sendApiError(res, 404, "MARKET_NOT_FOUND", `Market ${symbol} was not found`); + return; + } const engineResponse = await sendToEngine("get_depth", { symbol }); if (!engineResponse.success) { - res.status(400).json({ error: engineResponse.error }); + sendEngineError(res, engineResponse.error); return; } @@ -97,6 +106,10 @@ export async function getCandles(req: Request, res: Response) { const { symbol } = parsedParams.data; const { interval = "15M" } = parsedQueries.data; const bucket = intervalMap[interval]; + if (!marketStore.has(symbol)) { + sendApiError(res, 404, "MARKET_NOT_FOUND", `Market ${symbol} was not found`); + return; + } try { const candles = await prisma.$queryRaw` @@ -124,7 +137,7 @@ export async function getCandles(req: Request, res: Response) { }); } catch (err) { logger.error("Failed to fetch candles", err); - res.status(500).json({ error: "Internal server error" }); + sendApiError(res, 500, "INTERNAL_ERROR", "Candles could not be loaded"); } } @@ -142,7 +155,7 @@ export async function getTicker(req: Request, res: Response) { const market = marketStore.get(symbol); if (!market) { - res.status(404).json({ error: "Market not found" }); + sendApiError(res, 404, "MARKET_NOT_FOUND", `Market ${symbol} was not found`); return; } @@ -156,7 +169,7 @@ export async function getTicker(req: Request, res: Response) { res.status(200).json(formatTicker(JSON.parse(raw) as Record)); } catch (err) { logger.error("Failed to fetch ticker", err); - res.status(500).json({ error: "Internal server error" }); + sendApiError(res, 500, "INTERNAL_ERROR", "Ticker could not be loaded"); } } @@ -178,6 +191,6 @@ export async function getTickers(_req: Request, res: Response) { res.status(200).json(formatTickers(tickers)); } catch (err) { logger.error("Failed to fetch tickers", err); - res.status(500).json({ error: "Internal server error" }); + sendApiError(res, 500, "INTERNAL_ERROR", "Tickers could not be loaded"); } } diff --git a/backend/src/controllers/order.ts b/backend/src/controllers/order.ts index 8d7976e..a5dd43a 100644 --- a/backend/src/controllers/order.ts +++ b/backend/src/controllers/order.ts @@ -8,6 +8,7 @@ import { formatOrder, formatOrders, formatCancel } from "../utils/formatter"; import { getUserId } from "./user"; import { prisma } from "../db"; import { logger } from "../utils/logger"; +import { sendApiError, sendEngineError } from "../utils/apiError"; export async function createOrder(req: Request, res: Response) { const userId = getUserId(req); @@ -23,13 +24,26 @@ export async function createOrder(req: Request, res: Response) { const market = marketStore.get(symbol); if (!market) { - res.status(400).json({ error: `Unknown market: ${symbol}` }); + sendApiError(res, 404, "MARKET_NOT_FOUND", `Market ${symbol} was not found`); return; } - const scaledQty = toBigInt(qty, market.qtyPrecision); - const scaledPrice = - type === "MARKET" ? null : toBigInt(parsedBody.data.price, market.pricePrecision); + let scaledQty: bigint; + let scaledPrice: bigint | null; + try { + scaledQty = toBigInt(qty, market.qtyPrecision); + scaledPrice = + type === "MARKET" ? null : toBigInt(parsedBody.data.price, market.pricePrecision); + } catch (error) { + sendApiError( + res, + 400, + "VALIDATION_ERROR", + "Order precision is invalid for this market", + [{ message: error instanceof Error ? error.message : "Invalid order precision" }], + ); + return; + } const id = crypto.randomUUID(); const engineResponse = await sendToEngine("create_order", { @@ -43,11 +57,11 @@ export async function createOrder(req: Request, res: Response) { }); if (!engineResponse.success) { - res.status(400).json({ error: engineResponse.error }); + sendEngineError(res, engineResponse.error); return; } - res.status(200).json(formatOrder(engineResponse.data as Record)); + res.status(201).json(formatOrder(engineResponse.data as Record)); } export async function getOpenOrders(req: Request, res: Response) { @@ -56,7 +70,7 @@ export async function getOpenOrders(req: Request, res: Response) { const engineResponse = await sendToEngine("get_open_orders", { userId }); if (!engineResponse.success) { - res.status(400).json({ error: engineResponse.error }); + sendEngineError(res, engineResponse.error); return; } @@ -86,15 +100,10 @@ export async function getOrders(req: Request, res: Response) { orderBy: { createdAt: "desc" }, }); - if (!orders) { - res.status(404).json({ error: "Order not found" }); - return; - } - res.status(200).json(formatOrders(orders)); } catch (err) { logger.error("Failed to fetch order", err); - res.status(500).json({ error: "Internal server error" }); + sendApiError(res, 500, "INTERNAL_ERROR", "Order history could not be loaded"); } } @@ -118,14 +127,14 @@ export async function getOrderById(req: Request, res: Response) { }); if (!order) { - res.status(404).json({ error: "Order not found" }); + sendApiError(res, 404, "ORDER_NOT_FOUND", "Order not found"); return; } res.status(200).json(formatOrder(order as Record)); } catch (err) { logger.error("Failed to fetch order", err); - res.status(500).json({ error: "Internal server error" }); + sendApiError(res, 500, "INTERNAL_ERROR", "Order could not be loaded"); } } @@ -142,7 +151,7 @@ export async function cancelOrder(req: Request, res: Response) { const engineResponse = await sendToEngine("cancel_order", { userId, id: orderId }); if (!engineResponse.success) { - res.status(400).json({ error: engineResponse.error }); + sendEngineError(res, engineResponse.error); return; } diff --git a/backend/src/controllers/user.ts b/backend/src/controllers/user.ts index 947f366..83c8425 100644 --- a/backend/src/controllers/user.ts +++ b/backend/src/controllers/user.ts @@ -9,12 +9,13 @@ import { marketStore } from "../store/market"; import { calculatePortfolio, type Balance } from "../utils/portfolio"; import { getReferencePrices } from "../utils/referencePrice"; import { config } from "../config"; +import { ApiError, sendApiError, sendEngineError } from "../utils/apiError"; export function getUserId(req: Request): string { const userId = req.principal?.userId; if (!userId) { - throw new Error("Missing authenticated user"); + throw new ApiError(401, "AUTHENTICATION_REQUIRED", "Authentication required"); } return userId; @@ -31,10 +32,8 @@ export async function getUserData(req: Request, res: Response) { }); if (!user) { - res - .clearCookie("token", config.cookie) - .status(401) - .json({ error: "Authentication required" }); + res.clearCookie("token", config.cookie); + sendApiError(res, 401, "AUTHENTICATION_REQUIRED", "Authentication required"); return; } @@ -46,7 +45,7 @@ export async function getUserData(req: Request, res: Response) { }); } catch (e) { logger.error("getUserData failed", e); - res.status(500).json({ error: "Internal server error" }); + sendApiError(res, 500, "INTERNAL_ERROR", "User profile could not be loaded"); } } @@ -75,7 +74,7 @@ export async function getTradeHistory(req: Request, res: Response) { ); } catch (err) { logger.error("Failed to fetch trade history", err); - res.status(500).json({ error: "Internal server error" }); + sendApiError(res, 500, "INTERNAL_ERROR", "Trade history could not be loaded"); } } @@ -94,7 +93,7 @@ export async function getBalance(req: Request, res: Response) { const engineResponse = await sendToEngine("get_user_balance", { userId, asset }); if (!engineResponse.success) { - res.status(400).json({ error: engineResponse.error }); + sendEngineError(res, engineResponse.error); return; } @@ -108,7 +107,7 @@ export async function getPortfolio(req: Request, res: Response) { const engineResponse = await sendToEngine("get_user_balance", { userId }); if (!engineResponse.success) { - res.status(400).json({ error: engineResponse.error }); + sendEngineError(res, engineResponse.error); return; } @@ -131,20 +130,20 @@ export async function getPortfolio(req: Request, res: Response) { }); if (!user) { - res.status(404).json({ error: "User not found" }); + sendApiError(res, 401, "AUTHENTICATION_REQUIRED", "Authentication required"); return; } if (user.pnlBaseline == null || user.pnlBaselineAt == null) { logger.warn("Portfolio requested for user without a baseline", { userId }); - res.status(503).json({ error: "Portfolio baseline is not configured" }); + sendApiError(res, 503, "PORTFOLIO_UNAVAILABLE", "Portfolio baseline is not configured"); return; } res.status(200).json(formatPortfolio(portfolio, user.pnlBaseline, user.pnlBaselineAt)); } catch (error) { logger.error("Failed to calculate portfolio", error); - res.status(503).json({ error: "Portfolio valuation is temporarily unavailable" }); + sendApiError(res, 503, "PORTFOLIO_UNAVAILABLE", "Portfolio valuation is temporarily unavailable"); } } @@ -162,7 +161,7 @@ export async function createDeposit(req: Request, res: Response) { const engineResponse = await sendToEngine("create_deposit", { userId, amount, asset }); if (!engineResponse.success) { - res.status(400).json({ error: engineResponse.error }); + sendEngineError(res, engineResponse.error); return; } diff --git a/backend/src/middleware/auth.ts b/backend/src/middleware/auth.ts index c4c2791..58c377b 100644 --- a/backend/src/middleware/auth.ts +++ b/backend/src/middleware/auth.ts @@ -7,6 +7,7 @@ import { prisma } from "../db"; import { logger } from "../utils/logger"; import { API_KEY_PREFIX, type PrincipalType } from "../types/principal"; import type { Scope } from "../../generated/prisma/enums"; +import { sendApiError } from "../utils/apiError"; interface TokenPayload { id: string; @@ -54,7 +55,7 @@ export async function authenticate(req: Request, res: Response, next: NextFuncti }); if (!service) { - res.status(401).json({ error: "Invalid service email" }); + sendApiError(res, 401, "INVALID_API_KEY", "Invalid service credentials"); return; } @@ -68,14 +69,14 @@ export async function authenticate(req: Request, res: Response, next: NextFuncti } if (!credential.startsWith(API_KEY_PREFIX)) { - res.status(401).json({ error: "Malformed API key" }); + sendApiError(res, 401, "INVALID_API_KEY", "Malformed API key"); return; } const token = credential.slice(API_KEY_PREFIX.length); const separator = token.indexOf("."); if (separator <= 0) { - res.status(401).json({ error: "Malformed API key" }); + sendApiError(res, 401, "INVALID_API_KEY", "Malformed API key"); return; } @@ -84,7 +85,7 @@ export async function authenticate(req: Request, res: Response, next: NextFuncti const key = await prisma.apiKey.findUnique({ where: { id: keyId } }); if (!key || key.revokedAt || !secretsMatch(secret, key.hashedSecret)) { - res.status(401).json({ error: "Invalid API key" }); + sendApiError(res, 401, "INVALID_API_KEY", "Invalid API key"); return; } @@ -133,12 +134,12 @@ export function requireAccess({ types, scopes, allowUnverified = false }: Access const principal = req.principal; if (!principal) { - res.status(401).json({ error: "Authentication required" }); + sendApiError(res, 401, "AUTHENTICATION_REQUIRED", "Authentication required"); return; } if (types?.length && !types.includes(principal.type)) { - res.status(403).json({ error: "You do not have permission to perform this action" }); + sendApiError(res, 403, "FORBIDDEN", "You do not have permission to perform this action"); return; } @@ -153,7 +154,7 @@ export function requireAccess({ types, scopes, allowUnverified = false }: Access if (!user) { if (principal.type === "session") res.clearCookie("token", config.cookie); - res.status(401).json({ error: "Authentication required" }); + sendApiError(res, 401, "AUTHENTICATION_REQUIRED", "Authentication required"); return; } @@ -162,10 +163,7 @@ export function requireAccess({ types, scopes, allowUnverified = false }: Access } if (!emailVerified) { - res.status(403).json({ - error: "Verify your email to continue", - code: "EMAIL_NOT_VERIFIED", - }); + sendApiError(res, 403, "EMAIL_NOT_VERIFIED", "Verify your email to continue"); return; } } @@ -174,7 +172,7 @@ export function requireAccess({ types, scopes, allowUnverified = false }: Access const authorized = scopes.every((scope) => principal.scopes?.includes(scope)); if (!authorized) { - res.status(403).json({ error: "You do not have permission to perform this action" }); + sendApiError(res, 403, "FORBIDDEN", "You do not have permission to perform this action"); return; } } diff --git a/backend/src/middleware/errors.ts b/backend/src/middleware/errors.ts new file mode 100644 index 0000000..ca36bb6 --- /dev/null +++ b/backend/src/middleware/errors.ts @@ -0,0 +1,57 @@ +import type { ErrorRequestHandler, NextFunction, Request, Response } from "express"; +import { config } from "../config"; +import { ApiError, sendApiError } from "../utils/apiError"; +import { logger } from "../utils/logger"; + +export function requestContext(req: Request, res: Response, next: NextFunction) { + req.requestId = crypto.randomUUID(); + res.setHeader("X-Request-Id", req.requestId); + next(); +} + +export function notFoundHandler(req: Request, res: Response) { + sendApiError(res, 404, "RESOURCE_NOT_FOUND", `Route ${req.method} ${req.path} was not found`); +} + +function isBodyParserError(error: unknown): error is Error & { status: number; type?: string } { + return error instanceof Error && "status" in error && typeof error.status === "number"; +} + +export const errorHandler: ErrorRequestHandler = (error, req, res, _next) => { + if (res.headersSent) { + _next(error); + return; + } + + if (error instanceof ApiError) { + sendApiError(res, error.status, error.code, error.message, error.details); + return; + } + + if (isBodyParserError(error) && error.type === "entity.parse.failed") { + sendApiError(res, 400, "MALFORMED_JSON", "Request body contains invalid JSON"); + return; + } + + if (isBodyParserError(error) && error.status === 413) { + sendApiError(res, 413, "PAYLOAD_TOO_LARGE", "Request body is too large"); + return; + } + + const message = error instanceof Error ? error.message : "Unknown error"; + if (message === "Engine response timed out") { + sendApiError(res, 504, "ENGINE_TIMEOUT", "The matching engine did not respond in time"); + return; + } + + logger.error("Unhandled request error", { + requestId: req.requestId, + method: req.method, + path: req.originalUrl, + error: message, + }); + + const publicMessage = + config.app.env === "production" ? "An unexpected error occurred" : message; + sendApiError(res, 500, "INTERNAL_ERROR", publicMessage); +}; diff --git a/backend/src/middleware/rateLimit.ts b/backend/src/middleware/rateLimit.ts index 5aa8e2a..b46e128 100644 --- a/backend/src/middleware/rateLimit.ts +++ b/backend/src/middleware/rateLimit.ts @@ -4,6 +4,7 @@ import { RedisStore } from "rate-limit-redis"; import { config } from "../config"; import { rateLimiterClient } from "../redis"; import { logger } from "../utils/logger"; +import { sendApiError } from "../utils/apiError"; async function sendCommand(...args: string[]): Promise { if (!rateLimiterClient.isOpen) { @@ -34,7 +35,7 @@ function createLimiter(prefix: string, windowMs: number, limit: number) { skip: (req) => !config.rateLimit.enabled || req.principal?.type === "service", handler: (req, res) => { logger.warn("Rate limit exceeded", { key: bucketKey(req), path: req.originalUrl }); - res.status(429).json({ error: "Too many requests, please try again later" }); + sendApiError(res, 429, "RATE_LIMITED", "Too many requests; try again later"); }, }); } diff --git a/backend/src/store/pendingResponses.ts b/backend/src/store/pendingResponses.ts index 07598f1..5f5dbf2 100644 --- a/backend/src/store/pendingResponses.ts +++ b/backend/src/store/pendingResponses.ts @@ -34,3 +34,12 @@ export function resolveEngineResponse(response: EngineResponse): void { pendingResponses.delete(response.correlationId); pending.resolve(response); } + +export function rejectEngineResponse(correlationId: string, error: Error): void { + const pending = pendingResponses.get(correlationId); + if (!pending) return; + + clearTimeout(pending.timeout); + pendingResponses.delete(correlationId); + pending.reject(error); +} diff --git a/backend/src/types/express.d.ts b/backend/src/types/express.d.ts index 8c4196c..2987099 100644 --- a/backend/src/types/express.d.ts +++ b/backend/src/types/express.d.ts @@ -4,6 +4,7 @@ declare global { namespace Express { interface Request { principal?: Principal; + requestId: string; } } } diff --git a/backend/src/utils/apiError.ts b/backend/src/utils/apiError.ts new file mode 100644 index 0000000..08d27e3 --- /dev/null +++ b/backend/src/utils/apiError.ts @@ -0,0 +1,64 @@ +import type { Response } from "express"; + +export interface ApiErrorDetail { + field?: string; + message: string; +} + +export class ApiError extends Error { + constructor( + public readonly status: number, + public readonly code: string, + message: string, + public readonly details?: ApiErrorDetail[], + ) { + super(message); + this.name = "ApiError"; + } +} + +export function sendApiError( + res: Response, + status: number, + code: string, + message: string, + details?: ApiErrorDetail[], +) { + const requestId = res.req?.requestId ?? crypto.randomUUID(); + return res.status(status).json({ + error: { + code, + message, + ...(details?.length ? { details } : {}), + }, + requestId, + }); +} + +export function engineApiError(message = "The matching engine rejected the request") { + if (message.startsWith("Unknown market:")) { + return new ApiError(404, "MARKET_NOT_FOUND", message); + } + if (message.startsWith("Unknown asset:")) { + return new ApiError(404, "ASSET_NOT_FOUND", message); + } + if (message === "Order not found") { + return new ApiError(404, "ORDER_NOT_FOUND", message); + } + if (message === "Filled orders cannot be cancelled") { + return new ApiError(409, "ORDER_NOT_CANCELLABLE", message); + } + if (message === "Insufficient balance" || message.startsWith("Missing balance for asset:")) { + return new ApiError(422, "INSUFFICIENT_BALANCE", message); + } + if (message === "No liquidity") { + return new ApiError(422, "NO_LIQUIDITY", message); + } + + return new ApiError(422, "ORDER_REJECTED", message); +} + +export function sendEngineError(res: Response, message?: string) { + const error = engineApiError(message); + return sendApiError(res, error.status, error.code, error.message, error.details); +} diff --git a/backend/src/utils/engineClient.ts b/backend/src/utils/engineClient.ts index 3a94a10..5bf0836 100644 --- a/backend/src/utils/engineClient.ts +++ b/backend/src/utils/engineClient.ts @@ -1,10 +1,15 @@ import type { EngineCommandType, EngineResponse } from "../types/engine"; -import { resolveEngineResponse, waitForEngineResponse } from "../store/pendingResponses"; +import { + rejectEngineResponse, + resolveEngineResponse, + waitForEngineResponse, +} from "../store/pendingResponses"; import { activeSubscriptions } from "./websocket"; import { config } from "../config"; import { logger } from "./logger"; import { marketSubscriber, publisher, responsesubscriber } from "../redis"; import { formatCandle, formatDepth, formatTrade, formatTicker } from "./formatter"; +import { ApiError } from "./apiError"; export const engineAbortController = new AbortController(); @@ -15,25 +20,36 @@ export async function sendToEngine( const correlationId = crypto.randomUUID(); const responsePromise = waitForEngineResponse(correlationId, config.engine.timeout); - await publisher.xAdd( - config.engine.incomingStream, - "*", - { - correlationId, - responseQueue: config.engine.responseQueue, - type, - payload: JSON.stringify(payload), - }, - { - TRIM: { - strategy: "MINID", - strategyModifier: "=", - threshold: Date.now() - config.redis.retentionMs, + try { + await publisher.xAdd( + config.engine.incomingStream, + "*", + { + correlationId, + responseQueue: config.engine.responseQueue, + type, + payload: JSON.stringify(payload), }, - }, - ); + { + TRIM: { + strategy: "MINID", + strategyModifier: "=", + threshold: Date.now() - config.redis.retentionMs, + }, + }, + ); - return responsePromise; + return await responsePromise; + } catch (error) { + const cause = error instanceof Error ? error : new Error(String(error)); + if (cause.message === "Engine response timed out") { + throw new ApiError(504, "ENGINE_TIMEOUT", "The matching engine did not respond in time"); + } + + rejectEngineResponse(correlationId, cause); + void responsePromise.catch(() => undefined); + throw new ApiError(503, "ENGINE_UNAVAILABLE", "The matching engine is unavailable"); + } } export async function listenForEngineresponses(): Promise { diff --git a/backend/src/utils/formatter.ts b/backend/src/utils/formatter.ts index e0709e0..50792d2 100644 --- a/backend/src/utils/formatter.ts +++ b/backend/src/utils/formatter.ts @@ -120,7 +120,9 @@ export function formatTicker(ticker: Record) { quoteVolume: fmt(ticker.quoteVolume, m.pricePrecision + m.qtyPrecision), priceChange, priceChangePercent, - timestamp: new Date(Number(ticker.timestamp)).toISOString(), + timestamp: new Date( + ticker.timestamp == null ? Date.now() : Number(ticker.timestamp), + ).toISOString(), }; } @@ -221,6 +223,7 @@ export function formatCancel(result: Record) { return { id: result.id as string, symbol: result.symbol, + status: result.status, side: result.side, qty: fmt(result.qty, m.qtyPrecision), filledQty: fmt(result.filledQty, m.qtyPrecision), diff --git a/backend/src/utils/validation.ts b/backend/src/utils/validation.ts index 72407c6..28f19d1 100644 --- a/backend/src/utils/validation.ts +++ b/backend/src/utils/validation.ts @@ -1,12 +1,16 @@ import type { Response } from "express"; import type { ZodError } from "zod"; +import { sendApiError } from "./apiError"; -export async function sendValidationError(res: Response, error: ZodError) { - res.status(400).json({ - error: "validation_error", - issues: error.issues.map((issue) => ({ - path: issue.path.join("."), +export function sendValidationError(res: Response, error: ZodError) { + return sendApiError( + res, + 400, + "VALIDATION_ERROR", + "Request validation failed", + error.issues.map((issue) => ({ + field: issue.path.join(".") || undefined, message: issue.message, })), - }); + ); } diff --git a/backend/tests/access.test.ts b/backend/tests/access.test.ts index 0299700..c6b443f 100644 --- a/backend/tests/access.test.ts +++ b/backend/tests/access.test.ts @@ -45,8 +45,8 @@ test("requireAccess rejects users with unverified email", () => { expect(res.status).toHaveBeenCalledWith(403); expect(res.json).toHaveBeenCalledWith({ - error: "Verify your email to continue", - code: "EMAIL_NOT_VERIFIED", + error: { code: "EMAIL_NOT_VERIFIED", message: "Verify your email to continue" }, + requestId: expect.any(String), }); expect(next).not.toHaveBeenCalled(); }); @@ -89,7 +89,11 @@ test("requireAccess rejects api keys missing a required scope", () => { expect(res.status).toHaveBeenCalledWith(403); expect(res.json).toHaveBeenCalledWith({ - error: "You do not have permission to perform this action", + error: { + code: "FORBIDDEN", + message: "You do not have permission to perform this action", + }, + requestId: expect.any(String), }); expect(next).not.toHaveBeenCalled(); }); diff --git a/backend/tests/auth.test.ts b/backend/tests/auth.test.ts index 2a2c775..ded00d1 100644 --- a/backend/tests/auth.test.ts +++ b/backend/tests/auth.test.ts @@ -83,6 +83,9 @@ test("authenticate returns 401 for malformed api key", async () => { await authenticate(req, res, next); expect(res.status).toHaveBeenCalledWith(401); - expect(res.json).toHaveBeenCalledWith({ error: "Malformed API key" }); + expect(res.json).toHaveBeenCalledWith({ + error: { code: "INVALID_API_KEY", message: "Malformed API key" }, + requestId: expect.any(String), + }); expect(next).not.toHaveBeenCalled(); }); diff --git a/bot/src/client.ts b/bot/src/client.ts index 8a10310..63b36c8 100644 --- a/bot/src/client.ts +++ b/bot/src/client.ts @@ -29,7 +29,11 @@ async function api(path: string, options?: RequestInit): Promise { try { const data = await res.json(); - if (!res.ok) throw new Error((data as any)?.error || `API ${res.status}`); + if (!res.ok) { + const error = (data as any)?.error; + const message = typeof error === "object" ? error?.message : error; + throw new Error(message || `API ${res.status}`); + } return data as T; } catch (err) { log(`API error: ${err}`); diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index c54c46e..eaa0561 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -21,6 +21,9 @@ export class ApiError extends Error { constructor( message: string, public status: number, + public code?: string, + public details?: Array<{ field?: string; message: string }>, + public requestId?: string, ) { super(message); this.name = "ApiError"; @@ -47,13 +50,27 @@ async function request(path: string, options: RequestInit = {}): Promise { } if (!res.ok) { - const message = - typeof data?.error === "string" - ? data.error - : typeof data?.message === "string" - ? data.message - : `Request failed: ${res.status}`; - + const payloadError = data?.error; + if (payloadError && typeof payloadError === "object") { + const structured = payloadError as { + code?: string; + message?: string; + details?: Array<{ field?: string; message: string }>; + }; + throw new ApiError( + structured.message ?? `Request failed: ${res.status}`, + res.status, + structured.code, + structured.details, + typeof data.requestId === "string" ? data.requestId : undefined, + ); + } + + const message = typeof payloadError === "string" + ? payloadError + : typeof data?.message === "string" + ? data.message + : `Request failed: ${res.status}`; throw new ApiError(message, res.status); } diff --git a/tests/integration/api/http.test.ts b/tests/integration/api/http.test.ts index c885aa7..078e933 100644 --- a/tests/integration/api/http.test.ts +++ b/tests/integration/api/http.test.ts @@ -18,26 +18,30 @@ test("root exposes the public API contract", async () => { test("validation failures use the public JSON error contract", async () => { const response = await publicApi.request<{ - error: string; - issues: Array<{ path: string }>; + error: { code: string; message: string; details: Array<{ field: string }> }; + requestId: string; }>("/v1/auth/login", { method: "POST", body: JSON.stringify({ email: "not-an-email", password: "" }), }); expect(response.status).toBe(400); - expect(response.data.error).toBe("validation_error"); - expect(response.data.issues).toEqual( + expect(response.headers.get("x-request-id")).toBe(response.data.requestId); + expect(response.data.error).toMatchObject({ + code: "VALIDATION_ERROR", + message: "Request validation failed", + }); + expect(response.data.error.details).toEqual( expect.arrayContaining([ - expect.objectContaining({ path: "email" }), - expect.objectContaining({ path: "password" }), + expect.objectContaining({ field: "email" }), + expect.objectContaining({ field: "password" }), ]), ); }); test("anonymous users cannot access account or order operations", async () => { - const balance = await publicApi.request<{ error: string }>("/v1/balances"); - const order = await publicApi.request<{ error: string }>("/v1/orders", { + const balance = await publicApi.request>("/v1/balances"); + const order = await publicApi.request>("/v1/orders", { method: "POST", body: JSON.stringify({ type: "LIMIT", @@ -51,28 +55,54 @@ test("anonymous users cannot access account or order operations", async () => { for (const response of [balance, order]) { expect(response).toMatchObject({ status: 401, - data: { error: "Authentication required" }, + data: { + error: { code: "AUTHENTICATION_REQUIRED", message: "Authentication required" }, + requestId: expect.any(String), + }, }); } }); test("malformed API keys are rejected before protected handlers run", async () => { - const response = await publicApi.request<{ error: string }>("/v1/balances", { + const response = await publicApi.request>("/v1/balances", { headers: { "x-api-key": "not-a-paperdrill-key" }, }); - expect(response).toMatchObject({ status: 401, data: { error: "Malformed API key" } }); + expect(response).toMatchObject({ + status: 401, + data: { error: { code: "INVALID_API_KEY", message: "Malformed API key" } }, + }); }); test("invalid session cookies are cleared and cannot authorize a request", async () => { - const response = await publicApi.request<{ error: string }>("/v1/balances", { + const response = await publicApi.request>("/v1/balances", { headers: { cookie: "token=invalid-token" }, }); - expect(response).toMatchObject({ status: 401, data: { error: "Authentication required" } }); + expect(response).toMatchObject({ + status: 401, + data: { error: { code: "AUTHENTICATION_REQUIRED", message: "Authentication required" } }, + }); expect(response.headers.get("set-cookie")).toContain("token="); }); +test("malformed JSON and unknown routes return JSON errors", async () => { + const malformed = await publicApi.request>("/v1/auth/login", { + method: "POST", + body: "{not-json", + }); + const missing = await publicApi.request>("/v1/does-not-exist"); + + expect(malformed).toMatchObject({ + status: 400, + data: { error: { code: "MALFORMED_JSON", message: "Request body contains invalid JSON" } }, + }); + expect(missing).toMatchObject({ + status: 404, + data: { error: { code: "RESOURCE_NOT_FOUND" }, requestId: expect.any(String) }, + }); +}); + test("browser clients receive credentialed CORS preflight headers", async () => { const response = await fetch(`${process.env.API_BASE_URL}/v1/auth/login`, { method: "OPTIONS", diff --git a/tests/integration/api/orders.test.ts b/tests/integration/api/orders.test.ts index 501850c..fe44366 100644 --- a/tests/integration/api/orders.test.ts +++ b/tests/integration/api/orders.test.ts @@ -38,6 +38,9 @@ test("an account without base inventory cannot place a sell order", async () => qty: "0.1000", }); - expect(response).toMatchObject({ status: 400, data: { error: "Insufficient balance" } }); + expect(response).toMatchObject({ + status: 422, + data: { error: { code: "INSUFFICIENT_BALANCE", message: "Insufficient balance" } }, + }); expect(await client.getOpenOrders()).toMatchObject({ status: 200, data: [] }); }); diff --git a/tests/integration/flows/trading.test.ts b/tests/integration/flows/trading.test.ts index 3c9bec4..85c4893 100644 --- a/tests/integration/flows/trading.test.ts +++ b/tests/integration/flows/trading.test.ts @@ -45,9 +45,9 @@ test("a complete trade settles live balances and persists user-visible history", qty: "0.5000", }); - expect(sell).toMatchObject({ status: 200, data: { status: "OPEN", filledQty: "0.0000" } }); + expect(sell).toMatchObject({ status: 201, data: { status: "OPEN", filledQty: "0.0000" } }); expect(buy).toMatchObject({ - status: 200, + status: 201, data: { status: "FILLED", filledQty: "0.5000", averagePrice: "100.00" }, }); expect(await buyer.client.getBalances()).toMatchObject({ @@ -120,8 +120,13 @@ test("a complete trade settles live balances and persists user-visible history", }); expect(await buyer.client.cancelOrder(buy.data.id)).toMatchObject({ - status: 400, - data: { error: "Filled orders cannot be cancelled" }, + status: 409, + data: { + error: { + code: "ORDER_NOT_CANCELLABLE", + message: "Filled orders cannot be cancelled", + }, + }, }); }); @@ -145,7 +150,7 @@ test("a partial fill propagates through balances, open orders, fills, and persis qty: "4.00", }); - expect(buy).toMatchObject({ status: 200, data: { status: "FILLED", filledQty: "4.00" } }); + expect(buy).toMatchObject({ status: 201, data: { status: "FILLED", filledQty: "4.00" } }); expect(await seller.client.getOpenOrders()).toMatchObject({ status: 200, data: [ @@ -198,7 +203,7 @@ test("cancelling an open order releases funds and persists ownership-safe state" qty: "1.000", }); - expect(created).toMatchObject({ status: 200, data: { status: "OPEN" } }); + expect(created).toMatchObject({ status: 201, data: { status: "OPEN" } }); expect(await owner.client.getBalances()).toMatchObject({ data: { USD: { available: "9900.00", locked: "100.00" } }, }); @@ -206,11 +211,11 @@ test("cancelling an open order releases funds and persists ownership-safe state" await waitForOrder(created.data.id, "OPEN"); expect(await other.client.getOrder(created.data.id)).toMatchObject({ status: 404, - data: { error: "Order not found" }, + data: { error: { code: "ORDER_NOT_FOUND", message: "Order not found" } }, }); expect(await other.client.cancelOrder(created.data.id)).toMatchObject({ - status: 400, - data: { error: "Order not found" }, + status: 404, + data: { error: { code: "ORDER_NOT_FOUND", message: "Order not found" } }, }); expect(await owner.client.cancelOrder(created.data.id)).toMatchObject({ diff --git a/tests/integration/helpers/api.ts b/tests/integration/helpers/api.ts index 1a7af1c..079cf90 100644 --- a/tests/integration/helpers/api.ts +++ b/tests/integration/helpers/api.ts @@ -7,6 +7,15 @@ export interface ApiResponse { headers: Headers; } +export interface ErrorResponse { + error: { + code: string; + message: string; + details?: Array<{ field?: string; message: string }>; + }; + requestId: string; +} + export interface OrderInput { type: "LIMIT" | "MARKET"; side: "BUY" | "SELL"; @@ -79,7 +88,7 @@ export class ApiClient { } getOrder(id: string) { - return this.request(`/v1/orders/${id}`); + return this.request(`/v1/orders/${id}`); } getTrades() { diff --git a/tests/integration/redis/contracts.test.ts b/tests/integration/redis/contracts.test.ts index 7a48ee4..46443c6 100644 --- a/tests/integration/redis/contracts.test.ts +++ b/tests/integration/redis/contracts.test.ts @@ -22,7 +22,7 @@ test("backend writes complete place and cancel commands to the Redis broker", as qty: "0.2500", }); - expect(created).toMatchObject({ status: 200, data: { status: "OPEN" } }); + expect(created).toMatchObject({ status: 201, data: { status: "OPEN" } }); const placeCommand = await engine.findBrokerCommand( "create_order", From de1bebe7e1149aafb86d741b37f2e928afabe1f7 Mon Sep 17 00:00:00 2001 From: Harshil Date: Tue, 15 Sep 2026 20:50:23 +0530 Subject: [PATCH 03/20] feat: implement health check for engine and worker --- backend/src/app.ts | 6 +++--- compose.prod.yml | 28 ++++++++++++++++++++++++++++ compose.test.yml | 26 ++++++++++++++++++++++++++ engine/src/health.ts | 37 +++++++++++++++++++++++++++++++++++++ engine/src/index.ts | 3 +++ worker/src/health.ts | 37 +++++++++++++++++++++++++++++++++++++ worker/src/index.ts | 3 +++ 7 files changed, 137 insertions(+), 3 deletions(-) create mode 100644 engine/src/health.ts create mode 100644 worker/src/health.ts diff --git a/backend/src/app.ts b/backend/src/app.ts index a1388d1..55e7246 100644 --- a/backend/src/app.ts +++ b/backend/src/app.ts @@ -19,9 +19,6 @@ app.use(cors(config.cors)); app.use(express.json()); app.use(express.urlencoded({ extended: true })); -app.use(authenticate); -app.use(apiLimiter); - app.get("/", (_req: Request, res: Response) => { res.status(200).json({ message: "Welcome to PaperDrill", @@ -42,6 +39,9 @@ app.get("/health", async (_req: Request, res: Response) => { } }); +app.use(authenticate); +app.use(apiLimiter); + app.use("/v1", appRouter); app.use(notFoundHandler); app.use(errorHandler); diff --git a/compose.prod.yml b/compose.prod.yml index a240c1e..1859db6 100644 --- a/compose.prod.yml +++ b/compose.prod.yml @@ -27,6 +27,10 @@ services: depends_on: redis: condition: service_healthy + engine: + condition: service_healthy + worker: + condition: service_healthy networks: - paperdrill @@ -36,6 +40,18 @@ services: target: runner restart: unless-stopped env_file: ./.env.prod + healthcheck: + test: + [ + "CMD", + "bun", + "-e", + "const value = Number(await Bun.file('/tmp/health').text()); const age = Date.now() - value; if (!Number.isFinite(age) || age < 0 || age > 30000) process.exit(1)", + ] + interval: 15s + timeout: 5s + retries: 3 + start_period: 20s logging: driver: json-file options: @@ -55,6 +71,18 @@ services: target: runner restart: unless-stopped env_file: ./.env.prod + healthcheck: + test: + [ + "CMD", + "bun", + "-e", + "const value = Number(await Bun.file('/tmp/health').text()); const age = Date.now() - value; if (!Number.isFinite(age) || age < 0 || age > 30000) process.exit(1)", + ] + interval: 15s + timeout: 5s + retries: 3 + start_period: 20s logging: driver: json-file options: diff --git a/compose.test.yml b/compose.test.yml index 91800d0..d093c10 100644 --- a/compose.test.yml +++ b/compose.test.yml @@ -49,6 +49,10 @@ services: condition: service_healthy setup: condition: service_completed_successfully + engine: + condition: service_healthy + worker: + condition: service_healthy healthcheck: test: [ @@ -71,6 +75,17 @@ services: condition: service_healthy setup: condition: service_completed_successfully + healthcheck: + test: + [ + "CMD", + "bun", + "-e", + "const value = Number(await Bun.file('/tmp/health').text()); const age = Date.now() - value; if (!Number.isFinite(age) || age < 0 || age > 30000) process.exit(1)", + ] + interval: 2s + timeout: 5s + retries: 20 tmpfs: - /usr/src/app/snapshots @@ -84,3 +99,14 @@ services: condition: service_healthy setup: condition: service_completed_successfully + healthcheck: + test: + [ + "CMD", + "bun", + "-e", + "const value = Number(await Bun.file('/tmp/health').text()); const age = Date.now() - value; if (!Number.isFinite(age) || age < 0 || age > 30000) process.exit(1)", + ] + interval: 2s + timeout: 5s + retries: 20 diff --git a/engine/src/health.ts b/engine/src/health.ts new file mode 100644 index 0000000..2f5228d --- /dev/null +++ b/engine/src/health.ts @@ -0,0 +1,37 @@ +import { unlink } from "node:fs/promises"; +import { pool } from "./db"; +import { cacheClient } from "./redis/client"; +import { logger } from "./util/logger"; + +const HEALTH_FILE = "/tmp/health"; +const HEALTH_INTERVAL_MS = 10_000; + +let healthInterval: ReturnType | undefined; +let healthCheck: Promise | undefined; + +function refreshHealth() { + if (healthCheck) return healthCheck; + + healthCheck = (async () => { + await Promise.all([pool.query("SELECT 1"), cacheClient.ping()]); + await Bun.write(HEALTH_FILE, Date.now().toString()); + })().finally(() => { + healthCheck = undefined; + }); + + return healthCheck; +} + +export async function startHealth() { + await refreshHealth(); + healthInterval = setInterval(() => { + void refreshHealth().catch((error) => logger.error("Engine health check failed", error)); + }, HEALTH_INTERVAL_MS); +} + +export async function stopHealth() { + if (healthInterval) clearInterval(healthInterval); + healthInterval = undefined; + await healthCheck?.catch(() => undefined); + await unlink(HEALTH_FILE).catch(() => undefined); +} diff --git a/engine/src/index.ts b/engine/src/index.ts index 0bd3322..16c8d1d 100644 --- a/engine/src/index.ts +++ b/engine/src/index.ts @@ -14,6 +14,7 @@ import { connectDB, disconnectDB } from "./db"; import { bigintReplacer } from "./util"; import { initMarkets } from "./modules/market"; import { registerEventHandlers } from "./core/events"; +import { startHealth, stopHealth } from "./health"; const abortController = new AbortController(); @@ -22,6 +23,7 @@ await connectDB(); await initMarkets(); await loadSnapshot(); registerEventHandlers(); +await startHealth(); async function sendResponse(responseQueue: string, response: EngineResponse) { await streamProducer.lPush(responseQueue, JSON.stringify(response, bigintReplacer)); @@ -148,6 +150,7 @@ async function gracefulShutdown(signal: string) { }, 10000); abortController.abort(); + await stopHealth(); await disconnectRedis(); await disconnectDB(); diff --git a/worker/src/health.ts b/worker/src/health.ts new file mode 100644 index 0000000..b537d50 --- /dev/null +++ b/worker/src/health.ts @@ -0,0 +1,37 @@ +import { unlink } from "node:fs/promises"; +import { pool } from "./db"; +import { cacheClient } from "./redis"; +import { logger } from "./logger"; + +const HEALTH_FILE = "/tmp/health"; +const HEALTH_INTERVAL_MS = 10_000; + +let healthInterval: ReturnType | undefined; +let healthCheck: Promise | undefined; + +function refreshHealth() { + if (healthCheck) return healthCheck; + + healthCheck = (async () => { + await Promise.all([pool.query("SELECT 1"), cacheClient.ping()]); + await Bun.write(HEALTH_FILE, Date.now().toString()); + })().finally(() => { + healthCheck = undefined; + }); + + return healthCheck; +} + +export async function startHealth() { + await refreshHealth(); + healthInterval = setInterval(() => { + void refreshHealth().catch((error) => logger.error("Worker health check failed", error)); + }, HEALTH_INTERVAL_MS); +} + +export async function stopHealth() { + if (healthInterval) clearInterval(healthInterval); + healthInterval = undefined; + await healthCheck?.catch(() => undefined); + await unlink(HEALTH_FILE).catch(() => undefined); +} diff --git a/worker/src/index.ts b/worker/src/index.ts index cea1597..5de0d59 100644 --- a/worker/src/index.ts +++ b/worker/src/index.ts @@ -7,6 +7,7 @@ import { flushCandles } from "./candle"; import { warmUpTickers } from "./ticker"; import { flushBatch, queueFill, queueOrder, loadServiceUserIds } from "./persistence"; import type { StreamFill, StreamOrder } from "./types"; +import { startHealth, stopHealth } from "./health"; const abortController = new AbortController(); @@ -26,6 +27,7 @@ await loadServiceUserIds().catch((err) => { }); await warmUpTickers(); +await startHealth(); logger.info("Worker started, listening for stream events"); @@ -155,6 +157,7 @@ async function gracefulShutdown(signal: string) { abortController.abort(); clearTimeout(candleFlushInterval); clearInterval(batchInterval); + await stopHealth(); try { await flushCandles(); From b838f7f38624f6e93af87c1294a18c6925c324f9 Mon Sep 17 00:00:00 2001 From: Harshil Date: Wed, 16 Sep 2026 18:08:49 +0530 Subject: [PATCH 04/20] feat: integrate Sentry for error tracking in backend and engine --- .env.example | 6 ++- backend/bun.lock | 63 +++++++++++++++++++++++++++++ backend/package.json | 1 + backend/src/config/index.ts | 9 +++++ backend/src/index.ts | 7 +++- backend/src/instrument.ts | 30 ++++++++++++++ backend/src/middleware/errors.ts | 5 +++ backend/src/utils/logger.ts | 8 ++++ engine/bun.lock | 69 ++++++++++++++++++++++++++++++++ engine/package.json | 1 + engine/src/config/index.ts | 7 ++++ engine/src/index.ts | 35 ++++++++++------ engine/src/instrument.ts | 30 ++++++++++++++ engine/src/redis/client.ts | 10 ++--- engine/src/util/logger.ts | 10 ++++- 15 files changed, 271 insertions(+), 20 deletions(-) create mode 100644 backend/src/instrument.ts create mode 100644 engine/src/instrument.ts diff --git a/.env.example b/.env.example index cb72e60..94f262d 100644 --- a/.env.example +++ b/.env.example @@ -33,9 +33,13 @@ RATE_LIMIT_ORDER_WINDOW_MS=60000 LOG_LEVEL=debug +# Optional backend error reporting +SENTRY_DSN= +SENTRY_ENVIRONMENT=development + # Bot credentials BASE_URL=http://localhost:8000 SERVICE_TOKEN=your-service-token-here SERVICE_EMAIL=your-service-email-here MARKET=SOL_USD -MAX_ORDER_QTY=10 \ No newline at end of file +MAX_ORDER_QTY=10 diff --git a/backend/bun.lock b/backend/bun.lock index 6abdb9b..ae4145c 100644 --- a/backend/bun.lock +++ b/backend/bun.lock @@ -7,6 +7,7 @@ "dependencies": { "@prisma/adapter-pg": "^7.8.0", "@prisma/client": "^7.8.0", + "@sentry/bun": "^10.74.0", "bcrypt": "^6.0.0", "cookie": "^1.1.1", "cors": "^2.8.6", @@ -37,6 +38,10 @@ }, }, "packages": { + "@apm-js-collab/code-transformer": ["@apm-js-collab/code-transformer@0.18.1", "", { "dependencies": { "@types/estree": "^1.0.8", "astring": "^1.9.0", "esquery": "^1.7.0", "meriyah": "^6.1.4", "semifies": "^1.0.0", "source-map": "^0.6.0" }, "bin": { "code-transformer": "cli.js" } }, "sha512-u1Hb6bHjWtkSpiprwVP6YaHC1DTN4RAU3zYkUDUe7WMnJwdyU1pwTL9dFKiSJB9IiLue/EQovmyx6xhU7FFtAQ=="], + + "@apm-js-collab/code-transformer-bundler-plugins": ["@apm-js-collab/code-transformer-bundler-plugins@0.7.4", "", { "dependencies": { "@apm-js-collab/code-transformer": "^0.18.1", "es-module-lexer": "^2.1.0", "magic-string": "^0.30.21", "module-details-from-path": "^1.0.4" } }, "sha512-nAfOeZPSUAQvJa1iFT/5oCrTm5YQhMMrfCNthNnaXHZiOQhu1KGuLoIx7HtbAi3wfwaBYLaICPIeenIaEwcXIg=="], + "@electric-sql/pglite": ["@electric-sql/pglite@0.4.1", "", {}, "sha512-mZ9NzzUSYPOCnxHH1oAHPRzoMFJHY472raDKwXl/+6oPbpdJ7g8LsCN4FSaIIfkiCKHhb3iF/Zqo3NYxaIhU7Q=="], "@electric-sql/pglite-socket": ["@electric-sql/pglite-socket@0.1.1", "", { "peerDependencies": { "@electric-sql/pglite": "0.4.1" }, "bin": { "pglite-server": "dist/scripts/server.js" } }, "sha512-p2hoXw3Z3LQHwTeikdZNsFBOvXGqKY2hk51BBw+8NKND8eoH+8LFOtW9Z8CQKmTJ2qqGYu82ipqiyFZOTTXNfw=="], @@ -45,10 +50,26 @@ "@hono/node-server": ["@hono/node-server@1.19.11", "", { "peerDependencies": { "hono": "^4" } }, "sha512-dr8/3zEaB+p0D2n/IUrlPF1HZm586qgJNXK1a9fhg/PzdtkK7Ksd5l312tJX2yBuALqDYBlG20QEbayqPyxn+g=="], + "@jridgewell/sourcemap-codec": ["@jridgewell/sourcemap-codec@1.6.0", "", {}, "sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw=="], + "@kurkle/color": ["@kurkle/color@0.3.4", "", {}, "sha512-M5UknZPHRu3DEDWoipU6sE8PdkZ6Z/S+v4dD+Ke8IaNlpdSQah50lz1KtcFBa2vsdOnwbbnxJwVM4wty6udA5w=="], "@opentelemetry/api": ["@opentelemetry/api@1.9.1", "", {}, "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q=="], + "@opentelemetry/api-logs": ["@opentelemetry/api-logs@0.220.0", "", { "dependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-CmVa4ImJ+ynfrPMNaAXHET6Bhb44SwzmfyVJFq9ni2jgXJR/l7C6gfVFddNmHP+ZOkP9cf4f9DBe68qVLTHc9w=="], + + "@opentelemetry/core": ["@opentelemetry/core@2.11.0", "", { "dependencies": { "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-7YP44XH0tV6+Mb54x2YGf84i7yi+31MBZlE8JwvozkxyTvXbSp10X7cI7YE49ChJ3shMJoBmCJF3+1QFBJctGA=="], + + "@opentelemetry/instrumentation": ["@opentelemetry/instrumentation@0.220.0", "", { "dependencies": { "@opentelemetry/api-logs": "0.220.0", "import-in-the-middle": "^3.0.0", "require-in-the-middle": "^8.0.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-xQx3E2WxP1mDvKzxLxX+CTCtNLa560YJZ3087qYHerl2YmiKpv7AH+dAy7vmx+eVrZ5BwhfWUAVoKOoxCNHcpw=="], + + "@opentelemetry/resources": ["@opentelemetry/resources@2.11.0", "", { "dependencies": { "@opentelemetry/core": "2.11.0", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-Ie7+8q8MDF4FAEQCKVMTx3ReUvxiIAgIiiW3c9JdmP8+HMcDy20puT+AHjexnExgnbvBxjQ9fjkFDWrikJ2jQA=="], + + "@opentelemetry/sdk-trace": ["@opentelemetry/sdk-trace@2.11.0", "", { "dependencies": { "@opentelemetry/core": "2.11.0", "@opentelemetry/resources": "2.11.0", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-fFnTqGm8/G73GQVnxYi7LXa1ZVYEUvgL6XI1LpvV0bPC7WQ/ZGgKxCSl8FnlZBKto9JHHEFTO6s6CUpvvtwFrA=="], + + "@opentelemetry/sdk-trace-base": ["@opentelemetry/sdk-trace-base@2.11.0", "", { "dependencies": { "@opentelemetry/core": "2.11.0", "@opentelemetry/resources": "2.11.0", "@opentelemetry/sdk-trace": "2.11.0", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-H19x/TX/LZdqiYOjM7fqtSxwlplC5pgelavqbQdHbhdq0q/AI/TGkM2dfGuuynTXmJPeF2HoZVoPDu+TGoW78A=="], + + "@opentelemetry/semantic-conventions": ["@opentelemetry/semantic-conventions@1.43.0", "", {}, "sha512-eSYWTm620tTk45EKSedaUL8MFYI8hW164hIXsgIHyxu3VobUB3fFCu5t0hQby6OoWRPsG1KkKUG2M5UadiLiVg=="], + "@prisma/adapter-pg": ["@prisma/adapter-pg@7.8.0", "", { "dependencies": { "@prisma/driver-adapter-utils": "7.8.0", "@types/pg": "^8.16.0", "pg": "^8.16.3", "postgres-array": "3.0.4" } }, "sha512-ygb3UkerK3v8MDpXVgCISdRNDozpxh6+JVJgiIGbSr5KBgz10LLf5ejUskPGoXlsIjxsOu6nuy1JVQr2EKGSlg=="], "@prisma/client": ["@prisma/client@7.8.0", "", { "dependencies": { "@prisma/client-runtime-utils": "7.8.0" }, "peerDependencies": { "prisma": "*", "typescript": ">=5.4.0" }, "optionalPeers": ["prisma", "typescript"] }, "sha512-HFp3Dawv/3sU3JtlPha90IB+48lS7zHiH4LKZPjmcE8YH5P9DOXGPvo8dqOtO7MqLDd1p2hOWMcFlRT1DMblHw=="], @@ -103,6 +124,20 @@ "@redis/time-series": ["@redis/time-series@6.0.0", "", { "peerDependencies": { "@redis/client": "^6.0.0" } }, "sha512-QWhkYsg+3lhBrBf+cbzybtV8LQcSrk7iXIgTaGU+pHNFTkql7TpVRE24ROS6M2ybVIV6O/zxTqfxgxxYiqyw0Q=="], + "@sentry/bun": ["@sentry/bun@10.74.0", "", { "dependencies": { "@apm-js-collab/code-transformer-bundler-plugins": "^0.7.3", "@sentry/conventions": "^0.16.0", "@sentry/core": "10.74.0", "@sentry/node": "10.74.0", "@sentry/server-utils": "10.74.0" } }, "sha512-KDX+sBp0ISatJ31r+GN4Uk1FUogDM+e3xqTTD1sTATpmYH0PEmggqZC/RaG+VM2DjDxf7jxjdeNuLuC1xKhV3A=="], + + "@sentry/conventions": ["@sentry/conventions@0.16.0", "", {}, "sha512-fO9PLmHdVURcSPUpWCItWAtgKiMwGdJHbovoSEyLplX5sxs2ugvI4CBPTrkkgqhObnZOD0CnWBKDzSVQYBKEyQ=="], + + "@sentry/core": ["@sentry/core@10.74.0", "", { "dependencies": { "@sentry/conventions": "^0.16.0" } }, "sha512-u9rY8vcZfktccwm6LznfCZlqP5C9A+p76r4/pFS1grqpuTO0m21Cl8rosnlESrDGP/Xd9tfr91rWYk0jPH8jeQ=="], + + "@sentry/node": ["@sentry/node@10.74.0", "", { "dependencies": { "@opentelemetry/api": "^1.9.1", "@opentelemetry/instrumentation": "^0.220.0", "@opentelemetry/sdk-trace-base": "^2.9.0", "@sentry/conventions": "^0.16.0", "@sentry/core": "10.74.0", "@sentry/node-core": "10.74.0", "@sentry/opentelemetry": "10.74.0", "@sentry/server-utils": "10.74.0", "import-in-the-middle": "^3.0.0" } }, "sha512-u1wsarTOWHn9CCev81Da5T4IQHZgdcosXRfX2+4DMII/lVJMBYesKixTQuMwXexjVG2+pkf16zTmgjZDa+75jA=="], + + "@sentry/node-core": ["@sentry/node-core@10.74.0", "", { "dependencies": { "@sentry/conventions": "^0.16.0", "@sentry/core": "10.74.0", "@sentry/opentelemetry": "10.74.0", "import-in-the-middle": "^3.0.0" }, "peerDependencies": { "@opentelemetry/api": "^1.9.0", "@opentelemetry/core": "^1.30.1 || ^2.1.0", "@opentelemetry/exporter-trace-otlp-http": ">=0.57.0 <1", "@opentelemetry/instrumentation": ">=0.57.1 <1", "@opentelemetry/sdk-trace-base": "^1.30.1 || ^2.1.0" }, "optionalPeers": ["@opentelemetry/api", "@opentelemetry/core", "@opentelemetry/exporter-trace-otlp-http", "@opentelemetry/instrumentation", "@opentelemetry/sdk-trace-base"] }, "sha512-btgZXcGmkOGgojbxHo/gfGiyqYpzC9E8zVR78Q3MtM6Xnemk9gwJiQlhNmEX/FM+C36WBRPZrdcZcnMaJhbfJw=="], + + "@sentry/opentelemetry": ["@sentry/opentelemetry@10.74.0", "", { "dependencies": { "@sentry/conventions": "^0.16.0", "@sentry/core": "10.74.0" }, "peerDependencies": { "@opentelemetry/api": "^1.9.0", "@opentelemetry/core": "^1.30.1 || ^2.1.0", "@opentelemetry/sdk-trace-base": "^1.30.1 || ^2.1.0" } }, "sha512-ua5mt0NDBfye+/ACKjAw9Ad2i+y42lOLvhxXZepZXjszblMz80MEsIZflMB7uLZUCTNH7MbZN8tlzCy8KsJKmQ=="], + + "@sentry/server-utils": ["@sentry/server-utils@10.74.0", "", { "dependencies": { "@sentry/conventions": "^0.16.0", "@sentry/core": "10.74.0" } }, "sha512-AHmPIGE8yVRyywpZRhAkm/H0cHTgsQSPhFZFbaeQLcYHFE/eOP9bAMh8Nj/Apr7WMagFXYHd04tR/63BxtLgKw=="], + "@stablelib/base64": ["@stablelib/base64@1.0.1", "", {}, "sha512-1bnPQqSxSuc3Ii6MhBysoWCg58j97aUjuCSZrGSmDxNqtytIi0k8utUenAwTZN4V5mXXYGsVUI9zeBqy+jBOSQ=="], "@standard-schema/spec": ["@standard-schema/spec@1.1.0", "", {}, "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w=="], @@ -117,6 +152,8 @@ "@types/cors": ["@types/cors@2.8.19", "", { "dependencies": { "@types/node": "*" } }, "sha512-mFNylyeyqN93lfe/9CSxOGREz8cpzAhH+E93xJ4xWQf62V8sQ/24reV2nyzUWM6H6Xji+GGHpkbLe7pVoUEskg=="], + "@types/estree": ["@types/estree@1.0.9", "", {}, "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg=="], + "@types/express": ["@types/express@5.0.6", "", { "dependencies": { "@types/body-parser": "*", "@types/express-serve-static-core": "^5.0.0", "@types/serve-static": "^2" } }, "sha512-sKYVuV7Sv9fbPIt/442koC7+IIwK5olP1KWeD88e/idgoJqDm3JV/YUiPwkoKK92ylff2MGxSz1CSjsXelx0YA=="], "@types/express-serve-static-core": ["@types/express-serve-static-core@5.1.1", "", { "dependencies": { "@types/node": "*", "@types/qs": "*", "@types/range-parser": "*", "@types/send": "*" } }, "sha512-v4zIMr/cX7/d2BpAEX3KNKL/JrT1s43s96lLvvdTmza1oEvDudCqK9aF/djc/SWgy8Yh0h30TZx5VpzqFCxk5A=="], @@ -147,6 +184,8 @@ "ajv": ["ajv@8.20.0", "", { "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", "json-schema-traverse": "^1.0.0", "require-from-string": "^2.0.2" } }, "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA=="], + "astring": ["astring@1.9.0", "", { "bin": { "astring": "bin/astring" } }, "sha512-LElXdjswlqjWrPpJFg1Fx4wpkOCxj1TDHlSV4PlaRxHGWko024xICaa97ZkMfs6DRKlCguiAI+rbXv5GWwXIkg=="], + "aws-ssl-profiles": ["aws-ssl-profiles@1.1.2", "", {}, "sha512-NZKeq9AfyQvEeNlN0zSYAaWrmBffJh3IELMZfRpJVWgrpEbtEpnjvzqBPf+mxoI287JohRDoa+/nsfqqiZmF6g=="], "bcrypt": ["bcrypt@6.0.0", "", { "dependencies": { "node-addon-api": "^8.3.0", "node-gyp-build": "^4.8.4" } }, "sha512-cU8v/EGSrnH+HnxV2z0J7/blxH8gq7Xh2JFT6Aroax7UohdmiJJlxApMxtKfuI7z68NvvVcmR78k2LbT6efhRg=="], @@ -171,6 +210,8 @@ "chokidar": ["chokidar@5.0.0", "", { "dependencies": { "readdirp": "^5.0.0" } }, "sha512-TQMmc3w+5AxjpL8iIiwebF73dRDF4fBIieAqGn9RGCWaEVwQ6Fb2cGe31Yns0RRIzii5goJ1Y7xbMwo1TxMplw=="], + "cjs-module-lexer": ["cjs-module-lexer@2.2.1", "", {}, "sha512-Ca8swihM+/4yKecYHY52kgJd300hi2lADU/a1RxNTRe+RJ9jvqQlESpbz9DnG9mowez8qwXHB8qYdIUw9e+F5Q=="], + "cluster-key-slot": ["cluster-key-slot@1.1.2", "", {}, "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA=="], "confbox": ["confbox@0.2.4", "", {}, "sha512-ysOGlgTFbN2/Y6Cg3Iye8YKulHw+R2fNXHrgSmXISQdMnomY6eNDprVdW9R5xBguEqI954+S6709UyiO7B+6OQ=="], @@ -221,10 +262,16 @@ "es-errors": ["es-errors@1.3.0", "", {}, "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw=="], + "es-module-lexer": ["es-module-lexer@2.3.2", "", {}, "sha512-poHGpORABojJJucnV9KbOavETW8lBVnphkW77ER5/BQ5Fz7oXSoCNek7IH3vR5nRjdsEz926ibFYX8KtLQmdyw=="], + "es-object-atoms": ["es-object-atoms@1.1.1", "", { "dependencies": { "es-errors": "^1.3.0" } }, "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA=="], "escape-html": ["escape-html@1.0.3", "", {}, "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow=="], + "esquery": ["esquery@1.7.0", "", { "dependencies": { "estraverse": "^5.1.0" } }, "sha512-Ap6G0WQwcU/LHsvLwON1fAQX9Zp0A2Y6Y/cJBl9r/JbW90Zyg4/zbG6zzKa2OTALELarYHmKu0GhpM5EO+7T0g=="], + + "estraverse": ["estraverse@5.3.0", "", {}, "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA=="], + "etag": ["etag@1.8.1", "", {}, "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg=="], "express": ["express@5.2.1", "", { "dependencies": { "accepts": "^2.0.0", "body-parser": "^2.2.1", "content-disposition": "^1.0.0", "content-type": "^1.0.5", "cookie": "^0.7.1", "cookie-signature": "^1.2.1", "debug": "^4.4.0", "depd": "^2.0.0", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "etag": "^1.8.1", "finalhandler": "^2.1.0", "fresh": "^2.0.0", "http-errors": "^2.0.0", "merge-descriptors": "^2.0.0", "mime-types": "^3.0.0", "on-finished": "^2.4.1", "once": "^1.4.0", "parseurl": "^1.3.3", "proxy-addr": "^2.0.7", "qs": "^6.14.0", "range-parser": "^1.2.1", "router": "^2.2.0", "send": "^1.1.0", "serve-static": "^2.2.0", "statuses": "^2.0.1", "type-is": "^2.0.1", "vary": "^1.1.2" } }, "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw=="], @@ -281,6 +328,8 @@ "iconv-lite": ["iconv-lite@0.7.2", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw=="], + "import-in-the-middle": ["import-in-the-middle@3.5.1", "", { "dependencies": { "cjs-module-lexer": "^2.2.0", "es-module-lexer": "^3.0.2", "module-details-from-path": "^1.0.4" } }, "sha512-mPKuL8bPQzecui2KK6Gb+M8JvJoHnhS1FeYGa22QopBmlevF5F0FE6ued/B5EgHDeIoMTONIpDWWFKUPOG0DBQ=="], + "inherits": ["inherits@2.0.4", "", {}, "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="], "ip-address": ["ip-address@10.4.0", "", {}, "sha512-oSK96Grm3aP6OrS263xVxbNDGVL7rzBtYdpGqlDG8iQdoenDoTs/nkki+DflYbAEE8Xl6o5YxhxlrKvI3nqKXQ=="], @@ -321,16 +370,22 @@ "lru.min": ["lru.min@1.1.4", "", {}, "sha512-DqC6n3QQ77zdFpCMASA1a3Jlb64Hv2N2DciFGkO/4L9+q/IpIAuRlKOvCXabtRW6cQf8usbmM6BE/TOPysCdIA=="], + "magic-string": ["magic-string@0.30.21", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.5" } }, "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ=="], + "math-intrinsics": ["math-intrinsics@1.1.0", "", {}, "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g=="], "media-typer": ["media-typer@1.1.0", "", {}, "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw=="], "merge-descriptors": ["merge-descriptors@2.0.0", "", {}, "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g=="], + "meriyah": ["meriyah@6.1.4", "", {}, "sha512-Sz8FzjzI0kN13GK/6MVEsVzMZEPvOhnmmI1lU5+/1cGOiK3QUahntrNNtdVeihrO7t9JpoH75iMNXg6R6uWflQ=="], + "mime-db": ["mime-db@1.54.0", "", {}, "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ=="], "mime-types": ["mime-types@3.0.2", "", { "dependencies": { "mime-db": "^1.54.0" } }, "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A=="], + "module-details-from-path": ["module-details-from-path@1.0.4", "", {}, "sha512-EGWKgxALGMgzvxYF1UyGTy0HXX/2vHLkw6+NvDKW2jypWbHpjQuj4UMcqQWXHERJhVGKikolT06G3bcKe4fi7w=="], + "ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], "mysql2": ["mysql2@3.15.3", "", { "dependencies": { "aws-ssl-profiles": "^1.1.1", "denque": "^2.1.0", "generate-function": "^2.3.1", "iconv-lite": "^0.7.0", "long": "^5.2.1", "lru.min": "^1.0.0", "named-placeholders": "^1.1.3", "seq-queue": "^0.0.5", "sqlstring": "^2.3.2" } }, "sha512-FBrGau0IXmuqg4haEZRBfHNWB5mUARw6hNwPDXXGg0XzVJ50mr/9hb267lvpVMnhZ1FON3qNd4Xfcez1rbFwSg=="], @@ -423,6 +478,8 @@ "require-from-string": ["require-from-string@2.0.2", "", {}, "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw=="], + "require-in-the-middle": ["require-in-the-middle@8.0.1", "", { "dependencies": { "debug": "^4.3.5", "module-details-from-path": "^1.0.3" } }, "sha512-QT7FVMXfWOYFbeRBF6nu+I6tr2Tf3u0q8RIEjNob/heKY/nh7drD/k7eeMFmSQgnTtCzLDcCu/XEnpW2wk4xCQ=="], + "resend": ["resend@6.22.1", "", { "dependencies": { "postal-mime": "2.7.5", "standardwebhooks": "1.0.0" }, "peerDependencies": { "@react-email/render": "*" }, "optionalPeers": ["@react-email/render"] }, "sha512-VqYhB6zqsifujJfZsiTKBVcZaKih0XitdXHCte4x4d7KH4D0QvPx5GvqcypCn2y64tK0C/6OskpQvsaSJaHHBw=="], "retry": ["retry@0.12.0", "", {}, "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow=="], @@ -435,6 +492,8 @@ "scheduler": ["scheduler@0.27.0", "", {}, "sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q=="], + "semifies": ["semifies@1.0.0", "", {}, "sha512-xXR3KGeoxTNWPD4aBvL5NUpMTT7WMANr3EWnaS190QVkY52lqqcVRD7Q05UVbBhiWDGWMlJEUam9m7uFFGVScw=="], + "semver": ["semver@7.8.0", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-AcM7dV/5ul4EekoQ29Agm5vri8JNqRyj39o0qpX6vDF2GZrtutZl5RwgD1XnZjiTAfncsJhMI48QQH3sN87YNA=="], "send": ["send@1.2.1", "", { "dependencies": { "debug": "^4.4.3", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "etag": "^1.8.1", "fresh": "^2.0.0", "http-errors": "^2.0.1", "mime-types": "^3.0.2", "ms": "^2.1.3", "on-finished": "^2.4.1", "range-parser": "^1.2.1", "statuses": "^2.0.2" } }, "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ=="], @@ -459,6 +518,8 @@ "signal-exit": ["signal-exit@4.1.0", "", {}, "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw=="], + "source-map": ["source-map@0.6.1", "", {}, "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="], + "split2": ["split2@4.2.0", "", {}, "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg=="], "sqlstring": ["sqlstring@2.3.3", "", {}, "sha512-qC9iz2FlN7DQl3+wjwn3802RTyjCx7sDvfQEXchwa6CWOx07/WVfh91gBmQ9fahw8snwGEWU3xGzOt4tFyHLxg=="], @@ -503,6 +564,8 @@ "express/cookie": ["cookie@0.7.2", "", {}, "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w=="], + "import-in-the-middle/es-module-lexer": ["es-module-lexer@3.0.2", "", {}, "sha512-BuIB67FngDSyQ/dpQNOZybwdEBDUGJQvOqwWr4ha/ufYiqzuEwPkKO2zLhRAgay28tStRIHUeWmszZAJo3GCOg=="], + "pg-types/postgres-array": ["postgres-array@2.0.0", "", {}, "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA=="], "proper-lockfile/signal-exit": ["signal-exit@3.0.7", "", {}, "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ=="], diff --git a/backend/package.json b/backend/package.json index 1cfff9f..9e42918 100644 --- a/backend/package.json +++ b/backend/package.json @@ -26,6 +26,7 @@ "dependencies": { "@prisma/adapter-pg": "^7.8.0", "@prisma/client": "^7.8.0", + "@sentry/bun": "^10.74.0", "bcrypt": "^6.0.0", "cookie": "^1.1.1", "cors": "^2.8.6", diff --git a/backend/src/config/index.ts b/backend/src/config/index.ts index 1e18d81..51116df 100644 --- a/backend/src/config/index.ts +++ b/backend/src/config/index.ts @@ -34,6 +34,9 @@ const envSchema = z.object({ STREAM_RETENTION_MS: z.coerce.number().default(30 * 60 * 1000), LOG_LEVEL: z.enum(["debug", "info"]), + + SENTRY_DSN: z.string().trim().optional(), + SENTRY_ENVIRONMENT: z.string().trim().optional(), }); const env = envSchema.parse(process.env); @@ -102,4 +105,10 @@ export const config = { sameSite: isProduction ? "none" : "lax", path: "/", } satisfies CookieOptions, + + sentry: { + enabled: Boolean(env.SENTRY_DSN) && env.NODE_ENV !== "test", + dsn: env.SENTRY_DSN, + environment: env.SENTRY_ENVIRONMENT ?? env.NODE_ENV, + }, }; diff --git a/backend/src/index.ts b/backend/src/index.ts index d8a896c..ea3a24b 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -1,3 +1,4 @@ +import "./instrument"; import "dotenv"; import { engineAbortController, @@ -11,6 +12,7 @@ import { logger } from "./utils/logger"; import { prisma } from "./db"; import { loadMarkets } from "./store/market"; import { startLeaderboardRefresh, stopLeaderboardRefresh } from "./services/leaderboard"; +import { flushSentry } from "./instrument"; const { httpServer, wss } = createAppServer(); @@ -67,6 +69,8 @@ async function gracefulShutdown(signal: string) { logger.error("Error disconnecting Prisma", err); } + await flushSentry(); + clearTimeout(forceExit); process.exit(0); } @@ -83,7 +87,8 @@ process.on("uncaughtException", (err) => { gracefulShutdown("uncaughtException").then(() => process.exit(1)); }); -main().catch((err) => { +main().catch(async (err) => { logger.error("Fatal startup error", err); + await flushSentry(); process.exit(1); }); diff --git a/backend/src/instrument.ts b/backend/src/instrument.ts new file mode 100644 index 0000000..171a46a --- /dev/null +++ b/backend/src/instrument.ts @@ -0,0 +1,30 @@ +import "dotenv/config"; +import * as Sentry from "@sentry/bun"; +import { config } from "./config"; + +Sentry.init({ + dsn: config.sentry.dsn, + enabled: config.sentry.enabled, + environment: config.sentry.environment, + sendDefaultPii: false, + maxBreadcrumbs: 50, + initialScope: { tags: { service: "backend" } }, +}); + +export function captureBackendException( + error: unknown, + context?: { tags?: Record; extra?: Record }, +) { + if (!config.sentry.enabled) return; + + Sentry.withScope((scope) => { + if (context?.tags) scope.setTags(context.tags); + if (context?.extra) scope.setExtras(context.extra); + Sentry.captureException(error); + }); +} + +export async function flushSentry(timeoutMs = 2_000) { + if (!config.sentry.enabled) return true; + return Sentry.flush(timeoutMs); +} diff --git a/backend/src/middleware/errors.ts b/backend/src/middleware/errors.ts index ca36bb6..8271636 100644 --- a/backend/src/middleware/errors.ts +++ b/backend/src/middleware/errors.ts @@ -2,6 +2,7 @@ import type { ErrorRequestHandler, NextFunction, Request, Response } from "expre import { config } from "../config"; import { ApiError, sendApiError } from "../utils/apiError"; import { logger } from "../utils/logger"; +import { captureBackendException } from "../instrument"; export function requestContext(req: Request, res: Response, next: NextFunction) { req.requestId = crypto.randomUUID(); @@ -50,6 +51,10 @@ export const errorHandler: ErrorRequestHandler = (error, req, res, _next) => { path: req.originalUrl, error: message, }); + captureBackendException(error, { + tags: { source: "express", request_id: req.requestId }, + extra: { method: req.method, path: req.path }, + }); const publicMessage = config.app.env === "production" ? "An unexpected error occurred" : message; diff --git a/backend/src/utils/logger.ts b/backend/src/utils/logger.ts index a55e179..edc4412 100644 --- a/backend/src/utils/logger.ts +++ b/backend/src/utils/logger.ts @@ -1,4 +1,5 @@ import { config } from "../config"; +import { captureBackendException } from "../instrument"; type LogLevel = "debug" | "info" | "warn" | "error"; @@ -13,6 +14,13 @@ function log(level: LogLevel, message: string, meta?: unknown) { meta !== undefined ? `${prefix} ${message} ${JSON.stringify(meta)}` : `${prefix} ${message}`; if (level === "error") console.error(line); else console.log(line); + + if (level === "error" && meta instanceof Error) { + captureBackendException(meta, { + tags: { source: "logger" }, + extra: { logMessage: message }, + }); + } } export const logger = { diff --git a/engine/bun.lock b/engine/bun.lock index bd397a5..579f5a7 100644 --- a/engine/bun.lock +++ b/engine/bun.lock @@ -5,6 +5,7 @@ "": { "name": "engine", "dependencies": { + "@sentry/bun": "^10.74.0", "dotenv": "^17.4.2", "pg": "^8.23.0", "redis": "^6.0.0", @@ -20,6 +21,28 @@ }, }, "packages": { + "@apm-js-collab/code-transformer": ["@apm-js-collab/code-transformer@0.18.1", "", { "dependencies": { "@types/estree": "^1.0.8", "astring": "^1.9.0", "esquery": "^1.7.0", "meriyah": "^6.1.4", "semifies": "^1.0.0", "source-map": "^0.6.0" }, "bin": { "code-transformer": "cli.js" } }, "sha512-u1Hb6bHjWtkSpiprwVP6YaHC1DTN4RAU3zYkUDUe7WMnJwdyU1pwTL9dFKiSJB9IiLue/EQovmyx6xhU7FFtAQ=="], + + "@apm-js-collab/code-transformer-bundler-plugins": ["@apm-js-collab/code-transformer-bundler-plugins@0.7.4", "", { "dependencies": { "@apm-js-collab/code-transformer": "^0.18.1", "es-module-lexer": "^2.1.0", "magic-string": "^0.30.21", "module-details-from-path": "^1.0.4" } }, "sha512-nAfOeZPSUAQvJa1iFT/5oCrTm5YQhMMrfCNthNnaXHZiOQhu1KGuLoIx7HtbAi3wfwaBYLaICPIeenIaEwcXIg=="], + + "@jridgewell/sourcemap-codec": ["@jridgewell/sourcemap-codec@1.6.0", "", {}, "sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw=="], + + "@opentelemetry/api": ["@opentelemetry/api@1.9.1", "", {}, "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q=="], + + "@opentelemetry/api-logs": ["@opentelemetry/api-logs@0.220.0", "", { "dependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-CmVa4ImJ+ynfrPMNaAXHET6Bhb44SwzmfyVJFq9ni2jgXJR/l7C6gfVFddNmHP+ZOkP9cf4f9DBe68qVLTHc9w=="], + + "@opentelemetry/core": ["@opentelemetry/core@2.11.0", "", { "dependencies": { "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-7YP44XH0tV6+Mb54x2YGf84i7yi+31MBZlE8JwvozkxyTvXbSp10X7cI7YE49ChJ3shMJoBmCJF3+1QFBJctGA=="], + + "@opentelemetry/instrumentation": ["@opentelemetry/instrumentation@0.220.0", "", { "dependencies": { "@opentelemetry/api-logs": "0.220.0", "import-in-the-middle": "^3.0.0", "require-in-the-middle": "^8.0.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-xQx3E2WxP1mDvKzxLxX+CTCtNLa560YJZ3087qYHerl2YmiKpv7AH+dAy7vmx+eVrZ5BwhfWUAVoKOoxCNHcpw=="], + + "@opentelemetry/resources": ["@opentelemetry/resources@2.11.0", "", { "dependencies": { "@opentelemetry/core": "2.11.0", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-Ie7+8q8MDF4FAEQCKVMTx3ReUvxiIAgIiiW3c9JdmP8+HMcDy20puT+AHjexnExgnbvBxjQ9fjkFDWrikJ2jQA=="], + + "@opentelemetry/sdk-trace": ["@opentelemetry/sdk-trace@2.11.0", "", { "dependencies": { "@opentelemetry/core": "2.11.0", "@opentelemetry/resources": "2.11.0", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-fFnTqGm8/G73GQVnxYi7LXa1ZVYEUvgL6XI1LpvV0bPC7WQ/ZGgKxCSl8FnlZBKto9JHHEFTO6s6CUpvvtwFrA=="], + + "@opentelemetry/sdk-trace-base": ["@opentelemetry/sdk-trace-base@2.11.0", "", { "dependencies": { "@opentelemetry/core": "2.11.0", "@opentelemetry/resources": "2.11.0", "@opentelemetry/sdk-trace": "2.11.0", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-H19x/TX/LZdqiYOjM7fqtSxwlplC5pgelavqbQdHbhdq0q/AI/TGkM2dfGuuynTXmJPeF2HoZVoPDu+TGoW78A=="], + + "@opentelemetry/semantic-conventions": ["@opentelemetry/semantic-conventions@1.43.0", "", {}, "sha512-eSYWTm620tTk45EKSedaUL8MFYI8hW164hIXsgIHyxu3VobUB3fFCu5t0hQby6OoWRPsG1KkKUG2M5UadiLiVg=="], + "@redis/bloom": ["@redis/bloom@6.0.0", "", { "peerDependencies": { "@redis/client": "^6.0.0" } }, "sha512-P0n5NkV9IIdT6nYXOfMHG83sho8pE7Nay7yw27wOGVLv4DthgvzebpGz6m7VuMTizeJmw3LPw2Xek5wFUhGpVw=="], "@redis/client": ["@redis/client@6.0.0", "", { "dependencies": { "cluster-key-slot": "1.1.2" }, "peerDependencies": { "@node-rs/xxhash": "^1.1.0", "@opentelemetry/api": ">=1 <2" }, "optionalPeers": ["@node-rs/xxhash", "@opentelemetry/api"] }, "sha512-NS4iIT25r24sAjNQ2nSRdCW5jPJoV0rxkBee27oTeR+RXaOu89cjIsrww5rPBaYVGVdL1QCx9uz9141gZiSKdQ=="], @@ -30,18 +53,56 @@ "@redis/time-series": ["@redis/time-series@6.0.0", "", { "peerDependencies": { "@redis/client": "^6.0.0" } }, "sha512-QWhkYsg+3lhBrBf+cbzybtV8LQcSrk7iXIgTaGU+pHNFTkql7TpVRE24ROS6M2ybVIV6O/zxTqfxgxxYiqyw0Q=="], + "@sentry/bun": ["@sentry/bun@10.74.0", "", { "dependencies": { "@apm-js-collab/code-transformer-bundler-plugins": "^0.7.3", "@sentry/conventions": "^0.16.0", "@sentry/core": "10.74.0", "@sentry/node": "10.74.0", "@sentry/server-utils": "10.74.0" } }, "sha512-KDX+sBp0ISatJ31r+GN4Uk1FUogDM+e3xqTTD1sTATpmYH0PEmggqZC/RaG+VM2DjDxf7jxjdeNuLuC1xKhV3A=="], + + "@sentry/conventions": ["@sentry/conventions@0.16.0", "", {}, "sha512-fO9PLmHdVURcSPUpWCItWAtgKiMwGdJHbovoSEyLplX5sxs2ugvI4CBPTrkkgqhObnZOD0CnWBKDzSVQYBKEyQ=="], + + "@sentry/core": ["@sentry/core@10.74.0", "", { "dependencies": { "@sentry/conventions": "^0.16.0" } }, "sha512-u9rY8vcZfktccwm6LznfCZlqP5C9A+p76r4/pFS1grqpuTO0m21Cl8rosnlESrDGP/Xd9tfr91rWYk0jPH8jeQ=="], + + "@sentry/node": ["@sentry/node@10.74.0", "", { "dependencies": { "@opentelemetry/api": "^1.9.1", "@opentelemetry/instrumentation": "^0.220.0", "@opentelemetry/sdk-trace-base": "^2.9.0", "@sentry/conventions": "^0.16.0", "@sentry/core": "10.74.0", "@sentry/node-core": "10.74.0", "@sentry/opentelemetry": "10.74.0", "@sentry/server-utils": "10.74.0", "import-in-the-middle": "^3.0.0" } }, "sha512-u1wsarTOWHn9CCev81Da5T4IQHZgdcosXRfX2+4DMII/lVJMBYesKixTQuMwXexjVG2+pkf16zTmgjZDa+75jA=="], + + "@sentry/node-core": ["@sentry/node-core@10.74.0", "", { "dependencies": { "@sentry/conventions": "^0.16.0", "@sentry/core": "10.74.0", "@sentry/opentelemetry": "10.74.0", "import-in-the-middle": "^3.0.0" }, "peerDependencies": { "@opentelemetry/api": "^1.9.0", "@opentelemetry/core": "^1.30.1 || ^2.1.0", "@opentelemetry/exporter-trace-otlp-http": ">=0.57.0 <1", "@opentelemetry/instrumentation": ">=0.57.1 <1", "@opentelemetry/sdk-trace-base": "^1.30.1 || ^2.1.0" }, "optionalPeers": ["@opentelemetry/api", "@opentelemetry/core", "@opentelemetry/exporter-trace-otlp-http", "@opentelemetry/instrumentation", "@opentelemetry/sdk-trace-base"] }, "sha512-btgZXcGmkOGgojbxHo/gfGiyqYpzC9E8zVR78Q3MtM6Xnemk9gwJiQlhNmEX/FM+C36WBRPZrdcZcnMaJhbfJw=="], + + "@sentry/opentelemetry": ["@sentry/opentelemetry@10.74.0", "", { "dependencies": { "@sentry/conventions": "^0.16.0", "@sentry/core": "10.74.0" }, "peerDependencies": { "@opentelemetry/api": "^1.9.0", "@opentelemetry/core": "^1.30.1 || ^2.1.0", "@opentelemetry/sdk-trace-base": "^1.30.1 || ^2.1.0" } }, "sha512-ua5mt0NDBfye+/ACKjAw9Ad2i+y42lOLvhxXZepZXjszblMz80MEsIZflMB7uLZUCTNH7MbZN8tlzCy8KsJKmQ=="], + + "@sentry/server-utils": ["@sentry/server-utils@10.74.0", "", { "dependencies": { "@sentry/conventions": "^0.16.0", "@sentry/core": "10.74.0" } }, "sha512-AHmPIGE8yVRyywpZRhAkm/H0cHTgsQSPhFZFbaeQLcYHFE/eOP9bAMh8Nj/Apr7WMagFXYHd04tR/63BxtLgKw=="], + "@types/bun": ["@types/bun@1.3.14", "", { "dependencies": { "bun-types": "1.3.14" } }, "sha512-h1hFqFVcvAvD9j9K7ZW7vd82aSA+rTdznZa+5bwvCwqSB1jmmfLcbIWhOLx1/+boy/xmjgCs/OMUL8hRJSmnPw=="], + "@types/estree": ["@types/estree@1.0.9", "", {}, "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg=="], + "@types/node": ["@types/node@25.9.1", "", { "dependencies": { "undici-types": ">=7.24.0 <7.24.7" } }, "sha512-xfrlY7UD5rMJk3ZVJP8BNzS28J36YJg+xp+LPXV1TdWxr8uMH5A860QNxYDGQe/ylDSgjxE52Q9VnO7p75tJxg=="], "@types/pg": ["@types/pg@8.23.1", "", { "dependencies": { "@types/node": "*", "pg-protocol": "*", "pg-types": "^2.2.0" } }, "sha512-fKVHpikPdg4GKks3JuLEhvwSyvwzF23hnabPy6DD8ljVbC7+6J5dQzdv4arV6jqq57djnMgs1HKBxX4P8aBI3A=="], + "astring": ["astring@1.9.0", "", { "bin": { "astring": "bin/astring" } }, "sha512-LElXdjswlqjWrPpJFg1Fx4wpkOCxj1TDHlSV4PlaRxHGWko024xICaa97ZkMfs6DRKlCguiAI+rbXv5GWwXIkg=="], + "bun-types": ["bun-types@1.3.14", "", { "dependencies": { "@types/node": "*" } }, "sha512-4N0ig0fEomHt5R0KCFWjovxow98rIoRwKolrYdCcknNwMekCXRnWEUvgu5soYV8QXtVsrUD8B95MBOZGPvr6KQ=="], + "cjs-module-lexer": ["cjs-module-lexer@2.2.1", "", {}, "sha512-Ca8swihM+/4yKecYHY52kgJd300hi2lADU/a1RxNTRe+RJ9jvqQlESpbz9DnG9mowez8qwXHB8qYdIUw9e+F5Q=="], + "cluster-key-slot": ["cluster-key-slot@1.1.2", "", {}, "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA=="], + "debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="], + "dotenv": ["dotenv@17.4.2", "", {}, "sha512-nI4U3TottKAcAD9LLud4Cb7b2QztQMUEfHbvhTH09bqXTxnSie8WnjPALV/WMCrJZ6UV/qHJ6L03OqO3LcdYZw=="], + "es-module-lexer": ["es-module-lexer@2.3.2", "", {}, "sha512-poHGpORABojJJucnV9KbOavETW8lBVnphkW77ER5/BQ5Fz7oXSoCNek7IH3vR5nRjdsEz926ibFYX8KtLQmdyw=="], + + "esquery": ["esquery@1.7.0", "", { "dependencies": { "estraverse": "^5.1.0" } }, "sha512-Ap6G0WQwcU/LHsvLwON1fAQX9Zp0A2Y6Y/cJBl9r/JbW90Zyg4/zbG6zzKa2OTALELarYHmKu0GhpM5EO+7T0g=="], + + "estraverse": ["estraverse@5.3.0", "", {}, "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA=="], + + "import-in-the-middle": ["import-in-the-middle@3.5.1", "", { "dependencies": { "cjs-module-lexer": "^2.2.0", "es-module-lexer": "^3.0.2", "module-details-from-path": "^1.0.4" } }, "sha512-mPKuL8bPQzecui2KK6Gb+M8JvJoHnhS1FeYGa22QopBmlevF5F0FE6ued/B5EgHDeIoMTONIpDWWFKUPOG0DBQ=="], + + "magic-string": ["magic-string@0.30.21", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.5" } }, "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ=="], + + "meriyah": ["meriyah@6.1.4", "", {}, "sha512-Sz8FzjzI0kN13GK/6MVEsVzMZEPvOhnmmI1lU5+/1cGOiK3QUahntrNNtdVeihrO7t9JpoH75iMNXg6R6uWflQ=="], + + "module-details-from-path": ["module-details-from-path@1.0.4", "", {}, "sha512-EGWKgxALGMgzvxYF1UyGTy0HXX/2vHLkw6+NvDKW2jypWbHpjQuj4UMcqQWXHERJhVGKikolT06G3bcKe4fi7w=="], + + "ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], + "pg": ["pg@8.23.0", "", { "dependencies": { "pg-connection-string": "^2.14.0", "pg-pool": "^3.14.0", "pg-protocol": "^1.16.0", "pg-types": "2.2.0", "pgpass": "1.0.5" }, "optionalDependencies": { "pg-cloudflare": "^1.4.0" }, "peerDependencies": { "pg-native": ">=3.0.1" }, "optionalPeers": ["pg-native"] }, "sha512-Ip2EQCngowJLGOfCwkFhPXU7/ljlhn6Rxlmy4XYfL2Y+vyRM59+8uR2xqRWKdYmbXmxCFOAmKxBuSUCdF34qLg=="], "pg-cloudflare": ["pg-cloudflare@1.4.0", "", {}, "sha512-Vo7z/6rrQYxpNRylp4Tlob2elzbh+N/MOQbxFVWCxS7oEx6jF53GTJFxK2WWpKuBRkmiin4Mt+xofFDjx09R0A=="], @@ -68,6 +129,12 @@ "redis": ["redis@6.0.0", "", { "dependencies": { "@redis/bloom": "6.0.0", "@redis/client": "6.0.0", "@redis/json": "6.0.0", "@redis/search": "6.0.0", "@redis/time-series": "6.0.0" } }, "sha512-n9Thfc39OXleEoPT2k5gwKsqY+HfCww3YS71ofcr9KKbkn89bpjU9dToIlD+JRdM3/GYQkwMtVgTxLyed+LptQ=="], + "require-in-the-middle": ["require-in-the-middle@8.0.1", "", { "dependencies": { "debug": "^4.3.5", "module-details-from-path": "^1.0.3" } }, "sha512-QT7FVMXfWOYFbeRBF6nu+I6tr2Tf3u0q8RIEjNob/heKY/nh7drD/k7eeMFmSQgnTtCzLDcCu/XEnpW2wk4xCQ=="], + + "semifies": ["semifies@1.0.0", "", {}, "sha512-xXR3KGeoxTNWPD4aBvL5NUpMTT7WMANr3EWnaS190QVkY52lqqcVRD7Q05UVbBhiWDGWMlJEUam9m7uFFGVScw=="], + + "source-map": ["source-map@0.6.1", "", {}, "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="], + "split2": ["split2@4.2.0", "", {}, "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg=="], "typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="], @@ -77,5 +144,7 @@ "xtend": ["xtend@4.0.2", "", {}, "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ=="], "zod": ["zod@4.4.3", "", {}, "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ=="], + + "import-in-the-middle/es-module-lexer": ["es-module-lexer@3.0.2", "", {}, "sha512-BuIB67FngDSyQ/dpQNOZybwdEBDUGJQvOqwWr4ha/ufYiqzuEwPkKO2zLhRAgay28tStRIHUeWmszZAJo3GCOg=="], } } diff --git a/engine/package.json b/engine/package.json index c817f9e..60e0397 100644 --- a/engine/package.json +++ b/engine/package.json @@ -17,6 +17,7 @@ "typescript": "^5" }, "dependencies": { + "@sentry/bun": "^10.74.0", "dotenv": "^17.4.2", "pg": "^8.23.0", "redis": "^6.0.0", diff --git a/engine/src/config/index.ts b/engine/src/config/index.ts index 273c64b..0f87642 100644 --- a/engine/src/config/index.ts +++ b/engine/src/config/index.ts @@ -12,6 +12,8 @@ export const envSchema = z.object({ RECENT_TRADES_LIMIT: z.coerce.number().positive().default(50), STREAM_READ_BATCH_SIZE: z.coerce.number().positive().default(200), RESPONSE_BATCH_SIZE: z.coerce.number().positive().default(50), + SENTRY_DSN: z.string().trim().optional(), + SENTRY_ENVIRONMENT: z.string().trim().optional(), }); const env = envSchema.parse(process.env); @@ -27,4 +29,9 @@ export const config = { recentTradesLimit: env.RECENT_TRADES_LIMIT, streamReadBatchSize: env.STREAM_READ_BATCH_SIZE, responseBatchSize: env.RESPONSE_BATCH_SIZE, + sentry: { + enabled: Boolean(env.SENTRY_DSN) && env.NODE_ENV !== "test", + dsn: env.SENTRY_DSN, + environment: env.SENTRY_ENVIRONMENT ?? env.NODE_ENV, + }, }; diff --git a/engine/src/index.ts b/engine/src/index.ts index 16c8d1d..0ad1e29 100644 --- a/engine/src/index.ts +++ b/engine/src/index.ts @@ -1,3 +1,4 @@ +import { flushSentry } from "./instrument"; import type { EngineRequest, EngineResponse } from "./types/request"; import { cacheClient, @@ -18,13 +19,6 @@ import { startHealth, stopHealth } from "./health"; const abortController = new AbortController(); -await connectRedis(); -await connectDB(); -await initMarkets(); -await loadSnapshot(); -registerEventHandlers(); -await startHealth(); - async function sendResponse(responseQueue: string, response: EngineResponse) { await streamProducer.lPush(responseQueue, JSON.stringify(response, bigintReplacer)); } @@ -135,17 +129,27 @@ async function main() { } } -main().catch((err) => logger.error("Engine process error", err)); +async function start() { + await connectRedis(); + await connectDB(); + await initMarkets(); + await loadSnapshot(); + registerEventHandlers(); + await startHealth(); + + setInterval(() => { + snapshot().catch((err) => logger.error("Snapshot error", err)); + }, config.snapshotInterval).unref(); -setInterval(() => { - snapshot().catch((err) => logger.error("Snapshot error", err)); -}, config.snapshotInterval).unref(); + await main(); +} async function gracefulShutdown(signal: string) { logger.info(`Received ${signal}, shutting down...`); - const forceExit = setTimeout(() => { + const forceExit = setTimeout(async () => { logger.error("Graceful shutdown timed out, forcing exit"); + await flushSentry(1_000); process.exit(1); }, 10000); @@ -153,6 +157,7 @@ async function gracefulShutdown(signal: string) { await stopHealth(); await disconnectRedis(); await disconnectDB(); + await flushSentry(); clearTimeout(forceExit); process.exit(0); @@ -160,3 +165,9 @@ async function gracefulShutdown(signal: string) { process.on("SIGTERM", () => gracefulShutdown("SIGTERM")); process.on("SIGINT", () => gracefulShutdown("SIGINT")); + +start().catch(async (err) => { + logger.error("Engine process error", err); + await flushSentry(); + process.exit(1); +}); diff --git a/engine/src/instrument.ts b/engine/src/instrument.ts new file mode 100644 index 0000000..d99885e --- /dev/null +++ b/engine/src/instrument.ts @@ -0,0 +1,30 @@ +import "dotenv/config"; +import * as Sentry from "@sentry/bun"; +import { config } from "./config"; + +Sentry.init({ + dsn: config.sentry.dsn, + enabled: config.sentry.enabled, + environment: config.sentry.environment, + sendDefaultPii: false, + maxBreadcrumbs: 50, + initialScope: { tags: { service: "engine" } }, +}); + +export function captureEngineException( + error: unknown, + context?: { tags?: Record; extra?: Record }, +) { + if (!config.sentry.enabled) return; + + Sentry.withScope((scope) => { + if (context?.tags) scope.setTags(context.tags); + if (context?.extra) scope.setExtras(context.extra); + Sentry.captureException(error); + }); +} + +export async function flushSentry(timeoutMs = 2_000) { + if (!config.sentry.enabled) return true; + return Sentry.flush(timeoutMs); +} diff --git a/engine/src/redis/client.ts b/engine/src/redis/client.ts index 8557e88..199ed53 100644 --- a/engine/src/redis/client.ts +++ b/engine/src/redis/client.ts @@ -3,19 +3,19 @@ import { config } from "../config"; import { logger } from "../util/logger"; export const streamConsumer = createClient({ url: config.redisUrl }).on("error", (err) => - logger.error("Redis streamConsumer error", { error: (err as Error).message }), + logger.error("Redis streamConsumer error", err), ); export const streamProducer = createClient({ url: config.redisUrl }).on("error", (err) => - logger.error("Redis streamProducer error", { error: (err as Error).message }), + logger.error("Redis streamProducer error", err), ); export const publisher = createClient({ url: config.redisUrl }).on("error", (err) => - logger.error("Redis publisher error", { error: (err as Error).message }), + logger.error("Redis publisher error", err), ); export const cacheClient = createClient({ url: config.redisUrl }).on("error", (err) => - logger.error("Redis cacheClient error", { error: (err as Error).message }), + logger.error("Redis cacheClient error", err), ); export async function connectRedis() { @@ -29,7 +29,7 @@ export async function connectRedis() { logger.info(`Engine listening on Redis queue: ${config.incomingStream}`); } catch (err) { logger.error("Redis connection error", err); - process.exit(1); + throw err; } } diff --git a/engine/src/util/logger.ts b/engine/src/util/logger.ts index f6940f3..f1e9174 100644 --- a/engine/src/util/logger.ts +++ b/engine/src/util/logger.ts @@ -1,4 +1,5 @@ import { config } from "../config"; +import { captureEngineException } from "../instrument"; type LogLevel = "debug" | "info" | "warn" | "error"; @@ -6,11 +7,18 @@ const LEVEL = { debug: 0, info: 1, warn: 2, error: 3 }; function log(level: LogLevel, message: string, meta?: unknown) { if (LEVEL[level] < LEVEL[config.logLevel]) return; + if (level === "error" && meta instanceof Error) { + captureEngineException(meta, { tags: { source: "engine_logger" } }); + } const ts = new Date().toISOString(); const prefix = `[${ts}] [${level.toUpperCase()}]`; + const details = + meta instanceof Error ? { name: meta.name, message: meta.message, stack: meta.stack } : meta; const line = - meta !== undefined ? `${prefix} ${message} ${JSON.stringify(meta)}` : `${prefix} ${message}`; + details !== undefined + ? `${prefix} ${message} ${JSON.stringify(details)}` + : `${prefix} ${message}`; if (level === "error") console.error(line); else console.log(line); } From 48a45df13a514ed0c5a4cf12deaa68905ed5a80c Mon Sep 17 00:00:00 2001 From: Harshil Date: Thu, 17 Sep 2026 18:54:12 +0530 Subject: [PATCH 05/20] chore: docker setup and scripts for service orchestration --- .env.example | 8 +- Makefile | 112 +++++++++++++++----------- backend/prisma/seed.ts | 15 +++- benchmarks/run.sh | 14 +++- compose.bot.yml | 38 --------- compose.dev.yml | 77 +++++++++++------- compose.prod.yml | 168 +++++++++++++-------------------------- compose.test.yml | 77 +++--------------- compose.yml | 66 +++++++++++++++ scripts/compose.sh | 36 +++++++++ scripts/database.sh | 41 ++++++++++ scripts/deploy.sh | 11 +++ scripts/images.sh | 36 +++++++++ tests/run-integration.sh | 14 +++- 14 files changed, 408 insertions(+), 305 deletions(-) delete mode 100644 compose.bot.yml create mode 100644 compose.yml create mode 100755 scripts/compose.sh create mode 100755 scripts/database.sh create mode 100755 scripts/deploy.sh create mode 100755 scripts/images.sh diff --git a/.env.example b/.env.example index 94f262d..bd1abfd 100644 --- a/.env.example +++ b/.env.example @@ -33,9 +33,11 @@ RATE_LIMIT_ORDER_WINDOW_MS=60000 LOG_LEVEL=debug -# Optional backend error reporting -SENTRY_DSN= -SENTRY_ENVIRONMENT=development +# Production-only external services. They may be omitted in development and test. +# RESEND_API_KEY=re_... +# EMAIL_FROM=PaperDrill +# SENTRY_DSN=https://... +# SENTRY_ENVIRONMENT=production # Bot credentials BASE_URL=http://localhost:8000 diff --git a/Makefile b/Makefile index d8ddbd5..83222e6 100644 --- a/Makefile +++ b/Makefile @@ -1,67 +1,60 @@ -ENV ?= dev +SERVICE ?= +TAG ?= -DEV_FILE := compose.dev.yml -TEST_FILE := compose.test.yml -PROD_FILE := compose.prod.yml -BOT_FILE := compose.bot.yml +DEV_COMPOSE := ./scripts/compose.sh dev +PROD_COMPOSE := ./scripts/compose.sh prod -ifeq ($(ENV),dev) -COMPOSE := docker compose -f $(DEV_FILE) -endif - -ifeq ($(ENV),prod) -COMPOSE := docker compose -f $(PROD_FILE) -endif - -ifeq ($(ENV),test) -COMPOSE := docker compose -p paperdrill-test -f $(TEST_FILE) -endif - -ifeq ($(ENV),bot) -COMPOSE := docker compose -f $(BOT_FILE) -endif - -.PHONY: up build down deploy start restart stop status logs migrate-dev migrate-reset migrate-deploy seed test-unit test-integration test-e2e bench clean +.PHONY: up build down restart status logs shell bots-up bots-stop \ + db-prepare db-seed migrate-dev migrate-reset \ + test-unit test-integration bench \ + images-build images-push prod-config deploy prod-status prod-logs \ + prod-bots-up prod-bots-stop clean +# Development up: - $(COMPOSE) up -d + $(DEV_COMPOSE) up -d --wait postgres redis + ./scripts/database.sh dev prepare + $(DEV_COMPOSE) up -d --wait backend engine worker build: - $(COMPOSE) build + $(DEV_COMPOSE) build backend engine worker down: - $(COMPOSE) down - -deploy: - $(COMPOSE) up -d --build - -start: - $(COMPOSE) start + $(DEV_COMPOSE) down --remove-orphans restart: - $(COMPOSE) restart - -stop: - $(COMPOSE) stop + $(DEV_COMPOSE) restart backend engine worker status: - $(COMPOSE) ps + $(DEV_COMPOSE) ps logs: - $(COMPOSE) logs -f $(SERVICE) + $(DEV_COMPOSE) logs -f $(SERVICE) -migrate-dev: - $(COMPOSE) exec backend bun prisma migrate dev +shell: + @if [ -z "$(SERVICE)" ]; then echo "Set SERVICE, for example: make shell SERVICE=backend" >&2; exit 1; fi + $(DEV_COMPOSE) exec $(SERVICE) sh -migrate-reset: - $(COMPOSE) exec backend bun prisma migrate reset +bots-up: up + $(DEV_COMPOSE) --profile bots up -d --build btc-bot sol-bot eth-bot -migrate-deploy: - $(COMPOSE) exec backend bun prisma migrate deploy +bots-stop: + $(DEV_COMPOSE) --profile bots stop btc-bot sol-bot eth-bot + +# Database +db-prepare: + ./scripts/database.sh dev prepare db-seed: - $(COMPOSE) exec backend bun prisma db seed + ./scripts/database.sh dev seed + +migrate-dev: + $(DEV_COMPOSE) exec backend bun prisma migrate dev +migrate-reset: + $(DEV_COMPOSE) exec backend bun prisma migrate reset + +# Test test-unit: cd backend && set -a && . ../.env.test && set +a && bun test cd engine && set -a && . ../.env.test && set +a && bun test @@ -69,10 +62,35 @@ test-unit: test-integration: sh tests/run-integration.sh -test-e2e: - bench: sh benchmarks/run.sh +# Release images +images-build: + ./scripts/images.sh build "$(TAG)" + +images-push: + ./scripts/images.sh push "$(TAG)" + +# Production +prod-config: + $(PROD_COMPOSE) config --quiet + +deploy: + ./scripts/deploy.sh + +prod-status: + $(PROD_COMPOSE) ps + +prod-logs: + $(PROD_COMPOSE) logs -f $(SERVICE) + +prod-bots-up: + $(PROD_COMPOSE) --profile bots pull btc-bot sol-bot eth-bot + $(PROD_COMPOSE) --profile bots up -d btc-bot sol-bot eth-bot + +prod-bots-stop: + $(PROD_COMPOSE) --profile bots stop btc-bot sol-bot eth-bot + clean: - docker system prune -f + $(DEV_COMPOSE) down --volumes --remove-orphans diff --git a/backend/prisma/seed.ts b/backend/prisma/seed.ts index c7327d3..918d78e 100644 --- a/backend/prisma/seed.ts +++ b/backend/prisma/seed.ts @@ -37,6 +37,20 @@ async function seed() { qtyPrecision: 3, }, ]; + const serviceEmails = [ + "btc.service@paperdrill.dev", + "sol.service@paperdrill.dev", + "eth.service@paperdrill.dev", + ]; + const [marketCount, serviceCount] = await Promise.all([ + prisma.market.count({ where: { symbol: { in: markets.map((market) => market.symbol) } } }), + prisma.user.count({ where: { email: { in: serviceEmails }, type: UserType.SERVICE } }), + ]); + + if (marketCount === markets.length && serviceCount === serviceEmails.length) { + console.log("Seed data already exists, skipping"); + return; + } await prisma.market.createMany({ data: markets, @@ -74,7 +88,6 @@ async function seed() { }); console.log("Seeded: 3 markets, 3 services"); - process.exit(0); } seed() diff --git a/benchmarks/run.sh b/benchmarks/run.sh index ff60a59..df3a0ce 100644 --- a/benchmarks/run.sh +++ b/benchmarks/run.sh @@ -3,7 +3,7 @@ set -u compose() { - docker compose -p paperdrill-benchmark -f compose.test.yml "$@" + PAPERDRILL_PROJECT=paperdrill-benchmark ./scripts/compose.sh test "$@" } cleanup() { @@ -17,6 +17,14 @@ status=0 compose down --volumes --remove-orphans || true compose build || status=$? +if [ "$status" -eq 0 ]; then + compose up -d --wait postgres redis || status=$? +fi + +if [ "$status" -eq 0 ]; then + PAPERDRILL_PROJECT=paperdrill-benchmark ./scripts/database.sh test prepare || status=$? +fi + if [ "$status" -eq 0 ]; then compose up -d --wait backend engine worker || status=$? fi @@ -27,7 +35,7 @@ if [ "$status" -eq 0 ]; then . ./.env.test set +a - if [ "$NODE_ENV" != "test" ] || [ "$TEST_ENVIRONMENT" != "integration" ]; then + if [ "$NODE_ENV" != "test" ]; then echo "Refusing to benchmark outside the test environment" >&2 exit 1 fi @@ -41,7 +49,7 @@ fi if [ "$status" -ne 0 ]; then compose ps - compose logs --no-color backend engine worker postgres redis setup + compose logs --no-color backend engine worker fi exit "$status" diff --git a/compose.bot.yml b/compose.bot.yml deleted file mode 100644 index 4736bb8..0000000 --- a/compose.bot.yml +++ /dev/null @@ -1,38 +0,0 @@ -services: - btc-bot: - build: ./bot - restart: on-failure:5 - env_file: .env.prod - environment: - MARKET: BTC_USD - SERVICE_EMAIL: btc.service@paperdrill.dev - MAX_ORDER_QTY: 0.1 - networks: - - paperdrill - - sol-bot: - build: ./bot - restart: on-failure:5 - env_file: .env.prod - environment: - MARKET: SOL_USD - SERVICE_EMAIL: sol.service@paperdrill.dev - MAX_ORDER_QTY: 10 - networks: - - paperdrill - - eth-bot: - build: ./bot - restart: on-failure:5 - env_file: .env.prod - environment: - MARKET: ETH_USD - SERVICE_EMAIL: eth.service@paperdrill.dev - MAX_ORDER_QTY: 1 - networks: - - paperdrill - -networks: - paperdrill: - name: paperdrill-network - external: true diff --git a/compose.dev.yml b/compose.dev.yml index 5d39595..06e7676 100644 --- a/compose.dev.yml +++ b/compose.dev.yml @@ -1,77 +1,94 @@ +x-dev-bot: &dev-bot + build: ./bot + env_file: ./.env.dev + profiles: ["bots"] + restart: on-failure:5 + depends_on: + backend: + condition: service_healthy + services: backend: build: context: ./backend target: dev - restart: unless-stopped + env_file: ./.env.dev ports: - "8000:8000" - env_file: ./.env.dev volumes: - ./backend:/usr/src/app - /usr/src/app/node_modules depends_on: - - postgres - - redis - networks: - - paperdrill + postgres: + condition: service_healthy engine: build: context: ./engine target: dev - restart: unless-stopped env_file: ./.env.dev volumes: - ./engine:/usr/src/app - /usr/src/app/node_modules depends_on: - - postgres - - redis - networks: - - paperdrill + postgres: + condition: service_healthy worker: build: context: ./worker target: dev - restart: unless-stopped env_file: ./.env.dev volumes: - ./worker:/usr/src/app - /usr/src/app/node_modules depends_on: - - postgres - - redis - networks: - - paperdrill + postgres: + condition: service_healthy postgres: image: timescale/timescaledb:2.28.2-pg17 - restart: unless-stopped ports: - "5433:5432" environment: POSTGRES_DB: paperdrill POSTGRES_USER: postgres POSTGRES_PASSWORD: password + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres -d paperdrill"] + interval: 5s + timeout: 5s + retries: 10 volumes: - - postgres_data:/var/lib/postgresql/data - networks: - - paperdrill + - postgres-data:/var/lib/postgresql/data redis: - image: redis:7.2 - restart: unless-stopped ports: - "6380:6379" - networks: - - paperdrill -volumes: - postgres_data: + btc-bot: + <<: *dev-bot + environment: + BASE_URL: http://backend:8000 + MARKET: BTC_USD + SERVICE_EMAIL: btc.service@paperdrill.dev + MAX_ORDER_QTY: 0.1 -networks: - paperdrill: - name: paperdrill-network - external: true + sol-bot: + <<: *dev-bot + environment: + BASE_URL: http://backend:8000 + MARKET: SOL_USD + SERVICE_EMAIL: sol.service@paperdrill.dev + MAX_ORDER_QTY: 10 + + eth-bot: + <<: *dev-bot + environment: + BASE_URL: http://backend:8000 + MARKET: ETH_USD + SERVICE_EMAIL: eth.service@paperdrill.dev + MAX_ORDER_QTY: 1 + +volumes: + postgres-data: diff --git a/compose.prod.yml b/compose.prod.yml index 1859db6..cc3dc4b 100644 --- a/compose.prod.yml +++ b/compose.prod.yml @@ -1,100 +1,48 @@ +x-prod-runtime: &prod-runtime + restart: unless-stopped + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" + +x-prod-app: &prod-app + <<: *prod-runtime + env_file: ./.env.prod + +x-bot-service: &bot-service + <<: *prod-app + image: "ghcr.io/hars-21/paperdrill-bot:${PAPERDRILL_IMAGE_TAG:?Set PAPERDRILL_IMAGE_TAG in .env.prod}" + profiles: ["bots"] + restart: on-failure:5 + depends_on: + backend: + condition: service_healthy + services: backend: - build: - context: ./backend - target: runner - restart: unless-stopped - expose: - - 8000 - env_file: ./.env.prod - healthcheck: - test: - [ - "CMD", - "bun", - "-e", - "const r = await fetch('http://localhost:8000/health'); if (!r.ok) process.exit(1)", - ] - interval: 30s - timeout: 10s - retries: 3 - start_period: 40s - logging: - driver: json-file - options: - max-size: "10m" - max-file: "3" - depends_on: - redis: - condition: service_healthy - engine: - condition: service_healthy - worker: - condition: service_healthy - networks: - - paperdrill + <<: *prod-app + image: "ghcr.io/hars-21/paperdrill-backend:${PAPERDRILL_IMAGE_TAG:?Set PAPERDRILL_IMAGE_TAG in .env.prod}" engine: - build: - context: ./engine - target: runner - restart: unless-stopped - env_file: ./.env.prod - healthcheck: - test: - [ - "CMD", - "bun", - "-e", - "const value = Number(await Bun.file('/tmp/health').text()); const age = Date.now() - value; if (!Number.isFinite(age) || age < 0 || age > 30000) process.exit(1)", - ] - interval: 15s - timeout: 5s - retries: 3 - start_period: 20s - logging: - driver: json-file - options: - max-size: "10m" - max-file: "3" - depends_on: - redis: - condition: service_healthy + <<: *prod-app + image: "ghcr.io/hars-21/paperdrill-engine:${PAPERDRILL_IMAGE_TAG:?Set PAPERDRILL_IMAGE_TAG in .env.prod}" volumes: - engine-snapshots:/usr/src/app/snapshots - networks: - - paperdrill worker: - build: - context: ./worker - target: runner - restart: unless-stopped - env_file: ./.env.prod - healthcheck: - test: - [ - "CMD", - "bun", - "-e", - "const value = Number(await Bun.file('/tmp/health').text()); const age = Date.now() - value; if (!Number.isFinite(age) || age < 0 || age > 30000) process.exit(1)", - ] - interval: 15s - timeout: 5s - retries: 3 - start_period: 20s - logging: - driver: json-file - options: - max-size: "10m" - max-file: "3" - depends_on: - redis: - condition: service_healthy - networks: - - paperdrill + <<: *prod-app + image: "ghcr.io/hars-21/paperdrill-worker:${PAPERDRILL_IMAGE_TAG:?Set PAPERDRILL_IMAGE_TAG in .env.prod}" + + redis: + <<: *prod-runtime + image: redis:7.2 + command: redis-server --appendonly yes --maxmemory 768mb --maxmemory-policy noeviction --tcp-keepalive 60 --appendfsync everysec --auto-aof-rewrite-percentage 100 --auto-aof-rewrite-min-size 64mb + volumes: + - redis-data:/data nginx: + <<: *prod-runtime image: nginx:stable ports: - "80:80" @@ -105,34 +53,28 @@ services: volumes: - ./nginx/conf.d/:/etc/nginx/conf.d:ro - /etc/letsencrypt:/etc/letsencrypt:ro - networks: - - paperdrill - redis: - image: redis:7.2 - command: redis-server --appendonly yes --maxmemory 768mb --maxmemory-policy noeviction --tcp-keepalive 60 --appendfsync everysec --auto-aof-rewrite-percentage 100 --auto-aof-rewrite-min-size 64mb - restart: unless-stopped - healthcheck: - test: ["CMD", "redis-cli", "ping"] - interval: 10s - timeout: 5s - retries: 3 - start_period: 10s - logging: - driver: json-file - options: - max-size: "10m" - max-file: "3" - volumes: - - redis-data:/data - networks: - - paperdrill + btc-bot: + <<: *bot-service + environment: + MARKET: BTC_USD + SERVICE_EMAIL: btc.service@paperdrill.dev + MAX_ORDER_QTY: 0.1 + + sol-bot: + <<: *bot-service + environment: + MARKET: SOL_USD + SERVICE_EMAIL: sol.service@paperdrill.dev + MAX_ORDER_QTY: 10 + + eth-bot: + <<: *bot-service + environment: + MARKET: ETH_USD + SERVICE_EMAIL: eth.service@paperdrill.dev + MAX_ORDER_QTY: 1 volumes: engine-snapshots: redis-data: - -networks: - paperdrill: - name: paperdrill-network - external: true diff --git a/compose.test.yml b/compose.test.yml index d093c10..1e5b531 100644 --- a/compose.test.yml +++ b/compose.test.yml @@ -1,3 +1,10 @@ +x-test-app: &test-app + env_file: ./.env.test + healthcheck: + interval: 2s + timeout: 5s + retries: 20 + services: postgres: image: timescale/timescaledb:2.28.2-pg17 @@ -16,97 +23,33 @@ services: - /var/lib/postgresql/data redis: - image: redis:7.2 ports: - "6381:6379" healthcheck: - test: ["CMD", "redis-cli", "ping"] interval: 2s timeout: 5s retries: 15 tmpfs: - /data - setup: - build: - context: ./backend - target: dev - command: sh -c "bun prisma migrate deploy && bun prisma db seed" - env_file: ./.env.test - depends_on: - postgres: - condition: service_healthy - backend: + <<: *test-app build: context: ./backend target: runner ports: - "8001:8000" - env_file: ./.env.test - depends_on: - redis: - condition: service_healthy - setup: - condition: service_completed_successfully - engine: - condition: service_healthy - worker: - condition: service_healthy - healthcheck: - test: - [ - "CMD", - "bun", - "-e", - "const r = await fetch('http://localhost:8000/health'); if (!r.ok) process.exit(1)", - ] - interval: 2s - timeout: 5s - retries: 20 engine: + <<: *test-app build: context: ./engine target: runner - env_file: ./.env.test - depends_on: - redis: - condition: service_healthy - setup: - condition: service_completed_successfully - healthcheck: - test: - [ - "CMD", - "bun", - "-e", - "const value = Number(await Bun.file('/tmp/health').text()); const age = Date.now() - value; if (!Number.isFinite(age) || age < 0 || age > 30000) process.exit(1)", - ] - interval: 2s - timeout: 5s - retries: 20 tmpfs: - /usr/src/app/snapshots worker: + <<: *test-app build: context: ./worker target: runner - env_file: ./.env.test - depends_on: - redis: - condition: service_healthy - setup: - condition: service_completed_successfully - healthcheck: - test: - [ - "CMD", - "bun", - "-e", - "const value = Number(await Bun.file('/tmp/health').text()); const age = Date.now() - value; if (!Number.isFinite(age) || age < 0 || age > 30000) process.exit(1)", - ] - interval: 2s - timeout: 5s - retries: 20 diff --git a/compose.yml b/compose.yml new file mode 100644 index 0000000..6b6d88d --- /dev/null +++ b/compose.yml @@ -0,0 +1,66 @@ +services: + backend: + expose: + - "8000" + depends_on: + redis: + condition: service_healthy + engine: + condition: service_healthy + worker: + condition: service_healthy + healthcheck: + test: + [ + "CMD", + "bun", + "-e", + "const r = await fetch('http://localhost:8000/health'); if (!r.ok) process.exit(1)", + ] + interval: 30s + timeout: 10s + retries: 3 + start_period: 40s + + engine: + depends_on: + redis: + condition: service_healthy + healthcheck: + test: + [ + "CMD", + "bun", + "-e", + "const value = Number(await Bun.file('/tmp/health').text()); const age = Date.now() - value; if (!Number.isFinite(age) || age < 0 || age > 30000) process.exit(1)", + ] + interval: 15s + timeout: 5s + retries: 3 + start_period: 20s + + worker: + depends_on: + redis: + condition: service_healthy + healthcheck: + test: + [ + "CMD", + "bun", + "-e", + "const value = Number(await Bun.file('/tmp/health').text()); const age = Date.now() - value; if (!Number.isFinite(age) || age < 0 || age > 30000) process.exit(1)", + ] + interval: 15s + timeout: 5s + retries: 3 + start_period: 20s + + redis: + image: redis:7.2 + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 5s + retries: 3 + start_period: 10s diff --git a/scripts/compose.sh b/scripts/compose.sh new file mode 100755 index 0000000..511faa7 --- /dev/null +++ b/scripts/compose.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +set -euo pipefail + +repo_dir="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" +environment="${1:-}" + +if [[ -z "$environment" ]]; then + echo "Usage: scripts/compose.sh " >&2 + exit 2 +fi +shift + +case "$environment" in + dev | test | prod) ;; + *) + echo "Unknown environment: $environment (expected dev, test, or prod)" >&2 + exit 2 + ;; +esac + +env_file="$repo_dir/.env.$environment" +override_file="$repo_dir/compose.$environment.yml" +project_name="${PAPERDRILL_PROJECT:-paperdrill-$environment}" + +if [[ ! -f "$env_file" ]]; then + echo "Missing environment file: $env_file" >&2 + exit 1 +fi + +exec docker compose \ + --project-name "$project_name" \ + --env-file "$env_file" \ + --file "$repo_dir/compose.yml" \ + --file "$override_file" \ + "$@" diff --git a/scripts/database.sh b/scripts/database.sh new file mode 100755 index 0000000..f2d654e --- /dev/null +++ b/scripts/database.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +set -euo pipefail + +repo_dir="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" +environment="${1:-}" +action="${2:-}" +compose=("$repo_dir/scripts/compose.sh" "$environment") + +case "$environment" in + dev | test | prod) ;; + *) + echo "Usage: scripts/database.sh " >&2 + exit 2 + ;; +esac + +migrate() { + "${compose[@]}" run --rm --no-deps backend bun prisma migrate deploy +} + +seed() { + "${compose[@]}" run --rm --no-deps backend bun prisma db seed +} + +case "$action" in + migrate) + migrate + ;; + seed) + seed + ;; + prepare) + migrate + seed + ;; + *) + echo "Usage: scripts/database.sh " >&2 + exit 2 + ;; +esac diff --git a/scripts/deploy.sh b/scripts/deploy.sh new file mode 100755 index 0000000..c802f3d --- /dev/null +++ b/scripts/deploy.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -euo pipefail + +repo_dir="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" +compose=("$repo_dir/scripts/compose.sh" prod) + +"${compose[@]}" config --quiet +"${compose[@]}" pull backend engine worker +"$repo_dir/scripts/database.sh" prod prepare +"${compose[@]}" up -d --wait --remove-orphans diff --git a/scripts/images.sh b/scripts/images.sh new file mode 100755 index 0000000..41e2158 --- /dev/null +++ b/scripts/images.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +set -euo pipefail + +repo_dir="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" +action="${1:-}" +tag="${2:-}" +registry="ghcr.io/hars-21" +platform="linux/amd64" +services=(backend engine worker bot) + +if [[ "$action" != "build" && "$action" != "push" ]]; then + echo "Usage: scripts/images.sh " >&2 + exit 2 +fi + +if [[ -z "$tag" || "$tag" == "latest" ]]; then + echo "Use an explicit immutable release tag, for example v1.0.0 or sha-abc1234" >&2 + exit 2 +fi + +for service in "${services[@]}"; do + image="$registry/paperdrill-$service:$tag" + + if [[ "$action" == "build" ]]; then + build_args=(--platform "$platform" --pull --tag "$image") + if [[ "$service" != "bot" ]]; then + build_args+=(--target runner) + fi + docker build "${build_args[@]}" "$repo_dir/$service" + else + docker push "$image" + fi +done + +echo "$action complete for $registry/paperdrill-{backend,engine,worker,bot}:$tag" diff --git a/tests/run-integration.sh b/tests/run-integration.sh index f3068d7..43ec7d9 100644 --- a/tests/run-integration.sh +++ b/tests/run-integration.sh @@ -3,7 +3,7 @@ set -u compose() { - docker compose -p paperdrill-test -f compose.test.yml "$@" + ./scripts/compose.sh test "$@" } cleanup() { @@ -18,6 +18,14 @@ compose down --volumes --remove-orphans || true compose build || status=$? +if [ "$status" -eq 0 ]; then + compose up -d --wait postgres redis || status=$? +fi + +if [ "$status" -eq 0 ]; then + ./scripts/database.sh test prepare || status=$? +fi + if [ "$status" -eq 0 ]; then compose up -d --wait backend engine worker || status=$? fi @@ -39,7 +47,7 @@ fi if [ "$status" -ne 0 ]; then compose ps - compose logs --no-color backend engine worker postgres redis setup + compose logs --no-color backend engine worker fi -exit "$status" \ No newline at end of file +exit "$status" From 2de3992aebcdebd51f2ab634baf467bccad80d89 Mon Sep 17 00:00:00 2001 From: Harshil Date: Thu, 17 Sep 2026 19:25:51 +0530 Subject: [PATCH 06/20] perf: restructure benchmarks with new scripts --- benchmarks/api.bench.ts | 202 ----------------------------- benchmarks/engine.bench.ts | 193 ---------------------------- benchmarks/package.json | 4 +- benchmarks/run.sh | 3 +- benchmarks/src/client.ts | 55 ++++++++ benchmarks/src/config.ts | 29 +++++ benchmarks/src/index.ts | 255 +++++++++++++++++++++++++++++++++++++ benchmarks/src/metrics.ts | 74 +++++++++++ 8 files changed, 417 insertions(+), 398 deletions(-) delete mode 100644 benchmarks/api.bench.ts delete mode 100644 benchmarks/engine.bench.ts create mode 100644 benchmarks/src/client.ts create mode 100644 benchmarks/src/config.ts create mode 100644 benchmarks/src/index.ts create mode 100644 benchmarks/src/metrics.ts diff --git a/benchmarks/api.bench.ts b/benchmarks/api.bench.ts deleted file mode 100644 index 62e6bb0..0000000 --- a/benchmarks/api.bench.ts +++ /dev/null @@ -1,202 +0,0 @@ -import { performance } from "perf_hooks"; - -const API_URL = process.env.API_URL ?? "http://localhost:8000"; -const CONCURRENCY = parseInt(process.env.CONCURRENCY ?? "50", 10); -const DURATION_SEC = parseInt(process.env.DURATION_SEC ?? "10", 10); -const SERVICE_TOKEN = process.env.SERVICE_TOKEN; -const SERVICE_EMAIL = process.env.SERVICE_EMAIL; -const BENCHMARK_BALANCE = "1000000000000"; - -// --- Helpers --- - -function stats(times: number[]) { - if (times.length === 0) return { avg: 0, p50: 0, p95: 0, p99: 0, total: 0, count: 0 }; - times.sort((a, b) => a - b); - const sum = times.reduce((a, b) => a + b, 0); - const avg = sum / times.length; - const p50 = times[Math.floor(times.length * 0.5)]; - const p95 = times[Math.floor(times.length * 0.95)]; - const p99 = times[Math.floor(times.length * 0.99)]; - return { avg, p50, p95, p99, total: sum, count: times.length }; -} - -function report(name: string, times: number[], errors: number, elapsedMs: number) { - const s = stats(times); - const opsPerSec = elapsedMs === 0 ? 0 : (s.count / elapsedMs) * 1000; - const errRate = ((errors / (s.count + errors)) * 100).toFixed(1); - console.log(` ${name}`); - console.log( - ` ${s.count} ok, ${errors} err (${errRate}% err) | ${(elapsedMs / 1000).toFixed(1)}s`, - ); - console.log( - ` ${opsPerSec.toFixed(0)} req/sec | avg ${s.avg.toFixed(1)}ms | p50 ${s.p50?.toFixed(1)}ms | p95 ${s.p95?.toFixed(1)}ms | p99 ${s.p99?.toFixed(1)}ms`, - ); -} - -type RequestFn = () => Promise; - -async function runLoadTest(name: string, fn: RequestFn, durationMs: number) { - const times: number[] = []; - let errors = 0; - let running = true; - const startedAt = performance.now(); - - const worker = async () => { - while (running) { - const start = performance.now(); - const ok = await fn(); - const elapsed = performance.now() - start; - - if (ok) { - times.push(elapsed); - } else { - errors++; - } - } - }; - - const workers = Array.from({ length: CONCURRENCY }, () => worker()); - - const deadline = setTimeout(() => { - running = false; - }, durationMs); - - await Promise.all(workers); - clearTimeout(deadline); - - report(name, times, errors, performance.now() - startedAt); - return errors; -} - -// --- Load test scenarios --- - -async function benchMarkets() { - return runLoadTest( - "GET /v1/markets (public)", - async () => { - try { - const res = await fetch(`${API_URL}/v1/markets`); - return res.ok; - } catch { - return false; - } - }, - DURATION_SEC * 1000, - ); -} - -async function benchOrderbook() { - return runLoadTest( - "GET /v1/markets/SOL_USD/orderbook (public)", - async () => { - try { - const res = await fetch(`${API_URL}/v1/markets/SOL_USD/orderbook`); - return res.ok; - } catch { - return false; - } - }, - DURATION_SEC * 1000, - ); -} - -async function benchCreateOrder() { - return runLoadTest( - "POST /v1/orders (LIMIT, no match)", - async () => { - try { - const price = 50 + Math.floor(Math.random() * 10); - const res = await fetch(`${API_URL}/v1/orders`, { - method: "POST", - headers: { - "Content-Type": "application/json", - "X-Api-Key": SERVICE_TOKEN ?? "", - "X-Service-Email": SERVICE_EMAIL ?? "", - }, - body: JSON.stringify({ - type: "LIMIT", - side: "BUY", - symbol: "SOL_USD", - price: String(price), - qty: "0.1", - }), - }); - return res.ok; - } catch { - return false; - } - }, - DURATION_SEC * 1000, - ); -} - -async function seedServiceBalance() { - if (!SERVICE_TOKEN || !SERVICE_EMAIL) return; - - const res = await fetch(`${API_URL}/v1/deposits`, { - method: "POST", - headers: { - "Content-Type": "application/json", - "X-Api-Key": SERVICE_TOKEN, - "X-Service-Email": SERVICE_EMAIL, - }, - body: JSON.stringify({ asset: "USD", amount: BENCHMARK_BALANCE }), - }); - - if (!res.ok) { - throw new Error(`Failed to fund benchmark service account (${res.status}): ${await res.text()}`); - } -} - -async function benchTrades() { - return runLoadTest( - "GET /v1/markets/SOL_USD/trades (public)", - async () => { - try { - const res = await fetch(`${API_URL}/v1/markets/SOL_USD/trades?limit=50`); - return res.ok; - } catch { - return false; - } - }, - DURATION_SEC * 1000, - ); -} - -// --- Main --- - -console.log("=== API Load Tests ==="); -console.log(` URL: ${API_URL}`); -console.log(` Concurrency: ${CONCURRENCY}`); -console.log(` Duration: ${DURATION_SEC}s\n`); - -// Warm up — make sure backend is alive -try { - const res = await fetch(`${API_URL}/health`); - if (!res.ok) { - console.error(`Backend health check failed (${res.status}). Is it running?`); - process.exit(1); - } - console.log(" Backend is healthy\n"); -} catch (err) { - console.error(`Cannot reach backend at ${API_URL}. Is it running?`); - process.exit(1); -} - -let totalErrors = 0; - -totalErrors += await benchMarkets(); -console.log(); -totalErrors += await benchOrderbook(); -console.log(); -if (SERVICE_TOKEN && SERVICE_EMAIL) { - await seedServiceBalance(); - totalErrors += await benchCreateOrder(); - console.log(); -} else { - console.log(" POST /v1/orders skipped (no service token or email)\n"); -} -totalErrors += await benchTrades(); -console.log(); - -if (totalErrors > 0) process.exitCode = 1; diff --git a/benchmarks/engine.bench.ts b/benchmarks/engine.bench.ts deleted file mode 100644 index 099c76a..0000000 --- a/benchmarks/engine.bench.ts +++ /dev/null @@ -1,193 +0,0 @@ -import { performance } from "perf_hooks"; -import { resetState, placeOrder, cancelOrder } from "../engine/tests/utils"; -import { getDepth } from "../engine/src/modules/orderbook"; - -const N = 1000; - -function stats(times: number[]) { - times.sort((a, b) => a - b); - const sum = times.reduce((a, b) => a + b, 0); - const avg = sum / times.length; - const p50 = times[Math.floor(times.length * 0.5)]; - const p95 = times[Math.floor(times.length * 0.95)]; - const p99 = times[Math.floor(times.length * 0.99)]; - return { avg, p50, p95, p99, total: sum }; -} - -function report(name: string, times: number[]) { - const s = stats(times); - const opsPerSec = (times.length / s.total) * 1000; - console.log(` ${name}`); - console.log(` ${times.length} ops in ${s.total.toFixed(0)}ms`); - console.log( - ` ${opsPerSec.toFixed(0)} ops/sec | avg ${s.avg.toFixed(2)}ms | p50 ${s.p50?.toFixed(2)}ms | p95 ${s.p95?.toFixed(2)}ms | p99 ${s.p99?.toFixed(2)}ms`, - ); -} - -async function benchLimitNoMatch() { - resetState(); - const times: number[] = []; - - for (let i = 0; i < N; i++) { - const start = performance.now(); - await placeOrder({ - id: crypto.randomUUID(), - userId: "1", - side: "BUY", - type: "LIMIT", - symbol: "BTC_USD", - price: 60000n, - qty: 100n, - }); - times.push(performance.now() - start); - } - - report("Limit order (no match)", times); -} - -async function benchLimitMatch() { - resetState(); - const times: number[] = []; - - // Seed resting asks - for (let i = 0; i < N; i++) { - await placeOrder({ - id: crypto.randomUUID(), - userId: "1", - side: "SELL", - type: "LIMIT", - symbol: "BTC_USD", - price: 60000n + BigInt(i % 10), - qty: 100n, - }); - } - - // Cross with buys - for (let i = 0; i < N; i++) { - const start = performance.now(); - await placeOrder({ - id: crypto.randomUUID(), - userId: "2", - side: "BUY", - type: "LIMIT", - symbol: "BTC_USD", - price: 60000n + BigInt(i % 10), - qty: 100n, - }); - times.push(performance.now() - start); - } - - report("Limit order (with match)", times); -} - -async function benchMarketOrder() { - resetState(); - const times: number[] = []; - - // Seed resting asks - for (let i = 0; i < N; i++) { - await placeOrder({ - id: crypto.randomUUID(), - userId: "1", - side: "SELL", - type: "LIMIT", - symbol: "BTC_USD", - price: 60000n, - qty: 100n, - }); - } - - for (let i = 0; i < N; i++) { - const start = performance.now(); - await placeOrder({ - id: crypto.randomUUID(), - userId: "2", - side: "BUY", - type: "MARKET", - symbol: "BTC_USD", - price: null, - qty: 100n, - }); - times.push(performance.now() - start); - } - - report("Market order", times); -} - -async function benchGetDepth() { - resetState(); - - // Populate book - for (let i = 0; i < 50; i++) { - await placeOrder({ - id: crypto.randomUUID(), - userId: "1", - side: "SELL", - type: "LIMIT", - symbol: "BTC_USD", - price: 60000n + BigInt(i), - qty: 100n, - }); - await placeOrder({ - id: crypto.randomUUID(), - userId: "2", - side: "BUY", - type: "LIMIT", - symbol: "BTC_USD", - price: 59999n - BigInt(i), - qty: 100n, - }); - } - - const times: number[] = []; - for (let i = 0; i < N * 5; i++) { - const start = performance.now(); - await getDepth("BTC_USD"); - times.push(performance.now() - start); - } - - report("getDepth (100 levels each side)", times); -} - -async function benchCancelOrder() { - resetState(); - const times: number[] = []; - const ids: string[] = []; - - // Place orders - for (let i = 0; i < N; i++) { - const id = crypto.randomUUID(); - await placeOrder({ - id, - userId: "1", - side: "BUY", - type: "LIMIT", - symbol: "BTC_USD", - price: 50000n + BigInt(i), - qty: 100n, - }); - ids.push(id); - } - - // Cancel them - for (const id of ids) { - const start = performance.now(); - await cancelOrder("1", id); - times.push(performance.now() - start); - } - - report("cancelOrder", times); -} - -console.log("=== Engine Benchmarks ===\n"); - -await benchLimitNoMatch(); -console.log(); -await benchLimitMatch(); -console.log(); -await benchMarketOrder(); -console.log(); -await benchGetDepth(); -console.log(); -await benchCancelOrder(); -console.log(); diff --git a/benchmarks/package.json b/benchmarks/package.json index fd91f25..44795a8 100644 --- a/benchmarks/package.json +++ b/benchmarks/package.json @@ -2,8 +2,8 @@ "name": "benchmarks", "private": true, "scripts": { - "bench:api": "bun run api.bench.ts", - "bench:engine": "bun run engine.bench.ts" + "bench": "bun run src/index.ts", + "typecheck": "bunx tsc --noEmit" }, "devDependencies": { "@types/bun": "latest" diff --git a/benchmarks/run.sh b/benchmarks/run.sh index df3a0ce..12a3c80 100644 --- a/benchmarks/run.sh +++ b/benchmarks/run.sh @@ -41,9 +41,10 @@ if [ "$status" -eq 0 ]; then fi export API_URL="http://127.0.0.1:8001" + export WS_URL="ws://127.0.0.1:8001" cd benchmarks - bun run bench:api + bun run bench ) || status=$? fi diff --git a/benchmarks/src/client.ts b/benchmarks/src/client.ts new file mode 100644 index 0000000..73dbdd5 --- /dev/null +++ b/benchmarks/src/client.ts @@ -0,0 +1,55 @@ +import { config } from "./config"; + +export interface OrderInput { + type: "LIMIT" | "MARKET"; + side: "BUY" | "SELL"; + symbol: string; + price?: string; + qty: string; +} + +export interface OrderResponse { + id?: string; + status?: "OPEN" | "PARTIALLY_FILLED" | "FILLED" | "CANCELLED"; + filledQty?: string; + error?: string; +} + +export class ServiceClient { + constructor(private readonly email: string) {} + + private headers() { + return { + "content-type": "application/json", + "x-api-key": config.serviceToken, + "x-service-email": this.email, + }; + } + + async deposit(asset: string, amount: string) { + const response = await fetch(`${config.apiUrl}/v1/deposits`, { + method: "POST", + headers: this.headers(), + body: JSON.stringify({ asset, amount }), + }); + + if (!response.ok) { + throw new Error(`Deposit failed for ${this.email}: ${response.status} ${await response.text()}`); + } + } + + async createOrder(input: OrderInput): Promise<{ ok: boolean; data: OrderResponse }> { + const response = await fetch(`${config.apiUrl}/v1/orders`, { + method: "POST", + headers: this.headers(), + body: JSON.stringify(input), + }); + const data = (await response.json()) as OrderResponse; + return { ok: response.ok, data }; + } +} + +export async function assertHealthy() { + const response = await fetch(`${config.apiUrl}/health`); + if (!response.ok) throw new Error(`Backend health check failed with ${response.status}`); +} diff --git a/benchmarks/src/config.ts b/benchmarks/src/config.ts new file mode 100644 index 0000000..0034d32 --- /dev/null +++ b/benchmarks/src/config.ts @@ -0,0 +1,29 @@ +function positiveInteger(name: string, fallback: number): number { + const value = Number(process.env[name] ?? fallback); + if (!Number.isInteger(value) || value <= 0) { + throw new Error(`${name} must be a positive integer`); + } + return value; +} + +function required(name: string): string { + const value = process.env[name]?.trim(); + if (!value) throw new Error(`${name} is required`); + return value; +} + +if (process.env.NODE_ENV !== "test") { + throw new Error("Benchmarks only run against the isolated test environment"); +} + +export const config = { + apiUrl: required("API_URL").replace(/\/$/, ""), + wsUrl: required("WS_URL").replace(/\/$/, ""), + serviceToken: required("SERVICE_TOKEN"), + httpConcurrency: positiveInteger("HTTP_CONCURRENCY", 50), + unmatchedOrders: positiveInteger("UNMATCHED_ORDERS", 10_000), + matchedOrders: positiveInteger("MATCHED_ORDERS", 5_000), + sweepOrders: positiveInteger("SWEEP_ORDERS", 1_000), + websocketConnections: positiveInteger("WS_CONNECTIONS", 500), + websocketTimeoutMs: positiveInteger("WS_TIMEOUT_MS", 5_000), +}; diff --git a/benchmarks/src/index.ts b/benchmarks/src/index.ts new file mode 100644 index 0000000..bf74cad --- /dev/null +++ b/benchmarks/src/index.ts @@ -0,0 +1,255 @@ +import { arch, cpus, platform } from "node:os"; +import { performance } from "node:perf_hooks"; +import { ServiceClient, assertHealthy } from "./client"; +import { config } from "./config"; +import { percentile, printLoad, requireSuccessful, runLoad } from "./metrics"; + +const FUNDING_AMOUNT = "1000000000000000"; +const btcService = new ServiceClient("btc.service@paperdrill.dev"); +const solService = new ServiceClient("sol.service@paperdrill.dev"); +const ethService = new ServiceClient("eth.service@paperdrill.dev"); + +async function seedOrders(options: { + client: ServiceClient; + total: number; + input: (index: number) => Parameters[0]; +}) { + const result = await runLoad({ + total: options.total, + concurrency: config.httpConcurrency, + operation: async (index) => (await options.client.createOrder(options.input(index))).ok, + }); + requireSuccessful("Liquidity setup", result); +} + +async function benchmarkUnmatchedOrders() { + await btcService.deposit("USD", FUNDING_AMOUNT); + + const result = await runLoad({ + total: config.unmatchedOrders, + concurrency: config.httpConcurrency, + operation: async (index) => { + const cents = 100 + (index % 100); + const response = await btcService.createOrder({ + type: "LIMIT", + side: "BUY", + symbol: "BTC_USD", + price: `${Math.floor(cents / 100)}.${String(cents % 100).padStart(2, "0")}`, + qty: "0.0100", + }); + return response.ok && response.data.status === "OPEN"; + }, + }); + + printLoad("Unmatched limit-order load", result); + requireSuccessful("Unmatched order benchmark", result); +} + +async function benchmarkMatchedOrders() { + await Promise.all([ + solService.deposit("SOL", FUNDING_AMOUNT), + btcService.deposit("USD", FUNDING_AMOUNT), + ]); + + await seedOrders({ + client: solService, + total: config.matchedOrders, + input: () => ({ + type: "LIMIT", + side: "SELL", + symbol: "SOL_USD", + price: "100.00", + qty: "0.01", + }), + }); + + const result = await runLoad({ + total: config.matchedOrders, + concurrency: config.httpConcurrency, + operation: async () => { + const response = await btcService.createOrder({ + type: "LIMIT", + side: "BUY", + symbol: "SOL_USD", + price: "100.00", + qty: "0.01", + }); + return response.ok && response.data.status === "FILLED" && response.data.filledQty === "0.01"; + }, + }); + + printLoad("Matched limit-order load (one fill per taker)", result, "matched orders"); + requireSuccessful("Matched order benchmark", result); +} + +async function benchmarkSweep() { + await Promise.all([ + ethService.deposit("ETH", FUNDING_AMOUNT), + btcService.deposit("USD", FUNDING_AMOUNT), + ]); + + await seedOrders({ + client: ethService, + total: config.sweepOrders, + input: () => ({ + type: "LIMIT", + side: "SELL", + symbol: "ETH_USD", + price: "200.00", + qty: "0.001", + }), + }); + + const quantity = (config.sweepOrders / 1000).toFixed(3); + const startedAt = performance.now(); + const response = await btcService.createOrder({ + type: "LIMIT", + side: "BUY", + symbol: "ETH_USD", + price: "200.00", + qty: quantity, + }); + const elapsedMs = performance.now() - startedAt; + + if (!response.ok || response.data.status !== "FILLED" || response.data.filledQty !== quantity) { + throw new Error(`Sweep failed: ${JSON.stringify(response.data)}`); + } + + console.log(`\nSingle-order sweep`); + console.log(` ${config.sweepOrders} maker orders matched in ${elapsedMs.toFixed(1)} ms`); +} + +async function waitFor( + condition: () => boolean, + timeoutMs: number, + description: string, +): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (condition()) return; + await Bun.sleep(10); + } + throw new Error(`Timed out waiting for ${description}`); +} + +async function benchmarkWebSockets() { + const sockets: WebSocket[] = []; + const probeRecipients = new Set(); + const deliveryLatencies = new Map(); + let targetPrice = ""; + let measuring = false; + let deliveryStartedAt = 0; + + const connectStartedAt = performance.now(); + await Promise.all( + Array.from( + { length: config.websocketConnections }, + (_, index) => + new Promise((resolve, reject) => { + const socket = new WebSocket(config.wsUrl); + sockets.push(socket); + + const timeout = setTimeout( + () => reject(new Error(`WebSocket ${index} connection timed out`)), + config.websocketTimeoutMs, + ); + + socket.addEventListener("open", () => { + clearTimeout(timeout); + socket.send(JSON.stringify({ method: "SUBSCRIBE", params: ["depth:BTC_USD"] })); + resolve(); + }); + socket.addEventListener("error", () => { + clearTimeout(timeout); + reject(new Error(`WebSocket ${index} failed to connect`)); + }); + socket.addEventListener("message", (event) => { + try { + const message = JSON.parse(String(event.data)) as { + bids?: Array<{ price: string }>; + }; + if (!message.bids?.some((level) => level.price === targetPrice)) return; + + if (measuring) { + if (!deliveryLatencies.has(index)) { + deliveryLatencies.set(index, performance.now() - deliveryStartedAt); + } + } else { + probeRecipients.add(index); + } + } catch { + // Ignore unrelated or malformed messages during the benchmark. + } + }); + }), + ), + ); + const connectMs = performance.now() - connectStartedAt; + + for (let attempt = 0; attempt < 10 && probeRecipients.size < sockets.length; attempt++) { + targetPrice = `${10 + attempt}.00`; + await btcService.createOrder({ + type: "LIMIT", + side: "BUY", + symbol: "BTC_USD", + price: targetPrice, + qty: "1.0000", + }); + try { + await waitFor(() => probeRecipients.size === sockets.length, 500, "WebSocket subscriptions"); + } catch { + // A later probe gives subscriptions that were still being registered another chance. + } + } + + if (probeRecipients.size !== sockets.length) { + for (const socket of sockets) socket.close(); + throw new Error(`Only ${probeRecipients.size}/${sockets.length} WebSockets subscribed`); + } + + targetPrice = "25.00"; + measuring = true; + deliveryStartedAt = performance.now(); + const trigger = await btcService.createOrder({ + type: "LIMIT", + side: "BUY", + symbol: "BTC_USD", + price: targetPrice, + qty: "1.0000", + }); + if (!trigger.ok) + throw new Error(`WebSocket trigger order failed: ${JSON.stringify(trigger.data)}`); + + await waitFor( + () => deliveryLatencies.size === sockets.length, + config.websocketTimeoutMs, + "WebSocket depth delivery", + ); + + console.log(`\nWebSocket fan-out`); + console.log( + ` ${sockets.length} connections opened in ${connectMs.toFixed(1)} ms | ${deliveryLatencies.size}/${sockets.length} received update | p95 ${percentile([...deliveryLatencies.values()], 0.95).toFixed(1)} ms`, + ); + + for (const socket of sockets) socket.close(); +} + +async function main() { + await assertHealthy(); + + const cpu = cpus()[0]?.model ?? "unknown CPU"; + console.log("PaperDrill v1 benchmark"); + console.log(` Bun ${Bun.version} | ${platform()} ${arch()} | ${cpus().length} cores | ${cpu}`); + console.log( + ` HTTP concurrency ${config.httpConcurrency} | ${config.unmatchedOrders} unmatched | ${config.matchedOrders} matched | ${config.websocketConnections} WebSockets`, + ); + + await benchmarkUnmatchedOrders(); + await benchmarkMatchedOrders(); + await benchmarkSweep(); + await benchmarkWebSockets(); + + console.log("\nAll benchmark correctness checks passed."); +} + +await main(); diff --git a/benchmarks/src/metrics.ts b/benchmarks/src/metrics.ts new file mode 100644 index 0000000..f94ab74 --- /dev/null +++ b/benchmarks/src/metrics.ts @@ -0,0 +1,74 @@ +import { performance } from "node:perf_hooks"; + +interface LoadOptions { + total: number; + concurrency: number; + operation: (index: number) => Promise; +} + +export interface LoadResult { + total: number; + succeeded: number; + failed: number; + elapsedMs: number; + operationsPerSecond: number; + p95Ms: number; +} + +export function percentile(samples: number[], value: number): number { + if (samples.length === 0) return 0; + const sorted = samples.toSorted((a, b) => a - b); + const index = Math.min(sorted.length - 1, Math.ceil(sorted.length * value) - 1); + return sorted[Math.max(0, index)] ?? 0; +} + +export async function runLoad(options: LoadOptions): Promise { + const latencies: number[] = []; + let nextIndex = 0; + let succeeded = 0; + let failed = 0; + const startedAt = performance.now(); + + async function worker() { + for (;;) { + const index = nextIndex++; + if (index >= options.total) return; + + const requestStartedAt = performance.now(); + try { + if (await options.operation(index)) succeeded++; + else failed++; + } catch { + failed++; + } + latencies.push(performance.now() - requestStartedAt); + } + } + + await Promise.all( + Array.from({ length: Math.min(options.concurrency, options.total) }, () => worker()), + ); + + const elapsedMs = performance.now() - startedAt; + return { + total: options.total, + succeeded, + failed, + elapsedMs, + operationsPerSecond: (succeeded / elapsedMs) * 1000, + p95Ms: percentile(latencies, 0.95), + }; +} + +export function printLoad(label: string, result: LoadResult, unit = "orders") { + console.log(`\n${label}`); + console.log( + ` ${result.succeeded}/${result.total} succeeded | ${result.operationsPerSecond.toFixed(0)} ${unit}/sec | p95 ${result.p95Ms.toFixed(1)} ms`, + ); +} + +export function requireSuccessful(label: string, result: LoadResult) { + if (result.failed > 0) { + throw new Error(`${label} failed ${result.failed} of ${result.total} operations`); + } +} From e200c7295327f65693e2dd261bef8500ef50d728 Mon Sep 17 00:00:00 2001 From: Harshil Date: Thu, 17 Sep 2026 19:53:59 +0530 Subject: [PATCH 07/20] test: add integration tests for websocket --- tests/integration/helpers/websocket.ts | 67 ++++++++++++ tests/integration/setup/env.ts | 1 + tests/integration/websocket/events.test.ts | 116 +++++++++++++++++++++ tests/run-integration.sh | 1 + 4 files changed, 185 insertions(+) create mode 100644 tests/integration/helpers/websocket.ts create mode 100644 tests/integration/websocket/events.test.ts diff --git a/tests/integration/helpers/websocket.ts b/tests/integration/helpers/websocket.ts new file mode 100644 index 0000000..f0f8889 --- /dev/null +++ b/tests/integration/helpers/websocket.ts @@ -0,0 +1,67 @@ +import { env } from "../setup/env"; +import { waitFor } from "./wait-for"; + +export class WebSocketProbe { + private readonly messages: unknown[] = []; + + private constructor(readonly socket: WebSocket) { + socket.addEventListener("message", (event) => { + try { + this.messages.push(JSON.parse(String(event.data))); + } catch {} + }); + } + + static async connect(): Promise { + const socket = new WebSocket(env.wsUrl); + const probe = new WebSocketProbe(socket); + + await new Promise((resolve, reject) => { + const timeout = setTimeout(() => reject(new Error("Timed out opening WebSocket")), 5000); + socket.addEventListener( + "open", + () => { + clearTimeout(timeout); + resolve(); + }, + { once: true }, + ); + socket.addEventListener( + "error", + () => { + clearTimeout(timeout); + reject(new Error("WebSocket connection failed")); + }, + { once: true }, + ); + }); + + return probe; + } + + async subscribe(channel: string): Promise { + this.socket.send(JSON.stringify({ method: "SUBSCRIBE", params: [channel] })); + + this.socket.send("{"); + await this.next<{ error: string }>( + (message) => + typeof message === "object" && + message !== null && + "error" in message && + message.error === "Invalid JSON", + `subscription to ${channel}`, + ); + } + + next(accept: (message: unknown) => boolean, description: string): Promise { + return waitFor( + async () => this.messages.find(accept), + (message) => message !== undefined, + { description, timeoutMs: 8000, intervalMs: 25 }, + ) as Promise; + } + + close() { + this.socket.close(); + } +} diff --git a/tests/integration/setup/env.ts b/tests/integration/setup/env.ts index 0159708..ab8c35c 100644 --- a/tests/integration/setup/env.ts +++ b/tests/integration/setup/env.ts @@ -22,6 +22,7 @@ if (redisUrl.protocol !== "redis:" && redisUrl.protocol !== "rediss:") { export const env = { apiBaseUrl: required("API_BASE_URL").replace(/\/$/, ""), + wsUrl: required("WS_URL").replace(/\/$/, ""), databaseUrl: databaseUrl.toString(), redisUrl: redisUrl.toString(), incomingStream: required("INCOMING_STREAM"), diff --git a/tests/integration/websocket/events.test.ts b/tests/integration/websocket/events.test.ts new file mode 100644 index 0000000..2e7281a --- /dev/null +++ b/tests/integration/websocket/events.test.ts @@ -0,0 +1,116 @@ +import { beforeAll, expect, test } from "bun:test"; +import { waitForSystemReady } from "../helpers/api"; +import { EngineClient } from "../helpers/engine"; +import { createTestUser, seedBalance, type TestUser } from "../helpers/users"; +import { WebSocketProbe } from "../helpers/websocket"; + +beforeAll(waitForSystemReady); + +test("a real orderbook change reaches depth subscribers in the public format", async () => { + const probe = await WebSocketProbe.connect(); + const user = await createTestUser("websocket-depth"); + let orderId: string | undefined; + + try { + await probe.subscribe("depth:ETH_USD"); + const depthMessage = probe.next>((message) => { + if (typeof message !== "object" || message === null) return false; + const event = message as { event?: string; symbol?: string; bids?: unknown[] }; + return ( + event.event === "depth" && + event.symbol === "ETH_USD" && + event.bids?.some( + (level) => + typeof level === "object" && + level !== null && + "price" in level && + level.price === "123.45", + ) === true + ); + }, "formatted ETH depth update"); + + const created = await user.client.createOrder({ + type: "LIMIT", + side: "BUY", + symbol: "ETH_USD", + price: "123.45", + qty: "0.250", + }); + expect(created).toMatchObject({ status: 201, data: { status: "OPEN" } }); + orderId = created.data.id; + + expect(await depthMessage).toMatchObject({ + event: "depth", + symbol: "ETH_USD", + bids: [{ price: "123.45", qty: "0.250" }], + asks: [], + lastUpdateId: expect.any(Number), + timestamp: expect.any(Number), + }); + } finally { + if (orderId) await user.client.cancelOrder(orderId); + probe.close(); + } +}); + +test("a real match reaches trade subscribers with formatted price and quantity", async () => { + const probe = await WebSocketProbe.connect(); + const engine = new EngineClient(); + let seller: TestUser | undefined; + let sellOrderId: string | undefined; + + try { + await engine.connect(); + const buyer = await createTestUser("websocket-trade-buyer"); + seller = await createTestUser("websocket-trade-seller"); + await seedBalance(engine, seller.id, "ETH", "1000"); + await probe.subscribe("trade:ETH_USD"); + + const tradeMessage = probe.next>( + (message) => + typeof message === "object" && + message !== null && + "event" in message && + message.event === "trade" && + "symbol" in message && + message.symbol === "ETH_USD", + "ETH trade update", + ); + + const sell = await seller.client.createOrder({ + type: "LIMIT", + side: "SELL", + symbol: "ETH_USD", + price: "321.00", + qty: "0.250", + }); + expect(sell).toMatchObject({ status: 201, data: { status: "OPEN" } }); + sellOrderId = sell.data.id; + + const buy = await buyer.client.createOrder({ + type: "LIMIT", + side: "BUY", + symbol: "ETH_USD", + price: "321.00", + qty: "0.250", + }); + expect(buy).toMatchObject({ status: 201, data: { status: "FILLED" } }); + + expect(await tradeMessage).toMatchObject({ + event: "trade", + symbol: "ETH_USD", + id: expect.any(String), + price: "321.00", + qty: "0.250", + maker: false, + timestamp: expect.any(Number), + }); + sellOrderId = undefined; + } finally { + if (sellOrderId && seller) { + await seller.client.cancelOrder(sellOrderId); + } + await engine.close(); + probe.close(); + } +}); diff --git a/tests/run-integration.sh b/tests/run-integration.sh index 43ec7d9..92435ee 100644 --- a/tests/run-integration.sh +++ b/tests/run-integration.sh @@ -37,6 +37,7 @@ if [ "$status" -eq 0 ]; then set +a export API_BASE_URL="http://127.0.0.1:8001" + export WS_URL="ws://127.0.0.1:8001" export DATABASE_URL="postgresql://postgres:password@127.0.0.1:5434/paperdrill_test" export REDIS_URL="redis://127.0.0.1:6381" From 60efcb03264bd291b80127de32dc53d9e79b2f26 Mon Sep 17 00:00:00 2001 From: Harshil Date: Thu, 17 Sep 2026 21:03:00 +0530 Subject: [PATCH 08/20] ci: unified workflow for running unit and integration tests --- .env.test | 25 +++++++++++++++++ .github/workflows/{ci.yml => test.yml} | 38 ++++++++++++++++++-------- .gitignore | 1 + 3 files changed, 52 insertions(+), 12 deletions(-) create mode 100644 .env.test rename .github/workflows/{ci.yml => test.yml} (52%) diff --git a/.env.test b/.env.test new file mode 100644 index 0000000..83e4dd8 --- /dev/null +++ b/.env.test @@ -0,0 +1,25 @@ +NODE_ENV=test +PORT=8000 +CORS_ORIGIN=http://localhost:3000 +WS_URL=ws://127.0.0.1:8001 + +DATABASE_URL=postgresql://postgres:password@postgres:5432/paperdrill_test +REDIS_URL=redis://redis:6379 + +JWT_SECRET=paperdrill-integration-tests-only +SERVICE_TOKEN=YzW0PrwwnzD30jyadkz475WiuPAM+3W/LF8umuM/AOc= +SERVICE_TOKEN_HASH=83b9b13ea2ca0050245a4f8070d884968e71cb4e477436f36f305939d39768a4 +SERVICE_EMAIL=sol.service@paperdrill.dev + +INCOMING_STREAM=backend-to-engine-broker +BACKEND_QUEUE_ID=integration-tests +ENGINE_TIMEOUT_MS=5000 + +RATE_LIMIT_ENABLED=false +LOG_LEVEL=info + +STREAM_RETENTION_MS=1200000 +SNAPSHOT_INTERVAL=60000 +STREAM_READ_BATCH_SIZE=200 +RESPONSE_BATCH_SIZE=50 +FLUSH_INTERVAL_MS=250 diff --git a/.github/workflows/ci.yml b/.github/workflows/test.yml similarity index 52% rename from .github/workflows/ci.yml rename to .github/workflows/test.yml index bc4e252..419288e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/test.yml @@ -1,25 +1,18 @@ -name: CI +name: Tests on: push: pull_request: + workflow_dispatch: permissions: contents: read jobs: test: - name: Tests + name: Unit tests runs-on: ubuntu-latest timeout-minutes: 10 - env: - NODE_ENV: test - CORS_ORIGIN: http://localhost:3000 - REDIS_URL: redis://localhost:6379 - DATABASE_URL: postgresql://paperdrill:paperdrill@localhost:5432/paperdrill - JWT_SECRET: ci-jwt-secret - LOG_LEVEL: info - RATE_LIMIT_ENABLED: "false" steps: - name: Checkout repository @@ -42,5 +35,26 @@ jobs: working-directory: engine run: bun install --frozen-lockfile - - name: Run tests - run: bun test + - name: Run unit tests + run: make test-unit + + integration: + name: Integration tests + runs-on: ubuntu-latest + timeout-minutes: 15 + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Bun + uses: oven-sh/setup-bun@v2 + with: + bun-version: 1.3.3 + + - name: Install integration test dependencies + working-directory: tests + run: bun install --frozen-lockfile + + - name: Run integration tests + run: make test-integration diff --git a/.gitignore b/.gitignore index 5e7d2ce..42e3834 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json .env .env.* !.env.example +!.env.test # caches .eslintcache From 217860c3c0d0e78bec64cf09428dc4febe16dae6 Mon Sep 17 00:00:00 2001 From: Harshil Date: Fri, 18 Sep 2026 11:28:38 +0530 Subject: [PATCH 09/20] feat: implement daily credit system --- .../migration.sql | 25 +++++ backend/prisma/schema.prisma | 22 ++++ backend/src/controllers/user.ts | 13 ++- backend/src/routes/user.ts | 13 ++- backend/src/services/dailyCredit.ts | 106 ++++++++++++++++++ backend/src/types/engine.ts | 1 + engine/src/core/dispatcher.ts | 2 + engine/src/handlers/applyCredit.ts | 13 +++ engine/src/modules/balance.ts | 13 ++- engine/src/schema/index.ts | 4 + engine/src/store.ts | 1 + engine/src/types/request.ts | 1 + engine/src/util/snapshot.ts | 10 +- engine/tests/daily-credit.test.ts | 25 +++++ engine/tests/utils.ts | 11 +- frontend/src/context/AuthContext.tsx | 14 +++ frontend/src/hooks/use-balance.ts | 5 + frontend/src/lib/api.ts | 16 ++- frontend/src/pages/dashboard/balances.tsx | 25 +++-- frontend/src/pages/dashboard/overview.tsx | 32 ++++-- frontend/src/types/index.ts | 8 ++ tests/integration/flows/daily-credit.test.ts | 60 ++++++++++ tests/integration/helpers/api.ts | 24 ++++ tests/integration/setup/database.ts | 25 +++++ 24 files changed, 438 insertions(+), 31 deletions(-) create mode 100644 backend/prisma/migrations/20260917130000_add_daily_credit/migration.sql create mode 100644 backend/src/services/dailyCredit.ts create mode 100644 engine/src/handlers/applyCredit.ts create mode 100644 engine/tests/daily-credit.test.ts create mode 100644 tests/integration/flows/daily-credit.test.ts diff --git a/backend/prisma/migrations/20260917130000_add_daily_credit/migration.sql b/backend/prisma/migrations/20260917130000_add_daily_credit/migration.sql new file mode 100644 index 0000000..106da68 --- /dev/null +++ b/backend/prisma/migrations/20260917130000_add_daily_credit/migration.sql @@ -0,0 +1,25 @@ +-- CreateEnum +CREATE TYPE "DailyCreditStatus" AS ENUM ('PENDING', 'APPLIED'); + +-- CreateTable +CREATE TABLE "DailyCredit" ( + "id" TEXT NOT NULL, + "userId" TEXT NOT NULL, + "creditDate" DATE NOT NULL, + "asset" TEXT NOT NULL, + "amount" BIGINT NOT NULL, + "status" "DailyCreditStatus" NOT NULL DEFAULT 'PENDING', + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "appliedAt" TIMESTAMP(3), + + CONSTRAINT "DailyCredit_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE INDEX "DailyCredit_status_createdAt_idx" ON "DailyCredit"("status", "createdAt"); + +-- CreateIndex +CREATE UNIQUE INDEX "DailyCredit_userId_creditDate_key" ON "DailyCredit"("userId", "creditDate"); + +-- AddForeignKey +ALTER TABLE "DailyCredit" ADD CONSTRAINT "DailyCredit_userId_fkey" FOREIGN KEY ("userId") REFERENCES "User"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/backend/prisma/schema.prisma b/backend/prisma/schema.prisma index 89ebcaa..bf4a8c2 100644 --- a/backend/prisma/schema.prisma +++ b/backend/prisma/schema.prisma @@ -36,6 +36,11 @@ enum UserType { SERVICE } +enum DailyCreditStatus { + PENDING + APPLIED +} + model User { id String @id @default(uuid()) email String @unique @@ -53,6 +58,23 @@ model User { buyFills Fill[] @relation("FillBuyer") sellFills Fill[] @relation("FillSeller") leaderboardEntry LeaderboardEntry? + dailyCredits DailyCredit[] +} + +model DailyCredit { + id String @id @default(uuid()) + userId String + creditDate DateTime @db.Date + asset String + amount BigInt + status DailyCreditStatus @default(PENDING) + createdAt DateTime @default(now()) + appliedAt DateTime? + + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + + @@unique([userId, creditDate]) + @@index([status, createdAt]) } model LeaderboardEntry { diff --git a/backend/src/controllers/user.ts b/backend/src/controllers/user.ts index 83c8425..e21e4d6 100644 --- a/backend/src/controllers/user.ts +++ b/backend/src/controllers/user.ts @@ -10,6 +10,7 @@ import { calculatePortfolio, type Balance } from "../utils/portfolio"; import { getReferencePrices } from "../utils/referencePrice"; import { config } from "../config"; import { ApiError, sendApiError, sendEngineError } from "../utils/apiError"; +import { claimDailyCredit } from "../services/dailyCredit"; export function getUserId(req: Request): string { const userId = req.principal?.userId; @@ -143,7 +144,12 @@ export async function getPortfolio(req: Request, res: Response) { res.status(200).json(formatPortfolio(portfolio, user.pnlBaseline, user.pnlBaselineAt)); } catch (error) { logger.error("Failed to calculate portfolio", error); - sendApiError(res, 503, "PORTFOLIO_UNAVAILABLE", "Portfolio valuation is temporarily unavailable"); + sendApiError( + res, + 503, + "PORTFOLIO_UNAVAILABLE", + "Portfolio valuation is temporarily unavailable", + ); } } @@ -169,3 +175,8 @@ export async function createDeposit(req: Request, res: Response) { .status(200) .json(formatBalance(engineResponse.data as Record>)); } + +export async function claimDailyReward(req: Request, res: Response) { + const reward = await claimDailyCredit(getUserId(req)); + res.status(200).json(reward); +} diff --git a/backend/src/routes/user.ts b/backend/src/routes/user.ts index e59081a..6feab5e 100644 --- a/backend/src/routes/user.ts +++ b/backend/src/routes/user.ts @@ -1,7 +1,13 @@ import { Router } from "express"; import { requireAccess } from "../middleware/auth"; import { asyncHandler } from "../utils/asyncHandler"; -import { createDeposit, getBalance, getPortfolio, getUserData } from "../controllers/user"; +import { + claimDailyReward, + createDeposit, + getBalance, + getPortfolio, + getUserData, +} from "../controllers/user"; import { getTradeHistory } from "../controllers/user"; export const userRouter = Router(); @@ -27,3 +33,8 @@ userRouter.get( asyncHandler(getPortfolio), ); userRouter.post("/deposits", requireAccess({ types: ["service"] }), asyncHandler(createDeposit)); +userRouter.post( + "/users/me/daily-credit", + requireAccess({ types: ["session"] }), + asyncHandler(claimDailyReward), +); diff --git a/backend/src/services/dailyCredit.ts b/backend/src/services/dailyCredit.ts new file mode 100644 index 0000000..2631845 --- /dev/null +++ b/backend/src/services/dailyCredit.ts @@ -0,0 +1,106 @@ +import { Prisma } from "../../generated/prisma/client"; +import { DailyCreditStatus, UserType } from "../../generated/prisma/enums"; +import { prisma } from "../db"; +import { assetPrecision } from "../store/market"; +import { ApiError } from "../utils/apiError"; +import { fromBigInt, toBigInt } from "../utils/convert"; +import { sendToEngine } from "../utils/engineClient"; + +export const DAILY_CREDIT_ASSET = "USD"; +export const DAILY_CREDIT_AMOUNT = "100"; + +export function utcCreditDate(now = new Date()) { + return new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate())); +} + +export function nextUtcCreditDate(creditDate: Date) { + return new Date(creditDate.getTime() + 24 * 60 * 60 * 1000); +} + +async function findOrCreateCredit(userId: string, creditDate: Date, amount: bigint) { + try { + return await prisma.dailyCredit.create({ + data: { userId, creditDate, asset: DAILY_CREDIT_ASSET, amount }, + }); + } catch (error) { + if (!(error instanceof Prisma.PrismaClientKnownRequestError) || error.code !== "P2002") { + throw error; + } + + return prisma.dailyCredit.findUniqueOrThrow({ + where: { userId_creditDate: { userId, creditDate } }, + }); + } +} + +export async function claimDailyCredit(userId: string, now = new Date()) { + const user = await prisma.user.findUnique({ + where: { id: userId }, + select: { type: true, emailVerified: true, pnlBaseline: true }, + }); + + if (!user) throw new ApiError(401, "AUTHENTICATION_REQUIRED", "Authentication required"); + if (user.type !== UserType.USER) { + throw new ApiError(403, "DAILY_CREDIT_NOT_ELIGIBLE", "Service accounts are not eligible"); + } + if (!user.emailVerified) { + throw new ApiError(403, "EMAIL_NOT_VERIFIED", "Verify your email to claim daily credit"); + } + if (user.pnlBaseline == null) { + throw new ApiError(503, "DAILY_CREDIT_UNAVAILABLE", "PnL baseline is not configured"); + } + + const precision = assetPrecision.get(DAILY_CREDIT_ASSET); + if (precision == null) { + throw new ApiError(503, "DAILY_CREDIT_UNAVAILABLE", "Daily credit is temporarily unavailable"); + } + + const creditDate = utcCreditDate(now); + const configuredAmount = toBigInt(DAILY_CREDIT_AMOUNT, precision); + const credit = await findOrCreateCredit(userId, creditDate, configuredAmount); + + if (credit.status === DailyCreditStatus.APPLIED) { + return { + credited: false, + asset: credit.asset, + amount: fromBigInt(credit.amount, precision), + creditDate: creditDate.toISOString().slice(0, 10), + nextEligibleAt: nextUtcCreditDate(creditDate).toISOString(), + }; + } + + const engineResponse = await sendToEngine("apply_credit", { + creditId: credit.id, + userId, + asset: credit.asset, + amount: credit.amount.toString(), + }); + + if (!engineResponse.success) { + throw new ApiError(503, "DAILY_CREDIT_UNAVAILABLE", "Daily credit is temporarily unavailable"); + } + + const credited = await prisma.$transaction(async (transaction) => { + const applied = await transaction.dailyCredit.updateMany({ + where: { id: credit.id, status: DailyCreditStatus.PENDING }, + data: { status: DailyCreditStatus.APPLIED, appliedAt: new Date() }, + }); + + if (applied.count === 0) return false; + + await transaction.user.update({ + where: { id: userId }, + data: { pnlBaseline: { increment: credit.amount } }, + }); + + return true; + }); + + return { + credited, + asset: credit.asset, + amount: fromBigInt(credit.amount, precision), + creditDate: creditDate.toISOString().slice(0, 10), + nextEligibleAt: nextUtcCreditDate(creditDate).toISOString(), + }; +} diff --git a/backend/src/types/engine.ts b/backend/src/types/engine.ts index b7e46c8..1788dab 100644 --- a/backend/src/types/engine.ts +++ b/backend/src/types/engine.ts @@ -6,6 +6,7 @@ export type EngineCommandType = | "get_open_orders" | "get_trades" | "cancel_order" + | "apply_credit" | "create_deposit" | "initialize_balance"; diff --git a/engine/src/core/dispatcher.ts b/engine/src/core/dispatcher.ts index 3ecc825..5f621a8 100644 --- a/engine/src/core/dispatcher.ts +++ b/engine/src/core/dispatcher.ts @@ -8,6 +8,7 @@ import { getTradesHandler } from "../handlers/getTrades"; import { createDepositHandler } from "../handlers/createDeposit"; import { initializeBalanceHandler } from "../handlers/initializeBalance"; import { getAllBalancesHandler } from "../handlers/getAllBalances"; +import { applyCreditHandler } from "../handlers/applyCredit"; type EngineHandler = (payload: Record) => Promise; @@ -19,6 +20,7 @@ const handlers: Record = { get_all_balances: getAllBalancesHandler, get_open_orders: getOpenOrdersHandler, get_trades: getTradesHandler, + apply_credit: applyCreditHandler, create_deposit: createDepositHandler, initialize_balance: initializeBalanceHandler, }; diff --git a/engine/src/handlers/applyCredit.ts b/engine/src/handlers/applyCredit.ts new file mode 100644 index 0000000..6462a47 --- /dev/null +++ b/engine/src/handlers/applyCredit.ts @@ -0,0 +1,13 @@ +import { applyCredit } from "../modules/balance"; +import { creditPayloadSchema } from "../schema"; + +export async function applyCreditHandler(payload: Record) { + const parsed = creditPayloadSchema.safeParse(payload); + + if (!parsed.success) { + throw new Error(parsed.error.issues[0]?.message ?? "Invalid payload"); + } + + const { creditId, userId, asset, amount } = parsed.data; + return applyCredit(creditId, userId, asset, amount); +} diff --git a/engine/src/modules/balance.ts b/engine/src/modules/balance.ts index 296d4da..0fd49db 100644 --- a/engine/src/modules/balance.ts +++ b/engine/src/modules/balance.ts @@ -1,4 +1,4 @@ -import { ASSETS, BALANCES } from "../store"; +import { APPLIED_CREDITS, ASSETS, BALANCES } from "../store"; import type { CreateOrderInput, InternalOrder, Fill, UserBalance } from "../types/domain"; import { getMarket } from "./market"; @@ -59,6 +59,17 @@ export function addBalance(userId: string, asset: string, amount: bigint) { return { [asset]: assetBalance }; } +export function applyCredit(creditId: string, userId: string, asset: string, amount: bigint) { + if (APPLIED_CREDITS.has(creditId)) { + return { applied: false, balance: getUserBalance(userId, asset) }; + } + + const balance = addBalance(userId, asset, amount); + APPLIED_CREDITS.add(creditId); + + return { applied: true, balance }; +} + export function getUserBalance(userId: string, asset?: string): UserBalance { const userBalance = BALANCES[userId] ?? {}; diff --git a/engine/src/schema/index.ts b/engine/src/schema/index.ts index 4c3f0db..bab46b9 100644 --- a/engine/src/schema/index.ts +++ b/engine/src/schema/index.ts @@ -49,3 +49,7 @@ export const depositPayloadSchema = userPayloadSchema.extend({ }); export const initializeBalancePayloadSchema = depositPayloadSchema; + +export const creditPayloadSchema = depositPayloadSchema.extend({ + creditId: z.string().trim().min(1, "creditId is required"), +}); diff --git a/engine/src/store.ts b/engine/src/store.ts index d8dd5ea..2769ecf 100644 --- a/engine/src/store.ts +++ b/engine/src/store.ts @@ -28,6 +28,7 @@ import { config } from "./config"; */ export const BALANCES: Record = {}; export const ASSETS = new Set(); +export const APPLIED_CREDITS = new Set(); /* ORDERBOOK = { diff --git a/engine/src/types/request.ts b/engine/src/types/request.ts index b7e46c8..1788dab 100644 --- a/engine/src/types/request.ts +++ b/engine/src/types/request.ts @@ -6,6 +6,7 @@ export type EngineCommandType = | "get_open_orders" | "get_trades" | "cancel_order" + | "apply_credit" | "create_deposit" | "initialize_balance"; diff --git a/engine/src/util/snapshot.ts b/engine/src/util/snapshot.ts index 6bbd9c2..3ca5a77 100644 --- a/engine/src/util/snapshot.ts +++ b/engine/src/util/snapshot.ts @@ -1,10 +1,10 @@ import fs from "fs/promises"; -import { BALANCES, ORDERBOOK, ORDERS, RECENT_TRADES } from "../store"; +import { APPLIED_CREDITS, BALANCES, ORDERBOOK, ORDERS, RECENT_TRADES } from "../store"; import type { Fill, InternalOrder, PriceLevel } from "../types/domain"; import { logger } from "./logger"; import { cacheClient } from "../redis/client"; -const SNAPSHOT_VERSION = 3; +const SNAPSHOT_VERSION = 4; const SNAPSHOT_PATH = "snapshots/snapshot.json"; function bigintReplacer(_key: string, value: unknown) { @@ -45,6 +45,7 @@ export async function snapshot() { ), recentTrades: RECENT_TRADES, orders: Object.fromEntries(ORDERS), + appliedCreditIds: [...APPLIED_CREDITS], }; const tmpPath = `${SNAPSHOT_PATH}.tmp`; @@ -84,6 +85,11 @@ export async function loadSnapshot() { ORDERS.set(key, value as InternalOrder); } + APPLIED_CREDITS.clear(); + for (const creditId of parsed.appliedCreditIds ?? []) { + if (typeof creditId === "string") APPLIED_CREDITS.add(creditId); + } + if (parsed.recentTrades) { for (const symbol of Object.keys(ORDERBOOK)) { RECENT_TRADES[symbol] = (parsed.recentTrades[symbol] as Fill[]) ?? []; diff --git a/engine/tests/daily-credit.test.ts b/engine/tests/daily-credit.test.ts new file mode 100644 index 0000000..cb61c0a --- /dev/null +++ b/engine/tests/daily-credit.test.ts @@ -0,0 +1,25 @@ +import { beforeEach, expect, test } from "bun:test"; +import { applyCreditHandler } from "../src/handlers/applyCredit"; +import { APPLIED_CREDITS, BALANCES } from "../src/store"; +import { resetState } from "./utils"; + +beforeEach(resetState); + +test("a daily credit is applied only once for the same credit id", async () => { + const payload = { creditId: "daily-credit-1", userId: "3", asset: "USD", amount: "10000" }; + + const first = await applyCreditHandler(payload); + const duplicate = await applyCreditHandler(payload); + + expect(first.applied).toBe(true); + expect(duplicate.applied).toBe(false); + expect(BALANCES["3"]?.USD).toEqual({ available: 10000n, locked: 0n }); + expect(APPLIED_CREDITS.has("daily-credit-1")).toBe(true); +}); + +test("a rejected credit id remains retryable", async () => { + const payload = { creditId: "daily-credit-2", userId: "3", asset: "UNKNOWN", amount: "10000" }; + + await expect(applyCreditHandler(payload)).rejects.toThrow("Unknown asset: UNKNOWN"); + expect(APPLIED_CREDITS.has("daily-credit-2")).toBe(false); +}); diff --git a/engine/tests/utils.ts b/engine/tests/utils.ts index 35df954..14ff1a5 100644 --- a/engine/tests/utils.ts +++ b/engine/tests/utils.ts @@ -1,4 +1,12 @@ -import { ASSETS, BALANCES, ORDERBOOK, ORDERS, ARCHIVED_ORDERS, RECENT_TRADES } from "../src/store"; +import { + APPLIED_CREDITS, + ASSETS, + BALANCES, + ORDERBOOK, + ORDERS, + ARCHIVED_ORDERS, + RECENT_TRADES, +} from "../src/store"; import { createOrderHandler } from "../src/handlers/createOrder"; import { cancelOrderHandler } from "../src/handlers/cancelOrder"; @@ -6,6 +14,7 @@ export function resetState() { for (const symbol of Object.keys(ORDERBOOK)) delete ORDERBOOK[symbol]; for (const symbol of Object.keys(RECENT_TRADES)) delete RECENT_TRADES[symbol]; for (const userId of Object.keys(BALANCES)) delete BALANCES[userId]; + APPLIED_CREDITS.clear(); ASSETS.clear(); ASSETS.add("BTC"); ASSETS.add("USD"); diff --git a/frontend/src/context/AuthContext.tsx b/frontend/src/context/AuthContext.tsx index c40d42b..cce3424 100644 --- a/frontend/src/context/AuthContext.tsx +++ b/frontend/src/context/AuthContext.tsx @@ -1,5 +1,6 @@ import React, { createContext, useCallback, useContext, useEffect, useState } from "react"; import { api, isUnauthorized } from "@/lib/api"; +import { toast } from "sonner"; type User = { id: string; @@ -43,6 +44,19 @@ export const AuthProvider = ({ children }: { children: React.ReactNode }) => { refreshUser(); }, [refreshUser]); + useEffect(() => { + if (!user?.emailVerified) return; + + api + .claimDailyCredit() + .then((credit) => { + if (!credit.credited) return; + toast.success(`Daily credit: +${credit.amount} ${credit.asset}`); + window.dispatchEvent(new Event("paperdrill:account-updated")); + }) + .catch((error) => console.warn("Daily credit could not be claimed:", error)); + }, [user?.id, user?.emailVerified]); + return ( {children} diff --git a/frontend/src/hooks/use-balance.ts b/frontend/src/hooks/use-balance.ts index 0b7e086..fb1a24e 100644 --- a/frontend/src/hooks/use-balance.ts +++ b/frontend/src/hooks/use-balance.ts @@ -17,6 +17,11 @@ export function useBalance({ asset, enabled = true }: UseBalanceOptions = {}) { setRefreshCount((count) => count + 1); }, []); + useEffect(() => { + window.addEventListener("paperdrill:account-updated", refresh); + return () => window.removeEventListener("paperdrill:account-updated", refresh); + }, [refresh]); + useEffect(() => { if (!enabled) { setBalances({}); diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index eaa0561..b93eb5b 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -5,6 +5,7 @@ import type { ApiKeyScope, Candle, CreatedApiKey, + DailyCredit, LeaderboardResponse, Market, MyLeaderboardResponse, @@ -66,11 +67,12 @@ async function request(path: string, options: RequestInit = {}): Promise { ); } - const message = typeof payloadError === "string" - ? payloadError - : typeof data?.message === "string" - ? data.message - : `Request failed: ${res.status}`; + const message = + typeof payloadError === "string" + ? payloadError + : typeof data?.message === "string" + ? data.message + : `Request failed: ${res.status}`; throw new ApiError(message, res.status); } @@ -158,6 +160,10 @@ export const api = { return request("/portfolio"); }, + claimDailyCredit() { + return request("/users/me/daily-credit", { method: "POST" }); + }, + getLeaderboard(limit = 25, offset = 0) { return request(`/leaderboard?limit=${limit}&offset=${offset}`); }, diff --git a/frontend/src/pages/dashboard/balances.tsx b/frontend/src/pages/dashboard/balances.tsx index 5215d78..b87070d 100644 --- a/frontend/src/pages/dashboard/balances.tsx +++ b/frontend/src/pages/dashboard/balances.tsx @@ -1,4 +1,4 @@ -import { useEffect, useMemo, useState } from "react"; +import { useCallback, useEffect, useMemo, useState } from "react"; import { toast } from "sonner"; import { DashboardPage } from "@/components/dashboard-page"; import { AssetIcon, assetNames } from "@/components/icons/asset-icon"; @@ -26,16 +26,23 @@ export function DashboardBalancesPage() { const [portfolio, setPortfolio] = useState(null); const [portfolioLoading, setPortfolioLoading] = useState(true); - useEffect(() => { - api - .getPortfolio() - .then(setPortfolio) - .catch(() => { - toast.error("Portfolio valuation is temporarily unavailable"); - }) - .finally(() => setPortfolioLoading(false)); + const loadPortfolio = useCallback(async () => { + try { + setPortfolio(await api.getPortfolio()); + } catch { + toast.error("Portfolio valuation is temporarily unavailable"); + } finally { + setPortfolioLoading(false); + } }, []); + useEffect(() => { + void loadPortfolio(); + const refresh = () => void loadPortfolio(); + window.addEventListener("paperdrill:account-updated", refresh); + return () => window.removeEventListener("paperdrill:account-updated", refresh); + }, [loadPortfolio]); + const loading = balanceLoading || portfolioLoading; const precisionFor = (asset: string) => { diff --git a/frontend/src/pages/dashboard/overview.tsx b/frontend/src/pages/dashboard/overview.tsx index 8b0e2ab..2635c07 100644 --- a/frontend/src/pages/dashboard/overview.tsx +++ b/frontend/src/pages/dashboard/overview.tsx @@ -1,4 +1,4 @@ -import { useEffect, useMemo, useState } from "react"; +import { useCallback, useEffect, useMemo, useState } from "react"; import { ArrowRight, Database, KeyRound, WalletCards } from "lucide-react"; import { Link } from "react-router-dom"; import { toast } from "sonner"; @@ -26,18 +26,28 @@ export function DashboardOverviewPage() { const { balances, loading: balanceLoading } = useBalance(); const [data, setData] = useState(null); - useEffect(() => { - Promise.all([api.getOpenOrders(), api.getTradeHistory(5), api.getPortfolio()]) - .then(([openOrders, trades, portfolio]) => { - setData({ openOrders, trades, portfolio }); - }) - .catch((error) => { - console.error("Failed to load dashboard:", error); - setData({ openOrders: [], trades: [], portfolio: null }); - toast.error("Failed to load dashboard overview"); - }); + const loadDashboard = useCallback(async () => { + try { + const [openOrders, trades, portfolio] = await Promise.all([ + api.getOpenOrders(), + api.getTradeHistory(5), + api.getPortfolio(), + ]); + setData({ openOrders, trades, portfolio }); + } catch (error) { + console.error("Failed to load dashboard:", error); + setData({ openOrders: [], trades: [], portfolio: null }); + toast.error("Failed to load dashboard overview"); + } }, []); + useEffect(() => { + void loadDashboard(); + const refresh = () => void loadDashboard(); + window.addEventListener("paperdrill:account-updated", refresh); + return () => window.removeEventListener("paperdrill:account-updated", refresh); + }, [loadDashboard]); + const balanceEntries = useMemo(() => Object.entries(balances), [balances]); const portfolio = data?.portfolio; const pnl = Number(portfolio?.pnl ?? 0); diff --git a/frontend/src/types/index.ts b/frontend/src/types/index.ts index 3636fa4..8826c3e 100644 --- a/frontend/src/types/index.ts +++ b/frontend/src/types/index.ts @@ -18,6 +18,14 @@ export interface UserBalance { [asset: string]: Balance; } +export interface DailyCredit { + credited: boolean; + asset: string; + amount: string; + creditDate: string; + nextEligibleAt: string; +} + export interface PortfolioPosition { asset: string; available: string; diff --git a/tests/integration/flows/daily-credit.test.ts b/tests/integration/flows/daily-credit.test.ts new file mode 100644 index 0000000..b29675c --- /dev/null +++ b/tests/integration/flows/daily-credit.test.ts @@ -0,0 +1,60 @@ +import { afterAll, beforeAll, expect, test } from "bun:test"; +import { ApiClient, waitForSystemReady } from "../helpers/api"; +import { createTestUser } from "../helpers/users"; +import { TestDatabase } from "../setup/database"; + +const database = new TestDatabase(); + +beforeAll(waitForSystemReady); +afterAll(() => database.close()); + +test("daily credit is concurrent-safe and does not count as profit", async () => { + const user = await createTestUser("daily-credit"); + const claims = await Promise.all(Array.from({ length: 5 }, () => user.client.claimDailyCredit())); + + expect(claims.every((claim) => claim.status === 200)).toBe(true); + expect(claims.filter((claim) => claim.data.credited)).toHaveLength(1); + expect(claims[0]?.data).toMatchObject({ asset: "USD", amount: "100.00" }); + + expect(await user.client.getBalances()).toMatchObject({ + status: 200, + data: { USD: { available: "10100.00", locked: "0.00" } }, + }); + expect(await user.client.getPortfolio()).toMatchObject({ + status: 200, + data: { + quoteAsset: "USD", + equity: "10100.00", + baselineEquity: "10100.00", + pnl: "0.00", + pnlPercent: "0.00", + }, + }); + + expect(await user.client.claimDailyCredit()).toMatchObject({ + status: 200, + data: { credited: false, asset: "USD", amount: "100.00" }, + }); + expect(await database.dailyCredits(user.id)).toEqual([ + expect.objectContaining({ + userId: user.id, + asset: "USD", + amount: "10000", + status: "APPLIED", + }), + ]); + expect(await database.pnlBaseline(user.id)).toBe("1010000"); +}); + +test("daily credit requires an authenticated session", async () => { + const anonymous = new ApiClient(); + expect(await anonymous.claimDailyCredit()).toMatchObject({ + status: 401, + data: { + error: { + code: "AUTHENTICATION_REQUIRED", + message: "Authentication required", + }, + }, + }); +}); diff --git a/tests/integration/helpers/api.ts b/tests/integration/helpers/api.ts index 079cf90..cb70636 100644 --- a/tests/integration/helpers/api.ts +++ b/tests/integration/helpers/api.ts @@ -39,6 +39,22 @@ export interface OrderResponse { export type Balances = Record; +export interface DailyCreditResponse { + credited: boolean; + asset: string; + amount: string; + creditDate: string; + nextEligibleAt: string; +} + +export interface PortfolioResponse { + quoteAsset: string; + equity: string; + baselineEquity: string; + pnl: string; + pnlPercent: string; +} + export class ApiClient { private cookie: string | undefined; @@ -68,6 +84,14 @@ export class ApiClient { return this.request("/v1/balances"); } + getPortfolio() { + return this.request("/v1/portfolio"); + } + + claimDailyCredit() { + return this.request("/v1/users/me/daily-credit", { method: "POST" }); + } + createOrder(input: OrderInput) { return this.request("/v1/orders", { method: "POST", diff --git a/tests/integration/setup/database.ts b/tests/integration/setup/database.ts index b140796..3fd7953 100644 --- a/tests/integration/setup/database.ts +++ b/tests/integration/setup/database.ts @@ -25,6 +25,14 @@ export interface PersistedFill { sellerId: string; } +export interface PersistedDailyCredit { + userId: string; + creditDate: Date; + asset: string; + amount: string; + status: "PENDING" | "APPLIED"; +} + export class TestDatabase { private readonly pool = new Pool({ connectionString: env.databaseUrl }); @@ -59,4 +67,21 @@ export class TestDatabase { ); return Number(result.rows[0]?.count ?? 0); } + + async dailyCredits(userId: string): Promise { + const result = await this.pool.query( + `SELECT "userId", "creditDate", asset, amount::text, status + FROM "DailyCredit" WHERE "userId" = $1 ORDER BY "creditDate"`, + [userId], + ); + return result.rows; + } + + async pnlBaseline(userId: string): Promise { + const result = await this.pool.query<{ pnlBaseline: string }>( + `SELECT "pnlBaseline"::text FROM "User" WHERE id = $1`, + [userId], + ); + return result.rows[0]?.pnlBaseline; + } } From 19649369e830e8aaa1e2cd7ffbcaaae82eb742b7 Mon Sep 17 00:00:00 2001 From: Harshil Date: Fri, 18 Sep 2026 13:06:04 +0530 Subject: [PATCH 10/20] feat: integrate react-query for data fetching and state management --- frontend/bun.lock | 5 + frontend/package.json | 1 + frontend/src/components/market/data-panel.tsx | 66 +++++++------ frontend/src/components/market/trade-form.tsx | 40 +++----- frontend/src/context/AuthContext.tsx | 32 +++++-- frontend/src/frontend.tsx | 16 ++-- frontend/src/hooks/use-account.ts | 95 +++++++++++++++++++ frontend/src/hooks/use-balance.ts | 65 ++++--------- frontend/src/hooks/use-portfolio.ts | 26 +++++ frontend/src/lib/query-client.ts | 49 ++++++++++ frontend/src/pages/dashboard/balances.tsx | 26 ++--- frontend/src/pages/dashboard/overview.tsx | 64 +++++-------- frontend/src/pages/trade.tsx | 4 +- frontend/src/types/index.ts | 8 ++ 14 files changed, 309 insertions(+), 188 deletions(-) create mode 100644 frontend/src/hooks/use-account.ts create mode 100644 frontend/src/hooks/use-portfolio.ts create mode 100644 frontend/src/lib/query-client.ts diff --git a/frontend/bun.lock b/frontend/bun.lock index b6467e6..b5a0cbc 100644 --- a/frontend/bun.lock +++ b/frontend/bun.lock @@ -8,6 +8,7 @@ "@radix-ui/react-label": "^2.1.7", "@radix-ui/react-select": "^2.2.6", "@radix-ui/react-slot": "^1.2.3", + "@tanstack/react-query": "^5.103.1", "bun-plugin-tailwind": "^0.1.2", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", @@ -278,6 +279,10 @@ "@sindresorhus/merge-streams": ["@sindresorhus/merge-streams@4.0.0", "", {}, "sha512-tlqY9xq5ukxTUZBmoOp+m61cqwQD5pHJtFY3Mn8CA8ps6yghLH/Hw8UPdqg4OLmFW3IFlcXnQNmo/dh8HzXYIQ=="], + "@tanstack/query-core": ["@tanstack/query-core@5.103.1", "", {}, "sha512-rms8HqTGp6zA00dM+cUQ2eBcgzNJefuu5CAMB37i/6MiGT1zulPOytCFu2a0qjLqVR2n1jENPj9woqFQNuCzWA=="], + + "@tanstack/react-query": ["@tanstack/react-query@5.103.1", "", { "dependencies": { "@tanstack/query-core": "5.103.1" }, "peerDependencies": { "react": "^18 || ^19" } }, "sha512-rmAPPApNEK17VXRJhtE1rja53n23VV5w30C0saCgJhhX+EpdUnVqMGV/s5nnzV43X75KTHKzuoiuxSzGTBIkag=="], + "@ts-morph/common": ["@ts-morph/common@0.27.0", "", { "dependencies": { "fast-glob": "^3.3.3", "minimatch": "^10.0.1", "path-browserify": "^1.0.1" } }, "sha512-Wf29UqxWDpc+i61k3oIOzcUfQt79PIT9y/MWfAGlrkjg6lBC1hwDECLXPVJAhWjiGbfBCxZd65F/LIZF3+jeJQ=="], "@types/bun": ["@types/bun@1.3.14", "", { "dependencies": { "bun-types": "1.3.14" } }, "sha512-h1hFqFVcvAvD9j9K7ZW7vd82aSA+rTdznZa+5bwvCwqSB1jmmfLcbIWhOLx1/+boy/xmjgCs/OMUL8hRJSmnPw=="], diff --git a/frontend/package.json b/frontend/package.json index dc25780..5dc0ace 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -12,6 +12,7 @@ "@radix-ui/react-label": "^2.1.7", "@radix-ui/react-select": "^2.2.6", "@radix-ui/react-slot": "^1.2.3", + "@tanstack/react-query": "^5.103.1", "bun-plugin-tailwind": "^0.1.2", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", diff --git a/frontend/src/components/market/data-panel.tsx b/frontend/src/components/market/data-panel.tsx index 45de7be..ae7407e 100644 --- a/frontend/src/components/market/data-panel.tsx +++ b/frontend/src/components/market/data-panel.tsx @@ -14,8 +14,13 @@ import { toast } from "sonner"; import type { OrderRecord, UserBalance, UserTrade } from "@/types"; import { useAuth } from "@/context/AuthContext"; import { useMarkets } from "@/context/MarketContext"; +import { + useCancelOrder, + useOpenOrders, + useOrderHistory, + useTradeHistory, +} from "@/hooks/use-account"; import { useBalance } from "@/hooks/use-balance"; -import { api } from "@/lib/api"; import { cn } from "@/lib/utils"; import { formatDateTime, formatPrice, formatQty } from "@/utils/format"; import { AssetIcon } from "../icons/asset-icon"; @@ -44,11 +49,11 @@ type SortDirection = "asc" | "desc"; type DataPanelProps = { loading?: boolean; - refreshKey?: number; symbol?: string; }; const PAGE_SIZE = 10; +const ORDER_HISTORY_PARAMS = { limit: 100 } as const; function titleCase(value: string) { return value @@ -205,15 +210,10 @@ function Pagination({ ); } -export function DataPanel({ loading = false, refreshKey, symbol }: DataPanelProps) { +export function DataPanel({ loading = false, symbol }: DataPanelProps) { const { authenticated, verified } = useAuth(); const { markets } = useMarkets(); const [tab, setTab] = useState("open"); - const [openOrders, setOpenOrders] = useState([]); - const [orders, setOrders] = useState([]); - const [trades, setTrades] = useState([]); - const [fetching, setFetching] = useState(true); - const [cancelling, setCancelling] = useState(null); const [search, setSearch] = useState(""); const [currentMarketOnly, setCurrentMarketOnly] = useState(Boolean(symbol)); const [marketFilter, setMarketFilter] = useState("all"); @@ -228,29 +228,31 @@ export function DataPanel({ loading = false, refreshKey, symbol }: DataPanelProp loading: balanceLoading, error: balanceError, } = useBalance({ enabled: authenticated }); - - const fetchData = useCallback(async () => { - setFetching(true); - try { - const [open, orderHistory, tradeHistory] = await Promise.all([ - api.getOpenOrders(), - api.getOrders({ limit: 100 }), - api.getTradeHistory(100), - ]); - setOpenOrders(open); - setOrders(orderHistory); - setTrades(tradeHistory); - } catch (error) { - console.error("Failed to load account data:", error); - toast.error("Failed to load account data"); - } finally { - setFetching(false); - } - }, []); + const { + openOrders, + loading: openOrdersLoading, + error: openOrdersError, + } = useOpenOrders({ enabled: authenticated }); + const { + orders, + loading: ordersLoading, + error: ordersError, + } = useOrderHistory(ORDER_HISTORY_PARAMS, { enabled: authenticated }); + const { + trades, + loading: tradesLoading, + error: tradesError, + } = useTradeHistory(100, { enabled: authenticated }); + const cancelOrder = useCancelOrder(); + const cancelling = cancelOrder.isPending ? (cancelOrder.variables ?? null) : null; + const fetching = openOrdersLoading || ordersLoading || tradesLoading; + const accountDataError = balanceError ?? openOrdersError ?? ordersError ?? tradesError; useEffect(() => { - if (authenticated) fetchData(); - }, [authenticated, refreshKey, fetchData]); + if (!accountDataError) return; + console.error("Failed to load account data:", accountDataError); + toast.error("Failed to load account data"); + }, [accountDataError]); useEffect(() => { setPage(1); @@ -357,15 +359,11 @@ export function DataPanel({ loading = false, refreshKey, symbol }: DataPanelProp const handleCancel = async (orderId: string) => { if (!verified) return; - setCancelling(orderId); try { - await api.cancelOrder(orderId); + await cancelOrder.mutateAsync(orderId); toast.success("Order cancelled"); - await fetchData(); } catch (error) { toast.error(error instanceof Error ? error.message : "Failed to cancel order"); - } finally { - setCancelling(null); } }; diff --git a/frontend/src/components/market/trade-form.tsx b/frontend/src/components/market/trade-form.tsx index 0ce9457..0a90a51 100644 --- a/frontend/src/components/market/trade-form.tsx +++ b/frontend/src/components/market/trade-form.tsx @@ -3,8 +3,8 @@ import { Link } from "react-router-dom"; import { toast } from "sonner"; import { useAuth } from "@/context/AuthContext"; import { useMarket } from "@/context/MarketContext"; +import { useCreateOrder } from "@/hooks/use-account"; import { useBalance } from "@/hooks/use-balance"; -import { api } from "@/lib/api"; import { cn } from "@/lib/utils"; import { formatPrice, formatQty } from "@/utils/format"; import { Button } from "../ui/button"; @@ -19,7 +19,6 @@ interface TradeFormProps { lastPrice?: string | null; bestBid?: string | null; bestAsk?: string | null; - onOrderPlaced?: () => void; } function isPositive(value: number) { @@ -35,27 +34,17 @@ function editablePrice(value: number, precision: number) { return Number.isFinite(value) && value > 0 ? value.toFixed(precision) : ""; } -export function TradeForm({ - symbol, - loading, - lastPrice, - bestBid, - bestAsk, - onOrderPlaced, -}: TradeFormProps) { +export function TradeForm({ symbol, loading, lastPrice, bestBid, bestAsk }: TradeFormProps) { const [side, setSide] = useState<"BUY" | "SELL">("BUY"); const [orderType, setOrderType] = useState<"LIMIT" | "MARKET">("LIMIT"); const [price, setPrice] = useState(""); const [quantity, setQuantity] = useState(""); const [percent, setPercent] = useState(0); - const [submitting, setSubmitting] = useState(false); const { authenticated, verified, loading: authLoading } = useAuth(); + const createOrder = useCreateOrder(); + const submitting = createOrder.isPending; const canTrade = authenticated && verified; - const { - balances: balance, - loading: balanceLoading, - refresh: refreshBalance, - } = useBalance({ enabled: authenticated }); + const { balances: balance, loading: balanceLoading } = useBalance({ enabled: authenticated }); const market = useMarket(symbol); const base = market?.baseAsset ?? symbol.split("_")[0] ?? symbol; @@ -87,9 +76,7 @@ export function TradeForm({ const requestedQuantity = Number(quantity); const exceedsMaximum = - canTrade && - isPositive(requestedQuantity) && - requestedQuantity > maxQuantity + Number.EPSILON; + canTrade && isPositive(requestedQuantity) && requestedQuantity > maxQuantity + Number.EPSILON; const maximumMessage = side === "BUY" ? `You can buy a maximum of ${maximumQuantityText} ${base} with your available ${quote}.` @@ -155,15 +142,14 @@ export function TradeForm({ return; } - setSubmitting(true); try { - const result = await api.createOrder( + const result = await createOrder.mutateAsync({ side, - orderType, + type: orderType, symbol, - quantity, - orderType === "LIMIT" ? price : null, - ); + qty: quantity, + price: orderType === "LIMIT" ? price : null, + }); const statusMessage = result.status === "FILLED" @@ -175,12 +161,8 @@ export function TradeForm({ toast.success(statusMessage); setQuantity(""); setPercent(0); - refreshBalance(); - onOrderPlaced?.(); } catch (error) { toast.error(error instanceof Error ? error.message : "Order failed"); - } finally { - setSubmitting(false); } }; diff --git a/frontend/src/context/AuthContext.tsx b/frontend/src/context/AuthContext.tsx index cce3424..391b21f 100644 --- a/frontend/src/context/AuthContext.tsx +++ b/frontend/src/context/AuthContext.tsx @@ -1,5 +1,7 @@ -import React, { createContext, useCallback, useContext, useEffect, useState } from "react"; +import React, { createContext, useCallback, useContext, useEffect, useRef, useState } from "react"; +import { useMutation, useQueryClient } from "@tanstack/react-query"; import { api, isUnauthorized } from "@/lib/api"; +import { accountQueryKeys, invalidateAccountQueries } from "@/lib/query-client"; import { toast } from "sonner"; type User = { @@ -23,6 +25,16 @@ export const AuthContext = createContext(null); export const AuthProvider = ({ children }: { children: React.ReactNode }) => { const [user, setUser] = useState(null); const [loading, setLoading] = useState(true); + const queryClient = useQueryClient(); + const previousUserId = useRef(null); + const dailyCredit = useMutation({ + mutationFn: (_userId: string) => api.claimDailyCredit(), + onSuccess: async (credit, userId) => { + if (credit.credited) toast.success(`Daily credit: +${credit.amount} ${credit.asset}`); + await invalidateAccountQueries(queryClient, userId); + }, + onError: (error) => console.warn("Daily credit could not be claimed:", error), + }); const refreshUser = useCallback(async () => { try { @@ -45,16 +57,16 @@ export const AuthProvider = ({ children }: { children: React.ReactNode }) => { }, [refreshUser]); useEffect(() => { - if (!user?.emailVerified) return; + const previous = previousUserId.current; + if (previous && previous !== user?.id) { + queryClient.removeQueries({ queryKey: accountQueryKeys.all(previous) }); + } + previousUserId.current = user?.id ?? null; + }, [queryClient, user?.id]); - api - .claimDailyCredit() - .then((credit) => { - if (!credit.credited) return; - toast.success(`Daily credit: +${credit.amount} ${credit.asset}`); - window.dispatchEvent(new Event("paperdrill:account-updated")); - }) - .catch((error) => console.warn("Daily credit could not be claimed:", error)); + useEffect(() => { + if (!user?.emailVerified) return; + dailyCredit.mutate(user.id); }, [user?.id, user?.emailVerified]); return ( diff --git a/frontend/src/frontend.tsx b/frontend/src/frontend.tsx index 7051260..98e3044 100644 --- a/frontend/src/frontend.tsx +++ b/frontend/src/frontend.tsx @@ -1,22 +1,26 @@ import { StrictMode } from "react"; import { createRoot } from "react-dom/client"; import { BrowserRouter } from "react-router-dom"; +import { QueryClientProvider } from "@tanstack/react-query"; import { App } from "./App"; import { ErrorBoundary } from "./components/error-boundary"; import { ThemeProvider } from "./lib/theme-provider"; +import { queryClient } from "./lib/query-client"; import { Toaster } from "./components/ui/sonner"; const elem = document.getElementById("root")!; const app = ( - - - - - - + + + + + + + + ); diff --git a/frontend/src/hooks/use-account.ts b/frontend/src/hooks/use-account.ts new file mode 100644 index 0000000..b95e5ae --- /dev/null +++ b/frontend/src/hooks/use-account.ts @@ -0,0 +1,95 @@ +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { useAuth } from "@/context/AuthContext"; +import { api } from "@/lib/api"; +import { accountQueryKeys, invalidateTradingQueries } from "@/lib/query-client"; +import type { CreateOrderInput } from "@/types"; + +type QueryOptions = { + enabled?: boolean; +}; + +type OrderHistoryParams = { + symbol?: string; + status?: string; + limit?: number; + page?: number; +}; + +export function useOpenOrders({ enabled = true }: QueryOptions = {}) { + const { user } = useAuth(); + const active = enabled && Boolean(user?.id); + const query = useQuery({ + queryKey: accountQueryKeys.openOrders(user?.id ?? "anonymous"), + queryFn: api.getOpenOrders, + enabled: active, + refetchInterval: 10_000, + }); + + return { + openOrders: query.data ?? [], + loading: active && query.isPending, + error: query.data === undefined ? query.error : null, + }; +} + +export function useOrderHistory( + params: OrderHistoryParams = {}, + { enabled = true }: QueryOptions = {}, +) { + const { user } = useAuth(); + const active = enabled && Boolean(user?.id); + const query = useQuery({ + queryKey: accountQueryKeys.orders(user?.id ?? "anonymous", params), + queryFn: () => api.getOrders(params), + enabled: active, + refetchInterval: 15_000, + }); + + return { + orders: query.data ?? [], + loading: active && query.isPending, + error: query.data === undefined ? query.error : null, + }; +} + +export function useTradeHistory(limit = 100, { enabled = true }: QueryOptions = {}) { + const { user } = useAuth(); + const active = enabled && Boolean(user?.id); + const query = useQuery({ + queryKey: accountQueryKeys.trades(user?.id ?? "anonymous", limit), + queryFn: () => api.getTradeHistory(limit), + enabled: active, + refetchInterval: 15_000, + }); + + return { + trades: query.data ?? [], + loading: active && query.isPending, + error: query.data === undefined ? query.error : null, + }; +} + +export function useCreateOrder() { + const { user } = useAuth(); + const queryClient = useQueryClient(); + + return useMutation({ + mutationFn: (input: CreateOrderInput) => + api.createOrder(input.side, input.type, input.symbol, input.qty, input.price), + onSuccess: async () => { + if (user) await invalidateTradingQueries(queryClient, user.id); + }, + }); +} + +export function useCancelOrder() { + const { user } = useAuth(); + const queryClient = useQueryClient(); + + return useMutation({ + mutationFn: api.cancelOrder, + onSuccess: async () => { + if (user) await invalidateTradingQueries(queryClient, user.id); + }, + }); +} diff --git a/frontend/src/hooks/use-balance.ts b/frontend/src/hooks/use-balance.ts index fb1a24e..b9b23cd 100644 --- a/frontend/src/hooks/use-balance.ts +++ b/frontend/src/hooks/use-balance.ts @@ -1,6 +1,7 @@ -import { useCallback, useEffect, useState } from "react"; +import { useQuery } from "@tanstack/react-query"; +import { useAuth } from "@/context/AuthContext"; import { api } from "@/lib/api"; -import type { UserBalance } from "@/types"; +import { accountQueryKeys } from "@/lib/query-client"; type UseBalanceOptions = { asset?: string; @@ -8,49 +9,19 @@ type UseBalanceOptions = { }; export function useBalance({ asset, enabled = true }: UseBalanceOptions = {}) { - const [balances, setBalances] = useState({}); - const [loading, setLoading] = useState(enabled); - const [error, setError] = useState(null); - const [refreshCount, setRefreshCount] = useState(0); - - const refresh = useCallback(() => { - setRefreshCount((count) => count + 1); - }, []); - - useEffect(() => { - window.addEventListener("paperdrill:account-updated", refresh); - return () => window.removeEventListener("paperdrill:account-updated", refresh); - }, [refresh]); - - useEffect(() => { - if (!enabled) { - setBalances({}); - setError(null); - setLoading(false); - return; - } - - let active = true; - setLoading(true); - setError(null); - - api - .getBalance(asset) - .then((data) => { - if (active) setBalances(data); - }) - .catch((cause) => { - if (!active) return; - setError(cause instanceof Error ? cause : new Error("Failed to load balances")); - }) - .finally(() => { - if (active) setLoading(false); - }); - - return () => { - active = false; - }; - }, [asset, enabled, refreshCount]); - - return { balances, loading, error, refresh }; + const { user } = useAuth(); + const active = enabled && Boolean(user?.id); + const query = useQuery({ + queryKey: accountQueryKeys.balances(user?.id ?? "anonymous", asset), + queryFn: () => api.getBalance(asset), + enabled: active, + refetchInterval: 15_000, + }); + + return { + balances: query.data ?? {}, + loading: active && query.isPending, + error: query.data === undefined ? query.error : null, + refresh: query.refetch, + }; } diff --git a/frontend/src/hooks/use-portfolio.ts b/frontend/src/hooks/use-portfolio.ts new file mode 100644 index 0000000..7eba786 --- /dev/null +++ b/frontend/src/hooks/use-portfolio.ts @@ -0,0 +1,26 @@ +import { useQuery } from "@tanstack/react-query"; +import { useAuth } from "@/context/AuthContext"; +import { api } from "@/lib/api"; +import { accountQueryKeys } from "@/lib/query-client"; + +type UsePortfolioOptions = { + enabled?: boolean; +}; + +export function usePortfolio({ enabled = true }: UsePortfolioOptions = {}) { + const { user } = useAuth(); + const active = enabled && Boolean(user?.id); + const query = useQuery({ + queryKey: accountQueryKeys.portfolio(user?.id ?? "anonymous"), + queryFn: api.getPortfolio, + enabled: active, + refetchInterval: 15_000, + }); + + return { + portfolio: query.data ?? null, + loading: active && query.isPending, + error: query.data === undefined ? query.error : null, + refresh: query.refetch, + }; +} diff --git a/frontend/src/lib/query-client.ts b/frontend/src/lib/query-client.ts new file mode 100644 index 0000000..c38d392 --- /dev/null +++ b/frontend/src/lib/query-client.ts @@ -0,0 +1,49 @@ +import { QueryClient } from "@tanstack/react-query"; + +export const queryClient = new QueryClient({ + defaultOptions: { + queries: { + staleTime: 15_000, + gcTime: 5 * 60_000, + retry: 1, + refetchOnWindowFocus: true, + }, + mutations: { + retry: false, + }, + }, +}); + +export const accountQueryKeys = { + all: (userId: string) => ["account", userId], + balances: (userId: string, asset?: string) => ["account", userId, "balances", asset ?? "all"], + balancesPrefix: (userId: string) => ["account", userId, "balances"], + portfolio: (userId: string) => ["account", userId, "portfolio"], + openOrders: (userId: string) => ["account", userId, "open-orders"], + ordersPrefix: (userId: string) => ["account", userId, "orders"], + orders: (userId: string, params: Record) => [ + "account", + userId, + "orders", + params, + ], + tradesPrefix: (userId: string) => ["account", userId, "trades"], + trades: (userId: string, limit: number) => ["account", userId, "trades", { limit }], +}; + +export function invalidateAccountQueries(client: QueryClient, userId: string) { + return Promise.all([ + client.invalidateQueries({ queryKey: accountQueryKeys.balancesPrefix(userId) }), + client.invalidateQueries({ queryKey: accountQueryKeys.portfolio(userId) }), + ]); +} + +export function invalidateTradingQueries(client: QueryClient, userId: string) { + return Promise.all([ + client.invalidateQueries({ queryKey: accountQueryKeys.balancesPrefix(userId) }), + client.invalidateQueries({ queryKey: accountQueryKeys.portfolio(userId) }), + client.invalidateQueries({ queryKey: accountQueryKeys.openOrders(userId) }), + client.invalidateQueries({ queryKey: accountQueryKeys.ordersPrefix(userId) }), + client.invalidateQueries({ queryKey: accountQueryKeys.tradesPrefix(userId) }), + ]); +} diff --git a/frontend/src/pages/dashboard/balances.tsx b/frontend/src/pages/dashboard/balances.tsx index b87070d..f8c2efe 100644 --- a/frontend/src/pages/dashboard/balances.tsx +++ b/frontend/src/pages/dashboard/balances.tsx @@ -1,4 +1,4 @@ -import { useCallback, useEffect, useMemo, useState } from "react"; +import { useEffect, useMemo } from "react"; import { toast } from "sonner"; import { DashboardPage } from "@/components/dashboard-page"; import { AssetIcon, assetNames } from "@/components/icons/asset-icon"; @@ -13,9 +13,9 @@ import { } from "@/components/ui/table"; import { useMarkets } from "@/context/MarketContext"; import { useBalance } from "@/hooks/use-balance"; -import { api } from "@/lib/api"; +import { usePortfolio } from "@/hooks/use-portfolio"; import { cn } from "@/lib/utils"; -import type { Portfolio, PortfolioPosition } from "@/types"; +import type { PortfolioPosition } from "@/types"; import { formatPrice, formatQty } from "@/utils/format"; type BalanceRow = PortfolioPosition & { precision: number }; @@ -23,25 +23,11 @@ type BalanceRow = PortfolioPosition & { precision: number }; export function DashboardBalancesPage() { const { markets } = useMarkets(); const { balances, loading: balanceLoading } = useBalance(); - const [portfolio, setPortfolio] = useState(null); - const [portfolioLoading, setPortfolioLoading] = useState(true); - - const loadPortfolio = useCallback(async () => { - try { - setPortfolio(await api.getPortfolio()); - } catch { - toast.error("Portfolio valuation is temporarily unavailable"); - } finally { - setPortfolioLoading(false); - } - }, []); + const { portfolio, loading: portfolioLoading, error: portfolioError } = usePortfolio(); useEffect(() => { - void loadPortfolio(); - const refresh = () => void loadPortfolio(); - window.addEventListener("paperdrill:account-updated", refresh); - return () => window.removeEventListener("paperdrill:account-updated", refresh); - }, [loadPortfolio]); + if (portfolioError) toast.error("Portfolio valuation is temporarily unavailable"); + }, [portfolioError]); const loading = balanceLoading || portfolioLoading; diff --git a/frontend/src/pages/dashboard/overview.tsx b/frontend/src/pages/dashboard/overview.tsx index 2635c07..e20c233 100644 --- a/frontend/src/pages/dashboard/overview.tsx +++ b/frontend/src/pages/dashboard/overview.tsx @@ -1,4 +1,4 @@ -import { useCallback, useEffect, useMemo, useState } from "react"; +import { useEffect, useMemo } from "react"; import { ArrowRight, Database, KeyRound, WalletCards } from "lucide-react"; import { Link } from "react-router-dom"; import { toast } from "sonner"; @@ -9,47 +9,27 @@ import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; import { Skeleton } from "@/components/ui/skeleton"; import { useAuth } from "@/context/AuthContext"; import { useMarkets } from "@/context/MarketContext"; +import { useOpenOrders, useTradeHistory } from "@/hooks/use-account"; import { useBalance } from "@/hooks/use-balance"; -import { api } from "@/lib/api"; -import type { OrderRecord, Portfolio, UserTrade } from "@/types"; +import { usePortfolio } from "@/hooks/use-portfolio"; import { formatDateTime, formatPrice, formatQty } from "@/utils/format"; -type OverviewData = { - openOrders: OrderRecord[]; - trades: UserTrade[]; - portfolio: Portfolio | null; -}; - export function DashboardOverviewPage() { const { user } = useAuth(); const { markets } = useMarkets(); const { balances, loading: balanceLoading } = useBalance(); - const [data, setData] = useState(null); - - const loadDashboard = useCallback(async () => { - try { - const [openOrders, trades, portfolio] = await Promise.all([ - api.getOpenOrders(), - api.getTradeHistory(5), - api.getPortfolio(), - ]); - setData({ openOrders, trades, portfolio }); - } catch (error) { - console.error("Failed to load dashboard:", error); - setData({ openOrders: [], trades: [], portfolio: null }); - toast.error("Failed to load dashboard overview"); - } - }, []); + const { portfolio, loading: portfolioLoading, error: portfolioError } = usePortfolio(); + const { openOrders, loading: openOrdersLoading, error: openOrdersError } = useOpenOrders(); + const { trades, loading: tradesLoading, error: tradesError } = useTradeHistory(5); + const dashboardError = portfolioError ?? openOrdersError ?? tradesError; useEffect(() => { - void loadDashboard(); - const refresh = () => void loadDashboard(); - window.addEventListener("paperdrill:account-updated", refresh); - return () => window.removeEventListener("paperdrill:account-updated", refresh); - }, [loadDashboard]); + if (!dashboardError) return; + console.error("Failed to load dashboard:", dashboardError); + toast.error("Failed to load dashboard overview"); + }, [dashboardError]); const balanceEntries = useMemo(() => Object.entries(balances), [balances]); - const portfolio = data?.portfolio; const pnl = Number(portfolio?.pnl ?? 0); const pnlClassName = pnl >= 0 ? "text-green-text!" : "text-red-text!"; @@ -76,7 +56,7 @@ export function DashboardOverviewPage() { className="border-b sm:border-r xl:border-b-0" label="Portfolio value" value={ - !data + portfolioLoading ? null : portfolio ? `${formatPrice(portfolio.equity)} ${portfolio.quoteAsset}` @@ -87,7 +67,7 @@ export function DashboardOverviewPage() { className="border-b xl:border-r xl:border-b-0" label="Total PnL" value={ - !data + portfolioLoading ? null : portfolio ? `${pnl > 0 ? "+" : ""}${formatPrice(portfolio.pnl)} ${portfolio.quoteAsset}` @@ -98,10 +78,16 @@ export function DashboardOverviewPage() { 0 ? "+" : ""}${portfolio.pnlPercent}%` : "-"} + value={ + portfolioLoading + ? null + : portfolio + ? `${pnl > 0 ? "+" : ""}${portfolio.pnlPercent}%` + : "-" + } valueClassName={pnlClassName} /> - +
@@ -110,7 +96,7 @@ export function DashboardOverviewPage() { Balances - {!data || balanceLoading ? ( + {balanceLoading ? (
@@ -144,17 +130,17 @@ export function DashboardOverviewPage() { Recent trades - {!data ? ( + {tradesLoading ? (
- ) : data.trades.length === 0 ? ( + ) : trades.length === 0 ? (

Your completed trades will appear here.

) : ( - data.trades.map((trade) => { + trades.map((trade) => { const market = markets.find((item) => item.symbol === trade.symbol); return (
("book"); const { orderbook, loading: orderbookLoading, bestBid, bestAsk } = useOrderbook(symbol); const { trades, loading: tradesLoading } = useTrades(symbol); @@ -129,7 +128,6 @@ export function TradePage() { lastPrice={ticker?.lastPrice} bestBid={bestBid} bestAsk={bestAsk} - onOrderPlaced={() => setOrderbookRefreshKey((key) => key + 1)} />
@@ -140,7 +138,7 @@ export function TradePage() {
- +
diff --git a/frontend/src/types/index.ts b/frontend/src/types/index.ts index 8826c3e..7819712 100644 --- a/frontend/src/types/index.ts +++ b/frontend/src/types/index.ts @@ -172,3 +172,11 @@ export interface Candle { volume: string; symbol: string; } + +export interface CreateOrderInput { + side: "BUY" | "SELL"; + type: "LIMIT" | "MARKET"; + symbol: string; + qty: string; + price?: string | null; +} From c0589319a0698f1956b197660518d0a060b73165 Mon Sep 17 00:00:00 2001 From: Harshil Date: Fri, 18 Sep 2026 20:15:31 +0530 Subject: [PATCH 11/20] fix: update stream retention duration --- .env.example | 2 +- engine/src/modules/market.ts | 2 +- engine/src/util/snapshot.ts | 36 +++++++++++++++++-------- worker/src/index.ts | 51 ++++++++++++++++++++---------------- worker/src/ticker.ts | 26 +++++++++++++----- 5 files changed, 75 insertions(+), 42 deletions(-) diff --git a/.env.example b/.env.example index bd1abfd..de4bd80 100644 --- a/.env.example +++ b/.env.example @@ -17,7 +17,7 @@ BACKEND_QUEUE_ID=1 ENGINE_TIMEOUT_MS=30000 # Engine stream batching -STREAM_RETENTION_MS=1200000 +STREAM_RETENTION_MS=21600000 SNAPSHOT_INTERVAL=60000 STREAM_READ_BATCH_SIZE=200 RESPONSE_BATCH_SIZE=50 diff --git a/engine/src/modules/market.ts b/engine/src/modules/market.ts index 7100fd0..6b76996 100644 --- a/engine/src/modules/market.ts +++ b/engine/src/modules/market.ts @@ -52,7 +52,7 @@ export async function initMarkets() { break; } catch (err) { logger.error("Failed to load markets from database", err); - wait(5000); + await wait(5000); } } } diff --git a/engine/src/util/snapshot.ts b/engine/src/util/snapshot.ts index 3ca5a77..c3bab89 100644 --- a/engine/src/util/snapshot.ts +++ b/engine/src/util/snapshot.ts @@ -1,6 +1,6 @@ import fs from "fs/promises"; import { APPLIED_CREDITS, BALANCES, ORDERBOOK, ORDERS, RECENT_TRADES } from "../store"; -import type { Fill, InternalOrder, PriceLevel } from "../types/domain"; +import type { Fill, InternalOrder, PriceLevel, RestingOrder } from "../types/domain"; import { logger } from "./logger"; import { cacheClient } from "../redis/client"; @@ -55,6 +55,24 @@ export async function snapshot() { await fs.rename(tmpPath, SNAPSHOT_PATH); } +function restorePriceLevels(savedLevels: Record | undefined) { + return new Map( + Object.entries(savedLevels ?? {}).map(([price, level]) => [ + BigInt(price), + { + totalQty: level.totalQty, + orders: level.orders.map((savedOrder) => { + const order = ORDERS.get(savedOrder.id); + if (!order) { + throw new Error(`Snapshot order ${savedOrder.id} is missing from the order index`); + } + return order as RestingOrder; + }), + }, + ]), + ); +} + export async function loadSnapshot() { try { const parsed = await readSnapshotFile(SNAPSHOT_PATH); @@ -68,21 +86,17 @@ export async function loadSnapshot() { Object.assign(BALANCES, parsed.balances); + for (const [key, value] of Object.entries(parsed.orders)) { + ORDERS.set(key, value as InternalOrder); + } + for (const [symbol, market] of Object.entries(ORDERBOOK)) { const saved = parsed.orderbook?.[symbol]; if (!saved) continue; market.bestBid = saved.bestBid; market.bestAsk = saved.bestAsk; - market.bids = new Map( - Object.entries(saved.bids).map(([k, v]) => [BigInt(k), v as PriceLevel]), - ); - market.asks = new Map( - Object.entries(saved.asks).map(([k, v]) => [BigInt(k), v as PriceLevel]), - ); - } - - for (const [key, value] of Object.entries(parsed.orders)) { - ORDERS.set(key, value as InternalOrder); + market.bids = restorePriceLevels(saved.bids); + market.asks = restorePriceLevels(saved.asks); } APPLIED_CREDITS.clear(); diff --git a/worker/src/index.ts b/worker/src/index.ts index 5de0d59..3d0bbdf 100644 --- a/worker/src/index.ts +++ b/worker/src/index.ts @@ -56,32 +56,38 @@ const batchInterval = setInterval(() => { void flushAndCheckpoint(); }, config.flushIntervalMs); -let flushing = false; +let flushPromise: Promise | null = null; -async function flushAndCheckpoint() { - if (flushing) return; - flushing = true; +function flushAndCheckpoint() { + if (flushPromise) return flushPromise; - const fillCursor = lastFillId; - const orderCursor = lastOrderId; + flushPromise = (async () => { + const fillCursor = lastFillId; + const orderCursor = lastOrderId; - try { - await flushBatch(); - } catch { - flushing = false; - return; - } + try { + await flushBatch(); + } catch { + return; + } - try { - await Promise.all([ - cacheClient.set("worker:fill:last_id", fillCursor), - cacheClient.set("worker:order:last_id", orderCursor), - ]); - } catch (err) { - logger.error("Failed to checkpoint stream cursors", err); - } + try { + await Promise.all([ + cacheClient.set("worker:fill:last_id", fillCursor), + cacheClient.set("worker:order:last_id", orderCursor), + ]); + } catch (err) { + logger.error("Failed to checkpoint stream cursors", err); + } + })().finally(() => { + flushPromise = null; + }); + + return flushPromise; +} - flushing = false; +function wait(ms: number) { + return new Promise((resolve) => setTimeout(resolve, ms)); } async function main() { @@ -96,7 +102,7 @@ async function main() { { key: "stream:fill", id: lastFillId }, { key: "stream:order", id: lastOrderId }, ], - { BLOCK: 5000 }, + { BLOCK: 5000, COUNT: 500 }, ); if (signal.aborted) break; if (!streams) continue; @@ -140,6 +146,7 @@ async function main() { } catch (err) { if (signal.aborted) break; logger.error("Stream read error", err); + await wait(500); } } } diff --git a/worker/src/ticker.ts b/worker/src/ticker.ts index c73a26d..e266880 100644 --- a/worker/src/ticker.ts +++ b/worker/src/ticker.ts @@ -34,7 +34,12 @@ const tickers = new Map(); let warmingUp = false; -function createTickerState(symbol: string, price: bigint, qty: bigint, timestamp: number): TickerState { +function createTickerState( + symbol: string, + price: bigint, + qty: bigint, + timestamp: number, +): TickerState { return { symbol, trades: [{ price, qty, timestamp }], @@ -147,14 +152,21 @@ function toPayload(state: TickerState) { export async function publishTicker(state: TickerState) { if (!state.dirty) return; state.dirty = false; - state.lastPublishedAt = Date.now(); const payload = toPayload(state); - await Promise.all([ - publisher.publish(`${TICKER_CHANNEL_PREFIX}${state.symbol}`, payload), - publisher.set(`${TICKER_STATE_KEY_PREFIX}${state.symbol}`, payload), - ]); + try { + await Promise.all([ + publisher.publish(`${TICKER_CHANNEL_PREFIX}${state.symbol}`, payload), + publisher.set(`${TICKER_STATE_KEY_PREFIX}${state.symbol}`, payload), + ]); + state.lastPublishedAt = Date.now(); + } catch (error) { + state.dirty = true; + state.lastPublishedAt = Date.now(); + schedulePublish(state); + throw error; + } } export async function warmUpTickers() { @@ -198,4 +210,4 @@ export async function warmUpTickers() { err, ); } -} \ No newline at end of file +} From a7407851a7ce0d0a0a9b7f72123006023320b84a Mon Sep 17 00:00:00 2001 From: Harshil Date: Sat, 19 Sep 2026 13:43:15 +0530 Subject: [PATCH 12/20] feat: additional candle intervals with live updates --- backend/src/controllers/market.ts | 3 ++ backend/src/schema/exchange.ts | 2 +- frontend/src/hooks/use-candles.ts | 49 ++++++++++++++++------ tests/integration/websocket/events.test.ts | 21 ++++++++++ worker/src/candle.ts | 19 +++++---- 5 files changed, 73 insertions(+), 21 deletions(-) diff --git a/backend/src/controllers/market.ts b/backend/src/controllers/market.ts index 8b7a303..701b495 100644 --- a/backend/src/controllers/market.ts +++ b/backend/src/controllers/market.ts @@ -16,7 +16,10 @@ import { marketStore } from "../store/market"; import { sendApiError, sendEngineError } from "../utils/apiError"; const intervalMap = { + "1M": "1 minute", + "5M": "5 minutes", "15M": "15 minutes", + "30M": "30 minutes", "1H": "1 hour", "4H": "4 hours", "1D": "1 day", diff --git a/backend/src/schema/exchange.ts b/backend/src/schema/exchange.ts index d44e8de..051ad21 100644 --- a/backend/src/schema/exchange.ts +++ b/backend/src/schema/exchange.ts @@ -34,7 +34,7 @@ export const orderBodySchema = z.discriminatedUnion("type", [ ]); export const candleQuerySchema = z.object({ - interval: z.enum(["15M", "1H", "4H", "1D"]).optional(), + interval: z.enum(["1M", "5M", "15M", "30M", "1H", "4H", "1D"]).optional(), }); export const orderQuerySchema = z.object({ diff --git a/frontend/src/hooks/use-candles.ts b/frontend/src/hooks/use-candles.ts index 39bc1ff..0a66806 100644 --- a/frontend/src/hooks/use-candles.ts +++ b/frontend/src/hooks/use-candles.ts @@ -3,11 +3,14 @@ import type { Candle } from "@/types"; import { api } from "@/lib/api"; import { wsManager } from "@/lib/ws"; -export const CANDLE_INTERVALS = ["15M", "1H", "4H", "1D"] as const; +export const CANDLE_INTERVALS = ["1M", "5M", "15M", "30M", "1H", "4H", "1D"] as const; export type CandleInterval = (typeof CANDLE_INTERVALS)[number]; export const CANDLE_INTERVAL_MS: Record = { + "1M": 60 * 1000, + "5M": 5 * 60 * 1000, "15M": 15 * 60 * 1000, + "30M": 30 * 60 * 1000, "1H": 60 * 60 * 1000, "4H": 4 * 60 * 60 * 1000, "1D": 24 * 60 * 60 * 1000, @@ -44,7 +47,12 @@ function fillCandleGaps(candles: Candle[], intMs: number): Candle[] { return r; } -function mergeMinuteCandle(into: Candle[], inc: Candle, intMs: number): Candle[] { +function mergeMinuteCandle( + into: Candle[], + inc: Candle, + previousMinute: Candle | null, + intMs: number, +): Candle[] { const bs = getBucketStart(inc.time, intMs); const c = [...into]; const l = c[c.length - 1]; @@ -63,12 +71,17 @@ function mergeMinuteCandle(into: Candle[], inc: Candle, intMs: number): Candle[] g += intMs; } } - const b = c.find((x) => x.time === bs); + const bucketIndex = c.findIndex((x) => x.time === bs); + const b = c[bucketIndex]; if (b) { - b.high = Number(inc.high) > Number(b.high) ? inc.high : b.high; - b.low = Number(inc.low) < Number(b.low) ? inc.low : b.low; - b.close = inc.close; - b.volume = String(Number(b.volume) + Number(inc.volume)); + const previousVolume = previousMinute?.time === inc.time ? Number(previousMinute.volume) : 0; + c[bucketIndex] = { + ...b, + high: Number(inc.high) > Number(b.high) ? inc.high : b.high, + low: Number(inc.low) < Number(b.low) ? inc.low : b.low, + close: inc.close, + volume: String(Number(b.volume) + Number(inc.volume) - previousVolume), + }; } else { c.push({ ...inc, time: bs }); } @@ -81,6 +94,7 @@ export function useCandles(symbol: string, interval: CandleInterval = "1H") { const [loading, setLoading] = useState(true); const [error, setError] = useState(null); const candlesRef = useRef([]); + const liveMinuteRef = useRef(null); const intervalRef = useRef(interval); intervalRef.current = interval; @@ -91,6 +105,7 @@ export function useCandles(symbol: string, interval: CandleInterval = "1H") { setLastCandle(null); setError(null); candlesRef.current = []; + liveMinuteRef.current = null; const intMs = CANDLE_INTERVAL_MS[interval]; @@ -100,10 +115,12 @@ export function useCandles(symbol: string, interval: CandleInterval = "1H") { if (!active) return; const data = res?.data ?? []; const raw: Candle[] = data.map((c) => ({ ...c, symbol, time: c.time })); - const filled = fillCandleGaps(raw, intMs); - candlesRef.current = filled; - setCandles(filled); - const last = filled[filled.length - 1]; + let next = fillCandleGaps(raw, intMs); + const liveMinute = liveMinuteRef.current; + if (liveMinute) next = mergeMinuteCandle(next, liveMinute, null, intMs); + candlesRef.current = next; + setCandles(next); + const last = next[next.length - 1]; if (last) setLastCandle(last); }) .catch((err) => { @@ -119,10 +136,16 @@ export function useCandles(symbol: string, interval: CandleInterval = "1H") { const candle = raw as Candle; if (!candle.time || candle.open === undefined || candle.close === undefined) return; const m = CANDLE_INTERVAL_MS[intervalRef.current]; - const merged = mergeMinuteCandle(candlesRef.current, candle, m); + const merged = mergeMinuteCandle( + candlesRef.current, + candle, + liveMinuteRef.current, + m, + ); + liveMinuteRef.current = candle; candlesRef.current = merged; setCandles(merged); - setLastCandle(candle); + setLastCandle(merged[merged.length - 1] ?? candle); }); return () => { diff --git a/tests/integration/websocket/events.test.ts b/tests/integration/websocket/events.test.ts index 2e7281a..c006107 100644 --- a/tests/integration/websocket/events.test.ts +++ b/tests/integration/websocket/events.test.ts @@ -65,6 +65,7 @@ test("a real match reaches trade subscribers with formatted price and quantity", seller = await createTestUser("websocket-trade-seller"); await seedBalance(engine, seller.id, "ETH", "1000"); await probe.subscribe("trade:ETH_USD"); + await probe.subscribe("candle:ETH_USD"); const tradeMessage = probe.next>( (message) => @@ -76,6 +77,16 @@ test("a real match reaches trade subscribers with formatted price and quantity", message.symbol === "ETH_USD", "ETH trade update", ); + const candleMessage = probe.next>( + (message) => + typeof message === "object" && + message !== null && + "event" in message && + message.event === "candle" && + "symbol" in message && + message.symbol === "ETH_USD", + "live ETH candle update", + ); const sell = await seller.client.createOrder({ type: "LIMIT", @@ -105,6 +116,16 @@ test("a real match reaches trade subscribers with formatted price and quantity", maker: false, timestamp: expect.any(Number), }); + expect(await candleMessage).toMatchObject({ + event: "candle", + symbol: "ETH_USD", + open: "321.00", + high: "321.00", + low: "321.00", + close: "321.00", + volume: "0.250", + time: expect.any(Number), + }); sellOrderId = undefined; } finally { if (sellOrderId && seller) { diff --git a/worker/src/candle.ts b/worker/src/candle.ts index 2353cee..05cfe48 100644 --- a/worker/src/candle.ts +++ b/worker/src/candle.ts @@ -9,6 +9,16 @@ function getBucket(time: number) { return time - (time % 60000); } +function publishCandle(candle: Candle) { + const payload = JSON.stringify({ event: "candle", ...candle }, (_, value) => + typeof value === "bigint" ? value.toString() : value, + ); + + void publisher.publish(`candle:${candle.symbol}`, payload).catch((err) => { + logger.error("Failed to publish candle update", { symbol: candle.symbol, error: err }); + }); +} + export function deriveData(data: Trade) { const { symbol, price, qty, timestamp } = data; @@ -26,11 +36,13 @@ export function deriveData(data: Trade) { symbol, }; openCandles.set(`${symbol}_${bucket}`, candle); + publishCandle(candle); } else { currentCandle.high = price > currentCandle.high ? price : currentCandle.high; currentCandle.low = price < currentCandle.low ? price : currentCandle.low; currentCandle.close = price; currentCandle.volume += qty; + publishCandle(currentCandle); } } @@ -41,13 +53,6 @@ export async function flushCandles() { const timestamp = new Date(candle.time); try { - await publisher.publish( - `candle:${candle.symbol}`, - JSON.stringify({ event: "candle", ...candle }, (_, v) => - typeof v === "bigint" ? v.toString() : v, - ), - ); - await pool.query( `INSERT INTO "Candle" (symbol, open, high, low, close, volume, time) Values ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (symbol, time) DO UPDATE SET From 760a7c0ec66628d511ae00112719dd4fd02900d2 Mon Sep 17 00:00:00 2001 From: Harshil Date: Sat, 19 Sep 2026 14:14:01 +0530 Subject: [PATCH 13/20] feat: enhance logging with shutdown messages --- backend/src/index.ts | 1 + backend/src/utils/logger.ts | 10 +++++++++- engine/src/index.ts | 5 +++++ engine/src/util/snapshot.ts | 5 +++++ worker/src/index.ts | 1 + worker/src/logger.ts | 10 +++++++++- 6 files changed, 30 insertions(+), 2 deletions(-) diff --git a/backend/src/index.ts b/backend/src/index.ts index ea3a24b..d07b2e1 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -72,6 +72,7 @@ async function gracefulShutdown(signal: string) { await flushSentry(); clearTimeout(forceExit); + logger.info("Backend shutdown complete"); process.exit(0); } diff --git a/backend/src/utils/logger.ts b/backend/src/utils/logger.ts index edc4412..fe9339a 100644 --- a/backend/src/utils/logger.ts +++ b/backend/src/utils/logger.ts @@ -5,13 +5,21 @@ type LogLevel = "debug" | "info" | "warn" | "error"; const LEVEL = { debug: 0, info: 1, warn: 2, error: 3 }; +function serialize(meta: unknown) { + return JSON.stringify(meta, (_key, value) => + value instanceof Error + ? { name: value.name, message: value.message, stack: value.stack } + : value, + ); +} + function log(level: LogLevel, message: string, meta?: unknown) { if (LEVEL[level] < LEVEL[config.app.logLevel]) return; const ts = new Date().toISOString(); const prefix = `[${ts}] [${level.toUpperCase()}]`; const line = - meta !== undefined ? `${prefix} ${message} ${JSON.stringify(meta)}` : `${prefix} ${message}`; + meta !== undefined ? `${prefix} ${message} ${serialize(meta)}` : `${prefix} ${message}`; if (level === "error") console.error(line); else console.log(line); diff --git a/engine/src/index.ts b/engine/src/index.ts index 0ad1e29..8764cdf 100644 --- a/engine/src/index.ts +++ b/engine/src/index.ts @@ -136,6 +136,10 @@ async function start() { await loadSnapshot(); registerEventHandlers(); await startHealth(); + logger.info("Engine started", { + incomingStream: config.incomingStream, + snapshotIntervalMs: config.snapshotInterval, + }); setInterval(() => { snapshot().catch((err) => logger.error("Snapshot error", err)); @@ -160,6 +164,7 @@ async function gracefulShutdown(signal: string) { await flushSentry(); clearTimeout(forceExit); + logger.info("Engine shutdown complete"); process.exit(0); } diff --git a/engine/src/util/snapshot.ts b/engine/src/util/snapshot.ts index c3bab89..4645ccc 100644 --- a/engine/src/util/snapshot.ts +++ b/engine/src/util/snapshot.ts @@ -53,6 +53,11 @@ export async function snapshot() { await fs.mkdir("snapshots", { recursive: true }); await fs.writeFile(tmpPath, JSON.stringify(snapshotData, bigintReplacer)); await fs.rename(tmpPath, SNAPSHOT_PATH); + logger.info("Snapshot saved", { + jobsLastId, + orders: ORDERS.size, + appliedCredits: APPLIED_CREDITS.size, + }); } function restorePriceLevels(savedLevels: Record | undefined) { diff --git a/worker/src/index.ts b/worker/src/index.ts index 3d0bbdf..d4dde0b 100644 --- a/worker/src/index.ts +++ b/worker/src/index.ts @@ -177,6 +177,7 @@ async function gracefulShutdown(signal: string) { await pool.end(); clearTimeout(forceExit); + logger.info("Worker shutdown complete"); process.exit(0); } diff --git a/worker/src/logger.ts b/worker/src/logger.ts index 200eb86..9ea7982 100644 --- a/worker/src/logger.ts +++ b/worker/src/logger.ts @@ -4,13 +4,21 @@ type LogLevel = "debug" | "info" | "warn" | "error"; const LEVEL = { debug: 0, info: 1, warn: 2, error: 3 }; +function serialize(meta: unknown) { + return JSON.stringify(meta, (_key, value) => + value instanceof Error + ? { name: value.name, message: value.message, stack: value.stack } + : value, + ); +} + function log(level: LogLevel, message: string, meta?: unknown) { if (LEVEL[level] < LEVEL[config.logLevel]) return; const ts = new Date().toISOString(); const prefix = `[${ts}] [${level.toUpperCase()}]`; const line = - meta !== undefined ? `${prefix} ${message} ${JSON.stringify(meta)}` : `${prefix} ${message}`; + meta !== undefined ? `${prefix} ${message} ${serialize(meta)}` : `${prefix} ${message}`; if (level === "error") console.error(line); else console.log(line); } From be79bae17b956581d15dac20d4542419b4218e87 Mon Sep 17 00:00:00 2001 From: Harshil Date: Sat, 19 Sep 2026 15:16:10 +0530 Subject: [PATCH 14/20] fix(ui): loading, empty and error states --- frontend/src/components/chart/chart.tsx | 26 +++++-- frontend/src/components/market/data-panel.tsx | 71 +++++++++++++++---- .../src/components/market/market-header.tsx | 18 +++++ .../components/market/market-watchlist.tsx | 22 +++++- frontend/src/components/market/orderbook.tsx | 12 +++- frontend/src/components/market/trades.tsx | 9 ++- frontend/src/components/ui/badge.tsx | 2 +- frontend/src/components/ui/button.tsx | 6 +- frontend/src/components/ui/checkbox.tsx | 2 +- frontend/src/components/ui/input.tsx | 4 +- frontend/src/components/ui/select.tsx | 2 +- frontend/src/components/ui/sheet.tsx | 2 +- frontend/src/components/ui/sidebar.tsx | 10 +-- frontend/src/components/ui/textarea.tsx | 2 +- frontend/src/pages/trade.tsx | 14 ++-- 15 files changed, 153 insertions(+), 49 deletions(-) diff --git a/frontend/src/components/chart/chart.tsx b/frontend/src/components/chart/chart.tsx index 1008d25..5b00c1c 100644 --- a/frontend/src/components/chart/chart.tsx +++ b/frontend/src/components/chart/chart.tsx @@ -1,3 +1,4 @@ +import { Activity, CircleAlert } from "lucide-react"; import { useEffect, useRef, useState } from "react"; import type { Candle, OrderBook, Ticker } from "@/types"; import { type CandleInterval, useCandles } from "../../hooks/use-candles"; @@ -82,7 +83,7 @@ export function Chart({ symbol, orderbook, ticker }: ChartProps) { return (
-
+
{TABS.map((item) => ( + +
); } return ( -
+
-
+
{tabs.map((item) => (