diff --git a/__tests__/jobs/service.test.ts b/__tests__/jobs/service.test.ts new file mode 100644 index 0000000..39874f5 --- /dev/null +++ b/__tests__/jobs/service.test.ts @@ -0,0 +1,98 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { query } = vi.hoisted(() => ({ query: vi.fn() })) +vi.mock('@/lib/db', () => ({ sql: { query } })) + +import { JobQueueService } from '@/lib/jobs/service' + +const row = (overrides: Record = {}) => ({ + id: '33333333-3333-4333-8333-333333333333', + job_type: 'deadline_check', + dedupe_key: 'deadline:2026-09-28T09', + status: 'queued', + payload: {}, + result: null, + retry_count: 0, + max_retries: 3, + next_run_at: '2026-09-28T09:00:00Z', + completed_at: null, + created_at: '2026-09-28T08:00:00Z', + updated_at: '2026-09-28T08:00:00Z', + ...overrides, +}) + +describe('JobQueueService', () => { + beforeEach(() => query.mockReset()) + + it('queues a job once and returns it without running the handler', async () => { + query.mockResolvedValueOnce([row()]) + const service = new JobQueueService() + const result = await service.enqueue({ type: 'deadline_check', dedupeKey: 'deadline:2026-09-28T09' }) + expect(result.isDuplicate).toBe(false) + expect(result.job.state).toBe('pending') + expect(query.mock.calls[0][0]).toContain('ON CONFLICT (dedupe_key) DO NOTHING') + }) + + it('returns the existing job when the same dedupe key is queued again', async () => { + query.mockResolvedValueOnce([]).mockResolvedValueOnce([row({ status: 'completed' })]) + const service = new JobQueueService() + const result = await service.enqueue({ type: 'deadline_check', dedupeKey: 'deadline:2026-09-28T09' }) + expect(result.isDuplicate).toBe(true) + expect(result.job.status).toBe('completed') + expect(query).toHaveBeenCalledTimes(2) + }) + + it('rejects a dedupe key already bound to another job type', async () => { + query.mockResolvedValueOnce([]).mockResolvedValueOnce([row()]) + const service = new JobQueueService() + await expect(service.enqueue({ type: 'tx_monitor', dedupeKey: 'deadline:2026-09-28T09' })) + .rejects.toMatchObject({ code: 'JOB_DEDUPE_CONFLICT', httpStatus: 409 }) + }) + + it('claims due jobs with a row lock so another worker cannot take them', async () => { + query.mockResolvedValueOnce([row({ status: 'processing' })]) + query.mockResolvedValueOnce([]) + const service = new JobQueueService() + const result = await service.processBatch({ + workerId: 'worker-a', + handlers: { deadline_check: async () => ({ remindersSent: 1, overdueFlagged: 0 }) }, + }) + expect(query.mock.calls[0][0]).toContain('FOR UPDATE SKIP LOCKED') + expect(query.mock.calls[1][0]).toContain("status = 'completed'") + expect(result).toMatchObject({ claimed: 1, completed: 1, failed: 0 }) + }) + + it('records the stack and requeues with backoff while retries remain', async () => { + query.mockResolvedValueOnce([row()]).mockResolvedValueOnce([{ status: 'queued' }]) + const service = new JobQueueService() + const result = await service.processBatch({ + workerId: 'worker-a', + handlers: { deadline_check: async () => { throw new Error('horizon timeout') } }, + }) + const failureSql = query.mock.calls[1][0] as string + expect(failureSql).toContain('INSERT INTO background_job_errors') + expect(failureSql).toContain('INTERVAL \'1 millisecond\'') + expect(query.mock.calls[1][1][3]).toContain('horizon timeout') + expect(result).toMatchObject({ retried: 1, failed: 0 }) + }) + + it('marks the job failed once the retry limit is exhausted', async () => { + query.mockResolvedValueOnce([row({ retry_count: 2 })]).mockResolvedValueOnce([{ status: 'failed' }]) + const service = new JobQueueService() + const result = await service.processBatch({ + workerId: 'worker-a', + handlers: { deadline_check: async () => { throw new Error('still down') } }, + }) + expect(result).toMatchObject({ retried: 0, failed: 1 }) + }) + + it('returns job state and stored errors', async () => { + query.mockResolvedValueOnce([row({ + status: 'failed', + errors: [{ attempt: 3, message: 'still down', stack: 'Error: still down', metadata: { workerId: 'worker-a' }, createdAt: '2026-09-28T09:05:00Z' }], + })]) + const job = await new JobQueueService().getJob(row().id) + expect(job.state).toBe('failed') + expect(job.errors[0]).toMatchObject({ attempt: 3, message: 'still down', stack: 'Error: still down' }) + }) +}) diff --git a/app/api/jobs/[id]/route.ts b/app/api/jobs/[id]/route.ts new file mode 100644 index 0000000..f37c2bc --- /dev/null +++ b/app/api/jobs/[id]/route.ts @@ -0,0 +1,26 @@ +export const dynamic = 'force-dynamic' + +import { NextRequest, NextResponse } from 'next/server' +import { withAuthCtx } from '@/lib/auth/middleware' +import { JobQueueError, jobQueueService } from '@/lib/jobs' + +export const GET = withAuthCtx(async ( + _request: NextRequest, + _auth, + context: { params: Promise<{ id: string }> }, +) => { + const { id } = await context.params + if (!/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(id)) { + return NextResponse.json({ error: 'Job id must be a UUID.', code: 'INVALID_JOB_ID' }, { status: 422 }) + } + + try { + return NextResponse.json({ data: await jobQueueService.getJob(id) }) + } catch (error) { + if (error instanceof JobQueueError) { + return NextResponse.json({ error: error.message, code: error.code }, { status: error.httpStatus }) + } + console.error('[api/jobs/[id]] Error:', error) + return NextResponse.json({ error: 'Failed to load job', code: 'JOB_QUERY_FAILED' }, { status: 500 }) + } +}) diff --git a/app/api/jobs/process/route.ts b/app/api/jobs/process/route.ts new file mode 100644 index 0000000..4fbe144 --- /dev/null +++ b/app/api/jobs/process/route.ts @@ -0,0 +1,33 @@ +export const dynamic = 'force-dynamic' + +import { timingSafeEqual } from 'crypto' +import { NextRequest, NextResponse } from 'next/server' +import { createJobHandlers, jobQueueService } from '@/lib/jobs' + +function authorized(request: NextRequest): boolean { + const expected = process.env.JOB_WORKER_SECRET + const provided = request.headers.get('authorization')?.replace(/^Bearer\s+/i, '') + if (!expected || !provided) return false + const a = Buffer.from(expected) + const b = Buffer.from(provided) + return a.length === b.length && timingSafeEqual(a, b) +} + +export async function POST(request: NextRequest) { + if (!authorized(request)) { + return NextResponse.json({ error: 'Unauthorized', code: 'AUTH_REQUIRED' }, { status: 401 }) + } + try { + const body = await request.json().catch(() => ({})) + const batchSize = Number.isInteger(body.batchSize) ? body.batchSize : 10 + const data = await jobQueueService.processBatch({ + workerId: typeof body.workerId === 'string' && body.workerId.trim() ? body.workerId.trim() : 'api-worker', + batchSize, + handlers: createJobHandlers(), + }) + return NextResponse.json({ data }) + } catch (error) { + console.error('[api/jobs/process] Error:', error) + return NextResponse.json({ error: 'Job processing failed', code: 'JOB_PROCESS_FAILED' }, { status: 500 }) + } +} diff --git a/app/api/jobs/route.ts b/app/api/jobs/route.ts new file mode 100644 index 0000000..e69de29 diff --git a/lib/contract-sync/service.ts b/lib/contract-sync/service.ts index 1e1901e..4ea8a94 100644 --- a/lib/contract-sync/service.ts +++ b/lib/contract-sync/service.ts @@ -85,6 +85,11 @@ export class ContractSyncService { return this.listener } + /** Applies one Soroban event through the existing sync path. */ + async applyEvent(payload: SorobanEventPayload): Promise { + await this.processSync({ id: buildSyncDedupeKey(payload), payload }) + } + private async onEvent(payload: SorobanEventPayload): Promise { const dedupeKey = buildSyncDedupeKey(payload) diff --git a/lib/jobs/handlers.ts b/lib/jobs/handlers.ts new file mode 100644 index 0000000..b9ed3fe --- /dev/null +++ b/lib/jobs/handlers.ts @@ -0,0 +1,65 @@ +import type { JobHandler, JobType } from './types' +import type { SorobanEventPayload } from '@/lib/contract-sync/types' + +export function createJobHandlers(): Record { + return { + tx_monitor: async (job) => { + const { txRecoveryService } = await import('@/lib/tx-recovery') + const requested = job.payload.batchSize + const batchSize = typeof requested === 'number' ? requested : 25 + const result = await txRecoveryService.pollPendingQueue(batchSize) + return { ...result } + }, + notification: async (job) => { + const { createNotification } = await import('@/lib/notifications') + const { userId, title, message, type, eventType, payload } = job.payload + if (typeof userId !== 'number' && typeof userId !== 'string') { + throw new Error('notification job requires a userId') + } + if (typeof title !== 'string' || typeof message !== 'string' || typeof type !== 'string' || typeof eventType !== 'string') { + throw new Error('notification job requires title, message, type, and eventType') + } + const notification = await createNotification({ + userId: userId as number, + title, + message, + type, + eventType, + payload: payload && typeof payload === 'object' && !Array.isArray(payload) + ? payload as Record + : {}, + }) + return { notificationId: notification.id } + }, + contract_sync: async (job) => { + const { ContractSyncService } = await import('@/lib/contract-sync') + await new ContractSyncService().applyEvent(readSyncPayload(job.payload)) + return { synced: true } + }, + deadline_check: async () => { + const { DeadlineMonitorService } = await import('@/lib/deadline-monitor') + const result = await new DeadlineMonitorService().runCheck() + return { ...result } + }, + } +} + +function readSyncPayload(payload: Record): SorobanEventPayload { + const { event, contractAddress, txHash, data } = payload + if (typeof event !== 'string' || typeof contractAddress !== 'string' || typeof txHash !== 'string' || !Array.isArray(data)) { + throw new Error('contract_sync job requires event, contractAddress, txHash, and data') + } + return { + event: event as SorobanEventPayload['event'], + contractAddress, + txHash, + data, + ledgerSequence: typeof payload.ledgerSequence === 'number' ? payload.ledgerSequence : 0, + timestamp: typeof payload.timestamp === 'number' ? payload.timestamp : Date.now(), + milestoneId: typeof payload.milestoneId === 'number' ? payload.milestoneId : undefined, + disputeId: typeof payload.disputeId === 'number' ? payload.disputeId : undefined, + amount: typeof payload.amount === 'string' ? payload.amount : undefined, + actor: typeof payload.actor === 'string' ? payload.actor : undefined, + recipient: typeof payload.recipient === 'string' ? payload.recipient : undefined, + } +} diff --git a/lib/jobs/index.ts b/lib/jobs/index.ts new file mode 100644 index 0000000..eb691d2 --- /dev/null +++ b/lib/jobs/index.ts @@ -0,0 +1,13 @@ +export { JobQueueService, jobQueueService } from './service' +export { createJobHandlers } from './handlers' +export { JOB_TYPES, JOB_STATE, JobQueueError, getJobBackoffMs } from './types' +export type { + BackgroundJob, + EnqueueJobInput, + JobBatchResult, + JobErrorRecord, + JobHandler, + JobState, + JobStatus, + JobType, +} from './types' diff --git a/lib/jobs/service.ts b/lib/jobs/service.ts new file mode 100644 index 0000000..d1e43a2 --- /dev/null +++ b/lib/jobs/service.ts @@ -0,0 +1,245 @@ +import { sql } from '@/lib/db' +import { + JOB_STATE, + JobQueueError, + getJobBackoffMs, +} from './types' +import type { + BackgroundJob, + EnqueueJobInput, + JobBatchResult, + JobErrorRecord, + JobHandler, + JobStatus, + JobType, +} from './types' + +const SELECT_COLUMNS = `id, job_type, dedupe_key, status, payload, result, retry_count, + max_retries, next_run_at, completed_at, created_at, updated_at` + +export class JobQueueService { + async enqueue(input: EnqueueJobInput): Promise<{ job: BackgroundJob; isDuplicate: boolean }> { + const dedupeKey = input.dedupeKey.trim() + if (!dedupeKey || dedupeKey.length > 200) { + throw new JobQueueError('INVALID_DEDUPE_KEY', 'dedupeKey must be 1–200 characters.', 422) + } + const maxRetries = input.maxRetries ?? 5 + if (!Number.isInteger(maxRetries) || maxRetries < 1 || maxRetries > 20) { + throw new JobQueueError('INVALID_MAX_RETRIES', 'maxRetries must be an integer from 1 to 20.', 422) + } + + const inserted = await sql.query( + `INSERT INTO background_jobs (job_type, dedupe_key, payload, max_retries) + VALUES ($1, $2, $3::jsonb, $4) + ON CONFLICT (dedupe_key) DO NOTHING + RETURNING ${SELECT_COLUMNS}`, + [input.type, dedupeKey, JSON.stringify(input.payload ?? {}), maxRetries], + ) as Array> + + if (inserted.length) return { job: this.mapRow(inserted[0]), isDuplicate: false } + + const existing = await sql.query( + `SELECT ${SELECT_COLUMNS} FROM background_jobs WHERE dedupe_key = $1 LIMIT 1`, + [dedupeKey], + ) as Array> + if (!existing.length) throw new JobQueueError('JOB_ENQUEUE_FAILED', 'Job could not be queued.', 500) + + const job = this.mapRow(existing[0]) + if (job.type !== input.type) { + throw new JobQueueError('JOB_DEDUPE_CONFLICT', 'dedupeKey is already used by a different job type.', 409) + } + return { job, isDuplicate: true } + } + + async getJob(id: string): Promise { + const rows = await sql.query( + `SELECT ${SELECT_COLUMNS.split(', ').map((column) => `j.${column.trim()}`).join(', ')}, + COALESCE( + json_agg( + json_build_object( + 'attempt', e.attempt, + 'message', e.message, + 'stack', e.stack, + 'metadata', e.metadata, + 'createdAt', e.created_at + ) + ORDER BY e.created_at + ) FILTER (WHERE e.id IS NOT NULL), + '[]'::json + ) AS errors + FROM background_jobs j + LEFT JOIN background_job_errors e ON e.job_id = j.id + WHERE j.id = $1::uuid + GROUP BY j.id`, + [id], + ) as Array> + if (!rows.length) throw new JobQueueError('JOB_NOT_FOUND', 'Job not found.', 404) + return this.mapRow(rows[0], rows[0].errors) + } + + async processBatch(options: { + workerId: string + batchSize?: number + handlers: Partial> + }): Promise { + const workerId = options.workerId.trim() + if (!workerId) throw new JobQueueError('INVALID_WORKER', 'workerId is required.', 422) + const limit = Number.isInteger(options.batchSize) + ? Math.min(50, Math.max(1, options.batchSize as number)) + : 10 + + const claimed = await sql.query( + `WITH due AS ( + SELECT id FROM background_jobs + WHERE status = 'queued' AND next_run_at <= NOW() + ORDER BY next_run_at + LIMIT $1 + FOR UPDATE SKIP LOCKED + ) + UPDATE background_jobs j + SET status = 'processing', locked_by = $2, locked_at = NOW(), updated_at = NOW() + FROM due + WHERE j.id = due.id + RETURNING ${SELECT_COLUMNS.split(', ').map((column) => `j.${column.trim()}`).join(', ')}`, + [limit, workerId], + ) as Array> + + const result: JobBatchResult = { claimed: claimed.length, completed: 0, retried: 0, failed: 0, errors: [] } + for (const row of claimed) { + const job = this.mapRow(row) + try { + const handler = options.handlers[job.type] + if (!handler) throw new Error(`No handler registered for job type ${job.type}`) + const output = await handler(job) + await this.markCompleted(job.id, output ?? {}) + result.completed++ + } catch (error) { + const message = error instanceof Error ? error.message : String(error) + const stack = error instanceof Error ? error.stack ?? null : null + try { + const outcome = await this.recordFailure(job, workerId, message, stack) + if (outcome === 'failed') result.failed++ + else result.retried++ + } catch (persistError) { + result.errors.push(`Job ${job.id}: ${persistError instanceof Error ? persistError.message : String(persistError)}`) + } + result.errors.push(`Job ${job.id}: ${message}`) + } + } + return result + } + + private async markCompleted(id: string, output: Record): Promise { + await sql.query( + `UPDATE background_jobs + SET status = 'completed', result = $2::jsonb, completed_at = NOW(), + locked_by = NULL, locked_at = NULL, updated_at = NOW() + WHERE id = $1::uuid AND status = 'processing'`, + [id, JSON.stringify(output)], + ) + } + + private async recordFailure( + job: BackgroundJob, + workerId: string, + message: string, + stack: string | null, + ): Promise<'queued' | 'failed'> { + const attempt = job.retryCount + 1 + const rows = await sql.query( + `WITH err AS ( + INSERT INTO background_job_errors (job_id, attempt, message, stack, metadata) + VALUES ($1::uuid, $2, $3, $4, $5::jsonb) + RETURNING job_id + ) + UPDATE background_jobs j + SET retry_count = j.retry_count + 1, + status = CASE + WHEN j.retry_count + 1 >= j.max_retries THEN 'failed'::background_job_status + ELSE 'queued'::background_job_status + END, + next_run_at = CASE + WHEN j.retry_count + 1 >= j.max_retries THEN j.next_run_at + ELSE NOW() + ($6 * INTERVAL '1 millisecond') + END, + locked_by = NULL, + locked_at = NULL, + updated_at = NOW() + FROM err + WHERE j.id = err.job_id AND j.status = 'processing' + RETURNING j.status`, + [ + job.id, + attempt, + message, + stack, + JSON.stringify({ workerId, jobType: job.type, dedupeKey: job.dedupeKey }), + getJobBackoffMs(attempt), + ], + ) as Array<{ status: JobStatus }> + return rows[0]?.status === 'failed' ? 'failed' : 'queued' + } + + private mapRow(row: Record, errors: unknown = []): BackgroundJob { + const status = row.status as JobStatus + return { + id: String(row.id), + type: row.job_type as JobType, + dedupeKey: String(row.dedupe_key), + status, + state: JOB_STATE[status], + payload: asObject(row.payload), + result: row.result == null ? null : asObject(row.result), + retryCount: Number(row.retry_count), + maxRetries: Number(row.max_retries), + nextRunAt: asTimestamp(row.next_run_at), + completedAt: row.completed_at == null ? null : asTimestamp(row.completed_at), + createdAt: asTimestamp(row.created_at), + updatedAt: asTimestamp(row.updated_at), + errors: asErrors(errors), + } + } +} + +function asObject(value: unknown): Record { + if (value && typeof value === 'object' && !Array.isArray(value)) return value as Record + if (typeof value === 'string') { + try { + const parsed = JSON.parse(value) as unknown + if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) return parsed as Record + } catch { + return {} + } + } + return {} +} + +function asTimestamp(value: unknown): string { + if (value instanceof Date) return value.toISOString() + return String(value) +} + +function asErrors(value: unknown): JobErrorRecord[] { + const list = Array.isArray(value) ? value : typeof value === 'string' ? safeParseArray(value) : [] + return list.map((entry) => { + const row = asObject(entry) + return { + attempt: Number(row.attempt), + message: String(row.message ?? ''), + stack: row.stack == null ? null : String(row.stack), + metadata: asObject(row.metadata), + createdAt: asTimestamp(row.createdAt ?? row.created_at), + } + }) +} + +function safeParseArray(value: string): unknown[] { + try { + const parsed = JSON.parse(value) as unknown + return Array.isArray(parsed) ? parsed : [] + } catch { + return [] + } +} + +export const jobQueueService = new JobQueueService() diff --git a/lib/jobs/types.ts b/lib/jobs/types.ts new file mode 100644 index 0000000..373a3db --- /dev/null +++ b/lib/jobs/types.ts @@ -0,0 +1,71 @@ +export const JOB_TYPES = ['tx_monitor', 'notification', 'contract_sync', 'deadline_check'] as const + +export type JobType = (typeof JOB_TYPES)[number] +export type JobStatus = 'queued' | 'processing' | 'completed' | 'failed' +export type JobState = 'pending' | 'running' | 'completed' | 'failed' + +export const JOB_STATE: Record = { + queued: 'pending', + processing: 'running', + completed: 'completed', + failed: 'failed', +} + +export interface JobErrorRecord { + attempt: number + message: string + stack: string | null + metadata: Record + createdAt: string +} + +export interface BackgroundJob { + id: string + type: JobType + dedupeKey: string + status: JobStatus + state: JobState + payload: Record + result: Record | null + retryCount: number + maxRetries: number + nextRunAt: string + completedAt: string | null + createdAt: string + updatedAt: string + errors: JobErrorRecord[] +} + +export interface EnqueueJobInput { + type: JobType + dedupeKey: string + payload?: Record + maxRetries?: number +} + +export interface JobBatchResult { + claimed: number + completed: number + retried: number + failed: number + errors: string[] +} + +export type JobHandler = (job: BackgroundJob) => Promise | void> + +export class JobQueueError extends Error { + constructor( + readonly code: string, + message: string, + readonly httpStatus: number, + ) { + super(message) + this.name = 'JobQueueError' + } +} + +export function getJobBackoffMs(attempt: number): number { + const base = Number(process.env.JOB_BACKOFF_BASE_MS) || 1_000 + const cap = Number(process.env.JOB_BACKOFF_CAP_MS) || 60_000 + return Math.min(cap, base * 2 ** Math.max(0, attempt - 1)) +} diff --git a/package.json b/package.json index 5786834..4573606 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "sync-worker": "tsx scripts/sync-worker.ts", "deadline-monitor": "tsx scripts/deadline-monitor.ts", "tx-worker": "tsx scripts/tx-recovery-worker.ts", + "job-worker": "tsx scripts/job-worker.ts", "migrate": "tsx scripts/migrate.ts", "idempotency:cleanup": "tsx scripts/idempotency-cleanup.ts", "build:production": "npm run migrate && next build", diff --git a/scripts/job-worker.ts b/scripts/job-worker.ts new file mode 100644 index 0000000..64f37bf --- /dev/null +++ b/scripts/job-worker.ts @@ -0,0 +1,49 @@ +import { hostname } from 'os' +import { createJobHandlers, jobQueueService } from '@/lib/jobs' +import * as dotenv from 'dotenv' + +dotenv.config() + +if (!process.env.DATABASE_URL) { + console.error('FATAL: DATABASE_URL is not set. Job worker cannot connect to the database.') + process.exit(1) +} + +const POLL_INTERVAL_MS = Number(process.env.JOB_POLL_INTERVAL_MS) || 5_000 +const BATCH_SIZE = Number(process.env.JOB_BATCH_SIZE) || 10 +const WORKER_ID = process.env.JOB_WORKER_ID || `${hostname()}-${process.pid}` + +async function startJobWorker() { + console.log(`[JobWorker] Starting background job worker ${WORKER_ID}`) + console.log(`[JobWorker] Poll interval ${POLL_INTERVAL_MS}ms, batch size ${BATCH_SIZE}`) + const handlers = createJobHandlers() + + const run = async () => { + try { + const result = await jobQueueService.processBatch({ workerId: WORKER_ID, batchSize: BATCH_SIZE, handlers }) + if (result.claimed > 0) { + console.log( + `[JobWorker] Claimed ${result.claimed} — completed ${result.completed}, retried ${result.retried}, failed ${result.failed}`, + ) + } + if (result.errors.length > 0) console.error('[JobWorker] Errors:', result.errors) + } catch (error) { + console.error('[JobWorker] Poll cycle failed:', error) + } + } + + await run() + const interval = setInterval(run, POLL_INTERVAL_MS) + const shutdown = () => { + console.log('[JobWorker] Shutting down...') + clearInterval(interval) + process.exit(0) + } + process.on('SIGINT', shutdown) + process.on('SIGTERM', shutdown) +} + +startJobWorker().catch((error) => { + console.error('[FATAL JobWorker]', error) + process.exit(1) +})