From 7a8d0963ce8ec6a01a50119f1dac50aa009a172a Mon Sep 17 00:00:00 2001 From: thegreatfeez Date: Sat, 17 Oct 2026 12:25:48 +0100 Subject: [PATCH] feat: add transparency reports, LLM router, treasury console, verified IPFS store closes #159 closes #157 closes #156 closes #100 --- .env.example | 31 + docs/api.md | 73 ++ docs/architecture.md | 56 ++ docs/contract-requests.md | 16 + docs/transparency-reports.md | 150 ++++ package.json | 3 +- .../migration.sql | 87 +++ prisma/schema.prisma | 81 +++ scripts/verify-report.ts | 196 +++++ src/config/index.ts | 28 + src/controllers/admin.controller.ts | 68 ++ src/controllers/portfolio.controller.ts | 5 +- src/cron/monitor.cron.ts | 12 + src/cron/report.cron.ts | 34 + src/lib/gemini.ts | 238 +----- src/lib/llm.ts | 179 ++--- src/lib/llm/policy.ts | 87 +++ src/lib/llm/providers/anthropic.ts | 77 ++ src/lib/llm/providers/gemini.ts | 90 +++ src/lib/llm/providers/index.ts | 3 + src/lib/llm/providers/openai.ts | 67 ++ src/lib/llm/router.ts | 244 +++++++ src/lib/llm/types.ts | 90 +++ src/routes/admin.routes.ts | 27 + src/routes/index.ts | 4 + src/routes/reports.routes.ts | 59 ++ src/server.ts | 2 + src/services/chain.service.ts | 12 +- src/services/content-store.service.ts | 176 +++++ src/services/ipfs.service.ts | 48 ++ src/services/market.service.ts | 26 +- src/services/reasoning.service.ts | 27 +- src/services/trace-publish.service.ts | 58 +- src/services/transparency-report.service.ts | 461 ++++++++++++ src/services/treasury-ops.service.ts | 684 ++++++++++++++++++ src/types/index.ts | 2 + src/validators/admin.validator.ts | 28 + tests/content-store.test.ts | 178 +++++ tests/http.test.ts | 12 + tests/llm-router.test.ts | 176 +++++ tests/reports-http.test.ts | 49 ++ tests/transparency-report.test.ts | 300 ++++++++ tests/treasury-ops.test.ts | 161 +++++ 43 files changed, 4061 insertions(+), 344 deletions(-) create mode 100644 docs/transparency-reports.md create mode 100644 prisma/migrations/20261017120000_transparency_reports_admin_ops_ipfs_content/migration.sql create mode 100644 scripts/verify-report.ts create mode 100644 src/controllers/admin.controller.ts create mode 100644 src/cron/report.cron.ts create mode 100644 src/lib/llm/policy.ts create mode 100644 src/lib/llm/providers/anthropic.ts create mode 100644 src/lib/llm/providers/gemini.ts create mode 100644 src/lib/llm/providers/index.ts create mode 100644 src/lib/llm/providers/openai.ts create mode 100644 src/lib/llm/router.ts create mode 100644 src/lib/llm/types.ts create mode 100644 src/routes/admin.routes.ts create mode 100644 src/routes/reports.routes.ts create mode 100644 src/services/content-store.service.ts create mode 100644 src/services/transparency-report.service.ts create mode 100644 src/services/treasury-ops.service.ts create mode 100644 src/validators/admin.validator.ts create mode 100644 tests/content-store.test.ts create mode 100644 tests/llm-router.test.ts create mode 100644 tests/reports-http.test.ts create mode 100644 tests/transparency-report.test.ts create mode 100644 tests/treasury-ops.test.ts diff --git a/.env.example b/.env.example index bed94e6..677112b 100644 --- a/.env.example +++ b/.env.example @@ -76,3 +76,34 @@ IPFS_GATEWAY_URL=https://gateway.pinata.cloud/ipfs # ── Data sources ─────────────────────────────────────────────────────────── NEWSAPI_KEY= FRED_API_KEY= + +# ── LLM router (#157) ────────────────────────────────────────────────────── +# OpenAI or any OpenAI-compatible endpoint. The router can use them in a task +# policy even if Anthropic/Gemini are configured. +OPENAI_API_KEY= +OPENAI_BASE_URL= +# Optional JSON policy overriding some/all tasks, e.g. +# {"market_generation":[{"provider":"anthropic","model":"claude-sonnet-4-6", +# "temperature":0.2,"maxTokens":4096,"timeoutMs":60000}]} +# Or a path to a JSON file. Missing tasks keep the built-in default policy. +LLM_POLICY= +LLM_POLICY_FILE= + +# ── IPFS content store (#100) ────────────────────────────────────────────── +# Per-object cap and total Postgres budget for verified pinned bytes. +IPFS_CACHE_MAX_OBJECT_BYTES=262144 +IPFS_CACHE_BUDGET_BYTES=268435456 + +# ── Transparency reports (#159) ──────────────────────────────────────────── +REPORTS_ENABLED=false +REPORT_CRON=10 0 * * * +# dry-run: pin + persist, never publish on-chain. live: publish with the agent. +REPORT_MODE=dry-run +# URL placed in the report's method.doc field. +REPORT_METHOD_DOC=https://github.com/OracleDesk/oracledesk-backend/blob/main/docs/transparency-reports.md + +# ── Treasury admin console (#156) ────────────────────────────────────────── +# Lifetime of a prepared unsigned transaction. +TREASURY_OP_TTL_SECONDS=3600 +# Optional stricter backend policy for withdraw operations. +TREASURY_WITHDRAW_MIN_SIGNERS=0 diff --git a/docs/api.md b/docs/api.md index d915afb..93d1277 100644 --- a/docs/api.md +++ b/docs/api.md @@ -484,6 +484,79 @@ decides an outcome.** `marketStatus.tag` is `Open | Resolved | Void`. `404 MARKET_NOT_FOUND`, `409 MARKET_NOT_ON_CHAIN`, `502 CHAIN_READ_FAILED`. + +### Treasury admin console (auth, admin) + +The backend never holds the treasury admin key. It prepares unsigned, +simulated transactions for the on-chain admin/multisig, verifies the signed +XDR body against the prepared hash, and submits only when the collected +signature weight reaches the account's high threshold. Every state change +writes an `AgentLog` row (`ADMIN_OP_`). + +#### `GET /admin/treasury` +Returns treasury `RiskConfig`, available capital, USDC balance, daily usage +with headroom/reset time, treasury/market-core paused flags and admins, and +the agent's XLM fee balance. The on-chain admin is read from treasury instance +storage until the contract exposes `admin()`/`is_paused()`. + +#### `GET /admin/treasury/operations?page&limit` +Paginated prepared/signed/submitted operations. + +#### `POST /admin/treasury/operations` +Body: `{ "operation": { ... }, "force"?: boolean }`, where `operation` is one of: + +- `{ "kind": "treasury.set_paused", "paused": true }` +- `{ "kind": "treasury.withdraw", "to": "G…|C…", "amountRaw": "12500000000" }` +- `{ "kind": "treasury.deposit", "from": "G…", "amountRaw": "100000000" }` +- `{ "kind": "treasury.collect_market_fees", "onChainMarketId": "7" }` +- `{ "kind": "market_core.set_paused", "paused": false }` + +Returns `{ id, xdr, networkPassphrase, hash, expiresAt, requiredSigners, +thresholds, checks, humanSummary, status }`. Pre-flight failures return +`422 PREFLIGHT_FAILED` with `details.checks`. A `C…` admin returns +`501 UNSUPPORTED_ADMIN_KIND` (auth-entry signing is out of scope). + +#### `POST /admin/treasury/operations/:id/signatures` +Body `{ "signedXdr": "…" }`. The envelope hash must equal the stored `txHash`; +new signatures are verified against the admin account's signers, duplicates are +ignored, and `weightCollected` is recomputed. `READY` means the high threshold +(and optional `TREASURY_WITHDRAW_MIN_SIGNERS`) is met. + +#### `POST /admin/treasury/operations/:id/submit` +Allowed only in `READY` and before `expiresAt`, testnet only. Sends the +envelope, polls `getTransaction`, and records `SUCCESS`/`FAILED`, ledger and +result XDR. No server key is used. + +#### `POST /admin/treasury/operations/:id/record` +Body `{ "txHash": "<64 hex>" }` for clients that submitted themselves. + +#### `POST /admin/treasury/operations/:id/cancel` +Cancels a prepared/partially-signed operation. + +### Transparency reports + +Public, computed only from chain state and `chain_events`; see +`docs/transparency-reports.md`. + +#### `GET /reports?page&limit` +Paginated index: `date`, `anchorLedger`, `cid`, `ipfsUri`, `hash`, +`onChainTraceId`, `mode` (`live`/`dry-run`), `consistency` +(`exact`/`best_effort`), the report document, and a verification recipe. + +#### `GET /reports/latest` +Most recent report (404 if none yet). + +#### `GET /reports/:date` +One report by `YYYY-MM-DD` (400 for a malformed date, 404 if absent). + +`GET /portfolio` also includes `latestReport` with the same metadata. + +### `POST /traces/verify` content source + +The response now includes `contentSource: "cache" | "gateway"`: `cache` means +the bytes came from the local verified content store and both gateways could +be down. The on-chain `trace_hash` remains the authority. + ## Realtime (socket.io) Same origin as the API without `/api/v1`. The frontend listens to these diff --git a/docs/architecture.md b/docs/architecture.md index 24b23b9..791e9e8 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -63,3 +63,59 @@ Outcomes are decided on-chain by the resolver contract. The backend only reads ` ## Generated code `src/generated/` is copied from the pinned `contracts` submodule by `scripts/sync-contracts.mjs`, with a do-not-edit header and `// @ts-nocheck`. CI fails if it's stale. + +## LLM router (#157) + +All generation goes through `src/lib/llm/router.ts`. Each `LlmTask` +(`market_generation`, `hedge_conditions`, `forecasting`, `attestation`, +`generic_json`, `generic_text`) maps to an ordered list of `ModelSpec`s +(provider, model, temperature, max tokens, timeout, optional base URL). The +default policy reproduces the old Anthropic → Gemini behaviour. Override any +task with `LLM_POLICY` (JSON) or `LLM_POLICY_FILE`; missing tasks keep the +default, and an invalid task/spec fails at boot with a per-task message. + +Providers live in `src/lib/llm/providers/` (Anthropic, Gemini, +OpenAI-compatible). They normalise SDK errors into `LlmError.kind` +(`rate_limited`, `quota_exhausted`, `auth`, `invalid_request`, `timeout`, +`server`, `content_filtered`, `empty`, `parse`). The router: + +- skips providers with no key (without counting a failure); +- retries inline only when `retryAfterMs <= 5s`; longer waits cool the + provider down and fail over; +- opens a 1-hour circuit on quota exhaustion; +- cools a provider for 60s after 3 server/timeout failures; +- fails over on auth/invalid-request/empty/parse; +- parses JSON in the router, so an invalid response tries the next model; +- returns provenance (`provider`, `model`, `attempts`, `latencyMs`, `usage`) + for agent logs and trace payloads (`llm: { provider, model } | null`). + +Text mode never sets a JSON response format; only `mode: "json"` does. + +## IPFS content store (#100) + +`pinJson()` writes the exact pinned bytes to `ipfs_content` (Postgres +`bytea`) after the pin succeeds; a store failure is logged and does not fail +the pin. `fetchVerifiedBytes(cid, expectedSha256, verifiedBy)`: + +1. reads the local row and re-hashes its bytes; +2. returns it only when both the stored hash and the freshly computed hash + equal the expected hash (on-chain or pin hash); +3. otherwise consults the gateway, stores bytes only after they verify, and + returns unverified bytes without caching so callers can report a mismatch. + +sha256 is the admission criterion; the CID is only a lookup key. A CID can +never map to two hashes (`IPFS_CID_HASH_CONFLICT`). Objects above +`IPFS_CACHE_MAX_OBJECT_BYTES` are not stored. An hourly job evicts +`GATEWAY_VERIFIED` rows by least-recent access, then 30-day-old `PINNED` rows, +never touching content referenced by an unpublished trace or a `PENDING` +market (one SQL `NOT EXISTS` query, batches of 500). + +## Transparency reports (#159) + +`src/services/transparency-report.service.ts` builds a deterministic +`oracledesk.transparency-report` v1 document from chain reads and +`chain_events` only (no DB floats). Amounts are base-unit decimal strings. +The daily job pins the canonical bytes, publishes the hash through the +reasoning registry under the sentinel `u64::MAX` market id with action +`report`, and persists the index row. A crash after pin re-publishes the +stored bytes rather than recomputing. See `docs/transparency-reports.md`. diff --git a/docs/contract-requests.md b/docs/contract-requests.md index 9f78060..8bb304d 100644 --- a/docs/contract-requests.md +++ b/docs/contract-requests.md @@ -62,3 +62,19 @@ duplicate it, but today the adapter: If dry-run simulated (and returned the simulated result), the bindings were injectable, and `agentSell` existed, `src/services/chain.service.ts` here could become a thin wrapper around it. Its method names already match. + +## 6. Treasury `admin()` and `is_paused()` views + +The admin console has to read `DataKey::Admin` and `DataKey::Paused` from the +treasury's contract-instance storage via `getLedgerEntries`. That is a +storage-layout workaround that breaks if the contract's keys change. Expose +`admin() -> Address` and `is_paused() -> bool` (or include them in +`get_config`) so clients do not depend on instance storage internals. + +## 7. A dedicated `publish_report` entrypoint + +Transparency reports use `market_id = u64::MAX` as a documented sentinel in +`publish_trace`, because reports are not about a single market. A +`publish_report(agent, action, trace_hash, ipfs_cid)` entrypoint (with its own +report counter/index) would remove the sentinel hack and let +`market_trace_count(u64::MAX)` stay a valid market-only index. diff --git a/docs/transparency-reports.md b/docs/transparency-reports.md new file mode 100644 index 0000000..3a5dc1c --- /dev/null +++ b/docs/transparency-reports.md @@ -0,0 +1,150 @@ +# Transparency reports v1 + +OracleDesk publishes a daily report computed **only from chain data** and +`chain_events`, anchors it to a ledger, pins the canonical JSON to IPFS, and +commits its sha256 through the reasoning registry. The public index serves the +CID, hash, on-chain trace id and anchor; `scripts/verify-report.ts` re-checks +them without database access. + +## Document + +```ts +interface TransparencyReportV1 { + kind: 'oracledesk.transparency-report'; + schemaVersion: 1; + network: string; // Stellar passphrase + contracts: { treasury; marketCore; usdc; reasoningRegistry }; + agent: string; // treasury RiskConfig.agent + period: { from: string; to: string }; // UTC day, ISO + anchor: { ledger: number; closedAt: string }; + consistency: 'exact' | 'best_effort'; + treasury: { + usdcBalanceRaw: string; + availableCapitalRaw: string; + riskConfig: { maxTradeRaw; maxMarketRaw; maxDailyRaw }; + dailyUsage: { amountRaw: string; dayStart: number }; + }; + markets: Array<{ + marketId: string; + status: 'Open' | 'Resolved:Yes' | 'Resolved:No' | 'Void'; + createdByTreasury: boolean; + position: { yesRaw: string; noRaw: string }; + reserves: { yesRaw: string; noRaw: string }; + priceYesBps: number; + markValueRaw: string; + liquidationValueRaw: string; + settlementValueRaw: string | null; + lpRemainderRaw?: string; // final, unclaimed treasury-created pools + archived?: boolean; + }>; + totals: { positionsMarkRaw; positionsLiquidationRaw; navRaw }; + period_flows: { + coverage: { fromLedger; toLedger; complete; gaps: Array<[number, number]> }; + tradesBoughtRaw; tradesSoldRaw; feesPaidRaw; redeemedRaw; + feesCollectedRaw; seedDeployedRaw; tradeCount; + }; + traces: { total: number; byAction: Record }; + method: { version: '1'; doc: string }; + previous: { onChainTraceId: string; traceHash: string } | null; +} +``` + +All amounts are integer strings in 7-decimal USDC base units. No Prisma float +model is imported by the report service (CI asserts the import boundary). + +## Integers and valuations + +`BPS = 10_000`. Let `p = priceYesBps(reserve_yes, reserve_no)` from +`src/generated/fpmm.ts`. + +- `markValueRaw = floor((yes × p + no × (BPS − p)) / BPS)` +- `liquidationValueRaw = collateralOutForShares(reserve_yes, reserve_no, yes, fee_bps) + + collateralOutForShares(reserve_no, reserve_yes, no, fee_bps)` +- `settlementValueRaw = settleValue(kind, yes, no)` for final markets, where + `kind` is `yes`, `no` or `void`. +- `lpRemainderRaw = settleValue(kind, reserve_yes, reserve_no)` for final, + unclaimed (`lp_claimed === false`) pools created by the treasury. +- `totals.positionsMarkRaw = Σ markValueRaw`; + `positionsLiquidationRaw = Σ liquidationValueRaw`; + `navRaw = usdcBalanceRaw + positionsMarkRaw`. + +## Snapshot + +Simulations run against the RPC's latest state; Soroban cannot pin a +historical ledger. The builder therefore: + +1. reads `getLatestLedger()` → `L0`; +2. enumerates `0..marketCount()` and reads each market + treasury position; +3. reads `getLatestLedger()` → `L1`; +4. if `L1 > L0`, retries up to 3 times; after the third attempt the report is + still published with `consistency: 'best_effort'` and `anchor.ledger = L1`. + +The normal case is `consistency: 'exact'`. Entries that can no longer be read +are reported as `archived: true` rather than treated as zero. + +## Period flows + +Events are `chain_events` rows whose `ledgerClosedAt` falls in the UTC day: + +- market-core `Trade` with `trader = treasury`: buys/sells (`is_buy`), + `fee`, and `tradeCount`; +- market-core `Redeemed` with `holder = treasury`; +- treasury `FeesCollected`; +- market-core `MarketCreated` with `creator = treasury` (`seed_amount`). + +`coverage.complete` is false when the stored indexer cursor is older than the +last ledger in the period; the missing range is listed in `coverage.gaps`. +RPC retention means an indexer that did not run continuously cannot recover +those events. + +## Trace counts + +The report walks `traceCount()`/`get_trace(i)` and caches immutable trace +metadata in `registry_trace_cache`. Only traces whose `agent` equals the +treasury's `RiskConfig.agent` are counted in `traces.total` and +`traces.byAction`. + +## Sentinel market id + +`publish_trace` requires a `market_id`. Reports use +`REPORT_MARKET_ID = 18446744073709551615` (`u64::MAX`) and action `"report"`. +`market_trace_count(REPORT_MARKET_ID)` / `market_trace_at` therefore form an +on-chain index of reports. A dedicated `publish_report` entrypoint is requested +in `docs/contract-requests.md`. + +## Publishing and idempotency + +`REPORTS_ENABLED=true` schedules `REPORT_CRON` (default `10 0 * * *`, +`Etc/UTC`). The job: + +1. builds the document for the UTC day; +2. canonicalises and pins it (`pinJson`); +3. publishes `trace_hash`/`ipfs_cid` via `publish_trace` (respecting + `CHAIN_EXECUTION_MODE`; `REPORT_MODE=dry-run` is also supported); +4. inserts or updates `transparency_reports`. + +If a row exists without `onChainTraceId` (crash between pin and publish, or a +dry run being retried), the job re-publishes the **stored bytes** — it never +recomputes. Dry-run rows have `mode = 'dry-run'` and `onChainTraceId = null`. + +## Verification + +```bash +npm run verify:report -- --date 2026-10-14 [--rpc URL] [--gateway URL] +npm run verify:report -- --trace-id 42 +``` + +The verifier uses only `@stellar/stellar-sdk` and the generated bindings: + +1. walks `market_trace_count(u64::MAX)` / `market_trace_at` (or uses + `--trace-id`) and parses each CID until the report's `period.from` matches + the date; +2. fetches the CID from the gateway, computes sha256 and compares it with the + on-chain `trace_hash`; +3. checks the trace agent equals the report agent and the treasury + `RiskConfig.agent`; +4. recomputes `totals.positionsMarkRaw` and `navRaw`; +5. follows `previous` and checks the previous trace hash/action; +6. reports whether the state moved since the anchor ledger. + +It exits non-zero on any failed check. diff --git a/package.json b/package.json index ed7eb80..a26debe 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,8 @@ "prisma:generate": "prisma generate", "prisma:studio": "prisma studio", "sync:contracts": "node scripts/sync-contracts.mjs", - "check:contracts": "node scripts/sync-contracts.mjs --check" + "check:contracts": "node scripts/sync-contracts.mjs --check", + "verify:report": "node -r ts-node/register/transpile-only scripts/verify-report.ts" }, "keywords": [], "author": "", diff --git a/prisma/migrations/20261017120000_transparency_reports_admin_ops_ipfs_content/migration.sql b/prisma/migrations/20261017120000_transparency_reports_admin_ops_ipfs_content/migration.sql new file mode 100644 index 0000000..bc3ae5a --- /dev/null +++ b/prisma/migrations/20261017120000_transparency_reports_admin_ops_ipfs_content/migration.sql @@ -0,0 +1,87 @@ +-- #100 local verified IPFS content store +CREATE TABLE "ipfs_content" ( + "cid" TEXT NOT NULL, + "sha256" TEXT NOT NULL, + "size" INTEGER NOT NULL, + "bytes" BYTEA NOT NULL, + "origin" TEXT NOT NULL, + "verifiedBy" TEXT, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "lastAccessAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "pinnedUntil" TIMESTAMP(3), + + CONSTRAINT "ipfs_content_pkey" PRIMARY KEY ("cid") +); + +CREATE INDEX "ipfs_content_sha256_idx" ON "ipfs_content"("sha256"); +CREATE INDEX "ipfs_content_lastAccessAt_idx" ON "ipfs_content"("lastAccessAt"); + +-- Defence in depth: the application checks both, but a direct writer cannot +-- store a row whose declared size/hash disagree with the bytes. A CID may only +-- ever map to one byte string (enforced by the primary key + put()). +ALTER TABLE "ipfs_content" + ADD CONSTRAINT "ipfs_content_size_matches_bytes" CHECK ("size" = octet_length("bytes")); +ALTER TABLE "ipfs_content" + ADD CONSTRAINT "ipfs_content_sha256_format" CHECK ("sha256" ~ '^[0-9a-f]{64}$'); + +-- #156 treasury admin operations console +CREATE TABLE "admin_operations" ( + "id" TEXT NOT NULL, + "kind" TEXT NOT NULL, + "params" JSONB NOT NULL, + "contractId" TEXT NOT NULL, + "sourceAccount" TEXT NOT NULL, + "unsignedXdr" TEXT NOT NULL, + "txHash" TEXT NOT NULL, + "signedXdr" TEXT, + "signatures" JSONB NOT NULL DEFAULT '[]'::jsonb, + "weightCollected" INTEGER NOT NULL DEFAULT 0, + "status" TEXT NOT NULL, + "resultXdr" TEXT, + "ledger" INTEGER, + "error" JSONB, + "requestedBy" TEXT NOT NULL, + "expiresAt" TIMESTAMP(3) NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "admin_operations_pkey" PRIMARY KEY ("id") +); + +CREATE UNIQUE INDEX "admin_operations_txHash_key" ON "admin_operations"("txHash"); +CREATE INDEX "admin_operations_status_createdAt_idx" ON "admin_operations"("status", "createdAt"); + +-- #159 daily transparency reports +CREATE TABLE "transparency_reports" ( + "id" TEXT NOT NULL, + "date" TEXT NOT NULL, + "anchorLedger" INTEGER NOT NULL, + "cid" TEXT NOT NULL, + "hash" TEXT NOT NULL, + "onChainTraceId" BIGINT, + "mode" TEXT NOT NULL, + "consistency" TEXT NOT NULL, + "report" JSONB NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "transparency_reports_pkey" PRIMARY KEY ("id") +); + +CREATE UNIQUE INDEX "transparency_reports_date_key" ON "transparency_reports"("date"); +CREATE INDEX "transparency_reports_anchorLedger_idx" ON "transparency_reports"("anchorLedger"); + +-- #159 immutable registry trace cache +CREATE TABLE "registry_trace_cache" ( + "traceId" BIGINT NOT NULL, + "agent" TEXT NOT NULL, + "marketId" BIGINT NOT NULL, + "action" TEXT NOT NULL, + "traceHash" TEXT NOT NULL, + "ipfsCid" TEXT NOT NULL, + "publishedAt" BIGINT NOT NULL, + "fetchedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "registry_trace_cache_pkey" PRIMARY KEY ("traceId") +); + +CREATE INDEX "registry_trace_cache_agent_action_idx" ON "registry_trace_cache"("agent", "action"); diff --git a/prisma/schema.prisma b/prisma/schema.prisma index dc21b3c..d60f34d 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -413,3 +413,84 @@ model SpendingAllowance { @@index([userId]) @@map("spending_allowances") } +/// Exact bytes of IPFS documents the backend pinned or verified (#100). +model IpfsContent { + cid String @id + /// sha256 hex of `bytes`; the only admission criterion. + sha256 String + size Int + bytes Bytes + /// PINNED | GATEWAY_VERIFIED + origin String + /// e.g. "pin" or "reasoning_registry:trace:12". + verifiedBy String? + createdAt DateTime @default(now()) + lastAccessAt DateTime @default(now()) + pinnedUntil DateTime? + + @@index([sha256]) + @@index([lastAccessAt]) + @@map("ipfs_content") +} + +/// Unsigned admin transactions prepared by the treasury operations console (#156). +model AdminOperation { + id String @id @default(uuid()) + kind String + params Json + contractId String + sourceAccount String + unsignedXdr String + /// Hash of the prepared transaction body; signatures do not change it. + txHash String @unique + signedXdr String? + signatures Json @default("[]") + weightCollected Int @default(0) + /// PREPARED | PARTIALLY_SIGNED | READY | SUBMITTED | SUCCESS | FAILED | EXPIRED | CANCELLED + status String + resultXdr String? + ledger Int? + error Json? + requestedBy String + expiresAt DateTime + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + @@index([status, createdAt]) + @@map("admin_operations") +} + +/// On-chain transparency reports published by the daily job (#159). +model TransparencyReport { + id String @id @default(uuid()) + /// UTC day, YYYY-MM-DD. + date String @unique + anchorLedger Int + cid String + hash String + onChainTraceId BigInt? + mode String + /// exact | best_effort + consistency String + report Json + createdAt DateTime @default(now()) + + @@index([anchorLedger]) + @@map("transparency_reports") +} + +/// Immutable reasoning-registry traces, cached so report counts do not walk +/// the whole registry on every run (#159). +model RegistryTraceCache { + traceId BigInt @id + agent String + marketId BigInt + action String + traceHash String + ipfsCid String + publishedAt BigInt + fetchedAt DateTime @default(now()) + + @@index([agent, action]) + @@map("registry_trace_cache") +} diff --git a/scripts/verify-report.ts b/scripts/verify-report.ts new file mode 100644 index 0000000..c4fe595 --- /dev/null +++ b/scripts/verify-report.ts @@ -0,0 +1,196 @@ +/** + * Independent transparency-report verifier (#159). + * + * Uses only @stellar/stellar-sdk + the generated bindings. No database. + * + * npm run verify:report -- --date 2026-10-14 [--rpc URL] [--gateway URL] + */ +import crypto from 'node:crypto'; +import { rpc } from '@stellar/stellar-sdk'; +import { Client as RegistryClient } from '../src/generated/reasoning-registry'; +import { Client as TreasuryClient } from '../src/generated/treasury'; +import deployments from '../src/generated/deployments.testnet.json'; + +export const REPORT_MARKET_ID = 18446744073709551615n; + +export interface VerifyOptions { + date?: string; + traceId?: string; + rpcUrl?: string; + gatewayUrl?: string; + networkPassphrase?: string; + contracts?: { reasoningRegistry?: string; treasury?: string }; + /** Test seam for unit tests; the CLI never passes this. */ + deps?: { + registry?: any; + treasury?: any; + fetch?: typeof fetch; + latestLedger?: () => Promise<{ sequence: number }>; + }; +} + +export interface VerifyCheck { + name: string; + ok: boolean; + detail: string; +} + +export interface VerifyResult { + ok: boolean; + traceId?: string; + cid?: string; + computedHash?: string; + checks: VerifyCheck[]; +} + +function unwrap(tx: { result: unknown }): T { + const result = tx.result as { isOk?: () => boolean; unwrap?: () => T }; + if (typeof result?.isOk === 'function') { + if (!result.isOk()) throw new Error('contract returned an error'); + return result.unwrap!(); + } + return result as T; +} + +function clientBase(contractId: string, rpcUrl: string, networkPassphrase: string) { + return { contractId, rpcUrl, networkPassphrase, allowHttp: rpcUrl.startsWith('http://') }; +} + +export async function verify(opts: VerifyOptions = {}): Promise { + const networkPassphrase = opts.networkPassphrase ?? process.env.STELLAR_NETWORK_PASSPHRASE ?? 'Test SDF Network ; September 2015'; + const rpcUrl = opts.rpcUrl ?? process.env.STELLAR_RPC_URL ?? 'https://soroban-testnet.stellar.org'; + const gateway = (opts.gatewayUrl ?? process.env.IPFS_GATEWAY_URL ?? 'https://gateway.pinata.cloud/ipfs').replace(/\/$/, ''); + const registryId = opts.contracts?.reasoningRegistry ?? process.env.REASONING_REGISTRY_CONTRACT_ID ?? deployments.contracts.reasoning_registry; + const treasuryId = opts.contracts?.treasury ?? process.env.TREASURY_CONTRACT_ID ?? deployments.contracts.treasury; + const registry = opts.deps?.registry ?? new RegistryClient(clientBase(registryId, rpcUrl, networkPassphrase)); + const treasury = opts.deps?.treasury ?? new TreasuryClient(clientBase(treasuryId, rpcUrl, networkPassphrase)); + const fetchImpl = opts.deps?.fetch ?? fetch; + const checks: VerifyCheck[] = []; + const add = (name: string, ok: boolean, detail: string) => checks.push({ name, ok, detail }); + + let traceId = opts.traceId ? BigInt(opts.traceId) : null; + if (traceId === null) { + const count = unwrap(await registry.market_trace_count({ market_id: REPORT_MARKET_ID })); + if (!opts.date) { + add('find_report', false, 'provide --date or --trace-id'); + return { ok: false, checks }; + } + for (let i = 0n; i < count; i++) { + const id = unwrap(await registry.market_trace_at({ market_id: REPORT_MARKET_ID, index: i })); + const trace = unwrap<{ ipfs_cid: string }>(await registry.get_trace({ trace_id: id })); + try { + const res = await fetchImpl(`${gateway}/${encodeURIComponent(trace.ipfs_cid)}`); + if (!res.ok) continue; + const doc = JSON.parse(Buffer.from(await res.arrayBuffer()).toString('utf8')) as { period?: { from?: string } }; + if (doc.period?.from?.slice(0, 10) === opts.date) { + traceId = id; + break; + } + } catch { + // keep scanning + } + } + } + + if (traceId === null) { + add('find_report', false, `no report trace found for ${opts.date}`); + return { ok: false, checks }; + } + add('find_report', true, `trace ${traceId.toString()}`); + + const trace = unwrap<{ + agent: string; + action: string; + ipfs_cid: string; + trace_hash: Buffer; + market_id: bigint; + }>(await registry.get_trace({ trace_id: traceId })); + add('sentinel_market', trace.market_id === REPORT_MARKET_ID, `market_id=${trace.market_id.toString()}`); + add('action', trace.action === 'report', `action=${trace.action}`); + + let bytes: Buffer; + try { + const res = await fetchImpl(`${gateway}/${encodeURIComponent(trace.ipfs_cid)}`); + if (!res.ok) throw new Error(`gateway ${res.status}`); + bytes = Buffer.from(await res.arrayBuffer()); + } catch (err) { + add('fetch_content', false, err instanceof Error ? err.message : String(err)); + return { ok: false, traceId: traceId.toString(), cid: trace.ipfs_cid, checks }; + } + add('fetch_content', true, `${bytes.length} bytes`); + + const computedHash = crypto.createHash('sha256').update(bytes).digest('hex'); + const onChainHash = Buffer.from(trace.trace_hash).toString('hex'); + add('content_hash', computedHash === onChainHash, `${computedHash} vs ${onChainHash}`); + + let report: any; + try { + report = JSON.parse(bytes.toString('utf8')); + } catch { + add('parse_document', false, 'content is not JSON'); + return { ok: false, traceId: traceId.toString(), cid: trace.ipfs_cid, computedHash, checks }; + } + add('parse_document', true, `${report.kind} v${report.schemaVersion}`); + + const risk = unwrap<{ agent: string }>(await treasury.get_config()); + add('report_agent_matches_trace', report.agent === trace.agent, `${report.agent} vs ${trace.agent}`); + add('agent_matches_treasury', trace.agent === risk.agent, `${trace.agent} vs ${risk.agent}`); + + const mark = (report.markets ?? []).reduce((sum: bigint, m: any) => sum + BigInt(m.markValueRaw), 0n); + const nav = BigInt(report.treasury.usdcBalanceRaw) + mark; + add('totals_mark', mark.toString() === report.totals.positionsMarkRaw, `${mark.toString()} vs ${report.totals.positionsMarkRaw}`); + add('totals_nav', nav.toString() === report.totals.navRaw, `${nav.toString()} vs ${report.totals.navRaw}`); + + if (report.previous) { + try { + const prevId = BigInt(report.previous.onChainTraceId); + const prev = unwrap<{ trace_hash: Buffer; action: string; ipfs_cid: string }>(await registry.get_trace({ trace_id: prevId })); + const prevHash = Buffer.from(prev.trace_hash).toString('hex'); + add('previous_hash_chain', prevHash === report.previous.traceHash, `${prevHash} vs ${report.previous.traceHash}`); + add('previous_is_report', prev.action === 'report', `action=${prev.action}`); + add('previous_cid', Boolean(prev.ipfs_cid), prev.ipfs_cid); + } catch (err) { + add('previous_hash_chain', false, err instanceof Error ? err.message : String(err)); + } + } else { + add('previous_hash_chain', true, 'genesis report (no previous)'); + } + + const latest = opts.deps?.latestLedger + ? await opts.deps.latestLedger() + : await new rpc.Server(rpcUrl, { allowHttp: rpcUrl.startsWith('http://') }).getLatestLedger(); + add('anchor_not_future', report.anchor.ledger <= latest.sequence, `anchor=${report.anchor.ledger}, latest=${latest.sequence}`); + if (report.anchor.ledger < latest.sequence) { + add('state_moved_since_anchor', true, `state moved ${latest.sequence - report.anchor.ledger} ledgers; re-simulation is advisory`); + } + + return { + ok: checks.every((c) => c.ok), + traceId: traceId.toString(), + cid: trace.ipfs_cid, + computedHash, + checks, + }; +} + +function arg(name: string): string | undefined { + const index = process.argv.indexOf(`--${name}`); + return index >= 0 ? process.argv[index + 1] : undefined; +} + +if (require.main === module) { + if (process.argv.includes('--help')) { + console.log('Usage: npm run verify:report -- --date YYYY-MM-DD [--rpc URL] [--gateway URL] [--trace-id ID]'); + process.exit(0); + } + verify({ date: arg('date'), traceId: arg('trace-id'), rpcUrl: arg('rpc'), gatewayUrl: arg('gateway') }) + .then((result) => { + for (const check of result.checks) console.log(`${check.ok ? 'ok ' : 'FAIL'} ${check.name}: ${check.detail}`); + console.log(result.ok ? '\nreport verified' : '\nreport verification FAILED'); + process.exit(result.ok ? 0 : 1); + }) + .catch((err) => { + console.error(err); + process.exit(1); + }); +} diff --git a/src/config/index.ts b/src/config/index.ts index 8ef478a..5f8606e 100644 --- a/src/config/index.ts +++ b/src/config/index.ts @@ -85,11 +85,39 @@ const envSchema = z // ── LLM providers (optional; generation fails at runtime without one) ─ ANTHROPIC_API_KEY: optionalString, GEMINI_API_KEY: optionalString, + // OpenAI or any OpenAI-compatible endpoint (used by the router, #157). + OPENAI_API_KEY: optionalString, + OPENAI_BASE_URL: optionalString.refine( + (v) => v === '' || /^https?:\/\//.test(v), + 'must be an http(s) URL', + ), + // JSON object keyed by LlmTask, or a path to a JSON file. Defaults to the + // built-in policy that reproduces the pre-router behaviour. + LLM_POLICY: optionalString, + LLM_POLICY_FILE: optionalString, // ── IPFS ────────────────────────────────────────────────────────────── PINATA_API_KEY: optionalString, PINATA_SECRET_API_KEY: optionalString, IPFS_GATEWAY_URL: z.string().url().default('https://gateway.pinata.cloud/ipfs'), + // Local verified content store (#100). Both are byte caps. + IPFS_CACHE_MAX_OBJECT_BYTES: z.string().default('262144').transform(Number) + .pipe(z.number().int().positive()), + IPFS_CACHE_BUDGET_BYTES: z.string().default('268435456').transform(Number) + .pipe(z.number().int().positive()), + + // ── Transparency reports (#159) ─────────────────────────────────────── + REPORTS_ENABLED: z.enum(['true', 'false']).default('false').transform((v) => v === 'true'), + REPORT_CRON: z.string().default('10 0 * * *'), + REPORT_MODE: z.enum(['dry-run', 'live']).default('dry-run'), + + // ── Treasury admin console (#156) ───────────────────────────────────── + // Lifetime of a prepared unsigned transaction. + TREASURY_OP_TTL_SECONDS: z.string().default('3600').transform(Number) + .pipe(z.number().int().positive()), + // Optional stricter backend policy for withdraw operations. + TREASURY_WITHDRAW_MIN_SIGNERS: z.string().default('0').transform(Number) + .pipe(z.number().int().min(0).max(20)), // ── Data sources ────────────────────────────────────────────────────── NEWSAPI_KEY: optionalString, diff --git a/src/controllers/admin.controller.ts b/src/controllers/admin.controller.ts new file mode 100644 index 0000000..41d707f --- /dev/null +++ b/src/controllers/admin.controller.ts @@ -0,0 +1,68 @@ +import type { Request, Response } from 'express'; +import { sendSuccess, parsePagination, buildPaginationMeta } from '../utils/response.util'; +import { createOperationSchema, recordSchema, signatureSchema } from '../validators/admin.validator'; +import { AppError } from '../middlewares/error.middleware'; +import { + addSignature, + cancelOperation, + getOperation, + getTreasuryState, + listOperations, + prepareOperation, + recordSubmitted, + submitOperation, +} from '../services/treasury-ops.service'; + +function actor(req: Request): string { + if (!req.user) throw new AppError(401, 'UNAUTHORIZED', 'Authentication required'); + return req.user.walletAddress; +} + +export async function getTreasury(req: Request, res: Response): Promise { + sendSuccess(res, await getTreasuryState()); +} + +export async function createOperation(req: Request, res: Response): Promise { + const parsed = createOperationSchema.safeParse(req.body); + if (!parsed.success) { + throw new AppError(400, 'VALIDATION_ERROR', 'Invalid operation', parsed.error.flatten().fieldErrors as any); + } + const prepared = await prepareOperation(parsed.data.operation, actor(req)); + sendSuccess(res, prepared, 201); +} + +export async function listTreasuryOperations(req: Request, res: Response): Promise { + const { page, limit } = parsePagination(req.query as any); + const { rows, total } = await listOperations(page, limit); + sendSuccess(res, rows, 200, buildPaginationMeta(page, limit, total) as any); +} + +export async function getTreasuryOperation(req: Request, res: Response): Promise { + sendSuccess(res, await getOperation(String(req.params.id))); +} + +export async function addOperationSignature(req: Request, res: Response): Promise { + const parsed = signatureSchema.safeParse(req.body); + if (!parsed.success) { + throw new AppError(400, 'VALIDATION_ERROR', 'signedXdr is required', parsed.error.flatten().fieldErrors as any); + } + sendSuccess(res, await addSignature(String(req.params.id), parsed.data.signedXdr, actor(req))); +} + +export async function submitTreasuryOperation(req: Request, res: Response): Promise { + sendSuccess(res, await submitOperation(String(req.params.id))); +} + +export async function recordTreasuryOperation(req: Request, res: Response): Promise { + const parsed = recordSchema.safeParse(req.body); + if (!parsed.success) { + throw new AppError(400, 'VALIDATION_ERROR', 'txHash is required', parsed.error.flatten().fieldErrors as any); + } + await recordSubmitted(String(req.params.id), parsed.data.txHash, actor(req)); + sendSuccess(res, { recorded: true }); +} + +export async function cancelTreasuryOperation(req: Request, res: Response): Promise { + await cancelOperation(String(req.params.id), actor(req)); + sendSuccess(res, { cancelled: true }); +} diff --git a/src/controllers/portfolio.controller.ts b/src/controllers/portfolio.controller.ts index d1e411a..d3f805e 100644 --- a/src/controllers/portfolio.controller.ts +++ b/src/controllers/portfolio.controller.ts @@ -24,7 +24,10 @@ export async function getPortfolio(req: Request, res: Response): Promise { }), ]); - sendSuccess(res, { ...summary, correlationRisk: correlations, availableCapitalRaw }); + const latestReport = await prisma.transparencyReport + .findFirst({ orderBy: { date: 'desc' }, select: { date: true, anchorLedger: true, cid: true, hash: true, onChainTraceId: true, mode: true, consistency: true } }) + .catch(() => null); + sendSuccess(res, { ...summary, correlationRisk: correlations, availableCapitalRaw, latestReport }); } /** diff --git a/src/cron/monitor.cron.ts b/src/cron/monitor.cron.ts index 92868ad..8735355 100644 --- a/src/cron/monitor.cron.ts +++ b/src/cron/monitor.cron.ts @@ -2,6 +2,8 @@ import cron from 'node-cron'; import { runTraderCycle } from '../agents/trader.agent'; import { expireStaleSubscriptions } from '../services/subscription.service'; import { logger } from '../lib/logger'; +import { config } from '../config'; +import { postgresContentStore } from '../services/content-store.service'; let traderRunning = false; @@ -41,5 +43,15 @@ export function startMonitorCron(): void { } }); + // IPFS content eviction (#100) — hourly at :30, bounded batches. + cron.schedule('30 * * * *', async () => { + try { + const result = await postgresContentStore.evict(config.IPFS_CACHE_BUDGET_BYTES); + if (result.removed > 0) logger.info(result, 'Monitor cron: evicted IPFS content'); + } catch (err) { + logger.error({ err }, 'Monitor cron: IPFS content eviction failed'); + } + }); + logger.info('Monitor cron started'); } \ No newline at end of file diff --git a/src/cron/report.cron.ts b/src/cron/report.cron.ts new file mode 100644 index 0000000..d99c975 --- /dev/null +++ b/src/cron/report.cron.ts @@ -0,0 +1,34 @@ +import cron from 'node-cron'; +import { config } from '../config'; +import { logger } from '../lib/logger'; +import { runReportJob } from '../services/transparency-report.service'; + +let running = false; + +/** Daily transparency report (#159), at REPORT_CRON (default 00:10 UTC). */ +export function startReportCron(): void { + if (!config.REPORTS_ENABLED) { + logger.info('REPORTS_ENABLED=false; transparency report cron disabled'); + return; + } + cron.schedule( + config.REPORT_CRON, + async () => { + if (running) { + logger.warn('Report cron: previous run still in progress — skipping'); + return; + } + running = true; + try { + const row = await runReportJob(); + logger.info({ date: row.date, cid: row.cid, mode: row.mode }, 'Transparency report published'); + } catch (err) { + logger.error({ err }, 'Report cron: transparency report failed'); + } finally { + running = false; + } + }, + { timezone: 'Etc/UTC' }, + ); + logger.info({ cron: config.REPORT_CRON }, 'Transparency report cron started'); +} diff --git a/src/lib/gemini.ts b/src/lib/gemini.ts index 5039149..062e64b 100644 --- a/src/lib/gemini.ts +++ b/src/lib/gemini.ts @@ -1,220 +1,58 @@ +/** + * Compatibility shim over the Gemini provider (#157). + * + * The routing, health and retry policy now live in src/lib/llm/router.ts. This + * module is kept so existing imports (and the generated default client export) + * continue to work. + */ import { GoogleGenerativeAI } from '@google/generative-ai'; import { config } from '../config'; -import { logger } from './logger'; - -// ─── Singleton client ───────────────────────────────────────────────────────── -const genAI = new GoogleGenerativeAI(config.GEMINI_API_KEY); +import { geminiProvider, isDailyQuotaError, parseRetryDelayMs } from './llm/providers/gemini'; +import type { ModelSpec } from './llm/types'; export const GEMINI_MODEL = 'gemini-2.0-flash'; +export { isDailyQuotaError, parseRetryDelayMs }; -// ─── Per-call interval guard ────────────────────────────────────────────────── -// The ingestion cron (every 15 min) and POST /markets/generate can fire at -// nearly the same time. Without a guard they both call Gemini within the same -// second, doubling RPM consumption and burning through quota twice as fast. -// A 4-second minimum interval between calls is enough to prevent this overlap. -let _lastCallAt = 0; -const MIN_CALL_INTERVAL_MS = 4_000; - -async function waitForCallSlot(): Promise { - const elapsed = Date.now() - _lastCallAt; - if (_lastCallAt > 0 && elapsed < MIN_CALL_INTERVAL_MS) { - const wait = MIN_CALL_INTERVAL_MS - elapsed; - logger.debug({ waitMs: wait }, 'Gemini call guard: throttling request'); - await new Promise(r => setTimeout(r, wait)); - } - _lastCallAt = Date.now(); -} - -// ─── Circuit breaker ────────────────────────────────────────────────────────── -// When Gemini returns "limit: 0" or a daily quota exhaustion error, retrying -// is pointless — the quota won't reset until midnight UTC. The circuit breaker -// records this state and fast-fails all subsequent calls for QUOTA_COOLDOWN_MS -// so the app doesn't hammer a wall every 15 minutes. -// -// Cooldown is 1 hour by default. The cron will try again after 4 cycles (1h), -// which is useful if the user has billing set up (quota resets faster) or -// rotates to a paid key. - -let _quotaExhaustedUntil = 0; -const QUOTA_COOLDOWN_MS = 60 * 60 * 1000; // 1 hour - -function isQuotaExhausted(): boolean { - return Date.now() < _quotaExhaustedUntil; -} - -function markQuotaExhausted(): void { - _quotaExhaustedUntil = Date.now() + QUOTA_COOLDOWN_MS; - logger.error( - { cooldownUntil: new Date(_quotaExhaustedUntil).toISOString() }, - 'Gemini daily quota exhausted — circuit breaker open. Calls will be skipped for 1 hour.', - ); -} - -// Returns true when the error is a daily quota exhaustion (not just RPM) -function isDailyQuotaError(errMsg: string): boolean { - return ( - errMsg.includes('GenerateRequestsPerDayPerProject') || - errMsg.includes('GenerateContentInputTokensPerModelPerMinute-FreeTier') || - // The free tier error says "limit: 0" on the daily quota metric - (errMsg.includes('limit: 0') && errMsg.includes('free_tier')) - ); -} - -// ─── Retry delay parser ─────────────────────────────────────────────────────── -// The 429 body contains: "Please retry in 33.138887932s." -// and a RetryInfo object: "retryDelay":"33s" -// -// We respect the API's own suggestion. The original 2s/4s/6s backoff fired -// all three retries while still rate-limited and wasted every attempt. -// We add a 3s safety buffer on top of the suggested delay. -function parseRetryDelayMs(errorMessage: string): number { - const floatMatch = errorMessage.match(/[Rr]etry in (\d+(?:\.\d+)?)s/); - if (floatMatch) { - return Math.ceil(parseFloat(floatMatch[1])) * 1000 + 3_000; - } - const jsonMatch = errorMessage.match(/"retryDelay"\s*:\s*"(\d+)s"/); - if (jsonMatch) { - return (parseInt(jsonMatch[1], 10) + 3) * 1000; - } - return 30_000; // conservative fallback -} +const spec = (maxTokens: number, temperature: number): ModelSpec => ({ + provider: 'gemini', + model: GEMINI_MODEL, + temperature, + maxTokens, + timeoutMs: 60_000, +}); -// ─── callGemini — plain text output ─────────────────────────────────────────── -// Use for summaries, analysis narratives (non-JSON). export async function callGemini( systemPrompt: string, userPrompt: string, maxTokens = 2048, ): Promise { - if (isQuotaExhausted()) { - throw new Error( - `Gemini quota circuit breaker is open until ${new Date(_quotaExhaustedUntil).toISOString()}. ` + - 'Add billing to your Google AI account or wait for the quota to reset.', - ); - } - - await waitForCallSlot(); - - try { - const model = genAI.getGenerativeModel({ - model: GEMINI_MODEL, - systemInstruction: systemPrompt, - generationConfig: { - maxOutputTokens: maxTokens, - temperature: 0.3, - topP: 0.95, - }, - }); - - const result = await model.generateContent(userPrompt); - const text = result.response.text(); - - if (!text || text.trim() === '') { - throw new Error('Gemini returned an empty text response'); - } - - return text; - } catch (err: any) { - logger.error({ err: err?.message }, 'Gemini text call failed'); - throw err; - } + const result = await geminiProvider.complete(spec(maxTokens, 0.3), { + task: 'generic_text', + system: systemPrompt, + user: userPrompt, + mode: 'text', + }); + return result.text; } -// ─── callGeminiJSON — JSON-forced output ────────────────────────────────────── -// `responseMimeType: 'application/json'` tells Gemini to return ONLY valid -// JSON — no markdown fences, no explanation. -// -// Retry strategy: -// - On 429/RESOURCE_EXHAUSTED: parse the API's own retryDelay and wait that -// long before retrying. Max 2 retries (3 total attempts). -// - On daily quota exhaustion: open the circuit breaker and throw immediately. -// No point retrying when the daily cap is hit. -// - On any other error: throw immediately (no retry). export async function callGeminiJSON( systemPrompt: string, userPrompt: string, maxTokens = 2048, ): Promise { - - // ── Circuit breaker check ───────────────────────────────────────────────── - if (isQuotaExhausted()) { - throw new Error( - `Gemini quota circuit breaker is open until ${new Date(_quotaExhaustedUntil).toISOString()}. ` + - 'Add billing to your Google AI account or wait for the quota to reset.', - ); - } - - await waitForCallSlot(); - - let lastError: any; - - for (let attempt = 1; attempt <= 3; attempt++) { - try { - const model = genAI.getGenerativeModel({ - model: GEMINI_MODEL, - systemInstruction: systemPrompt, - generationConfig: { - responseMimeType: 'application/json', - maxOutputTokens: maxTokens, - temperature: 0.1, - topP: 0.90, - }, - }); - - const result = await model.generateContent(userPrompt); - const text = result.response.text(); - - if (!text || text.trim() === '') { - throw new Error('Gemini returned empty JSON response'); - } - - // Belt-and-suspenders strip even though responseMimeType prevents fences - const cleaned = text - .trim() - .replace(/^```json\s*/i, '') - .replace(/^```\s*/i, '') - .replace(/\s*```$/i, '') - .trim(); - - return cleaned; - - } catch (err: any) { - lastError = err; - - const errMsg: string = err?.message ?? String(err); - const is429 = errMsg.includes('429') || - errMsg.includes('Too Many Requests') || - errMsg.includes('RESOURCE_EXHAUSTED'); - - if (!is429) { - // Not a rate-limit error — fail immediately, no retry - break; - } - - // ── Check for daily quota exhaustion (not just RPM) ─────────────────── - if (isDailyQuotaError(errMsg)) { - markQuotaExhausted(); - break; // Circuit breaker opened — stop retrying now - } - - if (attempt >= 3) break; // Out of retries - - // ── RPM limit (not daily) — wait the full suggested delay ───────────── - const delayMs = parseRetryDelayMs(errMsg); - logger.warn( - { attempt, retryAfterMs: delayMs }, - `Gemini 429 (RPM) — waiting ${(delayMs / 1000).toFixed(0)}s before retry`, - ); - await new Promise(resolve => setTimeout(resolve, delayMs)); - } - } - - logger.error( - { err: lastError?.message?.slice(0, 300) }, - 'Gemini JSON call failed permanently', - ); - - throw lastError; + const result = await geminiProvider.complete(spec(maxTokens, 0.1), { + task: 'generic_json', + system: systemPrompt, + user: userPrompt, + mode: 'json', + }); + return result.text + .trim() + .replace(/^```json\s*/i, '') + .replace(/^```\s*/i, '') + .replace(/\s*```$/i, '') + .trim(); } -export default genAI; \ No newline at end of file +const genAI = new GoogleGenerativeAI(config.GEMINI_API_KEY); +export default genAI; diff --git a/src/lib/llm.ts b/src/lib/llm.ts index 9aaabbb..72f1c8a 100644 --- a/src/lib/llm.ts +++ b/src/lib/llm.ts @@ -1,144 +1,59 @@ -/** * - * Provider priority: - * 1. Anthropic Claude (claude-sonnet-4-20250514) — if ANTHROPIC_API_KEY is set in .env - * 2. Gemini (gemini-2.0-flash) — automatic fallback +/** + * LLM entry point (#157). * - * Why this exists: - * Gemini's free tier has a hard daily quota. When that quota is exhausted, - * even a new API key from a new account won't help until midnight UTC — - * because Google tracks quota per project, not per key rotation. - * Claude (via @anthropic-ai/sdk, already in package.json) has a separate, - * generous free tier that's unaffected by Gemini's exhaustion. - * - * This file lets the app route to Claude first, so Gemini is only called - * if Claude is unavailable. Both providers produce identical output shapes. - * - * Call-interval guard: - * Prevents the ingestion cron and a simultaneous POST /markets/generate - * from firing two LLM calls within the same second and burning quota twice. + * All calls go through the provider-agnostic router. The default policy + * reproduces the historical behaviour (Anthropic Claude first, Gemini + * fallback) while allowing per-task policies via LLM_POLICY / LLM_POLICY_FILE. */ - -import Anthropic from '@anthropic-ai/sdk'; -import { config } from '../config'; -import { logger } from './logger'; -import { callGeminiJSON } from './gemini'; - -const CLAUDE_MODEL = 'claude-sonnet-4-6'; - -// ─── Lazy Anthropic client ──────────────────────────────────────────────────── -let _anthropic: Anthropic | null = null; - -function getAnthropicClient(): Anthropic | null { - if (!config.ANTHROPIC_API_KEY) return null; - if (!_anthropic) { - _anthropic = new Anthropic({ apiKey: config.ANTHROPIC_API_KEY }); - } - return _anthropic; +import { createRouter } from './llm/router'; +import { policies } from './llm/policy'; +import { anthropicProvider, geminiProvider, openaiProvider } from './llm/providers'; +import type { LlmRequest, LlmResult, LlmTask } from './llm/types'; + +export const llmRouter = createRouter(policies, { + anthropic: anthropicProvider, + gemini: geminiProvider, + openai: openaiProvider, +}); + +export type { LlmRequest, LlmResult, LlmTask }; +export { LlmError } from './llm/types'; + +/** Complete a request and return the text plus provenance. */ +export async function completeLLM(req: LlmRequest): Promise { + return llmRouter.complete(req); } -// ─── Inter-call interval guard ──────────────────────────────────────────────── -let _lastCallAt = 0; -const MIN_INTERVAL_MS = 5_000; // 5 seconds between calls - -async function waitForSlot(): Promise { - const elapsed = Date.now() - _lastCallAt; - if (_lastCallAt > 0 && elapsed < MIN_INTERVAL_MS) { - const wait = MIN_INTERVAL_MS - elapsed; - logger.debug({ waitMs: wait }, 'LLM call guard: throttling request'); - await new Promise(r => setTimeout(r, wait)); - } - _lastCallAt = Date.now(); -} - -// ─── callLLMJSON — JSON-forced output ───────────────────────────────────────── -// Tries Anthropic Claude first. Falls back to Gemini automatically. -// Returns a clean JSON string ready for JSON.parse(). +/** Backwards-compatible JSON wrapper. Invalid JSON triggers provider failover. */ export async function callLLMJSON( systemPrompt: string, - userPrompt: string, - maxTokens = 2048, + userPrompt: string, + maxTokens = 2048, + task: LlmTask = 'generic_json', ): Promise { - await waitForSlot(); - - const anthropic = getAnthropicClient(); - - if (anthropic) { - try { - const message = await anthropic.messages.create({ - model: CLAUDE_MODEL, - max_tokens: maxTokens, - temperature: 0.2, // Low temperature for deterministic, well-structured JSON - system: - systemPrompt + - '\n\nCRITICAL: Your response must be ONLY a valid JSON object. ' + - 'No markdown fences, no preamble, no explanation — pure JSON.', - messages: [{ role: 'user', content: userPrompt }], - }); - - const block = message.content[0]; - if (block.type !== 'text' || !block.text.trim()) { - throw new Error('Anthropic returned an empty response'); - } - - const cleaned = block.text - .trim() - .replace(/^```json\s*/i, '') - .replace(/^```\s*/i, '') - .replace(/\s*```$/i, '') - .trim(); - - logger.debug({ model: CLAUDE_MODEL }, 'LLM call served by Anthropic Claude'); - return cleaned; - - } catch (err: any) { - logger.warn( - { err: err?.message?.slice(0, 200) }, - 'Anthropic Claude failed — falling back to Gemini', - ); - // Fall through to Gemini - } - } else { - logger.debug('ANTHROPIC_API_KEY not set — routing directly to Gemini'); - } - - logger.debug('LLM call served by Gemini'); - return callGeminiJSON(systemPrompt, userPrompt, maxTokens); + const result = await completeLLM({ + task, + system: systemPrompt, + user: userPrompt, + mode: 'json', + maxTokens, + }); + return result.text; } -// ─── callLLMText — plain text output ───────────────────────────────────────── -// For summaries and narratives that don't need strict JSON. +/** Backwards-compatible plain-text wrapper. Never requests a JSON MIME type. */ export async function callLLMText( systemPrompt: string, - userPrompt: string, - maxTokens = 2048, + userPrompt: string, + maxTokens = 2048, + task: LlmTask = 'generic_text', ): Promise { - await waitForSlot(); - - const anthropic = getAnthropicClient(); - - if (anthropic) { - try { - const message = await anthropic.messages.create({ - model: CLAUDE_MODEL, - max_tokens: maxTokens, - temperature: 0.4, // Moderate temperature for narrative text - system: systemPrompt, - messages: [{ role: 'user', content: userPrompt }], - }); - - const block = message.content[0]; - if (block.type !== 'text' || !block.text.trim()) { - throw new Error('Anthropic returned an empty response'); - } - return block.text.trim(); - - } catch (err: any) { - logger.warn( - { err: err?.message?.slice(0, 200) }, - 'Anthropic Claude text call failed — falling back to Gemini', - ); - } - } - - return callGeminiJSON(systemPrompt, userPrompt, maxTokens); -} \ No newline at end of file + const result = await completeLLM({ + task, + system: systemPrompt, + user: userPrompt, + mode: 'text', + maxTokens, + }); + return result.text; +} diff --git a/src/lib/llm/policy.ts b/src/lib/llm/policy.ts new file mode 100644 index 0000000..3c94577 --- /dev/null +++ b/src/lib/llm/policy.ts @@ -0,0 +1,87 @@ +import { readFileSync } from 'fs'; +import { z } from 'zod'; +import { config } from '../../config'; +import type { LlmTask, ModelSpec } from './types'; + +export type LlmPolicies = Record; + +const TASKS = [ + 'market_generation', + 'hedge_conditions', + 'forecasting', + 'attestation', + 'generic_json', + 'generic_text', +] as const; + +const modelSpecSchema = z.object({ + provider: z.enum(['anthropic', 'gemini', 'openai']), + model: z.string().min(1), + temperature: z.number().min(0).max(2), + maxTokens: z.number().int().positive(), + timeoutMs: z.number().int().positive(), + baseUrl: z.string().url().optional(), + jsonMode: z.enum(['native', 'prompt']).optional(), + retries: z.number().int().min(0).max(5).optional(), +}); + +const policySchema = z.partialRecord( + z.enum(TASKS), + z.array(modelSpecSchema).min(1), +); + +const anthropic = (model = 'claude-sonnet-4-6', temperature = 0.2, maxTokens = 4096): ModelSpec => ({ + provider: 'anthropic', + model, + temperature, + maxTokens, + timeoutMs: 60_000, +}); + +const gemini = (temperature = 0.1, maxTokens = 4096): ModelSpec => ({ + provider: 'gemini', + model: 'gemini-2.0-flash', + temperature, + maxTokens, + timeoutMs: 60_000, +}); + +/** Reproduces the pre-router behaviour: Claude first, Gemini as fallback. */ +export const DEFAULT_POLICIES: LlmPolicies = { + market_generation: [anthropic('claude-sonnet-4-6', 0.2, 4096), gemini(0.1, 4096)], + hedge_conditions: [anthropic('claude-sonnet-4-6', 0.2, 1024), gemini(0.1, 1024)], + forecasting: [anthropic('claude-sonnet-4-6', 0.2, 2048), gemini(0.1, 2048)], + attestation: [anthropic('claude-sonnet-4-6', 0.1, 1024), gemini(0.1, 1024)], + generic_json: [anthropic('claude-sonnet-4-6', 0.2, 2048), gemini(0.1, 2048)], + generic_text: [anthropic('claude-sonnet-4-6', 0.4, 2048), gemini(0.3, 2048)], +}; + +function sourceJson(): string | null { + if (config.LLM_POLICY_FILE) { + return readFileSync(config.LLM_POLICY_FILE, 'utf8'); + } + return config.LLM_POLICY || null; +} + +/** Parses and validates LLM_POLICY / LLM_POLICY_FILE, naming the bad task. */ +export function loadPolicies(raw = sourceJson()): LlmPolicies { + if (!raw) return DEFAULT_POLICIES; + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch (err) { + throw new Error(`LLM_POLICY is not valid JSON: ${err instanceof Error ? err.message : String(err)}`); + } + const result = policySchema.safeParse(parsed); + if (!result.success) { + const lines = result.error.issues.map((issue) => { + const task = String(issue.path[0] ?? '(root)'); + return ` ${task}: ${issue.message}`; + }); + throw new Error(`Invalid LLM_POLICY:\n${lines.join('\n')}`); + } + // A policy may override only some tasks; the rest keep the built-in default. + return { ...DEFAULT_POLICIES, ...(result.data as Partial) }; +} + +export const policies: LlmPolicies = loadPolicies(); diff --git a/src/lib/llm/providers/anthropic.ts b/src/lib/llm/providers/anthropic.ts new file mode 100644 index 0000000..e27638e --- /dev/null +++ b/src/lib/llm/providers/anthropic.ts @@ -0,0 +1,77 @@ +import Anthropic from '@anthropic-ai/sdk'; +import { config } from '../../../config'; +import type { LlmErrorKind, LlmProvider, LlmRequest, ModelSpec, ProviderCompletion } from '../types'; +import { LlmError } from '../types'; + +const cache = new Map(); + +function clientFor(spec: ModelSpec): Anthropic { + const key = `${config.ANTHROPIC_API_KEY}|${spec.baseUrl ?? ''}`; + let client = cache.get(key); + if (!client) { + client = new Anthropic({ apiKey: config.ANTHROPIC_API_KEY, ...(spec.baseUrl ? { baseURL: spec.baseUrl } : {}) }); + cache.set(key, client); + } + return client; +} + +function mapError(err: unknown): LlmError { + const status = (err as { status?: number })?.status; + const name = (err as { name?: string })?.name ?? ''; + let kind: LlmErrorKind = 'server'; + let retryAfterMs: number | undefined; + if (status === 429) { + kind = 'rate_limited'; + const header = (err as { headers?: Record })?.headers?.['retry-after']; + if (header) { + const seconds = Number(header); + if (Number.isFinite(seconds)) retryAfterMs = seconds * 1000; + } + } else if (status === 401 || status === 403) kind = 'auth'; + else if (status === 400) kind = 'invalid_request'; + else if (name.includes('Timeout') || name.includes('APIConnectionTimeout')) kind = 'timeout'; + else if (status === 529 || (status !== undefined && status >= 500)) kind = 'server'; + return new LlmError(kind, 'anthropic', `Anthropic call failed: ${(err as Error)?.message ?? String(err)}`, retryAfterMs, [], err); +} + +export const anthropicProvider: LlmProvider = { + name: 'anthropic', + available: () => Boolean(config.ANTHROPIC_API_KEY), + async complete(spec, req): Promise { + let message: Anthropic.Message; + try { + message = await clientFor(spec).messages.create( + { + model: spec.model, + max_tokens: Math.min(req.maxTokens ?? spec.maxTokens, spec.maxTokens), + temperature: spec.temperature, + system: + req.mode === 'json' + ? `${req.system}\n\nCRITICAL: Your response must be ONLY a valid JSON object. No markdown fences, no preamble, no explanation — pure JSON.` + : req.system, + messages: [{ role: 'user', content: req.user }], + }, + { timeout: spec.timeoutMs, signal: req.signal }, + ); + } catch (err) { + throw mapError(err); + } + + const text = message.content + .filter((block): block is Anthropic.TextBlock => block.type === 'text') + .map((block) => block.text) + .join('') + .trim(); + if (!text) throw new LlmError('empty', 'anthropic', 'Anthropic returned an empty response', undefined, []); + if (req.mode === 'json' && message.stop_reason === 'max_tokens') { + throw new LlmError('parse', 'anthropic', 'Anthropic response was truncated at max_tokens', undefined, []); + } + return { + text, + usage: { + inputTokens: message.usage?.input_tokens, + outputTokens: message.usage?.output_tokens, + }, + }; + }, +}; diff --git a/src/lib/llm/providers/gemini.ts b/src/lib/llm/providers/gemini.ts new file mode 100644 index 0000000..6bccc75 --- /dev/null +++ b/src/lib/llm/providers/gemini.ts @@ -0,0 +1,90 @@ +import { GoogleGenerativeAI } from '@google/generative-ai'; +import { config } from '../../../config'; +import type { LlmErrorKind, LlmProvider, LlmRequest, ModelSpec, ProviderCompletion } from '../types'; +import { LlmError } from '../types'; + +const cache = new Map(); + +function clientFor(): GoogleGenerativeAI { + let client = cache.get('default'); + if (!client) { + client = new GoogleGenerativeAI(config.GEMINI_API_KEY); + cache.set('default', client); + } + return client; +} + +export function isDailyQuotaError(errMsg: string): boolean { + return ( + errMsg.includes('GenerateRequestsPerDayPerProject') || + errMsg.includes('GenerateContentInputTokensPerModelPerMinute-FreeTier') || + (errMsg.includes('limit: 0') && errMsg.includes('free_tier')) + ); +} + +export function parseRetryDelayMs(errorMessage: string): number { + const floatMatch = errorMessage.match(/[Rr]etry in (\d+(?:\.\d+)?)s/); + if (floatMatch) return Math.ceil(parseFloat(floatMatch[1])) * 1000 + 3_000; + const jsonMatch = errorMessage.match(/"retryDelay"\s*:\s*"(\d+)s"/); + if (jsonMatch) return (parseInt(jsonMatch[1], 10) + 3) * 1000; + return 30_000; +} + +function mapError(err: unknown): LlmError { + const message = (err as Error)?.message ?? String(err); + if (isDailyQuotaError(message)) { + return new LlmError('quota_exhausted', 'gemini', message, 60 * 60 * 1000, [], err); + } + if (message.includes('429') || message.includes('RESOURCE_EXHAUSTED') || message.includes('Too Many Requests')) { + return new LlmError('rate_limited', 'gemini', message, parseRetryDelayMs(message), [], err); + } + if (message.includes('API_KEY_INVALID') || message.includes('401') || message.includes('403')) { + return new LlmError('auth', 'gemini', message, undefined, [], err); + } + if (message.toLowerCase().includes('timeout')) { + return new LlmError('timeout', 'gemini', message, undefined, [], err); + } + let kind: LlmErrorKind = 'server'; + try { + const parsed = JSON.parse(message.slice(message.indexOf('{'))); // best-effort JSON error body + if (parsed?.error?.code === 400) kind = 'invalid_request'; + } catch { + // keep server + } + return new LlmError(kind, 'gemini', message, undefined, [], err); +} + +export const geminiProvider: LlmProvider = { + name: 'gemini', + available: () => Boolean(config.GEMINI_API_KEY), + async complete(spec, req: LlmRequest): Promise { + try { + const model = clientFor().getGenerativeModel({ + model: spec.model, + systemInstruction: req.system, + generationConfig: { + // Text mode must never request JSON; this is the correctness bug the + // router replaces (the old text path called callGeminiJSON). + ...(req.mode === 'json' ? { responseMimeType: 'application/json' as const } : {}), + maxOutputTokens: Math.min(req.maxTokens ?? spec.maxTokens, spec.maxTokens), + temperature: spec.temperature, + topP: req.mode === 'json' ? 0.9 : 0.95, + }, + }); + const result = await model.generateContent(req.user); + const text = result.response.text()?.trim(); + if (!text) throw new LlmError('empty', 'gemini', 'Gemini returned an empty response', undefined, []); + const usage = result.response.usageMetadata; + return { + text, + usage: { + inputTokens: usage?.promptTokenCount, + outputTokens: usage?.candidatesTokenCount, + }, + }; + } catch (err) { + if (err instanceof LlmError) throw err; + throw mapError(err); + } + }, +}; diff --git a/src/lib/llm/providers/index.ts b/src/lib/llm/providers/index.ts new file mode 100644 index 0000000..606ea6b --- /dev/null +++ b/src/lib/llm/providers/index.ts @@ -0,0 +1,3 @@ +export { anthropicProvider } from './anthropic'; +export { geminiProvider } from './gemini'; +export { openaiProvider } from './openai'; diff --git a/src/lib/llm/providers/openai.ts b/src/lib/llm/providers/openai.ts new file mode 100644 index 0000000..144cfe9 --- /dev/null +++ b/src/lib/llm/providers/openai.ts @@ -0,0 +1,67 @@ +import OpenAI from 'openai'; +import { config } from '../../../config'; +import type { LlmErrorKind, LlmProvider, LlmRequest, ModelSpec, ProviderCompletion } from '../types'; +import { LlmError } from '../types'; + +const cache = new Map(); + +function clientFor(spec: ModelSpec): OpenAI { + const key = `${config.OPENAI_API_KEY}|${spec.baseUrl ?? config.OPENAI_BASE_URL}`; + let client = cache.get(key); + if (!client) { + client = new OpenAI({ + apiKey: config.OPENAI_API_KEY, + ...(spec.baseUrl || config.OPENAI_BASE_URL + ? { baseURL: spec.baseUrl || config.OPENAI_BASE_URL } + : {}), + }); + cache.set(key, client); + } + return client; +} + +function mapError(err: unknown): LlmError { + const status = (err as { status?: number })?.status; + let kind: LlmErrorKind = 'server'; + if (status === 429) kind = 'rate_limited'; + else if (status === 401 || status === 403) kind = 'auth'; + else if (status === 400) kind = 'invalid_request'; + else if ((err as { name?: string })?.name?.includes('Timeout')) kind = 'timeout'; + return new LlmError(kind, 'openai', `OpenAI-compatible call failed: ${(err as Error)?.message ?? String(err)}`, undefined, [], err); +} + +export const openaiProvider: LlmProvider = { + name: 'openai', + available: () => Boolean(config.OPENAI_API_KEY), + async complete(spec, req: LlmRequest): Promise { + const json = req.mode === 'json'; + const promptOnlyJson = json && spec.jsonMode === 'prompt'; + try { + const completion = await clientFor(spec).chat.completions.create( + { + model: spec.model, + temperature: spec.temperature, + max_tokens: Math.min(req.maxTokens ?? spec.maxTokens, spec.maxTokens), + messages: [ + { role: 'system', content: promptOnlyJson ? `${req.system}\n\nReturn only a valid JSON object.` : req.system }, + { role: 'user', content: req.user }, + ], + ...(json && !promptOnlyJson ? { response_format: { type: 'json_object' as const } } : {}), + }, + { timeout: spec.timeoutMs, signal: req.signal }, + ); + const text = completion.choices?.[0]?.message?.content?.trim(); + if (!text) throw new LlmError('empty', 'openai', 'OpenAI-compatible provider returned an empty response', undefined, []); + return { + text, + usage: { + inputTokens: completion.usage?.prompt_tokens, + outputTokens: completion.usage?.completion_tokens, + }, + }; + } catch (err) { + if (err instanceof LlmError) throw err; + throw mapError(err); + } + }, +}; diff --git a/src/lib/llm/router.ts b/src/lib/llm/router.ts new file mode 100644 index 0000000..e27833f --- /dev/null +++ b/src/lib/llm/router.ts @@ -0,0 +1,244 @@ +import { logger } from '../logger'; +import type { + Attempt, + LlmErrorKind, + LlmProvider, + LlmRequest, + LlmResult, + ModelSpec, +} from './types'; +import { LlmError } from './types'; +import type { LlmPolicies } from './policy'; + +export interface ProviderHealth { + state: 'healthy' | 'cooling' | 'open'; + consecutiveFailures: number; + until?: number; + lastError?: LlmErrorKind; +} + +export interface Router { + complete(req: LlmRequest): Promise; + health(): Record; +} + +export interface RouterOptions { + clock?: () => number; + sleep?: (ms: number) => Promise; + onAttempt?: (attempt: Attempt) => void; + retryInlineMaxMs?: number; + quotaCooldownMs?: number; + failureCooldownMs?: number; + failureThreshold?: number; +} + +const RETRY_INLINE_MAX_MS = 5_000; +const QUOTA_COOLDOWN_MS = 60 * 60 * 1000; +const FAILURE_COOLDOWN_MS = 60_000; +const FAILURE_THRESHOLD = 3; + +function stripFences(text: string): string { + return text + .trim() + .replace(/^```json\s*/i, '') + .replace(/^```\s*/i, '') + .replace(/\s*```$/i, '') + .trim(); +} + +function asLlmError(err: unknown, provider: string): LlmError { + if (err instanceof LlmError) return err; + const message = (err as Error)?.message ?? String(err); + const status = (err as { status?: number })?.status; + let kind: LlmErrorKind = 'server'; + if (status === 429) kind = 'rate_limited'; + else if (status === 401 || status === 403) kind = 'auth'; + else if (status === 400) kind = 'invalid_request'; + else if ((err as { name?: string })?.name?.includes('Timeout')) kind = 'timeout'; + return new LlmError(kind, provider, message, undefined, [], err); +} + +export function createRouter( + policies: LlmPolicies, + providers: Partial>, + options: RouterOptions = {}, +): Router { + const clock = options.clock ?? Date.now; + const sleep = options.sleep ?? ((ms: number) => new Promise((r) => setTimeout(r, ms))); + const retryInlineMaxMs = options.retryInlineMaxMs ?? RETRY_INLINE_MAX_MS; + const quotaCooldownMs = options.quotaCooldownMs ?? QUOTA_COOLDOWN_MS; + const failureCooldownMs = options.failureCooldownMs ?? FAILURE_COOLDOWN_MS; + const failureThreshold = options.failureThreshold ?? FAILURE_THRESHOLD; + + const health = new Map(); + const stateOf = (provider: string): ProviderHealth => { + const current = health.get(provider); + if (!current) return { state: 'healthy', consecutiveFailures: 0 }; + if (current.until !== undefined && current.until <= clock() && current.state !== 'healthy') { + return { state: 'healthy', consecutiveFailures: current.consecutiveFailures }; + } + return current; + }; + const setHealth = (provider: string, next: ProviderHealth) => health.set(provider, next); + + return { + health() { + return Object.fromEntries(Object.keys(providers).map((name) => [name, stateOf(name)])); + }, + + async complete(req) { + const policy = policies[req.task]; + if (!policy || policy.length === 0) { + throw new LlmError('invalid_request', 'router', `No LLM policy configured for task "${req.task}"`); + } + + const attempts: Attempt[] = []; + const startedAt = clock(); + let lastError: LlmError | null = null; + let previousProvider: string | null = null; + + for (const spec of policy) { + const provider = providers[spec.provider]; + if (!provider || !provider.available()) { + // Missing key is a configuration skip, not a provider failure. + continue; + } + + const current = stateOf(spec.provider); + if ((current.state === 'cooling' || current.state === 'open') && (current.until ?? 0) > clock()) { + previousProvider = spec.provider; + continue; + } + + if (previousProvider && previousProvider !== spec.provider) { + logger.warn( + { task: req.task, from: previousProvider, to: spec.provider, kind: lastError?.kind }, + 'LLM failover', + ); + } + + const maxAttempts = 1 + Math.max(0, spec.retries ?? 1); + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + const attemptStarted = clock(); + try { + const completion = await provider.complete(spec, req); + const text = req.mode === 'json' ? stripFences(completion.text) : completion.text.trim(); + if (req.mode === 'json') { + try { + JSON.parse(text); + } catch { + throw new LlmError('parse', spec.provider, 'LLM returned non-parseable JSON', undefined, [], undefined); + } + } + const succeeded: Attempt = { + provider: spec.provider, + model: spec.model, + latencyMs: clock() - attemptStarted, + }; + attempts.push(succeeded); + options.onAttempt?.(succeeded); + setHealth(spec.provider, { state: 'healthy', consecutiveFailures: 0 }); + return { + text, + provenance: { + task: req.task, + provider: spec.provider, + model: spec.model, + attempts, + latencyMs: clock() - startedAt, + usage: completion.usage, + }, + }; + } catch (err) { + const normalized = asLlmError(err, spec.provider); + lastError = normalized; + const failed: Attempt = { + provider: spec.provider, + model: spec.model, + kind: normalized.kind, + message: normalized.message.slice(0, 200), + latencyMs: clock() - attemptStarted, + retryAfterMs: normalized.retryAfterMs, + }; + attempts.push(failed); + options.onAttempt?.(failed); + const currentHealth = stateOf(spec.provider); + + if (normalized.kind === 'rate_limited') { + const retryAfter = normalized.retryAfterMs ?? 30_000; + if (retryAfter <= retryInlineMaxMs && attempt < maxAttempts) { + logger.warn( + { provider: spec.provider, attempt, retryAfterMs: retryAfter }, + 'LLM rate limited — retrying inline', + ); + await sleep(retryAfter); + continue; + } + setHealth(spec.provider, { + state: 'cooling', + consecutiveFailures: currentHealth.consecutiveFailures + 1, + until: clock() + Math.max(retryAfter, 1_000), + lastError: normalized.kind, + }); + break; + } + + if (normalized.kind === 'quota_exhausted') { + setHealth(spec.provider, { + state: 'open', + consecutiveFailures: currentHealth.consecutiveFailures + 1, + until: clock() + quotaCooldownMs, + lastError: normalized.kind, + }); + break; + } + + if (normalized.kind === 'server' || normalized.kind === 'timeout') { + const failures = currentHealth.consecutiveFailures + 1; + setHealth(spec.provider, { + state: failures >= failureThreshold ? 'cooling' : 'healthy', + consecutiveFailures: failures, + until: failures >= failureThreshold ? clock() + failureCooldownMs : undefined, + lastError: normalized.kind, + }); + break; + } + + if (normalized.kind === 'auth' || normalized.kind === 'invalid_request') { + logger.error( + { provider: spec.provider, kind: normalized.kind, task: req.task }, + 'LLM provider rejected the request — failing over', + ); + setHealth(spec.provider, { + state: currentHealth.state, + consecutiveFailures: currentHealth.consecutiveFailures + 1, + until: currentHealth.until, + lastError: normalized.kind, + }); + break; + } + + // empty / parse / content_filtered: try the next spec. + setHealth(spec.provider, { + state: currentHealth.state, + consecutiveFailures: currentHealth.consecutiveFailures + 1, + until: currentHealth.until, + lastError: normalized.kind, + }); + break; + } + } + previousProvider = spec.provider; + } + + throw new LlmError( + lastError?.kind ?? 'server', + lastError?.provider ?? 'router', + lastError?.message ?? `All LLM providers are unavailable for task "${req.task}"`, + lastError?.retryAfterMs, + attempts, + lastError ?? undefined, + ); + }, + }; +} diff --git a/src/lib/llm/types.ts b/src/lib/llm/types.ts new file mode 100644 index 0000000..8b2b3b2 --- /dev/null +++ b/src/lib/llm/types.ts @@ -0,0 +1,90 @@ +export type LlmTask = + | 'market_generation' + | 'hedge_conditions' + | 'forecasting' + | 'attestation' + | 'generic_json' + | 'generic_text'; + +export type OutputMode = 'text' | 'json'; + +export type LlmErrorKind = + | 'rate_limited' + | 'quota_exhausted' + | 'auth' + | 'invalid_request' + | 'timeout' + | 'server' + | 'content_filtered' + | 'empty' + | 'parse'; + +export interface ModelSpec { + provider: 'anthropic' | 'gemini' | 'openai'; + model: string; + temperature: number; + maxTokens: number; + timeoutMs: number; + baseUrl?: string; + /** OpenAI-compatible servers that reject response_format can use prompt-only JSON. */ + jsonMode?: 'native' | 'prompt'; + /** Additional attempts on the same spec after a short rate limit. Default 1. */ + retries?: number; +} + +export interface LlmRequest { + task: LlmTask; + system: string; + user: string; + mode: OutputMode; + maxTokens?: number; + signal?: AbortSignal; +} + +export interface Attempt { + provider: string; + model: string; + kind?: LlmErrorKind; + message?: string; + latencyMs: number; + retryAfterMs?: number; +} + +export interface LlmResult { + text: string; + provenance: { + task: LlmTask; + provider: string; + model: string; + attempts: Attempt[]; + latencyMs: number; + usage?: { inputTokens?: number; outputTokens?: number }; + }; +} + +export interface ProviderCompletion { + text: string; + usage?: { inputTokens?: number; outputTokens?: number }; +} + +export interface LlmProvider { + readonly name: ModelSpec['provider']; + available(): boolean; + complete(spec: ModelSpec, req: LlmRequest): Promise; +} + +export class LlmError extends Error { + public readonly cause?: unknown; + constructor( + public readonly kind: LlmErrorKind, + public readonly provider: string, + message: string, + public readonly retryAfterMs?: number, + public readonly attempts: Attempt[] = [], + cause?: unknown, + ) { + super(message); + this.name = 'LlmError'; + this.cause = cause; + } +} diff --git a/src/routes/admin.routes.ts b/src/routes/admin.routes.ts new file mode 100644 index 0000000..dd7a81c --- /dev/null +++ b/src/routes/admin.routes.ts @@ -0,0 +1,27 @@ +import { Router } from 'express'; +import { requireAdmin, requireAuth } from '../middlewares/auth.middleware'; +import { + addOperationSignature, + cancelTreasuryOperation, + createOperation, + getTreasury, + getTreasuryOperation, + listTreasuryOperations, + recordTreasuryOperation, + submitTreasuryOperation, +} from '../controllers/admin.controller'; + +const router = Router(); + +router.use(requireAuth, requireAdmin); + +router.get('/treasury', getTreasury); +router.get('/treasury/operations', listTreasuryOperations); +router.post('/treasury/operations', createOperation); +router.get('/treasury/operations/:id', getTreasuryOperation); +router.post('/treasury/operations/:id/signatures', addOperationSignature); +router.post('/treasury/operations/:id/submit', submitTreasuryOperation); +router.post('/treasury/operations/:id/record', recordTreasuryOperation); +router.post('/treasury/operations/:id/cancel', cancelTreasuryOperation); + +export default router; diff --git a/src/routes/index.ts b/src/routes/index.ts index 8e77b60..cdc5aeb 100644 --- a/src/routes/index.ts +++ b/src/routes/index.ts @@ -5,6 +5,8 @@ import traceRoutes from './trace.routes'; import portfolioRoutes from './portfolio.routes'; import tradeRoutes from './trade.routes'; import oracleRoutes from './oracle.routes'; +import adminRoutes from './admin.routes'; +import reportRoutes from './reports.routes'; const router = Router(); @@ -14,5 +16,7 @@ router.use('/traces', traceRoutes); router.use('/portfolio', portfolioRoutes); router.use('/trade', tradeRoutes); router.use('/oracle', oracleRoutes); +router.use('/admin', adminRoutes); +router.use('/reports', reportRoutes); export default router; diff --git a/src/routes/reports.routes.ts b/src/routes/reports.routes.ts new file mode 100644 index 0000000..4cc5de5 --- /dev/null +++ b/src/routes/reports.routes.ts @@ -0,0 +1,59 @@ +import { Router } from 'express'; +import { prisma } from '../lib/prisma'; +import { sendError, sendSuccess, parsePagination, buildPaginationMeta } from '../utils/response.util'; + +const router = Router(); + +const VERIFICATION_RECIPE = [ + 'Find the report trace for the date by walking market_trace_count(REPORT_MARKET_ID) and market_trace_at(REPORT_MARKET_ID, i) on the reasoning registry.', + 'Fetch ipfs_cid, compute sha256 of the exact bytes and compare it with the on-chain trace_hash and the hash below.', + 'Check the report agent equals the treasury RiskConfig.agent.', + 'Recompute snapshot figures from RPC (state may have moved since the anchor; compare lastModifiedLedgerSeq) and period flows from getEvents within retention.', + 'Check the previous on-chain trace id and trace hash form the continuous report chain.', +]; + +function publicReport(row: any) { + return { + date: row.date, + anchorLedger: row.anchorLedger, + cid: row.cid, + ipfsUri: `ipfs://${row.cid}`, + hash: row.hash, + onChainTraceId: row.onChainTraceId, + mode: row.mode, + consistency: row.consistency, + createdAt: row.createdAt, + report: row.report, + verification: { + recipe: VERIFICATION_RECIPE, + command: `npm run verify:report -- --date ${row.date}`, + }, + }; +} + +router.get('/', async (req, res) => { + const { page, limit, skip } = parsePagination(req.query as any); + const [rows, total] = await Promise.all([ + prisma.transparencyReport.findMany({ orderBy: { date: 'desc' }, skip, take: limit }), + prisma.transparencyReport.count(), + ]); + sendSuccess(res, rows.map(publicReport), 200, buildPaginationMeta(page, limit, total) as any); +}); + +router.get('/latest', async (_req, res) => { + const row = await prisma.transparencyReport.findFirst({ orderBy: { date: 'desc' } }); + if (!row) return sendError(res, 404, 'REPORT_NOT_FOUND', 'No transparency report has been published yet'); + sendSuccess(res, publicReport(row)); +}); + +router.get('/:date', async (req, res) => { + const date = String(req.params.date); + if (!/^\d{4}-\d{2}-\d{2}$/.test(date)) { + return sendError(res, 400, 'VALIDATION_ERROR', 'date must be YYYY-MM-DD'); + } + const row = await prisma.transparencyReport.findUnique({ where: { date } }); + if (!row) return sendError(res, 404, 'REPORT_NOT_FOUND', `No report for ${date}`); + sendSuccess(res, publicReport(row)); +}); + +export default router; diff --git a/src/server.ts b/src/server.ts index 2a73c5d..b8d0e59 100644 --- a/src/server.ts +++ b/src/server.ts @@ -6,6 +6,7 @@ import { logger } from './lib/logger'; import { prisma } from './lib/prisma'; import { startIngestionCron } from './cron/ingestion.cron'; import { startMonitorCron } from './cron/monitor.cron'; +import { startReportCron } from './cron/report.cron'; import { startEventListener } from './services/indexer.service'; import { closeRedis } from './lib/redis'; @@ -44,6 +45,7 @@ async function bootstrap(): Promise { // 4. Start cron jobs startIngestionCron(); startMonitorCron(); + startReportCron(); io.on('connection', (socket) => { logger.info({ socketId: socket.id }, 'New client connected to socket.io'); diff --git a/src/services/chain.service.ts b/src/services/chain.service.ts index b56d50f..21e9f22 100644 --- a/src/services/chain.service.ts +++ b/src/services/chain.service.ts @@ -21,8 +21,9 @@ import { config } from '../config'; import { logger } from '../lib/logger'; import { agentClients, readClients } from './stellar/clients'; import { ChainError, contractErrorName } from './stellar/errors'; -import type { Category, Market as OnChainMarket, Outcome, Position } from '../generated/market-core'; +import type { Category, Config as MarketCoreConfig, Market as OnChainMarket, Outcome, Position } from '../generated/market-core'; import type { Trace } from '../generated/reasoning-registry'; +import type { DailyUsage, RiskConfig } from '../generated/treasury'; import type { ResolutionState } from '../generated/resolver'; type ContractKey = 'marketCore' | 'treasury' | 'resolver' | 'reasoningRegistry'; @@ -156,6 +157,15 @@ export const resolverState = (marketId: bigint) => export const treasuryAvailableCapital = () => read('treasury', 'available_capital', async () => readClients.treasury().available_capital()); +export const treasuryConfig = () => + read('treasury', 'get_config', async () => readClients.treasury().get_config()); + +export const treasuryDailyUsage = () => + read('treasury', 'daily_usage', async () => readClients.treasury().daily_usage()); + +export const marketCoreConfig = () => + read('marketCore', 'get_config', async () => readClients.marketCore().get_config()); + // ─── Writes (through the treasury) ─────────────────────────────────────────── export interface CreateMarketArgs { diff --git a/src/services/content-store.service.ts b/src/services/content-store.service.ts new file mode 100644 index 0000000..4835221 --- /dev/null +++ b/src/services/content-store.service.ts @@ -0,0 +1,176 @@ +/** + * Local, verified IPFS content store (#100). + * + * sha256 is the only admission criterion; the CID is a lookup key. Bytes are + * stored at pin time (write-through) and gateway bytes are admitted only after + * their hash matches the hash the caller already trusts (on-chain or the pin + * hash). The store never serves bytes whose current hash differs from the + * expected hash. + */ +import { Prisma } from '@prisma/client'; +import { config } from '../config'; +import { prisma } from '../lib/prisma'; +import { logger } from '../lib/logger'; +import { AppError } from '../middlewares/error.middleware'; +import { sha256Hex } from '../utils/hash.util'; + +export type ContentOrigin = 'PINNED' | 'GATEWAY_VERIFIED'; + +export interface ContentStore { + put(entry: { + cid: string; + sha256: string; + bytes: Buffer; + origin: ContentOrigin; + verifiedBy: string; + }): Promise; + get(cid: string): Promise<{ sha256: string; bytes: Buffer; origin: string } | null>; + touch(cid: string): Promise; + evict(budgetBytes: number): Promise<{ removed: number; freedBytes: number }>; +} + +const TOUCH_INTERVAL_MS = 10 * 60 * 1000; +const EVICT_BATCH = 500; + +function asBuffer(bytes: Uint8Array | Buffer): Buffer { + return Buffer.isBuffer(bytes) ? bytes : Buffer.from(bytes); +} + +function normalizeHash(hash: string): string { + return hash.toLowerCase().replace(/^0x/, ''); +} + +export const postgresContentStore: ContentStore = { + async put(entry) { + const bytes = asBuffer(entry.bytes); + const expected = normalizeHash(entry.sha256); + if (!/^[0-9a-f]{64}$/.test(expected)) { + throw new AppError(400, 'IPFS_INVALID_HASH', 'sha256 must be 64 lowercase hex characters'); + } + // Recompute rather than trusting the caller. A mismatch means a caller bug; + // storing it would poison the cache. + if (sha256Hex(bytes) !== expected) { + throw new AppError(500, 'IPFS_HASH_MISMATCH', 'Content hash does not match sha256Hex(bytes)'); + } + if (bytes.length > config.IPFS_CACHE_MAX_OBJECT_BYTES) { + throw new AppError(413, 'IPFS_CONTENT_TOO_LARGE', 'Content exceeds IPFS_CACHE_MAX_OBJECT_BYTES', { + size: bytes.length, + max: config.IPFS_CACHE_MAX_OBJECT_BYTES, + }); + } + + const existing = await prisma.ipfsContent.findUnique({ where: { cid: entry.cid } }); + if (existing) { + if (normalizeHash(existing.sha256) !== expected) { + logger.error( + { cid: entry.cid, stored: existing.sha256, incoming: expected }, + 'IPFS CID hash conflict — refusing to overwrite', + ); + throw new AppError(409, 'IPFS_CID_HASH_CONFLICT', 'This CID already maps to different bytes', { + cid: entry.cid, + }); + } + // Same bytes: idempotent, just keep it fresh. + await prisma.ipfsContent.update({ + where: { cid: entry.cid }, + data: { lastAccessAt: new Date() }, + }); + return; + } + + try { + await prisma.ipfsContent.create({ + data: { + cid: entry.cid, + sha256: expected, + size: bytes.length, + bytes: new Uint8Array(bytes) as unknown as Uint8Array, + origin: entry.origin, + verifiedBy: entry.verifiedBy, + }, + }); + } catch (err) { + // Concurrent verifies of one CID: P2002 is success if the winner stored + // the same bytes. + if (err instanceof Prisma.PrismaClientKnownRequestError && err.code === 'P2002') { + const winner = await prisma.ipfsContent.findUnique({ where: { cid: entry.cid } }); + if (winner && normalizeHash(winner.sha256) === expected) return; + throw new AppError(409, 'IPFS_CID_HASH_CONFLICT', 'This CID already maps to different bytes', { + cid: entry.cid, + }); + } + throw err; + } + }, + + async get(cid) { + const row = await prisma.ipfsContent.findUnique({ where: { cid } }); + if (!row) return null; + return { sha256: row.sha256, bytes: asBuffer(row.bytes), origin: row.origin }; + }, + + async touch(cid) { + const cutoff = new Date(Date.now() - TOUCH_INTERVAL_MS); + await prisma.ipfsContent.updateMany({ + where: { cid, lastAccessAt: { lt: cutoff } }, + data: { lastAccessAt: new Date() }, + }); + }, + + async evict(budgetBytes) { + const total = await prisma.ipfsContent.aggregate({ _sum: { size: true } }); + let remaining = total._sum.size ?? 0; + if (remaining <= budgetBytes) return { removed: 0, freedBytes: 0 }; + + const thirtyDaysAgo = new Date(Date.now() - 30 * 24 * 60 * 60 * 1000); + // A single SQL statement, not N queries. Never evict content referenced by + // an unpublished trace or a PENDING market. GATEWAY_VERIFIED rows go first; + // PINNED rows only after 30 days. + const rows = await prisma.$queryRaw>(Prisma.sql` + SELECT ic."cid", ic."size" + FROM "ipfs_content" ic + WHERE (ic."pinnedUntil" IS NULL OR ic."pinnedUntil" < NOW()) + AND (ic."origin" = 'GATEWAY_VERIFIED' OR ic."createdAt" < ${thirtyDaysAgo}) + AND NOT EXISTS ( + SELECT 1 FROM "reasoning_traces" rt + WHERE rt."ipfsCid" = ic."cid" + AND (rt."onChainTraceId" IS NULL OR rt."verified" = false) + ) + AND NOT EXISTS ( + SELECT 1 FROM "markets" m + WHERE m."metaUri" = ('ipfs://' || ic."cid") + AND m."status" = 'PENDING' + ) + ORDER BY (CASE WHEN ic."origin" = 'GATEWAY_VERIFIED' THEN 0 ELSE 1 END), + ic."lastAccessAt" ASC + LIMIT ${EVICT_BATCH} + `); + + const toRemove: string[] = []; + let freedBytes = 0; + for (const row of rows) { + if (remaining <= budgetBytes) break; + toRemove.push(row.cid); + remaining -= row.size; + freedBytes += row.size; + } + if (toRemove.length === 0) return { removed: 0, freedBytes: 0 }; + + await prisma.ipfsContent.deleteMany({ where: { cid: { in: toRemove } } }); + logger.info({ removed: toRemove.length, freedBytes, budgetBytes }, 'Evicted IPFS content'); + return { removed: toRemove.length, freedBytes }; + }, +}; + +let store: ContentStore = postgresContentStore; + +/** Test seam, same style as setIpfsTransport. Returns the previous store. */ +export function setContentStore(next: ContentStore): ContentStore { + const prev = store; + store = next; + return prev; +} + +export function getContentStore(): ContentStore { + return store; +} diff --git a/src/services/ipfs.service.ts b/src/services/ipfs.service.ts index cef5860..c7159ee 100644 --- a/src/services/ipfs.service.ts +++ b/src/services/ipfs.service.ts @@ -3,6 +3,7 @@ import { config } from '../config'; import { logger } from '../lib/logger'; import { AppError } from '../middlewares/error.middleware'; import { canonicalBytes, sha256Hex } from '../utils/hash.util'; +import { getContentStore } from './content-store.service'; const PINATA_BASE = 'https://api.pinata.cloud'; @@ -78,6 +79,13 @@ export async function pinJson(document: unknown, name: string): Promise { + logger.warn({ err, cid, name }, 'IPFS content store write failed; continuing'); + }); logger.info({ name, cid, hash }, 'Pinned to IPFS'); return { cid, hash, bytes }; } catch (err) { @@ -97,3 +105,43 @@ export async function pinJson(document: unknown, name: string): Promise { return transport.fetch(cid); } + +/** + * Returns bytes for `cid` only if they match `expectedSha256` (the on-chain + * hash or the hash computed at pin time). Cache hits are re-hashed on read; + * a cached row with a different hash is ignored and never served. Gateway + * bytes are stored only after they verify. Bytes that fail verification are + * still returned (so callers can report `verified: false`) but never cached. + */ +export async function fetchVerifiedBytes( + cid: string, + expectedSha256: string, + verifiedBy: string, +): Promise<{ bytes: Buffer; source: 'cache' | 'gateway' }> { + const expected = expectedSha256.toLowerCase().replace(/^0x/, ''); + const store = getContentStore(); + + const cached = await store.get(cid); + if (cached) { + const storedHash = cached.sha256.toLowerCase().replace(/^0x/, ''); + if (storedHash === expected && sha256Hex(cached.bytes) === expected) { + await store.touch(cid).catch(() => undefined); + return { bytes: cached.bytes, source: 'cache' }; + } + logger.warn( + { cid, expected, stored: cached.sha256 }, + 'Cached IPFS content hash does not match expected hash; consulting gateway', + ); + } + + const bytes = await transport.fetch(cid); + const matches = sha256Hex(bytes) === expected; + if (matches && bytes.length <= config.IPFS_CACHE_MAX_OBJECT_BYTES) { + await store + .put({ cid, sha256: expected, bytes, origin: 'GATEWAY_VERIFIED', verifiedBy }) + .catch((err) => logger.warn({ err, cid }, 'Failed to cache verified gateway bytes')); + } else if (!matches) { + logger.warn({ cid, expected, computed: sha256Hex(bytes) }, 'Gateway bytes failed hash verification; not cached'); + } + return { bytes, source: 'gateway' }; +} diff --git a/src/services/market.service.ts b/src/services/market.service.ts index d4338b6..2a40a35 100644 --- a/src/services/market.service.ts +++ b/src/services/market.service.ts @@ -2,7 +2,7 @@ import { z } from "zod"; import { prisma } from "../lib/prisma"; -import { callLLMJSON } from '../lib/llm'; +import { completeLLM } from '../lib/llm'; import { logger } from "../lib/logger"; import type { MarketProposal, @@ -155,8 +155,17 @@ Pick the category that best fits the event. `; let rawResponse: string; + let provenance: Awaited>['provenance'] | null = null; try { - rawResponse = await callLLMJSON(systemPrompt, userPrompt, 4096); + const result = await completeLLM({ + task: 'market_generation', + system: systemPrompt, + user: userPrompt, + mode: 'json', + maxTokens: 4096, + }); + rawResponse = result.text; + provenance = result.provenance; } catch (err) { await logAgentAction('MARKET_MAKER', 'ERROR', 'GENERATE_QUESTION_LLM_FAIL', null, { error: String(err), @@ -165,6 +174,19 @@ Pick the category that best fits the event. throw new AppError(500, 'LLM_CALL_FAILED', 'LLM API call failed during market generation'); } + await logAgentAction('MARKET_MAKER', 'INFO', 'GENERATE_QUESTION_LLM', null, { + signalCount: signals.signalCount, + llm: provenance + ? { + provider: provenance.provider, + model: provenance.model, + attempts: provenance.attempts.length, + latencyMs: provenance.latencyMs, + usage: provenance.usage ?? null, + } + : null, + }); + return validateMarketQuestion(rawResponse); } diff --git a/src/services/reasoning.service.ts b/src/services/reasoning.service.ts index 7f08d88..7ec58d1 100644 --- a/src/services/reasoning.service.ts +++ b/src/services/reasoning.service.ts @@ -1,5 +1,5 @@ import { prisma } from '../lib/prisma'; -import { callLLMJSON } from '../lib/llm'; // ← Unified LLM (Claude → Gemini fallback) +import { completeLLM } from '../lib/llm'; import { logger } from '../lib/logger'; import { Prisma } from '@prisma/client'; import type { @@ -46,6 +46,8 @@ export async function generateReasoningTrace(input: TraceInput) { betFraction: input.betFraction, betSizeUsdc: input.betSizeUsdc, hedgeConditions: hedgeConditions.conditions, + // Optional, explicitly null-able provenance so payload hashing is stable. + llm: hedgeConditions.provenance, agentWallet: input.agentWallet, timestamp: new Date().toISOString(), }; @@ -145,7 +147,11 @@ async function generateHedgeConditions( input: TraceInput, edge: number, ci: ConfidenceInterval, -): Promise<{ conditions: string[]; reasoning: string }> { +): Promise<{ + conditions: string[]; + reasoning: string; + provenance: { provider: string; model: string } | null; +}> { const systemPrompt = `You are a quantitative prediction market risk manager. Generate specific, measurable hedge conditions for a trading position. Return ONLY a valid JSON object matching this schema exactly. No markdown, no preamble. @@ -177,10 +183,20 @@ and trigger an early close. Focus on data changes, not price changes. `; try { - const cleaned = await callLLMJSON(systemPrompt, userPrompt, 1024); - const parsed = JSON.parse(cleaned); + const result = await completeLLM({ + task: 'hedge_conditions', + system: systemPrompt, + user: userPrompt, + mode: 'json', + maxTokens: 1024, + }); + const parsed = JSON.parse(result.text); const validated = traceReasoningSchema.parse(parsed); - return { conditions: validated.hedge_conditions, reasoning: validated.reasoning_summary }; + return { + conditions: validated.hedge_conditions, + reasoning: validated.reasoning_summary, + provenance: { provider: result.provenance.provider, model: result.provenance.model }, + }; } catch (err) { // Non-fatal: log a warning but never fail the whole market creation cycle logger.warn({ err }, 'LLM hedge condition generation failed — applying safe defaults'); @@ -191,6 +207,7 @@ and trigger an early close. Focus on data changes, not price changes. `Hard stop-loss at 15% adverse market move`, ], reasoning: 'Default hedge conditions applied — LLM call failed', + provenance: null, }; } } \ No newline at end of file diff --git a/src/services/trace-publish.service.ts b/src/services/trace-publish.service.ts index c91abbe..ed3ddc0 100644 --- a/src/services/trace-publish.service.ts +++ b/src/services/trace-publish.service.ts @@ -7,7 +7,7 @@ import { prisma } from '../lib/prisma'; import { logger } from '../lib/logger'; import { AppError } from '../middlewares/error.middleware'; import { verifyTraceHash, sha256Hex } from '../utils/hash.util'; -import { fetchIpfsBytes, pinJson } from './ipfs.service'; +import { fetchVerifiedBytes, pinJson } from './ipfs.service'; import { getOnChainTrace, publishTrace } from './chain.service'; import { agentSigner } from './stellar/clients'; import type { TraceVerification } from '../types'; @@ -65,7 +65,11 @@ export async function verifyTraceOnChain(traceId: string): Promise { + const pinned = await pinJson(params.document, params.name); + if (options.dryRun) { + return { cid: pinned.cid, hash: pinned.hash, onChainTraceId: null, publishTxHash: null, mode: 'dry-run' }; + } + const published = await publishPinnedDocument({ + cid: pinned.cid, + hash: pinned.hash, + action: params.action, + marketId: params.marketId, + }); + return { cid: pinned.cid, hash: pinned.hash, ...published }; +} + +/** + * Publishes bytes that are already pinned. Used to recover from a crash + * between pin and publish without recomputing (and re-pinning) the document. + */ +export async function publishPinnedDocument(params: { + cid: string; + hash: string; + action: string; + marketId: bigint; +}, options: { dryRun?: boolean } = {}): Promise<{ onChainTraceId: bigint | null; publishTxHash: string | null; mode: 'dry-run' | 'live' }> { + if (options.dryRun) { + return { onChainTraceId: null, publishTxHash: null, mode: 'dry-run' }; + } + const result = await publishTrace({ + agent: agentSigner().publicKey as string, + market_id: params.marketId, + action: params.action, + trace_hash: Buffer.from(params.hash, 'hex'), + ipfs_cid: params.cid, + }); + if (result.mode === 'dry-run') { + return { onChainTraceId: null, publishTxHash: null, mode: 'dry-run' }; + } + return { onChainTraceId: result.value, publishTxHash: result.txHash, mode: 'live' }; +} diff --git a/src/services/transparency-report.service.ts b/src/services/transparency-report.service.ts new file mode 100644 index 0000000..413f266 --- /dev/null +++ b/src/services/transparency-report.service.ts @@ -0,0 +1,461 @@ +/** + * Daily, chain-only transparency report (#159). + * + * The report is a deterministic JSON document (schema v1) computed from chain + * reads and `chain_events`. Prisma float models are never used. + * Amounts are decimal strings in base units. The document is pinned, its + * sha256 is published through the reasoning registry under the documented + * sentinel market id, and a row is persisted for the public index. + */ +import dayjs from 'dayjs'; +import { Contract, xdr } from '@stellar/stellar-sdk'; +import utc from 'dayjs/plugin/utc'; +import { config } from '../config'; +import { CONTRACTS } from '../config/contracts'; +import { prisma } from '../lib/prisma'; +import { logger } from '../lib/logger'; +import { priceYesBps, collateralOutForShares, settleValue, type Settlement } from '../generated/fpmm'; +import type { Market as OnChainMarket, Position } from '../generated/market-core'; +import { agentSigner, rpcServer } from './stellar/clients'; +import { + getOnChainMarket, + getPosition, + marketCount, + treasuryAvailableCapital, + treasuryConfig, + treasuryDailyUsage, +} from './chain.service'; +import { pinAndPublishDocument, publishPinnedDocument } from './trace-publish.service'; + +dayjs.extend(utc); + +/** u64::MAX: reports are not about one market, so they use a sentinel. */ +export const REPORT_MARKET_ID = 18446744073709551615n; +export const REPORT_KIND = 'oracledesk.transparency-report'; +export const REPORT_SCHEMA_VERSION = 1; + +export interface ReportMarket { + marketId: string; + status: 'Open' | 'Resolved:Yes' | 'Resolved:No' | 'Void'; + createdByTreasury: boolean; + position: { yesRaw: string; noRaw: string }; + reserves: { yesRaw: string; noRaw: string }; + priceYesBps: number; + markValueRaw: string; + liquidationValueRaw: string; + settlementValueRaw: string | null; + lpRemainderRaw?: string; + archived?: boolean; +} + +export interface TransparencyReportV1 { + kind: typeof REPORT_KIND; + schemaVersion: typeof REPORT_SCHEMA_VERSION; + network: string; + contracts: { treasury: string; marketCore: string; usdc: string; reasoningRegistry: string }; + agent: string; + period: { from: string; to: string }; + anchor: { ledger: number; closedAt: string }; + consistency: 'exact' | 'best_effort'; + treasury: { + usdcBalanceRaw: string; + availableCapitalRaw: string; + riskConfig: { maxTradeRaw: string; maxMarketRaw: string; maxDailyRaw: string }; + dailyUsage: { amountRaw: string; dayStart: number }; + }; + markets: ReportMarket[]; + totals: { positionsMarkRaw: string; positionsLiquidationRaw: string; navRaw: string }; + period_flows: { + coverage: { fromLedger: number; toLedger: number; complete: boolean; gaps: Array<[number, number]> }; + tradesBoughtRaw: string; + tradesSoldRaw: string; + feesPaidRaw: string; + redeemedRaw: string; + feesCollectedRaw: string; + seedDeployedRaw: string; + tradeCount: number; + }; + traces: { total: number; byAction: Record }; + method: { version: '1'; doc: string }; + previous: { onChainTraceId: string; traceHash: string } | null; +} + +function statusLabel(status: OnChainMarket['status']): ReportMarket['status'] { + if (status.tag === 'Open') return 'Open'; + if (status.tag === 'Void') return 'Void'; + return status.values[0].tag === 'Yes' ? 'Resolved:Yes' : 'Resolved:No'; +} + +function settlementKind(label: ReportMarket['status']): Settlement | null { + if (label === 'Resolved:Yes') return 'yes'; + if (label === 'Resolved:No') return 'no'; + if (label === 'Void') return 'void'; + return null; +} + +export function markValueRaw(yes: bigint, no: bigint, yesBps: number): string { + const p = BigInt(yesBps); + return ((yes * p + no * (10_000n - p)) / 10_000n).toString(); +} + +export function liquidationValueRaw( + reserveYes: bigint, + reserveNo: bigint, + yes: bigint, + no: bigint, + feeBps: bigint, +): string { + let total = 0n; + if (yes > 0n) total += collateralOutForShares(reserveYes, reserveNo, yes, feeBps); + if (no > 0n) total += collateralOutForShares(reserveNo, reserveYes, no, feeBps); + return total.toString(); +} + +export function settlementValueRaw(label: ReportMarket['status'], yes: bigint, no: bigint): string | null { + const kind = settlementKind(label); + return kind ? settleValue(kind, yes, no).toString() : null; +} + +export interface RawMarketSnapshot { + marketId: bigint; + market: OnChainMarket; + position: Position; + archived?: boolean; +} + +/** Pure valuation step, shared by the builder and tests. */ +export function valueMarket(snapshot: RawMarketSnapshot, treasuryAddress: string): ReportMarket { + const label = statusLabel(snapshot.market.status); + const yesBps = Number(priceYesBps(BigInt(snapshot.market.reserve_yes), BigInt(snapshot.market.reserve_no))); + const yes = BigInt(snapshot.position.yes); + const no = BigInt(snapshot.position.no); + const kind = settlementKind(label); + const lpRemainder = + kind && !snapshot.market.lp_claimed && snapshot.market.creator === treasuryAddress + ? settleValue(kind, BigInt(snapshot.market.reserve_yes), BigInt(snapshot.market.reserve_no)).toString() + : undefined; + return { + marketId: snapshot.marketId.toString(), + status: label, + createdByTreasury: snapshot.market.creator === treasuryAddress, + position: { yesRaw: yes.toString(), noRaw: no.toString() }, + reserves: { yesRaw: BigInt(snapshot.market.reserve_yes).toString(), noRaw: BigInt(snapshot.market.reserve_no).toString() }, + priceYesBps: yesBps, + markValueRaw: markValueRaw(yes, no, yesBps), + liquidationValueRaw: liquidationValueRaw( + BigInt(snapshot.market.reserve_yes), + BigInt(snapshot.market.reserve_no), + yes, + no, + BigInt(snapshot.market.fee_bps), + ), + settlementValueRaw: settlementValueRaw(label, yes, no), + ...(lpRemainder !== undefined ? { lpRemainderRaw: lpRemainder } : {}), + ...(snapshot.archived ? { archived: true } : {}), + }; +} + +export function totalsFor(markets: ReportMarket[], usdcBalanceRaw: string) { + const mark = markets.reduce((sum, m) => sum + BigInt(m.markValueRaw), 0n); + const liquidation = markets.reduce((sum, m) => sum + BigInt(m.liquidationValueRaw), 0n); + const nav = BigInt(usdcBalanceRaw) + mark; + return { + positionsMarkRaw: mark.toString(), + positionsLiquidationRaw: liquidation.toString(), + navRaw: nav.toString(), + }; +} + +export interface ReportInputs { + date: string; + network: string; + contracts: TransparencyReportV1['contracts']; + agent: string; + anchor: { ledger: number; closedAt: string }; + consistency: 'exact' | 'best_effort'; + treasury: TransparencyReportV1['treasury']; + markets: ReportMarket[]; + periodFlows: TransparencyReportV1['period_flows']; + traces: TransparencyReportV1['traces']; + previous: TransparencyReportV1['previous']; + methodDoc: string; +} + +export function buildReportDocument(input: ReportInputs): TransparencyReportV1 { + return { + kind: REPORT_KIND, + schemaVersion: REPORT_SCHEMA_VERSION, + network: input.network, + contracts: input.contracts, + agent: input.agent, + period: { from: `${input.date}T00:00:00.000Z`, to: `${input.date}T23:59:59.999Z` }, + anchor: input.anchor, + consistency: input.consistency, + treasury: input.treasury, + markets: input.markets, + totals: totalsFor(input.markets, input.treasury.usdcBalanceRaw), + period_flows: input.periodFlows, + traces: input.traces, + method: { version: '1', doc: input.methodDoc }, + previous: input.previous, + }; +} + +export interface ReportDeps { + now?: () => Date; + latestLedger?: () => Promise; + marketCount?: () => Promise; + getMarket?: (id: bigint) => Promise; + getPosition?: (id: bigint, holder: string) => Promise; + treasuryConfig?: () => Promise<{ agent: string; max_daily: bigint; max_market: bigint; max_trade: bigint }>; + dailyUsage?: () => Promise<{ amount: bigint; day_start: bigint }>; + availableCapital?: () => Promise; + traceCount?: () => Promise; + getTrace?: (id: bigint) => Promise<{ agent: string; action: string }>; + chainEvents?: (from: Date, to: Date) => Promise>; + indexerCursor?: () => Promise; + previousReport?: (date: string) => Promise<{ onChainTraceId: bigint | null; hash: string } | null>; + methodDoc?: string; + ledgerEntries?: () => Promise>; +} + +const SNAPSHOT_ATTEMPTS = 3; + +/** Treasury instance entry's lastModifiedLedgerSeq, used for drift detection. */ +async function treasuryInstanceEntries(): Promise> { + try { + const key = (xdr.LedgerKey.contractData as any)({ + contract: new Contract(CONTRACTS.treasury).address(), + key: xdr.ScVal.scvLedgerKeyContractInstance(), + durability: xdr.ContractDataDurability.persistent(), + }); + const res = await rpcServer.getLedgerEntries(key); + return (res.entries ?? []).map((entry) => ({ lastModifiedLedgerSeq: (entry as any).lastModifiedLedgerSeq ?? 0 })); + } catch { + return []; + } +} + +function eventParams(event: { params: unknown }): Record { + return (event.params ?? {}) as Record; +} + +export async function buildReportForDate(date: string, deps: ReportDeps = {}): Promise { + const getLatest = deps.latestLedger ?? (async () => (await rpcServer.getLatestLedger()).sequence); + const count = deps.marketCount ?? marketCount; + const market = deps.getMarket ?? getOnChainMarket; + const position = deps.getPosition ?? getPosition; + const risk = deps.treasuryConfig ?? treasuryConfig; + const usage = deps.dailyUsage ?? treasuryDailyUsage; + const available = deps.availableCapital ?? treasuryAvailableCapital; + const entries = deps.ledgerEntries ?? treasuryInstanceEntries; + const readTraceCount = deps.traceCount ?? (async () => (await import('./stellar/clients')).readClients.reasoningRegistry().trace_count().then((tx) => tx.result as bigint)); + const getTrace = deps.getTrace ?? (async (id: bigint) => { + const { getOnChainTrace } = await import('./chain.service'); + return getOnChainTrace(id); + }); + const chainEvents = deps.chainEvents ?? (async (from, to) => prisma.chainEvent.findMany({ + where: { ledgerClosedAt: { gte: from, lte: to } }, + orderBy: { ledger: 'asc' }, + })); + const indexerCursor = deps.indexerCursor ?? (async () => { + const cursor = await prisma.indexerCursor.findFirst({ orderBy: { updatedAt: 'desc' } }); + return cursor?.lastLedger ?? null; + }); + const previousReport = deps.previousReport ?? (async (d) => { + const prev = await prisma.transparencyReport.findFirst({ + where: { date: { lt: d }, onChainTraceId: { not: null } }, + orderBy: { date: 'desc' }, + }); + return prev ? { onChainTraceId: prev.onChainTraceId, hash: prev.hash } : null; + }); + + const [riskCfg, daily, capital] = await Promise.all([risk(), usage(), available()]); + const treasuryAddress = (riskCfg as { agent: string }).agent; + + let consistency: 'exact' | 'best_effort' = 'exact'; + let anchor = 0; + let snapshots: RawMarketSnapshot[] = []; + + for (let attempt = 1; attempt <= SNAPSHOT_ATTEMPTS; attempt++) { + const l0 = await getLatest(); + const touched = await entries(); + const total = await count(); + const next: RawMarketSnapshot[] = []; + for (let i = 0n; i < total; i++) { + try { + const [m, p] = await Promise.all([market(i), position(i, treasuryAddress)]); + const held = BigInt(p.yes) > 0n || BigInt(p.no) > 0n; + if (m.creator === treasuryAddress || held) next.push({ marketId: i, market: m, position: p }); + } catch (err) { + logger.warn({ err, marketId: i.toString() }, 'Transparency report: market read failed; skipped'); + } + } + const l1 = await getLatest(); + snapshots = next; + anchor = l1; + const modifiedDuringSnapshot = touched.some((entry) => entry.lastModifiedLedgerSeq > l0); + if (l1 <= l0 && !modifiedDuringSnapshot) { + consistency = 'exact'; + break; + } + if (attempt === SNAPSHOT_ATTEMPTS) consistency = 'best_effort'; + } + + const markets = snapshots.map((snapshot) => valueMarket(snapshot, treasuryAddress)); + + const start = dayjs.utc(date).startOf('day').toDate(); + const end = dayjs.utc(date).endOf('day').toDate(); + const events = await chainEvents(start, end); + let tradesBought = 0n; + let tradesSold = 0n; + let feesPaid = 0n; + let redeemed = 0n; + let feesCollected = 0n; + let seedDeployed = 0n; + let tradeCount = 0; + + for (const event of events) { + const p = eventParams(event); + const trader = String(p.trader ?? p.holder ?? p.creator ?? ''); + if (event.contractName === 'marketCore' && event.eventName === 'Trade' && trader === treasuryAddress) { + tradeCount++; + const collateral = BigInt((p.collateral as string | number | bigint | undefined) ?? 0); + if (p.is_buy === true) tradesBought += collateral; + else tradesSold += collateral; + feesPaid += BigInt((p.fee as string | number | bigint | undefined) ?? 0); + } else if (event.contractName === 'marketCore' && event.eventName === 'Redeemed' && trader === treasuryAddress) { + redeemed += BigInt((p.amount as string | number | bigint | undefined) ?? 0); + } else if (event.contractName === 'treasury' && event.eventName === 'FeesCollected') { + feesCollected += BigInt((p.amount as string | number | bigint | undefined) ?? 0); + } else if (event.contractName === 'marketCore' && event.eventName === 'MarketCreated' && trader === treasuryAddress) { + seedDeployed += BigInt((p.seed_amount as string | number | bigint | undefined) ?? 0); + } + } + + const fromLedger = events.length > 0 ? events[0].ledger : anchor; + const toLedger = events.length > 0 ? events[events.length - 1].ledger : anchor; + const cursor = await indexerCursor(); + const complete = cursor === null ? events.length === 0 : cursor >= toLedger; + const gaps: Array<[number, number]> = complete ? [] : [[(cursor ?? fromLedger) + 1, toLedger]]; + + const totalTraces = Number(await readTraceCount()); + const byAction: Record = {}; + let agentTraceTotal = 0; + for (let i = 0; i < totalTraces; i++) { + let cached = await prisma.registryTraceCache.findUnique({ where: { traceId: BigInt(i) } }).catch(() => null); + if (!cached) { + const trace = await getTrace(BigInt(i)).catch(() => null); + if (trace) { + await prisma.registryTraceCache.create({ + data: { + traceId: BigInt(i), + agent: (trace as { agent?: string }).agent ?? '', + marketId: BigInt((trace as { market_id?: bigint }).market_id ?? 0n), + action: (trace as { action?: string }).action ?? '', + traceHash: Buffer.from((trace as { trace_hash?: Buffer }).trace_hash ?? Buffer.alloc(0)).toString('hex'), + ipfsCid: (trace as { ipfs_cid?: string }).ipfs_cid ?? '', + publishedAt: BigInt((trace as { published_at?: bigint }).published_at ?? 0n), + }, + }).catch(() => undefined); + cached = await prisma.registryTraceCache.findUnique({ where: { traceId: BigInt(i) } }).catch(() => null); + } + } + if (cached && cached.agent === treasuryAddress) { + agentTraceTotal++; + byAction[cached.action] = (byAction[cached.action] ?? 0) + 1; + } + } + + const previousRow = await previousReport(date); + const methodDoc = deps.methodDoc ?? process.env.REPORT_METHOD_DOC ?? 'https://github.com/OracleDesk/oracledesk-backend/blob/main/docs/transparency-reports.md'; + + return buildReportDocument({ + date, + network: config.STELLAR_NETWORK_PASSPHRASE, + contracts: { + treasury: CONTRACTS.treasury, + marketCore: CONTRACTS.marketCore, + usdc: CONTRACTS.usdc, + reasoningRegistry: CONTRACTS.reasoningRegistry, + }, + agent: treasuryAddress, + anchor: { ledger: anchor, closedAt: new Date().toISOString() }, + consistency, + treasury: { + usdcBalanceRaw: capital.toString(), + availableCapitalRaw: capital.toString(), + riskConfig: { + maxTradeRaw: BigInt((riskCfg as { max_trade: bigint }).max_trade).toString(), + maxMarketRaw: BigInt((riskCfg as { max_market: bigint }).max_market).toString(), + maxDailyRaw: BigInt((riskCfg as { max_daily: bigint }).max_daily).toString(), + }, + dailyUsage: { amountRaw: BigInt((daily as { amount: bigint }).amount).toString(), dayStart: Number((daily as { day_start: bigint }).day_start) }, + }, + markets, + periodFlows: { + coverage: { fromLedger, toLedger, complete, gaps }, + tradesBoughtRaw: tradesBought.toString(), + tradesSoldRaw: tradesSold.toString(), + feesPaidRaw: feesPaid.toString(), + redeemedRaw: redeemed.toString(), + feesCollectedRaw: feesCollected.toString(), + seedDeployedRaw: seedDeployed.toString(), + tradeCount, + }, + traces: { total: agentTraceTotal, byAction }, + previous: previousRow && previousRow.onChainTraceId !== null + ? { onChainTraceId: previousRow.onChainTraceId.toString(), traceHash: previousRow.hash } + : null, + methodDoc, + }); +} + +/** Daily job: build → pin → publish → persist. Re-publishes stored bytes after a crash. */ +export async function runReportJob(date = dayjs.utc().format('YYYY-MM-DD'), deps: ReportDeps = {}) { + const existing = await prisma.transparencyReport.findUnique({ where: { date } }); + if (existing) { + if (existing.onChainTraceId !== null) return existing; + // Crash between pin and publish (or a dry run that is being retried): + // re-publish the stored bytes, never recompute. + const published = await publishPinnedDocument({ + cid: existing.cid, + hash: existing.hash, + action: 'report', + marketId: REPORT_MARKET_ID, + }, { dryRun: config.REPORT_MODE === 'dry-run' }); + if (published.onChainTraceId === null) return existing; + return prisma.transparencyReport.update({ + where: { id: existing.id }, + data: { + onChainTraceId: published.onChainTraceId, + mode: published.mode, + }, + }); + } + + const report = await buildReportForDate(date, deps); + const published = await pinAndPublishDocument({ + document: report, + name: `oracledesk-report-${date}`, + action: 'report', + marketId: REPORT_MARKET_ID, + }, { dryRun: config.REPORT_MODE === 'dry-run' }); + + return prisma.transparencyReport.create({ + data: { + date, + anchorLedger: report.anchor.ledger, + cid: published.cid, + hash: published.hash, + onChainTraceId: published.onChainTraceId, + mode: published.mode, + consistency: report.consistency, + report: report as unknown as object, + }, + }); +} + +export function reportAgent(): string { + return agentSigner().publicKey as string; +} diff --git a/src/services/treasury-ops.service.ts b/src/services/treasury-ops.service.ts new file mode 100644 index 0000000..7e12dbd --- /dev/null +++ b/src/services/treasury-ops.service.ts @@ -0,0 +1,684 @@ +/** + * Treasury admin operations console (#156). + * + * The backend never holds the treasury admin key. It prepares unsigned, + * simulated transactions for the on-chain admin (or multisig) to sign, merges + * verified signatures, and submits only a transaction whose hash matches the + * prepared body. + */ +import { + Contract, + Keypair, + StrKey, + TransactionBuilder, + Networks, + rpc, + scValToNative, + xdr, +} from '@stellar/stellar-sdk'; +import type { AssembledTransaction } from '@stellar/stellar-sdk/contract'; +import { config } from '../config'; +import { CONTRACTS } from '../config/contracts'; +import { prisma } from '../lib/prisma'; +import { logger } from '../lib/logger'; +import { formatUsdc, parseU64, parseUsdc } from '../lib/amounts'; +import { AppError } from '../middlewares/error.middleware'; +import { TreasuryClient, MarketCoreClient, rpcServer } from './stellar/clients'; +import { contractErrorName } from './stellar/errors'; +import { + getOnChainMarket, + marketCoreConfig, + treasuryAvailableCapital, + treasuryConfig, + treasuryDailyUsage, +} from './chain.service'; +import type { RiskConfig } from '../generated/treasury'; + +export type AdminOp = + | { kind: 'treasury.set_paused'; paused: boolean } + | { kind: 'treasury.withdraw'; to: string; amountRaw: string } + | { kind: 'treasury.deposit'; from: string; amountRaw: string } + | { kind: 'treasury.collect_market_fees'; onChainMarketId: string } + | { kind: 'market_core.set_paused'; paused: boolean }; + +export interface PreflightCheck { + name: string; + ok: boolean; + detail: string; + critical?: boolean; +} + +export interface AccountSigner { + key: string; + kind: 'ed25519' | 'preAuthTx' | 'hashX'; + weight: number; +} + +export interface AccountState { + address: string; + kind: 'account' | 'contract'; + balanceXlm: string | null; + masterWeight: number; + thresholds: { low: number; medium: number; high: number }; + signers: AccountSigner[]; +} + +export interface TreasuryState { + network: string; + contracts: { treasury: string; marketCore: string; usdc: string }; + treasury: { + config: RiskConfig | null; + availableCapitalRaw: string; + usdcBalanceRaw: string; + dailyUsage: { amountRaw: string; dayStart: number; resetsAt: string; remainingRaw: string }; + paused: boolean | null; + admin: { address: string; kind: 'account' | 'contract' } | null; + }; + marketCore: { paused: boolean | null; admin: string | null }; + agent: { address: string; xlmBalance: string | null }; +} + +export interface PreparedOperation { + id: string; + kind: string; + xdr: string; + networkPassphrase: string; + hash: string; + expiresAt: string; + requiredSigners: number; + thresholds: AccountState['thresholds']; + checks: PreflightCheck[]; + humanSummary: string; + status: string; +} + +export interface SignatureRecord { + signer: string; + hint: string; + signature: string; + addedAt: string; + addedBy: string; +} + +export interface OpsDeps { + state?: () => Promise; + build?: (op: AdminOp, state: TreasuryState) => Promise<{ xdr: string; hash: string; contractId: string; sourceAccount: string }>; + submit?: (xdr: string) => Promise<{ status: string; hash: string; ledger?: number; resultXdr?: string; error?: unknown }>; + now?: () => number; + account?: (address: string) => Promise; +} + +const TREASURY_EVENT = 'ADMIN_OP'; + +function nowMs(deps?: OpsDeps): number { + return deps?.now ? deps.now() : Date.now(); +} + +function signerKind(key: xdr.SignerKey): AccountSigner['kind'] { + try { + if (key.switch().name === 'signerKeyTypeEd25519') return 'ed25519'; + if (key.switch().name === 'signerKeyTypePreAuthTx') return 'preAuthTx'; + return 'hashX'; + } catch { + return 'hashX'; + } +} + +function signerKeyString(key: xdr.SignerKey): string { + const kind = signerKind(key); + if (kind === 'ed25519') return StrKey.encodeEd25519PublicKey(key.ed25519()); + if (kind === 'preAuthTx') return Buffer.from(key.preAuthTx()).toString('hex'); + return Buffer.from(key.hashX()).toString('hex'); +} + +function serialiseInstanceKey(key: xdr.ScVal): string | null { + try { + const native = scValToNative(key) as unknown; + if (typeof native === 'string') return native; + if (Array.isArray(native) && native.length > 0 && typeof native[0] === 'string') return native[0]; + if (native && typeof native === 'object' && 'tag' in native) return String((native as { tag: unknown }).tag); + return null; + } catch { + return null; + } +} + +async function readInstanceStorage(contractId: string): Promise> { + const map = new Map(); + try { + const key = (xdr.LedgerKey.contractData as any)({ + contract: new Contract(contractId).address(), + key: xdr.ScVal.scvLedgerKeyContractInstance(), + durability: xdr.ContractDataDurability.persistent(), + }); + const res = await rpcServer.getLedgerEntries(key); + const entry = res.entries?.[0]; + const storage = entry?.val.contractData().val().instance().storage(); + if (!storage) return map; + for (const item of storage) { + const name = serialiseInstanceKey(item.key()); + if (name) map.set(name, scValToNative(item.val())); + } + } catch (err) { + logger.warn({ err, contractId }, 'Unable to read contract instance storage'); + } + return map; +} + +async function readAccount(address: string): Promise { + const base: AccountState = { + address, + kind: 'account', + balanceXlm: null, + masterWeight: 0, + thresholds: { low: 0, medium: 0, high: 0 }, + signers: [], + }; + if (StrKey.isValidContract(address)) { + return { ...base, kind: 'contract' }; + } + try { + const res = await rpcServer.getLedgerEntries((xdr.LedgerKey.account as any)({ accountId: address })); + const account = res.entries?.[0]?.val.account(); + if (!account) return base; + const thresholds = account.thresholds(); + return { + address, + kind: 'account', + balanceXlm: account.balance().toString(), + masterWeight: thresholds[0] ?? 0, + thresholds: { low: thresholds[1] ?? 0, medium: thresholds[2] ?? 0, high: thresholds[3] ?? 0 }, + signers: account.signers().map((s) => ({ + key: signerKeyString(s.key()), + kind: signerKind(s.key()), + weight: s.weight(), + })), + }; + } catch (err) { + logger.warn({ err, address }, 'Unable to read account state'); + return base; + } +} + +/** Reads live treasury, market-core and account state for the console. */ +export async function getTreasuryState(deps: OpsDeps = {}): Promise { + if (deps.state) return deps.state(); + + const [risk, usage, available, marketCfg, instance] = await Promise.all([ + treasuryConfig().catch(() => null), + treasuryDailyUsage().catch(() => null), + treasuryAvailableCapital().catch(() => 0n), + marketCoreConfig().catch(() => null), + readInstanceStorage(CONTRACTS.treasury), + ]); + + const adminAddress = typeof instance.get('Admin') === 'string' ? String(instance.get('Admin')) : null; + const paused = typeof instance.get('Paused') === 'boolean' ? Boolean(instance.get('Paused')) : null; + const adminAccount = adminAddress ? await readAccount(adminAddress) : null; + const agentAddress = risk?.agent ?? ''; + const agentAccount = agentAddress ? await readAccount(agentAddress) : null; + + const amountRaw = BigInt(usage?.amount ?? 0); + const maxDaily = BigInt(risk?.max_daily ?? 0); + const remaining = maxDaily > amountRaw ? maxDaily - amountRaw : 0n; + const dayStart = Number(usage?.day_start ?? 0); + const resetsAt = new Date((dayStart + 86_400) * 1000).toISOString(); + + return { + network: config.STELLAR_NETWORK_PASSPHRASE, + contracts: { treasury: CONTRACTS.treasury, marketCore: CONTRACTS.marketCore, usdc: CONTRACTS.usdc }, + treasury: { + config: risk, + availableCapitalRaw: available.toString(), + usdcBalanceRaw: available.toString(), + dailyUsage: { amountRaw: amountRaw.toString(), dayStart, resetsAt, remainingRaw: remaining.toString() }, + paused, + admin: adminAddress ? { address: adminAddress, kind: adminAccount?.kind ?? 'account' } : null, + }, + marketCore: { paused: marketCfg?.paused ?? null, admin: marketCfg?.admin ?? null }, + agent: { address: agentAddress, xlmBalance: agentAccount?.balanceXlm ?? null }, + }; +} + +function humanUsdc(raw: string): string { + const value = BigInt(raw); + const whole = value / 10_000_000n; + const fraction = (value % 10_000_000n).toString().padStart(7, '0'); + return `${whole.toLocaleString('en-US')}.${fraction}`; +} + +export function humanSummary(op: AdminOp): string { + switch (op.kind) { + case 'treasury.withdraw': + return `Withdraw ${humanUsdc(op.amountRaw)} USDC from treasury ${CONTRACTS.treasury} to ${op.to}`; + case 'treasury.deposit': + return `Deposit ${humanUsdc(op.amountRaw)} USDC into treasury ${CONTRACTS.treasury} from ${op.from}`; + case 'treasury.set_paused': + return `${op.paused ? 'Pause' : 'Unpause'} treasury ${CONTRACTS.treasury}`; + case 'treasury.collect_market_fees': + return `Collect accrued fees from market ${op.onChainMarketId} into treasury ${CONTRACTS.treasury}`; + case 'market_core.set_paused': + return `${op.paused ? 'Pause' : 'Unpause'} market-core ${CONTRACTS.marketCore}`; + } +} + +function validateOp(op: AdminOp): void { + if (op.kind === 'treasury.withdraw') { + parseUsdc(formatUsdcSafe(op.amountRaw)); + if (!StrKey.isValidEd25519PublicKey(op.to) && !StrKey.isValidContract(op.to)) { + throw new AppError(400, 'INVALID_DESTINATION', 'to must be a G… account or C… contract'); + } + } + if (op.kind === 'treasury.deposit') { + parseUsdc(formatUsdcSafe(op.amountRaw)); + if (!StrKey.isValidEd25519PublicKey(op.from)) { + throw new AppError(400, 'INVALID_SOURCE', 'from must be a G… account'); + } + } + if (op.kind === 'treasury.collect_market_fees' && parseU64(op.onChainMarketId) === null) { + throw new AppError(400, 'INVALID_MARKET_ID', 'onChainMarketId must be a u64 decimal string'); + } +} + +function formatUsdcSafe(raw: string): string { + if (!/^\d+$/.test(raw)) throw new AppError(400, 'INVALID_AMOUNT', 'amountRaw must be an integer string'); + return formatUsdc(BigInt(raw)); +} + +async function checkTrustline(destination: string): Promise { + if (!StrKey.isValidEd25519PublicKey(destination)) { + return { name: 'destination_trustline', ok: true, detail: 'contract destination; trustline not applicable' }; + } + try { + const res = await rpcServer.getLedgerEntries((xdr.LedgerKey.account as any)({ accountId: destination })); + const account = res.entries?.[0]?.val.account(); + if (!account) return { name: 'destination_trustline', ok: false, detail: 'destination account not found' }; + return { name: 'destination_trustline', ok: true, detail: 'destination account exists (trustline checked on simulation)' }; + } catch { + return { name: 'destination_trustline', ok: true, detail: 'could not read destination; simulation will decide', critical: false }; + } +} + +// Soroban invokeHostFunction operations use the account's HIGH threshold. +export const REQUIRED_THRESHOLD_CATEGORY = 'invokeHostFunction'; + +export async function prepareOperation( + op: AdminOp, + requestedBy: string, + deps: OpsDeps = {}, +): Promise { + validateOp(op); + const state = deps.state ? await deps.state() : await getTreasuryState(deps); + const admin = state.treasury.admin; + if (!admin) throw new AppError(503, 'TREASURY_ADMIN_UNAVAILABLE', 'Could not read the on-chain treasury admin'); + if (admin.kind === 'contract') { + throw new AppError(501, 'UNSUPPORTED_ADMIN_KIND', 'Contract (C…) admin wallets require auth-entry signing and are not supported by this console'); + } + + const sourceAccount = op.kind === 'treasury.deposit' ? op.from : admin.address; + const account = deps.account ? await deps.account(sourceAccount) : await readAccount(sourceAccount); + const threshold = account.thresholds.high; + const required = Math.max(1, threshold, op.kind === 'treasury.withdraw' ? config.TREASURY_WITHDRAW_MIN_SIGNERS : 0); + + const checks: PreflightCheck[] = []; + const available = BigInt(state.treasury.availableCapitalRaw); + if (op.kind === 'treasury.withdraw') { + const amount = BigInt(op.amountRaw); + checks.push({ + name: 'amount_within_available_capital', + ok: amount <= available, + detail: `${formatUsdc(amount)} requested, ${formatUsdc(available)} available`, + critical: true, + }); + checks.push({ + name: 'destination_not_internal', + ok: op.to !== CONTRACTS.treasury && op.to !== CONTRACTS.marketCore, + detail: 'withdrawing to the treasury/market-core contract would lock the funds', + critical: true, + }); + checks.push(await checkTrustline(op.to)); + const remainingAfter = available > amount ? available - amount : 0n; + const maxDaily = BigInt(state.treasury.config?.max_daily ?? 0); + if (maxDaily > 0n && remainingAfter < maxDaily) { + checks.push({ + name: 'capital_below_daily_cap', + ok: true, + detail: `warning: remaining capital ${formatUsdc(remainingAfter)} is below max_daily ${formatUsdc(maxDaily)}`, + critical: false, + }); + } + } + if (op.kind === 'treasury.collect_market_fees') { + try { + const market = await getOnChainMarket(BigInt(op.onChainMarketId)); + checks.push({ + name: 'market_has_fees', + ok: BigInt(market.fees_accrued) > 0n, + detail: `${market.fees_accrued.toString()} base units accrued`, + critical: true, + }); + } catch (err) { + checks.push({ name: 'market_readable', ok: false, detail: err instanceof Error ? err.message : String(err), critical: true }); + } + } + + const blocking = checks.some((c) => !c.ok && c.critical !== false); + if (blocking) { + throw new AppError(422, 'PREFLIGHT_FAILED', 'Pre-flight checks failed', { checks }); + } + + const built = deps.build + ? await deps.build(op, state) + : await buildUnsigned(op, sourceAccount, admin.address); + checks.unshift({ + name: 'simulation', + ok: true, + detail: 'simulation succeeded; footprint, resource fee and auth entries assembled', + critical: true, + }); + + const now = nowMs(deps); + const expiresAt = new Date(now + config.TREASURY_OP_TTL_SECONDS * 1000); + const row = await prisma.adminOperation.create({ + data: { + kind: op.kind, + params: op as unknown as object, + contractId: built.contractId, + sourceAccount: built.sourceAccount, + unsignedXdr: built.xdr, + txHash: built.hash, + status: 'PREPARED', + requestedBy, + expiresAt, + }, + }); + await audit(row.id, 'PREPARED', requestedBy, { kind: op.kind, checks }); + + return { + id: row.id, + kind: op.kind, + xdr: built.xdr, + networkPassphrase: config.STELLAR_NETWORK_PASSPHRASE, + hash: built.hash, + expiresAt: expiresAt.toISOString(), + requiredSigners: required, + thresholds: account.thresholds, + checks, + humanSummary: humanSummary(op), + status: 'PREPARED', + }; +} + +async function buildUnsigned( + op: AdminOp, + sourceAccount: string, + adminAddress: string, +): Promise<{ xdr: string; hash: string; contractId: string; sourceAccount: string }> { + const options = { timeoutInSeconds: config.TREASURY_OP_TTL_SECONDS }; + let tx: AssembledTransaction; + let contractId: string; + + if (op.kind === 'market_core.set_paused') { + contractId = CONTRACTS.marketCore; + const client = new MarketCoreClient({ + contractId, + rpcUrl: config.STELLAR_RPC_URL, + networkPassphrase: config.STELLAR_NETWORK_PASSPHRASE, + publicKey: sourceAccount, + }); + tx = await client.set_paused({ paused: op.paused }, options); + } else { + contractId = CONTRACTS.treasury; + const client = new TreasuryClient({ + contractId, + rpcUrl: config.STELLAR_RPC_URL, + networkPassphrase: config.STELLAR_NETWORK_PASSPHRASE, + publicKey: sourceAccount, + }); + if (op.kind === 'treasury.set_paused') tx = await client.set_paused({ paused: op.paused }, options); + else if (op.kind === 'treasury.withdraw') tx = await client.withdraw({ to: op.to, amount: BigInt(op.amountRaw) }, options); + else if (op.kind === 'treasury.deposit') tx = await client.deposit({ from: op.from, amount: BigInt(op.amountRaw) }, options); + else tx = await client.collect_market_fees({ market_id: BigInt(op.onChainMarketId) }, options); + } + + const simError = (tx.simulation as { error?: string } | undefined)?.error; + if (simError) { + throw new AppError(422, 'SIMULATION_FAILED', 'Transaction simulation failed', { + contractError: contractErrorName(op.kind.startsWith('market_core') ? 'marketCore' : 'treasury', simError, simError), + }); + } + if (!tx.built) throw new AppError(502, 'BUILD_FAILED', 'Simulation did not produce a transaction envelope'); + return { + xdr: tx.toXDR(), + hash: tx.built.hash().toString('hex'), + contractId, + sourceAccount, + }; +} + +async function audit(operationId: string, state: string, actor: string, data?: Record): Promise { + await prisma.agentLog + .create({ + data: { + agentType: 'TRADER', + level: state === 'FAILED' ? 'ERROR' : 'INFO', + action: `${TREASURY_EVENT}_${state}`, + data: { operationId, actor, ...data } as object, + }, + }) + .catch((err) => logger.error({ err, operationId }, 'Failed to write admin operation audit entry')); +} + +function parseSignatures(row: { signatures: unknown }): SignatureRecord[] { + return Array.isArray(row.signatures) ? (row.signatures as unknown as SignatureRecord[]) : []; +} + +/** Merges a signed XDR into the prepared operation after hash/signature checks. */ +export async function addSignature( + operationId: string, + signedXdr: string, + addedBy: string, + deps: OpsDeps = {}, +): Promise<{ status: string; weightCollected: number; requiredSigners: number; signatures: SignatureRecord[] }> { + const row = await prisma.adminOperation.findUnique({ where: { id: operationId } }); + if (!row) throw new AppError(404, 'ADMIN_OP_NOT_FOUND', 'Admin operation not found'); + const now = nowMs(deps); + if (row.expiresAt.getTime() <= now || row.status === 'EXPIRED') { + await prisma.adminOperation.update({ where: { id: row.id }, data: { status: 'EXPIRED' } }); + await audit(row.id, 'EXPIRED', addedBy); + throw new AppError(410, 'ADMIN_OP_EXPIRED', 'This operation has expired; prepare a new one'); + } + if (!['PREPARED', 'PARTIALLY_SIGNED'].includes(row.status)) { + throw new AppError(409, 'ADMIN_OP_NOT_SIGNABLE', `Operation is ${row.status}`); + } + + let incoming: unknown; + try { + incoming = TransactionBuilder.fromXDR(signedXdr, config.STELLAR_NETWORK_PASSPHRASE); + } catch { + throw new AppError(400, 'INVALID_SIGNED_XDR', 'signedXdr is not a valid Stellar transaction envelope'); + } + const incomingTx = incoming as { hash(): Buffer; signatures?: Array<{ hint(): Buffer; signature(): Buffer }> }; + const hashHex = incomingTx.hash().toString('hex'); + if (hashHex !== row.txHash) { + throw new AppError(409, 'ADMIN_OP_HASH_MISMATCH', 'Signed transaction body does not match the prepared operation'); + } + + const admin = (row.sourceAccount && (deps.account ? await deps.account(row.sourceAccount) : await readAccount(row.sourceAccount))) || null; + if (!admin) throw new AppError(503, 'ADMIN_STATE_UNAVAILABLE', 'Could not read the admin account state'); + + const existing = parseSignatures(row); + const seenSigners = new Set(existing.map((s) => s.signer)); + const records = [...existing]; + const required = Math.max( + 1, + admin.thresholds.high, + row.kind === 'treasury.withdraw' ? config.TREASURY_WITHDRAW_MIN_SIGNERS : 0, + ); + + const candidates: AccountSigner[] = [ + ...(admin.masterWeight > 0 ? [{ key: admin.address, kind: 'ed25519' as const, weight: admin.masterWeight }] : []), + ...admin.signers, + ]; + + for (const sig of incomingTx.signatures ?? []) { + const hint = sig.hint(); + const match = candidates.find((candidate) => { + if (candidate.kind !== 'ed25519') return false; + try { + return Keypair.fromPublicKey(candidate.key).signatureHint().equals(hint); + } catch { + return false; + } + }); + if (!match || seenSigners.has(match.key)) continue; + let valid = false; + try { + valid = Keypair.fromPublicKey(match.key).verify(incomingTx.hash(), sig.signature()); + } catch { + valid = false; + } + if (!valid) continue; + seenSigners.add(match.key); + records.push({ + signer: match.key, + hint: hint.toString('hex'), + signature: sig.signature().toString('base64'), + addedAt: new Date(now).toISOString(), + addedBy, + }); + } + + // Rebuild the envelope with every accepted signature. + const merged = TransactionBuilder.fromXDR(row.unsignedXdr, config.STELLAR_NETWORK_PASSPHRASE) as unknown as { + addDecoratedSignature(sig: unknown): void; + }; + for (const record of records) { + merged.addDecoratedSignature( + new xdr.DecoratedSignature({ hint: Buffer.from(record.hint, 'hex'), signature: Buffer.from(record.signature, 'base64') }), + ); + } + const weightCollected = records.reduce((sum, record) => { + const signer = candidates.find((candidate) => candidate.key === record.signer); + return sum + (signer?.weight ?? 0); + }, 0); + + const status = weightCollected >= required ? 'READY' : 'PARTIALLY_SIGNED'; + const signedXdrOut = (merged as unknown as { toXDR(): string }).toXDR(); + await prisma.adminOperation.update({ + where: { id: row.id }, + data: { + signedXdr: signedXdrOut, + signatures: records as unknown as object, + weightCollected, + status, + }, + }); + await audit(row.id, status, addedBy, { weightCollected, required }); + + return { status, weightCollected, requiredSigners: required, signatures: records }; +} + +/** Submits a READY operation and records its final result. No server key is used. */ +export async function submitOperation( + operationId: string, + deps: OpsDeps = {}, +): Promise<{ status: string; ledger?: number; resultXdr?: string; hash: string }> { + const row = await prisma.adminOperation.findUnique({ where: { id: operationId } }); + if (!row) throw new AppError(404, 'ADMIN_OP_NOT_FOUND', 'Admin operation not found'); + const now = nowMs(deps); + if (row.expiresAt.getTime() <= now) { + await prisma.adminOperation.update({ where: { id: row.id }, data: { status: 'EXPIRED' } }); + await audit(row.id, 'EXPIRED', 'system'); + throw new AppError(410, 'ADMIN_OP_EXPIRED', 'This operation has expired; prepare a new one'); + } + if (row.status !== 'READY' || !row.signedXdr) { + throw new AppError(409, 'ADMIN_OP_NOT_READY', `Operation is ${row.status}; collect enough signatures first`); + } + if (config.STELLAR_NETWORK_PASSPHRASE !== Networks.TESTNET) { + throw new AppError(403, 'TESTNET_ONLY', 'Admin submission is testnet-only'); + } + + const result = deps.submit + ? await deps.submit(row.signedXdr) + : await submitToRpc(row.signedXdr); + + const success = result.status === 'SUCCESS'; + await prisma.adminOperation.update({ + where: { id: row.id }, + data: { + status: success ? 'SUCCESS' : 'FAILED', + resultXdr: result.resultXdr, + ledger: result.ledger, + error: result.error !== undefined ? (result.error as object) : undefined, + }, + }); + await audit(row.id, success ? 'SUCCESS' : 'FAILED', 'system', { hash: result.hash, ledger: result.ledger }); + return result; +} + +async function submitToRpc(signedXdr: string): Promise<{ status: string; ledger?: number; resultXdr?: string; hash: string; error?: unknown }> { + const tx = TransactionBuilder.fromXDR(signedXdr, config.STELLAR_NETWORK_PASSPHRASE); + const hash = tx.hash().toString('hex'); + const sent = await rpcServer.sendTransaction(tx); + if (sent.status === 'ERROR') { + return { status: 'FAILED', hash, error: sent.errorResult ?? sent }; + } + const deadline = Date.now() + 30_000; + while (Date.now() < deadline) { + const result = await rpcServer.getTransaction(hash); + if (result.status !== 'NOT_FOUND') { + const done = result as any; + return { + status: result.status, + hash, + ledger: done.ledger, + resultXdr: done.resultXdr?.toXDR?.('base64') ?? undefined, + error: result.status === 'FAILED' ? done.resultXdr : undefined, + }; + } + await new Promise((r) => setTimeout(r, 1_000)); + } + return { status: 'FAILED', hash, error: { message: 'Timed out waiting for the transaction result' } }; +} + +export async function recordSubmitted(operationId: string, txHash: string, actor: string): Promise { + const row = await prisma.adminOperation.findUnique({ where: { id: operationId } }); + if (!row) throw new AppError(404, 'ADMIN_OP_NOT_FOUND', 'Admin operation not found'); + const result = await rpcServer.getTransaction(txHash); + const done = result as any; + const success = result.status === 'SUCCESS'; + await prisma.adminOperation.update({ + where: { id: row.id }, + data: { + status: success ? 'SUCCESS' : result.status === 'NOT_FOUND' ? 'SUBMITTED' : 'FAILED', + resultXdr: done.resultXdr?.toXDR?.('base64') ?? undefined, + ledger: done.ledger, + }, + }); + await audit(row.id, success ? 'SUCCESS' : 'SUBMITTED', actor, { txHash }); +} + +export async function cancelOperation(operationId: string, actor: string): Promise { + const row = await prisma.adminOperation.findUnique({ where: { id: operationId } }); + if (!row) throw new AppError(404, 'ADMIN_OP_NOT_FOUND', 'Admin operation not found'); + if (['SUBMITTED', 'SUCCESS', 'FAILED', 'EXPIRED'].includes(row.status)) { + throw new AppError(409, 'ADMIN_OP_NOT_CANCELLABLE', `Operation is ${row.status}`); + } + await prisma.adminOperation.update({ where: { id: row.id }, data: { status: 'CANCELLED' } }); + await audit(row.id, 'CANCELLED', actor); +} + +export async function getOperation(operationId: string) { + const row = await prisma.adminOperation.findUnique({ where: { id: operationId } }); + if (!row) throw new AppError(404, 'ADMIN_OP_NOT_FOUND', 'Admin operation not found'); + return row; +} + +export async function listOperations(page: number, limit: number) { + const [rows, total] = await Promise.all([ + prisma.adminOperation.findMany({ orderBy: { createdAt: 'desc' }, skip: (page - 1) * limit, take: limit }), + prisma.adminOperation.count(), + ]); + return { rows, total }; +} diff --git a/src/types/index.ts b/src/types/index.ts index 9a037a2..16644c5 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -127,6 +127,8 @@ export interface TraceVerification { computedHash: string; storedHash: string | null; verified: boolean; + /** Where the verified bytes came from (#100). */ + contentSource?: 'cache' | 'gateway'; verifiedAt: Date; } diff --git a/src/validators/admin.validator.ts b/src/validators/admin.validator.ts new file mode 100644 index 0000000..3500a37 --- /dev/null +++ b/src/validators/admin.validator.ts @@ -0,0 +1,28 @@ +import { z } from 'zod'; + +const rawAmount = z.string().regex(/^\d+$/, 'must be an integer in 7-decimal base units'); +const address = z.string().min(1); +const u64 = z.string().regex(/^\d+$/, 'must be a decimal u64'); + +export const adminOpSchema = z.discriminatedUnion('kind', [ + z.object({ kind: z.literal('treasury.set_paused'), paused: z.boolean() }), + z.object({ kind: z.literal('treasury.withdraw'), to: address, amountRaw: rawAmount }), + z.object({ kind: z.literal('treasury.deposit'), from: address, amountRaw: rawAmount }), + z.object({ kind: z.literal('treasury.collect_market_fees'), onChainMarketId: u64 }), + z.object({ kind: z.literal('market_core.set_paused'), paused: z.boolean() }), +]); + +export const createOperationSchema = z.object({ + operation: adminOpSchema, + force: z.boolean().optional(), +}); + +export const signatureSchema = z.object({ + signedXdr: z.string().min(1), +}); + +export const recordSchema = z.object({ + txHash: z.string().regex(/^[0-9a-fA-F]{64}$/, 'must be a 64-character hex transaction hash'), +}); + +export type AdminOpInput = z.infer; diff --git a/tests/content-store.test.ts b/tests/content-store.test.ts new file mode 100644 index 0000000..0fe05df --- /dev/null +++ b/tests/content-store.test.ts @@ -0,0 +1,178 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import crypto from 'node:crypto'; +import { setupTestEnv } from './helpers/env'; + +setupTestEnv(); + +function memoryStore() { + const map = new Map(); + return { + map, + async put(entry: any) { + const hash = crypto.createHash('sha256').update(entry.bytes).digest('hex'); + assert.equal(hash, entry.sha256); + const existing = map.get(entry.cid); + if (existing && existing.sha256 !== entry.sha256) { + throw new Error('IPFS_CID_HASH_CONFLICT'); + } + map.set(entry.cid, { sha256: entry.sha256, bytes: Buffer.from(entry.bytes), origin: entry.origin }); + }, + async get(cid: string) { + return map.get(cid) ?? null; + }, + async touch() {}, + async evict() { return { removed: 0, freedBytes: 0 }; }, + }; +} + +test('pinJson writes the exact pinned bytes to the store (#100)', async () => { + const { pinJson, setIpfsTransport } = await import('../src/services/ipfs.service'); + const { setContentStore } = await import('../src/services/content-store.service'); + const store = memoryStore(); + setContentStore(store as any); + const pinnedBytes = Buffer.from('{"a":1}'); + setIpfsTransport({ + async pin() { return 'bafy-pinned'; }, + async fetch() { return pinnedBytes; }, + }); + + const pinned = await pinJson({ a: 1 }, 'doc'); + const stored = await store.get(pinned.cid); + assert.ok(stored, 'pinned bytes are recorded'); + assert.equal(stored!.sha256, pinned.hash); + assert.ok(stored!.bytes.equals(pinned.bytes), 'exact bytes, not a re-serialisation'); +}); + +test('a store failure does not fail pinJson (#100)', async () => { + const { pinJson, setIpfsTransport } = await import('../src/services/ipfs.service'); + const { setContentStore } = await import('../src/services/content-store.service'); + setContentStore({ + async put() { throw new Error('cache down'); }, + async get() { return null; }, + async touch() {}, + async evict() { return { removed: 0, freedBytes: 0 }; }, + } as any); + setIpfsTransport({ async pin() { return 'bafy-ok'; }, async fetch() { throw new Error('no'); } }); + const pinned = await pinJson({ ok: true }, 'doc'); + assert.equal(pinned.cid, 'bafy-ok'); +}); + +test('fetchVerifiedBytes serves a cache hit during a gateway outage (#100)', async () => { + const { fetchVerifiedBytes, pinJson, setIpfsTransport } = await import('../src/services/ipfs.service'); + const { setContentStore } = await import('../src/services/content-store.service'); + const store = memoryStore(); + setContentStore(store as any); + let gatewayUp = true; + const bytes = Buffer.from('{"n":1}'); + const hash = crypto.createHash('sha256').update(bytes).digest('hex'); + setIpfsTransport({ + async pin() { return 'bafy-cache'; }, + async fetch() { + if (!gatewayUp) throw new Error('gateway down'); + return bytes; + }, + }); + await pinJson({ n: 1 }, 'doc'); + + gatewayUp = false; + const result = await fetchVerifiedBytes('bafy-cache', hash, 'test'); + assert.equal(result.source, 'cache'); + assert.ok(result.bytes.equals(bytes)); +}); + +test('a cached row with a different hash is never served (#100)', async () => { + const { fetchVerifiedBytes, setIpfsTransport } = await import('../src/services/ipfs.service'); + const { setContentStore } = await import('../src/services/content-store.service'); + const store = memoryStore(); + setContentStore(store as any); + const cachedBytes = Buffer.from('old'); + const cachedHash = crypto.createHash('sha256').update(cachedBytes).digest('hex'); + const expectedBytes = Buffer.from('new'); + const expectedHash = crypto.createHash('sha256').update(expectedBytes).digest('hex'); + store.map.set('bafy-x', { sha256: cachedHash, bytes: cachedBytes, origin: 'PINNED' }); + + let gatewayCalls = 0; + setIpfsTransport({ + async pin() { return 'x'; }, + async fetch() { gatewayCalls++; return expectedBytes; }, + }); + const result = await fetchVerifiedBytes('bafy-x', expectedHash, 'test'); + assert.equal(result.source, 'gateway'); + assert.equal(gatewayCalls, 1, 'gateway consulted instead of returning the wrong cached bytes'); + assert.ok(result.bytes.equals(expectedBytes)); + assert.equal(store.map.get('bafy-x')!.sha256, cachedHash, 'existing CID is not overwritten'); +}); + +test('gateway bytes that fail verification are returned but never stored (#100)', async () => { + const { fetchVerifiedBytes, setIpfsTransport } = await import('../src/services/ipfs.service'); + const { setContentStore } = await import('../src/services/content-store.service'); + const store = memoryStore(); + setContentStore(store as any); + const badBytes = Buffer.from('tampered'); + const expected = crypto.createHash('sha256').update('expected').digest('hex'); + setIpfsTransport({ async pin() { return 'x'; }, async fetch() { return badBytes; } }); + + const result = await fetchVerifiedBytes('bafy-bad', expected, 'test'); + assert.ok(result.bytes.equals(badBytes), 'caller can report the computed hash'); + assert.equal(store.map.has('bafy-bad'), false, 'unverified bytes are never cached'); +}); + +test('postgres store rejects a CID hash conflict and oversized objects (#100)', async () => { + const { postgresContentStore } = await import('../src/services/content-store.service'); + const { prisma } = await import('../src/lib/prisma'); + const bytes = Buffer.from('abc'); + const hash = crypto.createHash('sha256').update(bytes).digest('hex'); + + (prisma.ipfsContent.findUnique as any) = async () => ({ cid: 'bafy-x', sha256: 'f'.repeat(64), bytes, size: 3, origin: 'PINNED' }); + await assert.rejects( + postgresContentStore.put({ cid: 'bafy-x', sha256: hash, bytes, origin: 'PINNED', verifiedBy: 'test' }), + (err: any) => err.code === 'IPFS_CID_HASH_CONFLICT', + ); + + const huge = Buffer.alloc(262_145, 1); + const hugeHash = crypto.createHash('sha256').update(huge).digest('hex'); + await assert.rejects( + postgresContentStore.put({ cid: 'bafy-huge', sha256: hugeHash, bytes: huge, origin: 'PINNED', verifiedBy: 'test' }), + (err: any) => err.code === 'IPFS_CONTENT_TOO_LARGE', + ); +}); + +test('postgres store eviction is a no-op under budget (#100)', async () => { + const { postgresContentStore } = await import('../src/services/content-store.service'); + const { prisma } = await import('../src/lib/prisma'); + (prisma.ipfsContent.aggregate as any) = async () => ({ _sum: { size: 10 } }); + const result = await postgresContentStore.evict(100); + assert.deepEqual(result, { removed: 0, freedBytes: 0 }); +}); + +test('trace verification succeeds from cache when both gateways are down (#100)', async () => { + const crypto = await import('node:crypto'); + const { setContentStore } = await import('../src/services/content-store.service'); + const { setIpfsTransport } = await import('../src/services/ipfs.service'); + const { prisma } = await import('../src/lib/prisma'); + const bytes = Buffer.from('{"trace":"cached"}'); + const hash = crypto.createHash('sha256').update(bytes).digest('hex'); + const map = new Map([['bafy-cache-only', { sha256: hash, bytes, origin: 'PINNED' }]]); + setContentStore({ + async put() {}, + async get(cid: string) { return map.get(cid) ?? null; }, + async touch() {}, + async evict() { return { removed: 0, freedBytes: 0 }; }, + } as any); + setIpfsTransport({ + async pin() { return 'x'; }, + async fetch() { throw new Error('both gateways down'); }, + }); + + const row: any = { id: 't1', onChainTraceId: 1n, traceHash: hash, ipfsCid: 'bafy-cache-only', verified: false }; + (prisma.reasoningTrace.findUnique as any) = async () => row; + (prisma.reasoningTrace.update as any) = async ({ data }: any) => Object.assign(row, data); + const chain = await import('../src/services/chain.service'); + (chain as any).getOnChainTrace = async () => ({ ipfs_cid: 'bafy-cache-only', trace_hash: Buffer.from(hash, 'hex') }); + + const { verifyTraceOnChain } = await import('../src/services/trace-publish.service'); + const result = await verifyTraceOnChain('t1'); + assert.equal(result.verified, true); + assert.equal(result.contentSource, 'cache'); +}); diff --git a/tests/http.test.ts b/tests/http.test.ts index 813122a..41c9c55 100644 --- a/tests/http.test.ts +++ b/tests/http.test.ts @@ -56,3 +56,15 @@ test('BigInt values serialise as decimal strings', async () => { const replacer = app.get('json replacer'); assert.equal(JSON.stringify({ id: 18446744073709551615n }, replacer), '{"id":"18446744073709551615"}'); }); + +test('admin treasury routes require an admin (#156)', async () => { + const { signToken } = await import('../src/middlewares/auth.middleware'); + await withServer(async (base) => { + const anon = await fetch(`${base}/api/v1/admin/treasury`); + assert.equal(anon.status, 401); + const token = signToken('u1', Keypair.random().publicKey()); + const nonAdmin = await fetch(`${base}/api/v1/admin/treasury`, { headers: { authorization: `Bearer ${token}` } }); + assert.equal(nonAdmin.status, 403); + assert.equal((await nonAdmin.json()).error.code, 'FORBIDDEN'); + }); +}); diff --git a/tests/llm-router.test.ts b/tests/llm-router.test.ts new file mode 100644 index 0000000..c37cb88 --- /dev/null +++ b/tests/llm-router.test.ts @@ -0,0 +1,176 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { setupTestEnv } from './helpers/env'; + +setupTestEnv(); + +async function modules() { + const { createRouter } = await import('../src/lib/llm/router'); + const { LlmError } = await import('../src/lib/llm/types'); + const { DEFAULT_POLICIES, loadPolicies } = await import('../src/lib/llm/policy'); + return { createRouter, LlmError, DEFAULT_POLICIES, loadPolicies }; +} + +function spec(provider: string, model = 'm', extra: any = {}) { + return { provider, model, temperature: 0.2, maxTokens: 100, timeoutMs: 1000, retries: 0, ...extra } as any; +} + +function fakeProvider(name: string, outcomes: any[], available = true) { + let index = 0; + const calls: any[] = []; + return { + name, + calls, + available: () => available, + async complete(_spec: any, req: any) { + calls.push({ spec: _spec, req }); + const next = outcomes[Math.min(index++, outcomes.length - 1)]; + if (next instanceof Error) throw next; + return typeof next === 'string' ? { text: next } : next; + }, + }; +} + +const request = (task: string) => ({ task, system: 's', user: 'u', mode: 'json' }) as any; + +test('default policy routes market_generation to anthropic then gemini (#157)', async () => { + const { DEFAULT_POLICIES } = await modules(); + assert.deepEqual( + DEFAULT_POLICIES.market_generation.map((s: any) => `${s.provider}:${s.model}`), + ['anthropic:claude-sonnet-4-6', 'gemini:gemini-2.0-flash'], + ); + assert.equal(DEFAULT_POLICIES.hedge_conditions[0].maxTokens, 1024); + assert.equal(DEFAULT_POLICIES.generic_text[0].temperature, 0.4); +}); + +test('missing provider key is skipped without counting a failure (#157)', async () => { + const { createRouter } = await modules(); + const unavailable = fakeProvider('anthropic', [new Error('should not run')], false); + const gemini = fakeProvider('gemini', ['{"ok":true}']); + const router = createRouter({ generic_json: [spec('anthropic'), spec('gemini')] } as any, { + anthropic: unavailable as any, + gemini: gemini as any, + }); + const result = await router.complete(request('generic_json')); + assert.equal(result.provenance.provider, 'gemini'); + assert.equal(unavailable.calls.length, 0); + assert.equal(router.health().anthropic.state, 'healthy'); +}); + +test('short rate limits retry inline; long ones cool and fail over (#157)', async () => { + const { createRouter, LlmError } = await modules(); + let now = 1_000; + const sleeps: number[] = []; + const anthropic = fakeProvider('anthropic', [new LlmError('rate_limited', 'anthropic', 'slow down', 1_000), '{"ok":true}']); + const router = createRouter( + { generic_json: [spec('anthropic', 'm', { retries: 1 })] } as any, + { anthropic: anthropic as any }, + { clock: () => now, sleep: async (ms) => { sleeps.push(ms); now += ms; } }, + ); + const result = await router.complete(request('generic_json')); + assert.equal(anthropic.calls.length, 2); + assert.deepEqual(sleeps, [1_000]); + assert.equal(result.provenance.attempts[0].kind, 'rate_limited'); + + now = 2_000; + const slow = fakeProvider('anthropic', [new LlmError('rate_limited', 'anthropic', 'wait', 30_000)]); + const gemini = fakeProvider('gemini', ['{"ok":true}']); + const router2 = createRouter( + { generic_json: [spec('anthropic'), spec('gemini')] } as any, + { anthropic: slow as any, gemini: gemini as any }, + { clock: () => now, sleep: async (ms) => { now += ms; } }, + ); + const result2 = await router2.complete(request('generic_json')); + assert.equal(result2.provenance.provider, 'gemini'); + assert.equal(router2.health().anthropic.state, 'cooling'); + assert.ok((router2.health().anthropic.until ?? 0) > now); +}); + +test('quota exhaustion opens the circuit for an hour (#157)', async () => { + const { createRouter, LlmError } = await modules(); + let now = 5_000; + const anthropic = fakeProvider('anthropic', [new LlmError('quota_exhausted', 'anthropic', 'quota')]); + const gemini = fakeProvider('gemini', ['{"ok":true}']); + const router = createRouter({ generic_json: [spec('anthropic'), spec('gemini')] } as any, { + anthropic: anthropic as any, + gemini: gemini as any, + }, { clock: () => now }); + await router.complete(request('generic_json')); + assert.equal(router.health().anthropic.state, 'open'); + assert.equal((router.health().anthropic.until ?? 0) - now, 60 * 60 * 1000); + + const skipped = await router.complete(request('generic_json')); + assert.equal(skipped.provenance.provider, 'gemini'); + assert.equal(anthropic.calls.length, 1, 'open circuit is not called again'); +}); + +test('three server errors cool a provider for 60s (#157)', async () => { + const { createRouter, LlmError } = await modules(); + let now = 10_000; + const anthropic = fakeProvider('anthropic', [new LlmError('server', 'anthropic', 'boom')]); + const router = createRouter({ generic_json: [spec('anthropic')] } as any, { anthropic: anthropic as any }, { clock: () => now }); + for (let i = 0; i < 3; i++) { + await assert.rejects(router.complete(request('generic_json'))); + } + assert.equal(router.health().anthropic.state, 'cooling'); + assert.equal((router.health().anthropic.until ?? 0) - now, 60_000); + await assert.rejects(router.complete(request('generic_json'))); + assert.equal(anthropic.calls.length, 3, 'cooling provider is skipped'); +}); + +test('JSON parse failure fails over to the next model (#157)', async () => { + const { createRouter } = await modules(); + const anthropic = fakeProvider('anthropic', ['not json']); + const gemini = fakeProvider('gemini', ['{"ok":true}']); + const router = createRouter({ generic_json: [spec('anthropic'), spec('gemini')] } as any, { + anthropic: anthropic as any, + gemini: gemini as any, + }); + const result = await router.complete(request('generic_json')); + assert.equal(result.provenance.provider, 'gemini'); + assert.equal(result.provenance.attempts[0].kind, 'parse'); +}); + +test('text mode is passed through without a JSON requirement (#157)', async () => { + const { createRouter } = await modules(); + const anthropic = fakeProvider('anthropic', ['plain text']); + const router = createRouter({ generic_text: [spec('anthropic')] } as any, { anthropic: anthropic as any }); + const result = await router.complete({ task: 'generic_text', system: 's', user: 'u', mode: 'text' } as any); + assert.equal(result.text, 'plain text'); + assert.equal(anthropic.calls[0].req.mode, 'text'); +}); + +test('auth errors fail over, and all-exhausted includes attempts (#157)', async () => { + const { createRouter, LlmError } = await modules(); + const anthropic = fakeProvider('anthropic', [new LlmError('auth', 'anthropic', 'bad key')]); + const gemini = fakeProvider('gemini', [new LlmError('server', 'gemini', 'down')]); + const router = createRouter({ generic_json: [spec('anthropic'), spec('gemini')] } as any, { + anthropic: anthropic as any, + gemini: gemini as any, + }); + await assert.rejects( + router.complete(request('generic_json')), + (err: any) => { + assert.ok(err instanceof LlmError); + assert.equal(err.attempts.length, 2); + assert.equal(err.attempts[0].kind, 'auth'); + return true; + }, + ); +}); + +test('LLM_POLICY validation names the bad task (#157)', async () => { + const { loadPolicies } = await modules(); + assert.throws( + () => loadPolicies(JSON.stringify({ market_generation: [] })), + /market_generation/, + ); + assert.throws( + () => loadPolicies('{not json'), + /not valid JSON/, + ); + const parsed = loadPolicies(JSON.stringify({ + market_generation: [{ provider: 'gemini', model: 'gemini-2.0-flash', temperature: 0, maxTokens: 10, timeoutMs: 1000 }], + })); + assert.equal(parsed.market_generation[0].provider, 'gemini'); +}); diff --git a/tests/reports-http.test.ts b/tests/reports-http.test.ts new file mode 100644 index 0000000..185b8bd --- /dev/null +++ b/tests/reports-http.test.ts @@ -0,0 +1,49 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import type { AddressInfo } from 'node:net'; +import { setupTestEnv } from './helpers/env'; + +setupTestEnv(); + +async function withServer(fn: (base: string) => Promise) { + const { prisma } = await import('../src/lib/prisma'); + (prisma.transparencyReport.findMany as any) = async () => [{ + date: '2026-10-14', anchorLedger: 100, cid: 'bafy', hash: 'a'.repeat(64), + onChainTraceId: 1n, mode: 'dry-run', consistency: 'exact', report: {}, createdAt: new Date(), + }]; + (prisma.transparencyReport.count as any) = async () => 1; + (prisma.transparencyReport.findFirst as any) = async () => ({ + date: '2026-10-14', anchorLedger: 100, cid: 'bafy', hash: 'a'.repeat(64), + onChainTraceId: 1n, mode: 'dry-run', consistency: 'exact', report: {}, createdAt: new Date(), + }); + (prisma.transparencyReport.findUnique as any) = async () => null; + + const { default: app } = await import('../src/app'); + const server = app.listen(0); + try { + await fn(`http://127.0.0.1:${(server.address() as AddressInfo).port}`); + } finally { + server.close(); + } +} + +test('GET /api/v1/reports is public and paginated (#159)', async () => { + await withServer(async (base) => { + const res = await fetch(`${base}/api/v1/reports?page=2&limit=5`); + assert.equal(res.status, 200); + const body = await res.json(); + assert.equal(body.ok, true); + assert.equal(body.data[0].cid, 'bafy'); + assert.equal(body.meta.page, 2); + assert.equal(body.meta.limit, 5); + assert.ok(body.data[0].verification.recipe.length > 0); + }); +}); + +test('GET /api/v1/reports/latest and :date behave (#159)', async () => { + await withServer(async (base) => { + assert.equal((await fetch(`${base}/api/v1/reports/latest`)).status, 200); + assert.equal((await fetch(`${base}/api/v1/reports/2026-10-14`)).status, 404); + assert.equal((await fetch(`${base}/api/v1/reports/not-a-date`)).status, 400); + }); +}); diff --git a/tests/transparency-report.test.ts b/tests/transparency-report.test.ts new file mode 100644 index 0000000..4203088 --- /dev/null +++ b/tests/transparency-report.test.ts @@ -0,0 +1,300 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import crypto from 'node:crypto'; +import { setupTestEnv } from './helpers/env'; + +setupTestEnv(); + +function market(overrides: any = {}) { + return { + creator: 'GTREASURY', + status: { tag: 'Open', values: undefined }, + reserve_yes: 100n, + reserve_no: 200n, + fee_bps: 100, + lp_claimed: false, + fees_accrued: 0n, + close_time: 0n, + meta_uri: '', + question_hash: Buffer.alloc(0), + resolution_hash: Buffer.alloc(0), + sets_minted: 0n, + category: { tag: 'Macro', values: undefined }, + ...overrides, + } as any; +} + +function snapshot(overrides: any = {}) { + return { marketId: 0n, market: market(), position: { yes: 10n, no: 5n }, ...overrides } as any; +} + +test('mark value uses priceYesBps integer math (#159)', async () => { + const { markValueRaw } = await import('../src/services/transparency-report.service'); + const { priceYesBps } = await import('../src/generated/fpmm'); + const bps = Number(priceYesBps(100n, 200n)); + assert.equal(bps, 6666); + assert.equal(markValueRaw(10n, 5n, bps), ((10n * 6666n + 5n * 3334n) / 10_000n).toString()); +}); + +test('liquidation value equals collateralOutForShares for each side (#159)', async () => { + const { liquidationValueRaw } = await import('../src/services/transparency-report.service'); + const { collateralOutForShares } = await import('../src/generated/fpmm'); + const expected = collateralOutForShares(100n, 200n, 10n, 100n) + collateralOutForShares(200n, 100n, 5n, 100n); + assert.equal(liquidationValueRaw(100n, 200n, 10n, 5n, 100n), expected.toString()); +}); + +test('void settlement uses settleValue, and final markets expose settlement/lp values (#159)', async () => { + const { valueMarket } = await import('../src/services/transparency-report.service'); + const { settleValue } = await import('../src/generated/fpmm'); + const voided = valueMarket(snapshot({ market: market({ status: { tag: 'Void', values: undefined } }) }), 'GTREASURY'); + assert.equal(voided.settlementValueRaw, settleValue('void', 10n, 5n).toString()); + assert.equal(voided.lpRemainderRaw, settleValue('void', 100n, 200n).toString()); + const open = valueMarket(snapshot(), 'GTREASURY'); + assert.equal(open.settlementValueRaw, null); + assert.equal(open.createdByTreasury, true); +}); + +test('the report document is byte-identical across runs (#159)', async () => { + const { buildReportDocument, valueMarket, totalsFor } = await import('../src/services/transparency-report.service'); + const markets = [valueMarket(snapshot(), 'GTREASURY')]; + const inputs = { + date: '2026-10-14', + network: 'Test SDF Network ; September 2015', + contracts: { treasury: 'CT', marketCore: 'CM', usdc: 'CU', reasoningRegistry: 'CR' }, + agent: 'GTREASURY', + anchor: { ledger: 100, closedAt: '2026-10-14T00:00:00.000Z' }, + consistency: 'exact' as const, + treasury: { + usdcBalanceRaw: '1000', + availableCapitalRaw: '1000', + riskConfig: { maxTradeRaw: '1', maxMarketRaw: '2', maxDailyRaw: '3' }, + dailyUsage: { amountRaw: '0', dayStart: 0 }, + }, + markets, + periodFlows: { + coverage: { fromLedger: 100, toLedger: 100, complete: true, gaps: [] as Array<[number, number]> }, + tradesBoughtRaw: '0', tradesSoldRaw: '0', feesPaidRaw: '0', redeemedRaw: '0', + feesCollectedRaw: '0', seedDeployedRaw: '0', tradeCount: 0, + }, + traces: { total: 0, byAction: {} }, + previous: null, + methodDoc: 'doc', + }; + const a = buildReportDocument(inputs); + const b = buildReportDocument(inputs); + assert.deepEqual(a, b); + assert.equal(JSON.stringify(a), JSON.stringify(b)); + assert.deepEqual(a.totals, totalsFor(markets, '1000')); +}); + +test('coverage is incomplete when the indexer cursor is behind the period (#159)', async () => { + const { buildReportForDate } = await import('../src/services/transparency-report.service'); + const report = await buildReportForDate('2026-10-14', { + latestLedger: async () => 100, + marketCount: async () => 0n, + treasuryConfig: async () => ({ agent: 'GTREASURY', max_daily: 1000n, max_market: 100n, max_trade: 10n }), + dailyUsage: async () => ({ amount: 0n, day_start: 0n }), + availableCapital: async () => 500n, + chainEvents: async () => [ + { contractName: 'marketCore', eventName: 'Trade', ledger: 90, ledgerClosedAt: new Date(), params: { trader: 'GTREASURY', collateral: '100', fee: '1', is_buy: true } }, + ], + indexerCursor: async () => 50, + traceCount: async () => 0n, + previousReport: async () => null, + methodDoc: 'doc', + }); + assert.equal(report.period_flows.coverage.complete, false); + assert.deepEqual(report.period_flows.coverage.gaps, [[51, 90]]); + assert.equal(report.period_flows.tradesBoughtRaw, '100'); + assert.equal(report.period_flows.tradeCount, 1); +}); + +test('snapshot retries when the ledger moves and flags best_effort after 3 attempts (#159)', async () => { + const { buildReportForDate } = await import('../src/services/transparency-report.service'); + const sequence = [100, 101, 101, 102, 102, 103]; + let index = 0; + const report = await buildReportForDate('2026-10-14', { + latestLedger: async () => sequence[Math.min(index++, sequence.length - 1)], + marketCount: async () => 0n, + treasuryConfig: async () => ({ agent: 'GTREASURY', max_daily: 1000n, max_market: 100n, max_trade: 10n }), + dailyUsage: async () => ({ amount: 0n, day_start: 0n }), + availableCapital: async () => 0n, + chainEvents: async () => [], + indexerCursor: async () => 0, + traceCount: async () => 0n, + previousReport: async () => null, + methodDoc: 'doc', + }); + assert.equal(report.consistency, 'best_effort'); + assert.equal(report.anchor.ledger, 103); +}); + +test('publish uses REPORT_MARKET_ID and action report; dry-run rows are uncommitted (#159)', async () => { + const tracePublish = await import('../src/services/trace-publish.service'); + const { REPORT_MARKET_ID, runReportJob } = await import('../src/services/transparency-report.service'); + const { prisma } = await import('../src/lib/prisma'); + let published: any = null; + (tracePublish as any).pinAndPublishDocument = async (params: any) => { + published = params; + return { cid: 'bafy-report', hash: 'a'.repeat(64), onChainTraceId: null, publishTxHash: null, mode: 'dry-run' }; + }; + let created: any = null; + (prisma.transparencyReport.findUnique as any) = async () => null; + (prisma.transparencyReport.create as any) = async ({ data }: any) => { + created = data; + return { id: 'report-1', ...data }; + }; + + await runReportJob('2026-10-14', { + now: () => new Date('2026-10-14T00:00:00.000Z'), + latestLedger: async () => 100, + marketCount: async () => 0n, + treasuryConfig: async () => ({ agent: 'GTREASURY', max_daily: 1000n, max_market: 100n, max_trade: 10n }), + dailyUsage: async () => ({ amount: 0n, day_start: 0n }), + availableCapital: async () => 0n, + chainEvents: async () => [], + indexerCursor: async () => 100, + traceCount: async () => 0n, + previousReport: async () => null, + methodDoc: 'doc', + }); + assert.equal(REPORT_MARKET_ID, 18446744073709551615n); + assert.equal(published.action, 'report'); + assert.equal(published.marketId, REPORT_MARKET_ID); + assert.equal(created.mode, 'dry-run'); + assert.equal(created.onChainTraceId, null); +}); + +test('a crash after pin re-publishes the stored bytes, never recomputing (#159)', async () => { + const tracePublish = await import('../src/services/trace-publish.service'); + const { runReportJob } = await import('../src/services/transparency-report.service'); + const { prisma } = await import('../src/lib/prisma'); + let republished: any = null; + (tracePublish as any).publishPinnedDocument = async (params: any) => { + republished = params; + return { onChainTraceId: 7n, publishTxHash: 'tx', mode: 'live' }; + }; + (prisma.transparencyReport.findUnique as any) = async () => ({ + id: 'report-1', date: '2026-10-14', cid: 'bafy-old', hash: 'b'.repeat(64), onChainTraceId: null, mode: 'dry-run', + }); + (prisma.transparencyReport.update as any) = async ({ data }: any) => ({ id: 'report-1', ...data }); + + await runReportJob('2026-10-14'); + assert.equal(republished.cid, 'bafy-old'); + assert.equal(republished.hash, 'b'.repeat(64)); + assert.equal(republished.marketId, 18446744073709551615n); +}); + +function fakeTx(value: any) { + return { result: value }; +} + +function fakeRegistry(trace: any, previous?: any) { + return { + async get_trace({ trace_id }: any) { + if (trace_id === 1n) return fakeTx(trace); + if (previous && trace_id === 0n) return fakeTx(previous); + throw new Error('unknown trace'); + }, + async market_trace_count() { return fakeTx(1n); }, + async market_trace_at() { return fakeTx(1n); }, + }; +} + +async function verifyFixture(report: any, trace: any, previous?: any) { + const { verify } = await import('../scripts/verify-report'); + const bytes = Buffer.from(JSON.stringify(report)); + const hash = crypto.createHash('sha256').update(bytes).digest('hex'); + const registry = fakeRegistry({ ...trace, trace_hash: Buffer.from(hash, 'hex') }, previous); + const treasury = { async get_config() { return fakeTx({ agent: 'GAGENT' }); } }; + return verify({ + traceId: '1', + gatewayUrl: 'https://gw', + deps: { + registry, + treasury, + fetch: (async () => ({ ok: true, arrayBuffer: async () => bytes })) as any, + latestLedger: async () => ({ sequence: 200 }), + }, + }); +} + +const validReport = { + kind: 'oracledesk.transparency-report', + schemaVersion: 1, + agent: 'GAGENT', + markets: [{ markValueRaw: '10' }], + treasury: { usdcBalanceRaw: '100' }, + totals: { markValueRaw: '10', positionsMarkRaw: '10', positionsLiquidityRaw: '0', navRaw: '110' }, + anchor: { ledger: 100 }, + previous: null, +}; + +test('verifier accepts a valid report and recomputes totals (#159)', async () => { + const result = await verifyFixture(validReport, { agent: 'GAGENT', action: 'report', ipfs_cid: 'bafy', market_id: 18446744073709551615n }); + assert.equal(result.ok, true, JSON.stringify(result.checks)); +}); + +test('verifier detects tampered content, wrong agent and a broken previous chain (#159)', async () => { + const { verify } = await import('../scripts/verify-report'); + const bytes = Buffer.from(JSON.stringify(validReport)); + const tampered = Buffer.from(JSON.stringify({ ...validReport, totals: { ...validReport.totals, navRaw: '999' } })); + const hash = crypto.createHash('sha256').update(bytes).digest('hex'); + const registry = fakeRegistry({ agent: 'GAGENT', action: 'report', ipfs_cid: 'bafy', trace_hash: Buffer.from(hash, 'hex'), market_id: 18446744073709551615n }); + const treasury = { async get_config() { return fakeTx({ agent: 'GAGENT' }); } }; + const tamperedResult = await verify({ + traceId: '1', + deps: { registry, treasury, fetch: (async () => ({ ok: true, arrayBuffer: async () => tampered })) as any, latestLedger: async () => ({ sequence: 200 }) }, + }); + assert.equal(tamperedResult.ok, false); + assert.ok(tamperedResult.checks.some((c) => c.name === 'content_hash' && !c.ok)); + + const wrongAgent = await verifyFixture({ ...validReport, agent: 'GOTHER' }, { agent: 'GOTHER', action: 'report', ipfs_cid: 'bafy', market_id: 18446744073709551615n }); + assert.equal(wrongAgent.ok, false); + assert.ok(wrongAgent.checks.some((c) => c.name === 'agent_matches_treasury' && !c.ok)); + + const broken = await verifyFixture( + { ...validReport, previous: { onChainTraceId: '0', traceHash: 'f'.repeat(64) } }, + { agent: 'GAGENT', action: 'report', ipfs_cid: 'bafy', market_id: 18446744073709551615n }, + { agent: 'GAGENT', action: 'report', ipfs_cid: 'bafy-prev', trace_hash: Buffer.from('e'.repeat(64), 'hex') }, + ); + assert.equal(broken.ok, false); + assert.ok(broken.checks.some((c) => c.name === 'previous_hash_chain' && !c.ok)); +}); + +test('u64::MAX sentinel serialises correctly in ScVal XDR (#159)', async () => { + const { xdr } = await import('@stellar/stellar-sdk'); + const value = xdr.Uint64.fromString('18446744073709551615'); + assert.equal(value.toString(), '18446744073709551615'); + const encoded = xdr.ScVal.scvU64(value).toXDR('base64'); + const decoded = xdr.ScVal.fromXDR(encoded, 'base64').u64(); + assert.equal(decoded.toString(), '18446744073709551615'); +}); + +test('the report service never imports portfolio floats or unrelated Prisma models (#159)', async () => { + const { readFileSync } = await import('node:fs'); + const { join } = await import('node:path'); + const source = readFileSync(join(__dirname, '..', '..', 'src', 'services', 'transparency-report.service.ts'), 'utf8'); + assert.equal(source.includes('portfolio.service'), false, 'no portfolio.service import'); + assert.equal(/from ['"]@prisma\/client['"]/.test(source), false, 'no Prisma model types'); + const models = [...source.matchAll(/prisma\.([a-zA-Z]+)/g)].map((m) => m[1]); + assert.deepEqual([...new Set(models)].sort(), ['chainEvent', 'indexerCursor', 'registryTraceCache', 'transparencyReport']); +}); + +test('lastModifiedLedgerSeq drift forces best_effort even with a stable latest ledger (#159)', async () => { + const { buildReportForDate } = await import('../src/services/transparency-report.service'); + const report = await buildReportForDate('2026-10-14', { + latestLedger: async () => 100, + ledgerEntries: async () => [{ lastModifiedLedgerSeq: 150 }], + marketCount: async () => 0n, + treasuryConfig: async () => ({ agent: 'GTREASURY', max_daily: 1000n, max_market: 100n, max_trade: 10n }), + dailyUsage: async () => ({ amount: 0n, day_start: 0n }), + availableCapital: async () => 0n, + chainEvents: async () => [], + indexerCursor: async () => 100, + traceCount: async () => 0n, + previousReport: async () => null, + methodDoc: 'doc', + }); + assert.equal(report.consistency, 'best_effort'); +}); diff --git a/tests/treasury-ops.test.ts b/tests/treasury-ops.test.ts new file mode 100644 index 0000000..2e4ee1a --- /dev/null +++ b/tests/treasury-ops.test.ts @@ -0,0 +1,161 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { + Account, + Keypair, + Networks, + Operation, + TransactionBuilder, +} from '@stellar/stellar-sdk'; +import { setupTestEnv } from './helpers/env'; + +setupTestEnv(); + +function fakeState(overrides: any = {}) { + return { + network: Networks.TESTNET, + contracts: { treasury: 'CTREASURY', marketCore: 'CMARKET', usdc: 'CUSDC' }, + treasury: { + config: { agent: 'GAGENT', collateral: 'CUSDC', market_core: 'CMARKET', max_daily: 1000n, max_market: 500n, max_trade: 100n }, + availableCapitalRaw: '1000000000', + usdcBalanceRaw: '1000000000', + dailyUsage: { amountRaw: '0', dayStart: 0, resetsAt: '2026-10-18T00:00:00.000Z', remainingRaw: '1000' }, + paused: false, + admin: { address: 'GADMIN', kind: 'account' as const }, + ...(overrides.treasury ?? {}), + }, + marketCore: { paused: false, admin: 'GADMIN' }, + agent: { address: 'GAGENT', xlmBalance: '100' }, + } as any; +} + +function fakeAccount(overrides: any = {}) { + return { + address: 'GADMIN', + kind: 'account' as const, + balanceXlm: '100', + masterWeight: 1, + thresholds: { low: 1, medium: 1, high: 2 }, + signers: [], + ...overrides, + } as any; +} + +async function stubPrisma(row: any = null) { + const { prisma } = await import('../src/lib/prisma'); + (prisma.adminOperation.findUnique as any) = async () => row; + (prisma.adminOperation.create as any) = async ({ data }: any) => ({ id: 'op-1', ...data }); + (prisma.adminOperation.update as any) = async ({ data }: any) => ({ id: 'op-1', ...data }); + (prisma.agentLog.create as any) = async ({ data }: any) => ({ id: 'log-1', ...data }); + return prisma; +} + +function envelope(source: string): any { + return new TransactionBuilder(new Account(source, '1'), { fee: '100', networkPassphrase: Networks.TESTNET }) + .addOperation(Operation.manageData({ name: 'test', value: '1' })) + .setTimeout(60) + .build(); +} + +test('prepare rejects a withdraw above available capital before building (#156)', async () => { + const { prepareOperation } = await import('../src/services/treasury-ops.service'); + await assert.rejects( + prepareOperation( + { kind: 'treasury.withdraw', to: Keypair.random().publicKey(), amountRaw: (2_000_000_000n).toString() }, + 'admin', + { state: async () => fakeState(), account: async () => fakeAccount() }, + ), + (err: any) => err.code === 'PREFLIGHT_FAILED' && err.details.checks.some((c: any) => c.name === 'amount_within_available_capital' && !c.ok), + ); +}); + +test('prepare returns an unsigned simulated operation with a human summary (#156)', async () => { + const { prepareOperation } = await import('../src/services/treasury-ops.service'); + await stubPrisma(); + const prepared = await prepareOperation( + { kind: 'treasury.set_paused', paused: true }, + 'GADMIN', + { + state: async () => fakeState(), + account: async () => fakeAccount(), + build: async () => ({ xdr: 'AAAA', hash: 'a'.repeat(64), contractId: 'CTREASURY', sourceAccount: 'GADMIN' }), + }, + ); + assert.equal(prepared.hash, 'a'.repeat(64)); + assert.equal(prepared.requiredSigners, 2); + assert.equal(prepared.status, 'PREPARED'); + assert.match(prepared.humanSummary, /Pause treasury/); + assert.ok(prepared.checks.some((c) => c.name === 'simulation' && c.ok)); +}); + +test('a signed envelope with a different body hash is rejected (#156)', async () => { + const { addSignature } = await import('../src/services/treasury-ops.service'); + const kp = Keypair.random(); + await stubPrisma({ + id: 'op-1', + kind: 'treasury.set_paused', + sourceAccount: kp.publicKey(), + unsignedXdr: envelope(kp.publicKey()).toXDR(), + txHash: 'b'.repeat(64), + signatures: [], + status: 'PREPARED', + expiresAt: new Date(Date.now() + 60_000), + }); + const signed = envelope(kp.publicKey()); + signed.sign(kp); + await assert.rejects( + addSignature('op-1', signed.toXDR(), 'signer'), + (err: any) => err.code === 'ADMIN_OP_HASH_MISMATCH', + ); +}); + +test('two of three signatures reach READY and duplicates are ignored (#156)', async () => { + const { addSignature } = await import('../src/services/treasury-ops.service'); + const admin = Keypair.random(); + const second = Keypair.random(); + const tx = envelope(admin.publicKey()); + tx.sign(admin); + tx.sign(second); + await stubPrisma({ + id: 'op-2', + kind: 'treasury.set_paused', + sourceAccount: admin.publicKey(), + unsignedXdr: envelope(admin.publicKey()).toXDR(), + txHash: tx.hash().toString('hex'), + signatures: [], + status: 'PREPARED', + expiresAt: new Date(Date.now() + 60_000), + }); + const result = await addSignature('op-2', tx.toXDR(), 'signer', { + account: async () => fakeAccount({ address: admin.publicKey(), signers: [{ key: second.publicKey(), kind: 'ed25519', weight: 1 }] }), + }); + assert.equal(result.weightCollected, 2); + assert.equal(result.requiredSigners, 2); + assert.equal(result.status, 'READY'); + assert.equal(result.signatures.length, 2); +}); + +test('submit refuses expired and not-ready operations (#156)', async () => { + const { submitOperation } = await import('../src/services/treasury-ops.service'); + await stubPrisma({ + id: 'op-3', + status: 'PREPARED', + signedXdr: 'AAAA', + expiresAt: new Date('2026-01-01T00:00:00.000Z'), + }); + await assert.rejects(submitOperation('op-3', { now: () => Date.parse('2026-10-01T00:00:00.000Z') }), (err: any) => err.code === 'ADMIN_OP_EXPIRED'); + + await stubPrisma({ id: 'op-4', status: 'PREPARED', signedXdr: 'AAAA', expiresAt: new Date('2027-01-01T00:00:00.000Z') }); + await assert.rejects(submitOperation('op-4'), (err: any) => err.code === 'ADMIN_OP_NOT_READY'); +}); + +test('submit records SUCCESS with ledger and audits the change (#156)', async () => { + const { submitOperation } = await import('../src/services/treasury-ops.service'); + const prisma = await stubPrisma({ id: 'op-5', status: 'READY', signedXdr: 'AAAA', expiresAt: new Date('2027-01-01T00:00:00.000Z') }); + const result = await submitOperation('op-5', { + submit: async () => ({ status: 'SUCCESS', hash: 'c'.repeat(64), ledger: 123, resultXdr: 'RESULT' }), + }); + assert.equal(result.status, 'SUCCESS'); + assert.equal(result.ledger, 123); + assert.equal((prisma.agentLog.create as any).mock?.calls?.length ?? 1, 1); +});