diff --git a/.github/workflows/synthetic-monitor.yml b/.github/workflows/synthetic-monitor.yml new file mode 100644 index 0000000..01603f2 --- /dev/null +++ b/.github/workflows/synthetic-monitor.yml @@ -0,0 +1,36 @@ +name: Synthetic monitor + +# Probes the critical journey (account operation history and its older page) +# against live Horizon. See docs/OPERATIONAL_HEALTH.md#synthetic-monitoring-269. +on: + schedule: + - cron: "*/30 * * * *" + workflow_dispatch: + +permissions: + contents: read + +jobs: + probe: + name: Critical journey (${{ matrix.network }}) + runs-on: ubuntu-latest + timeout-minutes: 5 + strategy: + fail-fast: false + matrix: + network: [testnet, mainnet] + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: 24 + + # The probe imports no npm packages, so no install step is needed. + - name: Run synthetic monitor + env: + UTIX_USER_ROLE: service + run: npm run synthetic:monitor -- --network ${{ matrix.network }} diff --git a/core/__tests__/fuzz.test.ts b/core/__tests__/fuzz.test.ts new file mode 100644 index 0000000..c9167bc --- /dev/null +++ b/core/__tests__/fuzz.test.ts @@ -0,0 +1,235 @@ +import { describe, expect, it } from "vitest"; +import { hasAllPermissions, hasAnyPermission, hasPermission, ROLE_PERMISSIONS, type Permission, type Role } from "@/core/auth/types"; +import { + AMOUNT_DECIMALS, + amountToStroops, + INT64_MAX, + INT64_MIN, + parseAmount, + stroopsToAmount, + type AmountParseError +} from "@/core/format/amount"; +import { canonicalize, createIdempotencyStore, requestDigest } from "@/core/idempotency/idempotency"; +import { parseJsonRpcResponse } from "@/core/rpc/client"; + +/** + * Fuzz coverage for malformed payloads and boundary numeric values (#268). + * + * Inputs come from a seeded PRNG, so a failure reproduces exactly: rerun with + * `FUZZ_SEED= npm test -- core/__tests__/fuzz`. + * Each property states what must hold for *every* input — never throw, always + * answer with a declared code — rather than what one hand-picked input returns. + */ +const SEED = Number(process.env.FUZZ_SEED ?? 0x5eed_0268); +const RUNS = 1_000; + +/** mulberry32: tiny, fast, and good enough to spread inputs. Not crypto. */ +function prng(seed: number) { + let state = seed >>> 0; + const next = () => { + state = (state + 0x6d2b79f5) >>> 0; + let t = state; + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 2 ** 32; + }; + const int = (max: number) => Math.floor(next() * max); + const pick = (items: readonly T[]): T => items[int(items.length)]; + const string = (alphabet: string, maxLength: number) => + Array.from({ length: int(maxLength + 1) }, () => alphabet[int(alphabet.length)]).join(""); + /** Uniform over the full signed 64-bit range. */ + const int64 = () => { + const high = BigInt(int(2 ** 32)) << 32n; + return BigInt.asIntN(64, high | BigInt(int(2 ** 32))); + }; + return { next, int, pick, string, int64 }; +} + +const BOUNDARY_NUMBERS = [ + 0, -0, 1, -1, 0.1, 1.5, Number.NaN, Number.POSITIVE_INFINITY, Number.NEGATIVE_INFINITY, + Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER + 1, Number.MIN_SAFE_INTEGER, Number.EPSILON, + 2 ** 31, -(2 ** 31) - 1, 2 ** 32, 2 ** 53, -32_600, -32_700 +] as const; + +/** Any JSON-ish value plus the non-JSON values a buggy upstream can produce. */ +function arbitraryValue(random: ReturnType, depth = 0): unknown { + const leaves: (() => unknown)[] = [ + () => null, + () => undefined, + () => random.pick(BOUNDARY_NUMBERS), + () => random.string("abc 2.0-\u0000\u{1F680}\"\\", 8), + () => random.next() < 0.5, + () => random.int64(), + () => "2.0", + () => Symbol("payload") + ]; + if (depth >= 3 || random.next() < 0.4) return random.pick(leaves)(); + if (random.next() < 0.3) { + return Array.from({ length: random.int(4) }, () => arbitraryValue(random, depth + 1)); + } + const keys = ["jsonrpc", "id", "result", "error", "code", "message", "data", "__proto__", "constructor"]; + const object: Record = {}; + for (let index = random.int(5); index > 0; index -= 1) { + Object.defineProperty(object, random.pick(keys), { + value: arbitraryValue(random, depth + 1), + enumerable: true, + configurable: true, + writable: true + }); + } + if (random.next() < 0.1) object.self = object; + return object; +} + +const AMOUNT_ERRORS: readonly AmountParseError[] = [ + "empty", + "grouping_separator", + "invalid_format", + "too_many_decimals", + "negative", + "out_of_range" +]; + +describe("fuzz: Stellar amount boundaries (#268)", () => { + // developers.stellar.org: amounts are signed int64 stroops scaled by 10^7; + // the largest is 922,337,203,685.4775807 and the smallest unit is 0.0000001. + it.each([ + ["922337203685.4775807", false, INT64_MAX], + ["922337203685.4775808", false, "out_of_range"], + ["-922337203685.4775808", true, INT64_MIN], + ["-922337203685.4775809", true, "out_of_range"], + ["0.0000001", false, 1n], + ["0.00000001", false, "too_many_decimals"], + ["0", false, 0n], + ["-0.0000001", false, "negative"], + ["99999999999999999999999999", false, "out_of_range"] + ] as const)("parseAmount(%s, allowNegative=%s) → %s", (raw, allowNegative, expected) => { + const parsed = parseAmount(raw, { allowNegative }); + expect(parsed).toEqual(typeof expected === "bigint" ? { ok: true, value: expected } : { ok: false, code: expected }); + }); + + it("round-trips every int64 stroop value through its canonical amount string", () => { + const random = prng(SEED); + const samples = [INT64_MAX, INT64_MIN, 0n, 1n, -1n, ...Array.from({ length: RUNS }, random.int64)]; + for (const stroops of samples) { + const amount = stroopsToAmount(stroops); + expect(amount, `seed ${SEED}`).toMatch(new RegExp(`^-?\\d+\\.\\d{${AMOUNT_DECIMALS}}$`)); + expect(parseAmount(amount, { allowNegative: true }), `seed ${SEED}: ${amount}`).toEqual({ ok: true, value: stroops }); + expect(amountToStroops(amount), `seed ${SEED}: ${amount}`).toBe(stroops); + } + }); + + it("answers malformed amount input with a declared code and never throws", () => { + const random = prng(SEED + 1); + for (let run = 0; run < RUNS; run += 1) { + const raw = random.string("0123456789.-+eE ,_'  ٣\t", 30); + const parsed = parseAmount(raw, { allowNegative: random.next() < 0.5 }); + if (parsed.ok) { + expect(parsed.value <= INT64_MAX && parsed.value >= INT64_MIN, `seed ${SEED + 1}: ${JSON.stringify(raw)}`).toBe(true); + } else { + expect(AMOUNT_ERRORS, `seed ${SEED + 1}: ${JSON.stringify(raw)}`).toContain(parsed.code); + } + } + }); +}); + +describe("fuzz: malformed JSON-RPC payloads (#268)", () => { + it("rejects a non-integer error code (JSON-RPC 2.0 §5.1)", () => { + for (const code of [1.5, Number.NaN, Number.POSITIVE_INFINITY, "-32000", null]) { + const parsed = parseJsonRpcResponse({ jsonrpc: "2.0", id: 1, error: { code, message: "x" } }, 1); + expect(parsed).toMatchObject({ error: { code: -32600 } }); + } + }); + + it("always returns a well-formed envelope and never throws", () => { + const random = prng(SEED + 2); + for (let run = 0; run < RUNS; run += 1) { + const payload = arbitraryValue(random); + const expectedId = random.pick([undefined, 1, "req-1"]); + const parsed = parseJsonRpcResponse(payload, expectedId); + const context = `seed ${SEED + 2}, run ${run}`; + + expect(parsed.jsonrpc, context).toBe("2.0"); + expect(parsed.id === null || ["string", "number"].includes(typeof parsed.id), context).toBe(true); + expect("result" in parsed !== "error" in parsed, context).toBe(true); + if ("error" in parsed) { + expect(Number.isInteger(parsed.error.code), context).toBe(true); + expect(typeof parsed.error.message, context).toBe("string"); + } else if (expectedId !== undefined) { + expect(parsed.id, context).toBe(expectedId); + } + } + }); +}); + +describe("fuzz: idempotency payloads and keys (#268)", () => { + it("digests any payload — including BigInt and cycles — deterministically", () => { + const random = prng(SEED + 3); + for (let run = 0; run < RUNS; run += 1) { + const payload = arbitraryValue(random); + expect(() => requestDigest(payload), `seed ${SEED + 3}, run ${run}`).not.toThrow(); + expect(requestDigest(payload), `seed ${SEED + 3}, run ${run}`).toBe(requestDigest(payload)); + } + }); + + it("keeps BigInt and cycle markers distinct from the JSON values they resemble", () => { + const cyclic: Record = {}; + cyclic.self = cyclic; + const shared = { a: 1 }; + + expect(canonicalize({ amount: 10n })).toBe('{"amount":10n}'); + expect(requestDigest({ amount: 10n })).not.toBe(requestDigest({ amount: 10 })); + expect(requestDigest({ amount: 10n })).not.toBe(requestDigest({ amount: "10n" })); + expect(canonicalize(cyclic)).toBe('{"self":#cycle}'); + expect(canonicalize([shared, shared])).toBe('[{"a":1},{"a":1}]'); + }); + + it("answers any key with a claim or a declared rejection, and blocks the duplicate retry", () => { + const random = prng(SEED + 4); + const store = createIdempotencyStore({ now: () => 0 }); + const codes = ["idempotency_key_missing", "idempotency_key_invalid", "idempotency_in_flight"]; + for (let run = 0; run < RUNS; run += 1) { + const key = random.pick([undefined, "", random.string("Aa0._:-/ é", 140)]); + const request = { key, operation: "fuzz", request: arbitraryValue(random) }; + const first = store.begin(request); + if (first.ok) { + expect(store.begin(request), `seed ${SEED + 4}: ${JSON.stringify(key)}`).toEqual({ + ok: false, + code: "idempotency_in_flight" + }); + store.abandon(key!); + } else { + expect(codes, `seed ${SEED + 4}: ${JSON.stringify(key)}`).toContain(first.code); + } + } + }); +}); + +describe("fuzz: authorization with malformed roles (#268)", () => { + const permissions = Object.values(ROLE_PERMISSIONS).flat() as Permission[]; + const known = Object.keys(ROLE_PERMISSIONS); + + it("denies, never throws, for an unknown role — including prototype keys", () => { + const random = prng(SEED + 5); + const roles = [ + "admin", "VIEWER", "", " viewer", "toString", "__proto__", "constructor", "hasOwnProperty", + undefined, null, ...Array.from({ length: RUNS }, () => random.string("aeiorstvwnmc_ ", 12)) + ].filter((role) => !known.includes(role as string)) as unknown as Role[]; + + for (const role of roles) { + for (const permission of permissions) { + expect(hasPermission(role, permission), `role ${JSON.stringify(role)}`).toBe(false); + } + expect(hasAnyPermission(role, permissions)).toBe(false); + expect(hasAllPermissions(role, permissions)).toBe(false); + } + }); + + it("still grants exactly the declared permissions to every known role", () => { + for (const role of known as Role[]) { + for (const permission of permissions) { + expect(hasPermission(role, permission)).toBe(ROLE_PERMISSIONS[role].includes(permission)); + } + } + }); +}); diff --git a/core/auth/types.ts b/core/auth/types.ts index a35f64f..1dbd6e5 100644 --- a/core/auth/types.ts +++ b/core/auth/types.ts @@ -36,7 +36,9 @@ export const ROLE_PERMISSIONS: Record = { }; export function hasPermission(role: Role, permission: Permission): boolean { - return ROLE_PERMISSIONS[role].includes(permission); + // An unrecognised role (e.g. a mistyped UTIX_USER_ROLE) is denied, never a + // crash; `hasOwn` keeps prototype keys such as `toString` from resolving. + return Object.hasOwn(ROLE_PERMISSIONS, role) && ROLE_PERMISSIONS[role].includes(permission); } export function hasAnyPermission(role: Role, permissions: Permission[]): boolean { diff --git a/core/cache/__tests__/bounded-cache.test.ts b/core/cache/__tests__/bounded-cache.test.ts new file mode 100644 index 0000000..2be85e3 --- /dev/null +++ b/core/cache/__tests__/bounded-cache.test.ts @@ -0,0 +1,257 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { err, ok, type Result } from "@/core/result/result"; +import { + createCaptureSink, + resetTelemetrySink, + setTelemetrySink +} from "@/core/telemetry/telemetry"; +import { createBoundedCache, type BoundedCacheOptions } from "../bounded-cache"; +import { StaleProjectionDetector } from "../stale-detector"; + +const START = Date.parse("2026-10-17T00:00:00.000Z"); + +function harness(overrides: Partial = {}) { + let clock = START; + const cache = createBoundedCache({ + maxEntries: 3, + ttlMs: 1_000, + now: () => clock, + ...overrides + }); + return { cache, advance: (ms: number) => (clock += ms) }; +} + +/** A loader whose settlement the test controls. */ +function deferred() { + let resolve!: (value: T) => void; + const promise = new Promise((settle) => (resolve = settle)); + return { promise, resolve }; +} + +afterEach(() => resetTelemetrySink()); + +describe("bounded cache (#270)", () => { + describe("normal path", () => { + it("serves a stored value until its TTL elapses", () => { + const { cache, advance } = harness(); + cache.set("balance:GA", "100.0000000"); + + advance(1_000); + expect(cache.get("balance:GA")).toEqual(ok("100.0000000")); + advance(1); + expect(cache.get("balance:GA")).toEqual(err("cache_expired")); + expect(cache.get("balance:GA")).toEqual(err("cache_miss")); + }); + + it("never holds more than maxEntries and evicts the least recently used key", () => { + const { cache } = harness(); + cache.set("a", "1"); + cache.set("b", "2"); + cache.set("c", "3"); + cache.get("a"); + cache.set("d", "4"); + + expect(cache.get("b")).toEqual(err("cache_miss")); + expect(cache.snapshot().map((entry) => entry.key)).toEqual(["c", "a", "d"]); + expect(cache.stats()).toMatchObject({ size: 3, evictions: 1 }); + }); + + it("honours a per-entry TTL override", () => { + const { cache, advance } = harness(); + cache.set("ledger:head", "5247219", { ttlMs: 5_000 }); + advance(4_000); + expect(cache.get("ledger:head").ok).toBe(true); + }); + + it("rejects a per-entry TTL that could never expire or is already expired", () => { + const { cache } = harness(); + for (const ttlMs of [Number.NaN, Number.POSITIVE_INFINITY, 0, -1, 1.5]) { + expect(() => cache.set("k", "v", { ttlMs })).toThrow(RangeError); + } + expect(cache.get("k")).toEqual(err("cache_miss")); + }); + + it("rejects a non-positive or fractional bound at construction", () => { + expect(() => createBoundedCache({ maxEntries: 0, ttlMs: 1 })).toThrow(RangeError); + expect(() => createBoundedCache({ maxEntries: 1, ttlMs: 0.5 })).toThrow(RangeError); + expect(() => createBoundedCache({ maxEntries: 1, ttlMs: 1, schemaVersion: 0 })).toThrow(RangeError); + }); + }); + + describe("stale detection", () => { + it("stamps entries with the schema version so a bump is reported as skew", () => { + const { cache } = harness({ schemaVersion: 1 }); + cache.set("k", "v1-shape"); + + const report = StaleProjectionDetector.inspect(cache.snapshot(), [], { + now: START, + currentSchemaVersion: 2 + }); + expect(report.findings.map((finding) => finding.staleReason)).toEqual(["version_mismatch"]); + }); + + it("refuses a write that would move a projection to an older ledger", () => { + const { cache } = harness(); + expect(cache.set("account:GA", "seq-200", { ledgerSequence: 200 })).toEqual(ok(true)); + expect(cache.set("account:GA", "seq-199", { ledgerSequence: 199 })).toEqual(err("cache_stale_write")); + expect(cache.get("account:GA")).toEqual(ok("seq-200")); + expect(cache.set("account:GA", "seq-201", { ledgerSequence: 201 })).toEqual(ok(true)); + }); + + it("accepts an older ledger once the newer entry has itself expired", () => { + const { cache, advance } = harness(); + cache.set("account:GA", "seq-200", { ledgerSequence: 200 }); + advance(1_001); + expect(cache.set("account:GA", "seq-199", { ledgerSequence: 199 })).toEqual(ok(true)); + }); + + it("exposes a snapshot the existing StaleProjectionDetector can inspect", () => { + const { cache, advance } = harness(); + cache.set("account:GA", "a", { ledgerSequence: 100, sourceEntityId: "GA" }); + cache.set("account:GB", "b", { ledgerSequence: 190, sourceEntityId: "GB" }); + advance(10); + + const report = StaleProjectionDetector.inspect(cache.snapshot(), [], { + now: START + 10, + currentSchemaVersion: 1, + authoritativeLedgerSequence: 200, + maxToleratedDriftLedgers: 5 + }); + + expect(report.findings.map((finding) => [finding.targetKey, finding.staleReason])).toEqual([ + ["account:GA", "ledger_drift"], + ["account:GB", "ledger_drift"] + ]); + const evicted = cache.invalidateWhere((entry) => (entry.ledgerSequence ?? 0) < 200 - 5); + expect(evicted).toBe(2); + }); + }); + + describe("explicit invalidation", () => { + it("drops one key and reports whether it existed", () => { + const { cache } = harness(); + cache.set("a", "1"); + expect(cache.invalidate("a")).toBe(true); + expect(cache.invalidate("a")).toBe(false); + expect(cache.get("a")).toEqual(err("cache_miss")); + expect(cache.stats().invalidations).toBe(1); + }); + + it("drops every key a predicate selects", () => { + const { cache } = harness(); + cache.set("account:GA:balances", "1", { sourceEntityId: "GA" }); + cache.set("account:GA:signers", "2", { sourceEntityId: "GA" }); + cache.set("account:GB:balances", "3", { sourceEntityId: "GB" }); + + expect(cache.invalidateWhere((entry) => entry.sourceEntityId === "GA")).toBe(2); + expect(cache.snapshot().map((entry) => entry.key)).toEqual(["account:GB:balances"]); + }); + }); + + describe("failure and retry paths", () => { + it("coalesces concurrent loads of one key into a single request", async () => { + const { cache } = harness(); + const gate = deferred>(); + let calls = 0; + const load = () => { + calls += 1; + return gate.promise; + }; + + const first = cache.getOrLoad("k", load); + const second = cache.getOrLoad("k", load); + gate.resolve(ok("value")); + + expect(await Promise.all([first, second])).toEqual([ok("value"), ok("value")]); + await Promise.resolve(); + expect(calls).toBe(1); + expect(cache.get("k")).toEqual(ok("value")); + }); + + it("does not cache a failed load, so the next call retries it", async () => { + const { cache } = harness(); + const outcomes: Result[] = [err("horizon_unavailable"), ok("value")]; + const load = async () => outcomes.shift()!; + + expect(await cache.getOrLoad("k", load)).toEqual(err("horizon_unavailable")); + expect(cache.get("k")).toEqual(err("cache_miss")); + expect(await cache.getOrLoad("k", load)).toEqual(ok("value")); + expect(cache.get("k")).toEqual(ok("value")); + }); + + it("releases the in-flight slot when a loader throws", async () => { + const { cache } = harness(); + const thrower = () => { + throw new Error("transport"); + }; + + await expect(cache.getOrLoad("k", thrower)).rejects.toThrow("transport"); + expect(await cache.getOrLoad("k", async () => ok("recovered"))).toEqual(ok("recovered")); + }); + + it("does not store a load that was invalidated while it was in flight", async () => { + const { cache } = harness(); + const stale = deferred>(); + + const pending = cache.getOrLoad("k", () => stale.promise); + cache.invalidate("k"); + const fresh = cache.getOrLoad("k", async () => ok("fresh")); + stale.resolve(ok("stale")); + + expect(await pending).toEqual(ok("stale")); + expect(await fresh).toEqual(ok("fresh")); + expect(cache.get("k")).toEqual(ok("fresh")); + }); + + it("returns cache_stale_write when a slow load lost the race to a newer ledger", async () => { + const { cache } = harness(); + const slow = deferred>(); + + const pending = cache.getOrLoad("account:GA", () => slow.promise, { ledgerSequence: 290 }); + cache.set("account:GA", "seq-302", { ledgerSequence: 302 }); + slow.resolve(ok("seq-290")); + + expect(await pending).toEqual(err("cache_stale_write")); + expect(cache.get("account:GA")).toEqual(ok("seq-302")); + }); + }); + + describe("authorization path", () => { + it("refuses a full flush from a role without admin:recovery", () => { + const sink = createCaptureSink(); + setTelemetrySink(sink); + const { cache } = harness(); + cache.set("a", "1"); + + expect(cache.clear({ role: "viewer" }, "manual")).toEqual(err("cache_unauthorized")); + expect(cache.clear({ role: "service" }, "manual")).toEqual(err("cache_unauthorized")); + expect(cache.get("a")).toEqual(ok("1")); + expect(sink.events.at(-1)).toMatchObject({ + op: "cache.flushed", + result: "failure", + errorCode: "cache_unauthorized" + }); + }); + + it("lets a maintainer flush everything, including loads in flight", async () => { + const sink = createCaptureSink(); + setTelemetrySink(sink); + const { cache } = harness(); + cache.set("a", "1"); + cache.set("b", "2"); + const gate = deferred>(); + const pending = cache.getOrLoad("c", () => gate.promise); + + expect(cache.clear({ role: "maintainer" }, "schema rollback")).toEqual(ok(2)); + gate.resolve(ok("3")); + await pending; + + expect(cache.stats().size).toBe(0); + expect(sink.events.at(-1)).toMatchObject({ + op: "cache.flushed", + result: "success", + payload: { removed: 2, reason: "schema rollback" } + }); + }); + }); +}); diff --git a/core/cache/bounded-cache.ts b/core/cache/bounded-cache.ts new file mode 100644 index 0000000..c820c6c --- /dev/null +++ b/core/cache/bounded-cache.ts @@ -0,0 +1,271 @@ +/** + * Bounded cache with explicit invalidation and stale detection (#270) + * + * A least-recently-used cache for projections of ledger state (Horizon and RPC + * reads). It never grows past `maxEntries`, and it never serves a value that + * the stale rules in `StaleProjectionDetector` would flag: + * + * - an entry older than its TTL is `cache_expired`; + * - a write carrying an older ledger sequence than the entry it would replace + * is refused as `cache_stale_write`, so an out-of-order response cannot roll + * a projection back. + * + * Expired entries are dropped when they are read, so expiry costs nothing + * until the key is touched again. `snapshot()` exposes the live entries in the + * `CacheEntry` shape (stamped with `schemaVersion`), so the existing detector + * reports schema skew, ledger drift and orphans for this cache as-is. + * + * Complexity: `get`, `set`, `invalidate` and `getOrLoad` are O(1) — a `Map` + * keeps insertion order, so re-inserting on access makes its first key the + * least recently used. `invalidateWhere`, `clear` and `snapshot` are O(n). + * Space is O(maxEntries + loads in flight). + */ + +import { hasPermission } from "@/core/auth/types"; +import type { AuthContext } from "@/core/auth/middleware"; +import { err, ok, type Result } from "@/core/result/result"; +import { emitTelemetry, newCorrelationId } from "@/core/telemetry/telemetry"; +import type { CacheEntry } from "./types.ts"; + +export type CacheReadErrorCode = "cache_miss" | "cache_expired"; +export type CacheWriteErrorCode = "cache_stale_write"; +export type CacheFlushErrorCode = "cache_unauthorized"; + +export interface BoundedCacheOptions { + /** Upper bound on stored entries; the least recently used one is evicted. */ + readonly maxEntries: number; + /** Default time-to-live for an entry, in milliseconds. */ + readonly ttlMs: number; + /** Stamped on every entry for `StaleProjectionDetector`. Defaults to 1. */ + readonly schemaVersion?: number; + /** Telemetry label, e.g. `horizon.accounts`. */ + readonly name?: string; + /** Injected clock. Defaults to the wall clock. */ + readonly now?: () => number; +} + +export interface CacheWriteOptions { + /** Overrides the cache-wide TTL for this entry. */ + readonly ttlMs?: number; + /** Ledger the value was read at. Writes may not move it backwards. */ + readonly ledgerSequence?: number; + /** Primary record this value projects, for orphan detection. */ + readonly sourceEntityId?: string; +} + +export interface CacheStats { + readonly size: number; + readonly hits: number; + readonly misses: number; + readonly evictions: number; + readonly invalidations: number; +} + +export type CacheLoader = () => Promise>; + +export interface BoundedCache { + get(key: string): Result; + set(key: string, value: V, options?: CacheWriteOptions): Result; + /** + * Returns the cached value or runs `load` once for every concurrent caller. + * A failed load is never cached, so the next call retries it. A load that + * was invalidated while in flight returns its value but does not store it. + */ + getOrLoad( + key: string, + load: CacheLoader, + options?: CacheWriteOptions + ): Promise>; + /** Drops one key and any load in flight for it. */ + invalidate(key: string): boolean; + /** Drops every key the predicate selects and returns how many went. */ + invalidateWhere(predicate: (entry: CacheEntry) => boolean): number; + /** Drops everything. An operator action: requires `admin:recovery`. */ + clear(auth: AuthContext, reason: string): Result; + /** Live entries in the shape `StaleProjectionDetector.inspect` reads. */ + snapshot(): CacheEntry[]; + stats(): CacheStats; +} + +interface StoredEntry { + readonly value: V; + readonly cachedAt: number; + readonly ttlMs: number; + readonly version: number; + readonly ledgerSequence?: number; + readonly sourceEntityId?: string; +} + +function requirePositiveInteger(name: string, value: number): void { + if (!Number.isSafeInteger(value) || value < 1) { + throw new RangeError(`${name} must be a positive safe integer, received ${value}`); + } +} + +export function createBoundedCache(options: BoundedCacheOptions): BoundedCache { + const { maxEntries, ttlMs, schemaVersion = 1, name = "cache" } = options; + requirePositiveInteger("maxEntries", maxEntries); + requirePositiveInteger("ttlMs", ttlMs); + requirePositiveInteger("schemaVersion", schemaVersion); + const now = options.now ?? (() => Date.now()); + + const entries = new Map>(); + // The promise identity is the load's token: a load may only store its value + // while it is still the registered load for its key. + const inflight = new Map>>(); + let hits = 0; + let misses = 0; + let evictions = 0; + let invalidations = 0; + + /** Same boundary as `StaleProjectionDetector`: stale once age exceeds the TTL. */ + function isExpired(entry: StoredEntry, at: number): boolean { + return at - entry.cachedAt > entry.ttlMs; + } + + function toCacheEntry(key: string, entry: StoredEntry): CacheEntry { + return { + key, + value: entry.value, + cachedAt: entry.cachedAt, + ttlMs: entry.ttlMs, + version: entry.version, + ledgerSequence: entry.ledgerSequence, + sourceEntityId: entry.sourceEntityId + }; + } + + function get(key: string): Result { + const entry = entries.get(key); + if (!entry) { + misses += 1; + return err("cache_miss"); + } + if (isExpired(entry, now())) { + entries.delete(key); + misses += 1; + return err("cache_expired"); + } + // Re-insert so the Map's first key stays the least recently used. + entries.delete(key); + entries.set(key, entry); + hits += 1; + return ok(entry.value); + } + + function set(key: string, value: V, write: CacheWriteOptions = {}): Result { + // A NaN TTL would never expire (`age > NaN` is false), so it is refused like a bad bound. + if (write.ttlMs !== undefined) requirePositiveInteger("ttlMs", write.ttlMs); + const at = now(); + const current = entries.get(key); + if ( + current && + !isExpired(current, at) && + current.ledgerSequence !== undefined && + write.ledgerSequence !== undefined && + write.ledgerSequence < current.ledgerSequence + ) { + return err("cache_stale_write"); + } + + entries.delete(key); + entries.set(key, { + value, + cachedAt: at, + ttlMs: write.ttlMs ?? ttlMs, + version: schemaVersion, + ledgerSequence: write.ledgerSequence, + sourceEntityId: write.sourceEntityId + }); + if (entries.size > maxEntries) { + entries.delete(entries.keys().next().value as string); + evictions += 1; + } + return ok(true); + } + + async function getOrLoad( + key: string, + load: CacheLoader, + write?: CacheWriteOptions + ): Promise> { + const cached = get(key); + if (cached.ok) return cached; + + const pending = inflight.get(key) as Promise> | undefined; + if (pending) return pending; + + // `load` starts on the next microtask, so `run` is always assigned before + // either callback compares against it — even when `load` throws at once. + const run: Promise> = Promise.resolve() + .then(load) + .then((loaded) => { + if (!loaded.ok || inflight.get(key) !== run) return loaded; + const stored = set(key, loaded.value, write); + return stored.ok ? loaded : stored; + }) + .finally(() => { + if (inflight.get(key) === run) inflight.delete(key); + }); + inflight.set(key, run); + return run; + } + + function invalidate(key: string): boolean { + inflight.delete(key); + const removed = entries.delete(key); + if (removed) invalidations += 1; + return removed; + } + + function invalidateWhere(predicate: (entry: CacheEntry) => boolean): number { + let removed = 0; + for (const [key, entry] of entries) { + if (predicate(toCacheEntry(key, entry))) { + entries.delete(key); + inflight.delete(key); + removed += 1; + } + } + invalidations += removed; + return removed; + } + + function clear(auth: AuthContext, reason: string): Result { + const correlationId = newCorrelationId(); + if (!hasPermission(auth.role, "admin:recovery")) { + emitTelemetry({ + op: "cache.flushed", + actorType: "user", + result: "failure", + correlationId, + errorCode: "cache_unauthorized", + payload: { cache: name, role: auth.role } + }); + return err("cache_unauthorized"); + } + + const removed = entries.size; + entries.clear(); + inflight.clear(); + invalidations += removed; + emitTelemetry({ + op: "cache.flushed", + actorType: "user", + result: "success", + correlationId, + payload: { cache: name, role: auth.role, removed, reason } + }); + return ok(removed); + } + + function snapshot(): CacheEntry[] { + return Array.from(entries, ([key, entry]) => toCacheEntry(key, entry)); + } + + function stats(): CacheStats { + return { size: entries.size, hits, misses, evictions, invalidations }; + } + + return { get, set, getOrLoad, invalidate, invalidateWhere, clear, snapshot, stats }; +} diff --git a/core/cache/index.ts b/core/cache/index.ts index 6ea43e8..4864d79 100644 --- a/core/cache/index.ts +++ b/core/cache/index.ts @@ -1,2 +1,3 @@ export * from "./types.ts"; export * from "./stale-detector.ts"; +export * from "./bounded-cache.ts"; diff --git a/core/health/__tests__/synthetic.test.ts b/core/health/__tests__/synthetic.test.ts new file mode 100644 index 0000000..3638ec2 --- /dev/null +++ b/core/health/__tests__/synthetic.test.ts @@ -0,0 +1,269 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { NETWORK_PASSPHRASES } from "@/core/network/config"; +import { createCaptureSink, resetTelemetrySink, setTelemetrySink } from "@/core/telemetry/telemetry"; +import { + parseRetryAfter, + runSyntheticMonitor, + SYNTHETIC_PAGE_SIZE, + type SyntheticMonitorOptions +} from "../synthetic"; + +const HORIZON = "https://horizon.example"; +const ACCOUNT = `G${"A".repeat(55)}`; +const SERVICE = { role: "service" } as const; + +const root = (overrides: Record = {}) => ({ + network_passphrase: NETWORK_PASSPHRASES.testnet, + core_latest_ledger: 5_247_219, + history_latest_ledger: 5_247_219, + ...overrides +}); + +/** Paging tokens counting down from `from`, as Horizon returns them for `order=desc`. */ +const page = (from: number, count: number) => ({ + _embedded: { + records: Array.from({ length: count }, (_, index) => ({ + id: String(from - index), + paging_token: String(from - index), + source_account: ACCOUNT + })) + } +}); + +type Reply = { status?: number; body?: unknown; headers?: Record } | "transport" | "hang"; + +/** + * A scripted Horizon: each route answers from its queue (the last reply + * repeats), and every requested URL is kept for assertions. + */ +function horizon(routes: Record) { + const requests: URL[] = []; + const fetchImpl = (async (input: string | URL | Request, init?: RequestInit) => { + const url = new URL(String(input)); + requests.push(url); + const route = url.pathname === "/" ? "root" : url.searchParams.has("cursor") ? "older" : url.pathname.slice(1); + const queue = routes[route] ?? [{ status: 404, body: { status: 404 } }]; + const reply = queue.length > 1 ? queue.shift()! : queue[0]; + if (reply === "transport") throw new TypeError("fetch failed"); + if (reply === "hang") { + return new Promise((_, reject) => + init?.signal?.addEventListener("abort", () => reject(new DOMException("aborted", "AbortError"))) + ); + } + const body = typeof reply.body === "string" ? reply.body : JSON.stringify(reply.body ?? {}); + return new Response(body, { status: reply.status ?? 200, headers: reply.headers }); + }) as typeof fetch; + return { fetchImpl, requests }; +} + +function healthyRoutes(firstPageSize = SYNTHETIC_PAGE_SIZE): Record { + return { + root: [{ body: root() }], + operations: [{ body: page(900, 1) }], + [`accounts/${ACCOUNT}/operations`]: [{ body: page(800, firstPageSize) }], + older: [{ body: page(800 - firstPageSize, 5) }] + }; +} + +function options(fetchImpl: typeof fetch, overrides: Partial = {}) { + const sleeps: number[] = []; + let clock = 0; + return { + sleeps, + options: { + network: "testnet", + horizonUrl: HORIZON, + fetch: fetchImpl, + sleep: async (ms: number) => void sleeps.push(ms), + now: () => (clock += 1), + ...overrides + } satisfies SyntheticMonitorOptions + }; +} + +afterEach(() => resetTelemetrySink()); + +describe("synthetic critical-journey monitor (#269)", () => { + describe("normal path", () => { + it("walks the journey to the older page and reports healthy", async () => { + const { fetchImpl, requests } = horizon(healthyRoutes()); + const outcome = await runSyntheticMonitor(SERVICE, options(fetchImpl).options); + + expect(outcome.ok && outcome.value.status).toBe("healthy"); + expect(outcome.ok && outcome.value.steps.map((step) => [step.step, step.ok, step.attempts])).toEqual([ + ["horizon.root", true, 1], + ["operations.discover", true, 1], + ["account.operations", true, 1], + ["account.operations.older", true, 1] + ]); + expect(requests.map((url) => `${url.pathname}${url.search}`)).toEqual([ + "/", + "/operations?order=desc&limit=1", + `/accounts/${ACCOUNT}/operations?order=desc&limit=20`, + `/accounts/${ACCOUNT}/operations?order=desc&limit=20&cursor=781` + ]); + }); + + it("skips the older page when the first page is not full, as the UI does", async () => { + const { fetchImpl } = horizon(healthyRoutes(3)); + const outcome = await runSyntheticMonitor(SERVICE, options(fetchImpl).options); + + expect(outcome.ok && outcome.value.status).toBe("healthy"); + expect(outcome.ok && outcome.value.steps.map((step) => step.step)).not.toContain("account.operations.older"); + }); + + it("keeps the defaults when an option is passed as undefined", async () => { + const { fetchImpl } = horizon(healthyRoutes()); + const outcome = await runSyntheticMonitor( + SERVICE, + options(fetchImpl, { timeoutMs: undefined, maxAttempts: undefined }).options + ); + + expect(outcome.ok && outcome.value.status).toBe("healthy"); + }); + + it("emits one structured telemetry event per run", async () => { + const sink = createCaptureSink(); + setTelemetrySink(sink); + const { fetchImpl } = horizon(healthyRoutes()); + await runSyntheticMonitor(SERVICE, options(fetchImpl).options); + + expect(sink.events.filter((event) => event.op === "synthetic.journey")).toHaveLength(1); + expect(sink.events.at(-1)).toMatchObject({ actorType: "system", result: "success", payload: { status: "healthy" } }); + }); + }); + + describe("retry path", () => { + it("recovers from a transient 503 and reports degraded, not healthy", async () => { + const routes = healthyRoutes(); + routes.root = [{ status: 503 }, { body: root() }]; + const { fetchImpl } = horizon(routes); + const { options: run, sleeps } = options(fetchImpl); + + const outcome = await runSyntheticMonitor(SERVICE, run); + expect(outcome.ok && outcome.value.status).toBe("degraded"); + expect(outcome.ok && outcome.value.steps[0]).toMatchObject({ ok: true, attempts: 2 }); + expect(sleeps).toEqual([500]); + }); + + it("backs off exponentially and honours Retry-After within the ceiling", async () => { + const routes = healthyRoutes(); + routes.root = [ + { status: 429, headers: { "retry-after": "3" } }, + { status: 429, headers: { "retry-after": "120" } }, + { body: root() } + ]; + const { fetchImpl } = horizon(routes); + const { options: run, sleeps } = options(fetchImpl); + + await runSyntheticMonitor(SERVICE, run); + expect(sleeps).toEqual([3_000, 10_000]); + }); + + it("retries transport errors and timeouts", async () => { + const routes = healthyRoutes(); + routes.root = ["transport", "hang", { body: root() }]; + const { fetchImpl } = horizon(routes); + + const outcome = await runSyntheticMonitor(SERVICE, options(fetchImpl, { timeoutMs: 5 }).options); + expect(outcome.ok && outcome.value.steps[0]).toMatchObject({ ok: true, attempts: 3 }); + }); + }); + + describe("failure path", () => { + it.each([ + ["rate_limited", [{ status: 429 }]], + ["server_error", [{ status: 500 }]], + ["transport", ["transport"]], + ["timeout", ["hang"]] + ] as const)("reports %s as unhealthy once retries are exhausted", async (code, replies) => { + const routes = healthyRoutes(); + routes.root = [...replies] as Reply[]; + const { fetchImpl, requests } = horizon(routes); + + const outcome = await runSyntheticMonitor(SERVICE, options(fetchImpl, { timeoutMs: 5 }).options); + expect(outcome.ok && outcome.value).toMatchObject({ + status: "unhealthy", + steps: [{ step: "horizon.root", ok: false, attempts: 3, errorCode: code }] + }); + expect(requests).toHaveLength(3); + }); + + it("does not retry a contract failure", async () => { + const routes = healthyRoutes(); + routes.root = [{ body: root({ network_passphrase: NETWORK_PASSPHRASES.mainnet }) }]; + const { fetchImpl, requests } = horizon(routes); + + const outcome = await runSyntheticMonitor(SERVICE, options(fetchImpl).options); + expect(outcome.ok && outcome.value.steps).toEqual([ + expect.objectContaining({ step: "horizon.root", attempts: 1, errorCode: "network_mismatch" }) + ]); + expect(requests).toHaveLength(1); + }); + + it.each([ + ["a non-JSON root body", { root: [{ body: "" }] }, "horizon.root", "malformed_payload"], + ["a root without ledger numbers", { root: [{ body: { network_passphrase: NETWORK_PASSPHRASES.testnet } }] }, "horizon.root", "malformed_payload"], + ["a 404", { operations: [{ status: 404 }] }, "operations.discover", "http_status"], + ["an empty network", { operations: [{ body: page(1, 0) }] }, "operations.discover", "no_activity"], + ["a muxed or invalid source", { operations: [{ body: { _embedded: { records: [{ source_account: "M123" }] } } }] }, "operations.discover", "malformed_payload"], + ["an ascending first page", { [`accounts/${ACCOUNT}/operations`]: [{ body: { _embedded: { records: [{ paging_token: "1" }, { paging_token: "2" }] } } }] }, "account.operations", "pagination_broken"], + ["an older page that repeats the cursor", { older: [{ body: page(781, 2) }] }, "account.operations.older", "pagination_broken"] + ] as const)("flags %s", async (_, override, failedStep, code) => { + const { fetchImpl } = horizon({ ...healthyRoutes(), ...override } as Record); + const outcome = await runSyntheticMonitor(SERVICE, options(fetchImpl).options); + + expect(outcome.ok && outcome.value.status).toBe("unhealthy"); + expect(outcome.ok && outcome.value.steps.at(-1)).toMatchObject({ step: failedStep, ok: false, errorCode: code }); + }); + + it("reports degraded when ingestion lags or the run exceeds its latency budget", async () => { + const lagging = healthyRoutes(); + lagging.root = [{ body: root({ history_latest_ledger: 5_247_200 }) }]; + const lag = await runSyntheticMonitor(SERVICE, options(horizon(lagging).fetchImpl).options); + expect(lag.ok && lag.value).toMatchObject({ status: "degraded", ingestLagLedgers: 19 }); + + const slow = await runSyntheticMonitor( + SERVICE, + options(horizon(healthyRoutes()).fetchImpl, { latencyBudgetMs: 1 }).options + ); + expect(slow.ok && slow.value.status).toBe("degraded"); + }); + }); + + describe("authorization path", () => { + it("refuses a viewer or unknown role without making a request", async () => { + const sink = createCaptureSink(); + setTelemetrySink(sink); + const { fetchImpl, requests } = horizon(healthyRoutes()); + + for (const role of ["viewer", "admin"] as const) { + expect(await runSyntheticMonitor({ role: role as "viewer" }, options(fetchImpl).options)).toEqual({ + ok: false, + code: "synthetic_unauthorized" + }); + } + expect(requests).toHaveLength(0); + expect(sink.events.at(-1)).toMatchObject({ op: "synthetic.journey", errorCode: "synthetic_unauthorized" }); + }); + + it("runs for the service and maintainer roles", async () => { + for (const role of ["service", "maintainer"] as const) { + const outcome = await runSyntheticMonitor({ role }, options(horizon(healthyRoutes()).fetchImpl).options); + expect(outcome.ok).toBe(true); + } + }); + }); + + describe("parseRetryAfter (RFC 9110 §10.2.3)", () => { + it("reads delay-seconds and HTTP-dates, and ignores anything else", () => { + const now = Date.parse("2026-10-17T00:00:00Z"); + expect(parseRetryAfter("7", now)).toBe(7_000); + expect(parseRetryAfter("Sat, 17 Oct 2026 00:00:05 GMT", now)).toBe(5_000); + expect(parseRetryAfter("Fri, 16 Oct 2026 00:00:00 GMT", now)).toBe(0); + expect(parseRetryAfter("-1", now)).toBeUndefined(); + expect(parseRetryAfter("soon", now)).toBeUndefined(); + expect(parseRetryAfter(null, now)).toBeUndefined(); + }); + }); +}); diff --git a/core/health/synthetic.ts b/core/health/synthetic.ts new file mode 100644 index 0000000..34c74eb --- /dev/null +++ b/core/health/synthetic.ts @@ -0,0 +1,317 @@ +/** + * Synthetic monitoring for the critical end-to-end journey (#269) + * + * The critical journey (tests/e2e/critical-journey.spec.ts) is the operation + * browser: load an account's operation history newest-first, then follow the + * last paging token to the next older page. The e2e spec proves the UI against + * mocked Horizon; this probe proves the same Horizon contract against the live + * network on a schedule, so an upstream break is seen before a user reports it. + * + * Steps, each against the configured Horizon: + * 1. `horizon.root` `GET /` — reachable, right network passphrase, + * ingestion lag (`core_latest_ledger` − + * `history_latest_ledger`) within tolerance. + * 2. `operations.discover` `GET /operations?order=desc&limit=1` — finds a + * currently active account, so the probe needs + * no fixture that a testnet reset would delete. + * 3. `account.operations` `GET /accounts/{id}/operations?order=desc&limit=20` + * — the journey's first page. + * 4. `account.operations.older` the same with `cursor=`, + * only when page one was full — the "Load more" step. + * + * Transient failures (HTTP 429, 5xx, transport errors, timeouts) are retried + * with exponential backoff, honouring `Retry-After`. Contract failures (wrong + * passphrase, malformed body, broken pagination) are not retried: retrying + * cannot fix them. Running the probe is an operator action gated on + * `admin:health`. + * + * Complexity: O(steps × maxAttempts) requests; O(page size) time and space per + * step to validate one page. + */ + +import type { AuthContext } from "@/core/auth/middleware"; +import { hasPermission } from "@/core/auth/types"; +import type { HealthStatus } from "@/core/health/types"; +import { HORIZON_URLS, NETWORK_PASSPHRASES } from "@/core/network/config"; +import type { StellarNetwork } from "@/core/network/types"; +import { err, ok, type Result } from "@/core/result/result"; +import { emitTelemetry, newCorrelationId } from "@/core/telemetry/telemetry"; + +export type SyntheticStepName = + | "horizon.root" + | "operations.discover" + | "account.operations" + | "account.operations.older"; + +export type SyntheticErrorCode = + | "rate_limited" + | "server_error" + | "transport" + | "timeout" + | "http_status" + | "malformed_payload" + | "network_mismatch" + | "no_activity" + | "pagination_broken"; + +export interface SyntheticStepResult { + readonly step: SyntheticStepName; + readonly ok: boolean; + readonly attempts: number; + readonly latencyMs: number; + readonly errorCode?: SyntheticErrorCode; +} + +export interface SyntheticReport { + readonly network: StellarNetwork; + readonly status: HealthStatus; + readonly correlationId: string; + readonly startedAt: string; + readonly durationMs: number; + readonly steps: readonly SyntheticStepResult[]; + /** `core_latest_ledger − history_latest_ledger` from `GET /`. */ + readonly ingestLagLedgers?: number; +} + +export interface SyntheticMonitorOptions { + readonly network: StellarNetwork; + /** Defaults to the configured Horizon for `network`. */ + readonly horizonUrl?: string; + readonly fetch?: typeof fetch; + readonly sleep?: (ms: number) => Promise; + readonly now?: () => number; + /** Per-request timeout. */ + readonly timeoutMs?: number; + /** Attempts per step, including the first. */ + readonly maxAttempts?: number; + readonly baseDelayMs?: number; + /** Ceiling for any single backoff, including a server's `Retry-After`. */ + readonly maxDelayMs?: number; + /** A run slower than this is `degraded`. */ + readonly latencyBudgetMs?: number; + /** Ingestion lag above this is `degraded`. */ + readonly maxIngestLagLedgers?: number; +} + +/** Mirrors the operation browser's page size, so page two is exercised exactly as users reach it. */ +export const SYNTHETIC_PAGE_SIZE = 20; + +const DEFAULTS = { + timeoutMs: 10_000, + maxAttempts: 3, + baseDelayMs: 500, + maxDelayMs: 10_000, + latencyBudgetMs: 5_000, + maxIngestLagLedgers: 5 +} as const; + +const PUBLIC_KEY = /^G[A-Z2-7]{55}$/; +const PAGING_TOKEN = /^\d+$/; + +interface Attempt { + readonly result: Result; + readonly retryable: boolean; + readonly retryAfterMs?: number; +} + +/** + * `Retry-After` is delay-seconds or an HTTP-date (RFC 9110 §10.2.3). All three + * HTTP-date forms (IMF-fixdate, RFC 850, asctime) open with a day name, which + * keeps `Date.parse` from reading a value like `-1` as a year. + */ +export function parseRetryAfter(value: string | null, now: number): number | undefined { + if (!value) return undefined; + const trimmed = value.trim(); + if (/^\d+$/.test(trimmed)) return Number(trimmed) * 1_000; + if (!/^(Mon|Tue|Wed|Thu|Fri|Sat|Sun)/.test(trimmed)) return undefined; + const at = Date.parse(trimmed); + return Number.isNaN(at) ? undefined : Math.max(0, at - now); +} + +function records(body: unknown): Record[] | null { + const list = (body as { _embedded?: { records?: unknown } } | null)?._embedded?.records; + return Array.isArray(list) && list.every((item) => item !== null && typeof item === "object") + ? (list as Record[]) + : null; +} + +/** + * Paging tokens must be numeric, unique and strictly descending, and every one + * must sit below `cursor` when one was sent. O(n) over the page. + */ +function isDescendingPage(page: Record[], cursor?: string): boolean { + let previous = cursor === undefined ? undefined : BigInt(cursor); + for (const record of page) { + const token = record.paging_token; + if (typeof token !== "string" || !PAGING_TOKEN.test(token)) return false; + const value = BigInt(token); + if (previous !== undefined && value >= previous) return false; + previous = value; + } + return true; +} + +export async function runSyntheticMonitor( + auth: AuthContext, + options: SyntheticMonitorOptions +): Promise> { + const correlationId = newCorrelationId(); + if (!hasPermission(auth.role, "admin:health")) { + emitTelemetry({ + op: "synthetic.journey", + actorType: "system", + result: "failure", + correlationId, + errorCode: "synthetic_unauthorized", + payload: { network: options.network, role: auth.role } + }); + return err("synthetic_unauthorized"); + } + + // `??` per field, so an explicit `undefined` keeps the default instead of + // turning the timeout into an immediate abort. + const settings = { + timeoutMs: options.timeoutMs ?? DEFAULTS.timeoutMs, + maxAttempts: options.maxAttempts ?? DEFAULTS.maxAttempts, + baseDelayMs: options.baseDelayMs ?? DEFAULTS.baseDelayMs, + maxDelayMs: options.maxDelayMs ?? DEFAULTS.maxDelayMs, + latencyBudgetMs: options.latencyBudgetMs ?? DEFAULTS.latencyBudgetMs, + maxIngestLagLedgers: options.maxIngestLagLedgers ?? DEFAULTS.maxIngestLagLedgers + }; + const fetchImpl = options.fetch ?? fetch; + const sleep = options.sleep ?? ((ms: number) => new Promise((resolve) => setTimeout(resolve, ms))); + const now = options.now ?? (() => Date.now()); + const base = `${(options.horizonUrl ?? HORIZON_URLS[options.network]).replace(/\/+$/, "")}/`; + const steps: SyntheticStepResult[] = []; + const startedAt = now(); + + async function attempt(url: string): Promise { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), settings.timeoutMs); + try { + const response = await fetchImpl(url, { headers: { Accept: "application/json" }, signal: controller.signal }); + if (response.status === 429) { + return { + result: err("rate_limited"), + retryable: true, + retryAfterMs: parseRetryAfter(response.headers.get("retry-after"), now()) + }; + } + if (response.status >= 500) return { result: err("server_error"), retryable: true }; + if (!response.ok) return { result: err("http_status"), retryable: false }; + try { + return { result: ok(await response.json()), retryable: false }; + } catch { + return { result: err("malformed_payload"), retryable: false }; + } + } catch { + return { result: err(controller.signal.aborted ? "timeout" : "transport"), retryable: true }; + } finally { + clearTimeout(timer); + } + } + + /** Fetches with retries, validates, and records the step. Null means the journey stops. */ + async function step( + name: SyntheticStepName, + path: string, + query: Record, + validate: (body: unknown) => Result + ): Promise { + const url = new URL(path, base); + for (const [key, value] of Object.entries(query)) url.searchParams.set(key, value); + + const started = now(); + let attempts = 0; + let outcome: Attempt; + for (;;) { + attempts += 1; + outcome = await attempt(url.toString()); + if (!outcome.retryable || attempts >= settings.maxAttempts) break; + const backoff = settings.baseDelayMs * 2 ** (attempts - 1); + await sleep(Math.min(Math.max(backoff, outcome.retryAfterMs ?? 0), settings.maxDelayMs)); + } + + const validated = outcome.result.ok ? validate(outcome.result.value) : outcome.result; + steps.push({ + step: name, + ok: validated.ok, + attempts, + latencyMs: now() - started, + ...(validated.ok ? {} : { errorCode: validated.code }) + }); + return validated.ok ? validated.value : null; + } + + const expectedPassphrase = NETWORK_PASSPHRASES[options.network]; + const root = await step("horizon.root", "", {}, (body) => { + const { network_passphrase, core_latest_ledger, history_latest_ledger } = (body ?? {}) as Record; + if (typeof network_passphrase !== "string" || !Number.isSafeInteger(core_latest_ledger) || !Number.isSafeInteger(history_latest_ledger)) { + return err("malformed_payload"); + } + if (network_passphrase !== expectedPassphrase) return err("network_mismatch"); + return ok({ ingestLag: (core_latest_ledger as number) - (history_latest_ledger as number) }); + }); + + const accountId = + root && + (await step("operations.discover", "operations", { order: "desc", limit: "1" }, (body) => { + const page = records(body); + if (!page) return err("malformed_payload"); + if (page.length === 0) return err("no_activity"); + const source = page[0].source_account; + return typeof source === "string" && PUBLIC_KEY.test(source) ? ok(source) : err("malformed_payload"); + })); + + const operationsPath = accountId ? `accounts/${encodeURIComponent(accountId)}/operations` : ""; + const query = { order: "desc", limit: String(SYNTHETIC_PAGE_SIZE) }; + const firstPage = + accountId && + (await step("account.operations", operationsPath, query, (body) => { + const page = records(body); + if (!page) return err("malformed_payload"); + // The account was just discovered from its own operation, so an empty page is broken. + if (page.length === 0 || !isDescendingPage(page)) return err("pagination_broken"); + return ok(page); + })); + + if (firstPage && firstPage.length === SYNTHETIC_PAGE_SIZE) { + const cursor = firstPage[firstPage.length - 1].paging_token as string; + await step("account.operations.older", operationsPath, { ...query, cursor }, (body) => { + const page = records(body); + if (!page) return err("malformed_payload"); + return isDescendingPage(page, cursor) ? ok(page) : err("pagination_broken"); + }); + } + + const durationMs = now() - startedAt; + const failed = steps.find((result) => !result.ok); + const ingestLagLedgers = root?.ingestLag; + const status: HealthStatus = failed + ? "unhealthy" + : steps.some((result) => result.attempts > 1) || + durationMs > settings.latencyBudgetMs || + (ingestLagLedgers ?? 0) > settings.maxIngestLagLedgers + ? "degraded" + : "healthy"; + + emitTelemetry({ + op: "synthetic.journey", + actorType: "system", + result: failed ? "failure" : "success", + latencyMs: durationMs, + correlationId, + errorCode: failed?.errorCode, + payload: { network: options.network, status, steps: steps.map(({ step: name, ok: passed, attempts }) => ({ name, passed, attempts })) } + }); + + return ok({ + network: options.network, + status, + correlationId, + startedAt: new Date(startedAt).toISOString(), + durationMs, + steps, + ...(ingestLagLedgers === undefined ? {} : { ingestLagLedgers }) + }); +} diff --git a/core/idempotency/__tests__/idempotency.concurrency.test.ts b/core/idempotency/__tests__/idempotency.concurrency.test.ts new file mode 100644 index 0000000..6d80684 --- /dev/null +++ b/core/idempotency/__tests__/idempotency.concurrency.test.ts @@ -0,0 +1,177 @@ +import { describe, expect, it } from "vitest"; +import { + createIdempotencyStore, + withIdempotency, + type IdempotencyStorage +} from "@/core/idempotency/idempotency"; +import { err, ok, type Result } from "@/core/result/result"; + +/** + * Duplicate submissions under concurrency (#271). + * + * `withIdempotency` claims its key synchronously, before its first `await`, so + * the order in which two submissions reach `begin` is the order they were + * issued. Every test drives that order explicitly: a submission's work is held + * open on a gate the test releases, so "still in flight" is a fact of the test + * and not a guess about scheduling. + */ +const START = Date.parse("2026-10-17T00:00:00.000Z"); +const KEY = "submit-payment-csv-0001"; + +function sharedStorage(): IdempotencyStorage { + const values = new Map(); + return { + getItem: (key) => values.get(key) ?? null, + setItem: (key, value) => void values.set(key, value), + removeItem: (key) => void values.delete(key) + }; +} + +function gate() { + let release!: (value: T) => void; + const promise = new Promise((resolve) => (release = resolve)); + return { promise, release }; +} + +function counted(outcome: () => Promise>) { + let runs = 0; + return { + run: () => { + runs += 1; + return outcome(); + }, + get runs() { + return runs; + } + }; +} + +describe("idempotent submissions under concurrency (#271)", () => { + it("runs the work once when the same submission arrives twice at the same time", async () => { + const store = createIdempotencyStore({ storage: sharedStorage(), now: () => START }); + const held = gate>(); + const work = counted(() => held.promise); + const submit = () => withIdempotency(store, { key: KEY, operation: "export.generate", request: { rows: 3 }, run: work.run }); + + const first = submit(); + const duplicate = submit(); + held.release(ok("artifact-1")); + + expect(await duplicate).toEqual(err("idempotency_in_flight")); + expect(await first).toMatchObject({ ok: true, value: { value: "artifact-1", replayed: false } }); + expect(work.runs).toBe(1); + }); + + it("replays the first outcome to every retry once it has settled", async () => { + const store = createIdempotencyStore({ storage: sharedStorage(), now: () => START }); + const work = counted(async () => ok("artifact-1")); + const submit = () => withIdempotency(store, { key: KEY, operation: "export.generate", request: { rows: 3 }, run: work.run }); + + await submit(); + const retries = await Promise.all([submit(), submit(), submit()]); + + expect(retries.every((retry) => retry.ok && retry.value.replayed && retry.value.value === "artifact-1")).toBe(true); + expect(work.runs).toBe(1); + expect(store.lookup(KEY)).toMatchObject({ ok: true, value: { replays: 3 } }); + }); + + it("deduplicates across two tabs that share one storage", async () => { + const storage = sharedStorage(); + const tabA = createIdempotencyStore({ storage, now: () => START }); + const tabB = createIdempotencyStore({ storage, now: () => START }); + const held = gate>(); + const work = counted(() => held.promise); + + const fromA = withIdempotency(tabA, { key: KEY, operation: "notify.send", run: work.run }); + const fromB = withIdempotency(tabB, { key: KEY, operation: "notify.send", run: work.run }); + held.release(ok("sent")); + + expect(await fromB).toEqual(err("idempotency_in_flight")); + expect((await fromA).ok).toBe(true); + expect(await withIdempotency(tabB, { key: KEY, operation: "notify.send", run: work.run })).toMatchObject({ + ok: true, + value: { value: "sent", replayed: true } + }); + expect(work.runs).toBe(1); + }); + + it("replays a recorded failure instead of re-running the work on retry", async () => { + const store = createIdempotencyStore({ storage: sharedStorage(), now: () => START }); + const held = gate>(); + const work = counted(() => held.promise); + const submit = () => withIdempotency(store, { key: KEY, operation: "worker.enqueue", run: work.run }); + + const first = submit(); + const duplicate = submit(); + held.release(err("horizon_unavailable")); + + expect(await duplicate).toEqual(err("idempotency_in_flight")); + expect(await first).toEqual(err("horizon_unavailable")); + expect(await submit()).toEqual(err("horizon_unavailable")); + expect(work.runs).toBe(1); + }); + + it("records a thrown error, so a concurrent retry cannot slip a second run in", async () => { + const store = createIdempotencyStore({ storage: sharedStorage(), now: () => START }); + const work = counted(async () => { + throw new TypeError("network"); + }); + const submit = () => withIdempotency(store, { key: KEY, operation: "worker.enqueue", run: work.run }); + + expect(await Promise.all([submit(), submit()])).toEqual([err("TypeError"), err("idempotency_in_flight")]); + expect(await submit()).toEqual(err("TypeError")); + expect(work.runs).toBe(1); + }); + + it("lets a retry run again only after the stuck claim is explicitly abandoned", async () => { + const store = createIdempotencyStore({ storage: sharedStorage(), now: () => START }); + expect(store.begin({ key: KEY, operation: "worker.enqueue" }).ok).toBe(true); + + const work = counted(async () => ok("done")); + const submit = () => withIdempotency(store, { key: KEY, operation: "worker.enqueue", run: work.run }); + + expect(await submit()).toEqual(err("idempotency_in_flight")); + expect(store.abandon(KEY)).toEqual(ok(true)); + expect(await submit()).toMatchObject({ ok: true, value: { value: "done", replayed: false } }); + expect(work.runs).toBe(1); + }); + + it("rejects a concurrent submission that reuses the key for a different payload", async () => { + const store = createIdempotencyStore({ storage: sharedStorage(), now: () => START }); + const held = gate>(); + const work = counted(() => held.promise); + + const first = withIdempotency(store, { key: KEY, operation: "export.generate", request: { rows: 3 }, run: work.run }); + const altered = withIdempotency(store, { key: KEY, operation: "export.generate", request: { rows: 4 }, run: work.run }); + held.release(ok("artifact-1")); + + expect(await altered).toEqual(err("idempotency_key_conflict")); + expect((await first).ok).toBe(true); + expect(work.runs).toBe(1); + }); + + it("does not let another operation replay a recorded outcome under the same key", async () => { + const store = createIdempotencyStore({ storage: sharedStorage(), now: () => START }); + await withIdempotency(store, { key: KEY, operation: "export.generate", run: async () => ok("private-artifact") }); + + const foreign = await withIdempotency(store, { + key: KEY, + operation: "notify.send", + run: async () => ok("never") + }); + + expect(foreign).toEqual(err("idempotency_key_conflict")); + }); + + it("refuses a late duplicate after the replay window instead of running twice", async () => { + let clock = START; + const store = createIdempotencyStore({ storage: sharedStorage(), now: () => clock, ttlMs: 1_000 }); + const work = counted(async () => ok("done")); + const submit = () => withIdempotency(store, { key: KEY, operation: "worker.enqueue", run: work.run }); + + await submit(); + clock += 1_000; + expect(await submit()).toEqual(err("idempotency_key_expired")); + expect(work.runs).toBe(1); + }); +}); diff --git a/core/idempotency/idempotency.ts b/core/idempotency/idempotency.ts index 822da78..d3df084 100644 --- a/core/idempotency/idempotency.ts +++ b/core/idempotency/idempotency.ts @@ -137,14 +137,33 @@ function browserStorage(): IdempotencyStorage { return new MemoryStorage(); } -/** Sorted-key JSON, so `{a:1,b:2}` and `{b:2,a:1}` hash identically. */ +/** + * Sorted-key JSON, so `{a:1,b:2}` and `{b:2,a:1}` hash identically. + * + * Total over any input: a BigInt is written with its `n` suffix and a cycle as + * `#cycle`. Neither token is valid JSON, so neither collides with a JSON value, + * and every input `JSON.stringify` accepts keeps its previous form. Tracking + * only the current path (not every visited node) keeps shared, acyclic + * references expanded as before. O(n) time, O(depth) extra space. + */ export function canonicalize(value: unknown): string { - if (value === null || typeof value !== "object") return JSON.stringify(value) ?? "null"; - if (Array.isArray(value)) return `[${value.map(canonicalize).join(",")}]`; - const entries = Object.entries(value as Record) - .filter(([, entry]) => entry !== undefined) - .sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0)); - return `{${entries.map(([key, entry]) => `${JSON.stringify(key)}:${canonicalize(entry)}`).join(",")}}`; + const path = new Set(); + const visit = (node: unknown): string => { + if (typeof node === "bigint") return `${node}n`; + if (node === null || typeof node !== "object") return JSON.stringify(node) ?? "null"; + if (path.has(node)) return "#cycle"; + path.add(node); + const body = Array.isArray(node) + ? `[${node.map(visit).join(",")}]` + : `{${Object.entries(node as Record) + .filter(([, entry]) => entry !== undefined) + .sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0)) + .map(([key, entry]) => `${JSON.stringify(key)}:${visit(entry)}`) + .join(",")}}`; + path.delete(node); + return body; + }; + return visit(value); } /** FNV-1a over the canonical form. A digest, not a signature. */ diff --git a/core/notifications/__tests__/preferences.concurrency.test.ts b/core/notifications/__tests__/preferences.concurrency.test.ts new file mode 100644 index 0000000..da72d63 --- /dev/null +++ b/core/notifications/__tests__/preferences.concurrency.test.ts @@ -0,0 +1,127 @@ +import { describe, expect, it } from "vitest"; +import { NotificationStore, StalePreferencesVersionError } from "../store"; +import type { NotificationStorage } from "../types"; + +/** + * Stale-version writes to notification preferences (#271). + * + * Two tabs (or devices syncing one storage) each read the preferences, compute + * a change from what they read, and write it back. Without a version check the + * slower tab's write is computed from a stale read: toggling `muted` from a + * stale `false` re-mutes a recipient who has just unmuted elsewhere. Passing + * `expectedVersion` turns that into an explicit, retryable rejection. + */ +function sharedStorage(): NotificationStorage { + const values = new Map(); + return { + getItem: (key) => values.get(key) ?? null, + setItem: (key, value) => void values.set(key, value), + removeItem: (key) => void values.delete(key) + }; +} + +const alice = `account:G${"A".repeat(55)}` as const; +const bob = `account:G${"B".repeat(55)}` as const; + +function twoTabs() { + const storage = sharedStorage(); + return { storage, tabA: new NotificationStore(storage), tabB: new NotificationStore(storage) }; +} + +describe("notification preferences under concurrent writers (#271)", () => { + it("applies a write whose expected version is current and advances the version", () => { + const { tabA } = twoTabs(); + const before = tabA.getPreferences(alice); + + const after = tabA.updatePreferences(alice, { muted: true }, { expectedVersion: before.version }); + + expect(after.version).toBe(before.version + 1); + expect(tabA.getPreferences(alice)).toMatchObject({ muted: true, version: before.version + 1 }); + }); + + it("rejects the slower of two writers that read the same version", () => { + const { tabA, tabB } = twoTabs(); + const seenByA = tabA.getPreferences(alice); + const seenByB = tabB.getPreferences(alice); + + tabB.updatePreferences(alice, { muted: !seenByB.muted }, { expectedVersion: seenByB.version }); + + let rejection: unknown; + try { + tabA.updatePreferences(alice, { muted: !seenByA.muted }, { expectedVersion: seenByA.version }); + } catch (error) { + rejection = error; + } + + expect(rejection).toBeInstanceOf(StalePreferencesVersionError); + expect(rejection).toMatchObject({ recipient: alice, expectedVersion: 1, actualVersion: 2 }); + expect(tabA.getPreferences(alice)).toMatchObject({ muted: true, version: 2 }); + }); + + it("succeeds when the rejected writer re-reads and retries from the fresh version", () => { + const { tabA, tabB } = twoTabs(); + const stale = tabA.getPreferences(alice); + tabB.updatePreferences(alice, { muted: true }); + + expect(() => tabA.updatePreferences(alice, { enabledTones: { ...stale.enabledTones, info: false } }, { expectedVersion: stale.version })) + .toThrow(StalePreferencesVersionError); + + const fresh = tabA.getPreferences(alice); + const retried = tabA.updatePreferences( + alice, + { enabledTones: { ...fresh.enabledTones, info: false } }, + { expectedVersion: fresh.version } + ); + + expect(retried).toMatchObject({ muted: true, version: fresh.version + 1 }); + expect(retried.enabledTones.info).toBe(false); + }); + + it("applies a double-submitted toggle exactly once", () => { + const { tabA } = twoTabs(); + const seen = tabA.getPreferences(alice); + const toggle = () => tabA.updatePreferences(alice, { muted: !seen.muted }, { expectedVersion: seen.version }); + + toggle(); + expect(toggle).toThrow(StalePreferencesVersionError); + expect(tabA.getPreferences(alice)).toMatchObject({ muted: true, version: seen.version + 1 }); + }); + + it("keeps last-write-wins for callers that do not pass a version", () => { + const { tabA, tabB } = twoTabs(); + tabB.updatePreferences(alice, { muted: true }); + tabA.updatePreferences(alice, { muted: false }); + + expect(tabA.getPreferences(alice)).toMatchObject({ muted: false, version: 3 }); + }); + + it("scopes versions per recipient, so writers for different recipients never conflict", () => { + const { tabA, tabB } = twoTabs(); + const aliceSeen = tabA.getPreferences(alice); + const bobSeen = tabB.getPreferences(bob); + + tabB.updatePreferences(bob, { muted: true }, { expectedVersion: bobSeen.version }); + tabA.updatePreferences(alice, { muted: true }, { expectedVersion: aliceSeen.version }); + + expect(tabA.getPreferences(alice).version).toBe(2); + expect(tabA.getPreferences(bob).version).toBe(2); + }); + + it("refuses an invalid recipient before any version comparison", () => { + const { tabA } = twoTabs(); + const forged = "account:not-a-key" as typeof alice; + + expect(() => tabA.updatePreferences(forged, { muted: true }, { expectedVersion: 1 })).toThrow( + "Invalid notification recipient" + ); + }); + + it("treats a corrupt stored version as the default, so a writer can recover", () => { + const { storage, tabA } = twoTabs(); + storage.setItem(`utix:notification-preferences:v1:${alice}`, JSON.stringify({ muted: true, version: "x" })); + + const seen = tabA.getPreferences(alice); + expect(seen.version).toBe(1); + expect(tabA.updatePreferences(alice, { muted: false }, { expectedVersion: 1 }).version).toBe(2); + }); +}); diff --git a/core/notifications/index.ts b/core/notifications/index.ts index 3cf602b..32aad48 100644 --- a/core/notifications/index.ts +++ b/core/notifications/index.ts @@ -1,6 +1,6 @@ export { NotificationCenter } from "@/core/notifications/NotificationCenter"; export { NotificationProvider, useNotifications } from "@/core/notifications/NotificationProvider"; -export { NotificationStore } from "@/core/notifications/store"; +export { NotificationStore, StalePreferencesVersionError } from "@/core/notifications/store"; export type { Notification, NotificationEvent, diff --git a/core/notifications/store.ts b/core/notifications/store.ts index 2f21ea5..66cffce 100644 --- a/core/notifications/store.ts +++ b/core/notifications/store.ts @@ -154,6 +154,28 @@ function parseNotification(value: unknown): Notification | null { }; } +/** + * Thrown when `updatePreferences` is given an `expectedVersion` that is no + * longer the stored version: another tab or device wrote in between, so the + * caller's patch was computed from stale preferences (#271). + */ +export class StalePreferencesVersionError extends Error { + // Explicit fields, not parameter properties, so Node's type stripping can load this module. + readonly recipient: NotificationRecipient; + readonly expectedVersion: number; + readonly actualVersion: number; + + constructor(recipient: NotificationRecipient, expectedVersion: number, actualVersion: number) { + super( + `Stale preferences for ${recipient}: expected version ${expectedVersion}, stored version is ${actualVersion}` + ); + this.name = "StalePreferencesVersionError"; + this.recipient = recipient; + this.expectedVersion = expectedVersion; + this.actualVersion = actualVersion; + } +} + export class NotificationStore { private readonly storage: NotificationStorage; private readonly maxEntries: number; @@ -239,15 +261,23 @@ export class NotificationStore { /** * Updates durable preferences for a recipient and records an audit log. + * + * Pass `expectedVersion` (the `version` the patch was computed from) to make + * the write a compare-and-set: a stale version throws + * `StalePreferencesVersionError` instead of silently overwriting a newer write. */ updatePreferences( recipient: NotificationRecipient, - patch: Partial> + patch: Partial>, + { expectedVersion }: { expectedVersion?: number } = {} ): NotificationPreferences { if (!isNotificationRecipient(recipient)) { throw new Error(`Invalid notification recipient: ${String(recipient)}`); } const current = this.getPreferences(recipient); + if (expectedVersion !== undefined && expectedVersion !== current.version) { + throw new StalePreferencesVersionError(recipient, expectedVersion, current.version); + } const updated: NotificationPreferences = { recipient, muted: typeof patch.muted === "boolean" ? patch.muted : current.muted, diff --git a/core/rpc/client.ts b/core/rpc/client.ts index 7a15a8e..43d2fe1 100644 --- a/core/rpc/client.ts +++ b/core/rpc/client.ts @@ -122,7 +122,8 @@ export function parseJsonRpcResponse( if ( err === null || typeof err !== "object" || - typeof (err as Record).code !== "number" || + // JSON-RPC 2.0 §5.1: the error code MUST be an integer. + !Number.isInteger((err as Record).code) || typeof (err as Record).message !== "string" ) { return { @@ -130,7 +131,7 @@ export function parseJsonRpcResponse( id: rawId, error: { code: RPC_STANDARD_CODES.INVALID_REQUEST, - message: "Malformed JSON-RPC response: error field must be an object with numeric code and string message", + message: "Malformed JSON-RPC response: error field must be an object with integer code and string message", data: err } }; diff --git a/docs/FUZZING.md b/docs/FUZZING.md new file mode 100644 index 0000000..d9121e8 --- /dev/null +++ b/docs/FUZZING.md @@ -0,0 +1,40 @@ +# Fuzz Coverage for Malformed Payloads and Boundary Values (#268) + +`core/__tests__/fuzz.test.ts` drives core boundaries with generated input. A +seeded PRNG (mulberry32) generates the input, so every failure reproduces exactly: + +```bash +npm test -- core/__tests__/fuzz # default seed +FUZZ_SEED=12345 npm test -- core/__tests__/fuzz # explore another seed +``` + +Each property states what must hold for every input. A failing assertion +names the seed and the input. + +| Target | Property | +| :--- | :--- | +| `parseAmount` / `amountToStroops` / `stroopsToAmount` | Every int64 stroop value round-trips exactly. Amounts are signed int64 scaled by 10⁷, so the bounds are `922337203685.4775807` and `-922337203685.4775808`; one stroop past either is `out_of_range`, and an eighth decimal is `too_many_decimals`. Arbitrary input returns a declared `AmountParseError` and never throws. | +| `parseJsonRpcResponse` | Any value, including cycles, BigInt, symbols and prototype keys, yields a JSON-RPC 2.0 envelope with exactly one of `result`/`error`. `error.code` is always an integer (§5.1), and a success always carries the expected id. | +| `requestDigest` / `canonicalize` / `IdempotencyStore.begin` | Any payload digests deterministically. A key is either claimed or rejected with a declared code, and an immediate duplicate is `idempotency_in_flight`. | +| `hasPermission` / `hasAnyPermission` / `hasAllPermissions` | An unknown role, including `toString` and `__proto__`, is denied and never throws. Known roles keep their exact grants. | + +## Defects found and fixed + +| Defect | Impact | Fix | +| :--- | :--- | :--- | +| `hasPermission` threw `TypeError` for any role outside the table | A mistyped `UTIX_USER_ROLE` crashed every permission check, and prototype keys resolved to functions | `Object.hasOwn` guard: unknown roles are denied | +| `canonicalize` threw on `BigInt` and overflowed the stack on cycles | `IdempotencyStore.begin` threw instead of returning a `Result` | BigInt → `10n`; a back-reference → `#cycle` | +| `parseJsonRpcResponse` accepted `NaN` or fractional error codes | A malformed error envelope passed as valid | Error code must satisfy `Number.isInteger` | + +## Compatibility + +All three fixes only change inputs that used to throw or that the spec forbids: + +- Every input `JSON.stringify` accepted keeps its exact canonical form, so + persisted idempotency digests stay valid. `#cycle` and the `n` suffix are not + valid JSON, so they cannot collide with an existing digest. +- Every role in `ROLE_PERMISSIONS` keeps its exact grants. +- A valid JSON-RPC error has an integer code, so no valid envelope changes. + +No migration is needed. To roll back, revert the three one-function changes. +The fuzz suite then fails, which shows exactly which guard was removed. diff --git a/docs/IDEMPOTENCY.md b/docs/IDEMPOTENCY.md index 572814a..edf1646 100644 --- a/docs/IDEMPOTENCY.md +++ b/docs/IDEMPOTENCY.md @@ -124,3 +124,48 @@ mint a second artifact. npm test -- core/idempotency # replay, expiry, in-flight, collision, sweep npm test -- core/workers core/export ``` + +## Concurrency: duplicate submissions and stale versions (#271) + +`withIdempotency` claims its key **synchronously**, before its first `await`. +In one JavaScript realm, the first of two concurrent submissions always owns +the work, and the second gets `idempotency_in_flight`. The executable spec is in +`core/idempotency/__tests__/idempotency.concurrency.test.ts`: + +| Scenario | Outcome | +| :--- | :--- | +| Same key, same payload, concurrently | One run; duplicate gets `idempotency_in_flight` | +| Retry after the first settled | Recorded outcome replayed; work count stays 1 | +| Two tabs sharing one storage | Same as above: the claim is in shared storage | +| First attempt failed or threw | Failure is recorded and replayed, never re-run | +| Claim stuck in flight | Only `abandon()` (audited) frees the key for a re-run | +| Same key, different payload or operation | `idempotency_key_conflict`, no replay of another's outcome | +| Duplicate after the TTL | `idempotency_key_expired`, never a silent second run | + +Stale versions are covered in +`core/notifications/__tests__/preferences.concurrency.test.ts`. +`NotificationStore.updatePreferences` accepts an optional third argument, +`{ expectedVersion }`. When it is given and the stored `version` differs, the +write throws `StalePreferencesVersionError` (`recipient`, `expectedVersion`, +`actualVersion`) and storage is left unchanged. The caller re-reads, recomputes +its patch and retries. + +### Edge cases + +- **Cross-tab atomicity** is only as strong as `localStorage`. Browsers do not + lock it across tabs, so two tabs that read the key in the same instant can both + claim it. The in-realm guarantees above are exact. Cross-process exactly-once + needs a store with compare-and-set behind `IdempotencyStorage`. +- **Corrupt stored preferences** fall back to defaults (`version: 1`), so a + writer passing `expectedVersion: 1` recovers instead of being locked out. + +### Migration and rollback + +`expectedVersion` is optional, and omitting it keeps the previous last-write-wins +behaviour, so no existing caller changes. Rolling back removes the option and +the error class. Stored data is unaffected because the record shape and storage +keys are unchanged. + +```bash +npm test -- core/idempotency core/notifications +``` diff --git a/docs/OPERATIONAL_HEALTH.md b/docs/OPERATIONAL_HEALTH.md index 146b0ff..b746698 100644 --- a/docs/OPERATIONAL_HEALTH.md +++ b/docs/OPERATIONAL_HEALTH.md @@ -219,3 +219,68 @@ private async checkCustomComponent(): Promise { - Document resolution steps for common issues - Keep exception context minimal but actionable - Set reasonable thresholds for stale record alerts + +## Synthetic Monitoring (#269) + +`core/health/synthetic.ts` probes the critical journey against **live** Horizon. +The critical journey is the operation browser covered by +`tests/e2e/critical-journey.spec.ts`. That e2e spec proves the UI against mocked +Horizon. The probe proves the same Horizon contract upstream, on a schedule. + +```bash +UTIX_USER_ROLE=service npm run synthetic:monitor -- --network testnet [--json] +``` + +`.github/workflows/synthetic-monitor.yml` runs it every 30 minutes for testnet +and mainnet, and can also be started by hand (`workflow_dispatch`). It needs +Node 24 and no installed dependencies. + +| Step | Request | Checks | +| :--- | :--- | :--- | +| `horizon.root` | `GET /` | `network_passphrase` matches the network; `core_latest_ledger − history_latest_ledger` ≤ 5 | +| `operations.discover` | `GET /operations?order=desc&limit=1` | Finds a currently active `G…` account, so no fixture can be wiped by a testnet reset | +| `account.operations` | `GET /accounts/{id}/operations?order=desc&limit=20` | Non-empty page with numeric, strictly descending `paging_token`s | +| `account.operations.older` | same + `cursor=` | Runs only when page one is full (as "Load more" does); every token is below the cursor | + +| Status | Meaning | Exit code | +| :--- | :--- | :--- | +| `healthy` | Every step passed on its first attempt, within budget | 0 | +| `degraded` | Passed, but needed a retry, exceeded 5 s, or ingestion lags by > 5 ledgers | 0 | +| `unhealthy` | A step failed; the journey stops at that step | 1 | +| refused / crash | Role lacks `admin:health`, or the probe threw | 2 | + +Each run emits one `synthetic.journey` telemetry event that carries the +correlation id printed in the report. + +### Failure and retry behaviour + +- **Retried**, up to 3 attempts with backoff 500 ms → 1 s: HTTP 429 + (`rate_limited`), 5xx (`server_error`), network failure (`transport`), and + no response within 10 s (`timeout`). A `Retry-After` header (seconds or + HTTP-date) lengthens the wait. No single wait exceeds 10 s. +- **Not retried**, because a retry cannot fix them: `network_mismatch` (wrong + passphrase, meaning the URL points at the other network), `malformed_payload`, + `http_status` (another 4xx), `no_activity` and `pagination_broken`. +- **Authorization**: viewers and unknown roles get `synthetic_unauthorized` + before any request is made. `service` and `maintainer` hold `admin:health`. + +### Operational edge cases + +- **Rate limits**: one run makes at most 4 requests (12 with every retry). + Horizon's default limit (`PER_HOUR_RATE_LIMIT`) is 3,600 requests per hour + per IP, so a 30-minute schedule uses a negligible share of it. +- **Testnet resets** wipe accounts. Choosing the account from the latest + operation keeps the probe working across a reset. +- **An idle network** gives `no_activity`. That is unhealthy for the public + networks, but expected on a fresh private network. +- **Ingestion lag** of a few ledgers is normal under load. It is reported as + `degraded` above 5 ledgers, which matches the stale-cache tolerance. + +### Recovery and rollback + +- On `unhealthy`, read the failed step's `errorCode`. Rerun with `--json` and + find the run by its correlation id in telemetry. +- Point the probe at another Horizon with `NEXT_PUBLIC_HORIZON_TESTNET_URL` or + `NEXT_PUBLIC_HORIZON_MAINNET_URL`. +- The probe is read-only and changes no existing contract. To roll back, disable + or delete the workflow. Nothing else depends on it. diff --git a/docs/STALE_CACHE_REPAIR.md b/docs/STALE_CACHE_REPAIR.md index 5394ee5..77352de 100644 --- a/docs/STALE_CACHE_REPAIR.md +++ b/docs/STALE_CACHE_REPAIR.md @@ -52,3 +52,59 @@ The **Stale Projection Detector** (`core/cache/stale-detector.ts`) identifies, s 2. **Index Rebuild Recovery**: - If an automated index rebuild was interrupted, run the reindex task with the partition key or full rebuild flag. - All repairs log evidence under `projection.repaired` in the audit log for inspection. + +--- + +## Bounded Cache with Explicit Invalidation (#270) + +`core/cache/bounded-cache.ts` is the cache the detector above inspects. It is +an LRU bounded by `maxEntries`, with a TTL per entry. + +```ts +const cache = createBoundedCache({ maxEntries: 500, ttlMs: 30_000, name: "accounts" }); +const view = await cache.getOrLoad(`account:${id}`, () => loadAccount(id), { ledgerSequence }); +cache.invalidate(`account:${id}`); // one key, O(1) +cache.invalidateWhere((entry) => entry.sourceEntityId === id); // a scope, O(n) +cache.clear({ role: "maintainer" }, "schema rollback"); // everything, authorised +StaleProjectionDetector.inspect(cache.snapshot(), [], context); +``` + +| Operation | Time | Notes | +| :--- | :--- | :--- | +| `get` / `set` / `invalidate` / `getOrLoad` | O(1) | `Map` insertion order is the LRU order | +| `invalidateWhere` / `clear` / `snapshot` | O(n) | n ≤ `maxEntries` | +| Space | O(maxEntries + loads in flight) | | + +| Code | Meaning | Caller action | +| :--- | :--- | :--- | +| `cache_miss` | Key absent or evicted | Load from the source | +| `cache_expired` | Age exceeded the TTL; the entry was dropped | Load from the source | +| `cache_stale_write` | Write carried an older `ledgerSequence` than the live entry | Keep the newer entry; nothing to do | +| `cache_unauthorized` | `clear` called without `admin:recovery` | Ask a maintainer | + +### Operational edge cases + +1. **Concurrent misses** for one key share one load (`getOrLoad`), so a burst of + requests for the same account is one Horizon request, not N. +2. **Failed loads are never cached.** The next call retries the source. A loader + that throws releases its in-flight slot before the rejection reaches the caller. +3. **Invalidation during a load**: the in-flight load still answers its caller + but does not store its value, so a value read before the invalidation cannot + repopulate the cache after it. +4. **Out-of-order responses**: a write whose `ledgerSequence` is older than the + live entry's is refused (`cache_stale_write`). Once that entry expires, any + ledger is accepted again, so a testnet reset (which restarts sequences) heals + itself within one TTL. Flush with `clear` to heal it immediately. +5. **Clock skew**: age uses the injected clock only. Backwards clock jumps make + entries look younger, never older, so they cannot cause premature eviction. + +### Rollback and recovery + +- Every entry is a projection of Horizon/RPC state, so any eviction or flush is + safe: the next read reloads from the source. +- After changing a value's shape, raise `schemaVersion`. The cache is in-memory, + so a deploy starts it empty. When snapshots are compared across instances, the + detector reports entries stamped with the older version as `version_mismatch`. +- Rolling the module back is safe because nothing persists. The cache lives in + memory only, and no existing export or contract changed. The module only adds + new exports to `core/cache/index.ts`. diff --git a/package.json b/package.json index 5f05cbc..8e86002 100644 --- a/package.json +++ b/package.json @@ -40,6 +40,7 @@ "issues:drips": "node scripts/create-issues.mjs --target drips", "recovery:validate": "node scripts/validate-recovery.mjs", "health:check": "node scripts/health-check.mjs", + "synthetic:monitor": "node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON scripts/synthetic-monitor.mjs", "bundle:check": "node scripts/bundle-budget.mjs", "diagnostics:contributor": "node --loader ./scripts/loader.mjs scripts/contributor-diagnostics.mjs", "doctor": "node --loader ./scripts/loader.mjs scripts/contributor-diagnostics.mjs", diff --git a/scripts/synthetic-monitor.mjs b/scripts/synthetic-monitor.mjs new file mode 100644 index 0000000..94d1afe --- /dev/null +++ b/scripts/synthetic-monitor.mjs @@ -0,0 +1,55 @@ +#!/usr/bin/env node + +/** + * Synthetic monitor for the critical journey (#269). + * + * Runs the live Horizon probe in core/health/synthetic.ts. It needs Node 24 + * (built-in TypeScript type stripping) and no installed dependencies. + * + * Usage: + * UTIX_USER_ROLE=service node scripts/synthetic-monitor.mjs [--network testnet|mainnet] [--json] + * + * Exit codes: 0 healthy or degraded, 1 unhealthy, 2 unauthorized or crashed. + */ + +import { register } from "node:module"; + +register("./loader.mjs", import.meta.url); + +const { runSyntheticMonitor } = await import("../core/health/synthetic.ts"); +const { isStellarNetwork } = await import("../core/network/types.ts"); + +const args = process.argv.slice(2); +const networkFlag = args.indexOf("--network"); +const network = networkFlag === -1 ? "testnet" : args[networkFlag + 1]; + +if (!isStellarNetwork(network)) { + console.error(`Unknown network "${network}". Use testnet or mainnet.`); + process.exit(2); +} + +try { + const outcome = await runSyntheticMonitor({ role: process.env.UTIX_USER_ROLE ?? "viewer" }, { network }); + if (!outcome.ok) { + console.error("Synthetic monitor refused: the role needs admin:health (set UTIX_USER_ROLE=service)."); + process.exit(2); + } + + const report = outcome.value; + if (args.includes("--json")) { + console.log(JSON.stringify(report, null, 2)); + } else { + console.log(`Critical journey on ${report.network}: ${report.status.toUpperCase()} in ${report.durationMs}ms`); + for (const step of report.steps) { + const mark = step.ok ? "✓" : "✗"; + const failure = step.errorCode ? ` — ${step.errorCode}` : ""; + console.log(` ${mark} ${step.step} (${step.attempts} attempt(s), ${step.latencyMs}ms)${failure}`); + } + if (report.ingestLagLedgers !== undefined) console.log(` ingestion lag: ${report.ingestLagLedgers} ledger(s)`); + console.log(` correlation id: ${report.correlationId}`); + } + process.exit(report.status === "unhealthy" ? 1 : 0); +} catch (error) { + console.error("Synthetic monitor crashed:", error); + process.exit(2); +}