diff --git a/workers/api/migrations/0021_signal_core.sql b/workers/api/migrations/0021_signal_core.sql new file mode 100644 index 0000000..7acc741 --- /dev/null +++ b/workers/api/migrations/0021_signal_core.sql @@ -0,0 +1,124 @@ +-- Signal core: the source-agnostic layer that turns any personal data +-- stream into a mintable seed. +-- +-- The platform renders art from personal data (body, location, listening, +-- spend, calendar). Writing a bespoke path per source is unbounded work, +-- so every source normalises to one of five SIGNAL SHAPES — time series, +-- interval series, point events, spatial trace, categorical distribution +-- (see src/signals/shapes.ts). Adapters target those five; everything +-- downstream sees nothing else. Adding a source is additive. +-- +-- WHAT IS DELIBERATELY ABSENT FROM THIS SCHEMA: there is no column +-- anywhere below that holds raw personal data, a sample series, or a +-- feature vector. That is the point. Personal data is parsed in the +-- browser (file imports) or held only in transit (OAuth sources) and +-- reduced to an irreversible seed; only the seed, a hash commitment to +-- the features, and coarse descriptive metadata are persisted. Adding a +-- payload column here would silently convert this into a health-data +-- store subject to GDPR Article 9 — do not add one. +-- +-- The derivation is one-way by construction (src/signals/derive.ts): +-- seed = H(features ‖ salt). A seed cannot be inverted to recover the +-- night, the route, or the heartbeat that produced it. + +-- A registered origin of data for one user: either a file the user +-- imported, or an authorised third-party account. +CREATE TABLE IF NOT EXISTS data_sources ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, + -- Free-text provider key (apple_health, fitbit, garmin, strava, + -- spotify, manual, ...). Not constrained: the source set is expected to + -- grow continuously and a CHECK here would mean a migration per source. + provider TEXT NOT NULL, + -- file_import | oauth_api. These carry DIFFERENT privacy guarantees and + -- must not be conflated in copy: a file import is parsed client-side and + -- its bytes never reach the Worker, whereas an OAuth response + -- necessarily transits the Worker and is discarded after derivation. + source_class TEXT NOT NULL, + -- The user's own name for it ("Watch", "2019-2026 archive"). + label TEXT, + -- active | revoked. Revoking blocks further derivation but never + -- retroactively invalidates derivations already made — a minted token + -- cannot be recalled, and the schema should not pretend otherwise. + status TEXT NOT NULL DEFAULT 'active', + connected_at INTEGER NOT NULL, + revoked_at INTEGER, + created_at INTEGER NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_data_sources_user + ON data_sources(user_id, status, created_at DESC); + +-- One normalised signal, extracted from a source over a declared window. +-- Describes the signal; never contains it. +CREATE TABLE IF NOT EXISTS signal_imports ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, + source_id INTEGER NOT NULL REFERENCES data_sources(id) ON DELETE CASCADE, + -- One of the five shapes. See src/signals/shapes.ts. + shape TEXT NOT NULL, + -- What the signal is, in the source's own terms ("heart_rate", + -- "sleep_stage", "play_event", "commute"). Descriptive only. + signal_kind TEXT NOT NULL, + -- The window of the person's life this covers, unix seconds. + window_start INTEGER NOT NULL, + window_end INTEGER NOT NULL, + -- Coarse quality descriptors, so the UI can be honest about gaps + -- without retaining the series itself. + sample_count INTEGER NOT NULL DEFAULT 0, + -- 0..10000 basis points of the window actually covered by samples. + -- Integer to keep D1 comparisons exact. + coverage_bp INTEGER NOT NULL DEFAULT 0, + created_at INTEGER NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_signal_imports_user + ON signal_imports(user_id, created_at DESC); +CREATE INDEX IF NOT EXISTS idx_signal_imports_source + ON signal_imports(source_id, created_at DESC); + +-- The irreversible step: an import becomes a public seed. +CREATE TABLE IF NOT EXISTS derivations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, + import_id INTEGER NOT NULL REFERENCES signal_imports(id) ON DELETE CASCADE, + -- sha256 over the canonical feature encoding. A COMMITMENT, not the + -- features: it lets a holder later prove which feature set produced a + -- seed, without the platform retaining anything invertible. + feature_digest TEXT NOT NULL, + -- The public, mintable seed. Hex. Feeds the same slot the sketch lane + -- already mints against (migrations/0015_mint_seed.sql). + seed TEXT NOT NULL, + -- Which salt generation produced this seed, so the salt can be rotated + -- without stranding older derivations. + salt_version INTEGER NOT NULL DEFAULT 1, + -- Replay guard, mirroring token_ledger.idempotency_key. + idempotency_key TEXT NOT NULL, + created_at INTEGER NOT NULL, + UNIQUE(idempotency_key) +); + +CREATE INDEX IF NOT EXISTS idx_derivations_user + ON derivations(user_id, created_at DESC); +CREATE INDEX IF NOT EXISTS idx_derivations_import + ON derivations(import_id); + +-- Granular, individually revocable consent. Separate from data_sources +-- because one source can carry several independent permissions and a +-- person must be able to withdraw one without withdrawing the others. +CREATE TABLE IF NOT EXISTS consent_records ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, + -- Null for account-wide consent. + source_id INTEGER REFERENCES data_sources(id) ON DELETE CASCADE, + -- derive | publish_artwork | retain_metadata. Note there is no + -- 'retain_raw' scope: raw retention is not an option the system offers, + -- so it is not a consent the user can be asked for. + scope TEXT NOT NULL, + granted_at INTEGER NOT NULL, + revoked_at INTEGER, + created_at INTEGER NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_consent_user_scope + ON consent_records(user_id, scope, revoked_at); diff --git a/workers/api/src/signals/derive.ts b/workers/api/src/signals/derive.ts new file mode 100644 index 0000000..40feaeb --- /dev/null +++ b/workers/api/src/signals/derive.ts @@ -0,0 +1,103 @@ +/** + * Seed derivation — the one-way door. + * + * `deriveSeed` turns a feature set into the public, mintable seed that + * drives the artwork. It must be: + * + * - DETERMINISTIC: the same features always yield the same seed, or a + * token's provenance claim is meaningless. + * - IRREVERSIBLE: no feature value may be recoverable from a seed. The + * seed is published on-chain and pinned to IPFS forever; if it were + * invertible, minting would be permanent publication of health data. + * - SALTED: a per-deployment secret prevents an attacker from + * brute-forcing plausible feature sets against a public seed. Without + * it, the feature space is small enough (a dozen rounded numbers) that + * a dictionary attack is realistic — hashing alone is NOT enough here. + * + * The salt is versioned so it can be rotated without stranding existing + * derivations: a row records which generation produced it. + */ + +import type { Features } from "./features"; + +/** Fixed-point precision for canonical encoding. */ +const PRECISION = 6; + +/** + * Current salt generation. Bump alongside SIGNAL_SALT when rotating, and + * whenever the canonical encoding or an existing feature NAME changes — + * all of those alter the seed for identical input, and a stale version + * number would misrepresent how an old seed was produced. + */ +export const SALT_VERSION = 1; + +export class DerivationError extends Error { + constructor( + message: string, + public code: string, + ) { + super(message); + } +} + +/** + * Canonical text encoding of a feature set. + * + * Sorted by name and written at fixed precision so the encoding does not + * depend on extraction order or on floating-point noise: the same night + * parsed on two devices must hash identically, and IEEE-754 arithmetic + * differs in the last bits across engines. Rounding is what makes the + * determinism guarantee real rather than approximate. + */ +export function canonicalEncode(features: Features): string { + if (!features.length) { + throw new DerivationError("cannot derive from an empty feature set", "empty_features"); + } + const seen = new Set(); + const parts: string[] = []; + for (const f of [...features].sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0))) { + if (seen.has(f.name)) { + throw new DerivationError(`duplicate feature name: ${f.name}`, "duplicate_feature"); + } + seen.add(f.name); + if (!Number.isFinite(f.value)) { + throw new DerivationError(`feature ${f.name} is not finite`, "invalid_feature"); + } + // toFixed then Number-normalise so -0 and 0 encode identically. + const rounded = Number(f.value.toFixed(PRECISION)); + parts.push(`${f.name}=${rounded === 0 ? 0 : rounded}`); + } + return parts.join(";"); +} + +async function sha256Hex(input: string): Promise { + const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(input)); + return Array.from(new Uint8Array(digest), (b) => b.toString(16).padStart(2, "0")).join(""); +} + +/** + * Commitment to a feature set: sha256 over the canonical encoding, with + * no salt. Stored so a holder can later prove which features produced a + * seed by re-supplying them — the salt is deliberately excluded so this + * check does not require the platform's secret. + */ +export async function featureDigest(features: Features): Promise { + return sha256Hex(canonicalEncode(features)); +} + +/** + * The public seed. Salted, so the published value cannot be attacked by + * enumerating plausible feature sets. + */ +export async function deriveSeed(features: Features, salt: string): Promise { + if (!salt || salt.length < 16) { + // Failing closed matters more than convenience: a short or missing + // salt silently removes the only defence against brute-forcing a + // published seed back to someone's health data. + throw new DerivationError( + "signal salt is missing or too short (min 16 chars)", + "salt_unconfigured", + ); + } + return sha256Hex(`ga-signal-v${SALT_VERSION}|${salt}|${canonicalEncode(features)}`); +} diff --git a/workers/api/src/signals/features.ts b/workers/api/src/signals/features.ts new file mode 100644 index 0000000..1b6fe4d --- /dev/null +++ b/workers/api/src/signals/features.ts @@ -0,0 +1,255 @@ +/** + * Feature extraction — the irreversible narrowing step. + * + * A signal of arbitrary length becomes a fixed, small set of statistical + * descriptors. This is where personal data stops being personal data: + * you cannot reconstruct a night's sleep, a route, or a heartbeat from + * the aggregates below, and nothing here retains an individual sample. + * + * Features serve two purposes downstream: + * 1. Hashed into the public seed (src/signals/derive.ts). + * 2. Fed to the generator, so the artwork reflects the data's actual + * character rather than an arbitrary number. Descriptors like + * `masd` (mean absolute successive difference) and `fragmentation` + * exist specifically because they carry the QUALITY of a signal — + * a restless night versus a still one — which is what makes a piece + * recognisably someone's rather than decorative. + * + * Every function here is pure and deterministic: identical input must + * produce byte-identical output, because the seed depends on it. + */ + +import type { Signal } from "./shapes"; +import { timestampsOf } from "./shapes"; + +/** + * Ordered name/value pairs. An array rather than an object because the + * canonical encoding for hashing depends on a stable order, and object + * key order is a weak thing to rely on across engines. + */ +export type Features = { name: string; value: number }[]; + +const EARTH_RADIUS_M = 6_371_000; + +function mean(xs: number[]): number { + if (!xs.length) return 0; + let sum = 0; + for (const x of xs) sum += x; + return sum / xs.length; +} + +function stdDev(xs: number[]): number { + if (xs.length < 2) return 0; + const m = mean(xs); + let acc = 0; + for (const x of xs) acc += (x - m) ** 2; + return Math.sqrt(acc / (xs.length - 1)); +} + +/** Quantile by linear interpolation on a copy sorted ascending. */ +function quantile(xs: number[], q: number): number { + if (!xs.length) return 0; + const sorted = [...xs].sort((a, b) => a - b); + if (sorted.length === 1) return sorted[0]; + const pos = (sorted.length - 1) * q; + const lo = Math.floor(pos); + const hi = Math.ceil(pos); + if (lo === hi) return sorted[lo]; + return sorted[lo] + (sorted[hi] - sorted[lo]) * (pos - lo); +} + +/** + * Mean absolute successive difference — how much the signal moves + * between neighbouring samples. Two signals can share a mean and a + * spread while one is smooth and the other jagged; this separates them, + * and it is the descriptor that carries "irregularity" into the artwork. + */ +function masd(xs: number[]): number { + if (xs.length < 2) return 0; + let acc = 0; + for (let i = 1; i < xs.length; i++) acc += Math.abs(xs[i] - xs[i - 1]); + return acc / (xs.length - 1); +} + +/** Least-squares slope of v against t, per second. */ +function slope(ts: number[], vs: number[]): number { + const n = ts.length; + if (n < 2) return 0; + const mt = mean(ts); + const mv = mean(vs); + let num = 0; + let den = 0; + for (let i = 0; i < n; i++) { + const dt = ts[i] - mt; + num += dt * (vs[i] - mv); + den += dt * dt; + } + return den === 0 ? 0 : num / den; +} + +/** Shannon entropy in bits over a weight vector; 0 when degenerate. */ +function entropy(weights: number[]): number { + const total = weights.reduce((a, b) => a + b, 0); + if (total <= 0) return 0; + let h = 0; + for (const w of weights) { + if (w <= 0) continue; + const p = w / total; + h -= p * Math.log2(p); + } + return h; +} + +function haversineMetres( + a: { lat: number; lon: number }, + b: { lat: number; lon: number }, +): number { + const toRad = Math.PI / 180; + const dLat = (b.lat - a.lat) * toRad; + const dLon = (b.lon - a.lon) * toRad; + const lat1 = a.lat * toRad; + const lat2 = b.lat * toRad; + const h = + Math.sin(dLat / 2) ** 2 + + Math.cos(lat1) * Math.cos(lat2) * Math.sin(dLon / 2) ** 2; + return 2 * EARTH_RADIUS_M * Math.asin(Math.min(1, Math.sqrt(h))); +} + +/** + * Reduce a signal to its descriptors. + * + * Feature NAMES are part of the hashed encoding, so renaming one changes + * every seed derived after the change. Treat them as a versioned wire + * format: add new names freely, never rename or reorder existing ones + * without bumping the salt version. + */ +export function extractFeatures(signal: Signal): Features { + const ts = timestampsOf(signal); + const span = ts.length ? Math.max(...ts) - Math.min(...ts) : 0; + const base: Features = [{ name: "window_seconds", value: span }]; + + switch (signal.shape) { + case "time_series": { + const vs = signal.samples.map((s) => s.v); + const times = signal.samples.map((s) => s.t); + return [ + ...base, + { name: "count", value: vs.length }, + { name: "mean", value: mean(vs) }, + { name: "sd", value: stdDev(vs) }, + { name: "min", value: Math.min(...vs) }, + { name: "max", value: Math.max(...vs) }, + { name: "p25", value: quantile(vs, 0.25) }, + { name: "median", value: quantile(vs, 0.5) }, + { name: "p75", value: quantile(vs, 0.75) }, + { name: "masd", value: masd(vs) }, + { name: "slope_per_s", value: slope(times, vs) }, + ]; + } + + case "interval_series": { + const durations = signal.intervals.map((i) => i.end - i.start); + const total = durations.reduce((a, b) => a + b, 0); + // Distinct states and how the time divides between them — the + // shape of a night, without the night. + const byState = new Map(); + for (const i of signal.intervals) { + byState.set(i.state, (byState.get(i.state) ?? 0) + (i.end - i.start)); + } + let transitions = 0; + for (let i = 1; i < signal.intervals.length; i++) { + if (signal.intervals[i].state !== signal.intervals[i - 1].state) transitions++; + } + return [ + ...base, + { name: "count", value: signal.intervals.length }, + { name: "total_seconds", value: total }, + { name: "mean_seconds", value: mean(durations) }, + { name: "sd_seconds", value: stdDev(durations) }, + { name: "state_count", value: byState.size }, + { name: "transitions", value: transitions }, + // Transitions per hour of covered time: how broken up it was. + { + name: "fragmentation", + value: total > 0 ? transitions / (total / 3600) : 0, + }, + { name: "state_entropy", value: entropy([...byState.values()]) }, + ]; + } + + case "point_events": { + const times = [...signal.events.map((e) => e.t)].sort((a, b) => a - b); + const gaps: number[] = []; + for (let i = 1; i < times.length; i++) gaps.push(times[i] - times[i - 1]); + const meanGap = mean(gaps); + const sdGap = stdDev(gaps); + // Distribution across the 24-hour clock — when in the day this + // person does the thing, with no date attached. + const hours = new Array(24).fill(0); + for (const t of times) hours[Math.floor(t / 3600) % 24] += 1; + return [ + ...base, + { name: "count", value: times.length }, + { name: "mean_gap_s", value: meanGap }, + { name: "sd_gap_s", value: sdGap }, + // Coefficient of variation of gaps: >1 means bursty, ~0 regular. + { name: "burstiness", value: meanGap > 0 ? sdGap / meanGap : 0 }, + { name: "hour_entropy", value: entropy(hours) }, + ]; + } + + case "spatial_trace": { + const pts = signal.points; + let distance = 0; + for (let i = 1; i < pts.length; i++) distance += haversineMetres(pts[i - 1], pts[i]); + const lats = pts.map((p) => p.lat); + const lons = pts.map((p) => p.lon); + const straight = + pts.length > 1 ? haversineMetres(pts[0], pts[pts.length - 1]) : 0; + const alts = pts.map((p) => p.alt).filter((a): a is number => a !== undefined); + let gain = 0; + for (let i = 1; i < alts.length; i++) { + const d = alts[i] - alts[i - 1]; + if (d > 0) gain += d; + } + return [ + ...base, + { name: "count", value: pts.length }, + { name: "distance_m", value: distance }, + { name: "displacement_m", value: straight }, + // 1 = a straight line, →0 = a wandering loop. + { name: "straightness", value: distance > 0 ? straight / distance : 0 }, + { name: "mean_speed_ms", value: span > 0 ? distance / span : 0 }, + // Extent only — never the actual coordinates, which would + // identify a home address. + { name: "bbox_lat_deg", value: Math.max(...lats) - Math.min(...lats) }, + { name: "bbox_lon_deg", value: Math.max(...lons) - Math.min(...lons) }, + { name: "elevation_gain_m", value: gain }, + ]; + } + + case "categorical": { + const weights = signal.buckets.map((b) => b.weight); + const total = weights.reduce((a, b) => a + b, 0); + const sorted = [...weights].sort((a, b) => b - a); + const top = sorted[0] ?? 0; + // Gini: 0 = perfectly even, →1 = one bucket dominates. + const n = sorted.length; + let gini = 0; + if (n > 1 && total > 0) { + const asc = [...sorted].reverse(); + let weighted = 0; + for (let i = 0; i < n; i++) weighted += (i + 1) * asc[i]; + gini = (2 * weighted) / (n * total) - (n + 1) / n; + } + return [ + ...base, + { name: "count", value: n }, + { name: "total_weight", value: total }, + { name: "top_share", value: total > 0 ? top / total : 0 }, + { name: "entropy", value: entropy(weights) }, + { name: "gini", value: gini }, + ]; + } + } +} diff --git a/workers/api/src/signals/shapes.ts b/workers/api/src/signals/shapes.ts new file mode 100644 index 0000000..7c8e02a --- /dev/null +++ b/workers/api/src/signals/shapes.ts @@ -0,0 +1,250 @@ +/** + * The five normalized signal shapes. + * + * Personal data streams are unbounded in variety — a heart rate, a + * commute, a year of listening, a bank statement — but they are not + * unbounded in STRUCTURE. Every one reduces to one of the five shapes + * below. An adapter's entire job is to produce one of these from its + * source format; feature extraction, seed derivation and the generators + * see nothing else. + * + * That is what keeps "any personal data stream" tractable: adding a + * source is additive (one adapter), not multiplicative (a new path + * through every downstream stage). + * + * These types describe data IN TRANSIT — in the browser for file + * imports, or briefly in the Worker for OAuth sources. None of it is + * ever persisted; see the header of migrations/0021_signal_core.sql. + */ + +export const SIGNAL_SHAPES = [ + "time_series", + "interval_series", + "point_events", + "spatial_trace", + "categorical", +] as const; + +export type SignalShape = (typeof SIGNAL_SHAPES)[number]; + +export function isSignalShape(v: unknown): v is SignalShape { + return typeof v === "string" && (SIGNAL_SHAPES as readonly string[]).includes(v); +} + +/** A scalar sampled over time: heart rate, glucose, steps, spend. */ +export interface TimeSeries { + shape: "time_series"; + kind: string; + unit?: string; + /** Unix seconds and value. Need not be evenly spaced. */ + samples: { t: number; v: number }[]; +} + +/** A state occupying spans: sleep stages, calendar events, trips. */ +export interface IntervalSeries { + shape: "interval_series"; + kind: string; + intervals: { start: number; end: number; state: string }[]; +} + +/** Discrete moments: songs played, messages sent, transactions. */ +export interface PointEvents { + shape: "point_events"; + kind: string; + events: { t: number; label?: string }[]; +} + +/** A path through the world: a run, a commute, a year of travel. */ +export interface SpatialTrace { + shape: "spatial_trace"; + kind: string; + points: { t: number; lat: number; lon: number; alt?: number }[]; +} + +/** A distribution over labels: genres, merchants, contacts. */ +export interface Categorical { + shape: "categorical"; + kind: string; + buckets: { label: string; weight: number }[]; +} + +export type Signal = + | TimeSeries + | IntervalSeries + | PointEvents + | SpatialTrace + | Categorical; + +export class SignalError extends Error { + constructor( + message: string, + public code: string, + ) { + super(message); + } +} + +/** Finite real number — rejects NaN and ±Infinity, which JSON.parse admits. */ +function isNum(v: unknown): v is number { + return typeof v === "number" && Number.isFinite(v); +} + +function isUnixSeconds(v: unknown): v is number { + // Rejects milliseconds passed by mistake, a common adapter bug that + // would otherwise silently produce a window ~50,000 years wide. + return isNum(v) && v > 0 && v < 4_102_444_800; // < year 2100 +} + +function nonEmptyString(v: unknown, max = 120): v is string { + return typeof v === "string" && v.trim().length > 0 && v.length <= max; +} + +/** + * Validate and narrow an untrusted object to a Signal. + * + * Strict on purpose: a malformed signal that slips through produces a + * seed, and a seed is minted and irreversible. Failing here is cheap; + * failing later is permanent. + */ +export function parseSignal(input: unknown): Signal { + if (!input || typeof input !== "object") { + throw new SignalError("signal must be an object", "invalid_signal"); + } + const s = input as Record; + if (!isSignalShape(s.shape)) { + throw new SignalError( + `unknown signal shape: ${String(s.shape)}`, + "invalid_shape", + ); + } + if (!nonEmptyString(s.kind)) { + throw new SignalError("signal kind is required", "invalid_kind"); + } + const kind = (s.kind as string).trim(); + + switch (s.shape) { + case "time_series": { + const samples = requireArray(s.samples, "samples"); + const out = samples.map((raw, i) => { + const r = asObject(raw, "sample", i); + if (!isUnixSeconds(r.t)) throw at("sample.t must be unix seconds", "invalid_sample", i); + if (!isNum(r.v)) throw at("sample.v must be a finite number", "invalid_sample", i); + return { t: r.t, v: r.v }; + }); + const unit = nonEmptyString(s.unit, 32) ? (s.unit as string) : undefined; + return { shape: "time_series", kind, unit, samples: out }; + } + + case "interval_series": { + const intervals = requireArray(s.intervals, "intervals"); + const out = intervals.map((raw, i) => { + const r = asObject(raw, "interval", i); + if (!isUnixSeconds(r.start)) throw at("interval.start must be unix seconds", "invalid_interval", i); + if (!isUnixSeconds(r.end)) throw at("interval.end must be unix seconds", "invalid_interval", i); + // A zero-length or reversed interval means the adapter mapped the + // source wrong; silently sorting it would hide that. + if ((r.end as number) <= (r.start as number)) { + throw at("interval.end must be after start", "invalid_interval", i); + } + if (!nonEmptyString(r.state, 64)) throw at("interval.state is required", "invalid_interval", i); + return { start: r.start as number, end: r.end as number, state: (r.state as string).trim() }; + }); + return { shape: "interval_series", kind, intervals: out }; + } + + case "point_events": { + const events = requireArray(s.events, "events"); + const out = events.map((raw, i) => { + const r = asObject(raw, "event", i); + if (!isUnixSeconds(r.t)) throw at("event.t must be unix seconds", "invalid_event", i); + const label = nonEmptyString(r.label, 120) ? (r.label as string).trim() : undefined; + return { t: r.t as number, label }; + }); + return { shape: "point_events", kind, events: out }; + } + + case "spatial_trace": { + const points = requireArray(s.points, "points"); + const out = points.map((raw, i) => { + const r = asObject(raw, "point", i); + if (!isUnixSeconds(r.t)) throw at("point.t must be unix seconds", "invalid_point", i); + if (!isNum(r.lat) || r.lat < -90 || r.lat > 90) { + throw at("point.lat out of range", "invalid_point", i); + } + if (!isNum(r.lon) || r.lon < -180 || r.lon > 180) { + throw at("point.lon out of range", "invalid_point", i); + } + const alt = isNum(r.alt) ? r.alt : undefined; + return { t: r.t as number, lat: r.lat as number, lon: r.lon as number, alt }; + }); + return { shape: "spatial_trace", kind, points: out }; + } + + case "categorical": { + const buckets = requireArray(s.buckets, "buckets"); + const out = buckets.map((raw, i) => { + const r = asObject(raw, "bucket", i); + if (!nonEmptyString(r.label, 120)) throw at("bucket.label is required", "invalid_bucket", i); + // Negative weight has no meaning in a distribution and would + // corrupt the normalisation in feature extraction. + if (!isNum(r.weight) || r.weight < 0) { + throw at("bucket.weight must be a non-negative number", "invalid_bucket", i); + } + return { label: (r.label as string).trim(), weight: r.weight as number }; + }); + return { shape: "categorical", kind, buckets: out }; + } + } +} + +function requireArray(v: unknown, name: string): unknown[] { + if (!Array.isArray(v) || v.length === 0) { + throw new SignalError(`${name} must be a non-empty array`, "empty_signal"); + } + return v; +} + +function asObject(v: unknown, name: string, i: number): Record { + if (!v || typeof v !== "object") { + throw new SignalError(`${name}[${i}] must be an object`, "invalid_signal"); + } + return v as Record; +} + +function at(message: string, code: string, i: number): SignalError { + return new SignalError(`${message} (index ${i})`, code); +} + +/** The window a signal covers, and how densely — for signal_imports. */ +export function signalWindow(signal: Signal): { + start: number; + end: number; + sampleCount: number; +} { + const ts = timestampsOf(signal); + let start = ts[0]; + let end = ts[0]; + for (const t of ts) { + if (t < start) start = t; + if (t > end) end = t; + } + return { start, end, sampleCount: ts.length }; +} + +/** Every timestamp in a signal, in source order. */ +export function timestampsOf(signal: Signal): number[] { + switch (signal.shape) { + case "time_series": + return signal.samples.map((s) => s.t); + case "interval_series": + // Both edges: an interval's end is as much a moment as its start. + return signal.intervals.flatMap((i) => [i.start, i.end]); + case "point_events": + return signal.events.map((e) => e.t); + case "spatial_trace": + return signal.points.map((p) => p.t); + case "categorical": + // A distribution has no intrinsic time. Callers supply the window. + return []; + } +} diff --git a/workers/api/test/signals.core.test.mjs b/workers/api/test/signals.core.test.mjs new file mode 100644 index 0000000..df47203 --- /dev/null +++ b/workers/api/test/signals.core.test.mjs @@ -0,0 +1,315 @@ +// The signal core: shape validation, feature extraction, seed derivation. +// +// This is the highest-risk code in the data-art pipeline. A malformed +// signal that slips through validation still produces a seed, and a seed +// gets minted — permanently. And the derivation is the only thing +// standing between a published, on-chain value and someone's health +// data, so its irreversibility is a correctness property, not a nicety. + +import { build } from "esbuild"; +import { mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve, dirname } from "node:path"; +import { pathToFileURL, fileURLToPath } from "node:url"; +import assert from "node:assert/strict"; +import { test } from "node:test"; + +const srcDir = resolve(dirname(fileURLToPath(import.meta.url)), "..", "src"); +const tmp = mkdtempSync(join(tmpdir(), "ga-signals-")); +const outFile = join(tmp, "signals.mjs"); +const entry = join(tmp, "entry.ts"); + +writeFileSync( + entry, + `export { parseSignal, signalWindow, timestampsOf, SIGNAL_SHAPES } from ${JSON.stringify(join(srcDir, "signals", "shapes"))}; +export { extractFeatures } from ${JSON.stringify(join(srcDir, "signals", "features"))}; +export { deriveSeed, featureDigest, canonicalEncode, SALT_VERSION } from ${JSON.stringify(join(srcDir, "signals", "derive"))}; +`, +); + +await build({ + entryPoints: [entry], + bundle: true, + format: "esm", + platform: "node", + target: "node18", + outfile: outFile, +}); + +const { + parseSignal, + signalWindow, + extractFeatures, + deriveSeed, + featureDigest, + canonicalEncode, + SIGNAL_SHAPES, +} = await import(pathToFileURL(outFile).href); + +const SALT = "test-salt-at-least-16-chars-long"; +const T0 = 1_700_000_000; + +const hr = { + shape: "time_series", + kind: "heart_rate", + unit: "bpm", + samples: [ + { t: T0, v: 62 }, + { t: T0 + 60, v: 64 }, + { t: T0 + 120, v: 61 }, + { t: T0 + 180, v: 70 }, + ], +}; + +const sleep = { + shape: "interval_series", + kind: "sleep_stage", + intervals: [ + { start: T0, end: T0 + 3600, state: "light" }, + { start: T0 + 3600, end: T0 + 7200, state: "deep" }, + { start: T0 + 7200, end: T0 + 9000, state: "rem" }, + ], +}; + +const plays = { + shape: "point_events", + kind: "play_event", + events: [{ t: T0 }, { t: T0 + 200, label: "track b" }, { t: T0 + 900 }], +}; + +const run = { + shape: "spatial_trace", + kind: "run", + points: [ + { t: T0, lat: 46.204, lon: 6.143, alt: 375 }, + { t: T0 + 300, lat: 46.21, lon: 6.15, alt: 390 }, + { t: T0 + 600, lat: 46.215, lon: 6.16, alt: 385 }, + ], +}; + +const genres = { + shape: "categorical", + kind: "genre", + buckets: [ + { label: "ambient", weight: 120 }, + { label: "jazz", weight: 40 }, + { label: "techno", weight: 8 }, + ], +}; + +const ALL = [hr, sleep, plays, run, genres]; + +// ---- shape validation ---------------------------------------------------- + +test("all five shapes are covered by the fixtures", () => { + assert.deepEqual( + [...SIGNAL_SHAPES].sort(), + [...new Set(ALL.map((s) => s.shape))].sort(), + ); +}); + +test("valid signals of every shape parse", () => { + for (const s of ALL) { + const parsed = parseSignal(structuredClone(s)); + assert.equal(parsed.shape, s.shape); + assert.equal(parsed.kind, s.kind); + } +}); + +test("millisecond timestamps are rejected", () => { + // The classic adapter bug: ms instead of seconds silently yields a + // window ~50,000 years wide and a meaningless seed. + assert.throws( + () => parseSignal({ ...hr, samples: [{ t: T0 * 1000, v: 60 }] }), + /unix seconds/, + ); +}); + +test("non-finite values are rejected", () => { + // JSON.parse admits neither, but an adapter computing a rate can + // produce both. + assert.throws(() => parseSignal({ ...hr, samples: [{ t: T0, v: NaN }] }), /finite/); + assert.throws( + () => parseSignal({ ...hr, samples: [{ t: T0, v: Infinity }] }), + /finite/, + ); +}); + +test("empty signals are rejected", () => { + for (const s of ALL) { + const key = { time_series: "samples", interval_series: "intervals", point_events: "events", spatial_trace: "points", categorical: "buckets" }[s.shape]; + assert.throws(() => parseSignal({ ...s, [key]: [] }), /non-empty/); + } +}); + +test("reversed or zero-length intervals are rejected rather than sorted", () => { + assert.throws( + () => parseSignal({ ...sleep, intervals: [{ start: T0 + 10, end: T0, state: "deep" }] }), + /after start/, + ); + assert.throws( + () => parseSignal({ ...sleep, intervals: [{ start: T0, end: T0, state: "deep" }] }), + /after start/, + ); +}); + +test("out-of-range coordinates are rejected", () => { + assert.throws( + () => parseSignal({ ...run, points: [{ t: T0, lat: 91, lon: 0 }] }), + /lat out of range/, + ); + assert.throws( + () => parseSignal({ ...run, points: [{ t: T0, lat: 0, lon: -181 }] }), + /lon out of range/, + ); +}); + +test("negative categorical weights are rejected", () => { + assert.throws( + () => parseSignal({ ...genres, buckets: [{ label: "x", weight: -1 }] }), + /non-negative/, + ); +}); + +test("unknown shapes and missing kinds are rejected", () => { + assert.throws(() => parseSignal({ shape: "spectrogram", kind: "x" }), /unknown signal shape/); + assert.throws(() => parseSignal({ ...hr, kind: "" }), /kind is required/); +}); + +test("the window spans first to last sample", () => { + const w = signalWindow(parseSignal(structuredClone(hr))); + assert.equal(w.start, T0); + assert.equal(w.end, T0 + 180); + assert.equal(w.sampleCount, 4); +}); + +// ---- features ------------------------------------------------------------ + +test("feature extraction is deterministic", () => { + for (const s of ALL) { + const a = extractFeatures(parseSignal(structuredClone(s))); + const b = extractFeatures(parseSignal(structuredClone(s))); + assert.deepEqual(a, b); + } +}); + +test("masd separates a jagged signal from a smooth one with the same mean", () => { + // Two series, identical mean and range, different character. If these + // produced the same features the artwork could not reflect how the + // data actually behaved. + const smooth = parseSignal({ + shape: "time_series", kind: "hr", + samples: [60, 61, 62, 63, 64, 65].map((v, i) => ({ t: T0 + i * 60, v })), + }); + const jagged = parseSignal({ + shape: "time_series", kind: "hr", + samples: [60, 65, 61, 64, 62, 63].map((v, i) => ({ t: T0 + i * 60, v })), + }); + const f = (x) => Object.fromEntries(extractFeatures(x).map((k) => [k.name, k.value])); + const a = f(smooth), b = f(jagged); + assert.equal(a.mean, b.mean); + assert.ok(b.masd > a.masd * 2, `expected jagged masd >> smooth (${b.masd} vs ${a.masd})`); +}); + +test("no feature carries a raw sample value through verbatim", () => { + // Spot-check the intent of the layer: aggregates, not samples. + const feats = extractFeatures(parseSignal(structuredClone(run))); + const names = feats.map((f) => f.name); + for (const n of names) assert.ok(!/lat|lon|coord/.test(n) || /bbox/.test(n), `${n} may leak a coordinate`); +}); + +// ---- derivation ---------------------------------------------------------- + +test("canonical encoding is independent of feature order", () => { + const feats = extractFeatures(parseSignal(structuredClone(hr))); + const shuffled = [...feats].reverse(); + assert.equal(canonicalEncode(feats), canonicalEncode(shuffled)); +}); + +test("canonical encoding rounds away floating-point noise", () => { + // The same night parsed on two devices must hash identically; IEEE-754 + // arithmetic differs in the last bits across engines. + const a = canonicalEncode([{ name: "x", value: 1 / 3 }]); + const b = canonicalEncode([{ name: "x", value: 0.333333333333 }]); + assert.equal(a, b); +}); + +test("canonical encoding rejects duplicate names and non-finite values", () => { + assert.throws( + () => canonicalEncode([{ name: "x", value: 1 }, { name: "x", value: 2 }]), + /duplicate feature/, + ); + assert.throws(() => canonicalEncode([{ name: "x", value: NaN }]), /not finite/); + assert.throws(() => canonicalEncode([]), /empty feature set/); +}); + +test("seed derivation is deterministic", async () => { + const feats = extractFeatures(parseSignal(structuredClone(sleep))); + const a = await deriveSeed(feats, SALT); + const b = await deriveSeed(extractFeatures(parseSignal(structuredClone(sleep))), SALT); + assert.equal(a, b); + assert.match(a, /^[0-9a-f]{64}$/); +}); + +test("a near-identical night yields a completely different seed", async () => { + const base = extractFeatures(parseSignal(structuredClone(sleep))); + const nudged = parseSignal({ + ...sleep, + intervals: [ + { start: T0, end: T0 + 3601, state: "light" }, + { start: T0 + 3601, end: T0 + 7200, state: "deep" }, + { start: T0 + 7200, end: T0 + 9000, state: "rem" }, + ], + }); + const a = await deriveSeed(base, SALT); + const b = await deriveSeed(extractFeatures(nudged), SALT); + assert.notEqual(a, b); +}); + +test("the seed does not leak any feature value", async () => { + // The seed is published on-chain and pinned to IPFS forever. If a + // feature were recoverable from it, minting would be permanent + // publication of health data. + const feats = extractFeatures(parseSignal(structuredClone(hr))); + const seed = await deriveSeed(feats, SALT); + for (const f of feats) { + const rendered = String(Number(f.value.toFixed(6))); + const digits = rendered.replace(/[^0-9a-f]/g, ""); + if (digits.length >= 4) { + assert.ok(!seed.includes(digits), `seed appears to contain feature ${f.name}`); + } + } + assert.ok(!seed.includes(canonicalEncode(feats))); +}); + +test("changing the salt changes the seed but not the feature digest", async () => { + // The digest is a commitment a holder can verify without the + // platform's secret; the seed is salted so a published value cannot be + // brute-forced back to a plausible feature set. + const feats = extractFeatures(parseSignal(structuredClone(genres))); + const s1 = await deriveSeed(feats, SALT); + const s2 = await deriveSeed(feats, SALT + "-rotated"); + assert.notEqual(s1, s2); + + const d1 = await featureDigest(feats); + const d2 = await featureDigest(feats); + assert.equal(d1, d2); + assert.match(d1, /^[0-9a-f]{64}$/); + assert.notEqual(d1, s1); +}); + +test("derivation fails closed when the salt is missing or weak", async () => { + // Failing closed matters more than convenience here: a short salt + // silently removes the only defence against brute-forcing a seed. + const feats = extractFeatures(parseSignal(structuredClone(hr))); + await assert.rejects(() => deriveSeed(feats, ""), /salt is missing or too short/); + await assert.rejects(() => deriveSeed(feats, "short"), /salt is missing or too short/); +}); + +test("every shape derives a distinct seed from the others", async () => { + const seeds = new Set(); + for (const s of ALL) { + seeds.add(await deriveSeed(extractFeatures(parseSignal(structuredClone(s))), SALT)); + } + assert.equal(seeds.size, ALL.length); +});