diff --git a/README.md b/README.md index 605af49..11d8ed5 100644 --- a/README.md +++ b/README.md @@ -216,6 +216,16 @@ diagnostics and identity observations). Event export runs from the five-minute cron, up to ten batches of 100 per run, not after each upload, so an upload costs one D1 round trip. Each upload logs its status, event count and stage durations (`read_ms`, `rate_limit_ms`, `d1_ms`, `total_ms`) without identifiers. +A rejected upload's response and log carry a closed `reason` and `stage` +(`headers`, `body`, `batch`, `event`, `privacy` or `size`), listed in +`REJECTION_STAGES` in `workers/events/src/index.ts`. The log adds the batch +size, the rejected event's position, the client's `appVersion`, `buildChannel` +and `contractRevision` when they match their formats, and a user-agent family. +Nothing is authenticated when a batch is rejected, so it never logs event names, +property keys or values, identifiers, tokens or IP addresses. A failed upload logs `failed_step` +(`body`, `token`, `validation`, `rate-limit` or `storage`), the error's class +and name, and a closed `error_kind` for storage errors, never the error's +message. Desktop's uploader ignores the body of an unsuccessful response. The identity-link token is service-to-service authority. Rotate it if it is exposed, and never embed it in website or desktop bundles: diff --git a/workers/events/fixtures/contract-v11.json b/workers/events/fixtures/contract-v11.json index 3b0f9b9..d676a78 100644 --- a/workers/events/fixtures/contract-v11.json +++ b/workers/events/fixtures/contract-v11.json @@ -95,6 +95,7 @@ "queuedCountBucket": "unknown", "droppedCountBucket": "unknown", "retryCountBucket": "unknown", + "oldestEventAgeBucket": "unknown", "deliveryClass": "unknown", "appVersion": "0.6.8", "buildChannel": "stable", @@ -110,6 +111,7 @@ "queuedCountBucket": "4-10", "droppedCountBucket": "2-3", "retryCountBucket": "1", + "oldestEventAgeBucket": "unknown", "deliveryClass": "network", "appVersion": "0.6.8", "buildChannel": "stable", @@ -361,6 +363,10 @@ "crossProvider": true, "completionWake": "settled_only", "durationSeconds": 90, + "failureClass": "permission", + "failureCode": "retry_stalled", + "failurePhase": "provider", + "httpStatus": 400, "appVersion": "0.6.8", "buildChannel": "stable", "contractRevision": "11" @@ -566,6 +572,7 @@ "durationBucket": "5-15s", "usedTools": true, "hasAttachment": true, + "retryAttempts": 2, "appVersion": "0.6.8", "buildChannel": "stable", "contractRevision": "11" @@ -594,6 +601,9 @@ "properties": { "provider": "antigravity", "modelKey": "gpt-5.6-sol", + "retryAttempts": 2, + "failureCode": "retry_stalled", + "httpStatus": 400, "failureClass": "permission", "durationBucket": "5-15s", "appVersion": "0.6.8", @@ -861,6 +871,7 @@ "stage": "unknown", "failureClass": "unknown", "durationBucket": "unknown", + "trigger": "user", "appVersion": "0.6.8", "buildChannel": "stable", "contractRevision": "11" @@ -878,6 +889,11 @@ "stage": "downloading", "failureClass": "permission", "durationBucket": "5-15s", + "trigger": "agent", + "attempt": 2, + "downloadSource": "fallback", + "compressed": true, + "resumedBucket": "10-50mb", "appVersion": "0.6.8", "buildChannel": "stable", "contractRevision": "11" @@ -895,6 +911,7 @@ "stage": "unknown", "failureClass": "unknown", "durationBucket": "unknown", + "trigger": "user", "appVersion": "0.6.8", "buildChannel": "stable", "contractRevision": "11" @@ -912,6 +929,11 @@ "stage": "downloading", "failureClass": "permission", "durationBucket": "5-15s", + "trigger": "agent", + "attempt": 2, + "downloadSource": "fallback", + "compressed": true, + "resumedBucket": "10-50mb", "appVersion": "0.6.8", "buildChannel": "stable", "contractRevision": "11" @@ -929,6 +951,7 @@ "stage": "unknown", "failureClass": "unknown", "durationBucket": "unknown", + "trigger": "user", "appVersion": "0.6.8", "buildChannel": "stable", "contractRevision": "11" @@ -946,6 +969,11 @@ "stage": "downloading", "failureClass": "permission", "durationBucket": "5-15s", + "trigger": "agent", + "attempt": 2, + "downloadSource": "fallback", + "compressed": true, + "resumedBucket": "10-50mb", "appVersion": "0.6.8", "buildChannel": "stable", "contractRevision": "11" @@ -963,6 +991,7 @@ "stage": "unknown", "failureClass": "unknown", "durationBucket": "unknown", + "trigger": "user", "appVersion": "0.6.8", "buildChannel": "stable", "contractRevision": "11" @@ -980,6 +1009,38 @@ "stage": "downloading", "failureClass": "permission", "durationBucket": "5-15s", + "trigger": "agent", + "attempt": 2, + "downloadSource": "fallback", + "compressed": true, + "resumedBucket": "10-50mb", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "11" + } + }, + { + "case": "provider.runtime.first_pick:fallbacks", + "name": "provider.runtime.first_pick", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "provider": "other", + "state": "unknown", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "11" + } + }, + { + "case": "provider.runtime.first_pick:representative", + "name": "provider.runtime.first_pick", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "provider": "antigravity", + "state": "ready", + "percentBucket": "50-74", "appVersion": "0.6.8", "buildChannel": "stable", "contractRevision": "11" @@ -1924,6 +1985,66 @@ "contractRevision": "11" } }, + { + "case": "provider.turn.failed:failure-cause-hostile", + "name": "provider.turn.failed", + "privacyLevel": "essential", + "consentLevel": "essential", + "properties": { + "provider": "scient", + "modelKey": "gpt-5.6-sol", + "failureClass": "provider_error", + "durationBucket": "unknown", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "11" + } + }, + { + "case": "mcp.delegated_task.finished:delegated-failure-hostile", + "name": "mcp.delegated_task.finished", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "status": "failed", + "callerProvider": "claudeAgent", + "targetProvider": "scient", + "failurePhase": "unknown", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "11" + } + }, + { + "case": "provider.turn.failed:failure-code-unknown-identifier", + "name": "provider.turn.failed", + "privacyLevel": "essential", + "consentLevel": "essential", + "properties": { + "provider": "scient", + "modelKey": "gpt-5.6-sol", + "failureClass": "provider_error", + "durationBucket": "unknown", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "11" + } + }, + { + "case": "mcp.delegated_task.finished:delegated-failure-unknown-identifier", + "name": "mcp.delegated_task.finished", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "status": "failed", + "callerProvider": "claudeAgent", + "targetProvider": "codex", + "failurePhase": "provider", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "11" + } + }, { "case": "provider.commands.observed:command-hostile", "name": "provider.commands.observed", @@ -2243,6 +2364,45 @@ "contractRevision": "11" } }, + { + "case": "panel.viewed:file-csv", + "name": "panel.viewed", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "category": "file-preview", + "fileType": "csv", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "11" + } + }, + { + "case": "panel.viewed:file-none", + "name": "panel.viewed", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "category": "file-preview", + "fileType": "none", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "11" + } + }, + { + "case": "panel.viewed:file-private", + "name": "panel.viewed", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "category": "file-preview", + "fileType": "other", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "11" + } + }, { "case": "provider.turn.sent:scient", "name": "provider.turn.sent", @@ -2272,6 +2432,7 @@ "durationBucket": "5-15s", "usedTools": true, "hasAttachment": true, + "retryAttempts": 2, "appVersion": "0.6.8", "buildChannel": "stable", "contractRevision": "11" @@ -2341,6 +2502,11 @@ "stage": "downloading", "failureClass": "permission", "durationBucket": "5-15s", + "trigger": "agent", + "attempt": 2, + "downloadSource": "fallback", + "compressed": true, + "resumedBucket": "10-50mb", "appVersion": "0.6.8", "buildChannel": "stable", "contractRevision": "11" @@ -2375,6 +2541,7 @@ "durationBucket": "5-15s", "usedTools": true, "hasAttachment": true, + "retryAttempts": 2, "appVersion": "0.6.8", "buildChannel": "stable", "contractRevision": "11" @@ -2444,6 +2611,11 @@ "stage": "downloading", "failureClass": "permission", "durationBucket": "5-15s", + "trigger": "agent", + "attempt": 2, + "downloadSource": "fallback", + "compressed": true, + "resumedBucket": "10-50mb", "appVersion": "0.6.8", "buildChannel": "stable", "contractRevision": "11" @@ -2478,6 +2650,7 @@ "durationBucket": "5-15s", "usedTools": true, "hasAttachment": true, + "retryAttempts": 2, "appVersion": "0.6.8", "buildChannel": "stable", "contractRevision": "11" @@ -2547,6 +2720,11 @@ "stage": "downloading", "failureClass": "permission", "durationBucket": "5-15s", + "trigger": "agent", + "attempt": 2, + "downloadSource": "fallback", + "compressed": true, + "resumedBucket": "10-50mb", "appVersion": "0.6.8", "buildChannel": "stable", "contractRevision": "11" @@ -2581,6 +2759,7 @@ "durationBucket": "5-15s", "usedTools": true, "hasAttachment": true, + "retryAttempts": 2, "appVersion": "0.6.8", "buildChannel": "stable", "contractRevision": "11" @@ -2650,6 +2829,11 @@ "stage": "downloading", "failureClass": "permission", "durationBucket": "5-15s", + "trigger": "agent", + "attempt": 2, + "downloadSource": "fallback", + "compressed": true, + "resumedBucket": "10-50mb", "appVersion": "0.6.8", "buildChannel": "stable", "contractRevision": "11" diff --git a/workers/events/src/index.test.ts b/workers/events/src/index.test.ts index bb993c7..1277491 100644 --- a/workers/events/src/index.test.ts +++ b/workers/events/src/index.test.ts @@ -278,12 +278,13 @@ describe("event gateway routes", () => { properties: event.properties, })), ); - for (const [key, value] of [ - ["path", "PRIVATE document"], - ["error", "PRIVATE document"], - ["text", "PRIVATE document"], - ["documentId", "/Users/someone/PRIVATE.tex"], - ["revision", "https://example.com/PRIVATE"], + const log = vi.spyOn(console, "log").mockImplementation(() => undefined); + for (const [key, value, reason] of [ + ["path", "PRIVATE document", "property-content-key"], + ["error", "PRIVATE document", "property-content-key"], + ["text", "PRIVATE document", "property-content-key"], + ["documentId", "/Users/someone/PRIVATE.tex", "property-private-value"], + ["revision", "https://example.com/PRIVATE", "property-private-value"], ]) { const rejected = await submit([ { @@ -292,6 +293,18 @@ describe("event gateway routes", () => { }, ]); expect(rejected.status).toBe(400); + expect(await rejected.json()).toMatchObject({ reason, stage: "privacy" }); + const entry = String(log.mock.lastCall?.[0]); + expect(JSON.parse(entry)).toMatchObject({ + reason, + stage: "privacy", + rejected_event_index: 0, + app_version: "0.6.25", + build_channel: "stable", + contract_revision: "6", + }); + expect(entry).not.toContain("PRIVATE"); + expect(entry).not.toContain("feature.viewed"); } expect(database.sqlite.prepare("SELECT count(*) AS n FROM analytics_events").get()!.n).toBe(3); }); @@ -313,14 +326,21 @@ describe("event gateway routes", () => { expect(database.batch).not.toHaveBeenCalled(); }); - it("requires an installation-owned token", async () => { + it("requires an installation-owned token and logs why without identifiers or values", async () => { const database = createDatabase(); + const log = vi.spyOn(console, "log").mockImplementation(() => undefined); + const payload = validPayload(); + // Nothing is authenticated yet, so no event name is logged, even one of a valid shape. + const named = { + ...payload, + events: [...payload.events, { ...payload.events[0], name: "patient.192.0.2.10" }], + }; const response = await worker.fetch!( incoming( new Request("https://events.scientfactory.com/v1/events", { method: "POST", headers: { "Content-Type": "application/json" }, - body: JSON.stringify(validPayload()), + body: JSON.stringify(named), }), ), gatewayEnv(database.database), @@ -328,7 +348,69 @@ describe("event gateway routes", () => { ); expect(response.status).toBe(400); + expect(await response.json()).toEqual({ + error: "Invalid installation token", + reason: "installation-token-missing", + stage: "headers", + }); expect(database.batch).not.toHaveBeenCalled(); + const entry = String(log.mock.lastCall?.[0]); + expect(JSON.parse(entry)).toMatchObject({ + status: 400, + error_class: "validation", + reason: "installation-token-missing", + stage: "headers", + schema_version: 1, + source: "desktop", + batch_size: 2, + app_version: "0.0.32", + build_channel: "development", + }); + for (const secret of [ + "installation:", + "session:", + "8e0ee7d5", + "codex", + "gpt-5.6", + "192.0.2", + "provider.turn.sent", + ]) { + expect(entry).not.toContain(secret); + } + }); + + it("names the failed storage step without logging the error message", async () => { + const database = createDatabase(); + database.batch.mockRejectedValueOnce(new Error("D1_ERROR: Network connection lost.")); + const failure = vi.spyOn(console, "error").mockImplementation(() => undefined); + vi.spyOn(console, "log").mockImplementation(() => undefined); + const response = await worker.fetch!( + incoming( + new Request("https://events.scientfactory.com/v1/events", { + method: "POST", + headers: { + "Content-Type": "application/json", + "X-Scient-Installation-Token": INSTALLATION_TOKEN, + }, + body: JSON.stringify(validPayload()), + }), + ), + gatewayEnv(database.database), + { waitUntil: vi.fn() } as unknown as ExecutionContext, + ); + + expect(response.status).toBe(500); + expect(await response.json()).toEqual({ + error: "Analytics ingestion failed", + }); + const entry = String(failure.mock.lastCall?.[0]); + expect(JSON.parse(entry)).toEqual({ + message: "Analytics ingestion failed", + failed_step: "storage", + error_class: "internal", + error_name: "Error", + error_kind: "network-lost", + }); }); it("acknowledges deletion when the installation has never uploaded data", async () => { diff --git a/workers/events/src/index.ts b/workers/events/src/index.ts index 926bdd1..2916b5d 100644 --- a/workers/events/src/index.ts +++ b/workers/events/src/index.ts @@ -104,9 +104,72 @@ interface PendingDeletionRow { readonly posthog_submitted_at: string | null; } -class RequestValidationError extends Error {} +// Closed rejection reasons. They are logged and returned with every 400 so a +// rejection is diagnosable without property values, identifiers or IP addresses. +const REJECTION_STAGES = { + "installation-token-missing": "headers", + "installation-token-invalid": "headers", + "body-too-large": "size", + "body-invalid-json": "body", + "payload-unsupported": "batch", + "batch-empty": "batch", + "batch-too-large": "size", + "batch-mixed-installations": "batch", + "event-not-object": "event", + "event-id-invalid": "event", + "event-name-invalid": "event", + "privacy-level-invalid": "event", + "consent-level-invalid": "event", + "occurred-at-invalid": "event", + "occurred-at-future": "event", + "occurred-at-too-old": "event", + "distinct-id-invalid": "event", + "installation-identity-required": "event", + "session-id-invalid": "event", + "properties-invalid": "event", + "property-key-invalid": "event", + "property-value-invalid": "event", + "contract-violation": "event", + "consent-below-privacy": "privacy", + "property-content-key": "privacy", + "property-private-value": "privacy", + "properties-too-many": "size", + "properties-too-large": "size", + "request-invalid": "request", +} as const; +type RejectionReason = keyof typeof REJECTION_STAGES; + +class RequestValidationError extends Error { + eventIndex?: number; + constructor( + message: string, + readonly reason: RejectionReason = "request-invalid", + ) { + super(message); + } +} class InstallationAuthenticationError extends Error {} +/** Maps the generated contract guard's messages onto the closed reasons. */ +const CONTRACT_VIOLATION_REASONS: ReadonlyArray = [ + ["Invalid event name", "event-name-invalid"], + ["Invalid privacy level", "privacy-level-invalid"], + ["Invalid consent level", "consent-level-invalid"], + ["Consent level", "consent-below-privacy"], + ["Too many event properties", "properties-too-many"], + ["Invalid event property name", "property-key-invalid"], + ["Invalid event property", "property-value-invalid"], + ["Private event property", "property-content-key"], + ["Private value in event property", "property-private-value"], + ["Event properties are too large", "properties-too-large"], +]; +function contractViolationReason(message: string): RejectionReason { + return ( + CONTRACT_VIOLATION_REASONS.find(([prefix]) => message.startsWith(prefix))?.[1] ?? + "contract-violation" + ); +} + function jsonResponse(body: unknown, status = 200, origin?: string | null): Response { const headers = new Headers({ "Cache-Control": "no-store", @@ -123,38 +186,55 @@ function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } -function requireString(value: unknown, field: string, pattern: RegExp): string { +function requireString( + value: unknown, + field: string, + pattern: RegExp, + reason?: RejectionReason, +): string { if (typeof value !== "string" || !pattern.test(value)) { - throw new RequestValidationError(`Invalid ${field}`); + throw new RequestValidationError(`Invalid ${field}`, reason); } return value; } function parseEvent(value: unknown, identityEnabled = false): AcceptedEvent { - if (!isRecord(value)) throw new RequestValidationError("Each event must be an object"); + if (!isRecord(value)) + throw new RequestValidationError("Each event must be an object", "event-not-object"); const properties = value.properties; - if (!isRecord(properties)) throw new RequestValidationError("Invalid event properties"); + if (!isRecord(properties)) + throw new RequestValidationError("Invalid event properties", "properties-invalid"); if (Object.keys(properties).length > 64) { - throw new RequestValidationError("Too many event properties"); + throw new RequestValidationError("Too many event properties", "properties-too-many"); } if (new TextEncoder().encode(JSON.stringify(properties)).byteLength > MAX_PROPERTIES_BYTES) { - throw new RequestValidationError("Event properties are too large"); + throw new RequestValidationError("Event properties are too large", "properties-too-large"); } - const privacyLevel = requireString(value.privacy_level, "privacy_level", /^[a-z]+$/); + const privacyLevel = requireString( + value.privacy_level, + "privacy_level", + /^[a-z]+$/, + "privacy-level-invalid", + ); if (!PRIVACY_LEVELS.includes(privacyLevel as PrivacyLevel)) { - throw new RequestValidationError("Invalid privacy_level"); + throw new RequestValidationError("Invalid privacy_level", "privacy-level-invalid"); } - const occurredAtInput = requireString(value.occurred_at, "occurred_at", /^.{10,40}$/); + const occurredAtInput = requireString( + value.occurred_at, + "occurred_at", + /^.{10,40}$/, + "occurred-at-invalid", + ); const occurredAtDate = new Date(occurredAtInput); if (Number.isNaN(occurredAtDate.valueOf())) { - throw new RequestValidationError("Invalid occurred_at"); + throw new RequestValidationError("Invalid occurred_at", "occurred-at-invalid"); } const now = Date.now(); if (occurredAtDate.valueOf() > now + 24 * 60 * 60 * 1000) { - throw new RequestValidationError("occurred_at is too far in the future"); + throw new RequestValidationError("occurred_at is too far in the future", "occurred-at-future"); } const earliestAllowed = identityEnabled ? diagnosticObservation(String(value.name), privacyLevel, properties) @@ -164,30 +244,54 @@ function parseEvent(value: unknown, identityEnabled = false): AcceptedEvent { (privacyLevel === "diagnostic" ? DIAGNOSTIC_EVENT_RETENTION_DAYS : RAW_EVENT_RETENTION_DAYS) * 86400000; if (occurredAtDate.valueOf() < earliestAllowed) { - throw new RequestValidationError("occurred_at is too old"); + throw new RequestValidationError("occurred_at is too old", "occurred-at-too-old"); } - const distinctId = requireString(value.distinct_id, "distinct_id", IDENTIFIER_PATTERN); + const distinctId = requireString( + value.distinct_id, + "distinct_id", + IDENTIFIER_PATTERN, + "distinct-id-invalid", + ); if (!INSTALLATION_ID_PATTERN.test(distinctId)) { - throw new RequestValidationError("Desktop events require an installation identity"); + throw new RequestValidationError( + "Desktop events require an installation identity", + "installation-identity-required", + ); } - const sessionId = requireString(value.session_id, "session_id", SESSION_ID_PATTERN); - const consentLevel = requireString(value.consent_level, "consent_level", /^[a-z]+$/); + const sessionId = requireString( + value.session_id, + "session_id", + SESSION_ID_PATTERN, + "session-id-invalid", + ); + const consentLevel = requireString( + value.consent_level, + "consent_level", + /^[a-z]+$/, + "consent-level-invalid", + ); if (!PRIVACY_LEVELS.includes(consentLevel as PrivacyLevel)) { - throw new RequestValidationError("Invalid consent_level"); + throw new RequestValidationError("Invalid consent_level", "consent-level-invalid"); } - const name = requireString(value.name, "event name", /^[a-z][a-z0-9_.-]{0,79}$/); + const name = requireString( + value.name, + "event name", + /^[a-z][a-z0-9_.-]{0,79}$/, + "event-name-invalid", + ); const contractViolation = eventContractViolation({ name, privacyLevel: privacyLevel as PrivacyLevel, consentLevel: consentLevel as PrivacyLevel, properties, }); - if (contractViolation) throw new RequestValidationError(contractViolation); + if (contractViolation) + throw new RequestValidationError(contractViolation, contractViolationReason(contractViolation)); return { - id: requireString(value.id, "event id", IDENTIFIER_PATTERN), + id: requireString(value.id, "event id", IDENTIFIER_PATTERN, "event-id-invalid"), name, distinctId, identityType: "desktop_installation", @@ -204,27 +308,45 @@ export function validateIngestionPayload( identityEnabled = false, ): ReadonlyArray { if (!isRecord(value) || value.schema_version !== 1 || value.source !== "desktop") { - throw new RequestValidationError("Unsupported event payload"); + throw new RequestValidationError("Unsupported event payload", "payload-unsupported"); } if (!Array.isArray(value.events) || value.events.length < 1) { - throw new RequestValidationError("At least one event is required"); + throw new RequestValidationError("At least one event is required", "batch-empty"); } if (value.events.length > MAX_BATCH_SIZE) { - throw new RequestValidationError(`At most ${MAX_BATCH_SIZE} events are accepted`); + throw new RequestValidationError( + `At most ${MAX_BATCH_SIZE} events are accepted`, + "batch-too-large", + ); } - const events = value.events.map((event) => parseEvent(event, identityEnabled)); + const events = value.events.map((event, index) => { + try { + return parseEvent(event, identityEnabled); + } catch (error) { + if (error instanceof RequestValidationError) error.eventIndex = index; + throw error; + } + }); const installationId = events[0]?.distinctId; if (events.some((event) => event.distinctId !== installationId)) { - throw new RequestValidationError("A batch must contain one installation identity"); + throw new RequestValidationError( + "A batch must contain one installation identity", + "batch-mixed-installations", + ); } return events; } function requireInstallationToken(request: Request): string { + const token = request.headers.get(INSTALLATION_TOKEN_HEADER); + if (token === null) { + throw new RequestValidationError("Invalid installation token", "installation-token-missing"); + } return requireString( - request.headers.get(INSTALLATION_TOKEN_HEADER), + token, "installation token", INSTALLATION_TOKEN_PATTERN, + "installation-token-invalid", ); } @@ -237,12 +359,96 @@ async function readJsonBody(request: Request): Promise { try { return await readBoundedJson(request, MAX_REQUEST_BYTES); } catch (error) { - throw new RequestValidationError( - error instanceof TransportFailure && error.kind === "body-too-large" - ? "Request body is too large" - : "Request body must be valid JSON", - ); + throw error instanceof TransportFailure && error.kind === "body-too-large" + ? new RequestValidationError("Request body is too large", "body-too-large") + : new RequestValidationError("Request body must be valid JSON", "body-invalid-json"); + } +} + +const APP_VERSION_PATTERN = + /^(?:\d{1,4}\.\d{1,4}\.\d{1,4}(?:-[a-z]{1,16}(?:\.\d{1,14}){0,3})?|unknown)$/; +const BUILD_CHANNEL_PATTERN = /^(?:stable|beta|nightly|development|unknown)$/; +const CONTRACT_REVISION_PATTERN = /^\d{1,4}$/; + +/** + * A privacy-safe description of a submitted batch for rejection and failure logs: + * its size and the client's release fields when they match their closed formats. + * Never event names, property keys or values, identifiers, tokens or IP addresses. + */ +function batchDiagnostics(value: unknown, request: Request): Record { + const userAgent = request.headers.get("User-Agent") ?? ""; + const diagnostics: Record = { + user_agent_family: !userAgent + ? "none" + : /^node\b/i.test(userAgent) + ? "node" + : /\bElectron\//.test(userAgent) + ? "electron" + : /^Mozilla\//.test(userAgent) + ? "browser" + : /^curl\//i.test(userAgent) + ? "curl" + : "other", + }; + if (!isRecord(value)) return diagnostics; + if (typeof value.schema_version === "number") diagnostics.schema_version = value.schema_version; + if (typeof value.source === "string" && /^[a-z]{1,16}$/.test(value.source)) + diagnostics.source = value.source; + if (!Array.isArray(value.events)) return diagnostics; + diagnostics.batch_size = value.events.length; + // Nothing is authenticated when a batch is rejected, so no name or key the client chose is + // logged: an event name of a valid shape can still carry a name or an address. + for (const event of value.events.slice(0, MAX_BATCH_SIZE)) { + const properties = isRecord(event) && isRecord(event.properties) ? event.properties : {}; + for (const [field, logged, pattern] of [ + ["appVersion", "app_version", APP_VERSION_PATTERN], + ["buildChannel", "build_channel", BUILD_CHANNEL_PATTERN], + ["contractRevision", "contract_revision", CONTRACT_REVISION_PATTERN], + ] as const) { + const fieldValue = properties[field]; + if (!(logged in diagnostics) && typeof fieldValue === "string" && pattern.test(fieldValue)) + diagnostics[logged] = fieldValue; + } } + return diagnostics; +} + +/** The offending event's position in the batch, never its name or contents. */ +function rejectionDiagnostics(error: RequestValidationError): Record { + const diagnostics: Record = {}; + if (error.eventIndex === undefined) return diagnostics; + diagnostics.rejected_event_index = error.eventIndex; + return diagnostics; +} + +/** Error class, runtime name and a closed storage kind; never the message, which may carry SQL or data. */ +function errorDiagnostics(error: unknown): { + readonly error_class: string; + readonly error_name: string; + readonly error_kind?: string; +} { + const name = error instanceof Error ? error.name : typeof error; + const message = error instanceof Error ? error.message : ""; + const kind = /network connection lost/i.test(message) + ? "network-lost" + : /timed? ?out/i.test(message) + ? "timeout" + : /overloaded/i.test(message) + ? "overloaded" + : /SQLITE_BUSY|database is locked/i.test(message) + ? "busy" + : /constraint/i.test(message) + ? "constraint" + : /no such (?:table|column)/i.test(message) + ? "schema" + : /^D1_/.test(message) + ? "d1-other" + : null; + return { + error_class: error instanceof TransportFailure ? error.kind : "internal", + error_name: /^[A-Za-z][A-Za-z0-9_]{0,63}$/.test(name) ? name : "unknown", + ...(kind ? { error_kind: kind } : {}), + }; } async function persistEvents( @@ -987,7 +1193,7 @@ async function handleIdentityLink( console.error( JSON.stringify({ message: "Identity linking failed", - error_class: error instanceof TransportFailure ? error.kind : "internal", + ...errorDiagnostics(error), }), ); return jsonResponse({ error: "Identity linking failed" }, 500); @@ -1136,7 +1342,12 @@ async function handleInstallationDeletion(request: Request, env: AnalyticsEnv): if (error instanceof RequestValidationError) { return jsonResponse({ error: error.message }, 400); } - console.error(JSON.stringify({ message: "Analytics deletion request failed" })); + console.error( + JSON.stringify({ + message: "Analytics deletion request failed", + ...errorDiagnostics(error), + }), + ); return jsonResponse({ error: "Analytics deletion request failed" }, 500); } } @@ -1164,13 +1375,21 @@ async function handleIngestion(request: Request, env: AnalyticsEnv): Promise { + let body: unknown; + // The ingestion step in progress, so an internal failure names where it happened. + let step: "body" | "token" | "validation" | "rate-limit" | "storage" = "body"; + const respond = ( + response: Response, + errorClass?: string, + diagnostics: Record = {}, + ) => { console.log( JSON.stringify({ message: "Analytics ingestion", status: response.status, events: eventCount, ...(errorClass ? { error_class: errorClass } : {}), + ...diagnostics, ...Object.fromEntries(Object.entries(stages).map(([name, ms]) => [`${name}_ms`, ms])), total_ms: Date.now() - started, }), @@ -1179,28 +1398,34 @@ async function handleIngestion(request: Request, env: AnalyticsEnv): Promise