diff --git a/.changeset/tangy-colts-spend.md b/.changeset/tangy-colts-spend.md new file mode 100644 index 000000000..a845151cc --- /dev/null +++ b/.changeset/tangy-colts-spend.md @@ -0,0 +1,2 @@ +--- +--- diff --git a/packages/cli-core/src/lib/config.ts b/packages/cli-core/src/lib/config.ts index a7e1f085b..67450ea36 100644 --- a/packages/cli-core/src/lib/config.ts +++ b/packages/cli-core/src/lib/config.ts @@ -223,6 +223,12 @@ export async function ensureMachineUuid(): Promise { return config.machineUuid; } +/** The machine id if this machine has one; never creates it (a preview on an opted-out machine must not). */ +export async function peekMachineUuid(): Promise { + const config = await readConfig(); + return config.machineUuid ?? null; +} + /** Read-only peek at the notice flag (the User-Agent gating needs it). */ export async function getTelemetryNoticeShown(): Promise { const config = await readConfig(); diff --git a/packages/cli-core/src/lib/fetch.ts b/packages/cli-core/src/lib/fetch.ts index 1d73ea9ff..f879acff2 100644 --- a/packages/cli-core/src/lib/fetch.ts +++ b/packages/cli-core/src/lib/fetch.ts @@ -29,7 +29,7 @@ function resolveUserAgent(): Promise { // The AIAgent segment exists purely for analytics classification, so it // honors the telemetry opt-outs (env vars and `clerk telemetry disable`) // and, like the events, stays absent until the disclosure notice has been - // shown — CI exempt, matching maybeShowTelemetryNotice. + // shown — CI exempt, matching showTelemetryNoticeOnce. userAgentPromise ??= (async () => { const beforeDisclosure = !process.env.CI && !(await getTelemetryNoticeShown().catch(() => false)); diff --git a/packages/cli-core/src/lib/telemetry.test.ts b/packages/cli-core/src/lib/telemetry.test.ts index 66def2591..a985a1288 100644 --- a/packages/cli-core/src/lib/telemetry.test.ts +++ b/packages/cli-core/src/lib/telemetry.test.ts @@ -1,13 +1,16 @@ -import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test"; import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { _setConfigDir, markTelemetryNoticeShown, setTelemetryDisabled } from "./config.ts"; +import * as config from "./config.ts"; import { declareSoftExitError, declareSoftExitOutcome, finalizeAndSendTelemetry, getTelemetryStatus, + sendTelemetryEvent, + showTelemetryNoticeOnce, setTelemetryPauseStep, setTelemetryDomainComponents, setTelemetryOAuthComplete, @@ -837,3 +840,135 @@ describe("finalizeAndSendTelemetry", () => { }); }); }); + +// The shared sender enforces the opt-out itself, so a caller that forgets the +// check cannot send from an opted-out machine. The per-run flush keeps its own +// earlier check (it must also skip the notice) and ignores this result. +describe("sendTelemetryEvent", () => { + const originalFetch = globalThis.fetch; + let calls = 0; + let status = 200; + + beforeEach(() => { + delete process.env.CLERK_TELEMETRY_URL; + delete process.env.CLERK_TELEMETRY_DISABLED; + delete process.env.DO_NOT_TRACK; + process.env.CLERK_TELEMETRY_URL = "https://capture.invalid/v1/event"; + calls = 0; + status = 200; + globalThis.fetch = (async () => { + calls += 1; + return new Response("{}", { status }); + }) as unknown as typeof fetch; + }); + afterEach(() => { + globalThis.fetch = originalFetch; + delete process.env.CLERK_TELEMETRY_URL; + delete process.env.CLERK_TELEMETRY_DISABLED; + delete process.env.DO_NOT_TRACK; + }); + + const send = async () => + sendTelemetryEvent({ event: "X", payload: {} }, { signal: new AbortController().signal }); + + test("accepted on a 2xx, rejected on a 5xx", async () => { + expect(await send()).toBe("accepted"); + status = 500; + expect(await send()).toBe("rejected"); + expect(calls).toBe(2); + }); + + // A 4xx is the endpoint refusing the event itself, which a caller must be + // able to tell from a server failure: the refused event was not stored. + test.each([[400], [404], [499]])("refused on a %i", async (code) => { + status = code; + expect(await send()).toBe("refused"); + }); + + test.each([[399], [500]])("not refused on a %i", async (code) => { + status = code; + expect(await send()).toBe("rejected"); + }); + + test.each([ + ["DO_NOT_TRACK", async () => void (process.env.DO_NOT_TRACK = "1")], + ["CLERK_TELEMETRY_DISABLED", async () => void (process.env.CLERK_TELEMETRY_DISABLED = "1")], + ["clerk telemetry disable", async () => setTelemetryDisabled(true)], + ])("disabled by %s: no request at all", async (_label, optOut) => { + await optOut(); + expect(await send()).toBe("disabled"); + expect(calls).toBe(0); + }); +}); + +// The exported sender always reads the opt-out itself: no caller can hand it +// a status and skip the check. The per-run flush avoids a second read by +// calling the module-internal post step after its own check. +describe("sendTelemetryEvent always reads the opt-out itself", () => { + const originalFetch = globalThis.fetch; + beforeEach(() => { + delete process.env.CLERK_TELEMETRY_DISABLED; + delete process.env.DO_NOT_TRACK; + process.env.CLERK_TELEMETRY_URL = "https://capture.invalid/v1/event"; + globalThis.fetch = (async () => new Response("{}")) as unknown as typeof fetch; + }); + afterEach(() => { + globalThis.fetch = originalFetch; + delete process.env.CLERK_TELEMETRY_URL; + }); + + const signal = () => new AbortController().signal; + + test("reads once per call, whatever extra options a caller passes", async () => { + await sendTelemetryEvent({ event: "X", payload: {} }, { signal: signal() }); // warms the User-Agent cache + const reads = spyOn(config, "getTelemetryDisabled"); + try { + const smuggled = { signal: signal(), telemetryStatus: { enabled: true } } as unknown as { + signal: AbortSignal; + }; + expect(await sendTelemetryEvent({ event: "X", payload: {} }, smuggled)).toBe("accepted"); + expect(reads).toHaveBeenCalledTimes(1); + } finally { + reads.mockRestore(); + } + }); + + test("the per-run flush reads the opt-out once", async () => { + await markTelemetryNoticeShown(); + await sendTelemetryEvent({ event: "X", payload: {} }, { signal: signal() }); // warms the User-Agent cache + const reads = spyOn(config, "getTelemetryDisabled"); + try { + startCommandTelemetry(fakeTelemetryCommand("list")); + await finalizeAndSendTelemetry({ outcome: "success", exitCode: 0 }); + expect(reads).toHaveBeenCalledTimes(1); + } finally { + reads.mockRestore(); + } + }); +}); + +// The end-of-run flush shows the notice through showTelemetryNoticeOnce on +// every command, so it must cost one config read: the CI check, then the one +// read-and-mark. +describe("showTelemetryNoticeOnce reads the config once", () => { + const originalCi = process.env.CI; + afterEach(() => { + if (originalCi === undefined) delete process.env.CI; + else process.env.CI = originalCi; + }); + + test.each([ + ["a machine that has shown the notice", true], + ["a fresh machine", false], + ])("%s: no separate read of the notice flag", async (_label, alreadyShown) => { + delete process.env.CI; + if (alreadyShown) await markTelemetryNoticeShown(); + const peek = spyOn(config, "getTelemetryNoticeShown"); + try { + await showTelemetryNoticeOnce(); + expect(peek).not.toHaveBeenCalled(); + } finally { + peek.mockRestore(); + } + }); +}); diff --git a/packages/cli-core/src/lib/telemetry.ts b/packages/cli-core/src/lib/telemetry.ts index a8681571e..98f0bbf4d 100644 --- a/packages/cli-core/src/lib/telemetry.ts +++ b/packages/cli-core/src/lib/telemetry.ts @@ -17,6 +17,7 @@ import { ensureMachineUuid, getTelemetryDisabled, markTelemetryNoticeShown, + peekMachineUuid, resolveProfile, } from "./config.ts"; import { @@ -33,7 +34,7 @@ import { getCurrentEnvName } from "./environment.ts"; import { ApiError, CliError, EXIT_CODE, UserAbortError } from "./errors.ts"; import { loggedFetch } from "./fetch.ts"; import { log } from "./log.ts"; -import { getMode } from "../mode.ts"; +import { getMode, type Mode } from "../mode.ts"; import { CURRENT_VERSION, IS_DEV_BUILD } from "./version.ts"; /** @@ -420,72 +421,209 @@ async function buildAndSend( ): Promise { // Re-checked here (not just at start) so `clerk telemetry disable` itself // sees the freshly persisted opt-out and sends nothing. - if (!(await getTelemetryStatus()).enabled) return; + const telemetryStatus = await getTelemetryStatus(); + if (!telemetryStatus.enabled) return; // The notice tells the user "Nothing has been sent during this run" — honor it. - if (await maybeShowTelemetryNotice()) return; + if (await showTelemetryNoticeOnce()) return; + + const envelope = await buildTelemetryEnvelope(); + + // The internal post step, not the exported sender: the opt-out was read + // just above, and this whole job runs under a one-second deadline. + await postTelemetryEvent( + { + event: "CLI_COMMAND_EXECUTED", + payload: { + command: current.command, + flags: current.flags, + outcome: result.outcome, + exit_code: result.exitCode, + error_code: result.errorCode ?? null, + // `pause_step` and `components` are deploy's; null on other commands. + stage: current.stage, + pause_step: current.pauseStep, + components: current.components, + duration_ms: Date.now() - current.startedAt, + ...envelope, + }, + }, + { + signal, + // Only the shutdown flush reports the interrupt, so only it may outlive + // one. A normal end-of-command flush must stay interruptible: bypassing + // the signal there would let a Ctrl-C mid-POST record the run's success + // event. Being aborted is how it hands the run to the shutdown flush, + // which finds the context still in place and re-sends it as an abort. + ignoreInterrupt: outlivesInterrupt, + }, + ); +} - const machineUuid = await ensureMachineUuid(); - const resolved = await resolveProfile(process.cwd()).catch(() => undefined); +/** + * The fields that describe the machine and the session, carried by every + * event the CLI sends. Keys are warehouse contracts (see `env-signals.ts`). + */ +export type TelemetryEnvelope = { + machine_uuid: string; + install_method: string; + ai_agent: string; + terminal_program: string; + mode: Mode; + os: string; + arch: string; + ci: boolean; + in_tmux: boolean; + in_screen: boolean; + env: string; + workspace_id: string | null; + app_id: string | null; +}; - const event = { - sdk: "clerk-cli", - sdkv: CURRENT_VERSION, - event: "CLI_COMMAND_EXECUTED", - payload: { - command: current.command, - flags: current.flags, - outcome: result.outcome, - exit_code: result.exitCode, - error_code: result.errorCode ?? null, - // `pause_step` and `components` are deploy's; null on other commands. - stage: current.stage, - pause_step: current.pauseStep, - components: current.components, - duration_ms: Date.now() - current.startedAt, - machine_uuid: machineUuid, - install_method: detectInstallMethod(process.env, process.execPath), - ai_agent: detectAiAgent(process.env), - terminal_program: detectTerminalProgram(process.env), - mode: getMode(), - os: process.platform, - arch: process.arch, - ci: Boolean(process.env.CI), - in_tmux: detectInTmux(process.env), - in_screen: detectInScreen(process.env), - env: getCurrentEnvName(), - workspace_id: resolved?.profile.workspaceId ?? null, - app_id: resolved?.profile.appId ?? null, - }, +/** {@link TelemetryEnvelope} for a preview on a machine that may have no id yet. */ +export type TelemetryEnvelopePreview = Omit & { + machine_uuid: string | null; +}; + +/** + * Read the envelope fields. Creates the machine id on a machine that has + * none, so a caller that only previews an event still writes the config file; + * a preview that must not do that uses {@link peekTelemetryEnvelope}. + */ +export async function buildTelemetryEnvelope(): Promise { + return envelopeWith(await ensureMachineUuid()); +} + +/** + * The envelope with the machine id this machine already has; never creates + * one. The id is null when the machine has none, or when the config file + * cannot be read: this is for a preview, which must always be able to show + * something. + */ +export async function peekTelemetryEnvelope(): Promise { + return envelopeWith(await peekMachineUuid().catch(() => null)); +} + +async function envelopeWith( + machineUuid: Id, +): Promise & { machine_uuid: Id }> { + const resolved = await resolveProfile(process.cwd()).catch(() => undefined); + return { + machine_uuid: machineUuid, + install_method: detectInstallMethod(process.env, process.execPath), + ai_agent: detectAiAgent(process.env), + terminal_program: detectTerminalProgram(process.env), + mode: getMode(), + os: process.platform, + arch: process.arch, + ci: Boolean(process.env.CI), + in_tmux: detectInTmux(process.env), + in_screen: detectInScreen(process.env), + env: getCurrentEnvName(), + workspace_id: resolved?.profile.workspaceId ?? null, + app_id: resolved?.profile.appId ?? null, }; +} + +export type TelemetryEvent = { + event: string; + payload: Record; +}; + +/** An event as it is posted: wrapped with the sender name and the CLI version. */ +export type PostedTelemetryEvent = { + sdk: "clerk-cli"; + sdkv: string; +} & E; + +/** Wrap an event exactly as {@link sendTelemetryEvent} posts it, so a preview can show the same object. */ +export function asPostedTelemetryEvent( + event: E, +): PostedTelemetryEvent { + return { sdk: "clerk-cli", sdkv: CURRENT_VERSION, ...event }; +} + +/** + * `accepted`: the endpoint answered 2xx. `refused`: it answered 4xx, refusing + * the event itself, so the event was not stored. `rejected`: it answered + * anything else (a 5xx), which may have come after the event was stored. + * `disabled`: telemetry is off on this machine, so nothing was sent. + */ +export type SendOutcome = "accepted" | "refused" | "rejected" | "disabled"; - log.debug(`telemetry: event ${JSON.stringify(event)}`); +/** + * Wrap one event in `{ sdk, sdkv }`, print it at debug level (this is what + * `--verbose` shows, and the README promises it for every event), and POST it + * to the telemetry endpoint. Resolves to a {@link SendOutcome} + * (`loggedFetch` returns 4xx and 5xx without throwing). Rejects on a network + * failure or an abort: the caller decides what that means, since the + * command-event flush keeps its context for the shutdown flush while the + * feedback command reports the failure to the user. + * + * The opt-out is enforced here, at the one place every outside caller passes, + * so a caller that forgets to check it cannot send from an opted-out machine + * (the README promises that nothing is), and nothing a caller passes can skip + * the check. The one exception is inside this module: the per-run flush reads + * the status itself (it must, to decide about the first-run notice) and then + * calls {@link postTelemetryEvent} directly, so its one-second job pays no + * second config read. + * + * Shared by the per-run `CLI_COMMAND_EXECUTED` flush and by `clerk feedback`, + * whose `CLI_FEEDBACK` event is a second, differently named event sent by the + * command's own action. It never goes through `finalizeAndSendTelemetry`: the + * `finalized`/`context` latch there keeps a run to one command event, and the + * feedback event must not consume or be consumed by it. + */ +export async function sendTelemetryEvent( + event: TelemetryEvent, + options: { signal: AbortSignal; ignoreInterrupt?: boolean; onPost?: () => void }, +): Promise { + if (!(await getTelemetryStatus()).enabled) return "disabled"; + // Tells the caller the opt-out check passed and the request is being made, + // which a rejection alone cannot: `clerk feedback` prints the report id + // only when a POST was attempted. + options.onPost?.(); + return postTelemetryEvent(event, options); +} + +/** The POST itself, with no opt-out check: only for a caller in this module that has just made one. */ +async function postTelemetryEvent( + event: TelemetryEvent, + options: { signal: AbortSignal; ignoreInterrupt?: boolean }, +): Promise> { + const wrapped = asPostedTelemetryEvent(event); + log.debug(`telemetry: event ${JSON.stringify(wrapped)}`); const url = process.env.CLERK_TELEMETRY_URL ?? DEFAULT_TELEMETRY_ENDPOINT; - await loggedFetch(url, { + const response = await loggedFetch(url, { tag: "telemetry", method: "POST", headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ events: [event] }), - signal, + body: JSON.stringify({ events: [wrapped] }), + signal: options.signal, bestEffort: true, - // Only the shutdown flush reports the interrupt, so only it may outlive - // one. A normal end-of-command flush must stay interruptible: bypassing - // the signal there would let a Ctrl-C mid-POST record the run's success - // event. Being aborted is how it hands the run to the shutdown flush, - // which finds the context still in place and re-sends it as an abort. - ignoreInterrupt: outlivesInterrupt, + ignoreInterrupt: options.ignoreInterrupt ?? false, }); + if (response.ok) return "accepted"; + return response.status >= 400 && response.status < 500 ? "refused" : "rejected"; } +const TELEMETRY_NOTICE_LAST_LINE = "Nothing has been sent during this run."; + /** * One-time stderr disclosure for humans and agents alike. Returns true when - * the notice was just shown — that run sends nothing, so disclosure always - * precedes a machine's first event. CI is exempt from both the notice and - * the grace run: ephemeral CI machines are always on their "first run", so - * a grace there would mean CI never sends at all. + * the notice was just shown — on the command-event path that run sends + * nothing, so disclosure always precedes a machine's first event. CI is exempt + * from both the notice and the grace run: ephemeral CI machines are always on + * their "first run", so a grace there would mean CI never sends at all. + * + * `clerk feedback` is the one caller that sends on the run that shows the + * notice, because that send is explicit; it passes its own `lastLine` saying + * what the run sends, so the notice never claims a silence that is not true. */ -async function maybeShowTelemetryNotice(): Promise { +export async function showTelemetryNoticeOnce( + lastLine: string = TELEMETRY_NOTICE_LAST_LINE, +): Promise { if (process.env.CI) return false; if (!(await markTelemetryNoticeShown())) return false; log.blank(); @@ -496,7 +634,7 @@ async function maybeShowTelemetryNotice(): Promise { "duration, outcome, the step a multi-step command reached, a random machine identifier —", ); log.info("and your workspace and app IDs when a project is linked."); - log.info("Nothing has been sent during this run."); + log.info(lastLine); log.info("Opt out: `clerk telemetry disable` — details: https://clerk.com/docs/telemetry"); log.blank(); return true; diff --git a/packages/cli-core/src/test/lib/stubs.ts b/packages/cli-core/src/test/lib/stubs.ts index dd3fa6b4f..39943579f 100644 --- a/packages/cli-core/src/test/lib/stubs.ts +++ b/packages/cli-core/src/test/lib/stubs.ts @@ -185,6 +185,7 @@ export const configStubs = { profileLabel: (profile: { appName?: string; appId: string }) => profile.appName ? `${profile.appName} (${profile.appId})` : profile.appId, ensureMachineUuid: async () => "00000000-0000-4000-8000-000000000000", + peekMachineUuid: async () => "00000000-0000-4000-8000-000000000000", markTelemetryNoticeShown: async () => false, getTelemetryNoticeShown: async () => true, getTelemetryDisabled: async () => false,