From 406d5a156556631ad7d03a3372febf67a4498770 Mon Sep 17 00:00:00 2001 From: betegon Date: Fri, 30 Oct 2026 14:20:25 +0100 Subject: [PATCH] feat(cli): tag CI invocations in telemetry --- packages/cli/DEVELOPMENT.md | 33 +++++++++++ packages/cli/src/commands/build/upload.ts | 2 +- packages/cli/src/commands/snapshots/upload.ts | 2 +- packages/cli/src/lib/build/vcs.ts | 33 ----------- packages/cli/src/lib/ci.ts | 33 +++++++++++ packages/cli/src/lib/telemetry.ts | 6 +- packages/cli/test/lib/build/vcs.test.ts | 15 ----- packages/cli/test/lib/ci.test.ts | 23 ++++++++ packages/cli/test/lib/telemetry-ci.test.ts | 58 +++++++++++++++++++ 9 files changed, 154 insertions(+), 51 deletions(-) create mode 100644 packages/cli/src/lib/ci.ts create mode 100644 packages/cli/test/lib/ci.test.ts create mode 100644 packages/cli/test/lib/telemetry-ci.test.ts diff --git a/packages/cli/DEVELOPMENT.md b/packages/cli/DEVELOPMENT.md index 12ffb6e2f..aec2f0dd4 100644 --- a/packages/cli/DEVELOPMENT.md +++ b/packages/cli/DEVELOPMENT.md @@ -157,3 +157,36 @@ When creating a bound `createSentryClient` instead, register Passing an existing client in its constructor config does not transfer interceptors. Keep the runtime SDK import outside `sentry-client.ts`, which is also loaded synchronously by telemetry's database integration. + +### Invocation telemetry + +When telemetry is enabled, `is_ci` records `true` or `false` for each invocation, +independently of the `agent` and `is_tty` tags. CI detection reuses the environment +variable detector used for VCS metadata collection, including `CI` and known +provider markers. Telemetry records the detection result without sending raw +environment variable values. The signals can overlap: an agent running in CI +retains its agent identity and also has `is_ci:true`. + +For command usage widgets, retain the existing base query: + +```text +!span.description:cli-execution !span.description:[sentry.auth.login,sentry.auth.logout,sentry.auth.refresh] span.op:cli.command +``` + +Keep the `count(span.duration)` aggregate and append the relevant condition: + +| Segment | Additional condition | +|---------|----------------------| +| Agent | `!agent:""` | +| CI | `agent:"" is_ci:true` | +| Human | `agent:"" is_ci:false` | +| Unknown | `agent:"" is_ci:""` | + +These segments give detected agents priority, including agents running in CI. +To inspect that subset, append `!agent:"" is_ci:true` to the base query. +The Human label remains an approximation: no detected CI or agent does not prove +human execution; local scripts and undetected agents can also fall into this +segment. Older telemetry without `is_ci` stays Unknown when no agent was detected. +Do not treat an absent `is_ci` tag as `false`. + +Update usage dashboards after releasing the CLI version that emits `is_ci`. diff --git a/packages/cli/src/commands/build/upload.ts b/packages/cli/src/commands/build/upload.ts index 20cea601b..9fb380a39 100644 --- a/packages/cli/src/commands/build/upload.ts +++ b/packages/cli/src/commands/build/upload.ts @@ -27,10 +27,10 @@ import { } from "../../lib/build/index.js"; import { collectVcsMetadata, - isCi, type VcsFlags, vcsInfoToBody, } from "../../lib/build/vcs.js"; +import { isCi } from "../../lib/ci.js"; import { buildCommand } from "../../lib/command.js"; import { ContextError, ValidationError } from "../../lib/errors.js"; import { diff --git a/packages/cli/src/commands/snapshots/upload.ts b/packages/cli/src/commands/snapshots/upload.ts index dbaf23390..c93c40be1 100644 --- a/packages/cli/src/commands/snapshots/upload.ts +++ b/packages/cli/src/commands/snapshots/upload.ts @@ -16,11 +16,11 @@ import { } from "../../lib/api/preprod-artifacts.js"; import { collectVcsMetadata, - isCi, type VcsFlags, type VcsInfo, vcsInfoToBody, } from "../../lib/build/vcs.js"; +import { isCi } from "../../lib/ci.js"; import { buildCommand } from "../../lib/command.js"; import { ContextError, ValidationError } from "../../lib/errors.js"; import { diff --git a/packages/cli/src/lib/build/vcs.ts b/packages/cli/src/lib/build/vcs.ts index 79cc36000..dcea44d87 100644 --- a/packages/cli/src/lib/build/vcs.ts +++ b/packages/cli/src/lib/build/vcs.ts @@ -59,39 +59,6 @@ export type VcsFlags = { "no-git-metadata"?: boolean; }; -/** Environment variables commonly set by CI providers. */ -const CI_ENV_VARS = [ - "CI", - "JENKINS_URL", - "HUDSON_URL", - "TEAMCITY_VERSION", - "CIRCLE_BUILD_URL", - "bamboo_resultsUrl", - "GITHUB_ACTIONS", - "GITLAB_CI", - "TRAVIS_JOB_ID", - "BITRISE_BUILD_URL", - "GO_SERVER_URL", - "TF_BUILD", - "BUILDKITE", -]; - -/** - * Detect whether we are running in a CI environment. - * - * @param env - Environment variables (defaults to `process.env`). - */ -export function isCi(env: NodeJS.ProcessEnv = process.env): boolean { - return CI_ENV_VARS.some((name) => { - const value = env[name]; - // Treat explicit opt-out values ("", "false", "0") as not-in-CI so a local - // `CI=false` does not silently enable git-metadata collection. - return ( - value !== undefined && value !== "" && value !== "false" && value !== "0" - ); - }); -} - /** A 40-character lowercase hex SHA-1. */ const SHA1_RE = /^[0-9a-f]{40}$/; diff --git a/packages/cli/src/lib/ci.ts b/packages/cli/src/lib/ci.ts new file mode 100644 index 000000000..f12d538e6 --- /dev/null +++ b/packages/cli/src/lib/ci.ts @@ -0,0 +1,33 @@ +/** Shared CI detection for command telemetry and VCS metadata collection. */ + +/** Environment variables commonly set by CI providers. */ +const CI_ENV_VARS = [ + "CI", + "JENKINS_URL", + "HUDSON_URL", + "TEAMCITY_VERSION", + "CIRCLE_BUILD_URL", + "bamboo_resultsUrl", + "GITHUB_ACTIONS", + "GITLAB_CI", + "TRAVIS_JOB_ID", + "BITRISE_BUILD_URL", + "GO_SERVER_URL", + "TF_BUILD", + "BUILDKITE", +]; + +/** + * Detect CI from generic or provider-specific environment markers. + * Empty strings, "false", and "0" are ignored for each marker. + * + * @param env - Environment variables (defaults to `process.env`). + */ +export function isCi(env: NodeJS.ProcessEnv = process.env): boolean { + return CI_ENV_VARS.some((name) => { + const value = env[name]; + return ( + value !== undefined && value !== "" && value !== "false" && value !== "0" + ); + }); +} diff --git a/packages/cli/src/lib/telemetry.ts b/packages/cli/src/lib/telemetry.ts index b3ddc694a..f4a878f7b 100644 --- a/packages/cli/src/lib/telemetry.ts +++ b/packages/cli/src/lib/telemetry.ts @@ -17,6 +17,7 @@ import * as Sentry from "@sentry/node"; const _require = createRequire(import.meta.url); import { isMusl } from "./binary.js"; +import { isCi } from "./ci.js"; import { CLI_VERSION, getCliEnvironment, @@ -757,7 +758,10 @@ export function initSentry( // Tag whether targeting self-hosted Sentry (not SaaS) Sentry.setTag("is_self_hosted", !isSentrySaasUrl(getSentryBaseUrl())); - // Tag whether running in an interactive terminal or agent/CI environment + // CI and agent identity are independent: an agent can run inside CI. + Sentry.setTag("is_ci", isCi(getEnv())); + + // Tag whether stdout is a terminal, independently of CI or agent detection. Sentry.setTag("is_tty", !!process.stdout.isTTY); // Tag the C library variant on Linux (musl vs glibc). diff --git a/packages/cli/test/lib/build/vcs.test.ts b/packages/cli/test/lib/build/vcs.test.ts index a2ce813fe..585654907 100644 --- a/packages/cli/test/lib/build/vcs.test.ts +++ b/packages/cli/test/lib/build/vcs.test.ts @@ -25,7 +25,6 @@ vi.mock("../../../src/lib/git.js", () => gitMock); import { ValidationError } from "../../../src/lib/errors.js"; import { collectVcsMetadata, - isCi, vcsInfoToBody, } from "../../../src/lib/build/vcs.js"; @@ -46,20 +45,6 @@ afterEach(() => { } }); -describe("isCi", () => { - test("true when a CI variable is set", () => { - expect(isCi({ GITHUB_ACTIONS: "true" })).toBe(true); - expect(isCi({ CI: "1" })).toBe(true); - }); - - test("false with no CI variables (and ignores opt-out values)", () => { - expect(isCi({})).toBe(false); - expect(isCi({ CI: "" })).toBe(false); - expect(isCi({ CI: "false" })).toBe(false); - expect(isCi({ CI: "0" })).toBe(false); - }); -}); - describe("vcsInfoToBody", () => { test("flattens to snake_case and renames provider", () => { expect( diff --git a/packages/cli/test/lib/ci.test.ts b/packages/cli/test/lib/ci.test.ts new file mode 100644 index 000000000..8d35dcd98 --- /dev/null +++ b/packages/cli/test/lib/ci.test.ts @@ -0,0 +1,23 @@ +/** Tests for the shared CI environment detector. */ + +import { describe, expect, test } from "vitest"; +import { isCi } from "../../src/lib/ci.js"; + +describe("isCi", () => { + test("true when a CI variable is set", () => { + expect(isCi({ GITHUB_ACTIONS: "true" })).toBe(true); + expect(isCi({ TF_BUILD: "True" })).toBe(true); + expect(isCi({ CI: "1" })).toBe(true); + }); + + test("false with no CI variables (and ignores opt-out values)", () => { + expect(isCi({})).toBe(false); + expect(isCi({ CI: "" })).toBe(false); + expect(isCi({ CI: "false" })).toBe(false); + expect(isCi({ CI: "0" })).toBe(false); + }); + + test("detects provider markers even when the generic marker is disabled", () => { + expect(isCi({ CI: "false", GITHUB_ACTIONS: "true" })).toBe(true); + }); +}); diff --git a/packages/cli/test/lib/telemetry-ci.test.ts b/packages/cli/test/lib/telemetry-ci.test.ts new file mode 100644 index 000000000..362bf272e --- /dev/null +++ b/packages/cli/test/lib/telemetry-ci.test.ts @@ -0,0 +1,58 @@ +/** Verify CI and agent attribution on outgoing command telemetry. */ + +import type { Transport } from "@sentry/core"; +import { captureMessage, startSpan, withIsolationScope } from "@sentry/node"; +import { afterEach, describe, expect, test, vi } from "vitest"; +import { withEnv } from "../../src/lib/env.js"; +// oxlint-disable-next-line sentry-cli/no-namespace-import -- intercept the outbound transport factory +import * as transportModule from "../../src/lib/telemetry/zstd-transport.js"; +import { initSentry } from "../../src/lib/telemetry.js"; + +let client: ReturnType; + +afterEach(async () => { + await client?.close(0); + vi.restoreAllMocks(); +}); + +describe("CI telemetry", () => { + test.each([ + { env: { CI: "true" }, expected: true }, + { env: { GITHUB_ACTIONS: "true" }, expected: true }, + { env: { CI: "false" }, expected: false }, + { env: {}, expected: false }, + ])( + "records is_ci=$expected for $env without losing agent identity", + async ({ env, expected }) => { + const send = vi.fn().mockResolvedValue({}); + vi.spyOn(transportModule, "makeCompressedTransport").mockReturnValue({ + send, + flush: async () => true, + }); + + await withEnv({ ...env, AI_AGENT: "codex" }, () => + withIsolationScope(async () => { + client = initSentry(true, { libraryMode: true }); + if (!client) { + throw new Error("Expected the telemetry client"); + } + + captureMessage("CI attribution"); + startSpan( + { name: "sentry.info", op: "cli.command", forceTransaction: true }, + () => {}, + ); + await client.flush(1000); + }), + ); + + const items = send.mock.calls.flatMap(([envelope]) => envelope[1]); + for (const type of ["event", "transaction"]) { + const event = items.find(([header]) => header.type === type)?.[1]; + expect(event).toMatchObject({ + tags: { is_ci: expected, agent: "codex" }, + }); + } + }, + ); +});