diff --git a/CHANGELOG.md b/CHANGELOG.md index afdf86ff0f..fadd96bdf9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,9 @@ This is the log of notable changes to EAS CLI and related packages. ### 🎉 New features +- [eas-cli] Add experimentation client. ([#4558](https://github.com/expo/eas-cli/pull/4558) by [@williamgrosset](https://github.com/williamgrosset)) +- [eas-cli] Add a concurrency limit message experiment to `eas build`. ([#4611](https://github.com/expo/eas-cli/pull/4611) by [@williamgrosset](https://github.com/williamgrosset)) + ### 🐛 Bug fixes ### 🧹 Chores diff --git a/packages/eas-cli/package.json b/packages/eas-cli/package.json index 0d0b09e12d..09f30d4402 100644 --- a/packages/eas-cli/package.json +++ b/packages/eas-cli/package.json @@ -51,6 +51,7 @@ "@expo/eas-build-job": "24.13.0", "@expo/eas-json": "24.13.0", "@expo/env": "^1.0.0", + "@expo/experimentation": "0.0.5", "@expo/json-file": "8.3.3", "@expo/logger": "24.13.0", "@expo/multipart-body-parser": "2.0.0", diff --git a/packages/eas-cli/src/__tests__/commands/utils-test.ts b/packages/eas-cli/src/__tests__/commands/utils-test.ts new file mode 100644 index 0000000000..53cdf4dc72 --- /dev/null +++ b/packages/eas-cli/src/__tests__/commands/utils-test.ts @@ -0,0 +1,26 @@ +import EasCommand from '../../commandUtils/EasCommand'; +import { mockCommandContext } from './utils'; + +jest.mock('fs'); + +class TestCommand extends EasCommand { + static override contextDefinition = { + ...this.ContextOptions.LoggedIn, + ...this.ContextOptions.Experimentation, + }; + + async runAsync(): Promise {} +} + +describe(mockCommandContext, () => { + it('provides a disabled experimentation client that returns defaults', () => { + const ctx = mockCommandContext(TestCommand, {}); + expect(ctx.experimentation.getUserNamespace('any').getParam('variant', 'control')).toBe( + 'control' + ); + expect( + ctx.experimentation.getAccountNamespace('any', { accountId: 'a' }).getParam('enabled', true) + ).toBe(true); + expect(ctx.experimentation.getDeviceNamespace('any').getParam('count', 3)).toBe(3); + }); +}); diff --git a/packages/eas-cli/src/__tests__/commands/utils.ts b/packages/eas-cli/src/__tests__/commands/utils.ts index f5fb7463a1..b6ad014e24 100644 --- a/packages/eas-cli/src/__tests__/commands/utils.ts +++ b/packages/eas-cli/src/__tests__/commands/utils.ts @@ -7,6 +7,8 @@ import { v4 as uuidv4 } from 'uuid'; import { ContextInput, ContextOutput } from '../../commandUtils/EasCommand'; import { DynamicConfigContextFn } from '../../commandUtils/context/DynamicProjectConfigContextField'; +import { ExperimentOverrides } from '../../experimentation/ExperimentOverrides'; +import { ExperimentationClient } from '../../experimentation/ExperimentationClient'; import { AppFragment, Role, @@ -97,6 +99,9 @@ export function mockCommandContext< if (contextKey === 'sessionManager') { result.sessionManager = {}; } + if (contextKey === 'experimentation') { + result.experimentation = ExperimentationClient.createDisabled(ExperimentOverrides.empty()); + } if (contextKey === 'projectConfig') { result.projectConfig = { exp: overrides.exp ?? getMockExpoConfig(), diff --git a/packages/eas-cli/src/analytics/AnalyticsManager.ts b/packages/eas-cli/src/analytics/AnalyticsManager.ts index dd7a1b0ecb..426012f358 100644 --- a/packages/eas-cli/src/analytics/AnalyticsManager.ts +++ b/packages/eas-cli/src/analytics/AnalyticsManager.ts @@ -21,12 +21,17 @@ export type AnalyticsEvent = | SubmissionEvent | MetadataEvent | BillingEvent - | SimulatorEvent; + | SimulatorEvent + | ExperimentationEvent; export enum CommandEvent { ACTION = 'action', // generic event type which is used to determine the 'daily active user' stat, include an `action: eas ${subcommand}` property inside of the event properties object } +export enum ExperimentationEvent { + EXPERIMENT_VIEWED = 'Experiment Viewed', +} + export enum BillingEvent { SUBSCRIBE_COMMAND = 'billing cli subscribe command', } @@ -76,6 +81,8 @@ export enum BuildEvent { CREDENTIALS_SYNC_UPDATE_REMOTE_FAIL = 'build cli credentials sync update remote fail', ANDROID_KEYSTORE_CREATE = 'build cli credentials keystore create', + + CONCURRENCY_LIMIT_MESSAGE_SHOWN = 'build cli concurrency limit message shown', } export enum SimulatorEvent { @@ -105,6 +112,8 @@ export interface Analytics { export interface AnalyticsWithOrchestration extends Analytics { setActor(actor: Actor): void; flushAsync(): Promise; + /** The persisted RudderStack anonymous ID, or null when analytics is disabled. */ + getDeviceId(): string | null; } const USER_SETTINGS_KEY_AMPLITUDE_ENABLED = 'amplitudeEnabled'; @@ -183,6 +192,9 @@ class NoOpAnalytics implements AnalyticsWithOrchestration { logEvent(): void {} setActor(): void {} async flushAsync(): Promise {} + getDeviceId(): null { + return null; + } } const RudderstackAnalyticsConfig = @@ -234,6 +246,10 @@ class RudderstackAnalytics implements AnalyticsWithOrchestration { this.identifiedActor = actor; } + public getDeviceId(): string { + return this.persistentDeviceId; + } + public logEvent(name: AnalyticsEvent, properties: AnalyticsEventProperties): void { const userId = this.identifiedActor?.id; const deviceId = this.persistentDeviceId; diff --git a/packages/eas-cli/src/analytics/__tests__/AnalyticsManager-test.ts b/packages/eas-cli/src/analytics/__tests__/AnalyticsManager-test.ts index 0847588f19..86744c08b4 100644 --- a/packages/eas-cli/src/analytics/__tests__/AnalyticsManager-test.ts +++ b/packages/eas-cli/src/analytics/__tests__/AnalyticsManager-test.ts @@ -140,3 +140,55 @@ describe(isAnalyticsOptedOutAsync, () => { await expect(isAnalyticsOptedOutAsync()).resolves.toBe(true); }); }); + +describe('getDeviceId', () => { + // Backs UserSettings with an in-memory store so a value written by createAnalyticsAsync is read back. + function useInMemoryUserSettings(initial: Record): void { + const stored: Record = { ...initial }; + (userSettingsMock.setAsync as jest.Mock).mockImplementation( + async (key: string, value: unknown) => { + stored[key] = value; + return {}; + } + ); + (userSettingsMock.getAsync as jest.Mock).mockImplementation( + async (key: string, defaultValue: unknown) => (key in stored ? stored[key] : defaultValue) + ); + } + + it('returns the persisted analytics device ID', async () => { + const analytics = await createAnalyticsAsync(); + expect(analytics.getDeviceId()).toBe('persistent-device-id'); + }); + + it('creates and persists a device ID when none is stored', async () => { + userSettingsMock.getAsync.mockImplementation(async (key, defaultValue) => + key === 'analyticsDeviceId' ? null : defaultValue + ); + const analytics = await createAnalyticsAsync(); + const deviceId = analytics.getDeviceId(); + expect(deviceId).toMatch(/^[0-9a-f-]{36}$/); + expect(userSettingsMock.setAsync).toHaveBeenCalledWith('analyticsDeviceId', deviceId); + }); + + it('is null when DISABLE_EAS_ANALYTICS is set', async () => { + useInMemoryUserSettings({ analyticsDeviceId: 'persistent-device-id' }); + process.env.DISABLE_EAS_ANALYTICS = '1'; + const analytics = await createAnalyticsAsync(); + expect(analytics.getDeviceId()).toBeNull(); + }); + + it('is null when running behind https_proxy', async () => { + process.env.https_proxy = 'http://proxy.local:8080'; + const analytics = await createAnalyticsAsync(); + expect(analytics.getDeviceId()).toBeNull(); + }); + + it('is null when analytics were turned off with eas analytics off', async () => { + userSettingsMock.getAsync.mockImplementation(async (key, defaultValue) => + key === 'analyticsEnabled' ? false : defaultValue + ); + const analytics = await createAnalyticsAsync(); + expect(analytics.getDeviceId()).toBeNull(); + }); +}); diff --git a/packages/eas-cli/src/build/__tests__/concurrencyLimitMessage-test.ts b/packages/eas-cli/src/build/__tests__/concurrencyLimitMessage-test.ts new file mode 100644 index 0000000000..824634fb02 --- /dev/null +++ b/packages/eas-cli/src/build/__tests__/concurrencyLimitMessage-test.ts @@ -0,0 +1,431 @@ +import type { ExperimentationDefinition } from '@expo/experimentation'; + +import { BuildEvent } from '../../analytics/AnalyticsManager'; +import { ExpoGraphqlClient } from '../../commandUtils/context/contextUtils/createGraphqlClient'; +import { ExperimentOverrides } from '../../experimentation/ExperimentOverrides'; +import { + EMPTY_EXPERIMENTATION_DEFINITION, + ExperimentationClient, + createLoggerConfig, +} from '../../experimentation/ExperimentationClient'; +import { createMockAnalytics, findUnitsByValue } from '../../experimentation/__tests__/fixtures'; +import { AccountConcurrencyUsage, AccountQuery } from '../../graphql/queries/AccountQuery'; +import Log from '../../log'; +import { + CONCURRENCY_LIMIT_EXPERIMENT_NAMESPACE, + ConcurrencyLimitMessage, + ConcurrencyLimitMessageOptions, + formatConcurrencyLimitMessage, +} from '../concurrencyLimitMessage'; + +jest.mock('../../graphql/queries/AccountQuery'); +jest.mock('../../log', () => ({ + __esModule: true, + ...jest.requireActual('../../log'), + link: jest.fn((url: string, opts?: { text?: string }) => opts?.text ?? url), +})); + +const graphqlClient = {} as unknown as ExpoGraphqlClient; + +function createClientWithVariant(variant: string): ExperimentationClient { + return ExperimentationClient.createDisabled( + new ExperimentOverrides( + new Map([[CONCURRENCY_LIMIT_EXPERIMENT_NAMESPACE, new Map([['variant', variant]])]]) + ) + ); +} + +function createUsage({ + planId = 'price_starter', + planName = 'Starter', + total = 2 as number | null, + activeCount = 2, + jobRunCount = 0, +}: { + planId?: string; + planName?: string; + total?: number | null; + activeCount?: number; + jobRunCount?: number; +} = {}): AccountConcurrencyUsage { + return { + __typename: 'Account', + id: 'account-id', + subscription: { + __typename: 'SubscriptionDetails', + id: 'subscription-id', + name: planName, + planId, + concurrencies: + total === null + ? null + : { __typename: 'Concurrencies', total, android: total, ios: total }, + }, + concurrencyConsumers: [ + ...Array.from({ length: activeCount }, (_, i) => ({ + __typename: 'Build' as const, + id: `build-${i}`, + })), + ...Array.from({ length: jobRunCount }, (_, i) => ({ + __typename: 'JobRun' as const, + id: `job-run-${i}`, + })), + ], + }; +} + +function createMessage(overrides: Partial = {}): { + message: ConcurrencyLimitMessage; + analytics: ReturnType; +} { + const analytics = createMockAnalytics(); + const message = new ConcurrencyLimitMessage({ + graphqlClient, + experimentation: createClientWithVariant('treatment'), + analytics, + accountId: 'account-id', + accountName: 'acme', + buildIds: ['build-id'], + nonInteractive: false, + analyticsEventProperties: { tracking_id: 'tracking-id', platform: 'android' }, + ...overrides, + }); + return { message, analytics }; +} + +function loggedMessage(): string { + return jest.mocked(Log.log).mock.calls.map(call => call.join(' ')).join('\n'); +} + +describe(ConcurrencyLimitMessage, () => { + beforeEach(() => { + jest.clearAllMocks(); + jest.spyOn(Log, 'log').mockImplementation(() => {}); + jest.spyOn(Log, 'newLine').mockImplementation(() => {}); + jest.spyOn(Log, 'debug').mockImplementation(() => {}); + jest.mocked(AccountQuery.getConcurrencyUsageAsync).mockResolvedValue(createUsage()); + }); + + it('is not built when the command has no experimentation client', () => { + expect( + ConcurrencyLimitMessage.forBuildRun({ + experimentation: undefined, + graphqlClient, + analytics: createMockAnalytics(), + startedBuilds: [{ build: { id: 'build-id' } }], + nonInteractive: false, + buildCtx: { + accountId: 'account-id', + accountName: 'acme', + analyticsEventProperties: {}, + }, + }) + ).toBeUndefined(); + }); + + it('takes the account and event properties from the build context', async () => { + const analytics = createMockAnalytics(); + const message = ConcurrencyLimitMessage.forBuildRun({ + experimentation: createClientWithVariant('treatment'), + graphqlClient, + analytics, + startedBuilds: [{ build: { id: 'build-a' } }, { build: { id: 'build-b' } }], + nonInteractive: true, + buildCtx: { + accountId: 'ctx-account-id', + accountName: 'ctx-account', + analyticsEventProperties: { tracking_id: 'ctx-tracking-id' }, + }, + }); + + await expect(message?.maybeShowAsync()).resolves.toBe(true); + + expect(AccountQuery.getConcurrencyUsageAsync).toHaveBeenCalledWith( + graphqlClient, + 'ctx-account-id' + ); + expect(loggedMessage()).toContain('/accounts/ctx-account/settings/billing'); + expect(analytics.logEvent).toHaveBeenCalledWith( + BuildEvent.CONCURRENCY_LIMIT_MESSAGE_SHOWN, + expect.objectContaining({ + tracking_id: 'ctx-tracking-id', + build_ids: 'build-a,build-b', + build_count: 2, + non_interactive: true, + }) + ); + }); + + it('does nothing for the control variant', async () => { + const { message, analytics } = createMessage({ + experimentation: createClientWithVariant('control'), + }); + + await expect(message.maybeShowAsync()).resolves.toBe(false); + + expect(AccountQuery.getConcurrencyUsageAsync).not.toHaveBeenCalled(); + expect(Log.log).not.toHaveBeenCalled(); + expect(analytics.logEvent).not.toHaveBeenCalled(); + }); + + it('does nothing when the namespace is unknown', async () => { + const { message, analytics } = createMessage({ + experimentation: ExperimentationClient.createDisabled(ExperimentOverrides.empty()), + }); + + await expect(message.maybeShowAsync()).resolves.toBe(false); + + expect(AccountQuery.getConcurrencyUsageAsync).not.toHaveBeenCalled(); + expect(analytics.logEvent).not.toHaveBeenCalled(); + }); + + it('prints the paid-plan message once and logs the event', async () => { + const { message, analytics } = createMessage(); + + await expect(message.maybeShowAsync()).resolves.toBe(true); + + expect(AccountQuery.getConcurrencyUsageAsync).toHaveBeenCalledTimes(1); + expect(AccountQuery.getConcurrencyUsageAsync).toHaveBeenCalledWith(graphqlClient, 'account-id'); + expect(Log.log).toHaveBeenCalledTimes(1); + const text = loggedMessage(); + expect(text).toContain('2 builds are already running and the Starter plan includes 2.'); + expect(text).toContain('Add concurrencies ($50/mo each)'); + expect(text).toContain( + 'https://expo.dev/accounts/acme/settings/billing?utm_source=eas-cli&utm_campaign=cli-build-concurrency-limit' + ); + expect(text).not.toContain('billing:subscribe'); + expect(analytics.logEvent).toHaveBeenCalledTimes(1); + expect(analytics.logEvent).toHaveBeenCalledWith( + BuildEvent.CONCURRENCY_LIMIT_MESSAGE_SHOWN, + expect.objectContaining({ + tracking_id: 'tracking-id', + platform: 'android', + experiment_name: CONCURRENCY_LIMIT_EXPERIMENT_NAMESPACE, + variant: 'treatment', + build_ids: 'build-id', + build_count: 1, + active_consumer_count: 2, + active_build_count: 2, + active_job_run_count: 0, + concurrency_limit: 2, + has_paid_plan: true, + plan_id: 'price_starter', + plan_name: 'Starter', + non_interactive: false, + }) + ); + }); + + it('prints the upgrade path for a free plan', async () => { + jest + .mocked(AccountQuery.getConcurrencyUsageAsync) + .mockResolvedValue(createUsage({ planId: 'price_free', planName: 'Free', total: 1, activeCount: 1 })); + const { message, analytics } = createMessage({ nonInteractive: true }); + + await expect(message.maybeShowAsync()).resolves.toBe(true); + + const text = loggedMessage(); + expect(text).toContain('1 build is already running and the Free plan includes 1.'); + expect(text).toContain('Upgrade to Starter or Production'); + expect(text).toContain( + 'https://expo.dev/accounts/acme/settings/billing?utm_source=eas-cli&utm_campaign=cli-build-concurrency-limit' + ); + expect(text).toContain('eas billing:subscribe starter --account acme'); + expect(text).not.toContain('$50/mo'); + expect(analytics.logEvent).toHaveBeenCalledWith( + BuildEvent.CONCURRENCY_LIMIT_MESSAGE_SHOWN, + expect.objectContaining({ has_paid_plan: false, non_interactive: true }) + ); + }); + + it('names workflow jobs that hold a concurrency', async () => { + jest + .mocked(AccountQuery.getConcurrencyUsageAsync) + .mockResolvedValue(createUsage({ total: 3, activeCount: 1, jobRunCount: 2 })); + const { message, analytics } = createMessage(); + + await expect(message.maybeShowAsync()).resolves.toBe(true); + + expect(loggedMessage()).toContain( + '1 build and 2 workflow jobs are already running and the Starter plan includes 3.' + ); + expect(analytics.logEvent).toHaveBeenCalledWith( + BuildEvent.CONCURRENCY_LIMIT_MESSAGE_SHOWN, + expect.objectContaining({ + active_consumer_count: 3, + active_build_count: 1, + active_job_run_count: 2, + }) + ); + }); + + it('shows the upgrade path and omits plan fields when the account has no subscription', async () => { + jest + .mocked(AccountQuery.getConcurrencyUsageAsync) + .mockResolvedValue({ ...createUsage({ activeCount: 1 }), subscription: null }); + const { message, analytics } = createMessage(); + + await expect(message.maybeShowAsync()).resolves.toBe(true); + + const text = loggedMessage(); + expect(text).toContain('All concurrencies in your plan are in use.'); + expect(text).toContain('Upgrade to Starter or Production'); + expect(analytics.logEvent).toHaveBeenCalledTimes(1); + const [, properties] = jest.mocked(analytics.logEvent).mock.calls[0]; + expect(properties).toEqual( + expect.objectContaining({ + active_consumer_count: 1, + concurrency_limit: -1, + has_paid_plan: false, + }) + ); + expect(properties).not.toHaveProperty('plan_id'); + expect(properties).not.toHaveProperty('plan_name'); + }); + + it('uses the count-free copy when the plan concurrencies are unknown', async () => { + jest.mocked(AccountQuery.getConcurrencyUsageAsync).mockResolvedValue(createUsage({ total: null })); + const { message, analytics } = createMessage(); + + await expect(message.maybeShowAsync()).resolves.toBe(true); + + expect(loggedMessage()).toContain('All concurrencies in your plan are in use.'); + expect(analytics.logEvent).toHaveBeenCalledWith( + BuildEvent.CONCURRENCY_LIMIT_MESSAGE_SHOWN, + expect.objectContaining({ concurrency_limit: -1 }) + ); + }); + + it('uses the count-free copy when the totals do not confirm the limit', async () => { + jest + .mocked(AccountQuery.getConcurrencyUsageAsync) + .mockResolvedValue(createUsage({ total: 4, activeCount: 2 })); + const { message } = createMessage(); + + await expect(message.maybeShowAsync()).resolves.toBe(true); + + const text = loggedMessage(); + expect(text).toContain('All concurrencies in your plan are in use.'); + expect(text).not.toContain('already running'); + }); + + it('falls back silently when the account query fails', async () => { + jest + .mocked(AccountQuery.getConcurrencyUsageAsync) + .mockRejectedValue(new Error('network down')); + const { message, analytics } = createMessage(); + + await expect(message.maybeShowAsync()).resolves.toBe(false); + + expect(Log.debug).toHaveBeenCalledTimes(1); + expect(Log.log).not.toHaveBeenCalled(); + expect(analytics.logEvent).not.toHaveBeenCalled(); + }); + + it('runs the query, the print, and the event only once across repeated calls', async () => { + const { message, analytics } = createMessage(); + + await expect( + Promise.all([message.maybeShowAsync(), message.maybeShowAsync(), message.maybeShowAsync()]) + ).resolves.toEqual([true, true, true]); + await expect(message.maybeShowAsync()).resolves.toBe(true); + + expect(AccountQuery.getConcurrencyUsageAsync).toHaveBeenCalledTimes(1); + expect(Log.log).toHaveBeenCalledTimes(1); + expect(analytics.logEvent).toHaveBeenCalledTimes(1); + }); + + it('logs the exposure on the first call, not on construction', async () => { + const definition: ExperimentationDefinition = { + experiments: [ + { + experimentName: CONCURRENCY_LIMIT_EXPERIMENT_NAMESPACE, + paramDefinitions: { + variant: { type: 'uniform_choice', choices: ['control', 'treatment'] }, + }, + exposureLoggerType: 'rudderstack', + }, + ], + namespaces: [ + { + name: CONCURRENCY_LIMIT_EXPERIMENT_NAMESPACE, + numSegments: 1000, + segmentExperimentSetupDefinitions: [ + { + method: 'ADD', + name: 'full rollout', + experimentName: CONCURRENCY_LIMIT_EXPERIMENT_NAMESPACE, + numSegments: 1000, + }, + ], + }, + ], + }; + const createClient = ( + analytics: ReturnType + ): ExperimentationClient => + new ExperimentationClient( + { + user: EMPTY_EXPERIMENTATION_DEFINITION, + account: definition, + device: EMPTY_EXPERIMENTATION_DEFINITION, + }, + { user: null, device: null }, + ExperimentOverrides.empty(), + createLoggerConfig(analytics) + ); + const probeClient = createClient(createMockAnalytics()); + const units = findUnitsByValue( + accountId => + probeClient.getAccountNamespace(CONCURRENCY_LIMIT_EXPERIMENT_NAMESPACE, { accountId }), + 'variant', + 'control', + ['control', 'treatment'] + ); + + const analytics = createMockAnalytics(); + const { message } = createMessage({ + analytics, + experimentation: createClient(analytics), + accountId: units.get('treatment'), + }); + expect(analytics.logEvent).not.toHaveBeenCalled(); + + await expect(message.maybeShowAsync()).resolves.toBe(true); + await expect(message.maybeShowAsync()).resolves.toBe(true); + + const eventNames = analytics.logEvent.mock.calls.map(([name]) => name); + expect(eventNames.filter(name => name === 'Experiment Viewed')).toHaveLength(1); + expect(analytics.logEvent).toHaveBeenCalledWith('Experiment Viewed', { + experimentName: CONCURRENCY_LIMIT_EXPERIMENT_NAMESPACE, + variationName: 'treatment', + unit: units.get('treatment'), + }); + }); +}); + +describe(formatConcurrencyLimitMessage, () => { + it('names the current plan when the plan name is unknown', () => { + const text = formatConcurrencyLimitMessage({ + accountName: 'acme', + planName: null, + activeBuildCount: 3, + activeJobRunCount: 0, + concurrencyLimit: 3, + hasPaidPlan: true, + }); + expect(text).toContain('3 builds are already running and the current plan includes 3.'); + }); + + it('uses the singular form for one workflow job', () => { + const text = formatConcurrencyLimitMessage({ + accountName: 'acme', + planName: 'Free', + activeBuildCount: 0, + activeJobRunCount: 1, + concurrencyLimit: 1, + hasPaidPlan: false, + }); + expect(text).toContain('1 workflow job is already running and the Free plan includes 1.'); + }); +}); diff --git a/packages/eas-cli/src/build/__tests__/waitForBuildEndAsync-test.ts b/packages/eas-cli/src/build/__tests__/waitForBuildEndAsync-test.ts index 90accc38c3..3a2297e001 100644 --- a/packages/eas-cli/src/build/__tests__/waitForBuildEndAsync-test.ts +++ b/packages/eas-cli/src/build/__tests__/waitForBuildEndAsync-test.ts @@ -1,4 +1,5 @@ import { BuildPhase } from '@expo/eas-build-job'; +import MockDate from 'mockdate'; import { v4 as uuid } from 'uuid'; import { ExpoGraphqlClient } from '../../commandUtils/context/contextUtils/createGraphqlClient'; @@ -13,11 +14,14 @@ import { BuildStatus, } from '../../graphql/generated'; import { BuildQuery } from '../../graphql/queries/BuildQuery'; +import Log from '../../log'; import { Ora, isSpinnerEnabled, ora } from '../../ora'; import { createRealtimeLogsClient } from '../../utils/centrifuge'; import { sleepAsync } from '../../utils/promise'; import { waitForBuildEndAsync } from '../build'; +import { ConcurrencyLimitMessage, TREATMENT_SPINNER_TEXT } from '../concurrencyLimitMessage'; +jest.mock('cli-progress'); jest.mock('../../ora', () => ({ ...jest.requireActual('../../ora'), ora: jest.fn(), @@ -41,14 +45,31 @@ jest.mock('../../utils/promise', () => ({ const graphqlClient = {} as unknown as ExpoGraphqlClient; -function createBuildFragment(status: BuildStatus): BuildFragment { +function createBuildFragment( + status: BuildStatus, + { + id = 'build-id', + platform = AppPlatform.Android, + priority = BuildPriority.Normal, + queue, + }: { + id?: string; + platform?: AppPlatform; + priority?: BuildPriority; + queue?: Pick< + BuildFragment, + 'initialQueuePosition' | 'queuePosition' | 'estimatedWaitTimeLeftSeconds' + >; + } = {} +): BuildFragment { return { - id: 'build-id', + id, createdAt: new Date().toISOString(), updatedAt: new Date().toISOString(), - platform: AppPlatform.Android, + platform, logFiles: [], - priority: BuildPriority.Normal, + priority, + ...queue, app: { __typename: 'App', slug: 'test-project', @@ -87,6 +108,37 @@ function setBuildStatuses(statuses: BuildStatus[]): void { } } +/** One status queue per build ID, consumed in order on each poll. */ +function setBuildStatusesById(statusesById: Record): void { + const queues = new Map(Object.entries(statusesById).map(([id, s]) => [id, [...s]])); + jest.mocked(BuildQuery.byIdAsync).mockImplementation(async (_client, buildId) => { + const status = queues.get(buildId)?.shift(); + if (!status) { + throw new Error(`No more statuses for ${buildId}`); + } + const platform = buildId === 'build-ios' ? AppPlatform.Ios : AppPlatform.Android; + return createBuildFragment(status, { id: buildId, platform }); + }); +} + +function createConcurrencyLimitMessage(shown: boolean): ConcurrencyLimitMessage & { + maybeShowAsync: jest.Mock; +} { + return { maybeShowAsync: jest.fn(async () => shown) } as unknown as ConcurrencyLimitMessage & { + maybeShowAsync: jest.Mock; + }; +} + +/** Moves the clock past the NEW-status grace period on every poll. */ +function advanceClockOnSleep(): void { + jest.mocked(sleepAsync).mockImplementation(async () => { + MockDate.set(Date.now() + 16_000); + }); +} + +const GRACE_TEXT = 'Waiting for build to get enqueued…'; +const CONTROL_TEXT_PREFIX = 'Build concurrency limit reached for your account.'; + describe(waitForBuildEndAsync, () => { let spinner: Ora; let publish: () => void; @@ -118,6 +170,11 @@ describe(waitForBuildEndAsync, () => { jest.mocked(sleepAsync).mockImplementation(async () => { publish(); }); + MockDate.set('2026-10-12T00:00:00.000Z'); + }); + + afterEach(() => { + MockDate.reset(); }); it('does not create a watcher when the spinner is disabled', async () => { @@ -160,4 +217,120 @@ describe(waitForBuildEndAsync, () => { expect(jest.mocked(sleepAsync).mock.calls.length).toBe(2); expect(spinner.text).toBe('Build queued...'); }); + + it('suggests the priority queue when a free-tier build gets a queue position', async () => { + jest.mocked(isSpinnerEnabled).mockReturnValue(false); + const logSpy = jest.spyOn(Log, 'log').mockImplementation(() => {}); + const queue = { initialQueuePosition: 4, queuePosition: 2, estimatedWaitTimeLeftSeconds: 60 }; + jest + .mocked(BuildQuery.byIdAsync) + .mockResolvedValueOnce(createBuildFragment(BuildStatus.InQueue, { queue })) + .mockResolvedValueOnce(createBuildFragment(BuildStatus.Finished)); + + await waitForBuildEndAsync(graphqlClient, { buildIds: ['build-id'], accountName: 'acct' }); + + const logged = logSpy.mock.calls.map(call => call.join(' ')).join('\n'); + expect(logged).toContain('Start builds sooner in the priority queue.'); + expect(logged).toContain('Sign up for a paid plan at'); + expect(logged).toContain('/accounts/acct/settings/billing'); + expect(logged).toContain('Waiting in Free tier queue'); + logSpy.mockRestore(); + }); + + describe('concurrency limit message', () => { + beforeEach(() => { + jest.mocked(isSpinnerEnabled).mockReturnValue(false); + }); + + it('shows the grace text and does not consult the message inside the grace period', async () => { + const message = createConcurrencyLimitMessage(true); + setBuildStatuses([BuildStatus.New, BuildStatus.Finished]); + + await waitForBuildEndAsync(graphqlClient, { + buildIds: ['build-id'], + accountName: 'acct', + concurrencyLimitMessage: message, + }); + + expect(spinner.text).toBe(GRACE_TEXT); + expect(message.maybeShowAsync).not.toHaveBeenCalled(); + }); + + it('keeps the existing text when no message is given', async () => { + advanceClockOnSleep(); + setBuildStatuses([BuildStatus.New, BuildStatus.New, BuildStatus.Finished]); + + await waitForBuildEndAsync(graphqlClient, { buildIds: ['build-id'], accountName: 'acct' }); + + expect(spinner.text).toContain(CONTROL_TEXT_PREFIX); + expect(spinner.text).toContain('/accounts/acct/settings/billing'); + }); + + it('switches to the treatment spinner text when the message was shown', async () => { + advanceClockOnSleep(); + const message = createConcurrencyLimitMessage(true); + setBuildStatuses([ + BuildStatus.New, + BuildStatus.New, + BuildStatus.New, + BuildStatus.Finished, + ]); + + await waitForBuildEndAsync(graphqlClient, { + buildIds: ['build-id'], + accountName: 'acct', + concurrencyLimitMessage: message, + }); + + expect(spinner.text).toBe(TREATMENT_SPINNER_TEXT); + expect(message.maybeShowAsync).toHaveBeenCalledTimes(2); + }); + + it('keeps the existing text when the message was not shown', async () => { + advanceClockOnSleep(); + const message = createConcurrencyLimitMessage(false); + setBuildStatuses([BuildStatus.New, BuildStatus.New, BuildStatus.Finished]); + + await waitForBuildEndAsync(graphqlClient, { + buildIds: ['build-id'], + accountName: 'acct', + concurrencyLimitMessage: message, + }); + + expect(spinner.text).toContain(CONTROL_TEXT_PREFIX); + expect(message.maybeShowAsync).toHaveBeenCalledTimes(1); + }); + + it('consults the message once per poll for multi-platform builds', async () => { + advanceClockOnSleep(); + const message = createConcurrencyLimitMessage(true); + setBuildStatusesById({ + 'build-android': [BuildStatus.New, BuildStatus.New, BuildStatus.Finished], + 'build-ios': [BuildStatus.New, BuildStatus.New, BuildStatus.Finished], + }); + + await waitForBuildEndAsync(graphqlClient, { + buildIds: ['build-android', 'build-ios'], + accountName: 'acct', + concurrencyLimitMessage: message, + }); + + expect(message.maybeShowAsync).toHaveBeenCalledTimes(1); + expect(spinner.text).toContain('Android build - status:'); + expect(spinner.text).toContain('iOS build - status:'); + expect(spinner.text.split('(concurrency limit reached)')).toHaveLength(3); + }); + + it('resets the NEW-status timer between runs', async () => { + advanceClockOnSleep(); + setBuildStatuses([BuildStatus.New, BuildStatus.New, BuildStatus.Finished]); + await waitForBuildEndAsync(graphqlClient, { buildIds: ['build-id'], accountName: 'acct' }); + expect(spinner.text).toContain(CONTROL_TEXT_PREFIX); + + setBuildStatuses([BuildStatus.New, BuildStatus.Finished]); + await waitForBuildEndAsync(graphqlClient, { buildIds: ['build-id'], accountName: 'acct' }); + + expect(spinner.text).toBe(GRACE_TEXT); + }); + }); }); diff --git a/packages/eas-cli/src/build/build.ts b/packages/eas-cli/src/build/build.ts index d35b53b460..30eba6b836 100644 --- a/packages/eas-cli/src/build/build.ts +++ b/packages/eas-cli/src/build/build.ts @@ -13,6 +13,7 @@ import fs from 'fs-extra'; import { GraphQLError } from 'graphql/error'; import nullthrows from 'nullthrows'; +import { ConcurrencyLimitMessage, TREATMENT_SPINNER_TEXT } from './concurrencyLimitMessage'; import { BuildContext } from './context'; import { EasBuildDownForMaintenanceError, @@ -38,6 +39,7 @@ import { } from './logs'; import { collectMetadataAsync } from './metadata'; import { printDeprecationWarnings } from './utils/printBuildInfo'; +import { getAccountBillingUrl } from './utils/url'; import { LocalFile, assertProjectTarballSizeDoesNotExceedLimit, @@ -48,7 +50,6 @@ import { } from './utils/repository'; import { BuildEvent } from '../analytics/AnalyticsManager'; import { withAnalyticsAsync } from '../analytics/common'; -import { getExpoWebsiteBaseUrl } from '../api'; import { formatStarterSubscribeCommand } from '../billing/plans'; import { ExpoGraphqlClient } from '../commandUtils/context/contextUtils/createGraphqlClient'; import { EasCommandError } from '../commandUtils/errors'; @@ -534,9 +535,20 @@ export type MaybeBuildFragment = BuildFragment | null; export async function waitForBuildEndAsync( graphqlClient: ExpoGraphqlClient, - { buildIds, accountName }: { buildIds: string[]; accountName: string }, + { + buildIds, + accountName, + concurrencyLimitMessage, + }: { + buildIds: string[]; + accountName: string; + /** Prints the concurrency-limit experiment message once per run. Absent means control. */ + concurrencyLimitMessage?: ConcurrencyLimitMessage; + }, { intervalSec = 10 } = {} ): Promise { + statusNewSetAt = null; + queueProgressBarStarted = false; let spinner; let originalSpinnerText; if (buildIds.length === 1) { @@ -568,11 +580,11 @@ export async function waitForBuildEndAsync( const { refetch } = builds.length === 1 ? await handleSingleBuildProgressAsync( - { build: builds[0], accountName }, + { build: builds[0], accountName, concurrencyLimitMessage }, { spinner, logsStates, installRender } ) : await handleMultipleBuildsProgressAsync( - { builds }, + { builds, concurrencyLimitMessage }, { spinner, originalSpinnerText, logsStates, installRender } ); if (!refetch) { @@ -642,9 +654,11 @@ async function handleSingleBuildProgressAsync( { build, accountName, + concurrencyLimitMessage, }: { build: MaybeBuildFragment; accountName: string; + concurrencyLimitMessage?: ConcurrencyLimitMessage; }, { spinner, logsStates, installRender }: { spinner: Ora } & BuildLogsProgressOptions ): Promise { @@ -678,10 +692,13 @@ async function handleSingleBuildProgressAsync( if (newStatusDurationMs < NEW_STATUS_GRACE_PERIOD_MS) { updateSpinnerText(spinner, { text: 'Waiting for build to get enqueued…' }); } else { + const shownTreatmentMessage = (await concurrencyLimitMessage?.maybeShowAsync()) ?? false; updateSpinnerText(spinner, { - text: `Build concurrency limit reached for your account. Build will enter queue once a concurrency becomes available. Add additional concurrencies at ${link( - formatAccountBillingUrl(accountName) - )}.`, + text: shownTreatmentMessage + ? TREATMENT_SPINNER_TEXT + : `Build concurrency limit reached for your account. Build will enter queue once a concurrency becomes available. Add additional concurrencies at ${link( + getAccountBillingUrl(accountName) + )}.`, }); } break; @@ -703,7 +720,7 @@ async function handleSingleBuildProgressAsync( if (build.priority !== BuildPriority.High) { Log.newLine(); Log.log('Start builds sooner in the priority queue.'); - Log.log(`Sign up for a paid plan at ${link(formatAccountBillingUrl(accountName))}`); + Log.log(`Sign up for a paid plan at ${link(getAccountBillingUrl(accountName))}`); } Log.newLine(); @@ -768,7 +785,10 @@ const statusToDisplayName: Record = { const platforms = [AppPlatform.Android, AppPlatform.Ios]; async function handleMultipleBuildsProgressAsync( - { builds: maybeBuilds }: { builds: MaybeBuildFragment[] }, + { + builds: maybeBuilds, + concurrencyLimitMessage, + }: { builds: MaybeBuildFragment[]; concurrencyLimitMessage?: ConcurrencyLimitMessage }, { spinner, originalSpinnerText, @@ -808,6 +828,9 @@ async function handleMultipleBuildsProgressAsync( someNew && statusNewSetAt !== null && Date.now() - statusNewSetAt >= NEW_STATUS_GRACE_PERIOD_MS; + if (showConcurrencyWarning) { + await concurrencyLimitMessage?.maybeShowAsync(); + } const pendingBuildsText = formatPendingBuildsText( originalSpinnerText, builds, @@ -895,10 +918,6 @@ function formatEstimatedWaitTime(estimatedWaitTimeLeftSeconds: number): string { } } -function formatAccountBillingUrl(accountName: string): string { - return new URL(`/accounts/${accountName}/settings/billing`, getExpoWebsiteBaseUrl()).toString(); -} - async function computeAndMaybeUploadRuntimeAndFingerprintMetadataAsync( ctx: BuildContext ): Promise<{ diff --git a/packages/eas-cli/src/build/concurrencyLimitMessage.ts b/packages/eas-cli/src/build/concurrencyLimitMessage.ts new file mode 100644 index 0000000000..bc71f6f7b8 --- /dev/null +++ b/packages/eas-cli/src/build/concurrencyLimitMessage.ts @@ -0,0 +1,198 @@ +import chalk from 'chalk'; + +import { getAccountBillingUrl } from './utils/url'; +import { Analytics, AnalyticsEventProperties, BuildEvent } from '../analytics/AnalyticsManager'; +import { formatStarterSubscribeCommand, hasPaidSubscription } from '../billing/plans'; +import { ExpoGraphqlClient } from '../commandUtils/context/contextUtils/createGraphqlClient'; +import { ExperimentationClient } from '../experimentation/ExperimentationClient'; +import { AccountQuery } from '../graphql/queries/AccountQuery'; +import Log, { link } from '../log'; + +/** Must match the namespace in the server's account experimentation config. */ +export const CONCURRENCY_LIMIT_EXPERIMENT_NAMESPACE = 'cli-build-concurrency-limit'; + +type ConcurrencyLimitVariant = 'control' | 'treatment'; + +/** Spinner text for the treatment arm. The details are in the persisted line above the spinner. */ +export const TREATMENT_SPINNER_TEXT = 'Waiting for a concurrency to become available…'; + +export type ConcurrencyLimitMessageOptions = { + graphqlClient: ExpoGraphqlClient; + experimentation: ExperimentationClient; + analytics: Analytics; + accountId: string; + accountName: string; + buildIds: string[]; + nonInteractive: boolean; + /** Properties of the `build cli build command` event, so the two events share a `tracking_id`. */ + analyticsEventProperties: AnalyticsEventProperties; +}; + +/** + * Prints the concurrency-limit experiment message at most once per `eas build` run. + * + * Call `maybeShowAsync` only once a build has waited in NEW for the grace period, because the first + * call reads the variant and logs the exposure. Control accounts get no network call and no output. + * Treatment accounts get one account query, one persisted line, and one analytics event. + */ +export class ConcurrencyLimitMessage { + private pending: Promise | null = null; + + constructor(private readonly options: ConcurrencyLimitMessageOptions) {} + + /** Builds the message for a run, or nothing when the command has no experimentation client. */ + public static forBuildRun({ + experimentation, + startedBuilds, + buildCtx, + ...options + }: Omit< + ConcurrencyLimitMessageOptions, + 'experimentation' | 'buildIds' | 'accountId' | 'accountName' | 'analyticsEventProperties' + > & { + experimentation: ExperimentationClient | undefined; + startedBuilds: { build: { id: string } }[]; + buildCtx: Pick< + ConcurrencyLimitMessageOptions, + 'accountId' | 'accountName' | 'analyticsEventProperties' + >; + }): ConcurrencyLimitMessage | undefined { + if (!experimentation) { + return undefined; + } + return new ConcurrencyLimitMessage({ + ...options, + experimentation, + buildIds: startedBuilds.map(({ build }) => build.id), + accountId: buildCtx.accountId, + accountName: buildCtx.accountName, + analyticsEventProperties: buildCtx.analyticsEventProperties, + }); + } + + /** Never throws. Resolves to true when the treatment line was printed. */ + public async maybeShowAsync(): Promise { + this.pending ??= this.showOnceAsync(); + return await this.pending; + } + + private async showOnceAsync(): Promise { + const { graphqlClient, experimentation, analytics, accountId, accountName } = this.options; + try { + const variant = experimentation + .getAccountNamespace(CONCURRENCY_LIMIT_EXPERIMENT_NAMESPACE, { accountId }) + .getParam('variant', 'control'); + if (variant !== 'treatment') { + return false; + } + + const usage = await AccountQuery.getConcurrencyUsageAsync(graphqlClient, accountId); + const subscription = usage.subscription ?? null; + const activeBuildCount = usage.concurrencyConsumers.filter( + consumer => consumer.__typename === 'Build' + ).length; + const activeJobRunCount = usage.concurrencyConsumers.length - activeBuildCount; + const concurrencyLimit = subscription?.concurrencies?.total ?? null; + const hasPaidPlan = hasPaidSubscription(subscription); + const planName = subscription?.name ?? null; + + Log.newLine(); + Log.log( + formatConcurrencyLimitMessage({ + accountName, + planName, + activeBuildCount, + activeJobRunCount, + concurrencyLimit, + hasPaidPlan, + }) + ); + Log.newLine(); + + analytics.logEvent(BuildEvent.CONCURRENCY_LIMIT_MESSAGE_SHOWN, { + ...this.options.analyticsEventProperties, + experiment_name: CONCURRENCY_LIMIT_EXPERIMENT_NAMESPACE, + variant, + build_ids: this.options.buildIds.join(','), + build_count: this.options.buildIds.length, + active_consumer_count: activeBuildCount + activeJobRunCount, + active_build_count: activeBuildCount, + active_job_run_count: activeJobRunCount, + concurrency_limit: concurrencyLimit ?? -1, + has_paid_plan: hasPaidPlan, + ...(subscription?.planId ? { plan_id: subscription.planId } : {}), + ...(planName ? { plan_name: planName } : {}), + non_interactive: this.options.nonInteractive, + }); + return true; + } catch (error) { + Log.debug( + '[concurrency-limit] Failed to show the concurrency limit message. Keeping the default text.', + error + ); + return false; + } + } +} + +export function formatConcurrencyLimitMessage({ + accountName, + planName, + activeBuildCount, + activeJobRunCount, + concurrencyLimit, + hasPaidPlan, +}: { + accountName: string; + planName: string | null; + activeBuildCount: number; + activeJobRunCount: number; + concurrencyLimit: number | null; + hasPaidPlan: boolean; +}): string { + // The counts only explain the wait when they confirm it. A per-platform pool can be full while + // the totals still show a free slot, so fall back to a count-free sentence in that case. + const activeCount = activeBuildCount + activeJobRunCount; + const countsConfirmLimit = concurrencyLimit !== null && activeCount >= concurrencyLimit; + const reason = countsConfirmLimit + ? `${formatActiveConsumers(activeBuildCount, activeJobRunCount)} already running and the ${ + planName ?? 'current' + } plan includes ${concurrencyLimit}.` + : 'All concurrencies in your plan are in use.'; + const billingLink = link(getTreatmentBillingUrl(accountName), { dim: false }); + + const lines = [`Your build is waiting for a free concurrency. ${reason}`]; + if (hasPaidPlan) { + lines.push( + `Add concurrencies ($50/mo each) so your team can build at the same time: ${billingLink}` + ); + } else { + lines.push(`Upgrade to Starter or Production to build more at once: ${billingLink}`); + lines.push(`Or run ${chalk.bold(formatStarterSubscribeCommand(accountName))}`); + } + return lines.join('\n'); +} + +/** + * The billing URL with source tags, so the website can attribute a visit to this message. + * Only the treatment arm uses it. The control arm keeps the plain billing URL. + */ +export function getTreatmentBillingUrl(accountName: string): string { + const url = new URL(getAccountBillingUrl(accountName)); + url.searchParams.set('utm_source', 'eas-cli'); + url.searchParams.set('utm_campaign', CONCURRENCY_LIMIT_EXPERIMENT_NAMESPACE); + return url.toString(); +} + +/** "2 builds are", "1 workflow job is", or "1 build and 2 workflow jobs are". */ +function formatActiveConsumers(buildCount: number, jobRunCount: number): string { + const parts: string[] = []; + if (buildCount > 0) { + parts.push(`${buildCount} ${buildCount === 1 ? 'build' : 'builds'}`); + } + if (jobRunCount > 0) { + parts.push(`${jobRunCount} ${jobRunCount === 1 ? 'workflow job' : 'workflow jobs'}`); + } + const verb = buildCount + jobRunCount === 1 ? 'is' : 'are'; + return `${parts.join(' and ')} ${verb}`; +} diff --git a/packages/eas-cli/src/build/runBuildAndSubmit.ts b/packages/eas-cli/src/build/runBuildAndSubmit.ts index ca0e8ad8ca..49191592fa 100644 --- a/packages/eas-cli/src/build/runBuildAndSubmit.ts +++ b/packages/eas-cli/src/build/runBuildAndSubmit.ts @@ -16,6 +16,7 @@ import nullthrows from 'nullthrows'; import { prepareAndroidBuildAsync } from './android/build'; import { BuildRequestSender, MaybeBuildFragment, waitForBuildEndAsync } from './build'; +import { ConcurrencyLimitMessage } from './concurrencyLimitMessage'; import { ensureProjectConfiguredAsync } from './configure'; import { BuildContext } from './context'; import { createBuildContextAsync } from './createContext'; @@ -30,6 +31,7 @@ import { Analytics } from '../analytics/AnalyticsManager'; import { createAndLinkChannelAsync, doesChannelExistAsync } from '../channel/queries'; import { DynamicConfigContextFn } from '../commandUtils/context/DynamicProjectConfigContextField'; import { ExpoGraphqlClient } from '../commandUtils/context/contextUtils/createGraphqlClient'; +import { ExperimentationClient } from '../experimentation/ExperimentationClient'; import { AppPlatform, BuildFragment, @@ -118,6 +120,7 @@ export async function runBuildAndSubmitAsync({ getDynamicPrivateProjectConfigAsync, downloadSimBuildAutoConfirm, envOverride, + experimentation, }: { graphqlClient: ExpoGraphqlClient; analytics: Analytics; @@ -128,6 +131,8 @@ export async function runBuildAndSubmitAsync({ getDynamicPrivateProjectConfigAsync: DynamicConfigContextFn; downloadSimBuildAutoConfirm?: boolean; envOverride?: Env; + /** Enables the concurrency-limit experiment while waiting for the build. */ + experimentation?: ExperimentationClient; }): Promise<{ buildIds: string[]; buildProfiles?: ProfileData<'build'>[]; @@ -324,6 +329,14 @@ export async function runBuildAndSubmitAsync({ const builds = await waitForBuildEndAsync(graphqlClient, { buildIds: startedBuilds.map(({ build }) => build.id), accountName, + concurrencyLimitMessage: ConcurrencyLimitMessage.forBuildRun({ + experimentation, + graphqlClient, + analytics, + startedBuilds, + nonInteractive: flags.nonInteractive, + buildCtx: Object.values(buildCtxByPlatform)[0], + }), }); if (!flags.json) { printBuildResults(builds); diff --git a/packages/eas-cli/src/build/utils/url.ts b/packages/eas-cli/src/build/utils/url.ts index f49fef662d..c882b2aaa7 100644 --- a/packages/eas-cli/src/build/utils/url.ts +++ b/packages/eas-cli/src/build/utils/url.ts @@ -27,6 +27,13 @@ export function getBuildLogsUrl(build: BuildFragment, hash?: string): string { return new URL(url, getExpoWebsiteBaseUrl()).toString(); } +export function getAccountBillingUrl(accountName: string): string { + return new URL( + `/accounts/${encodeURIComponent(accountName)}/settings/billing`, + getExpoWebsiteBaseUrl() + ).toString(); +} + export function getArtifactUrl(artifactId: string): string { return new URL(`/artifacts/${artifactId}`, getExpoWebsiteBaseUrl()).toString(); } diff --git a/packages/eas-cli/src/commandUtils/EasCommand.ts b/packages/eas-cli/src/commandUtils/EasCommand.ts index d167880247..ef3d46c846 100644 --- a/packages/eas-cli/src/commandUtils/EasCommand.ts +++ b/packages/eas-cli/src/commandUtils/EasCommand.ts @@ -7,6 +7,7 @@ import nullthrows from 'nullthrows'; import AnalyticsContextField from './context/AnalyticsContextField'; import ContextField from './context/ContextField'; import DynamicLoggedInContextField from './context/DynamicLoggedInContextField'; +import ExperimentationContextField from './context/ExperimentationContextField'; import { DynamicPrivateProjectConfigContextField, DynamicPublicProjectConfigContextField, @@ -148,6 +149,14 @@ export default abstract class EasCommand extends Command { ProjectId: { projectId: new ProjectIdContextField(), }, + /** + * Experimentation client for user, account, and device experiments. + * Declare a login field first so user experiments see the actor. + * Account experiments take the account ID from the command. + */ + Experimentation: { + experimentation: new ExperimentationContextField(), + }, }; /** diff --git a/packages/eas-cli/src/commandUtils/__tests__/EasCommand-test.ts b/packages/eas-cli/src/commandUtils/__tests__/EasCommand-test.ts index a3c74cd36d..63ab364413 100644 --- a/packages/eas-cli/src/commandUtils/__tests__/EasCommand-test.ts +++ b/packages/eas-cli/src/commandUtils/__tests__/EasCommand-test.ts @@ -48,6 +48,7 @@ const analytics: AnalyticsWithOrchestration = { logEvent: jest.fn((): void => {}), setActor: jest.fn((): void => {}), flushAsync: jest.fn(async (): Promise => {}), + getDeviceId: jest.fn((): string | null => null), }; beforeEach(() => { diff --git a/packages/eas-cli/src/commandUtils/context/ExperimentationContextField.ts b/packages/eas-cli/src/commandUtils/context/ExperimentationContextField.ts new file mode 100644 index 0000000000..0de0e30d62 --- /dev/null +++ b/packages/eas-cli/src/commandUtils/context/ExperimentationContextField.ts @@ -0,0 +1,106 @@ +import ContextField, { ContextOptions } from './ContextField'; +import { createGraphqlClient } from './contextUtils/createGraphqlClient'; +import { AnalyticsWithOrchestration } from '../../analytics/AnalyticsManager'; +import { ExperimentOverrides } from '../../experimentation/ExperimentOverrides'; +import { + ExperimentationClient, + createLoggerConfig, +} from '../../experimentation/ExperimentationClient'; +import { ExperimentationConfigQuery } from '../../graphql/queries/ExperimentationConfigQuery'; +import Log from '../../log'; + +/** Caps how long the config fetch can delay a command. */ +const REQUEST_TIMEOUT_MS = 3000; + +const CI_ENVIRONMENT_FLAGS = ['EAS_BUILD', 'CI']; + +function isRunningInCi(): boolean { + return CI_ENVIRONMENT_FLAGS.some(flag => !!process.env[flag]); +} + +/** + * Never prompts, throws, writes files, or reads the app config. A failed config fetch disables + * all scopes. CI runs skip the fetch and get defaults. + */ +export default class ExperimentationContextField extends ContextField { + // One client per command run, so repeated getContextAsync calls cannot log duplicate exposures. + private readonly clientByAnalytics = new WeakMap< + AnalyticsWithOrchestration, + Promise + >(); + + async getValueAsync(options: ContextOptions): Promise { + let pending = this.clientByAnalytics.get(options.analytics); + if (!pending) { + pending = this.buildClientAsync(options); + this.clientByAnalytics.set(options.analytics, pending); + } + return await pending; + } + + private async buildClientAsync({ + sessionManager, + analytics, + }: ContextOptions): Promise { + const overrides = ExperimentOverrides.fromEnv(); + + if (isRunningInCi()) { + Log.debug('[experimentation] Running in CI. Experiments return defaults.'); + return ExperimentationClient.createDisabled(overrides); + } + + const deviceId = analytics.getDeviceId(); + if (deviceId === null) { + Log.debug('[experimentation] Analytics is disabled. Experiments return defaults.'); + return ExperimentationClient.createDisabled(overrides); + } + + try { + const graphqlClient = createGraphqlClient( + { + accessToken: sessionManager.getAccessToken(), + sessionSecret: sessionManager.getSessionSecret(), + }, + { requestTimeoutMs: REQUEST_TIMEOUT_MS } + ); + + const [actorResult, configsResult] = await Promise.allSettled([ + sessionManager.getUserAsync(), + ExperimentationConfigQuery.getConfigsAsync(graphqlClient), + ]); + + if (configsResult.status === 'rejected') { + Log.debug( + '[experimentation] Failed to fetch the experiment config. Experiments return defaults.', + errorMessage(configsResult.reason) + ); + return ExperimentationClient.createDisabled(overrides); + } + + const configs = configsResult.value; + return new ExperimentationClient( + { + user: configs.userConfig, + account: configs.accountConfig, + device: configs.deviceConfig, + }, + { + user: actorResult.status === 'fulfilled' ? (actorResult.value?.id ?? null) : null, + device: deviceId, + }, + overrides, + createLoggerConfig(analytics) + ); + } catch (error) { + Log.debug( + '[experimentation] Failed to initialize experiments. Experiments return defaults.', + errorMessage(error) + ); + return ExperimentationClient.createDisabled(overrides); + } + } +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} diff --git a/packages/eas-cli/src/commandUtils/context/__tests__/ExperimentationContextField-test.ts b/packages/eas-cli/src/commandUtils/context/__tests__/ExperimentationContextField-test.ts new file mode 100644 index 0000000000..1c55a41922 --- /dev/null +++ b/packages/eas-cli/src/commandUtils/context/__tests__/ExperimentationContextField-test.ts @@ -0,0 +1,262 @@ +import { instance, mock, verify, when } from 'ts-mockito'; + +import { AnalyticsWithOrchestration } from '../../../analytics/AnalyticsManager'; +import { TEST_DEFINITION } from '../../../experimentation/__tests__/fixtures'; +import { ExperimentationConfigQuery } from '../../../graphql/queries/ExperimentationConfigQuery'; +import Log from '../../../log'; +import SessionManager from '../../../user/SessionManager'; +import { Actor } from '../../../user/User'; +import { ContextOptions } from '../ContextField'; +import ExperimentationContextField from '../ExperimentationContextField'; +import { createGraphqlClient } from '../contextUtils/createGraphqlClient'; + +jest.mock('../../../env', () => ({ + __esModule: true, + default: { experimentOverride: undefined as string | undefined }, +})); +jest.mock('../../../graphql/queries/ExperimentationConfigQuery'); +jest.mock('../../../log'); +jest.mock('../contextUtils/createGraphqlClient'); + +const mockEnv = jest.requireMock('../../../env').default as { experimentOverride?: string }; +const getConfigsAsyncMock = jest.mocked(ExperimentationConfigQuery.getConfigsAsync); +const fakeGraphqlClient = { fake: true } as any; + +const actor = { __typename: 'User', id: 'user-1' } as unknown as Actor; +const emptyDefinition = { experiments: [], namespaces: [] }; +const allConfigs = { + userConfig: TEST_DEFINITION, + accountConfig: TEST_DEFINITION, + deviceConfig: TEST_DEFINITION, +}; + +function createAnalytics(deviceId: string | null = 'device-1'): AnalyticsWithOrchestration & { + logEvent: jest.Mock; +} { + return { + logEvent: jest.fn(), + setActor: jest.fn(), + flushAsync: jest.fn(), + getDeviceId: jest.fn(() => deviceId), + }; +} + +function createSessionManager({ loggedOut = false }: { loggedOut?: boolean } = {}): { + sessionManager: SessionManager; + mocked: SessionManager; +} { + const mocked = mock(); + when(mocked.getAccessToken()).thenReturn(null); + when(mocked.getSessionSecret()).thenReturn(loggedOut ? null : 'secret'); + when(mocked.getUserAsync()).thenResolve(loggedOut ? undefined : actor); + return { sessionManager: instance(mocked), mocked }; +} + +function options(overrides: Partial = {}): ContextOptions { + return { + nonInteractive: false, + sessionManager: createSessionManager().sessionManager, + analytics: createAnalytics(), + ...overrides, + } as ContextOptions; +} + +async function flushPromisesAsync(): Promise { + await new Promise(resolve => setImmediate(resolve)); +} + +afterAll(() => { + process.env = ORIGINAL_ENV; +}); + +const ORIGINAL_ENV = process.env; + +beforeEach(() => { + jest.clearAllMocks(); + process.env = { ...ORIGINAL_ENV }; + delete process.env.CI; + delete process.env.EAS_BUILD; + mockEnv.experimentOverride = undefined; + jest.mocked(createGraphqlClient).mockReturnValue(fakeGraphqlClient); + getConfigsAsyncMock.mockResolvedValue(allConfigs); +}); + +describe(ExperimentationContextField, () => { + it('returns a disabled client and fetches nothing when analytics is disabled', async () => { + const { sessionManager, mocked } = createSessionManager(); + const client = await new ExperimentationContextField().getValueAsync( + options({ analytics: createAnalytics(null), sessionManager }) + ); + + expect(client.getUserNamespace('cli-full').getParam('variant', 'control')).toBe('control'); + expect(client.getDeviceNamespace('cli-full').getParam('variant', 'control')).toBe('control'); + expect(getConfigsAsyncMock).not.toHaveBeenCalled(); + expect(createGraphqlClient).not.toHaveBeenCalled(); + verify(mocked.getUserAsync()).never(); + }); + + it.each(['CI', 'EAS_BUILD'])( + 'returns a disabled client and fetches nothing when %s is set', + async flag => { + process.env[flag] = '1'; + const { sessionManager, mocked } = createSessionManager(); + const client = await new ExperimentationContextField().getValueAsync( + options({ sessionManager }) + ); + + expect(client.getUserNamespace('cli-full').getParam('variant', 'control')).toBe('control'); + expect(client.getDeviceNamespace('cli-full').getParam('variant', 'control')).toBe('control'); + expect(getConfigsAsyncMock).not.toHaveBeenCalled(); + expect(createGraphqlClient).not.toHaveBeenCalled(); + verify(mocked.getUserAsync()).never(); + } + ); + + it('still honors EAS_EXPERIMENT_OVERRIDE in CI', async () => { + process.env.CI = 'true'; + mockEnv.experimentOverride = 'cli-full:variant=treatment'; + const client = await new ExperimentationContextField().getValueAsync(options()); + expect(client.getUserNamespace('cli-full').getParam('variant', 'control')).toBe('treatment'); + expect(getConfigsAsyncMock).not.toHaveBeenCalled(); + }); + + it('still honors EAS_EXPERIMENT_OVERRIDE when analytics is disabled', async () => { + mockEnv.experimentOverride = 'cli-full:variant=treatment'; + const client = await new ExperimentationContextField().getValueAsync( + options({ analytics: createAnalytics(null) }) + ); + expect(client.getUserNamespace('cli-full').getParam('variant', 'control')).toBe('treatment'); + }); + + it('buckets each scope on its own unit and logs exposures through analytics', async () => { + const analytics = createAnalytics('device-1'); + const client = await new ExperimentationContextField().getValueAsync(options({ analytics })); + + client.getUserNamespace('cli-full').getParam('variant', 'control'); + client + .getAccountNamespace('cli-full', { accountId: 'account-1' }) + .getParam('variant', 'control'); + client.getDeviceNamespace('cli-full').getParam('variant', 'control'); + + const units = analytics.logEvent.mock.calls.map(([, properties]) => properties.unit); + expect(units).toEqual(['user-1', 'account-1', 'device-1']); + expect(createGraphqlClient).toHaveBeenCalledWith( + { accessToken: null, sessionSecret: 'secret' }, + { requestTimeoutMs: 3000 } + ); + }); + + it('gives logged-out users defaults for user experiments but still runs device experiments', async () => { + const analytics = createAnalytics('device-1'); + const { sessionManager } = createSessionManager({ loggedOut: true }); + const client = await new ExperimentationContextField().getValueAsync( + options({ analytics, sessionManager }) + ); + + expect(client.getUserNamespace('cli-full').getParam('variant', 'control')).toBe('control'); + expect(['control', 'treatment']).toContain( + client.getDeviceNamespace('cli-full').getParam('variant', 'x') + ); + expect(analytics.logEvent).toHaveBeenCalledTimes(1); + }); + + it('fails open to defaults when the config fetch rejects', async () => { + getConfigsAsyncMock.mockRejectedValue(new Error('offline')); + const analytics = createAnalytics(); + const client = await new ExperimentationContextField().getValueAsync(options({ analytics })); + + expect(client.getUserNamespace('cli-full').getParam('variant', 'control')).toBe('control'); + expect(client.getDeviceNamespace('cli-full').getParam('variant', 'control')).toBe('control'); + expect(analytics.logEvent).not.toHaveBeenCalled(); + expect(Log.debug).toHaveBeenCalledWith( + expect.stringContaining('Failed to fetch'), + expect.anything() + ); + expect(Log.error).not.toHaveBeenCalled(); + expect(Log.warn).not.toHaveBeenCalled(); + }); + + it('returns defaults and logs nothing for empty configs, as the server sends for robots', async () => { + getConfigsAsyncMock.mockResolvedValue({ + userConfig: emptyDefinition, + accountConfig: emptyDefinition, + deviceConfig: emptyDefinition, + }); + const analytics = createAnalytics(); + const client = await new ExperimentationContextField().getValueAsync(options({ analytics })); + + expect(client.getUserNamespace('cli-full').getParam('variant', 'control')).toBe('control'); + expect(analytics.logEvent).not.toHaveBeenCalled(); + }); + + it('fails open to defaults when the server config is invalid', async () => { + getConfigsAsyncMock.mockResolvedValue({ + ...allConfigs, + userConfig: { + experiments: TEST_DEFINITION.experiments, + namespaces: [ + { + name: 'too-many', + numSegments: 10, + segmentExperimentSetupDefinitions: [ + { method: 'ADD', name: 'all', experimentName: 'cli-test', numSegments: 20 }, + ], + }, + ], + }, + }); + const client = await new ExperimentationContextField().getValueAsync(options()); + + expect(client.getDeviceNamespace('cli-full').getParam('variant', 'control')).toBe('control'); + expect(Log.debug).toHaveBeenCalledWith( + expect.stringContaining('Failed to initialize'), + expect.anything() + ); + }); + + it('logs account exposures on the account ID the command passes', async () => { + const analytics = createAnalytics(); + const client = await new ExperimentationContextField().getValueAsync(options({ analytics })); + client + .getAccountNamespace('cli-full', { accountId: 'picked-account' }) + .getParam('variant', 'x'); + expect(analytics.logEvent).toHaveBeenCalledWith( + 'Experiment Viewed', + expect.objectContaining({ unit: 'picked-account' }) + ); + }); + + it('runs the actor lookup and the config fetch in parallel', async () => { + let resolveConfigs!: (value: typeof allConfigs) => void; + let resolveActor!: (value: Actor) => void; + getConfigsAsyncMock.mockReturnValue(new Promise(resolve => (resolveConfigs = resolve))); + const mocked = mock(); + when(mocked.getAccessToken()).thenReturn(null); + when(mocked.getSessionSecret()).thenReturn('secret'); + when(mocked.getUserAsync()).thenReturn(new Promise(resolve => (resolveActor = resolve))); + + const pending = new ExperimentationContextField().getValueAsync( + options({ sessionManager: instance(mocked) }) + ); + await flushPromisesAsync(); + + expect(getConfigsAsyncMock).toHaveBeenCalledTimes(1); + verify(mocked.getUserAsync()).once(); + + resolveConfigs(allConfigs); + resolveActor(actor); + await expect(pending).resolves.toBeDefined(); + }); + + it('caches one client per analytics instance so repeated context calls do not log duplicate exposures', async () => { + const field = new ExperimentationContextField(); + const analytics = createAnalytics(); + const first = await field.getValueAsync(options({ analytics })); + const second = await field.getValueAsync(options({ analytics })); + const other = await field.getValueAsync(options({ analytics: createAnalytics() })); + + expect(second).toBe(first); + expect(other).not.toBe(first); + expect(getConfigsAsyncMock).toHaveBeenCalledTimes(2); + }); +}); diff --git a/packages/eas-cli/src/commandUtils/context/contextUtils/__tests__/createGraphqlClient-test.ts b/packages/eas-cli/src/commandUtils/context/contextUtils/__tests__/createGraphqlClient-test.ts new file mode 100644 index 0000000000..c3d0f3c389 --- /dev/null +++ b/packages/eas-cli/src/commandUtils/context/contextUtils/__tests__/createGraphqlClient-test.ts @@ -0,0 +1,124 @@ +import { createRequest } from '@urql/core'; +import http from 'http'; +import { AddressInfo } from 'net'; + +import { createGraphqlClient } from '../createGraphqlClient'; + +jest.mock('../../../../api', () => ({ + getExpoApiBaseUrl: () => process.env.CREATE_GRAPHQL_CLIENT_TEST_URL, +})); + +// A plain string keeps this document out of GraphQL codegen, which scans gql tags in src/**. +const TEST_QUERY = 'query CreateGraphqlClientTest { __typename }'; + +function fetchOptionsOf(client: ReturnType): RequestInit { + const operation = client.createRequestOperation('query', createRequest(TEST_QUERY, {})); + const { fetchOptions } = operation.context; + return typeof fetchOptions === 'function' ? fetchOptions() : (fetchOptions ?? {}); +} + +/** Serves one GraphQL response after `delayMs`. Returns the base URL. */ +async function startSlowServerAsync(server: http.Server, delayMs: number): Promise { + server.on('request', (_req, res) => { + setTimeout(() => { + res.setHeader('content-type', 'application/json'); + res.end(JSON.stringify({ data: { __typename: 'RootQuery' } })); + }, delayMs); + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + const { port } = server.address() as AddressInfo; + return `http://127.0.0.1:${port}`; +} + +describe(createGraphqlClient, () => { + it('sends the access token as a bearer header', () => { + const init = fetchOptionsOf(createGraphqlClient({ accessToken: 'token', sessionSecret: null })); + expect(init.headers).toEqual({ authorization: 'Bearer token' }); + }); + + it('sends the session secret when there is no access token', () => { + const init = fetchOptionsOf( + createGraphqlClient({ accessToken: null, sessionSecret: 'secret' }) + ); + expect(init.headers).toEqual({ 'expo-session': 'secret' }); + }); + + it('keeps the auth headers when a request timeout is set', () => { + const init = fetchOptionsOf( + createGraphqlClient({ accessToken: 'token', sessionSecret: null }, { requestTimeoutMs: 50 }) + ); + expect(init.headers).toEqual({ authorization: 'Bearer token' }); + }); + + describe('against a slow server', () => { + const server = http.createServer(); + const serverDelayMs = 2000; + + beforeAll(async () => { + process.env.CREATE_GRAPHQL_CLIENT_TEST_URL = await startSlowServerAsync( + server, + serverDelayMs + ); + }); + + afterAll(async () => { + delete process.env.CREATE_GRAPHQL_CLIENT_TEST_URL; + server.closeAllConnections(); + await new Promise(resolve => { + server.close(() => { + resolve(); + }); + }); + }); + + it('aborts the request when the timeout elapses', async () => { + // Only a real request proves the timeout works. + const client = createGraphqlClient( + { accessToken: 'token', sessionSecret: null }, + { requestTimeoutMs: 100 } + ); + const start = Date.now(); + const result = await client.query(TEST_QUERY, {}, { noRetry: true }).toPromise(); + expect(Date.now() - start).toBeLessThan(serverDelayMs); + expect(result.data).toBeUndefined(); + expect(result.error?.networkError?.name).toBe('TimeoutError'); + expect(result.error?.networkError?.message).toBe('Request timed out after 100 ms'); + }); + + it('waits for the response when no timeout is set', async () => { + const client = createGraphqlClient({ accessToken: 'token', sessionSecret: null }); + const result = await client.query(TEST_QUERY, {}, { noRetry: true }).toPromise(); + expect(result.error).toBeUndefined(); + expect(result.data).toEqual({ __typename: 'RootQuery' }); + }); + }); + + describe('against a closed port', () => { + beforeAll(async () => { + // Listen once to reserve a free port, then close so the port refuses connections. + const server = http.createServer(); + process.env.CREATE_GRAPHQL_CLIENT_TEST_URL = await startSlowServerAsync(server, 0); + await new Promise(resolve => { + server.close(() => { + resolve(); + }); + }); + }); + + afterAll(() => { + delete process.env.CREATE_GRAPHQL_CLIENT_TEST_URL; + }); + + it('passes through fetch errors that are not timeouts', async () => { + const client = createGraphqlClient( + { accessToken: 'token', sessionSecret: null }, + { requestTimeoutMs: 5000 } + ); + const result = await client.query(TEST_QUERY, {}, { noRetry: true }).toPromise(); + expect(result.data).toBeUndefined(); + expect(result.error?.networkError).toBeDefined(); + expect(result.error?.networkError?.name).not.toBe('TimeoutError'); + expect(result.error?.networkError?.message).toContain('ECONNREFUSED'); + }); + }); +}); diff --git a/packages/eas-cli/src/commandUtils/context/contextUtils/createGraphqlClient.ts b/packages/eas-cli/src/commandUtils/context/contextUtils/createGraphqlClient.ts index 633e090453..98272e203d 100644 --- a/packages/eas-cli/src/commandUtils/context/contextUtils/createGraphqlClient.ts +++ b/packages/eas-cli/src/commandUtils/context/contextUtils/createGraphqlClient.ts @@ -1,6 +1,10 @@ import { Client, cacheExchange, createClient as createUrqlClient, fetchExchange } from '@urql/core'; import { retryExchange } from '@urql/exchange-retry'; -import fetch from 'node-fetch'; +import fetch, { + RequestInfo as NodeFetchRequestInfo, + RequestInit as NodeFetchRequestInit, + Response as NodeFetchResponse, +} from 'node-fetch'; import { getExpoApiBaseUrl } from '../../../api'; import { httpsProxyAgent } from '../../../fetch'; @@ -8,10 +12,46 @@ import { httpsProxyAgent } from '../../../fetch'; // Extend the urql Client to allow for future customizations or type additions ahead of upstream type support. export interface ExpoGraphqlClient extends Client {} -export function createGraphqlClient(authInfo: { - accessToken: string | null; - sessionSecret: string | null; -}): ExpoGraphqlClient { +export function createGraphqlClient( + authInfo: { + accessToken: string | null; + sessionSecret: string | null; + }, + options: { + /** Request timeout. Kept at the client level: a per-operation `fetchOptions` in urql would replace the auth headers. */ + requestTimeoutMs?: number; + } = {} +): ExpoGraphqlClient { + const { requestTimeoutMs } = options; + // urql overwrites `fetchOptions.signal`, so the timeout must be added inside fetch. + const fetchWithTimeoutAsync = async ( + url: NodeFetchRequestInfo, + init?: NodeFetchRequestInit + ): Promise => { + if (!requestTimeoutMs) { + return await fetch(url, init); + } + const timeoutSignal = AbortSignal.timeout(requestTimeoutMs); + try { + return await fetch(url, { + ...init, + signal: AbortSignal.any([ + ...(init?.signal ? [init.signal as AbortSignal] : []), + timeoutSignal, + ]), + }); + } catch (error) { + if (timeoutSignal.aborted) { + const timeoutError = new Error(`Request timed out after ${requestTimeoutMs} ms`, { + cause: error, + }); + timeoutError.name = 'TimeoutError'; + throw timeoutError; + } + throw error; + } + }; + return createUrqlClient({ url: getExpoApiBaseUrl() + '/graphql', exchanges: [ @@ -29,7 +69,7 @@ export function createGraphqlClient(authInfo: { fetchExchange, ], // @ts-expect-error Type 'typeof fetch' is not assignable to type '(input: RequestInfo, init?: RequestInit | undefined) => Promise'. - fetch, + fetch: fetchWithTimeoutAsync, fetchOptions: (): RequestInit => { const headers: Record = {}; if (authInfo.accessToken) { diff --git a/packages/eas-cli/src/commands/account/__tests__/login.test.ts b/packages/eas-cli/src/commands/account/__tests__/login.test.ts index 30717e51a9..ada438fd5d 100644 --- a/packages/eas-cli/src/commands/account/__tests__/login.test.ts +++ b/packages/eas-cli/src/commands/account/__tests__/login.test.ts @@ -28,6 +28,7 @@ const sessionManager = new SessionManager({ logEvent: jest.fn(), setActor: jest.fn(), flushAsync: jest.fn(), + getDeviceId: jest.fn(), }); const originalIsTTY = process.stdin.isTTY; const originalCI = process.env.CI; diff --git a/packages/eas-cli/src/commands/analytics.ts b/packages/eas-cli/src/commands/analytics.ts index 5975137389..e147acfbf8 100644 --- a/packages/eas-cli/src/commands/analytics.ts +++ b/packages/eas-cli/src/commands/analytics.ts @@ -4,7 +4,8 @@ import EasCommand from '../commandUtils/EasCommand'; import Log from '../log'; export default class AnalyticsView extends EasCommand { - static override description = 'display or change analytics settings'; + static override description = + 'display or change analytics settings (disabling analytics also disables experiments)'; static override args = { STATUS: Args.string({ diff --git a/packages/eas-cli/src/commands/build/index.ts b/packages/eas-cli/src/commands/build/index.ts index 77460b0d0a..7ffa923214 100644 --- a/packages/eas-cli/src/commands/build/index.ts +++ b/packages/eas-cli/src/commands/build/index.ts @@ -140,6 +140,7 @@ export default class Build extends EasCommand { ...this.ContextOptions.ProjectDir, ...this.ContextOptions.Analytics, ...this.ContextOptions.Vcs, + ...this.ContextOptions.Experimentation, }; async runAsync(): Promise { @@ -156,6 +157,7 @@ export default class Build extends EasCommand { projectDir, analytics, vcsClient, + experimentation, } = await this.getContextAsync(Build, { nonInteractive: flags.nonInteractive, withServerSideEnvironment: null, @@ -182,6 +184,7 @@ export default class Build extends EasCommand { flags: flagsWithPlatform, actor, getDynamicPrivateProjectConfigAsync, + experimentation, }); this.maybeSuggestUsingEasBuildDev({ diff --git a/packages/eas-cli/src/env.ts b/packages/eas-cli/src/env.ts index 42ff6eb234..55258d6a89 100644 --- a/packages/eas-cli/src/env.ts +++ b/packages/eas-cli/src/env.ts @@ -21,4 +21,10 @@ export default { * Comma separated list of feature gate keys of feature gates to evaluate override to false. */ featureGateDisable: process.env.EAS_FG_DISABLE, + /** + * Comma separated list of experiment overrides in the form `namespace:param=value`. + * Overrides apply in every scope, even when analytics is disabled. A namespace with any override + * returns defaults for its other params and logs no exposure. + */ + experimentOverride: process.env.EAS_EXPERIMENT_OVERRIDE, }; diff --git a/packages/eas-cli/src/experimentation/AnalyticsExposureLogger.ts b/packages/eas-cli/src/experimentation/AnalyticsExposureLogger.ts new file mode 100644 index 0000000000..84359f2f29 --- /dev/null +++ b/packages/eas-cli/src/experimentation/AnalyticsExposureLogger.ts @@ -0,0 +1,18 @@ +import type { ExposureData, ExposureLogger } from '@expo/experimentation'; + +import { Analytics, ExperimentationEvent } from '../analytics/AnalyticsManager'; + +export class AnalyticsExposureLogger implements ExposureLogger> { + constructor(private readonly analytics: Analytics) {} + + public logExposure(data: ExposureData>): void { + const { variant } = data.params; + this.analytics.logEvent(ExperimentationEvent.EXPERIMENT_VIEWED, { + experimentName: data.name, + // Analytics properties cannot be undefined. Fall back to all params when there is no variant. + variationName: variant !== undefined ? String(variant) : JSON.stringify(data.params), + // The unit is always a single value (actor ID, account ID, or device ID). + unit: String(data.unit[0]), + }); + } +} diff --git a/packages/eas-cli/src/experimentation/ExperimentNamespace.ts b/packages/eas-cli/src/experimentation/ExperimentNamespace.ts new file mode 100644 index 0000000000..7b68d85c17 --- /dev/null +++ b/packages/eas-cli/src/experimentation/ExperimentNamespace.ts @@ -0,0 +1,79 @@ +import type { Namespace } from '@expo/experimentation'; + +import { ExperimentOverrides } from './ExperimentOverrides'; +import Log from '../log'; + +export type ExperimentNamespace = { + getParam(paramName: string, defaultValue: T): T; + isOverridden: boolean; +}; + +export const FALLBACK_NAMESPACE: ExperimentNamespace = { + getParam(_paramName, defaultValue) { + return defaultValue; + }, + isOverridden: false, +}; + +/** Serves overrides without consulting the real namespace, so no exposure is logged. */ +export class OverrideFallbackNamespace implements ExperimentNamespace { + public readonly isOverridden = true; + + constructor( + private readonly namespaceName: string, + private readonly overrides: ExperimentOverrides + ) {} + + public getParam(paramName: string, defaultValue: T): T { + if (this.overrides.hasOverride(this.namespaceName, paramName)) { + return coerceOverrideValue( + this.overrides.getOverride(this.namespaceName, paramName), + defaultValue + ); + } + return defaultValue; + } +} + +/** Returns the default instead of throwing when the library rejects a param on first read. */ +export class SafeNamespace implements ExperimentNamespace { + public readonly isOverridden = false; + + constructor( + private readonly namespace: Namespace, + private readonly namespaceName: string + ) {} + + public getParam(paramName: string, defaultValue: T): T { + try { + return this.namespace.getParam(paramName, defaultValue); + } catch (error) { + Log.debug( + `[experimentation] Failed to read param "${paramName}" from namespace "${this.namespaceName}". Returning the default.`, + error + ); + return defaultValue; + } + } +} + +export function coerceOverrideValue(overrideValue: string | undefined, defaultValue: T): T { + if (overrideValue === undefined || overrideValue.trim() === '') { + return defaultValue; + } + if (typeof defaultValue === 'boolean') { + const normalized = overrideValue.toLowerCase(); + if (normalized === 'true') { + return true as T; + } + if (normalized === 'false') { + return false as T; + } + return defaultValue; + } + if (typeof defaultValue === 'number') { + const parsed = Number(overrideValue); + return (Number.isFinite(parsed) ? parsed : defaultValue) as T; + } + return overrideValue as T; +} diff --git a/packages/eas-cli/src/experimentation/ExperimentOverrides.ts b/packages/eas-cli/src/experimentation/ExperimentOverrides.ts new file mode 100644 index 0000000000..c9409c8d06 --- /dev/null +++ b/packages/eas-cli/src/experimentation/ExperimentOverrides.ts @@ -0,0 +1,73 @@ +import env from '../env'; + +export type ExperimentOverrideMap = ReadonlyMap>; + +/** + * Parses `EAS_EXPERIMENT_OVERRIDE`: comma-separated `namespace:param=value` entries. Values may + * contain `:` and `=` but not `,`. Overrides are keyed by namespace name and apply in every scope. + * + * @example EAS_EXPERIMENT_OVERRIDE="onboarding:variant=treatment,cli-hints:enabled=true" + */ +export function parseExperimentOverrides(rawValue: string | undefined): ExperimentOverrideMap { + const overrideMap = new Map>(); + if (!rawValue) { + return overrideMap; + } + + for (const entry of rawValue.split(',')) { + const override = entry.trim(); + if (!override) { + continue; + } + + const firstEqualsIndex = override.indexOf('='); + if (firstEqualsIndex === -1) { + continue; + } + const namespaceAndParam = override.slice(0, firstEqualsIndex); + const value = override.slice(firstEqualsIndex + 1).trim(); + + const firstColonIndex = namespaceAndParam.indexOf(':'); + if (firstColonIndex === -1) { + continue; + } + const namespace = namespaceAndParam.slice(0, firstColonIndex).trim(); + const param = namespaceAndParam.slice(firstColonIndex + 1).trim(); + if (!namespace || !param) { + continue; + } + + let params = overrideMap.get(namespace); + if (!params) { + params = new Map(); + overrideMap.set(namespace, params); + } + params.set(param, value); + } + + return overrideMap; +} + +export class ExperimentOverrides { + constructor(private readonly overrideMap: ExperimentOverrideMap) {} + + public static fromEnv(): ExperimentOverrides { + return new ExperimentOverrides(parseExperimentOverrides(env.experimentOverride)); + } + + public static empty(): ExperimentOverrides { + return new ExperimentOverrides(new Map()); + } + + public hasNamespaceOverride(namespace: string): boolean { + return this.overrideMap.has(namespace); + } + + public hasOverride(namespace: string, param: string): boolean { + return this.overrideMap.get(namespace)?.has(param) ?? false; + } + + public getOverride(namespace: string, param: string): string | undefined { + return this.overrideMap.get(namespace)?.get(param); + } +} diff --git a/packages/eas-cli/src/experimentation/ExperimentationClient.ts b/packages/eas-cli/src/experimentation/ExperimentationClient.ts new file mode 100644 index 0000000000..4773214efd --- /dev/null +++ b/packages/eas-cli/src/experimentation/ExperimentationClient.ts @@ -0,0 +1,175 @@ +import { + type ExperimentUnit, + type ExperimentationDefinition, + type LoggerConfig, + type Namespace, + NoOpExposureLogger, + parseExperimentationConfig, +} from '@expo/experimentation'; +import { createHash } from 'node:crypto'; + +import { AnalyticsExposureLogger } from './AnalyticsExposureLogger'; +import { + type ExperimentNamespace, + FALLBACK_NAMESPACE, + OverrideFallbackNamespace, + SafeNamespace, +} from './ExperimentNamespace'; +import { ExperimentOverrides } from './ExperimentOverrides'; +import { Analytics } from '../analytics/AnalyticsManager'; +import Log from '../log'; + +export type ExperimentationScope = 'user' | 'account' | 'device'; + +export type ExperimentationConfigs = Record; + +/** Commands pass the account unit on each `getAccountNamespace` call. */ +export type ExperimentationUnits = Record, string | null>; + +export const EMPTY_EXPERIMENTATION_DEFINITION: ExperimentationDefinition = { + experiments: [], + namespaces: [], +}; + +const EMPTY_CONFIGS: ExperimentationConfigs = { + user: EMPTY_EXPERIMENTATION_DEFINITION, + account: EMPTY_EXPERIMENTATION_DEFINITION, + device: EMPTY_EXPERIMENTATION_DEFINITION, +}; + +const NULL_UNITS: ExperimentationUnits = { user: null, device: null }; + +function sha1(input: string): string { + return createHash('sha1').update(input).digest('hex'); +} + +export function createLoggerConfig(analytics: Analytics): LoggerConfig { + return { + flavorMap: new Map([ + ['no-op', new NoOpExposureLogger()], + ['rudderstack', new AnalyticsExposureLogger(analytics)], + ]), + fallbackLoggerType: 'no-op', + }; +} + +const NO_OP_LOGGER_CONFIG: LoggerConfig = { + flavorMap: new Map([['no-op', new NoOpExposureLogger()]]), + fallbackLoggerType: 'no-op', +}; + +/** Cached per (unit, name): the library deduplicates exposures per Namespace instance. */ +class ScopedNamespaces { + private readonly factories: ReadonlyMap Namespace>; + private readonly instances = new Map>(); + + constructor( + definition: ExperimentationDefinition, + loggerConfig: LoggerConfig, + defaultUnit: string | null + ) { + this.factories = parseExperimentationConfig(definition, { sha1Fn: sha1 }, loggerConfig); + // Build every namespace now so an invalid server config throws here, in the fail-open path. + if (defaultUnit) { + for (const name of this.factories.keys()) { + this.getNamespace(name, defaultUnit); + } + } + } + + public getNamespace(name: string, unit: string): Namespace | null { + const factory = this.factories.get(name); + if (!factory) { + return null; + } + let byName = this.instances.get(unit); + if (!byName) { + byName = new Map(); + this.instances.set(unit, byName); + } + let namespace = byName.get(name); + if (!namespace) { + namespace = factory([unit]); + byName.set(name, namespace); + } + return namespace; + } +} + +/** + * Resolves experiment params for the current CLI run. Units: user is the actor ID, account is the + * ID the command passes to `getAccountNamespace`, device is the analytics device ID. + * User and account units are shared across surfaces. The device unit is specific to the CLI, so a + * device namespace must not be shared with another surface. + * The first `getParam` on a namespace logs an exposure, so call it only where the variant is used. + */ +export class ExperimentationClient { + private readonly scopes: Record; + + constructor( + configs: ExperimentationConfigs, + private readonly units: ExperimentationUnits, + private readonly overrides: ExperimentOverrides, + loggerConfig: LoggerConfig + ) { + this.scopes = { + user: new ScopedNamespaces(configs.user, loggerConfig, units.user), + account: new ScopedNamespaces(configs.account, loggerConfig, null), + device: new ScopedNamespaces(configs.device, loggerConfig, units.device), + }; + } + + /** Env overrides still apply, so QA can force a variant even with analytics disabled. */ + public static createDisabled( + overrides: ExperimentOverrides = ExperimentOverrides.fromEnv() + ): ExperimentationClient { + return new ExperimentationClient(EMPTY_CONFIGS, NULL_UNITS, overrides, NO_OP_LOGGER_CONFIG); + } + + public getUserNamespace(name: string): ExperimentNamespace { + return this.resolve('user', name, this.units.user); + } + + /** Takes the account the command already resolved, so no extra lookup or config read happens here. */ + public getAccountNamespace( + name: string, + { accountId }: { accountId: string } + ): ExperimentNamespace { + return this.resolve('account', name, accountId); + } + + public getDeviceNamespace(name: string): ExperimentNamespace { + return this.resolve('device', name, this.units.device); + } + + private resolve( + scope: ExperimentationScope, + name: string, + unit: string | null + ): ExperimentNamespace { + if (this.overrides.hasNamespaceOverride(name)) { + return new OverrideFallbackNamespace(name, this.overrides); + } + if (!unit) { + return FALLBACK_NAMESPACE; + } + + let namespace: Namespace | null; + try { + namespace = this.scopes[scope].getNamespace(name, unit); + } catch (error) { + Log.debug( + `[experimentation] Failed to build namespace "${name}" in the ${scope} config. Returning defaults.`, + error + ); + return FALLBACK_NAMESPACE; + } + if (!namespace) { + Log.debug( + `[experimentation] Namespace "${name}" not found in the ${scope} config. Returning defaults.` + ); + return FALLBACK_NAMESPACE; + } + return new SafeNamespace(namespace, name); + } +} diff --git a/packages/eas-cli/src/experimentation/__tests__/AnalyticsExposureLogger-test.ts b/packages/eas-cli/src/experimentation/__tests__/AnalyticsExposureLogger-test.ts new file mode 100644 index 0000000000..cdb43b7fc5 --- /dev/null +++ b/packages/eas-cli/src/experimentation/__tests__/AnalyticsExposureLogger-test.ts @@ -0,0 +1,124 @@ +import RudderAnalytics from '@expo/rudder-sdk-node'; + +import UserSettings from '../../user/UserSettings'; +import { createAnalyticsAsync } from '../../analytics/AnalyticsManager'; +import { AnalyticsExposureLogger } from '../AnalyticsExposureLogger'; +import { createMockAnalytics } from './fixtures'; + +const mockIdentify = jest.fn(); +const mockTrack = jest.fn(); +const mockFlush = jest.fn(); + +jest.mock('@expo/rudder-sdk-node', () => + jest.fn().mockImplementation(() => ({ + identify: mockIdentify, + track: mockTrack, + flush: mockFlush, + })) +); +jest.mock('../../user/UserSettings', () => ({ + __esModule: true, + default: { + deleteKeyAsync: jest.fn(), + getAsync: jest.fn(), + setAsync: jest.fn(), + }, +})); +jest.mock('../../analytics/agent', () => ({ getAgentTelemetryContext: jest.fn() })); +jest.mock('../../analytics/sandbox', () => ({ getSandboxTelemetryContext: jest.fn() })); + +const originalDisableEasAnalytics = process.env.DISABLE_EAS_ANALYTICS; +const originalHttpsProxy = process.env.https_proxy; + +afterAll(() => { + if (originalDisableEasAnalytics === undefined) { + delete process.env.DISABLE_EAS_ANALYTICS; + } else { + process.env.DISABLE_EAS_ANALYTICS = originalDisableEasAnalytics; + } + if (originalHttpsProxy === undefined) { + delete process.env.https_proxy; + } else { + process.env.https_proxy = originalHttpsProxy; + } +}); + +const exposure = { + name: 'cli-test', + time: 1700000000, + salt: 'cli-full.cli-test.cli-test', +}; + +describe(AnalyticsExposureLogger, () => { + it('logs Experiment Viewed with experimentName, variationName, and unit', () => { + const analytics = createMockAnalytics(); + new AnalyticsExposureLogger(analytics).logExposure({ + ...exposure, + unit: ['account-1'], + params: { variant: 'treatment' }, + }); + expect(analytics.logEvent).toHaveBeenCalledWith('Experiment Viewed', { + experimentName: 'cli-test', + variationName: 'treatment', + unit: 'account-1', + }); + }); + + it('never sends an undefined variationName', () => { + const analytics = createMockAnalytics(); + new AnalyticsExposureLogger(analytics).logExposure({ + ...exposure, + unit: ['account-1'], + params: { enabled: true, count: 2 }, + }); + expect(analytics.logEvent).toHaveBeenCalledWith( + 'Experiment Viewed', + expect.objectContaining({ variationName: JSON.stringify({ enabled: true, count: 2 }) }) + ); + }); + + it('sends numeric units and variants as strings', () => { + const analytics = createMockAnalytics(); + new AnalyticsExposureLogger(analytics).logExposure({ + ...exposure, + unit: [42], + params: { variant: 3 }, + }); + expect(analytics.logEvent).toHaveBeenCalledWith( + 'Experiment Viewed', + expect.objectContaining({ unit: '42', variationName: '3' }) + ); + }); + + it('reaches RudderStack with the CLI source and anonymous ID', async () => { + jest.mocked(RudderAnalytics).mockClear(); + mockTrack.mockClear(); + jest + .mocked(UserSettings.getAsync) + .mockImplementation(async (key, defaultValue) => + key === 'analyticsDeviceId' ? 'persistent-device-id' : defaultValue + ); + delete process.env.DISABLE_EAS_ANALYTICS; + delete process.env.https_proxy; + + const analytics = await createAnalyticsAsync(); + new AnalyticsExposureLogger(analytics).logExposure({ + ...exposure, + unit: ['account-1'], + params: { variant: 'treatment' }, + }); + + expect(mockTrack).toHaveBeenCalledWith( + expect.objectContaining({ + event: 'Experiment Viewed', + anonymousId: 'persistent-device-id', + properties: expect.objectContaining({ + experimentName: 'cli-test', + variationName: 'treatment', + unit: 'account-1', + source: 'eas cli', + }), + }) + ); + }); +}); diff --git a/packages/eas-cli/src/experimentation/__tests__/ExperimentNamespace-test.ts b/packages/eas-cli/src/experimentation/__tests__/ExperimentNamespace-test.ts new file mode 100644 index 0000000000..749ec23395 --- /dev/null +++ b/packages/eas-cli/src/experimentation/__tests__/ExperimentNamespace-test.ts @@ -0,0 +1,74 @@ +import type { Namespace } from '@expo/experimentation'; + +import Log from '../../log'; +import { + FALLBACK_NAMESPACE, + OverrideFallbackNamespace, + SafeNamespace, + coerceOverrideValue, +} from '../ExperimentNamespace'; +import { ExperimentOverrides, parseExperimentOverrides } from '../ExperimentOverrides'; + +jest.mock('../../log'); + +describe(coerceOverrideValue, () => { + it('coerces booleans only from "true" and "false"', () => { + expect(coerceOverrideValue('false', true)).toBe(false); + expect(coerceOverrideValue('TRUE', false)).toBe(true); + expect(coerceOverrideValue('1', false)).toBe(false); + expect(coerceOverrideValue('yes', true)).toBe(true); + }); + + it('coerces finite numbers and falls back otherwise', () => { + expect(coerceOverrideValue('2', 1)).toBe(2); + expect(coerceOverrideValue('2.5', 1)).toBe(2.5); + expect(coerceOverrideValue('x', 1)).toBe(1); + expect(coerceOverrideValue('Infinity', 1)).toBe(1); + }); + + it('passes strings through and returns the default for empty input', () => { + expect(coerceOverrideValue('b', 'a')).toBe('b'); + expect(coerceOverrideValue('', 'a')).toBe('a'); + expect(coerceOverrideValue(' ', 'a')).toBe('a'); + expect(coerceOverrideValue(undefined, 'a')).toBe('a'); + }); +}); + +describe('FALLBACK_NAMESPACE', () => { + it('returns the default for every param', () => { + expect(FALLBACK_NAMESPACE.getParam('variant', 'control')).toBe('control'); + expect(FALLBACK_NAMESPACE.isOverridden).toBe(false); + }); +}); + +describe(OverrideFallbackNamespace, () => { + const overrides = new ExperimentOverrides(parseExperimentOverrides('ns:variant=treatment')); + const namespace = new OverrideFallbackNamespace('ns', overrides); + + it('returns the coerced override for overridden params', () => { + expect(namespace.getParam('variant', 'control')).toBe('treatment'); + expect(namespace.isOverridden).toBe(true); + }); + + it('returns the default for params without an override', () => { + expect(namespace.getParam('enabled', false)).toBe(false); + }); +}); + +describe(SafeNamespace, () => { + it('delegates to the wrapped namespace', () => { + const inner = { getParam: jest.fn().mockReturnValue('treatment') } as unknown as Namespace; + expect(new SafeNamespace(inner, 'ns').getParam('variant', 'control')).toBe('treatment'); + expect(inner.getParam).toHaveBeenCalledWith('variant', 'control'); + }); + + it('returns the default and logs at debug level when the wrapped namespace throws', () => { + const inner = { + getParam: jest.fn(() => { + throw new Error('bad operator'); + }), + } as unknown as Namespace; + expect(new SafeNamespace(inner, 'ns').getParam('variant', 'control')).toBe('control'); + expect(Log.debug).toHaveBeenCalled(); + }); +}); diff --git a/packages/eas-cli/src/experimentation/__tests__/ExperimentOverrides-test.ts b/packages/eas-cli/src/experimentation/__tests__/ExperimentOverrides-test.ts new file mode 100644 index 0000000000..15e25d2f46 --- /dev/null +++ b/packages/eas-cli/src/experimentation/__tests__/ExperimentOverrides-test.ts @@ -0,0 +1,72 @@ +import { ExperimentOverrides, parseExperimentOverrides } from '../ExperimentOverrides'; + +jest.mock('../../env', () => ({ + __esModule: true, + default: { experimentOverride: 'from-env:variant=treatment' }, +})); + +describe(parseExperimentOverrides, () => { + it('parses a single override', () => { + const map = parseExperimentOverrides('pricing:variant=treatment'); + expect(map.get('pricing')?.get('variant')).toBe('treatment'); + }); + + it('parses multiple overrides and multiple params per namespace', () => { + const map = parseExperimentOverrides('a:variant=x,b:limit=3,a:enabled=true'); + expect(map.get('a')?.get('variant')).toBe('x'); + expect(map.get('a')?.get('enabled')).toBe('true'); + expect(map.get('b')?.get('limit')).toBe('3'); + }); + + it('returns an empty map for undefined or empty input', () => { + expect(parseExperimentOverrides(undefined).size).toBe(0); + expect(parseExperimentOverrides('').size).toBe(0); + expect(parseExperimentOverrides(' , ').size).toBe(0); + }); + + it('skips entries without "=" or ":" and entries with an empty namespace or param', () => { + const map = parseExperimentOverrides( + 'bogus,ns:x,variant=1,:variant=1,ns:=1,ok:variant=control' + ); + expect([...map.keys()]).toEqual(['ok']); + }); + + it('keeps "=" and ":" inside values', () => { + const map = parseExperimentOverrides('ns:url=https://expo.dev/a=b'); + expect(map.get('ns')?.get('url')).toBe('https://expo.dev/a=b'); + }); + + it('trims whitespace and tolerates a trailing comma', () => { + const map = parseExperimentOverrides(' ns : variant = treatment , '); + expect(map.get('ns')?.get('variant')).toBe('treatment'); + expect(map.size).toBe(1); + }); + + it('lets later duplicates win', () => { + const map = parseExperimentOverrides('ns:variant=a,ns:variant=b'); + expect(map.get('ns')?.get('variant')).toBe('b'); + }); +}); + +describe(ExperimentOverrides, () => { + it('exposes namespace and param lookups', () => { + const overrides = new ExperimentOverrides(parseExperimentOverrides('ns:variant=treatment')); + expect(overrides.hasNamespaceOverride('ns')).toBe(true); + expect(overrides.hasNamespaceOverride('other')).toBe(false); + expect(overrides.hasOverride('ns', 'variant')).toBe(true); + expect(overrides.hasOverride('ns', 'enabled')).toBe(false); + expect(overrides.hasOverride('other', 'variant')).toBe(false); + expect(overrides.getOverride('ns', 'variant')).toBe('treatment'); + expect(overrides.getOverride('ns', 'enabled')).toBeUndefined(); + expect(overrides.getOverride('other', 'variant')).toBeUndefined(); + }); + + it('reads EAS_EXPERIMENT_OVERRIDE through env', () => { + const overrides = ExperimentOverrides.fromEnv(); + expect(overrides.getOverride('from-env', 'variant')).toBe('treatment'); + }); + + it('empty() has no overrides', () => { + expect(ExperimentOverrides.empty().hasNamespaceOverride('from-env')).toBe(false); + }); +}); diff --git a/packages/eas-cli/src/experimentation/__tests__/ExperimentationClient-test.ts b/packages/eas-cli/src/experimentation/__tests__/ExperimentationClient-test.ts new file mode 100644 index 0000000000..59861712b4 --- /dev/null +++ b/packages/eas-cli/src/experimentation/__tests__/ExperimentationClient-test.ts @@ -0,0 +1,376 @@ +import type { ExperimentationDefinition } from '@expo/experimentation'; + +import Log from '../../log'; +import { + EMPTY_EXPERIMENTATION_DEFINITION, + ExperimentationClient, + ExperimentationConfigs, + ExperimentationUnits, + createLoggerConfig, +} from '../ExperimentationClient'; +import { ExperimentOverrides, parseExperimentOverrides } from '../ExperimentOverrides'; +import { TEST_DEFINITION, createMockAnalytics, findUnitsByValue } from './fixtures'; +import { FALLBACK_NAMESPACE } from '../ExperimentNamespace'; + +jest.mock('../../log'); +jest.mock('../../env', () => ({ + __esModule: true, + default: { experimentOverride: 'cli-full:variant=from-env' }, +})); + +function createClient({ + configs = {}, + units = {}, + overrides = ExperimentOverrides.empty(), + analytics = createMockAnalytics(), +}: { + configs?: Partial; + units?: Partial; + overrides?: ExperimentOverrides; + analytics?: ReturnType; +} = {}): { client: ExperimentationClient; analytics: ReturnType } { + const client = new ExperimentationClient( + { + user: configs.user ?? EMPTY_EXPERIMENTATION_DEFINITION, + account: configs.account ?? EMPTY_EXPERIMENTATION_DEFINITION, + device: configs.device ?? EMPTY_EXPERIMENTATION_DEFINITION, + }, + { user: units.user ?? null, device: units.device ?? null }, + overrides, + createLoggerConfig(analytics) + ); + return { client, analytics }; +} + +function userClientForUnit(unit: string): ExperimentationClient { + return createClient({ configs: { user: TEST_DEFINITION }, units: { user: unit } }).client; +} + +const unitsByVariant = findUnitsByValue( + unit => userClientForUnit(unit).getUserNamespace('cli-full'), + 'variant', + 'control', + ['control', 'treatment'] +); +const controlUnit = unitsByVariant.get('control')!; +const treatmentUnit = unitsByVariant.get('treatment')!; + +beforeEach(() => { + jest.clearAllMocks(); +}); + +describe(findUnitsByValue, () => { + it('throws when no unit produces an expected value', () => { + expect(() => findUnitsByValue(() => FALLBACK_NAMESPACE, 'variant', 'x', ['never'])).toThrow( + 'Could not find units for all of: never' + ); + }); +}); + +describe(ExperimentationClient, () => { + describe('assignment', () => { + it('gives the same unit the same variant across clients', () => { + const first = userClientForUnit(treatmentUnit).getUserNamespace('cli-full'); + const second = userClientForUnit(treatmentUnit).getUserNamespace('cli-full'); + expect(first.getParam('variant', 'control')).toBe('treatment'); + expect(second.getParam('variant', 'control')).toBe('treatment'); + expect( + userClientForUnit(controlUnit).getUserNamespace('cli-full').getParam('variant', 'x') + ).toBe('control'); + }); + + it('returns the default and logs at debug level for an unknown namespace', () => { + const { client, analytics } = createClient({ + configs: { user: TEST_DEFINITION }, + units: { user: treatmentUnit }, + }); + expect(client.getUserNamespace('missing').getParam('variant', 'control')).toBe('control'); + expect(Log.debug).toHaveBeenCalledWith(expect.stringContaining('"missing"')); + expect(analytics.logEvent).not.toHaveBeenCalled(); + }); + + it('returns the default and logs no exposure when the scope has no unit', () => { + const { client, analytics } = createClient({ configs: { user: TEST_DEFINITION } }); + expect(client.getUserNamespace('cli-full').getParam('variant', 'control')).toBe('control'); + expect(analytics.logEvent).not.toHaveBeenCalled(); + expect(Log.debug).not.toHaveBeenCalled(); + }); + + it('treats an empty string unit like a missing unit', () => { + const { client, analytics } = createClient({ configs: { account: TEST_DEFINITION } }); + expect( + client.getAccountNamespace('cli-full', { accountId: '' }).getParam('variant', 'control') + ).toBe('control'); + expect(analytics.logEvent).not.toHaveBeenCalled(); + }); + + it('buckets account experiments on the account ID the command passes', () => { + const { client, analytics } = createClient({ configs: { account: TEST_DEFINITION } }); + expect( + client.getAccountNamespace('cli-full', { accountId: controlUnit }).getParam('variant', 'x') + ).toBe('control'); + expect( + client + .getAccountNamespace('cli-full', { accountId: treatmentUnit }) + .getParam('variant', 'x') + ).toBe('treatment'); + expect(analytics.logEvent).toHaveBeenCalledTimes(2); + }); + + it('resolves the same namespace name independently per scope', () => { + const scoped = (choice: string): ExperimentationDefinition => ({ + experiments: [ + { + experimentName: 'shared', + paramDefinitions: { variant: { type: 'uniform_choice', choices: [choice] } }, + exposureLoggerType: 'no-op', + }, + ], + namespaces: [ + { + name: 'shared', + numSegments: 10, + segmentExperimentSetupDefinitions: [ + { method: 'ADD', name: 'all', experimentName: 'shared', numSegments: 10 }, + ], + }, + ], + }); + const { client } = createClient({ + configs: { + user: scoped('from-user'), + account: scoped('from-account'), + device: scoped('from-device'), + }, + units: { user: 'u', device: 'd' }, + }); + expect(client.getUserNamespace('shared').getParam('variant', 'x')).toBe('from-user'); + expect( + client.getAccountNamespace('shared', { accountId: 'a' }).getParam('variant', 'x') + ).toBe('from-account'); + expect(client.getDeviceNamespace('shared').getParam('variant', 'x')).toBe('from-device'); + }); + + it('returns the default and logs no exposure for a unit outside the allocated segments', () => { + const unallocated = findUnitsByValue( + unit => userClientForUnit(unit).getUserNamespace('cli-half'), + 'variant', + 'not-allocated', + ['not-allocated'] + ).get('not-allocated')!; + const { client, analytics } = createClient({ + configs: { user: TEST_DEFINITION }, + units: { user: unallocated }, + }); + expect(client.getUserNamespace('cli-half').getParam('variant', 'control')).toBe('control'); + expect(analytics.logEvent).not.toHaveBeenCalled(); + }); + }); + + describe('exposure logging', () => { + it('logs one exposure per namespace per client', () => { + const { client, analytics } = createClient({ + configs: { user: TEST_DEFINITION }, + units: { user: treatmentUnit }, + }); + const namespace = client.getUserNamespace('cli-full'); + namespace.getParam('variant', 'control'); + namespace.getParam('enabled', false); + client.getUserNamespace('cli-full').getParam('variant', 'control'); + + expect(analytics.logEvent).toHaveBeenCalledTimes(1); + expect(analytics.logEvent).toHaveBeenCalledWith('Experiment Viewed', { + experimentName: 'cli-test', + variationName: 'treatment', + unit: treatmentUnit, + }); + }); + + it('logs again for a second client with the same unit, so clients must be cached per command', () => { + const analytics = createMockAnalytics(); + createClient({ + configs: { user: TEST_DEFINITION }, + units: { user: treatmentUnit }, + analytics, + }) + .client.getUserNamespace('cli-full') + .getParam('variant', 'control'); + createClient({ + configs: { user: TEST_DEFINITION }, + units: { user: treatmentUnit }, + analytics, + }) + .client.getUserNamespace('cli-full') + .getParam('variant', 'control'); + expect(analytics.logEvent).toHaveBeenCalledTimes(2); + }); + + it('logs nothing for experiments that use the no-op logger', () => { + const { client, analytics } = createClient({ + configs: { user: TEST_DEFINITION }, + units: { user: 'unit-1' }, + }); + expect(['a', 'b']).toContain( + client.getUserNamespace('cli-quiet-ns').getParam('variant', 'x') + ); + expect(analytics.logEvent).not.toHaveBeenCalled(); + }); + + it('falls back to the no-op logger for an unknown exposureLoggerType', () => { + const definition: ExperimentationDefinition = { + experiments: [ + { + experimentName: 'odd', + paramDefinitions: { variant: { type: 'uniform_choice', choices: ['only'] } }, + exposureLoggerType: 'something-else', + }, + ], + namespaces: [ + { + name: 'odd', + numSegments: 10, + segmentExperimentSetupDefinitions: [ + { method: 'ADD', name: 'all', experimentName: 'odd', numSegments: 10 }, + ], + }, + ], + }; + const { client, analytics } = createClient({ + configs: { user: definition }, + units: { user: 'u' }, + }); + expect(client.getUserNamespace('odd').getParam('variant', 'x')).toBe('only'); + expect(analytics.logEvent).not.toHaveBeenCalled(); + }); + }); + + describe('overrides', () => { + it('returns the override instead of the assigned variant and logs no exposure', () => { + const { client, analytics } = createClient({ + configs: { user: TEST_DEFINITION }, + units: { user: controlUnit }, + overrides: new ExperimentOverrides(parseExperimentOverrides('cli-full:variant=treatment')), + }); + const namespace = client.getUserNamespace('cli-full'); + expect(namespace.isOverridden).toBe(true); + expect(namespace.getParam('variant', 'control')).toBe('treatment'); + expect(analytics.logEvent).not.toHaveBeenCalled(); + }); + + it('returns the default for other params of an overridden namespace without logging an exposure', () => { + const { client, analytics } = createClient({ + configs: { user: TEST_DEFINITION }, + units: { user: controlUnit }, + overrides: new ExperimentOverrides(parseExperimentOverrides('cli-full:variant=treatment')), + }); + expect(client.getUserNamespace('cli-full').getParam('enabled', false)).toBe(false); + expect(analytics.logEvent).not.toHaveBeenCalled(); + }); + + it('applies overrides to namespaces that are not in the server config', () => { + const { client } = createClient({ + overrides: new ExperimentOverrides( + parseExperimentOverrides('not-deployed:variant=treatment') + ), + }); + expect(client.getUserNamespace('not-deployed').getParam('variant', 'control')).toBe( + 'treatment' + ); + }); + + it('applies the same override in every scope', () => { + const { client } = createClient({ + overrides: new ExperimentOverrides(parseExperimentOverrides('ns:enabled=true')), + }); + expect(client.getUserNamespace('ns').getParam('enabled', false)).toBe(true); + expect(client.getAccountNamespace('ns', { accountId: 'a' }).getParam('enabled', false)).toBe( + true + ); + expect(client.getDeviceNamespace('ns').getParam('enabled', false)).toBe(true); + }); + }); + + describe('invalid server config', () => { + it('returns the default without throwing when a param uses an unknown operator', () => { + const definition = { + experiments: [ + { + experimentName: 'broken', + paramDefinitions: { variant: { type: 'not-an-operator' } }, + exposureLoggerType: 'rudderstack', + }, + ], + namespaces: [ + { + name: 'broken', + numSegments: 10, + segmentExperimentSetupDefinitions: [ + { method: 'ADD', name: 'all', experimentName: 'broken', numSegments: 10 }, + ], + }, + ], + } as unknown as ExperimentationDefinition; + const { client } = createClient({ configs: { user: definition }, units: { user: 'u' } }); + expect(client.getUserNamespace('broken').getParam('variant', 'control')).toBe('control'); + expect(Log.debug).toHaveBeenCalled(); + }); + + const overAllocated: ExperimentationDefinition = { + experiments: TEST_DEFINITION.experiments, + namespaces: [ + { + name: 'too-many', + numSegments: 10, + segmentExperimentSetupDefinitions: [ + { method: 'ADD', name: 'all', experimentName: 'cli-test', numSegments: 20 }, + ], + }, + ], + }; + + it('throws during construction when a namespace over-allocates segments', () => { + expect(() => + createClient({ configs: { user: overAllocated }, units: { user: 'u' } }) + ).toThrow(/requested 20 segments/); + }); + + it('returns the default when a namespace fails to build for an account ID', () => { + // Account namespaces have no default unit, so the failure surfaces on lookup, not construction. + const { client, analytics } = createClient({ configs: { account: overAllocated } }); + expect( + client.getAccountNamespace('too-many', { accountId: 'a' }).getParam('variant', 'control') + ).toBe('control'); + expect(Log.debug).toHaveBeenCalledWith( + expect.stringContaining('Failed to build namespace'), + expect.anything() + ); + expect(analytics.logEvent).not.toHaveBeenCalled(); + }); + }); + + describe('createDisabled', () => { + it('returns defaults for every scope and logs no exposure', () => { + const client = ExperimentationClient.createDisabled(ExperimentOverrides.empty()); + expect(client.getUserNamespace('cli-full').getParam('variant', 'control')).toBe('control'); + expect(client.getDeviceNamespace('cli-full').getParam('variant', 'control')).toBe('control'); + expect(Log.debug).not.toHaveBeenCalled(); + // The account scope always has a unit, so the empty config reports the namespace as missing. + expect( + client.getAccountNamespace('cli-full', { accountId: 'a' }).getParam('variant', 'control') + ).toBe('control'); + expect(Log.debug).toHaveBeenCalledWith(expect.stringContaining('not found')); + }); + + it('reads overrides from the environment by default', () => { + const client = ExperimentationClient.createDisabled(); + expect(client.getUserNamespace('cli-full').getParam('variant', 'control')).toBe('from-env'); + }); + + it('still honors overrides', () => { + const client = ExperimentationClient.createDisabled( + new ExperimentOverrides(parseExperimentOverrides('cli-full:variant=treatment')) + ); + expect(client.getUserNamespace('cli-full').getParam('variant', 'control')).toBe('treatment'); + }); + }); +}); diff --git a/packages/eas-cli/src/experimentation/__tests__/fixtures.ts b/packages/eas-cli/src/experimentation/__tests__/fixtures.ts new file mode 100644 index 0000000000..4aeb03ed14 --- /dev/null +++ b/packages/eas-cli/src/experimentation/__tests__/fixtures.ts @@ -0,0 +1,79 @@ +import type { ExperimentationDefinition } from '@expo/experimentation'; + +import { Analytics } from '../../analytics/AnalyticsManager'; +import { ExperimentNamespace } from '../ExperimentNamespace'; + +/** + * - `cli-full`: every unit is in the `cli-test` experiment (50% control, 50% treatment). + * - `cli-half`: half of the units are in `cli-test`, the rest get defaults. + * - `cli-quiet-ns`: experiment with the `no-op` exposure logger. + */ +export const TEST_DEFINITION: ExperimentationDefinition = { + experiments: [ + { + experimentName: 'cli-test', + paramDefinitions: { + variant: { type: 'uniform_choice', choices: ['control', 'treatment'] }, + enabled: { type: 'bernoulli', p: 0.5 }, + }, + exposureLoggerType: 'rudderstack', + }, + { + experimentName: 'cli-quiet', + paramDefinitions: { + variant: { type: 'uniform_choice', choices: ['a', 'b'] }, + }, + exposureLoggerType: 'no-op', + }, + ], + namespaces: [ + { + name: 'cli-full', + numSegments: 1000, + segmentExperimentSetupDefinitions: [ + { method: 'ADD', name: 'cli-test full', experimentName: 'cli-test', numSegments: 1000 }, + ], + }, + { + name: 'cli-half', + numSegments: 1000, + segmentExperimentSetupDefinitions: [ + { method: 'ADD', name: 'cli-test half', experimentName: 'cli-test', numSegments: 500 }, + ], + }, + { + name: 'cli-quiet-ns', + numSegments: 1000, + segmentExperimentSetupDefinitions: [ + { method: 'ADD', name: 'cli-quiet full', experimentName: 'cli-quiet', numSegments: 1000 }, + ], + }, + ], +}; + +export function createMockAnalytics(): Analytics & { logEvent: jest.Mock } { + return { logEvent: jest.fn() }; +} + +/** + * Finds one unit per expected value by probing the real library, instead of hard-coding hashes. + */ +export function findUnitsByValue( + getNamespaceForUnit: (unit: string) => ExperimentNamespace, + paramName: string, + defaultValue: T, + expectedValues: T[] +): Map { + const found = new Map(); + for (let i = 0; i < 1000 && found.size < expectedValues.length; i++) { + const unit = `unit-${i}`; + const value = getNamespaceForUnit(unit).getParam(paramName, defaultValue); + if (expectedValues.includes(value) && !found.has(value)) { + found.set(value, unit); + } + } + if (found.size !== expectedValues.length) { + throw new Error(`Could not find units for all of: ${expectedValues.join(', ')}`); + } + return found; +} diff --git a/packages/eas-cli/src/graphql/generated.ts b/packages/eas-cli/src/graphql/generated.ts index 33033c8151..41b5338064 100644 --- a/packages/eas-cli/src/graphql/generated.ts +++ b/packages/eas-cli/src/graphql/generated.ts @@ -16498,6 +16498,16 @@ export type AccountSubscriptionQueryVariables = Exact<{ export type AccountSubscriptionQuery = { __typename?: 'RootQuery', account: { __typename?: 'AccountQuery', byId: { __typename?: 'Account', id: string, subscription?: { __typename?: 'SubscriptionDetails', id: string, name?: string | null, planId?: string | null } | null } } }; +export type AccountConcurrencyUsageQueryVariables = Exact<{ + accountId: Scalars['String']['input']; +}>; + + +export type AccountConcurrencyUsageQuery = { __typename?: 'RootQuery', account: { __typename?: 'AccountQuery', byId: { __typename?: 'Account', id: string, subscription?: { __typename?: 'SubscriptionDetails', id: string, name?: string | null, planId?: string | null, concurrencies?: { __typename?: 'Concurrencies', total: number, android: number, ios: number } | null } | null, concurrencyConsumers: Array< + | { __typename: 'Build', id: string } + | { __typename: 'JobRun', id: string } + > } } }; + export type AppByIdQueryVariables = Exact<{ appId: Scalars['String']['input']; }>; @@ -16910,6 +16920,11 @@ export type EnvironmentVariablesSharedWithSensitiveQueryVariables = Exact<{ export type EnvironmentVariablesSharedWithSensitiveQuery = { __typename?: 'RootQuery', app: { __typename?: 'AppQuery', byId: { __typename?: 'App', id: string, ownerAccount: { __typename?: 'Account', id: string, environmentVariablesIncludingSensitive: Array<{ __typename?: 'EnvironmentVariableWithSecret', id: string, name: string, value?: string | null, environments?: Array | null, createdAt: any, updatedAt: any, scope: EnvironmentVariableScope, visibility: EnvironmentVariableVisibility, type: EnvironmentSecretType, valueWithFileContent?: string | null }> } } } }; +export type ExperimentationConfigsQueryVariables = Exact<{ [key: string]: never; }>; + + +export type ExperimentationConfigsQuery = { __typename?: 'RootQuery', experimentation: { __typename?: 'ExperimentationQuery', userConfig: any, accountConfig: any, deviceConfig: any } }; + export type FingerprintsByAppIdQueryVariables = Exact<{ appId: Scalars['String']['input']; after?: InputMaybe; diff --git a/packages/eas-cli/src/graphql/queries/AccountQuery.ts b/packages/eas-cli/src/graphql/queries/AccountQuery.ts index f8ab12a3cf..1ba510fbc8 100644 --- a/packages/eas-cli/src/graphql/queries/AccountQuery.ts +++ b/packages/eas-cli/src/graphql/queries/AccountQuery.ts @@ -6,6 +6,8 @@ import { withErrorHandlingAsync } from '../client'; import { AccountByNameQuery as AccountByNameQueryType, AccountByNameQueryVariables, + AccountConcurrencyUsageQuery as AccountConcurrencyUsageQueryType, + AccountConcurrencyUsageQueryVariables, AccountFullUsageQuery as AccountFullUsageQueryType, AccountFullUsageQueryVariables, AccountSubscriptionQuery as AccountSubscriptionQueryType, @@ -16,6 +18,7 @@ import { import { AccountUsageMetricFragmentNode, BillingPeriodFragmentNode, + ConcurrenciesFragmentNode, SubscriptionDetailsFragmentNode, UsageMetricTotalFragmentNode, } from '../types/Account'; @@ -24,6 +27,7 @@ export type AccountFullUsageData = NonNullable['subscription'] >; +export type AccountConcurrencyUsage = AccountConcurrencyUsageQueryType['account']['byId']; export const AccountQuery = { async getByNameAsync( @@ -317,4 +321,48 @@ export const AccountQuery = { return data.account.byId?.subscription ?? null; }, + + /** Plan concurrency total and the builds and workflow jobs that hold a concurrency right now. */ + async getConcurrencyUsageAsync( + graphqlClient: ExpoGraphqlClient, + accountId: string + ): Promise { + const data = await withErrorHandlingAsync( + graphqlClient + .query( + gql` + query AccountConcurrencyUsage($accountId: String!) { + account { + byId(accountId: $accountId) { + id + subscription { + id + name + planId + concurrencies { + ...ConcurrenciesFragment + } + } + concurrencyConsumers { + __typename + ... on Build { + id + } + ... on JobRun { + id + } + } + } + } + } + ${print(ConcurrenciesFragmentNode)} + `, + { accountId }, + { additionalTypenames: ['Account', 'SubscriptionDetails', 'Build', 'JobRun'] } + ) + .toPromise() + ); + + return data.account.byId; + }, }; diff --git a/packages/eas-cli/src/graphql/queries/ExperimentationConfigQuery.ts b/packages/eas-cli/src/graphql/queries/ExperimentationConfigQuery.ts new file mode 100644 index 0000000000..4bfabb1993 --- /dev/null +++ b/packages/eas-cli/src/graphql/queries/ExperimentationConfigQuery.ts @@ -0,0 +1,66 @@ +import type { ExperimentationDefinition } from '@expo/experimentation'; +import { OperationResult } from '@urql/core'; +import gql from 'graphql-tag'; + +import { ExpoGraphqlClient } from '../../commandUtils/context/contextUtils/createGraphqlClient'; +import { EMPTY_EXPERIMENTATION_DEFINITION } from '../../experimentation/ExperimentationClient'; +import { ExperimentationConfigsQuery, ExperimentationConfigsQueryVariables } from '../generated'; + +export type ExperimentationConfigsResult = { + userConfig: ExperimentationDefinition; + accountConfig: ExperimentationDefinition; + deviceConfig: ExperimentationDefinition; +}; + +/** Like `withErrorHandlingAsync`, but never logs. Callers fail open. */ +async function quietQueryAsync(promise: Promise>): Promise { + const { data, error } = await promise; + if (error) { + throw error; + } + if (!data) { + throw new Error('Returned query result data is null!'); + } + return data; +} + +/** The server sends an untyped JSONObject. Anything but the expected shape means no experiments. */ +function toExperimentationDefinition(value: unknown): ExperimentationDefinition { + if ( + value && + typeof value === 'object' && + Array.isArray((value as { experiments?: unknown }).experiments) && + Array.isArray((value as { namespaces?: unknown }).namespaces) + ) { + return value as ExperimentationDefinition; + } + return EMPTY_EXPERIMENTATION_DEFINITION; +} + +export const ExperimentationConfigQuery = { + async getConfigsAsync(graphqlClient: ExpoGraphqlClient): Promise { + const data = await quietQueryAsync( + graphqlClient + .query( + gql` + query ExperimentationConfigs { + experimentation { + userConfig + accountConfig + deviceConfig + } + } + `, + {}, + // A failing network must not delay the command. + { additionalTypenames: ['ExperimentationQuery'], noRetry: true } + ) + .toPromise() + ); + return { + userConfig: toExperimentationDefinition(data.experimentation.userConfig), + accountConfig: toExperimentationDefinition(data.experimentation.accountConfig), + deviceConfig: toExperimentationDefinition(data.experimentation.deviceConfig), + }; + }, +}; diff --git a/packages/eas-cli/src/graphql/queries/__tests__/AccountQuery-test.ts b/packages/eas-cli/src/graphql/queries/__tests__/AccountQuery-test.ts new file mode 100644 index 0000000000..8fb4111f57 --- /dev/null +++ b/packages/eas-cli/src/graphql/queries/__tests__/AccountQuery-test.ts @@ -0,0 +1,59 @@ +import { print } from 'graphql'; + +import { ExpoGraphqlClient } from '../../../commandUtils/context/contextUtils/createGraphqlClient'; +import { AccountQuery } from '../AccountQuery'; + +function makeGraphqlClient(result: { data?: unknown; error?: Error }): ExpoGraphqlClient & { + query: jest.Mock; +} { + return { + query: jest.fn().mockReturnValue({ toPromise: jest.fn().mockResolvedValue(result) }), + } as unknown as ExpoGraphqlClient & { query: jest.Mock }; +} + +describe('AccountQuery.getConcurrencyUsageAsync', () => { + const account = { + __typename: 'Account', + id: 'account-id', + subscription: { + __typename: 'SubscriptionDetails', + id: 'subscription-id', + name: 'Starter', + planId: 'price_starter', + concurrencies: { __typename: 'Concurrencies', total: 2, android: 2, ios: 2 }, + }, + concurrencyConsumers: [ + { __typename: 'Build', id: 'build-1' }, + { __typename: 'JobRun', id: 'job-run-1' }, + ], + }; + + it('returns the account subscription and concurrency consumers', async () => { + const graphqlClient = makeGraphqlClient({ data: { account: { byId: account } } }); + + await expect( + AccountQuery.getConcurrencyUsageAsync(graphqlClient, 'account-id') + ).resolves.toEqual(account); + + const [document, variables, options] = graphqlClient.query.mock.calls[0]; + const text = print(document); + expect(variables).toEqual({ accountId: 'account-id' }); + expect(text).toContain('concurrencyConsumers'); + expect(text).toContain('concurrencies'); + expect(text).toContain('planId'); + expect(options.additionalTypenames).toEqual([ + 'Account', + 'SubscriptionDetails', + 'Build', + 'JobRun', + ]); + }); + + it('throws when the query fails', async () => { + const graphqlClient = makeGraphqlClient({ error: new Error('boom') }); + + await expect( + AccountQuery.getConcurrencyUsageAsync(graphqlClient, 'account-id') + ).rejects.toThrow(); + }); +}); diff --git a/packages/eas-cli/src/graphql/queries/__tests__/ExperimentationConfigQuery-test.ts b/packages/eas-cli/src/graphql/queries/__tests__/ExperimentationConfigQuery-test.ts new file mode 100644 index 0000000000..30c4b292b6 --- /dev/null +++ b/packages/eas-cli/src/graphql/queries/__tests__/ExperimentationConfigQuery-test.ts @@ -0,0 +1,89 @@ +import { CombinedError } from '@urql/core'; +import { print } from 'graphql'; + +import { ExpoGraphqlClient } from '../../../commandUtils/context/contextUtils/createGraphqlClient'; +import Log from '../../../log'; +import { ExperimentationConfigQuery } from '../ExperimentationConfigQuery'; + +jest.mock('../../../log'); + +const definition = { + experiments: [{ experimentName: 'x', paramDefinitions: {}, exposureLoggerType: 'rudderstack' }], + namespaces: [], +}; + +function makeGraphqlClient(result: { data?: unknown; error?: CombinedError }): ExpoGraphqlClient & { + query: jest.Mock; +} { + return { + query: jest.fn().mockReturnValue({ toPromise: jest.fn().mockResolvedValue(result) }), + } as unknown as ExpoGraphqlClient & { query: jest.Mock }; +} + +describe('ExperimentationConfigQuery.getConfigsAsync', () => { + it('returns the three configs', async () => { + const graphqlClient = makeGraphqlClient({ + data: { + experimentation: { + userConfig: definition, + accountConfig: definition, + deviceConfig: definition, + }, + }, + }); + await expect(ExperimentationConfigQuery.getConfigsAsync(graphqlClient)).resolves.toEqual({ + userConfig: definition, + accountConfig: definition, + deviceConfig: definition, + }); + }); + + it('replaces malformed configs with an empty definition', async () => { + const graphqlClient = makeGraphqlClient({ + data: { + experimentation: { + userConfig: { experiments: 'nope', namespaces: [] }, + accountConfig: null, + deviceConfig: definition, + }, + }, + }); + await expect(ExperimentationConfigQuery.getConfigsAsync(graphqlClient)).resolves.toEqual({ + userConfig: { experiments: [], namespaces: [] }, + accountConfig: { experiments: [], namespaces: [] }, + deviceConfig: definition, + }); + }); + + it('disables retries and selects only the config fields', async () => { + const graphqlClient = makeGraphqlClient({ + data: { experimentation: { userConfig: {}, accountConfig: {}, deviceConfig: {} } }, + }); + await ExperimentationConfigQuery.getConfigsAsync(graphqlClient); + + const [document, variables, context] = graphqlClient.query.mock.calls[0]; + const printed = print(document); + expect(printed).toContain('experimentation'); + expect(printed).toContain('userConfig'); + expect(printed).toContain('accountConfig'); + expect(printed).toContain('deviceConfig'); + expect(printed).not.toContain('deviceExperimentationUnit'); + expect(variables).toEqual({}); + expect(context).toEqual({ additionalTypenames: ['ExperimentationQuery'], noRetry: true }); + }); + + it('rejects on a GraphQL error without logging', async () => { + const error = new CombinedError({ networkError: new Error('offline') }); + const graphqlClient = makeGraphqlClient({ error }); + await expect(ExperimentationConfigQuery.getConfigsAsync(graphqlClient)).rejects.toBe(error); + expect(Log.error).not.toHaveBeenCalled(); + expect(Log.warn).not.toHaveBeenCalled(); + }); + + it('rejects when the result has no data', async () => { + const graphqlClient = makeGraphqlClient({ data: null }); + await expect(ExperimentationConfigQuery.getConfigsAsync(graphqlClient)).rejects.toThrow( + 'Returned query result data is null!' + ); + }); +}); diff --git a/packages/eas-cli/src/user/__tests__/SessionManager-test.ts b/packages/eas-cli/src/user/__tests__/SessionManager-test.ts index 5840df0f79..3cbe81c383 100644 --- a/packages/eas-cli/src/user/__tests__/SessionManager-test.ts +++ b/packages/eas-cli/src/user/__tests__/SessionManager-test.ts @@ -41,6 +41,7 @@ const analytics: AnalyticsWithOrchestration = { logEvent: jest.fn((): void => {}), setActor: jest.fn((): void => {}), flushAsync: jest.fn(async (): Promise => {}), + getDeviceId: jest.fn((): string | null => null), }; beforeEach(() => { diff --git a/packages/eas-cli/src/user/__tests__/deviceLogin-test.ts b/packages/eas-cli/src/user/__tests__/deviceLogin-test.ts index 1cf4370fc2..361a0eab0e 100644 --- a/packages/eas-cli/src/user/__tests__/deviceLogin-test.ts +++ b/packages/eas-cli/src/user/__tests__/deviceLogin-test.ts @@ -18,7 +18,12 @@ const now = Date.parse('2026-09-29T12:00:00Z'); let directory: string; function manager(): SessionManager { - return new SessionManager({ logEvent: jest.fn(), setActor: jest.fn(), flushAsync: jest.fn() }); + return new SessionManager({ + logEvent: jest.fn(), + setActor: jest.fn(), + flushAsync: jest.fn(), + getDeviceId: jest.fn(), + }); } async function startAsync(): Promise { diff --git a/yarn.lock b/yarn.lock index 83724cc8a9..eeafde8c56 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2642,6 +2642,15 @@ __metadata: languageName: node linkType: hard +"@expo/experimentation@npm:0.0.5": + version: 0.0.5 + resolution: "@expo/experimentation@npm:0.0.5" + dependencies: + nullthrows: "npm:^1.1.1" + checksum: 10c0/bc054e1bcc0583f2a09e8b56d2f95bb62cf2f4eaf78edac4eb709c73cfd1fdb93626b948f4184f6dc8096a44b6c12c97109a84622bb56620484b4e61856007fd + languageName: node + linkType: hard + "@expo/image-utils@npm:^0.7.6": version: 0.7.6 resolution: "@expo/image-utils@npm:0.7.6" @@ -10478,6 +10487,7 @@ __metadata: "@expo/eas-build-job": "npm:24.13.0" "@expo/eas-json": "npm:24.13.0" "@expo/env": "npm:^1.0.0" + "@expo/experimentation": "npm:0.0.5" "@expo/json-file": "npm:8.3.3" "@expo/logger": "npm:24.13.0" "@expo/multipart-body-parser": "npm:2.0.0"