diff --git a/packages/assets-controller/src/AssetsController.snap-balance-errors.test.ts b/packages/assets-controller/src/AssetsController.snap-balance-errors.test.ts new file mode 100644 index 00000000000..a6a535f23b4 --- /dev/null +++ b/packages/assets-controller/src/AssetsController.snap-balance-errors.test.ts @@ -0,0 +1,180 @@ +import type { InternalAccount } from '@metamask/keyring-internal-api'; + +import { + createMockInternalAccount, + createMockMessengers, + registerAssetsControllerActions, +} from './__fixtures__/MockAssetControllerMessenger.js'; +import { createTestApiClient } from './__fixtures__/mockTokenApi.js'; +import { AssetsController } from './AssetsController.js'; +import type { + SnapBalanceFetchStatusUpdate, + SnapDataSourceOptions, +} from './data-sources/SnapDataSource.js'; +import { SnapDataSource } from './data-sources/SnapDataSource.js'; +import type { ChainId } from './types.js'; + +/** + * Covers how the controller surfaces per-account snap (non-EVM) balance fetch + * failures in `assetsBalanceErrors` and exposes a manual retry. Lives outside + * `AssetsController.test.ts`, which is already very large. + */ + +const capturedOptions: SnapDataSourceOptions[] = []; + +jest.mock('./data-sources/SnapDataSource.js', () => { + const actual = jest.requireActual('./data-sources/SnapDataSource.js'); + class CapturingSnapDataSource extends actual.SnapDataSource { + constructor(options: SnapDataSourceOptions) { + super(options); + capturedOptions.push(options); + } + } + return { ...actual, SnapDataSource: CapturingSnapDataSource }; +}); + +const BTC_MAINNET = 'bip122:000000000019d6689c085ae165831e93' as ChainId; +const BTC_TESTNET = 'bip122:000000000933ea01ad0ee984209779ba' as ChainId; + +const ERROR = { + status: 'unavailable' as const, + error: 'the account has not been synchronized yet', + attempts: 1, + failedAt: 1_000, + nextRetryAt: 31_000, +}; + +async function withController( + fn: (args: { + controller: AssetsController; + emitStatus: (update: SnapBalanceFetchStatusUpdate) => void; + }) => Promise | ReturnValue, +): Promise { + capturedOptions.length = 0; + const accounts = [ + createMockInternalAccount({ + id: 'btc-account', + metadata: { name: 'Account 1' } as InternalAccount['metadata'], + }), + ]; + const { assetsControllerMessenger } = createMockMessengers({ + registerCustomRootActions: (messenger) => + registerAssetsControllerActions(messenger, { + accounts, + enabledNetworkMap: { eip155: { '1': true } }, + nativeAssetIdentifiers: {}, + remoteFeatureFlags: {}, + }), + }); + const controller = new AssetsController({ + messenger: assetsControllerMessenger, + queryApiClient: createTestApiClient(), + isBasicFunctionality: (): boolean => true, + }); + const options = capturedOptions[0]; + if (!options?.onBalanceFetchStatusChange) { + throw new Error('SnapDataSource was not given a status callback'); + } + const emitStatus = options.onBalanceFetchStatusChange; + try { + return await fn({ controller, emitStatus }); + } finally { + controller.destroy(); + } +} + +describe('AssetsController snap balance fetch errors', () => { + it('passes a status callback to the SnapDataSource', async () => { + await withController(({ controller }) => { + expect(capturedOptions).toHaveLength(1); + expect(controller.state.assetsBalanceErrors).toBeUndefined(); + }); + }); + + it('records an unavailable status per account and chain', async () => { + await withController(({ controller, emitStatus }) => { + emitStatus({ + accountId: 'btc-account', + chainIds: [BTC_MAINNET, BTC_TESTNET], + error: ERROR, + }); + + expect(controller.state.assetsBalanceErrors).toStrictEqual({ + 'btc-account': { [BTC_MAINNET]: ERROR, [BTC_TESTNET]: ERROR }, + }); + }); + }); + + it('clears the status for the chains that recovered', async () => { + await withController(({ controller, emitStatus }) => { + emitStatus({ + accountId: 'btc-account', + chainIds: [BTC_MAINNET, BTC_TESTNET], + error: ERROR, + }); + emitStatus({ + accountId: 'btc-account', + chainIds: [BTC_TESTNET], + error: null, + }); + + expect(controller.state.assetsBalanceErrors).toStrictEqual({ + 'btc-account': { [BTC_MAINNET]: ERROR }, + }); + + emitStatus({ + accountId: 'btc-account', + chainIds: [BTC_MAINNET], + error: null, + }); + + expect(controller.state.assetsBalanceErrors).toStrictEqual({}); + }); + }); + + it('ignores a recovery for an account without an error', async () => { + await withController(({ controller, emitStatus }) => { + emitStatus({ + accountId: 'btc-account', + chainIds: [BTC_MAINNET], + error: null, + }); + + expect(controller.state.assetsBalanceErrors).toBeUndefined(); + }); + }); + + it('does not persist the balance errors', async () => { + await withController(({ controller, emitStatus }) => { + emitStatus({ + accountId: 'btc-account', + chainIds: [BTC_MAINNET], + error: ERROR, + }); + + expect(controller.metadata.assetsBalanceErrors).toStrictEqual({ + persist: false, + includeInStateLogs: true, + includeInDebugSnapshot: true, + usedInUi: true, + }); + }); + }); + + it('retrySnapBalances delegates to the SnapDataSource', async () => { + const retrySpy = jest + .spyOn(SnapDataSource.prototype, 'retryFailedAccounts') + .mockResolvedValue(undefined); + try { + await withController(async ({ controller }) => { + await controller.retrySnapBalances(['btc-account']); + await controller.retrySnapBalances(); + + expect(retrySpy).toHaveBeenNthCalledWith(1, ['btc-account']); + expect(retrySpy).toHaveBeenNthCalledWith(2, undefined); + }); + } finally { + retrySpy.mockRestore(); + } + }); +}); diff --git a/packages/assets-controller/src/AssetsController.ts b/packages/assets-controller/src/AssetsController.ts index 17d814a2b2e..bddbd5c7136 100644 --- a/packages/assets-controller/src/AssetsController.ts +++ b/packages/assets-controller/src/AssetsController.ts @@ -100,6 +100,7 @@ import type { RpcDataSourceConfig } from './data-sources/RpcDataSource.js'; import { RpcDataSource } from './data-sources/RpcDataSource.js'; import type { AccountsControllerAccountBalancesUpdatedEvent } from './data-sources/SnapDataSource.js'; import { SnapDataSource } from './data-sources/SnapDataSource.js'; +import type { SnapBalanceFetchStatusUpdate } from './data-sources/SnapDataSource.js'; import type { StakedBalanceDataSourceConfig } from './data-sources/StakedBalanceDataSource.js'; import { StakedBalanceDataSource } from './data-sources/StakedBalanceDataSource.js'; import { @@ -502,6 +503,12 @@ const stateMetadata: StateMetadata = { includeInDebugSnapshot: true, usedInUi: true, }, + assetsBalanceErrors: { + persist: false, + includeInStateLogs: true, + includeInDebugSnapshot: true, + usedInUi: true, + }, }; // ============================================================================ @@ -1025,6 +1032,8 @@ export class AssetsController extends BaseController< isBalanceV6Enabled: (): boolean => this.#isBalanceV6Enabled(), getAssetVisibility: this.#getAssetVisibility.bind(this), getAssetsState: (): AssetsControllerState => this.state, + onBalanceFetchStatusChange: (update): void => + this.#handleBalanceFetchStatusChange(update), }); this.#rpcDataSource = new RpcDataSource({ messenger: this.messenger, @@ -2283,6 +2292,61 @@ export class AssetsController extends BaseController< }); } + // ============================================================================ + // SNAP BALANCE FETCH STATUS + // ============================================================================ + + /** + * Retry balance fetches for snap (non-EVM) accounts whose last fetch failed. + * Accounts that never failed are ignored. Success clears the matching + * `assetsBalanceErrors` entries; failure re-schedules the backoff retry. + * + * @param accountIds - Optional account IDs to retry. Defaults to all failed accounts. + */ + async retrySnapBalances(accountIds?: string[]): Promise { + await this.#snapDataSource.retryFailedAccounts(accountIds); + } + + /** + * Apply a per-account balance fetch status change from the SnapDataSource to + * `assetsBalanceErrors`. While an entry exists, any balance stored for that + * account/chain (including a seeded native `0`) is a placeholder or stale + * value and must not be presented as a confirmed balance. + * + * @param update - The status change. + */ + #handleBalanceFetchStatusChange(update: SnapBalanceFetchStatusUpdate): void { + const { accountId, chainIds, error } = update; + const current = this.state.assetsBalanceErrors?.[accountId]; + if (!error && !current) { + return; + } + this.update((state) => { + const errors = (state.assetsBalanceErrors ??= {}); + if (error) { + const accountErrors = (errors[accountId] ??= {}); + for (const chainId of chainIds) { + accountErrors[chainId] = { ...error }; + } + return; + } + const accountErrors = errors[accountId]; + if (!accountErrors) { + return; + } + if (chainIds.length === 0) { + delete errors[accountId]; + return; + } + for (const chainId of chainIds) { + delete accountErrors[chainId]; + } + if (Object.keys(accountErrors).length === 0) { + delete errors[accountId]; + } + }); + } + // ============================================================================ // CUSTOM ASSETS MANAGEMENT // ============================================================================ diff --git a/packages/assets-controller/src/data-sources/SnapDataSource.test.ts b/packages/assets-controller/src/data-sources/SnapDataSource.test.ts index 1f33b4588df..766697e0698 100644 --- a/packages/assets-controller/src/data-sources/SnapDataSource.test.ts +++ b/packages/assets-controller/src/data-sources/SnapDataSource.test.ts @@ -31,6 +31,7 @@ import { getChainIdsCaveat, KEYRING_PERMISSION, ASSETS_PERMISSION, + getSnapBalanceRetryDelay, } from './SnapDataSource.js'; // Test chain IDs @@ -207,6 +208,8 @@ function setupController( isBalanceV6Enabled?: () => boolean; getAssetVisibility?: GetAssetVisibility; getAssetsState?: () => AssetsControllerState; + onBalanceFetchStatusChange?: SnapDataSourceOptions['onBalanceFetchStatusChange']; + retryBackoff?: SnapDataSourceOptions['retryBackoff']; } = {}, ): SetupResult { const { @@ -295,6 +298,8 @@ function setupController( isBalanceV6Enabled, getAssetVisibility, getAssetsState, + onBalanceFetchStatusChange: options.onBalanceFetchStatusChange, + retryBackoff: options.retryBackoff, }; const controller = new SnapDataSource(controllerOptions); @@ -1369,6 +1374,358 @@ describe('SnapDataSource', () => { expect(true).toBe(true); }); + describe('per-account balance fetch errors and retry', () => { + const SOL_BALANCE = { amount: '1000000000', unit: 'SOL' }; + + const failingHandler = + (failing: { value: boolean }) => + (params: { request?: { method: string } }): Promise => { + if (params.request?.method === 'keyring_listAccountAssets') { + return Promise.resolve([MOCK_SOL_ASSET]); + } + if (params.request?.method === 'keyring_getAccountBalances') { + return failing.value + ? Promise.reject( + new Error('the account has not been synchronized yet'), + ) + : Promise.resolve({ [MOCK_SOL_ASSET]: SOL_BALANCE }); + } + return Promise.resolve(null); + }; + + const setupFailing = ( + options: { + isBalanceV6Enabled?: () => boolean; + retryBackoff?: SnapDataSourceOptions['retryBackoff']; + onBalanceFetchStatusChange?: jest.Mock; + } = {}, + ): SetupResult & { + failing: { value: boolean }; + onStatus: jest.Mock; + } => { + const onStatus = options.onBalanceFetchStatusChange ?? jest.fn(); + const failing = { value: true }; + const setup = setupController({ + installedSnaps: { + [SOLANA_SNAP_ID]: { version: '1.0.0', chainIds: [SOLANA_MAINNET] }, + }, + isBalanceV6Enabled: options.isBalanceV6Enabled ?? ((): boolean => true), + getAssetVisibility: () => ({ + visibleAssetIds: [MOCK_SOL_ASSET], + hiddenAssetIds: [], + }), + onBalanceFetchStatusChange: onStatus, + // Long default so automatic retries never fire unless a test wants them. + retryBackoff: options.retryBackoff ?? { initialDelayMs: 60_000 }, + }); + setup.mockHandleRequest.mockImplementation(failingHandler(failing)); + return { ...setup, failing, onStatus }; + }; + + const flush = async (ms = 0): Promise => { + await new Promise((resolve) => setTimeout(resolve, ms)); + }; + + it('getSnapBalanceRetryDelay grows exponentially and is capped', () => { + expect(getSnapBalanceRetryDelay(1, 1000, 10_000)).toBe(1000); + expect(getSnapBalanceRetryDelay(2, 1000, 10_000)).toBe(2000); + expect(getSnapBalanceRetryDelay(3, 1000, 10_000)).toBe(4000); + expect(getSnapBalanceRetryDelay(10, 1000, 10_000)).toBe(10_000); + expect(getSnapBalanceRetryDelay(0)).toBe(30_000); + expect(getSnapBalanceRetryDelay(100)).toBe(300_000); + }); + + it.each([ + ['v6', true], + ['v5', false], + ])( + 'reports the account as unavailable instead of swallowing the error (%s)', + async (_label, v6) => { + const { controller, onStatus, cleanup } = setupFailing({ + isBalanceV6Enabled: (): boolean => v6, + }); + await flush(); + + const response = await controller.fetch(createDataRequest()); + + expect(response.assetsBalance).toStrictEqual({}); + expect(onStatus).toHaveBeenLastCalledWith({ + accountId: 'mock-account-id', + chainIds: [SOLANA_MAINNET], + error: expect.objectContaining({ + status: 'unavailable', + error: 'the account has not been synchronized yet', + attempts: expect.any(Number), + failedAt: expect.any(Number), + nextRetryAt: expect.any(Number), + }), + }); + expect(controller.getAccountBalanceErrors()).toStrictEqual({ + 'mock-account-id': { + chainIds: [SOLANA_MAINNET], + attempts: expect.any(Number), + }, + }); + + cleanup(); + }, + ); + + const lastError = ( + onStatus: jest.Mock, + ): { + attempts: number; + failedAt: number; + nextRetryAt: number | null; + error: string; + } => { + const calls = onStatus.mock.calls.filter( + ([update]) => update.accountId === 'mock-account-id' && update.error, + ); + return calls[calls.length - 1]?.[0].error; + }; + + it('does not count ordinary failed fetches as retry attempts, and schedules the first retry after the initial delay', async () => { + const { controller, onStatus, cleanup } = setupFailing({ + retryBackoff: { initialDelayMs: 30_000, maxDelayMs: 300_000 }, + }); + await flush(); + + // Onboarding: many fetches fail within seconds (subscriptions, polling, + // other controllers). None of them is a retry. + for (let i = 0; i < 13; i++) { + await controller.fetch(createDataRequest()); + } + + expect( + controller.getAccountBalanceErrors()['mock-account-id']?.attempts, + ).toBe(0); + const status = lastError(onStatus); + expect(status.attempts).toBe(0); + expect(status.nextRetryAt).not.toBeNull(); + // The retry scheduled by the first failure is kept: ~30 s, not the cap. + expect( + (status.nextRetryAt as number) - status.failedAt, + ).toBeLessThanOrEqual(30_000); + expect((status.nextRetryAt as number) - Date.now()).toBeGreaterThan( + 25_000, + ); + + cleanup(); + }); + + it('notifies repeated ordinary failures only when the error changes', async () => { + const { controller, onStatus, cleanup } = setupFailing(); + await flush(); + await controller.fetch(createDataRequest()); + const callsAfterFirst = onStatus.mock.calls.length; + + await controller.fetch(createDataRequest()); + await controller.fetch(createDataRequest()); + + expect(onStatus.mock.calls).toHaveLength(callsAfterFirst); + + cleanup(); + }); + + it('only scheduled retries advance the backoff (30 s, 60 s, ... capped)', async () => { + const { controller, onStatus, cleanup } = setupFailing({ + retryBackoff: { initialDelayMs: 20, maxDelayMs: 80 }, + }); + await flush(); + await controller.fetch(createDataRequest()); + const first = lastError(onStatus); + expect(first.attempts).toBe(0); + expect((first.nextRetryAt as number) - first.failedAt).toBe(20); + + // First scheduled retry fires after ~20 ms and fails again. + await flush(35); + const second = lastError(onStatus); + expect(second.attempts).toBe(1); + expect((second.nextRetryAt as number) - second.failedAt).toBe(40); + + // Second scheduled retry fires after ~40 ms more. + await flush(55); + const third = lastError(onStatus); + expect(third.attempts).toBe(2); + expect((third.nextRetryAt as number) - third.failedAt).toBe(80); + + cleanup(); + }); + + it('a failed manual retry resets the backoff to the initial delay', async () => { + const { controller, onStatus, cleanup } = setupFailing({ + retryBackoff: { initialDelayMs: 20, maxDelayMs: 80 }, + }); + await flush(); + await controller.fetch(createDataRequest()); + await flush(35); + expect(lastError(onStatus).attempts).toBe(1); + + await controller.retryFailedAccounts(['mock-account-id']); + + const status = lastError(onStatus); + expect(status.attempts).toBe(0); + expect((status.nextRetryAt as number) - status.failedAt).toBe(20); + + cleanup(); + }); + + it('a manual retry calls the snap again (so it can start a sync)', async () => { + const { controller, mockHandleRequest, cleanup } = setupFailing(); + await flush(); + await controller.fetch(createDataRequest()); + mockHandleRequest.mockClear(); + + await controller.retryFailedAccounts(); + + expect(mockHandleRequest).toHaveBeenCalledWith( + expect.objectContaining({ + request: expect.objectContaining({ + method: 'keyring_getAccountBalances', + }), + }), + ); + + cleanup(); + }); + + it('retryFailedAccounts clears the error and reports balances on success', async () => { + const { controller, failing, onStatus, assetsUpdateHandler, cleanup } = + setupFailing(); + await flush(); + await controller.fetch(createDataRequest()); + assetsUpdateHandler.mockClear(); + + failing.value = false; + await controller.retryFailedAccounts(['mock-account-id']); + + expect(onStatus).toHaveBeenLastCalledWith({ + accountId: 'mock-account-id', + chainIds: [SOLANA_MAINNET], + error: null, + }); + expect(controller.getAccountBalanceErrors()).toStrictEqual({}); + expect(assetsUpdateHandler).toHaveBeenCalledWith( + expect.objectContaining({ + assetsBalance: { + 'mock-account-id': { [MOCK_SOL_ASSET]: { amount: '1000000000' } }, + }, + }), + ); + + cleanup(); + }); + + it('retryFailedAccounts ignores accounts that did not fail', async () => { + const { controller, mockHandleRequest, cleanup } = setupFailing(); + await flush(); + mockHandleRequest.mockClear(); + + await controller.retryFailedAccounts(['some-other-account']); + + expect(mockHandleRequest).not.toHaveBeenCalled(); + + cleanup(); + }); + + it('a successful fetch clears a previous failure', async () => { + const { controller, failing, onStatus, cleanup } = setupFailing(); + await flush(); + await controller.fetch(createDataRequest()); + + failing.value = false; + const response = await controller.fetch(createDataRequest()); + + expect(response.assetsBalance).toStrictEqual({ + 'mock-account-id': { [MOCK_SOL_ASSET]: { amount: '1000000000' } }, + }); + expect(onStatus).toHaveBeenLastCalledWith( + expect.objectContaining({ error: null }), + ); + expect(controller.getAccountBalanceErrors()).toStrictEqual({}); + + cleanup(); + }); + + it('a snap balances-updated event for the account clears the failure', async () => { + const { controller, onStatus, triggerBalancesUpdated, cleanup } = + setupFailing(); + await flush(); + await controller.fetch(createDataRequest()); + + triggerBalancesUpdated({ + balances: { + 'mock-account-id': { [MOCK_SOL_ASSET]: SOL_BALANCE }, + }, + }); + await flush(); + + expect(onStatus).toHaveBeenLastCalledWith( + expect.objectContaining({ accountId: 'mock-account-id', error: null }), + ); + expect(controller.getAccountBalanceErrors()).toStrictEqual({}); + + cleanup(); + }); + + it('retries automatically with backoff until the fetch succeeds', async () => { + const { controller, failing, onStatus, assetsUpdateHandler, cleanup } = + setupFailing({ retryBackoff: { initialDelayMs: 20, maxDelayMs: 40 } }); + await flush(); + await controller.fetch(createDataRequest()); + failing.value = false; + assetsUpdateHandler.mockClear(); + + await flush(150); + + expect(onStatus).toHaveBeenLastCalledWith( + expect.objectContaining({ error: null }), + ); + expect(controller.getAccountBalanceErrors()).toStrictEqual({}); + expect(assetsUpdateHandler).toHaveBeenCalledWith( + expect.objectContaining({ + assetsBalance: { + 'mock-account-id': { [MOCK_SOL_ASSET]: { amount: '1000000000' } }, + }, + }), + ); + + cleanup(); + }); + + it('destroy cancels pending retries', async () => { + const { controller, mockHandleRequest, cleanup } = setupFailing({ + retryBackoff: { initialDelayMs: 20 }, + }); + await flush(); + await controller.fetch(createDataRequest()); + cleanup(); + mockHandleRequest.mockClear(); + + await flush(100); + + expect(mockHandleRequest).not.toHaveBeenCalled(); + }); + + it('a throwing status listener does not break the fetch', async () => { + const onStatus = jest.fn(() => { + throw new Error('listener failed'); + }); + const { controller, cleanup } = setupFailing({ + onBalanceFetchStatusChange: onStatus, + }); + await flush(); + + const response = await controller.fetch(createDataRequest()); + + expect(response.assetsBalance).toStrictEqual({}); + expect(onStatus).toHaveBeenCalled(); + + cleanup(); + }); + }); + it('createSnapDataSource factory creates instance', async () => { const rootMessenger = new Messenger< MockAnyNamespace, diff --git a/packages/assets-controller/src/data-sources/SnapDataSource.ts b/packages/assets-controller/src/data-sources/SnapDataSource.ts index a90c7f2b2d7..0093532e5c4 100644 --- a/packages/assets-controller/src/data-sources/SnapDataSource.ts +++ b/packages/assets-controller/src/data-sources/SnapDataSource.ts @@ -20,7 +20,9 @@ import type { Json, JsonRpcRequest } from '@metamask/utils'; import type { AssetsControllerMessenger } from '../AssetsController.js'; import { projectLogger, createModuleLogger } from '../logger.js'; import type { + AccountWithSupportedChains, AssetBalance, + AssetsBalanceFetchError, AssetsControllerState, ChainId, Caip19AssetId, @@ -78,6 +80,40 @@ export const KEYRING_PERMISSION = 'endowment:keyring'; /** The permission name for snap assets endowment (contains chainIds) */ export const ASSETS_PERMISSION = 'endowment:assets'; +/** First automatic retry delay after a failed snap balance fetch. */ +export const SNAP_BALANCE_RETRY_INITIAL_DELAY_MS = 30_000; + +/** Maximum automatic retry delay after repeated snap balance fetch failures. */ +export const SNAP_BALANCE_RETRY_MAX_DELAY_MS = 5 * 60_000; + +/** + * Exponential backoff for automatic snap balance retries. + * + * @param attempts - 1-based index of the upcoming automatic retry (1 for the + * retry scheduled right after the first failure, 2 after one failed retry...). + * @param initialDelayMs - Delay after the first failure. + * @param maxDelayMs - Upper bound. + * @returns The delay before the next retry, in ms. + */ +export function getSnapBalanceRetryDelay( + attempts: number, + initialDelayMs: number = SNAP_BALANCE_RETRY_INITIAL_DELAY_MS, + maxDelayMs: number = SNAP_BALANCE_RETRY_MAX_DELAY_MS, +): number { + const exponent = Math.max(0, attempts - 1); + return Math.min(maxDelayMs, initialDelayMs * 2 ** Math.min(exponent, 20)); +} + +/** + * Per-account balance fetch status change reported by the SnapDataSource. + * `error: null` means the balance is available again. + */ +export type SnapBalanceFetchStatusUpdate = { + accountId: string; + chainIds: ChainId[]; + error: AssetsBalanceFetchError | null; +}; + // ============================================================================ // PERMISSION UTILITIES // ============================================================================ @@ -189,6 +225,13 @@ export type SnapDataSourceOptions = { configuredNetworks?: ChainId[]; /** Default polling interval in ms for subscriptions */ pollInterval?: number; + /** + * Called when a snap account's balance becomes unavailable (a fetch failed) + * or available again (a fetch succeeded or the snap pushed balances). + */ + onBalanceFetchStatusChange?: (update: SnapBalanceFetchStatusUpdate) => void; + /** Automatic retry backoff for failed per-account fetches. */ + retryBackoff?: { initialDelayMs?: number; maxDelayMs?: number }; /** Initial state */ state?: Partial; }; @@ -197,6 +240,28 @@ export type SnapDataSourceOptions = { // SNAP DATA SOURCE // ============================================================================ +/** + * Balance fetch failure tracked per account. + */ +type FailedAccountEntry = { + /** + * Retries made so far (scheduled automatic retries only; a manual retry + * resets the backoff). Ordinary fetches (polling, subscriptions, other + * controllers) that fail do not count: they never advance the backoff. + */ + attempts: number; + chainIds: ChainId[]; + /** Narrowed request used to retry the account. */ + request: DataRequest; + timer: ReturnType | null; + /** Message of the last failure. */ + error: string; + failedAt: number; + nextRetryAt: number | null; + /** Kind of retry whose fetch is in progress, if any. */ + retryInProgress: 'scheduled' | 'manual' | null; +}; + /** * Unified Snap data source that routes requests to the appropriate wallet snap * based on the chain ID prefix. @@ -245,6 +310,22 @@ export class SnapDataSource extends AbstractDataSource< /** Cache of KeyringClient instances per snap ID to avoid re-instantiation */ readonly #keyringClientCache: Map = new Map(); + readonly #onBalanceFetchStatusChange?: ( + update: SnapBalanceFetchStatusUpdate, + ) => void; + + readonly #retryInitialDelayMs: number; + + readonly #retryMaxDelayMs: number; + + /** + * Accounts whose last balance fetch failed, with the narrowed request used + * to retry them and the pending retry timer. + */ + readonly #failedAccounts: Map = new Map(); + + #destroyed = false; + constructor(options: SnapDataSourceOptions) { super(SNAP_DATA_SOURCE_NAME, { ...defaultSnapState, @@ -257,6 +338,12 @@ export class SnapDataSource extends AbstractDataSource< this.#isBalanceV6Enabled = options.isBalanceV6Enabled; this.#getAssetVisibility = options.getAssetVisibility; this.#getAssetsState = options.getAssetsState; + this.#onBalanceFetchStatusChange = options.onBalanceFetchStatusChange; + this.#retryInitialDelayMs = + options.retryBackoff?.initialDelayMs ?? + SNAP_BALANCE_RETRY_INITIAL_DELAY_MS; + this.#retryMaxDelayMs = + options.retryBackoff?.maxDelayMs ?? SNAP_BALANCE_RETRY_MAX_DELAY_MS; // Bind handlers for cleanup in destroy() this.#handleSnapBalancesUpdatedBound = @@ -330,6 +417,8 @@ export class SnapDataSource extends AbstractDataSource< if (accountAssets) { assetsBalance ??= {}; assetsBalance[accountId] = accountAssets; + // The snap pushed balances for this account: it is synchronized again. + this.#markAccountSucceeded(accountId); } } @@ -500,7 +589,8 @@ export class SnapDataSource extends AbstractDataSource< }; // Fetch balances for each account using its snap ID from metadata - for (const { account } of request.accountsWithSupportedChains) { + for (const entry of request.accountsWithSupportedChains) { + const { account } = entry; // Skip accounts without snap metadata (non-snap accounts) const snapId = account.metadata.snap?.id; if (!snapId) { @@ -508,10 +598,10 @@ export class SnapDataSource extends AbstractDataSource< } // Skip accounts whose snap doesn't support any of the requested chains - const snapSupportsRequestedChains = request.chainIds.some( + const supportedChainIds = request.chainIds.filter( (chainId) => this.state.chainToSnap[chainId] === snapId, ); - if (!snapSupportsRequestedChains) { + if (supportedChainIds.length === 0) { continue; } @@ -524,6 +614,7 @@ export class SnapDataSource extends AbstractDataSource< // If no assets, skip to next account if (!accountAssets || accountAssets.length === 0) { + this.#markAccountSucceeded(accountId); continue; } @@ -544,8 +635,13 @@ export class SnapDataSource extends AbstractDataSource< } } } - } catch { - // Expected when account doesn't belong to this snap + this.#markAccountSucceeded(accountId); + } catch (error) { + // The account's snap was resolved from its own metadata, so this is a + // real failure (e.g. the snap has not synchronized the account yet, or + // its network calls fail), not a foreign account. Keep the previous + // balance, but record the failure and retry with backoff. + this.#markAccountFailed(entry, supportedChainIds, request, error); } } @@ -574,7 +670,8 @@ export class SnapDataSource extends AbstractDataSource< }; // Fetch balances for each account using its snap ID from metadata - for (const { account } of request.accountsWithSupportedChains) { + for (const entry of request.accountsWithSupportedChains) { + const { account } = entry; // Skip accounts without snap metadata (non-snap accounts) const snapId = account.metadata.snap?.id; if (!snapId) { @@ -641,9 +738,12 @@ export class SnapDataSource extends AbstractDataSource< if (results.assetsBalance) { results.assetsBalance[accountId] = accountBalances; } - } catch { - // Snap failed or the account does not belong to this snap. Contribute - // nothing so previous balances stay in state. + this.#markAccountSucceeded(accountId); + } catch (error) { + // Snap failed. Contribute nothing so previous balances stay in state, + // but record the failure (the balance is unavailable, not 0) and retry + // with backoff. + this.#markAccountFailed(entry, supportedChainIds, request, error); } } @@ -868,6 +968,218 @@ export class SnapDataSource extends AbstractDataSource< } } + // ============================================================================ + // PER-ACCOUNT FAILURES AND RETRY + // ============================================================================ + + /** + * Accounts whose last balance fetch failed, keyed by account ID. + * + * @returns The current failures. + */ + getAccountBalanceErrors(): Record< + string, + { chainIds: ChainId[]; attempts: number } + > { + const errors: Record = + {}; + for (const [accountId, failure] of this.#failedAccounts) { + errors[accountId] = { + chainIds: [...failure.chainIds], + attempts: failure.attempts, + }; + } + return errors; + } + + /** + * Retry failed balance fetches now, instead of waiting for the backoff. + * A manual retry resets the backoff: if it fails, the next automatic retry + * comes after the initial delay again. + * + * @param accountIds - Accounts to retry; all failed accounts when omitted. + */ + async retryFailedAccounts(accountIds?: string[]): Promise { + const targets = accountIds + ? accountIds.filter((accountId) => this.#failedAccounts.has(accountId)) + : Array.from(this.#failedAccounts.keys()); + await Promise.all( + targets.map(async (accountId) => this.#retryAccount(accountId, 'manual')), + ); + } + + #markAccountSucceeded(accountId: string): void { + const failure = this.#failedAccounts.get(accountId); + if (!failure) { + return; + } + if (failure.timer) { + clearTimeout(failure.timer); + } + this.#failedAccounts.delete(accountId); + this.#notifyStatus({ accountId, chainIds: failure.chainIds, error: null }); + } + + /** + * Records a failed balance fetch for an account. + * + * Only retries drive the backoff: the first failure schedules the first + * retry after the initial delay; a failed scheduled retry schedules the next + * one with a doubled delay (capped); a failed manual retry starts over at + * the initial delay. Any other failing fetch of an account that is already + * failing (polling, subscriptions, the legacy controller...) keeps the + * pending retry and its counter, and only refreshes the error. + * + * @param entry - The account and its supported chains. + * @param chainIds - The chains whose balance could not be fetched. + * @param request - The originating request. + * @param error - The failure. + */ + #markAccountFailed( + entry: AccountWithSupportedChains, + chainIds: ChainId[], + request: DataRequest, + error: unknown, + ): void { + const accountId = entry.account.id; + const previous = this.#failedAccounts.get(accountId); + const message = error instanceof Error ? error.message : String(error); + const now = Date.now(); + const retryRequest: DataRequest = { + ...request, + chainIds, + accountsWithSupportedChains: [entry], + forceUpdate: true, + }; + + const isRetryResult = Boolean(previous?.retryInProgress); + if (previous && !isRetryResult && (previous.timer || this.#destroyed)) { + // Not a retry: keep the scheduled retry and the backoff step. + const changed = + previous.error !== message || + previous.chainIds.length !== chainIds.length || + previous.chainIds.some((chainId) => !chainIds.includes(chainId)); + previous.error = message; + previous.failedAt = now; + previous.chainIds = chainIds; + previous.request = retryRequest; + log('Snap balance fetch failed (retry already scheduled)', { + accountId, + chainIds, + attempts: previous.attempts, + error, + }); + if (changed) { + this.#notifyFailure(accountId, previous); + } + return; + } + + if (previous?.timer) { + clearTimeout(previous.timer); + } + // `attempts` was already advanced by `#retryAccount` for a scheduled + // retry and reset for a manual one; a first failure starts at 0. + const attempts = previous?.attempts ?? 0; + const delay = getSnapBalanceRetryDelay( + attempts + 1, + this.#retryInitialDelayMs, + this.#retryMaxDelayMs, + ); + log('Snap balance fetch failed', { + accountId, + chainIds, + attempts, + nextRetryInMs: delay, + error, + }); + + const failure: FailedAccountEntry = { + attempts, + chainIds, + request: retryRequest, + timer: null, + error: message, + failedAt: now, + nextRetryAt: null, + retryInProgress: previous?.retryInProgress ?? null, + }; + if (!this.#destroyed) { + failure.timer = setTimeout(() => { + failure.timer = null; + this.#retryAccount(accountId, 'scheduled').catch((retryError) => + log('Snap balance retry failed', { accountId, error: retryError }), + ); + }, delay); + // Do not keep a Node process (tests, scripts) alive for a retry. + (failure.timer as { unref?: () => void }).unref?.(); + failure.nextRetryAt = now + delay; + } + this.#failedAccounts.set(accountId, failure); + this.#notifyFailure(accountId, failure); + } + + #notifyFailure(accountId: string, failure: FailedAccountEntry): void { + this.#notifyStatus({ + accountId, + chainIds: failure.chainIds, + error: { + status: 'unavailable', + error: failure.error, + attempts: failure.attempts, + failedAt: failure.failedAt, + nextRetryAt: failure.nextRetryAt, + }, + }); + } + + /** + * Retries the balance fetch of a failed account. The snap is asked for the + * balance again, which also lets snaps that answer from local state start a + * sync of the account (the Bitcoin and Stellar snaps kick off a + * deduplicated sync / full scan when asked for an unknown balance). + * + * @param accountId - The account ID. + * @param kind - `scheduled` (automatic, advances the backoff) or `manual` + * (user-initiated, resets the backoff). + */ + async #retryAccount( + accountId: string, + kind: 'scheduled' | 'manual', + ): Promise { + const failure = this.#failedAccounts.get(accountId); + if (!failure || this.#destroyed) { + return; + } + if (failure.timer) { + clearTimeout(failure.timer); + failure.timer = null; + } + failure.attempts = kind === 'scheduled' ? failure.attempts + 1 : 0; + failure.retryInProgress = kind; + try { + // `fetch` records the outcome (success clears the failure, another + // failure reschedules the next retry). + const response = await this.fetch(failure.request); + if (Object.keys(response.assetsBalance ?? {}).length > 0) { + await this.#onAssetsUpdate(response); + } + } finally { + const current = this.#failedAccounts.get(accountId); + if (current) { + current.retryInProgress = null; + } + } + } + + #notifyStatus(update: SnapBalanceFetchStatusUpdate): void { + try { + this.#onBalanceFetchStatusChange?.(update); + } catch (error) { + log('Balance fetch status listener failed', { error }); + } + } + // ============================================================================ // KEYRING CLIENT // ============================================================================ @@ -933,6 +1245,15 @@ export class SnapDataSource extends AbstractDataSource< // Clear keyring client cache this.#keyringClientCache.clear(); + + // Stop pending retries + this.#destroyed = true; + for (const failure of this.#failedAccounts.values()) { + if (failure.timer) { + clearTimeout(failure.timer); + } + } + this.#failedAccounts.clear(); } } diff --git a/packages/assets-controller/src/index.ts b/packages/assets-controller/src/index.ts index 3c328054c1c..752072c3874 100644 --- a/packages/assets-controller/src/index.ts +++ b/packages/assets-controller/src/index.ts @@ -94,6 +94,7 @@ export type { DataResponse, AssetsUpdateMode, AssetsLoadingStatus, + AssetsBalanceFetchError, // Middleware types Context, NextFunction, diff --git a/packages/assets-controller/src/types.ts b/packages/assets-controller/src/types.ts index 37e7663eb37..18fb53f847f 100644 --- a/packages/assets-controller/src/types.ts +++ b/packages/assets-controller/src/types.ts @@ -421,6 +421,26 @@ export type AssetsUpdateMode = 'full' | 'merge'; export type AssetsLoadingStatus = 'loading' | 'loaded'; +/** + * Why a balance could not be fetched for an account on a chain (non-EVM snap + * accounts today). Present only while the balance is unavailable: the entry is + * removed once a fetch succeeds or the snap pushes balances for the account. + * A balance in `assetsBalance` for the same account and chain is then the + * last-known value (or a placeholder), not a confirmed amount. + */ +export type AssetsBalanceFetchError = { + /** Always `'unavailable'` for now; kept as a field to allow more states. */ + status: 'unavailable'; + /** The error message of the last failed attempt. */ + error: string; + /** Number of consecutive failed attempts. */ + attempts: number; + /** When the last attempt failed (ms since epoch). */ + failedAt: number; + /** When the next automatic retry is scheduled (ms since epoch), if any. */ + nextRetryAt: number | null; +}; + // ============================================================================ // DATA SOURCE <-> CONTROLLER (DIRECT CALLS, NO MESSENGER PER SOURCE) // ============================================================================ @@ -503,6 +523,14 @@ export type AssetsControllerState = { /** Currently-active ISO 4217 currency code */ selectedCurrency: SupportedCurrency; assetsLoadingStatus?: Record; + /** + * Per-account, per-chain balance fetch failures (not persisted). See + * {@link AssetsBalanceFetchError}. + */ + assetsBalanceErrors?: Record< + AccountId, + Record + >; }; // ============================================================================