diff --git a/packages/tron-wallet-snap/src/clients/tron-http/TronHttpClient.test.ts b/packages/tron-wallet-snap/src/clients/tron-http/TronHttpClient.test.ts index 5685a496b..2233f240a 100644 --- a/packages/tron-wallet-snap/src/clients/tron-http/TronHttpClient.test.ts +++ b/packages/tron-wallet-snap/src/clients/tron-http/TronHttpClient.test.ts @@ -1,5 +1,6 @@ import { BaseConfigProvider, + BrokenCircuitError, FetchPolicyErrorName, } from '@metamask/snap-networks-utils'; @@ -120,4 +121,73 @@ describe('TronHttpClient', () => { expect(fetchMock).toHaveBeenCalledTimes(2); }); }); + + describe('open circuit bypass (v5)', () => { + /** + * Opens the fetch policy circuit with non-retryable server failures. + * + * @param client - The client. + */ + async function openCircuit(client: TronHttpClient): Promise { + jest + .spyOn(global, 'fetch') + .mockImplementation( + async () => new globalThis.Response('', { status: 501 }), + ); + for (let attempt = 0; attempt < 30; attempt++) { + try { + await client.getReward(Network.Mainnet, `address-${attempt}`); + } catch (error) { + if (error instanceof BrokenCircuitError) { + return; + } + } + } + throw new Error('The circuit did not open'); + } + + it('lets an explicit request reach the network once while the circuit is open', async () => { + const client = new TronHttpClient({ configProvider }); + await openCircuit(client); + const fetchMock = jest.spyOn(global, 'fetch').mockImplementation( + async () => + new globalThis.Response(JSON.stringify({ reward: 7 }), { + status: 200, + }), + ); + + // The spy is reused: forget the calls that opened the circuit. + fetchMock.mockClear(); + + await expect( + client.getReward(Network.Mainnet, 'address'), + ).rejects.toBeInstanceOf(BrokenCircuitError); + expect(fetchMock).not.toHaveBeenCalled(); + + expect( + await client.getReward(Network.Mainnet, 'address', { + bypassOpenCircuit: true, + }), + ).toBe(7); + expect(fetchMock).toHaveBeenCalledTimes(1); + + // The next explicit call for the same request (the balances call that + // follows `listAccountAssets`) reuses the direct result. + expect( + await client.getReward(Network.Mainnet, 'address', { + bypassOpenCircuit: true, + }), + ).toBe(7); + expect(fetchMock).toHaveBeenCalledTimes(1); + + // The service answered: the circuit is closed again for every read. + + expect( + await client.getAccountResources(Network.Mainnet, 'address', { + bypassOpenCircuit: true, + }), + ).toStrictEqual({ reward: 7 }); + expect(fetchMock).toHaveBeenCalledTimes(2); + }); + }); }); diff --git a/packages/tron-wallet-snap/src/clients/tron-http/TronHttpClient.ts b/packages/tron-wallet-snap/src/clients/tron-http/TronHttpClient.ts index ac3207b5d..87ba80735 100644 --- a/packages/tron-wallet-snap/src/clients/tron-http/TronHttpClient.ts +++ b/packages/tron-wallet-snap/src/clients/tron-http/TronHttpClient.ts @@ -14,6 +14,11 @@ import { Network } from '../../constants'; import type { ConfigProvider } from '../../services/config'; import { hexToString } from '../../utils/hex'; import logger, { noOpLogger } from '../../utils/logger'; +import { OpenCircuitBypass } from '../../utils/openCircuitBypass'; +import type { + BypassablePolicy, + ExplicitRequestOptions, +} from '../../utils/openCircuitBypass'; import { AccountResourcesStruct, ChainParametersResponseStruct, @@ -55,7 +60,9 @@ export class TronHttpClient { } > = new Map(); - readonly #policies: Map; + readonly #policies: Map; + + readonly #openCircuitBypass = new OpenCircuitBypass(); readonly #requestCache: ( network: Network, @@ -172,12 +179,14 @@ export class TronHttpClient { * @see https://developers.tron.network/reference/getaccountresource * @param network - Network to query. * @param accountAddress - Account address in base58 format. + * @param options - Options of an explicit client request. * @returns Promise - Account resources (energy, bandwidth, etc.). * @throws Error - HTTP errors or configuration errors. */ async getAccountResources( network: Network, accountAddress: string, + options?: ExplicitRequestOptions, ): Promise { const client = this.#clients.get(network); if (!client) { @@ -195,11 +204,17 @@ export class TronHttpClient { visible: true, }); - const response = await this.#fetch(network, url, { - method: 'POST', - headers, - body, - }); + const response = await this.#fetch( + network, + url, + { + method: 'POST', + headers, + body, + }, + [], + options, + ); const accountResources: AccountResources = JSON.parse(response.body); @@ -434,9 +449,14 @@ export class TronHttpClient { * @see https://developers.tron.network/reference/getreward * @param network - Network to query * @param accountAddress - Account address in base58 format + * @param options - Options of an explicit client request. * @returns Promise - Unclaimed rewards in sun (0 if no rewards) */ - async getReward(network: Network, accountAddress: string): Promise { + async getReward( + network: Network, + accountAddress: string, + options?: ExplicitRequestOptions, + ): Promise { const client = this.#clients.get(network); if (!client) { throw new Error(`No client configured for network: ${network}`); @@ -453,11 +473,17 @@ export class TronHttpClient { visible: true, }); - const response = await this.#fetch(network, url, { - method: 'POST', - headers, - body, - }); + const response = await this.#fetch( + network, + url, + { + method: 'POST', + headers, + body, + }, + [], + options, + ); const rewardResponse: GetRewardResponse = JSON.parse(response.body); @@ -472,14 +498,25 @@ export class TronHttpClient { url: string, init: RequestInit, allowedStatuses: number[] = [], + options?: ExplicitRequestOptions, ): Promise<{ body: string; status: number }> { - return this.#requestCache( - network, - url, - init.method ?? 'GET', - init.headers as Record | undefined, - typeof init.body === 'string' ? init.body : '', - allowedStatuses, + const method = init.method ?? 'GET'; + const headers = init.headers as Record | undefined; + const body = typeof init.body === 'string' ? init.body : ''; + return this.#openCircuitBypass.run( + `${network} ${method} ${url} ${body}`, + options, + async () => + this.#requestCache( + network, + url, + method, + headers, + body, + allowedStatuses, + ), + async () => this.#request(url, method, headers, body, allowedStatuses), + this.#policies.get(network), ); } @@ -496,26 +533,36 @@ export class TronHttpClient { throw new Error(`No policy configured for network: ${network}`); } - return executeWithFetchPolicy(policy, async () => { - const response = await fetch(url, { - method, - headers, - ...(body ? { body } : {}), - signal: AbortSignal.timeout(10_000), - }); - if (!response.ok && !allowedStatuses.includes(response.status)) { - throw createFetchPolicyError(`HTTP error! status: ${response.status}`, { - httpStatus: response.status, - retryAfterMs: parseRetryAfter(response.headers.get('Retry-After')), - }); - } - if (allowedStatuses.includes(response.status)) { - return { body: '', status: response.status }; - } - return { - body: await response.text(), - status: response.status, - }; + return executeWithFetchPolicy(policy, async () => + this.#request(url, method, headers, body, allowedStatuses), + ); + } + + async #request( + url: string, + method: string, + headers: Record | undefined, + body: string, + allowedStatuses: number[], + ): Promise<{ body: string; status: number }> { + const response = await fetch(url, { + method, + headers, + ...(body ? { body } : {}), + signal: AbortSignal.timeout(10_000), }); + if (!response.ok && !allowedStatuses.includes(response.status)) { + throw createFetchPolicyError(`HTTP error! status: ${response.status}`, { + httpStatus: response.status, + retryAfterMs: parseRetryAfter(response.headers.get('Retry-After')), + }); + } + if (allowedStatuses.includes(response.status)) { + return { body: '', status: response.status }; + } + return { + body: await response.text(), + status: response.status, + }; } } diff --git a/packages/tron-wallet-snap/src/clients/trongrid/TrongridApiClient.test.ts b/packages/tron-wallet-snap/src/clients/trongrid/TrongridApiClient.test.ts index 84a3d00cb..87296a5b2 100644 --- a/packages/tron-wallet-snap/src/clients/trongrid/TrongridApiClient.test.ts +++ b/packages/tron-wallet-snap/src/clients/trongrid/TrongridApiClient.test.ts @@ -1,5 +1,6 @@ import { BaseConfigProvider, + BrokenCircuitError, InMemoryCache, } from '@metamask/snap-networks-utils'; import type { ICache, Serializable } from '@metamask/snap-networks-utils'; @@ -737,4 +738,106 @@ describe('TrongridApiClient', () => { }); }); }); + + describe('open circuit bypass (v5)', () => { + const mockAddress = 'TGJn1wnUYHJbvN88cynZbsAz2EMeZq73yx'; + const mockTrc20Balances: Trc20Balance[] = [ + { TR7NHqjeKQxGTCi8q8ZY4pL8otSzgjLj6t: '24249143' }, + ]; + + /** + * Opens the fetch policy circuit with non-retryable server failures. + * + * @param client - The client. + */ + async function openCircuit(client: TrongridApiClient): Promise { + jest + .spyOn(global, 'fetch') + // eslint-disable-next-line no-restricted-globals + .mockImplementation(async () => new Response('', { status: 501 })); + for (let attempt = 0; attempt < 30; attempt++) { + try { + await client.getTrc20BalancesByAddress( + Network.Mainnet, + `address-${attempt}`, + ); + } catch (error) { + if (error instanceof BrokenCircuitError) { + return; + } + } + } + throw new Error('The circuit did not open'); + } + + it('lets an explicit request reach the network once while the circuit is open', async () => { + await withTrongridApiClient(async ({ client }) => { + await openCircuit(client); + const fetchMock = jest.spyOn(global, 'fetch').mockImplementation( + async () => + // eslint-disable-next-line no-restricted-globals + new Response( + JSON.stringify({ + data: mockTrc20Balances, + success: true, + meta: { at: 1770121997373, page_size: 4 }, + }), + { status: 200, headers: { 'Content-Type': 'application/json' } }, + ), + ); + + // The spy is reused: forget the calls that opened the circuit. + fetchMock.mockClear(); + + // Background reads stay short-circuited. + await expect( + client.getTrc20BalancesByAddress(Network.Mainnet, mockAddress), + ).rejects.toBeInstanceOf(BrokenCircuitError); + expect(fetchMock).not.toHaveBeenCalled(); + + // Concurrent identical explicit requests share one attempt. + const results = await Promise.all([ + client.getTrc20BalancesByAddress(Network.Mainnet, mockAddress, { + bypassOpenCircuit: true, + }), + client.getTrc20BalancesByAddress(Network.Mainnet, mockAddress, { + bypassOpenCircuit: true, + }), + ]); + expect(results).toStrictEqual([mockTrc20Balances, mockTrc20Balances]); + expect(fetchMock).toHaveBeenCalledTimes(1); + + // The next explicit call reuses the direct result. + expect( + await client.getTrc20BalancesByAddress(Network.Mainnet, mockAddress, { + bypassOpenCircuit: true, + }), + ).toStrictEqual(mockTrc20Balances); + expect(fetchMock).toHaveBeenCalledTimes(1); + + // The service answered: background reads are no longer + // short-circuited. + expect( + await client.getTrc20BalancesByAddress(Network.Mainnet, mockAddress), + ).toStrictEqual(mockTrc20Balances); + expect(fetchMock).toHaveBeenCalledTimes(2); + }); + }); + + it('does not bypass the circuit for other failures', async () => { + await withTrongridApiClient(async ({ client }) => { + const fetchMock = jest + .spyOn(global, 'fetch') + // eslint-disable-next-line no-restricted-globals + .mockImplementation(async () => new Response('', { status: 404 })); + + await expect( + client.getTrc20BalancesByAddress(Network.Mainnet, mockAddress, { + bypassOpenCircuit: true, + }), + ).rejects.toThrow(Error); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + }); + }); }); diff --git a/packages/tron-wallet-snap/src/clients/trongrid/TrongridApiClient.ts b/packages/tron-wallet-snap/src/clients/trongrid/TrongridApiClient.ts index 488759c88..2e86db82c 100644 --- a/packages/tron-wallet-snap/src/clients/trongrid/TrongridApiClient.ts +++ b/packages/tron-wallet-snap/src/clients/trongrid/TrongridApiClient.ts @@ -18,6 +18,11 @@ import { assert } from '@metamask/superstruct'; import { Network } from '../../constants'; import type { ConfigProvider } from '../../services/config'; import logger, { noOpLogger } from '../../utils/logger'; +import { OpenCircuitBypass } from '../../utils/openCircuitBypass'; +import type { + BypassablePolicy, + ExplicitRequestOptions, +} from '../../utils/openCircuitBypass'; import type { TronHttpClient } from '../tron-http/TronHttpClient'; import type { ChainParameter } from '../tron-http/types'; import { @@ -76,7 +81,9 @@ export class TrongridApiClient { readonly #cache: ICache; - readonly #policies: Map; + readonly #policies: Map; + + readonly #openCircuitBypass = new OpenCircuitBypass(); readonly #requestCache: ( scope: Network, @@ -173,6 +180,7 @@ export class TrongridApiClient { * @see https://developers.tron.network/reference/get-account-info-by-address * @param scope - The network to query (e.g., 'mainnet', 'shasta') * @param address - The TRON address to query + * @param options - Options of an explicit client request. * @returns Promise - Account data including balances. * @throws TrongridAccountNotFoundError - for inactive accounts. * @throws Error - HTTP errors or API failures. @@ -180,6 +188,7 @@ export class TrongridApiClient { async getAccountInfoByAddress( scope: Network, address: string, + options?: ExplicitRequestOptions, ): Promise { const client = this.#clients.get(scope); if (!client) { @@ -193,7 +202,7 @@ export class TrongridApiClient { pathParams: { address }, }); - const response = await this.#fetch(scope, url, headers); + const response = await this.#fetch(scope, url, headers, options); const rawData: TrongridApiResponse = JSON.parse(response); @@ -324,11 +333,13 @@ export class TrongridApiClient { * @see https://developers.tron.network/reference/get-trc20-token-holder-balances * @param scope - The network to query (e.g., 'mainnet', 'shasta') * @param address - The TRON address to query + * @param options - Options of an explicit client request. * @returns Promise - Array of TRC20 balances (contract address -> balance) */ async getTrc20BalancesByAddress( scope: Network, address: string, + options?: ExplicitRequestOptions, ): Promise { const client = this.#clients.get(scope); if (!client) { @@ -342,7 +353,7 @@ export class TrongridApiClient { pathParams: { address }, }); - const response = await this.#fetch(scope, url, headers); + const response = await this.#fetch(scope, url, headers, options); const rawData: TrongridApiResponse = JSON.parse(response); @@ -403,8 +414,15 @@ export class TrongridApiClient { scope: Network, url: string, headers: Record, + options?: ExplicitRequestOptions, ): Promise { - return this.#requestCache(scope, url, headers); + return this.#openCircuitBypass.run( + `${scope} GET ${url}`, + options, + async () => this.#requestCache(scope, url, headers), + async () => this.#request(url, headers), + this.#policies.get(scope), + ); } async #fetchUncached( @@ -417,15 +435,22 @@ export class TrongridApiClient { throw new Error(`No policy configured for network: ${scope}`); } - return executeWithFetchPolicy(policy, async () => { - const response = await fetch(url, { - headers, - signal: AbortSignal.timeout(10_000), - }); - if (!response.ok) { - throw createTrongridHttpError(response); - } - return response.text(); + return executeWithFetchPolicy(policy, async () => + this.#request(url, headers), + ); + } + + async #request( + url: string, + headers: Record, + ): Promise { + const response = await fetch(url, { + headers, + signal: AbortSignal.timeout(10_000), }); + if (!response.ok) { + throw createTrongridHttpError(response); + } + return response.text(); } } diff --git a/packages/tron-wallet-snap/src/handlers/keyring/keyring.openCircuit.test.ts b/packages/tron-wallet-snap/src/handlers/keyring/keyring.openCircuit.test.ts new file mode 100644 index 000000000..7cc022895 --- /dev/null +++ b/packages/tron-wallet-snap/src/handlers/keyring/keyring.openCircuit.test.ts @@ -0,0 +1,346 @@ +import { KeyringRpcMethod } from '@metamask/keyring-api'; +import type { Json, JsonRpcRequest } from '@metamask/utils'; +import { + BaseConfigProvider, + BrokenCircuitError, + InMemoryCache, + InMemoryState, +} from '@metamask/snap-networks-utils'; + +import type { PriceApiClient } from '../../clients/price-api/PriceApiClient'; +import type { SnapClient } from '../../clients/snap/SnapClient'; +import type { TokenApiClient } from '../../clients/token-api/TokenApiClient'; +import { TronHttpClient } from '../../clients/tron-http/TronHttpClient'; +import { TrongridApiClient } from '../../clients/trongrid/TrongridApiClient'; +import { KnownCaip19Id, Network } from '../../constants'; +import type { TronKeyringAccount } from '../../entities/tronKeyringAccount'; +import type { AccountsService } from '../../services/accounts/AccountsService'; +import { SnapAssetsAdapter } from '../../services/assets/adapters/SnapAssetsAdapter'; +import type { CoreAssetsAdapter } from '../../services/assets/adapters/CoreAssetsAdapter'; +import type { AssetsRepository } from '../../services/assets/AssetsRepository'; +import { AssetsService } from '../../services/assets/AssetsService'; +import { + ConfigStruct, + ENVIRONMENT, +} from '../../services/config/ConfigProvider'; +import type { ConfirmationHandler } from '../../services/confirmation/ConfirmationHandler'; +import type { UnencryptedStateValue } from '../../services/state/stateTypes'; +import type { TransactionsService } from '../../services/transactions/TransactionsService'; +import type { WalletService } from '../../services/wallet/WalletService'; +import { shouldTrack } from '../../utils/errors'; +import { + ACCOUNT_INFO_FETCH_FAILED_MESSAGE, + failureStreakGate, +} from '../../utils/failureStreakGate'; +import { mockLogger } from '../../utils/mockLogger'; +import { OPEN_CIRCUIT_ERROR_MESSAGE } from '../../utils/openCircuitBypass'; +import { KeyringHandler } from './keyring'; + +/* eslint-disable no-restricted-globals */ + +/** + * The URL of a fetch input. + * + * @param input - The fetch input. + * @returns The URL. + */ +function urlOf(input: RequestInfo | URL): string { + return input instanceof Request ? input.url : input.toString(); +} + +const ADDRESS = 'TJRabPrwbZy45sbavfcjinPJC18kjpRTv8'; + +const account: TronKeyringAccount = { + id: '123e4567-e89b-42d3-a456-426614174000', + address: ADDRESS, + options: {}, + methods: [], + type: 'tron:eoa', + scopes: [Network.Mainnet], + entropySource: 'entropy-source-1', + derivationPath: "m/44'/195'/0'/0/0", + index: 0, +}; + +/** + * A healthy upstream: TronGrid account info, account resources and rewards. + * + * @param input - The fetch input. + * @returns The response. + */ +async function healthyFetch(input: RequestInfo | URL): Promise { + const url = urlOf(input); + const json = (body: unknown): Response => + new Response(JSON.stringify(body), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + if (url.includes('/v1/accounts/')) { + return json({ + data: [{ address: ADDRESS, balance: 2_500_000 }], + success: true, + meta: { at: 1770121997373, page_size: 1 }, + }); + } + if (url.includes('/wallet/getaccountresource')) { + return json({}); + } + if (url.includes('/wallet/getReward')) { + return json({ reward: 0 }); + } + throw new Error(`Unexpected request: ${url}`); +} + +/** + * Builds the real keyring path of a balance refresh: keyring handler (through + * the keyring SDK's request dispatch), assets service, snap assets adapter and + * the real TronGrid / Tron HTTP clients with their real fetch policies. Only + * `fetch` and unrelated collaborators are mocked. + * + * @returns The handler and the clients. + */ +function setup(): { + keyringHandler: KeyringHandler; + trongridApiClient: TrongridApiClient; + tronHttpClient: TronHttpClient; +} { + const baseUrls = { + [Network.Mainnet]: 'https://api.trongrid.io', + [Network.Nile]: 'https://nile.trongrid.io', + [Network.Shasta]: 'https://api.shasta.trongrid.io', + }; + const configProvider = new BaseConfigProvider( + { + ...ENVIRONMENT, + trongridApi: { baseUrls }, + tronHttpApi: { baseUrls }, + }, + ConfigStruct, + ); + const tronHttpClient = new TronHttpClient({ configProvider }); + const trongridApiClient = new TrongridApiClient({ + configProvider, + tronHttpClient, + cache: new InMemoryCache(mockLogger), + }); + + const snapClient = { + trackError: jest.fn(), + } as unknown as SnapClient; + const snapAssetsAdapter = new SnapAssetsAdapter({ + logger: mockLogger, + state: new InMemoryState( + {} as UnencryptedStateValue, + ), + assetsRepository: { + saveMany: jest.fn(), + } as unknown as AssetsRepository, + trongridApiClient, + tronHttpClient, + priceApiClient: { + getMultipleSpotPrices: jest.fn().mockResolvedValue({}), + } as unknown as PriceApiClient, + tokenApiClient: { + getTokensMetadata: jest.fn().mockResolvedValue({}), + } as unknown as TokenApiClient, + snapClient, + }); + const assetsService = new AssetsService({ + snapAdapter: snapAssetsAdapter, + coreAdapter: {} as CoreAssetsAdapter, + remoteFeatureFlagsProvider: { + getFeatureFlag: jest.fn(), + } as never, + }); + + const keyringHandler = new KeyringHandler({ + logger: mockLogger, + snapClient, + accountsService: { + findById: jest.fn().mockResolvedValue(account), + } as unknown as AccountsService, + assetsService, + transactionsService: {} as TransactionsService, + walletService: {} as WalletService, + confirmationHandler: {} as ConfirmationHandler, + }); + + return { keyringHandler, trongridApiClient, tronHttpClient }; +} + +/** + * Opens the circuit of the mainnet TronGrid and Tron HTTP fetch policies the + * way a broken connection does: repeated failing background reads (no bypass) + * until the policy short-circuits them. + * + * @param clients - The clients. + * @param clients.trongridApiClient - The TronGrid client. + * @param clients.tronHttpClient - The Tron HTTP client. + */ +async function openCircuits({ + trongridApiClient, + tronHttpClient, +}: { + trongridApiClient: TrongridApiClient; + tronHttpClient: TronHttpClient; +}): Promise { + jest + .spyOn(global, 'fetch') + .mockImplementation(async () => new Response('', { status: 501 })); + const reads = [ + async (index: number): Promise => + trongridApiClient.getAccountInfoByAddress( + Network.Mainnet, + `address-${index}`, + ), + async (index: number): Promise => + tronHttpClient.getAccountResources(Network.Mainnet, `address-${index}`), + ]; + for (const read of reads) { + let opened = false; + for (let index = 0; index < 30 && !opened; index++) { + try { + await read(index); + } catch (error) { + opened = error instanceof BrokenCircuitError; + } + } + if (!opened) { + throw new Error('The circuit did not open'); + } + } +} + +const request = ( + method: string, + params: Record, +): JsonRpcRequest => ({ + jsonrpc: '2.0', + id: 1, + method, + params, +}); + +describe('KeyringHandler balance refresh while the fetch circuit is open (v6)', () => { + const originalFetch = global.fetch; + + afterEach(() => { + global.fetch = originalFetch; + jest.restoreAllMocks(); + failureStreakGate.clear(); + }); + + it('reaches the network for listAccountAssets then getAccountBalances (the Retry path)', async () => { + const clients = setup(); + await openCircuits(clients); + + // Background reads are short-circuited without a request. + const fetchMock = jest + .spyOn(global, 'fetch') + .mockImplementation(healthyFetch); + fetchMock.mockClear(); + await expect( + clients.trongridApiClient.getAccountInfoByAddress( + Network.Mainnet, + ADDRESS, + ), + ).rejects.toBeInstanceOf(BrokenCircuitError); + expect(fetchMock).not.toHaveBeenCalled(); + + // A failure streak is in progress (already reported once). + expect( + failureStreakGate.shouldReport(ACCOUNT_INFO_FETCH_FAILED_MESSAGE), + ).toBe(true); + + // The client's refresh: keyring_listAccountAssets, then + // keyring_getAccountBalances, through the keyring SDK dispatch. + const assets = await clients.keyringHandler.handle( + 'metamask', + request(KeyringRpcMethod.ListAccountAssets, { id: account.id }), + ); + expect(assets).toContain(KnownCaip19Id.TrxMainnet); + // TronGrid answered: the streak ended, the next failure is reported. + expect( + failureStreakGate.shouldReport(ACCOUNT_INFO_FETCH_FAILED_MESSAGE), + ).toBe(true); + + const requestedUrls = fetchMock.mock.calls.map(([input]) => urlOf(input)); + expect(requestedUrls).toStrictEqual( + expect.arrayContaining([ + `https://api.trongrid.io/v1/accounts/${ADDRESS}`, + 'https://api.trongrid.io/wallet/getaccountresource', + 'https://api.trongrid.io/wallet/getReward', + ]), + ); + + const balances = (await clients.keyringHandler.handle( + 'metamask', + request(KeyringRpcMethod.GetAccountBalances, { + id: account.id, + assets: [KnownCaip19Id.TrxMainnet], + }), + )) as Record; + expect(balances[KnownCaip19Id.TrxMainnet]).toStrictEqual({ + amount: '2.5', + unit: 'TRX', + }); + }); + + it('makes the request when the circuit is open before the call (state pre-check)', async () => { + const clients = setup(); + await openCircuits(clients); + const fetchMock = jest + .spyOn(global, 'fetch') + .mockImplementation(healthyFetch); + fetchMock.mockClear(); + + await clients.keyringHandler.handle( + 'metamask', + request(KeyringRpcMethod.GetAccountBalances, { + id: account.id, + assets: [KnownCaip19Id.TrxMainnet], + }), + ); + + expect( + fetchMock.mock.calls.filter(([input]) => + urlOf(input).includes('/v1/accounts/'), + ), + ).toHaveLength(1); + }); + + it('lets one direct attempt through per interval when it fails', async () => { + const clients = setup(); + await openCircuits(clients); + const fetchMock = jest + .spyOn(global, 'fetch') + .mockImplementation(async () => { + throw new TypeError('Failed to fetch'); + }); + // The spy is reused: forget the calls that opened the circuits. + fetchMock.mockClear(); + + // The direct attempt fails (still offline)... + const first = clients.keyringHandler.handle( + 'metamask', + request(KeyringRpcMethod.ListAccountAssets, { id: account.id }), + ); + await expect(first).rejects.toThrow('Failed to fetch account info'); + const attempts = fetchMock.mock.calls.length; + expect(attempts).toBeGreaterThan(0); + + // ...and an immediate repeat is not let through again. + await expect( + clients.keyringHandler.handle( + 'metamask', + request(KeyringRpcMethod.ListAccountAssets, { id: account.id }), + ), + ).rejects.toThrow('Failed to fetch account info'); + expect(fetchMock.mock.calls).toHaveLength(attempts); + }); + + it('does not track the open-circuit error rethrown by the keyring SDK as a plain Error', () => { + // `handleKeyringRequest` rethrows non-snap errors as `new Error(message)`. + expect(shouldTrack(new Error(OPEN_CIRCUIT_ERROR_MESSAGE))).toBe(false); + }); +}); diff --git a/packages/tron-wallet-snap/src/handlers/keyring/keyring.test.ts b/packages/tron-wallet-snap/src/handlers/keyring/keyring.test.ts index f75fc4366..447169f06 100644 --- a/packages/tron-wallet-snap/src/handlers/keyring/keyring.test.ts +++ b/packages/tron-wallet-snap/src/handlers/keyring/keyring.test.ts @@ -700,7 +700,7 @@ describe('KeyringHandler', () => { ]); expect( mockAssetsService.fetchAccountAssetsFromTrongrid, - ).toHaveBeenCalledWith(mockAccount); + ).toHaveBeenCalledWith(mockAccount, { bypassOpenCircuit: true }); expect(mockAssetsService.saveMany).toHaveBeenCalledWith([ mockLiveAsset, mockShastaAsset, @@ -740,7 +740,7 @@ describe('KeyringHandler', () => { }); expect( mockAssetsService.fetchAccountAssetsFromTrongrid, - ).toHaveBeenCalledWith(mockAccount); + ).toHaveBeenCalledWith(mockAccount, { bypassOpenCircuit: true }); expect(mockAssetsService.saveMany).toHaveBeenCalledWith([ mockLiveAsset, mockShastaAsset, diff --git a/packages/tron-wallet-snap/src/handlers/keyring/keyring.ts b/packages/tron-wallet-snap/src/handlers/keyring/keyring.ts index e1a746de9..13e0fbced 100644 --- a/packages/tron-wallet-snap/src/handlers/keyring/keyring.ts +++ b/packages/tron-wallet-snap/src/handlers/keyring/keyring.ts @@ -185,8 +185,15 @@ export class KeyringHandler implements KeyringSnapRpc { this.#logger.info('Getting account assets', { accountId }); + // Clients call `keyring_listAccountAssets` before + // `keyring_getAccountBalances` on every balance refresh, including the + // user's Retry. It is an explicit request too: without the bypass, an + // open circuit fails it without a request and the balances call that + // would bypass the circuit is never made. const assetEntities = - await this.#assetsService.fetchAccountAssetsFromTrongrid(account); + await this.#assetsService.fetchAccountAssetsFromTrongrid(account, { + bypassOpenCircuit: true, + }); await this.#assetsService.saveMany(assetEntities); @@ -278,8 +285,14 @@ export class KeyringHandler implements KeyringSnapRpc { const account = await this.#getAccountOrThrow(accountId); + // An explicit client request (e.g. the balance Retry): it may reach the + // network while a fetch policy's circuit is open (rate limited), so a + // retry made during the 30 s open window is not answered from the + // circuit alone. const assetsList = - await this.#assetsService.fetchAccountAssetsFromTrongrid(account); + await this.#assetsService.fetchAccountAssetsFromTrongrid(account, { + bypassOpenCircuit: true, + }); await this.#assetsService.saveMany(assetsList); diff --git a/packages/tron-wallet-snap/src/services/accounts/AccountsService.test.ts b/packages/tron-wallet-snap/src/services/accounts/AccountsService.test.ts index f8167c5b1..a4acf94ca 100644 --- a/packages/tron-wallet-snap/src/services/accounts/AccountsService.test.ts +++ b/packages/tron-wallet-snap/src/services/accounts/AccountsService.test.ts @@ -11,7 +11,7 @@ import type { import { AccountCreationType, TrxAccountType } from '@metamask/keyring-api'; import { getSelectedAccounts } from '@metamask/keyring-snap-sdk'; import type { Logger } from '@metamask/snap-networks-utils'; -import { LogLevel } from '@metamask/snap-networks-utils'; +import { BrokenCircuitError, LogLevel } from '@metamask/snap-networks-utils'; import type { SnapClient } from '../../clients/snap/SnapClient'; import { Network } from '../../constants'; @@ -1169,6 +1169,52 @@ describe('AccountsService', () => { ); }); + it('does not report accounts short-circuited by an open circuit (v5)', async () => { + const shortCircuited: TronKeyringAccount = { + id: 'sync-open-circuit-id', + address: 'TSyncOpenCircuit12345678901', + type: TrxAccountType.Eoa, + options: {}, + methods: [], + scopes: [], + entropySource: 'e1', + derivationPath: "m/44'/195'/0'/0/0", + index: 0, + }; + const failing: TronKeyringAccount = { + ...shortCircuited, + id: 'sync-fail-id', + address: 'TSyncFail123456789012345678', + derivationPath: "m/44'/195'/0'/0/1", + index: 1, + }; + + await withAccountsService( + async ({ accountsService, mockConfigProvider, mockAssetsService }) => { + mockConfigProvider.config = { + ...MOCK_CONFIG, + activeNetworks: [Network.Mainnet], + }; + mockAssetsService.fetchAccountAssets + .mockRejectedValueOnce( + new Error('Failed to fetch account info', { + cause: new BrokenCircuitError(), + }), + ) + .mockRejectedValueOnce(new Error('grpc unavailable')); + + await accountsService.synchronize([shortCircuited, failing]); + + expect(trackError).toHaveBeenCalledTimes(1); + const tracked = (trackError as jest.Mock).mock.calls[0][0] as Error; + expect(tracked.message).toBe( + 'Failed to fetch assets (1 failed): ' + + 'sync-fail-id: Error: grpc unavailable', + ); + }, + ); + }); + it('survives a hostile rejection reason that cannot be stringified', async () => { const account: TronKeyringAccount = { id: 'sync-hostile-id', diff --git a/packages/tron-wallet-snap/src/services/accounts/AccountsService.ts b/packages/tron-wallet-snap/src/services/accounts/AccountsService.ts index b6a207bec..735835d1f 100644 --- a/packages/tron-wallet-snap/src/services/accounts/AccountsService.ts +++ b/packages/tron-wallet-snap/src/services/accounts/AccountsService.ts @@ -33,6 +33,7 @@ import { createTronBip44KeypairDeriver, } from '../../utils/deriveTronFromCoinTypeNode'; import { sanitizeSensitiveError, trackError } from '../../utils/errors'; +import { isOpenCircuitError } from '../../utils/openCircuitBypass'; import { DerivationPathStruct } from '../../validation/structs'; import type { AssetsService } from '../assets/AssetsService'; import type { ConfigProvider } from '../config'; @@ -638,7 +639,16 @@ export class AccountsService { accountIds: string[], ): Promise { try { - const failures = getSyncFailuresFromSettledResult(results, accountIds); + // A fetch short-circuited by an open circuit made no request: not a + // failure to report (the failures that opened it were reported). + const failures = getSyncFailuresFromSettledResult( + results.map((result) => + result.status === 'rejected' && isOpenCircuitError(result.reason) + ? { status: 'fulfilled' as const, value: undefined } + : result, + ), + accountIds, + ); if (failures.length === 0) { return; diff --git a/packages/tron-wallet-snap/src/services/assets/AssetsService.test.ts b/packages/tron-wallet-snap/src/services/assets/AssetsService.test.ts index 37b3a2a52..5bd72b2c1 100644 --- a/packages/tron-wallet-snap/src/services/assets/AssetsService.test.ts +++ b/packages/tron-wallet-snap/src/services/assets/AssetsService.test.ts @@ -389,10 +389,20 @@ describe('AssetsService', () => { ).toHaveBeenCalledTimes(2); expect( mockTrongridApiClient.getAccountInfoByAddress, - ).toHaveBeenNthCalledWith(1, Network.Mainnet, mockAccount.address); + ).toHaveBeenNthCalledWith( + 1, + Network.Mainnet, + mockAccount.address, + undefined, + ); expect( mockTrongridApiClient.getAccountInfoByAddress, - ).toHaveBeenNthCalledWith(2, Network.Shasta, mockAccount.address); + ).toHaveBeenNthCalledWith( + 2, + Network.Shasta, + mockAccount.address, + undefined, + ); expect(findAsset(assets, KnownCaip19Id.TrxMainnet)?.rawAmount).toBe( '1000000', ); diff --git a/packages/tron-wallet-snap/src/services/assets/AssetsService.ts b/packages/tron-wallet-snap/src/services/assets/AssetsService.ts index 7e021f28d..852201aad 100644 --- a/packages/tron-wallet-snap/src/services/assets/AssetsService.ts +++ b/packages/tron-wallet-snap/src/services/assets/AssetsService.ts @@ -11,6 +11,7 @@ import { Networks } from '../../constants'; import type { Network } from '../../constants'; import type { AssetEntity } from '../../entities/assets'; import { extractScopesFromAssetIds } from '../../utils/caip'; +import type { ExplicitRequestOptions } from '../../utils/openCircuitBypass'; import type { CoreAssetsAdapter } from './adapters/CoreAssetsAdapter'; import { SnapAssetsAdapter } from './adapters/SnapAssetsAdapter'; @@ -113,12 +114,14 @@ export class AssetsService { * flag. * * @param account - The account to fetch live assets for. + * @param options - Options of an explicit client request. * @returns The live assets. */ async fetchAccountAssetsFromTrongrid( account: KeyringAccount, + options?: ExplicitRequestOptions, ): Promise { - return this.#snapAdapter.fetchAccountAssets(account); + return this.#snapAdapter.fetchAccountAssets(account, options); } /** diff --git a/packages/tron-wallet-snap/src/services/assets/adapters/SnapAssetsAdapter.ts b/packages/tron-wallet-snap/src/services/assets/adapters/SnapAssetsAdapter.ts index c80ba0788..ef272a737 100644 --- a/packages/tron-wallet-snap/src/services/assets/adapters/SnapAssetsAdapter.ts +++ b/packages/tron-wallet-snap/src/services/assets/adapters/SnapAssetsAdapter.ts @@ -40,6 +40,11 @@ import { import type { AssetEntity } from '../../../entities/assets'; import { parseTronCaipAssetType } from '../../../utils/caip'; import { toUiAmount } from '../../../utils/conversion'; +import { + ACCOUNT_INFO_FETCH_FAILED_MESSAGE, + failureStreakGate, +} from '../../../utils/failureStreakGate'; +import type { ExplicitRequestOptions } from '../../../utils/openCircuitBypass'; import type { UnencryptedStateValue } from '../../state/stateTypes'; import type { AssetsRepository } from '../AssetsRepository'; import { AssetFetchError } from '../errors'; @@ -172,6 +177,8 @@ export class SnapAssetsAdapter { * * @param scope - The network to query. * @param account - The keyring account. + * @param options - Options of an explicit client request: its balance + * reads may reach the network once while a fetch policy's circuit is open. * @throws {@link AssetFetchError} When the account info request fails for * reasons other than "account not found", since the resulting balance * state is unknown. @@ -180,6 +187,7 @@ export class SnapAssetsAdapter { async fetchAssetsAndBalancesForAccount( scope: Network, account: KeyringAccount, + options?: ExplicitRequestOptions, ): Promise { this.#logger.info('Fetching assets and balances by account', { account, @@ -191,16 +199,27 @@ export class SnapAssetsAdapter { tronAccountResourcesRequest, stakingRewardsRequest, ] = await Promise.allSettled([ - this.#trongridApiClient.getAccountInfoByAddress(scope, account.address), - this.#tronHttpClient.getAccountResources(scope, account.address), - this.#tronHttpClient.getReward(scope, account.address), + this.#trongridApiClient.getAccountInfoByAddress( + scope, + account.address, + options, + ), + this.#tronHttpClient.getAccountResources(scope, account.address, options), + this.#tronHttpClient.getReward(scope, account.address, options), ]); const isInactiveAccount = tronAccountInfoRequest.status === 'rejected'; + if ( + tronAccountInfoRequest.status === 'fulfilled' || + tronAccountInfoRequest.reason instanceof TrongridAccountNotFoundError + ) { + // TronGrid answered: a later account info failure starts a new streak. + failureStreakGate.endStreak(ACCOUNT_INFO_FETCH_FAILED_MESSAGE); + } if (isInactiveAccount) { const { reason } = tronAccountInfoRequest; if (!(reason instanceof TrongridAccountNotFoundError)) { - throw new AssetFetchError('Failed to fetch account info', { + throw new AssetFetchError(ACCOUNT_INFO_FETCH_FAILED_MESSAGE, { cause: reason, }); } @@ -213,7 +232,7 @@ export class SnapAssetsAdapter { const trc20BalancesFallback = isInactiveAccount ? await this.#trongridApiClient - .getTrc20BalancesByAddress(scope, account.address) + .getTrc20BalancesByAddress(scope, account.address, options) .catch(async (error) => { await this.#snapClient.trackError( new Error('Failed to fetch TRC20 balances for inactive account', { @@ -262,10 +281,17 @@ export class SnapAssetsAdapter { return this.#filterTokensWithoutPriceData(enrichedAssets, spotPrices); } - async fetchAccountAssets(account: KeyringAccount): Promise { + async fetchAccountAssets( + account: KeyringAccount, + options?: ExplicitRequestOptions, + ): Promise { const results = await Promise.all( account.scopes.map((scope) => - this.fetchAssetsAndBalancesForAccount(scope as Network, account), + this.fetchAssetsAndBalancesForAccount( + scope as Network, + account, + options, + ), ), ); diff --git a/packages/tron-wallet-snap/src/utils/errors.test.ts b/packages/tron-wallet-snap/src/utils/errors.test.ts index e890ff98e..b8b799945 100644 --- a/packages/tron-wallet-snap/src/utils/errors.test.ts +++ b/packages/tron-wallet-snap/src/utils/errors.test.ts @@ -1,6 +1,12 @@ +import { BrokenCircuitError } from '@metamask/snap-networks-utils'; import { SnapError, UserRejectedRequestError } from '@metamask/snaps-sdk'; -import { trackError, withCatchAndThrowSnapError } from './errors'; +import { shouldTrack, trackError, withCatchAndThrowSnapError } from './errors'; +import { + ACCOUNT_INFO_FETCH_FAILED_MESSAGE, + FAILURE_STREAK_REMINDER_MS, + failureStreakGate, +} from './failureStreakGate'; import { mockLogger } from './mockLogger'; jest.mock('./logger', () => ({ @@ -31,6 +37,73 @@ describe('errors', () => { expect(mockSnapRequest).not.toHaveBeenCalled(); }); + it('does not track the open-circuit short-circuit, even wrapped (v5)', async () => { + const { mockSnapRequest } = setupTest(); + + expect(await trackError(new BrokenCircuitError())).toBeUndefined(); + expect( + await trackError( + new Error('Failed to fetch account info', { + cause: new BrokenCircuitError(), + }), + ), + ).toBeUndefined(); + + expect(mockSnapRequest).not.toHaveBeenCalled(); + }); + + it('does not track the open-circuit error rethrown as a plain Error by the keyring SDK (v6)', async () => { + const { mockSnapRequest } = setupTest(); + + // The onboarding event: `handleKeyringRequest` rethrew the + // `BrokenCircuitError` as `new Error(message)`, and the handler wrapper + // tracked it. + await expect( + withCatchAndThrowSnapError(async () => { + throw new Error( + 'Execution prevented because the circuit breaker is open', + ); + }), + ).rejects.toThrow('circuit breaker is open'); + + expect(mockSnapRequest).not.toHaveBeenCalled(); + }); + + it('tracks "Failed to fetch account info" once per failure streak (v6)', async () => { + const { mockSnapRequest } = setupTest(); + mockSnapRequest.mockResolvedValue('id'); + failureStreakGate.clear(); + let now = 5_000_000; + const dateSpy = jest.spyOn(Date, 'now').mockImplementation(() => now); + const fail = async (): Promise => + withCatchAndThrowSnapError(async () => { + // What reaches the wrapper from a keyring call. + throw new Error(ACCOUNT_INFO_FETCH_FAILED_MESSAGE); + }).catch(() => undefined); + + try { + await fail(); + now += 15_000; + await fail(); + now += 15_000; + await fail(); + expect(mockSnapRequest).toHaveBeenCalledTimes(1); + + // Still failing after the reminder interval: reported again. + now += FAILURE_STREAK_REMINDER_MS; + await fail(); + expect(mockSnapRequest).toHaveBeenCalledTimes(2); + + // A success ends the streak: the next failure is reported. + failureStreakGate.endStreak(ACCOUNT_INFO_FETCH_FAILED_MESSAGE); + await fail(); + expect(mockSnapRequest).toHaveBeenCalledTimes(3); + } finally { + dateSpy.mockRestore(); + failureStreakGate.clear(); + } + }); + it('sanitizes errors before tracking', async () => { const { mockSnapRequest } = setupTest(); mockSnapRequest.mockResolvedValue('tracked-error-id'); @@ -91,4 +164,17 @@ describe('errors', () => { expect(mockSnapRequest).not.toHaveBeenCalled(); }); }); + + describe('shouldTrack', () => { + it('tracks other errors', () => { + expect(shouldTrack(new Error('Failed to fetch account info'))).toBe(true); + }); + + it('skips user rejections and open-circuit errors', () => { + expect(shouldTrack(new UserRejectedRequestError())).toBe(false); + expect( + shouldTrack(new Error('wrapped', { cause: new BrokenCircuitError() })), + ).toBe(false); + }); + }); }); diff --git a/packages/tron-wallet-snap/src/utils/errors.ts b/packages/tron-wallet-snap/src/utils/errors.ts index b8b2a7de1..e30e5af51 100644 --- a/packages/tron-wallet-snap/src/utils/errors.ts +++ b/packages/tron-wallet-snap/src/utils/errors.ts @@ -1,14 +1,42 @@ import { createSnapErrorHandling } from '@metamask/snap-networks-utils'; +import { UserRejectedRequestError } from '@metamask/snaps-sdk'; import { getSnapProvider } from '../clients/snap/getSnapProvider'; +import { failureStreakGate, getStreakGatedMessage } from './failureStreakGate'; import logger from './logger'; +import { isOpenCircuitError } from './openCircuitBypass'; import { sanitizeSensitiveError } from './sensitiveErrors'; export { isSnapRpcError, sanitizeSensitiveError } from './sensitiveErrors'; +/** + * Whether an error is reported to Sentry. Not reported: + * - user rejections (the default); + * - the fetch policy's open-circuit short-circuit, detected by class, flag, + * name or message through the cause chain (the keyring SDK rethrows it as a + * plain `Error` with the same message): no request was made, and the + * failures that opened the circuit were already reported; + * - repeats of a streak-gated failure ('Failed to fetch account info'): only + * the first failure of a streak is reported, then a reminder at most every + * 15 minutes while it lasts (see `FailureStreakGate`). + * + * @param error - The error. + * @returns `true` if the error should be tracked. + */ +export function shouldTrack(error: unknown): boolean { + if (error instanceof UserRejectedRequestError || isOpenCircuitError(error)) { + return false; + } + const gatedMessage = getStreakGatedMessage(error); + return ( + gatedMessage === undefined || failureStreakGate.shouldReport(gatedMessage) + ); +} + export const { trackError, withCatchAndThrowSnapError } = createSnapErrorHandling({ getSnapProvider, logError: logger.error.bind(logger), prepareError: sanitizeSensitiveError, + shouldTrack, }); diff --git a/packages/tron-wallet-snap/src/utils/failureStreakGate.test.ts b/packages/tron-wallet-snap/src/utils/failureStreakGate.test.ts new file mode 100644 index 000000000..9f26b9142 --- /dev/null +++ b/packages/tron-wallet-snap/src/utils/failureStreakGate.test.ts @@ -0,0 +1,60 @@ +import { + ACCOUNT_INFO_FETCH_FAILED_MESSAGE, + FAILURE_STREAK_REMINDER_MS, + FailureStreakGate, + getStreakGatedMessage, +} from './failureStreakGate'; + +describe('FailureStreakGate', () => { + let now: number; + + beforeEach(() => { + now = 1_000_000; + jest.spyOn(Date, 'now').mockImplementation(() => now); + }); + + afterEach(() => { + jest.restoreAllMocks(); + }); + + it('reports the first failure of a streak, then a reminder per interval', () => { + const gate = new FailureStreakGate(); + + expect(gate.shouldReport('a')).toBe(true); + now += 30_000; + expect(gate.shouldReport('a')).toBe(false); + now += FAILURE_STREAK_REMINDER_MS - 30_000 - 1; + expect(gate.shouldReport('a')).toBe(false); + now += 1; + expect(gate.shouldReport('a')).toBe(true); + expect(gate.shouldReport('a')).toBe(false); + }); + + it('starts a new streak after a success', () => { + const gate = new FailureStreakGate(); + + expect(gate.shouldReport('a')).toBe(true); + expect(gate.shouldReport('a')).toBe(false); + gate.endStreak('a'); + expect(gate.shouldReport('a')).toBe(true); + }); + + it('keeps keys independent', () => { + const gate = new FailureStreakGate(); + + expect(gate.shouldReport('a')).toBe(true); + expect(gate.shouldReport('b')).toBe(true); + gate.clear(); + expect(gate.shouldReport('a')).toBe(true); + }); +}); + +describe('getStreakGatedMessage', () => { + it('returns the gated message only', () => { + expect( + getStreakGatedMessage(new Error(ACCOUNT_INFO_FETCH_FAILED_MESSAGE)), + ).toBe(ACCOUNT_INFO_FETCH_FAILED_MESSAGE); + expect(getStreakGatedMessage(new Error('other'))).toBeUndefined(); + expect(getStreakGatedMessage('x')).toBeUndefined(); + }); +}); diff --git a/packages/tron-wallet-snap/src/utils/failureStreakGate.ts b/packages/tron-wallet-snap/src/utils/failureStreakGate.ts new file mode 100644 index 000000000..b286d9318 --- /dev/null +++ b/packages/tron-wallet-snap/src/utils/failureStreakGate.ts @@ -0,0 +1,97 @@ +/** + * The message of the error thrown when an account's on-chain info cannot be + * fetched (see `SnapAssetsAdapter.fetchAssetsAndBalancesForAccount`). Every + * balance refresh of a client (`listAccountAssets`, `getAccountBalances`, + * revalidation every 15 s, the user's Retry) fails with it while TronGrid is + * unreachable, so it is reported once per failure streak instead of once per + * call. + */ +export const ACCOUNT_INFO_FETCH_FAILED_MESSAGE = 'Failed to fetch account info'; + +/** + * While a failure streak lasts, it is reported again at most this often, so a + * long outage still shows up in Sentry as ongoing. + */ +export const FAILURE_STREAK_REMINDER_MS = 15 * 60_000; + +/** + * Reports a repeated failure once per streak: the first failure is reported, + * the following ones are not until the streak ends (a success for the same + * key) or {@link FAILURE_STREAK_REMINDER_MS} has passed since the last report. + * + * Keyed by message, not by account: the error reaching the snap's error + * handler from a keyring call is rethrown by the keyring SDK as a plain + * `Error(message)` without its cause or the account, and an upstream outage + * fails every account at once, so one event per outage carries the signal. + */ +export class FailureStreakGate { + readonly #reminderMs: number; + + readonly #reportedAt = new Map(); + + constructor(reminderMs: number = FAILURE_STREAK_REMINDER_MS) { + this.#reminderMs = reminderMs; + } + + /** + * Whether a failure should be reported, recording the report if so. + * + * @param key - The failure key (the error message). + * @returns `true` for the first failure of a streak or a due reminder. + */ + shouldReport(key: string): boolean { + const now = Date.now(); + const reportedAt = this.#reportedAt.get(key); + if (reportedAt !== undefined && now - reportedAt < this.#reminderMs) { + return false; + } + this.#reportedAt.set(key, now); + return true; + } + + /** + * Ends the failure streak of a key (the operation succeeded): its next + * failure is reported. + * + * @param key - The failure key. + */ + endStreak(key: string): void { + this.#reportedAt.delete(key); + } + + /** + * Forgets every streak. + */ + clear(): void { + this.#reportedAt.clear(); + } +} + +/** + * Failures reported once per streak (see {@link FailureStreakGate}). + */ +export const STREAK_GATED_FAILURE_MESSAGES: readonly string[] = [ + ACCOUNT_INFO_FETCH_FAILED_MESSAGE, +]; + +/** + * The snap-wide gate used by `shouldTrack`. + */ +export const failureStreakGate = new FailureStreakGate(); + +/** + * The streak-gated message of an error, if any. + * + * @param error - The error. + * @returns The gated message, or `undefined`. + */ +export function getStreakGatedMessage(error: unknown): string | undefined { + const message = + typeof error === 'object' && error !== null + ? (error as { message?: unknown }).message + : undefined; + return typeof message === 'string' && + STREAK_GATED_FAILURE_MESSAGES.includes(message) + ? message + : undefined; +} diff --git a/packages/tron-wallet-snap/src/utils/openCircuitBypass.test.ts b/packages/tron-wallet-snap/src/utils/openCircuitBypass.test.ts new file mode 100644 index 000000000..d084d6d76 --- /dev/null +++ b/packages/tron-wallet-snap/src/utils/openCircuitBypass.test.ts @@ -0,0 +1,308 @@ +import { BrokenCircuitError } from '@metamask/snap-networks-utils'; + +import { + isCircuitOpen, + isOpenCircuitError, + OPEN_CIRCUIT_BYPASS_INTERVAL_MS, + OPEN_CIRCUIT_BYPASS_RESULT_TTL_MS, + OPEN_CIRCUIT_ERROR_MESSAGE, + OpenCircuitBypass, +} from './openCircuitBypass'; + +describe('isOpenCircuitError', () => { + it('detects the open-circuit error and wrapping errors', () => { + const open = new BrokenCircuitError(); + + expect(isOpenCircuitError(open)).toBe(true); + expect( + isOpenCircuitError( + new Error('Failed to fetch account info', { + cause: new Error('wrapped', { cause: open }), + }), + ), + ).toBe(true); + }); + + it('does not match other errors', () => { + expect(isOpenCircuitError(new Error('HTTP 500'))).toBe(false); + expect( + isOpenCircuitError(new Error('outer', { cause: new Error('inner') })), + ).toBe(false); + expect(isOpenCircuitError(undefined)).toBe(false); + expect(isOpenCircuitError('BrokenCircuitError')).toBe(false); + }); +}); + +describe('OpenCircuitBypass', () => { + const key = 'tron:728126428 GET https://example.test/v1/accounts/T1'; + + let now: number; + + beforeEach(() => { + now = 1_000_000; + jest.spyOn(Date, 'now').mockImplementation(() => now); + }); + + it('returns the guarded result without a direct attempt', async () => { + const bypass = new OpenCircuitBypass(); + const direct = jest.fn(); + + expect( + await bypass.run( + key, + { bypassOpenCircuit: true }, + async () => 'guarded', + direct, + ), + ).toBe('guarded'); + expect(direct).not.toHaveBeenCalled(); + }); + + it('makes one direct attempt when the circuit is open on an explicit request', async () => { + const bypass = new OpenCircuitBypass(); + const direct = jest.fn().mockResolvedValue('direct'); + + expect( + await bypass.run( + key, + { bypassOpenCircuit: true }, + async () => { + throw new BrokenCircuitError(); + }, + direct, + ), + ).toBe('direct'); + expect(direct).toHaveBeenCalledTimes(1); + }); + + it('does not bypass the circuit for background reads', async () => { + const bypass = new OpenCircuitBypass(); + const direct = jest.fn(); + const guarded = async (): Promise => { + throw new BrokenCircuitError(); + }; + + await expect( + bypass.run(key, undefined, guarded, direct), + ).rejects.toBeInstanceOf(BrokenCircuitError); + await expect( + bypass.run(key, { bypassOpenCircuit: false }, guarded, direct), + ).rejects.toBeInstanceOf(BrokenCircuitError); + expect(direct).not.toHaveBeenCalled(); + }); + + it('rethrows other errors without a direct attempt', async () => { + const bypass = new OpenCircuitBypass(); + const direct = jest.fn(); + const failure = new Error('HTTP 500'); + + await expect( + bypass.run( + key, + { bypassOpenCircuit: true }, + async () => { + throw failure; + }, + direct, + ), + ).rejects.toBe(failure); + expect(direct).not.toHaveBeenCalled(); + }); + + it('bypasses the circuit at most once per interval for the same request', async () => { + const bypass = new OpenCircuitBypass(); + const direct = jest.fn().mockRejectedValue(new Error('HTTP 429')); + const guarded = async (): Promise => { + throw new BrokenCircuitError(); + }; + const run = async (requestKey: string): Promise => + bypass.run(requestKey, { bypassOpenCircuit: true }, guarded, direct); + + await expect(run(key)).rejects.toThrow('HTTP 429'); + now += OPEN_CIRCUIT_BYPASS_INTERVAL_MS - 1; + await expect(run(key)).rejects.toBeInstanceOf(BrokenCircuitError); + expect(direct).toHaveBeenCalledTimes(1); + + // Another upstream request has its own interval. + await expect(run(`${key}-other`)).rejects.toThrow('HTTP 429'); + expect(direct).toHaveBeenCalledTimes(2); + + now += 1; + await expect(run(key)).rejects.toThrow('HTTP 429'); + expect(direct).toHaveBeenCalledTimes(3); + }); + + it('shares an in-flight direct attempt between identical requests', async () => { + const bypass = new OpenCircuitBypass(); + let resolveDirect: (value: string) => void = () => undefined; + const direct = jest.fn( + async () => + new Promise((resolve) => { + resolveDirect = resolve; + }), + ); + const guarded = async (): Promise => { + throw new BrokenCircuitError(); + }; + + const first = bypass.run(key, { bypassOpenCircuit: true }, guarded, direct); + const second = bypass.run( + key, + { bypassOpenCircuit: true }, + guarded, + direct, + ); + // Let both guarded calls fail. + await new Promise((resolve) => setImmediate(resolve)); + resolveDirect('direct'); + + expect(await first).toBe('direct'); + expect(await second).toBe('direct'); + expect(direct).toHaveBeenCalledTimes(1); + }); +}); + +describe('open-circuit detection without the class (v6)', () => { + it('matches by message, flag or name, also in the cause chain', () => { + // What the keyring SDK rethrows: a plain Error with the same message. + expect(isOpenCircuitError(new Error(OPEN_CIRCUIT_ERROR_MESSAGE))).toBe( + true, + ); + expect(isOpenCircuitError({ isBrokenCircuitError: true })).toBe(true); + const named = new Error('x'); + named.name = 'IsolatedCircuitError'; + expect(isOpenCircuitError(named)).toBe(true); + expect( + isOpenCircuitError( + new Error('Failed to fetch account info', { + cause: { message: OPEN_CIRCUIT_ERROR_MESSAGE }, + }), + ), + ).toBe(true); + expect(isOpenCircuitError(new Error('Failed to fetch account info'))).toBe( + false, + ); + }); + + it('reads the circuit state from the policy', () => { + expect(isCircuitOpen(undefined)).toBe(false); + expect(isCircuitOpen({})).toBe(false); + expect(isCircuitOpen({ getRemainingCircuitOpenDuration: () => null })).toBe( + false, + ); + expect(isCircuitOpen({ getRemainingCircuitOpenDuration: () => 0 })).toBe( + false, + ); + expect( + isCircuitOpen({ getRemainingCircuitOpenDuration: () => 12_000 }), + ).toBe(true); + }); +}); + +describe('OpenCircuitBypass (v6)', () => { + const key = 'tron:728126428 GET https://example.test/v1/accounts/T1'; + + let now: number; + + beforeEach(() => { + now = 2_000_000; + jest.spyOn(Date, 'now').mockImplementation(() => now); + }); + + it('bypasses an error that only carries the open-circuit message', async () => { + const bypass = new OpenCircuitBypass(); + const direct = jest.fn().mockResolvedValue('direct'); + + expect( + await bypass.run( + key, + { bypassOpenCircuit: true }, + async () => { + throw new Error(OPEN_CIRCUIT_ERROR_MESSAGE); + }, + direct, + ), + ).toBe('direct'); + expect(direct).toHaveBeenCalledTimes(1); + }); + + it('goes direct without the guarded call when the policy reports an open circuit', async () => { + const bypass = new OpenCircuitBypass(); + const guarded = jest.fn(); + const direct = jest.fn().mockResolvedValue('direct'); + const policy = { + getRemainingCircuitOpenDuration: (): number => 20_000, + reset: jest.fn(), + }; + + expect( + await bypass.run( + key, + { bypassOpenCircuit: true }, + guarded, + direct, + policy, + ), + ).toBe('direct'); + expect(guarded).not.toHaveBeenCalled(); + expect(policy.reset).toHaveBeenCalledTimes(1); + }); + + it('does not check the state of background reads', async () => { + const bypass = new OpenCircuitBypass(); + const guarded = jest.fn().mockResolvedValue('guarded'); + const direct = jest.fn(); + + expect( + await bypass.run(key, undefined, guarded, direct, { + getRemainingCircuitOpenDuration: (): number => 20_000, + }), + ).toBe('guarded'); + expect(direct).not.toHaveBeenCalled(); + }); + + it('reuses a successful direct result for the result TTL', async () => { + const bypass = new OpenCircuitBypass(); + const guarded = async (): Promise => { + throw new BrokenCircuitError(); + }; + const direct = jest + .fn() + .mockResolvedValueOnce('first') + .mockResolvedValueOnce('second'); + const run = async (): Promise => + bypass.run(key, { bypassOpenCircuit: true }, guarded, direct); + + expect(await run()).toBe('first'); + now += OPEN_CIRCUIT_BYPASS_RESULT_TTL_MS - 1; + expect(await run()).toBe('first'); + expect(direct).toHaveBeenCalledTimes(1); + + // Expired, and within the bypass interval: the circuit error again. + now += 1; + await expect(run()).rejects.toBeInstanceOf(BrokenCircuitError); + + now += OPEN_CIRCUIT_BYPASS_INTERVAL_MS; + expect(await run()).toBe('second'); + }); + + it('does not reset the policy after a failed direct attempt', async () => { + const bypass = new OpenCircuitBypass(); + const policy = { reset: jest.fn() }; + + await expect( + bypass.run( + key, + { bypassOpenCircuit: true }, + async () => { + throw new BrokenCircuitError(); + }, + async () => { + throw new Error('offline'); + }, + policy, + ), + ).rejects.toThrow('offline'); + expect(policy.reset).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/tron-wallet-snap/src/utils/openCircuitBypass.ts b/packages/tron-wallet-snap/src/utils/openCircuitBypass.ts new file mode 100644 index 000000000..dfb2e8b09 --- /dev/null +++ b/packages/tron-wallet-snap/src/utils/openCircuitBypass.ts @@ -0,0 +1,239 @@ +import { BrokenCircuitError } from '@metamask/snap-networks-utils'; + +/** + * Minimum time between two requests that skip the open circuit of a fetch + * policy for the same upstream request (see {@link OpenCircuitBypass}). + */ +export const OPEN_CIRCUIT_BYPASS_INTERVAL_MS = 10_000; + +/** + * How long the result of a successful direct (bypassing) request is reused for + * the same upstream request. A balance refresh makes two keyring calls in a + * row (`keyring_listAccountAssets`, then `keyring_getAccountBalances`) that + * read the same endpoints; both must see the result of the one direct request. + */ +export const OPEN_CIRCUIT_BYPASS_RESULT_TTL_MS = 5_000; + +/** + * The message of cockatiel's `BrokenCircuitError` (and its subclasses), thrown + * by the fetch policy when it short-circuits a call while its circuit is open. + * The keyring SDK rethrows non-snap errors as `new Error(message)`, dropping + * the class and the cause, so the message is the only marker left by the time + * the error reaches the snap's error handler. + */ +export const OPEN_CIRCUIT_ERROR_MESSAGE = + 'Execution prevented because the circuit breaker is open'; + +/** + * Options of a read made on behalf of an explicit client request. + */ +export type ExplicitRequestOptions = { + /** + * The request was explicitly made by the client (a `listAccountAssets` or + * `getAccountBalances` keyring call, such as the balance Retry): it may reach + * the network once while the fetch policy's circuit is open, instead of + * failing without a request until the circuit half-opens. + */ + bypassOpenCircuit?: boolean; +}; + +/** + * The parts of a fetch policy (`createFetchPolicy`) the bypass uses. Optional, + * so a bare `{ execute }` policy (tests) still works. + */ +export type BypassablePolicy = { + getRemainingCircuitOpenDuration?: () => number | null; + reset?: () => void; +}; + +/** + * Whether a single error (not its cause chain) is an open-circuit error. + * + * Checked by class, by cockatiel's `isBrokenCircuitError` flag, by name, and by + * message, because the class identity does not survive every boundary (the + * keyring SDK rethrows a plain `Error` with the same message; bundling can + * duplicate the class). + * + * @param error - The error. + * @returns `true` if the error is an open-circuit error. + */ +function isOwnOpenCircuitError(error: unknown): boolean { + if (error instanceof BrokenCircuitError) { + return true; + } + if (typeof error !== 'object' || error === null) { + return false; + } + const { isBrokenCircuitError, name, message } = error as { + isBrokenCircuitError?: unknown; + name?: unknown; + message?: unknown; + }; + return ( + isBrokenCircuitError === true || + name === 'BrokenCircuitError' || + name === 'IsolatedCircuitError' || + (typeof message === 'string' && + message.includes(OPEN_CIRCUIT_ERROR_MESSAGE)) + ); +} + +/** + * Whether an error, or any error in its `cause` chain, is the fetch policy's + * open-circuit short-circuit (no request was made). + * + * @param error - The error to check. + * @returns `true` if the error comes from an open circuit. + */ +export function isOpenCircuitError(error: unknown): boolean { + let current: unknown = error; + for (let depth = 0; depth < 10 && current !== undefined; depth++) { + if (isOwnOpenCircuitError(current)) { + return true; + } + current = + typeof current === 'object' && current !== null + ? (current as { cause?: unknown }).cause + : undefined; + } + return false; +} + +/** + * Whether the policy's circuit is open (calls fail without a request). + * + * @param policy - The fetch policy. + * @returns `true` if the circuit is open. + */ +export function isCircuitOpen(policy: BypassablePolicy | undefined): boolean { + const remaining = policy?.getRemainingCircuitOpenDuration?.(); + return typeof remaining === 'number' && remaining > 0; +} + +/** + * Lets an explicit client request reach the network while a fetch policy's + * circuit is open. + * + * The circuit opens after repeated failures and then fails every call without + * a request for 30 seconds, so a user-initiated retry made meanwhile could + * never succeed. With this gate, such a request makes one direct attempt (no + * retries, not counted by the policy) when the circuit is open: either known + * up front from the policy's state, or detected from the guarded call's + * open-circuit error. Concurrent identical requests share the attempt, a + * successful attempt is reused for {@link OPEN_CIRCUIT_BYPASS_RESULT_TTL_MS} + * and closes the circuit (the service answered), and each upstream request + * bypasses the circuit at most once per {@link OPEN_CIRCUIT_BYPASS_INTERVAL_MS}, + * so polling cannot hammer a failing service; background reads never bypass + * it. + */ +export class OpenCircuitBypass { + readonly #intervalMs: number; + + readonly #resultTtlMs: number; + + readonly #lastAttemptAt = new Map(); + + readonly #lastResult = new Map(); + + readonly #inFlight = new Map>(); + + constructor( + intervalMs: number = OPEN_CIRCUIT_BYPASS_INTERVAL_MS, + resultTtlMs: number = OPEN_CIRCUIT_BYPASS_RESULT_TTL_MS, + ) { + this.#intervalMs = intervalMs; + this.#resultTtlMs = resultTtlMs; + } + + /** + * Runs the guarded read, and the direct one if allowed when the circuit is + * open. + * + * @param key - Identifies the upstream request (network, method, URL, body). + * @param options - The request options. + * @param guarded - The read under the fetch policy. + * @param direct - The same read, once, without the policy. + * @param policy - The fetch policy guarding the read, used to check whether + * its circuit is open before the call and to close it after a successful + * direct read. + * @returns The read result. + */ + async run( + key: string, + options: ExplicitRequestOptions | undefined, + guarded: () => Promise, + direct: () => Promise, + policy?: BypassablePolicy, + ): Promise { + if (!options?.bypassOpenCircuit) { + return guarded(); + } + + const now = Date.now(); + const lastResult = this.#lastResult.get(key); + if (lastResult && now - lastResult.at < this.#resultTtlMs) { + return lastResult.value as Result; + } + + const inFlight = this.#inFlight.get(key); + if (inFlight) { + return (await inFlight) as Result; + } + + let openCircuitError: unknown = new BrokenCircuitError(); + if (!isCircuitOpen(policy)) { + try { + return await guarded(); + } catch (error) { + if (!isOpenCircuitError(error)) { + throw error; + } + openCircuitError = error; + } + } + + return this.#runDirect(key, direct, openCircuitError, policy); + } + + async #runDirect( + key: string, + direct: () => Promise, + openCircuitError: unknown, + policy: BypassablePolicy | undefined, + ): Promise { + const inFlight = this.#inFlight.get(key); + if (inFlight) { + return (await inFlight) as Result; + } + + const now = Date.now(); + const lastAttemptAt = this.#lastAttemptAt.get(key); + if (lastAttemptAt !== undefined && now - lastAttemptAt < this.#intervalMs) { + throw openCircuitError; + } + for (const [otherKey, otherAt] of this.#lastAttemptAt) { + if (now - otherAt >= this.#intervalMs) { + this.#lastAttemptAt.delete(otherKey); + } + } + for (const [otherKey, { at }] of this.#lastResult) { + if (now - at >= this.#resultTtlMs) { + this.#lastResult.delete(otherKey); + } + } + this.#lastAttemptAt.set(key, now); + + const attempt = (async (): Promise => { + const value = await direct(); + this.#lastResult.set(key, { at: Date.now(), value }); + // The service answered: close the circuit so background reads resume + // now instead of after the next half-open probe. + policy?.reset?.(); + return value; + })().finally(() => { + this.#inFlight.delete(key); + }); + this.#inFlight.set(key, attempt); + return await attempt; + } +}