From 3a0cfd9f15403617c68b7d798a0d9ba4f72d5f11 Mon Sep 17 00:00:00 2001 From: local Date: Mon, 26 Oct 2026 10:02:49 +0100 Subject: [PATCH] fix(tron): let balance Retry reach the network while the circuit breaker is open and dedupe failure reports Explicit keyring balance and asset requests (the wallet's Retry) bypass an open circuit breaker with one direct attempt per upstream request at most every 10 s, shared by concurrent callers. Background reads never bypass. Open-circuit short-circuits are no longer sent to trackError, and 'Failed to fetch account info' is reported once per failure streak. --- .../clients/tron-http/TronHttpClient.test.ts | 70 ++++ .../src/clients/tron-http/TronHttpClient.ts | 125 +++++-- .../trongrid/TrongridApiClient.test.ts | 103 ++++++ .../src/clients/trongrid/TrongridApiClient.ts | 51 ++- .../keyring/keyring.openCircuit.test.ts | 346 ++++++++++++++++++ .../src/handlers/keyring/keyring.test.ts | 4 +- .../src/handlers/keyring/keyring.ts | 17 +- .../services/accounts/AccountsService.test.ts | 48 ++- .../src/services/accounts/AccountsService.ts | 12 +- .../src/services/assets/AssetsService.test.ts | 14 +- .../src/services/assets/AssetsService.ts | 5 +- .../assets/adapters/SnapAssetsAdapter.ts | 40 +- .../tron-wallet-snap/src/utils/errors.test.ts | 88 ++++- packages/tron-wallet-snap/src/utils/errors.ts | 28 ++ .../src/utils/failureStreakGate.test.ts | 60 +++ .../src/utils/failureStreakGate.ts | 97 +++++ .../src/utils/openCircuitBypass.test.ts | 308 ++++++++++++++++ .../src/utils/openCircuitBypass.ts | 239 ++++++++++++ 18 files changed, 1586 insertions(+), 69 deletions(-) create mode 100644 packages/tron-wallet-snap/src/handlers/keyring/keyring.openCircuit.test.ts create mode 100644 packages/tron-wallet-snap/src/utils/failureStreakGate.test.ts create mode 100644 packages/tron-wallet-snap/src/utils/failureStreakGate.ts create mode 100644 packages/tron-wallet-snap/src/utils/openCircuitBypass.test.ts create mode 100644 packages/tron-wallet-snap/src/utils/openCircuitBypass.ts diff --git a/packages/tron-wallet-snap/src/clients/tron-http/TronHttpClient.test.ts b/packages/tron-wallet-snap/src/clients/tron-http/TronHttpClient.test.ts index 5685a496b..2233f240a 100644 --- a/packages/tron-wallet-snap/src/clients/tron-http/TronHttpClient.test.ts +++ b/packages/tron-wallet-snap/src/clients/tron-http/TronHttpClient.test.ts @@ -1,5 +1,6 @@ import { BaseConfigProvider, + BrokenCircuitError, FetchPolicyErrorName, } from '@metamask/snap-networks-utils'; @@ -120,4 +121,73 @@ describe('TronHttpClient', () => { expect(fetchMock).toHaveBeenCalledTimes(2); }); }); + + describe('open circuit bypass (v5)', () => { + /** + * Opens the fetch policy circuit with non-retryable server failures. + * + * @param client - The client. + */ + async function openCircuit(client: TronHttpClient): Promise { + jest + .spyOn(global, 'fetch') + .mockImplementation( + async () => new globalThis.Response('', { status: 501 }), + ); + for (let attempt = 0; attempt < 30; attempt++) { + try { + await client.getReward(Network.Mainnet, `address-${attempt}`); + } catch (error) { + if (error instanceof BrokenCircuitError) { + return; + } + } + } + throw new Error('The circuit did not open'); + } + + it('lets an explicit request reach the network once while the circuit is open', async () => { + const client = new TronHttpClient({ configProvider }); + await openCircuit(client); + const fetchMock = jest.spyOn(global, 'fetch').mockImplementation( + async () => + new globalThis.Response(JSON.stringify({ reward: 7 }), { + status: 200, + }), + ); + + // The spy is reused: forget the calls that opened the circuit. + fetchMock.mockClear(); + + await expect( + client.getReward(Network.Mainnet, 'address'), + ).rejects.toBeInstanceOf(BrokenCircuitError); + expect(fetchMock).not.toHaveBeenCalled(); + + expect( + await client.getReward(Network.Mainnet, 'address', { + bypassOpenCircuit: true, + }), + ).toBe(7); + expect(fetchMock).toHaveBeenCalledTimes(1); + + // The next explicit call for the same request (the balances call that + // follows `listAccountAssets`) reuses the direct result. + expect( + await client.getReward(Network.Mainnet, 'address', { + bypassOpenCircuit: true, + }), + ).toBe(7); + expect(fetchMock).toHaveBeenCalledTimes(1); + + // The service answered: the circuit is closed again for every read. + + expect( + await client.getAccountResources(Network.Mainnet, 'address', { + bypassOpenCircuit: true, + }), + ).toStrictEqual({ reward: 7 }); + expect(fetchMock).toHaveBeenCalledTimes(2); + }); + }); }); diff --git a/packages/tron-wallet-snap/src/clients/tron-http/TronHttpClient.ts b/packages/tron-wallet-snap/src/clients/tron-http/TronHttpClient.ts index ac3207b5d..87ba80735 100644 --- a/packages/tron-wallet-snap/src/clients/tron-http/TronHttpClient.ts +++ b/packages/tron-wallet-snap/src/clients/tron-http/TronHttpClient.ts @@ -14,6 +14,11 @@ import { Network } from '../../constants'; import type { ConfigProvider } from '../../services/config'; import { hexToString } from '../../utils/hex'; import logger, { noOpLogger } from '../../utils/logger'; +import { OpenCircuitBypass } from '../../utils/openCircuitBypass'; +import type { + BypassablePolicy, + ExplicitRequestOptions, +} from '../../utils/openCircuitBypass'; import { AccountResourcesStruct, ChainParametersResponseStruct, @@ -55,7 +60,9 @@ export class TronHttpClient { } > = new Map(); - readonly #policies: Map; + readonly #policies: Map; + + readonly #openCircuitBypass = new OpenCircuitBypass(); readonly #requestCache: ( network: Network, @@ -172,12 +179,14 @@ export class TronHttpClient { * @see https://developers.tron.network/reference/getaccountresource * @param network - Network to query. * @param accountAddress - Account address in base58 format. + * @param options - Options of an explicit client request. * @returns Promise - Account resources (energy, bandwidth, etc.). * @throws Error - HTTP errors or configuration errors. */ async getAccountResources( network: Network, accountAddress: string, + options?: ExplicitRequestOptions, ): Promise { const client = this.#clients.get(network); if (!client) { @@ -195,11 +204,17 @@ export class TronHttpClient { visible: true, }); - const response = await this.#fetch(network, url, { - method: 'POST', - headers, - body, - }); + const response = await this.#fetch( + network, + url, + { + method: 'POST', + headers, + body, + }, + [], + options, + ); const accountResources: AccountResources = JSON.parse(response.body); @@ -434,9 +449,14 @@ export class TronHttpClient { * @see https://developers.tron.network/reference/getreward * @param network - Network to query * @param accountAddress - Account address in base58 format + * @param options - Options of an explicit client request. * @returns Promise - Unclaimed rewards in sun (0 if no rewards) */ - async getReward(network: Network, accountAddress: string): Promise { + async getReward( + network: Network, + accountAddress: string, + options?: ExplicitRequestOptions, + ): Promise { const client = this.#clients.get(network); if (!client) { throw new Error(`No client configured for network: ${network}`); @@ -453,11 +473,17 @@ export class TronHttpClient { visible: true, }); - const response = await this.#fetch(network, url, { - method: 'POST', - headers, - body, - }); + const response = await this.#fetch( + network, + url, + { + method: 'POST', + headers, + body, + }, + [], + options, + ); const rewardResponse: GetRewardResponse = JSON.parse(response.body); @@ -472,14 +498,25 @@ export class TronHttpClient { url: string, init: RequestInit, allowedStatuses: number[] = [], + options?: ExplicitRequestOptions, ): Promise<{ body: string; status: number }> { - return this.#requestCache( - network, - url, - init.method ?? 'GET', - init.headers as Record | undefined, - typeof init.body === 'string' ? init.body : '', - allowedStatuses, + const method = init.method ?? 'GET'; + const headers = init.headers as Record | undefined; + const body = typeof init.body === 'string' ? init.body : ''; + return this.#openCircuitBypass.run( + `${network} ${method} ${url} ${body}`, + options, + async () => + this.#requestCache( + network, + url, + method, + headers, + body, + allowedStatuses, + ), + async () => this.#request(url, method, headers, body, allowedStatuses), + this.#policies.get(network), ); } @@ -496,26 +533,36 @@ export class TronHttpClient { throw new Error(`No policy configured for network: ${network}`); } - return executeWithFetchPolicy(policy, async () => { - const response = await fetch(url, { - method, - headers, - ...(body ? { body } : {}), - signal: AbortSignal.timeout(10_000), - }); - if (!response.ok && !allowedStatuses.includes(response.status)) { - throw createFetchPolicyError(`HTTP error! status: ${response.status}`, { - httpStatus: response.status, - retryAfterMs: parseRetryAfter(response.headers.get('Retry-After')), - }); - } - if (allowedStatuses.includes(response.status)) { - return { body: '', status: response.status }; - } - return { - body: await response.text(), - status: response.status, - }; + return executeWithFetchPolicy(policy, async () => + this.#request(url, method, headers, body, allowedStatuses), + ); + } + + async #request( + url: string, + method: string, + headers: Record | undefined, + body: string, + allowedStatuses: number[], + ): Promise<{ body: string; status: number }> { + const response = await fetch(url, { + method, + headers, + ...(body ? { body } : {}), + signal: AbortSignal.timeout(10_000), }); + if (!response.ok && !allowedStatuses.includes(response.status)) { + throw createFetchPolicyError(`HTTP error! status: ${response.status}`, { + httpStatus: response.status, + retryAfterMs: parseRetryAfter(response.headers.get('Retry-After')), + }); + } + if (allowedStatuses.includes(response.status)) { + return { body: '', status: response.status }; + } + return { + body: await response.text(), + status: response.status, + }; } } diff --git a/packages/tron-wallet-snap/src/clients/trongrid/TrongridApiClient.test.ts b/packages/tron-wallet-snap/src/clients/trongrid/TrongridApiClient.test.ts index 84a3d00cb..87296a5b2 100644 --- a/packages/tron-wallet-snap/src/clients/trongrid/TrongridApiClient.test.ts +++ b/packages/tron-wallet-snap/src/clients/trongrid/TrongridApiClient.test.ts @@ -1,5 +1,6 @@ import { BaseConfigProvider, + BrokenCircuitError, InMemoryCache, } from '@metamask/snap-networks-utils'; import type { ICache, Serializable } from '@metamask/snap-networks-utils'; @@ -737,4 +738,106 @@ describe('TrongridApiClient', () => { }); }); }); + + describe('open circuit bypass (v5)', () => { + const mockAddress = 'TGJn1wnUYHJbvN88cynZbsAz2EMeZq73yx'; + const mockTrc20Balances: Trc20Balance[] = [ + { TR7NHqjeKQxGTCi8q8ZY4pL8otSzgjLj6t: '24249143' }, + ]; + + /** + * Opens the fetch policy circuit with non-retryable server failures. + * + * @param client - The client. + */ + async function openCircuit(client: TrongridApiClient): Promise { + jest + .spyOn(global, 'fetch') + // eslint-disable-next-line no-restricted-globals + .mockImplementation(async () => new Response('', { status: 501 })); + for (let attempt = 0; attempt < 30; attempt++) { + try { + await client.getTrc20BalancesByAddress( + Network.Mainnet, + `address-${attempt}`, + ); + } catch (error) { + if (error instanceof BrokenCircuitError) { + return; + } + } + } + throw new Error('The circuit did not open'); + } + + it('lets an explicit request reach the network once while the circuit is open', async () => { + await withTrongridApiClient(async ({ client }) => { + await openCircuit(client); + const fetchMock = jest.spyOn(global, 'fetch').mockImplementation( + async () => + // eslint-disable-next-line no-restricted-globals + new Response( + JSON.stringify({ + data: mockTrc20Balances, + success: true, + meta: { at: 1770121997373, page_size: 4 }, + }), + { status: 200, headers: { 'Content-Type': 'application/json' } }, + ), + ); + + // The spy is reused: forget the calls that opened the circuit. + fetchMock.mockClear(); + + // Background reads stay short-circuited. + await expect( + client.getTrc20BalancesByAddress(Network.Mainnet, mockAddress), + ).rejects.toBeInstanceOf(BrokenCircuitError); + expect(fetchMock).not.toHaveBeenCalled(); + + // Concurrent identical explicit requests share one attempt. + const results = await Promise.all([ + client.getTrc20BalancesByAddress(Network.Mainnet, mockAddress, { + bypassOpenCircuit: true, + }), + client.getTrc20BalancesByAddress(Network.Mainnet, mockAddress, { + bypassOpenCircuit: true, + }), + ]); + expect(results).toStrictEqual([mockTrc20Balances, mockTrc20Balances]); + expect(fetchMock).toHaveBeenCalledTimes(1); + + // The next explicit call reuses the direct result. + expect( + await client.getTrc20BalancesByAddress(Network.Mainnet, mockAddress, { + bypassOpenCircuit: true, + }), + ).toStrictEqual(mockTrc20Balances); + expect(fetchMock).toHaveBeenCalledTimes(1); + + // The service answered: background reads are no longer + // short-circuited. + expect( + await client.getTrc20BalancesByAddress(Network.Mainnet, mockAddress), + ).toStrictEqual(mockTrc20Balances); + expect(fetchMock).toHaveBeenCalledTimes(2); + }); + }); + + it('does not bypass the circuit for other failures', async () => { + await withTrongridApiClient(async ({ client }) => { + const fetchMock = jest + .spyOn(global, 'fetch') + // eslint-disable-next-line no-restricted-globals + .mockImplementation(async () => new Response('', { status: 404 })); + + await expect( + client.getTrc20BalancesByAddress(Network.Mainnet, mockAddress, { + bypassOpenCircuit: true, + }), + ).rejects.toThrow(Error); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + }); + }); }); diff --git a/packages/tron-wallet-snap/src/clients/trongrid/TrongridApiClient.ts b/packages/tron-wallet-snap/src/clients/trongrid/TrongridApiClient.ts index 488759c88..2e86db82c 100644 --- a/packages/tron-wallet-snap/src/clients/trongrid/TrongridApiClient.ts +++ b/packages/tron-wallet-snap/src/clients/trongrid/TrongridApiClient.ts @@ -18,6 +18,11 @@ import { assert } from '@metamask/superstruct'; import { Network } from '../../constants'; import type { ConfigProvider } from '../../services/config'; import logger, { noOpLogger } from '../../utils/logger'; +import { OpenCircuitBypass } from '../../utils/openCircuitBypass'; +import type { + BypassablePolicy, + ExplicitRequestOptions, +} from '../../utils/openCircuitBypass'; import type { TronHttpClient } from '../tron-http/TronHttpClient'; import type { ChainParameter } from '../tron-http/types'; import { @@ -76,7 +81,9 @@ export class TrongridApiClient { readonly #cache: ICache; - readonly #policies: Map; + readonly #policies: Map; + + readonly #openCircuitBypass = new OpenCircuitBypass(); readonly #requestCache: ( scope: Network, @@ -173,6 +180,7 @@ export class TrongridApiClient { * @see https://developers.tron.network/reference/get-account-info-by-address * @param scope - The network to query (e.g., 'mainnet', 'shasta') * @param address - The TRON address to query + * @param options - Options of an explicit client request. * @returns Promise - Account data including balances. * @throws TrongridAccountNotFoundError - for inactive accounts. * @throws Error - HTTP errors or API failures. @@ -180,6 +188,7 @@ export class TrongridApiClient { async getAccountInfoByAddress( scope: Network, address: string, + options?: ExplicitRequestOptions, ): Promise { const client = this.#clients.get(scope); if (!client) { @@ -193,7 +202,7 @@ export class TrongridApiClient { pathParams: { address }, }); - const response = await this.#fetch(scope, url, headers); + const response = await this.#fetch(scope, url, headers, options); const rawData: TrongridApiResponse = JSON.parse(response); @@ -324,11 +333,13 @@ export class TrongridApiClient { * @see https://developers.tron.network/reference/get-trc20-token-holder-balances * @param scope - The network to query (e.g., 'mainnet', 'shasta') * @param address - The TRON address to query + * @param options - Options of an explicit client request. * @returns Promise - Array of TRC20 balances (contract address -> balance) */ async getTrc20BalancesByAddress( scope: Network, address: string, + options?: ExplicitRequestOptions, ): Promise { const client = this.#clients.get(scope); if (!client) { @@ -342,7 +353,7 @@ export class TrongridApiClient { pathParams: { address }, }); - const response = await this.#fetch(scope, url, headers); + const response = await this.#fetch(scope, url, headers, options); const rawData: TrongridApiResponse = JSON.parse(response); @@ -403,8 +414,15 @@ export class TrongridApiClient { scope: Network, url: string, headers: Record, + options?: ExplicitRequestOptions, ): Promise { - return this.#requestCache(scope, url, headers); + return this.#openCircuitBypass.run( + `${scope} GET ${url}`, + options, + async () => this.#requestCache(scope, url, headers), + async () => this.#request(url, headers), + this.#policies.get(scope), + ); } async #fetchUncached( @@ -417,15 +435,22 @@ export class TrongridApiClient { throw new Error(`No policy configured for network: ${scope}`); } - return executeWithFetchPolicy(policy, async () => { - const response = await fetch(url, { - headers, - signal: AbortSignal.timeout(10_000), - }); - if (!response.ok) { - throw createTrongridHttpError(response); - } - return response.text(); + return executeWithFetchPolicy(policy, async () => + this.#request(url, headers), + ); + } + + async #request( + url: string, + headers: Record, + ): Promise { + const response = await fetch(url, { + headers, + signal: AbortSignal.timeout(10_000), }); + if (!response.ok) { + throw createTrongridHttpError(response); + } + return response.text(); } } diff --git a/packages/tron-wallet-snap/src/handlers/keyring/keyring.openCircuit.test.ts b/packages/tron-wallet-snap/src/handlers/keyring/keyring.openCircuit.test.ts new file mode 100644 index 000000000..7cc022895 --- /dev/null +++ b/packages/tron-wallet-snap/src/handlers/keyring/keyring.openCircuit.test.ts @@ -0,0 +1,346 @@ +import { KeyringRpcMethod } from '@metamask/keyring-api'; +import type { Json, JsonRpcRequest } from '@metamask/utils'; +import { + BaseConfigProvider, + BrokenCircuitError, + InMemoryCache, + InMemoryState, +} from '@metamask/snap-networks-utils'; + +import type { PriceApiClient } from '../../clients/price-api/PriceApiClient'; +import type { SnapClient } from '../../clients/snap/SnapClient'; +import type { TokenApiClient } from '../../clients/token-api/TokenApiClient'; +import { TronHttpClient } from '../../clients/tron-http/TronHttpClient'; +import { TrongridApiClient } from '../../clients/trongrid/TrongridApiClient'; +import { KnownCaip19Id, Network } from '../../constants'; +import type { TronKeyringAccount } from '../../entities/tronKeyringAccount'; +import type { AccountsService } from '../../services/accounts/AccountsService'; +import { SnapAssetsAdapter } from '../../services/assets/adapters/SnapAssetsAdapter'; +import type { CoreAssetsAdapter } from '../../services/assets/adapters/CoreAssetsAdapter'; +import type { AssetsRepository } from '../../services/assets/AssetsRepository'; +import { AssetsService } from '../../services/assets/AssetsService'; +import { + ConfigStruct, + ENVIRONMENT, +} from '../../services/config/ConfigProvider'; +import type { ConfirmationHandler } from '../../services/confirmation/ConfirmationHandler'; +import type { UnencryptedStateValue } from '../../services/state/stateTypes'; +import type { TransactionsService } from '../../services/transactions/TransactionsService'; +import type { WalletService } from '../../services/wallet/WalletService'; +import { shouldTrack } from '../../utils/errors'; +import { + ACCOUNT_INFO_FETCH_FAILED_MESSAGE, + failureStreakGate, +} from '../../utils/failureStreakGate'; +import { mockLogger } from '../../utils/mockLogger'; +import { OPEN_CIRCUIT_ERROR_MESSAGE } from '../../utils/openCircuitBypass'; +import { KeyringHandler } from './keyring'; + +/* eslint-disable no-restricted-globals */ + +/** + * The URL of a fetch input. + * + * @param input - The fetch input. + * @returns The URL. + */ +function urlOf(input: RequestInfo | URL): string { + return input instanceof Request ? input.url : input.toString(); +} + +const ADDRESS = 'TJRabPrwbZy45sbavfcjinPJC18kjpRTv8'; + +const account: TronKeyringAccount = { + id: '123e4567-e89b-42d3-a456-426614174000', + address: ADDRESS, + options: {}, + methods: [], + type: 'tron:eoa', + scopes: [Network.Mainnet], + entropySource: 'entropy-source-1', + derivationPath: "m/44'/195'/0'/0/0", + index: 0, +}; + +/** + * A healthy upstream: TronGrid account info, account resources and rewards. + * + * @param input - The fetch input. + * @returns The response. + */ +async function healthyFetch(input: RequestInfo | URL): Promise { + const url = urlOf(input); + const json = (body: unknown): Response => + new Response(JSON.stringify(body), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + if (url.includes('/v1/accounts/')) { + return json({ + data: [{ address: ADDRESS, balance: 2_500_000 }], + success: true, + meta: { at: 1770121997373, page_size: 1 }, + }); + } + if (url.includes('/wallet/getaccountresource')) { + return json({}); + } + if (url.includes('/wallet/getReward')) { + return json({ reward: 0 }); + } + throw new Error(`Unexpected request: ${url}`); +} + +/** + * Builds the real keyring path of a balance refresh: keyring handler (through + * the keyring SDK's request dispatch), assets service, snap assets adapter and + * the real TronGrid / Tron HTTP clients with their real fetch policies. Only + * `fetch` and unrelated collaborators are mocked. + * + * @returns The handler and the clients. + */ +function setup(): { + keyringHandler: KeyringHandler; + trongridApiClient: TrongridApiClient; + tronHttpClient: TronHttpClient; +} { + const baseUrls = { + [Network.Mainnet]: 'https://api.trongrid.io', + [Network.Nile]: 'https://nile.trongrid.io', + [Network.Shasta]: 'https://api.shasta.trongrid.io', + }; + const configProvider = new BaseConfigProvider( + { + ...ENVIRONMENT, + trongridApi: { baseUrls }, + tronHttpApi: { baseUrls }, + }, + ConfigStruct, + ); + const tronHttpClient = new TronHttpClient({ configProvider }); + const trongridApiClient = new TrongridApiClient({ + configProvider, + tronHttpClient, + cache: new InMemoryCache(mockLogger), + }); + + const snapClient = { + trackError: jest.fn(), + } as unknown as SnapClient; + const snapAssetsAdapter = new SnapAssetsAdapter({ + logger: mockLogger, + state: new InMemoryState( + {} as UnencryptedStateValue, + ), + assetsRepository: { + saveMany: jest.fn(), + } as unknown as AssetsRepository, + trongridApiClient, + tronHttpClient, + priceApiClient: { + getMultipleSpotPrices: jest.fn().mockResolvedValue({}), + } as unknown as PriceApiClient, + tokenApiClient: { + getTokensMetadata: jest.fn().mockResolvedValue({}), + } as unknown as TokenApiClient, + snapClient, + }); + const assetsService = new AssetsService({ + snapAdapter: snapAssetsAdapter, + coreAdapter: {} as CoreAssetsAdapter, + remoteFeatureFlagsProvider: { + getFeatureFlag: jest.fn(), + } as never, + }); + + const keyringHandler = new KeyringHandler({ + logger: mockLogger, + snapClient, + accountsService: { + findById: jest.fn().mockResolvedValue(account), + } as unknown as AccountsService, + assetsService, + transactionsService: {} as TransactionsService, + walletService: {} as WalletService, + confirmationHandler: {} as ConfirmationHandler, + }); + + return { keyringHandler, trongridApiClient, tronHttpClient }; +} + +/** + * Opens the circuit of the mainnet TronGrid and Tron HTTP fetch policies the + * way a broken connection does: repeated failing background reads (no bypass) + * until the policy short-circuits them. + * + * @param clients - The clients. + * @param clients.trongridApiClient - The TronGrid client. + * @param clients.tronHttpClient - The Tron HTTP client. + */ +async function openCircuits({ + trongridApiClient, + tronHttpClient, +}: { + trongridApiClient: TrongridApiClient; + tronHttpClient: TronHttpClient; +}): Promise { + jest + .spyOn(global, 'fetch') + .mockImplementation(async () => new Response('', { status: 501 })); + const reads = [ + async (index: number): Promise => + trongridApiClient.getAccountInfoByAddress( + Network.Mainnet, + `address-${index}`, + ), + async (index: number): Promise => + tronHttpClient.getAccountResources(Network.Mainnet, `address-${index}`), + ]; + for (const read of reads) { + let opened = false; + for (let index = 0; index < 30 && !opened; index++) { + try { + await read(index); + } catch (error) { + opened = error instanceof BrokenCircuitError; + } + } + if (!opened) { + throw new Error('The circuit did not open'); + } + } +} + +const request = ( + method: string, + params: Record, +): JsonRpcRequest => ({ + jsonrpc: '2.0', + id: 1, + method, + params, +}); + +describe('KeyringHandler balance refresh while the fetch circuit is open (v6)', () => { + const originalFetch = global.fetch; + + afterEach(() => { + global.fetch = originalFetch; + jest.restoreAllMocks(); + failureStreakGate.clear(); + }); + + it('reaches the network for listAccountAssets then getAccountBalances (the Retry path)', async () => { + const clients = setup(); + await openCircuits(clients); + + // Background reads are short-circuited without a request. + const fetchMock = jest + .spyOn(global, 'fetch') + .mockImplementation(healthyFetch); + fetchMock.mockClear(); + await expect( + clients.trongridApiClient.getAccountInfoByAddress( + Network.Mainnet, + ADDRESS, + ), + ).rejects.toBeInstanceOf(BrokenCircuitError); + expect(fetchMock).not.toHaveBeenCalled(); + + // A failure streak is in progress (already reported once). + expect( + failureStreakGate.shouldReport(ACCOUNT_INFO_FETCH_FAILED_MESSAGE), + ).toBe(true); + + // The client's refresh: keyring_listAccountAssets, then + // keyring_getAccountBalances, through the keyring SDK dispatch. + const assets = await clients.keyringHandler.handle( + 'metamask', + request(KeyringRpcMethod.ListAccountAssets, { id: account.id }), + ); + expect(assets).toContain(KnownCaip19Id.TrxMainnet); + // TronGrid answered: the streak ended, the next failure is reported. + expect( + failureStreakGate.shouldReport(ACCOUNT_INFO_FETCH_FAILED_MESSAGE), + ).toBe(true); + + const requestedUrls = fetchMock.mock.calls.map(([input]) => urlOf(input)); + expect(requestedUrls).toStrictEqual( + expect.arrayContaining([ + `https://api.trongrid.io/v1/accounts/${ADDRESS}`, + 'https://api.trongrid.io/wallet/getaccountresource', + 'https://api.trongrid.io/wallet/getReward', + ]), + ); + + const balances = (await clients.keyringHandler.handle( + 'metamask', + request(KeyringRpcMethod.GetAccountBalances, { + id: account.id, + assets: [KnownCaip19Id.TrxMainnet], + }), + )) as Record; + expect(balances[KnownCaip19Id.TrxMainnet]).toStrictEqual({ + amount: '2.5', + unit: 'TRX', + }); + }); + + it('makes the request when the circuit is open before the call (state pre-check)', async () => { + const clients = setup(); + await openCircuits(clients); + const fetchMock = jest + .spyOn(global, 'fetch') + .mockImplementation(healthyFetch); + fetchMock.mockClear(); + + await clients.keyringHandler.handle( + 'metamask', + request(KeyringRpcMethod.GetAccountBalances, { + id: account.id, + assets: [KnownCaip19Id.TrxMainnet], + }), + ); + + expect( + fetchMock.mock.calls.filter(([input]) => + urlOf(input).includes('/v1/accounts/'), + ), + ).toHaveLength(1); + }); + + it('lets one direct attempt through per interval when it fails', async () => { + const clients = setup(); + await openCircuits(clients); + const fetchMock = jest + .spyOn(global, 'fetch') + .mockImplementation(async () => { + throw new TypeError('Failed to fetch'); + }); + // The spy is reused: forget the calls that opened the circuits. + fetchMock.mockClear(); + + // The direct attempt fails (still offline)... + const first = clients.keyringHandler.handle( + 'metamask', + request(KeyringRpcMethod.ListAccountAssets, { id: account.id }), + ); + await expect(first).rejects.toThrow('Failed to fetch account info'); + const attempts = fetchMock.mock.calls.length; + expect(attempts).toBeGreaterThan(0); + + // ...and an immediate repeat is not let through again. + await expect( + clients.keyringHandler.handle( + 'metamask', + request(KeyringRpcMethod.ListAccountAssets, { id: account.id }), + ), + ).rejects.toThrow('Failed to fetch account info'); + expect(fetchMock.mock.calls).toHaveLength(attempts); + }); + + it('does not track the open-circuit error rethrown by the keyring SDK as a plain Error', () => { + // `handleKeyringRequest` rethrows non-snap errors as `new Error(message)`. + expect(shouldTrack(new Error(OPEN_CIRCUIT_ERROR_MESSAGE))).toBe(false); + }); +}); diff --git a/packages/tron-wallet-snap/src/handlers/keyring/keyring.test.ts b/packages/tron-wallet-snap/src/handlers/keyring/keyring.test.ts index f75fc4366..447169f06 100644 --- a/packages/tron-wallet-snap/src/handlers/keyring/keyring.test.ts +++ b/packages/tron-wallet-snap/src/handlers/keyring/keyring.test.ts @@ -700,7 +700,7 @@ describe('KeyringHandler', () => { ]); expect( mockAssetsService.fetchAccountAssetsFromTrongrid, - ).toHaveBeenCalledWith(mockAccount); + ).toHaveBeenCalledWith(mockAccount, { bypassOpenCircuit: true }); expect(mockAssetsService.saveMany).toHaveBeenCalledWith([ mockLiveAsset, mockShastaAsset, @@ -740,7 +740,7 @@ describe('KeyringHandler', () => { }); expect( mockAssetsService.fetchAccountAssetsFromTrongrid, - ).toHaveBeenCalledWith(mockAccount); + ).toHaveBeenCalledWith(mockAccount, { bypassOpenCircuit: true }); expect(mockAssetsService.saveMany).toHaveBeenCalledWith([ mockLiveAsset, mockShastaAsset, diff --git a/packages/tron-wallet-snap/src/handlers/keyring/keyring.ts b/packages/tron-wallet-snap/src/handlers/keyring/keyring.ts index e1a746de9..13e0fbced 100644 --- a/packages/tron-wallet-snap/src/handlers/keyring/keyring.ts +++ b/packages/tron-wallet-snap/src/handlers/keyring/keyring.ts @@ -185,8 +185,15 @@ export class KeyringHandler implements KeyringSnapRpc { this.#logger.info('Getting account assets', { accountId }); + // Clients call `keyring_listAccountAssets` before + // `keyring_getAccountBalances` on every balance refresh, including the + // user's Retry. It is an explicit request too: without the bypass, an + // open circuit fails it without a request and the balances call that + // would bypass the circuit is never made. const assetEntities = - await this.#assetsService.fetchAccountAssetsFromTrongrid(account); + await this.#assetsService.fetchAccountAssetsFromTrongrid(account, { + bypassOpenCircuit: true, + }); await this.#assetsService.saveMany(assetEntities); @@ -278,8 +285,14 @@ export class KeyringHandler implements KeyringSnapRpc { const account = await this.#getAccountOrThrow(accountId); + // An explicit client request (e.g. the balance Retry): it may reach the + // network while a fetch policy's circuit is open (rate limited), so a + // retry made during the 30 s open window is not answered from the + // circuit alone. const assetsList = - await this.#assetsService.fetchAccountAssetsFromTrongrid(account); + await this.#assetsService.fetchAccountAssetsFromTrongrid(account, { + bypassOpenCircuit: true, + }); await this.#assetsService.saveMany(assetsList); diff --git a/packages/tron-wallet-snap/src/services/accounts/AccountsService.test.ts b/packages/tron-wallet-snap/src/services/accounts/AccountsService.test.ts index f8167c5b1..a4acf94ca 100644 --- a/packages/tron-wallet-snap/src/services/accounts/AccountsService.test.ts +++ b/packages/tron-wallet-snap/src/services/accounts/AccountsService.test.ts @@ -11,7 +11,7 @@ import type { import { AccountCreationType, TrxAccountType } from '@metamask/keyring-api'; import { getSelectedAccounts } from '@metamask/keyring-snap-sdk'; import type { Logger } from '@metamask/snap-networks-utils'; -import { LogLevel } from '@metamask/snap-networks-utils'; +import { BrokenCircuitError, LogLevel } from '@metamask/snap-networks-utils'; import type { SnapClient } from '../../clients/snap/SnapClient'; import { Network } from '../../constants'; @@ -1169,6 +1169,52 @@ describe('AccountsService', () => { ); }); + it('does not report accounts short-circuited by an open circuit (v5)', async () => { + const shortCircuited: TronKeyringAccount = { + id: 'sync-open-circuit-id', + address: 'TSyncOpenCircuit12345678901', + type: TrxAccountType.Eoa, + options: {}, + methods: [], + scopes: [], + entropySource: 'e1', + derivationPath: "m/44'/195'/0'/0/0", + index: 0, + }; + const failing: TronKeyringAccount = { + ...shortCircuited, + id: 'sync-fail-id', + address: 'TSyncFail123456789012345678', + derivationPath: "m/44'/195'/0'/0/1", + index: 1, + }; + + await withAccountsService( + async ({ accountsService, mockConfigProvider, mockAssetsService }) => { + mockConfigProvider.config = { + ...MOCK_CONFIG, + activeNetworks: [Network.Mainnet], + }; + mockAssetsService.fetchAccountAssets + .mockRejectedValueOnce( + new Error('Failed to fetch account info', { + cause: new BrokenCircuitError(), + }), + ) + .mockRejectedValueOnce(new Error('grpc unavailable')); + + await accountsService.synchronize([shortCircuited, failing]); + + expect(trackError).toHaveBeenCalledTimes(1); + const tracked = (trackError as jest.Mock).mock.calls[0][0] as Error; + expect(tracked.message).toBe( + 'Failed to fetch assets (1 failed): ' + + 'sync-fail-id: Error: grpc unavailable', + ); + }, + ); + }); + it('survives a hostile rejection reason that cannot be stringified', async () => { const account: TronKeyringAccount = { id: 'sync-hostile-id', diff --git a/packages/tron-wallet-snap/src/services/accounts/AccountsService.ts b/packages/tron-wallet-snap/src/services/accounts/AccountsService.ts index b6a207bec..735835d1f 100644 --- a/packages/tron-wallet-snap/src/services/accounts/AccountsService.ts +++ b/packages/tron-wallet-snap/src/services/accounts/AccountsService.ts @@ -33,6 +33,7 @@ import { createTronBip44KeypairDeriver, } from '../../utils/deriveTronFromCoinTypeNode'; import { sanitizeSensitiveError, trackError } from '../../utils/errors'; +import { isOpenCircuitError } from '../../utils/openCircuitBypass'; import { DerivationPathStruct } from '../../validation/structs'; import type { AssetsService } from '../assets/AssetsService'; import type { ConfigProvider } from '../config'; @@ -638,7 +639,16 @@ export class AccountsService { accountIds: string[], ): Promise { try { - const failures = getSyncFailuresFromSettledResult(results, accountIds); + // A fetch short-circuited by an open circuit made no request: not a + // failure to report (the failures that opened it were reported). + const failures = getSyncFailuresFromSettledResult( + results.map((result) => + result.status === 'rejected' && isOpenCircuitError(result.reason) + ? { status: 'fulfilled' as const, value: undefined } + : result, + ), + accountIds, + ); if (failures.length === 0) { return; diff --git a/packages/tron-wallet-snap/src/services/assets/AssetsService.test.ts b/packages/tron-wallet-snap/src/services/assets/AssetsService.test.ts index 37b3a2a52..5bd72b2c1 100644 --- a/packages/tron-wallet-snap/src/services/assets/AssetsService.test.ts +++ b/packages/tron-wallet-snap/src/services/assets/AssetsService.test.ts @@ -389,10 +389,20 @@ describe('AssetsService', () => { ).toHaveBeenCalledTimes(2); expect( mockTrongridApiClient.getAccountInfoByAddress, - ).toHaveBeenNthCalledWith(1, Network.Mainnet, mockAccount.address); + ).toHaveBeenNthCalledWith( + 1, + Network.Mainnet, + mockAccount.address, + undefined, + ); expect( mockTrongridApiClient.getAccountInfoByAddress, - ).toHaveBeenNthCalledWith(2, Network.Shasta, mockAccount.address); + ).toHaveBeenNthCalledWith( + 2, + Network.Shasta, + mockAccount.address, + undefined, + ); expect(findAsset(assets, KnownCaip19Id.TrxMainnet)?.rawAmount).toBe( '1000000', ); diff --git a/packages/tron-wallet-snap/src/services/assets/AssetsService.ts b/packages/tron-wallet-snap/src/services/assets/AssetsService.ts index 7e021f28d..852201aad 100644 --- a/packages/tron-wallet-snap/src/services/assets/AssetsService.ts +++ b/packages/tron-wallet-snap/src/services/assets/AssetsService.ts @@ -11,6 +11,7 @@ import { Networks } from '../../constants'; import type { Network } from '../../constants'; import type { AssetEntity } from '../../entities/assets'; import { extractScopesFromAssetIds } from '../../utils/caip'; +import type { ExplicitRequestOptions } from '../../utils/openCircuitBypass'; import type { CoreAssetsAdapter } from './adapters/CoreAssetsAdapter'; import { SnapAssetsAdapter } from './adapters/SnapAssetsAdapter'; @@ -113,12 +114,14 @@ export class AssetsService { * flag. * * @param account - The account to fetch live assets for. + * @param options - Options of an explicit client request. * @returns The live assets. */ async fetchAccountAssetsFromTrongrid( account: KeyringAccount, + options?: ExplicitRequestOptions, ): Promise { - return this.#snapAdapter.fetchAccountAssets(account); + return this.#snapAdapter.fetchAccountAssets(account, options); } /** diff --git a/packages/tron-wallet-snap/src/services/assets/adapters/SnapAssetsAdapter.ts b/packages/tron-wallet-snap/src/services/assets/adapters/SnapAssetsAdapter.ts index c80ba0788..ef272a737 100644 --- a/packages/tron-wallet-snap/src/services/assets/adapters/SnapAssetsAdapter.ts +++ b/packages/tron-wallet-snap/src/services/assets/adapters/SnapAssetsAdapter.ts @@ -40,6 +40,11 @@ import { import type { AssetEntity } from '../../../entities/assets'; import { parseTronCaipAssetType } from '../../../utils/caip'; import { toUiAmount } from '../../../utils/conversion'; +import { + ACCOUNT_INFO_FETCH_FAILED_MESSAGE, + failureStreakGate, +} from '../../../utils/failureStreakGate'; +import type { ExplicitRequestOptions } from '../../../utils/openCircuitBypass'; import type { UnencryptedStateValue } from '../../state/stateTypes'; import type { AssetsRepository } from '../AssetsRepository'; import { AssetFetchError } from '../errors'; @@ -172,6 +177,8 @@ export class SnapAssetsAdapter { * * @param scope - The network to query. * @param account - The keyring account. + * @param options - Options of an explicit client request: its balance + * reads may reach the network once while a fetch policy's circuit is open. * @throws {@link AssetFetchError} When the account info request fails for * reasons other than "account not found", since the resulting balance * state is unknown. @@ -180,6 +187,7 @@ export class SnapAssetsAdapter { async fetchAssetsAndBalancesForAccount( scope: Network, account: KeyringAccount, + options?: ExplicitRequestOptions, ): Promise { this.#logger.info('Fetching assets and balances by account', { account, @@ -191,16 +199,27 @@ export class SnapAssetsAdapter { tronAccountResourcesRequest, stakingRewardsRequest, ] = await Promise.allSettled([ - this.#trongridApiClient.getAccountInfoByAddress(scope, account.address), - this.#tronHttpClient.getAccountResources(scope, account.address), - this.#tronHttpClient.getReward(scope, account.address), + this.#trongridApiClient.getAccountInfoByAddress( + scope, + account.address, + options, + ), + this.#tronHttpClient.getAccountResources(scope, account.address, options), + this.#tronHttpClient.getReward(scope, account.address, options), ]); const isInactiveAccount = tronAccountInfoRequest.status === 'rejected'; + if ( + tronAccountInfoRequest.status === 'fulfilled' || + tronAccountInfoRequest.reason instanceof TrongridAccountNotFoundError + ) { + // TronGrid answered: a later account info failure starts a new streak. + failureStreakGate.endStreak(ACCOUNT_INFO_FETCH_FAILED_MESSAGE); + } if (isInactiveAccount) { const { reason } = tronAccountInfoRequest; if (!(reason instanceof TrongridAccountNotFoundError)) { - throw new AssetFetchError('Failed to fetch account info', { + throw new AssetFetchError(ACCOUNT_INFO_FETCH_FAILED_MESSAGE, { cause: reason, }); } @@ -213,7 +232,7 @@ export class SnapAssetsAdapter { const trc20BalancesFallback = isInactiveAccount ? await this.#trongridApiClient - .getTrc20BalancesByAddress(scope, account.address) + .getTrc20BalancesByAddress(scope, account.address, options) .catch(async (error) => { await this.#snapClient.trackError( new Error('Failed to fetch TRC20 balances for inactive account', { @@ -262,10 +281,17 @@ export class SnapAssetsAdapter { return this.#filterTokensWithoutPriceData(enrichedAssets, spotPrices); } - async fetchAccountAssets(account: KeyringAccount): Promise { + async fetchAccountAssets( + account: KeyringAccount, + options?: ExplicitRequestOptions, + ): Promise { const results = await Promise.all( account.scopes.map((scope) => - this.fetchAssetsAndBalancesForAccount(scope as Network, account), + this.fetchAssetsAndBalancesForAccount( + scope as Network, + account, + options, + ), ), ); diff --git a/packages/tron-wallet-snap/src/utils/errors.test.ts b/packages/tron-wallet-snap/src/utils/errors.test.ts index e890ff98e..b8b799945 100644 --- a/packages/tron-wallet-snap/src/utils/errors.test.ts +++ b/packages/tron-wallet-snap/src/utils/errors.test.ts @@ -1,6 +1,12 @@ +import { BrokenCircuitError } from '@metamask/snap-networks-utils'; import { SnapError, UserRejectedRequestError } from '@metamask/snaps-sdk'; -import { trackError, withCatchAndThrowSnapError } from './errors'; +import { shouldTrack, trackError, withCatchAndThrowSnapError } from './errors'; +import { + ACCOUNT_INFO_FETCH_FAILED_MESSAGE, + FAILURE_STREAK_REMINDER_MS, + failureStreakGate, +} from './failureStreakGate'; import { mockLogger } from './mockLogger'; jest.mock('./logger', () => ({ @@ -31,6 +37,73 @@ describe('errors', () => { expect(mockSnapRequest).not.toHaveBeenCalled(); }); + it('does not track the open-circuit short-circuit, even wrapped (v5)', async () => { + const { mockSnapRequest } = setupTest(); + + expect(await trackError(new BrokenCircuitError())).toBeUndefined(); + expect( + await trackError( + new Error('Failed to fetch account info', { + cause: new BrokenCircuitError(), + }), + ), + ).toBeUndefined(); + + expect(mockSnapRequest).not.toHaveBeenCalled(); + }); + + it('does not track the open-circuit error rethrown as a plain Error by the keyring SDK (v6)', async () => { + const { mockSnapRequest } = setupTest(); + + // The onboarding event: `handleKeyringRequest` rethrew the + // `BrokenCircuitError` as `new Error(message)`, and the handler wrapper + // tracked it. + await expect( + withCatchAndThrowSnapError(async () => { + throw new Error( + 'Execution prevented because the circuit breaker is open', + ); + }), + ).rejects.toThrow('circuit breaker is open'); + + expect(mockSnapRequest).not.toHaveBeenCalled(); + }); + + it('tracks "Failed to fetch account info" once per failure streak (v6)', async () => { + const { mockSnapRequest } = setupTest(); + mockSnapRequest.mockResolvedValue('id'); + failureStreakGate.clear(); + let now = 5_000_000; + const dateSpy = jest.spyOn(Date, 'now').mockImplementation(() => now); + const fail = async (): Promise => + withCatchAndThrowSnapError(async () => { + // What reaches the wrapper from a keyring call. + throw new Error(ACCOUNT_INFO_FETCH_FAILED_MESSAGE); + }).catch(() => undefined); + + try { + await fail(); + now += 15_000; + await fail(); + now += 15_000; + await fail(); + expect(mockSnapRequest).toHaveBeenCalledTimes(1); + + // Still failing after the reminder interval: reported again. + now += FAILURE_STREAK_REMINDER_MS; + await fail(); + expect(mockSnapRequest).toHaveBeenCalledTimes(2); + + // A success ends the streak: the next failure is reported. + failureStreakGate.endStreak(ACCOUNT_INFO_FETCH_FAILED_MESSAGE); + await fail(); + expect(mockSnapRequest).toHaveBeenCalledTimes(3); + } finally { + dateSpy.mockRestore(); + failureStreakGate.clear(); + } + }); + it('sanitizes errors before tracking', async () => { const { mockSnapRequest } = setupTest(); mockSnapRequest.mockResolvedValue('tracked-error-id'); @@ -91,4 +164,17 @@ describe('errors', () => { expect(mockSnapRequest).not.toHaveBeenCalled(); }); }); + + describe('shouldTrack', () => { + it('tracks other errors', () => { + expect(shouldTrack(new Error('Failed to fetch account info'))).toBe(true); + }); + + it('skips user rejections and open-circuit errors', () => { + expect(shouldTrack(new UserRejectedRequestError())).toBe(false); + expect( + shouldTrack(new Error('wrapped', { cause: new BrokenCircuitError() })), + ).toBe(false); + }); + }); }); diff --git a/packages/tron-wallet-snap/src/utils/errors.ts b/packages/tron-wallet-snap/src/utils/errors.ts index b8b2a7de1..e30e5af51 100644 --- a/packages/tron-wallet-snap/src/utils/errors.ts +++ b/packages/tron-wallet-snap/src/utils/errors.ts @@ -1,14 +1,42 @@ import { createSnapErrorHandling } from '@metamask/snap-networks-utils'; +import { UserRejectedRequestError } from '@metamask/snaps-sdk'; import { getSnapProvider } from '../clients/snap/getSnapProvider'; +import { failureStreakGate, getStreakGatedMessage } from './failureStreakGate'; import logger from './logger'; +import { isOpenCircuitError } from './openCircuitBypass'; import { sanitizeSensitiveError } from './sensitiveErrors'; export { isSnapRpcError, sanitizeSensitiveError } from './sensitiveErrors'; +/** + * Whether an error is reported to Sentry. Not reported: + * - user rejections (the default); + * - the fetch policy's open-circuit short-circuit, detected by class, flag, + * name or message through the cause chain (the keyring SDK rethrows it as a + * plain `Error` with the same message): no request was made, and the + * failures that opened the circuit were already reported; + * - repeats of a streak-gated failure ('Failed to fetch account info'): only + * the first failure of a streak is reported, then a reminder at most every + * 15 minutes while it lasts (see `FailureStreakGate`). + * + * @param error - The error. + * @returns `true` if the error should be tracked. + */ +export function shouldTrack(error: unknown): boolean { + if (error instanceof UserRejectedRequestError || isOpenCircuitError(error)) { + return false; + } + const gatedMessage = getStreakGatedMessage(error); + return ( + gatedMessage === undefined || failureStreakGate.shouldReport(gatedMessage) + ); +} + export const { trackError, withCatchAndThrowSnapError } = createSnapErrorHandling({ getSnapProvider, logError: logger.error.bind(logger), prepareError: sanitizeSensitiveError, + shouldTrack, }); diff --git a/packages/tron-wallet-snap/src/utils/failureStreakGate.test.ts b/packages/tron-wallet-snap/src/utils/failureStreakGate.test.ts new file mode 100644 index 000000000..9f26b9142 --- /dev/null +++ b/packages/tron-wallet-snap/src/utils/failureStreakGate.test.ts @@ -0,0 +1,60 @@ +import { + ACCOUNT_INFO_FETCH_FAILED_MESSAGE, + FAILURE_STREAK_REMINDER_MS, + FailureStreakGate, + getStreakGatedMessage, +} from './failureStreakGate'; + +describe('FailureStreakGate', () => { + let now: number; + + beforeEach(() => { + now = 1_000_000; + jest.spyOn(Date, 'now').mockImplementation(() => now); + }); + + afterEach(() => { + jest.restoreAllMocks(); + }); + + it('reports the first failure of a streak, then a reminder per interval', () => { + const gate = new FailureStreakGate(); + + expect(gate.shouldReport('a')).toBe(true); + now += 30_000; + expect(gate.shouldReport('a')).toBe(false); + now += FAILURE_STREAK_REMINDER_MS - 30_000 - 1; + expect(gate.shouldReport('a')).toBe(false); + now += 1; + expect(gate.shouldReport('a')).toBe(true); + expect(gate.shouldReport('a')).toBe(false); + }); + + it('starts a new streak after a success', () => { + const gate = new FailureStreakGate(); + + expect(gate.shouldReport('a')).toBe(true); + expect(gate.shouldReport('a')).toBe(false); + gate.endStreak('a'); + expect(gate.shouldReport('a')).toBe(true); + }); + + it('keeps keys independent', () => { + const gate = new FailureStreakGate(); + + expect(gate.shouldReport('a')).toBe(true); + expect(gate.shouldReport('b')).toBe(true); + gate.clear(); + expect(gate.shouldReport('a')).toBe(true); + }); +}); + +describe('getStreakGatedMessage', () => { + it('returns the gated message only', () => { + expect( + getStreakGatedMessage(new Error(ACCOUNT_INFO_FETCH_FAILED_MESSAGE)), + ).toBe(ACCOUNT_INFO_FETCH_FAILED_MESSAGE); + expect(getStreakGatedMessage(new Error('other'))).toBeUndefined(); + expect(getStreakGatedMessage('x')).toBeUndefined(); + }); +}); diff --git a/packages/tron-wallet-snap/src/utils/failureStreakGate.ts b/packages/tron-wallet-snap/src/utils/failureStreakGate.ts new file mode 100644 index 000000000..b286d9318 --- /dev/null +++ b/packages/tron-wallet-snap/src/utils/failureStreakGate.ts @@ -0,0 +1,97 @@ +/** + * The message of the error thrown when an account's on-chain info cannot be + * fetched (see `SnapAssetsAdapter.fetchAssetsAndBalancesForAccount`). Every + * balance refresh of a client (`listAccountAssets`, `getAccountBalances`, + * revalidation every 15 s, the user's Retry) fails with it while TronGrid is + * unreachable, so it is reported once per failure streak instead of once per + * call. + */ +export const ACCOUNT_INFO_FETCH_FAILED_MESSAGE = 'Failed to fetch account info'; + +/** + * While a failure streak lasts, it is reported again at most this often, so a + * long outage still shows up in Sentry as ongoing. + */ +export const FAILURE_STREAK_REMINDER_MS = 15 * 60_000; + +/** + * Reports a repeated failure once per streak: the first failure is reported, + * the following ones are not until the streak ends (a success for the same + * key) or {@link FAILURE_STREAK_REMINDER_MS} has passed since the last report. + * + * Keyed by message, not by account: the error reaching the snap's error + * handler from a keyring call is rethrown by the keyring SDK as a plain + * `Error(message)` without its cause or the account, and an upstream outage + * fails every account at once, so one event per outage carries the signal. + */ +export class FailureStreakGate { + readonly #reminderMs: number; + + readonly #reportedAt = new Map(); + + constructor(reminderMs: number = FAILURE_STREAK_REMINDER_MS) { + this.#reminderMs = reminderMs; + } + + /** + * Whether a failure should be reported, recording the report if so. + * + * @param key - The failure key (the error message). + * @returns `true` for the first failure of a streak or a due reminder. + */ + shouldReport(key: string): boolean { + const now = Date.now(); + const reportedAt = this.#reportedAt.get(key); + if (reportedAt !== undefined && now - reportedAt < this.#reminderMs) { + return false; + } + this.#reportedAt.set(key, now); + return true; + } + + /** + * Ends the failure streak of a key (the operation succeeded): its next + * failure is reported. + * + * @param key - The failure key. + */ + endStreak(key: string): void { + this.#reportedAt.delete(key); + } + + /** + * Forgets every streak. + */ + clear(): void { + this.#reportedAt.clear(); + } +} + +/** + * Failures reported once per streak (see {@link FailureStreakGate}). + */ +export const STREAK_GATED_FAILURE_MESSAGES: readonly string[] = [ + ACCOUNT_INFO_FETCH_FAILED_MESSAGE, +]; + +/** + * The snap-wide gate used by `shouldTrack`. + */ +export const failureStreakGate = new FailureStreakGate(); + +/** + * The streak-gated message of an error, if any. + * + * @param error - The error. + * @returns The gated message, or `undefined`. + */ +export function getStreakGatedMessage(error: unknown): string | undefined { + const message = + typeof error === 'object' && error !== null + ? (error as { message?: unknown }).message + : undefined; + return typeof message === 'string' && + STREAK_GATED_FAILURE_MESSAGES.includes(message) + ? message + : undefined; +} diff --git a/packages/tron-wallet-snap/src/utils/openCircuitBypass.test.ts b/packages/tron-wallet-snap/src/utils/openCircuitBypass.test.ts new file mode 100644 index 000000000..d084d6d76 --- /dev/null +++ b/packages/tron-wallet-snap/src/utils/openCircuitBypass.test.ts @@ -0,0 +1,308 @@ +import { BrokenCircuitError } from '@metamask/snap-networks-utils'; + +import { + isCircuitOpen, + isOpenCircuitError, + OPEN_CIRCUIT_BYPASS_INTERVAL_MS, + OPEN_CIRCUIT_BYPASS_RESULT_TTL_MS, + OPEN_CIRCUIT_ERROR_MESSAGE, + OpenCircuitBypass, +} from './openCircuitBypass'; + +describe('isOpenCircuitError', () => { + it('detects the open-circuit error and wrapping errors', () => { + const open = new BrokenCircuitError(); + + expect(isOpenCircuitError(open)).toBe(true); + expect( + isOpenCircuitError( + new Error('Failed to fetch account info', { + cause: new Error('wrapped', { cause: open }), + }), + ), + ).toBe(true); + }); + + it('does not match other errors', () => { + expect(isOpenCircuitError(new Error('HTTP 500'))).toBe(false); + expect( + isOpenCircuitError(new Error('outer', { cause: new Error('inner') })), + ).toBe(false); + expect(isOpenCircuitError(undefined)).toBe(false); + expect(isOpenCircuitError('BrokenCircuitError')).toBe(false); + }); +}); + +describe('OpenCircuitBypass', () => { + const key = 'tron:728126428 GET https://example.test/v1/accounts/T1'; + + let now: number; + + beforeEach(() => { + now = 1_000_000; + jest.spyOn(Date, 'now').mockImplementation(() => now); + }); + + it('returns the guarded result without a direct attempt', async () => { + const bypass = new OpenCircuitBypass(); + const direct = jest.fn(); + + expect( + await bypass.run( + key, + { bypassOpenCircuit: true }, + async () => 'guarded', + direct, + ), + ).toBe('guarded'); + expect(direct).not.toHaveBeenCalled(); + }); + + it('makes one direct attempt when the circuit is open on an explicit request', async () => { + const bypass = new OpenCircuitBypass(); + const direct = jest.fn().mockResolvedValue('direct'); + + expect( + await bypass.run( + key, + { bypassOpenCircuit: true }, + async () => { + throw new BrokenCircuitError(); + }, + direct, + ), + ).toBe('direct'); + expect(direct).toHaveBeenCalledTimes(1); + }); + + it('does not bypass the circuit for background reads', async () => { + const bypass = new OpenCircuitBypass(); + const direct = jest.fn(); + const guarded = async (): Promise => { + throw new BrokenCircuitError(); + }; + + await expect( + bypass.run(key, undefined, guarded, direct), + ).rejects.toBeInstanceOf(BrokenCircuitError); + await expect( + bypass.run(key, { bypassOpenCircuit: false }, guarded, direct), + ).rejects.toBeInstanceOf(BrokenCircuitError); + expect(direct).not.toHaveBeenCalled(); + }); + + it('rethrows other errors without a direct attempt', async () => { + const bypass = new OpenCircuitBypass(); + const direct = jest.fn(); + const failure = new Error('HTTP 500'); + + await expect( + bypass.run( + key, + { bypassOpenCircuit: true }, + async () => { + throw failure; + }, + direct, + ), + ).rejects.toBe(failure); + expect(direct).not.toHaveBeenCalled(); + }); + + it('bypasses the circuit at most once per interval for the same request', async () => { + const bypass = new OpenCircuitBypass(); + const direct = jest.fn().mockRejectedValue(new Error('HTTP 429')); + const guarded = async (): Promise => { + throw new BrokenCircuitError(); + }; + const run = async (requestKey: string): Promise => + bypass.run(requestKey, { bypassOpenCircuit: true }, guarded, direct); + + await expect(run(key)).rejects.toThrow('HTTP 429'); + now += OPEN_CIRCUIT_BYPASS_INTERVAL_MS - 1; + await expect(run(key)).rejects.toBeInstanceOf(BrokenCircuitError); + expect(direct).toHaveBeenCalledTimes(1); + + // Another upstream request has its own interval. + await expect(run(`${key}-other`)).rejects.toThrow('HTTP 429'); + expect(direct).toHaveBeenCalledTimes(2); + + now += 1; + await expect(run(key)).rejects.toThrow('HTTP 429'); + expect(direct).toHaveBeenCalledTimes(3); + }); + + it('shares an in-flight direct attempt between identical requests', async () => { + const bypass = new OpenCircuitBypass(); + let resolveDirect: (value: string) => void = () => undefined; + const direct = jest.fn( + async () => + new Promise((resolve) => { + resolveDirect = resolve; + }), + ); + const guarded = async (): Promise => { + throw new BrokenCircuitError(); + }; + + const first = bypass.run(key, { bypassOpenCircuit: true }, guarded, direct); + const second = bypass.run( + key, + { bypassOpenCircuit: true }, + guarded, + direct, + ); + // Let both guarded calls fail. + await new Promise((resolve) => setImmediate(resolve)); + resolveDirect('direct'); + + expect(await first).toBe('direct'); + expect(await second).toBe('direct'); + expect(direct).toHaveBeenCalledTimes(1); + }); +}); + +describe('open-circuit detection without the class (v6)', () => { + it('matches by message, flag or name, also in the cause chain', () => { + // What the keyring SDK rethrows: a plain Error with the same message. + expect(isOpenCircuitError(new Error(OPEN_CIRCUIT_ERROR_MESSAGE))).toBe( + true, + ); + expect(isOpenCircuitError({ isBrokenCircuitError: true })).toBe(true); + const named = new Error('x'); + named.name = 'IsolatedCircuitError'; + expect(isOpenCircuitError(named)).toBe(true); + expect( + isOpenCircuitError( + new Error('Failed to fetch account info', { + cause: { message: OPEN_CIRCUIT_ERROR_MESSAGE }, + }), + ), + ).toBe(true); + expect(isOpenCircuitError(new Error('Failed to fetch account info'))).toBe( + false, + ); + }); + + it('reads the circuit state from the policy', () => { + expect(isCircuitOpen(undefined)).toBe(false); + expect(isCircuitOpen({})).toBe(false); + expect(isCircuitOpen({ getRemainingCircuitOpenDuration: () => null })).toBe( + false, + ); + expect(isCircuitOpen({ getRemainingCircuitOpenDuration: () => 0 })).toBe( + false, + ); + expect( + isCircuitOpen({ getRemainingCircuitOpenDuration: () => 12_000 }), + ).toBe(true); + }); +}); + +describe('OpenCircuitBypass (v6)', () => { + const key = 'tron:728126428 GET https://example.test/v1/accounts/T1'; + + let now: number; + + beforeEach(() => { + now = 2_000_000; + jest.spyOn(Date, 'now').mockImplementation(() => now); + }); + + it('bypasses an error that only carries the open-circuit message', async () => { + const bypass = new OpenCircuitBypass(); + const direct = jest.fn().mockResolvedValue('direct'); + + expect( + await bypass.run( + key, + { bypassOpenCircuit: true }, + async () => { + throw new Error(OPEN_CIRCUIT_ERROR_MESSAGE); + }, + direct, + ), + ).toBe('direct'); + expect(direct).toHaveBeenCalledTimes(1); + }); + + it('goes direct without the guarded call when the policy reports an open circuit', async () => { + const bypass = new OpenCircuitBypass(); + const guarded = jest.fn(); + const direct = jest.fn().mockResolvedValue('direct'); + const policy = { + getRemainingCircuitOpenDuration: (): number => 20_000, + reset: jest.fn(), + }; + + expect( + await bypass.run( + key, + { bypassOpenCircuit: true }, + guarded, + direct, + policy, + ), + ).toBe('direct'); + expect(guarded).not.toHaveBeenCalled(); + expect(policy.reset).toHaveBeenCalledTimes(1); + }); + + it('does not check the state of background reads', async () => { + const bypass = new OpenCircuitBypass(); + const guarded = jest.fn().mockResolvedValue('guarded'); + const direct = jest.fn(); + + expect( + await bypass.run(key, undefined, guarded, direct, { + getRemainingCircuitOpenDuration: (): number => 20_000, + }), + ).toBe('guarded'); + expect(direct).not.toHaveBeenCalled(); + }); + + it('reuses a successful direct result for the result TTL', async () => { + const bypass = new OpenCircuitBypass(); + const guarded = async (): Promise => { + throw new BrokenCircuitError(); + }; + const direct = jest + .fn() + .mockResolvedValueOnce('first') + .mockResolvedValueOnce('second'); + const run = async (): Promise => + bypass.run(key, { bypassOpenCircuit: true }, guarded, direct); + + expect(await run()).toBe('first'); + now += OPEN_CIRCUIT_BYPASS_RESULT_TTL_MS - 1; + expect(await run()).toBe('first'); + expect(direct).toHaveBeenCalledTimes(1); + + // Expired, and within the bypass interval: the circuit error again. + now += 1; + await expect(run()).rejects.toBeInstanceOf(BrokenCircuitError); + + now += OPEN_CIRCUIT_BYPASS_INTERVAL_MS; + expect(await run()).toBe('second'); + }); + + it('does not reset the policy after a failed direct attempt', async () => { + const bypass = new OpenCircuitBypass(); + const policy = { reset: jest.fn() }; + + await expect( + bypass.run( + key, + { bypassOpenCircuit: true }, + async () => { + throw new BrokenCircuitError(); + }, + async () => { + throw new Error('offline'); + }, + policy, + ), + ).rejects.toThrow('offline'); + expect(policy.reset).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/tron-wallet-snap/src/utils/openCircuitBypass.ts b/packages/tron-wallet-snap/src/utils/openCircuitBypass.ts new file mode 100644 index 000000000..dfb2e8b09 --- /dev/null +++ b/packages/tron-wallet-snap/src/utils/openCircuitBypass.ts @@ -0,0 +1,239 @@ +import { BrokenCircuitError } from '@metamask/snap-networks-utils'; + +/** + * Minimum time between two requests that skip the open circuit of a fetch + * policy for the same upstream request (see {@link OpenCircuitBypass}). + */ +export const OPEN_CIRCUIT_BYPASS_INTERVAL_MS = 10_000; + +/** + * How long the result of a successful direct (bypassing) request is reused for + * the same upstream request. A balance refresh makes two keyring calls in a + * row (`keyring_listAccountAssets`, then `keyring_getAccountBalances`) that + * read the same endpoints; both must see the result of the one direct request. + */ +export const OPEN_CIRCUIT_BYPASS_RESULT_TTL_MS = 5_000; + +/** + * The message of cockatiel's `BrokenCircuitError` (and its subclasses), thrown + * by the fetch policy when it short-circuits a call while its circuit is open. + * The keyring SDK rethrows non-snap errors as `new Error(message)`, dropping + * the class and the cause, so the message is the only marker left by the time + * the error reaches the snap's error handler. + */ +export const OPEN_CIRCUIT_ERROR_MESSAGE = + 'Execution prevented because the circuit breaker is open'; + +/** + * Options of a read made on behalf of an explicit client request. + */ +export type ExplicitRequestOptions = { + /** + * The request was explicitly made by the client (a `listAccountAssets` or + * `getAccountBalances` keyring call, such as the balance Retry): it may reach + * the network once while the fetch policy's circuit is open, instead of + * failing without a request until the circuit half-opens. + */ + bypassOpenCircuit?: boolean; +}; + +/** + * The parts of a fetch policy (`createFetchPolicy`) the bypass uses. Optional, + * so a bare `{ execute }` policy (tests) still works. + */ +export type BypassablePolicy = { + getRemainingCircuitOpenDuration?: () => number | null; + reset?: () => void; +}; + +/** + * Whether a single error (not its cause chain) is an open-circuit error. + * + * Checked by class, by cockatiel's `isBrokenCircuitError` flag, by name, and by + * message, because the class identity does not survive every boundary (the + * keyring SDK rethrows a plain `Error` with the same message; bundling can + * duplicate the class). + * + * @param error - The error. + * @returns `true` if the error is an open-circuit error. + */ +function isOwnOpenCircuitError(error: unknown): boolean { + if (error instanceof BrokenCircuitError) { + return true; + } + if (typeof error !== 'object' || error === null) { + return false; + } + const { isBrokenCircuitError, name, message } = error as { + isBrokenCircuitError?: unknown; + name?: unknown; + message?: unknown; + }; + return ( + isBrokenCircuitError === true || + name === 'BrokenCircuitError' || + name === 'IsolatedCircuitError' || + (typeof message === 'string' && + message.includes(OPEN_CIRCUIT_ERROR_MESSAGE)) + ); +} + +/** + * Whether an error, or any error in its `cause` chain, is the fetch policy's + * open-circuit short-circuit (no request was made). + * + * @param error - The error to check. + * @returns `true` if the error comes from an open circuit. + */ +export function isOpenCircuitError(error: unknown): boolean { + let current: unknown = error; + for (let depth = 0; depth < 10 && current !== undefined; depth++) { + if (isOwnOpenCircuitError(current)) { + return true; + } + current = + typeof current === 'object' && current !== null + ? (current as { cause?: unknown }).cause + : undefined; + } + return false; +} + +/** + * Whether the policy's circuit is open (calls fail without a request). + * + * @param policy - The fetch policy. + * @returns `true` if the circuit is open. + */ +export function isCircuitOpen(policy: BypassablePolicy | undefined): boolean { + const remaining = policy?.getRemainingCircuitOpenDuration?.(); + return typeof remaining === 'number' && remaining > 0; +} + +/** + * Lets an explicit client request reach the network while a fetch policy's + * circuit is open. + * + * The circuit opens after repeated failures and then fails every call without + * a request for 30 seconds, so a user-initiated retry made meanwhile could + * never succeed. With this gate, such a request makes one direct attempt (no + * retries, not counted by the policy) when the circuit is open: either known + * up front from the policy's state, or detected from the guarded call's + * open-circuit error. Concurrent identical requests share the attempt, a + * successful attempt is reused for {@link OPEN_CIRCUIT_BYPASS_RESULT_TTL_MS} + * and closes the circuit (the service answered), and each upstream request + * bypasses the circuit at most once per {@link OPEN_CIRCUIT_BYPASS_INTERVAL_MS}, + * so polling cannot hammer a failing service; background reads never bypass + * it. + */ +export class OpenCircuitBypass { + readonly #intervalMs: number; + + readonly #resultTtlMs: number; + + readonly #lastAttemptAt = new Map(); + + readonly #lastResult = new Map(); + + readonly #inFlight = new Map>(); + + constructor( + intervalMs: number = OPEN_CIRCUIT_BYPASS_INTERVAL_MS, + resultTtlMs: number = OPEN_CIRCUIT_BYPASS_RESULT_TTL_MS, + ) { + this.#intervalMs = intervalMs; + this.#resultTtlMs = resultTtlMs; + } + + /** + * Runs the guarded read, and the direct one if allowed when the circuit is + * open. + * + * @param key - Identifies the upstream request (network, method, URL, body). + * @param options - The request options. + * @param guarded - The read under the fetch policy. + * @param direct - The same read, once, without the policy. + * @param policy - The fetch policy guarding the read, used to check whether + * its circuit is open before the call and to close it after a successful + * direct read. + * @returns The read result. + */ + async run( + key: string, + options: ExplicitRequestOptions | undefined, + guarded: () => Promise, + direct: () => Promise, + policy?: BypassablePolicy, + ): Promise { + if (!options?.bypassOpenCircuit) { + return guarded(); + } + + const now = Date.now(); + const lastResult = this.#lastResult.get(key); + if (lastResult && now - lastResult.at < this.#resultTtlMs) { + return lastResult.value as Result; + } + + const inFlight = this.#inFlight.get(key); + if (inFlight) { + return (await inFlight) as Result; + } + + let openCircuitError: unknown = new BrokenCircuitError(); + if (!isCircuitOpen(policy)) { + try { + return await guarded(); + } catch (error) { + if (!isOpenCircuitError(error)) { + throw error; + } + openCircuitError = error; + } + } + + return this.#runDirect(key, direct, openCircuitError, policy); + } + + async #runDirect( + key: string, + direct: () => Promise, + openCircuitError: unknown, + policy: BypassablePolicy | undefined, + ): Promise { + const inFlight = this.#inFlight.get(key); + if (inFlight) { + return (await inFlight) as Result; + } + + const now = Date.now(); + const lastAttemptAt = this.#lastAttemptAt.get(key); + if (lastAttemptAt !== undefined && now - lastAttemptAt < this.#intervalMs) { + throw openCircuitError; + } + for (const [otherKey, otherAt] of this.#lastAttemptAt) { + if (now - otherAt >= this.#intervalMs) { + this.#lastAttemptAt.delete(otherKey); + } + } + for (const [otherKey, { at }] of this.#lastResult) { + if (now - at >= this.#resultTtlMs) { + this.#lastResult.delete(otherKey); + } + } + this.#lastAttemptAt.set(key, now); + + const attempt = (async (): Promise => { + const value = await direct(); + this.#lastResult.set(key, { at: Date.now(), value }); + // The service answered: close the circuit so background reads resume + // now instead of after the next half-open probe. + policy?.reset?.(); + return value; + })().finally(() => { + this.#inFlight.delete(key); + }); + this.#inFlight.set(key, attempt); + return await attempt; + } +}