diff --git a/packages/bitcoin-wallet-snap/src/entities/account.ts b/packages/bitcoin-wallet-snap/src/entities/account.ts index 067a3be3b..2dbe6b243 100644 --- a/packages/bitcoin-wallet-snap/src/entities/account.ts +++ b/packages/bitcoin-wallet-snap/src/entities/account.ts @@ -126,6 +126,15 @@ export type BitcoinAccount = { */ applyUpdate(update: Update): void; + /** + * Whether the account's wallet has applied at least one chain update (sync + * or full scan). A wallet that never synced still sits on the genesis + * checkpoint (height 0), and its zero balance means "unknown", not "empty". + * + * @returns true if the wallet has synchronized with the chain at least once + */ + hasSynchronized(): boolean; + /** * Extract the change set if it exists. * diff --git a/packages/bitcoin-wallet-snap/src/handlers/CronHandler.test.ts b/packages/bitcoin-wallet-snap/src/handlers/CronHandler.test.ts index 2a0d16cbb..cee872038 100644 --- a/packages/bitcoin-wallet-snap/src/handlers/CronHandler.test.ts +++ b/packages/bitcoin-wallet-snap/src/handlers/CronHandler.test.ts @@ -1,5 +1,4 @@ import type { Amount, WalletTx } from '@metamask/bitcoindevkit'; -import { TransactionType } from '@metamask/keyring-api'; import { getSelectedAccounts } from '@metamask/keyring-snap-sdk'; import { SynchronizationError } from '@metamask/snap-networks-utils'; import type { SnapsProvider, JsonRpcRequest } from '@metamask/snaps-sdk'; @@ -12,7 +11,7 @@ import type { SyncResult, } from '../entities'; import type { SendFlowUseCases, AccountUseCases } from '../use-cases'; -import { CronHandler, CronMethod } from './CronHandler'; +import { CronHandler, CronMethod, MAX_FULL_SCANS_PER_RUN } from './CronHandler'; jest.mock('@metamask/keyring-snap-sdk', () => ({ getSelectedAccounts: jest.fn(), @@ -41,9 +40,12 @@ describe('CronHandler', () => { clientVersion: '1.0.0', platformVersion: '1.0.0', }); - // Default the repair rescan to nothing pending so existing tests don't - // trigger a repair scan. + // Default to no queued full scan and known balances so existing tests + // don't trigger a full scan. mockSnapClient.getState.mockResolvedValue({ pending: [] }); + mockAccountUseCases.listDueFullScans.mockResolvedValue([]); + mockAccountUseCases.isBalanceUnknown.mockResolvedValue(false); + mockAccountUseCases.isFullScanInFlight.mockReturnValue(false); }); describe('synchronizeAccounts', () => { @@ -271,7 +273,7 @@ describe('CronHandler', () => { ); }); - describe('repair rescan', () => { + describe('queued full scans (first scan and repair)', () => { const buildTx = (txid: string): WalletTx => mock({ txid: mock({ toString: () => txid }), @@ -295,156 +297,94 @@ describe('CronHandler', () => { mockAccount2.listTransactions.mockReturnValue([]); }); - it('on first run, persists the full pending list, scans only the first account, and emits for missed transactions', async () => { - mockSnapClient.getState.mockResolvedValue(null); + it('asks for at most MAX_FULL_SCANS_PER_RUN due accounts and does nothing when none is due', async () => { + mockAccountUseCases.listDueFullScans.mockResolvedValue([]); + + await handler.route(request); + + expect(mockAccountUseCases.listDueFullScans).toHaveBeenCalledWith( + mockAccounts, + MAX_FULL_SCANS_PER_RUN, + ); + expect(mockAccountUseCases.fullScan).not.toHaveBeenCalled(); + expect(mockSnapClient.emitTrackingError).not.toHaveBeenCalled(); + // The regular sync still runs. + expect(mockAccountUseCases.synchronize).toHaveBeenCalledTimes(2); + }); + + it('full-scans every due account, emits balances and missed transactions', async () => { + mockAccountUseCases.listDueFullScans.mockResolvedValue([ + mockAccount1, + mockAccount2, + ]); const txBefore = buildTx('txid-existing'); - const txNew = buildTx('txid-new'); + const txMissed = buildTx('txid-missed'); mockAccount1.listTransactions .mockReturnValueOnce([txBefore]) - .mockReturnValueOnce([txBefore, txNew]); - mockAccountUseCases.fullScan.mockResolvedValue({ - account: mockAccount1, + .mockReturnValueOnce([txBefore, txMissed]); + mockAccountUseCases.fullScan.mockImplementation(async (account) => ({ + account, transactionsToNotify: [], - }); + })); await handler.route(request); - expect(mockSnapClient.getState).toHaveBeenCalledWith('rescanV1'); - expect(mockAccountUseCases.fullScan).toHaveBeenCalledTimes(1); expect(mockAccountUseCases.fullScan).toHaveBeenCalledWith(mockAccount1); + expect(mockAccountUseCases.fullScan).toHaveBeenCalledWith(mockAccount2); + expect( + mockSnapClient.emitAccountBalancesUpdatedEvent, + ).toHaveBeenCalledWith([mockAccount1, mockAccount2]); expect(mockSnapClient.emitTrackingEvent).toHaveBeenCalledTimes(1); expect(mockSnapClient.emitTrackingEvent).toHaveBeenCalledWith( - 'Missed Transactions Discovered', + expect.anything(), mockAccount1, - txNew, + txMissed, 'cron', - TransactionType.Receive, + expect.anything(), ); - expect( - mockSnapClient.emitAccountBalancesUpdatedEvent, - ).toHaveBeenCalledWith([mockAccount1]); - expect(mockSnapClient.setState).toHaveBeenCalledWith('rescanV1', { - pending: ['account-1', 'account-2'], - }); - expect(mockSnapClient.setState).toHaveBeenCalledWith('rescanV1', { - pending: ['account-2'], - }); - - // The initial full list is persisted before the first scan, and the - // shortened list is persisted only after the scan succeeds. - expect(mockSnapClient.setState).toHaveBeenCalledTimes(2); - const [initialPersistOrder, shortenedPersistOrder] = - mockSnapClient.setState.mock.invocationCallOrder; - const scanOrder = - mockAccountUseCases.fullScan.mock.invocationCallOrder[0]; - expect(initialPersistOrder).toBeLessThan(scanOrder as number); - expect(scanOrder).toBeLessThan(shortenedPersistOrder as number); - - // The normal sync flow still runs afterwards. - expect(mockAccountUseCases.synchronize).toHaveBeenCalled(); + expect(mockSnapClient.emitTrackingError).not.toHaveBeenCalled(); }); - it('on a subsequent run, scans only the next pending account', async () => { - mockSnapClient.getState.mockResolvedValue({ pending: ['account-2'] }); - mockAccountUseCases.fullScan.mockResolvedValue({ - account: mockAccount2, - transactionsToNotify: [], - }); - - await handler.route(request); - - expect(mockAccountUseCases.fullScan).toHaveBeenCalledTimes(1); - expect(mockAccountUseCases.fullScan).toHaveBeenCalledWith(mockAccount2); - expect(mockSnapClient.setState).toHaveBeenCalledWith('rescanV1', { - pending: [], - }); - }); - - it('prunes deleted accounts from the pending list before scanning', async () => { - mockSnapClient.getState.mockResolvedValue({ - pending: ['gone', 'account-2'], - }); - mockAccountUseCases.fullScan.mockResolvedValue({ - account: mockAccount2, - transactionsToNotify: [], + it('does not let a failing account block the next one (no head-of-line blocking)', async () => { + mockAccountUseCases.listDueFullScans.mockResolvedValue([ + mockAccount1, + mockAccount2, + ]); + const scanError = new Error('Failed to perform initial full scan'); + mockAccountUseCases.fullScan.mockImplementation(async (account) => { + if (account === mockAccount1) { + throw scanError; + } + return { account, transactionsToNotify: [] }; }); await handler.route(request); - expect(mockSnapClient.setState).toHaveBeenCalledWith('rescanV1', { - pending: ['account-2'], - }); expect(mockAccountUseCases.fullScan).toHaveBeenCalledWith(mockAccount2); - - // The pruned list is persisted before the scan runs. - const pruneOrder = mockSnapClient.setState.mock.invocationCallOrder[0]; - const scanOrder = - mockAccountUseCases.fullScan.mock.invocationCallOrder[0]; - expect(pruneOrder).toBeLessThan(scanOrder as number); - }); - - it('does not scan when every pending account has been deleted', async () => { - mockSnapClient.getState.mockResolvedValue({ pending: ['gone'] }); - - await handler.route(request); - - // The pending list is pruned to nothing, and the repair pass ends - // without scanning anything. - expect(mockSnapClient.setState).toHaveBeenCalledWith('rescanV1', { - pending: [], - }); - expect(mockAccountUseCases.fullScan).not.toHaveBeenCalled(); - }); - - it('does nothing when nothing is pending, short-circuiting before list()', async () => { - mockSnapClient.getState.mockResolvedValue({ pending: [] }); - - await handler.route(request); - - expect(mockAccountUseCases.fullScan).not.toHaveBeenCalled(); - expect(mockSnapClient.setState).not.toHaveBeenCalled(); - // The regular sync flow is the only caller of `list()` here — the - // repair path short-circuits on the empty pending list before - // reaching it. - expect(mockAccountUseCases.list).toHaveBeenCalledTimes(1); + expect( + mockSnapClient.emitAccountBalancesUpdatedEvent, + ).toHaveBeenCalledWith([mockAccount2]); + expect(mockSnapClient.emitTrackingError).toHaveBeenCalledTimes(1); + const [error] = mockSnapClient.emitTrackingError.mock.calls[0] as [ + Error, + ]; + expect(error.message).toBe('Account repair scan failed'); + expect(error.cause).toBe(scanError); + // The regular sync still runs. + expect(mockAccountUseCases.synchronize).toHaveBeenCalledTimes(2); }); - it('reports a scan failure without shortening the pending list, and still runs the regular sync', async () => { - mockSnapClient.getState.mockResolvedValue({ - pending: ['account-1', 'account-2'], - }); - const scanError = new Error('scan failed'); - mockAccountUseCases.fullScan.mockRejectedValue(scanError); + it('reports a failure of the queue itself and still runs the regular sync', async () => { + mockAccountUseCases.listDueFullScans.mockRejectedValue( + new Error('state unavailable'), + ); await handler.route(request); expect(mockSnapClient.emitTrackingError).toHaveBeenCalledWith( - expect.objectContaining({ - message: 'Account repair scan failed', - cause: scanError, - }), + expect.objectContaining({ message: 'Account repair scan failed' }), ); - // The failure is contained: the pending list is not shortened and - // the regular sync still runs. - expect(mockSnapClient.setState).not.toHaveBeenCalledWith('rescanV1', { - pending: ['account-2'], - }); - expect(mockAccountUseCases.synchronize).toHaveBeenCalled(); - }); - - it('treats malformed stored state as unset and reinitializes pending from the account list', async () => { - mockSnapClient.getState.mockResolvedValue(true); - mockAccountUseCases.fullScan.mockResolvedValue({ - account: mockAccount1, - transactionsToNotify: [], - }); - - await handler.route(request); - - expect(mockSnapClient.setState).toHaveBeenCalledWith('rescanV1', { - pending: ['account-1', 'account-2'], - }); - expect(mockAccountUseCases.fullScan).toHaveBeenCalledWith(mockAccount1); + expect(mockAccountUseCases.synchronize).toHaveBeenCalledTimes(2); }); }); }); @@ -752,6 +692,10 @@ describe('CronHandler', () => { transactionsToNotify: mockTxs, }; mockAccountUseCases.get.mockResolvedValue(mockAccount); + // Unknown before the scan, known after it. + mockAccountUseCases.isBalanceUnknown + .mockResolvedValueOnce(true) + .mockResolvedValue(false); mockAccountUseCases.fullScan.mockResolvedValue(mockResult); await handler.route(request); @@ -789,10 +733,39 @@ describe('CronHandler', () => { it('propagates errors from fullScan', async () => { const error = new Error('fullScan failed'); mockAccountUseCases.get.mockResolvedValue(mockAccount); + mockAccountUseCases.isBalanceUnknown.mockResolvedValue(true); mockAccountUseCases.fullScan.mockRejectedValue(error); await expect(handler.route(request)).rejects.toThrow(error); }); + + it('skips an on-demand scan whose purpose was already fulfilled (balance known)', async () => { + mockAccountUseCases.get.mockResolvedValue(mockAccount); + mockAccountUseCases.isFullScanInFlight.mockReturnValue(false); + mockAccountUseCases.isBalanceUnknown.mockResolvedValue(false); + + await handler.route(request); + + expect(mockAccountUseCases.fullScan).not.toHaveBeenCalled(); + expect( + mockSnapClient.emitAccountBalancesUpdatedEvent, + ).not.toHaveBeenCalled(); + }); + + it('joins a scan that is already in flight', async () => { + const mockResult: SyncResult = { + account: mockAccount, + transactionsToNotify: [], + }; + mockAccountUseCases.get.mockResolvedValue(mockAccount); + mockAccountUseCases.isFullScanInFlight.mockReturnValue(true); + mockAccountUseCases.isBalanceUnknown.mockResolvedValue(false); + mockAccountUseCases.fullScan.mockResolvedValue(mockResult); + + await handler.route(request); + + expect(mockAccountUseCases.fullScan).toHaveBeenCalledWith(mockAccount); + }); }); describe('route', () => { diff --git a/packages/bitcoin-wallet-snap/src/handlers/CronHandler.ts b/packages/bitcoin-wallet-snap/src/handlers/CronHandler.ts index 8214d50c5..a46eaeeca 100644 --- a/packages/bitcoin-wallet-snap/src/handlers/CronHandler.ts +++ b/packages/bitcoin-wallet-snap/src/handlers/CronHandler.ts @@ -6,14 +6,19 @@ import { getSyncFailuresFromSettledResult, } from '@metamask/snap-networks-utils'; import type { JsonRpcRequest, SnapsProvider } from '@metamask/snaps-sdk'; -import { array, assert, is, object, string } from 'superstruct'; +import { array, assert, object, string } from 'superstruct'; import { InexistentMethodError, mapToTransactionType, TrackingSnapEvent, } from '../entities'; -import type { Logger, SnapClient, SyncResult } from '../entities'; +import type { + BitcoinAccount, + Logger, + SnapClient, + SyncResult, +} from '../entities'; import type { SendFlowUseCases, AccountUseCases } from '../use-cases'; export const CronMethod = { @@ -37,9 +42,10 @@ export const FullScanAccountRequest = object({ accountId: string(), }); -const RescanState = object({ - pending: array(string()), -}); +/** + * Maximum number of queued full scans run per sync run. + */ +export const MAX_FULL_SCANS_PER_RUN = 2; export class CronHandler { readonly #accountsUseCases: AccountUseCases; @@ -137,8 +143,10 @@ export class CronHandler { // that coalesced callers share the run's outcome, including the reported // synchronization failures. await this.#syncCoalescer.run('synchronizeAccounts', async () => { + const allAccounts = await this.#accountsUseCases.list(); + try { - await this.#repairNextAccount(); + await this.#runDueFullScans(allAccounts); } catch (error) { await this.#snapClient.emitTrackingError( new Error('Account repair scan failed', { cause: error }), @@ -149,11 +157,9 @@ export class CronHandler { await getSelectedAccounts(this.#snap), ); - const accounts = (await this.#accountsUseCases.list()).filter( - (account) => { - return selectedAccounts.has(account.id); - }, - ); + const accounts = allAccounts.filter((account) => { + return selectedAccounts.has(account.id); + }); const results = await Promise.allSettled( accounts.map(async (account) => @@ -262,56 +268,70 @@ export class CronHandler { async fullScanAccount(accountId: string): Promise { const account = await this.#accountsUseCases.get(accountId); + // On-demand scans (new account, balance request, network recovery) can + // be requested more than once; skip the ones whose purpose was already + // fulfilled by another scan in the meantime. + if ( + !this.#accountsUseCases.isFullScanInFlight(accountId) && + !(await this.#accountsUseCases.isBalanceUnknown(account)) + ) { + this.#logger.debug('Full scan no longer needed: %s', accountId); + return; + } const result = await this.#accountsUseCases.fullScan(account); await this.#emitSyncEvents([result]); } /** - * Repair one previously-unwatched account per sync run. Advances the - * pending list only after the scan succeeds, so a scan that never runs - * (client locked/inactive) or fails leaves the account pending for the - * next sync run instead of being marked done prematurely. + * Runs the full scans that are due: the first full scan of new accounts, + * of accounts that never synchronized, and the one-off repair scan of + * previously-unwatched accounts. + * + * @param accounts - All live accounts. + * + * Up to {@link MAX_FULL_SCANS_PER_RUN} accounts are scanned concurrently per + * sync run. A failed scan moves the account to the tail of the queue and + * backs it off exponentially (see `FullScanQueue`), so a single failing + * account can no longer block every account queued behind it. */ - async #repairNextAccount(): Promise { - const stored = await this.#snapClient.getState('rescanV1'); - if (is(stored, RescanState) && stored.pending.length === 0) { + async #runDueFullScans(accounts: BitcoinAccount[]): Promise { + const due = await this.#accountsUseCases.listDueFullScans( + accounts, + MAX_FULL_SCANS_PER_RUN, + ); + if (due.length === 0) { return; } - const accounts = await this.#accountsUseCases.list(); - const liveIds = new Set(accounts.map((account) => account.id)); - - const pending = is(stored, RescanState) - ? stored.pending.filter((id) => liveIds.has(id)) - : accounts.map((account) => account.id); - - if (!is(stored, RescanState) || pending.length !== stored.pending.length) { - await this.#snapClient.setState('rescanV1', { pending }); - } + const results = await Promise.allSettled( + due.map(async (account) => { + const before = new Set( + account.listTransactions().map((tx) => tx.txid.toString()), + ); + const result = await this.#accountsUseCases.fullScan(account); + + for (const tx of account.listTransactions()) { + if (!before.has(tx.txid.toString())) { + await this.#snapClient.emitTrackingEvent( + TrackingSnapEvent.MissedTransactionsDiscovered, + account, + tx, + 'cron', + mapToTransactionType(account, tx.tx), + ); + } + } + return result; + }), + ); - const account = accounts.find(({ id }) => id === pending[0]); - if (!account) { - return; - } + await this.#finishSync(results); - const before = new Set( - account.listTransactions().map((tx) => tx.txid.toString()), + const firstFailure = results.find( + (result): result is PromiseRejectedResult => result.status === 'rejected', ); - const result = await this.#accountsUseCases.fullScan(account); - - for (const tx of account.listTransactions()) { - if (!before.has(tx.txid.toString())) { - await this.#snapClient.emitTrackingEvent( - TrackingSnapEvent.MissedTransactionsDiscovered, - account, - tx, - 'cron', - mapToTransactionType(account, tx.tx), - ); - } + if (firstFailure) { + throw firstFailure.reason; } - - await this.#emitSyncEvents([result]); - await this.#snapClient.setState('rescanV1', { pending: pending.slice(1) }); } } diff --git a/packages/bitcoin-wallet-snap/src/handlers/KeyringHandler.test.ts b/packages/bitcoin-wallet-snap/src/handlers/KeyringHandler.test.ts index c27159d9e..22ea6d0da 100644 --- a/packages/bitcoin-wallet-snap/src/handlers/KeyringHandler.test.ts +++ b/packages/bitcoin-wallet-snap/src/handlers/KeyringHandler.test.ts @@ -679,6 +679,32 @@ describe('KeyringHandler', () => { ); expect(mockAccounts.get).toHaveBeenCalled(); }); + + it('kicks off a full scan when the balance is still unknown', async () => { + mockAccounts.get.mockResolvedValue(mockAccount); + mockAccounts.isBalanceUnknown.mockResolvedValue(true); + mockAccounts.requestFullScan.mockResolvedValue(true); + + const result = await handler.getAccountBalances(mockAccount.id); + + expect(mockAccounts.isBalanceUnknown).toHaveBeenCalledWith(mockAccount); + expect(mockAccounts.requestFullScan).toHaveBeenCalledWith( + mockAccount, + 'balance-request', + ); + expect(result).toStrictEqual({ + [Caip19Asset.Bitcoin]: { amount: '1', unit: 'BTC' }, + }); + }); + + it('does not request a full scan when the balance is known', async () => { + mockAccounts.get.mockResolvedValue(mockAccount); + mockAccounts.isBalanceUnknown.mockResolvedValue(false); + + await handler.getAccountBalances(mockAccount.id); + + expect(mockAccounts.requestFullScan).not.toHaveBeenCalled(); + }); }); describe('getAccount', () => { diff --git a/packages/bitcoin-wallet-snap/src/handlers/KeyringHandler.ts b/packages/bitcoin-wallet-snap/src/handlers/KeyringHandler.ts index 084f80ecc..fda22113b 100644 --- a/packages/bitcoin-wallet-snap/src/handlers/KeyringHandler.ts +++ b/packages/bitcoin-wallet-snap/src/handlers/KeyringHandler.ts @@ -277,6 +277,16 @@ export class KeyringHandler implements KeyringSnapRpc { id: string, ): Promise> { const account = await this.#accountsUseCases.get(id); + + // An account that never synchronized, or never completed its first full + // scan, has an unknown balance: its local state alone would never + // recover, so the request kicks off a full scan (deduplicated and + // single-flight per account, bounded backoff); its balances-updated event + // then delivers the real balance. + if (await this.#accountsUseCases.isBalanceUnknown(account)) { + await this.#accountsUseCases.requestFullScan(account, 'balance-request'); + } + const balance = Amount.from_sat(computeDisplayBalanceSats(account)) .to_btc() .toString(); diff --git a/packages/bitcoin-wallet-snap/src/infra/BdkAccountAdapter.ts b/packages/bitcoin-wallet-snap/src/infra/BdkAccountAdapter.ts index 2171e1b94..7977f9dc0 100644 --- a/packages/bitcoin-wallet-snap/src/infra/BdkAccountAdapter.ts +++ b/packages/bitcoin-wallet-snap/src/infra/BdkAccountAdapter.ts @@ -183,6 +183,12 @@ export class BdkAccountAdapter implements BitcoinAccount { return this.#wallet.apply_update(update); } + hasSynchronized(): boolean { + // A freshly created wallet only holds the genesis checkpoint (height 0); + // every successful sync or full scan applies a chain update to the tip. + return this.#wallet.latest_checkpoint.height > 0; + } + takeStaged(): ChangeSet | undefined { const staged = this.#collectStaged(); this.#staged = undefined; diff --git a/packages/bitcoin-wallet-snap/src/infra/StoredAccountAdapter.ts b/packages/bitcoin-wallet-snap/src/infra/StoredAccountAdapter.ts index edb5164c6..3f4d12f7b 100644 --- a/packages/bitcoin-wallet-snap/src/infra/StoredAccountAdapter.ts +++ b/packages/bitcoin-wallet-snap/src/infra/StoredAccountAdapter.ts @@ -123,6 +123,13 @@ export class StoredAccountAdapter implements BitcoinAccount { this.#unsupported(); } + hasSynchronized(): boolean { + // Metadata-only view: the wallet (and its checkpoint) is not loaded. + // Callers that need the sync state load the full account; never report + // this view as unsynchronized, which would hide a known balance. + return true; + } + takeStaged(): ChangeSet | undefined { return this.#unsupported(); } diff --git a/packages/bitcoin-wallet-snap/src/store/FullScanQueue.test.ts b/packages/bitcoin-wallet-snap/src/store/FullScanQueue.test.ts new file mode 100644 index 000000000..94e516d0b --- /dev/null +++ b/packages/bitcoin-wallet-snap/src/store/FullScanQueue.test.ts @@ -0,0 +1,231 @@ +import type { SnapClient } from '../entities'; +import { + FULL_SCAN_QUEUE_KEY, + FULL_SCAN_RETRY_BASE_MS, + FULL_SCAN_RETRY_KEY, + FULL_SCAN_RETRY_MAX_MS, + FullScanQueue, + getFullScanRetryDelayMs, +} from './FullScanQueue'; + +const clone = (value: Value): Value => + JSON.parse(JSON.stringify(value)) as Value; + +const createStateClient = ( + initial: Record = {}, +): { client: SnapClient; state: Record } => { + const state: Record = clone(initial); + const client = { + getState: jest.fn(async (key: string) => + Object.hasOwn(state, key) ? clone(state[key]) : null, + ), + setState: jest.fn(async (key: string, value: unknown) => { + state[key] = clone(value); + }), + } as unknown as SnapClient; + return { client, state }; +}; + +describe('getFullScanRetryDelayMs', () => { + it('grows exponentially from the base delay and caps it', () => { + expect(getFullScanRetryDelayMs(1)).toBe(FULL_SCAN_RETRY_BASE_MS); + expect(getFullScanRetryDelayMs(2)).toBe(FULL_SCAN_RETRY_BASE_MS * 2); + expect(getFullScanRetryDelayMs(3)).toBe(FULL_SCAN_RETRY_BASE_MS * 4); + expect(getFullScanRetryDelayMs(50)).toBe(FULL_SCAN_RETRY_MAX_MS); + // Bounded to minutes, not half an hour: an account whose first scan + // failed must recover soon after the network comes back. + expect(FULL_SCAN_RETRY_MAX_MS).toBe(5 * 60_000); + expect(getFullScanRetryDelayMs(5)).toBe(FULL_SCAN_RETRY_MAX_MS); + }); +}); + +describe('FullScanQueue', () => { + let now: number; + const clock = (): number => now; + + beforeEach(() => { + now = 1_000_000; + }); + + it('initializes the queue with every live account when it does not exist (repair behaviour)', async () => { + const { client, state } = createStateClient(); + const queue = new FullScanQueue(client, clock); + + expect(await queue.due(['a', 'b', 'c'], 2)).toStrictEqual(['a', 'b']); + expect(state[FULL_SCAN_QUEUE_KEY]).toStrictEqual({ + pending: ['a', 'b', 'c'], + }); + }); + + it('prunes accounts that no longer exist', async () => { + const { client, state } = createStateClient({ + [FULL_SCAN_QUEUE_KEY]: { pending: ['a', 'gone'] }, + [FULL_SCAN_RETRY_KEY]: { + gone: { attempts: 1, nextAttemptAt: 0 }, + }, + }); + const queue = new FullScanQueue(client, clock); + + expect(await queue.due(['a'], 5)).toStrictEqual(['a']); + expect(state[FULL_SCAN_QUEUE_KEY]).toStrictEqual({ pending: ['a'] }); + expect(state[FULL_SCAN_RETRY_KEY]).toStrictEqual({}); + }); + + it('moves a failed account to the tail so it does not block the others', async () => { + const { client } = createStateClient({ + [FULL_SCAN_QUEUE_KEY]: { pending: ['a', 'b'] }, + }); + const queue = new FullScanQueue(client, clock); + + await queue.markFailed('a', new Error('unreachable')); + + expect(await queue.due(['a', 'b'], 2)).toStrictEqual(['b']); + }); + + it('retries a failed account once its backoff has elapsed, with growing delays', async () => { + const { client, state } = createStateClient({ + [FULL_SCAN_QUEUE_KEY]: { pending: ['a'] }, + }); + const queue = new FullScanQueue(client, clock); + + await queue.markFailed('a', new Error('boom')); + expect(await queue.due(['a'], 1)).toStrictEqual([]); + now += FULL_SCAN_RETRY_BASE_MS; + expect(await queue.due(['a'], 1)).toStrictEqual(['a']); + + await queue.markFailed('a', 'again'); + expect(state[FULL_SCAN_RETRY_KEY]).toStrictEqual({ + a: { + attempts: 2, + nextAttemptAt: now + FULL_SCAN_RETRY_BASE_MS * 2, + lastError: 'again', + }, + }); + }); + + it('removes an account and its retry metadata after a successful scan', async () => { + const { client, state } = createStateClient({ + [FULL_SCAN_QUEUE_KEY]: { pending: ['a', 'b'] }, + [FULL_SCAN_RETRY_KEY]: { + a: { attempts: 1, nextAttemptAt: 0, initial: true }, + }, + }); + const queue = new FullScanQueue(client, clock); + + await queue.markSucceeded('a'); + + expect(state[FULL_SCAN_QUEUE_KEY]).toStrictEqual({ pending: ['b'] }); + expect(state[FULL_SCAN_RETRY_KEY]).toStrictEqual({}); + expect(await queue.isInitialScanPending('a')).toBe(false); + }); + + it('flags new accounts as pending their initial scan until it succeeds, across failures', async () => { + const { client } = createStateClient({ + [FULL_SCAN_QUEUE_KEY]: { pending: [] }, + }); + const queue = new FullScanQueue(client, clock); + + await queue.enqueueInitial(['new']); + expect(await queue.isInitialScanPending('new')).toBe(true); + expect(await queue.due(['new'], 2)).toStrictEqual(['new']); + + await queue.markFailed('new', new Error('Failed to fetch')); + expect(await queue.isInitialScanPending('new')).toBe(true); + + await queue.markSucceeded('new'); + expect(await queue.isInitialScanPending('new')).toBe(false); + }); + + it('records initial accounts even before the queue exists, and due() then includes them', async () => { + const { client, state } = createStateClient(); + const queue = new FullScanQueue(client, clock); + + await queue.enqueueInitial(['new']); + + expect(state[FULL_SCAN_QUEUE_KEY]).toBeUndefined(); + expect(await queue.isInitialScanPending('new')).toBe(true); + expect(await queue.due(['old', 'new'], 5)).toStrictEqual(['old', 'new']); + }); + + it('does nothing when enqueueing or ensuring no account', async () => { + const { client } = createStateClient(); + const queue = new FullScanQueue(client, clock); + + await queue.enqueueInitial([]); + expect(await queue.ensureQueued([])).toStrictEqual([]); + expect(client.setState).not.toHaveBeenCalled(); + }); + + describe('ensureQueued', () => { + it('queues never-synchronized accounts and flags them as initial', async () => { + const { client, state } = createStateClient({ + [FULL_SCAN_QUEUE_KEY]: { pending: [] }, + }); + const queue = new FullScanQueue(client, clock); + + expect(await queue.ensureQueued(['legacy'])).toStrictEqual(['legacy']); + expect(state[FULL_SCAN_QUEUE_KEY]).toStrictEqual({ + pending: ['legacy'], + }); + expect(await queue.isInitialScanPending('legacy')).toBe(true); + }); + + it('keeps the position and backoff of an already-queued account', async () => { + const retryEntry = { attempts: 3, nextAttemptAt: now + 60_000 }; + const { client, state } = createStateClient({ + [FULL_SCAN_QUEUE_KEY]: { pending: ['legacy', 'other'] }, + [FULL_SCAN_RETRY_KEY]: { legacy: retryEntry }, + }); + const queue = new FullScanQueue(client, clock); + + expect(await queue.ensureQueued(['legacy'])).toStrictEqual([]); + expect(state[FULL_SCAN_QUEUE_KEY]).toStrictEqual({ + pending: ['legacy', 'other'], + }); + expect(state[FULL_SCAN_RETRY_KEY]).toStrictEqual({ + legacy: { ...retryEntry, initial: true }, + }); + expect(await queue.due(['legacy', 'other'], 2)).toStrictEqual(['other']); + }); + + it('does not write when nothing changes', async () => { + const { client } = createStateClient({ + [FULL_SCAN_QUEUE_KEY]: { pending: ['legacy'] }, + [FULL_SCAN_RETRY_KEY]: { + legacy: { attempts: 0, nextAttemptAt: 0, initial: true }, + }, + }); + const queue = new FullScanQueue(client, clock); + + await queue.ensureQueued(['legacy']); + + expect(client.setState).not.toHaveBeenCalled(); + }); + }); + + it('serializes concurrent operations so no update is lost', async () => { + const { client, state } = createStateClient({ + [FULL_SCAN_QUEUE_KEY]: { pending: ['a', 'b', 'c'] }, + }); + const queue = new FullScanQueue(client, clock); + + await Promise.all([ + queue.markSucceeded('a'), + queue.markSucceeded('b'), + queue.markFailed('c', new Error('x')), + ]); + + expect(state[FULL_SCAN_QUEUE_KEY]).toStrictEqual({ pending: ['c'] }); + }); + + it('ignores a corrupted stored state', async () => { + const { client } = createStateClient({ + [FULL_SCAN_QUEUE_KEY]: { pending: 'not-an-array' }, + [FULL_SCAN_RETRY_KEY]: 42, + }); + const queue = new FullScanQueue(client, clock); + + expect(await queue.due(['a'], 1)).toStrictEqual(['a']); + expect(await queue.isInitialScanPending('a')).toBe(false); + }); +}); diff --git a/packages/bitcoin-wallet-snap/src/store/FullScanQueue.ts b/packages/bitcoin-wallet-snap/src/store/FullScanQueue.ts new file mode 100644 index 000000000..3ae758f4c --- /dev/null +++ b/packages/bitcoin-wallet-snap/src/store/FullScanQueue.ts @@ -0,0 +1,302 @@ +import type { Json } from '@metamask/utils'; +import { + array, + boolean, + is, + number, + object, + optional, + record, + string, +} from 'superstruct'; + +import type { SnapClient } from '../entities'; + +/** + * State key of the full-scan queue. The name predates this class: it was the + * one-off "repair previously-unwatched accounts" list, whose shape + * (`{ pending: string[] }`) is kept so existing installs continue where they + * left off. + */ +export const FULL_SCAN_QUEUE_KEY = 'rescanV1'; + +/** + * State key of the per-account retry metadata of the full-scan queue. + */ +export const FULL_SCAN_RETRY_KEY = 'fullScanRetryV1'; + +/** First retry delay after a failed full scan. */ +export const FULL_SCAN_RETRY_BASE_MS = 30_000; + +/** + * Upper bound of the full scan retry delay. Kept short: an account whose + * first scan failed has an unknown balance until a scan succeeds, so after + * an outage it must be retried within minutes (on-demand scans triggered by + * balance requests and successful syncs usually recover it sooner). + */ +export const FULL_SCAN_RETRY_MAX_MS = 5 * 60_000; + +const QueueState = object({ + pending: array(string()), +}); + +const RetryEntryStruct = object({ + attempts: number(), + nextAttemptAt: number(), + initial: optional(boolean()), + lastError: optional(string()), +}); + +const RetryState = record(string(), RetryEntryStruct); + +export type FullScanRetryEntry = { + attempts: number; + nextAttemptAt: number; + /** + * Whether the account has never completed a full scan since it was created + * (as opposed to a legacy account queued for a repair scan). Its balance is + * unknown, not zero, until the scan succeeds. + */ + initial?: boolean; + lastError?: string; +}; + +/** + * Computes the delay before the next full scan attempt. + * + * @param attempts - Number of failed attempts so far (>= 1). + * @returns The delay in milliseconds. + */ +export function getFullScanRetryDelayMs(attempts: number): number { + const exponent = Math.max(0, attempts - 1); + return Math.min( + FULL_SCAN_RETRY_BASE_MS * 2 ** Math.min(exponent, 16), + FULL_SCAN_RETRY_MAX_MS, + ); +} + +/** + * Persistent queue of accounts that still need a full scan. + * + * - Accounts are processed in queue order, skipping those whose backoff has + * not elapsed, so one failing account never blocks the others (no + * head-of-line blocking). + * - A failed account is moved to the tail and retried with exponential + * backoff ({@link getFullScanRetryDelayMs}). + * - When the queue state does not exist yet, every existing account is + * queued (the original `rescanV1` repair behaviour). + * + * All operations are serialized so concurrent keyring requests and cron runs + * don't overwrite each other's state. + */ +export class FullScanQueue { + readonly #snapClient: SnapClient; + + readonly #now: () => number; + + #tail: Promise = Promise.resolve(); + + constructor(snapClient: SnapClient, now: () => number = Date.now) { + this.#snapClient = snapClient; + this.#now = now; + } + + async #serialize(task: () => Promise): Promise { + const run = this.#tail.then(task, task); + this.#tail = run.catch(() => undefined); + return run; + } + + async #readPending(): Promise { + const stored = await this.#snapClient.getState(FULL_SCAN_QUEUE_KEY); + return is(stored, QueueState) ? stored.pending : null; + } + + async #readRetry(): Promise> { + const stored = await this.#snapClient.getState(FULL_SCAN_RETRY_KEY); + return is(stored, RetryState) ? { ...stored } : {}; + } + + async #write( + pending: string[], + retry?: Record, + ): Promise { + await this.#snapClient.setState(FULL_SCAN_QUEUE_KEY, { pending }); + if (retry) { + await this.#snapClient.setState( + FULL_SCAN_RETRY_KEY, + retry as unknown as Json, + ); + } + } + + /** + * Queues newly created accounts for their first full scan. + * + * @param accountIds - The new account IDs. + */ + async enqueueInitial(accountIds: string[]): Promise { + if (accountIds.length === 0) { + return; + } + await this.#serialize(async () => { + const pending = await this.#readPending(); + const retry = await this.#readRetry(); + const now = this.#now(); + for (const id of accountIds) { + if (pending && !pending.includes(id)) { + pending.push(id); + } + retry[id] = { attempts: 0, nextAttemptAt: now, initial: true }; + } + if (pending) { + await this.#write(pending, retry); + } else { + // The queue doesn't exist yet: `due()` will initialize it with every + // live account, these included. + await this.#snapClient.setState( + FULL_SCAN_RETRY_KEY, + retry as unknown as Json, + ); + } + }); + } + + /** + * Queues accounts that never synchronized with the chain (e.g. created + * before this queue existed, whose first scan failed and was never + * retried). Accounts already queued keep their position and backoff. + * They are flagged as `initial`: their balance is unknown until the scan + * succeeds. + * + * @param accountIds - IDs of the never-synchronized accounts. + * @returns The IDs that were newly queued. + */ + async ensureQueued(accountIds: string[]): Promise { + if (accountIds.length === 0) { + return []; + } + return this.#serialize(async () => { + const pending = (await this.#readPending()) ?? []; + const retry = await this.#readRetry(); + const now = this.#now(); + const added: string[] = []; + let retryChanged = false; + for (const id of accountIds) { + if (!pending.includes(id)) { + pending.push(id); + added.push(id); + } + if (!retry[id]) { + retry[id] = { attempts: 0, nextAttemptAt: now, initial: true }; + retryChanged = true; + } else if (!retry[id].initial) { + retry[id] = { ...retry[id], initial: true }; + retryChanged = true; + } + } + if (added.length > 0 || retryChanged) { + await this.#write(pending, retryChanged ? retry : undefined); + } + return added; + }); + } + + /** + * @param accountId - The account ID. + * @returns Whether the account never completed its first full scan. + */ + async isInitialScanPending(accountId: string): Promise { + const retry = await this.#readRetry(); + return retry[accountId]?.initial === true; + } + + /** + * Returns the queued accounts that are due for a scan, pruning accounts + * that no longer exist. Initializes the queue with every live account when + * it doesn't exist yet. + * + * @param liveAccountIds - IDs of the accounts that currently exist. + * @param max - Maximum number of accounts to return. + * @returns The due account IDs, in queue order. + */ + async due(liveAccountIds: string[], max: number): Promise { + return this.#serialize(async () => { + const live = new Set(liveAccountIds); + const stored = await this.#readPending(); + const pending = stored + ? stored.filter((id) => live.has(id)) + : [...liveAccountIds]; + const retry = await this.#readRetry(); + const retryIds = Object.keys(retry); + const staleRetry = retryIds.filter((id) => !live.has(id)); + for (const id of staleRetry) { + delete retry[id]; + } + if (pending.length !== stored?.length || staleRetry.length > 0) { + await this.#write(pending, staleRetry.length > 0 ? retry : undefined); + } + + const now = this.#now(); + return pending + .filter((id) => (retry[id]?.nextAttemptAt ?? 0) <= now) + .slice(0, max); + }); + } + + /** + * Removes an account from the queue after a successful full scan. + * + * @param accountId - The account ID. + */ + async markSucceeded(accountId: string): Promise { + await this.#serialize(async () => { + const pending = await this.#readPending(); + const retry = await this.#readRetry(); + const hadRetry = Object.hasOwn(retry, accountId); + delete retry[accountId]; + if (pending === null) { + if (hadRetry) { + await this.#snapClient.setState( + FULL_SCAN_RETRY_KEY, + retry as unknown as Json, + ); + } + return; + } + await this.#write( + pending.filter((id) => id !== accountId), + hadRetry ? retry : undefined, + ); + }); + } + + /** + * Records a failed full scan: moves the account to the tail of the queue + * and schedules its next attempt with exponential backoff. + * + * @param accountId - The account ID. + * @param error - The failure. + */ + async markFailed(accountId: string, error: unknown): Promise { + await this.#serialize(async () => { + const pending = (await this.#readPending()) ?? []; + const retry = await this.#readRetry(); + const previous = retry[accountId]; + const attempts = (previous?.attempts ?? 0) + 1; + const entry: FullScanRetryEntry = { + attempts, + nextAttemptAt: this.#now() + getFullScanRetryDelayMs(attempts), + lastError: error instanceof Error ? error.message : String(error), + }; + if (previous?.initial) { + entry.initial = true; + } + retry[accountId] = entry; + await this.#write( + [...pending.filter((id) => id !== accountId), accountId], + retry, + ); + }); + } +} diff --git a/packages/bitcoin-wallet-snap/src/store/index.ts b/packages/bitcoin-wallet-snap/src/store/index.ts index c23e9e145..bb00c0f59 100644 --- a/packages/bitcoin-wallet-snap/src/store/index.ts +++ b/packages/bitcoin-wallet-snap/src/store/index.ts @@ -1,2 +1,3 @@ export * from './BdkAccountRepository'; export * from './JSXSendFlowRepository'; +export * from './FullScanQueue'; diff --git a/packages/bitcoin-wallet-snap/src/use-cases/AccountUseCases.test.ts b/packages/bitcoin-wallet-snap/src/use-cases/AccountUseCases.test.ts index 24911957a..0258b860b 100644 --- a/packages/bitcoin-wallet-snap/src/use-cases/AccountUseCases.test.ts +++ b/packages/bitcoin-wallet-snap/src/use-cases/AccountUseCases.test.ts @@ -43,6 +43,12 @@ import type { } from './AccountUseCases'; import { AccountUseCases, + DISCOVERY_SCAN_REUSE_TTL_MS, + FULL_SCAN_ON_DEMAND_MAX_INTERVAL_MS, + FULL_SCAN_ON_DEMAND_MIN_INTERVAL_MS, + FULL_SCAN_RECOVERY_MIN_INTERVAL_MS, + getOnDemandFullScanIntervalMs, + MAX_IMMEDIATE_FULL_SCANS_ON_CREATE, SENDER_LOOKUP_CONCURRENCY, SENDER_RESOLUTION_LIMIT, } from './AccountUseCases'; @@ -2711,4 +2717,937 @@ describe('AccountUseCases', () => { expect(JSON.stringify(result)).not.toContain(sensitiveError); }); }); + describe('balance-unknown tracking and the full scan queue', () => { + const buildStatefulUseCases = (): { + useCases: AccountUseCases; + snapClient: ReturnType>; + state: Record; + } => { + const state: Record = {}; + const snapClient = mock(); + snapClient.getState.mockImplementation(async (key?: string) => + key !== undefined && Object.hasOwn(state, key) + ? (JSON.parse(JSON.stringify(state[key])) as never) + : (null as never), + ); + snapClient.setState.mockImplementation(async (key, value) => { + state[key as string] = JSON.parse(JSON.stringify(value)); + }); + return { + useCases: new AccountUseCases( + mockLogger, + snapClient, + mockRepository, + mockConfirmationRepository, + mockChain, + fallbackFeeRate, + targetBlocksConfirmation, + undefined, + ), + snapClient, + state, + }; + }; + + const buildAccount = ( + id: string, + synchronized: boolean, + ): BitcoinAccount => { + const account = mock({ id, network: 'bitcoin' }); + account.hasSynchronized.mockReturnValue(synchronized); + account.listTransactions.mockReturnValue([]); + return account; + }; + + it('reports the balance of a never-synchronized account as unknown', async () => { + const { useCases: testUseCases } = buildStatefulUseCases(); + + expect( + await testUseCases.isBalanceUnknown(buildAccount('a', false)), + ).toBe(true); + expect(await testUseCases.isBalanceUnknown(buildAccount('b', true))).toBe( + false, + ); + }); + + it('keeps a new account unknown until its first full scan succeeds, and retries a failed scan', async () => { + const { useCases: testUseCases } = buildStatefulUseCases(); + const account = buildAccount('new-id', true); + mockRepository.getByDerivationPaths.mockResolvedValue({ + accounts: [null], + snapshot: { accounts: null, derivationPaths: null }, + }); + mockRepository.createMany.mockResolvedValue([account]); + mockRepository.update.mockResolvedValue(); + + await testUseCases.createMany([ + { + network: 'bitcoin', + entropySource: 'some-source', + index: 0, + addressType: 'p2wpkh', + synchronize: false, + }, + ]); + expect(await testUseCases.isBalanceUnknown(account)).toBe(true); + expect(await testUseCases.isInitialFullScanPending('new-id')).toBe(true); + + mockChain.fullScan.mockRejectedValueOnce(new Error('Failed to fetch')); + await expect(testUseCases.fullScan(account)).rejects.toThrow( + 'Failed to fetch', + ); + expect(await testUseCases.isBalanceUnknown(account)).toBe(true); + // Backoff: not due right after the failure. + expect(await testUseCases.listDueFullScans([account], 2)).toStrictEqual( + [], + ); + + mockChain.fullScan.mockResolvedValueOnce(); + await testUseCases.fullScan(account); + expect(await testUseCases.isBalanceUnknown(account)).toBe(false); + }); + + it('queues never-synchronized accounts and returns at most `max` due accounts', async () => { + const { useCases: testUseCases, state } = buildStatefulUseCases(); + // The queue already exists (e.g. the repair scan finished). + state.rescanV1 = { pending: [] }; + const synced = buildAccount('synced', true); + const legacy1 = buildAccount('legacy-1', false); + const legacy2 = buildAccount('legacy-2', false); + const legacy3 = buildAccount('legacy-3', false); + + const due = await testUseCases.listDueFullScans( + [synced, legacy1, legacy2, legacy3], + 2, + ); + + expect(due).toStrictEqual([legacy1, legacy2]); + expect(state.rescanV1).toStrictEqual({ + pending: ['legacy-1', 'legacy-2', 'legacy-3'], + }); + }); + }); + + describe('full scan coordination (single-flight, on-demand, recovery)', () => { + const NOW = 1_700_000_000_000; + let nowSpy: jest.SpyInstance; + + const buildUseCases = (): { + useCases: AccountUseCases; + snapClient: ReturnType>; + } => { + const state: Record = {}; + const snapClient = mock(); + snapClient.getState.mockImplementation(async (key?: string) => + key !== undefined && Object.hasOwn(state, key) + ? (JSON.parse(JSON.stringify(state[key])) as never) + : (null as never), + ); + snapClient.setState.mockImplementation(async (key, value) => { + state[key as string] = JSON.parse(JSON.stringify(value)); + }); + return { + useCases: new AccountUseCases( + mockLogger, + snapClient, + mockRepository, + mockConfirmationRepository, + mockChain, + fallbackFeeRate, + targetBlocksConfirmation, + undefined, + ), + snapClient, + }; + }; + + const buildAccount = (id: string, synchronized = true): BitcoinAccount => { + const account = mock({ id, network: 'bitcoin' }); + account.hasSynchronized.mockReturnValue(synchronized); + account.listTransactions.mockReturnValue([]); + return account; + }; + + const fullScanEvents = ( + snapClient: ReturnType>, + ): unknown[] => + snapClient.scheduleBackgroundEvent.mock.calls.filter( + ([event]) => event.method === CronMethod.FullScanAccount, + ); + + beforeEach(() => { + mockChain.fullScan.mockReset(); + mockChain.sync.mockReset(); + mockRepository.update.mockResolvedValue(); + nowSpy = jest.spyOn(Date, 'now').mockReturnValue(NOW); + }); + + afterEach(() => { + nowSpy.mockRestore(); + }); + + it('computes a bounded on-demand interval', () => { + expect(getOnDemandFullScanIntervalMs(0)).toBe( + FULL_SCAN_ON_DEMAND_MIN_INTERVAL_MS, + ); + expect(getOnDemandFullScanIntervalMs(1)).toBe(30_000); + expect(getOnDemandFullScanIntervalMs(2)).toBe(60_000); + expect(getOnDemandFullScanIntervalMs(3)).toBe( + FULL_SCAN_ON_DEMAND_MAX_INTERVAL_MS, + ); + expect(getOnDemandFullScanIntervalMs(50)).toBe( + FULL_SCAN_ON_DEMAND_MAX_INTERVAL_MS, + ); + expect(FULL_SCAN_ON_DEMAND_MAX_INTERVAL_MS).toBe(2 * 60_000); + }); + + it('runs one full scan per account at a time (single-flight)', async () => { + const { useCases: testUseCases } = buildUseCases(); + const account = buildAccount('a'); + let finishScan: () => void = () => undefined; + mockChain.fullScan.mockReturnValue( + new Promise((resolve) => { + finishScan = resolve; + }), + ); + + const first = testUseCases.fullScan(account); + const second = testUseCases.fullScan(account); + expect(testUseCases.isFullScanInFlight('a')).toBe(true); + finishScan(); + const [firstResult, secondResult] = await Promise.all([first, second]); + + expect(mockChain.fullScan).toHaveBeenCalledTimes(1); + expect(secondResult).toBe(firstResult); + expect(testUseCases.isFullScanInFlight('a')).toBe(false); + + // A later scan runs again. + mockChain.fullScan.mockResolvedValue(); + await testUseCases.fullScan(account); + expect(mockChain.fullScan).toHaveBeenCalledTimes(2); + }); + + it('makes a sync join an in-flight full scan instead of fetching concurrently', async () => { + const { useCases: testUseCases } = buildUseCases(); + const scanned = buildAccount('a'); + const stale = buildAccount('a'); + let finishScan: () => void = () => undefined; + mockChain.fullScan.mockReturnValue( + new Promise((resolve) => { + finishScan = resolve; + }), + ); + + const scan = testUseCases.fullScan(scanned); + const sync = testUseCases.synchronize(stale, 'cron'); + finishScan(); + await scan; + + expect(await sync).toStrictEqual({ + account: scanned, + transactionsToNotify: [], + }); + expect(mockChain.sync).not.toHaveBeenCalled(); + }); + + it('excludes in-flight and just-scanned accounts from the cron queue', async () => { + const { useCases: testUseCases } = buildUseCases(); + const account = buildAccount('a', false); + let finishScan: () => void = () => undefined; + mockChain.fullScan.mockReturnValue( + new Promise((resolve) => { + finishScan = resolve; + }), + ); + + const scan = testUseCases.fullScan(account); + expect(await testUseCases.listDueFullScans([account], 2)).toStrictEqual( + [], + ); + finishScan(); + await scan; + }); + + it('starts the first full scan of new accounts right away, even without `synchronize`', async () => { + const { useCases: testUseCases, snapClient } = buildUseCases(); + const accounts = ['n1', 'n2', 'n3'].map((id) => buildAccount(id)); + mockRepository.getByDerivationPaths.mockResolvedValue({ + accounts: [null, null, null], + snapshot: { accounts: null, derivationPaths: null }, + }); + mockRepository.createMany.mockResolvedValue(accounts); + mockRepository.insertMany.mockResolvedValue(accounts); + + await testUseCases.createMany( + [0, 1, 2].map((index) => ({ + network: 'bitcoin' as const, + entropySource: 'some-source', + index, + addressType: 'p2wpkh' as const, + synchronize: false, + })), + ); + + expect(fullScanEvents(snapClient)).toStrictEqual( + accounts.slice(0, MAX_IMMEDIATE_FULL_SCANS_ON_CREATE).map(({ id }) => [ + { + duration: 'PT1S', + method: CronMethod.FullScanAccount, + params: { accountId: id }, + }, + ]), + ); + // The scheduled accounts are not picked again by the cron queue. + const due = await testUseCases.listDueFullScans(accounts, 3); + expect(due.map(({ id }) => id)).toStrictEqual(['n3']); + }); + + it('deduplicates on-demand scan requests from balance requests', async () => { + const { useCases: testUseCases, snapClient } = buildUseCases(); + const account = buildAccount('a'); + + expect( + await testUseCases.requestFullScan(account, 'balance-request'), + ).toBe(true); + expect( + await testUseCases.requestFullScan(account, 'balance-request'), + ).toBe(false); + expect( + await testUseCases.requestFullScan(account, 'sync-succeeded'), + ).toBe(false); + expect(fullScanEvents(snapClient)).toHaveLength(1); + + nowSpy.mockReturnValue(NOW + FULL_SCAN_ON_DEMAND_MIN_INTERVAL_MS); + expect( + await testUseCases.requestFullScan(account, 'balance-request'), + ).toBe(true); + expect(fullScanEvents(snapClient)).toHaveLength(2); + }); + + it('does not request a scan while one is running', async () => { + const { useCases: testUseCases, snapClient } = buildUseCases(); + const account = buildAccount('a'); + let finishScan: () => void = () => undefined; + mockChain.fullScan.mockReturnValue( + new Promise((resolve) => { + finishScan = resolve; + }), + ); + + const scan = testUseCases.fullScan(account); + expect( + await testUseCases.requestFullScan(account, 'balance-request'), + ).toBe(false); + finishScan(); + await scan; + expect(fullScanEvents(snapClient)).toHaveLength(0); + }); + + it('backs off on-demand scans after failures, bounded to 2 minutes', async () => { + const { useCases: testUseCases, snapClient } = buildUseCases(); + const account = buildAccount('a'); + mockChain.fullScan.mockRejectedValue(new Error('Failed to fetch')); + + for (let i = 0; i < 4; i++) { + await expect(testUseCases.fullScan(account)).rejects.toThrow( + 'Failed to fetch', + ); + } + + nowSpy.mockReturnValue(NOW + FULL_SCAN_ON_DEMAND_MAX_INTERVAL_MS - 1); + expect( + await testUseCases.requestFullScan(account, 'balance-request'), + ).toBe(false); + nowSpy.mockReturnValue(NOW + FULL_SCAN_ON_DEMAND_MAX_INTERVAL_MS); + expect( + await testUseCases.requestFullScan(account, 'balance-request'), + ).toBe(true); + expect(fullScanEvents(snapClient)).toHaveLength(1); + }); + + it('runs the pending full scan promptly once syncs succeed again after an outage', async () => { + const { useCases: testUseCases, snapClient } = buildUseCases(); + const account = buildAccount('a'); + mockRepository.getByDerivationPaths.mockResolvedValue({ + accounts: [null], + snapshot: { accounts: null, derivationPaths: null }, + }); + mockRepository.createMany.mockResolvedValue([account]); + mockRepository.insertMany.mockResolvedValue([account]); + await testUseCases.createMany([ + { + network: 'bitcoin', + entropySource: 'some-source', + index: 0, + addressType: 'p2wpkh', + synchronize: false, + }, + ]); + snapClient.scheduleBackgroundEvent.mockClear(); + + // Outage: repeated scans fail (on-demand interval now 2 minutes) and + // the cron syncs fail too. + mockChain.fullScan.mockRejectedValue(new Error('Failed to fetch')); + for (let i = 0; i < 4; i++) { + await expect(testUseCases.fullScan(account)).rejects.toThrow( + 'Failed to fetch', + ); + } + mockChain.sync.mockRejectedValueOnce(new Error('Failed to fetch')); + await expect(testUseCases.synchronize(account, 'cron')).rejects.toThrow( + 'Failed to fetch', + ); + + // The network comes back: the next cron sync (30 s later) succeeds and + // runs the full scan right away, without waiting for the backoff. + nowSpy.mockReturnValue(NOW + 30_000); + mockChain.sync.mockResolvedValueOnce(); + await testUseCases.synchronize(account, 'cron'); + + expect(fullScanEvents(snapClient)).toStrictEqual([ + [ + { + duration: 'PT1S', + method: CronMethod.FullScanAccount, + params: { accountId: 'a' }, + }, + ], + ]); + }); + + it('lets a successful sync trigger the pending scan, bounded by the on-demand backoff', async () => { + const { useCases: testUseCases, snapClient } = buildUseCases(); + const account = buildAccount('a'); + mockRepository.getByDerivationPaths.mockResolvedValue({ + accounts: [null], + snapshot: { accounts: null, derivationPaths: null }, + }); + mockRepository.createMany.mockResolvedValue([account]); + mockRepository.insertMany.mockResolvedValue([account]); + await testUseCases.createMany([ + { + network: 'bitcoin', + entropySource: 'some-source', + index: 0, + addressType: 'p2wpkh', + synchronize: false, + }, + ]); + snapClient.scheduleBackgroundEvent.mockClear(); + mockChain.fullScan.mockRejectedValue(new Error('Failed to fetch')); + for (let i = 0; i < 3; i++) { + await expect(testUseCases.fullScan(account)).rejects.toThrow( + 'Failed to fetch', + ); + } + mockChain.sync.mockResolvedValue(); + + nowSpy.mockReturnValue(NOW + 60_000); + await testUseCases.synchronize(account, 'cron'); + expect(fullScanEvents(snapClient)).toHaveLength(0); + + nowSpy.mockReturnValue(NOW + FULL_SCAN_ON_DEMAND_MAX_INTERVAL_MS); + await testUseCases.synchronize(account, 'cron'); + expect(fullScanEvents(snapClient)).toHaveLength(1); + }); + + it('does not request scans from syncs of accounts with a known balance', async () => { + const { useCases: testUseCases, snapClient } = buildUseCases(); + const account = buildAccount('a'); + mockChain.sync.mockResolvedValue(); + + await testUseCases.synchronize(account, 'cron'); + + expect(fullScanEvents(snapClient)).toHaveLength(0); + }); + + it('treats a failed full scan joined by a sync as a failed sync, so the next successful sync recovers promptly (v3 item 6)', async () => { + const { useCases: testUseCases, snapClient } = buildUseCases(); + const account = buildAccount('a'); + mockRepository.getByDerivationPaths.mockResolvedValue({ + accounts: [null], + snapshot: { accounts: null, derivationPaths: null }, + }); + mockRepository.createMany.mockResolvedValue([account]); + mockRepository.insertMany.mockResolvedValue([account]); + await testUseCases.createMany([ + { + network: 'bitcoin', + entropySource: 'some-source', + index: 0, + addressType: 'p2wpkh', + synchronize: false, + }, + ]); + snapClient.scheduleBackgroundEvent.mockClear(); + + // Outage: scans keep failing (on-demand interval now 2 minutes). + mockChain.fullScan.mockRejectedValue(new Error('Failed to fetch')); + for (let i = 0; i < 3; i++) { + await expect(testUseCases.fullScan(account)).rejects.toThrow( + 'Failed to fetch', + ); + } + // The cron sync joins the next (failing) scan instead of syncing. + let failScan: (error: Error) => void = () => undefined; + mockChain.fullScan.mockReturnValueOnce( + new Promise((_resolve, reject) => { + failScan = reject; + }), + ); + const scan = testUseCases.fullScan(account); + const joinedSynchronization = testUseCases.synchronize(account, 'cron'); + failScan(new Error('Failed to fetch')); + await expect(scan).rejects.toThrow('Failed to fetch'); + await expect(joinedSynchronization).rejects.toThrow('Failed to fetch'); + expect(mockChain.sync).not.toHaveBeenCalled(); + + // The connection heals: the next cron sync (30 s later) succeeds and + // requests the full scan right away, not after the 2-minute backoff. + nowSpy.mockReturnValue(NOW + 30_000); + mockChain.sync.mockResolvedValueOnce(); + await testUseCases.synchronize(account, 'cron'); + + expect(fullScanEvents(snapClient)).toStrictEqual([ + [ + { + duration: 'PT1S', + method: CronMethod.FullScanAccount, + params: { accountId: 'a' }, + }, + ], + ]); + }); + + describe('first heal of a never-synchronized account (v7)', () => { + const createNeverSyncedAccount = async ( + testUseCases: AccountUseCases, + snapClient: ReturnType>, + ): Promise => { + const account = buildAccount('a', false); + mockRepository.getByDerivationPaths.mockResolvedValue({ + accounts: [null], + snapshot: { accounts: null, derivationPaths: null }, + }); + mockRepository.createMany.mockResolvedValue([account]); + mockRepository.insertMany.mockResolvedValue([account]); + await testUseCases.createMany([ + { + network: 'bitcoin', + entropySource: 'some-source', + index: 0, + addressType: 'p2wpkh', + synchronize: false, + }, + ]); + snapClient.scheduleBackgroundEvent.mockClear(); + return account; + }; + + const at = (offsetMs: number): void => { + nowSpy.mockReturnValue(NOW + offsetMs); + }; + + /** + * Seconds until the scheduled full scans run, from their ISO durations. + * + * @param snapClient - The snap client mock. + * @returns The delays in seconds, one per scheduled full scan. + */ + const scheduledScanDelays = ( + snapClient: ReturnType>, + ): number[] => + snapClient.scheduleBackgroundEvent.mock.calls + .filter(([event]) => event.method === CronMethod.FullScanAccount) + .map(([event]) => Number(/^PT(\d+)S$/u.exec(event.duration ?? '')?.[1])); + + it('requests the full scan within seconds when the first successful sync comes 25 s after a failed scan (A-v6 22:49:13)', async () => { + const { useCases: testUseCases, snapClient } = buildUseCases(); + const account = await createNeverSyncedAccount(testUseCases, snapClient); + mockChain.fullScan.mockRejectedValue(new Error('Failed to fetch')); + mockChain.sync.mockRejectedValue(new Error('Failed to fetch')); + + // Outage, with the A-v6 timeline (seconds after onboarding): the + // first scans fail at +0 and +2, on-demand retries at +31 and +92, + // and the 30 s cron syncs fail in between. + for (const [offset, kind] of [ + [0, 'scan'], + [2, 'scan'], + [26, 'sync'], + [31, 'scan'], + [56, 'sync'], + [86, 'sync'], + [92, 'scan'], + ] as const) { + at(offset * 1000); + const step = + kind === 'scan' + ? testUseCases.fullScan(account) + : testUseCases.synchronize(account, 'cron'); + await expect(step).rejects.toThrow('Failed to fetch'); + } + // The backoff has grown: a balance request (client retry) is now + // held back by the 2-minute on-demand interval. + at(116_000); + expect( + await testUseCases.requestFullScan(account, 'balance-request'), + ).toBe(false); + expect(fullScanEvents(snapClient)).toHaveLength(0); + + // Heal: the next cron sync, 25 s after the last failed scan, succeeds. + at(117_000); + mockChain.sync.mockResolvedValue(); + await testUseCases.synchronize(account, 'cron'); + + const delays = scheduledScanDelays(snapClient); + expect(delays).toHaveLength(1); + expect(delays[0]).toBeLessThanOrEqual(5); + expect(fullScanEvents(snapClient)).toStrictEqual([ + [ + { + duration: 'PT1S', + method: CronMethod.FullScanAccount, + params: { accountId: 'a' }, + }, + ], + ]); + + // No duplicate scan before the scheduled one runs: neither the cron + // queue, nor later syncs, nor balance requests add another. + at(118_000); + expect(await testUseCases.listDueFullScans([account], 2)).toStrictEqual( + [], + ); + await testUseCases.synchronize(account, 'cron'); + expect( + await testUseCases.requestFullScan(account, 'balance-request'), + ).toBe(false); + expect(fullScanEvents(snapClient)).toHaveLength(1); + + // The scheduled scan runs and succeeds: the balance is known and the + // following syncs request nothing more. + mockChain.fullScan.mockResolvedValue(); + await testUseCases.fullScan(account); + jest.mocked(account.hasSynchronized).mockReturnValue(true); + expect(await testUseCases.isBalanceUnknown(account)).toBe(false); + at(147_000); + await testUseCases.synchronize(account, 'cron'); + expect(fullScanEvents(snapClient)).toHaveLength(1); + expect(mockChain.fullScan).toHaveBeenCalledTimes(5); + }); + + it('defers, rather than drops, a recovery that arrives right after a failed scan', async () => { + const { useCases: testUseCases, snapClient } = buildUseCases(); + const account = await createNeverSyncedAccount(testUseCases, snapClient); + mockChain.fullScan.mockRejectedValue(new Error('Failed to fetch')); + mockChain.sync.mockRejectedValueOnce(new Error('Failed to fetch')); + for (let i = 0; i < 3; i++) { + await expect(testUseCases.fullScan(account)).rejects.toThrow( + 'Failed to fetch', + ); + } + await expect(testUseCases.synchronize(account, 'cron')).rejects.toThrow( + 'Failed to fetch', + ); + + // The sync succeeds 2 s after the last failed scan. + at(2_000); + mockChain.sync.mockResolvedValue(); + await testUseCases.synchronize(account, 'cron'); + + expect(scheduledScanDelays(snapClient)).toStrictEqual([ + (FULL_SCAN_RECOVERY_MIN_INTERVAL_MS - 2_000) / 1000, + ]); + // Still a single scan while it is pending. + at(4_000); + await testUseCases.synchronize(account, 'cron'); + expect(await testUseCases.listDueFullScans([account], 2)).toStrictEqual( + [], + ); + expect(fullScanEvents(snapClient)).toHaveLength(1); + }); + + it('keeps the on-demand backoff for successful syncs that are not a recovery', async () => { + const { useCases: testUseCases, snapClient } = buildUseCases(); + const account = await createNeverSyncedAccount(testUseCases, snapClient); + mockChain.fullScan.mockRejectedValue(new Error('Failed to fetch')); + for (let i = 0; i < 3; i++) { + await expect(testUseCases.fullScan(account)).rejects.toThrow( + 'Failed to fetch', + ); + } + // Syncs never failed (only the scans did): no recovery, so the + // 2-minute on-demand interval still applies. + mockChain.sync.mockResolvedValue(); + at(25_000); + await testUseCases.synchronize(account, 'cron'); + expect(fullScanEvents(snapClient)).toHaveLength(0); + }); + }); + }); + + describe('discovery scan reuse and the discovery frontier (v3 item 1)', () => { + const NOW = 1_700_000_000_000; + let nowSpy: jest.SpyInstance; + + const buildUseCases = (): { + useCases: AccountUseCases; + snapClient: ReturnType>; + } => { + const state: Record = {}; + const snapClient = mock(); + snapClient.getState.mockImplementation(async (key?: string) => + key !== undefined && Object.hasOwn(state, key) + ? (JSON.parse(JSON.stringify(state[key])) as never) + : (null as never), + ); + snapClient.setState.mockImplementation(async (key, value) => { + state[key as string] = JSON.parse(JSON.stringify(value)); + }); + return { + useCases: new AccountUseCases( + mockLogger, + snapClient, + mockRepository, + mockConfirmationRepository, + mockChain, + fallbackFeeRate, + targetBlocksConfirmation, + undefined, + ), + snapClient, + }; + }; + + const buildAccount = ( + id: string, + { synchronized = true, transactions = 0 } = {}, + ): BitcoinAccount => { + const account = mock({ id, network: 'bitcoin' }); + account.hasSynchronized.mockReturnValue(synchronized); + account.listTransactions.mockReturnValue( + Array.from({ length: transactions }, () => mock()), + ); + return account; + }; + + const request = (index: number): CreateAccountParams => ({ + network: 'bitcoin', + entropySource: 'some-source', + index, + addressType: 'p2wpkh', + synchronize: false, + }); + + const eventsOf = ( + snapClient: ReturnType>, + method: string, + ): unknown[] => + snapClient.scheduleBackgroundEvent.mock.calls + .filter(([event]) => event.method === method) + .map(([event]) => event); + + const discover = async ( + testUseCases: AccountUseCases, + account: BitcoinAccount, + index: number, + ): Promise => { + mockRepository.getByDerivationPath.mockResolvedValueOnce(null); + mockRepository.create.mockResolvedValueOnce(account); + mockChain.fullScan.mockResolvedValueOnce(); + await testUseCases.discover({ + network: 'bitcoin', + entropySource: 'some-source', + index, + addressType: 'p2wpkh', + }); + }; + + beforeEach(() => { + mockChain.fullScan.mockReset(); + mockChain.sync.mockReset(); + mockRepository.createMany.mockReset(); + mockRepository.insertMany.mockReset(); + mockRepository.update.mockResolvedValue(); + nowSpy = jest.spyOn(Date, 'now').mockReturnValue(NOW); + }); + + afterEach(() => { + nowSpy.mockRestore(); + }); + + it('reuses the account discovery just scanned as its first synchronization', async () => { + const { useCases: testUseCases, snapClient } = buildUseCases(); + const discovered = buildAccount('d1'); + await discover(testUseCases, discovered, 1); + mockRepository.getByDerivationPaths.mockResolvedValue({ + accounts: [null], + snapshot: { accounts: null, derivationPaths: null }, + }); + mockRepository.insertMany.mockResolvedValue([discovered]); + + const created = await testUseCases.createMany([request(1)]); + + expect(created).toStrictEqual([discovered]); + expect(mockRepository.createMany).not.toHaveBeenCalled(); + expect(mockRepository.insertMany).toHaveBeenCalledWith( + [discovered], + expect.anything(), + ); + expect(discovered.revealNextAddress).toHaveBeenCalled(); + // No second full scan: not scheduled, not queued. + expect(mockChain.fullScan).toHaveBeenCalledTimes(1); + expect(eventsOf(snapClient, CronMethod.FullScanAccount)).toHaveLength(0); + expect(await testUseCases.isInitialFullScanPending('d1')).toBe(false); + expect(await testUseCases.isBalanceUnknown(discovered)).toBe(false); + }); + + it('syncs (instead of full-scanning) new accounts above the lowest index discovery found unused', async () => { + const { useCases: testUseCases, snapClient } = buildUseCases(); + // Onboarding: discovery finds index 1 unused, then alignment creates + // groups 1..4 because other networks have activity there. + const discovered = buildAccount('d1'); + await discover(testUseCases, discovered, 1); + const fresh = ['a2', 'a3', 'a4'].map((id) => + buildAccount(id, { synchronized: false }), + ); + mockRepository.getByDerivationPaths.mockResolvedValue({ + accounts: [null, null, null, null], + snapshot: { accounts: null, derivationPaths: null }, + }); + mockRepository.createMany.mockResolvedValue(fresh); + mockRepository.insertMany.mockResolvedValue([discovered, ...fresh]); + + await testUseCases.createMany([1, 2, 3, 4].map(request)); + + expect(mockRepository.createMany).toHaveBeenCalledTimes(1); + expect( + mockRepository.createMany.mock.calls[0]?.[0].map( + ({ derivationPath }) => derivationPath[3], + ), + ).toStrictEqual(["2'", "3'", "4'"]); + expect(eventsOf(snapClient, CronMethod.FullScanAccount)).toHaveLength(0); + expect( + eventsOf(snapClient, CronMethod.SyncSelectedAccounts), + ).toStrictEqual([ + { + duration: 'PT1S', + method: CronMethod.SyncSelectedAccounts, + params: { accountIds: ['a2', 'a3', 'a4'] }, + }, + ]); + expect(await testUseCases.isInitialFullScanPending('a2')).toBe(false); + // Not picked by the cron full-scan queue either. + expect(await testUseCases.listDueFullScans(fresh, 5)).toStrictEqual([]); + // Discovery's own scan is the only full scan. + expect(mockChain.fullScan).toHaveBeenCalledTimes(1); + }); + + it('answers a balance request for a fresh account with a sync, and full-scans normally once it synced', async () => { + const { useCases: testUseCases, snapClient } = buildUseCases(); + await discover(testUseCases, buildAccount('d1'), 1); + const fresh = buildAccount('a2', { synchronized: false }); + mockRepository.getByDerivationPaths.mockResolvedValue({ + accounts: [null], + snapshot: { accounts: null, derivationPaths: null }, + }); + mockRepository.createMany.mockResolvedValue([fresh]); + mockRepository.insertMany.mockResolvedValue([fresh]); + await testUseCases.createMany([request(2)]); + snapClient.scheduleBackgroundEvent.mockClear(); + + expect(await testUseCases.requestFullScan(fresh, 'balance-request')).toBe( + true, + ); + expect(snapClient.scheduleBackgroundEvent).toHaveBeenCalledWith({ + duration: 'PT1S', + method: CronMethod.SyncSelectedAccounts, + params: { accountIds: ['a2'] }, + }); + + // After its first successful sync, it is a regular account again. + mockChain.sync.mockResolvedValueOnce(); + jest.mocked(fresh.hasSynchronized).mockReturnValue(true); + await testUseCases.synchronize(fresh, 'cron'); + snapClient.scheduleBackgroundEvent.mockClear(); + nowSpy.mockReturnValue(NOW + 60_000); + await testUseCases.requestFullScan(fresh, 'balance-request'); + expect(snapClient.scheduleBackgroundEvent).toHaveBeenCalledWith({ + duration: 'PT1S', + method: CronMethod.FullScanAccount, + params: { accountId: 'a2' }, + }); + }); + + it('still full-scans accounts at or below an index with activity', async () => { + const { useCases: testUseCases, snapClient } = buildUseCases(); + // Discovery found activity at index 1 (frontier not set by it). + await discover(testUseCases, buildAccount('d1', { transactions: 2 }), 1); + const next = buildAccount('a2', { synchronized: false }); + mockRepository.getByDerivationPaths.mockResolvedValue({ + accounts: [null], + snapshot: { accounts: null, derivationPaths: null }, + }); + mockRepository.createMany.mockResolvedValue([next]); + mockRepository.insertMany.mockResolvedValue([next]); + + await testUseCases.createMany([request(2)]); + + expect( + eventsOf(snapClient, CronMethod.SyncSelectedAccounts), + ).toHaveLength(0); + expect(eventsOf(snapClient, CronMethod.FullScanAccount)).toStrictEqual([ + { + duration: 'PT1S', + method: CronMethod.FullScanAccount, + params: { accountId: 'a2' }, + }, + ]); + }); + + it('requests a standard-gap scan for a reused discovery scan that found activity', async () => { + const { useCases: testUseCases, snapClient } = buildUseCases(); + const discovered = buildAccount('d1', { transactions: 3 }); + await discover(testUseCases, discovered, 1); + mockRepository.getByDerivationPaths.mockResolvedValue({ + accounts: [null], + snapshot: { accounts: null, derivationPaths: null }, + }); + + await testUseCases.createMany([request(1)]); + + expect(mockRepository.createMany).not.toHaveBeenCalled(); + expect(eventsOf(snapClient, CronMethod.FullScanAccount)).toStrictEqual([ + { + duration: 'PT1S', + method: CronMethod.FullScanAccount, + params: { accountId: 'd1' }, + }, + ]); + // Its balance is known from the discovery scan meanwhile. + expect(await testUseCases.isInitialFullScanPending('d1')).toBe(false); + }); + + it('does not reuse discovery results older than the reuse TTL', async () => { + const { useCases: testUseCases, snapClient } = buildUseCases(); + await discover(testUseCases, buildAccount('d1'), 1); + nowSpy.mockReturnValue(NOW + DISCOVERY_SCAN_REUSE_TTL_MS); + const derived = buildAccount('n1', { synchronized: false }); + const fresh = buildAccount('n2', { synchronized: false }); + mockRepository.getByDerivationPaths.mockResolvedValue({ + accounts: [null, null], + snapshot: { accounts: null, derivationPaths: null }, + }); + mockRepository.createMany.mockResolvedValue([derived, fresh]); + mockRepository.insertMany.mockResolvedValue([derived, fresh]); + + await testUseCases.createMany([request(1), request(2)]); + + expect(mockRepository.createMany.mock.calls[0]?.[0]).toHaveLength(2); + expect( + eventsOf(snapClient, CronMethod.SyncSelectedAccounts), + ).toHaveLength(0); + expect(eventsOf(snapClient, CronMethod.FullScanAccount)).toHaveLength(2); + }); + }); }); diff --git a/packages/bitcoin-wallet-snap/src/use-cases/AccountUseCases.ts b/packages/bitcoin-wallet-snap/src/use-cases/AccountUseCases.ts index b9771a4ab..2a347cc27 100644 --- a/packages/bitcoin-wallet-snap/src/use-cases/AccountUseCases.ts +++ b/packages/bitcoin-wallet-snap/src/use-cases/AccountUseCases.ts @@ -42,6 +42,7 @@ import { WalletError, } from '../entities'; import { CronMethod } from '../handlers/CronHandler'; +import { FullScanQueue } from '../store/FullScanQueue'; import { resolveFeeRate } from '../utils/fee-rate'; export type DiscoverAccountParams = { @@ -225,6 +226,109 @@ class SenderLookupLimiter { } } +/** + * Minimum interval between two on-demand full scans of the same account (and + * between a scheduled-but-not-started scan and a new request for it). + */ +export const FULL_SCAN_ON_DEMAND_MIN_INTERVAL_MS = 30_000; + +/** + * Upper bound of the on-demand full scan interval while scans keep failing: + * client retries and successful syncs can trigger a scan at most this often, + * so recovery after the network comes back is bounded by it. + */ +export const FULL_SCAN_ON_DEMAND_MAX_INTERVAL_MS = 2 * 60_000; + +/** + * Minimum time between a failed full scan and the scan requested by the + * network recovery that follows it (the first successful sync after failing + * syncs). The failed attempt says nothing about the network now that a sync + * reached it, so only this short floor applies (it keeps a scan that failed + * an instant ago from being retried back-to-back). A recovery that arrives + * inside the floor schedules the scan for when it elapses instead of being + * dropped: the recovery is seen only once, and dropping it left the account + * waiting for the on-demand backoff (up to 2 minutes). + */ +export const FULL_SCAN_RECOVERY_MIN_INTERVAL_MS = 5_000; + +/** + * Maximum number of new accounts per `createMany` call whose first full scan + * is started right away; the others are picked up by the cron queue. + */ +export const MAX_IMMEDIATE_FULL_SCANS_ON_CREATE = 2; + +/** + * How long the result of an account-discovery scan counts as the account's + * first synchronization: an account created for a derivation path discovery + * scanned less than this long ago reuses the scanned wallet instead of + * running a second full scan. + */ +export const DISCOVERY_SCAN_REUSE_TTL_MS = 10 * 60_000; + +/** + * Upper bound on the discovery results kept in memory for reuse. + */ +export const MAX_REUSABLE_DISCOVERY_SCANS = 64; + +type DiscoveryScan = { + account: BitcoinAccount; + scannedAt: number; +}; + +type DiscoveryFrontier = { + /** Lowest account index discovery found without on-chain activity. */ + index: number; + scannedAt: number; +}; + +/** + * @param req - The discovery or creation request. + * @returns The key identifying the request's wallet (entropy source, address + * type, network), shared by all its account indexes. + */ +function getDiscoveryFrontierKey(req: DiscoverAccountParams): string { + return `${req.network}|${req.addressType}|${req.entropySource}`; +} + +/** + * Why a full scan is requested outside the cron queue. + * - `created`: first scan of a newly created account, started immediately so + * the first balance is not delayed. + * - `balance-request`: the client asked for the balance of an account whose + * balance is unknown (initial fetch, scheduled or manual retry). + * - `sync-succeeded`: a sync of an account whose initial scan is pending + * succeeded, so the network works again. + * - `network-recovered`: same, right after failing syncs (the network just + * came back): only the minimum interval applies. + */ +export type FullScanRequestReason = + | 'created' + | 'balance-request' + | 'sync-succeeded' + | 'network-recovered'; + +type FullScanAttemptInfo = { + lastAttemptAt: number; + consecutiveFailures: number; +}; + +/** + * @param consecutiveFailures - Consecutive failed full scans of the account. + * @returns The minimum interval before the next on-demand full scan. + */ +export function getOnDemandFullScanIntervalMs( + consecutiveFailures: number, +): number { + if (consecutiveFailures <= 0) { + return FULL_SCAN_ON_DEMAND_MIN_INTERVAL_MS; + } + return Math.min( + FULL_SCAN_ON_DEMAND_MIN_INTERVAL_MS * + 2 ** Math.min(consecutiveFailures - 1, 16), + FULL_SCAN_ON_DEMAND_MAX_INTERVAL_MS, + ); +} + export class AccountUseCases { readonly #logger: Logger; @@ -246,6 +350,44 @@ export class AccountUseCases { #accountMutationQueue: Promise = Promise.resolve(); + readonly #fullScanQueue: FullScanQueue; + + /** + * In-flight full scans per account (single-flight): concurrent requests + * (cron queue, scheduled scan, balance requests, syncs) join the running + * scan instead of starting another one. + */ + readonly #fullScansInFlight = new Map>(); + + /** Last full scan attempt and consecutive failures per account. */ + readonly #fullScanAttempts = new Map(); + + /** When an on-demand full scan was scheduled and has not started yet. */ + readonly #fullScanRequestedAt = new Map(); + + /** Accounts whose last sync failed (used to detect network recovery). */ + readonly #failingSyncs = new Set(); + + /** + * Recent discovery scans per network and derivation path: discovery already + * ran a full scan of these wallets, so creating the account reuses them + * (counting as its first synchronization) instead of scanning again. + */ + readonly #discoveryScans = new Map(); + + /** + * Per wallet, the lowest account index discovery found without activity. + * Like BIP-44 account discovery, accounts above it are treated as unused: + * creating them needs a regular sync, not a full address-gap scan. + */ + readonly #discoveryFrontiers = new Map(); + + /** + * New accounts that only need a regular sync to know their balance (see + * {@link AccountUseCases.#discoveryFrontiers}), until that sync succeeds. + */ + readonly #syncOnlyAccounts = new Set(); + constructor( logger: Logger, snapClient: SnapClient, @@ -264,6 +406,198 @@ export class AccountUseCases { this.#fallbackFeeRate = fallbackFeeRate; this.#targetBlocksConfirmation = targetBlocksConfirmation; this.#metaProtocols = metaProtocols; + this.#fullScanQueue = new FullScanQueue(snapClient); + } + + /** + * @param id - The account ID. + * @returns Whether the account never completed its first full scan, in + * which case its balance is unknown rather than zero. + */ + async isInitialFullScanPending(id: string): Promise { + return this.#fullScanQueue.isInitialScanPending(id); + } + + /** + * Whether the account's balance is unknown rather than zero: the wallet + * never synchronized with the chain, or its first full scan has not + * succeeded yet (only the revealed addresses may have been synced, so the + * balance can be partial). + * + * @param account - The account. + * @returns `true` if the balance must not be reported as known. + */ + async isBalanceUnknown(account: BitcoinAccount): Promise { + if (!account.hasSynchronized()) { + return true; + } + return this.#fullScanQueue.isInitialScanPending(account.id); + } + + /** + * Returns the accounts due for a (re)tried full scan, in queue order. + * + * @param accounts - The live accounts. + * @param max - Maximum number of accounts to return. + * @returns The due accounts. + */ + async listDueFullScans( + accounts: BitcoinAccount[], + max: number, + ): Promise { + // Accounts that never synchronized at all (genesis checkpoint) are + // (re)queued: before the queue existed, a failed first scan was never + // retried and such accounts kept reporting a zero balance forever. + // Fresh accounts beyond the discovery frontier only wait for a regular + // sync (scheduled at creation), not a full scan. + const neverSynchronized = accounts + .filter( + (account) => + !account.hasSynchronized() && !this.#syncOnlyAccounts.has(account.id), + ) + .map(({ id }) => id); + await this.#fullScanQueue.ensureQueued(neverSynchronized); + + const dueIds = await this.#fullScanQueue.due( + accounts.map(({ id }) => id), + max, + ); + const byId = new Map(accounts.map((account) => [account.id, account])); + // Sync-only accounts are skipped even when queued (e.g. by the one-off + // repair scan of every live account): discovery already covered them. + return dueIds + .filter( + (id) => + !this.#syncOnlyAccounts.has(id) && !this.#isFullScanBusyOrRecent(id), + ) + .map((id) => byId.get(id)) + .filter((account): account is BitcoinAccount => account !== undefined); + } + + /** + * @param accountId - The account ID. + * @returns Whether a full scan of the account is running. + */ + isFullScanInFlight(accountId: string): boolean { + return this.#fullScansInFlight.has(accountId); + } + + /** + * Whether a full scan of the account is running, scheduled to start, or + * was attempted less than {@link FULL_SCAN_ON_DEMAND_MIN_INTERVAL_MS} ago: + * the cron queue skips such accounts so scans never overlap or repeat + * back-to-back. + * + * @param accountId - The account ID. + * @returns `true` if the queue must not scan the account now. + */ + #isFullScanBusyOrRecent(accountId: string): boolean { + if (this.#fullScansInFlight.has(accountId)) { + return true; + } + const now = Date.now(); + const requestedAt = this.#fullScanRequestedAt.get(accountId); + if ( + requestedAt !== undefined && + now - requestedAt < FULL_SCAN_ON_DEMAND_MIN_INTERVAL_MS + ) { + return true; + } + const attempt = this.#fullScanAttempts.get(accountId); + return ( + attempt !== undefined && + now - attempt.lastAttemptAt < FULL_SCAN_ON_DEMAND_MIN_INTERVAL_MS + ); + } + + /** + * Requests a full scan outside the cron queue by scheduling a + * `FullScanAccount` background event, deduplicated per account: + * - never while a scan of the account is running or already scheduled; + * - for `balance-request` and `sync-succeeded`, at most once per + * {@link getOnDemandFullScanIntervalMs} (30 s, doubling per consecutive + * failure up to {@link FULL_SCAN_ON_DEMAND_MAX_INTERVAL_MS}), bypassing the + * longer persistent queue backoff; + * - for `network-recovered`, never dropped because of a previous attempt: + * after a failed scan it runs once {@link FULL_SCAN_RECOVERY_MIN_INTERVAL_MS} + * has elapsed since that attempt (after a successful one, + * {@link FULL_SCAN_ON_DEMAND_MIN_INTERVAL_MS}); a request inside that + * window is scheduled for when it ends. + * + * Never throws. + * + * @param account - The account. + * @param reason - Why the scan is requested. + * @returns Whether a scan was scheduled. + */ + async requestFullScan( + account: BitcoinAccount, + reason: FullScanRequestReason, + ): Promise { + const { id } = account; + if (this.#fullScansInFlight.has(id)) { + return false; + } + const now = Date.now(); + let delayMs = 0; + if (reason !== 'created') { + const requestedAt = this.#fullScanRequestedAt.get(id); + if ( + requestedAt !== undefined && + now - requestedAt < FULL_SCAN_ON_DEMAND_MIN_INTERVAL_MS + ) { + // A scan is already scheduled and has not started yet. + return false; + } + const attempt = this.#fullScanAttempts.get(id); + if (attempt) { + const elapsed = now - attempt.lastAttemptAt; + if (reason === 'network-recovered') { + const interval = + attempt.consecutiveFailures > 0 + ? FULL_SCAN_RECOVERY_MIN_INTERVAL_MS + : FULL_SCAN_ON_DEMAND_MIN_INTERVAL_MS; + delayMs = Math.max(0, interval - elapsed); + } else if ( + elapsed < getOnDemandFullScanIntervalMs(attempt.consecutiveFailures) + ) { + return false; + } + } + } + + // Marks the scan as scheduled until it starts (`#runFullScan` clears it), + // so the cron queue and other requests don't add a second one. + this.#fullScanRequestedAt.set(id, now + delayMs); + const duration = `PT${Math.max(1, Math.ceil(delayMs / 1000))}S`; + try { + // An account known to be fresh (beyond the discovery frontier) only + // needs a regular sync to know its balance. + await this.#snapClient.scheduleBackgroundEvent( + this.#syncOnlyAccounts.has(id) + ? { + duration, + method: CronMethod.SyncSelectedAccounts, + params: { accountIds: [id] }, + } + : { + duration, + method: CronMethod.FullScanAccount, + params: { accountId: id }, + }, + ); + this.#logger.debug( + 'Full scan requested (%s, in %ss): %s', + reason, + duration.slice(2, -1), + id, + ); + return true; + } catch (error) { + this.#fullScanRequestedAt.delete(id); + this.#logger.warn('Failed to schedule a full scan', error); + return false; + } } async list(): Promise { @@ -328,6 +662,12 @@ export class AccountUseCases { // has any previous activity since later on we filter out // accounts with no tx history await this.#chain.fullScan(newAccount, 'discovery'); + try { + this.#rememberDiscoveryScan(req, derivationPath, newAccount); + } catch (error) { + // Only an optimization: never fail discovery because of it. + this.#logger.warn('Failed to remember the discovery scan', error); + } this.#logger.info( 'Bitcoin account discovered successfully. Request: %o', @@ -336,6 +676,97 @@ export class AccountUseCases { return newAccount; } + /** + * Keeps a discovery scan so that creating the same account shortly after + * (account alignment right after discovery) counts it as the account's + * first synchronization, and records the discovery frontier. + * + * @param req - The discovery request. + * @param derivationPath - The scanned account's derivation path. + * @param account - The scanned, not persisted, account. + */ + #rememberDiscoveryScan( + req: DiscoverAccountParams, + derivationPath: string[], + account: BitcoinAccount, + ): void { + const now = Date.now(); + this.#pruneDiscoveryScans(now); + + const key = `${req.network}|${getDerivationPathKey(derivationPath)}`; + this.#discoveryScans.delete(key); + this.#discoveryScans.set(key, { account, scannedAt: now }); + while (this.#discoveryScans.size > MAX_REUSABLE_DISCOVERY_SCANS) { + const oldest = this.#discoveryScans.keys().next().value as string; + this.#discoveryScans.delete(oldest); + } + + const transactions = account.listTransactions() ?? []; + if (transactions.length === 0) { + const frontierKey = getDiscoveryFrontierKey(req); + const frontier = this.#discoveryFrontiers.get(frontierKey); + if (!frontier || req.index <= frontier.index) { + this.#discoveryFrontiers.set(frontierKey, { + index: req.index, + scannedAt: now, + }); + } + } + } + + #pruneDiscoveryScans(now: number): void { + for (const [key, { scannedAt }] of this.#discoveryScans) { + if (now - scannedAt >= DISCOVERY_SCAN_REUSE_TTL_MS) { + this.#discoveryScans.delete(key); + } + } + for (const [key, { scannedAt }] of this.#discoveryFrontiers) { + if (now - scannedAt >= DISCOVERY_SCAN_REUSE_TTL_MS) { + this.#discoveryFrontiers.delete(key); + } + } + } + + /** + * Takes the recent discovery scan of an account to create, if any. + * + * @param req - The creation request. + * @param pathKey - The account's derivation path key. + * @returns The scanned account, or `undefined`. + */ + #takeDiscoveryScan( + req: CreateAccountParams, + pathKey: string, + ): BitcoinAccount | undefined { + const key = `${req.network}|${pathKey}`; + const scan = this.#discoveryScans.get(key); + if (!scan) { + return undefined; + } + this.#discoveryScans.delete(key); + if ( + Date.now() - scan.scannedAt >= DISCOVERY_SCAN_REUSE_TTL_MS || + scan.account.network !== req.network + ) { + return undefined; + } + return scan.account; + } + + /** + * @param req - The creation request. + * @returns Whether discovery recently found a lower account index of the + * same wallet without activity, so this new account is unused. + */ + #isBeyondDiscoveryFrontier(req: CreateAccountParams): boolean { + const frontier = this.#discoveryFrontiers.get(getDiscoveryFrontierKey(req)); + return ( + frontier !== undefined && + Date.now() - frontier.scannedAt < DISCOVERY_SCAN_REUSE_TTL_MS && + req.index > frontier.index + ); + } + async createMany(reqs: CreateAccountParams[]): Promise { if (reqs.length === 0) { return []; @@ -343,134 +774,223 @@ export class AccountUseCases { const startMs = Date.now(); - const { accounts, createdAccountKeys } = await this.#runAccountMutation( - async () => { - const lookupStartMs = Date.now(); - const entries = reqs.map((req, index) => { - const derivationPath = getAccountDerivationPath(req); - return { - req, - index, - derivationPath, - pathKey: getDerivationPathKey(derivationPath), - }; - }); + const { + accounts, + createdAccountKeys, + discoveredAccountKeys, + syncOnlyAccountKeys, + } = await this.#runAccountMutation(async () => { + const lookupStartMs = Date.now(); + const entries = reqs.map((req, index) => { + const derivationPath = getAccountDerivationPath(req); + return { + req, + index, + derivationPath, + pathKey: getDerivationPathKey(derivationPath), + }; + }); - const uniqueEntriesByPath = new Map(); - for (const entry of entries) { - if (!uniqueEntriesByPath.has(entry.pathKey)) { - uniqueEntriesByPath.set(entry.pathKey, entry); - } + const uniqueEntriesByPath = new Map(); + for (const entry of entries) { + if (!uniqueEntriesByPath.has(entry.pathKey)) { + uniqueEntriesByPath.set(entry.pathKey, entry); } - const uniqueEntries = [...uniqueEntriesByPath.values()]; - - const { accounts: existingAccounts, snapshot } = - await this.#repository.getByDerivationPaths( - uniqueEntries.map(({ derivationPath }) => derivationPath), - ); - const existingAccountsByPath = new Map(); - - uniqueEntries.forEach((entry, index) => { - const account = existingAccounts[index]; - if (account?.network === entry.req.network) { - existingAccountsByPath.set(entry.pathKey, account); - } - }); + } + const uniqueEntries = [...uniqueEntriesByPath.values()]; - const entriesToCreate = uniqueEntries.filter( - ({ pathKey }) => !existingAccountsByPath.has(pathKey), + const { accounts: existingAccounts, snapshot } = + await this.#repository.getByDerivationPaths( + uniqueEntries.map(({ derivationPath }) => derivationPath), ); - const lookupMs = Date.now() - lookupStartMs; - - // Batch-create so entropy is fetched once per parent path instead of - // once per account; remaining per-account work is local derivation - // plus synchronous WASM wallet construction, so no throttling needed. - const deriveStartMs = Date.now(); - const newAccounts = - entriesToCreate.length > 0 - ? await this.#repository.createMany( - entriesToCreate.map(({ derivationPath, req }) => ({ - derivationPath, - network: req.network, - addressType: req.addressType, - })), - ) - : []; - - for (const newAccount of newAccounts) { - newAccount.revealNextAddress(); + const existingAccountsByPath = new Map(); + + uniqueEntries.forEach((entry, index) => { + const account = existingAccounts[index]; + if (account?.network === entry.req.network) { + existingAccountsByPath.set(entry.pathKey, account); } - const deriveMs = Date.now() - deriveStartMs; - - const persistStartMs = Date.now(); - if (newAccounts.length > 0) { - // Reuse the lookup's derivation-path snapshot: account lifecycle - // mutations are serialized, while `insertMany` refreshes the - // accounts map to preserve concurrent sync updates. - await this.#repository.insertMany(newAccounts, snapshot); + }); + + const entriesToCreate = uniqueEntries.filter( + ({ pathKey }) => !existingAccountsByPath.has(pathKey), + ); + const lookupMs = Date.now() - lookupStartMs; + + // Batch-create so entropy is fetched once per parent path instead of + // once per account; remaining per-account work is local derivation + // plus synchronous WASM wallet construction, so no throttling needed. + const deriveStartMs = Date.now(); + this.#pruneDiscoveryScans(Date.now()); + // Accounts discovery just scanned reuse that scan as their first + // synchronization instead of deriving and scanning them again. + const discoveredByPath = new Map(); + for (const { req, pathKey } of entriesToCreate) { + const discovered = this.#takeDiscoveryScan(req, pathKey); + if (discovered) { + discoveredByPath.set(pathKey, discovered); } - const persistMs = Date.now() - persistStartMs; - - // Stringified so the values survive in the console after the snap's - // execution environment is torn down. - this.#logger.info( - `[createMany] Phase timings ${JSON.stringify({ - requested: reqs.length, - created: newAccounts.length, - lookupMs, - deriveMs, - persistMs, - totalMs: Date.now() - startMs, - })}`, - ); + } + const entriesToDerive = entriesToCreate.filter( + ({ pathKey }) => !discoveredByPath.has(pathKey), + ); + const derivedAccounts = + entriesToDerive.length > 0 + ? await this.#repository.createMany( + entriesToDerive.map(({ derivationPath, req }) => ({ + derivationPath, + network: req.network, + addressType: req.addressType, + })), + ) + : []; + const derivedByPath = new Map( + entriesToDerive.map((entry, index) => [ + entry.pathKey, + derivedAccounts[index] as BitcoinAccount, + ]), + ); + const newAccounts = entriesToCreate.map( + ({ pathKey }) => + (discoveredByPath.get(pathKey) ?? + derivedByPath.get(pathKey)) as BitcoinAccount, + ); - const newAccountsByPath = new Map( - entriesToCreate.map((entry, index) => [ - entry.pathKey, - newAccounts[index] as BitcoinAccount, - ]), - ); + for (const newAccount of newAccounts) { + newAccount.revealNextAddress(); + } + const deriveMs = Date.now() - deriveStartMs; + + const persistStartMs = Date.now(); + if (newAccounts.length > 0) { + // Reuse the lookup's derivation-path snapshot: account lifecycle + // mutations are serialized, while `insertMany` refreshes the + // accounts map to preserve concurrent sync updates. + await this.#repository.insertMany(newAccounts, snapshot); + } + const persistMs = Date.now() - persistStartMs; - const accountsInOrder = entries.map((entry) => { - const account = - existingAccountsByPath.get(entry.pathKey) ?? - newAccountsByPath.get(entry.pathKey); + // Stringified so the values survive in the console after the snap's + // execution environment is torn down. + this.#logger.info( + `[createMany] Phase timings ${JSON.stringify({ + requested: reqs.length, + created: newAccounts.length, + lookupMs, + deriveMs, + persistMs, + totalMs: Date.now() - startMs, + })}`, + ); - if (!account) { - throw new AssertionError('Failed to create account', { - index: entry.index, - derivationPath: entry.derivationPath, - }); - } + const newAccountsByPath = new Map( + entriesToCreate.map((entry, index) => [ + entry.pathKey, + newAccounts[index] as BitcoinAccount, + ]), + ); - return account; - }); + const accountsInOrder = entries.map((entry) => { + const account = + existingAccountsByPath.get(entry.pathKey) ?? + newAccountsByPath.get(entry.pathKey); - return { - accounts: accountsInOrder, - createdAccountKeys: new Set(newAccountsByPath.keys()), - }; - }, + if (!account) { + throw new AssertionError('Failed to create account', { + index: entry.index, + derivationPath: entry.derivationPath, + }); + } + + return account; + }); + + return { + accounts: accountsInOrder, + createdAccountKeys: new Set(newAccountsByPath.keys()), + discoveredAccountKeys: new Set(discoveredByPath.keys()), + syncOnlyAccountKeys: new Set( + entriesToDerive + .filter(({ req }) => this.#isBeyondDiscoveryFrontier(req)) + .map(({ pathKey }) => pathKey), + ), + }; + }); + + // Every new account needs one first synchronization covering its whole + // address gap; until it succeeds its balance is reported as unknown: + // - accounts discovery just scanned already had it (reused scan); + // - accounts above the lowest index discovery found unused are fresh + // (BIP-44 account discovery), so a regular sync is enough; + // - any other account needs a full scan: syncs only watch revealed + // addresses. They are all queued (also those scheduled below, in case + // the background event fails) so the cron retries with backoff. + const createdKeyByIndex = reqs.map((req) => + getDerivationPathKey(getAccountDerivationPath(req)), ); + const createdAccountsByKind = ( + predicate: (pathKey: string) => boolean, + ): BitcoinAccount[] => [ + ...new Map( + accounts + .filter((_account, index) => { + const pathKey = createdKeyByIndex[index] as string; + return createdAccountKeys.has(pathKey) && predicate(pathKey); + }) + .map((account) => [account.id, account]), + ).values(), + ]; + const syncOnlyAccounts = createdAccountsByKind((pathKey) => + syncOnlyAccountKeys.has(pathKey), + ); + const createdAccounts = createdAccountsByKind( + (pathKey) => + !discoveredAccountKeys.has(pathKey) && + !syncOnlyAccountKeys.has(pathKey), + ); + const createdAccountIds = createdAccounts.map(({ id }) => id); + try { + await this.#fullScanQueue.enqueueInitial(createdAccountIds); + } catch (error) { + this.#logger.warn('Failed to queue the initial full scan', error); + } - const scheduledAccountIds = new Set(); - for (const [index, account] of accounts.entries()) { - const req = reqs[index] as CreateAccountParams; - const pathKey = getDerivationPathKey(getAccountDerivationPath(req)); - if ( - req.synchronize && - createdAccountKeys.has(pathKey) && - !scheduledAccountIds.has(account.id) - ) { - scheduledAccountIds.add(account.id); + if (syncOnlyAccounts.length > 0) { + const syncOnlyIds = syncOnlyAccounts.map(({ id }) => id); + for (const id of syncOnlyIds) { + this.#syncOnlyAccounts.add(id); + } + try { await this.#snapClient.scheduleBackgroundEvent({ duration: 'PT1S', - method: CronMethod.FullScanAccount, - params: { accountId: account.id }, + method: CronMethod.SyncSelectedAccounts, + params: { accountIds: syncOnlyIds }, }); + } catch (error) { + this.#logger.warn('Failed to schedule the first sync', error); } } + // Start the first full scan right away (deduplicated, single-flight) so + // the first balance is not delayed until the next cron run; the queue + // above is the fallback if it fails or the background event is lost. + for (const account of createdAccounts.slice( + 0, + MAX_IMMEDIATE_FULL_SCANS_ON_CREATE, + )) { + await this.requestFullScan(account, 'created'); + } + + // A reused discovery scan only covered the discovery gap: an account where + // it found activity gets the standard-gap scan (its balance is already + // known, so it is not queued as an initial scan). + const discoveredWithActivity = createdAccountsByKind((pathKey) => + discoveredAccountKeys.has(pathKey), + ).filter((account) => (account.listTransactions() ?? []).length > 0); + for (const account of discoveredWithActivity) { + await this.requestFullScan(account, 'created'); + } + return accounts; } @@ -480,8 +1000,36 @@ export class AccountUseCases { ): Promise { this.#logger.debug('Synchronizing account: %s', account.id); + // A running full scan covers this sync: join it instead of fetching the + // same addresses concurrently. Its result carries the scanned account; + // its transactions are notified by the scan's own caller. + const inFlightScan = this.#fullScansInFlight.get(account.id); + if (inFlightScan) { + this.#logger.debug( + 'Full scan in progress, joining it instead of syncing: %s', + account.id, + ); + try { + const scanned = await inFlightScan; + return { account: scanned.account, transactionsToNotify: [] }; + } catch (error) { + // The joined scan failed: this sync failed too. Record it, so that + // the next successful sync is seen as a network recovery and runs the + // pending full scan promptly instead of waiting for its backoff. + this.#failingSyncs.add(account.id); + throw error; + } + } + const txsBeforeSync = account.listTransactions(); - await this.#chain.sync(account); + try { + await this.#chain.sync(account); + } catch (error) { + this.#failingSyncs.add(account.id); + throw error; + } + const networkRecovered = this.#failingSyncs.delete(account.id); + this.#syncOnlyAccounts.delete(account.id); const txsAfterSync = account.listTransactions(); // If new transactions appeared, fetch inscriptions; otherwise, just update. @@ -557,6 +1105,16 @@ export class AccountUseCases { this.#logger.debug('Account synchronized successfully: %s', account.id); + // The sync reached the chain, so the network works: if the first full + // scan is still pending (e.g. it failed while the network was down and is + // now backing off), run it promptly instead of waiting for the backoff. + if (await this.isBalanceUnknown(account)) { + await this.requestFullScan( + account, + networkRecovered ? 'network-recovered' : 'sync-succeeded', + ); + } + return { account, transactionsToNotify: txsToNotify, @@ -568,15 +1126,62 @@ export class AccountUseCases { }; } + /** + * Performs a full scan of the account. Single-flight per account: a call + * made while a scan of the same account is running returns that scan's + * result instead of starting a concurrent one. + * + * @param account - The account. + * @returns The scan result. + */ async fullScan(account: BitcoinAccount): Promise { + const inFlight = this.#fullScansInFlight.get(account.id); + if (inFlight) { + this.#logger.debug('Joining in-flight full scan: %s', account.id); + return inFlight; + } + + const run = this.#runFullScan(account).finally(() => { + this.#fullScansInFlight.delete(account.id); + }); + this.#fullScansInFlight.set(account.id, run); + return run; + } + + async #runFullScan(account: BitcoinAccount): Promise { this.#logger.debug('Performing initial full scan: %s', account.id); - await this.#chain.fullScan(account); + this.#fullScanRequestedAt.delete(account.id); + const previousAttempt = this.#fullScanAttempts.get(account.id); + const attempt: FullScanAttemptInfo = { + lastAttemptAt: Date.now(), + consecutiveFailures: previousAttempt?.consecutiveFailures ?? 0, + }; + this.#fullScanAttempts.set(account.id, attempt); + + try { + await this.#chain.fullScan(account); + } catch (error) { + attempt.consecutiveFailures += 1; + await this.#fullScanQueue + .markFailed(account.id, error) + .catch((stateError: unknown) => + this.#logger.warn('Failed to record full scan failure', stateError), + ); + throw error; + } const inscriptions = this.#metaProtocols ? await this.#metaProtocols.fetchInscriptions(account) : []; await this.#repository.update(account, inscriptions); + attempt.consecutiveFailures = 0; + this.#failingSyncs.delete(account.id); + await this.#fullScanQueue + .markSucceeded(account.id) + .catch((stateError: unknown) => + this.#logger.warn('Failed to record full scan success', stateError), + ); const transactionsToNotify = account.listTransactions(); @@ -599,6 +1204,7 @@ export class AccountUseCases { async delete(id: string): Promise { this.#logger.debug('Deleting account: %s', id); + this.#syncOnlyAccounts.delete(id); await this.#runAccountMutation(async () => { const account = await this.#repository.get(id); if (!account) {