diff --git a/Cargo.lock b/Cargo.lock index 31d35d477..cbe84b457 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1076,6 +1076,7 @@ dependencies = [ "solana-transaction", "solana-transaction-status-client-types", "thiserror 2.0.18", + "time", "tokio", ] diff --git a/Cargo.toml b/Cargo.toml index d2c389421..cd008b2a3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -76,6 +76,7 @@ solana-transaction-status-client-types = "3.1.8" tar = "0.4.44" tempfile = "3.26.0" thiserror = "2.0.18" +time = "0.3.47" tokio = "1.49.0" [profile.release] diff --git a/integration_tests/tests/tests.rs b/integration_tests/tests/tests.rs index 53ba6ae5f..ef26a4bd8 100644 --- a/integration_tests/tests/tests.rs +++ b/integration_tests/tests/tests.rs @@ -10,9 +10,9 @@ use cksol_int_tests::{ validator::FEE_PER_SIGNATURE, }; use cksol_types::{ - DepositSolArgs, DepositSolError, DepositSolStatus, GetDepositAddressArgs, Lamport, MinterInfo, - TxFinalizedStatus, WatchDepositAddressError, WatchDepositAddressStatus, WithdrawSolArgs, - WithdrawSolError, WithdrawSolStatus, + DepositSolArgs, DepositSolError, DepositSolId, DepositSolStatus, GetDepositAddressArgs, + Lamport, MinterInfo, Signature, TxFinalizedStatus, WatchDepositAddressError, + WatchDepositAddressStatus, WithdrawSolArgs, WithdrawSolError, WithdrawSolStatus, }; use cksol_types_internal::{ UpgradeArgs, @@ -49,7 +49,14 @@ async fn deposit_and_credit_funds(setup: &Setup) { .await .expect("deposit_sol should queue a sweep"); - // Sweep + let sweep_signature = sweep_deposit(setup, deposit_id).await; + finalize_and_credit_sweep(setup, &sweep_signature, SWEEPABLE_AMOUNT).await; +} + +/// Runs the sweep timer on the deposit queued by the first batch of HTTP outcalls (IDs 0 to +/// 3, a `getBalance` of `deposit_sol` or a `getMultipleAccounts` of the scan timer) and +/// returns the signature of its sweep. +async fn sweep_deposit(setup: &Setup, deposit_id: DepositSolId) -> Signature { setup.advance_time(SWEEP_DEPOSITS_DELAY).await; setup .execute_http_mocks( @@ -58,19 +65,23 @@ async fn deposit_and_credit_funds(setup: &Setup) { .build(), ) .await; - let sweep_signature = assert_matches!( + assert_matches!( setup.minter().deposit_sol_status(deposit_id).await, DepositSolStatus::Swept { signature } => signature - ); + ) +} - // Finalize and credit +/// Runs the finalization timer on the sweep submitted by [`sweep_deposit`], reporting that +/// the minter's main account received `received`, which credits the sweep if it matches +/// its plan. +async fn finalize_and_credit_sweep(setup: &Setup, sweep_signature: &Signature, received: Lamport) { setup.advance_time(FINALIZE_TRANSACTIONS_DELAY).await; - let sweep = setup.minter().signed_transaction(&sweep_signature).await; + let sweep = setup.minter().signed_transaction(sweep_signature).await; setup .execute_http_mocks( MockBuilder::with_start_id(16) - .finalize_transaction(&sweep_signature, SUBMISSION_BLOCK_HEIGHT) - .get_sweep_transaction(&sweep, SWEEPABLE_AMOUNT) + .finalize_transaction(sweep_signature, SUBMISSION_BLOCK_HEIGHT) + .get_sweep_transaction(&sweep, received) .build(), ) .await; @@ -975,19 +986,7 @@ mod deposit_sol_tests { .await .expect("deposit_sol should queue a sweep"); - setup.advance_time(SWEEP_DEPOSITS_DELAY).await; - setup - .execute_http_mocks( - MockBuilder::with_start_id(4) - .submit_transaction(SUBMISSION_BLOCK_HEIGHT) - .build(), - ) - .await; - - let sweep_signature = assert_matches!( - setup.minter().deposit_sol_status(deposit_id).await, - DepositSolStatus::Swept { signature } => signature - ); + let sweep_signature = sweep_deposit(&setup, deposit_id).await; setup.minter().assert_that_events().await.satisfy(|events| { check!(events.iter().any(|e| matches!( e, @@ -999,16 +998,7 @@ mod deposit_sol_tests { ))); }); - setup.advance_time(FINALIZE_TRANSACTIONS_DELAY).await; - let sweep = setup.minter().signed_transaction(&sweep_signature).await; - setup - .execute_http_mocks( - MockBuilder::with_start_id(16) - .finalize_transaction(&sweep_signature, SUBMISSION_BLOCK_HEIGHT) - .get_sweep_transaction(&sweep, SWEEPABLE_AMOUNT) - .build(), - ) - .await; + finalize_and_credit_sweep(&setup, &sweep_signature, SWEEPABLE_AMOUNT).await; setup.wait_for_deposit_minted(deposit_id).await; @@ -1041,18 +1031,7 @@ mod deposit_sol_tests { .deposit_sol(DEFAULT_CALLER_ACCOUNT) .await .expect("deposit_sol should queue a sweep"); - setup.advance_time(SWEEP_DEPOSITS_DELAY).await; - setup - .execute_http_mocks( - MockBuilder::with_start_id(4) - .submit_transaction(SUBMISSION_BLOCK_HEIGHT) - .build(), - ) - .await; - let sweep_signature = assert_matches!( - setup.minter().deposit_sol_status(deposit_id).await, - DepositSolStatus::Swept { signature } => signature - ); + let sweep_signature = sweep_deposit(&setup, deposit_id).await; setup.advance_time(FINALIZE_TRANSACTIONS_DELAY).await; setup @@ -1078,7 +1057,8 @@ mod deposit_sol_tests { let setup = setup .check_metrics() .await - .assert_contains_metric_matching(r"dropped_deposits 1 \d+") + .assert_contains_metric_matching(r#"dropped_deposits\{origin="manual"\} 1 \d+"#) + .assert_contains_metric_matching(r#"dropped_deposits\{origin="automatic"\} 0 \d+"#) .into(); let deposit_id_after_drop = setup @@ -1109,30 +1089,10 @@ mod deposit_sol_tests { .deposit_sol(DEFAULT_CALLER_ACCOUNT) .await .expect("deposit_sol should queue a sweep"); - setup.advance_time(SWEEP_DEPOSITS_DELAY).await; - setup - .execute_http_mocks( - MockBuilder::with_start_id(4) - .submit_transaction(SUBMISSION_BLOCK_HEIGHT) - .build(), - ) - .await; - let sweep_signature = assert_matches!( - setup.minter().deposit_sol_status(deposit_id).await, - DepositSolStatus::Swept { signature } => signature - ); + let sweep_signature = sweep_deposit(&setup, deposit_id).await; - setup.advance_time(FINALIZE_TRANSACTIONS_DELAY).await; let amount_not_in_the_plan = BALANCE_ABOVE_MINIMUM - RENT_EXEMPTION_THRESHOLD + 1; - let sweep = setup.minter().signed_transaction(&sweep_signature).await; - setup - .execute_http_mocks( - MockBuilder::with_start_id(16) - .finalize_transaction(&sweep_signature, SUBMISSION_BLOCK_HEIGHT) - .get_sweep_transaction(&sweep, amount_not_in_the_plan) - .build(), - ) - .await; + finalize_and_credit_sweep(&setup, &sweep_signature, amount_not_in_the_plan).await; assert_eq!( setup.minter().deposit_sol_status(deposit_id).await, @@ -1192,6 +1152,7 @@ mod automatic_deposit_sol_tests { const SCAN_WATCHED_ADDRESSES_DELAY: Duration = Duration::from_mins(1); const BALANCE_ABOVE_MINIMUM: Lamport = Setup::DEFAULT_MINIMUM_DEPOSIT_AMOUNT + 1; + const BALANCE_BELOW_MINIMUM: Lamport = Setup::DEFAULT_MINIMUM_DEPOSIT_AMOUNT - 1; #[tokio::test] async fn should_register_the_caller() { @@ -1387,11 +1348,106 @@ mod automatic_deposit_sol_tests { setup .check_metrics() .await + .assert_contains_metric_matching(r#"watched_deposit_addresses 0 \d+"#) .assert_contains_metric_matching(r#"deposit_address_scans\{outcome="queued"\} 1 \d+"#) .into() .drop() .await; } + + #[tokio::test] + async fn should_sweep_finalize_and_mint_the_detected_deposit_minus_the_automatic_fee() { + const SWEEPABLE_AMOUNT: Lamport = BALANCE_ABOVE_MINIMUM - RENT_EXEMPTION_THRESHOLD; + const FEE: Lamport = Setup::DEFAULT_AUTOMATIC_DEPOSIT_FEE; + let setup = SetupBuilder::new().build().await; + setup + .minter() + .watch_deposit_address(None) + .await + .expect("watch_deposit_address should register the caller"); + setup.advance_time(SCAN_WATCHED_ADDRESSES_DELAY).await; + setup + .execute_http_mocks( + MockBuilder::new() + .get_multiple_accounts(&[BALANCE_ABOVE_MINIMUM]) + .build(), + ) + .await; + + let sweep_signature = sweep_deposit(&setup, 0).await; + finalize_and_credit_sweep(&setup, &sweep_signature, SWEEPABLE_AMOUNT).await; + setup.wait_for_deposit_minted(0).await; + + let minted_amount = SWEEPABLE_AMOUNT - FEE_PER_SIGNATURE - FEE; + assert_eq!( + setup.minter().deposit_sol_status(0).await, + DepositSolStatus::Minted { + block_index: 0, + minted_amount, + } + ); + assert_eq!( + setup.ledger().balance_of(DEFAULT_CALLER_ACCOUNT).await, + minted_amount + ); + assert_eq!( + setup + .minter() + .get_minter_info() + .await + .collected_automatic_deposit_fees, + Some(FEE) + ); + setup + .check_metrics() + .await + .assert_contains_metric_matching(format!(r"collected_automatic_deposit_fees {FEE} \d+")) + .into() + .drop() + .await; + } + + #[tokio::test] + async fn should_report_a_due_scan_in_the_metrics_until_it_runs() { + let setup = SetupBuilder::new().build().await; + setup + .minter() + .watch_deposit_address(None) + .await + .expect("watch_deposit_address should register the caller"); + + // The tick starts the scan, whose call stays pending until the mocks below answer it. + setup.advance_time(SCAN_WATCHED_ADDRESSES_DELAY).await; + setup.tick().await; + let setup = setup + .check_metrics() + .await + .assert_contains_metric_matching(r#"watched_deposit_addresses 1 \d+"#) + .assert_contains_metric_matching( + r#"oldest_due_deposit_address_scan_age_seconds 60 \d+"#, + ) + .into(); + + setup + .execute_http_mocks( + MockBuilder::new() + .get_multiple_accounts(&[BALANCE_BELOW_MINIMUM]) + .build(), + ) + .await; + + setup + .check_metrics() + .await + .assert_contains_metric_matching(r#"watched_deposit_addresses 1 \d+"#) + .assert_contains_metric_matching(r#"oldest_due_deposit_address_scan_age_seconds 0 \d+"#) + .assert_contains_metric_matching( + r#"deposit_address_scans\{outcome="below_minimum"\} 1 \d+"#, + ) + .into() + .drop() + .await; + } } mod anonymous_caller_tests { @@ -1497,6 +1553,24 @@ mod metrics_tests { .assert_contains_metric_matching(r#"minter_balance 0 \d+"#) .assert_contains_metric_matching(r#"oldest_in_flight_deposit_age_seconds 0 \d+"#) .assert_contains_metric_matching(r#"oldest_pending_mint_age_seconds 0 \d+"#) + .assert_contains_metric_matching(r#"watched_deposit_addresses 0 \d+"#) + .assert_contains_metric_matching(r#"watched_deposit_address_capacity 1000 \d+"#) + .assert_contains_metric_matching(r#"oldest_due_deposit_address_scan_age_seconds 0 \d+"#) + .assert_contains_metric_matching(r#"collected_automatic_deposit_fees 0 \d+"#) + .assert_contains_metric_matching(r"# TYPE collected_automatic_deposit_fees counter") + .assert_contains_metric_matching(r#"deposit_address_scans\{outcome="queued"\} 0 \d+"#) + .assert_contains_metric_matching( + r#"deposit_address_scans\{outcome="below_minimum"\} 0 \d+"#, + ) + .assert_contains_metric_matching(r#"deposit_address_scans\{outcome="failed"\} 0 \d+"#) + .assert_contains_metric_matching( + r#"automatic_deposit_rearms\{outcome="rearmed"\} 0 \d+"#, + ) + .assert_contains_metric_matching( + r#"automatic_deposit_rearms\{outcome="watchlist_full"\} 0 \d+"#, + ) + .assert_contains_metric_matching(r#"dropped_deposits\{origin="manual"\} 0 \d+"#) + .assert_contains_metric_matching(r#"dropped_deposits\{origin="automatic"\} 0 \d+"#) .assert_contains_metric_matching( r#"failed_credit_attempts\{reason="not_found"\} 0 \d+"#, ) diff --git a/minter/Cargo.toml b/minter/Cargo.toml index 11a661ccc..6cd08bb32 100644 --- a/minter/Cargo.toml +++ b/minter/Cargo.toml @@ -57,6 +57,7 @@ solana-system-interface = { workspace = true, features = ["bincode"] } solana-transaction = { workspace = true, features = ["bincode", "verify"] } solana-transaction-status-client-types = { workspace = true } thiserror = { workspace = true } +time = { workspace = true } [dev-dependencies] agave-reserved-account-keys = { workspace = true } diff --git a/minter/src/dashboard/mod.rs b/minter/src/dashboard/mod.rs index 499dc7483..8695826a0 100644 --- a/minter/src/dashboard/mod.rs +++ b/minter/src/dashboard/mod.rs @@ -1,6 +1,7 @@ use crate::{ address::minter_address, state::{QuarantineCause, State}, + utils::timed_sized_map::Timestamp, }; use askama::Template; use candid::Principal; @@ -21,6 +22,23 @@ pub fn lamports_to_sol(lamports: u64) -> String { } } +/// Formats a time as a UTC date and time to the second, e.g. `2023-11-14T22:13:20+00:00`, as +/// on the dashboard of the ckETH minter. +fn format_timestamp(timestamp: Timestamp) -> String { + let datetime = + time::OffsetDateTime::from_unix_timestamp_nanos(i128::from(timestamp.as_nanos())) + .expect("BUG: nanoseconds since the epoch that fit a u64 are a valid time"); + format!( + "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}+00:00", + datetime.year(), + u8::from(datetime.month()), + datetime.day(), + datetime.hour(), + datetime.minute(), + datetime.second() + ) +} + fn solscan_cluster_suffix(network: SolanaNetwork) -> &'static str { match network { SolanaNetwork::Mainnet => "", @@ -41,6 +59,7 @@ pub struct DashboardPaginationParameters { pub quarantined_swept_deposits_start: usize, pub minted_sweeps_start: usize, pub withdrawals_start: usize, + pub watched_deposit_addresses_start: usize, } impl DashboardPaginationParameters { @@ -57,6 +76,7 @@ impl DashboardPaginationParameters { quarantined_swept_deposits_start: parse(req, "quarantined_swept_deposits_start")?, minted_sweeps_start: parse(req, "minted_sweeps_start")?, withdrawals_start: parse(req, "withdrawals_start")?, + watched_deposit_addresses_start: parse(req, "watched_deposit_addresses_start")?, }) } @@ -69,6 +89,10 @@ impl DashboardPaginationParameters { ), ("minted_sweeps_start", self.minted_sweeps_start), ("withdrawals_start", self.withdrawals_start), + ( + "watched_deposit_addresses_start", + self.watched_deposit_addresses_start, + ), ] .into_iter() .filter(|(name, _)| *name != exclude) @@ -229,6 +253,20 @@ pub struct DashboardQuarantinedDeposit { pub amount_to_mint: String, } +/// A deposit address registered for the automatic deposit flow, with its scan progress. +#[derive(Clone)] +pub struct DashboardWatchedAddress { + pub account: String, + pub address: String, + pub valid_until: String, + pub scan_count: String, + /// The time of the last scan, or `-` before the first scan. + pub last_scanned_at: String, + /// When the next scan is due, or `-` once no slot is left before the window closes: a + /// time well in the past shows that the scan timer is behind. + pub next_scan_at: String, +} + /// A swept deposit whose ckSOL mint landed on the ledger. #[derive(Clone)] pub struct DashboardMintedSweep { @@ -251,13 +289,20 @@ pub struct DashboardTemplate { pub minimum_deposit_amount: String, pub minimum_withdrawal_amount: String, pub balance: String, + pub automatic_deposit_fee: String, + pub collected_automatic_deposit_fees: String, pub quarantined_swept_deposits_table: DashboardPaginatedTable, pub minted_sweeps_table: DashboardPaginatedTable, pub withdrawals_table: DashboardPaginatedTable, + pub watched_deposit_addresses_table: DashboardPaginatedTable, } impl DashboardTemplate { - pub fn from_state(state: &State, pagination: DashboardPaginationParameters) -> Self { + pub fn from_state( + state: &State, + now: Timestamp, + pagination: DashboardPaginationParameters, + ) -> Self { let minter_address = state .minter_public_key() .map(|key| minter_address(key).to_string()) @@ -383,6 +428,32 @@ impl DashboardTemplate { pagination.other_params("withdrawals_start"), ); + let watched_deposit_addresses: Vec = state + .watchlist() + .live(now) + .map(|watched| DashboardWatchedAddress { + account: watched.account.to_string(), + address: watched.address.to_string(), + valid_until: format_timestamp(watched.valid_until), + scan_count: watched.scan_count.to_string(), + last_scanned_at: watched + .last_scanned_at + .map_or_else(|| "-".to_string(), format_timestamp), + next_scan_at: watched + .next_scan_at + .map_or_else(|| "-".to_string(), format_timestamp), + }) + .collect(); + let watched_deposit_addresses_table = DashboardPaginatedTable::from_items( + &watched_deposit_addresses, + pagination.watched_deposit_addresses_start, + DEFAULT_PAGE_SIZE, + 6, + "watched-deposit-addresses", + "watched_deposit_addresses_start", + pagination.other_params("watched_deposit_addresses_start"), + ); + let network = state.solana_network(); DashboardTemplate { solana_cluster: format!("{:?}", network), @@ -395,9 +466,16 @@ impl DashboardTemplate { minimum_deposit_amount: lamports_to_sol(state.minimum_deposit_amount()), minimum_withdrawal_amount: lamports_to_sol(state.minimum_withdrawal_amount()), balance: lamports_to_sol(state.balance()), + automatic_deposit_fee: state + .automatic_deposit_fee() + .map_or_else(|| "N/A".to_string(), lamports_to_sol), + collected_automatic_deposit_fees: lamports_to_sol( + state.collected_automatic_deposit_fees(), + ), quarantined_swept_deposits_table, minted_sweeps_table, withdrawals_table, + watched_deposit_addresses_table, } } } diff --git a/minter/src/dashboard/tests.rs b/minter/src/dashboard/tests.rs index 0aeb4622a..1ac76dc64 100644 --- a/minter/src/dashboard/tests.rs +++ b/minter/src/dashboard/tests.rs @@ -1,24 +1,30 @@ use crate::dashboard::{DashboardPaginationParameters, DashboardTemplate, lamports_to_sol}; -use crate::state::read_state; +use crate::state::{mutate_state, read_state}; use crate::test_fixtures::{ - MINIMUM_DEPOSIT_AMOUNT, MINIMUM_WITHDRAWAL_AMOUNT, WITHDRAWAL_FEE, account, + AUTOMATIC_DEPOSIT_FEE, HOUR_NANOS, MINIMUM_DEPOSIT_AMOUNT, MINIMUM_WITHDRAWAL_AMOUNT, + MINUTE_NANOS, REGISTERED_AT, WATCH_WINDOW_NANOS, WITHDRAWAL_FEE, account, at, deposit_address, events::{ accept_withdrawal, fail_transaction, quarantine_sweep, queue_deposit, submit_sweep, submit_withdrawal, succeed_transaction, }, flow::deposit::DepositFlow, init_balance, init_schnorr_master_key, init_state, init_state_with_args, ledger_canister_id, - signature, sol_rpc_canister_id, valid_init_args, + signature, sol_rpc_canister_id, valid_init_args, watch, }; +use crate::utils::timed_sized_map::Timestamp; use askama::Template; -use cksol_types_internal::SolanaNetwork; +use cksol_types_internal::{InitArgs, SolanaNetwork}; fn dashboard() -> DashboardTemplate { dashboard_with_pagination(DashboardPaginationParameters::default()) } fn dashboard_with_pagination(pagination: DashboardPaginationParameters) -> DashboardTemplate { - read_state(|state| DashboardTemplate::from_state(state, pagination)) + dashboard_at(at(REGISTERED_AT), pagination) +} + +fn dashboard_at(now: Timestamp, pagination: DashboardPaginationParameters) -> DashboardTemplate { + read_state(|state| DashboardTemplate::from_state(state, now, pagination)) } fn init_state_with_network(network: SolanaNetwork) { @@ -56,16 +62,177 @@ fn should_display_metadata() { "#minimum-withdrawal-amount > td", &lamports_to_sol(MINIMUM_WITHDRAWAL_AMOUNT), "wrong minimum withdrawal amount", + ) + .has_string_value( + "#automatic-deposit-fee > td", + &lamports_to_sol(AUTOMATIC_DEPOSIT_FEE), + "wrong automatic deposit fee", + ) + .has_string_value( + "#collected-automatic-deposit-fees > td", + "0", + "wrong collected automatic deposit fees", ); } +#[test] +fn should_display_an_unconfigured_automatic_deposit_fee_as_not_available() { + init_state_with_args(InitArgs { + automatic_deposit_fee: None, + ..valid_init_args() + }); + + DashboardAssert::assert_that(dashboard()).has_string_value( + "#automatic-deposit-fee > td", + "N/A", + "expected N/A when the automatic deposit fee is not configured", + ); +} + +#[test] +fn should_display_the_collected_automatic_deposit_fees() { + init_state(); + init_schnorr_master_key(); + DepositFlow::queue_automatic(account(1), 400_000_000, AUTOMATIC_DEPOSIT_FEE) + .sweep(signature(0xAA)) + .succeed() + .credit(); + + DashboardAssert::assert_that(dashboard()).has_string_value( + "#collected-automatic-deposit-fees > td", + &lamports_to_sol(AUTOMATIC_DEPOSIT_FEE), + "wrong collected automatic deposit fees", + ); +} + +#[test] +fn should_display_watched_deposit_addresses_by_ascending_window_end() { + init_state(); + init_schnorr_master_key(); + watch(at(REGISTERED_AT + 1), account(1)); + watch(at(REGISTERED_AT), account(2)); + watch(at(REGISTERED_AT + 2), account(3)); + mutate_state(|s| { + s.watchlist_mut() + .record_scan(at(REGISTERED_AT + MINUTE_NANOS), &account(2)); + s.watchlist_mut() + .record_scan(at(REGISTERED_AT + WATCH_WINDOW_NANOS), &account(3)); + }); + + DashboardAssert::assert_that(dashboard()) + .has_table_row_value( + "#watched-deposit-addresses + table > tbody > tr:nth-child(1)", + &[ + &account(2).to_string(), + &deposit_address(account(2)).to_string(), + "2023-11-15T22:13:20+00:00", + "1", + "2023-11-14T22:14:20+00:00", + "2023-11-14T22:15:20+00:00", + ], + "watched deposit addresses", + ) + .has_table_row_value( + "#watched-deposit-addresses + table > tbody > tr:nth-child(2)", + &[ + &account(1).to_string(), + &deposit_address(account(1)).to_string(), + "2023-11-15T22:13:20+00:00", + "0", + "-", + "2023-11-14T22:13:20+00:00", + ], + "watched deposit addresses", + ) + .has_table_row_value( + "#watched-deposit-addresses + table > tbody > tr:nth-child(3)", + &[ + &account(3).to_string(), + &deposit_address(account(3)).to_string(), + "2023-11-15T22:13:20+00:00", + "1", + "2023-11-15T22:13:20+00:00", + "-", + ], + "watched deposit addresses", + ) + .has_link_matching( + "#watched-deposit-addresses + table > tbody > tr:nth-child(1) a", + |href| href == format!("https://solscan.io/account/{}", deposit_address(account(2))), + ); +} + +#[test] +fn should_display_only_the_live_watched_deposit_addresses() { + init_state(); + init_schnorr_master_key(); + watch(at(REGISTERED_AT), account(1)); + watch(at(REGISTERED_AT + HOUR_NANOS), account(2)); + let expired_at = REGISTERED_AT + WATCH_WINDOW_NANOS + 1; + + let page = dashboard_at(at(expired_at), DashboardPaginationParameters::default()); + + let listed: Vec<&str> = page + .watched_deposit_addresses_table + .current_page + .iter() + .map(|watched| watched.account.as_str()) + .collect(); + assert_eq!(listed, vec![account(2).to_string()]); + assert_eq!(read_state(|s| s.watchlist().len()), 2); + DashboardAssert::assert_that(dashboard_at( + at(expired_at + HOUR_NANOS), + DashboardPaginationParameters::default(), + )) + .has_no_elements_matching("#watched-deposit-addresses + table"); +} + +#[test] +fn should_paginate_watched_deposit_addresses_across_multiple_pages() { + use crate::dashboard::DEFAULT_PAGE_SIZE; + + init_state(); + init_schnorr_master_key(); + for index in 0..DEFAULT_PAGE_SIZE + 1 { + watch(at(REGISTERED_AT), account(index)); + } + + let page1 = dashboard(); + assert_eq!( + page1.watched_deposit_addresses_table.current_page.len(), + DEFAULT_PAGE_SIZE + ); + assert_eq!( + page1.watched_deposit_addresses_table.pagination.pages.len(), + 2 + ); + assert_eq!( + page1.watched_deposit_addresses_table.pagination.table_width, + 6 + ); + + let page2 = dashboard_with_pagination(DashboardPaginationParameters { + watched_deposit_addresses_start: DEFAULT_PAGE_SIZE, + ..Default::default() + }); + assert_eq!(page2.watched_deposit_addresses_table.current_page.len(), 1); + assert_eq!( + page2 + .watched_deposit_addresses_table + .pagination + .current_page_index, + 2 + ); +} + #[test] fn should_display_empty_state() { init_state(); DashboardAssert::assert_that(dashboard()) .has_no_elements_matching("#minted-sweeps + table") - .has_no_elements_matching("#withdrawals + table"); + .has_no_elements_matching("#withdrawals + table") + .has_no_elements_matching("#watched-deposit-addresses + table"); } #[test] diff --git a/minter/src/main.rs b/minter/src/main.rs index f0eb836c3..6d187010f 100644 --- a/minter/src/main.rs +++ b/minter/src/main.rs @@ -12,6 +12,7 @@ use cksol_minter::{ monitor::{FINALIZE_TRANSACTIONS_DELAY, finalize_transactions}, runtime::IcCanisterRuntime, state::read_state, + utils::timed_sized_map::Timestamp, withdraw::{WITHDRAWAL_PROCESSING_DELAY, process_pending_withdrawals}, }; use cksol_types::{ @@ -166,7 +167,9 @@ fn http_request(request: HttpRequest) -> HttpResponse { .build(); } }; - let dashboard = read_state(|state| DashboardTemplate::from_state(state, pagination)); + let now = Timestamp::from_nanos(ic_cdk::api::time()); + let dashboard = + read_state(|state| DashboardTemplate::from_state(state, now, pagination)); HttpResponseBuilder::ok() .header("Content-Type", "text/html; charset=utf-8") .with_body_and_content_length(dashboard.render().unwrap()) diff --git a/minter/src/metrics.rs b/minter/src/metrics.rs index ad207ffa2..5f550d439 100644 --- a/minter/src/metrics.rs +++ b/minter/src/metrics.rs @@ -1,7 +1,9 @@ -use crate::state::{QuarantineCause, State}; +use crate::state::{DepositOrigin, QuarantineCause, State}; use crate::storage::{ - self, FailedCreditReason, FailedMintReason, ScanOutcome, UndecidedWithdrawalReason, + self, FailedCreditReason, FailedMintReason, RearmOutcome, ScanOutcome, + UndecidedWithdrawalReason, }; +use crate::utils::timed_sized_map::Timestamp; use ic_metrics_encoder::MetricsEncoder; const WASM_PAGE_SIZE_IN_BYTES: usize = 65536; @@ -48,11 +50,20 @@ pub fn encode_metrics(w: &mut MetricsEncoder>, s: &State) -> std::io::Re s.deposits().minted().len().metric_value(), "Number of swept deposits whose ckSOL mint landed on the ledger.", )?; - w.encode_gauge( + let mut dropped_manual: usize = 0; + let mut dropped_automatic: usize = 0; + for dropped in s.deposits().dropped().values() { + match dropped.deposit.origin { + DepositOrigin::Manual => dropped_manual += 1, + DepositOrigin::Automatic => dropped_automatic += 1, + } + } + w.gauge_vec( "dropped_deposits", - s.deposits().dropped().len().metric_value(), - "Number of deposits whose sweep transaction failed or expired.", - )?; + "Number of deposits whose sweep transaction failed or expired, by origin: queued by deposit_sol or by a balance scan of the automatic flow.", + )? + .value(&[("origin", "manual")], dropped_manual.metric_value())? + .value(&[("origin", "automatic")], dropped_automatic.metric_value())?; let mut sweep_unreadable: usize = 0; let mut mint_unresolved: usize = 0; for quarantined in s.deposits().quarantined().values() { @@ -114,6 +125,32 @@ pub fn encode_metrics(w: &mut MetricsEncoder>, s: &State) -> std::io::Re age_seconds(s.deposits().oldest_pending_mint_created_at()).metric_value(), "Age of the oldest pending ckSOL mint in seconds, from the created_at_time the ledger deduplicates it by. Returns 0 if there are no pending mints.", )?; + let now = Timestamp::from_nanos(ic_cdk::api::time()); + w.encode_gauge( + "watched_deposit_addresses", + s.watchlist().live_len(now).metric_value(), + "Number of deposit addresses registered for the automatic deposit flow whose window is still open.", + )?; + w.encode_gauge( + "watched_deposit_address_capacity", + s.watchlist().capacity().metric_value(), + "Number of live registrations the watchlist of the automatic deposit flow admits at most.", + )?; + w.encode_gauge( + "oldest_due_deposit_address_scan_age_seconds", + age_seconds( + s.watchlist() + .oldest_due_scan_at(now) + .map(Timestamp::as_nanos), + ) + .metric_value(), + "Time in seconds since the earliest due scan of a watched deposit address that has not run yet. Returns 0 if no scan is due.", + )?; + w.encode_counter( + "collected_automatic_deposit_fees", + s.collected_automatic_deposit_fees().metric_value(), + "Sum in Lamports of the automatic deposit fees deducted from the mints of credited sweeps.", + )?; let mut failed_credit_attempts = w.counter_vec( "failed_credit_attempts", "Number of failed attempts to credit the deposits of a finalized sweep, by reason.", @@ -144,6 +181,16 @@ pub fn encode_metrics(w: &mut MetricsEncoder>, s: &State) -> std::io::Re storage::deposit_address_scan_count(outcome).metric_value(), )?; } + let mut automatic_deposit_rearms = w.counter_vec( + "automatic_deposit_rearms", + "Number of automatic deposits dropped with their sweep since the last upgrade, by outcome: their account put back on the watchlist for a fresh window, or not because the watchlist was full.", + )?; + for outcome in RearmOutcome::ALL { + automatic_deposit_rearms = automatic_deposit_rearms.value( + &[("outcome", outcome.label())], + storage::automatic_deposit_rearm_count(outcome).metric_value(), + )?; + } w.encode_gauge( "minter_balance", s.balance().metric_value(), diff --git a/minter/src/state/audit.rs b/minter/src/state/audit.rs index 5c3e0cfea..c35c10dac 100644 --- a/minter/src/state/audit.rs +++ b/minter/src/state/audit.rs @@ -48,17 +48,23 @@ pub fn process_dropped_sweep( let now = Timestamp::from_nanos(runtime.time()); for (deposit_id, deposit) in deposits { match state.watchlist_mut().rearm(now, &deposit) { - Ok(_) => log!( - Priority::Info, - "Watching account {} again after its deposit {deposit_id} was dropped with sweep {signature}", - deposit.account - ), + Ok(_) => { + storage::record_automatic_deposit_rearm(storage::RearmOutcome::Rearmed); + log!( + Priority::Info, + "Watching account {} again after its deposit {deposit_id} was dropped with sweep {signature}", + deposit.account + ) + } Err(RearmError::ManualDeposit) => {} - Err(RearmError::WatchlistFull) => log!( - Priority::Info, - "Failed to watch account {} again after its deposit {deposit_id} was dropped with sweep {signature}: the watchlist is full", - deposit.account - ), + Err(RearmError::WatchlistFull) => { + storage::record_automatic_deposit_rearm(storage::RearmOutcome::WatchlistFull); + log!( + Priority::Info, + "Failed to watch account {} again after its deposit {deposit_id} was dropped with sweep {signature}: the watchlist is full", + deposit.account + ) + } } } } diff --git a/minter/src/state/tests.rs b/minter/src/state/tests.rs index c45cf423a..00e625063 100644 --- a/minter/src/state/tests.rs +++ b/minter/src/state/tests.rs @@ -4,6 +4,7 @@ use crate::{ rpc::BlockHeight, sol_transfer::MAX_SIGNATURES, state::{audit::process_event, read_state}, + storage::{RearmOutcome, automatic_deposit_rearm_count}, test_fixtures::{ AUTOMATIC_DEPOSIT_FEE, DEPOSIT_SOL_FEE, DEPOSIT_SOL_REQUIRED_CYCLES, MINIMUM_DEPOSIT_AMOUNT, MINIMUM_WITHDRAWAL_AMOUNT, WITHDRAWAL_FEE, account, address, @@ -347,6 +348,7 @@ mod swept_deposits { let automatic = DepositFlow::queue_automatic(account(2), 20_000_000, AUTOMATIC_DEPOSIT_FEE); SweepFlow::of([manual, automatic]).submit(sweep_signature); + let rearms_before = automatic_deposit_rearm_count(RearmOutcome::Rearmed); mutate_state(|state| { process_dropped_sweep( @@ -377,6 +379,16 @@ mod swept_deposits { "{drop_event:?}" ); }); + assert_eq!( + automatic_deposit_rearm_count(RearmOutcome::Rearmed), + rearms_before + 1, + "{drop_event:?}" + ); + assert_eq!( + automatic_deposit_rearm_count(RearmOutcome::WatchlistFull), + 0, + "{drop_event:?}" + ); let mut live_without_rearm = read_state(State::clone); *live_without_rearm.watchlist_mut() = Watchlist::default(); assert_eq!( @@ -412,6 +424,11 @@ mod swept_deposits { assert_eq!(s.watchlist().len(), MAX_WATCHED_ADDRESSES.get()); assert_eq!(s.deposits().in_flight_id(&account(0)), None); }); + assert_eq!( + automatic_deposit_rearm_count(RearmOutcome::WatchlistFull), + 1 + ); + assert_eq!(automatic_deposit_rearm_count(RearmOutcome::Rearmed), 0); } #[test] diff --git a/minter/src/state/watchlist/mod.rs b/minter/src/state/watchlist/mod.rs index 98dc6247d..42fcb1afd 100644 --- a/minter/src/state/watchlist/mod.rs +++ b/minter/src/state/watchlist/mod.rs @@ -260,11 +260,28 @@ impl Watchlist { self.entries.is_empty() } + /// The number of live registrations the watchlist admits, which is + /// [`MAX_WATCHED_ADDRESSES`] after a rebuild too. + pub fn capacity(&self) -> usize { + self.entries.capacity().get() + } + pub fn live_len(&self, now: Timestamp) -> usize { self.live(now).count() } - fn live(&self, now: Timestamp) -> impl Iterator { + /// When the earliest due scan was due, if any scan is due: since the due time of a + /// registration only moves when a scan is recorded, this falls behind `now` by more than a + /// tick only when the scan timer does not keep up. + pub fn oldest_due_scan_at(&self, now: Timestamp) -> Option { + self.due(now) + .into_iter() + .filter_map(|watched| watched.next_scan_at) + .min() + } + + /// The live registrations, by ascending window end. + pub fn live(&self, now: Timestamp) -> impl Iterator { self.entries .iter_by_expiry() .filter(move |(_, entry)| entry.is_live(now)) diff --git a/minter/src/state/watchlist/tests.rs b/minter/src/state/watchlist/tests.rs index d44a57113..89cba90df 100644 --- a/minter/src/state/watchlist/tests.rs +++ b/minter/src/state/watchlist/tests.rs @@ -57,6 +57,7 @@ fn should_reject_a_registration_when_full_of_live_entries() { assert_eq!(result, Err(WatchlistFull)); assert_eq!(watchlist.len(), MAX_WATCHED_ADDRESSES.get()); + assert_eq!(watchlist.capacity(), MAX_WATCHED_ADDRESSES.get()); } #[test] @@ -178,6 +179,44 @@ fn should_count_the_live_registrations() { assert_eq!(watchlist.len(), 2); } +#[test] +fn should_report_the_earliest_due_scan_until_it_is_recorded() { + let mut watchlist = Watchlist::default(); + register(&mut watchlist, at(REGISTERED_AT), account(1)); + register(&mut watchlist, at(REGISTERED_AT + MINUTE_NANOS), account(2)); + + assert_eq!( + watchlist.oldest_due_scan_at(at(REGISTERED_AT + HOUR_NANOS)), + Some(at(REGISTERED_AT)) + ); + + watchlist.record_scan(at(REGISTERED_AT + HOUR_NANOS), &account(1)); + + assert_eq!( + watchlist.oldest_due_scan_at(at(REGISTERED_AT + HOUR_NANOS)), + Some(at(REGISTERED_AT + MINUTE_NANOS)) + ); + + watchlist.record_scan(at(REGISTERED_AT + HOUR_NANOS), &account(2)); + + assert_eq!( + watchlist.oldest_due_scan_at(at(REGISTERED_AT + HOUR_NANOS)), + None + ); +} + +#[test] +fn should_report_no_due_scan_of_an_expired_registration() { + let mut watchlist = Watchlist::default(); + register(&mut watchlist, at(REGISTERED_AT), account(1)); + + assert_eq!( + watchlist.oldest_due_scan_at(at(VALID_UNTIL)), + Some(at(REGISTERED_AT)) + ); + assert_eq!(watchlist.oldest_due_scan_at(at(VALID_UNTIL + 1)), None); +} + #[test] fn should_rearm_an_account_for_a_fresh_window_from_the_first_slot() { let mut watchlist = Watchlist::default(); @@ -295,6 +334,7 @@ fn should_keep_expired_registrations_and_exceed_the_capacity_when_rebuilding() { let mut rebuilt = Watchlist::from_snapshot(&snapshot); assert_eq!(rebuilt.len(), entries.len()); + assert_eq!(rebuilt.capacity(), MAX_WATCHED_ADDRESSES.get()); assert_eq!( rebuilt.register( at(REGISTERED_AT), diff --git a/minter/src/storage/mod.rs b/minter/src/storage/mod.rs index c09e01ad2..b54a15aba 100644 --- a/minter/src/storage/mod.rs +++ b/minter/src/storage/mod.rs @@ -41,6 +41,7 @@ pub(crate) struct Metrics { pub withdrawal_transaction_rebroadcasts: u64, pub undecided_withdrawal_transactions: BTreeMap, pub deposit_address_scans: BTreeMap, + pub automatic_deposit_rearms: BTreeMap, } impl Metrics { @@ -52,6 +53,7 @@ impl Metrics { withdrawal_transaction_rebroadcasts: 0, undecided_withdrawal_transactions: BTreeMap::new(), deposit_address_scans: BTreeMap::new(), + automatic_deposit_rearms: BTreeMap::new(), } } } @@ -84,6 +86,38 @@ pub(crate) fn deposit_address_scan_count(outcome: ScanOutcome) -> u64 { with_unstable_metrics(|m| m.deposit_address_scans.get(&outcome).copied().unwrap_or(0)) } +/// Whether the account of an automatic deposit dropped with its sweep was put back on the +/// watchlist. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub enum RearmOutcome { + Rearmed, + WatchlistFull, +} + +impl RearmOutcome { + pub const ALL: [Self; 2] = [Self::Rearmed, Self::WatchlistFull]; + + pub fn label(self) -> &'static str { + match self { + Self::Rearmed => "rearmed", + Self::WatchlistFull => "watchlist_full", + } + } +} + +pub(crate) fn record_automatic_deposit_rearm(outcome: RearmOutcome) { + with_unstable_metrics_mut(|m| *m.automatic_deposit_rearms.entry(outcome).or_insert(0) += 1); +} + +pub(crate) fn automatic_deposit_rearm_count(outcome: RearmOutcome) -> u64 { + with_unstable_metrics(|m| { + m.automatic_deposit_rearms + .get(&outcome) + .copied() + .unwrap_or(0) + }) +} + /// Why an attempt to credit the deposits of a finalized sweep failed. #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] pub enum FailedCreditReason { diff --git a/minter/templates/dashboard.html b/minter/templates/dashboard.html index 7f681029b..393433032 100644 --- a/minter/templates/dashboard.html +++ b/minter/templates/dashboard.html @@ -93,8 +93,45 @@

Metadata

Balance (SOL) {{ balance }} + + Automatic deposit fee (SOL) + {{ automatic_deposit_fee }} + + + Collected automatic deposit fees (SOL) + {{ collected_automatic_deposit_fees }} + + + + + {% if !watched_deposit_addresses_table.is_empty() %} +

Watched Deposit Addresses

+ + + + + + + + + + + + + {% for watched in watched_deposit_addresses_table.current_page %} + + + + + + + + + {% endfor %} + {% if watched_deposit_addresses_table.has_more_than_one_page() %}{{ watched_deposit_addresses_table.pagination|safe }}{% endif %}
AccountDeposit AddressValid UntilScansLast Scanned AtNext Scan At
{{ watched.account }}{% call solscan_address_link(watched.address) %}{{ watched.valid_until }}{{ watched.scan_count }}{{ watched.last_scanned_at }}{{ watched.next_scan_at }}
+ {% endif %} {% if !quarantined_swept_deposits_table.is_empty() %}

Quarantined Swept Deposits