diff --git a/.github/workflows/deploy-program.yml b/.github/workflows/deploy-program.yml index f343cdc..aa1d77a 100644 --- a/.github/workflows/deploy-program.yml +++ b/.github/workflows/deploy-program.yml @@ -68,7 +68,7 @@ permissions: contents: read env: - ARCH_KIT_VERSION: 0.1.8 + ARCH_KIT_VERSION: 0.1.9 ANZA_VERSION: v3.1.9 CARGO_TERM_COLOR: always @@ -101,7 +101,19 @@ jobs: libclang-dev - name: Install arch-kit - run: cargo install arch-kit --version "${ARCH_KIT_VERSION}" --locked + # crates.io and the GitHub release API answer with transient errors + # often enough that one attempt fails a deploy a few times a month. + run: | + set -euo pipefail + for attempt in 1 2 3; do + if cargo install arch-kit --version "${ARCH_KIT_VERSION}" --locked; then + exit 0 + fi + echo "arch-kit install attempt ${attempt}/3 failed; retrying in 20 s" >&2 + sleep 20 + done + echo "::error::arch-kit ${ARCH_KIT_VERSION} could not be installed after 3 attempts" >&2 + exit 1 - name: Stage deployment keys id: keys @@ -129,10 +141,25 @@ jobs: echo "authority-key=${authority_key}" >> "${GITHUB_OUTPUT}" - name: Install SBF toolchain + # The Anza installer and the platform-tools download behind + # `cargo build-sbf` both go through the GitHub release API, which + # intermittently returns bodies the installer cannot parse. Fetching + # platform-tools here, with retries, keeps that failure out of the + # build step. run: | set -euo pipefail - curl -sSfL "https://release.anza.xyz/${ANZA_VERSION}/install" | sh - echo "${HOME}/.local/share/solana/install/active_release/bin" >> "${GITHUB_PATH}" + sbf_bin="${HOME}/.local/share/solana/install/active_release/bin" + for attempt in 1 2 3; do + if curl -sSfL "https://release.anza.xyz/${ANZA_VERSION}/install" | sh \ + && "${sbf_bin}/cargo-build-sbf" --install-only; then + echo "${sbf_bin}" >> "${GITHUB_PATH}" + exit 0 + fi + echo "SBF toolchain install attempt ${attempt}/3 failed; retrying in 20 s" >&2 + sleep 20 + done + echo "::error::SBF toolchain ${ANZA_VERSION} could not be installed after 3 attempts" >&2 + exit 1 - name: Build program ELF id: program @@ -226,7 +253,20 @@ jobs: env: RPC_URL: ${{ inputs.rpc-url }} BITCOIN_NETWORK: ${{ inputs.bitcoin-network }} - run: arch-kit --rpc-url "${RPC_URL}" --bitcoin-network "${BITCOIN_NETWORK}" health + # arch-kit >= 0.1.9 waits up to 20 s for the height to move; the + # retries here cover a pinned older arch-kit and a node that is + # restarting, and fail only when the chain is actually stalled. + run: | + set -euo pipefail + for attempt in 1 2 3 4 5; do + if arch-kit --rpc-url "${RPC_URL}" --bitcoin-network "${BITCOIN_NETWORK}" health; then + exit 0 + fi + echo "health probe ${attempt}/5 failed; retrying in 10 s" >&2 + sleep 10 + done + echo "::error::Arch node at ${RPC_URL} failed the health probe 5 times" >&2 + exit 1 - name: Deploy program and IDL env: diff --git a/Cargo.lock b/Cargo.lock index 8545027..5efb6bb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -136,7 +136,7 @@ dependencies = [ [[package]] name = "arch-kit" -version = "0.1.8" +version = "0.1.9" dependencies = [ "aes-gcm", "apl-associated-token-account", diff --git a/Cargo.toml b/Cargo.toml index 2c33fd6..17423c4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "arch-kit" -version = "0.1.8" +version = "0.1.9" edition = "2024" rust-version = "1.89" description = "Command-line toolkit for developing, deploying, and interacting with Arch Network programs" diff --git a/README.md b/README.md index bc5be15..6fa146a 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ with the reusable workflow: ```yaml jobs: deploy: - uses: Arch-Network/arch-kit/.github/workflows/deploy-program.yml@v0.1.8 + uses: Arch-Network/arch-kit/.github/workflows/deploy-program.yml@v0.1.9 with: program-path: program bitcoin-network: testnet @@ -218,9 +218,12 @@ declared instructions and accounts. arch-kit health ``` -The command checks validator readiness, reports RPC latency, and samples the -block height twice. It exits successfully only when the node is ready and the -height increases during its two-second observation window. +The command checks validator readiness, reports RPC latency, and polls the +block height once a second for up to twenty seconds. It exits successfully as +soon as the node is ready and the height has moved, and fails only when the +height does not move within the timeout. A healthy chain still has multi-second +gaps between consecutive blocks, which is why a single short window is not +enough. ## Generate keys diff --git a/src/client.rs b/src/client.rs index 5848fb5..a6aae31 100644 --- a/src/client.rs +++ b/src/client.rs @@ -8,7 +8,12 @@ use bitcoin::Network; use thiserror::Error; const IS_NODE_READY: &str = "is_node_ready"; -const BLOCK_PROGRESS_WINDOW: Duration = Duration::from_secs(2); +/// A healthy chain still has multi-second gaps between blocks (3.5 s between +/// two consecutive blocks was observed on a chain averaging nine blocks per +/// second), so progress is polled until the height moves, bounded by this +/// timeout, instead of being judged from one fixed window. +const BLOCK_PROGRESS_TIMEOUT: Duration = Duration::from_secs(20); +const BLOCK_PROGRESS_POLL_INTERVAL: Duration = Duration::from_secs(1); /// The smallest reusable Arch client, currently limited to node health. #[derive(Clone, Debug)] @@ -26,8 +31,9 @@ impl ArchKitClient { Ok(Self { rpc_url }) } - /// Check that the node is ready and produces a new block during the - /// observation window. + /// Check that the node is ready and produces a new block within the + /// progress timeout. The returned `observation_window` is the time it + /// took to see the height move. pub fn health(&self) -> Result { let started_at = Instant::now(); let config = self.rpc_config(); @@ -43,23 +49,36 @@ impl ArchKitClient { let initial_block_height = client .get_block_count() .map_err(|source| self.rpc_error(source))?; - thread::sleep(BLOCK_PROGRESS_WINDOW); - let final_block_height = client - .get_block_count() - .map_err(|source| self.rpc_error(source))?; - require_progress( + let observation_started_at = Instant::now(); + let progress = wait_for_progress( initial_block_height, - final_block_height, - BLOCK_PROGRESS_WINDOW.as_secs(), - &self.rpc_url, + BLOCK_PROGRESS_TIMEOUT, + BLOCK_PROGRESS_POLL_INTERVAL, + || { + client + .get_block_count() + .map_err(|source| self.rpc_error(source)) + }, )?; + let observation_window = observation_started_at.elapsed(); + let final_block_height = match progress { + Progress::Advanced(height) => height, + Progress::Stalled(height) => { + return Err(ArchKitError::BlocksNotProgressing { + rpc_url: self.rpc_url.clone(), + initial_height: initial_block_height, + final_height: height, + observation_seconds: observation_window.as_secs(), + }); + } + }; Ok(HealthStatus { rpc_url: self.rpc_url.clone(), initial_block_height, final_block_height, rpc_latency, - observation_window: BLOCK_PROGRESS_WINDOW, + observation_window, total_elapsed: started_at.elapsed(), }) } @@ -144,21 +163,34 @@ fn require_ready(readiness: Option, rpc_url: &str) -> Result<(), ArchKitEr } } -fn require_progress( +/// Outcome of watching the block height after an initial sample. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum Progress { + /// The height rose above the initial sample; carries the height seen. + Advanced(u64), + /// The timeout passed without the height rising; carries the last height. + Stalled(u64), +} + +/// Poll `sample` every `poll_interval` until it exceeds `initial_height` or +/// `timeout` elapses. A sampling error ends the wait immediately, since a +/// node that stops answering is not healthy either. +fn wait_for_progress( initial_height: u64, - final_height: u64, - observation_seconds: u64, - rpc_url: &str, -) -> Result<(), ArchKitError> { - if final_height > initial_height { - Ok(()) - } else { - Err(ArchKitError::BlocksNotProgressing { - rpc_url: rpc_url.to_string(), - initial_height, - final_height, - observation_seconds, - }) + timeout: Duration, + poll_interval: Duration, + mut sample: impl FnMut() -> Result, +) -> Result { + let started_at = Instant::now(); + loop { + thread::sleep(poll_interval); + let height = sample()?; + if height > initial_height { + return Ok(Progress::Advanced(height)); + } + if started_at.elapsed() >= timeout { + return Ok(Progress::Stalled(height)); + } } } @@ -188,20 +220,48 @@ mod tests { } #[test] - fn requires_the_block_height_to_increase() { - assert!(require_progress(100, 101, 2, "http://node").is_ok()); - - for final_height in [100, 99] { - assert!(matches!( - require_progress(100, final_height, 2, "http://node"), - Err(ArchKitError::BlocksNotProgressing { - initial_height: 100, - final_height: observed, - observation_seconds: 2, - .. - }) if observed == final_height - )); - } + fn progress_is_reported_as_soon_as_the_height_moves() { + let mut samples = [100, 100, 101].into_iter(); + let progress = wait_for_progress::<()>( + 100, + Duration::from_secs(10), + Duration::from_millis(1), + || Ok(samples.next().expect("sample")), + ); + assert_eq!(progress, Ok(Progress::Advanced(101))); + } + + #[test] + fn a_stalled_height_is_reported_after_the_timeout() { + let progress = wait_for_progress::<()>( + 100, + Duration::from_millis(5), + Duration::from_millis(1), + || Ok(100), + ); + assert_eq!(progress, Ok(Progress::Stalled(100))); + } + + #[test] + fn a_lower_height_never_counts_as_progress() { + let progress = wait_for_progress::<()>( + 100, + Duration::from_millis(5), + Duration::from_millis(1), + || Ok(99), + ); + assert_eq!(progress, Ok(Progress::Stalled(99))); + } + + #[test] + fn a_sampling_error_ends_the_wait() { + let progress = wait_for_progress( + 100, + Duration::from_secs(10), + Duration::from_millis(1), + || Err("rpc down"), + ); + assert_eq!(progress, Err("rpc down")); } #[test]