Poll for block progress and retry toolchain installs

.github/workflows/deploy-program.yml

@@ -68,7 +68,7 @@ permissions:
   contents: read
 
 env:
-  ARCH_KIT_VERSION: 0.1.8
+  ARCH_KIT_VERSION: 0.1.9
   ANZA_VERSION: v3.1.9
   CARGO_TERM_COLOR: always
 
@@ -101,7 +101,19 @@ jobs:
             libclang-dev
 
       - name: Install arch-kit
-        run: cargo install arch-kit --version "${ARCH_KIT_VERSION}" --locked
+        # crates.io and the GitHub release API answer with transient errors
+        # often enough that one attempt fails a deploy a few times a month.
+        run: |
+          set -euo pipefail
+          for attempt in 1 2 3; do
+            if cargo install arch-kit --version "${ARCH_KIT_VERSION}" --locked; then
+              exit 0
+            fi
+            echo "arch-kit install attempt ${attempt}/3 failed; retrying in 20 s" >&2
+            sleep 20
+          done
+          echo "::error::arch-kit ${ARCH_KIT_VERSION} could not be installed after 3 attempts" >&2
+          exit 1
 
       - name: Stage deployment keys
         id: keys
@@ -129,10 +141,25 @@ jobs:
           echo "authority-key=${authority_key}" >> "${GITHUB_OUTPUT}"
 
       - name: Install SBF toolchain
+        # The Anza installer and the platform-tools download behind
+        # `cargo build-sbf` both go through the GitHub release API, which
+        # intermittently returns bodies the installer cannot parse. Fetching
+        # platform-tools here, with retries, keeps that failure out of the
+        # build step.
         run: |
           set -euo pipefail
-          curl -sSfL "https://release.anza.xyz/${ANZA_VERSION}/install" | sh
-          echo "${HOME}/.local/share/solana/install/active_release/bin" >> "${GITHUB_PATH}"
+          sbf_bin="${HOME}/.local/share/solana/install/active_release/bin"
+          for attempt in 1 2 3; do
+            if curl -sSfL "https://release.anza.xyz/${ANZA_VERSION}/install" | sh \
+              && "${sbf_bin}/cargo-build-sbf" --install-only; then
+              echo "${sbf_bin}" >> "${GITHUB_PATH}"
+              exit 0
+            fi
+            echo "SBF toolchain install attempt ${attempt}/3 failed; retrying in 20 s" >&2
+            sleep 20
+          done
+          echo "::error::SBF toolchain ${ANZA_VERSION} could not be installed after 3 attempts" >&2
+          exit 1
 
       - name: Build program ELF
         id: program
@@ -226,7 +253,20 @@ jobs:
         env:
           RPC_URL: ${{ inputs.rpc-url }}
           BITCOIN_NETWORK: ${{ inputs.bitcoin-network }}
-        run: arch-kit --rpc-url "${RPC_URL}" --bitcoin-network "${BITCOIN_NETWORK}" health
+        # arch-kit >= 0.1.9 waits up to 20 s for the height to move; the
+        # retries here cover a pinned older arch-kit and a node that is
+        # restarting, and fail only when the chain is actually stalled.
+        run: |
+          set -euo pipefail
+          for attempt in 1 2 3 4 5; do
+            if arch-kit --rpc-url "${RPC_URL}" --bitcoin-network "${BITCOIN_NETWORK}" health; then
+              exit 0
+            fi
+            echo "health probe ${attempt}/5 failed; retrying in 10 s" >&2
+            sleep 10
+          done
+          echo "::error::Arch node at ${RPC_URL} failed the health probe 5 times" >&2
+          exit 1
 
       - name: Deploy program and IDL
         env:

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "arch-kit"
-version = "0.1.8"
+version = "0.1.9"
 edition = "2024"
 rust-version = "1.89"
 description = "Command-line toolkit for developing, deploying, and interacting with Arch Network programs"
@@ -27,9 +27,9 @@
 base64 = "0.22"
 bitcoin = "=0.32.7"
 bs58 = "0.5"
 clap = { version = "4.5", features = ["derive", "env"] }
 flate2 = "1.1"
 hex = "0.4"
 hkdf = "0.12"
 include_dir = "0.7"
 minijinja = "2"

README.md

@@ -45,7 +45,7 @@ with the reusable workflow:
 ```yaml
 jobs:
   deploy:
-    uses: Arch-Network/arch-kit/.github/workflows/deploy-program.yml@v0.1.8
+    uses: Arch-Network/arch-kit/.github/workflows/deploy-program.yml@v0.1.9
     with:
       program-path: program
       bitcoin-network: testnet
@@ -218,9 +218,12 @@ declared instructions and accounts.
 arch-kit health
 ```
 
-The command checks validator readiness, reports RPC latency, and samples the
-block height twice. It exits successfully only when the node is ready and the
-height increases during its two-second observation window.
+The command checks validator readiness, reports RPC latency, and polls the
+block height once a second for up to twenty seconds. It exits successfully as
+soon as the node is ready and the height has moved, and fails only when the
+height does not move within the timeout. A healthy chain still has multi-second
+gaps between consecutive blocks, which is why a single short window is not
+enough.
 
 ## Generate keys
 

src/client.rs

@@ -8,7 +8,12 @@ use bitcoin::Network;
 use thiserror::Error;
 
 const IS_NODE_READY: &str = "is_node_ready";
-const BLOCK_PROGRESS_WINDOW: Duration = Duration::from_secs(2);
+/// A healthy chain still has multi-second gaps between blocks (3.5 s between
+/// two consecutive blocks was observed on a chain averaging nine blocks per
+/// second), so progress is polled until the height moves, bounded by this
+/// timeout, instead of being judged from one fixed window.
+const BLOCK_PROGRESS_TIMEOUT: Duration = Duration::from_secs(20);
+const BLOCK_PROGRESS_POLL_INTERVAL: Duration = Duration::from_secs(1);
 
 /// The smallest reusable Arch client, currently limited to node health.
 #[derive(Clone, Debug)]
@@ -26,8 +31,9 @@ impl ArchKitClient {
         Ok(Self { rpc_url })
     }
 
-    /// Check that the node is ready and produces a new block during the
-    /// observation window.
+    /// Check that the node is ready and produces a new block within the
+    /// progress timeout. The returned `observation_window` is the time it
+    /// took to see the height move.
     pub fn health(&self) -> Result<HealthStatus, ArchKitError> {
         let started_at = Instant::now();
         let config = self.rpc_config();
@@ -43,23 +49,36 @@ impl ArchKitClient {
         let initial_block_height = client
             .get_block_count()
             .map_err(|source| self.rpc_error(source))?;
-        thread::sleep(BLOCK_PROGRESS_WINDOW);
-        let final_block_height = client
-            .get_block_count()
-            .map_err(|source| self.rpc_error(source))?;
-        require_progress(
+        let observation_started_at = Instant::now();
+        let progress = wait_for_progress(
             initial_block_height,
-            final_block_height,
-            BLOCK_PROGRESS_WINDOW.as_secs(),
-            &self.rpc_url,
+            BLOCK_PROGRESS_TIMEOUT,
+            BLOCK_PROGRESS_POLL_INTERVAL,
+            || {
+                client
+                    .get_block_count()
+                    .map_err(|source| self.rpc_error(source))
+            },
         )?;
+        let observation_window = observation_started_at.elapsed();
+        let final_block_height = match progress {
+            Progress::Advanced(height) => height,
+            Progress::Stalled(height) => {
+                return Err(ArchKitError::BlocksNotProgressing {
+                    rpc_url: self.rpc_url.clone(),
+                    initial_height: initial_block_height,
+                    final_height: height,
+                    observation_seconds: observation_window.as_secs(),
+                });
+            }
+        };
 
         Ok(HealthStatus {
             rpc_url: self.rpc_url.clone(),
             initial_block_height,
             final_block_height,
             rpc_latency,
-            observation_window: BLOCK_PROGRESS_WINDOW,
+            observation_window,
             total_elapsed: started_at.elapsed(),
         })
     }
@@ -144,21 +163,34 @@ fn require_ready(readiness: Option<bool>, rpc_url: &str) -> Result<(), ArchKitEr
     }
 }
 
-fn require_progress(
+/// Outcome of watching the block height after an initial sample.
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+enum Progress {
+    /// The height rose above the initial sample; carries the height seen.
+    Advanced(u64),
+    /// The timeout passed without the height rising; carries the last height.
+    Stalled(u64),
+}
+
+/// Poll `sample` every `poll_interval` until it exceeds `initial_height` or
+/// `timeout` elapses. A sampling error ends the wait immediately, since a
+/// node that stops answering is not healthy either.
+fn wait_for_progress<E>(
     initial_height: u64,
-    final_height: u64,
-    observation_seconds: u64,
-    rpc_url: &str,
-) -> Result<(), ArchKitError> {
-    if final_height > initial_height {
-        Ok(())
-    } else {
-        Err(ArchKitError::BlocksNotProgressing {
-            rpc_url: rpc_url.to_string(),
-            initial_height,
-            final_height,
-            observation_seconds,
-        })
+    timeout: Duration,
+    poll_interval: Duration,
+    mut sample: impl FnMut() -> Result<u64, E>,
+) -> Result<Progress, E> {
+    let started_at = Instant::now();
+    loop {
+        thread::sleep(poll_interval);
+        let height = sample()?;
+        if height > initial_height {
+            return Ok(Progress::Advanced(height));
+        }
+        if started_at.elapsed() >= timeout {
+            return Ok(Progress::Stalled(height));
+        }
     }
 }
 
@@ -188,20 +220,48 @@ mod tests {
     }
 
     #[test]
-    fn requires_the_block_height_to_increase() {
-        assert!(require_progress(100, 101, 2, "http://node").is_ok());
-
-        for final_height in [100, 99] {
-            assert!(matches!(
-                require_progress(100, final_height, 2, "http://node"),
-                Err(ArchKitError::BlocksNotProgressing {
-                    initial_height: 100,
-                    final_height: observed,
-                    observation_seconds: 2,
-                    ..
-                }) if observed == final_height
-            ));
-        }
+    fn progress_is_reported_as_soon_as_the_height_moves() {
+        let mut samples = [100, 100, 101].into_iter();
+        let progress = wait_for_progress::<()>(
+            100,
+            Duration::from_secs(10),
+            Duration::from_millis(1),
+            || Ok(samples.next().expect("sample")),
+        );
+        assert_eq!(progress, Ok(Progress::Advanced(101)));
+    }
+
+    #[test]
+    fn a_stalled_height_is_reported_after_the_timeout() {
+        let progress = wait_for_progress::<()>(
+            100,
+            Duration::from_millis(5),
+            Duration::from_millis(1),
+            || Ok(100),
+        );
+        assert_eq!(progress, Ok(Progress::Stalled(100)));
+    }
+
+    #[test]
+    fn a_lower_height_never_counts_as_progress() {
+        let progress = wait_for_progress::<()>(
+            100,
+            Duration::from_millis(5),
+            Duration::from_millis(1),
+            || Ok(99),
+        );
+        assert_eq!(progress, Ok(Progress::Stalled(99)));
+    }
+
+    #[test]
+    fn a_sampling_error_ends_the_wait() {
+        let progress = wait_for_progress(
+            100,
+            Duration::from_secs(10),
+            Duration::from_millis(1),
+            || Err("rpc down"),
+        );
+        assert_eq!(progress, Err("rpc down"));
     }
 
     #[test]