From 3d7c7c998eca91f2faeefb83566fd677a4040684 Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Thu, 8 Oct 2026 15:26:06 -0400 Subject: [PATCH 01/36] feat(daemon): improt enriched snapshots --- CHANGELOG.md | 2 + src/daemon/db_util.rs | 229 +++++++++++++++++++++++++++++++++++++++++- src/daemon/mod.rs | 14 ++- 3 files changed, 241 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e9fbdef4d9..fa78d9ddcfc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,8 @@ ### Added +- [#6802](https://github.com/ChainSafe/forest/issues/6802): When importing a snapshot from a URL, Forest now checks the snapshot's `.metadata.json` and, if extended snapshots are advertised, downloads and imports the receipts events and the tipset lookup alongside it. + ### Changed ### Removed diff --git a/src/daemon/db_util.rs b/src/daemon/db_util.rs index 2895c2c86fe..e2570db52d6 100644 --- a/src/daemon/db_util.rs +++ b/src/daemon/db_util.rs @@ -1,12 +1,13 @@ // Copyright 2019-2026 ChainSafe Systems // SPDX-License-Identifier: Apache-2.0, MIT -use crate::blocks::Tipset; -use crate::db::SettingsStoreExt; +use crate::blocks::{Tipset, TipsetKey}; +use crate::chain::{TIPSET_LOOKUP_HAMT_BIT_WIDTH, index::ChainIndex}; use crate::db::car::forest::{ FOREST_CAR_FILE_EXTENSION, TEMP_FOREST_CAR_FILE_EXTENSION, new_forest_car_temp_path_in, }; use crate::db::car::{ForestCar, ManyCar}; +use crate::db::{EthMappingsStore, MemoryDB, SettingsStoreExt}; use crate::ipld::ChainExportState; use crate::message::SignedMessage; use crate::networks::ChainConfig; @@ -14,10 +15,11 @@ use crate::prelude::*; use crate::rpc::sync::SnapshotProgressTracker; use crate::shim::clock::ChainEpoch; use crate::state_manager::StateManager; -use crate::utils::db::car_stream::CarStream; +use crate::utils::db::car_stream::{CarBlock, CarStream}; use crate::utils::io::EitherMmapOrRandomAccessFile; use crate::utils::net::{DownloadFileOption, download_to}; use anyhow::{Context, bail}; +use fil_actors_shared::fvm_ipld_hamt::Hamt; use futures::TryStreamExt; use serde::{Deserialize, Serialize}; use std::sync::LazyLock; @@ -333,6 +335,189 @@ async fn transcode_into_forest_car(from: &Path, to: &Path) -> anyhow::Result<()> Ok(()) } +/// The subset of the snapshot `.metadata.json` file published in the Forest archive +/// that is needed to locate the extended snapshots. +#[derive(Debug, Deserialize)] +struct SnapshotMetadataFile { + #[serde(rename = "Snapshot")] + snapshot: SnapshotMetadataSection, +} + +#[derive(Debug, Deserialize)] +struct SnapshotMetadataSection { + #[serde(rename = "Extended", default)] + extended: Option, +} + +/// Auxiliary snapshots that enrich a base snapshot, see `Forest.ChainExport`. +#[derive(Debug, Default, Deserialize, PartialEq)] +struct ExtendedSnapshots { + /// Message receipts and events + #[serde(rename = "Augmented", default)] + receipts_events: Option, + /// Tipset lookup table (checkpoint epoch to tipset key) + #[serde(rename = "Tipset lookup", default)] + tipset_lookup: Option, +} + +#[derive(Debug, Deserialize, PartialEq)] +struct ExtendedSnapshotFile { + #[serde(rename = "Filename")] + filename: String, + #[serde(rename = "Sha256")] + sha256: String, +} + +/// Fetches the `.metadata.json` published next to `snapshot_url` and returns its extended +/// snapshots, if any. A missing metadata file is not an error. +async fn fetch_extended_snapshots(snapshot_url: &Url) -> anyhow::Result> { + let metadata_url = Url::parse(&format!("{snapshot_url}{}", ".metadata.json"))?; + let response = crate::utils::net::global_http_client() + .get(metadata_url.clone()) + .send() + .await?; + if response.status() == reqwest::StatusCode::NOT_FOUND { + debug!("snapshot metadata not found at {metadata_url}"); + return Ok(None); + } + let metadata: SnapshotMetadataFile = response + .error_for_status()? + .json() + .await + .with_context(|| format!("failed to parse snapshot metadata at {metadata_url}"))?; + Ok(metadata.snapshot.extended) +} + +/// Downloads an extended snapshot that sits next to `snapshot_url` into a temporary file in +/// `temp_dir` and verifies its `SHA-256` checksum. +async fn download_extended_snapshot( + snapshot_url: &Url, + file: &ExtendedSnapshotFile, + temp_dir: &Path, +) -> anyhow::Result { + use sha2::{Digest as _, Sha256}; + + anyhow::ensure!( + !file.filename.is_empty() && !file.filename.contains(['/', '\\', '?', '#']), + "invalid extended snapshot filename: {}", + file.filename + ); + let url = snapshot_url.join(&file.filename)?; + let path = new_forest_car_temp_path_in(temp_dir)?; + info!("Downloading extended snapshot: {url}"); + download_to(&url, &path, DownloadFileOption::Resumable, None).await?; + let actual = crate::utils::encoding::hex::encode(Sha256::digest(tokio::fs::read(&path).await?)); + anyhow::ensure!( + actual.eq_ignore_ascii_case(&file.sha256), + "checksum mismatch for {}, expected: {}, actual: {actual}", + file.filename, + file.sha256 + ); + Ok(path) +} + +/// Imports message receipts and events from a receipts and events snapshot into `db`. +async fn import_receipts_events_snapshot( + db: &impl Blockstore, + path: &Path, +) -> anyhow::Result { + let mut car_stream = CarStream::new_from_path(path).await?; + let mut n_blocks = 0; + while let Some(CarBlock { cid, data }) = car_stream.try_next().await? { + db.put_keyed(&cid, &data)?; + n_blocks += 1; + } + Ok(n_blocks) +} + +/// Populates the tipset lookup table in `db` from a tipset lookup snapshot. +/// Fails without writing anything if an entry is not a checkpoint at or below `head`. +async fn import_tipset_lookup_snapshot( + db: &impl EthMappingsStore, + path: &Path, + head: &Tipset, +) -> anyhow::Result { + let store = MemoryDB::default(); + let mut car_stream = CarStream::new_from_path(path).await?; + let hamt_root = *car_stream.header_v1.roots.first(); + while let Some(CarBlock { cid, data }) = car_stream.try_next().await? { + store.put_keyed(&cid, &data)?; + } + let hamt: Hamt<_, TipsetKey, ChainEpoch> = + Hamt::load_with_bit_width(&hamt_root, &store, TIPSET_LOOKUP_HAMT_BIT_WIDTH) + .context("failed to load tipset lookup HAMT")?; + // Validate all entries before writing any of them. + let mut entries = vec![]; + hamt.for_each_cacheless(|&epoch, tsk| { + anyhow::ensure!( + ChainIndex::is_tipset_lookup_checkpoint(epoch) && epoch <= head.epoch(), + "invalid tipset lookup entry at epoch {epoch}, head epoch: {}", + head.epoch() + ); + entries.push((epoch, tsk.clone())); + Ok(()) + })?; + for (epoch, tsk) in &entries { + db.set_tipset_key_at_epoch_raw(*epoch, tsk)?; + } + Ok(entries.len()) +} + +/// Downloads the extended snapshots (message receipts and events, tipset lookup table) +/// advertised in the metadata of the snapshot at `snapshot_url` and imports them into `db`. +/// This is best-effort: failures are logged and never fail the base snapshot import. +pub async fn maybe_import_extended_snapshots( + db: &(impl Blockstore + EthMappingsStore), + snapshot_url: &Url, + head: &Tipset, + temp_dir: &Path, +) { + let extended = match fetch_extended_snapshots(snapshot_url).await { + Ok(Some(extended)) => extended, + Ok(None) => { + info!("No extended snapshots advertised for {snapshot_url}"); + return; + } + Err(e) => { + warn!("Failed to fetch snapshot metadata for {snapshot_url}: {e:#}"); + return; + } + }; + + if let Some(file) = &extended.receipts_events { + let result = async { + let path = download_extended_snapshot(snapshot_url, file, temp_dir).await?; + import_receipts_events_snapshot(db, &path).await + } + .await; + match result { + Ok(n) => info!( + "Imported {n} message receipt and event blocks from {}", + file.filename + ), + Err(e) => warn!( + "Failed to import receipts and events snapshot {}: {e:#}", + file.filename + ), + } + } + + if let Some(file) = &extended.tipset_lookup { + let result = async { + let path = download_extended_snapshot(snapshot_url, file, temp_dir).await?; + import_tipset_lookup_snapshot(db, &path, head).await + } + .await; + match result { + Ok(n) => info!("Imported {n} tipset lookup entries from {}", file.filename), + Err(e) => warn!( + "Failed to import tipset lookup snapshot {}: {e:#}", + file.filename + ), + } + } +} + /// Settings-store key under which index backfill persists the epoch of the last committed /// batch, so an interrupted backfill can be resumed from where it left off. pub const BACKFILL_CHECKPOINT_KEY: &str = "/index/backfill/checkpoint"; @@ -862,6 +1047,44 @@ pub async fn run_backfill( #[cfg(test)] mod test { use super::*; + + #[test] + fn parse_snapshot_metadata_with_extended() { + let json = r#"{ + "Snapshot": { + "Snapshot version": "2", + "Epoch": 4137600, + "Extended": { + "Augmented": { + "Filename": "forest_snapshot_calibnet_2026-10-08_height_4137600_receipts_events.forest.car.zst", + "Sha256": "9585fa94f3c84a117929dd1942014a6f4f79f47dbfdbf693122ff9acfa1311cc" + }, + "Tipset lookup": { + "Filename": "forest_snapshot_calibnet_2026-10-08_height_4137600_tipset_lookup.forest.car.zst", + "Sha256": "710262581a4fdea6a8cd162e4a65b42727475b83c44b0b0fe05aff72ce70c2f2" + } + } + }, + "Build Information": {} + }"#; + let metadata: SnapshotMetadataFile = serde_json::from_str(json).unwrap(); + let extended = metadata.snapshot.extended.unwrap(); + assert_eq!( + extended.receipts_events.unwrap().filename, + "forest_snapshot_calibnet_2026-10-08_height_4137600_receipts_events.forest.car.zst" + ); + assert_eq!( + extended.tipset_lookup.unwrap().sha256, + "710262581a4fdea6a8cd162e4a65b42727475b83c44b0b0fe05aff72ce70c2f2" + ); + } + + #[test] + fn parse_snapshot_metadata_without_extended() { + let json = r#"{ "Snapshot": { "Snapshot version": "2", "Epoch": 4137600 } }"#; + let metadata: SnapshotMetadataFile = serde_json::from_str(json).unwrap(); + assert!(metadata.snapshot.extended.is_none()); + } use rstest::rstest; // The backfill guard shares the chain-export single-flight slot, so serialize with the export diff --git a/src/daemon/mod.rs b/src/daemon/mod.rs index dedf803645e..f75cc7a1405 100644 --- a/src/daemon/mod.rs +++ b/src/daemon/mod.rs @@ -16,7 +16,10 @@ use crate::cli_shared::{ cli::{CliOpts, Config}, delete_chain_data, }; -use crate::daemon::{context::AppContext, db_util::import_chain_as_forest_car}; +use crate::daemon::{ + context::AppContext, + db_util::{import_chain_as_forest_car, maybe_import_extended_snapshots}, +}; use crate::db::gc::SnapshotGarbageCollector; use crate::db::ttl::EthMappingCollector; use crate::libp2p::{Libp2pService, PeerManager}; @@ -156,6 +159,15 @@ async fn maybe_import_snapshot( .await?; ctx.db .read_only_files(std::iter::once(car_db_path.clone()))?; + if let Ok(snapshot_url) = url::Url::parse(&path.display().to_string()) { + maybe_import_extended_snapshots( + &ctx.db, + &snapshot_url, + &ts, + &ctx.db_meta_data.get_forest_car_db_dir(), + ) + .await; + } let ts_epoch = ts.epoch(); // Explicitly set heaviest tipset here in case HEAD_KEY has already been set // in the current setting store From 7af312da5f0a6446efaf14d30fe4fee0d58064e0 Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Thu, 8 Oct 2026 16:10:16 -0400 Subject: [PATCH 02/36] chore: remove backfill from rpc-checks --- scripts/tests/external-rpc-checks/init.sh | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/scripts/tests/external-rpc-checks/init.sh b/scripts/tests/external-rpc-checks/init.sh index de1b89a5122..d032c3f6b18 100755 --- a/scripts/tests/external-rpc-checks/init.sh +++ b/scripts/tests/external-rpc-checks/init.sh @@ -1,14 +1,11 @@ #!/usr/bin/env bash -# Runs in the `init` service: imports the snapshot ./resolve.rb picked and -# back-fills the chain index, all before the daemon starts. +# Runs in the `init` service: imports the snapshot ./resolve.rb picked, along +# with its extended snapshots (receipts and events, tipset lookup), before the +# daemon starts. set -euo pipefail url=$(< /data/snapshot-url) -epoch=$(< /data/snapshot-epoch) -read -r chain start _ < /data/check-target +read -r chain _ < /data/check-target forest --chain="${chain}" --encrypt-keystore=false --import-snapshot="${url}" --halt-after-import - -# inclusive on both ends -forest-tool index backfill --chain="${chain}" --from="${epoch}" --to="${start}" From 3f4494ef86205e274f749b0cb52d442ea866b023 Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Fri, 9 Oct 2026 07:39:25 -0400 Subject: [PATCH 03/36] chore: changes to metadata url format --- src/daemon/db_util.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/daemon/db_util.rs b/src/daemon/db_util.rs index e2570db52d6..b2b76d200de 100644 --- a/src/daemon/db_util.rs +++ b/src/daemon/db_util.rs @@ -39,9 +39,6 @@ use walkdir::WalkDir; #[cfg(doc)] use crate::rpc::eth::types::EthHash; -#[cfg(doc)] -use crate::blocks::TipsetKey; - #[cfg(doc)] use cid::Cid; @@ -371,7 +368,7 @@ struct ExtendedSnapshotFile { /// Fetches the `.metadata.json` published next to `snapshot_url` and returns its extended /// snapshots, if any. A missing metadata file is not an error. async fn fetch_extended_snapshots(snapshot_url: &Url) -> anyhow::Result> { - let metadata_url = Url::parse(&format!("{snapshot_url}{}", ".metadata.json"))?; + let metadata_url = Url::parse(&format!("{snapshot_url}.metadata.json"))?; let response = crate::utils::net::global_http_client() .get(metadata_url.clone()) .send() From 094843485594add57df12329f65fb9b37ce50e55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ecl=C3=A9sio=20Junior?= Date: Thu, 15 Oct 2026 08:00:00 -0400 Subject: [PATCH 04/36] Update CHANGELOG.md Co-authored-by: Shashank <99187193+sudo-shashank@users.noreply.github.com> --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 707f309bb81..5c05a69c2d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,7 +33,7 @@ ### Added -- [#6802](https://github.com/ChainSafe/forest/issues/6802): When importing a snapshot from a URL, Forest now checks the snapshot's `.metadata.json` and, if extended snapshots are advertised, downloads and imports the receipts events and the tipset lookup alongside it. +- [#6802](https://github.com/ChainSafe/forest/issues/6802): When importing a snapshot from a URL, Forest now checks the snapshot's `.metadata.json` and, if extended snapshots are advertised, downloads and imports the receipts, events, and the tipset lookup alongside it. ### Changed From 6c1c8ea089411081ad4aa0e813eba8249b55d85b Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Fri, 16 Oct 2026 09:05:37 -0400 Subject: [PATCH 05/36] chore: batch writes to db to improve speed --- src/daemon/db_util.rs | 107 +++++++++++++++++++++++++++-------------- src/db/car/many.rs | 7 +++ src/db/mod.rs | 12 +++++ src/db/parity_db.rs | 41 ++++++++++++++++ src/db/parity_db/gc.rs | 7 +++ 5 files changed, 137 insertions(+), 37 deletions(-) diff --git a/src/daemon/db_util.rs b/src/daemon/db_util.rs index b2b76d200de..39c9151d20f 100644 --- a/src/daemon/db_util.rs +++ b/src/daemon/db_util.rs @@ -366,7 +366,7 @@ struct ExtendedSnapshotFile { } /// Fetches the `.metadata.json` published next to `snapshot_url` and returns its extended -/// snapshots, if any. A missing metadata file is not an error. +/// snapshots, if any. async fn fetch_extended_snapshots(snapshot_url: &Url) -> anyhow::Result> { let metadata_url = Url::parse(&format!("{snapshot_url}.metadata.json"))?; let response = crate::utils::net::global_http_client() @@ -385,8 +385,10 @@ async fn fetch_extended_snapshots(snapshot_url: &Url) -> anyhow::Result anyhow::Result { +/// Imports message receipts and events from a receipts and events snapshot into `db`, +/// committing [`EXTENDED_SNAPSHOT_IMPORT_BATCH_SIZE`] blocks at a time. +async fn import_receipts_events_snapshot(db: &Arc, path: &Path) -> anyhow::Result +where + DB: Blockstore + Send + Sync + 'static, +{ + async fn flush( + db: &Arc, + batch: Vec<(Cid, bytes::Bytes)>, + ) -> anyhow::Result<()> { + let db = db.clone(); + tokio::task::spawn_blocking(move || db.put_many_keyed(batch)).await? + } + let mut car_stream = CarStream::new_from_path(path).await?; let mut n_blocks = 0; + let mut batch = Vec::with_capacity(EXTENDED_SNAPSHOT_IMPORT_BATCH_SIZE); while let Some(CarBlock { cid, data }) = car_stream.try_next().await? { - db.put_keyed(&cid, &data)?; + batch.push((cid, data)); n_blocks += 1; + if batch.len() == EXTENDED_SNAPSHOT_IMPORT_BATCH_SIZE { + flush( + db, + std::mem::replace( + &mut batch, + Vec::with_capacity(EXTENDED_SNAPSHOT_IMPORT_BATCH_SIZE), + ), + ) + .await?; + } + } + if !batch.is_empty() { + flush(db, batch).await?; } Ok(n_blocks) } /// Populates the tipset lookup table in `db` from a tipset lookup snapshot. -/// Fails without writing anything if an entry is not a checkpoint at or below `head`. -async fn import_tipset_lookup_snapshot( - db: &impl EthMappingsStore, +async fn import_tipset_lookup_snapshot( + db: &Arc, path: &Path, - head: &Tipset, -) -> anyhow::Result { + head_epoch: ChainEpoch, +) -> anyhow::Result +where + DB: EthMappingsStore + Send + Sync + 'static, +{ let store = MemoryDB::default(); let mut car_stream = CarStream::new_from_path(path).await?; let hamt_root = *car_stream.header_v1.roots.first(); while let Some(CarBlock { cid, data }) = car_stream.try_next().await? { store.put_keyed(&cid, &data)?; } - let hamt: Hamt<_, TipsetKey, ChainEpoch> = - Hamt::load_with_bit_width(&hamt_root, &store, TIPSET_LOOKUP_HAMT_BIT_WIDTH) - .context("failed to load tipset lookup HAMT")?; - // Validate all entries before writing any of them. - let mut entries = vec![]; - hamt.for_each_cacheless(|&epoch, tsk| { - anyhow::ensure!( - ChainIndex::is_tipset_lookup_checkpoint(epoch) && epoch <= head.epoch(), - "invalid tipset lookup entry at epoch {epoch}, head epoch: {}", - head.epoch() - ); - entries.push((epoch, tsk.clone())); - Ok(()) - })?; - for (epoch, tsk) in &entries { - db.set_tipset_key_at_epoch_raw(*epoch, tsk)?; - } - Ok(entries.len()) + + let db = db.clone(); + tokio::task::spawn_blocking(move || { + let hamt: Hamt<_, TipsetKey, ChainEpoch> = + Hamt::load_with_bit_width(&hamt_root, &store, TIPSET_LOOKUP_HAMT_BIT_WIDTH) + .context("failed to load tipset lookup HAMT")?; + // Validate all entries before writing any of them. + let mut entries = vec![]; + hamt.for_each_cacheless(|&epoch, tsk| { + anyhow::ensure!( + ChainIndex::is_tipset_lookup_checkpoint(epoch) && epoch <= head_epoch, + "invalid tipset lookup entry at epoch {epoch}, head epoch: {head_epoch}", + ); + entries.push((epoch, tsk.clone())); + Ok(()) + })?; + for batch in entries.chunks(EXTENDED_SNAPSHOT_IMPORT_BATCH_SIZE) { + db.set_tipset_keys_at_epochs_raw(batch)?; + } + Ok(entries.len()) + }) + .await? } /// Downloads the extended snapshots (message receipts and events, tipset lookup table) /// advertised in the metadata of the snapshot at `snapshot_url` and imports them into `db`. -/// This is best-effort: failures are logged and never fail the base snapshot import. -pub async fn maybe_import_extended_snapshots( - db: &(impl Blockstore + EthMappingsStore), +pub async fn maybe_import_extended_snapshots( + db: &Arc, snapshot_url: &Url, head: &Tipset, temp_dir: &Path, -) { +) where + DB: Blockstore + EthMappingsStore + Send + Sync + 'static, +{ let extended = match fetch_extended_snapshots(snapshot_url).await { Ok(Some(extended)) => extended, Ok(None) => { @@ -502,7 +535,7 @@ pub async fn maybe_import_extended_snapshots( if let Some(file) = &extended.tipset_lookup { let result = async { let path = download_extended_snapshot(snapshot_url, file, temp_dir).await?; - import_tipset_lookup_snapshot(db, &path, head).await + import_tipset_lookup_snapshot(db, &path, head.epoch()).await } .await; match result { diff --git a/src/db/car/many.rs b/src/db/car/many.rs index 2e870be35be..96a3a44aee7 100644 --- a/src/db/car/many.rs +++ b/src/db/car/many.rs @@ -321,6 +321,13 @@ impl EthMappingsStore for ManyCar { ) -> anyhow::Result<()> { EthMappingsStore::set_tipset_key_at_epoch_raw(self.writer(), epoch, tsk) } + + fn set_tipset_keys_at_epochs_raw( + &self, + entries: &[(ChainEpoch, TipsetKey)], + ) -> anyhow::Result<()> { + EthMappingsStore::set_tipset_keys_at_epochs_raw(self.writer(), entries) + } } impl EthBlockBloomStore for ManyCar { diff --git a/src/db/mod.rs b/src/db/mod.rs index 90c0f466423..4d6c53d018c 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -123,6 +123,18 @@ pub trait EthMappingsStore { fn set_tipset_key_at_epoch_raw(&self, epoch: ChainEpoch, tsk: &TipsetKey) -> anyhow::Result<()>; + /// Writes the tipset keys for the given epochs(heights) to the store. Implementations + /// should commit them in a single batch. + fn set_tipset_keys_at_epochs_raw( + &self, + entries: &[(ChainEpoch, TipsetKey)], + ) -> anyhow::Result<()> { + for (epoch, tsk) in entries { + EthMappingsStore::set_tipset_key_at_epoch_raw(self, *epoch, tsk)?; + } + Ok(()) + } + /// Writes the tipset key for a given epoch(height) to the store. fn set_tipset_key_at_epoch(&self, ts: &Tipset) -> anyhow::Result<()> { EthMappingsStore::set_tipset_key_at_epoch_raw(self, ts.epoch(), ts.key()) diff --git a/src/db/parity_db.rs b/src/db/parity_db.rs index 684ba2f11c9..33dda17f76e 100644 --- a/src/db/parity_db.rs +++ b/src/db/parity_db.rs @@ -263,6 +263,25 @@ impl EthMappingsStore for ParityDb { let bytes = fvm_ipld_encoding::to_vec(tsk)?; self.write_to_column(key, bytes, DbColumn::EthMappings) } + + fn set_tipset_keys_at_epochs_raw( + &self, + entries: &[(ChainEpoch, TipsetKey)], + ) -> anyhow::Result<()> { + let tx = entries + .iter() + .map(|(epoch, tsk)| { + Ok(( + DbColumn::EthMappings as u8, + epoch.to_le_bytes(), + Some(fvm_ipld_encoding::to_vec(tsk)?), + )) + }) + .collect::>>()?; + self.db + .commit(tx) + .with_context(|| format!("error writing to column {}", DbColumn::EthMappings)) + } } impl EthBlockBloomStore for ParityDb { @@ -636,4 +655,26 @@ mod test { assert!(db.write_ops_broadcast_tx.read().is_none()); } + + #[test] + fn set_tipset_keys_at_epochs_raw_test() { + let db = TempParityDB::new(); + let tsk = |n: u64| { + TipsetKey::from(nunny::vec![Cid::new_v1( + DAG_CBOR, + MultihashCode::Blake2b256.digest(&n.to_le_bytes()) + )]) + }; + let entries = [(20, tsk(1)), (40, tsk(2)), (60, tsk(3))]; + db.set_tipset_keys_at_epochs_raw(&entries).unwrap(); + for (epoch, expected) in &entries { + assert_eq!( + db.tipset_key_by_epoch(*epoch).unwrap().as_ref(), + Some(expected) + ); + } + assert!(db.tipset_key_by_epoch(80).unwrap().is_none()); + // Empty batches are a no-op. + db.set_tipset_keys_at_epochs_raw(&[]).unwrap(); + } } diff --git a/src/db/parity_db/gc.rs b/src/db/parity_db/gc.rs index bbecc51284d..757160cc241 100644 --- a/src/db/parity_db/gc.rs +++ b/src/db/parity_db/gc.rs @@ -167,6 +167,13 @@ impl EthMappingsStore for GarbageCollectableParityDb { ) -> anyhow::Result<()> { EthMappingsStore::set_tipset_key_at_epoch_raw(&*self.db.read(), epoch, tsk) } + + fn set_tipset_keys_at_epochs_raw( + &self, + entries: &[(ChainEpoch, TipsetKey)], + ) -> anyhow::Result<()> { + EthMappingsStore::set_tipset_keys_at_epochs_raw(&*self.db.read(), entries) + } } impl EthBlockBloomStore for GarbageCollectableParityDb { From 9713afe869dfa84d792d237e1bb3ab670361b525 Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Mon, 19 Oct 2026 11:38:41 -0400 Subject: [PATCH 06/36] chore: address comments --- src/daemon/db_util.rs | 271 ++------------------------------ src/daemon/extended_snapshot.rs | 258 ++++++++++++++++++++++++++++++ src/daemon/mod.rs | 21 ++- src/db/car/many.rs | 7 - src/db/mod.rs | 12 -- src/db/parity_db.rs | 41 ----- src/db/parity_db/gc.rs | 7 - 7 files changed, 278 insertions(+), 339 deletions(-) create mode 100644 src/daemon/extended_snapshot.rs diff --git a/src/daemon/db_util.rs b/src/daemon/db_util.rs index 39c9151d20f..e265731c0eb 100644 --- a/src/daemon/db_util.rs +++ b/src/daemon/db_util.rs @@ -1,13 +1,12 @@ // Copyright 2019-2026 ChainSafe Systems // SPDX-License-Identifier: Apache-2.0, MIT -use crate::blocks::{Tipset, TipsetKey}; -use crate::chain::{TIPSET_LOOKUP_HAMT_BIT_WIDTH, index::ChainIndex}; +use crate::blocks::Tipset; +use crate::db::SettingsStoreExt; use crate::db::car::forest::{ FOREST_CAR_FILE_EXTENSION, TEMP_FOREST_CAR_FILE_EXTENSION, new_forest_car_temp_path_in, }; use crate::db::car::{ForestCar, ManyCar}; -use crate::db::{EthMappingsStore, MemoryDB, SettingsStoreExt}; use crate::ipld::ChainExportState; use crate::message::SignedMessage; use crate::networks::ChainConfig; @@ -15,11 +14,10 @@ use crate::prelude::*; use crate::rpc::sync::SnapshotProgressTracker; use crate::shim::clock::ChainEpoch; use crate::state_manager::StateManager; -use crate::utils::db::car_stream::{CarBlock, CarStream}; +use crate::utils::db::car_stream::CarStream; use crate::utils::io::EitherMmapOrRandomAccessFile; use crate::utils::net::{DownloadFileOption, download_to}; use anyhow::{Context, bail}; -use fil_actors_shared::fvm_ipld_hamt::Hamt; use futures::TryStreamExt; use serde::{Deserialize, Serialize}; use std::sync::LazyLock; @@ -153,7 +151,7 @@ pub async fn import_chain_as_forest_car( let forest_car_db_path = forest_car_db_path.clone(); async move { let downloaded_car_temp_path = new_forest_car_temp_path_in(forest_car_db_dir)?; - if let Ok(url) = Url::parse(&from_path.display().to_string()) { + if let Some(url) = path_as_url(from_path) { download_to( &url, &downloaded_car_temp_path, @@ -193,7 +191,7 @@ pub async fn import_chain_as_forest_car( match import_mode { ImportMode::Auto => { - if Url::parse(&from_path.display().to_string()).is_ok() { + if path_as_url(from_path).is_some() { // Fallback to move if from_path is url move_or_copy(ImportMode::Move).await?; } else if ForestCar::is_valid(&EitherMmapOrRandomAccessFile::open(from_path)?) { @@ -291,6 +289,11 @@ pub async fn import_chain_as_forest_car( Ok((forest_car_db_path, ts)) } +/// Parses `path` as a URL, returning `None` for local paths. +pub(super) fn path_as_url(path: &Path) -> Option { + path.to_str().and_then(|s| Url::parse(s).ok()) +} + fn move_or_copy_file(from: &Path, to: &Path, import_mode: ImportMode) -> anyhow::Result<()> { match import_mode { ImportMode::Move => { @@ -332,222 +335,6 @@ async fn transcode_into_forest_car(from: &Path, to: &Path) -> anyhow::Result<()> Ok(()) } -/// The subset of the snapshot `.metadata.json` file published in the Forest archive -/// that is needed to locate the extended snapshots. -#[derive(Debug, Deserialize)] -struct SnapshotMetadataFile { - #[serde(rename = "Snapshot")] - snapshot: SnapshotMetadataSection, -} - -#[derive(Debug, Deserialize)] -struct SnapshotMetadataSection { - #[serde(rename = "Extended", default)] - extended: Option, -} - -/// Auxiliary snapshots that enrich a base snapshot, see `Forest.ChainExport`. -#[derive(Debug, Default, Deserialize, PartialEq)] -struct ExtendedSnapshots { - /// Message receipts and events - #[serde(rename = "Augmented", default)] - receipts_events: Option, - /// Tipset lookup table (checkpoint epoch to tipset key) - #[serde(rename = "Tipset lookup", default)] - tipset_lookup: Option, -} - -#[derive(Debug, Deserialize, PartialEq)] -struct ExtendedSnapshotFile { - #[serde(rename = "Filename")] - filename: String, - #[serde(rename = "Sha256")] - sha256: String, -} - -/// Fetches the `.metadata.json` published next to `snapshot_url` and returns its extended -/// snapshots, if any. -async fn fetch_extended_snapshots(snapshot_url: &Url) -> anyhow::Result> { - let metadata_url = Url::parse(&format!("{snapshot_url}.metadata.json"))?; - let response = crate::utils::net::global_http_client() - .get(metadata_url.clone()) - .send() - .await?; - if response.status() == reqwest::StatusCode::NOT_FOUND { - debug!("snapshot metadata not found at {metadata_url}"); - return Ok(None); - } - let metadata: SnapshotMetadataFile = response - .error_for_status()? - .json() - .await - .with_context(|| format!("failed to parse snapshot metadata at {metadata_url}"))?; - Ok(metadata.snapshot.extended) -} - -/// Number of blocks or tipset lookup entries written to the database per commit when importing -/// extended snapshots. -const EXTENDED_SNAPSHOT_IMPORT_BATCH_SIZE: usize = 1_000; - -async fn download_extended_snapshot( - snapshot_url: &Url, - file: &ExtendedSnapshotFile, - temp_dir: &Path, -) -> anyhow::Result { - use sha2::{Digest as _, Sha256}; - - anyhow::ensure!( - !file.filename.is_empty() && !file.filename.contains(['/', '\\', '?', '#']), - "invalid extended snapshot filename: {}", - file.filename - ); - let url = snapshot_url.join(&file.filename)?; - let path = new_forest_car_temp_path_in(temp_dir)?; - info!("Downloading extended snapshot: {url}"); - download_to(&url, &path, DownloadFileOption::Resumable, None).await?; - - let actual = crate::utils::encoding::hex::encode(Sha256::digest(tokio::fs::read(&path).await?)); - anyhow::ensure!( - actual.eq_ignore_ascii_case(&file.sha256), - "checksum mismatch for {}, expected: {}, actual: {actual}", - file.filename, - file.sha256 - ); - Ok(path) -} - -/// Imports message receipts and events from a receipts and events snapshot into `db`, -/// committing [`EXTENDED_SNAPSHOT_IMPORT_BATCH_SIZE`] blocks at a time. -async fn import_receipts_events_snapshot(db: &Arc, path: &Path) -> anyhow::Result -where - DB: Blockstore + Send + Sync + 'static, -{ - async fn flush( - db: &Arc, - batch: Vec<(Cid, bytes::Bytes)>, - ) -> anyhow::Result<()> { - let db = db.clone(); - tokio::task::spawn_blocking(move || db.put_many_keyed(batch)).await? - } - - let mut car_stream = CarStream::new_from_path(path).await?; - let mut n_blocks = 0; - let mut batch = Vec::with_capacity(EXTENDED_SNAPSHOT_IMPORT_BATCH_SIZE); - while let Some(CarBlock { cid, data }) = car_stream.try_next().await? { - batch.push((cid, data)); - n_blocks += 1; - if batch.len() == EXTENDED_SNAPSHOT_IMPORT_BATCH_SIZE { - flush( - db, - std::mem::replace( - &mut batch, - Vec::with_capacity(EXTENDED_SNAPSHOT_IMPORT_BATCH_SIZE), - ), - ) - .await?; - } - } - if !batch.is_empty() { - flush(db, batch).await?; - } - Ok(n_blocks) -} - -/// Populates the tipset lookup table in `db` from a tipset lookup snapshot. -async fn import_tipset_lookup_snapshot( - db: &Arc, - path: &Path, - head_epoch: ChainEpoch, -) -> anyhow::Result -where - DB: EthMappingsStore + Send + Sync + 'static, -{ - let store = MemoryDB::default(); - let mut car_stream = CarStream::new_from_path(path).await?; - let hamt_root = *car_stream.header_v1.roots.first(); - while let Some(CarBlock { cid, data }) = car_stream.try_next().await? { - store.put_keyed(&cid, &data)?; - } - - let db = db.clone(); - tokio::task::spawn_blocking(move || { - let hamt: Hamt<_, TipsetKey, ChainEpoch> = - Hamt::load_with_bit_width(&hamt_root, &store, TIPSET_LOOKUP_HAMT_BIT_WIDTH) - .context("failed to load tipset lookup HAMT")?; - // Validate all entries before writing any of them. - let mut entries = vec![]; - hamt.for_each_cacheless(|&epoch, tsk| { - anyhow::ensure!( - ChainIndex::is_tipset_lookup_checkpoint(epoch) && epoch <= head_epoch, - "invalid tipset lookup entry at epoch {epoch}, head epoch: {head_epoch}", - ); - entries.push((epoch, tsk.clone())); - Ok(()) - })?; - for batch in entries.chunks(EXTENDED_SNAPSHOT_IMPORT_BATCH_SIZE) { - db.set_tipset_keys_at_epochs_raw(batch)?; - } - Ok(entries.len()) - }) - .await? -} - -/// Downloads the extended snapshots (message receipts and events, tipset lookup table) -/// advertised in the metadata of the snapshot at `snapshot_url` and imports them into `db`. -pub async fn maybe_import_extended_snapshots( - db: &Arc, - snapshot_url: &Url, - head: &Tipset, - temp_dir: &Path, -) where - DB: Blockstore + EthMappingsStore + Send + Sync + 'static, -{ - let extended = match fetch_extended_snapshots(snapshot_url).await { - Ok(Some(extended)) => extended, - Ok(None) => { - info!("No extended snapshots advertised for {snapshot_url}"); - return; - } - Err(e) => { - warn!("Failed to fetch snapshot metadata for {snapshot_url}: {e:#}"); - return; - } - }; - - if let Some(file) = &extended.receipts_events { - let result = async { - let path = download_extended_snapshot(snapshot_url, file, temp_dir).await?; - import_receipts_events_snapshot(db, &path).await - } - .await; - match result { - Ok(n) => info!( - "Imported {n} message receipt and event blocks from {}", - file.filename - ), - Err(e) => warn!( - "Failed to import receipts and events snapshot {}: {e:#}", - file.filename - ), - } - } - - if let Some(file) = &extended.tipset_lookup { - let result = async { - let path = download_extended_snapshot(snapshot_url, file, temp_dir).await?; - import_tipset_lookup_snapshot(db, &path, head.epoch()).await - } - .await; - match result { - Ok(n) => info!("Imported {n} tipset lookup entries from {}", file.filename), - Err(e) => warn!( - "Failed to import tipset lookup snapshot {}: {e:#}", - file.filename - ), - } - } -} - /// Settings-store key under which index backfill persists the epoch of the last committed /// batch, so an interrupted backfill can be resumed from where it left off. pub const BACKFILL_CHECKPOINT_KEY: &str = "/index/backfill/checkpoint"; @@ -1077,44 +864,6 @@ pub async fn run_backfill( #[cfg(test)] mod test { use super::*; - - #[test] - fn parse_snapshot_metadata_with_extended() { - let json = r#"{ - "Snapshot": { - "Snapshot version": "2", - "Epoch": 4137600, - "Extended": { - "Augmented": { - "Filename": "forest_snapshot_calibnet_2026-10-08_height_4137600_receipts_events.forest.car.zst", - "Sha256": "9585fa94f3c84a117929dd1942014a6f4f79f47dbfdbf693122ff9acfa1311cc" - }, - "Tipset lookup": { - "Filename": "forest_snapshot_calibnet_2026-10-08_height_4137600_tipset_lookup.forest.car.zst", - "Sha256": "710262581a4fdea6a8cd162e4a65b42727475b83c44b0b0fe05aff72ce70c2f2" - } - } - }, - "Build Information": {} - }"#; - let metadata: SnapshotMetadataFile = serde_json::from_str(json).unwrap(); - let extended = metadata.snapshot.extended.unwrap(); - assert_eq!( - extended.receipts_events.unwrap().filename, - "forest_snapshot_calibnet_2026-10-08_height_4137600_receipts_events.forest.car.zst" - ); - assert_eq!( - extended.tipset_lookup.unwrap().sha256, - "710262581a4fdea6a8cd162e4a65b42727475b83c44b0b0fe05aff72ce70c2f2" - ); - } - - #[test] - fn parse_snapshot_metadata_without_extended() { - let json = r#"{ "Snapshot": { "Snapshot version": "2", "Epoch": 4137600 } }"#; - let metadata: SnapshotMetadataFile = serde_json::from_str(json).unwrap(); - assert!(metadata.snapshot.extended.is_none()); - } use rstest::rstest; // The backfill guard shares the chain-export single-flight slot, so serialize with the export diff --git a/src/daemon/extended_snapshot.rs b/src/daemon/extended_snapshot.rs new file mode 100644 index 00000000000..41a66e098e9 --- /dev/null +++ b/src/daemon/extended_snapshot.rs @@ -0,0 +1,258 @@ +// Copyright 2019-2026 ChainSafe Systems +// SPDX-License-Identifier: Apache-2.0, MIT + +//! Download and import of the extended snapshots (receipts/events and tipset lookup) +//! published next to a snapshot in the Forest archive. + +use crate::blocks::{Tipset, TipsetKey}; +use crate::chain::{TIPSET_LOOKUP_HAMT_BIT_WIDTH, index::ChainIndex}; +use crate::daemon::db_util::path_as_url; +use crate::db::car::forest::new_forest_car_temp_path_in; +use crate::db::{EthMappingsStore, MemoryDB}; +use crate::prelude::*; +use crate::shim::clock::ChainEpoch; +use crate::utils::db::car_stream::{CarBlock, CarStream}; +use crate::utils::net::{DownloadFileOption, download_to}; +use anyhow::Context; +use fil_actors_shared::fvm_ipld_hamt::Hamt; +use futures::TryStreamExt; +use regex::Regex; +use serde::Deserialize; +use std::path::Path; +use std::sync::{Arc, LazyLock}; +use tracing::{info, warn}; +use url::Url; + +static FILENAME_RE: LazyLock = + LazyLock::new(|| Regex::new(r"^[A-Za-z0-9_.-]+\.forest\.car\.zst$").expect("valid regex")); + +/// The subset of the snapshot `.metadata.json` file published in the Forest archive +/// that is needed to locate the extended snapshots. +#[derive(Debug, Deserialize)] +struct SnapshotMetadataFile { + #[serde(rename = "Snapshot")] + snapshot: SnapshotMetadataSection, +} + +#[derive(Debug, Deserialize)] +struct SnapshotMetadataSection { + #[serde(rename = "Extended", default)] + extended: Option, +} + +/// Auxiliary snapshots that enrich a base snapshot, see `Forest.ChainExport`. +#[derive(Debug, Default, Deserialize, PartialEq)] +struct ExtendedSnapshots { + /// Message receipts and events + #[serde(rename = "Augmented", default)] + receipts_events: Option, + /// Tipset lookup table (checkpoint epoch to tipset key) + #[serde(rename = "Tipset lookup", default)] + tipset_lookup: Option, +} + +#[derive(Debug, Deserialize, PartialEq)] +struct ExtendedSnapshotFile { + #[serde(rename = "Filename")] + filename: String, + #[serde(rename = "Sha256")] + #[serde(with = "crate::utils::encoding::hex::serde")] + sha256: [u8; 32], +} + +/// Fetches the `.metadata.json` published next to `snapshot_url` and returns its extended +/// snapshots, if any. +async fn fetch_extended_snapshots(snapshot_url: &Url) -> anyhow::Result> { + let metadata_url = Url::parse(&format!("{snapshot_url}.metadata.json"))?; + let response = crate::utils::net::global_http_client() + .get(metadata_url.clone()) + .send() + .await?; + let metadata: SnapshotMetadataFile = response + .error_for_status()? + .json() + .await + .with_context(|| format!("failed to parse snapshot metadata at {metadata_url}"))?; + Ok(metadata.snapshot.extended) +} + +async fn download_extended_snapshot( + snapshot_url: &Url, + file: &ExtendedSnapshotFile, + temp_dir: &Path, +) -> anyhow::Result { + use sha2::{Digest as _, Sha256}; + + anyhow::ensure!( + FILENAME_RE.is_match(&file.filename), + "invalid extended snapshot filename: {}", + file.filename + ); + let url = snapshot_url.join(&file.filename)?; + let path = new_forest_car_temp_path_in(temp_dir)?; + info!("Downloading extended snapshot: {url}"); + download_to(&url, &path, DownloadFileOption::Resumable, None).await?; + + let actual: [u8; 32] = Sha256::digest(tokio::fs::read(&path).await?).into(); + if actual != file.sha256 { + anyhow::bail!("checksum mismatch for {}", file.filename); + } + Ok(path) +} + +/// Imports message receipts and events from a receipts and events snapshot into `db`. +async fn import_receipts_events_snapshot(db: &DB, path: &Path) -> anyhow::Result +where + DB: Blockstore, +{ + let mut car_stream = CarStream::new_from_path(path).await?; + let mut n_blocks = 0; + while let Some(CarBlock { cid, data }) = car_stream.try_next().await? { + db.put_keyed(&cid, &data)?; + n_blocks += 1; + } + Ok(n_blocks) +} + +/// Populates the tipset lookup table in `db` from a tipset lookup snapshot. +async fn import_tipset_lookup_snapshot( + db: &Arc, + path: &Path, + head_epoch: ChainEpoch, +) -> anyhow::Result +where + DB: EthMappingsStore + Send + Sync + 'static, +{ + let store = MemoryDB::default(); + let mut car_stream = CarStream::new_from_path(path).await?; + let hamt_root = *car_stream.header_v1.roots.first(); + while let Some(CarBlock { cid, data }) = car_stream.try_next().await? { + store.put_keyed(&cid, &data)?; + } + + let db = db.clone(); + tokio::task::spawn_blocking(move || { + let hamt: Hamt<_, TipsetKey, ChainEpoch> = + Hamt::load_with_bit_width(&hamt_root, &store, TIPSET_LOOKUP_HAMT_BIT_WIDTH) + .context("failed to load tipset lookup HAMT")?; + // Validate all entries before writing any of them. + let mut n_entries = 0; + hamt.for_each_cacheless(|&epoch, _| { + anyhow::ensure!( + ChainIndex::is_tipset_lookup_checkpoint(epoch) && epoch <= head_epoch, + "invalid tipset lookup entry at epoch {epoch}, head epoch: {head_epoch}", + ); + n_entries += 1; + Ok(()) + })?; + hamt.for_each_cacheless(|&epoch, tsk| db.set_tipset_key_at_epoch_raw(epoch, tsk))?; + Ok(n_entries) + }) + .await? +} + +/// Downloads the extended snapshots (message receipts and events, tipset lookup table) +/// advertised in the metadata of the snapshot at `snapshot_path` and imports them into `db`. +/// Does nothing when `snapshot_path` is a local file rather than a URL. +pub async fn maybe_import_extended_snapshots( + db: &Arc, + snapshot_path: &Path, + head: &Tipset, + temp_dir: &Path, +) where + DB: Blockstore + EthMappingsStore + Send + Sync + 'static, +{ + // Extended snapshots are only published next to remote snapshots. + let Some(snapshot_url) = path_as_url(snapshot_path) else { + return; + }; + let snapshot_url = &snapshot_url; + let extended = match fetch_extended_snapshots(snapshot_url).await { + Ok(Some(extended)) => extended, + Ok(None) => { + info!("No extended snapshots advertised for {snapshot_url}"); + return; + } + Err(e) => { + warn!("Failed to fetch snapshot metadata for {snapshot_url}: {e:#}"); + return; + } + }; + + if let Some(file) = &extended.receipts_events { + let result = async { + let path = download_extended_snapshot(snapshot_url, file, temp_dir).await?; + import_receipts_events_snapshot(db, &path).await + } + .await; + match result { + Ok(n) => info!( + "Imported {n} message receipt and event blocks from {}", + file.filename + ), + Err(e) => warn!( + "Failed to import receipts and events snapshot {}: {e:#}", + file.filename + ), + } + } + + if let Some(file) = &extended.tipset_lookup { + let result = async { + let path = download_extended_snapshot(snapshot_url, file, temp_dir).await?; + import_tipset_lookup_snapshot(db, &path, head.epoch()).await + } + .await; + match result { + Ok(n) => info!("Imported {n} tipset lookup entries from {}", file.filename), + Err(e) => warn!( + "Failed to import tipset lookup snapshot {}: {e:#}", + file.filename + ), + } + } +} + +#[cfg(test)] +mod test { + use super::*; + use crate::utils::encoding::hex; + + #[test] + fn parse_snapshot_metadata_with_extended() { + let json = r#"{ + "Snapshot": { + "Snapshot version": "2", + "Epoch": 4137600, + "Extended": { + "Augmented": { + "Filename": "forest_snapshot_calibnet_2026-10-08_height_4137600_receipts_events.forest.car.zst", + "Sha256": "9585fa94f3c84a117929dd1942014a6f4f79f47dbfdbf693122ff9acfa1311cc" + }, + "Tipset lookup": { + "Filename": "forest_snapshot_calibnet_2026-10-08_height_4137600_tipset_lookup.forest.car.zst", + "Sha256": "710262581a4fdea6a8cd162e4a65b42727475b83c44b0b0fe05aff72ce70c2f2" + } + } + }, + "Build Information": {} + }"#; + let metadata: SnapshotMetadataFile = serde_json::from_str(json).unwrap(); + let extended = metadata.snapshot.extended.unwrap(); + assert_eq!( + extended.receipts_events.unwrap().filename, + "forest_snapshot_calibnet_2026-10-08_height_4137600_receipts_events.forest.car.zst" + ); + assert_eq!( + hex::encode(extended.tipset_lookup.unwrap().sha256), + "710262581a4fdea6a8cd162e4a65b42727475b83c44b0b0fe05aff72ce70c2f2" + ); + } + + #[test] + fn parse_snapshot_metadata_without_extended() { + let json = r#"{ "Snapshot": { "Snapshot version": "2", "Epoch": 4137600 } }"#; + let metadata: SnapshotMetadataFile = serde_json::from_str(json).unwrap(); + assert!(metadata.snapshot.extended.is_none()); + } +} diff --git a/src/daemon/mod.rs b/src/daemon/mod.rs index 3b3933d43a7..ff395c43b13 100644 --- a/src/daemon/mod.rs +++ b/src/daemon/mod.rs @@ -4,6 +4,7 @@ pub mod bundle; mod context; pub mod db_util; +mod extended_snapshot; pub mod main; use crate::blocks::TipsetKey; @@ -17,8 +18,8 @@ use crate::cli_shared::{ delete_chain_data, }; use crate::daemon::{ - context::AppContext, - db_util::{import_chain_as_forest_car, maybe_import_extended_snapshots}, + context::AppContext, db_util::import_chain_as_forest_car, + extended_snapshot::maybe_import_extended_snapshots, }; use crate::db::gc::SnapshotGarbageCollector; use crate::db::ttl::EthMappingCollector; @@ -158,15 +159,13 @@ async fn maybe_import_snapshot( .await?; ctx.db .read_only_files(std::iter::once(car_db_path.clone()))?; - if let Ok(snapshot_url) = url::Url::parse(&path.display().to_string()) { - maybe_import_extended_snapshots( - &ctx.db, - &snapshot_url, - &ts, - &ctx.db_meta_data.get_forest_car_db_dir(), - ) - .await; - } + maybe_import_extended_snapshots( + &ctx.db, + path, + &ts, + &ctx.db_meta_data.get_forest_car_db_dir(), + ) + .await; let ts_epoch = ts.epoch(); // Explicitly set heaviest tipset here in case HEAD_KEY has already been set // in the current setting store diff --git a/src/db/car/many.rs b/src/db/car/many.rs index 96a3a44aee7..2e870be35be 100644 --- a/src/db/car/many.rs +++ b/src/db/car/many.rs @@ -321,13 +321,6 @@ impl EthMappingsStore for ManyCar { ) -> anyhow::Result<()> { EthMappingsStore::set_tipset_key_at_epoch_raw(self.writer(), epoch, tsk) } - - fn set_tipset_keys_at_epochs_raw( - &self, - entries: &[(ChainEpoch, TipsetKey)], - ) -> anyhow::Result<()> { - EthMappingsStore::set_tipset_keys_at_epochs_raw(self.writer(), entries) - } } impl EthBlockBloomStore for ManyCar { diff --git a/src/db/mod.rs b/src/db/mod.rs index 4d6c53d018c..90c0f466423 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -123,18 +123,6 @@ pub trait EthMappingsStore { fn set_tipset_key_at_epoch_raw(&self, epoch: ChainEpoch, tsk: &TipsetKey) -> anyhow::Result<()>; - /// Writes the tipset keys for the given epochs(heights) to the store. Implementations - /// should commit them in a single batch. - fn set_tipset_keys_at_epochs_raw( - &self, - entries: &[(ChainEpoch, TipsetKey)], - ) -> anyhow::Result<()> { - for (epoch, tsk) in entries { - EthMappingsStore::set_tipset_key_at_epoch_raw(self, *epoch, tsk)?; - } - Ok(()) - } - /// Writes the tipset key for a given epoch(height) to the store. fn set_tipset_key_at_epoch(&self, ts: &Tipset) -> anyhow::Result<()> { EthMappingsStore::set_tipset_key_at_epoch_raw(self, ts.epoch(), ts.key()) diff --git a/src/db/parity_db.rs b/src/db/parity_db.rs index 33dda17f76e..684ba2f11c9 100644 --- a/src/db/parity_db.rs +++ b/src/db/parity_db.rs @@ -263,25 +263,6 @@ impl EthMappingsStore for ParityDb { let bytes = fvm_ipld_encoding::to_vec(tsk)?; self.write_to_column(key, bytes, DbColumn::EthMappings) } - - fn set_tipset_keys_at_epochs_raw( - &self, - entries: &[(ChainEpoch, TipsetKey)], - ) -> anyhow::Result<()> { - let tx = entries - .iter() - .map(|(epoch, tsk)| { - Ok(( - DbColumn::EthMappings as u8, - epoch.to_le_bytes(), - Some(fvm_ipld_encoding::to_vec(tsk)?), - )) - }) - .collect::>>()?; - self.db - .commit(tx) - .with_context(|| format!("error writing to column {}", DbColumn::EthMappings)) - } } impl EthBlockBloomStore for ParityDb { @@ -655,26 +636,4 @@ mod test { assert!(db.write_ops_broadcast_tx.read().is_none()); } - - #[test] - fn set_tipset_keys_at_epochs_raw_test() { - let db = TempParityDB::new(); - let tsk = |n: u64| { - TipsetKey::from(nunny::vec![Cid::new_v1( - DAG_CBOR, - MultihashCode::Blake2b256.digest(&n.to_le_bytes()) - )]) - }; - let entries = [(20, tsk(1)), (40, tsk(2)), (60, tsk(3))]; - db.set_tipset_keys_at_epochs_raw(&entries).unwrap(); - for (epoch, expected) in &entries { - assert_eq!( - db.tipset_key_by_epoch(*epoch).unwrap().as_ref(), - Some(expected) - ); - } - assert!(db.tipset_key_by_epoch(80).unwrap().is_none()); - // Empty batches are a no-op. - db.set_tipset_keys_at_epochs_raw(&[]).unwrap(); - } } diff --git a/src/db/parity_db/gc.rs b/src/db/parity_db/gc.rs index 757160cc241..bbecc51284d 100644 --- a/src/db/parity_db/gc.rs +++ b/src/db/parity_db/gc.rs @@ -167,13 +167,6 @@ impl EthMappingsStore for GarbageCollectableParityDb { ) -> anyhow::Result<()> { EthMappingsStore::set_tipset_key_at_epoch_raw(&*self.db.read(), epoch, tsk) } - - fn set_tipset_keys_at_epochs_raw( - &self, - entries: &[(ChainEpoch, TipsetKey)], - ) -> anyhow::Result<()> { - EthMappingsStore::set_tipset_keys_at_epochs_raw(&*self.db.read(), entries) - } } impl EthBlockBloomStore for GarbageCollectableParityDb { From 995fa7d82e1088e4b6b6e1fd4c1416227ea359da Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Mon, 19 Oct 2026 16:36:35 -0400 Subject: [PATCH 07/36] chore: share import_tipset_lookup_snapshot & backfill_receipts_events --- src/daemon/extended_snapshot.rs | 130 ++++++++++++++++++++++---------- src/daemon/mod.rs | 5 +- src/tool/subcommands/db_cmd.rs | 37 ++------- 3 files changed, 98 insertions(+), 74 deletions(-) diff --git a/src/daemon/extended_snapshot.rs b/src/daemon/extended_snapshot.rs index 41a66e098e9..e6e3b9044b0 100644 --- a/src/daemon/extended_snapshot.rs +++ b/src/daemon/extended_snapshot.rs @@ -6,11 +6,15 @@ use crate::blocks::{Tipset, TipsetKey}; use crate::chain::{TIPSET_LOOKUP_HAMT_BIT_WIDTH, index::ChainIndex}; -use crate::daemon::db_util::path_as_url; +use crate::daemon::db_util::{ + BackfillGuard, BackfillOptions, RangeSpec, path_as_url, run_backfill, +}; +use crate::db::EthMappingsStore; +use crate::db::car::AnyCar; use crate::db::car::forest::new_forest_car_temp_path_in; -use crate::db::{EthMappingsStore, MemoryDB}; use crate::prelude::*; use crate::shim::clock::ChainEpoch; +use crate::state_manager::StateManager; use crate::utils::db::car_stream::{CarBlock, CarStream}; use crate::utils::net::{DownloadFileOption, download_to}; use anyhow::Context; @@ -114,41 +118,79 @@ where Ok(n_blocks) } -/// Populates the tipset lookup table in `db` from a tipset lookup snapshot. -async fn import_tipset_lookup_snapshot( - db: &Arc, +/// Populates the tipset lookup table in `db` from a tipset lookup snapshot, calling +/// `on_imported` after each entry is written. Every entry is validated against the tipsets +/// in `db` before any of them is written. +pub fn import_tipset_lookup_snapshot( + db: &DB, path: &Path, - head_epoch: ChainEpoch, -) -> anyhow::Result + mut on_imported: impl FnMut(), +) -> anyhow::Result<()> where - DB: EthMappingsStore + Send + Sync + 'static, + DB: Blockstore + EthMappingsStore, { - let store = MemoryDB::default(); - let mut car_stream = CarStream::new_from_path(path).await?; - let hamt_root = *car_stream.header_v1.roots.first(); - while let Some(CarBlock { cid, data }) = car_stream.try_next().await? { - store.put_keyed(&cid, &data)?; - } + let hamt_car = AnyCar::try_from(path)?; + anyhow::ensure!( + hamt_car.header_v1().roots.len() == 1, + "hamt car should have a single root" + ); + let hamt_root = *hamt_car.header_v1().roots.first(); + let hamt: Hamt<_, TipsetKey, ChainEpoch> = + Hamt::load_with_bit_width(&hamt_root, hamt_car, TIPSET_LOOKUP_HAMT_BIT_WIDTH) + .context("failed to load tipset lookup HAMT")?; + let mut n_entries = 0; + hamt.for_each_cacheless(|&epoch, tsk| { + anyhow::ensure!( + ChainIndex::is_tipset_lookup_checkpoint(epoch), + "epoch {epoch} is not a tipset lookup checkpoint" + ); + let ts = Tipset::load_required(db, tsk)?; + anyhow::ensure!( + ts.epoch() == epoch, + "epochs do not match, {epoch} in hamt, {} in database", + ts.epoch() + ); + n_entries += 1; + Ok(()) + })?; + hamt.for_each_cacheless(|&epoch, tsk| { + db.set_tipset_key_at_epoch_raw(epoch, tsk)?; + on_imported(); + Ok(()) + })?; + Ok(()) +} - let db = db.clone(); - tokio::task::spawn_blocking(move || { - let hamt: Hamt<_, TipsetKey, ChainEpoch> = - Hamt::load_with_bit_width(&hamt_root, &store, TIPSET_LOOKUP_HAMT_BIT_WIDTH) - .context("failed to load tipset lookup HAMT")?; - // Validate all entries before writing any of them. - let mut n_entries = 0; - hamt.for_each_cacheless(|&epoch, _| { - anyhow::ensure!( - ChainIndex::is_tipset_lookup_checkpoint(epoch) && epoch <= head_epoch, - "invalid tipset lookup entry at epoch {epoch}, head epoch: {head_epoch}", - ); - n_entries += 1; - Ok(()) - })?; - hamt.for_each_cacheless(|&epoch, tsk| db.set_tipset_key_at_epoch_raw(epoch, tsk))?; - Ok(n_entries) - }) - .await? +/// Indexes the tipsets whose receipts and events were just imported (block logs blooms and +/// Ethereum mappings), as `forest-tool index backfill` does. +async fn backfill_receipts_events( + state_manager: &StateManager, + head: &Tipset, +) -> anyhow::Result<()> { + let db = state_manager.db(); + let Some(to) = head + .shallow_clone() + .chain(db) + .take_while(|ts| db.has(ts.parent_state()).unwrap_or_default()) + .last() + .map(|ts| ts.epoch()) + else { + return Ok(()); + }; + let guard = BackfillGuard::try_start()?; + let options = BackfillOptions { + allow_recompute: false, + ..Default::default() + }; + let result = run_backfill(state_manager, head, RangeSpec::To(to), options, &guard).await; + let report = guard.finish(result)?; + info!( + "Indexed {} tipsets from epoch {} to {to} (skipped: {})", + report.indexed, + head.epoch(), + report.skipped + ); + Ok(()) } /// Downloads the extended snapshots (message receipts and events, tipset lookup table) @@ -156,8 +198,9 @@ where /// Does nothing when `snapshot_path` is a local file rather than a URL. pub async fn maybe_import_extended_snapshots( db: &Arc, - snapshot_path: &Path, + state_manager: &StateManager, head: &Tipset, + snapshot_path: &Path, temp_dir: &Path, ) where DB: Blockstore + EthMappingsStore + Send + Sync + 'static, @@ -186,10 +229,15 @@ pub async fn maybe_import_extended_snapshots( } .await; match result { - Ok(n) => info!( - "Imported {n} message receipt and event blocks from {}", - file.filename - ), + Ok(n) => { + info!( + "Imported {n} message receipt and event blocks from {}", + file.filename + ); + if let Err(e) = backfill_receipts_events(state_manager, head).await { + warn!("Failed to index imported receipts and events: {e:#}"); + } + } Err(e) => warn!( "Failed to import receipts and events snapshot {}: {e:#}", file.filename @@ -200,11 +248,13 @@ pub async fn maybe_import_extended_snapshots( if let Some(file) = &extended.tipset_lookup { let result = async { let path = download_extended_snapshot(snapshot_url, file, temp_dir).await?; - import_tipset_lookup_snapshot(db, &path, head.epoch()).await + let db = db.clone(); + tokio::task::spawn_blocking(move || import_tipset_lookup_snapshot(&*db, &path, || {})) + .await? } .await; match result { - Ok(n) => info!("Imported {n} tipset lookup entries from {}", file.filename), + Ok(()) => info!("Imported tipset lookup entries from {}", file.filename), Err(e) => warn!( "Failed to import tipset lookup snapshot {}: {e:#}", file.filename diff --git a/src/daemon/mod.rs b/src/daemon/mod.rs index ff395c43b13..fe97cde9ccc 100644 --- a/src/daemon/mod.rs +++ b/src/daemon/mod.rs @@ -4,7 +4,7 @@ pub mod bundle; mod context; pub mod db_util; -mod extended_snapshot; +pub mod extended_snapshot; pub mod main; use crate::blocks::TipsetKey; @@ -161,8 +161,9 @@ async fn maybe_import_snapshot( .read_only_files(std::iter::once(car_db_path.clone()))?; maybe_import_extended_snapshots( &ctx.db, - path, + &ctx.state_manager, &ts, + path, &ctx.db_meta_data.get_forest_car_db_dir(), ) .await; diff --git a/src/tool/subcommands/db_cmd.rs b/src/tool/subcommands/db_cmd.rs index b2e10d98198..8cb4978b6b0 100644 --- a/src/tool/subcommands/db_cmd.rs +++ b/src/tool/subcommands/db_cmd.rs @@ -4,19 +4,17 @@ use std::path::PathBuf; use std::time::Instant; -use crate::blocks::{Tipset, TipsetKey}; -use crate::chain::TIPSET_LOOKUP_HAMT_BIT_WIDTH; use crate::cli::subcommands::prompt_confirm; use crate::cli_shared::{chain_path, delete_chain_data, read_config}; use crate::daemon::db_util::load_all_forest_cars; -use crate::db::car::{AnyCar, ManyCar}; +use crate::daemon::extended_snapshot::import_tipset_lookup_snapshot; +use crate::db::car::ManyCar; use crate::db::db_engine::{db_root, open_db}; -use crate::db::{BlockstoreWithWriteBuffer, CAR_DB_DIR_NAME, EthMappingsStore}; +use crate::db::{BlockstoreWithWriteBuffer, CAR_DB_DIR_NAME}; use crate::networks::NetworkChain; use crate::prelude::*; use crate::utils::db::car_stream::CarStream; use clap::Subcommand; -use fil_actors_shared::fvm_ipld_hamt::Hamt; use indicatif::{ProgressBar, ProgressStyle}; use tokio_stream::StreamExt; @@ -166,17 +164,6 @@ impl DBCommands { db, } => { let start = Instant::now(); - println!("Loading tipset lookup hamt..."); - let hamt_car = AnyCar::try_from(snapshot.as_path())?; - anyhow::ensure!( - hamt_car.header_v1().roots.len() == 1, - "hamt car should have a single root" - ); - let hamt_root = *hamt_car.header_v1().roots.first(); - let hamt: Hamt<_, TipsetKey, ChainEpoch> = - Hamt::load_with_bit_width(&hamt_root, hamt_car, TIPSET_LOOKUP_HAMT_BIT_WIDTH)?; - println!("Loaded tipset lookup hamt"); - let db_root_path = if let Some(db) = db { db } else { @@ -190,27 +177,13 @@ impl DBCommands { load_all_forest_cars(&db, &forest_car_db_dir)?; db }; - println!("Validating tipset lookup hamt..."); - hamt.for_each_cacheless(|&epoch, tsk| { - let ts = Tipset::load_required(&db, tsk)?; - anyhow::ensure!( - ts.epoch() == epoch, - "epochs do not match, {epoch} in hamt, {} in database", - ts.epoch() - ); - anyhow::Ok(()) - })?; - println!("Successfully validated tipset lookup hamt"); + println!("Validating and importing tipset lookup hamt..."); let pb = ProgressBar::new_spinner().with_style( ProgressStyle::with_template("{spinner} {pos} entries imported") .expect("indicatif template must be valid"), ); pb.enable_steady_tick(std::time::Duration::from_millis(100)); - hamt.for_each_cacheless(|&epoch, tsk| { - db.set_tipset_key_at_epoch_raw(epoch, tsk)?; - pb.inc(1); - anyhow::Ok(()) - })?; + import_tipset_lookup_snapshot(&db, &snapshot, || pb.inc(1))?; drop(db); let total = pb.position(); pb.finish(); From 8f85fbf8a61207e7405f981df8562a7fc35a262a Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Mon, 19 Oct 2026 17:04:50 -0400 Subject: [PATCH 08/36] chore: resolve redirect urls --- src/daemon/extended_snapshot.rs | 9 +++++---- src/daemon/mod.rs | 21 +++++++++++++++++++-- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/src/daemon/extended_snapshot.rs b/src/daemon/extended_snapshot.rs index e6e3b9044b0..f0e995a0b7e 100644 --- a/src/daemon/extended_snapshot.rs +++ b/src/daemon/extended_snapshot.rs @@ -24,6 +24,7 @@ use regex::Regex; use serde::Deserialize; use std::path::Path; use std::sync::{Arc, LazyLock}; +use std::time::Duration; use tracing::{info, warn}; use url::Url; @@ -70,6 +71,7 @@ async fn fetch_extended_snapshots(snapshot_url: &Url) -> anyhow::Result( let Some(snapshot_url) = path_as_url(snapshot_path) else { return; }; - let snapshot_url = &snapshot_url; - let extended = match fetch_extended_snapshots(snapshot_url).await { + let extended = match fetch_extended_snapshots(&snapshot_url).await { Ok(Some(extended)) => extended, Ok(None) => { info!("No extended snapshots advertised for {snapshot_url}"); @@ -224,7 +225,7 @@ pub async fn maybe_import_extended_snapshots( if let Some(file) = &extended.receipts_events { let result = async { - let path = download_extended_snapshot(snapshot_url, file, temp_dir).await?; + let path = download_extended_snapshot(&snapshot_url, file, temp_dir).await?; import_receipts_events_snapshot(db, &path).await } .await; @@ -247,7 +248,7 @@ pub async fn maybe_import_extended_snapshots( if let Some(file) = &extended.tipset_lookup { let result = async { - let path = download_extended_snapshot(snapshot_url, file, temp_dir).await?; + let path = download_extended_snapshot(&snapshot_url, file, temp_dir).await?; let db = db.clone(); tokio::task::spawn_blocking(move || import_tipset_lookup_snapshot(&*db, &path, || {})) .await? diff --git a/src/daemon/mod.rs b/src/daemon/mod.rs index fe97cde9ccc..78d1ba765c3 100644 --- a/src/daemon/mod.rs +++ b/src/daemon/mod.rs @@ -18,7 +18,8 @@ use crate::cli_shared::{ delete_chain_data, }; use crate::daemon::{ - context::AppContext, db_util::import_chain_as_forest_car, + context::AppContext, + db_util::{import_chain_as_forest_car, path_as_url}, extended_snapshot::maybe_import_extended_snapshots, }; use crate::db::gc::SnapshotGarbageCollector; @@ -42,7 +43,7 @@ use anyhow::{Context as _, bail}; use backon::{ExponentialBuilder, Retryable}; use dialoguer::theme::ColorfulTheme; use futures::{Future, FutureExt}; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::sync::Arc; use std::sync::OnceLock; use std::time::{Duration, Instant}; @@ -147,6 +148,7 @@ async fn maybe_import_snapshot( if !opts.skip_load.unwrap_or_default() && let Some(path) = &config.client.snapshot_path { + let path = &resolve_snapshot_path(path).await?; let (car_db_path, ts) = import_chain_as_forest_car( path, &ctx.db_meta_data.get_forest_car_db_dir(), @@ -988,6 +990,21 @@ where tokio::task::spawn_blocking(f).then(|res| async { res.expect("spawned task panicked") }) } +/// Follows redirects when `path` is a URL (e.g. a stable `latest` snapshot URL) +async fn resolve_snapshot_path(path: &Path) -> anyhow::Result { + let Some(url) = path_as_url(path) else { + return Ok(path.to_owned()); + }; + let response = crate::utils::net::global_http_client() + .head(url.clone()) + .timeout(Duration::from_secs(30);) + .send() + .await + .and_then(|r| r.error_for_status()) + .with_context(|| format!("failed to resolve snapshot URL {url}"))?; + Ok(response.url().as_str().into()) +} + #[cfg(test)] mod tests { use rstest::rstest; From d8766e43922e837aaec53b17dba54f7ddbeca7f0 Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Mon, 19 Oct 2026 17:13:51 -0400 Subject: [PATCH 09/36] chore: solve problems --- src/daemon/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/daemon/mod.rs b/src/daemon/mod.rs index 78d1ba765c3..3ba4712bcf6 100644 --- a/src/daemon/mod.rs +++ b/src/daemon/mod.rs @@ -997,7 +997,7 @@ async fn resolve_snapshot_path(path: &Path) -> anyhow::Result { }; let response = crate::utils::net::global_http_client() .head(url.clone()) - .timeout(Duration::from_secs(30);) + .timeout(Duration::from_secs(30)) .send() .await .and_then(|r| r.error_for_status()) From 9bab5cc0dc34211554b14314326e083901664a13 Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Mon, 19 Oct 2026 18:19:26 -0400 Subject: [PATCH 10/36] chore: fix backfill problems --- src/daemon/db_util.rs | 2 ++ src/daemon/mod.rs | 18 ++++++++++-------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/daemon/db_util.rs b/src/daemon/db_util.rs index e265731c0eb..0f36a606cda 100644 --- a/src/daemon/db_util.rs +++ b/src/daemon/db_util.rs @@ -34,6 +34,8 @@ use tracing::{debug, info, warn}; use url::Url; use walkdir::WalkDir; +#[cfg(doc)] +use crate::blocks::TipsetKey; #[cfg(doc)] use crate::rpc::eth::types::EthHash; diff --git a/src/daemon/mod.rs b/src/daemon/mod.rs index 3ba4712bcf6..9fc48e77473 100644 --- a/src/daemon/mod.rs +++ b/src/daemon/mod.rs @@ -161,6 +161,16 @@ async fn maybe_import_snapshot( .await?; ctx.db .read_only_files(std::iter::once(car_db_path.clone()))?; + let ts_epoch = ts.epoch(); + // Explicitly set heaviest tipset here in case HEAD_KEY has already been set + // in the current setting store + ctx.state_manager + .chain_store() + .set_heaviest_tipset(ts.shallow_clone())?; + debug!( + "Loaded car DB at {} and set current head to epoch {ts_epoch}", + car_db_path.display(), + ); maybe_import_extended_snapshots( &ctx.db, &ctx.state_manager, @@ -169,14 +179,6 @@ async fn maybe_import_snapshot( &ctx.db_meta_data.get_forest_car_db_dir(), ) .await; - let ts_epoch = ts.epoch(); - // Explicitly set heaviest tipset here in case HEAD_KEY has already been set - // in the current setting store - ctx.state_manager.chain_store().set_heaviest_tipset(ts)?; - debug!( - "Loaded car DB at {} and set current head to epoch {ts_epoch}", - car_db_path.display(), - ); } // If the snapshot progress state is not completed, From 5ca56425fb106c188f8534c8a4409c21b97bfe91 Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Tue, 20 Oct 2026 11:07:06 -0400 Subject: [PATCH 11/36] chore: include few logs --- src/daemon/extended_snapshot.rs | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/src/daemon/extended_snapshot.rs b/src/daemon/extended_snapshot.rs index f0e995a0b7e..046047e4f1b 100644 --- a/src/daemon/extended_snapshot.rs +++ b/src/daemon/extended_snapshot.rs @@ -82,6 +82,8 @@ async fn fetch_extended_snapshots(snapshot_url: &Url) -> anyhow::Result 1 { + warn!( + "Indexed {} tipsets from epoch {} to {to}, {} could not be indexed", + report.indexed, + head.epoch(), + report.skipped + ); + } else { + info!( + "Indexed {} tipsets from epoch {} to {to}", + report.indexed, + head.epoch() + ); + } Ok(()) } @@ -218,7 +229,7 @@ pub async fn maybe_import_extended_snapshots( return; } Err(e) => { - warn!("Failed to fetch snapshot metadata for {snapshot_url}: {e:#}"); + warn!("No snapshot metadata available for {snapshot_url}: {e:#}"); return; } }; From 6ae1502b23e32bc90fc750a9c43a228b8156b4da Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Tue, 20 Oct 2026 13:21:46 -0400 Subject: [PATCH 12/36] use a simple regex on download_extended_snapshot --- src/daemon/extended_snapshot.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/daemon/extended_snapshot.rs b/src/daemon/extended_snapshot.rs index 046047e4f1b..f5fddd39d9f 100644 --- a/src/daemon/extended_snapshot.rs +++ b/src/daemon/extended_snapshot.rs @@ -23,14 +23,11 @@ use futures::TryStreamExt; use regex::Regex; use serde::Deserialize; use std::path::Path; -use std::sync::{Arc, LazyLock}; +use std::sync::Arc; use std::time::Duration; use tracing::{info, warn}; use url::Url; -static FILENAME_RE: LazyLock = - LazyLock::new(|| Regex::new(r"^[A-Za-z0-9_.-]+\.forest\.car\.zst$").expect("valid regex")); - /// The subset of the snapshot `.metadata.json` file published in the Forest archive /// that is needed to locate the extended snapshots. #[derive(Debug, Deserialize)] @@ -91,8 +88,9 @@ async fn download_extended_snapshot( ) -> anyhow::Result { use sha2::{Digest as _, Sha256}; + let filename_re = Regex::new(r"^[A-Za-z0-9_.-]+\.forest\.car\.zst$")?; anyhow::ensure!( - FILENAME_RE.is_match(&file.filename), + filename_re.is_match(&file.filename), "invalid extended snapshot filename: {}", file.filename ); From 78d87aab93238b3910d90ba29f77bfbcddf3c942 Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Tue, 20 Oct 2026 15:38:10 -0400 Subject: [PATCH 13/36] chore: introduce SnapshotImportPath --- src/cli_shared/snapshot.rs | 13 ++++ src/daemon/db_util.rs | 130 +++++++++++++++++++++----------- src/daemon/extended_snapshot.rs | 16 ++-- src/daemon/mod.rs | 25 ++---- 4 files changed, 112 insertions(+), 72 deletions(-) diff --git a/src/cli_shared/snapshot.rs b/src/cli_shared/snapshot.rs index 68059207492..38f78ea70a4 100644 --- a/src/cli_shared/snapshot.rs +++ b/src/cli_shared/snapshot.rs @@ -5,6 +5,7 @@ use std::{ fmt::Display, path::{Path, PathBuf}, str::FromStr, + time::Duration, }; use crate::prelude::*; @@ -112,6 +113,18 @@ pub async fn peek( )) } +/// Returns the final URL after redirection(s) of `url`, e.g. a stable `latest` snapshot URL. +pub async fn peek_url(url: Url) -> anyhow::Result { + let response = crate::utils::net::global_http_client() + .head(url) + .timeout(Duration::from_secs(30)) + .send() + .await? + .error_for_status() + .context("server returned an error response")?; + Ok(response.url().clone()) +} + // Extract file paths from content-disposition values: // "attachment; filename=\"911520_2023_09_14T06_13_00Z.car.zst\"" // => "911520_2023_09_14T06_13_00Z.car.zst" diff --git a/src/daemon/db_util.rs b/src/daemon/db_util.rs index afb76b6ee63..00eabf5a7bc 100644 --- a/src/daemon/db_util.rs +++ b/src/daemon/db_util.rs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0, MIT use crate::blocks::Tipset; +use crate::cli_shared::snapshot; use crate::db::SettingsStoreExt; use crate::db::car::forest::{ FOREST_CAR_FILE_EXTENSION, TEMP_FOREST_CAR_FILE_EXTENSION, new_forest_car_temp_path_in, @@ -129,10 +130,43 @@ pub enum ImportMode { Hardlink, } -/// This function validates and stores the CAR binary from `from_path`(either local path or URL) into the `{DB_ROOT}/car_db/` +/// A snapshot to import (`--import-snapshot`), either a URL or a local file. +#[derive(Debug, Clone)] +pub enum SnapshotImportPath { + /// A remote snapshot, downloaded before importing + Url(Url), + /// A local snapshot file + FilePath(PathBuf), +} + +impl SnapshotImportPath { + /// Parses `path` as a URL, resolved to the final URL after redirection(s), or else as a + /// local file. + pub async fn parse(path: PathBuf) -> anyhow::Result { + match path.to_str().and_then(|s| Url::parse(s).ok()) { + Some(url) => Ok(Self::Url( + snapshot::peek_url(url.clone()) + .await + .with_context(|| format!("failed to resolve snapshot URL {url}"))?, + )), + None => Ok(Self::FilePath(path)), + } + } +} + +impl std::fmt::Display for SnapshotImportPath { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Url(url) => write!(f, "{url}"), + Self::FilePath(path) => write!(f, "{}", path.display()), + } + } +} + +/// This function validates and stores the CAR binary from `from`(either local path or URL) into the `{DB_ROOT}/car_db/` /// (automatically trans-code into `.forest.car.zst` format when needed), and returns its final file path and the heaviest tipset. pub async fn import_chain_as_forest_car( - from_path: &Path, + from: &SnapshotImportPath, forest_car_db_dir: &Path, import_mode: ImportMode, rpc_endpoint: Url, @@ -140,7 +174,7 @@ pub async fn import_chain_as_forest_car( chain_config: &ChainConfig, snapshot_progress_tracker: &SnapshotProgressTracker, ) -> anyhow::Result<(PathBuf, Tipset)> { - info!("Importing chain from snapshot at: {}", from_path.display()); + info!("Importing chain from snapshot at: {from}"); let stopwatch = time::Instant::now(); @@ -153,25 +187,29 @@ pub async fn import_chain_as_forest_car( let forest_car_db_path = forest_car_db_path.clone(); async move { let downloaded_car_temp_path = new_forest_car_temp_path_in(forest_car_db_dir)?; - if let Some(url) = path_as_url(from_path) { - download_to( - &url, - &downloaded_car_temp_path, - DownloadFileOption::Resumable, - snapshot_progress_tracker.create_callback(), - ) - .await?; + match from { + SnapshotImportPath::Url(url) => { + download_to( + url, + &downloaded_car_temp_path, + DownloadFileOption::Resumable, + snapshot_progress_tracker.create_callback(), + ) + .await?; - snapshot_progress_tracker.completed(); - } else { - snapshot_progress_tracker.not_required(); - if ForestCar::is_valid(&EitherMmapOrRandomAccessFile::open(from_path)?) { - move_or_copy_file(from_path, &downloaded_car_temp_path, mode)?; - } else { - // For a local snapshot, we transcode directly instead of copying & transcoding. - transcode_into_forest_car(from_path, &downloaded_car_temp_path).await?; - if mode == ImportMode::Move { - std::fs::remove_file(from_path).context("Error removing original file")?; + snapshot_progress_tracker.completed(); + } + SnapshotImportPath::FilePath(from_path) => { + snapshot_progress_tracker.not_required(); + if ForestCar::is_valid(&EitherMmapOrRandomAccessFile::open(from_path)?) { + move_or_copy_file(from_path, &downloaded_car_temp_path, mode)?; + } else { + // For a local snapshot, we transcode directly instead of copying & transcoding. + transcode_into_forest_car(from_path, &downloaded_car_temp_path).await?; + if mode == ImportMode::Move { + std::fs::remove_file(from_path) + .context("Error removing original file")?; + } } } } @@ -192,31 +230,35 @@ pub async fn import_chain_as_forest_car( }; match import_mode { - ImportMode::Auto => { - if path_as_url(from_path).is_some() { - // Fallback to move if from_path is url - move_or_copy(ImportMode::Move).await?; - } else if ForestCar::is_valid(&EitherMmapOrRandomAccessFile::open(from_path)?) { - tracing::info!( - "Hardlinking {} to {}", - from_path.display(), - forest_car_db_path.display() - ); - if std::fs::hard_link(from_path, &forest_car_db_path).is_err() { - tracing::warn!("Error creating hardlink, fallback to copy"); + ImportMode::Auto => match from { + // Fallback to move if from is url + SnapshotImportPath::Url(_) => move_or_copy(ImportMode::Move).await?, + SnapshotImportPath::FilePath(from_path) => { + if ForestCar::is_valid(&EitherMmapOrRandomAccessFile::open(from_path)?) { + tracing::info!( + "Hardlinking {} to {}", + from_path.display(), + forest_car_db_path.display() + ); + if std::fs::hard_link(from_path, &forest_car_db_path).is_err() { + tracing::warn!("Error creating hardlink, fallback to copy"); + move_or_copy(ImportMode::Copy).await?; + } + } else { + tracing::warn!( + "Snapshot file is not a valid forest.car.zst file, fallback to copy" + ); move_or_copy(ImportMode::Copy).await?; } - } else { - tracing::warn!( - "Snapshot file is not a valid forest.car.zst file, fallback to copy" - ); - move_or_copy(ImportMode::Copy).await?; } - } + }, ImportMode::Copy | ImportMode::Move => { move_or_copy(import_mode).await?; } ImportMode::Symlink => { + let SnapshotImportPath::FilePath(from_path) = from else { + bail!("Symlink import mode requires a local snapshot file"); + }; let from_path = std::path::absolute(from_path)?; if ForestCar::is_valid(&EitherMmapOrRandomAccessFile::open(&from_path)?) { tracing::info!( @@ -231,6 +273,9 @@ pub async fn import_chain_as_forest_car( } } ImportMode::Hardlink => { + let SnapshotImportPath::FilePath(from_path) = from else { + bail!("Hardlink import mode requires a local snapshot file"); + }; if ForestCar::is_valid(&EitherMmapOrRandomAccessFile::open(from_path)?) { tracing::info!( "Hardlinking {} to {}", @@ -291,11 +336,6 @@ pub async fn import_chain_as_forest_car( Ok((forest_car_db_path, ts)) } -/// Parses `path` as a URL, returning `None` for local paths. -pub(super) fn path_as_url(path: &Path) -> Option { - path.to_str().and_then(|s| Url::parse(s).ok()) -} - fn move_or_copy_file(from: &Path, to: &Path, import_mode: ImportMode) -> anyhow::Result<()> { match import_mode { ImportMode::Move => { @@ -1038,7 +1078,7 @@ mod test { let temp_db_dir = tempfile::Builder::new().tempdir()?; let (path, ts) = import_chain_as_forest_car( - file_path, + &SnapshotImportPath::FilePath(file_path.to_owned()), temp_db_dir.path(), import_mode, "http://127.0.0.1:2345/rpc/v1".parse().unwrap(), diff --git a/src/daemon/extended_snapshot.rs b/src/daemon/extended_snapshot.rs index f5fddd39d9f..f3864ab989b 100644 --- a/src/daemon/extended_snapshot.rs +++ b/src/daemon/extended_snapshot.rs @@ -7,7 +7,7 @@ use crate::blocks::{Tipset, TipsetKey}; use crate::chain::{TIPSET_LOOKUP_HAMT_BIT_WIDTH, index::ChainIndex}; use crate::daemon::db_util::{ - BackfillGuard, BackfillOptions, RangeSpec, path_as_url, run_backfill, + BackfillGuard, BackfillOptions, RangeSpec, SnapshotImportPath, run_backfill, }; use crate::db::EthMappingsStore; use crate::db::car::AnyCar; @@ -64,7 +64,9 @@ struct ExtendedSnapshotFile { /// Fetches the `.metadata.json` published next to `snapshot_url` and returns its extended /// snapshots, if any. -async fn fetch_extended_snapshots(snapshot_url: &Url) -> anyhow::Result> { +async fn fetch_extended_snapshots_info( + snapshot_url: &Url, +) -> anyhow::Result> { let metadata_url = Url::parse(&format!("{snapshot_url}.metadata.json"))?; let response = crate::utils::net::global_http_client() .get(metadata_url.clone()) @@ -211,16 +213,16 @@ pub async fn maybe_import_extended_snapshots( db: &Arc, state_manager: &StateManager, head: &Tipset, - snapshot_path: &Path, + snapshot_path: &SnapshotImportPath, temp_dir: &Path, ) where DB: Blockstore + EthMappingsStore + Send + Sync + 'static, { // Extended snapshots are only published next to remote snapshots. - let Some(snapshot_url) = path_as_url(snapshot_path) else { + let SnapshotImportPath::Url(snapshot_url) = snapshot_path else { return; }; - let extended = match fetch_extended_snapshots(&snapshot_url).await { + let extended = match fetch_extended_snapshots_info(snapshot_url).await { Ok(Some(extended)) => extended, Ok(None) => { info!("No extended snapshots advertised for {snapshot_url}"); @@ -234,7 +236,7 @@ pub async fn maybe_import_extended_snapshots( if let Some(file) = &extended.receipts_events { let result = async { - let path = download_extended_snapshot(&snapshot_url, file, temp_dir).await?; + let path = download_extended_snapshot(snapshot_url, file, temp_dir).await?; import_receipts_events_snapshot(db, &path).await } .await; @@ -257,7 +259,7 @@ pub async fn maybe_import_extended_snapshots( if let Some(file) = &extended.tipset_lookup { let result = async { - let path = download_extended_snapshot(&snapshot_url, file, temp_dir).await?; + let path = download_extended_snapshot(snapshot_url, file, temp_dir).await?; let db = db.clone(); tokio::task::spawn_blocking(move || import_tipset_lookup_snapshot(&*db, &path, || {})) .await? diff --git a/src/daemon/mod.rs b/src/daemon/mod.rs index 9fc48e77473..86627110938 100644 --- a/src/daemon/mod.rs +++ b/src/daemon/mod.rs @@ -19,7 +19,7 @@ use crate::cli_shared::{ }; use crate::daemon::{ context::AppContext, - db_util::{import_chain_as_forest_car, path_as_url}, + db_util::{SnapshotImportPath, import_chain_as_forest_car}, extended_snapshot::maybe_import_extended_snapshots, }; use crate::db::gc::SnapshotGarbageCollector; @@ -43,7 +43,7 @@ use anyhow::{Context as _, bail}; use backon::{ExponentialBuilder, Retryable}; use dialoguer::theme::ColorfulTheme; use futures::{Future, FutureExt}; -use std::path::{Path, PathBuf}; +use std::path::Path; use std::sync::Arc; use std::sync::OnceLock; use std::time::{Duration, Instant}; @@ -148,9 +148,9 @@ async fn maybe_import_snapshot( if !opts.skip_load.unwrap_or_default() && let Some(path) = &config.client.snapshot_path { - let path = &resolve_snapshot_path(path).await?; + let path = SnapshotImportPath::parse(path.clone()).await?; let (car_db_path, ts) = import_chain_as_forest_car( - path, + &path, &ctx.db_meta_data.get_forest_car_db_dir(), config.client.import_mode, config.client.rpc_v1_endpoint()?, @@ -175,7 +175,7 @@ async fn maybe_import_snapshot( &ctx.db, &ctx.state_manager, &ts, - path, + &path, &ctx.db_meta_data.get_forest_car_db_dir(), ) .await; @@ -992,21 +992,6 @@ where tokio::task::spawn_blocking(f).then(|res| async { res.expect("spawned task panicked") }) } -/// Follows redirects when `path` is a URL (e.g. a stable `latest` snapshot URL) -async fn resolve_snapshot_path(path: &Path) -> anyhow::Result { - let Some(url) = path_as_url(path) else { - return Ok(path.to_owned()); - }; - let response = crate::utils::net::global_http_client() - .head(url.clone()) - .timeout(Duration::from_secs(30)) - .send() - .await - .and_then(|r| r.error_for_status()) - .with_context(|| format!("failed to resolve snapshot URL {url}"))?; - Ok(response.url().as_str().into()) -} - #[cfg(test)] mod tests { use rstest::rstest; From e6b59d1308cd06fdbb79a8f7b46e695667308987 Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Tue, 20 Oct 2026 15:42:53 -0400 Subject: [PATCH 14/36] chore: use `ensure!` --- src/daemon/extended_snapshot.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/daemon/extended_snapshot.rs b/src/daemon/extended_snapshot.rs index f3864ab989b..c7c7fa64840 100644 --- a/src/daemon/extended_snapshot.rs +++ b/src/daemon/extended_snapshot.rs @@ -102,9 +102,11 @@ async fn download_extended_snapshot( download_to(&url, &path, DownloadFileOption::Resumable, None).await?; let actual: [u8; 32] = Sha256::digest(tokio::fs::read(&path).await?).into(); - if actual != file.sha256 { - anyhow::bail!("checksum mismatch for {}", file.filename); - } + anyhow::ensure!( + actual == file.sha256, + "checksum mismatch for {}", + file.filename + ); Ok(path) } From c77e9f61e6343a794d02247fa696d06616ee2bc9 Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Tue, 20 Oct 2026 17:16:01 -0400 Subject: [PATCH 15/36] chore: remove counter --- src/daemon/extended_snapshot.rs | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/daemon/extended_snapshot.rs b/src/daemon/extended_snapshot.rs index c7c7fa64840..35785cb0b01 100644 --- a/src/daemon/extended_snapshot.rs +++ b/src/daemon/extended_snapshot.rs @@ -111,17 +111,15 @@ async fn download_extended_snapshot( } /// Imports message receipts and events from a receipts and events snapshot into `db`. -async fn import_receipts_events_snapshot(db: &DB, path: &Path) -> anyhow::Result +async fn import_receipts_events_snapshot(db: &DB, path: &Path) -> anyhow::Result<()> where DB: Blockstore, { let mut car_stream = CarStream::new_from_path(path).await?; - let mut n_blocks = 0; while let Some(CarBlock { cid, data }) = car_stream.try_next().await? { db.put_keyed(&cid, &data)?; - n_blocks += 1; } - Ok(n_blocks) + Ok(()) } /// Populates the tipset lookup table in `db` from a tipset lookup snapshot, calling @@ -144,7 +142,6 @@ where let hamt: Hamt<_, TipsetKey, ChainEpoch> = Hamt::load_with_bit_width(&hamt_root, hamt_car, TIPSET_LOOKUP_HAMT_BIT_WIDTH) .context("failed to load tipset lookup HAMT")?; - let mut n_entries = 0; hamt.for_each_cacheless(|&epoch, tsk| { anyhow::ensure!( ChainIndex::is_tipset_lookup_checkpoint(epoch), @@ -156,7 +153,6 @@ where "epochs do not match, {epoch} in hamt, {} in database", ts.epoch() ); - n_entries += 1; Ok(()) })?; hamt.for_each_cacheless(|&epoch, tsk| { @@ -243,9 +239,9 @@ pub async fn maybe_import_extended_snapshots( } .await; match result { - Ok(n) => { + Ok(()) => { info!( - "Imported {n} message receipt and event blocks from {}", + "Imported message receipts and events from {}", file.filename ); if let Err(e) = backfill_receipts_events(state_manager, head).await { From 40d6cce6d4584b6e70f3156e1b955db393c43606 Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Wed, 21 Oct 2026 08:50:29 -0400 Subject: [PATCH 16/36] chore: remove out-of-scope changes --- src/daemon/db_util.rs | 130 +++++++++++--------------------- src/daemon/extended_snapshot.rs | 12 +-- src/daemon/mod.rs | 13 ++-- 3 files changed, 59 insertions(+), 96 deletions(-) diff --git a/src/daemon/db_util.rs b/src/daemon/db_util.rs index 00eabf5a7bc..afb76b6ee63 100644 --- a/src/daemon/db_util.rs +++ b/src/daemon/db_util.rs @@ -2,7 +2,6 @@ // SPDX-License-Identifier: Apache-2.0, MIT use crate::blocks::Tipset; -use crate::cli_shared::snapshot; use crate::db::SettingsStoreExt; use crate::db::car::forest::{ FOREST_CAR_FILE_EXTENSION, TEMP_FOREST_CAR_FILE_EXTENSION, new_forest_car_temp_path_in, @@ -130,43 +129,10 @@ pub enum ImportMode { Hardlink, } -/// A snapshot to import (`--import-snapshot`), either a URL or a local file. -#[derive(Debug, Clone)] -pub enum SnapshotImportPath { - /// A remote snapshot, downloaded before importing - Url(Url), - /// A local snapshot file - FilePath(PathBuf), -} - -impl SnapshotImportPath { - /// Parses `path` as a URL, resolved to the final URL after redirection(s), or else as a - /// local file. - pub async fn parse(path: PathBuf) -> anyhow::Result { - match path.to_str().and_then(|s| Url::parse(s).ok()) { - Some(url) => Ok(Self::Url( - snapshot::peek_url(url.clone()) - .await - .with_context(|| format!("failed to resolve snapshot URL {url}"))?, - )), - None => Ok(Self::FilePath(path)), - } - } -} - -impl std::fmt::Display for SnapshotImportPath { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::Url(url) => write!(f, "{url}"), - Self::FilePath(path) => write!(f, "{}", path.display()), - } - } -} - -/// This function validates and stores the CAR binary from `from`(either local path or URL) into the `{DB_ROOT}/car_db/` +/// This function validates and stores the CAR binary from `from_path`(either local path or URL) into the `{DB_ROOT}/car_db/` /// (automatically trans-code into `.forest.car.zst` format when needed), and returns its final file path and the heaviest tipset. pub async fn import_chain_as_forest_car( - from: &SnapshotImportPath, + from_path: &Path, forest_car_db_dir: &Path, import_mode: ImportMode, rpc_endpoint: Url, @@ -174,7 +140,7 @@ pub async fn import_chain_as_forest_car( chain_config: &ChainConfig, snapshot_progress_tracker: &SnapshotProgressTracker, ) -> anyhow::Result<(PathBuf, Tipset)> { - info!("Importing chain from snapshot at: {from}"); + info!("Importing chain from snapshot at: {}", from_path.display()); let stopwatch = time::Instant::now(); @@ -187,29 +153,25 @@ pub async fn import_chain_as_forest_car( let forest_car_db_path = forest_car_db_path.clone(); async move { let downloaded_car_temp_path = new_forest_car_temp_path_in(forest_car_db_dir)?; - match from { - SnapshotImportPath::Url(url) => { - download_to( - url, - &downloaded_car_temp_path, - DownloadFileOption::Resumable, - snapshot_progress_tracker.create_callback(), - ) - .await?; + if let Some(url) = path_as_url(from_path) { + download_to( + &url, + &downloaded_car_temp_path, + DownloadFileOption::Resumable, + snapshot_progress_tracker.create_callback(), + ) + .await?; - snapshot_progress_tracker.completed(); - } - SnapshotImportPath::FilePath(from_path) => { - snapshot_progress_tracker.not_required(); - if ForestCar::is_valid(&EitherMmapOrRandomAccessFile::open(from_path)?) { - move_or_copy_file(from_path, &downloaded_car_temp_path, mode)?; - } else { - // For a local snapshot, we transcode directly instead of copying & transcoding. - transcode_into_forest_car(from_path, &downloaded_car_temp_path).await?; - if mode == ImportMode::Move { - std::fs::remove_file(from_path) - .context("Error removing original file")?; - } + snapshot_progress_tracker.completed(); + } else { + snapshot_progress_tracker.not_required(); + if ForestCar::is_valid(&EitherMmapOrRandomAccessFile::open(from_path)?) { + move_or_copy_file(from_path, &downloaded_car_temp_path, mode)?; + } else { + // For a local snapshot, we transcode directly instead of copying & transcoding. + transcode_into_forest_car(from_path, &downloaded_car_temp_path).await?; + if mode == ImportMode::Move { + std::fs::remove_file(from_path).context("Error removing original file")?; } } } @@ -230,35 +192,31 @@ pub async fn import_chain_as_forest_car( }; match import_mode { - ImportMode::Auto => match from { - // Fallback to move if from is url - SnapshotImportPath::Url(_) => move_or_copy(ImportMode::Move).await?, - SnapshotImportPath::FilePath(from_path) => { - if ForestCar::is_valid(&EitherMmapOrRandomAccessFile::open(from_path)?) { - tracing::info!( - "Hardlinking {} to {}", - from_path.display(), - forest_car_db_path.display() - ); - if std::fs::hard_link(from_path, &forest_car_db_path).is_err() { - tracing::warn!("Error creating hardlink, fallback to copy"); - move_or_copy(ImportMode::Copy).await?; - } - } else { - tracing::warn!( - "Snapshot file is not a valid forest.car.zst file, fallback to copy" - ); + ImportMode::Auto => { + if path_as_url(from_path).is_some() { + // Fallback to move if from_path is url + move_or_copy(ImportMode::Move).await?; + } else if ForestCar::is_valid(&EitherMmapOrRandomAccessFile::open(from_path)?) { + tracing::info!( + "Hardlinking {} to {}", + from_path.display(), + forest_car_db_path.display() + ); + if std::fs::hard_link(from_path, &forest_car_db_path).is_err() { + tracing::warn!("Error creating hardlink, fallback to copy"); move_or_copy(ImportMode::Copy).await?; } + } else { + tracing::warn!( + "Snapshot file is not a valid forest.car.zst file, fallback to copy" + ); + move_or_copy(ImportMode::Copy).await?; } - }, + } ImportMode::Copy | ImportMode::Move => { move_or_copy(import_mode).await?; } ImportMode::Symlink => { - let SnapshotImportPath::FilePath(from_path) = from else { - bail!("Symlink import mode requires a local snapshot file"); - }; let from_path = std::path::absolute(from_path)?; if ForestCar::is_valid(&EitherMmapOrRandomAccessFile::open(&from_path)?) { tracing::info!( @@ -273,9 +231,6 @@ pub async fn import_chain_as_forest_car( } } ImportMode::Hardlink => { - let SnapshotImportPath::FilePath(from_path) = from else { - bail!("Hardlink import mode requires a local snapshot file"); - }; if ForestCar::is_valid(&EitherMmapOrRandomAccessFile::open(from_path)?) { tracing::info!( "Hardlinking {} to {}", @@ -336,6 +291,11 @@ pub async fn import_chain_as_forest_car( Ok((forest_car_db_path, ts)) } +/// Parses `path` as a URL, returning `None` for local paths. +pub(super) fn path_as_url(path: &Path) -> Option { + path.to_str().and_then(|s| Url::parse(s).ok()) +} + fn move_or_copy_file(from: &Path, to: &Path, import_mode: ImportMode) -> anyhow::Result<()> { match import_mode { ImportMode::Move => { @@ -1078,7 +1038,7 @@ mod test { let temp_db_dir = tempfile::Builder::new().tempdir()?; let (path, ts) = import_chain_as_forest_car( - &SnapshotImportPath::FilePath(file_path.to_owned()), + file_path, temp_db_dir.path(), import_mode, "http://127.0.0.1:2345/rpc/v1".parse().unwrap(), diff --git a/src/daemon/extended_snapshot.rs b/src/daemon/extended_snapshot.rs index 35785cb0b01..fb4ced58fdf 100644 --- a/src/daemon/extended_snapshot.rs +++ b/src/daemon/extended_snapshot.rs @@ -7,7 +7,7 @@ use crate::blocks::{Tipset, TipsetKey}; use crate::chain::{TIPSET_LOOKUP_HAMT_BIT_WIDTH, index::ChainIndex}; use crate::daemon::db_util::{ - BackfillGuard, BackfillOptions, RangeSpec, SnapshotImportPath, run_backfill, + BackfillGuard, BackfillOptions, RangeSpec, path_as_url, run_backfill, }; use crate::db::EthMappingsStore; use crate::db::car::AnyCar; @@ -211,16 +211,16 @@ pub async fn maybe_import_extended_snapshots( db: &Arc, state_manager: &StateManager, head: &Tipset, - snapshot_path: &SnapshotImportPath, + snapshot_path: &Path, temp_dir: &Path, ) where DB: Blockstore + EthMappingsStore + Send + Sync + 'static, { // Extended snapshots are only published next to remote snapshots. - let SnapshotImportPath::Url(snapshot_url) = snapshot_path else { + let Some(snapshot_url) = path_as_url(snapshot_path) else { return; }; - let extended = match fetch_extended_snapshots_info(snapshot_url).await { + let extended = match fetch_extended_snapshots_info(&snapshot_url).await { Ok(Some(extended)) => extended, Ok(None) => { info!("No extended snapshots advertised for {snapshot_url}"); @@ -234,7 +234,7 @@ pub async fn maybe_import_extended_snapshots( if let Some(file) = &extended.receipts_events { let result = async { - let path = download_extended_snapshot(snapshot_url, file, temp_dir).await?; + let path = download_extended_snapshot(&snapshot_url, file, temp_dir).await?; import_receipts_events_snapshot(db, &path).await } .await; @@ -257,7 +257,7 @@ pub async fn maybe_import_extended_snapshots( if let Some(file) = &extended.tipset_lookup { let result = async { - let path = download_extended_snapshot(snapshot_url, file, temp_dir).await?; + let path = download_extended_snapshot(&snapshot_url, file, temp_dir).await?; let db = db.clone(); tokio::task::spawn_blocking(move || import_tipset_lookup_snapshot(&*db, &path, || {})) .await? diff --git a/src/daemon/mod.rs b/src/daemon/mod.rs index 86627110938..7e492e4360d 100644 --- a/src/daemon/mod.rs +++ b/src/daemon/mod.rs @@ -19,7 +19,7 @@ use crate::cli_shared::{ }; use crate::daemon::{ context::AppContext, - db_util::{SnapshotImportPath, import_chain_as_forest_car}, + db_util::{import_chain_as_forest_car, path_as_url}, extended_snapshot::maybe_import_extended_snapshots, }; use crate::db::gc::SnapshotGarbageCollector; @@ -43,7 +43,7 @@ use anyhow::{Context as _, bail}; use backon::{ExponentialBuilder, Retryable}; use dialoguer::theme::ColorfulTheme; use futures::{Future, FutureExt}; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::sync::Arc; use std::sync::OnceLock; use std::time::{Duration, Instant}; @@ -148,9 +148,12 @@ async fn maybe_import_snapshot( if !opts.skip_load.unwrap_or_default() && let Some(path) = &config.client.snapshot_path { - let path = SnapshotImportPath::parse(path.clone()).await?; + let path = &match path_as_url(path) { + Some(url) => PathBuf::from(snapshot::peek_url(url).await?.as_str()), + None => path.clone(), + }; let (car_db_path, ts) = import_chain_as_forest_car( - &path, + path, &ctx.db_meta_data.get_forest_car_db_dir(), config.client.import_mode, config.client.rpc_v1_endpoint()?, @@ -175,7 +178,7 @@ async fn maybe_import_snapshot( &ctx.db, &ctx.state_manager, &ts, - &path, + path, &ctx.db_meta_data.get_forest_car_db_dir(), ) .await; From 4cbdcf3375df541b7ad18a9da32b7f77246c2e3d Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Wed, 21 Oct 2026 10:17:23 -0400 Subject: [PATCH 17/36] backfill only after receipts_events and tipset lookup import --- src/daemon/extended_snapshot.rs | 50 ++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/src/daemon/extended_snapshot.rs b/src/daemon/extended_snapshot.rs index fb4ced58fdf..86a48b7c2d6 100644 --- a/src/daemon/extended_snapshot.rs +++ b/src/daemon/extended_snapshot.rs @@ -227,33 +227,37 @@ pub async fn maybe_import_extended_snapshots( return; } Err(e) => { - warn!("No snapshot metadata available for {snapshot_url}: {e:#}"); + warn!("Failed to fetch extended snapshots for {snapshot_url}: {e:#}"); return; } }; - if let Some(file) = &extended.receipts_events { - let result = async { - let path = download_extended_snapshot(&snapshot_url, file, temp_dir).await?; - import_receipts_events_snapshot(db, &path).await - } - .await; - match result { - Ok(()) => { - info!( - "Imported message receipts and events from {}", - file.filename - ); - if let Err(e) = backfill_receipts_events(state_manager, head).await { - warn!("Failed to index imported receipts and events: {e:#}"); + let receipts_events_imported = match &extended.receipts_events { + Some(file) => { + let result = async { + let path = download_extended_snapshot(&snapshot_url, file, temp_dir).await?; + import_receipts_events_snapshot(db, &path).await + } + .await; + match result { + Ok(()) => { + info!( + "Imported message receipts and events from {}", + file.filename + ); + true + } + Err(e) => { + warn!( + "Failed to import receipts and events snapshot {}: {e:#}", + file.filename + ); + false } } - Err(e) => warn!( - "Failed to import receipts and events snapshot {}: {e:#}", - file.filename - ), } - } + None => false, + }; if let Some(file) = &extended.tipset_lookup { let result = async { @@ -271,6 +275,12 @@ pub async fn maybe_import_extended_snapshots( ), } } + + // Index once both snapshots are imported, so each import's logs stay together. + if receipts_events_imported && let Err(e) = backfill_receipts_events(state_manager, head).await + { + warn!("Failed to index imported receipts and events: {e:#}"); + } } #[cfg(test)] From f122381ea58c0f7c35b6280ec512f69a1c6a30ce Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Wed, 21 Oct 2026 10:26:34 -0400 Subject: [PATCH 18/36] chore: avoid warn log by starting backfill at the heads parent --- src/daemon/extended_snapshot.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/daemon/extended_snapshot.rs b/src/daemon/extended_snapshot.rs index 86a48b7c2d6..9f87e5b1c96 100644 --- a/src/daemon/extended_snapshot.rs +++ b/src/daemon/extended_snapshot.rs @@ -170,7 +170,8 @@ async fn backfill_receipts_events( head: &Tipset, ) -> anyhow::Result<()> { let db = state_manager.db(); - let Some(to) = head + let from = Tipset::load_required(db, head.parents())?; + let Some(to) = from .shallow_clone() .chain(db) .take_while(|ts| db.has(ts.parent_state()).unwrap_or_default()) @@ -184,21 +185,20 @@ async fn backfill_receipts_events( allow_recompute: false, ..Default::default() }; - let result = run_backfill(state_manager, head, RangeSpec::To(to), options, &guard).await; + let result = run_backfill(state_manager, &from, RangeSpec::To(to), options, &guard).await; let report = guard.finish(result)?; - // The head is always skipped: its receipts are only known once its child is synced. - if report.skipped > 1 { + if report.skipped > 0 { warn!( "Indexed {} tipsets from epoch {} to {to}, {} could not be indexed", report.indexed, - head.epoch(), + from.epoch(), report.skipped ); } else { info!( "Indexed {} tipsets from epoch {} to {to}", report.indexed, - head.epoch() + from.epoch() ); } Ok(()) From 935fd256ee95808abc16adcee30775cc693ec43b Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Wed, 21 Oct 2026 10:39:47 -0400 Subject: [PATCH 19/36] chore: update documentation --- docs/docs/users/knowledge_base/enriched_snapshots.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/docs/users/knowledge_base/enriched_snapshots.md b/docs/docs/users/knowledge_base/enriched_snapshots.md index 2160bc68c6c..4175c97d914 100644 --- a/docs/docs/users/knowledge_base/enriched_snapshots.md +++ b/docs/docs/users/knowledge_base/enriched_snapshots.md @@ -61,6 +61,15 @@ at least one must be given. ## Importing into Forest +### Automatically on startup + +When Forest imports a snapshot from a URL on startup, with `--auto-download-snapshot` or +`--import-snapshot `, it imports the enriched snapshot too. +This step is best-effort: if a companion file can't be downloaded, verified, or +imported, Forest logs a warning and starts without it. + +### Manually + `--import-snapshot` accepts a single file and the companion CARs have no chain spine of their own, so only the base goes through it. The companions are loaded with `forest-tool db`, which writes directly into the node's key-value store and From 5f6401db3911d34c34942822b2519af88ed458d0 Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Wed, 21 Oct 2026 13:21:46 -0400 Subject: [PATCH 20/36] implement `CarBlock::validate` the same check as `forest-tool db import` performs --- src/daemon/extended_snapshot.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/daemon/extended_snapshot.rs b/src/daemon/extended_snapshot.rs index 9f87e5b1c96..58d8fe744d6 100644 --- a/src/daemon/extended_snapshot.rs +++ b/src/daemon/extended_snapshot.rs @@ -15,7 +15,7 @@ use crate::db::car::forest::new_forest_car_temp_path_in; use crate::prelude::*; use crate::shim::clock::ChainEpoch; use crate::state_manager::StateManager; -use crate::utils::db::car_stream::{CarBlock, CarStream}; +use crate::utils::db::car_stream::CarStream; use crate::utils::net::{DownloadFileOption, download_to}; use anyhow::Context; use fil_actors_shared::fvm_ipld_hamt::Hamt; @@ -116,8 +116,9 @@ where DB: Blockstore, { let mut car_stream = CarStream::new_from_path(path).await?; - while let Some(CarBlock { cid, data }) = car_stream.try_next().await? { - db.put_keyed(&cid, &data)?; + while let Some(block) = car_stream.try_next().await? { + block.validate()?; + db.put_keyed(&block.cid, &block.data)?; } Ok(()) } From 6a57c04462c43a1625ade51f3f87a0b24dcc5325 Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Wed, 21 Oct 2026 13:30:48 -0400 Subject: [PATCH 21/36] chore: remove unneded `resolve.rb` as we auto backfill on node startup --- scripts/tests/external-rpc-checks/resolve.rb | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/tests/external-rpc-checks/resolve.rb b/scripts/tests/external-rpc-checks/resolve.rb index 1efd8f5a86e..06b01c5e283 100644 --- a/scripts/tests/external-rpc-checks/resolve.rb +++ b/scripts/tests/external-rpc-checks/resolve.rb @@ -1,8 +1,8 @@ # frozen_string_literal: true # Picks the ${FOREST_CHAIN} snapshot published DAYS_AGO days ago (UTC), by the date in its name, -# and records under /data what the rest of the harness needs: its URL, its head epoch, and the -# chain plus the ${FOREST_CHECK_EPOCHS}-epoch range the checks cover. +# and records under /data what the rest of the harness needs: its URL, and the chain plus the +# ${FOREST_CHECK_EPOCHS}-epoch range the checks cover. # Runs in the checks image, which has Ruby, so the Forest image needs no curl. require 'date' @@ -49,6 +49,5 @@ def fetch(uri, attempts: 3) epoch = Integer(url[/_height_(\d+)/, 1]) File.write('/data/snapshot-url', "#{url}\n") -File.write('/data/snapshot-epoch', "#{epoch}\n") File.write('/data/check-target', "#{chain} #{epoch - epochs} #{epoch - 1}\n") puts "#{chain} snapshot for #{day}: #{url}" From 6bf80a11bde00b7de1c98f4c6fc273bcc474fb7d Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Wed, 21 Oct 2026 13:36:43 -0400 Subject: [PATCH 22/36] `maybe_import_extended_snapshots` no longer takes a db parameter --- src/daemon/extended_snapshot.rs | 18 +++++++----------- src/daemon/mod.rs | 1 - 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/src/daemon/extended_snapshot.rs b/src/daemon/extended_snapshot.rs index 58d8fe744d6..454ae046712 100644 --- a/src/daemon/extended_snapshot.rs +++ b/src/daemon/extended_snapshot.rs @@ -23,7 +23,6 @@ use futures::TryStreamExt; use regex::Regex; use serde::Deserialize; use std::path::Path; -use std::sync::Arc; use std::time::Duration; use tracing::{info, warn}; use url::Url; @@ -206,17 +205,14 @@ async fn backfill_receipts_events( } /// Downloads the extended snapshots (message receipts and events, tipset lookup table) -/// advertised in the metadata of the snapshot at `snapshot_path` and imports them into `db`. -/// Does nothing when `snapshot_path` is a local file rather than a URL. -pub async fn maybe_import_extended_snapshots( - db: &Arc, +/// advertised in the metadata of the snapshot at `snapshot_path` and imports them into the +/// database of `state_manager`. Does nothing when `snapshot_path` is a local file rather than a URL. +pub(super) async fn maybe_import_extended_snapshots( state_manager: &StateManager, head: &Tipset, snapshot_path: &Path, temp_dir: &Path, -) where - DB: Blockstore + EthMappingsStore + Send + Sync + 'static, -{ +) { // Extended snapshots are only published next to remote snapshots. let Some(snapshot_url) = path_as_url(snapshot_path) else { return; @@ -237,7 +233,7 @@ pub async fn maybe_import_extended_snapshots( Some(file) => { let result = async { let path = download_extended_snapshot(&snapshot_url, file, temp_dir).await?; - import_receipts_events_snapshot(db, &path).await + import_receipts_events_snapshot(state_manager.db(), &path).await } .await; match result { @@ -263,8 +259,8 @@ pub async fn maybe_import_extended_snapshots( if let Some(file) = &extended.tipset_lookup { let result = async { let path = download_extended_snapshot(&snapshot_url, file, temp_dir).await?; - let db = db.clone(); - tokio::task::spawn_blocking(move || import_tipset_lookup_snapshot(&*db, &path, || {})) + let db = state_manager.db_owned(); + tokio::task::spawn_blocking(move || import_tipset_lookup_snapshot(&db, &path, || {})) .await? } .await; diff --git a/src/daemon/mod.rs b/src/daemon/mod.rs index 7e492e4360d..07d89011ad0 100644 --- a/src/daemon/mod.rs +++ b/src/daemon/mod.rs @@ -175,7 +175,6 @@ async fn maybe_import_snapshot( car_db_path.display(), ); maybe_import_extended_snapshots( - &ctx.db, &ctx.state_manager, &ts, path, From 2c57904b09b778b891cf2692a4a088e784870a04 Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Wed, 21 Oct 2026 15:01:42 -0400 Subject: [PATCH 23/36] chore: address doc lint --- docs/docs/users/knowledge_base/enriched_snapshots.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs/users/knowledge_base/enriched_snapshots.md b/docs/docs/users/knowledge_base/enriched_snapshots.md index 4175c97d914..f004f0ded0c 100644 --- a/docs/docs/users/knowledge_base/enriched_snapshots.md +++ b/docs/docs/users/knowledge_base/enriched_snapshots.md @@ -64,7 +64,7 @@ at least one must be given. ### Automatically on startup When Forest imports a snapshot from a URL on startup, with `--auto-download-snapshot` or -`--import-snapshot `, it imports the enriched snapshot too. +`--import-snapshot `, it imports the enriched snapshot too. This step is best-effort: if a companion file can't be downloaded, verified, or imported, Forest logs a warning and starts without it. From 0353b9408f77b5561bcbe9a3dd9cf713c95ec1af Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Fri, 23 Oct 2026 15:12:29 -0400 Subject: [PATCH 24/36] chore: show download progress --- src/daemon/extended_snapshot.rs | 28 +++++++++++++++++--- src/daemon/mod.rs | 1 + src/utils/net/download_file.rs | 45 ++++++++++++++++++++++----------- 3 files changed, 56 insertions(+), 18 deletions(-) diff --git a/src/daemon/extended_snapshot.rs b/src/daemon/extended_snapshot.rs index 454ae046712..8f890ef63bc 100644 --- a/src/daemon/extended_snapshot.rs +++ b/src/daemon/extended_snapshot.rs @@ -13,6 +13,7 @@ use crate::db::EthMappingsStore; use crate::db::car::AnyCar; use crate::db::car::forest::new_forest_car_temp_path_in; use crate::prelude::*; +use crate::rpc::sync::SnapshotProgressTracker; use crate::shim::clock::ChainEpoch; use crate::state_manager::StateManager; use crate::utils::db::car_stream::CarStream; @@ -86,6 +87,7 @@ async fn download_extended_snapshot( snapshot_url: &Url, file: &ExtendedSnapshotFile, temp_dir: &Path, + snapshot_progress_tracker: &SnapshotProgressTracker, ) -> anyhow::Result { use sha2::{Digest as _, Sha256}; @@ -98,7 +100,14 @@ async fn download_extended_snapshot( let url = snapshot_url.join(&file.filename)?; let path = new_forest_car_temp_path_in(temp_dir)?; info!("Downloading extended snapshot: {url}"); - download_to(&url, &path, DownloadFileOption::Resumable, None).await?; + download_to( + &url, + &path, + DownloadFileOption::Resumable, + snapshot_progress_tracker.create_callback(), + ) + .await?; + snapshot_progress_tracker.completed(); let actual: [u8; 32] = Sha256::digest(tokio::fs::read(&path).await?).into(); anyhow::ensure!( @@ -212,6 +221,7 @@ pub(super) async fn maybe_import_extended_snapshots( head: &Tipset, snapshot_path: &Path, temp_dir: &Path, + snapshot_progress_tracker: &SnapshotProgressTracker, ) { // Extended snapshots are only published next to remote snapshots. let Some(snapshot_url) = path_as_url(snapshot_path) else { @@ -232,7 +242,13 @@ pub(super) async fn maybe_import_extended_snapshots( let receipts_events_imported = match &extended.receipts_events { Some(file) => { let result = async { - let path = download_extended_snapshot(&snapshot_url, file, temp_dir).await?; + let path = download_extended_snapshot( + &snapshot_url, + file, + temp_dir, + snapshot_progress_tracker, + ) + .await?; import_receipts_events_snapshot(state_manager.db(), &path).await } .await; @@ -258,7 +274,13 @@ pub(super) async fn maybe_import_extended_snapshots( if let Some(file) = &extended.tipset_lookup { let result = async { - let path = download_extended_snapshot(&snapshot_url, file, temp_dir).await?; + let path = download_extended_snapshot( + &snapshot_url, + file, + temp_dir, + snapshot_progress_tracker, + ) + .await?; let db = state_manager.db_owned(); tokio::task::spawn_blocking(move || import_tipset_lookup_snapshot(&db, &path, || {})) .await? diff --git a/src/daemon/mod.rs b/src/daemon/mod.rs index 07d89011ad0..a407317d301 100644 --- a/src/daemon/mod.rs +++ b/src/daemon/mod.rs @@ -179,6 +179,7 @@ async fn maybe_import_snapshot( &ts, path, &ctx.db_meta_data.get_forest_car_db_dir(), + &snapshot_tracker, ) .await; } diff --git a/src/utils/net/download_file.rs b/src/utils/net/download_file.rs index 75414881a83..8893a1ae074 100644 --- a/src/utils/net/download_file.rs +++ b/src/utils/net/download_file.rs @@ -91,6 +91,22 @@ fn call_progress_callback( } } +fn log_download_progress(downloaded: u64, total_size: u64, bytes_per_sec: f64, elapsed: Duration) { + let percent = downloaded + .checked_mul(100) + .and_then(|v| v.checked_div(total_size)) + .unwrap_or(0); + tracing::info!( + target: "forest::progress", + "Loading {} / {}, {}%, {}/s, elapsed time: {}", + downloaded.human_count_bytes(), + total_size.human_count_bytes(), + percent, + bytes_per_sec.human_count_bytes(), + format_duration(Duration::from_secs(elapsed.as_secs())) + ); +} + #[derive(Debug, Copy, Clone)] pub enum DownloadFileOption { NonResumable, @@ -304,6 +320,8 @@ async fn download_http_parallel( // Store elapsed millis since start_time to avoid needing a Mutex. let last_logged_millis = Arc::new(std::sync::atomic::AtomicU64::new(0)); let start_time = Instant::now(); + // Log the start and the end too, so a download shorter than the update frequency is visible. + log_download_progress(0, total_size, 0.0, Duration::ZERO); const UPDATE_FREQUENCY: Duration = Duration::from_secs(5); const UPDATE_FREQUENCY_MS: u64 = UPDATE_FREQUENCY.as_millis() as u64; @@ -377,24 +395,14 @@ async fn download_http_parallel( .is_ok() { let last_bytes = last_logged_bytes.load(Ordering::Relaxed); - let elapsed_secs = elapsed_ms as f64 / 1000.0; let seconds_since_last = (elapsed_ms - prev_ms) as f64 / 1000.0; let speed = downloaded.saturating_sub(last_bytes) as f64 / seconds_since_last.max(0.1); - let percent = downloaded - .checked_mul(100) - .and_then(|v| v.checked_div(total_size)) - .unwrap_or(0); - tracing::info!( - target: "forest::progress", - "Loading {} / {}, {}%, {}/s, elapsed time: {}", - downloaded.human_count_bytes(), - total_size.human_count_bytes(), - percent, - speed.human_count_bytes(), - format_duration(Duration::from_secs( - elapsed_secs as u64 - )) + log_download_progress( + downloaded, + total_size, + speed, + Duration::from_millis(elapsed_ms), ); last_logged_bytes.store(downloaded, Ordering::Relaxed); @@ -440,6 +448,13 @@ async fn download_http_parallel( for (i, result) in results.into_iter().enumerate() { result.with_context(|| format!("Chunk {i} failed"))?; } + let elapsed = start_time.elapsed(); + log_download_progress( + total_size, + total_size, + total_size as f64 / elapsed.as_secs_f64().max(0.001), + elapsed, + ); // Rename to final destination tokio::fs::rename(&tmp_dst_path, &dst_path) From 7962684806961c5e30cba03705d7d87fb314a026 Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Fri, 23 Oct 2026 23:19:25 -0400 Subject: [PATCH 25/36] chore: remove double loop while importing tipset_lookup --- src/daemon/extended_snapshot.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/daemon/extended_snapshot.rs b/src/daemon/extended_snapshot.rs index 8f890ef63bc..04a17d5a86d 100644 --- a/src/daemon/extended_snapshot.rs +++ b/src/daemon/extended_snapshot.rs @@ -162,9 +162,7 @@ where "epochs do not match, {epoch} in hamt, {} in database", ts.epoch() ); - Ok(()) - })?; - hamt.for_each_cacheless(|&epoch, tsk| { + db.set_tipset_key_at_epoch_raw(epoch, tsk)?; on_imported(); Ok(()) From 56ada40d726bcea951a96a45823585d866e02577 Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Tue, 27 Oct 2026 15:22:20 -0400 Subject: [PATCH 26/36] chore: addressing comments --- CHANGELOG.md | 4 +- .../knowledge_base/enriched_snapshots.md | 10 +-- src/daemon/extended_snapshot.rs | 63 ++++++++++--------- 3 files changed, 41 insertions(+), 36 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f7e6b09fe8..628f2b9ed45 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,8 @@ ### Added +- [#6802](https://github.com/ChainSafe/forest/issues/6802): When importing a snapshot from a URL, Forest now checks the snapshot's `.metadata.json` and, if extended snapshots are advertised, downloads and imports the receipts, events, and the tipset lookup alongside it + ### Changed ### Removed @@ -47,8 +49,6 @@ Non-mandatory release for all node operators. It includes fixes and improvements - [#7780](https://github.com/ChainSafe/forest/pull/7780): Removed the unused `chunk_size`, `buffer_size` and `skip_load` keys from the `[client]` config section. A config file that still sets them is rejected at startup; delete them from the file. The `--skip-load` flag is unchanged. -- [#6802](https://github.com/ChainSafe/forest/issues/6802): When importing a snapshot from a URL, Forest now checks the snapshot's `.metadata.json` and, if extended snapshots are advertised, downloads and imports the receipts, events, and the tipset lookup alongside it. - ### Changed - [#7702](https://github.com/ChainSafe/forest/issues/7702): `eth_estimateGas` matches Lotus v1.38.0 and reth. The estimate never exceeds the call's `gas` or, with `gasPrice` or `maxFeePerGas` set, the gas the sender can pay for after the value; without a price the sender's balance does not limit the gas. A revert that more gas fixes is searched past, and failures use the Ethereum codes: `-32000` when the cap is below the message inclusion cost or the sender affords no gas, `-32003` when the call runs out of gas within the cap or, with a price, the value exceeds the balance, `-32602` when both prices are set. See [lotus#13911](https://github.com/filecoin-project/lotus/pull/13911). diff --git a/docs/docs/users/knowledge_base/enriched_snapshots.md b/docs/docs/users/knowledge_base/enriched_snapshots.md index f004f0ded0c..abfcdc37d50 100644 --- a/docs/docs/users/knowledge_base/enriched_snapshots.md +++ b/docs/docs/users/knowledge_base/enriched_snapshots.md @@ -29,9 +29,9 @@ forest_snapshot_mainnet_2026-08-14_height_6279840_tipset_lookup.forest.car.zst ``` :::info -Enriched snapshots are a recent addition and are not yet listed by the snapshot -service. Discovery is still to be defined; for now the companion files follow the -suffix convention above next to their base snapshot. +The companion files are listed, with their SHA-256, under `Snapshot.Extended` in +the base snapshot's `.metadata.json`, published next to it as +`.metadata.json`. ::: ## Generating @@ -64,7 +64,9 @@ at least one must be given. ### Automatically on startup When Forest imports a snapshot from a URL on startup, with `--auto-download-snapshot` or -`--import-snapshot `, it imports the enriched snapshot too. +`--import-snapshot `, it imports the enriched snapshot too, then indexes the +imported receipts and events (block `logsBloom` and Ethereum mappings), so +`FOREST_ETH_RPC_COMPUTE_BLOOM_ON_MISS=1` is only needed for the manual recipe below. This step is best-effort: if a companion file can't be downloaded, verified, or imported, Forest logs a warning and starts without it. diff --git a/src/daemon/extended_snapshot.rs b/src/daemon/extended_snapshot.rs index 04a17d5a86d..858b1196ec2 100644 --- a/src/daemon/extended_snapshot.rs +++ b/src/daemon/extended_snapshot.rs @@ -24,7 +24,7 @@ use futures::TryStreamExt; use regex::Regex; use serde::Deserialize; use std::path::Path; -use std::time::Duration; +use std::time::{Duration, Instant}; use tracing::{info, warn}; use url::Url; @@ -38,22 +38,22 @@ struct SnapshotMetadataFile { #[derive(Debug, Deserialize)] struct SnapshotMetadataSection { - #[serde(rename = "Extended", default)] + #[serde(rename = "Extended")] extended: Option, } /// Auxiliary snapshots that enrich a base snapshot, see `Forest.ChainExport`. -#[derive(Debug, Default, Deserialize, PartialEq)] +#[derive(Debug, Deserialize)] struct ExtendedSnapshots { /// Message receipts and events - #[serde(rename = "Augmented", default)] + #[serde(rename = "Augmented")] receipts_events: Option, /// Tipset lookup table (checkpoint epoch to tipset key) - #[serde(rename = "Tipset lookup", default)] + #[serde(rename = "Tipset lookup")] tipset_lookup: Option, } -#[derive(Debug, Deserialize, PartialEq)] +#[derive(Debug, Deserialize)] struct ExtendedSnapshotFile { #[serde(rename = "Filename")] filename: String, @@ -67,7 +67,9 @@ struct ExtendedSnapshotFile { async fn fetch_extended_snapshots_info( snapshot_url: &Url, ) -> anyhow::Result> { - let metadata_url = Url::parse(&format!("{snapshot_url}.metadata.json"))?; + // Append to the path, not the whole URL, so dont affect any query string. + let mut metadata_url = snapshot_url.clone(); + metadata_url.set_path(&format!("{}.metadata.json", snapshot_url.path())); let response = crate::utils::net::global_http_client() .get(metadata_url.clone()) .timeout(Duration::from_secs(30)) @@ -123,17 +125,21 @@ async fn import_receipts_events_snapshot(db: &DB, path: &Path) -> anyhow::Re where DB: Blockstore, { + let started = Instant::now(); let mut car_stream = CarStream::new_from_path(path).await?; while let Some(block) = car_stream.try_next().await? { block.validate()?; db.put_keyed(&block.cid, &block.data)?; } + info!( + "Imported message receipts and events in {:.1}s", + started.elapsed().as_secs_f64() + ); Ok(()) } /// Populates the tipset lookup table in `db` from a tipset lookup snapshot, calling -/// `on_imported` after each entry is written. Every entry is validated against the tipsets -/// in `db` before any of them is written. +/// `on_imported` after each entry is written. pub fn import_tipset_lookup_snapshot( db: &DB, path: &Path, @@ -142,6 +148,7 @@ pub fn import_tipset_lookup_snapshot( where DB: Blockstore + EthMappingsStore, { + let started = Instant::now(); let hamt_car = AnyCar::try_from(path)?; anyhow::ensure!( hamt_car.header_v1().roots.len() == 1, @@ -167,6 +174,10 @@ where on_imported(); Ok(()) })?; + info!( + "Imported tipset lookup entries in {:.1}s", + started.elapsed().as_secs_f64() + ); Ok(()) } @@ -187,6 +198,7 @@ async fn backfill_receipts_events( else { return Ok(()); }; + let started = Instant::now(); let guard = BackfillGuard::try_start()?; let options = BackfillOptions { allow_recompute: false, @@ -203,9 +215,10 @@ async fn backfill_receipts_events( ); } else { info!( - "Indexed {} tipsets from epoch {} to {to}", + "Indexed {} tipsets from epoch {} to {to} in {:.1}s", report.indexed, - from.epoch() + from.epoch(), + started.elapsed().as_secs_f64() ); } Ok(()) @@ -250,22 +263,13 @@ pub(super) async fn maybe_import_extended_snapshots( import_receipts_events_snapshot(state_manager.db(), &path).await } .await; - match result { - Ok(()) => { - info!( - "Imported message receipts and events from {}", - file.filename - ); - true - } - Err(e) => { - warn!( - "Failed to import receipts and events snapshot {}: {e:#}", - file.filename - ); - false - } + if let Err(e) = &result { + warn!( + "Failed to import receipts and events snapshot {}: {e:#}", + file.filename + ); } + result.is_ok() } None => false, }; @@ -284,12 +288,11 @@ pub(super) async fn maybe_import_extended_snapshots( .await? } .await; - match result { - Ok(()) => info!("Imported tipset lookup entries from {}", file.filename), - Err(e) => warn!( + if let Err(e) = result { + warn!( "Failed to import tipset lookup snapshot {}: {e:#}", file.filename - ), + ); } } From a9bdaff2184bc65bad2d26c65f8006c5d9725cc4 Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Wed, 28 Oct 2026 11:09:24 -0400 Subject: [PATCH 27/36] chore: show tipset import entries progress --- src/daemon/extended_snapshot.rs | 36 +++++++++------------------------ src/daemon/mod.rs | 1 - src/utils/io/progress_log.rs | 4 ++-- 3 files changed, 11 insertions(+), 30 deletions(-) diff --git a/src/daemon/extended_snapshot.rs b/src/daemon/extended_snapshot.rs index 858b1196ec2..6ebc18134d9 100644 --- a/src/daemon/extended_snapshot.rs +++ b/src/daemon/extended_snapshot.rs @@ -13,10 +13,10 @@ use crate::db::EthMappingsStore; use crate::db::car::AnyCar; use crate::db::car::forest::new_forest_car_temp_path_in; use crate::prelude::*; -use crate::rpc::sync::SnapshotProgressTracker; use crate::shim::clock::ChainEpoch; use crate::state_manager::StateManager; use crate::utils::db::car_stream::CarStream; +use crate::utils::io::progress_log::Progress; use crate::utils::net::{DownloadFileOption, download_to}; use anyhow::Context; use fil_actors_shared::fvm_ipld_hamt::Hamt; @@ -89,7 +89,6 @@ async fn download_extended_snapshot( snapshot_url: &Url, file: &ExtendedSnapshotFile, temp_dir: &Path, - snapshot_progress_tracker: &SnapshotProgressTracker, ) -> anyhow::Result { use sha2::{Digest as _, Sha256}; @@ -102,14 +101,7 @@ async fn download_extended_snapshot( let url = snapshot_url.join(&file.filename)?; let path = new_forest_car_temp_path_in(temp_dir)?; info!("Downloading extended snapshot: {url}"); - download_to( - &url, - &path, - DownloadFileOption::Resumable, - snapshot_progress_tracker.create_callback(), - ) - .await?; - snapshot_progress_tracker.completed(); + download_to(&url, &path, DownloadFileOption::Resumable, None).await?; let actual: [u8; 32] = Sha256::digest(tokio::fs::read(&path).await?).into(); anyhow::ensure!( @@ -232,7 +224,6 @@ pub(super) async fn maybe_import_extended_snapshots( head: &Tipset, snapshot_path: &Path, temp_dir: &Path, - snapshot_progress_tracker: &SnapshotProgressTracker, ) { // Extended snapshots are only published next to remote snapshots. let Some(snapshot_url) = path_as_url(snapshot_path) else { @@ -253,13 +244,7 @@ pub(super) async fn maybe_import_extended_snapshots( let receipts_events_imported = match &extended.receipts_events { Some(file) => { let result = async { - let path = download_extended_snapshot( - &snapshot_url, - file, - temp_dir, - snapshot_progress_tracker, - ) - .await?; + let path = download_extended_snapshot(&snapshot_url, file, temp_dir).await?; import_receipts_events_snapshot(state_manager.db(), &path).await } .await; @@ -276,16 +261,13 @@ pub(super) async fn maybe_import_extended_snapshots( if let Some(file) = &extended.tipset_lookup { let result = async { - let path = download_extended_snapshot( - &snapshot_url, - file, - temp_dir, - snapshot_progress_tracker, - ) - .await?; + let path = download_extended_snapshot(&snapshot_url, file, temp_dir).await?; let db = state_manager.db_owned(); - tokio::task::spawn_blocking(move || import_tipset_lookup_snapshot(&db, &path, || {})) - .await? + tokio::task::spawn_blocking(move || { + let mut progress = Progress::new("Importing tipset lookup entries"); + import_tipset_lookup_snapshot(&db, &path, || progress.inc(1)) + }) + .await? } .await; if let Err(e) = result { diff --git a/src/daemon/mod.rs b/src/daemon/mod.rs index a407317d301..07d89011ad0 100644 --- a/src/daemon/mod.rs +++ b/src/daemon/mod.rs @@ -179,7 +179,6 @@ async fn maybe_import_snapshot( &ts, path, &ctx.db_meta_data.get_forest_car_db_dir(), - &snapshot_tracker, ) .await; } diff --git a/src/utils/io/progress_log.rs b/src/utils/io/progress_log.rs index 0b65864ec49..a7dbaf8e3f0 100644 --- a/src/utils/io/progress_log.rs +++ b/src/utils/io/progress_log.rs @@ -111,7 +111,7 @@ enum ItemType { } impl Progress { - fn new(message: &str) -> Self { + pub(crate) fn new(message: &str) -> Self { let now = Instant::now(); Self { completed_items: 0, @@ -135,7 +135,7 @@ impl Progress { self } - fn inc(&mut self, value: u64) { + pub(crate) fn inc(&mut self, value: u64) { self.completed_items += value; self.emit_log_if_required(); From 22146db84a8636ccc5605a7caf28d0c652f037ba Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Tue, 3 Nov 2026 09:01:33 -0400 Subject: [PATCH 28/36] chore: fallback to url when HEAD request cannot redirect successfully --- src/cli_shared/snapshot.rs | 50 ++++++++++++++++++++++++++++++++------ src/daemon/mod.rs | 2 +- 2 files changed, 43 insertions(+), 9 deletions(-) diff --git a/src/cli_shared/snapshot.rs b/src/cli_shared/snapshot.rs index 38f78ea70a4..3c6f1d246b9 100644 --- a/src/cli_shared/snapshot.rs +++ b/src/cli_shared/snapshot.rs @@ -113,16 +113,22 @@ pub async fn peek( )) } -/// Returns the final URL after redirection(s) of `url`, e.g. a stable `latest` snapshot URL. -pub async fn peek_url(url: Url) -> anyhow::Result { +/// Returns the final URL after redirection(s). +/// if server rejects `HEAD` requests returns the `url` itself +pub async fn peek_url(url: Url) -> Url { let response = crate::utils::net::global_http_client() - .head(url) + .head(url.clone()) .timeout(Duration::from_secs(30)) .send() - .await? - .error_for_status() - .context("server returned an error response")?; - Ok(response.url().clone()) + .await + .and_then(|response| response.error_for_status()); + match response { + Ok(response) => response.url().clone(), + Err(e) => { + tracing::debug!("Couldn't follow the redirects of {url}: {e:#}"); + url + } + } } // Extract file paths from content-disposition values: @@ -447,8 +453,36 @@ mod parse { #[cfg(test)] mod tests { - use super::parse_content_disposition; + use super::{parse_content_disposition, peek_url}; use reqwest::header::HeaderValue; + use url::Url; + + #[rstest::rstest] + #[case::follows_redirect("/latest", "/snapshot.forest.car.zst")] + #[case::head_rejected("/post-only", "/post-only")] + #[case::not_found("/missing", "/missing")] + #[tokio::test] + async fn peek_url_falls_back_to_given_url(#[case] path: &str, #[case] expected: &str) { + use axum::{ + Router, + response::Redirect, + routing::{get, post}, + }; + + let app = Router::new() + .route( + "/latest", + get(|| async { Redirect::temporary("/snapshot.forest.car.zst") }), + ) + .route("/snapshot.forest.car.zst", get(|| async { "snapshot" })) + .route("/post-only", post(|| async { "post" })); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { axum::serve(listener, app).await }); + + let url = Url::parse(&format!("http://{addr}{path}")).unwrap(); + assert_eq!(peek_url(url).await.path(), expected); + } #[test] fn content_disposition_forest() { diff --git a/src/daemon/mod.rs b/src/daemon/mod.rs index 07d89011ad0..12e2e746b95 100644 --- a/src/daemon/mod.rs +++ b/src/daemon/mod.rs @@ -149,7 +149,7 @@ async fn maybe_import_snapshot( && let Some(path) = &config.client.snapshot_path { let path = &match path_as_url(path) { - Some(url) => PathBuf::from(snapshot::peek_url(url).await?.as_str()), + Some(url) => PathBuf::from(snapshot::peek_url(url).await.as_str()), None => path.clone(), }; let (car_db_path, ts) = import_chain_as_forest_car( From ca6a40be55f125ad8f7f9932a5ea0a6707e6d560 Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Tue, 3 Nov 2026 09:26:28 -0400 Subject: [PATCH 29/36] chore: url join + query so it dont drop any url info --- src/daemon/extended_snapshot.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/daemon/extended_snapshot.rs b/src/daemon/extended_snapshot.rs index 6ebc18134d9..4b5ada4f39c 100644 --- a/src/daemon/extended_snapshot.rs +++ b/src/daemon/extended_snapshot.rs @@ -98,7 +98,9 @@ async fn download_extended_snapshot( "invalid extended snapshot filename: {}", file.filename ); - let url = snapshot_url.join(&file.filename)?; + let mut url = snapshot_url.join(&file.filename)?; + url.set_query(snapshot_url.query()); + let path = new_forest_car_temp_path_in(temp_dir)?; info!("Downloading extended snapshot: {url}"); download_to(&url, &path, DownloadFileOption::Resumable, None).await?; From ad737e614d83d382ccd3727904c8c2886d741e8f Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Tue, 3 Nov 2026 09:54:21 -0400 Subject: [PATCH 30/36] backfill cover only the number of state roots advertised at snapshot metadata --- src/daemon/extended_snapshot.rs | 52 ++++++++++++++++----------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/src/daemon/extended_snapshot.rs b/src/daemon/extended_snapshot.rs index 4b5ada4f39c..698ea475bad 100644 --- a/src/daemon/extended_snapshot.rs +++ b/src/daemon/extended_snapshot.rs @@ -38,6 +38,9 @@ struct SnapshotMetadataFile { #[derive(Debug, Deserialize)] struct SnapshotMetadataSection { + /// Number of epochs below the head with state, which the receipts and events also cover + #[serde(rename = "State-roots")] + state_roots: ChainEpoch, #[serde(rename = "Extended")] extended: Option, } @@ -62,11 +65,10 @@ struct ExtendedSnapshotFile { sha256: [u8; 32], } -/// Fetches the `.metadata.json` published next to `snapshot_url` and returns its extended -/// snapshots, if any. -async fn fetch_extended_snapshots_info( +/// Fetches the `.metadata.json` published next to `snapshot_url`. +async fn fetch_metadata_snapshots_info( snapshot_url: &Url, -) -> anyhow::Result> { +) -> anyhow::Result { // Append to the path, not the whole URL, so dont affect any query string. let mut metadata_url = snapshot_url.clone(); metadata_url.set_path(&format!("{}.metadata.json", snapshot_url.path())); @@ -80,7 +82,7 @@ async fn fetch_extended_snapshots_info( .json() .await .with_context(|| format!("failed to parse snapshot metadata at {metadata_url}"))?; - Ok(metadata.snapshot.extended) + Ok(metadata.snapshot) } /// Downloads the extended snapshot `file`, published next to `snapshot_url`, into `temp_dir` @@ -100,7 +102,7 @@ async fn download_extended_snapshot( ); let mut url = snapshot_url.join(&file.filename)?; url.set_query(snapshot_url.query()); - + let path = new_forest_car_temp_path_in(temp_dir)?; info!("Downloading extended snapshot: {url}"); download_to(&url, &path, DownloadFileOption::Resumable, None).await?; @@ -176,22 +178,16 @@ where } /// Indexes the tipsets whose receipts and events were just imported (block logs blooms and -/// Ethereum mappings), as `forest-tool index backfill` does. +/// Ethereum mappings), as `forest-tool index backfill` does: those within `depth` epochs below +/// `head`, which the snapshot holds state for. async fn backfill_receipts_events( state_manager: &StateManager, head: &Tipset, + depth: ChainEpoch, ) -> anyhow::Result<()> { - let db = state_manager.db(); - let from = Tipset::load_required(db, head.parents())?; - let Some(to) = from - .shallow_clone() - .chain(db) - .take_while(|ts| db.has(ts.parent_state()).unwrap_or_default()) - .last() - .map(|ts| ts.epoch()) - else { - return Ok(()); - }; + let from = Tipset::load_required(state_manager.db(), head.parents())?; + // Inclusive: the lowest tipset's receipts and resulting state are in its child. + let to = (head.epoch() - depth).max(0); let started = Instant::now(); let guard = BackfillGuard::try_start()?; let options = BackfillOptions { @@ -231,17 +227,17 @@ pub(super) async fn maybe_import_extended_snapshots( let Some(snapshot_url) = path_as_url(snapshot_path) else { return; }; - let extended = match fetch_extended_snapshots_info(&snapshot_url).await { - Ok(Some(extended)) => extended, - Ok(None) => { - info!("No extended snapshots advertised for {snapshot_url}"); - return; - } + let metadata = match fetch_metadata_snapshots_info(&snapshot_url).await { + Ok(metadata) => metadata, Err(e) => { warn!("Failed to fetch extended snapshots for {snapshot_url}: {e:#}"); return; } }; + let Some(extended) = metadata.extended else { + info!("No extended snapshots advertised for {snapshot_url}"); + return; + }; let receipts_events_imported = match &extended.receipts_events { Some(file) => { @@ -281,7 +277,8 @@ pub(super) async fn maybe_import_extended_snapshots( } // Index once both snapshots are imported, so each import's logs stay together. - if receipts_events_imported && let Err(e) = backfill_receipts_events(state_manager, head).await + if receipts_events_imported + && let Err(e) = backfill_receipts_events(state_manager, head, metadata.state_roots).await { warn!("Failed to index imported receipts and events: {e:#}"); } @@ -298,6 +295,7 @@ mod test { "Snapshot": { "Snapshot version": "2", "Epoch": 4137600, + "State-roots": 2000, "Extended": { "Augmented": { "Filename": "forest_snapshot_calibnet_2026-10-08_height_4137600_receipts_events.forest.car.zst", @@ -312,6 +310,7 @@ mod test { "Build Information": {} }"#; let metadata: SnapshotMetadataFile = serde_json::from_str(json).unwrap(); + assert_eq!(metadata.snapshot.state_roots, 2000); let extended = metadata.snapshot.extended.unwrap(); assert_eq!( extended.receipts_events.unwrap().filename, @@ -325,7 +324,8 @@ mod test { #[test] fn parse_snapshot_metadata_without_extended() { - let json = r#"{ "Snapshot": { "Snapshot version": "2", "Epoch": 4137600 } }"#; + let json = + r#"{ "Snapshot": { "Snapshot version": "2", "Epoch": 4137600, "State-roots": 2000 } }"#; let metadata: SnapshotMetadataFile = serde_json::from_str(json).unwrap(); assert!(metadata.snapshot.extended.is_none()); } From dc08f7083c9958a79d76cfe9a6e7f7a87ba0cfec Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Tue, 3 Nov 2026 10:24:05 -0400 Subject: [PATCH 31/36] chore: validate before writing --- src/daemon/extended_snapshot.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/daemon/extended_snapshot.rs b/src/daemon/extended_snapshot.rs index 698ea475bad..85052e99537 100644 --- a/src/daemon/extended_snapshot.rs +++ b/src/daemon/extended_snapshot.rs @@ -154,6 +154,7 @@ where let hamt: Hamt<_, TipsetKey, ChainEpoch> = Hamt::load_with_bit_width(&hamt_root, hamt_car, TIPSET_LOOKUP_HAMT_BIT_WIDTH) .context("failed to load tipset lookup HAMT")?; + info!("Validating tipset lookup entries against the chain, this can take a while"); hamt.for_each_cacheless(|&epoch, tsk| { anyhow::ensure!( ChainIndex::is_tipset_lookup_checkpoint(epoch), @@ -165,7 +166,9 @@ where "epochs do not match, {epoch} in hamt, {} in database", ts.epoch() ); - + Ok(()) + })?; + hamt.for_each_cacheless(|&epoch, tsk| { db.set_tipset_key_at_epoch_raw(epoch, tsk)?; on_imported(); Ok(()) From 440506f5bd90c57a253ac364c439eb1ea14ba9cd Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Tue, 3 Nov 2026 11:31:00 -0400 Subject: [PATCH 32/36] place a max cap on backfill & resolve fail if no extended snapshots are present --- scripts/tests/external-rpc-checks/resolve.rb | 6 +++++- src/daemon/extended_snapshot.rs | 7 +++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/scripts/tests/external-rpc-checks/resolve.rb b/scripts/tests/external-rpc-checks/resolve.rb index a56e82a2eae..ab8762518ca 100644 --- a/scripts/tests/external-rpc-checks/resolve.rb +++ b/scripts/tests/external-rpc-checks/resolve.rb @@ -2,7 +2,8 @@ # Picks the newest ${FOREST_CHAIN} snapshot whose head lies within the first 2000 epochs of # the UTC day: importing it indexes its receipts and events back to the day's start, -# and the node syncs the rest. Writes its URL and the day's epoch range under /data. +# and the node syncs the rest. Writes its URL and the day's epoch range under /data, and fails +# if the snapshot advertises no receipts and events snapshot. require 'date' require 'json' @@ -47,6 +48,9 @@ def fetch(uri, attempts: 3) exit 2 end +metadata = JSON.parse(fetch(URI("#{url}.metadata.json"))) +abort "#{url} advertises no receipts and events snapshot" unless metadata.dig('Snapshot', 'Extended', 'Augmented') + File.write('/data/snapshot-url', "#{url}\n") File.write('/data/check-target', "#{chain} #{start} #{finish}\n") puts "#{chain} #{day.iso8601} is epochs #{start}..#{finish}; snapshot at #{epoch}: #{url}" diff --git a/src/daemon/extended_snapshot.rs b/src/daemon/extended_snapshot.rs index 85052e99537..4277711278a 100644 --- a/src/daemon/extended_snapshot.rs +++ b/src/daemon/extended_snapshot.rs @@ -181,16 +181,15 @@ where } /// Indexes the tipsets whose receipts and events were just imported (block logs blooms and -/// Ethereum mappings), as `forest-tool index backfill` does: those within `depth` epochs below -/// `head`, which the snapshot holds state for. +/// Ethereum mappings), as `forest-tool index backfill` does async fn backfill_receipts_events( state_manager: &StateManager, head: &Tipset, depth: ChainEpoch, ) -> anyhow::Result<()> { let from = Tipset::load_required(state_manager.db(), head.parents())?; - // Inclusive: the lowest tipset's receipts and resulting state are in its child. - let to = (head.epoch() - depth).max(0); + // the lowest tipset that can be indexed is at `head - depth + 1` + let to = head.epoch() - depth.clamp(0, head.epoch()) + 1; let started = Instant::now(); let guard = BackfillGuard::try_start()?; let options = BackfillOptions { From 1aca6c29e09e0b1ee102ccec35a68344e46dac64 Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Tue, 3 Nov 2026 15:07:18 -0400 Subject: [PATCH 33/36] chore: use `BlockstoreWithWriteBuffer` --- src/daemon/extended_snapshot.rs | 4 +++- src/db/blockstore_with_write_buffer.rs | 3 +++ src/db/mod.rs | 2 +- src/tool/subcommands/db_cmd.rs | 4 +--- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/daemon/extended_snapshot.rs b/src/daemon/extended_snapshot.rs index 4277711278a..d2af834ad1a 100644 --- a/src/daemon/extended_snapshot.rs +++ b/src/daemon/extended_snapshot.rs @@ -9,9 +9,9 @@ use crate::chain::{TIPSET_LOOKUP_HAMT_BIT_WIDTH, index::ChainIndex}; use crate::daemon::db_util::{ BackfillGuard, BackfillOptions, RangeSpec, path_as_url, run_backfill, }; -use crate::db::EthMappingsStore; use crate::db::car::AnyCar; use crate::db::car::forest::new_forest_car_temp_path_in; +use crate::db::{BlockstoreWithWriteBuffer, DB_WRITE_BUFFER_CAPACITY, EthMappingsStore}; use crate::prelude::*; use crate::shim::clock::ChainEpoch; use crate::state_manager::StateManager; @@ -122,11 +122,13 @@ where DB: Blockstore, { let started = Instant::now(); + let db = BlockstoreWithWriteBuffer::new_with_capacity(db, DB_WRITE_BUFFER_CAPACITY); let mut car_stream = CarStream::new_from_path(path).await?; while let Some(block) = car_stream.try_next().await? { block.validate()?; db.put_keyed(&block.cid, &block.data)?; } + drop(db); info!( "Imported message receipts and events in {:.1}s", started.elapsed().as_secs_f64() diff --git a/src/db/blockstore_with_write_buffer.rs b/src/db/blockstore_with_write_buffer.rs index afb4b99acf6..1a60ab1d3d3 100644 --- a/src/db/blockstore_with_write_buffer.rs +++ b/src/db/blockstore_with_write_buffer.rs @@ -5,6 +5,9 @@ use crate::prelude::*; use ahash::HashMap; use parking_lot::RwLock; +/// Number of blocks buffered between writes when bulk-importing CAR files. +pub const DB_WRITE_BUFFER_CAPACITY: usize = 10_000; + pub struct BlockstoreWithWriteBuffer { inner: DB, buffer: RwLock>>, diff --git a/src/db/mod.rs b/src/db/mod.rs index 30f489196d3..671eeb66708 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -15,7 +15,7 @@ pub mod parity_db_config; pub mod ttl; pub use blockstore_with_read_cache::*; -pub use blockstore_with_write_buffer::BlockstoreWithWriteBuffer; +pub use blockstore_with_write_buffer::{BlockstoreWithWriteBuffer, DB_WRITE_BUFFER_CAPACITY}; pub use db_impl::DbImpl; pub use either::Either; pub use fvm_ipld_blockstore::Blockstore; diff --git a/src/tool/subcommands/db_cmd.rs b/src/tool/subcommands/db_cmd.rs index 8cb4978b6b0..fb7852744ba 100644 --- a/src/tool/subcommands/db_cmd.rs +++ b/src/tool/subcommands/db_cmd.rs @@ -10,7 +10,7 @@ use crate::daemon::db_util::load_all_forest_cars; use crate::daemon::extended_snapshot::import_tipset_lookup_snapshot; use crate::db::car::ManyCar; use crate::db::db_engine::{db_root, open_db}; -use crate::db::{BlockstoreWithWriteBuffer, CAR_DB_DIR_NAME}; +use crate::db::{BlockstoreWithWriteBuffer, CAR_DB_DIR_NAME, DB_WRITE_BUFFER_CAPACITY}; use crate::networks::NetworkChain; use crate::prelude::*; use crate::utils::db::car_stream::CarStream; @@ -121,8 +121,6 @@ impl DBCommands { db, skip_validation: no_validation, } => { - const DB_WRITE_BUFFER_CAPACITY: usize = 10000; - let db_root_path = if let Some(db) = db { db } else { From 6f776ec1ac4a7d9535b74dff25e303a88518adc0 Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Wed, 4 Nov 2026 13:37:46 -0400 Subject: [PATCH 34/36] chore: include tests --- src/daemon/extended_snapshot.rs | 110 ++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) diff --git a/src/daemon/extended_snapshot.rs b/src/daemon/extended_snapshot.rs index d2af834ad1a..814378b9c00 100644 --- a/src/daemon/extended_snapshot.rs +++ b/src/daemon/extended_snapshot.rs @@ -291,7 +291,117 @@ pub(super) async fn maybe_import_extended_snapshots( #[cfg(test)] mod test { use super::*; + use crate::chain::index::tests::{genesis_tipset, persist_tipset, tipset_child}; + use crate::db::MemoryDB; use crate::utils::encoding::hex; + use rstest::rstest; + + // Imports a lookup table with the single entry `hamt_epoch -> tipset at tipset_epoch` into a + // chain of tipsets at epochs 20, 21 and 40; `None` points at a tipset missing from the chain. + #[rstest] + #[case::checkpoint(40, Some(40), None)] + #[case::non_checkpoint_epoch(21, Some(21), Some("is not a tipset lookup checkpoint"))] + #[case::epoch_mismatch(40, Some(20), Some("epochs do not match"))] + #[case::unknown_tipset(40, None, Some("Required tipset missing"))] + #[tokio::test] + async fn import_tipset_lookup( + #[case] hamt_epoch: ChainEpoch, + #[case] tipset_epoch: Option, + #[case] expected_err: Option<&str>, + ) { + let db = MemoryDB::default(); + let mut parent = genesis_tipset(); + persist_tipset(&parent, &db); + let mut chain = vec![]; + for epoch in [20, 21, 40] { + let ts = tipset_child(&parent, epoch); + persist_tipset(&ts, &db); + chain.push(ts.shallow_clone()); + parent = ts; + } + let tsk = match tipset_epoch { + Some(epoch) => chain + .iter() + .find(|ts| ts.epoch() == epoch) + .unwrap() + .key() + .clone(), + None => tipset_child(&parent, 40).key().clone(), + }; + + let hamt_store = MemoryDB::default(); + let mut hamt: Hamt<_, TipsetKey, ChainEpoch> = + Hamt::new_with_bit_width(&hamt_store, TIPSET_LOOKUP_HAMT_BIT_WIDTH); + hamt.set(hamt_epoch, tsk.clone()).unwrap(); + let root = hamt.flush().unwrap(); + let mut car = vec![]; + hamt_store + .export_forest_car_with_roots(nunny::vec![root], &mut car) + .await + .unwrap(); + let temp_dir = tempfile::tempdir().unwrap(); + let car_path = temp_dir.path().join("tipset_lookup.forest.car.zst"); + std::fs::write(&car_path, car).unwrap(); + + let result = import_tipset_lookup_snapshot(&db, &car_path, || {}); + match expected_err { + None => { + result.unwrap(); + assert_eq!(db.tipset_key_by_epoch(hamt_epoch).unwrap(), Some(tsk)); + } + Some(expected) => { + let err = format!("{:#}", result.unwrap_err()); + assert!(err.contains(expected), "{err}"); + // Validation happens before any write. + assert_eq!(db.tipset_key_by_epoch(hamt_epoch).unwrap(), None); + } + } + } + + #[rstest] + #[case::ok("companion.forest.car.zst", true, None)] + #[case::checksum_mismatch("companion.forest.car.zst", false, Some("checksum mismatch"))] + #[case::path_traversal( + "../companion.forest.car.zst", + true, + Some("invalid extended snapshot filename") + )] + #[tokio::test] + async fn download_extended_snapshot_verifies( + #[case] filename: &str, + #[case] matching_checksum: bool, + #[case] expected_err: Option<&str>, + ) { + use sha2::{Digest as _, Sha256}; + let dummy_content: &[u8] = b"companion snapshot bytes" + + let app = axum::Router::new().route( + "/companion.forest.car.zst", + axum::routing::get(|| async { dummy_content }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { axum::serve(listener, app).await }); + + let snapshot_url = Url::parse(&format!("http://{addr}/base.forest.car.zst")).unwrap(); + let file = ExtendedSnapshotFile { + filename: filename.into(), + sha256: if matching_checksum { + Sha256::digest(dummy_content).into() + } else { + [0; 32] + }, + }; + let temp_dir = tempfile::tempdir().unwrap(); + let result = download_extended_snapshot(&snapshot_url, &file, temp_dir.path()).await; + match expected_err { + None => assert_eq!(std::fs::read(result.unwrap()).unwrap(), dummy_content), + Some(expected) => { + let err = format!("{:#}", result.unwrap_err()); + assert!(err.contains(expected), "{err}"); + } + } + } #[test] fn parse_snapshot_metadata_with_extended() { From 46655734823d7465d007dbc8bfcaf478037f292f Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Wed, 4 Nov 2026 14:17:31 -0400 Subject: [PATCH 35/36] chore: fix lint --- src/daemon/extended_snapshot.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/daemon/extended_snapshot.rs b/src/daemon/extended_snapshot.rs index 814378b9c00..29eefd118d4 100644 --- a/src/daemon/extended_snapshot.rs +++ b/src/daemon/extended_snapshot.rs @@ -373,11 +373,11 @@ mod test { #[case] expected_err: Option<&str>, ) { use sha2::{Digest as _, Sha256}; - let dummy_content: &[u8] = b"companion snapshot bytes" + let dummy_content: &'static [u8] = b"companion snapshot bytes"; let app = axum::Router::new().route( "/companion.forest.car.zst", - axum::routing::get(|| async { dummy_content }), + axum::routing::get(move || async move { dummy_content }), ); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); From 297524768230ddd2def04482a41d006ac0ccc7c5 Mon Sep 17 00:00:00 2001 From: EclesioMeloJunior Date: Mon, 9 Nov 2026 19:23:06 -0400 Subject: [PATCH 36/36] chore: addressing most of the comments --- scripts/tests/external-rpc-checks/verify.rb | 36 +- src/chain/mod.rs | 62 ++- src/chain/tests.rs | 70 +++ src/cli_shared/snapshot.rs | 7 +- src/daemon/db_util.rs | 10 +- src/daemon/extended_snapshot.rs | 482 +++++++++++++------- src/daemon/mod.rs | 28 +- src/db/blockstore_with_write_buffer.rs | 11 +- src/db/mod.rs | 2 +- src/tool/subcommands/db_cmd.rs | 26 +- src/tool/subcommands/snapshot_cmd.rs | 8 +- src/tool/subcommands/snapshot_cmd/tests.rs | 4 +- src/utils/db/car_util.rs | 71 +++ src/utils/net/download_file.rs | 49 +- 14 files changed, 628 insertions(+), 238 deletions(-) diff --git a/scripts/tests/external-rpc-checks/verify.rb b/scripts/tests/external-rpc-checks/verify.rb index 57987db1c5f..7894a0fda39 100644 --- a/scripts/tests/external-rpc-checks/verify.rb +++ b/scripts/tests/external-rpc-checks/verify.rb @@ -1,31 +1,35 @@ # frozen_string_literal: true -# Confirms the node indexed the epochs the checks query. +# Confirms the node indexed the epochs the checks query: the tipset is served and the receipts +# of its messages are in the store, read back through the standard chain API. require 'json' require 'net/http' url = URI("http://#{ENV.fetch('FOREST_RPC_URL')}") -ARGV.each do |argument| - epoch = Integer(argument) +def rpc(url, method, params) request = Net::HTTP::Post.new(url, 'Content-Type' => 'application/json') - request.body = { - jsonrpc: '2.0', - id: 1, - method: 'eth_getBlockByNumber', - params: ["0x#{epoch.to_s(16)}", false] - }.to_json + request.body = { jsonrpc: '2.0', id: 1, method: method, params: params }.to_json + JSON.parse(Net::HTTP.start(url.hostname, url.port) { |http| http.request(request) }.body) +end - response = Net::HTTP.start(url.hostname, url.port) { |http| http.request(request) } - body = JSON.parse(response.body) +ARGV.each do |argument| + epoch = Integer(argument) + block = rpc(url, 'eth_getBlockByNumber', ["0x#{epoch.to_s(16)}", false]) # Error code 12 denotes a null round across implementations. # https://github.com/ChainSafe/forest/blob/8c980e679b6606c534c6eb5bd8aa03ff8cc6f5c0/src/rpc/methods/eth/errors.rs#L22 - null_round = body.dig('error', 'code') == 12 - - unless body.dig('result', 'number') || null_round - abort "epoch #{epoch} is not indexed: #{response.code} #{response.body}" + if block.dig('error', 'code') == 12 + puts "epoch #{epoch} was a null round" + next end + abort "epoch #{epoch} is not indexed: #{block}" unless block.dig('result', 'number') - puts "epoch #{epoch} #{null_round ? 'was a null round' : 'is indexed'}" + # The receipts of an epoch's messages are stored under its child tipset, the first non-null + # tipset above it. Reading them back is the same check any client would make. + child = rpc(url, 'Filecoin.ChainGetTipSetAfterHeight', [epoch + 1, []]) + abort "epoch #{epoch} has no tipset above it: #{child['error']}" if child['error'] + receipts = rpc(url, 'Filecoin.ChainGetParentReceipts', [child.dig('result', 'Cids', 0)]) + abort "epoch #{epoch} has no receipts: #{receipts['error']}" if receipts['error'] + puts "epoch #{epoch} is indexed" end diff --git a/src/chain/mod.rs b/src/chain/mod.rs index ccc7aa5a430..4e37c84e0c9 100644 --- a/src/chain/mod.rs +++ b/src/chain/mod.rs @@ -13,8 +13,9 @@ pub use self::{snapshot_format::*, store::*, weight::*}; use crate::blocks::{Tipset, TipsetKey}; use crate::chain::index::{ChainIndex, ResolveNullTipset}; use crate::cid_collections::{CidHashSet, CidHashSetLike, FileBackedCidHashSet}; -use crate::db::IndexMapBlockstore; +use crate::db::car::AnyCar; use crate::db::car::forest::{self, ForestCarFrame, finalize_frame}; +use crate::db::{EthMappingsStore, IndexMapBlockstore}; use crate::ipld::{IpldStream, should_save_block_to_snapshot, stream_chain}; use crate::prelude::*; use crate::shim::executor::Receipt; @@ -481,3 +482,62 @@ pub async fn export_receipts_events_to_forest_car( Ok(()) } + +/// Check that the entry is at a checkpoint epoch and that the stored tipset is at that epoch. +pub fn validate_tipset_lookup_entry( + db: &DB, + epoch: ChainEpoch, + tsk: &TipsetKey, +) -> anyhow::Result +where + DB: Blockstore, +{ + anyhow::ensure!( + ChainIndex::is_tipset_lookup_checkpoint(epoch), + "epoch {epoch} is not a tipset lookup checkpoint" + ); + let ts: Tipset = Tipset::load_required(db, tsk)?; + anyhow::ensure!( + ts.epoch() == epoch, + "epochs do not match, {epoch} in hamt, {} in database", + ts.epoch() + ); + Ok(ts) +} + +/// Imports a tipset lookup snapshot into `db`, calling `on_imported` after each entry is written. +/// Every entry is validated against the stored chain before the first write. +pub fn import_tipset_lookup_snapshot( + db: &DB, + path: &Path, + mut on_imported: impl FnMut(), +) -> anyhow::Result<()> +where + DB: Blockstore + EthMappingsStore, +{ + let started = Instant::now(); + let hamt_car = AnyCar::try_from(path)?; + anyhow::ensure!( + hamt_car.header_v1().roots.len() == 1, + "hamt car should have a single root" + ); + let hamt_root = *hamt_car.header_v1().roots.first(); + let hamt: Hamt<_, TipsetKey, ChainEpoch> = + Hamt::load_with_bit_width(&hamt_root, hamt_car, TIPSET_LOOKUP_HAMT_BIT_WIDTH) + .context("failed to load tipset lookup HAMT")?; + tracing::info!("Validating tipset lookup entries against the chain, this can take a while"); + hamt.for_each_cacheless(|&epoch, tsk| { + validate_tipset_lookup_entry(db, epoch, tsk)?; + Ok(()) + })?; + hamt.for_each_cacheless(|&epoch, tsk| { + db.set_tipset_key_at_epoch_raw(epoch, tsk)?; + on_imported(); + Ok(()) + })?; + tracing::info!( + "Imported tipset lookup entries in {:.1}s", + started.elapsed().as_secs_f64() + ); + Ok(()) +} diff --git a/src/chain/tests.rs b/src/chain/tests.rs index b552e173e3c..c34045896f3 100644 --- a/src/chain/tests.rs +++ b/src/chain/tests.rs @@ -604,3 +604,73 @@ async fn ensure_recent_roots_present_checks_oldest_walked_tipset( expected ); } + +// Imports a lookup table with the given `hamt_epoch -> tipset at tipset_epoch` entries into a +// chain of tipsets at epochs 20, 21 and 40; `None` points at a tipset missing from the chain. +// On an error nothing may have been written, so one invalid entry rejects the valid ones too. +#[rstest] +#[case::checkpoint(&[(40, Some(40))], None)] +#[case::non_checkpoint_epoch( + &[(20, Some(20)), (21, Some(21)), (40, Some(40))], + Some("is not a tipset lookup checkpoint") +)] +#[case::epoch_mismatch(&[(40, Some(20))], Some("epochs do not match"))] +#[case::unknown_tipset(&[(40, None)], Some("Required tipset missing"))] +#[tokio::test] +async fn import_tipset_lookup( + #[case] entries: &[(ChainEpoch, Option)], + #[case] expected_err: Option<&str>, +) { + use crate::chain::index::tests::{genesis_tipset, persist_tipset, tipset_child}; + + let db = MemoryDB::default(); + let mut parent = genesis_tipset(); + persist_tipset(&parent, &db); + let mut chain = vec![]; + for epoch in [20, 21, 40] { + let ts = tipset_child(&parent, epoch); + persist_tipset(&ts, &db); + chain.push(ts.shallow_clone()); + parent = ts; + } + + let hamt_store = MemoryDB::default(); + let mut hamt: Hamt<_, TipsetKey, ChainEpoch> = + Hamt::new_with_bit_width(&hamt_store, TIPSET_LOOKUP_HAMT_BIT_WIDTH); + let mut expected = vec![]; + for &(hamt_epoch, tipset_epoch) in entries { + let tsk = match tipset_epoch { + Some(epoch) => chain + .iter() + .find(|ts| ts.epoch() == epoch) + .unwrap() + .key() + .clone(), + None => tipset_child(&parent, 40).key().clone(), + }; + hamt.set(hamt_epoch, tsk.clone()).unwrap(); + expected.push((hamt_epoch, tsk)); + } + let root = hamt.flush().unwrap(); + let mut car = vec![]; + hamt_store + .export_forest_car_with_roots(nunny::vec![root], &mut car) + .await + .unwrap(); + let temp_dir = tempfile::tempdir().unwrap(); + let car_path = temp_dir.path().join("tipset_lookup.forest.car.zst"); + std::fs::write(&car_path, car).unwrap(); + + let result = import_tipset_lookup_snapshot(&db, &car_path, || {}); + match expected_err { + None => result.unwrap(), + Some(expected) => { + let err = format!("{:#}", result.unwrap_err()); + assert!(err.contains(expected), "{err}"); + } + } + for (hamt_epoch, tsk) in expected { + let written = expected_err.is_none().then_some(tsk); + assert_eq!(db.tipset_key_by_epoch(hamt_epoch).unwrap(), written); + } +} diff --git a/src/cli_shared/snapshot.rs b/src/cli_shared/snapshot.rs index 3c6f1d246b9..a321764bdef 100644 --- a/src/cli_shared/snapshot.rs +++ b/src/cli_shared/snapshot.rs @@ -461,6 +461,7 @@ mod tests { #[case::follows_redirect("/latest", "/snapshot.forest.car.zst")] #[case::head_rejected("/post-only", "/post-only")] #[case::not_found("/missing", "/missing")] + #[case::redirect_to_rejected("/stale", "/stale")] #[tokio::test] async fn peek_url_falls_back_to_given_url(#[case] path: &str, #[case] expected: &str) { use axum::{ @@ -475,7 +476,11 @@ mod tests { get(|| async { Redirect::temporary("/snapshot.forest.car.zst") }), ) .route("/snapshot.forest.car.zst", get(|| async { "snapshot" })) - .route("/post-only", post(|| async { "post" })); + .route("/post-only", post(|| async { "post" })) + .route( + "/stale", + get(|| async { Redirect::temporary("/post-only") }), + ); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); tokio::spawn(async move { axum::serve(listener, app).await }); diff --git a/src/daemon/db_util.rs b/src/daemon/db_util.rs index afb76b6ee63..5e2f818798e 100644 --- a/src/daemon/db_util.rs +++ b/src/daemon/db_util.rs @@ -34,11 +34,12 @@ use tracing::{debug, info, warn}; use url::Url; use walkdir::WalkDir; -#[cfg(doc)] -use crate::blocks::TipsetKey; #[cfg(doc)] use crate::rpc::eth::types::EthHash; +#[cfg(doc)] +use crate::blocks::TipsetKey; + #[cfg(doc)] use cid::Cid; @@ -863,6 +864,9 @@ pub async fn run_backfill( Ok(report) } +#[cfg(test)] +pub(crate) use test::test_state_manager; + #[cfg(test)] mod test { use super::*; @@ -938,7 +942,7 @@ mod test { drop(bf); } - fn test_state_manager() -> StateManager { + pub(crate) fn test_state_manager() -> StateManager { use crate::blocks::{CachingBlockHeader, RawBlockHeader}; use crate::chain::ChainStore; use crate::networks::ChainConfig; diff --git a/src/daemon/extended_snapshot.rs b/src/daemon/extended_snapshot.rs index 29eefd118d4..c8d61c02aa2 100644 --- a/src/daemon/extended_snapshot.rs +++ b/src/daemon/extended_snapshot.rs @@ -4,25 +4,21 @@ //! Download and import of the extended snapshots (receipts/events and tipset lookup) //! published next to a snapshot in the Forest archive. -use crate::blocks::{Tipset, TipsetKey}; -use crate::chain::{TIPSET_LOOKUP_HAMT_BIT_WIDTH, index::ChainIndex}; +use crate::blocks::Tipset; +use crate::chain::{ChainEpochDelta, import_tipset_lookup_snapshot}; use crate::daemon::db_util::{ - BackfillGuard, BackfillOptions, RangeSpec, path_as_url, run_backfill, + BackfillGuard, BackfillOptions, BackfillReport, RangeSpec, run_backfill, }; -use crate::db::car::AnyCar; use crate::db::car::forest::new_forest_car_temp_path_in; -use crate::db::{BlockstoreWithWriteBuffer, DB_WRITE_BUFFER_CAPACITY, EthMappingsStore}; use crate::prelude::*; -use crate::shim::clock::ChainEpoch; use crate::state_manager::StateManager; -use crate::utils::db::car_stream::CarStream; +use crate::utils::db::car_util::import_car_blocks; use crate::utils::io::progress_log::Progress; -use crate::utils::net::{DownloadFileOption, download_to}; +use crate::utils::net::{DownloadFileOption, download_to_with_timeout}; use anyhow::Context; -use fil_actors_shared::fvm_ipld_hamt::Hamt; -use futures::TryStreamExt; use regex::Regex; use serde::Deserialize; +use sha2::Sha256; use std::path::Path; use std::time::{Duration, Instant}; use tracing::{info, warn}; @@ -40,7 +36,7 @@ struct SnapshotMetadataFile { struct SnapshotMetadataSection { /// Number of epochs below the head with state, which the receipts and events also cover #[serde(rename = "State-roots")] - state_roots: ChainEpoch, + state_roots: ChainEpochDelta, #[serde(rename = "Extended")] extended: Option, } @@ -65,10 +61,11 @@ struct ExtendedSnapshotFile { sha256: [u8; 32], } -/// Fetches the `.metadata.json` published next to `snapshot_url`. +/// Fetches the `.metadata.json` published next to `snapshot_url`, or `None` when the host has +/// none (a 404), which is normal for a snapshot without extended snapshots. async fn fetch_metadata_snapshots_info( snapshot_url: &Url, -) -> anyhow::Result { +) -> anyhow::Result> { // Append to the path, not the whole URL, so dont affect any query string. let mut metadata_url = snapshot_url.clone(); metadata_url.set_path(&format!("{}.metadata.json", snapshot_url.path())); @@ -77,12 +74,15 @@ async fn fetch_metadata_snapshots_info( .timeout(Duration::from_secs(30)) .send() .await?; + if response.status() != reqwest::StatusCode::OK { + return Ok(None); + } let metadata: SnapshotMetadataFile = response .error_for_status()? .json() .await .with_context(|| format!("failed to parse snapshot metadata at {metadata_url}"))?; - Ok(metadata.snapshot) + Ok(Some(metadata.snapshot)) } /// Downloads the extended snapshot `file`, published next to `snapshot_url`, into `temp_dir` @@ -92,8 +92,6 @@ async fn download_extended_snapshot( file: &ExtendedSnapshotFile, temp_dir: &Path, ) -> anyhow::Result { - use sha2::{Digest as _, Sha256}; - let filename_re = Regex::new(r"^[A-Za-z0-9_.-]+\.forest\.car\.zst$")?; anyhow::ensure!( filename_re.is_match(&file.filename), @@ -105,9 +103,24 @@ async fn download_extended_snapshot( let path = new_forest_car_temp_path_in(temp_dir)?; info!("Downloading extended snapshot: {url}"); - download_to(&url, &path, DownloadFileOption::Resumable, None).await?; + download_to_with_timeout( + &url, + &path, + DownloadFileOption::Resumable, + Duration::from_mins(10), + None, + ) + .await + .with_context(|| format!("failed to download {}", file.filename))?; - let actual: [u8; 32] = Sha256::digest(tokio::fs::read(&path).await?).into(); + let path_for_hash = path.to_path_buf(); + let actual: [u8; 32] = tokio::task::spawn_blocking(move || { + crate::utils::hash::digest_file::(&path_for_hash) + }) + .await + .with_context(|| format!("failed to hash {}", file.filename))? + .with_context(|| format!("failed to read {} for hashing", file.filename))? + .into(); anyhow::ensure!( actual == file.sha256, "checksum mismatch for {}", @@ -122,13 +135,7 @@ where DB: Blockstore, { let started = Instant::now(); - let db = BlockstoreWithWriteBuffer::new_with_capacity(db, DB_WRITE_BUFFER_CAPACITY); - let mut car_stream = CarStream::new_from_path(path).await?; - while let Some(block) = car_stream.try_next().await? { - block.validate()?; - db.put_keyed(&block.cid, &block.data)?; - } - drop(db); + import_car_blocks(db, path, true, || {}).await?; info!( "Imported message receipts and events in {:.1}s", started.elapsed().as_secs_f64() @@ -136,59 +143,13 @@ where Ok(()) } -/// Populates the tipset lookup table in `db` from a tipset lookup snapshot, calling -/// `on_imported` after each entry is written. -pub fn import_tipset_lookup_snapshot( - db: &DB, - path: &Path, - mut on_imported: impl FnMut(), -) -> anyhow::Result<()> -where - DB: Blockstore + EthMappingsStore, -{ - let started = Instant::now(); - let hamt_car = AnyCar::try_from(path)?; - anyhow::ensure!( - hamt_car.header_v1().roots.len() == 1, - "hamt car should have a single root" - ); - let hamt_root = *hamt_car.header_v1().roots.first(); - let hamt: Hamt<_, TipsetKey, ChainEpoch> = - Hamt::load_with_bit_width(&hamt_root, hamt_car, TIPSET_LOOKUP_HAMT_BIT_WIDTH) - .context("failed to load tipset lookup HAMT")?; - info!("Validating tipset lookup entries against the chain, this can take a while"); - hamt.for_each_cacheless(|&epoch, tsk| { - anyhow::ensure!( - ChainIndex::is_tipset_lookup_checkpoint(epoch), - "epoch {epoch} is not a tipset lookup checkpoint" - ); - let ts = Tipset::load_required(db, tsk)?; - anyhow::ensure!( - ts.epoch() == epoch, - "epochs do not match, {epoch} in hamt, {} in database", - ts.epoch() - ); - Ok(()) - })?; - hamt.for_each_cacheless(|&epoch, tsk| { - db.set_tipset_key_at_epoch_raw(epoch, tsk)?; - on_imported(); - Ok(()) - })?; - info!( - "Imported tipset lookup entries in {:.1}s", - started.elapsed().as_secs_f64() - ); - Ok(()) -} - /// Indexes the tipsets whose receipts and events were just imported (block logs blooms and -/// Ethereum mappings), as `forest-tool index backfill` does +/// Ethereum mappings), as `forest-tool index backfill` does. async fn backfill_receipts_events( state_manager: &StateManager, head: &Tipset, - depth: ChainEpoch, -) -> anyhow::Result<()> { + depth: ChainEpochDelta, +) -> anyhow::Result { let from = Tipset::load_required(state_manager.db(), head.parents())?; // the lowest tipset that can be indexed is at `head - depth + 1` let to = head.epoch() - depth.clamp(0, head.epoch()) + 1; @@ -200,6 +161,12 @@ async fn backfill_receipts_events( }; let result = run_backfill(state_manager, &from, RangeSpec::To(to), options, &guard).await; let report = guard.finish(result)?; + + // index ethereum events from the head + let chain_store = state_manager.chain_store(); + let head_delegated_messages = + chain_store.headers_delegated_messages(head.block_headers().iter())?; + chain_store.process_signed_messages(&head_delegated_messages, true)?; if report.skipped > 0 { warn!( "Indexed {} tipsets from epoch {} to {to}, {} could not be indexed", @@ -215,26 +182,29 @@ async fn backfill_receipts_events( started.elapsed().as_secs_f64() ); } - Ok(()) + Ok(report) } /// Downloads the extended snapshots (message receipts and events, tipset lookup table) -/// advertised in the metadata of the snapshot at `snapshot_path` and imports them into the -/// database of `state_manager`. Does nothing when `snapshot_path` is a local file rather than a URL. +/// advertised in the metadata of the snapshot at `snapshot_url` and imports them into the +/// database of `state_manager`. pub(super) async fn maybe_import_extended_snapshots( state_manager: &StateManager, head: &Tipset, - snapshot_path: &Path, + snapshot_url: &Url, temp_dir: &Path, ) { - // Extended snapshots are only published next to remote snapshots. - let Some(snapshot_url) = path_as_url(snapshot_path) else { - return; - }; - let metadata = match fetch_metadata_snapshots_info(&snapshot_url).await { - Ok(metadata) => metadata, + let metadata = match fetch_metadata_snapshots_info(snapshot_url).await { + Ok(Some(metadata)) => metadata, + Ok(None) => { + warn!("No snapshot metadata at {snapshot_url}, skipping extended snapshots"); + return; + } Err(e) => { - warn!("Failed to fetch extended snapshots for {snapshot_url}: {e:#}"); + warn!( + "Failed to fetch extended snapshots for {snapshot_url}: {e:#}; continuing without \ + receipts and events, import them manually, see the enriched snapshots docs" + ); return; } }; @@ -246,13 +216,14 @@ pub(super) async fn maybe_import_extended_snapshots( let receipts_events_imported = match &extended.receipts_events { Some(file) => { let result = async { - let path = download_extended_snapshot(&snapshot_url, file, temp_dir).await?; + let path = download_extended_snapshot(snapshot_url, file, temp_dir).await?; import_receipts_events_snapshot(state_manager.db(), &path).await } .await; if let Err(e) = &result { warn!( - "Failed to import receipts and events snapshot {}: {e:#}", + "Failed to import receipts and events snapshot {}: {e:#}; continuing without \ + them, import them manually, see the enriched snapshots docs", file.filename ); } @@ -263,7 +234,7 @@ pub(super) async fn maybe_import_extended_snapshots( if let Some(file) = &extended.tipset_lookup { let result = async { - let path = download_extended_snapshot(&snapshot_url, file, temp_dir).await?; + let path = download_extended_snapshot(snapshot_url, file, temp_dir).await?; let db = state_manager.db_owned(); tokio::task::spawn_blocking(move || { let mut progress = Progress::new("Importing tipset lookup entries"); @@ -274,7 +245,8 @@ pub(super) async fn maybe_import_extended_snapshots( .await; if let Err(e) = result { warn!( - "Failed to import tipset lookup snapshot {}: {e:#}", + "Failed to import tipset lookup snapshot {}: {e:#}; continuing without the tipset \ + lookup table, import it manually, see the enriched snapshots docs", file.filename ); } @@ -284,93 +256,229 @@ pub(super) async fn maybe_import_extended_snapshots( if receipts_events_imported && let Err(e) = backfill_receipts_events(state_manager, head, metadata.state_roots).await { - warn!("Failed to index imported receipts and events: {e:#}"); + warn!( + "Failed to index imported receipts and events: {e:#}; continuing without the bloom \ + index and Ethereum mappings, run forest-tool index backfill, see the enriched \ + snapshots docs" + ); } } #[cfg(test)] mod test { use super::*; + use crate::blocks::{CachingBlockHeader, RawBlockHeader}; use crate::chain::index::tests::{genesis_tipset, persist_tipset, tipset_child}; - use crate::db::MemoryDB; + use crate::chain_sync::TipsetValidator; + use crate::rpc::eth::eth_tx_from_signed_eth_message; + use crate::test_utils::{construct_eth_messages, dummy_ticket}; + use crate::utils::db::CborStoreExt as _; + use crate::utils::db::car_stream::CarBlock; use crate::utils::encoding::hex; + use crate::utils::multihash::prelude::*; + use fvm_ipld_encoding::DAG_CBOR; + use nunny::vec as nonempty; use rstest::rstest; - // Imports a lookup table with the single entry `hamt_epoch -> tipset at tipset_epoch` into a - // chain of tipsets at epochs 20, 21 and 40; `None` points at a tipset missing from the chain. - #[rstest] - #[case::checkpoint(40, Some(40), None)] - #[case::non_checkpoint_epoch(21, Some(21), Some("is not a tipset lookup checkpoint"))] - #[case::epoch_mismatch(40, Some(20), Some("epochs do not match"))] - #[case::unknown_tipset(40, None, Some("Required tipset missing"))] - #[tokio::test] - async fn import_tipset_lookup( - #[case] hamt_epoch: ChainEpoch, - #[case] tipset_epoch: Option, - #[case] expected_err: Option<&str>, + // The default fixture: a head with a single parent. + fn state_manager_with_delegated_head() -> ( + StateManager, + Tipset, + crate::message::SignedMessage, + crate::rpc::eth::types::EthHash, ) { - let db = MemoryDB::default(); - let mut parent = genesis_tipset(); - persist_tipset(&parent, &db); - let mut chain = vec![]; - for epoch in [20, 21, 40] { - let ts = tipset_child(&parent, epoch); - persist_tipset(&ts, &db); - chain.push(ts.shallow_clone()); - parent = ts; + state_manager_with_delegated_head_chain(1) + } + + // Builds a chain of `parents_below` tipsets ending in a head that carries a single delegated + // message, and returns the state manager, the head, the message and the message's Ethereum + // hash. + fn state_manager_with_delegated_head_chain( + parents_below: ChainEpoch, + ) -> ( + StateManager, + Tipset, + crate::message::SignedMessage, + crate::rpc::eth::types::EthHash, + ) { + let sm = crate::daemon::db_util::test_state_manager(); + let db = sm.db(); + + // A delegated message carried by the head block. + let (_, smsg) = construct_eth_messages(0); + db.put_cbor_default(&smsg).unwrap(); + + // The fixture uses mainnet, which forks to Hygge at 2_683_348; stay above it so + // `headers_delegated_messages` does not filter the head out. + let head_epoch = 2_700_000; + let mut parent = tipset_child(&genesis_tipset(), head_epoch - parents_below); + persist_tipset(&parent, db); + for epoch in (head_epoch - parents_below + 1)..head_epoch { + parent = tipset_child(&parent, epoch); + persist_tipset(&parent, db); } - let tsk = match tipset_epoch { - Some(epoch) => chain - .iter() - .find(|ts| ts.epoch() == epoch) - .unwrap() - .key() - .clone(), - None => tipset_child(&parent, 40).key().clone(), - }; + let head = Tipset::from(CachingBlockHeader::new(RawBlockHeader { + parents: parent.key().clone(), + epoch: head_epoch, + ticket: dummy_ticket(1), + messages: TipsetValidator::compute_msg_root( + db, + std::iter::empty(), + [smsg.signed_cid()], + ) + .unwrap(), + ..Default::default() + })); + persist_tipset(&head, db); + + let (_, tx) = + eth_tx_from_signed_eth_message(&smsg, sm.chain_store().chain_config().eth_chain_id) + .unwrap(); + let hash = tx.eth_hash().unwrap().into(); + (sm, head, smsg, hash) + } + + // The walk covers the head's parent down to `head - depth + 1`, both inclusive. The fixture + // tipsets carry no executed state, so each one is skipped and the counters give the range. + // The head is the one tipset the walk cannot reach, so its delegated message is indexed + // separately. + #[tokio::test] + #[serial_test::serial(chain_export)] + async fn backfill_receipts_events_indexes_range_and_head() { + let (sm, head, smsg, hash) = state_manager_with_delegated_head_chain(3); - let hamt_store = MemoryDB::default(); - let mut hamt: Hamt<_, TipsetKey, ChainEpoch> = - Hamt::new_with_bit_width(&hamt_store, TIPSET_LOOKUP_HAMT_BIT_WIDTH); - hamt.set(hamt_epoch, tsk.clone()).unwrap(); - let root = hamt.flush().unwrap(); - let mut car = vec![]; - hamt_store - .export_forest_car_with_roots(nunny::vec![root], &mut car) + let report = backfill_receipts_events(&sm, &head, 3).await.unwrap(); + + assert_eq!(report.skipped, 2); + assert_eq!(report.indexed, 0); + assert_eq!( + sm.chain_store().get_mapping(&hash).unwrap(), + Some(smsg.cid()) + ); + } + + // A receipts and events CAR with one valid block. The orchestration tests only need the import + // to succeed; the fixture's tipsets carry no executed state, so the walk indexes no tipset + // keys and the head mapping is the observable effect of the backfill. + async fn receipts_events_car() -> Vec { + let data = b"receipts and events".to_vec(); + let cid = Cid::new_v1(DAG_CBOR, MultihashCode::Blake2b256.digest(&data)); + let block = CarBlock { + cid, + data: data.into(), + }; + let frames = + crate::db::car::forest::Encoder::compress_stream_default(futures::stream::iter([ + Ok::<_, anyhow::Error>(block), + ])); + let mut writer = vec![]; + crate::db::car::forest::Encoder::write(&mut writer, nonempty![cid], frames) .await .unwrap(); + writer + } + + fn extended_metadata_json(state_roots: ChainEpoch, sha256_hex: &str) -> String { + format!( + r#"{{ + "Snapshot": {{ + "State-roots": {state_roots}, + "Extended": {{ + "Augmented": {{ + "Filename": + "receipts.forest.car.zst", + "Sha256": "{sha256_hex}" + }} + }} + }} + }}"# + ) + } + + // Serves `.metadata.json` and the receipts companion next to `snapshot_url`. Both + // handlers require the `token=x` query, so a dropped query string fails the import. + async fn serve_extended_snapshot(metadata: String, car: Vec) -> Url { + use axum::response::IntoResponse as _; + + let app = axum::Router::new() + .route( + "/base.forest.car.zst.metadata.json", + axum::routing::get(move |uri: axum::http::Uri| { + let metadata = metadata.clone(); + async move { + if uri.query() != Some("token=x") { + return (axum::http::StatusCode::FORBIDDEN, String::new()) + .into_response(); + } + (axum::http::StatusCode::OK, metadata).into_response() + } + }), + ) + .route( + "/receipts.forest.car.zst", + axum::routing::get(move |uri: axum::http::Uri| { + let car = car.clone(); + async move { + if uri.query() != Some("token=x") { + return (axum::http::StatusCode::FORBIDDEN, Vec::new()).into_response(); + } + (axum::http::StatusCode::OK, car).into_response() + } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { axum::serve(listener, app).await }); + Url::parse(&format!("http://{addr}/base.forest.car.zst?token=x")).unwrap() + } + + // The whole orchestration: metadata lookup and companion download keep the snapshot's query + // string, the receipts import gates the backfill, and the head tipset's delegated message + // ends up indexed. + #[tokio::test] + #[serial_test::serial(chain_export)] + async fn maybe_import_extended_snapshots_indexes_head() { + use sha2::{Digest as _, Sha256}; + + let (sm, head, smsg, hash) = state_manager_with_delegated_head(); + let car = receipts_events_car().await; + let metadata = extended_metadata_json(1, &hex::encode(Sha256::digest(&car))); + let snapshot_url = serve_extended_snapshot(metadata, car).await; + let temp_dir = tempfile::tempdir().unwrap(); - let car_path = temp_dir.path().join("tipset_lookup.forest.car.zst"); - std::fs::write(&car_path, car).unwrap(); - - let result = import_tipset_lookup_snapshot(&db, &car_path, || {}); - match expected_err { - None => { - result.unwrap(); - assert_eq!(db.tipset_key_by_epoch(hamt_epoch).unwrap(), Some(tsk)); - } - Some(expected) => { - let err = format!("{:#}", result.unwrap_err()); - assert!(err.contains(expected), "{err}"); - // Validation happens before any write. - assert_eq!(db.tipset_key_by_epoch(hamt_epoch).unwrap(), None); - } - } + maybe_import_extended_snapshots(&sm, &head, &snapshot_url, temp_dir.path()).await; + + assert_eq!( + sm.chain_store().get_mapping(&hash).unwrap(), + Some(smsg.cid()) + ); + } + + // A failed receipts import must skip the backfill, so the head's mapping is not written. + #[tokio::test] + #[serial_test::serial(chain_export)] + async fn maybe_import_extended_snapshots_skips_backfill_without_receipts() { + let (sm, head, _, hash) = state_manager_with_delegated_head(); + let car = receipts_events_car().await; + // A wrong checksum makes the companion import fail. + let metadata = extended_metadata_json(1, &hex::encode([0u8; 32])); + let snapshot_url = serve_extended_snapshot(metadata, car).await; + + let temp_dir = tempfile::tempdir().unwrap(); + maybe_import_extended_snapshots(&sm, &head, &snapshot_url, temp_dir.path()).await; + + assert_eq!(sm.chain_store().get_mapping(&hash).unwrap(), None); } #[rstest] - #[case::ok("companion.forest.car.zst", true, None)] - #[case::checksum_mismatch("companion.forest.car.zst", false, Some("checksum mismatch"))] - #[case::path_traversal( - "../companion.forest.car.zst", - true, - Some("invalid extended snapshot filename") - )] + #[case::ok("companion.forest.car.zst", true, true)] + #[case::checksum_mismatch("companion.forest.car.zst", false, false)] + #[case::path_traversal("../companion.forest.car.zst", true, false)] #[tokio::test] async fn download_extended_snapshot_verifies( #[case] filename: &str, #[case] matching_checksum: bool, - #[case] expected_err: Option<&str>, + #[case] succeeds: bool, ) { use sha2::{Digest as _, Sha256}; let dummy_content: &'static [u8] = b"companion snapshot bytes"; @@ -394,15 +502,38 @@ mod test { }; let temp_dir = tempfile::tempdir().unwrap(); let result = download_extended_snapshot(&snapshot_url, &file, temp_dir.path()).await; - match expected_err { - None => assert_eq!(std::fs::read(result.unwrap()).unwrap(), dummy_content), - Some(expected) => { - let err = format!("{:#}", result.unwrap_err()); - assert!(err.contains(expected), "{err}"); - } + assert_eq!(result.is_ok(), succeeds, "{result:?}"); + if let Ok(path) = result { + assert_eq!(std::fs::read(path).unwrap(), dummy_content); } } + // A stalled companion download must surface as a timeout instead of blocking startup. + #[tokio::test(start_paused = true)] + async fn download_extended_snapshot_times_out() { + let app = axum::Router::new().route( + "/companion.forest.car.zst", + axum::routing::get(|| async { std::future::pending::<()>().await }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { axum::serve(listener, app).await }); + + let snapshot_url = Url::parse(&format!("http://{addr}/base.forest.car.zst")).unwrap(); + let file = ExtendedSnapshotFile { + filename: "companion.forest.car.zst".into(), + sha256: [0; 32], + }; + let temp_dir = tempfile::tempdir().unwrap(); + let err = download_extended_snapshot(&snapshot_url, &file, temp_dir.path()) + .await + .unwrap_err(); + assert_eq!( + err.downcast_ref(), + Some(&crate::utils::RetryError::TimeoutExceeded) + ); + } + #[test] fn parse_snapshot_metadata_with_extended() { let json = r#"{ @@ -443,4 +574,37 @@ mod test { let metadata: SnapshotMetadataFile = serde_json::from_str(json).unwrap(); assert!(metadata.snapshot.extended.is_none()); } + + // A host without a `.metadata.json` (a 404) is normal, not a warning. Any other metadata + // response is still parsed, and a real failure stays an error. + #[tokio::test] + async fn fetch_metadata_snapshots_info_missing_is_none() { + let app = axum::Router::new() + .route( + "/base.forest.car.zst.metadata.json", + axum::routing::get(|| async { r#"{ "Snapshot": { "State-roots": 2000 } }"# }), + ) + .route( + "/missing.forest.car.zst.metadata.json", + axum::routing::get(|| async { axum::http::StatusCode::NOT_FOUND }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { axum::serve(listener, app).await }); + + let present = Url::parse(&format!("http://{addr}/base.forest.car.zst")).unwrap(); + let metadata = fetch_metadata_snapshots_info(&present) + .await + .unwrap() + .unwrap(); + assert_eq!(metadata.state_roots, 2000); + + let missing = Url::parse(&format!("http://{addr}/missing.forest.car.zst")).unwrap(); + assert!( + fetch_metadata_snapshots_info(&missing) + .await + .unwrap() + .is_none() + ); + } } diff --git a/src/daemon/mod.rs b/src/daemon/mod.rs index 12e2e746b95..6d39e9355f7 100644 --- a/src/daemon/mod.rs +++ b/src/daemon/mod.rs @@ -4,7 +4,7 @@ pub mod bundle; mod context; pub mod db_util; -pub mod extended_snapshot; +mod extended_snapshot; pub mod main; use crate::blocks::TipsetKey; @@ -148,8 +148,14 @@ async fn maybe_import_snapshot( if !opts.skip_load.unwrap_or_default() && let Some(path) = &config.client.snapshot_path { - let path = &match path_as_url(path) { - Some(url) => PathBuf::from(snapshot::peek_url(url).await.as_str()), + // Resolve a moving alias once so the base and its extended snapshots come from the same + // snapshot. + let url = match path_as_url(path) { + Some(url) => Some(snapshot::peek_url(url).await), + None => None, + }; + let path = &match &url { + Some(url) => PathBuf::from(url.as_str()), None => path.clone(), }; let (car_db_path, ts) = import_chain_as_forest_car( @@ -174,13 +180,15 @@ async fn maybe_import_snapshot( "Loaded car DB at {} and set current head to epoch {ts_epoch}", car_db_path.display(), ); - maybe_import_extended_snapshots( - &ctx.state_manager, - &ts, - path, - &ctx.db_meta_data.get_forest_car_db_dir(), - ) - .await; + if let Some(url) = &url { + maybe_import_extended_snapshots( + &ctx.state_manager, + &ts, + url, + &ctx.db_meta_data.get_forest_car_db_dir(), + ) + .await; + } } // If the snapshot progress state is not completed, diff --git a/src/db/blockstore_with_write_buffer.rs b/src/db/blockstore_with_write_buffer.rs index 1a60ab1d3d3..7c3585e7a94 100644 --- a/src/db/blockstore_with_write_buffer.rs +++ b/src/db/blockstore_with_write_buffer.rs @@ -5,9 +5,6 @@ use crate::prelude::*; use ahash::HashMap; use parking_lot::RwLock; -/// Number of blocks buffered between writes when bulk-importing CAR files. -pub const DB_WRITE_BUFFER_CAPACITY: usize = 10_000; - pub struct BlockstoreWithWriteBuffer { inner: DB, buffer: RwLock>>, @@ -44,7 +41,9 @@ impl BlockstoreWithWriteBuffer { } } - fn flush_buffer(&self) -> anyhow::Result<()> { + /// Writes the buffered blocks to the inner store. Dropping the buffer also flushes, but only + /// logs a failure, so callers that must know the blocks were written call this first. + pub fn flush(&self) -> anyhow::Result<()> { let records = { let mut buffer = self.buffer.write(); buffer.drain().collect_vec() @@ -54,7 +53,7 @@ impl BlockstoreWithWriteBuffer { fn flush_buffer_if_needed(&self) -> anyhow::Result<()> { if self.buffer.read().len() >= self.buffer_capacity { - self.flush_buffer() + self.flush() } else { Ok(()) } @@ -63,7 +62,7 @@ impl BlockstoreWithWriteBuffer { impl Drop for BlockstoreWithWriteBuffer { fn drop(&mut self) { - if let Err(e) = self.flush_buffer() { + if let Err(e) = self.flush() { tracing::warn!("{e:#}"); } } diff --git a/src/db/mod.rs b/src/db/mod.rs index 671eeb66708..30f489196d3 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -15,7 +15,7 @@ pub mod parity_db_config; pub mod ttl; pub use blockstore_with_read_cache::*; -pub use blockstore_with_write_buffer::{BlockstoreWithWriteBuffer, DB_WRITE_BUFFER_CAPACITY}; +pub use blockstore_with_write_buffer::BlockstoreWithWriteBuffer; pub use db_impl::DbImpl; pub use either::Either; pub use fvm_ipld_blockstore::Blockstore; diff --git a/src/tool/subcommands/db_cmd.rs b/src/tool/subcommands/db_cmd.rs index fb7852744ba..a208ff37db5 100644 --- a/src/tool/subcommands/db_cmd.rs +++ b/src/tool/subcommands/db_cmd.rs @@ -4,19 +4,18 @@ use std::path::PathBuf; use std::time::Instant; +use crate::chain::import_tipset_lookup_snapshot; use crate::cli::subcommands::prompt_confirm; use crate::cli_shared::{chain_path, delete_chain_data, read_config}; use crate::daemon::db_util::load_all_forest_cars; -use crate::daemon::extended_snapshot::import_tipset_lookup_snapshot; +use crate::db::CAR_DB_DIR_NAME; use crate::db::car::ManyCar; use crate::db::db_engine::{db_root, open_db}; -use crate::db::{BlockstoreWithWriteBuffer, CAR_DB_DIR_NAME, DB_WRITE_BUFFER_CAPACITY}; use crate::networks::NetworkChain; use crate::prelude::*; -use crate::utils::db::car_stream::CarStream; +use crate::utils::db::car_util::import_car_blocks; use clap::Subcommand; use indicatif::{ProgressBar, ProgressStyle}; -use tokio_stream::StreamExt; #[derive(Debug, Subcommand)] pub enum DBCommands { @@ -128,10 +127,7 @@ impl DBCommands { db_root(&chain_path(&config))? }; println!("Opening parity-db at {}", db_root_path.display()); - let db_writer = BlockstoreWithWriteBuffer::new_with_capacity( - open_db(db_root_path, &Default::default())?, - DB_WRITE_BUFFER_CAPACITY, - ); + let db = open_db(db_root_path, &Default::default())?; let pb = ProgressBar::new_spinner().with_style( ProgressStyle::with_template("{spinner} {msg}") @@ -141,18 +137,12 @@ impl DBCommands { let mut total = 0; for snap in snapshot_files { - let mut car = CarStream::new_from_path(&snap).await?; - while let Some(b) = car.try_next().await? { - if !no_validation { - b.validate()?; - } - db_writer.put_keyed(&b.cid, &b.data)?; + import_car_blocks(&db, &snap, !no_validation, || { total += 1; - let text = format!("{total} blocks imported"); - pb.set_message(text); - } + pb.set_message(format!("{total} blocks imported")); + }) + .await?; } - drop(db_writer); pb.finish(); Ok(()) } diff --git a/src/tool/subcommands/snapshot_cmd.rs b/src/tool/subcommands/snapshot_cmd.rs index 1e66fc51a4f..871efc806e1 100644 --- a/src/tool/subcommands/snapshot_cmd.rs +++ b/src/tool/subcommands/snapshot_cmd.rs @@ -9,6 +9,7 @@ use crate::blocks::{Tipset, TipsetKey}; use crate::chain::{ TIPSET_LOOKUP_HAMT_BIT_WIDTH, index::{ChainIndex, ResolveNullTipset}, + validate_tipset_lookup_entry, }; use crate::cid_collections::CidHashSet; use crate::cli_shared::snapshot; @@ -386,14 +387,9 @@ fn validate_tipset_lookup_hamt( { if let Some(tsk) = hamt.get(&checkpoint)? { n_checkpoints += 1; - let ts = Tipset::load_required(&store, tsk).with_context(|| { + let ts = validate_tipset_lookup_entry(&store, checkpoint, tsk).with_context(|| { format!("failed to lookup tipset at checkpoint, epoch: {checkpoint}, key: {tsk}") })?; - anyhow::ensure!( - ts.epoch() == checkpoint, - "tipset mismatch, checkpoint epoch: {checkpoint}, tipset epoch: {}, tipset key: {tsk}", - ts.epoch() - ); null_checkpoint_search_from_ts = ts; } else { n_null_checkpoints += 1; diff --git a/src/tool/subcommands/snapshot_cmd/tests.rs b/src/tool/subcommands/snapshot_cmd/tests.rs index 016af522afe..7d0720d6d97 100644 --- a/src/tool/subcommands/snapshot_cmd/tests.rs +++ b/src/tool/subcommands/snapshot_cmd/tests.rs @@ -18,12 +18,12 @@ enum HamtEntries { #[case::bad_checkpoint_entry( HamtEntries::Checkpoints, Some(40), - Some("tipset mismatch, checkpoint epoch: 40") + Some("epochs do not match, 40 in hamt") )] #[case::null_checkpoint_entry( HamtEntries::Checkpoints, Some(20), - Some("tipset mismatch, checkpoint epoch: 20") + Some("epochs do not match, 20 in hamt") )] fn test_validate_tipset_lookup_hamt( #[case] entries: HamtEntries, diff --git a/src/utils/db/car_util.rs b/src/utils/db/car_util.rs index 7b7002c9a93..9f53cef918f 100644 --- a/src/utils/db/car_util.rs +++ b/src/utils/db/car_util.rs @@ -1,11 +1,14 @@ // Copyright 2019-2026 ChainSafe Systems // SPDX-License-Identifier: Apache-2.0, MIT +use std::path::Path; + use crate::prelude::*; use futures::{Stream, StreamExt as _, TryStreamExt as _}; use tokio::io::{AsyncBufRead, AsyncSeek, BufReader}; use crate::cid_collections::CidHashSet; +use crate::db::BlockstoreWithWriteBuffer; use crate::utils::db::car_stream::{CarBlock, CarStream, CarV1Header}; /// Stream key-value pairs from a CAR archive into a block store. @@ -21,6 +24,26 @@ where Ok(stream.header_v1) } +/// Streams the blocks of the CAR archive at `path` into `db` through a write buffer, validating +/// each block when `validate` is set and calling `on_block` after each block is buffered. +pub async fn import_car_blocks( + db: &impl Blockstore, + path: &Path, + validate: bool, + mut on_block: impl FnMut(), +) -> anyhow::Result<()> { + let db = BlockstoreWithWriteBuffer::new_with_capacity(db, 10_000); + let mut car_stream = CarStream::new_from_path(path).await?; + while let Some(block) = car_stream.try_next().await? { + if validate { + block.validate()?; + } + db.put_keyed(&block.cid, &block.data)?; + on_block(); + } + db.flush() +} + pub fn merge_car_streams( car_streams: Vec>, ) -> impl Stream> @@ -196,4 +219,52 @@ mod tests { Ok::<_, anyhow::Error>(()) }) } + + #[tokio::test] + async fn import_car_blocks_writes_blocks_and_reports_each() { + use crate::db::MemoryDB; + + let blocks = Blocks(vec![ + CarBlock { + cid: Cid::new_v1(DAG_CBOR, MultihashCode::Blake2b256.digest(b"one")), + data: b"one".to_vec().into(), + }, + CarBlock { + cid: Cid::new_v1(DAG_CBOR, MultihashCode::Blake2b256.digest(b"two")), + data: b"two".to_vec().into(), + }, + ]); + let car = blocks.clone().into_forest_car_zst_bytes().await; + let temp_dir = tempfile::tempdir().unwrap(); + let path = temp_dir.path().join("blocks.forest.car.zst"); + std::fs::write(&path, car).unwrap(); + + let db = MemoryDB::default(); + let mut imported = 0; + import_car_blocks(&db, &path, true, || imported += 1) + .await + .unwrap(); + + assert_eq!(imported, 2); + for block in &blocks.0 { + assert_eq!(db.get(&block.cid).unwrap(), Some(block.data.to_vec())); + } + + // The trailing buffer is flushed before returning, so a failed write is an error. + import_car_blocks(&FailingBlockstore, &path, true, || {}) + .await + .unwrap_err(); + } + + struct FailingBlockstore; + + impl Blockstore for FailingBlockstore { + fn get(&self, _k: &Cid) -> anyhow::Result>> { + Ok(None) + } + + fn put_keyed(&self, _k: &Cid, _block: &[u8]) -> anyhow::Result<()> { + anyhow::bail!("write failed") + } + } } diff --git a/src/utils/net/download_file.rs b/src/utils/net/download_file.rs index 8893a1ae074..0a07d72fa56 100644 --- a/src/utils/net/download_file.rs +++ b/src/utils/net/download_file.rs @@ -567,26 +567,45 @@ pub async fn download_to( option: DownloadFileOption, callback: Option>, ) -> anyhow::Result<()> { - download_file_with_retry( - url, - destination.parent().with_context(|| { - format!( - "Error getting the parent directory of {}", - destination.display() - ) - })?, - destination - .file_name() - .and_then(OsStr::to_str) - .with_context(|| format!("Error getting the file name of {}", destination.display()))?, - option, - callback, + let (directory, filename) = destination_parts(destination)?; + download_file_with_retry(url, directory, filename, option, callback).await?; + Ok(()) +} + +/// Like [`download_to`], but with `timeout` argument, retries included. +pub async fn download_to_with_timeout( + url: &Url, + destination: &Path, + option: DownloadFileOption, + timeout: Duration, + callback: Option>, +) -> anyhow::Result<()> { + let (directory, filename) = destination_parts(destination)?; + retry( + RetryArgs { + timeout: Some(timeout), + ..Default::default() + }, + || download_http(url, directory, filename, option, callback.clone()), ) .await?; - Ok(()) } +fn destination_parts(destination: &Path) -> anyhow::Result<(&Path, &str)> { + let directory = destination.parent().with_context(|| { + format!( + "Error getting the parent directory of {}", + destination.display() + ) + })?; + let filename = destination + .file_name() + .and_then(OsStr::to_str) + .with_context(|| format!("Error getting the file name of {}", destination.display()))?; + Ok((directory, filename)) +} + #[cfg(test)] mod test { use super::*;