feat(daemon): import enriched snapshots

CHANGELOG.md

@@ -31,6 +31,8 @@
 
 ### Added
 
+- [#6802](https://github.com/ChainSafe/forest/issues/6802): When importing a snapshot from a URL, Forest now checks the snapshot's `.metadata.json` and, if extended snapshots are advertised, downloads and imports the receipts, events, and the tipset lookup alongside it
+
 - [#7802](https://github.com/ChainSafe/forest/pull/7802): Added `--target <EPOCH>` to `forest-cli sync wait`. The command exits once the node head reaches the given epoch.
 
 - [#7781](https://github.com/ChainSafe/forest/pull/7781): Actor bundles are now verified against their CIDs, and missing or tampered blocks are replaced.

docs/docs/users/knowledge_base/enriched_snapshots.md

@@ -29,9 +29,9 @@ forest_snapshot_mainnet_2026-08-14_height_6279840_tipset_lookup.forest.car.zst
 ```
 
 :::info
-Enriched snapshots are a recent addition and are not yet listed by the snapshot
-service. Discovery is still to be defined; for now the companion files follow the
-suffix convention above next to their base snapshot.
+The companion files are listed, with their SHA-256, under `Snapshot.Extended` in
+the base snapshot's `.metadata.json`, published next to it as
+`<snapshot-url>.metadata.json`.
 :::
 
 ## Generating
@@ -61,6 +61,17 @@ at least one must be given.
 
 ## Importing into Forest
 
+### Automatically on startup
+
+When Forest imports a snapshot from a URL on startup, with `--auto-download-snapshot` or
+`--import-snapshot <url>`, it imports the enriched snapshot too, then indexes the
+imported receipts and events (block `logsBloom` and Ethereum mappings), so
+`FOREST_ETH_RPC_COMPUTE_BLOOM_ON_MISS=1` is only needed for the manual recipe below.
+This step is best-effort: if a companion file can't be downloaded, verified, or
+imported, Forest logs a warning and starts without it.
+
+### Manually
+
 `--import-snapshot` accepts a single file and the companion CARs have no chain
 spine of their own, so only the base goes through it. The companions are loaded
 with `forest-tool db`, which writes directly into the node's key-value store and

scripts/tests/external-rpc-checks/init.sh

@@ -1,14 +1,11 @@
 #!/usr/bin/env bash
-# Runs in the `init` service: imports the snapshot ./resolve.rb picked and
-# back-fills the chain index, all before the daemon starts.
+# Runs in the `init` service: imports the snapshot ./resolve.rb picked, along
+# with its extended snapshots (receipts and events, tipset lookup), before the
+# daemon starts.
 
 set -euo pipefail
 
 url=$(< /data/snapshot-url)
-epoch=$(< /data/snapshot-epoch)
-read -r chain start _ < /data/check-target
+read -r chain _ < /data/check-target
 
 forest --chain="${chain}" --encrypt-keystore=false --import-snapshot="${url}" --halt-after-import
-
-# inclusive on both ends
-forest-tool index backfill --chain="${chain}" --from="${epoch}" --to="${start}"

scripts/tests/external-rpc-checks/resolve.rb

@@ -1,8 +1,9 @@
 # frozen_string_literal: true
 
 # Picks the newest ${FOREST_CHAIN} snapshot whose head lies within the first 2000 epochs of
-# the UTC day: it holds the state the backfill needs to reach the day's start,
-# and the node syncs the rest. Writes its URL, head epoch and the day's epoch range under /data.
+# the UTC day: importing it indexes its receipts and events back to the day's start,
+# and the node syncs the rest. Writes its URL and the day's epoch range under /data, and fails
+# if the snapshot advertises no receipts and events snapshot.
 
 require 'date'
 require 'json'
@@ -47,7 +48,9 @@ def fetch(uri, attempts: 3)
   exit 2
 end
 
+metadata = JSON.parse(fetch(URI("#{url}.metadata.json")))
+abort "#{url} advertises no receipts and events snapshot" unless metadata.dig('Snapshot', 'Extended', 'Augmented')
+
 File.write('/data/snapshot-url', "#{url}\n")
-File.write('/data/snapshot-epoch', "#{epoch}\n")
 File.write('/data/check-target', "#{chain} #{start} #{finish}\n")
 puts "#{chain} #{day.iso8601} is epochs #{start}..#{finish}; snapshot at #{epoch}: #{url}"

scripts/tests/external-rpc-checks/verify.rb

@@ -1,31 +1,35 @@
 # frozen_string_literal: true
 
-# Confirms the node indexed the epochs the checks query.
+# Confirms the node indexed the epochs the checks query: the tipset is served and the receipts
+# of its messages are in the store, read back through the standard chain API.
 
 require 'json'
 require 'net/http'
 
 url = URI("http://#{ENV.fetch('FOREST_RPC_URL')}")
 
-ARGV.each do |argument|
-  epoch = Integer(argument)
+def rpc(url, method, params)
   request = Net::HTTP::Post.new(url, 'Content-Type' => 'application/json')
-  request.body = {
-    jsonrpc: '2.0',
-    id: 1,
-    method: 'eth_getBlockByNumber',
-    params: ["0x#{epoch.to_s(16)}", false]
-  }.to_json
+  request.body = { jsonrpc: '2.0', id: 1, method: method, params: params }.to_json
+  JSON.parse(Net::HTTP.start(url.hostname, url.port) { |http| http.request(request) }.body)
+end
 
-  response = Net::HTTP.start(url.hostname, url.port) { |http| http.request(request) }
-  body = JSON.parse(response.body)
+ARGV.each do |argument|
+  epoch = Integer(argument)
+  block = rpc(url, 'eth_getBlockByNumber', ["0x#{epoch.to_s(16)}", false])
   # Error code 12 denotes a null round across implementations.
   # https://github.com/ChainSafe/forest/blob/8c980e679b6606c534c6eb5bd8aa03ff8cc6f5c0/src/rpc/methods/eth/errors.rs#L22
-  null_round = body.dig('error', 'code') == 12
-
-  unless body.dig('result', 'number') || null_round
-    abort "epoch #{epoch} is not indexed: #{response.code} #{response.body}"
+  if block.dig('error', 'code') == 12
+    puts "epoch #{epoch} was a null round"
+    next
   end
+  abort "epoch #{epoch} is not indexed: #{block}" unless block.dig('result', 'number')
 
-  puts "epoch #{epoch} #{null_round ? 'was a null round' : 'is indexed'}"
+  # The receipts of an epoch's messages are stored under its child tipset, the first non-null
+  # tipset above it. Reading them back is the same check any client would make.
+  child = rpc(url, 'Filecoin.ChainGetTipSetAfterHeight', [epoch + 1, []])
+  abort "epoch #{epoch} has no tipset above it: #{child['error']}" if child['error']
+  receipts = rpc(url, 'Filecoin.ChainGetParentReceipts', [child.dig('result', 'Cids', 0)])
+  abort "epoch #{epoch} has no receipts: #{receipts['error']}" if receipts['error']
+  puts "epoch #{epoch} is indexed"
 end

src/chain/mod.rs

@@ -13,8 +13,9 @@ pub use self::{snapshot_format::*, store::*, weight::*};
 use crate::blocks::{Tipset, TipsetKey};
 use crate::chain::index::{ChainIndex, ResolveNullTipset};
 use crate::cid_collections::{CidHashSet, CidHashSetLike, FileBackedCidHashSet};
-use crate::db::IndexMapBlockstore;
+use crate::db::car::AnyCar;
 use crate::db::car::forest::{self, ForestCarFrame, finalize_frame};
+use crate::db::{EthMappingsStore, IndexMapBlockstore};
 use crate::ipld::{IpldStream, should_save_block_to_snapshot, stream_chain};
 use crate::prelude::*;
 use crate::shim::executor::Receipt;
@@ -481,3 +482,62 @@ pub async fn export_receipts_events_to_forest_car(
 
     Ok(())
 }
+
+/// Check that the entry is at a checkpoint epoch and that the stored tipset is at that epoch.
+pub fn validate_tipset_lookup_entry<DB>(
+    db: &DB,
+    epoch: ChainEpoch,
+    tsk: &TipsetKey,
+) -> anyhow::Result<Tipset>
+where
+    DB: Blockstore,
+{
+    anyhow::ensure!(
+        ChainIndex::is_tipset_lookup_checkpoint(epoch),
+        "epoch {epoch} is not a tipset lookup checkpoint"
+    );
+    let ts: Tipset = Tipset::load_required(db, tsk)?;
+    anyhow::ensure!(
+        ts.epoch() == epoch,
+        "epochs do not match, {epoch} in hamt, {} in database",
+        ts.epoch()
+    );
+    Ok(ts)
+}
+
+/// Imports a tipset lookup snapshot into `db`, calling `on_imported` after each entry is written.
+/// Every entry is validated against the stored chain before the first write.
+pub fn import_tipset_lookup_snapshot<DB>(
+    db: &DB,
+    path: &Path,
+    mut on_imported: impl FnMut(),
+) -> anyhow::Result<()>
+where
+    DB: Blockstore + EthMappingsStore,
+{
+    let started = Instant::now();
+    let hamt_car = AnyCar::try_from(path)?;
+    anyhow::ensure!(
+        hamt_car.header_v1().roots.len() == 1,
+        "hamt car should have a single root"
+    );
+    let hamt_root = *hamt_car.header_v1().roots.first();
+    let hamt: Hamt<_, TipsetKey, ChainEpoch> =
+        Hamt::load_with_bit_width(&hamt_root, hamt_car, TIPSET_LOOKUP_HAMT_BIT_WIDTH)
+            .context("failed to load tipset lookup HAMT")?;
+    tracing::info!("Validating tipset lookup entries against the chain, this can take a while");
+    hamt.for_each_cacheless(|&epoch, tsk| {
+        validate_tipset_lookup_entry(db, epoch, tsk)?;
+        Ok(())
+    })?;
+    hamt.for_each_cacheless(|&epoch, tsk| {
+        db.set_tipset_key_at_epoch_raw(epoch, tsk)?;
+        on_imported();
+        Ok(())
+    })?;
+    tracing::info!(
+        "Imported tipset lookup entries in {:.1}s",
+        started.elapsed().as_secs_f64()
+    );
+    Ok(())
+}

src/chain/tests.rs

@@ -604,3 +604,73 @@ async fn ensure_recent_roots_present_checks_oldest_walked_tipset(
         expected
     );
 }
+
+// Imports a lookup table with the given `hamt_epoch -> tipset at tipset_epoch` entries into a
+// chain of tipsets at epochs 20, 21 and 40; `None` points at a tipset missing from the chain.
+// On an error nothing may have been written, so one invalid entry rejects the valid ones too.
+#[rstest]
+#[case::checkpoint(&[(40, Some(40))], None)]
+#[case::non_checkpoint_epoch(
+    &[(20, Some(20)), (21, Some(21)), (40, Some(40))],
+    Some("is not a tipset lookup checkpoint")
+)]
+#[case::epoch_mismatch(&[(40, Some(20))], Some("epochs do not match"))]
+#[case::unknown_tipset(&[(40, None)], Some("Required tipset missing"))]
+#[tokio::test]
+async fn import_tipset_lookup(
+    #[case] entries: &[(ChainEpoch, Option<ChainEpoch>)],
+    #[case] expected_err: Option<&str>,
+) {
+    use crate::chain::index::tests::{genesis_tipset, persist_tipset, tipset_child};
+
+    let db = MemoryDB::default();
+    let mut parent = genesis_tipset();
+    persist_tipset(&parent, &db);
+    let mut chain = vec![];
+    for epoch in [20, 21, 40] {
+        let ts = tipset_child(&parent, epoch);
+        persist_tipset(&ts, &db);
+        chain.push(ts.shallow_clone());
+        parent = ts;
+    }
+
+    let hamt_store = MemoryDB::default();
+    let mut hamt: Hamt<_, TipsetKey, ChainEpoch> =
+        Hamt::new_with_bit_width(&hamt_store, TIPSET_LOOKUP_HAMT_BIT_WIDTH);
+    let mut expected = vec![];
+    for &(hamt_epoch, tipset_epoch) in entries {
+        let tsk = match tipset_epoch {
+            Some(epoch) => chain
+                .iter()
+                .find(|ts| ts.epoch() == epoch)
+                .unwrap()
+                .key()
+                .clone(),
+            None => tipset_child(&parent, 40).key().clone(),
+        };
+        hamt.set(hamt_epoch, tsk.clone()).unwrap();
+        expected.push((hamt_epoch, tsk));
+    }
+    let root = hamt.flush().unwrap();
+    let mut car = vec![];
+    hamt_store
+        .export_forest_car_with_roots(nunny::vec![root], &mut car)
+        .await
+        .unwrap();
+    let temp_dir = tempfile::tempdir().unwrap();
+    let car_path = temp_dir.path().join("tipset_lookup.forest.car.zst");
+    std::fs::write(&car_path, car).unwrap();
+
+    let result = import_tipset_lookup_snapshot(&db, &car_path, || {});
+    match expected_err {
+        None => result.unwrap(),
+        Some(expected) => {
+            let err = format!("{:#}", result.unwrap_err());
+            assert!(err.contains(expected), "{err}");
+        }
+    }
+    for (hamt_epoch, tsk) in expected {
+        let written = expected_err.is_none().then_some(tsk);
+        assert_eq!(db.tipset_key_by_epoch(hamt_epoch).unwrap(), written);
+    }
+}

src/cli_shared/snapshot.rs

@@ -5,6 +5,7 @@ use std::{
     fmt::Display,
     path::{Path, PathBuf},
     str::FromStr,
+    time::Duration,
 };
 
 use crate::prelude::*;
@@ -112,6 +113,24 @@ pub async fn peek(
     ))
 }
 
+/// Returns the final URL after redirection(s).
+/// if server rejects `HEAD` requests returns the `url` itself
+pub async fn peek_url(url: Url) -> Url {
+    let response = crate::utils::net::global_http_client()
+        .head(url.clone())
+        .timeout(Duration::from_secs(30))
+        .send()
+        .await
+        .and_then(|response| response.error_for_status());
+    match response {
+        Ok(response) => response.url().clone(),
+        Err(e) => {
+            tracing::debug!("Couldn't follow the redirects of {url}: {e:#}");
+            url
+        }
+    }
+}
+
 // Extract file paths from content-disposition values:
 //   "attachment; filename=\"911520_2023_09_14T06_13_00Z.car.zst\""
 // => "911520_2023_09_14T06_13_00Z.car.zst"
@@ -434,8 +453,41 @@ mod parse {
 
 #[cfg(test)]
 mod tests {
-    use super::parse_content_disposition;
+    use super::{parse_content_disposition, peek_url};
     use reqwest::header::HeaderValue;
+    use url::Url;
+
+    #[rstest::rstest]
+    #[case::follows_redirect("/latest", "/snapshot.forest.car.zst")]
+    #[case::head_rejected("/post-only", "/post-only")]
+    #[case::not_found("/missing", "/missing")]
+    #[case::redirect_to_rejected("/stale", "/stale")]
+    #[tokio::test]
+    async fn peek_url_falls_back_to_given_url(#[case] path: &str, #[case] expected: &str) {
+        use axum::{
+            Router,
+            response::Redirect,
+            routing::{get, post},
+        };
+
+        let app = Router::new()
+            .route(
+                "/latest",
+                get(|| async { Redirect::temporary("/snapshot.forest.car.zst") }),
+            )
+            .route("/snapshot.forest.car.zst", get(|| async { "snapshot" }))
+            .route("/post-only", post(|| async { "post" }))
+            .route(
+                "/stale",
+                get(|| async { Redirect::temporary("/post-only") }),
+            );
+        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+        let addr = listener.local_addr().unwrap();
+        tokio::spawn(async move { axum::serve(listener, app).await });
+
+        let url = Url::parse(&format!("http://{addr}{path}")).unwrap();
+        assert_eq!(peek_url(url).await.path(), expected);
+    }
 
     #[test]
     fn content_disposition_forest() {