feat(tool): forest-tool api generate-test-snapshot (#5074)

Cargo.toml

@@ -241,6 +241,7 @@ glob = "0.3"
 http-range-header = "0.4"
 insta = { version = "1", features = ["yaml"] }
 libp2p-swarm-test = { workspace = true }
+md5 = { package = "md-5", version = "0.10" }
 num-bigint = { version = "0.4", features = ['quickcheck'] }
 petgraph = "0.7"
 predicates = "3"

src/chain/store/chain_store.rs

@@ -234,7 +234,7 @@ where
                 &self
                     .settings
                     .require_obj::<TipsetKey>(HEAD_KEY)
-                    .expect("failed to load heaviest tipset"),
+                    .expect("failed to load heaviest tipset key"),
             )
             .expect("failed to load heaviest tipset")
     }

src/chain/store/index.rs

@@ -7,6 +7,7 @@ use crate::beacon::{BeaconEntry, IGNORE_DRAND_VAR};
 use crate::blocks::{Tipset, TipsetKey};
 use crate::metrics;
 use crate::shim::clock::ChainEpoch;
+use crate::utils::misc::env::is_env_truthy;
 use fvm_ipld_blockstore::Blockstore;
 use itertools::Itertools;
 use lru::LruCache;
@@ -47,11 +48,13 @@ impl<DB: Blockstore> ChainIndex<DB> {
     /// Loads a tipset from memory given the tipset keys and cache. Semantically
     /// identical to [`Tipset::load`] but the result is cached.
     pub fn load_tipset(&self, tsk: &TipsetKey) -> Result<Option<Arc<Tipset>>, Error> {
-        if let Some(ts) = self.ts_cache.lock().get(tsk) {
-            metrics::LRU_CACHE_HIT
-                .get_or_create(&metrics::values::TIPSET)
-                .inc();
-            return Ok(Some(ts.clone()));
+        if !is_env_truthy("FOREST_TIPSET_CACHE_DISABLED") {
+            if let Some(ts) = self.ts_cache.lock().get(tsk) {
+                metrics::LRU_CACHE_HIT
+                    .get_or_create(&metrics::values::TIPSET)
+                    .inc();
+                return Ok(Some(ts.clone()));
+            }
         }
 
         let ts_opt = Tipset::load(&self.db, tsk)?.map(Arc::new);

src/cli_shared/snapshot.rs

@@ -134,6 +134,9 @@ fn parse_content_disposition(value: &reqwest::header::HeaderValue) -> Option<Str
 
 /// Download the file at `url` with a private HTTP client, returning the path to the downloaded file
 async fn download_http(url: &Url, directory: &Path, filename: &str) -> anyhow::Result<PathBuf> {
+    if !directory.is_dir() {
+        std::fs::create_dir_all(directory)?;
+    }
     let dst_path = directory.join(filename);
     let destination = dst_path.display();
     event!(target: "forest::snapshot", tracing::Level::INFO, %url, %destination, "downloading snapshot");

src/db/memory.rs

@@ -1,33 +1,63 @@
 // Copyright 2019-2025 ChainSafe Systems
 // SPDX-License-Identifier: Apache-2.0, MIT
 
+use super::{EthMappingsStore, SettingsStore, SettingsStoreExt};
+use crate::blocks::TipsetKey;
 use crate::cid_collections::CidHashSet;
 use crate::db::{GarbageCollectable, PersistentStore};
 use crate::libp2p_bitswap::{BitswapStoreRead, BitswapStoreReadWrite};
 use crate::rpc::eth::types::EthHash;
+use crate::utils::db::car_stream::CarBlock;
 use crate::utils::multihash::prelude::*;
 use ahash::HashMap;
+use anyhow::Context as _;
 use cid::Cid;
 use fvm_ipld_blockstore::Blockstore;
 use itertools::Itertools;
 use parking_lot::RwLock;
 
-use super::{EthMappingsStore, SettingsStore};
-
 #[derive(Debug, Default)]
 pub struct MemoryDB {
-    blockchain_db: RwLock<HashMap<Vec<u8>, Vec<u8>>>,
-    blockchain_persistent_db: RwLock<HashMap<Vec<u8>, Vec<u8>>>,
+    blockchain_db: RwLock<HashMap<Cid, Vec<u8>>>,
+    blockchain_persistent_db: RwLock<HashMap<Cid, Vec<u8>>>,
     settings_db: RwLock<HashMap<String, Vec<u8>>>,
     eth_mappings_db: RwLock<HashMap<EthHash, Vec<u8>>>,
 }
 
+impl MemoryDB {
+    pub async fn export_forest_car<W: tokio::io::AsyncWrite + Unpin>(
+        &self,
+        writer: &mut W,
+    ) -> anyhow::Result<()> {
+        let roots =
+            SettingsStoreExt::read_obj::<TipsetKey>(self, crate::db::setting_keys::HEAD_KEY)?
+                .context("chain head is not tracked and cannot be exported")?
+                .into_cids();
+        let blocks = {
+            let blockchain_db = self.blockchain_db.read();
+            let blockchain_persistent_db = self.blockchain_persistent_db.read();
+            blockchain_db
+                .iter()
+                .chain(blockchain_persistent_db.iter())
+                .map(|(&cid, data)| {
+                    anyhow::Ok(CarBlock {
+                        cid,
+                        data: data.clone(),
+                    })
+                })
+                .collect_vec()
+        };
+        let frames =
+            crate::db::car::forest::Encoder::compress_stream_default(futures::stream::iter(blocks));
+        crate::db::car::forest::Encoder::write(writer, roots, frames).await
+    }
+}
+
 impl GarbageCollectable<CidHashSet> for MemoryDB {
     fn get_keys(&self) -> anyhow::Result<CidHashSet> {
         let mut set = CidHashSet::new();
-        for key in self.blockchain_db.read().keys() {
-            let cid = Cid::try_from(key.as_slice())?;
-            set.insert(cid);
+        for &key in self.blockchain_db.read().keys() {
+            set.insert(key);
         }
         Ok(set)
     }
@@ -36,17 +66,11 @@ impl GarbageCollectable<CidHashSet> for MemoryDB {
         let mut db = self.blockchain_db.write();
         let mut deleted = 0;
         db.retain(|key, _| {
-            let cid = Cid::try_from(key.as_slice());
-            match cid {
-                Ok(cid) => {
-                    let retain = !keys.contains(&cid);
-                    if !retain {
-                        deleted += 1;
-                    }
-                    retain
-                }
-                _ => true,
+            let retain = !keys.contains(key);
+            if !retain {
+                deleted += 1;
             }
+            retain
         });
         Ok(deleted)
     }
@@ -111,22 +135,15 @@ impl EthMappingsStore for MemoryDB {
 
 impl Blockstore for MemoryDB {
     fn get(&self, k: &Cid) -> anyhow::Result<Option<Vec<u8>>> {
-        Ok(self
-            .blockchain_db
+        Ok(self.blockchain_db.read().get(k).cloned().or(self
+            .blockchain_persistent_db
             .read()
-            .get(&k.to_bytes())
-            .cloned()
-            .or(self
-                .blockchain_persistent_db
-                .read()
-                .get(&k.to_bytes())
-                .cloned()))
+            .get(k)
+            .cloned()))
     }
 
     fn put_keyed(&self, k: &Cid, block: &[u8]) -> anyhow::Result<()> {
-        self.blockchain_db
-            .write()
-            .insert(k.to_bytes(), block.to_vec());
+        self.blockchain_db.write().insert(*k, block.to_vec());
         Ok(())
     }
 }
@@ -135,14 +152,14 @@ impl PersistentStore for MemoryDB {
     fn put_keyed_persistent(&self, k: &Cid, block: &[u8]) -> anyhow::Result<()> {
         self.blockchain_persistent_db
             .write()
-            .insert(k.to_bytes(), block.to_vec());
+            .insert(*k, block.to_vec());
         Ok(())
     }
 }
 
 impl BitswapStoreRead for MemoryDB {
     fn contains(&self, cid: &Cid) -> anyhow::Result<bool> {
-        Ok(self.blockchain_db.read().contains_key(&cid.to_bytes()))
+        Ok(self.blockchain_db.read().contains_key(cid))
     }
 
     fn get(&self, cid: &Cid) -> anyhow::Result<Option<Vec<u8>>> {
@@ -157,3 +174,43 @@ impl BitswapStoreReadWrite for MemoryDB {
         self.put_keyed(block.cid(), block.data())
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::db::{car::ForestCar, setting_keys::HEAD_KEY};
+    use fvm_ipld_encoding::DAG_CBOR;
+    use multihash_codetable::Code::Blake2b256;
+    use nunny::vec as nonempty;
+
+    #[tokio::test]
+    async fn test_export_forest_car() {
+        let db = MemoryDB::default();
+        let record1 = b"non-persistent";
+        let key1 = Cid::new_v1(DAG_CBOR, Blake2b256.digest(record1.as_slice()));
+        db.put_keyed(&key1, record1.as_slice()).unwrap();
+
+        let record2 = b"persistent";
+        let key2 = Cid::new_v1(DAG_CBOR, Blake2b256.digest(record2.as_slice()));
+        db.put_keyed_persistent(&key2, record2.as_slice()).unwrap();
+
+        let mut car_db_bytes = vec![];
+        assert!(db
+            .export_forest_car(&mut car_db_bytes)
+            .await
+            .unwrap_err()
+            .to_string()
+            .contains("chain head is not tracked and cannot be exported"));
+
+        db.write_obj(HEAD_KEY, &TipsetKey::from(nonempty![key1]))
+            .unwrap();
+
+        car_db_bytes.clear();
+        db.export_forest_car(&mut car_db_bytes).await.unwrap();
+
+        let car = ForestCar::new(car_db_bytes).unwrap();
+        assert_eq!(car.roots(), &nonempty![key1]);
+        assert!(car.has(&key1).unwrap());
+        assert!(car.has(&key2).unwrap());
+    }
+}

src/rpc/auth_layer.rs

@@ -32,7 +32,7 @@ static METHOD_NAME2REQUIRED_PERMISSION: Lazy<HashMap<&str, Permission>> = Lazy::
             }
         };
     }
-    super::for_each_method!(insert);
+    super::for_each_rpc_method!(insert);
 
     access.insert(chain::CHAIN_NOTIFY, Permission::Read);
     access.insert(CANCEL_METHOD_NAME, Permission::Read);