iroh_blobs/format/
collection.rs

1//! The collection type used by iroh
2use std::{collections::BTreeMap, future::Future};
3
4// n0_error::Context is no longer exported; use explicit mapping instead.
5use bao_tree::blake3;
6use bytes::Bytes;
7use n0_error::{Result, StdResultExt};
8use serde::{Deserialize, Serialize};
9
10use crate::{
11    api::{blobs::AddBytesOptions, Store},
12    get::{fsm, Stats},
13    hashseq::HashSeq,
14    util::temp_tag::TempTag,
15    BlobFormat, Hash,
16};
17
18/// A collection of blobs
19///
20/// Note that the format is subject to change.
21#[derive(Clone, Debug, PartialEq, Deserialize, Serialize, Default)]
22pub struct Collection {
23    /// Links to the blobs in this collection
24    blobs: Vec<(String, Hash)>,
25}
26
27impl std::ops::Index<usize> for Collection {
28    type Output = (String, Hash);
29
30    fn index(&self, index: usize) -> &Self::Output {
31        &self.blobs[index]
32    }
33}
34
35impl<K, V> Extend<(K, V)> for Collection
36where
37    K: Into<String>,
38    V: Into<Hash>,
39{
40    fn extend<T: IntoIterator<Item = (K, V)>>(&mut self, iter: T) {
41        self.blobs
42            .extend(iter.into_iter().map(|(k, v)| (k.into(), v.into())));
43    }
44}
45
46impl<K, V> FromIterator<(K, V)> for Collection
47where
48    K: Into<String>,
49    V: Into<Hash>,
50{
51    fn from_iter<T: IntoIterator<Item = (K, V)>>(iter: T) -> Self {
52        let mut res = Self::default();
53        res.extend(iter);
54        res
55    }
56}
57
58impl IntoIterator for Collection {
59    type Item = (String, Hash);
60    type IntoIter = std::vec::IntoIter<Self::Item>;
61
62    fn into_iter(self) -> Self::IntoIter {
63        self.blobs.into_iter()
64    }
65}
66
67/// A simple store trait for loading blobs
68pub trait SimpleStore {
69    /// Load a blob from the store
70    fn load(&self, hash: Hash) -> impl Future<Output = Result<Bytes>> + Send + '_;
71}
72
73impl SimpleStore for crate::api::Store {
74    async fn load(&self, hash: Hash) -> Result<Bytes> {
75        Ok(self.get_bytes(hash).await?)
76    }
77}
78
79/// Metadata for a collection
80///
81/// This is the wire format for the metadata blob.
82#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)]
83struct CollectionMeta {
84    header: [u8; 13], // Must contain "CollectionV0."
85    names: Vec<String>,
86}
87
88impl Collection {
89    /// The header for the collection format.
90    ///
91    /// This is the start of the metadata blob.
92    pub const HEADER: &'static [u8; 13] = b"CollectionV0.";
93
94    /// Convert the collection to an iterator of blobs, with the last being the
95    /// root blob.
96    ///
97    /// To persist the collection, write all the blobs to storage, and use the
98    /// hash of the last blob as the collection hash.
99    pub fn to_blobs(&self) -> impl DoubleEndedIterator<Item = Bytes> {
100        let meta = CollectionMeta {
101            header: *Self::HEADER,
102            names: self.names(),
103        };
104        let meta_bytes = postcard::to_stdvec(&meta).unwrap();
105        let meta_bytes_hash = blake3::hash(&meta_bytes).into();
106        let links = std::iter::once(meta_bytes_hash)
107            .chain(self.links())
108            .collect::<HashSeq>();
109        let links_bytes = links.into_inner();
110        [meta_bytes.into(), links_bytes].into_iter()
111    }
112
113    /// Read the collection from a get fsm.
114    ///
115    /// Returns the fsm at the start of the first child blob (if any),
116    /// the links array, and the collection.
117    pub async fn read_fsm(
118        fsm_at_start_root: fsm::AtStartRoot,
119    ) -> Result<(fsm::EndBlobNext, HashSeq, Collection)> {
120        let (next, links) = {
121            let curr = fsm_at_start_root.next();
122            let (curr, data) = curr.concatenate_into_vec().await?;
123            let links = HashSeq::new(data.into())
124                .ok_or_else(|| n0_error::anyerr!("links could not be parsed"))?;
125            (curr.next(), links)
126        };
127        let fsm::EndBlobNext::MoreChildren(at_meta) = next else {
128            n0_error::bail_any!("expected meta");
129        };
130        let (next, collection) = {
131            let mut children = links.clone();
132            let meta_link = children
133                .pop_front()
134                .ok_or_else(|| n0_error::anyerr!("meta link not found"))?;
135            let curr = at_meta.next(meta_link);
136            let (curr, names) = curr.concatenate_into_vec().await?;
137            let names = postcard::from_bytes::<CollectionMeta>(&names).anyerr()?;
138            n0_error::ensure_any!(
139                names.header == *Self::HEADER,
140                "expected header {:?}, got {:?}",
141                Self::HEADER,
142                names.header
143            );
144            let collection = Collection::from_parts(children, names);
145            (curr.next(), collection)
146        };
147        Ok((next, links, collection))
148    }
149
150    /// Read the collection and all it's children from a get fsm.
151    ///
152    /// Returns the collection, a map from blob offsets to bytes, and the stats.
153    pub async fn read_fsm_all(
154        fsm_at_start_root: crate::get::fsm::AtStartRoot,
155    ) -> Result<(Collection, BTreeMap<u64, Bytes>, Stats)> {
156        let (next, links, collection) = Self::read_fsm(fsm_at_start_root).await?;
157        let mut res = BTreeMap::new();
158        let mut curr = next;
159        let end = loop {
160            match curr {
161                fsm::EndBlobNext::MoreChildren(more) => {
162                    let child_offset = more.offset() - 1;
163                    let Some(hash) = links.get(usize::try_from(child_offset).anyerr()?) else {
164                        break more.finish();
165                    };
166                    let header = more.next(hash);
167                    let (next, blob) = header.concatenate_into_vec().await?;
168                    res.insert(child_offset - 1, blob.into());
169                    curr = next.next();
170                }
171                fsm::EndBlobNext::Closing(closing) => break closing,
172            }
173        };
174        let stats = end.next().await?;
175        Ok((collection, res, stats))
176    }
177
178    /// Create a new collection from a hash sequence and metadata.
179    pub async fn load(root: Hash, store: &impl SimpleStore) -> Result<Self> {
180        let hs = store.load(root).await?;
181        let hs = HashSeq::try_from(hs)?;
182        let meta_hash = hs
183            .iter()
184            .next()
185            .ok_or_else(|| n0_error::anyerr!("empty hash seq"))?;
186        let meta = store.load(meta_hash).await?;
187        let meta: CollectionMeta = postcard::from_bytes(&meta).anyerr()?;
188        n0_error::ensure_any!(
189            meta.names.len() + 1 == hs.len(),
190            "names and links length mismatch"
191        );
192        Ok(Self::from_parts(hs.into_iter().skip(1), meta))
193    }
194
195    /// Store a collection in a store. returns the root hash of the collection
196    /// as a TempTag.
197    pub async fn store(self, db: &Store) -> Result<TempTag> {
198        let (links, meta) = self.into_parts();
199        let meta_bytes = postcard::to_stdvec(&meta).anyerr()?;
200        let meta_tag = db.add_bytes(meta_bytes).temp_tag().await?;
201        let links_bytes = std::iter::once(meta_tag.hash())
202            .chain(links)
203            .collect::<HashSeq>();
204        let links_tag = db
205            .add_bytes_with_opts(AddBytesOptions {
206                data: links_bytes.into(),
207                format: BlobFormat::HashSeq,
208            })
209            .temp_tag()
210            .await?;
211        Ok(links_tag)
212    }
213
214    /// Split a collection into a sequence of links and metadata
215    fn into_parts(self) -> (Vec<Hash>, CollectionMeta) {
216        let mut names = Vec::with_capacity(self.blobs.len());
217        let mut links = Vec::with_capacity(self.blobs.len());
218        for (name, hash) in self.blobs {
219            names.push(name);
220            links.push(hash);
221        }
222        let meta = CollectionMeta {
223            header: *Self::HEADER,
224            names,
225        };
226        (links, meta)
227    }
228
229    /// Create a new collection from a list of hashes and metadata
230    fn from_parts(links: impl IntoIterator<Item = Hash>, meta: CollectionMeta) -> Self {
231        meta.names.into_iter().zip(links).collect()
232    }
233
234    /// Get the links to the blobs in this collection
235    fn links(&self) -> impl Iterator<Item = Hash> + '_ {
236        self.blobs.iter().map(|(_name, hash)| *hash)
237    }
238
239    /// Get the names of the blobs in this collection
240    fn names(&self) -> Vec<String> {
241        self.blobs.iter().map(|(name, _)| name.clone()).collect()
242    }
243
244    /// Iterate over the blobs in this collection
245    pub fn iter(&self) -> impl Iterator<Item = &(String, Hash)> {
246        self.blobs.iter()
247    }
248
249    /// Get the number of blobs in this collection
250    pub fn len(&self) -> usize {
251        self.blobs.len()
252    }
253
254    /// Check if this collection is empty
255    pub fn is_empty(&self) -> bool {
256        self.blobs.is_empty()
257    }
258
259    /// Add the given blob to the collection.
260    pub fn push(&mut self, name: String, hash: Hash) {
261        self.blobs.push((name, hash));
262    }
263}
264
265#[cfg(test)]
266mod tests {
267    use n0_error::{Result, StackResultExt};
268
269    use super::*;
270
271    #[test]
272    fn roundtrip_blob() {
273        let b = (
274            "test".to_string(),
275            blake3::Hash::from_hex(
276                "3aa61c409fd7717c9d9c639202af2fae470c0ef669be7ba2caea5779cb534e9d",
277            )
278            .unwrap()
279            .into(),
280        );
281
282        let mut buf = bytes::BytesMut::zeroed(1024);
283        postcard::to_slice(&b, &mut buf).unwrap();
284        let deserialize_b: (String, Hash) = postcard::from_bytes(&buf).unwrap();
285        assert_eq!(b, deserialize_b);
286    }
287
288    #[test]
289    fn roundtrip_collection_meta() {
290        let expected = CollectionMeta {
291            header: *Collection::HEADER,
292            names: vec!["test".to_string(), "a".to_string(), "b".to_string()],
293        };
294        let mut buf = bytes::BytesMut::zeroed(1024);
295        postcard::to_slice(&expected, &mut buf).unwrap();
296        let actual: CollectionMeta = postcard::from_bytes(&buf).unwrap();
297        assert_eq!(expected, actual);
298    }
299
300    #[tokio::test]
301    async fn collection_store_load() -> testresult::TestResult {
302        let collection = (0..3)
303            .map(|i| {
304                (
305                    format!("blob{i}"),
306                    crate::Hash::from(blake3::hash(&[i as u8])),
307                )
308            })
309            .collect::<Collection>();
310        let mut root = None;
311        let store = collection
312            .to_blobs()
313            .map(|data| {
314                let hash = crate::Hash::from(blake3::hash(&data));
315                root = Some(hash);
316                (hash, data)
317            })
318            .collect::<TestStore>();
319        let collection2 = Collection::load(root.unwrap(), &store).await?;
320        assert_eq!(collection, collection2);
321        Ok(())
322    }
323
324    /// An implementation of a [SimpleStore] for testing
325    struct TestStore(BTreeMap<Hash, Bytes>);
326
327    impl FromIterator<(Hash, Bytes)> for TestStore {
328        fn from_iter<T: IntoIterator<Item = (Hash, Bytes)>>(iter: T) -> Self {
329            Self(iter.into_iter().collect())
330        }
331    }
332
333    impl SimpleStore for TestStore {
334        async fn load(&self, hash: Hash) -> Result<Bytes> {
335            self.0.get(&hash).cloned().context("not found")
336        }
337    }
338}
iroh_blobs/format/collection.rs

iroh_blobs/format/
collection.rs
