use super::*; use generational_arena::{ Arena, Index as ArenaIndex, }; use std::convert::{TryFrom, TryInto}; use std::collections::BTreeMap; use std::collections::HashSet; use std::path::{Path, PathBuf}; use std::fs::File; use std::ffi::OsString; use memmap::Mmap; use bytes::Bytes; use cryptohelpers::{aes, sha256}; mod entry; pub use entry::Entry; pub use entry::builder::EntryBuilder; mod cache; pub use cache::DataCacheState; mod metadata; pub use metadata::StoreMetadata; mod freeze; pub use freeze::Freeze; mod search; pub use search::*; mod mutation; pub use mutation::*; mod clean; pub use clean::*; #[cfg(test)] mod test; /// The key used to look up a single entry in `O(1)` time. /// /// # Notes /// If you change this, make sure to change the `BuildHasher` back to `RandomState`. pub type EntryKey = sha256::Sha256Hash; /// The hasher used for the entry data set. /// /// # Notes /// Change this back to `RandomState` if you change the type of `EntryKey`. pub type BuildHasher = std::collections::hash_map::RandomState;//Sha256TopBuildHasher; <-- bug here #[derive(Debug, Copy, Clone, Ord, PartialOrd, Eq, PartialEq, Hash, Default)] struct PurgeTrack { /// Number of removals since last purge num_removals: usize, /// Max number of removals before an arena index purge. /// /// If set to 0, purge will be performed on every removal of a tag. num_removals_max: usize, } impl PurgeTrack { /// Create a new purge tracker with this purge threshold. #[inline] pub fn new(max: usize) -> Self { Self { num_removals: 0, num_removals_max: max, } } /// Should we purge now? #[inline] pub fn should_purge(&self) -> bool { self.num_removals >= self.num_removals_max } } #[derive(Debug)] pub struct Store { metadata: StoreMetadata, purge_track: PurgeTrack, data: HashSet, // The entry sha256 hash is used as the `key` here, as `Entry` both hasshes to, and `Borrow`s to `Sha256Hash`. data_hashes: Arena, // used to lookup in `data`. tag_mappings: Arena>, tags: BTreeMap, // string (tags) -> index (tag_mappings) -> index (data_hashes) -> hash used for lookup (data) } // Creating impl Store { /// Create a new empty store with this metadata. /// /// # Panics /// If the root directory specified in `metadata` does not exist or is not a directory. pub fn new(metadata: StoreMetadata) -> Self { assert!(metadata.root.exists() && metadata.root.is_dir(), "Metadata root {:?} passed to `new` not existant or not a directory", metadata.root); Self { metadata, purge_track: PurgeTrack::new(16), data: HashSet::with_hasher(Default::default()), data_hashes: Arena::new(), tag_mappings: Arena::new(), tags: BTreeMap::new(), } } /// Create a new empty store with this metadata and initial storage capacity /// /// # Panics /// If the root directory specified in `metadata` does not exist or is not a directory. pub fn with_capacity(metadata: StoreMetadata, cap: usize) -> Self { assert!(metadata.root.exists() && metadata.root.is_dir(), "Metadata root {:?} passed to `with_capacity` not existant or not a directory", metadata.root); Self { metadata, purge_track: PurgeTrack::new(16), data: HashSet::with_capacity_and_hasher(cap, Default::default()), data_hashes: Arena::with_capacity(cap), tag_mappings: Arena::with_capacity(cap), tags: BTreeMap::new(), } } } // Freezing impl Store { /// Create a snapshot of this store, cloning all data into a frozen and serialisable version of it. /// # Notes /// This method clones the entire store into the new `Freeze`. To avoid this, use `into_freeze` if the store is no longer used after the freeze. #[inline] pub fn freeze(&self) -> Freeze { Freeze::new_ref(self) } /// Consume into a snapshot of this store, moving all data into a frozen and serializable version of it. #[inline] pub fn into_freeze(self) -> Freeze { Freeze::new_moved(self) } /// Create a new store instance by cloning from a frozen snapshot of it. /// # Notes /// This method clones the entire `Freeze` into the new store. To avoid this, use `from_freeze` if the snapshot is no longer used after the unfreeze. #[inline] pub fn unfreeze(freeze: &Freeze) -> Self { freeze.create_new() } /// Consume a store snapshot and move its entries into a new store. #[inline] pub fn from_freeze(freeze: Freeze) -> Self { freeze.into_new() } } // Primitive access impl Store { /// Look up a single entry in `O(1)` time with its key. #[inline] pub fn get(&self, key: &EntryKey) -> Option<&Entry> { self.data.get(key) } }