You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
177 lines
4.7 KiB
177 lines
4.7 KiB
use super::*;
|
|
use generational_arena::{
|
|
Arena, Index as ArenaIndex,
|
|
};
|
|
|
|
use std::convert::{TryFrom, TryInto};
|
|
|
|
use std::collections::BTreeMap;
|
|
use std::collections::HashSet;
|
|
|
|
use std::path::{Path, PathBuf};
|
|
use std::fs::File;
|
|
use std::ffi::OsString;
|
|
use memmap::Mmap;
|
|
use bytes::Bytes;
|
|
use cryptohelpers::{aes, sha256};
|
|
|
|
|
|
mod entry;
|
|
pub use entry::Entry;
|
|
pub use entry::builder::EntryBuilder;
|
|
|
|
mod cache;
|
|
pub use cache::DataCacheState;
|
|
|
|
mod metadata;
|
|
pub use metadata::StoreMetadata;
|
|
|
|
mod freeze;
|
|
pub use freeze::Freeze;
|
|
|
|
mod search;
|
|
pub use search::*;
|
|
|
|
mod mutation;
|
|
pub use mutation::*;
|
|
|
|
#[cfg(test)] mod test;
|
|
|
|
/// The key used to look up a single entry in `O(1)` time.
|
|
///
|
|
/// # Notes
|
|
/// If you change this, make sure to change the `BuildHasher` back to `RandomState`.
|
|
pub type EntryKey = sha256::Sha256Hash;
|
|
|
|
/// The hasher used for the entry data set.
|
|
///
|
|
/// # Notes
|
|
/// Change this back to `RandomState` if you change the type of `EntryKey`.
|
|
pub type BuildHasher = std::collections::hash_map::RandomState;//Sha256TopBuildHasher; <-- bug here
|
|
|
|
#[derive(Debug, Copy, Clone, Ord, PartialOrd, Eq, PartialEq, Hash, Default)]
|
|
struct PurgeTrack
|
|
{
|
|
/// Number of removals since last purge
|
|
num_removals: usize,
|
|
/// Max number of removals before an arena index purge.
|
|
///
|
|
/// If set to 0, purge will be performed on every removal of a tag.
|
|
num_removals_max: usize,
|
|
}
|
|
|
|
impl PurgeTrack
|
|
{
|
|
/// Create a new purge tracker with this purge threshold.
|
|
#[inline] pub fn new(max: usize) -> Self
|
|
{
|
|
Self {
|
|
num_removals: 0,
|
|
num_removals_max: max,
|
|
}
|
|
}
|
|
|
|
/// Should we purge now?
|
|
#[inline] pub fn should_purge(&self) -> bool
|
|
{
|
|
self.num_removals >= self.num_removals_max
|
|
}
|
|
}
|
|
|
|
#[derive(Debug)]
|
|
pub struct Store
|
|
{
|
|
metadata: StoreMetadata,
|
|
|
|
purge_track: PurgeTrack,
|
|
|
|
data: HashSet<Entry, BuildHasher>, // The entry sha256 hash is used as the `key` here, as `Entry` both hasshes to, and `Borrow`s to `Sha256Hash`.
|
|
data_hashes: Arena<EntryKey>, // used to lookup in `data`.
|
|
|
|
tag_mappings: Arena<HashSet<ArenaIndex>>,
|
|
tags: BTreeMap<String, ArenaIndex>, // string (tags) -> index (tag_mappings) -> index (data_hashes) -> hash used for lookup (data)
|
|
}
|
|
|
|
// Creating
|
|
impl Store
|
|
{
|
|
/// Create a new empty store with this metadata.
|
|
///
|
|
/// # Panics
|
|
/// If the root directory specified in `metadata` does not exist or is not a directory.
|
|
pub fn new(metadata: StoreMetadata) -> Self
|
|
{
|
|
assert!(metadata.root.exists() && metadata.root.is_dir(), "Metadata root {:?} passed to `new` not existant or not a directory", metadata.root);
|
|
Self {
|
|
metadata,
|
|
|
|
purge_track: PurgeTrack::new(16),
|
|
|
|
data: HashSet::with_hasher(Default::default()),
|
|
data_hashes: Arena::new(),
|
|
|
|
tag_mappings: Arena::new(),
|
|
tags: BTreeMap::new(),
|
|
}
|
|
}
|
|
/// Create a new empty store with this metadata and initial storage capacity
|
|
///
|
|
/// # Panics
|
|
/// If the root directory specified in `metadata` does not exist or is not a directory.
|
|
pub fn with_capacity(metadata: StoreMetadata, cap: usize) -> Self
|
|
{
|
|
assert!(metadata.root.exists() && metadata.root.is_dir(), "Metadata root {:?} passed to `with_capacity` not existant or not a directory", metadata.root);
|
|
Self {
|
|
metadata,
|
|
|
|
purge_track: PurgeTrack::new(16),
|
|
|
|
data: HashSet::with_capacity_and_hasher(cap, Default::default()),
|
|
data_hashes: Arena::with_capacity(cap),
|
|
|
|
tag_mappings: Arena::with_capacity(cap),
|
|
tags: BTreeMap::new(),
|
|
}
|
|
}
|
|
}
|
|
|
|
// Freezing
|
|
impl Store
|
|
{
|
|
/// Create a snapshot of this store, cloning all data into a frozen and serialisable version of it.
|
|
/// # Notes
|
|
/// This method clones the entire store into the new `Freeze`. To avoid this, use `into_freeze` if the store is no longer used after the freeze.
|
|
#[inline] pub fn freeze(&self) -> Freeze
|
|
{
|
|
Freeze::new_ref(self)
|
|
}
|
|
/// Consume into a snapshot of this store, moving all data into a frozen and serializable version of it.
|
|
#[inline] pub fn into_freeze(self) -> Freeze
|
|
{
|
|
Freeze::new_moved(self)
|
|
}
|
|
|
|
/// Create a new store instance by cloning from a frozen snapshot of it.
|
|
/// # Notes
|
|
/// This method clones the entire `Freeze` into the new store. To avoid this, use `from_freeze` if the snapshot is no longer used after the unfreeze.
|
|
#[inline] pub fn unfreeze(freeze: &Freeze) -> Self
|
|
{
|
|
freeze.create_new()
|
|
}
|
|
/// Consume a store snapshot and move its entries into a new store.
|
|
#[inline] pub fn from_freeze(freeze: Freeze) -> Self
|
|
{
|
|
freeze.into_new()
|
|
}
|
|
}
|
|
|
|
// Primitive access
|
|
impl Store
|
|
{
|
|
/// Look up a single entry in `O(1)` time with its key.
|
|
#[inline] pub fn get(&self, key: &EntryKey) -> Option<&Entry>
|
|
{
|
|
self.data.get(key)
|
|
}
|
|
}
|