mtfse/src/data/mod.rs

use super::*;
use generational_arena::{
    Arena, Index as ArenaIndex,
};

use std::convert::{TryFrom, TryInto};

use std::collections::BTreeMap;
use std::collections::HashSet;

use std::path::{Path, PathBuf};
use std::fs::File;
use std::ffi::OsString;
use memmap::Mmap;
use bytes::Bytes;
use cryptohelpers::{aes, sha256};


mod entry;
pub use entry::Entry;
pub use entry::builder::EntryBuilder;

mod cache;
pub use cache::DataCacheState;

mod metadata;
pub use metadata::StoreMetadata;

mod freeze;
pub use freeze::Freeze;

mod search;
pub use search::*;

mod mutation;
pub use mutation::*;

mod clean;
pub use clean::*;

#[cfg(test)] mod test;

/// The key used to look up a single entry in `O(1)` time.
///
/// # Notes
/// If you change this, make sure to change the `BuildHasher` back to `RandomState`.
pub type EntryKey = sha256::Sha256Hash;

/// The hasher used for the entry data set.
///
/// # Notes
/// Change this back to `RandomState` if you change the type of `EntryKey`.
pub type BuildHasher = std::collections::hash_map::RandomState;//Sha256TopBuildHasher; <-- bug here

#[derive(Debug, Copy, Clone, Ord, PartialOrd, Eq, PartialEq, Hash, Default)]
struct PurgeTrack
{
    /// Number of removals since last purge
    num_removals: usize,
    /// Max number of removals before an arena index purge.
    ///
    /// If set to 0, purge will be performed on every removal of a tag.
    num_removals_max: usize,
}

impl PurgeTrack
{
    /// Create a new purge tracker with this purge threshold.
    #[inline] pub fn new(max: usize) -> Self
    {
	Self {
	    num_removals: 0,
	    num_removals_max: max,
	}
    }

    /// Should we purge now?
    #[inline] pub fn should_purge(&self) -> bool
    {
	self.num_removals >= self.num_removals_max
    }
}

#[derive(Debug)]
pub struct Store
{
    metadata: StoreMetadata,

    purge_track: PurgeTrack,

    data: HashSet<Entry, BuildHasher>, // The entry sha256 hash is used as the `key` here, as `Entry` both hasshes to, and `Borrow`s to `Sha256Hash`.
    data_hashes: Arena<EntryKey>, // used to lookup in `data`.
    
    tag_mappings: Arena<HashSet<ArenaIndex>>,
    tags: BTreeMap<String, ArenaIndex>, // string (tags) -> index (tag_mappings) -> index (data_hashes) -> hash used for lookup (data)
}

// Creating
impl Store
{
    /// Create a new empty store with this metadata.
    ///
    /// # Panics
    /// If the root directory specified in `metadata` does not exist or is not a directory.
    pub fn new(metadata: StoreMetadata) -> Self
    {
	assert!(metadata.root.exists() && metadata.root.is_dir(), "Metadata root {:?} passed to `new` not existant or not a directory", metadata.root);
	Self {
	    metadata,

	    purge_track: PurgeTrack::new(16),

	    data: HashSet::with_hasher(Default::default()),
	    data_hashes: Arena::new(),

	    tag_mappings: Arena::new(),
	    tags: BTreeMap::new(),
	}
    }
    /// Create a new empty store with this metadata and initial storage capacity
    ///
    /// # Panics
    /// If the root directory specified in `metadata` does not exist or is not a directory.
    pub fn with_capacity(metadata: StoreMetadata, cap: usize) -> Self
    {
	assert!(metadata.root.exists() && metadata.root.is_dir(), "Metadata root {:?} passed to `with_capacity` not existant or not a directory", metadata.root);
	Self {
	    metadata,

	    purge_track: PurgeTrack::new(16),

	    data: HashSet::with_capacity_and_hasher(cap, Default::default()),
	    data_hashes: Arena::with_capacity(cap),

	    tag_mappings: Arena::with_capacity(cap),
	    tags: BTreeMap::new(),
	}
    }
}

// Freezing
impl Store
{
    /// Create a snapshot of this store, cloning all data into a frozen and serialisable version of it.
    /// # Notes
    /// This method clones the entire store into the new `Freeze`. To avoid this, use `into_freeze` if the store is no longer used after the freeze.
    #[inline] pub fn freeze(&self) -> Freeze
    {
	Freeze::new_ref(self)
    }
    /// Consume into a snapshot of this store, moving all data into a frozen and serializable version of it.
    #[inline] pub fn into_freeze(self) -> Freeze
    {
	Freeze::new_moved(self)
    }

    /// Create a new store instance by cloning from a frozen snapshot of it.
    /// # Notes
    /// This method clones the entire `Freeze` into the new store. To avoid this, use `from_freeze` if the snapshot is no longer used after the unfreeze.
    #[inline] pub fn unfreeze(freeze: &Freeze) -> Self
    {
	freeze.create_new()
    }
    /// Consume a store snapshot and move its entries into a new store.
    #[inline] pub fn from_freeze(freeze: Freeze) -> Self
    {
	freeze.into_new()
    }
}

// Primitive access
impl Store
{
    /// Look up a single entry in `O(1)` time with its key.
    #[inline] pub fn get(&self, key: &EntryKey) -> Option<&Entry>
    {
	self.data.get(key)
    }
}