started byte cache

new
Avril 3 years ago
parent 1fb08b35a0
commit 674dab36d1
Signed by: flanchan
GPG Key ID: 284488987C31F630

61
Cargo.lock generated

@ -55,9 +55,9 @@ checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38"
[[package]]
name = "bytes"
version = "0.6.0"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e0dcbc35f504eb6fc275a6d20e4ebcda18cf50d40ba6fabff8c711fa16cb3b16"
checksum = "b700ce4376041dcd0a327fd0097c41095743c4c8af8887265942faf1100bd040"
[[package]]
name = "cc"
@ -93,6 +93,20 @@ dependencies = [
"tokio",
]
[[package]]
name = "chrono"
version = "0.4.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73"
dependencies = [
"libc",
"num-integer",
"num-traits",
"serde",
"time",
"winapi 0.3.9",
]
[[package]]
name = "cpuid-bool"
version = "0.1.2"
@ -120,9 +134,9 @@ dependencies = [
[[package]]
name = "cryptohelpers"
version = "1.7.2"
version = "1.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "33c34ac8437a348d0c23e71327d6d8affe4509cc91e33dd22c1e38f7c9da8070"
checksum = "488ef2d3d94ab2ec791e4c1decf766b4b9beac2d49b41d184db937adc8d54a91"
dependencies = [
"crc",
"futures",
@ -323,7 +337,7 @@ checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8"
dependencies = [
"cfg-if 1.0.0",
"libc",
"wasi 0.10.2+wasi-snapshot-preview1",
"wasi 0.10.0+wasi-snapshot-preview1",
]
[[package]]
@ -510,8 +524,9 @@ version = "0.1.0"
dependencies = [
"ad-hoc-iter",
"bitflags",
"bytes 0.6.0",
"bytes 1.0.1",
"chacha20stream",
"chrono",
"cryptohelpers",
"futures",
"generational-arena",
@ -541,6 +556,25 @@ dependencies = [
"winapi 0.3.9",
]
[[package]]
name = "num-integer"
version = "0.1.44"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db"
dependencies = [
"autocfg",
"num-traits",
]
[[package]]
name = "num-traits"
version = "0.2.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290"
dependencies = [
"autocfg",
]
[[package]]
name = "num_cpus"
version = "1.13.0"
@ -901,6 +935,17 @@ dependencies = [
"unicode-xid",
]
[[package]]
name = "time"
version = "0.1.44"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255"
dependencies = [
"libc",
"wasi 0.10.0+wasi-snapshot-preview1",
"winapi 0.3.9",
]
[[package]]
name = "tokio"
version = "0.2.24"
@ -978,9 +1023,9 @@ checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519"
[[package]]
name = "wasi"
version = "0.10.2+wasi-snapshot-preview1"
version = "0.10.0+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"
checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f"
[[package]]
name = "winapi"

@ -10,8 +10,9 @@ edition = "2018"
[dependencies]
ad-hoc-iter = "0.2.2"
bitflags = "1.2.1"
bytes = "0.6.0"
bytes = "1.0.1"
chacha20stream = {version = "1.0", features = ["async", "serde"]}
chrono = {version = "0.4.19", features= ["serde"]}
cryptohelpers = {version = "1.7", features = ["full", "async", "serde"]}
futures = "0.3.8"
generational-arena = "0.2.8"

@ -2,6 +2,7 @@ use super::*;
use std::hash::{BuildHasherDefault, Hasher};
use smallvec::SmallVec;
use cryptohelpers::sha256;
use ::bytes::Buf;
/// A hasher that takes the first 8 bytes from SHA256 hash as its output.
///
@ -34,3 +35,29 @@ impl Hasher for Sha256TopHasher
self.0.extend_from_slice(bytes);
}
}
pub trait Sha256HashExt
{
fn compute_sha256_hash(&self) -> sha256::Sha256Hash;
}
pub trait Sha256HashOwnedExt: Sized
{
fn into_sha256_hash(self) -> sha256::Sha256Hash;
}
impl<T: ?Sized> Sha256HashExt for T
where T: AsRef<[u8]>
{
#[inline] fn compute_sha256_hash(&self) -> sha256::Sha256Hash {
sha256::compute_slice(self.as_ref())
}
}
impl<T: Buf> Sha256HashOwnedExt for T
{
#[inline] fn into_sha256_hash(self) -> sha256::Sha256Hash {
sha256::compute_sync(self.reader()).unwrap()
}
}

@ -0,0 +1,85 @@
//! Service cacheing
use super::*;
use std::{
hash::Hash,
marker::{Send, Sync},
fmt,
};
use std::sync::{
RwLock,
atomic::AtomicUsize,
};
use std::path::PathBuf;
use std::num::NonZeroUsize;
use std::collections::HashMap;
use chrono::DateTime;
use ::bytes::Bytes;
pub type Timezone = chrono::Utc;
/// A type that can be used for the cache key
pub trait Key: Sized +
Send +
Sync +
Hash +
PartialEq +
Eq +
fmt::Display +
Serialize +
for<'a> Deserialize<'a> +
'static
{}
basic_enum!(pub PurgeOrder; "How should a cache determine what to purge": Oldest => "Purge the oldest entries first", LeastUsed => "Purge the least accessed entries first", OldestUsed => "Purge the oldest accessed entries first");
default!(PurgeOrder: Self::LeastUsed);
/// Config for a cache
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Config
{
// General
// Disk cache
/// The directory to save to data to.
/// If `None`, se memory cache only.
pub disk_location: Option<PathBuf>,
// Memory cache
/// Max size of an entry to cache in memory
pub mem_max_ent_size: Option<NonZeroUsize>,
/// Max entries in memory cache
pub mem_max_ent: Option<NonZeroUsize>,
/// Max total bytes to keep in memcache before purging older entries.
pub mem_max_total_size: Option<NonZeroUsize>,
/// When purging entries from memcache, how to select ones to purge
pub mem_purge_order: PurgeOrder,
}
/// The memory hold of a cahce.
#[derive(Debug, Clone, PartialEq, Eq, Hash, Default)]
struct Memory(Bytes);
/// An entry in a `ByteCache`.
#[derive(Debug)]
pub struct CacheEntry //<K: Key> // `K` is the key in `entries`.
{
tm_accessed: RwLock<DateTime<Timezone>>, // Can be mutated when read, so must be mutable by the reader, hence the `RwLock`.
tm_created: DateTime<Timezone>,
accesses: AtomicUsize, // Can be mutated when read, hence the atomic.
memory: Option<Memory>,
// Pathname is computed from `key`.
}
/// A byte-stream cache. Caches both to memory and also writes entries to disk.
#[derive(Debug)]
pub struct ByteCache<K: Key>
{
// Config is big, box it.
cfg: Box<Config>,
/// Frozen entries.
entries: HashMap<K, CacheEntry>,
}

@ -25,6 +25,8 @@ pub mod config;
mod builder;
pub use builder::*;
pub mod cache;
mod host;
/// Handle to a running service. Can be used to join it or create `Channel`s.

Loading…
Cancel
Save