start dedup and container

dedup
Avril 4 years ago
parent 5c784482f6
commit 9a7234806f
Signed by: flanchan
GPG Key ID: 284488987C31F630

144
Cargo.lock generated

@ -103,6 +103,12 @@ dependencies = [
"safemem", "safemem",
] ]
[[package]]
name = "build_const"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39092a32794787acd8525ee150305ff051b0aa6cc2abaf193924f5ab05425f39"
[[package]] [[package]]
name = "byte-tools" name = "byte-tools"
version = "0.3.1" version = "0.3.1"
@ -175,6 +181,44 @@ version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8aebca1129a03dc6dc2b127edd729435bbc4a37e1d5f4d7513165089ceb02634" checksum = "8aebca1129a03dc6dc2b127edd729435bbc4a37e1d5f4d7513165089ceb02634"
[[package]]
name = "crc"
version = "1.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d663548de7f5cca343f1e0a48d14dcfb0e9eb4e079ec58883b7251539fa10aeb"
dependencies = [
"build_const",
]
[[package]]
name = "crypto-mac"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "58bcd97a54c7ca5ce2f6eb16f6bede5b0ab5f0055fedc17d2f0b4466e21671ca"
dependencies = [
"generic-array 0.14.4",
"subtle",
]
[[package]]
name = "cryptohelpers"
version = "1.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2cfc491baaffd7cbd6acc02ebd23564760d83a2c17e1a47e6a04a8d5a86e7fb5"
dependencies = [
"crc",
"getrandom",
"hex-literal",
"hmac",
"libc",
"openssl",
"pbkdf2",
"serde",
"serde_derive",
"sha2",
"tokio",
]
[[package]] [[package]]
name = "digest" name = "digest"
version = "0.8.1" version = "0.8.1"
@ -236,6 +280,21 @@ version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
[[package]]
name = "foreign-types"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
dependencies = [
"foreign-types-shared",
]
[[package]]
name = "foreign-types-shared"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
[[package]] [[package]]
name = "fuchsia-cprng" name = "fuchsia-cprng"
version = "0.1.1" version = "0.1.1"
@ -457,6 +516,22 @@ dependencies = [
"libc", "libc",
] ]
[[package]]
name = "hex-literal"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5af1f635ef1bc545d78392b136bfe1c9809e029023c84a3638a864a10b8819c8"
[[package]]
name = "hmac"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "deae6d9dbb35ec2c502d62b8f7b1c000a0822c3b0794ba36b3149c0a1c840dff"
dependencies = [
"crypto-mac",
"digest 0.9.0",
]
[[package]] [[package]]
name = "http" name = "http"
version = "0.2.1" version = "0.2.1"
@ -621,6 +696,7 @@ dependencies = [
"async-compression", "async-compression",
"bzip2-sys", "bzip2-sys",
"cfg-if 1.0.0", "cfg-if 1.0.0",
"cryptohelpers",
"futures", "futures",
"hyper", "hyper",
"lazy_static", "lazy_static",
@ -632,6 +708,7 @@ dependencies = [
"rustc_version", "rustc_version",
"serde", "serde",
"serde_cbor", "serde_cbor",
"sha2",
"smallmap", "smallmap",
"tokio", "tokio",
"toml", "toml",
@ -802,6 +879,48 @@ version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
[[package]]
name = "openssl"
version = "0.10.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8d575eff3665419f9b83678ff2815858ad9d11567e082f5ac1814baba4e2bcb4"
dependencies = [
"bitflags",
"cfg-if 0.1.10",
"foreign-types",
"lazy_static",
"libc",
"openssl-sys",
]
[[package]]
name = "openssl-sys"
version = "0.9.58"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a842db4709b604f0fe5d1170ae3565899be2ad3d9cbc72dedc789ac0511f78de"
dependencies = [
"autocfg 1.0.1",
"cc",
"libc",
"pkg-config",
"vcpkg",
]
[[package]]
name = "pbkdf2"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7170d73bf11f39b4ce1809aabc95bf5c33564cdc16fc3200ddda17a5f6e5e48b"
dependencies = [
"base64",
"crypto-mac",
"hmac",
"rand 0.7.3",
"rand_core 0.5.1",
"sha2",
"subtle",
]
[[package]] [[package]]
name = "percent-encoding" name = "percent-encoding"
version = "2.1.0" version = "2.1.0"
@ -1229,6 +1348,19 @@ dependencies = [
"opaque-debug 0.3.0", "opaque-debug 0.3.0",
] ]
[[package]]
name = "sha2"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2933378ddfeda7ea26f48c555bdad8bb446bf8a3d17832dc83e380d444cfb8c1"
dependencies = [
"block-buffer 0.9.0",
"cfg-if 0.1.10",
"cpuid-bool",
"digest 0.9.0",
"opaque-debug 0.3.0",
]
[[package]] [[package]]
name = "signal-hook-registry" name = "signal-hook-registry"
version = "1.2.1" version = "1.2.1"
@ -1266,6 +1398,12 @@ dependencies = [
"winapi 0.3.9", "winapi 0.3.9",
] ]
[[package]]
name = "subtle"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "343f3f510c2915908f155e94f17220b19ccfacf2a64a2a5d8004f2c3e311e7fd"
[[package]] [[package]]
name = "syn" name = "syn"
version = "1.0.42" version = "1.0.42"
@ -1536,6 +1674,12 @@ version = "0.7.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "05e42f7c18b8f902290b009cde6d651262f956c98bc51bca4cd1d511c9cd85c7" checksum = "05e42f7c18b8f902290b009cde6d651262f956c98bc51bca4cd1d511c9cd85c7"
[[package]]
name = "vcpkg"
version = "0.2.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6454029bf181f092ad1b853286f23e2c507d8e8194d01d92da4a55c274a5508c"
[[package]] [[package]]
name = "version_check" name = "version_check"
version = "0.9.2" version = "0.9.2"

@ -68,6 +68,8 @@ smallmap = "1.1.5"
lazy_static = "1.4.0" lazy_static = "1.4.0"
once_cell = "1.4.1" once_cell = "1.4.1"
bzip2-sys = {version = "0.1.9", optional = true} bzip2-sys = {version = "0.1.9", optional = true}
cryptohelpers = {version = "1.5.1", features= ["sha256", "async"]}
sha2 = "0.9.1"
[build-dependencies] [build-dependencies]
rustc_version = "0.2" rustc_version = "0.2"

@ -0,0 +1,9 @@
//! Container for chain
use super::*;
use chain::{Chain, Chainable};
pub struct ChainContainer<T: Chainable>
{
chain: Chain<T>,
inputs_exact: dedup::HashRefSet<[T]>,
}

@ -0,0 +1,94 @@
//! De-duplicating inputs
use super::*;
use std::{
hash::{Hash, Hasher},
collections::{HashSet, hash_set},
marker::PhantomData,
borrow::Borrow,
};
use sha2::{Sha256, Digest};
use cryptohelpers::sha256::Sha256Hash;
fn compute<T: Hash>(thing: &T) -> Sha256Hash
{
use std::mem::size_of;
struct Sha256Hasher(Sha256);
impl Hasher for Sha256Hasher
{
fn write(&mut self, bytes: &[u8])
{
self.0.update(bytes);
}
fn finish(&self) -> u64
{
let ar = self.0.clone().finalize();
let mut rest = [0u8; size_of::<u64>()];
bytes::copy_slice(&mut rest[..], &ar[..]);
u64::from_le_bytes(rest)
}
}
let mut hasher = Sha256Hasher(Sha256::new());
thing.hash(&mut hasher);
hasher.0.into()
}
/// A HashSet that doesn't own its items.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct HashRefSet<T: Hash + ?Sized>(HashSet<Sha256Hash>, PhantomData<HashSet<*const T>>);
unsafe impl<T: Send + Hash + ?Sized> Send for HashRefSet<T>{}
unsafe impl<T: Send + Sync + Hash + ?Sized> Sync for HashRefSet<T>{}
impl<T: Hash + ?Sized> HashRefSet<T>
{
pub fn new() -> Self
{
Self(HashSet::new(), PhantomData)
}
pub fn with_capacity(cap: usize) -> Self
{
Self(HashSet::with_capacity(cap), PhantomData)
}
}
impl<T: Hash + ?Sized> util::NewCapacity for HashRefSet<T>
{
fn new() -> Self
{
Self::new()
}
fn with_capacity(cap: usize) -> Self
{
Self::with_capacity(cap)
}
}
impl<T: Hash> HashRefSet<T>
{
pub fn insert<U>(&mut self, value: &U) -> bool
where U: Borrow<T>
{
self.0.insert(compute(value.borrow()))
}
pub fn remove<U>(&mut self, value: &U) -> bool
where U: Borrow<T>
{
self.0.remove(&compute(value.borrow()))
}
pub fn contains<U>(&self, value: &U) -> bool
where U: Borrow<T>
{
self.0.contains(&compute(value.borrow()))
}
pub fn len(&self) -> usize
{
self.0.len()
}
pub fn hashes(&self) -> hash_set::Iter<'_, Sha256Hash>
{
self.0.iter()
}
}

@ -79,6 +79,8 @@ mod save;
mod forwarded_list; mod forwarded_list;
use forwarded_list::XForwardedFor; use forwarded_list::XForwardedFor;
mod handle; mod handle;
mod dedup;
mod container;
mod feed; mod feed;
mod gen; mod gen;

Loading…
Cancel
Save