merge from master

ddc contain
start dedup and container
6 changed files with 301 additions and 1 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@ -109,6 +109,12 @@ dependencies = [
 "safemem",
 ]

+[[package]]
+name = "build_const"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "39092a32794787acd8525ee150305ff051b0aa6cc2abaf193924f5ab05425f39"
+
 [[package]]
 name = "byte-tools"
 version = "0.3.1"
@ -191,6 +197,44 @@ version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8aebca1129a03dc6dc2b127edd729435bbc4a37e1d5f4d7513165089ceb02634"

+[[package]]
+name = "crc"
+version = "1.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d663548de7f5cca343f1e0a48d14dcfb0e9eb4e079ec58883b7251539fa10aeb"
+dependencies = [
+ "build_const",
+]
+
+[[package]]
+name = "crypto-mac"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "58bcd97a54c7ca5ce2f6eb16f6bede5b0ab5f0055fedc17d2f0b4466e21671ca"
+dependencies = [
+ "generic-array 0.14.4",
+ "subtle",
+]
+
+[[package]]
+name = "cryptohelpers"
+version = "1.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2cfc491baaffd7cbd6acc02ebd23564760d83a2c17e1a47e6a04a8d5a86e7fb5"
+dependencies = [
+ "crc",
+ "getrandom",
+ "hex-literal",
+ "hmac",
+ "libc",
+ "openssl",
+ "pbkdf2",
+ "serde",
+ "serde_derive",
+ "sha2",
+ "tokio",
+]
+
 [[package]]
 name = "digest"
 version = "0.8.1"
@ -252,6 +296,21 @@ version = "1.0.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"

+[[package]]
+name = "foreign-types"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
+dependencies = [
+ "foreign-types-shared",
+]
+
+[[package]]
+name = "foreign-types-shared"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
+
 [[package]]
 name = "fuchsia-cprng"
 version = "0.1.1"
@ -473,6 +532,22 @@ dependencies = [
 "libc",
 ]

+[[package]]
+name = "hex-literal"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5af1f635ef1bc545d78392b136bfe1c9809e029023c84a3638a864a10b8819c8"
+
+[[package]]
+name = "hmac"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "deae6d9dbb35ec2c502d62b8f7b1c000a0822c3b0794ba36b3149c0a1c840dff"
+dependencies = [
+ "crypto-mac",
+ "digest 0.9.0",
+]
+
 [[package]]
 name = "http"
 version = "0.2.1"
@ -637,7 +712,11 @@ dependencies = [
 "async-compression",
 "bzip2-sys",
 "cfg-if 1.0.0",
+<<<<<<< HEAD
+ "cryptohelpers",
+=======
 "cidr",
+>>>>>>> master
 "futures",
 "hyper",
 "lazy_static",
@ -649,6 +728,7 @@ dependencies = [
 "rustc_version",
 "serde",
 "serde_cbor",
+ "sha2",
 "smallmap",
 "tokio",
 "toml",
@ -819,6 +899,48 @@ version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"

+[[package]]
+name = "openssl"
+version = "0.10.30"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8d575eff3665419f9b83678ff2815858ad9d11567e082f5ac1814baba4e2bcb4"
+dependencies = [
+ "bitflags",
+ "cfg-if 0.1.10",
+ "foreign-types",
+ "lazy_static",
+ "libc",
+ "openssl-sys",
+]
+
+[[package]]
+name = "openssl-sys"
+version = "0.9.58"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a842db4709b604f0fe5d1170ae3565899be2ad3d9cbc72dedc789ac0511f78de"
+dependencies = [
+ "autocfg 1.0.1",
+ "cc",
+ "libc",
+ "pkg-config",
+ "vcpkg",
+]
+
+[[package]]
+name = "pbkdf2"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7170d73bf11f39b4ce1809aabc95bf5c33564cdc16fc3200ddda17a5f6e5e48b"
+dependencies = [
+ "base64",
+ "crypto-mac",
+ "hmac",
+ "rand 0.7.3",
+ "rand_core 0.5.1",
+ "sha2",
+ "subtle",
+]
+
 [[package]]
 name = "percent-encoding"
 version = "2.1.0"
@ -1246,6 +1368,19 @@ dependencies = [
 "opaque-debug 0.3.0",
 ]

+[[package]]
+name = "sha2"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2933378ddfeda7ea26f48c555bdad8bb446bf8a3d17832dc83e380d444cfb8c1"
+dependencies = [
+ "block-buffer 0.9.0",
+ "cfg-if 0.1.10",
+ "cpuid-bool",
+ "digest 0.9.0",
+ "opaque-debug 0.3.0",
+]
+
 [[package]]
 name = "signal-hook-registry"
 version = "1.2.1"
@ -1283,6 +1418,12 @@ dependencies = [
 "winapi 0.3.9",
 ]

+[[package]]
+name = "subtle"
+version = "2.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "343f3f510c2915908f155e94f17220b19ccfacf2a64a2a5d8004f2c3e311e7fd"
+
 [[package]]
 name = "syn"
 version = "1.0.42"
@ -1553,6 +1694,12 @@ version = "0.7.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "05e42f7c18b8f902290b009cde6d651262f956c98bc51bca4cd1d511c9cd85c7"

+[[package]]
+name = "vcpkg"
+version = "0.2.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6454029bf181f092ad1b853286f23e2c507d8e8194d01d92da4a55c274a5508c"
+
 [[package]]
 name = "version_check"
 version = "0.9.2"
--- a/Cargo.toml
+++ b/Cargo.toml
@ -68,6 +68,8 @@ smallmap = "1.1.5"
 lazy_static = "1.4.0"
 once_cell = "1.4.1"
 bzip2-sys = {version = "0.1.9", optional = true}
+cryptohelpers = {version = "1.5.1", features= ["sha256", "async"]}
+sha2 = "0.9.1"
 cidr = {version = "0.1.1", features = ["serde"]}

 [build-dependencies]
--- a/src/container.rs
+++ b/src/container.rs
@ -0,0 +1,55 @@
+//! Container for chain
+use super::*;
+use chain::{Chain, Chainable};
+use dedup::HashRefSet;
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct ChainContainer<T: Chainable>
+{
+    chain: Chain<T>,
+    inputs_exact: HashRefSet<[T]>,
+}
+
+impl<T: Chainable> ChainContainer<T>
+{
+    pub fn new() -> Self
+    {
+	Self {
+	    chain: Chain::new(),
+	    inputs_exact: HashRefSet::new(),
+	}
+    }
+
+    pub fn force_feed<U: AsRef<[T]>>(&mut self, tokens: U)
+    {
+	self.inputs_exact.insert(&tokens.as_ref());
+	self.chain.feed(tokens);
+    }
+    pub fn feed<U: AsRef<[T]>>(&mut self, tokens: U) -> bool
+    {
+	let tokens=tokens.as_ref();
+	if self.inputs_exact.insert(&tokens) {
+	    self.chain.feed(tokens);
+	    true
+	} else {
+	    trace!("Exact duplication input disallowed");
+	    false
+	}
+    }
+
+    pub fn chain(&self) -> &Chain<T>
+    {
+	&self.chain
+    }
+}
+
+impl<T: Chainable> From<Chain<T>> for ChainContainer<T>
+{
+    fn from(chain: Chain<T>) -> Self
+    {
+	Self {
+	    chain,
+	    inputs_exact: HashRefSet::new(),
+	}
+    }
+}
--- a/src/dedup.rs
+++ b/src/dedup.rs
@ -0,0 +1,94 @@
+//! De-duplicating inputs
+use super::*;
+use std::{
+    hash::{Hash, Hasher},
+    collections::{HashSet, hash_set},
+    marker::PhantomData,
+    borrow::Borrow,
+};
+use sha2::{Sha256, Digest};
+use cryptohelpers::sha256::Sha256Hash;
+
+fn compute<T: Hash + ?Sized>(thing: &T) -> Sha256Hash
+{
+    use std::mem::size_of;
+    struct Sha256Hasher(Sha256);
+    impl Hasher for Sha256Hasher
+    {
+	fn write(&mut self, bytes: &[u8])
+	{
+	    self.0.update(bytes);
+	}
+	fn finish(&self) -> u64
+	{
+	    let ar = self.0.clone().finalize();
+	    let mut rest = [0u8; size_of::<u64>()];
+	    bytes::copy_slice(&mut rest[..], &ar[..]);
+	    u64::from_le_bytes(rest)
+	}
+    }
+
+    let mut hasher = Sha256Hasher(Sha256::new());
+    thing.hash(&mut hasher);
+    hasher.0.into()
+}
+
+/// A HashSet that doesn't own its items.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct HashRefSet<T: Hash + ?Sized>(HashSet<Sha256Hash>, PhantomData<HashSet<*const T>>);
+
+unsafe impl<T: Send + Hash + ?Sized> Send for HashRefSet<T>{}
+unsafe impl<T: Send + Sync + Hash + ?Sized> Sync for HashRefSet<T>{}
+
+impl<T: Hash + ?Sized> HashRefSet<T>
+{
+    pub fn new() -> Self
+    {
+	Self(HashSet::new(), PhantomData)
+    }
+    pub fn with_capacity(cap: usize) -> Self
+    {
+	Self(HashSet::with_capacity(cap), PhantomData)
+    }
+}
+
+impl<T: Hash + ?Sized> util::NewCapacity for HashRefSet<T>
+{
+    fn new() -> Self
+    {
+	Self::new()
+    }
+    
+    fn with_capacity(cap: usize) -> Self
+    {
+	Self::with_capacity(cap)
+    }
+}
+
+impl<T: Hash + ?Sized> HashRefSet<T>
+{
+    pub fn insert<U>(&mut self, value: &U) -> bool
+    where U: Borrow<T>
+    {
+	self.0.insert(compute(value.borrow()))
+    }
+    pub fn remove<U>(&mut self, value: &U) -> bool
+	where U: Borrow<T>
+    {
+	self.0.remove(&compute(value.borrow()))
+    }
+    pub fn contains<U>(&self, value: &U) -> bool
+    where U: Borrow<T>
+    {
+	self.0.contains(&compute(value.borrow()))
+    }
+    pub fn len(&self) -> usize
+    {
+	self.0.len()
+    }
+    pub fn hashes(&self) -> hash_set::Iter<'_, Sha256Hash>
+    {
+	self.0.iter()
+    }
+}
+
--- a/src/handle.rs
+++ b/src/handle.rs
@ -245,7 +245,7 @@ pub async fn host(from: ChainHandle<String>)
    trace!("Begin polling on child");
    tokio::select!{
 	v = &mut child => {
-	    match v  {
+	    match v {
 		#[cold] Ok(_) => {warn!("Child exited before we have? This should probably never happen.")},//Should never happen.
 		Err(e) => {error!("Child exited abnormally. Aborting: {}", e)}, //Child panic or cancel.
 	    }
--- a/src/main.rs
+++ b/src/main.rs
@ -80,6 +80,8 @@ mod ipfilt;
 mod forwarded_list;
 use forwarded_list::XForwardedFor;
 mod handle;
+mod dedup;
+mod container;

 mod feed;
 mod gen;
Author	SHA1	Message	Date
Avril	29cf9660d0	merge from master	4 years ago
Avril	6aa033ebbb	ddc contain	4 years ago
Avril	9a7234806f	start dedup and container	4 years ago