added paranoid-dedup

Cargo.lock

@@ -61,6 +61,15 @@ version = "1.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
 
+[[package]]
+name = "block-buffer"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4"
+dependencies = [
+ "generic-array",
+]
+
 [[package]]
 name = "bytes"
 version = "0.5.6"
@@ -86,6 +95,21 @@ dependencies = [
  "owo-colors",
 ]
 
+[[package]]
+name = "cpuid-bool"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8aebca1129a03dc6dc2b127edd729435bbc4a37e1d5f4d7513165089ceb02634"
+
+[[package]]
+name = "digest"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066"
+dependencies = [
+ "generic-array",
+]
+
 [[package]]
 name = "env_logger"
 version = "0.7.1"
@@ -226,6 +250,16 @@ dependencies = [
  "slab",
 ]
 
+[[package]]
+name = "generic-array"
+version = "0.14.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "501466ecc8a30d1d3b7fc9229b122b2ce8ed6e9d9223f1138d4babb253e51817"
+dependencies = [
+ "typenum",
+ "version_check",
+]
+
 [[package]]
 name = "getrandom"
 version = "0.1.15"
@@ -387,6 +421,12 @@ version = "1.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "260e51e7efe62b592207e9e13a68e43692a7a279171d6ba57abd208bf23645ad"
 
+[[package]]
+name = "opaque-debug"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
+
 [[package]]
 name = "owo-colors"
 version = "1.1.3"
@@ -590,10 +630,24 @@ dependencies = [
  "pretty_env_logger",
  "recolored",
  "rustc_version",
+ "sha2",
  "tokio",
  "uuid",
 ]
 
+[[package]]
+name = "sha2"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2933378ddfeda7ea26f48c555bdad8bb446bf8a3d17832dc83e380d444cfb8c1"
+dependencies = [
+ "block-buffer",
+ "cfg-if",
+ "cpuid-bool",
+ "digest",
+ "opaque-debug",
+]
+
 [[package]]
 name = "slab"
 version = "0.4.2"
@@ -657,6 +711,12 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "typenum"
+version = "1.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "373c8a200f9e67a0c95e62a4f52fbf80c23b4381c05a17845531982fa99e6b33"
+
 [[package]]
 name = "unicode-xid"
 version = "0.2.1"
@@ -672,6 +732,12 @@ dependencies = [
  "rand",
 ]
 
+[[package]]
+name = "version_check"
+version = "0.9.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed"
+
 [[package]]
 name = "wasi"
 version = "0.9.0+wasi-snapshot-preview1"

Cargo.toml

@@ -7,8 +7,6 @@ edition = "2018"
 readme = "README.org"
 license = "GPL-3.0-or-later"
 
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
 [features]
 default = ["parallel", "limit-concurrency", "splash", "limit-recursion"]
 
@@ -16,14 +14,16 @@ default = ["parallel", "limit-concurrency", "splash", "limit-recursion"]
 limit-concurrency = ["parallel"]
 # Handle directories recursively
 recursive = []
+# Limit the max recursion depth to 256
 limit-recursion = ["recursive"]
+# Show splash message when ran with no arguments
 splash = []
+# Run operations in parallel
 parallel = ["tokio", "futures", "pin-project"]
+# Run on a thread pool
 threads = ["parallel", "tokio/rt-threaded"]
-
+# Use SHA256 hash algorithm for argument file deduping instead of default hasher.
-# use PRETTY_ENV_LOGGER I guess
+paranoid-dedup = ["sha2"]
-# On trace: Display no. of hardlinks for each file
-# On debug: Display reason why why metadata() failes, instead of just "warn: stat failed"
 
 [dependencies]
 cfg-if = "0.1.10"
@@ -36,6 +36,7 @@ lazy_static = "1.4.0"
 uuid = {version = "0.8.1", features = ["v4"]}
 recolored = "1.9.3"
 pin-project = {version = "0.4.26", optional=true}
+sha2 = {version = "0.9.1", optional=true}
 
 [build-dependencies]
 rustc_version = "0.2"

src/map.rs

@@ -6,13 +6,57 @@ use std::{
     hash::Hash,
 };
 
-//TODO: Feature flag for SHA256 hashing
+cfg_if!{
-pub type HashOutput = u64;
+    if #[cfg(feature="paranoid-dedup")] {
+	pub const HASH_SIZE: usize  = 256;
+	pub type HashOutput = [u8; HASH_SIZE];
+    } else {
+	pub const HASH_SIZE: usize = 8;
+	pub type HashOutput = u64;
+    }
 
+}
 pub fn compute<H: Hash>(what: &H) -> HashOutput
 {
     use std::hash::Hasher;
-    let mut hasher = std::collections::hash_map::DefaultHasher::new();
+
+    let mut hasher = {
+	cfg_if!{
+            if #[cfg(feature="paranoid-dedup")] {
+		use sha2::{Sha256, Digest,};
+		struct Sha256Hasher(Sha256);
+		impl Hasher for Sha256Hasher
+		{
+		    fn write(&mut self, bytes: &[u8])
+		    {
+			self.0.update(bytes);
+		    }
+		    fn finish(&self) -> u64
+		    {
+			unimplemented!("This shouldn't really be called tbh")
+		    }
+		}
+
+		impl Sha256Hasher
+		{
+		    fn finish(self) -> HashOutput
+		    {
+			let mut output = [0u8; HASH_SIZE];
+			let finish = self.0.finalize();
+			for (d, s) in output.iter_mut().zip(finish.into_iter())
+			{
+			    *d = s;
+			}
+			output
+		    }
+		}
+
+		Sha256Hasher(Sha256::new())
+	    } else {
+		std::collections::hash_map::DefaultHasher::new()
+	    }
+	}
+    };
     what.hash(&mut hasher);
     hasher.finish()
 }

src/splash.rs

@@ -61,6 +61,7 @@ Made by {} with <3 (Licensed GPL 3.0 or later)"#, arg::program_name(), env!("CAR
     feature!(off "threads", "\tUsing thread-pool");
     feature!(on "recursive", "\tRecursivly process files up to {} directories deep", recurse::MAX_DEPTH);
     feature!(on "limit-recursion", "Concurrency is capped");
+    feature!(off "paranoid-dedup", "Use SHA256 for argument dedup instead of basic hashing");
     
     std::process::exit(1)
 }