fixed stupid bug

Cargo.lock

@@ -466,7 +466,7 @@ checksum = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84"
 
 [[package]]
 name = "rmdupe"
-version = "0.1.0"
+version = "1.0.0"
 dependencies = [
  "chrono",
  "futures",

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "rmdupe"
-version = "0.1.0"
+version = "1.0.0"
 authors = ["Avril <flanchan@cumallover.me>"]
 edition = "2018"
 

src/bytes.rs

@@ -12,3 +12,20 @@ T: Clone
     }
     sz
 }
+
+#[inline]
+pub fn reinterpret<T>(src: &T) -> &[u8]
+    where T: ?Sized
+{
+    unsafe {
+	std::slice::from_raw_parts(src as *const T as *const u8, std::mem::size_of_val(src))
+    }
+}
+
+pub fn reinterpret_back<T>(src: &[u8]) -> &T
+    where T: ?Sized + Copy
+{
+    unsafe {
+	&*(&src[0] as *const u8 as *const T)
+    }
+}

src/container.rs

@@ -1,42 +1,83 @@
 use super::*;
 use std::{
-    collections::HashSet,
+    collections::{HashMap, HashSet},
     io::{
 	self,
 	Write,
 	Read,
     },
+    path::{
+	Path,
+	PathBuf
+    },
 };
 
 #[derive(Clone, PartialEq, Eq, Debug)]
-pub struct DupeMap(HashSet<hash::Sha256Hash>);
+pub struct DupeMap
+{
+    iteration: HashSet<hash::Sha256Hash>, // What we calculate
+    table: HashMap<PathBuf, hash::Sha256Hash>, // What we save and load
+}
+
+/// Do we care about windows? nah
+#[inline]
+fn path_bytes(path: &Path) -> &[u8]
+{
+    use std::os::unix::ffi::OsStrExt;
+    path.as_os_str().as_bytes()
+}
+
+#[inline]
+fn bytes_path(bytes: &[u8]) -> &Path
+{
+    use std::os::unix::ffi::OsStrExt;
+    std::ffi::OsStr::from_bytes(bytes).as_ref()
+}
+
+const ENTRY_HEADER: &[u8] = &[0x00, 0xde, 0xad];
 
 impl DupeMap
 {
     /// Create a new empty dupe map
     pub fn new() -> Self
     {
-	Self(HashSet::new())
+	Self{iteration: HashSet::new(), table: HashMap::new()}
     }
 
-    /// Iterator over all added keys
+    /// Iterator over all hashes
     pub fn iter(&self) -> std::collections::hash_set::Iter<hash::Sha256Hash>
     {
-	self.0.iter()
+	self.iteration.iter()
     }
 
-    /// Is this hash in the set?
-    pub fn peek(&self, hash: &hash::Sha256Hash) -> bool {
-	self.0.contains(hash)
+    /// Cache this path's hash
+    ///
+    /// # Returns
+    ///
+    /// True if caching was okay, false if key already added.
+    pub fn cache<T: AsRef<Path>>(&mut self, id: T, hash: hash::Sha256Hash) -> bool
+    {
+	if self.table.contains_key(id.as_ref()) {
+	    false
+	} else {
+	    self.table.insert(id.as_ref().to_owned(), hash);
+	    true
+	}
     }
 
-    /// Try to add an entry, returns true if was not a dupe, false if it was.
+    /// Look for path `id` in cache.
+    pub fn get_cache<T: AsRef<Path>>(&self, id: T) -> Option<&hash::Sha256Hash>
+    {
+	self.table.get(id.as_ref())
+    }
+
+    /// Try to add to store. True if adding was oke, false if already exists.
     pub fn try_add(&mut self, hash: hash::Sha256Hash) -> bool
     {
-	if self.0.contains(&hash) {
+	if self.iteration.contains(&hash) {
 	    false
 	} else {
-	    self.0.insert(hash);
+	    self.iteration.insert(hash);
 	    true
 	}
     }
@@ -45,9 +86,15 @@ impl DupeMap
     pub fn save<W: Write>(&self, to: &mut W) -> io::Result<usize>
     {
 	let mut done=0;
-	for x in self.0.iter()
+	for (path, hash) in self.table.iter()
 	{
-	    to.write(x.as_ref())?;
+	    let path = path_bytes(path.as_ref());
+	    let hash: &[u8] = hash.as_ref();
+
+	    to.write(ENTRY_HEADER)?;
+	    to.write(bytes::reinterpret(&path.len()))?;
+	    to.write(path)?;
+	    to.write(hash)?;
 	    done+=1;
 	}
 	Ok(done)
@@ -60,9 +107,15 @@ impl DupeMap
 	use tokio::prelude::*;
 	
 	let mut done=0;
-	for x in self.0.iter()
+	for (path, hash) in self.table.iter()
 	{
-	    to.write(x.as_ref()).await?;
+	    let path = path_bytes(path.as_ref());
+	    let hash: &[u8] = hash.as_ref();
+
+	    to.write(ENTRY_HEADER).await?;
+	    to.write(bytes::reinterpret(&path.len())).await?;
+	    to.write(path).await?;
+	    to.write(hash).await?;
 	    done+=1;
 	}
 	Ok(done)
@@ -73,15 +126,26 @@ impl DupeMap
     {
 	let mut done=0;
 	let mut read;
-	let mut buffer = [0u8; hash::SHA256_SIZE];
+	let mut header_buffer = [0u8; ENTRY_HEADER.len() + std::mem::size_of::<usize>()];
+	let mut hash_buffer = [0u8; hash::SHA256_SIZE];
 	
-	while {read = from.read(&mut buffer[..])?; read==hash::SHA256_SIZE} {
-	    done += if self.try_add(hash::Sha256Hash::new(buffer)) {
-		1
-	    } else {
-		0
-	    };
+	while {read = from.read(&mut header_buffer[..])?; read == header_buffer.len() && &header_buffer[..ENTRY_HEADER.len()] == ENTRY_HEADER}
+	{
+	    let sz = *bytes::reinterpret_back(&header_buffer[ENTRY_HEADER.len()..]);
+	    if sz > 0 {
+		let mut path = vec![0u8; sz];
+		if from.read(&mut path[..])? == sz {
+		    let path = bytes_path(&path[..]);
+		    if from.read(&mut hash_buffer[..])? == hash::SHA256_SIZE
+		    {
+			if self.cache(path, hash::Sha256Hash::new(hash_buffer)) {
+			    done +=1;
 			}
+		    }
+		}
+	    }
+	}
+	
 	Ok(done)
     }
 
@@ -94,15 +158,26 @@ impl DupeMap
 	
 	let mut done=0;
 	let mut read;
-	let mut buffer = [0u8; hash::SHA256_SIZE];
+	let mut header_buffer = [0u8; ENTRY_HEADER.len() + std::mem::size_of::<usize>()];
+	let mut hash_buffer = [0u8; hash::SHA256_SIZE];
 	
-	while {read = from.read(&mut buffer[..]).await?; read==hash::SHA256_SIZE} {
-	    done += if self.try_add(hash::Sha256Hash::new(buffer)) {
-		1
-	    } else {
-		0
-	    };
+	while {read = from.read(&mut header_buffer[..]).await?; read == header_buffer.len() && &header_buffer[..ENTRY_HEADER.len()] == ENTRY_HEADER}
+	{
+	    let sz = *bytes::reinterpret_back(&header_buffer[ENTRY_HEADER.len()..]);
+	    if sz > 0 {
+		let mut path = vec![0u8; sz];
+		if from.read(&mut path[..]).await? == sz {
+		    let path = bytes_path(&path[..]);
+		    if from.read(&mut hash_buffer[..]).await? == hash::SHA256_SIZE
+		    {
+			if self.cache(path, hash::Sha256Hash::new(hash_buffer)) {
+			    done +=1;
 			}
+		    }
+		}
+	    }
+	}
+	
 	Ok(done)
     }
 }

src/proc.rs

@@ -96,40 +96,55 @@ impl Default for DupeCount
 }
 
 /// Process a file and add it to the table, returns true if is not a dupe.
-pub fn process_file<P: AsRef<Path>>(file: P, set: &mut container::DupeMap) -> Result<bool, error::Error>
+pub fn process_file<P: AsRef<Path>>(path: P, set: &mut container::DupeMap) -> Result<bool, error::Error>
 {
+    let path = path.as_ref();
+    if let Some(&hash) = set.get_cache(path) {
+	Ok(set.try_add(hash))
+    } else {
 	let mut file = OpenOptions::new()
 	    .read(true)
-	.open(file)?;
+	    .open(path)?;
 	let sz: usize = file.metadata()?.len().try_into().or(Err(error::Error::Arch(Some("Filesize is too large to be known. you have likely compiled the binary for 32-bit architecture or less. This shouldn't happen on 64-bit systems."))))?;
 	
 	let mut result = hash::Sha256Hash::default();
 	error::check_size(sz, hash::compute(&mut file, &mut result)?)?;
-
+	set.cache(path, result);
 	Ok(set.try_add(result))
     }
+}
 
 
 /// Process a file and add it to the table, returns true if is not a dupe.
 #[cfg(feature="threads")]
-pub async fn process_file_async<P: AsRef<Path>>(file: P, set: &std::sync::Arc<tokio::sync::Mutex<container::DupeMap>>) -> Result<bool, error::Error>
+pub async fn process_file_async<P: AsRef<Path>>(path: P, set: &std::sync::Arc<tokio::sync::Mutex<container::DupeMap>>) -> Result<bool, error::Error>
 {
     use tokio::{
 	fs::{
 	    OpenOptions,
 	},
     };
+    let path = path.as_ref();
+    if let Some(hash) = {
+	let set = set.lock().await;
+	set.get_cache(path).and_then(|&h| Some(h))
+    } {
+	let mut set = set.lock().await;
+	Ok(set.try_add(hash))
+    } else {
 	let mut file = OpenOptions::new()
 	    .read(true)
-	.open(file).await?;
+	    .open(path).await?;
 	let sz: usize = file.metadata().await?.len().try_into().or(Err(error::Error::Arch(Some("Filesize is too large to be known. you have likely compiled the binary for 32-bit architecture or less. This shouldn't happen on 64-bit systems."))))?;
 	
 	let mut result = hash::Sha256Hash::default();
 	error::check_size(sz, hash::compute_async(&mut file, &mut result).await?)?;
 
 	let mut set = set.lock().await;
+	set.cache(path, result);
 	Ok(set.try_add(result))
     }
+}
 
 /// Walk a dir structure and remove all dupes in it
 pub fn do_dir<P: AsRef<Path>>(dir: P, depth: usize, set: &mut container::DupeMap, mode: &config::Mode) -> Result<DupeCount, error::Error>