master
Avril 4 years ago
parent e77e8c4290
commit a9e9e48633
Signed by: flanchan
GPG Key ID: 284488987C31F630

1
.gitignore vendored

@ -1,3 +1,4 @@
/target /target
*~ *~
test-inpu* test-inpu*
.rmdupe

@ -39,6 +39,8 @@ pub fn usage() -> !
println!(" --\t\t\tStop reading args"); println!(" --\t\t\tStop reading args");
println!("Other:"); println!("Other:");
println!(" --help -h:\t\tPrint this message"); println!(" --help -h:\t\tPrint this message");
#[cfg(feature="threads")]
println!("Compiled with threading support");
std::process::exit(1) std::process::exit(1)
} }

@ -1,4 +1,3 @@
use super::*;
pub fn copy_slice<T,S,D>(mut dst: D, src: S) -> usize pub fn copy_slice<T,S,D>(mut dst: D, src: S) -> usize
where S: AsRef<[T]>, where S: AsRef<[T]>,

@ -1,7 +1,4 @@
use super::*; use super::*;
use std::{
};
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub enum RecursionMode pub enum RecursionMode

@ -8,7 +8,7 @@ use std::{
}, },
}; };
#[derive(Clone, PartialEq, Eq)] #[derive(Clone, PartialEq, Eq, Debug)]
pub struct DupeMap(HashSet<hash::Sha256Hash>); pub struct DupeMap(HashSet<hash::Sha256Hash>);
impl DupeMap impl DupeMap

@ -1,7 +1,7 @@
use super::*; use super::*;
use std::{ use std::{
hash::Hash, hash::Hash,
collections::{HashSet, HashMap}, collections::HashMap,
mem, mem,
fmt, fmt,
}; };
@ -40,23 +40,25 @@ where T: Hash + Default + Eq
} }
pub trait ErrorLogForgetExt<T, E>: Sized pub trait ErrorLogForgetExt<T, E>: Sized
{ {
fn log_and_forget(self, mode: &log::Mode, level: log::Level) -> Option<T>; fn log_and_forget(self, mode: &log::Mode, level: log::Level) -> Result<Option<T>, E>;
} }
impl<T,E> ErrorLogForgetExt<T,E> for Result<T,E> impl<T,E> ErrorLogForgetExt<T,E> for Result<T,E>
where E: fmt::Display, where E: fmt::Display,
{ {
fn log_and_forget(self, mode: &log::Mode, level: log::Level) -> Option<T> fn log_and_forget(self, mode: &log::Mode, level: log::Level) -> Result<Option<T>, E>
{ {
match self { Ok(match self {
Err(e) => { Err(e) => {
log!(mode.level(level), "{}", &e); log!(mode.level(level), "{}", &e);
if let log::Level::Fatal = level { if let log::Level::Fatal = level {
std::process::exit(1); std::process::exit(1);
} else if log::Level::Error == level {
return Err(e);
} }
None None
}, },
Ok(v) => Some(v), Ok(v) => Some(v),
} })
} }
} }
pub trait ErrorLogExt: Sized pub trait ErrorLogExt: Sized

@ -1,4 +1,3 @@
use super::*;
use std::{ use std::{
fmt, fmt,
}; };
@ -94,12 +93,23 @@ impl fmt::Display for Level
Level::Warning => write!(f, "WARN"), Level::Warning => write!(f, "WARN"),
Level::Error => write!(f, "ERROR"), Level::Error => write!(f, "ERROR"),
Level::Fatal => write!(f, "FATAL"), Level::Fatal => write!(f, "FATAL"),
#[allow(unreachable_patterns)]
_ => write!(f, "(unbound)"), _ => write!(f, "(unbound)"),
} }
} }
} }
macro_rules! log { macro_rules! log {
($level:tt, $mode:expr => $format:expr) => {
{
if let Some(level) = $mode.level($crate::log::Level::$level) {
println!("{} [{}]: {}", $crate::log::timestamp(), level, $format);
true
} else {
false
};
}
};
($level:tt, $mode:expr => $format:expr, $($rest:expr),*) => { ($level:tt, $mode:expr => $format:expr, $($rest:expr),*) => {
{ {
if let Some(level) = $mode.level($crate::log::Level::$level) { if let Some(level) = $mode.level($crate::log::Level::$level) {

@ -88,10 +88,13 @@ async fn main() -> Result<(), Box<dyn std::error::Error>>
fs::{ fs::{
OpenOptions, OpenOptions,
}, },
sync, sync::{
Mutex
},
}; };
use std::{ use std::{
path::Path, path::Path,
sync::Arc,
}; };
let args = parse_args().into_string()?; let args = parse_args().into_string()?;
let lmode = &args.mode.logging_mode; let lmode = &args.mode.logging_mode;
@ -110,9 +113,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error>>
if load.is_file() { if load.is_file() {
if let Some(mut file) = OpenOptions::new() if let Some(mut file) = OpenOptions::new()
.read(true) .read(true)
.open(load).await.log_and_forget(lmode, log::Level::Warning) .open(load).await.log_and_forget(lmode, log::Level::Warning)?
{ {
log!(Info, lmode => "Hashes loading from {:?}", load);
args.mode.error_mode.handle(hashes.load_async(&mut file).await).log_and_forget(lmode, log::Level::Warning)?;
} }
} else { } else {
log!(Warning, lmode => "Exclusing directory from load path {:?}", load); log!(Warning, lmode => "Exclusing directory from load path {:?}", load);
@ -122,24 +126,122 @@ async fn main() -> Result<(), Box<dyn std::error::Error>>
} }
} }
log!(Debug, lmode => "Loaded hashes: {:?}", hashes);
log!(Info, lmode => "Starting checks (threaded)");
let hashes = Arc::new(Mutex::new(hashes));
for path in args.paths.iter() for path in args.paths.iter()
{ {
let path = Path::new(path); let path = Path::new(path);
if path.is_dir() { if path.is_dir() {
log!(Debug, lmode => "Spawning for {:?}", path);
let mode = args.mode.clone();
let path = path.to_owned();
let hashes= Arc::clone(&hashes);
children.push(tokio::task::spawn(async move { children.push(tokio::task::spawn(async move {
//proc::do_dir_async() log!(Debug, mode.logging_mode => " + {:?}", path);
let res = mode.error_mode.handle(proc::do_dir_async(path.clone(), 0, hashes, mode.clone()).await).log_and_forget(&mode.logging_mode, log::Level::Error);
log!(Info, mode.logging_mode => " - {:?}", path);
res
})); }));
} }
} }
log!(Info, lmode => "Waiting on children");
let mut done = proc::DupeCount::default();
for child in children.into_iter()
{
done += args.mode.error_mode.handle(child.await?)?.unwrap_or_default().unwrap_or_default().unwrap_or_default();
}
log!(Info, lmode => "Found: {:?}", done);
let hashes = hashes.lock().await;
log!(Debug, lmode => "New hashes: {:?}", hashes);
for save in args.save.iter()
{
let save = Path::new(save);
log!(Info, lmode => "Saving hashes to {:?}", save);
if let Some(mut file) = OpenOptions::new()
.create(true)
//.append(true)
.truncate(true)
.write(true)
.open(save).await.log_and_forget(lmode, log::Level::Warning)?
{
args.mode.error_mode.handle(hashes.save_async(&mut file).await).log_and_forget(lmode, log::Level::Warning)?;
}
}
Ok(()) Ok(())
} }
#[cfg(not(feature="threads"))] #[cfg(not(feature="threads"))]
fn main() -> Result<(), error::Error> fn main() -> Result<(), Box<dyn std::error::Error>>
{ {
let args = parse_args()?; use std::{
path::Path,
fs::{
OpenOptions,
},
};
let args = parse_args().into_string()?;
let lmode = &args.mode.logging_mode;
log!(Debug, lmode => "Args parsed: {:?}", args);
let mut hashes = container::DupeMap::new();
// Load hashes
for load in args.load.iter()
{
let load = Path::new(load);
if load.exists() {
if load.is_file() {
if let Some(mut file) = OpenOptions::new()
.read(true)
.open(load).log_and_forget(lmode, log::Level::Warning)?
{
log!(Info, lmode => "Hashes loading from {:?}", load);
args.mode.error_mode.handle(hashes.load(&mut file)).log_and_forget(lmode, log::Level::Warning)?;
}
} else {
log!(Warning, lmode => "Exclusing directory from load path {:?}", load);
}
} else {
log!(Info, lmode => "Ignoring non-existant load path {:?}", load);
}
}
log!(Debug, lmode => "Loaded hashes: {:?}", hashes);
log!(Info, lmode => "Starting checks (threaded)");
let mut done = proc::DupeCount::default();
for path in args.paths.iter()
{
let path = Path::new(path);
if path.is_dir() {
log!(Debug, lmode => " + {:?}", path);
done += args.mode.error_mode.handle(proc::do_dir(path.clone(), 0, &mut hashes, &args.mode)).log_and_forget(lmode, log::Level::Error)?.unwrap_or_default().unwrap_or_default();
log!(Info, lmode => " - {:?}", path);
}
}
log!(Info, lmode => "Found: {:?}", done);
log!(Debug, lmode => "New hashes: {:?}", hashes);
for save in args.save.iter()
{
let save = Path::new(save);
log!(Info, lmode => "Saving hashes to {:?}", save);
if let Some(mut file) = OpenOptions::new()
.create(true)
//.append(true)
.truncate(true)
.write(true)
.open(save).log_and_forget(lmode, log::Level::Warning)?
{
args.mode.error_mode.handle(hashes.save(&mut file)).log_and_forget(lmode, log::Level::Warning)?;
}
}
log!(Fatal, log::Mode::Error => "{:?}", args);
Ok(()) Ok(())
} }

@ -1,8 +1,5 @@
use super::*; use super::*;
use std::{ use std::{
io::{
self, Read,
},
path::{ path::{
Path Path
}, },
@ -19,20 +16,35 @@ use std::{
}; };
/// Handle a detected dupe /// Handle a detected dupe
fn handle_dupe<P>(path: P, _mode: &config::Mode) -> Result<(), error::Error> fn handle_dupe<P>(path: P, mode: &config::Mode) -> Result<(), error::Error>
where P: AsRef<Path> where P: AsRef<Path>
{ {
println!(" -> {:?}", path.as_ref()); log!(Info, mode.logging_mode => " -> {:?}", path.as_ref());
match mode.operation_mode
{
config::OperationMode::Delete => {
mode.error_mode.handle(std::fs::remove_file(path.as_ref()))?;
},
_ => (),
}
Ok(()) Ok(())
} }
/// Handle a detected dupe async /// Handle a detected dupe async
#[inline(always)] #[inline(always)]
#[cfg(feature="threads")] #[cfg(feature="threads")]
async fn handle_dupe_async<P>(path: P, mode: &config::Mode) -> Result<(), error::Error> async fn handle_dupe_async<P>(path: P, mode: &config::Mode) -> Result<(), error::Error>
where P: AsRef<Path> where P: AsRef<Path>
{ {
handle_dupe(path, mode) log!(Info, mode.logging_mode => " -> {:?}", path.as_ref());
match mode.operation_mode
{
config::OperationMode::Delete => {
mode.error_mode.handle(tokio::fs::remove_file(path.as_ref()).await)?;
},
_ => (),
}
Ok(())
} }
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
@ -103,7 +115,6 @@ pub fn process_file<P: AsRef<Path>>(file: P, set: &mut container::DupeMap) -> Re
pub async fn process_file_async<P: AsRef<Path>>(file: P, set: &std::sync::Arc<tokio::sync::Mutex<container::DupeMap>>) -> Result<bool, error::Error> pub async fn process_file_async<P: AsRef<Path>>(file: P, set: &std::sync::Arc<tokio::sync::Mutex<container::DupeMap>>) -> Result<bool, error::Error>
{ {
use tokio::{ use tokio::{
prelude::*,
fs::{ fs::{
OpenOptions, OpenOptions,
}, },
@ -141,6 +152,7 @@ pub fn do_dir<P: AsRef<Path>>(dir: P, depth: usize, set: &mut container::DupeMap
count += mode.handle(do_dir(obj, depth+1, set, cmode))?.unwrap_or_default(); count += mode.handle(do_dir(obj, depth+1, set, cmode))?.unwrap_or_default();
} else { } else {
count += if mode.handle(process_file(&obj, set))?.unwrap_or_default() { count += if mode.handle(process_file(&obj, set))?.unwrap_or_default() {
log!(Info, cmode.logging_mode => "OK {:?}", obj);
DupeCount{total: 1, dupes: 0} DupeCount{total: 1, dupes: 0}
} else { } else {
mode.handle(handle_dupe(obj, &cmode))?; mode.handle(handle_dupe(obj, &cmode))?;
@ -183,6 +195,7 @@ pub fn do_dir_async<P: AsRef<Path> + std::marker::Send + std::marker::Sync + 'st
let cmode = cmode.clone(); let cmode = cmode.clone();
let mode = mode.clone(); let mode = mode.clone();
children.push(tokio::task::spawn(async move { children.push(tokio::task::spawn(async move {
log!(Info, cmode.logging_mode => "OK {:?}", obj);
match mode.handle(do_dir_async(obj, depth+1, set, cmode).await) { match mode.handle(do_dir_async(obj, depth+1, set, cmode).await) {
Ok(v) => Ok(v.unwrap_or_default()), Ok(v) => Ok(v.unwrap_or_default()),
Err(v) => Err(v), Err(v) => Err(v),
@ -196,6 +209,7 @@ pub fn do_dir_async<P: AsRef<Path> + std::marker::Send + std::marker::Sync + 'st
match mode.handle(process_file_async(&obj, &set).await) { match mode.handle(process_file_async(&obj, &set).await) {
Ok(v) => { Ok(v) => {
if v.unwrap_or_default() { if v.unwrap_or_default() {
log!(Info, cmode.logging_mode => "OK {:?}", obj);
Ok(true) Ok(true)
} else { } else {
if let Err(e) = mode.handle(handle_dupe_async(obj, &cmode).await) { if let Err(e) = mode.handle(handle_dupe_async(obj, &cmode).await) {

Loading…
Cancel
Save