logging; improve dedup

master
Avril 4 years ago
parent 05dbc6a354
commit 649e98b9cc
Signed by: flanchan
GPG Key ID: 284488987C31F630

47
Cargo.lock generated

@ -6,6 +6,12 @@ version = "0.4.7"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4d25d88fd6b8041580a654f9d0c581a047baee2b3efee13275f2fc392fc75034" checksum = "4d25d88fd6b8041580a654f9d0c581a047baee2b3efee13275f2fc392fc75034"
[[package]]
name = "autocfg"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d"
[[package]] [[package]]
name = "bitflags" name = "bitflags"
version = "1.2.1" version = "1.2.1"
@ -42,6 +48,17 @@ version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"
[[package]]
name = "chrono"
version = "0.4.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c74d84029116787153e02106bf53e66828452a4b325cc8652b788b5967c0a0b6"
dependencies = [
"num-integer",
"num-traits",
"time",
]
[[package]] [[package]]
name = "cpuid-bool" name = "cpuid-bool"
version = "0.1.0" version = "0.1.0"
@ -338,6 +355,25 @@ dependencies = [
"winapi 0.3.9", "winapi 0.3.9",
] ]
[[package]]
name = "num-integer"
version = "0.1.43"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8d59457e662d541ba17869cf51cf177c0b5f0cbf476c66bdc90bf1edac4f875b"
dependencies = [
"autocfg",
"num-traits",
]
[[package]]
name = "num-traits"
version = "0.2.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac267bcc07f48ee5f8935ab0d24f316fb722d7a1292e2913f0cc196b29ffd611"
dependencies = [
"autocfg",
]
[[package]] [[package]]
name = "num_cpus" name = "num_cpus"
version = "1.13.0" version = "1.13.0"
@ -432,6 +468,7 @@ checksum = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84"
name = "rmdupe" name = "rmdupe"
version = "0.1.0" version = "0.1.0"
dependencies = [ dependencies = [
"chrono",
"futures", "futures",
"lazy_static", "lazy_static",
"sha2", "sha2",
@ -521,6 +558,16 @@ dependencies = [
"unicode-xid", "unicode-xid",
] ]
[[package]]
name = "time"
version = "0.1.43"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438"
dependencies = [
"libc",
"winapi 0.3.9",
]
[[package]] [[package]]
name = "tokio" name = "tokio"
version = "0.2.21" version = "0.2.21"

@ -22,3 +22,4 @@ tokio = { version = "0.2", features = ["full"], optional = true }
sha2 = "0.9" sha2 = "0.9"
futures = { version = "0.3", optional = true } futures = { version = "0.3", optional = true }
lazy_static = "1.4" lazy_static = "1.4"
chrono = "0.4"

@ -24,20 +24,21 @@ pub fn usage() -> !
println!("OPTIONS:"); println!("OPTIONS:");
println!(" --load -l:\t\tLoad the hashes from `load-file` if possible."); println!(" --load -l:\t\tLoad the hashes from `load-file` if possible.");
println!(" --save -s:\t\tSave the hashes to `save-file` if possible."); println!(" --save -s:\t\tSave the hashes to `save-file` if possible.");
println!(" --load-file <filename>\tSpecify file for `--load`"); println!(" --load-file <file>\tSpecify file for `--load`");
println!(" --save-file <filename>\tSpecify file for `--save`"); println!(" --save-file <file>\tSpecify file for `--save`");
println!(" --load-save <filename>\tSpecify file for `--save` and `--load`"); println!(" --load-save <file>\tSpecify file for `--save` and `--load`");
println!(" --delete -d\t\tDelete dupes instead of printing"); println!(" --delete -d\t\tDelete dupes instead of printing");
println!(" --verbose -v\t\tPrint verbose info"); println!(" --verbose -v\t\tPrint verbose info");
println!(" --debug -V\t\tPrint very verbose info");
println!(" --error-mode <mode>\tSpecify error handling mode, one of: [IGNORE|WARN|CANCEL|TERMINATE]. Default is `WARN`"); println!(" --error-mode <mode>\tSpecify error handling mode, one of: [IGNORE|WARN|CANCEL|TERMINATE]. Default is `WARN`");
println!(" --quiet -q\t\tAlias for `--error-mode IGNORE`"); println!(" --quiet -q\t\tAlias for `--error-mode IGNORE`");
println!(" --warn -i\t\tAlias for `--error-mode WARN`"); println!(" --warn -i\t\tAlias for `--error-mode WARN`");
println!(" --cancel -w\t\tAlias for `--error-mode CANCEL`"); println!(" --cancel -w\t\tAlias for `--error-mode CANCEL`");
println!(" --error -W\t\tAlias for `--error-mode TERMINATE`"); println!(" --error -W\t\tAlias for `--error-mode TERMINATE`");
println!(" --recursive <max depth>|inf\t\tRecursive mode."); println!(" --recurse <max>|inf\tRecursive mode, give max depth or infinite.");
println!(" --\t\t\tStop reading args"); println!(" --\t\t\tStop reading args");
println!("Other:"); println!("Other:");
println!(" --help:\t\tPrint this message"); println!(" --help -h:\t\tPrint this message");
std::process::exit(1) std::process::exit(1)
} }
@ -84,13 +85,20 @@ where P: AsRef<Path>
/// Try to parse `std::env::args()` /// Try to parse `std::env::args()`
#[inline] #[inline]
pub fn parse_args() -> Result<config::Config, Error> pub fn parse_args() -> Result<Output, Error>
{ {
parse(std::env::args().skip(1)) parse(std::env::args().skip(1))
} }
#[derive(Debug)]
pub enum Output
{
Normal(config::Config),
Help,
}
/// Try to parse args /// Try to parse args
pub fn parse<I>(args: I) -> Result<config::Config, Error> pub fn parse<I>(args: I) -> Result<Output, Error>
where I: IntoIterator<Item=String> where I: IntoIterator<Item=String>
{ {
let mut args = args.into_iter(); let mut args = args.into_iter();
@ -99,10 +107,10 @@ where I: IntoIterator<Item=String>
let mut paths = Vec::new(); let mut paths = Vec::new();
let mut load = Vec::new(); let mut load = Vec::new();
let mut save = Vec::new(); let mut save = Vec::new();
let mut verbose = false;
let mut delete = false; let mut delete = false;
let mut mode_er = error::Mode::Cancel; let mut mode_er = error::Mode::Cancel;
let mut mode_rec = config::RecursionMode::None; let mut mode_rec = config::RecursionMode::None;
let mut mode_log = log::Mode::Warn;
macro_rules! push { macro_rules! push {
($arg:expr) => { ($arg:expr) => {
@ -123,10 +131,13 @@ where I: IntoIterator<Item=String>
} }
} }
} }
while let Some(arg) = args.next() while let Some(arg) = args.next()
{ {
if reading && arg.chars().next().unwrap_or('\0') == '-' { if reading && arg.chars().next().unwrap_or('\0') == '-' {
match &arg[..] { match &arg[..] {
"--help" => return Ok(Output::Help),
"--" => reading = false, "--" => reading = false,
"--load" => { "--load" => {
@ -146,7 +157,8 @@ where I: IntoIterator<Item=String>
save.push(validate_path(&one, Ensure::Dir, false)?.to_owned()); save.push(validate_path(&one, Ensure::Dir, false)?.to_owned());
}, },
"--verbose" => verbose = true, "--verbose" => mode_log = log::Mode::Verbose,
"--debug" => mode_log = log::Mode::Debug,
"--delete" => delete = true, "--delete" => delete = true,
"--error" => mode_er = error::Mode::Terminate, "--error" => mode_er = error::Mode::Terminate,
@ -160,11 +172,11 @@ where I: IntoIterator<Item=String>
"ignore" => error::Mode::Ignore, "ignore" => error::Mode::Ignore,
_ => return Err(Error::UnknownErrorMode(one)), _ => return Err(Error::UnknownErrorMode(one)),
}, },
"--recursive" if take_one!() => { "--recurse" if take_one!() => {
if one.to_lowercase().trim() == "inf" { if one.to_lowercase().trim() == "inf" {
mode_rec = config::RecursionMode::All; mode_rec = config::RecursionMode::All;
} else if let Ok(sz) = one.parse::<usize>() { } else if let Ok(sz) = one.parse::<usize>() {
mode_rec = config::RecursionMode::N(sz); mode_rec = if sz == 0 {config::RecursionMode::None} else {config::RecursionMode::N(sz)};
} else { } else {
return Err(Error::Parse(one, "number")); return Err(Error::Parse(one, "number"));
} }
@ -177,13 +189,16 @@ where I: IntoIterator<Item=String>
's' => save.push(validate_path(config::DEFAULT_HASHNAME, Ensure::File, false)?.to_owned()), 's' => save.push(validate_path(config::DEFAULT_HASHNAME, Ensure::File, false)?.to_owned()),
'd' => delete = true, 'd' => delete = true,
'v' => verbose = true, 'v' => mode_log = log::Mode::Verbose,
'V' => mode_log = log::Mode::Debug,
'W' => mode_er = error::Mode::Terminate, 'W' => mode_er = error::Mode::Terminate,
'w' => mode_er = error::Mode::Cancel, 'w' => mode_er = error::Mode::Cancel,
'i' => mode_er = error::Mode::Warn, 'i' => mode_er = error::Mode::Warn,
'q' => mode_er = error::Mode::Ignore, 'q' => mode_er = error::Mode::Ignore,
'h' => return Ok(Output::Help),
'r' => mode_rec = config::RecursionMode::All, 'r' => mode_rec = config::RecursionMode::All,
_ => return Err(Error::UnknownArgChar(argchar)), _ => return Err(Error::UnknownArgChar(argchar)),
} }
@ -196,28 +211,32 @@ where I: IntoIterator<Item=String>
} }
} }
// Remove dupes. Load order doesn't really matter // Remove dupes & keeps ordering
load.sort_unstable(); load.dedup_full();
load.dedup(); save.dedup_full();
save.sort_unstable(); paths.dedup_full();
save.dedup();
if paths.len() < 1 {
return Err(Error::NoInput);
}
Ok(config::Config{ Ok(Output::Normal(config::Config{
paths: paths, paths: paths,
mode: config::Mode{ mode: config::Mode{
error_mode: mode_er, error_mode: mode_er,
recursion_mode: mode_rec, recursion_mode: mode_rec,
operation_mode: if delete { config::OperationMode::Delete } else { config::OperationMode::Print }, operation_mode: if delete { config::OperationMode::Delete } else { config::OperationMode::Print },
logging_mode: mode_log,
}, },
save, save,
load, load,
verbose, }))
})
} }
#[derive(Debug)] #[derive(Debug)]
pub enum Error pub enum Error
{ {
NoInput,
Parse(String, &'static str), Parse(String, &'static str),
UnknownArg(String), UnknownArg(String),
UnknownArgChar(char), UnknownArgChar(char),
@ -235,6 +254,7 @@ impl fmt::Display for Error
{ {
write!(f, "failed to parse args: ")?; write!(f, "failed to parse args: ")?;
match self { match self {
Error::NoInput => write!(f, "need at least one input"),
Error::Parse(value, typ) => write!(f, "expected a {}, got `{}'", typ, value), Error::Parse(value, typ) => write!(f, "expected a {}, got `{}'", typ, value),
Error::UnknownArg(arg) => write!(f, "i don't know how to `{}'", arg), Error::UnknownArg(arg) => write!(f, "i don't know how to `{}'", arg),
Error::UnknownArgChar(arg) => write!(f, "i don't know how to `-{}'", arg), Error::UnknownArgChar(arg) => write!(f, "i don't know how to `-{}'", arg),

@ -18,12 +18,14 @@ pub enum OperationMode
Delete, Delete,
} }
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct Mode pub struct Mode
{ {
pub error_mode: error::Mode, pub error_mode: error::Mode,
pub recursion_mode: RecursionMode, pub recursion_mode: RecursionMode,
pub operation_mode: OperationMode, pub operation_mode: OperationMode,
pub logging_mode: log::Mode,
} }
impl Default for Mode impl Default for Mode
@ -34,6 +36,7 @@ impl Default for Mode
error_mode: error::Mode::Cancel, error_mode: error::Mode::Cancel,
recursion_mode: RecursionMode::None, recursion_mode: RecursionMode::None,
operation_mode: OperationMode::Print, operation_mode: OperationMode::Print,
logging_mode: log::Mode::Warn,
} }
} }
} }
@ -51,6 +54,4 @@ pub struct Config
pub save: Vec<String>, pub save: Vec<String>,
/// Load hashes from /// Load hashes from
pub load: Vec<String>, pub load: Vec<String>,
/// Print verbose info
pub verbose: bool,
} }

@ -1,3 +1,4 @@
use super::*;
use std::{ use std::{
error, error,
fmt, fmt,
@ -39,6 +40,7 @@ pub enum Error
Arch(Option<&'static str>), Arch(Option<&'static str>),
IO(io::Error), IO(io::Error),
Size{expected: usize, got: usize}, Size{expected: usize, got: usize},
ArgParse(arg::Error),
Internal(Box<(dyn error::Error + std::marker::Send)>), Internal(Box<(dyn error::Error + std::marker::Send)>),
} }
@ -60,6 +62,7 @@ impl fmt::Display for Error
write!(f, "error: ")?; write!(f, "error: ")?;
match self { match self {
Self::ArgParse(arg) => write!(f, "arg: {}", arg),
Self::Arch(Some(expl)) => write!(f, "bad arch: {}", expl), Self::Arch(Some(expl)) => write!(f, "bad arch: {}", expl),
Self::Arch(_) => write!(f, "bad arch (this is usually an indicator of an improperly compiled binary.)"), Self::Arch(_) => write!(f, "bad arch (this is usually an indicator of an improperly compiled binary.)"),
Self::Size{expected, got} => write!(f, "size mismatch: expected {}, got {}", expected, got), Self::Size{expected, got} => write!(f, "size mismatch: expected {}, got {}", expected, got),
@ -86,6 +89,14 @@ impl From<io::Error> for Error
} }
} }
impl From<arg::Error> for Error
{
fn from(er: arg::Error) -> Self
{
Self::ArgParse(er)
}
}
// Helper functions: // Helper functions:
/// Return error for size if needed /// Return error for size if needed

@ -0,0 +1,68 @@
use super::*;
use std::{
hash::Hash,
collections::{HashSet, HashMap},
mem,
};
pub trait DedupFullExt
{
fn dedup_full(&mut self);
}
impl<T> DedupFullExt for Vec<T>
where T: Hash + Default + Eq
{
fn dedup_full(&mut self)
{
let mut set: HashMap<T, usize> = HashMap::new();
for (i,x) in (0..).zip(self.iter_mut())
{
if !set.contains_key(x) {
let us = mem::replace(x, Default::default());
set.insert(us, i);
}
}
// To preserve order:
let mut tmp = Vec::with_capacity(self.len());
for item in set.into_iter()
{
tmp.push(item);
}
tmp.sort_by(move |a, b| a.1.partial_cmp(&b.1).unwrap());
self.truncate(tmp.len());
for (d,s) in self.iter_mut().zip(tmp.into_iter())
{
*d = s.0;
}
}
}
#[cfg(test)]
mod test
{
use super::*;
#[test]
fn dedup_full()
{
let mut vec = vec![
"hello",
"how",
"are",
"you",
"hello",
"hello",
"today",
"today",
"you",
"how",
"hello",
];
vec.dedup_full();
assert_eq!(vec.len(), 5);
assert_eq!(&vec[..], &["hello","how", "are", "you", "today"]);
}
}

@ -0,0 +1,131 @@
use super::*;
use std::{
fmt,
};
use lazy_static::lazy_static;
/// Logging mode
#[derive(Debug,Clone,PartialEq,Eq,Hash)]
pub enum Mode
{
/// Output nothing
None,
/// Only print errors
Error,
/// Only print warnings
Warn,
/// Verbose message
Verbose,
/// Debug messages
Debug,
}
#[derive(Debug, Clone, Copy, PartialEq,Eq,Hash)]
pub enum Level
{
Debug,
Info,
Warning,
Error,
Fatal,
}
const LEVELS_FOR: &[(Mode, &[Level]); 5] = &[
(Mode::None, &[]),
(Mode::Error, &[Level::Fatal, Level::Error]),
(Mode::Warn, &[Level::Fatal, Level::Error, Level::Warning]),
(Mode::Verbose, &[Level::Fatal, Level::Error, Level::Warning, Level::Info]),
(Mode::Debug, &[Level::Fatal, Level::Error, Level::Warning, Level::Info, Level::Debug]),
];
impl Mode
{
/// Can we print this level?
#[inline]
pub fn level(&self, lv: Level) -> Option<Level>
{
if lv.mode_ok(self) {
Some(lv)
} else {
None
}
}
}
impl Level
{
/// Can we print for `mode`?
pub fn mode_ok(&self, mode: &Mode) -> bool
{
use std::collections::{HashMap, HashSet};
lazy_static! {
static ref MAP: HashMap<&'static Mode, HashSet<&'static Level>> = {
let mut m = HashMap::new();
for (key, values) in LEVELS_FOR.iter() {
let mut set = HashSet::new();
for value in values.iter() {
set.insert(value);
}
m.insert(key, set);
}
m
};
}
if let Some(values) = MAP.get(mode) {
if values.contains(self) {
true
} else {
false
}
} else {
unreachable!()
}
}
}
impl fmt::Display for Level
{
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result
{
match self {
Level::Debug => write!(f, "DEBUG"),
Level::Info => write!(f, "INFO"),
Level::Warning => write!(f, "WARN"),
Level::Error => write!(f, "ERROR"),
Level::Fatal => write!(f, "FATAL"),
_ => write!(f, "(unbound)"),
}
}
}
macro_rules! log {
($level:tt, $mode:expr => $format:expr, $($rest:expr),*) => {
{
if let Some(level) = $mode.level($crate::log::Level::$level) {
println!("{} [{}]: {}", $crate::log::timestamp(), level, format!($format, $($rest)*));
true
} else {
false
};
}
};
($mode:expr => $format:expr, $($rest:expr),*) => {
{
log!(Info, $mode => $format, $($rest)*);
}
};
}
pub fn timestamp() -> String
{
use chrono::{
offset::Utc,
DateTime,
};
let time: DateTime<Utc> = std::time::SystemTime::now().into();
time.format("%Y.%m.%d %h:%M:%S").to_string()
}
//log!(Level::Debug, cmode => "Hello {}", "hi");

@ -3,9 +3,13 @@
pub const BUFFER_SIZE: usize = 4096; pub const BUFFER_SIZE: usize = 4096;
mod bytes; mod bytes;
mod ext;
pub use ext::*;
mod error; mod error;
mod hash; mod hash;
mod container; mod container;
#[macro_use]
mod log;
mod config; mod config;
mod arg; mod arg;
mod proc; mod proc;
@ -68,6 +72,33 @@ mod test {
} }
} }
fn main() { fn parse_args() -> Result<config::Config, error::Error>
{
match arg::parse_args()? {
arg::Output::Normal(conf) => Ok(conf),
_ => arg::usage(),
}
}
#[cfg_attr(feature="threads", tokio::main)]
#[cfg(feature="threads")]
async fn main() -> Result<(), error::Error>
{
let args = parse_args()?;
let lmode = &args.mode.logging_mode;
log!(Debug, lmode => "Args parsed: {:?}", args);
Ok(())
}
#[cfg(not(feature="threads"))]
fn main() -> Result<(), error::Error>
{
let args = parse_args()?;
log!(Fatal, log::Mode::Error => "{:?}", args);
Ok(())
} }

Loading…
Cancel
Save