From 61c78c89eec49c090c5d236d0f1b935da3c1a8f4 Mon Sep 17 00:00:00 2001 From: Avril Date: Sat, 31 Oct 2020 00:07:23 +0000 Subject: [PATCH] start delete --- Cargo.lock | 96 ++++++++++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 2 ++ src/config.rs | 7 +++- src/delete.rs | 47 ++++++++++++++++++++++++ src/main.rs | 20 +++++++---- src/restore.rs | 18 ++++++++++ src/state.rs | 47 ++++++++++++++++++++++++ 7 files changed, 230 insertions(+), 7 deletions(-) create mode 100644 src/delete.rs create mode 100644 src/restore.rs create mode 100644 src/state.rs diff --git a/Cargo.lock b/Cargo.lock index 7aae4e4..cd88fac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -15,6 +15,15 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e" +[[package]] +name = "aho-corasick" +version = "0.7.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b476ce7103678b0c6d3d395dbbae31d48ff910bd28be979ba5d48c6351131d0d" +dependencies = [ + "memchr", +] + [[package]] name = "ansi_term" version = "0.11.0" @@ -39,6 +48,17 @@ version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d25d88fd6b8041580a654f9d0c581a047baee2b3efee13275f2fc392fc75034" +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi 0.3.9", +] + [[package]] name = "autocfg" version = "1.0.0" @@ -144,6 +164,19 @@ dependencies = [ "generic-array", ] +[[package]] +name = "env_logger" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" +dependencies = [ + "atty", + "humantime", + "log", + "regex", + "termcolor", +] + [[package]] name = "eyre" version = "0.6.0" @@ -296,6 +329,15 @@ dependencies = [ "libc", ] +[[package]] +name = "humantime" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f" +dependencies = [ + "quick-error", +] + [[package]] name = "indenter" version = "0.3.0" @@ -511,6 +553,16 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pretty_env_logger" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "926d36b9553851b8b0005f1275891b392ee4d2d833852c417ed025477350fb9d" +dependencies = [ + "env_logger", + "log", +] + [[package]] name = "proc-macro-hack" version = "0.5.18" @@ -532,6 +584,12 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + [[package]] name = "quote" version = "1.0.7" @@ -547,6 +605,24 @@ version = "0.1.57" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" +[[package]] +name = "regex" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8963b85b8ce3074fecffde43b4b0dded83ce2f367dc8d363afc56679f3ee820b" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", + "thread_local", +] + +[[package]] +name = "regex-syntax" +version = "0.6.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cab7a364d15cde1e505267766a2d3c4e22a843e1a601f0fa7564c0f82ced11c" + [[package]] name = "rustc-demangle" version = "0.1.16" @@ -647,6 +723,15 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "termcolor" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb6bfa289a4d7c5766392812c0a1f4c1ba45afa1ad47803c11e1f407d846d75f" +dependencies = [ + "winapi-util", +] + [[package]] name = "thread_local" version = "1.0.1" @@ -781,6 +866,8 @@ dependencies = [ "color-eyre", "futures", "lazy_static", + "log", + "pretty_env_logger", "rustc_version", "sha2", "smallmap", @@ -815,6 +902,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi 0.3.9", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" diff --git a/Cargo.toml b/Cargo.toml index 221e698..70afdf0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,6 +26,8 @@ color-eyre = "0.5.1" lazy_static = "1.4.0" futures = "0.3.6" smallmap = "^1.1.6" +log = "0.4.11" +pretty_env_logger = "0.4.0" [build-dependencies] rustc_version = "0.2" diff --git a/src/config.rs b/src/config.rs index 89d876b..eccbf63 100644 --- a/src/config.rs +++ b/src/config.rs @@ -5,14 +5,18 @@ use std::{ }, error, fmt, - io + io, + num::NonZeroUsize, }; use tokio::fs; #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct Config { + /// The base dir of the database pub base_dir: PathBuf, + /// Maximum number of concurrent operations + pub limit: Option, } impl Default for Config @@ -22,6 +26,7 @@ impl Default for Config { Self { base_dir: PathBuf::from("/tmp/videl"), + limit: Some(unsafe{NonZeroUsize::new_unchecked(4096)}), } } } diff --git a/src/delete.rs b/src/delete.rs new file mode 100644 index 0000000..b311592 --- /dev/null +++ b/src/delete.rs @@ -0,0 +1,47 @@ +//! Handle deletion +//! +//! # Handling paths +//! Deletion of single files is trivial, deletion of directories is not as trivial. +//! Other types of filesystem object are ignored. +//! +//! ## Directories +//! With directories the deletion processes all files containing recursively. +//! See `restore` for restoration of directories +use super::*; +use std::{ + path::{ + Path, + PathBuf, + }, +}; + +/// Process a single file. +/// +/// `path` is known to be a file at this point. +async fn process_single(state: Arc, path: &Path) -> eyre::Result<()> +{ + let _g = state.lock().await; + debug!("{:?} Processing", path); + Ok(()) +} + +/// Process this path +/// +/// This will not return until all its children finish too (if any) +pub async fn process(state: Arc, path: impl AsRef) -> eyre::Result<()> +{ + let path = path.as_ref(); + if path.is_dir() { + trace!("{:?} Spawning children", path); + //TODO! Walk dir tree + todo!() + } else if path.is_file() { + process_single(state, path).await + .wrap_err(eyre!("Processing file failed")) + .with_section(|| format!("{:?}", path).header("Path was")) + } else { + error!("{:?} is not a recognised FS object", path); + return Err(eyre!("Unsupported FS object")) + .with_section(|| format!("{:?}", path).header("Path was")) + } +} diff --git a/src/main.rs b/src/main.rs index ed942bc..8a35a98 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,6 +2,8 @@ #![allow(dead_code)] +#[macro_use] extern crate log; + use cfg_if::cfg_if; use lazy_static::lazy_static; //use futures::prelude::*; @@ -34,6 +36,9 @@ mod database; mod args; mod config; +mod state; +mod delete; +mod restore; cfg_if!{ if #[cfg(nightly)] { @@ -52,8 +57,10 @@ fn install() -> eyre::Result<()> Ok(()) } -async fn process(config: Arc, file: String) +/// Process a path for deletion +async fn process(state: Arc, file: String) { + let config = state.config(); println!(" -> {:?}", file); let dbdir = resolve::mangle_path(&config, &file); println!("Database path for this file {:?}", dbdir); @@ -77,12 +84,12 @@ async fn validate_config(config: config::Config) -> eyre::Result async fn begin() -> eyre::Result { install()?; - let config = Arc::new(validate_config(config::Config::default()).await - .wrap_err(eyre!("Failed to validate config"))?); - println!("Validated config OK"); + let state = Arc::new(state::State::new(validate_config(config::Config::default()).await + .wrap_err(eyre!("Failed to validate config"))?)); + info!("Validated config OK"); if args::process(|file| { - let config = Arc::clone(&config); - process(config, file) + let state = Arc::clone(&state); + process(state, file) }).await .wrap_err(eyre!("One or more child workers failed to complete successfully"))? .len() == 0 @@ -96,6 +103,7 @@ async fn begin() -> eyre::Result #[tokio::main] async fn main() -> eyre::Result<()> { + pretty_env_logger::init(); std::process::exit(match begin().await { Ok(0) => return Ok(()), Err(err) => { diff --git a/src/restore.rs b/src/restore.rs new file mode 100644 index 0000000..f9addbc --- /dev/null +++ b/src/restore.rs @@ -0,0 +1,18 @@ +//! Handles restoration +//! +//! # Handling paths +//! For single files, restoration is trivial. +//! +//! ## Directories +//! In restoration, directories are restored recursively. All deleted children of a directory path are restored in this operation. +//! +//! # Process +//! When determining if the recursive (directory) stratergy is needed, this happens +//! * If the path does not exist in the database as its own atom +//! * If there are one or more atoms prefixed with this exact path +//! +//! Then we use the directory stratergy +//! ## Issues +//! This can cause issues if a file is deleted and replaced with a directory which is then deleted. The file will be recovered before the directory. +//! This is a bug or design flaw. (Maybe we can mitigate this with a flag to prefer the directory stratergy, or check timestamps to determine if the file or the directory was the newest one.) +use super::*; diff --git a/src/state.rs b/src/state.rs new file mode 100644 index 0000000..b86898f --- /dev/null +++ b/src/state.rs @@ -0,0 +1,47 @@ +//! Keeping state +use super::*; +use tokio::{ + sync::{ + Semaphore, + SemaphorePermit, + }, +}; +use futures::{ + future::OptionFuture, +}; + +/// Contains state for a set of operations +#[derive(Debug)] +pub struct State +{ + cfg: config::Config, + mtx: Option, +} + +/// Guard for operations inside state +#[derive(Debug)] +pub struct Permit<'a>(Option>); + +impl State +{ + /// Create a new state + pub fn new(cfg: config::Config) -> Self + { + Self { + mtx: cfg.limit.map(|x| Semaphore::new(x.into())), + cfg, + } + } + + /// Acquire a permit for concurrent work, yielding the task if needed. + pub async fn lock(&self) -> Permit<'_> + { + Permit(OptionFuture::from(self.mtx.as_ref().map(Semaphore::acquire)).await) + } + + /// The config object + #[inline] pub fn config(&self) -> &config::Config + { + &self.cfg + } +}