flanchan
/
dirstat
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

task limit implemented

Browse Source 
tested, graph builds ok
redo-gragh
Avril 4 years ago
parent 47b5e97006
commit e9a72f7513
Signed by: flanchan
GPG Key ID: 284488987C31F630
7 changed files with 270 additions and 42 deletions
Show Stats Download Patch File Download Diff File

100
Cargo.lock generated
Unescape View File

@ -1,5 +1,40 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
[[package]]
name = "addr2line"
version = "0.14.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a55f82cfe485775d02112886f4169bde0c5894d75e79ead7eafe7e40a25e45f7"
dependencies = [
"gimli",
]
[[package]]
name = "adler"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e"
[[package]]
name = "autocfg"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
[[package]]
name = "backtrace"
version = "0.3.56"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d117600f438b1707d4e4ae15d3595657288f8235a0eb593e80ecc98ab34e1bc"
dependencies = [
"addr2line",
"cfg-if 1.0.0",
"libc",
"miniz_oxide",
"object",
"rustc-demangle",
]
[[package]]
name = "bitflags"
version = "1.2.1"
@ -24,15 +59,40 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "color-eyre"
version = "0.5.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b29030875fd8376e4a28ef497790d5b4a7843d8d1396bf08ce46f5eec562c5c"
dependencies = [
"backtrace",
"eyre",
"indenter",
"once_cell",
"owo-colors",
]
[[package]]
name = "dirstat"
version = "0.1.0"
dependencies = [
"color-eyre",
"futures",
"num_cpus",
"pin-project",
"tokio",
]
[[package]]
name = "eyre"
version = "0.6.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "221239d1d5ea86bf5d6f91c9d6bc3646ffe471b08ff9b0f91c44f115ac969d2b"
dependencies = [
"indenter",
"once_cell",
]
[[package]]
name = "fnv"
version = "1.0.7"
@ -150,6 +210,12 @@ dependencies = [
"slab",
]
[[package]]
name = "gimli"
version = "0.23.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f6503fe142514ca4799d4c26297c4248239fe8838d827db6bd6065c6ed29a6ce"
[[package]]
name = "hermit-abi"
version = "0.1.18"
@ -159,6 +225,12 @@ dependencies = [
"libc",
]
[[package]]
name = "indenter"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f4d5eb2e114fec2b7fe0fadc22888ad2658789bb7acac4dbee9cf8389f971ec8"
[[package]]
name = "iovec"
version = "0.1.4"
@ -205,6 +277,16 @@ version = "2.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525"
[[package]]
name = "miniz_oxide"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0f2d26ec3309788e423cfbf68ad1800f061638098d76a83681af979dc4eda19d"
dependencies = [
"adler",
"autocfg",
]
[[package]]
name = "mio"
version = "0.6.23"
@ -290,12 +372,24 @@ dependencies = [
"libc",
]
[[package]]
name = "object"
version = "0.23.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a9a7ab5d64814df0fe4a4b5ead45ed6c5f181ee3ff04ba344313a6c80446c5d4"
[[package]]
name = "once_cell"
version = "1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13bd41f508810a131401606d54ac32a467c97172d74ba7662562ebba5ad07fa0"
[[package]]
name = "owo-colors"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2386b4ebe91c2f7f51082d4cefa145d030e33a1842a96b12e4885cc3c01f7a55"
[[package]]
name = "pin-project"
version = "1.0.5"
@ -364,6 +458,12 @@ dependencies = [
"proc-macro2",
]
[[package]]
name = "rustc-demangle"
version = "0.1.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e3bad0ee36814ca07d7968269dd4b7ec89ec2da10c4bb613928d3077083c232"
[[package]]
name = "signal-hook-registry"
version = "1.3.0"

2
Cargo.toml
Unescape View File

@ -7,6 +7,8 @@ edition = "2018"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
color-eyre = {version = "0.5.10", default-features=false}
futures = "0.3.12"
num_cpus = "1.13.0"
pin-project = "1.0.5"
tokio = {version = "0.2", features=["full"]}

68
src/config.rs
Unescape View File

@ -1,11 +1,12 @@
use std::path::PathBuf;
use std::num::NonZeroUsize;
use std::{fmt,error};
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Recursion
{
None,
Limited(usize),
Limited(NonZeroUsize),
Unlimited,
}
@ -23,19 +24,80 @@ impl Recursion
/// Can we run at this depth?
pub fn can_run(&self, depth: usize) -> bool
{
debug_assert!(depth > 0, "Depth of 0 is invalid");
match self {
Self::None => depth == 1,
Self::Limited(limit) => depth <= *limit,
Self::Limited(limit) => depth <= limit.get(),
Self::Unlimited => true
}
}
}
/// Configuration for this run
#[derive(Debug, Clone, PartialEq, Eq, Default)]
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Config
{
pub paths: Vec<PathBuf>,
pub recursive: Recursion,
pub max_tasks: Option<NonZeroUsize>,
}
impl Default for Config
{
#[inline]
fn default() -> Self
{
Self {
paths: Vec::new(),
recursive: Default::default(),
max_tasks: NonZeroUsize::new(num_cpus::get()),
}
}
}
impl Config
{
/// Validate this configuration instance.
pub fn validate(self) -> Result<Self, InvalidConfigError>
{
if self.paths.len() < 1 {
return Err(InvalidConfigError::NoPaths);
}
let paths: Result<Vec<_>, _> = self.paths.into_iter()
.map(|path| if !path.exists() { Err(InvalidConfigError::PathNotFound(path)) } else { Ok(path) })
.collect();
Ok(Self{
paths: paths?,
..self
})
}
}
/// Error type for an invalid instance of `Config`.
#[derive(Debug)]
#[non_exhaustive]
pub enum InvalidConfigError
{
/// No paths were given.
NoPaths,
/// Non-existant path was given.
PathNotFound(PathBuf),
/// Unknown error
Other,
}
impl error::Error for InvalidConfigError{}
impl fmt::Display for InvalidConfigError
{
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result
{
match self {
Self::NoPaths => write!(f, "No input paths were given. Cannot do anything"),
Self::PathNotFound(path) => write!(f, "Root path {:?} not found", path),
#[cold] _ => write!(f, "Unknown error"),
}
}
}

4
src/data/graph.rs
Unescape View File

@ -20,5 +20,7 @@ impl INodeInfoGraph
paths
}
}
//TODO: Order by largest size, get and iter fns etc
//TODO: Get whole directory structure. Find largest, etc.
//TODO: Order by largest file size, get, iter, etc
//TODO: Group children to parent (child FSInfos all have references to their parent INode, but parents don't have references to their children. Top level FsInfos will also have parent INodes that don't appear in the map as keys.)
}

32
src/main.rs
Unescape View File

@ -3,6 +3,17 @@
#[macro_use] extern crate pin_project;
use color_eyre::{
eyre::{
self,
eyre,
WrapErr as _,
},
Help as _,
};
#[macro_use] mod ext;
pub use ext::prelude::*;
@ -11,7 +22,24 @@ mod config;
mod state;
mod work;
async fn read_config() -> eyre::Result<config::Config>
{
Ok(config::Config::default()) //TODO: read config
}
#[tokio::main]
async fn main() {
println!("Hello, world!");
async fn main() -> eyre::Result<()> {
color_eyre::install()?;
let state = state::State::new(read_config().await
.wrap_err(eyre!("Failed to load config"))?
.validate()
.wrap_err(eyre!("Invalid config"))
.with_suggestion(|| "Try running `--help`")?);
let graph = work::work_on_all(state).await;
println!("{:?}", graph);
Ok(())
}

30
src/state.rs
Unescape View File

@ -1,6 +1,13 @@
use super::*;
use std::sync::Arc;
use tokio::sync::{
Semaphore,
SemaphorePermit
};
use futures::Future;
use futures::future::OptionFuture;
use data::Cache;
use config::Config;
@ -11,6 +18,20 @@ pub struct State
config: Arc<Config>,
cache: Cache,
depth: usize,
throttle: Option<Arc<Semaphore>>,
}
#[derive(Debug)]
pub struct TaskLock<'a>(&'a Option<Arc<Semaphore>>);
impl<'a> TaskLock<'a>
{
/// Enter this lock section.
pub fn enter(self) -> OptionFuture<impl Future<Output = SemaphorePermit<'a>> + 'a>
{
self.0.as_ref().map(|x| x.acquire()).into()
}
}
impl State
@ -19,12 +40,19 @@ impl State
pub fn new(cfg: Config) -> Self
{
Self {
config: Arc::new(cfg),
throttle: cfg.max_tasks.map(|max| Arc::new(Semaphore::new(max.get()))),
cache: Cache::new(),
depth: 1,
config: Arc::new(cfg),
}
}
/// Get the semaphore for this state
pub fn lock(&self) -> TaskLock<'_>
{
TaskLock(&self.throttle)
}
/// Current depth of this tree
pub fn depth(&self) -> usize
{

16
src/work.rs
Unescape View File

@ -73,11 +73,13 @@ pub async fn work_on_all(state: State) -> INodeInfoGraph
{
for (path, ino) in join_root(&root, res)
{
if let Some(_) = ino_map.get(&ino) {
// If this inode is not in the map, this is a top-level path.
//if let Some(_) = ino_map.get(&ino) {
output.insert(path, ino);
} else {
eprintln!("No ino entry for {:?} ({:?})", path, ino);
}
//} else {
// eprintln!("No ino entry for {:?} ({:?})", path, ino);
//}
}
}
}
@ -99,6 +101,9 @@ fn walk(state: State, root: PathBuf, root_ino: INode) -> BoxFuture<'static, Hash
let mut output = HashMap::new();
let mut children: Vec<JoinHandle<HashMap<PathBuf, INode>>> = Vec::new();
async move {
{
let _guard = state.lock().enter().await;
println!(" -> {:?}", root);
match fs::read_dir(&root).await
{
Ok(mut dir) => {
@ -135,7 +140,8 @@ fn walk(state: State, root: PathBuf, root_ino: INode) -> BoxFuture<'static, Hash
},
Err(err) => eprintln!("Failed to walk {:?}: {}", root, err),
}
// drop work guard here
}
// Join all children
for child in join_all(children.into_iter()).await
{

Write Preview
Loading…
Cancel
Save