task limit implemented

tested, graph builds ok

Cargo.lock

@@ -1,5 +1,40 @@
 # This file is automatically @generated by Cargo.
 # It is not intended for manual editing.
+[[package]]
+name = "addr2line"
+version = "0.14.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a55f82cfe485775d02112886f4169bde0c5894d75e79ead7eafe7e40a25e45f7"
+dependencies = [
+ "gimli",
+]
+
+[[package]]
+name = "adler"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e"
+
+[[package]]
+name = "autocfg"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
+
+[[package]]
+name = "backtrace"
+version = "0.3.56"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9d117600f438b1707d4e4ae15d3595657288f8235a0eb593e80ecc98ab34e1bc"
+dependencies = [
+ "addr2line",
+ "cfg-if 1.0.0",
+ "libc",
+ "miniz_oxide",
+ "object",
+ "rustc-demangle",
+]
+
 [[package]]
 name = "bitflags"
 version = "1.2.1"
@@ -24,15 +59,40 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
+[[package]]
+name = "color-eyre"
+version = "0.5.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b29030875fd8376e4a28ef497790d5b4a7843d8d1396bf08ce46f5eec562c5c"
+dependencies = [
+ "backtrace",
+ "eyre",
+ "indenter",
+ "once_cell",
+ "owo-colors",
+]
+
 [[package]]
 name = "dirstat"
 version = "0.1.0"
 dependencies = [
+ "color-eyre",
  "futures",
+ "num_cpus",
  "pin-project",
  "tokio",
 ]
 
+[[package]]
+name = "eyre"
+version = "0.6.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "221239d1d5ea86bf5d6f91c9d6bc3646ffe471b08ff9b0f91c44f115ac969d2b"
+dependencies = [
+ "indenter",
+ "once_cell",
+]
+
 [[package]]
 name = "fnv"
 version = "1.0.7"
@@ -150,6 +210,12 @@ dependencies = [
  "slab",
 ]
 
+[[package]]
+name = "gimli"
+version = "0.23.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f6503fe142514ca4799d4c26297c4248239fe8838d827db6bd6065c6ed29a6ce"
+
 [[package]]
 name = "hermit-abi"
 version = "0.1.18"
@@ -159,6 +225,12 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "indenter"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f4d5eb2e114fec2b7fe0fadc22888ad2658789bb7acac4dbee9cf8389f971ec8"
+
 [[package]]
 name = "iovec"
 version = "0.1.4"
@@ -205,6 +277,16 @@ version = "2.3.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525"
 
+[[package]]
+name = "miniz_oxide"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0f2d26ec3309788e423cfbf68ad1800f061638098d76a83681af979dc4eda19d"
+dependencies = [
+ "adler",
+ "autocfg",
+]
+
 [[package]]
 name = "mio"
 version = "0.6.23"
@@ -290,12 +372,24 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "object"
+version = "0.23.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a9a7ab5d64814df0fe4a4b5ead45ed6c5f181ee3ff04ba344313a6c80446c5d4"
+
 [[package]]
 name = "once_cell"
 version = "1.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "13bd41f508810a131401606d54ac32a467c97172d74ba7662562ebba5ad07fa0"
 
+[[package]]
+name = "owo-colors"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2386b4ebe91c2f7f51082d4cefa145d030e33a1842a96b12e4885cc3c01f7a55"
+
 [[package]]
 name = "pin-project"
 version = "1.0.5"
@@ -364,6 +458,12 @@ dependencies = [
  "proc-macro2",
 ]
 
+[[package]]
+name = "rustc-demangle"
+version = "0.1.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6e3bad0ee36814ca07d7968269dd4b7ec89ec2da10c4bb613928d3077083c232"
+
 [[package]]
 name = "signal-hook-registry"
 version = "1.3.0"

Cargo.toml

@@ -7,6 +7,8 @@ edition = "2018"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+color-eyre = {version = "0.5.10", default-features=false}
 futures = "0.3.12"
+num_cpus = "1.13.0"
 pin-project = "1.0.5"
 tokio = {version = "0.2", features=["full"]}

src/config.rs

@@ -1,11 +1,12 @@
 use std::path::PathBuf;
 use std::num::NonZeroUsize;
+use std::{fmt,error};
 
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub enum Recursion
 {
     None,
-    Limited(usize),
+    Limited(NonZeroUsize),
     Unlimited,
 }
 
@@ -23,19 +24,80 @@ impl Recursion
     /// Can we run at this depth?
     pub fn can_run(&self, depth: usize) -> bool
     {
+	debug_assert!(depth > 0,  "Depth of 0 is invalid");
 	match self {
 	    Self::None => depth == 1,
-	    Self::Limited(limit) => depth <= *limit,
+	    Self::Limited(limit) => depth <= limit.get(),
 	    Self::Unlimited => true
 	}
     }
+
 }
 
 /// Configuration for this run
-#[derive(Debug, Clone, PartialEq, Eq, Default)]
+#[derive(Debug, Clone, PartialEq, Eq)]
 pub struct Config
 {
     pub paths: Vec<PathBuf>,
     pub recursive: Recursion,
     pub max_tasks: Option<NonZeroUsize>,
 }
+
+impl Default for Config
+{
+    #[inline]
+    fn default() -> Self
+    {
+	Self {
+	    paths: Vec::new(),
+	    recursive: Default::default(),
+	    max_tasks: NonZeroUsize::new(num_cpus::get()),
+	}
+    }
+}
+
+
+impl Config
+{
+    /// Validate this configuration instance.
+    pub fn validate(self) -> Result<Self, InvalidConfigError>
+    {
+	if self.paths.len() < 1 {
+	    return Err(InvalidConfigError::NoPaths);
+	}
+	let paths: Result<Vec<_>, _> = self.paths.into_iter()
+	    .map(|path| if !path.exists() { Err(InvalidConfigError::PathNotFound(path)) } else { Ok(path) })
+	    .collect();
+	Ok(Self{
+	    paths: paths?,
+	    ..self
+	})
+    }
+}
+
+/// Error type for an invalid instance of `Config`.
+#[derive(Debug)]
+#[non_exhaustive]
+pub enum InvalidConfigError
+{
+    /// No paths were given.
+    NoPaths,
+    /// Non-existant path was given.
+    PathNotFound(PathBuf),
+
+    /// Unknown error
+    Other,
+}
+
+impl error::Error for InvalidConfigError{}
+impl fmt::Display for InvalidConfigError
+{
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result
+    {
+	match self {
+	    Self::NoPaths => write!(f, "No input paths were given. Cannot do anything"),
+	    Self::PathNotFound(path) => write!(f, "Root path {:?} not found", path),
+	    #[cold] _ => write!(f, "Unknown error"),
+	}
+    }
+}

src/data/graph.rs

@@ -20,5 +20,7 @@ impl INodeInfoGraph
 	    paths
 	}
     }
-    //TODO: Order by largest size, get and iter fns etc
+    //TODO: Get whole directory structure. Find largest, etc.
+    //TODO: Order by largest file size, get, iter, etc
+    //TODO: Group children to parent (child FSInfos all have references to their parent INode, but parents don't have references to their children. Top level FsInfos will also have parent INodes that don't appear in the map as keys.)
 }

src/main.rs

@@ -3,6 +3,17 @@
 
 #[macro_use] extern crate pin_project;
 
+use color_eyre::{
+    eyre::{
+	self,
+	eyre,
+
+	WrapErr as _,
+    },
+    
+    Help as _,
+};
+
 #[macro_use] mod ext;
 pub use ext::prelude::*;
 
@@ -11,7 +22,24 @@ mod config;
 mod state;
 mod work;
 
+async fn read_config() -> eyre::Result<config::Config>
+{
+    Ok(config::Config::default()) //TODO: read config
+}
+
 #[tokio::main]
-async fn main() {
+async fn main() -> eyre::Result<()> {
-    println!("Hello, world!");
+    color_eyre::install()?;
+
+    let state = state::State::new(read_config().await
+				  .wrap_err(eyre!("Failed to load config"))?
+				  .validate()
+				  .wrap_err(eyre!("Invalid config"))
+				  .with_suggestion(|| "Try running `--help`")?);
+    
+    let graph = work::work_on_all(state).await;
+
+    println!("{:?}", graph);
+    
+    Ok(())
 }

src/state.rs

@@ -1,6 +1,13 @@
 use super::*;
 use std::sync::Arc;
 
+use tokio::sync::{
+    Semaphore,
+    SemaphorePermit
+};
+use futures::Future;
+use futures::future::OptionFuture;
+
 use data::Cache;
 use config::Config;
 
@@ -11,6 +18,20 @@ pub struct State
     config: Arc<Config>,
     cache: Cache,
     depth: usize,
+
+    throttle: Option<Arc<Semaphore>>,
+}
+
+#[derive(Debug)]
+pub struct TaskLock<'a>(&'a Option<Arc<Semaphore>>);
+
+impl<'a> TaskLock<'a>
+{
+    /// Enter this lock section.
+    pub fn enter(self) -> OptionFuture<impl Future<Output = SemaphorePermit<'a>> + 'a>
+    {
+	self.0.as_ref().map(|x| x.acquire()).into()
+    }
 }
 
 impl State
@@ -19,12 +40,19 @@ impl State
     pub fn new(cfg: Config) -> Self
     {
 	Self {
-	    config: Arc::new(cfg),
+	    throttle: cfg.max_tasks.map(|max| Arc::new(Semaphore::new(max.get()))),
 	    cache: Cache::new(),
 	    depth: 1,
+	    config: Arc::new(cfg),
 	}
     }
 
+    /// Get the semaphore for this state
+    pub fn lock(&self) -> TaskLock<'_>
+    {
+	TaskLock(&self.throttle)
+    }
+
     /// Current depth of this tree
     pub fn depth(&self) -> usize
     {

src/work.rs

@@ -73,11 +73,13 @@ pub async fn work_on_all(state: State) -> INodeInfoGraph
 	{
 	    for (path, ino) in join_root(&root, res)
 	    {
-		if let Some(_) = ino_map.get(&ino) {
+		// If this inode is not in the map, this is a top-level path.
+		
+		//if let Some(_) = ino_map.get(&ino) { 
 		output.insert(path, ino);
-		} else {
+		//} else {
-		    eprintln!("No ino entry for {:?} ({:?})", path, ino);
+		//    eprintln!("No ino entry for {:?} ({:?})", path, ino);
-		}
+		//}
 	    }
 	}
     }
@@ -99,6 +101,9 @@ fn walk(state: State, root: PathBuf, root_ino: INode) -> BoxFuture<'static, Hash
     let mut output = HashMap::new();
     let mut children: Vec<JoinHandle<HashMap<PathBuf, INode>>> = Vec::new();
     async move {
+	{
+	    let _guard = state.lock().enter().await;
+	    println!(" -> {:?}", root);
 	    match fs::read_dir(&root).await
 	    {
 		Ok(mut dir) => {
@@ -135,7 +140,8 @@ fn walk(state: State, root: PathBuf, root_ino: INode) -> BoxFuture<'static, Hash
 		},
 		Err(err) => eprintln!("Failed to walk {:?}: {}", root, err),
 	    }
-	
+	    // drop work guard here
+	}
 	// Join all children
 	for child in join_all(children.into_iter()).await
 	{