task limit implemented

tested, graph builds ok
redo-gragh
Avril 3 years ago
parent 47b5e97006
commit e9a72f7513
Signed by: flanchan
GPG Key ID: 284488987C31F630

100
Cargo.lock generated

@ -1,5 +1,40 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
[[package]]
name = "addr2line"
version = "0.14.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a55f82cfe485775d02112886f4169bde0c5894d75e79ead7eafe7e40a25e45f7"
dependencies = [
"gimli",
]
[[package]]
name = "adler"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e"
[[package]]
name = "autocfg"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
[[package]]
name = "backtrace"
version = "0.3.56"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d117600f438b1707d4e4ae15d3595657288f8235a0eb593e80ecc98ab34e1bc"
dependencies = [
"addr2line",
"cfg-if 1.0.0",
"libc",
"miniz_oxide",
"object",
"rustc-demangle",
]
[[package]]
name = "bitflags"
version = "1.2.1"
@ -24,15 +59,40 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "color-eyre"
version = "0.5.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b29030875fd8376e4a28ef497790d5b4a7843d8d1396bf08ce46f5eec562c5c"
dependencies = [
"backtrace",
"eyre",
"indenter",
"once_cell",
"owo-colors",
]
[[package]]
name = "dirstat"
version = "0.1.0"
dependencies = [
"color-eyre",
"futures",
"num_cpus",
"pin-project",
"tokio",
]
[[package]]
name = "eyre"
version = "0.6.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "221239d1d5ea86bf5d6f91c9d6bc3646ffe471b08ff9b0f91c44f115ac969d2b"
dependencies = [
"indenter",
"once_cell",
]
[[package]]
name = "fnv"
version = "1.0.7"
@ -150,6 +210,12 @@ dependencies = [
"slab",
]
[[package]]
name = "gimli"
version = "0.23.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f6503fe142514ca4799d4c26297c4248239fe8838d827db6bd6065c6ed29a6ce"
[[package]]
name = "hermit-abi"
version = "0.1.18"
@ -159,6 +225,12 @@ dependencies = [
"libc",
]
[[package]]
name = "indenter"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f4d5eb2e114fec2b7fe0fadc22888ad2658789bb7acac4dbee9cf8389f971ec8"
[[package]]
name = "iovec"
version = "0.1.4"
@ -205,6 +277,16 @@ version = "2.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525"
[[package]]
name = "miniz_oxide"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0f2d26ec3309788e423cfbf68ad1800f061638098d76a83681af979dc4eda19d"
dependencies = [
"adler",
"autocfg",
]
[[package]]
name = "mio"
version = "0.6.23"
@ -290,12 +372,24 @@ dependencies = [
"libc",
]
[[package]]
name = "object"
version = "0.23.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a9a7ab5d64814df0fe4a4b5ead45ed6c5f181ee3ff04ba344313a6c80446c5d4"
[[package]]
name = "once_cell"
version = "1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13bd41f508810a131401606d54ac32a467c97172d74ba7662562ebba5ad07fa0"
[[package]]
name = "owo-colors"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2386b4ebe91c2f7f51082d4cefa145d030e33a1842a96b12e4885cc3c01f7a55"
[[package]]
name = "pin-project"
version = "1.0.5"
@ -364,6 +458,12 @@ dependencies = [
"proc-macro2",
]
[[package]]
name = "rustc-demangle"
version = "0.1.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e3bad0ee36814ca07d7968269dd4b7ec89ec2da10c4bb613928d3077083c232"
[[package]]
name = "signal-hook-registry"
version = "1.3.0"

@ -7,6 +7,8 @@ edition = "2018"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
color-eyre = {version = "0.5.10", default-features=false}
futures = "0.3.12"
num_cpus = "1.13.0"
pin-project = "1.0.5"
tokio = {version = "0.2", features=["full"]}

@ -1,11 +1,12 @@
use std::path::PathBuf;
use std::num::NonZeroUsize;
use std::{fmt,error};
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Recursion
{
None,
Limited(usize),
Limited(NonZeroUsize),
Unlimited,
}
@ -23,19 +24,80 @@ impl Recursion
/// Can we run at this depth?
pub fn can_run(&self, depth: usize) -> bool
{
debug_assert!(depth > 0, "Depth of 0 is invalid");
match self {
Self::None => depth == 1,
Self::Limited(limit) => depth <= *limit,
Self::Limited(limit) => depth <= limit.get(),
Self::Unlimited => true
}
}
}
/// Configuration for this run
#[derive(Debug, Clone, PartialEq, Eq, Default)]
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Config
{
pub paths: Vec<PathBuf>,
pub recursive: Recursion,
pub max_tasks: Option<NonZeroUsize>,
}
impl Default for Config
{
#[inline]
fn default() -> Self
{
Self {
paths: Vec::new(),
recursive: Default::default(),
max_tasks: NonZeroUsize::new(num_cpus::get()),
}
}
}
impl Config
{
/// Validate this configuration instance.
pub fn validate(self) -> Result<Self, InvalidConfigError>
{
if self.paths.len() < 1 {
return Err(InvalidConfigError::NoPaths);
}
let paths: Result<Vec<_>, _> = self.paths.into_iter()
.map(|path| if !path.exists() { Err(InvalidConfigError::PathNotFound(path)) } else { Ok(path) })
.collect();
Ok(Self{
paths: paths?,
..self
})
}
}
/// Error type for an invalid instance of `Config`.
#[derive(Debug)]
#[non_exhaustive]
pub enum InvalidConfigError
{
/// No paths were given.
NoPaths,
/// Non-existant path was given.
PathNotFound(PathBuf),
/// Unknown error
Other,
}
impl error::Error for InvalidConfigError{}
impl fmt::Display for InvalidConfigError
{
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result
{
match self {
Self::NoPaths => write!(f, "No input paths were given. Cannot do anything"),
Self::PathNotFound(path) => write!(f, "Root path {:?} not found", path),
#[cold] _ => write!(f, "Unknown error"),
}
}
}

@ -20,5 +20,7 @@ impl INodeInfoGraph
paths
}
}
//TODO: Order by largest size, get and iter fns etc
//TODO: Get whole directory structure. Find largest, etc.
//TODO: Order by largest file size, get, iter, etc
//TODO: Group children to parent (child FSInfos all have references to their parent INode, but parents don't have references to their children. Top level FsInfos will also have parent INodes that don't appear in the map as keys.)
}

@ -3,6 +3,17 @@
#[macro_use] extern crate pin_project;
use color_eyre::{
eyre::{
self,
eyre,
WrapErr as _,
},
Help as _,
};
#[macro_use] mod ext;
pub use ext::prelude::*;
@ -11,7 +22,24 @@ mod config;
mod state;
mod work;
async fn read_config() -> eyre::Result<config::Config>
{
Ok(config::Config::default()) //TODO: read config
}
#[tokio::main]
async fn main() {
println!("Hello, world!");
async fn main() -> eyre::Result<()> {
color_eyre::install()?;
let state = state::State::new(read_config().await
.wrap_err(eyre!("Failed to load config"))?
.validate()
.wrap_err(eyre!("Invalid config"))
.with_suggestion(|| "Try running `--help`")?);
let graph = work::work_on_all(state).await;
println!("{:?}", graph);
Ok(())
}

@ -1,6 +1,13 @@
use super::*;
use std::sync::Arc;
use tokio::sync::{
Semaphore,
SemaphorePermit
};
use futures::Future;
use futures::future::OptionFuture;
use data::Cache;
use config::Config;
@ -11,6 +18,20 @@ pub struct State
config: Arc<Config>,
cache: Cache,
depth: usize,
throttle: Option<Arc<Semaphore>>,
}
#[derive(Debug)]
pub struct TaskLock<'a>(&'a Option<Arc<Semaphore>>);
impl<'a> TaskLock<'a>
{
/// Enter this lock section.
pub fn enter(self) -> OptionFuture<impl Future<Output = SemaphorePermit<'a>> + 'a>
{
self.0.as_ref().map(|x| x.acquire()).into()
}
}
impl State
@ -19,12 +40,19 @@ impl State
pub fn new(cfg: Config) -> Self
{
Self {
config: Arc::new(cfg),
throttle: cfg.max_tasks.map(|max| Arc::new(Semaphore::new(max.get()))),
cache: Cache::new(),
depth: 1,
config: Arc::new(cfg),
}
}
/// Get the semaphore for this state
pub fn lock(&self) -> TaskLock<'_>
{
TaskLock(&self.throttle)
}
/// Current depth of this tree
pub fn depth(&self) -> usize
{

@ -73,11 +73,13 @@ pub async fn work_on_all(state: State) -> INodeInfoGraph
{
for (path, ino) in join_root(&root, res)
{
if let Some(_) = ino_map.get(&ino) {
output.insert(path, ino);
} else {
eprintln!("No ino entry for {:?} ({:?})", path, ino);
}
// If this inode is not in the map, this is a top-level path.
//if let Some(_) = ino_map.get(&ino) {
output.insert(path, ino);
//} else {
// eprintln!("No ino entry for {:?} ({:?})", path, ino);
//}
}
}
}
@ -99,43 +101,47 @@ fn walk(state: State, root: PathBuf, root_ino: INode) -> BoxFuture<'static, Hash
let mut output = HashMap::new();
let mut children: Vec<JoinHandle<HashMap<PathBuf, INode>>> = Vec::new();
async move {
match fs::read_dir(&root).await
{
Ok(mut dir) => {
while let Some(entry) = dir.next().await
{
match entry
let _guard = state.lock().enter().await;
println!(" -> {:?}", root);
match fs::read_dir(&root).await
{
Ok(mut dir) => {
while let Some(entry) = dir.next().await
{
Ok(entry) => {
let ino = entry.inode();
// Check cache for this
if state.cache().get(&ino).await.is_none() {
// Not added, process.
match process_entry(&entry, root_ino).await {
Ok(fsinfo) => {
if fsinfo.is_dir()
{
if let Some(next) = state.deeper()
match entry
{
Ok(entry) => {
let ino = entry.inode();
// Check cache for this
if state.cache().get(&ino).await.is_none() {
// Not added, process.
match process_entry(&entry, root_ino).await {
Ok(fsinfo) => {
if fsinfo.is_dir()
{
children.push(tokio::spawn(
walk(next, entry.path(), ino)
));
if let Some(next) = state.deeper()
{
children.push(tokio::spawn(
walk(next, entry.path(), ino)
));
}
}
}
let mut cache = state.cache_sub();
cache.insert(ino, fsinfo).await;
},
Err(err) => eprintln!("Failed to stat {:?}: {}", entry.path(), err),
let mut cache = state.cache_sub();
cache.insert(ino, fsinfo).await;
},
Err(err) => eprintln!("Failed to stat {:?}: {}", entry.path(), err),
}
}
}
},
Err(err) => eprintln!("Walking {:?} failed: {}", root, err),
},
Err(err) => eprintln!("Walking {:?} failed: {}", root, err),
}
}
}
},
Err(err) => eprintln!("Failed to walk {:?}: {}", root, err),
},
Err(err) => eprintln!("Failed to walk {:?}: {}", root, err),
}
// drop work guard here
}
// Join all children
for child in join_all(children.into_iter()).await
{

Loading…
Cancel
Save