Compare commits

..

1 Commits

Author SHA1 Message Date
Avril 946a3e6cc3
fuck you
4 years ago

1
.gitignore vendored

@ -1,5 +1,4 @@
/target /target
*~ *~
*.dump *.dump
*.dump.raw
profiling/ profiling/

147
Cargo.lock generated

@ -2,12 +2,6 @@
# It is not intended for manual editing. # It is not intended for manual editing.
version = 3 version = 3
[[package]]
name = "ad-hoc-iter"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "90a8dd76beceb5313687262230fcbaaf8d4e25c37541350cf0932e9adb8309c8"
[[package]] [[package]]
name = "addr2line" name = "addr2line"
version = "0.14.1" version = "0.14.1"
@ -120,34 +114,11 @@ dependencies = [
"owo-colors", "owo-colors",
] ]
[[package]]
name = "dirs-next"
version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1"
dependencies = [
"cfg-if 1.0.0",
"dirs-sys-next",
]
[[package]]
name = "dirs-sys-next"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d"
dependencies = [
"libc",
"redox_users",
"winapi 0.3.9",
]
[[package]] [[package]]
name = "dirstat" name = "dirstat"
version = "0.1.0" version = "0.1.0"
dependencies = [ dependencies = [
"ad-hoc-iter",
"async-compression", "async-compression",
"cfg-if 1.0.0",
"color-eyre", "color-eyre",
"futures", "futures",
"jemallocator", "jemallocator",
@ -157,12 +128,9 @@ dependencies = [
"num_cpus", "num_cpus",
"once_cell", "once_cell",
"pin-project", "pin-project",
"rustyline",
"serde", "serde",
"serde_cbor", "serde_cbor",
"smallvec",
"tokio", "tokio",
"treemap",
] ]
[[package]] [[package]]
@ -181,16 +149,6 @@ version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
[[package]]
name = "fs2"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213"
dependencies = [
"libc",
"winapi 0.3.9",
]
[[package]] [[package]]
name = "fs_extra" name = "fs_extra"
version = "1.2.0" version = "1.2.0"
@ -308,17 +266,6 @@ dependencies = [
"slab", "slab",
] ]
[[package]]
name = "getrandom"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8"
dependencies = [
"cfg-if 1.0.0",
"libc",
"wasi",
]
[[package]] [[package]]
name = "gimli" name = "gimli"
version = "0.23.0" version = "0.23.0"
@ -508,18 +455,6 @@ dependencies = [
"winapi 0.3.9", "winapi 0.3.9",
] ]
[[package]]
name = "nix"
version = "0.19.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b2ccba0cfe4fdf15982d1674c69b1fd80bad427d293849982668dfe454bd61f2"
dependencies = [
"bitflags",
"cc",
"cfg-if 1.0.0",
"libc",
]
[[package]] [[package]]
name = "num_cpus" name = "num_cpus"
version = "1.13.0" version = "1.13.0"
@ -622,58 +557,12 @@ dependencies = [
"proc-macro2", "proc-macro2",
] ]
[[package]]
name = "redox_syscall"
version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94341e4e44e24f6b591b59e47a8a027df12e008d73fd5672dbea9cc22f4507d9"
dependencies = [
"bitflags",
]
[[package]]
name = "redox_users"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "528532f3d801c87aec9def2add9ca802fe569e44a544afe633765267840abe64"
dependencies = [
"getrandom",
"redox_syscall",
]
[[package]] [[package]]
name = "rustc-demangle" name = "rustc-demangle"
version = "0.1.18" version = "0.1.18"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e3bad0ee36814ca07d7968269dd4b7ec89ec2da10c4bb613928d3077083c232" checksum = "6e3bad0ee36814ca07d7968269dd4b7ec89ec2da10c4bb613928d3077083c232"
[[package]]
name = "rustyline"
version = "7.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8227301bfc717136f0ecbd3d064ba8199e44497a0bdd46bb01ede4387cfd2cec"
dependencies = [
"bitflags",
"cfg-if 1.0.0",
"dirs-next",
"fs2",
"libc",
"log",
"memchr",
"nix",
"scopeguard",
"unicode-segmentation",
"unicode-width",
"utf8parse",
"winapi 0.3.9",
]
[[package]]
name = "scopeguard"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
[[package]] [[package]]
name = "serde" name = "serde"
version = "1.0.123" version = "1.0.123"
@ -719,12 +608,6 @@ version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8"
[[package]]
name = "smallvec"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e"
[[package]] [[package]]
name = "socket2" name = "socket2"
version = "0.3.19" version = "0.3.19"
@ -782,42 +665,12 @@ dependencies = [
"syn", "syn",
] ]
[[package]]
name = "treemap"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1571f89da27a5e1aa83304ee1ab9519ea8c6432b4c8903aaaa6c9a9eecb6f36"
[[package]]
name = "unicode-segmentation"
version = "1.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb0d2e7be6ae3a5fa87eed5fb451aff96f2573d2694942e40543ae0bbe19c796"
[[package]]
name = "unicode-width"
version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3"
[[package]] [[package]]
name = "unicode-xid" name = "unicode-xid"
version = "0.2.1" version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564"
[[package]]
name = "utf8parse"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "936e4b492acfd135421d8dca4b1aa80a7bfc26e702ef3af710e0752684df5372"
[[package]]
name = "wasi"
version = "0.10.2+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"
[[package]] [[package]]
name = "winapi" name = "winapi"
version = "0.2.8" version = "0.2.8"

@ -13,10 +13,7 @@ codegen-units = 1
panic = "unwind" panic = "unwind"
[features] [features]
default = ["splash", "inspect", "defer-drop", "jemalloc", "prealloc", "treemap"] default = ["splash", "inspect", "defer-drop", "jemalloc", "prealloc"]
# When using the REPL to inspect graphs, save command history.
save-history = []
# Use jemalloc as global allocator instead of system allocator. # Use jemalloc as global allocator instead of system allocator.
# May potentially cause some speedups and better memory profile on large runs. # May potentially cause some speedups and better memory profile on large runs.
@ -35,9 +32,7 @@ defer-drop = []
splash = [] splash = []
[dependencies] [dependencies]
ad-hoc-iter = "0.2.3"
async-compression = {version = "0.3", features=["tokio-02", "bzip2"], optional=true} async-compression = {version = "0.3", features=["tokio-02", "bzip2"], optional=true}
cfg-if = "1.0.0"
color-eyre = {version = "0.5.10", default-features=false} color-eyre = {version = "0.5.10", default-features=false}
futures = "0.3.12" futures = "0.3.12"
jemallocator = {version = "0.3.2", optional = true} jemallocator = {version = "0.3.2", optional = true}
@ -47,9 +42,6 @@ memmap = {version = "0.7.0", optional = true}
num_cpus = "1.13.0" num_cpus = "1.13.0"
once_cell = "1.5.2" once_cell = "1.5.2"
pin-project = "1.0.5" pin-project = "1.0.5"
rustyline = "7.1.0"
serde = {version = "1.0.123", features=["derive"], optional=true} serde = {version = "1.0.123", features=["derive"], optional=true}
serde_cbor = {version = "0.11.1", optional=true} serde_cbor = {version = "0.11.1", optional=true}
smallvec = "1.6.1"
tokio = {version = "0.2", features=["full"]} tokio = {version = "0.2", features=["full"]}
treemap = {version = "0.3.2", optional=true}

@ -5,8 +5,6 @@ use std::fmt;
use config::Config; use config::Config;
mod parsing;
/// Executable name /// Executable name
pub fn program_name() -> &'static str pub fn program_name() -> &'static str
{ {
@ -35,19 +33,10 @@ const OPTIONS_NORMAL: &'static [&'static str] = &[
"-v Verbose mode. Output extra information.", "-v Verbose mode. Output extra information.",
#[cfg(feature="inspect")] "--save <file> Dump the collected data to this file for further inspection.", #[cfg(feature="inspect")] "--save <file> Dump the collected data to this file for further inspection.",
#[cfg(feature="inspect")] "-D Dump the collected data to `stdout` (see `--save`.)", #[cfg(feature="inspect")] "-D Dump the collected data to `stdout` (see `--save`.)",
#[cfg(feature="inspect")] "--save-raw <file> Dump the collected data to this file uncompressed. (see `--save`.)", #[cfg(feature="prealloc")] "--save-raw <file> Dump the collected data to this file uncompressed.",
#[cfg(feature="inspect")] "-R Dump the collected data to standard output uncompressed. (see `--save-raw`.)",
"- Stop parsing arguments, treat all the rest as paths.", "- Stop parsing arguments, treat all the rest as paths.",
]; ];
const NOTES: &'static [&'static str] = &[
"The first time a non-option argument is encountered, the program stops parsing arguments and assumes the rest of the arguments are paths.",
"If parallelism is set to unlimited, there can be a huge syscall overhead. It is recommended to use `-m` in large runs.",
"",
"Symlinks are ignored while collecting stat data. They will fail with message 'Unknown file type'. Symlinks are generally very small in the actual data they contain themselves, so this is *usually* unimportant.",
#[cfg(feature="inspect")] "\nThe save formats of `--save` (`-D`) and `--save-raw` (`-Dr`) are incompatible. The former is bzip2 compressed the latter is uncompressed.",
];
fn get_opt_normal() -> impl fmt::Display fn get_opt_normal() -> impl fmt::Display
{ {
#[derive(Debug)] #[derive(Debug)]
@ -81,6 +70,7 @@ pub fn usage()
println!(r#" println!(r#"
OPTIONS: OPTIONS:
{} {}
--help Print this message and exit. --help Print this message and exit.
NOTES: NOTES:
@ -115,23 +105,74 @@ pub enum Mode
fn parse<I: IntoIterator<Item=String>>(args: I) -> eyre::Result<Mode> fn parse<I: IntoIterator<Item=String>>(args: I) -> eyre::Result<Mode>
{ {
//let mut cfg = config::Config::default(); let suggestion_intended_arg = || "If this was intended as a path instead of an option, use option `-` before it.";
let mut buffer = parsing::Output::new();
let mut args = args.into_iter(); let mut args = args.into_iter();
let mut cfg = Config::default();
while let Some(arg) = args.next() let mut reading = true;
while let Some(opt) = args.next()
{ {
match parsing::parse_next(&mut args, &mut buffer, arg)? { if reading {
parsing::Continue::No => { match opt.trim()
parsing::consume(args, &mut buffer); {
break; "--help" => return Ok(Mode::Help),
}, "-" => reading = false,
parsing::Continue::Abort(Some(change_to)) => return Ok(*change_to),
parsing::Continue::Abort(_) => break, "--threads" => {
_ => (), let max = args.next().ok_or(eyre!("`--threads` expects a parameter"))
.with_suggestion(suggestion_intended_arg.clone())?;
cfg.max_tasks = NonZeroUsize::new(max.parse::<usize>()
.wrap_err(eyre!("`--threads` expects a non-negative number"))
.with_suggestion(suggestion_intended_arg.clone())
.with_section(move || max.header("Parameter given was"))?);
},
"-M" => cfg.max_tasks = None, // this is the default, but it is possible an earlier command mutated it, so doing nothing here would be a bug for that corner case
"-m" => {
cfg.max_tasks = config::max_tasks_cpus();
},
"-q" => {
cfg.output_level = config::OutputLevel::Quiet;
},
"-Q" => {
cfg.output_level = config::OutputLevel::Silent;
},
"-v" => {
cfg.output_level = config::OutputLevel::Verbose;
},
#[cfg(feature="inspect")] "-D" => {
cfg.serialise_output = Some(config::OutputSerialisationMode::Stdout);
},
#[cfg(feature="inspect")] "--save" => {
let file = args.next().ok_or(eyre!("`--save` expects a filename parameter"))
.with_suggestion(suggestion_intended_arg.clone())?;
cfg.serialise_output = Some(config::OutputSerialisationMode::File(file.into()));
},
#[cfg(feature="prealloc")] "--save-raw" => {
let file = args.next().ok_or(eyre!("`--save-raw` expects a filename parameter"))
.with_suggestion(suggestion_intended_arg.clone())?;
cfg.serialise_output = Some(config::OutputSerialisationMode::PreallocFile(file.into()));
},
"--recursive" => {
let max = args.next().ok_or(eyre!("`--recursive` expects a parameter"))
.with_suggestion(suggestion_intended_arg.clone())?;
cfg.recursive = max.parse::<usize>()
.wrap_err(eyre!("`--recursive` expects a non-negative number"))
.with_suggestion(suggestion_intended_arg.clone())
.with_section(move || max.header("Parameter given was"))?.into();
},
"-r" => cfg.recursive = config::Recursion::Unlimited,
_ => {
cfg.paths.push(opt.into());
reading = false;
}
}
continue;
} else {
cfg.paths.push(opt.into());
} }
} }
Ok(Mode::Normal(cfg))
parsing::into_mode(buffer)
} }

@ -1,515 +0,0 @@
//! For parsing arguments
use super::*;
use std::collections::{HashMap, HashSet};
use std::mem::Discriminant;
use std::fmt;
#[cfg(feature="inspect")] use config::OutputSerialisationMode;
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub enum InspectKind
{
Treemap(Option<(u64, u64)>),
}
impl fmt::Display for InspectKind
{
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result
{
match self {
Self::Treemap(None) => write!(f, "treemap"),
Self::Treemap(Some((x,y))) => write!(f, "treemap:{}:{}", x, y), // Width and height.
}
}
}
impl std::str::FromStr for InspectKind
{
type Err = eyre::Report;
fn from_str(s: &str) -> Result<Self, Self::Err>
{
todo!()
}
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub enum Argument
{
ModeChangeHelp,
LimitConcMaxProc,
LimitConc(NonZeroUsize),
UnlimitConc,
Save(String),
SaveStdout,
SaveRaw(String),
SaveRawStdout,
LimitRecurse(NonZeroUsize),
UnlimitRecurse,
LogVerbose,
LogQuiet,
LogSilent,
StopReading,
Inspect(InspectKind),
Input(String),
}
/// Kinds of modes of operation for the program.
///
/// These map to `super::Mode`.
#[derive(Debug, Clone, PartialEq, Eq, Hash, Ord, PartialOrd, Copy)]
#[non_exhaustive]
enum ModeKind
{
Normal,
Help
}
impl Default for ModeKind
{
#[inline]
fn default() -> Self
{
Self::Normal
}
}
impl Argument
{
/// What mode does this argument change to, if any?
fn mode_change_kind(&self) -> Option<ModeKind>
{
Some(match self
{
Self::ModeChangeHelp => ModeKind::Help,
_ => return None,
})
}
/// Insert this `Argument` into config
pub fn insert_into_cfg(self, cfg: &mut Config)
{
use Argument::*;
match self {
Inspect(InspectKind::Treemap(None)) => cfg.inspection.treemap = Some((640, 480)),
Inspect(InspectKind::Treemap(x)) => cfg.inspection.treemap = x,
LimitConcMaxProc => cfg.max_tasks = config::max_tasks_cpus(),
LimitConc(max) => cfg.max_tasks = Some(max),
UnlimitConc => cfg.max_tasks = None,
#[cfg(feature="inspect")] Save(output) => cfg.serialise_output = Some(OutputSerialisationMode::File(output.into())),
#[cfg(feature="inspect")] SaveStdout => cfg.serialise_output = Some(OutputSerialisationMode::Stdout),
#[cfg(feature="inspect")] SaveRaw(output) => {
cfg_if! {
if #[cfg(feature="prealloc")] {
cfg.serialise_output = Some(OutputSerialisationMode::PreallocFile(output.into()));
} else {
cfg.serialise_output = Some(OutputSerialisationMode::RawFile(output.into()));
}
}
},
#[cfg(feature="inspect")] SaveRawStdout => cfg.serialise_output = Some(OutputSerialisationMode::RawStdout),
LimitRecurse(limit) => cfg.recursive = if limit.get() == 1 { config::Recursion::None } else { config::Recursion::Limited(limit) },
UnlimitRecurse => cfg.recursive = config::Recursion::Unlimited,
LogVerbose => cfg.output_level = config::OutputLevel::Verbose,
LogQuiet => cfg.output_level = config::OutputLevel::Quiet,
LogSilent => cfg.output_level = config::OutputLevel::Silent,
Input(path) => cfg.paths.push(path.into()),
_ => (), //unreachable()! // Do nothing instead of panic.
}
}
}
impl fmt::Display for Argument
{
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result
{
use Argument::*;
match self
{
Inspect(ins) => write!(f, "--inspect {}", ins),
ModeChangeHelp => write!(f, "--help"),
LimitConcMaxProc => write!(f, "-m"),
LimitConc(limit) => write!(f, "--threads {}", limit),
UnlimitConc => write!(f, "-M (--threads 0)"),
Save(s) => write!(f, "--save {:?}", s),
SaveStdout => write!(f, "-D"),
SaveRaw(s) => write!(f, "--save-raw {:?}", s),
SaveRawStdout => write!(f, "-R"),
LimitRecurse(rec) => write!(f, "--recursive {}", rec),
UnlimitRecurse => write!(f, "-r (--recursive 0)"),
LogVerbose => write!(f, "-v"),
LogQuiet => write!(f, "-q"),
LogSilent => write!(f, "-Q"),
StopReading => write!(f, "-"),
Input(input) => write!(f, "<{}>", input),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Hash, Copy)]
enum MX
{
None,
Itself,
All,
Only(Discriminant<Argument>),
Many(&'static [Discriminant<Argument>]),
}
impl Default for MX
{
#[inline]
fn default() -> Self
{
Self::Itself
}
}
impl MX
{
/// Is this argument discriminant mutually exclusive with this other argument?
pub fn is_mx(&self, this: Discriminant<Argument>, other: &Argument) -> bool
{
use std::mem::discriminant;
let other = discriminant(other);
match self
{
Self::Itself if other == this => true,
Self::All => true,
Self::Only(disc) if other == *disc => true,
Self::Many(discs) if discs.contains(&other) => true,
_ => false,
}
}
}
impl Argument
{
/// Is this `Argument` mutually exclusive with another?
pub fn is_mx_with(&self, other: &Self) -> bool
{
use std::mem::discriminant;
lazy_static! {
static ref MX_REF: HashMap<Discriminant<Argument>, MaybeVec<MX>> = {
let mut out = HashMap::new();
macro_rules! mx {
(@) => {
std::iter::empty()
};
(@ self $($tt:tt)*) => {
iter![MX::Itself].chain(mx!(@ $($tt)*))
};
(@ [$inner:expr] $($tt:tt)*) => {
iter![MX::Only(discriminant(&$inner))].chain(mx!(@ $($tt)*))
};
(@ [$($inner:expr),*] $($tt:tt)*) => {
iter![MX::Many(vec![$(discriminant(&$inner)),*].leak())].chain(mx!(@ $($tt)*))
};
(@ $ident:ident $($tt:tt)*) => {
iter![MX::$ident].chain(mx!(@ $($tt)*))
};
($disc:expr => $($tt:tt)*) => {
out.insert(discriminant(&$disc), mx!(@ $($tt)*).collect());
};
}
mx!(Argument::ModeChangeHelp => All);
mx!(Argument::LimitConcMaxProc => self [Argument::UnlimitConc,
Argument::LimitConc(unsafe{NonZeroUsize::new_unchecked(1)})]);
mx!(Argument::UnlimitConc => self [Argument::LimitConcMaxProc, Argument::LimitConc(unsafe{NonZeroUsize::new_unchecked(1)})]);
mx!(Argument::LimitConc(unsafe{NonZeroUsize::new_unchecked(1)}) => self [Argument::LimitConcMaxProc, Argument::UnlimitConc]);
mx!(Argument::Save(String::default()) => self [Argument::SaveStdout,
Argument::SaveRaw(Default::default()),
Argument::SaveRawStdout]);
mx!(Argument::SaveStdout => self [Argument::Save(String::default()),
Argument::SaveRaw(Default::default()),
Argument::SaveRawStdout]);
mx!(Argument::SaveRaw(Default::default()) => self [Argument::Save(String::default()),
Argument::SaveStdout,
Argument::SaveRawStdout]);
mx!(Argument::SaveRawStdout => self [Argument::Save(String::default()),
Argument::SaveRaw(String::default()),
Argument::SaveStdout]);
mx!(Argument::LimitRecurse(unsafe{NonZeroUsize::new_unchecked(1)}) => self [Argument::UnlimitRecurse]);
mx!(Argument::UnlimitRecurse => self [Argument::LimitRecurse(unsafe{NonZeroUsize::new_unchecked(1)})]);
mx!(Argument::LogVerbose => self [Argument::LogQuiet, Argument::LogSilent]);
mx!(Argument::LogQuiet => self [Argument::LogVerbose, Argument::LogSilent]);
mx!(Argument::LogSilent => self [Argument::LogQuiet, Argument::LogVerbose]);
mx!(Argument::StopReading => All);
mx!(Argument::Input(String::default()) => None);
out
};
}
let this = discriminant(self);
match MX_REF.get(&this) {
Some(mx) if mx.iter().filter(|mx| mx.is_mx(this, other)).next().is_some() => true,
_ => false,
}
}
}
/// Should we continue parsing and/or reading arguments?
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Continue
{
/// Keep parsing the arguments
Yes,
/// Stop parsing arguments, add the rest of args as `Input`s
No,
/// On mode change, we don't need to parse the rest of the argument. Stop reading entirely, and optionally return the last one here, which must be a mode change argument.
///
/// Returning this when the contained value is `Some` immediately terminates parsing and precedes to mode-switch. However, if it is `None`, parsing of chained short args is allowed to continue, although `Abort(None)` will be returned at the end regardless of subsequent `Continue` results from that change (unless one is an `Abort(Some(_))`, which immediately returns itself.)
// Box `Argument` to reduce the size of `Continue`, as it is returned from functions often and when its value is set to `Some` it will always be the last `Argument` processed anyway and the only one to be boxed here at all.
//TODO: Deprecate the early return of an `Argument` here. Either change it to `Mode`, or have no early return. Mode change happens at the bottom in `into_mode` now.
Abort(Option<Box<Mode>>),
}
impl Continue
{
/// Should we keep *parsing* args?
#[inline] pub fn keep_reading(&self) -> bool
{
if let Self::Yes = self {
true
} else {
false
}
}
/// Is this an abort?
#[inline] pub fn is_abort(&self) -> bool
{
if let Self::Abort(_) = self {
true
} else {
false
}
}
}
impl Default for Continue
{
#[inline]
fn default() -> Self
{
Self::Yes
}
}
impl From<bool> for Continue
{
fn from(from: bool) -> Self
{
if from {
Self::Yes
} else {
Self::No
}
}
}
pub type Output = HashSet<Argument>;
#[inline] const fn suggestion_intended_arg() -> &'static str {
"If this was intended as a path instead of an option, use option `-` before it."
}
fn save_output(output: &mut Output, item: Argument) -> eyre::Result<()>
{
if let Some(mx) = output.iter().filter(|arg| item.is_mx_with(arg)).next() {
return Err(eyre!("Arguments are mutually exclusive"))
.with_section(|| item.header("Trying to addargument "))
.with_section(|| mx.to_string().header("Which is mutually exclusive with previously added"));
}
output.insert(item); //TODO: Warn when adding duplicate?
Ok(())
}
fn parse_single<I>(_args: &mut I, output: &mut Output, this: char) -> eyre::Result<Continue>
where I: Iterator<Item=String>
{
let item = match this
{
'r' => Argument::UnlimitRecurse,
#[cfg(feature="inspect")] 'D' => Argument::SaveStdout,
#[cfg(feature="inspect")] 'R' => Argument::SaveRawStdout,
'v' => Argument::LogVerbose,
'q' => Argument::LogQuiet,
'Q' => Argument::LogSilent,
'm' => Argument::LimitConcMaxProc,
'M' => Argument::UnlimitConc,
unknown => {
return Err(eyre!("Unknown short argument {:?}", unknown))
.with_suggestion(suggestion_intended_arg.clone());
},
};
save_output(output, item)
.with_section(|| this.header("Short argument was"))?;
Ok(Continue::Yes)
}
/// Consume this iterator into `Input`s
pub fn consume<I>(args: I, output: &mut Output)
where I: IntoIterator<Item=String>
{
output.extend(args.into_iter().map(Argument::Input));
}
pub fn parse_next<I>(args: &mut I, output: &mut Output, this: String) -> eyre::Result<Continue>
where I: Iterator<Item=String>
{
let mut keep_reading = Continue::Yes;
let item = match this.trim()
{
"--inspect" => {
let ins = args.next().ok_or(eyre!("`--inspect` expects a parameter"))
.with_suggestion(suggestion_intended_arg.clone())?;
Argument::Inspect(ins.parse().wrap_err(eyre!("Failed to parse parameter for `--inspect`"))?)
},
" --threads" => {
let max = args.next().ok_or(eyre!("`--threads` expects a parameter"))
.with_suggestion(suggestion_intended_arg.clone())?;
match NonZeroUsize::new(max.parse::<usize>()
.wrap_err(eyre!("`--threads` expects a non-negative number"))
.with_suggestion(suggestion_intended_arg.clone())
.with_section(move || max.header("Parameter given was"))?)
{
Some(max) => Argument::LimitConc(max),
None => Argument::UnlimitConc,
}
},
"--recursive" => {
let max = args.next().ok_or(eyre!("`--recursive` expects a parameter"))
.with_suggestion(suggestion_intended_arg.clone())?;
match NonZeroUsize::new(max.parse::<usize>().wrap_err(eyre!("`--recursive` expects a non-negative number"))
.with_suggestion(suggestion_intended_arg.clone())
.with_section(move || max.header("Parameter given was"))?)
{
Some(x) => Argument::LimitRecurse(x),
None => Argument::UnlimitRecurse,
}
},
"--help" => {
return Ok(Continue::Abort(Some(Box::new(Mode::Help))));
},
"-" => {
return Ok(Continue::No);
},
#[cfg(feature="inspect")] "--save" => {
let file = args.next().ok_or(eyre!("`--save` expects a parameter"))
.with_suggestion(suggestion_intended_arg.clone())?;
Argument::Save(file)
},
#[cfg(feature="inspect")] "--save-raw" => {
let file = args.next().ok_or(eyre!("`--save` expects a parameter"))
.with_suggestion(suggestion_intended_arg.clone())?;
Argument::SaveRaw(file)
},
single if single.starts_with("-") => {
for ch in single.chars().skip(1) {
match parse_single(args, output, ch)
.wrap_err(eyre!("Error parsing short argument"))
.with_section(|| this.clone().header("Full short argument chain was"))? {
abort @ Continue::Abort(Some(_)) => return Ok(abort),
x @ Continue::No |
x @ Continue::Abort(_) if !x.is_abort() => keep_reading = x,
_ => (),
}
}
return Ok(keep_reading);
},
_ => {
keep_reading = Continue::No;
Argument::Input(this)
}
};
save_output(output, item)?;
Ok(keep_reading)
}
/// Converts parsed argument lists into a respective mode.
///
/// # Notes
/// These functions assume the mode has already been correctly calculated to be the mode pertaining to that function.
mod modes {
use super::*;
use config::Config;
/// Consume a parsed list of arguments in `Normal` mode into a `Normal` mode `Config` object.
pub fn normal(args: Output) -> eyre::Result<config::Config>
{
let mut cfg = Config::default();
for arg in args.into_iter()
{
arg.insert_into_cfg(&mut cfg);
}
Ok(cfg)
}
}
/// Consume this parsed list of arguments into a `Mode` and return it
pub fn into_mode(args: Output) -> eyre::Result<Mode>
{
let mut mode_kind = ModeKind::default(); //Normal.
for arg in args.iter() {
//find any mode change Argument (with `Argument::mode_change_kind()`) in `args`, changing `mode_kind` in turn. There should be at most 1.
if let Some(mode) = arg.mode_change_kind()
{
mode_kind = mode;
break;
}
}
//pass `args` to the respective mode generation function in mode `modes`, and wrap that mode around its return value.
match mode_kind
{
ModeKind::Normal => modes::normal(args).map(Mode::Normal),
ModeKind::Help => Ok(Mode::Help),
}
}

@ -98,48 +98,13 @@ impl From<usize> for Recursion
#[cfg(feature="inspect")] #[cfg(feature="inspect")]
#[derive(Debug, Clone, PartialEq, Eq)] #[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub enum OutputSerialisationMode pub enum OutputSerialisationMode
{ {
Stdout, Stdout,
File(PathBuf), File(PathBuf),
RawFile(PathBuf),
RawStdout,
#[cfg(feature="prealloc")] PreallocFile(PathBuf), #[cfg(feature="prealloc")] PreallocFile(PathBuf),
} }
#[cfg(feature="inspect")]
impl OutputSerialisationMode
{
/// Should this serialisation mode be compressed?
#[inline] pub fn should_compress(&self) -> bool
{
match self {
Self::File(_) | Self::Stdout => true,
_ => false,
}
}
}
/// What to do with the graph afterwards?
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Inspection
{
pub treemap: Option<(u64, u64)>, //w and h
}
impl Default for Inspection
{
#[inline]
fn default() -> Self
{
Self {
treemap: None
}
}
}
/// Configuration for this run /// Configuration for this run
#[derive(Debug, Clone, PartialEq, Eq)] #[derive(Debug, Clone, PartialEq, Eq)]
pub struct Config pub struct Config
@ -152,8 +117,6 @@ pub struct Config
#[cfg(feature="inspect")] #[cfg(feature="inspect")]
pub serialise_output: Option<OutputSerialisationMode>, pub serialise_output: Option<OutputSerialisationMode>,
pub inspection: Inspection,
} }
impl Config impl Config
@ -176,11 +139,11 @@ impl Config
#[inline] pub fn is_using_stdout(&self) -> bool #[inline] pub fn is_using_stdout(&self) -> bool
{ {
#[cfg(feature="inspect")] { #[cfg(feature="inspect")] {
return match self.serialise_output { return if let Some(OutputSerialisationMode::Stdout) = self.serialise_output {
Some(OutputSerialisationMode::Stdout) | true
Some(OutputSerialisationMode::RawStdout) => true, } else {
_ => false, false
} };
} }
#[cfg(not(feature="inspect"))] { #[cfg(not(feature="inspect"))] {
false false
@ -209,8 +172,6 @@ impl Default for Config
output_level: Default::default(), output_level: Default::default(),
#[cfg(feature="inspect")] #[cfg(feature="inspect")]
serialise_output: None, serialise_output: None,
inspection: Default::default(),
} }
} }
} }

@ -15,123 +15,8 @@ pub struct INodeInfoGraph
children: HashMap<INode, Vec<INode>>, //reverse lookup for directory INodes and their parent INodes children: HashMap<INode, Vec<INode>>, //reverse lookup for directory INodes and their parent INodes
} }
#[derive(Debug, Clone)]
pub struct INodeInfoGraphEntry<'a>
{
master: &'a INodeInfoGraph,
inode: INode,
}
impl<'a> INodeInfoGraphEntry<'a>
{
/// Create an iterator over children of this graph item.
///
/// # Note
/// Returns `None` if this is not a directory item.
pub fn level(&self) -> Option<Level<'a>>
{
match self.master.inodes.get(&self.inode)
{
Some(FsInfo::Directory(parent)) => {
Some(
Level{
master: self.master,
inode: self.inode.clone(),
children: match self.master.children.get(&self.inode) {
Some(iter) => iter.iter(),
_ => [].iter(),
},
}
)
},
Some(FsInfo::File(_, _)) => None,
None => {
panic!("No lookup information for inode {:?}", self.inode)
}//Some(Level{master: self.master, inode: None, children: [].iter()}),
}
}
/// The `FsInfo` for this item.
pub fn info(&self) -> &FsInfo
{
self.master.inodes.get(&self.inode).unwrap()
}
/// The path for this inode
pub fn path(&self) -> &PathBuf
{
self.master.paths_reverse.get(&self.inode).unwrap()
}
}
#[derive(Debug, Clone)]
pub struct Level<'a>
{
master: &'a INodeInfoGraph,
inode: INode,// Should only ever be `Directory` for `Level`.
children: std::slice::Iter<'a, INode>,
}
#[derive(Debug, Clone)]
pub struct TopLevel<'a>
{
master: &'a INodeInfoGraph,
children: std::vec::IntoIter<&'a INode>,
}
impl<'a> Iterator for TopLevel<'a>
{
type Item = INodeInfoGraphEntry<'a>;
fn next(&mut self) -> Option<Self::Item>
{
let master = self.master;
self.children.next().map(|inode| {
INodeInfoGraphEntry{
master,
inode: inode.clone(),
}
})
}
}
impl<'a> Iterator for Level<'a>
{
type Item = INodeInfoGraphEntry<'a>;
fn next(&mut self) -> Option<Self::Item>
{
let master = self.master;
self.children.next().map(|inode| {
INodeInfoGraphEntry{
master,
inode: inode.clone(),
}
})
}
}
impl INodeInfoGraph impl INodeInfoGraph
{ {
/// Total size of all files
pub fn total_size(&self) -> u64
{
self.inodes.iter().map(|(_, v)| {
match v {
FsInfo::File(len, _) => *len,
_ => 0,
}
}).sum()
}
/// An iterator over the top level of items
pub fn top_level(&self) -> TopLevel<'_>
{
let top_level = self.inodes.iter().filter(|(node, _)| {
!self.inodes.contains_key(node)
});
TopLevel {
master: self,
children: top_level.map(|(k, _)| k).collect::<Vec<_>>().into_iter(),
}
}
/// Create a new graph from these linked `HashMap`s /// Create a new graph from these linked `HashMap`s
#[inline] pub fn new(inodes: HashMap<INode, FsInfo>, paths: HashMap<PathBuf, INode>) -> Self #[inline] pub fn new(inodes: HashMap<INode, FsInfo>, paths: HashMap<PathBuf, INode>) -> Self
{ {
@ -163,11 +48,6 @@ impl INodeInfoGraph
self.paths_reverse.extend(self.paths.iter().map(|(x,y)| (*y,x.clone()))); self.paths_reverse.extend(self.paths.iter().map(|(x,y)| (*y,x.clone())));
self self
} }
/// Total number of items in the graph
#[inline] pub fn len(&self) -> usize
{
self.children.len()
}
/// Get the FsInfo of this `INode` /// Get the FsInfo of this `INode`
#[inline] pub fn get_info(&self, node: impl Borrow<INode>) -> Option<&FsInfo> #[inline] pub fn get_info(&self, node: impl Borrow<INode>) -> Option<&FsInfo>
{ {
@ -177,17 +57,17 @@ impl INodeInfoGraph
/// An iterator over top-level children of this node /// An iterator over top-level children of this node
pub fn children_of(&self, node: impl Borrow<INode>) -> !//Children<'_> pub fn children_of(&self, node: impl Borrow<INode>) -> !//Children<'_>
{ {
todo!(); todo!();
/*Children(self, match self.children.get(node.borrow()) { /*Children(self, match self.children.get(node.borrow()) {
Some(slc) => slc.iter(), Some(slc) => slc.iter(),
_ => [].iter(), _ => [].iter(),
})*/ })*/
} }
/// An iterator over all the directories in this /// An iterator over all the directories in this
pub fn directories(&self) -> !//Directories<'_> pub fn directories(&self) -> !//Directories<'_>
{ {
todo!()//Directories(self, self.children.keys()) todo!()//Directories(self, self.children.keys())
}*/ }*/
/// Convert into a hierarchical representation /// Convert into a hierarchical representation
pub fn into_hierarchical(self) -> HierarchicalINodeGraph pub fn into_hierarchical(self) -> HierarchicalINodeGraph
@ -384,12 +264,6 @@ impl HierarchicalINodeGraph
hi.len().map(|x| (path.as_path(), x)) hi.len().map(|x| (path.as_path(), x))
}).max_by_key(|x| x.1) }).max_by_key(|x| x.1)
} }
/// Complete number of items in the tree
#[inline] pub fn len(&self) -> usize
{
self.table.len()
}
} }
impl From<INodeInfoGraph> for HierarchicalINodeGraph impl From<INodeInfoGraph> for HierarchicalINodeGraph

@ -18,7 +18,6 @@ pub use graph::{
INodeInfoGraph, INodeInfoGraph,
HierarchicalINodeGraph, HierarchicalINodeGraph,
FsObjKind as FsKind, FsObjKind as FsKind,
INodeInfoGraphEntry,
}; };
/// A raw file or directory inode number /// A raw file or directory inode number

@ -19,9 +19,10 @@ pub mod prelude
pub use super::StreamLagExt as _; pub use super::StreamLagExt as _;
pub use super::INodeExt as _; pub use super::INodeExt as _;
pub use super::OptionIterator; pub use super::async_write_ext::{
EitherWrite,
pub use super::MaybeVec; DeadSink,
};
} }
pub trait INodeExt pub trait INodeExt
@ -205,63 +206,6 @@ where S: Stream
} }
} }
/// An iterator that can be constructed from an `Option<Iterator>`.
#[derive(Debug, Clone)]
pub struct OptionIterator<I>(Option<I>);
impl<I> OptionIterator<I>
{
/// Consume into the inner `Option`.
#[inline] pub fn into_inner(self) -> Option<I>
{
self.0
}
/// Does this `OptionIterator` have a value?
pub fn is_some(&self) -> bool
{
self.0.is_some()
}
}
impl<I: Iterator> From<Option<I>> for OptionIterator<I>
{
fn from(from: Option<I>) -> Self
{
Self(from)
}
}
impl<I: Iterator> Iterator for OptionIterator<I>
{
type Item = I::Item;
fn next(&mut self) -> Option<Self::Item>
{
self.0.as_mut().map(|x| x.next()).flatten()
}
fn size_hint(&self) -> (usize, Option<usize>) {
match self.0.as_ref() {
Some(i) => i.size_hint(),
_ => (0, Some(0)),
}
}
}
impl<I: Iterator> std::iter::FusedIterator for OptionIterator<I>
where I: std::iter::FusedIterator{}
impl<I: Iterator> ExactSizeIterator for OptionIterator<I>
where I: ExactSizeIterator{}
impl<I: Iterator> DoubleEndedIterator for OptionIterator<I>
where I: DoubleEndedIterator
{
fn next_back(&mut self) -> Option<Self::Item> {
self.0.as_mut().map(|x| x.next_back()).flatten()
}
}
/// Create a duration with time suffix `h`, `m`, `s`, `ms` or `ns`. /// Create a duration with time suffix `h`, `m`, `s`, `ms` or `ns`.
/// ///
/// # Combination /// # Combination
@ -408,4 +352,146 @@ impl fmt::Display for SoftAssertionFailedError
}; };
} }
pub type MaybeVec<T> = smallvec::SmallVec<[T; 1]>; mod async_write_ext {
use std::ops::{Deref, DerefMut};
use tokio::io::{
self,
AsyncWrite, AsyncRead,
};
use std::{
pin::Pin,
task::{Poll, Context},
};
use std::marker::PhantomData;
#[derive(Debug, Clone)]
pub enum EitherWrite<'a, T,U>
{
First(T),
Second(U, PhantomData<&'a mut U>),
}
impl<'a, T, U> Deref for EitherWrite<'a, T,U>
where T: AsyncWrite + Unpin + 'a,
U: AsyncWrite + Unpin + 'a
{
type Target = dyn AsyncWrite + Unpin + 'a;
fn deref(&self) -> &Self::Target {
match self {
Self::First(t) => t,
Self::Second(u, _) => u,
}
}
}
impl<'a, T, U> DerefMut for EitherWrite<'a, T,U>
where T: AsyncWrite + Unpin + 'a,
U: AsyncWrite + Unpin + 'a
{
fn deref_mut(&mut self) -> &mut Self::Target {
match self {
Self::First(t) => t,
Self::Second(u, _) => u,
}
}
}
impl<'a, T, U> From<Result<T,U>> for EitherWrite<'a, T, U>
where T: AsyncWrite + Unpin + 'a,
U: AsyncWrite + Unpin + 'a
{
#[inline] fn from(from: Result<T,U>) -> Self
{
match from {
Ok(v) => Self::First(v),
Err(v) => Self::Second(v, PhantomData),
}
}
}
impl<'a, T> EitherWrite<'a, T, DeadSink>
{
#[inline] fn as_first_infallible(&mut self) -> &mut T
{
match self {
Self::Second(_, _) => unsafe { core::hint::unreachable_unchecked() },
Self::First(t) => t
}
}
}
impl<'a, U> EitherWrite<'a, DeadSink, U>
{
#[inline] fn as_second_infallible(&mut self) -> &mut U
{
match self {
Self::First(_) => unsafe { core::hint::unreachable_unchecked() },
Self::Second(t, _) => t
}
}
}
/* impl<'a, T> AsyncWrite for EitherWrite<'a, T, DeadSink>
where T: AsyncWrite + Unpin + 'a
{
#[inline] fn poll_write(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &[u8]) -> Poll<Result<usize, io::Error>> {
let this = unsafe { self.map_unchecked_mut(|x| x.as_first_infallible()) };
this.poll_write(cx, buf)
}
#[inline] fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), io::Error>> {
let this = unsafe { self.map_unchecked_mut(|x| x.as_first_infallible()) };
this.poll_flush(cx)
}
fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), io::Error>> {
let this = unsafe { self.map_unchecked_mut(|x| x.as_first_infallible()) };
this.poll_shutdown(cx)
}
}*/
impl<'a, U> AsyncWrite for EitherWrite<'a, DeadSink, U>
where U: AsyncWrite + Unpin + 'a
{
#[inline] fn poll_write(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &[u8]) -> Poll<Result<usize, io::Error>> {
let this = unsafe { self.map_unchecked_mut(|x| x.as_second_infallible()) };
this.poll_write(cx, buf)
}
#[inline] fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), io::Error>> {
let this = unsafe { self.map_unchecked_mut(|x| x.as_second_infallible()) };
this.poll_flush(cx)
}
fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), io::Error>> {
let this = unsafe { self.map_unchecked_mut(|x| x.as_second_infallible()) };
this.poll_shutdown(cx)
}
}
/// An `Infallible` type for `AsyncWrite` & `AsyncRead`
#[derive(Debug)]
pub enum DeadSink { }
impl AsyncWrite for DeadSink
{
#[inline] fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Result<(), io::Error>> {
unreachable!();
}
#[inline] fn poll_write(self: Pin<&mut Self>, _cx: &mut Context<'_>, _buf: &[u8]) -> Poll<Result<usize, io::Error>> {
unreachable!();
}
#[inline] fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Result<(), io::Error>> {
unreachable!();
}
}
impl AsyncRead for DeadSink
{
#[inline] fn poll_read(self: Pin<&mut Self>, _cx: &mut Context<'_>, _buf: &mut [u8]) -> Poll<io::Result<usize>> {
unreachable!();
}
}
}

@ -1,124 +0,0 @@
use super::*;
use treemap::{
Rect,
Mappable,
TreemapLayout
};
use data::{FsInfo, INodeInfoGraph, INodeInfoGraphEntry};
/// A treemap of all **files** in the graph.
#[derive(Debug, Clone, PartialEq)]
pub struct Treemap
{
//layout: TreemapLayout,
nodes: Vec<MapNode>,
}
impl Treemap
{
/// All nodes of the map.
#[inline] pub fn nodes(&self) -> &[MapNode]
{
&self.nodes[..]
}
}
#[derive(Debug, Clone, PartialEq)]
pub struct MapNode
{
name: String,
vw_size: f64, // Should be halved each iteration
vw_bounds: Rect, // should be Rect::new() before aligntment
}
impl MapNode
{
/// The calculated bounds of the node
#[inline] pub fn bounds(&self) -> &Rect
{
&self.vw_bounds
}
/// The name of the node
#[inline] pub fn name(&self) -> &str
{
&self.name[..]
}
#[inline] fn size(&self) -> f64 //Is this useful for consumer?
{
self.vw_size
}
#[inline] fn new(name: String) -> Self
{
Self {
vw_size: 1.0,
vw_bounds: Rect::new(),
name,
}
}
}
/// Create a treemap from this graph.
pub fn treemap(_cfg: &Config, graph: &INodeInfoGraph, (w, h): (f64, f64)) -> Treemap
{
let layout = TreemapLayout::new();
let mut nodes = Vec::with_capacity(graph.len());
//TODO: Recursively walk the graph, halving size with each iteration. (Maybe we need `INodeInfoGraph` here, not `Hierarchicalinodegraph`?)
let total_size = graph.total_size();
let size = 1.0;
fn calc_path<'a, I: IntoIterator<Item = INodeInfoGraphEntry<'a>>>(insert: &'a mut Vec<MapNode>, from: I, total_size: u64, size: f64, scale: f64)
{
for top in from {
let path = top.path();
match top.info() {
FsInfo::Directory(_) => {
//TODO: Do we add dir itself? I think not?
// Add children
let size = size * 0.5;
calc_path(insert, top.level().unwrap(), total_size, size, scale);
},
&FsInfo::File(sz, _) => {
let fract = (sz as f64) / (total_size as f64);
insert.push(MapNode {
name: path.to_string_lossy().into_owned(),
vw_size: fract * scale,
vw_bounds: Rect::new(),
})
},
}
}
}
calc_path(&mut nodes, graph.top_level(), total_size, size, 1.0);
layout.layout_items(&mut nodes[..], Rect {
x: 0.0,
y: 0.0,
w, h,
});
Treemap {
//layout,
nodes
}
}
impl Mappable for MapNode
{
fn size(&self) -> f64
{
self.vw_size
}
fn bounds(&self) -> &Rect
{
&self.vw_bounds
}
fn set_bounds(&mut self, bounds: Rect)
{
self.vw_bounds = bounds;
}
}

@ -1,21 +0,0 @@
//! Prints the information found in graph in different ways
use super::*;
use data::HierarchicalINodeGraph;
use config::Config;
//pub mod repl;
/// Print the most basic info
pub fn print_basic_max_info(cfg: &Config, graph: &HierarchicalINodeGraph)
{
cfg_println!(Quiet; cfg, "Max size file: {:?}", graph.path_max_size_for(data::FsKind::File));
cfg_println!(Quiet; cfg, "Max size dir: {:?}", graph.path_max_size_for(data::FsKind::Directory));
cfg_println!(Quiet; cfg, "Max size all: {:?}", graph.path_max_size());
}
#[cfg(feature="treemap")]
mod map;
#[cfg(feature="treemap")]
pub use map::*;

@ -1,131 +0,0 @@
//! Graph inspection REPL
use super::*;
use std::{fmt, error};
use std::path::PathBuf;
use std::io;
use rustyline::error::ReadlineError;
use rustyline::Editor;
mod env;
mod command;
mod opcodes;
/// Default history file name
///
/// # Path lookup
/// * To make this an absolute path, start it with `/`
/// * To make this path relative to the user's home directory, start it with `~/` (Note: If we are unable to find the user's home directory, it is considered a lookup **failure** (*not* a **disable**) and `calculate_history_path()` will return `Err`.)
/// * Otherwise, the path is taken relative to the current working directory
///
/// # Notes
/// This is only used when the `save-history` feature is enabled.
const DEFAULT_HISTORY_FILE: &'static str = "~/.dirstat_history";
/// Get the path to the history file.
///
/// # Lookup
/// * If the `DIRSTAT_HISTORY` envvar is set and not empty, use this file path.
/// * If the `DIRSTAT_HISTORY` envvar is set and empty, saving history is considered **disabled**, we return `Ok(None)`.
/// * Otherwise, refer to lookup rules for `DEFAULT_HISTORY_FILE`.
pub fn calculate_history_path() -> io::Result<Option<PathBuf>>
{
cfg_if! {
if #[cfg(feature="save-history")] {
todo!()
} else {
unreachable!("Tried to calculate repl history path when binary was compiled with history saving perma-disabled.")
}
}
}
/// Inspect the graph with commands
///
/// # Note
/// This function synchronously blocks the current thread.
pub fn inspect(cfg: &Config, graph: &HierarchicalINodeGraph) -> Result<(), ReplExitError>
{
let mut repl = Editor::<()>::new(); //TODO: Change `()` to our completer, when we have a decent idea of how they'll work.
cfg_if! {
if #[cfg(feature="save-history")] {
let history_path = match calculate_history_path() {
Ok(Some(path)) => {
if let Err(err) = repl.load_history(&path)
{
cfg_eprintln!(cfg, "Failed to load repl history from {:?}: {}", path, err);
}
Some(path)
},
Ok(None) => None,
Err(err)
{
cfg_eprintln!(cfg, "Failed to find repl history: {}", err);
None
}
}
}
}
let res: Result<(), ReplExitError> = try {
loop {
let line = repl.readline("> ")?;
repl.add_history_entry(&line);
//TODO: How to interpret commands?
todo!("Interpret commands from `line`.");
}
};
cfg_if! {
if #[cfg(feature="save-history")] {
if let Some(path) = history_path {
if let Err(err) = repl.save_history(&path)
{
cfg_eprintln!(cfg, "Failed to save repl history to {:?}: {}", path, err);
}
}
}
}
res
}
/// When the inspection repl exists abnormally.
#[derive(Debug)]
pub enum ReplExitError
{
ReadLine(ReadlineError),
}
impl From<ReadlineError> for ReplExitError
{
#[inline] fn from(from: ReadlineError) -> Self
{
Self::ReadLine(from)
}
}
impl error::Error for ReplExitError
{
fn source(&self) -> Option<&(dyn error::Error + 'static)> {
Some(match &self
{
Self::ReadLine(rl) => rl
})
}
}
impl fmt::Display for ReplExitError
{
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result
{
match self
{
Self::ReadLine(ReadlineError::Eof) |
Self::ReadLine(ReadlineError::Interrupted) => write!(f, "exit"),
Self::ReadLine(_) => write!(f, "readline error"),
}
}
}

@ -1,54 +0,0 @@
//! Repl commands
use super::*;
use std::str::FromStr;
use super::env::*;
#[derive(Debug)]
pub struct Context<'a>
{
/// Environment containing variable name mappings.
env: &'a mut Lexenv,
}
/// Trait for commands.
///
/// # Defining commands
/// A command object should be created once only, and then referenced and executed using `params` and through mutating `cx`.
pub trait Command: fmt::Debug
{
fn execute(&self, cx: &mut Context<'_>, params: Vec<Value>) -> eyre::Result<()>;
}
/// Command structurally parsed.
///
/// Can be converted into `Command` with the `TryInto` trait.
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct IR
{
op: String,
params: Vec<Value>,
}
impl FromStr for IR
{
type Err = CommandParseError;
fn from_str(s: &str) -> Result<Self, Self::Err> {
todo!()
}
}
/// Error when parsing a command into `IR`.
#[derive(Debug)]
pub struct CommandParseError(String);
impl error::Error for CommandParseError{}
impl fmt::Display for CommandParseError
{
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result
{
write!(f, "failed to parse command from {:?}", self.0)
}
}

@ -1,242 +0,0 @@
//! Execution environment for repl
use super::*;
use std::str::FromStr;
use std::collections::{BTreeMap, HashMap};
#[derive(Debug)]
pub struct Lexenv
{
/// Maps symbol name to value in generations.
kvstack: BTreeMap<usize, HashMap<String, Value>>,
/// Current generation of the satck
current_generation: usize,
}
impl Lexenv
{
/// The generation number of the current level.
pub fn depth(&self) -> usize
{
self.current_generation
}
/// Create a new empty lexenv
pub fn new() -> Self
{
Self {
kvstack: BTreeMap::new(),
current_generation: 0,
}
}
/// All valid symbols at this level.
///
/// # Ordering
/// Each symbol's level will appear in the order from level 0 to the current level, however the order of intra-level symbols is undefined.
pub fn symbols(&self) -> impl Iterator<Item = &'_ Value> + '_
{
self.kvstack.range(0..=self.current_generation).flat_map(|(_, v)| v.values())
}
/// All valid symbols **in** this level.
pub fn symbols_local(&self) -> impl Iterator<Item = &'_ Value> + '_
{
OptionIterator::from(self.kvstack.get(&self.current_generation).map(|x| x.values()))
}
/// Remove the current level, but leave its memory allocated for further use.
pub fn pop(&mut self)
{
self.kvstack.entry(self.current_generation).or_insert_with(|| HashMap::new()).clear();
if self.current_generation > 0 {
self.current_generation-=1;
}
}
/// Remove a symbol from the **current** level.
pub fn remove(&mut self, name: &str) -> Option<Value>
{
self.kvstack.entry(self.current_generation).or_insert_with(|| HashMap::new()).remove(name)
}
/// Insert a new value mapping into the current level.
pub fn insert(&mut self, name: String, value: Value)
{
self.kvstack.entry(self.current_generation).or_insert_with(|| HashMap::new()).insert(name, value);
}
/// Look up a symbol in this or any of the above levels.
pub fn lookup(&self, name: &str) -> Option<&Value>
{
for (_, lvmap) in self.kvstack.range(0..=self.current_generation).rev()
{
let m = lvmap.get(name);
if m.is_some() {
return m;
}
}
None
}
/// Look up a symbol in this level.
pub fn lookup_local(&self, name: &str) -> Option<&Value>
{
self.kvstack.get(&self.current_generation).map(|map| map.get(name)).flatten()
}
/// Create a new, empty level.
pub fn push(&mut self)
{
self.current_generation+=1;
}
/// Remove the current level, deallocating any memory it was using.
pub fn pop_clear(&mut self)
{
if self.current_generation > 0 {
self.kvstack.remove(&self.current_generation);
self.current_generation -=1;
} else {
self.kvstack.entry(0).or_insert_with(|| HashMap::new()).clear();
}
}
}
/// The value type
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum Value
{
String(String),
Symbol(String),
List(Vec<Value>),
}
impl Value
{
/// Parse from an iterator of `char`s.
pub fn parse_chars<T>(ch: &mut T) -> Result<Self, ValueParseError>
where T: Iterator<Item = char>
{
match ch.next()
{
Some('(') => {
todo!("list");
},
Some('"') => {
todo!("string");
},
Some(first_chr) => {
todo!("symbol");
},
_ => Err(ValueParseError(String::default())),
}
}
/// Parse a `Value` from this string and then return the rest of the string.
#[deprecated]
pub fn parse_running(s: &str) -> Result<(Self, &'_ str), ValueParseError>
{
match s.trim().as_bytes()
{
& [b'(', ..] => {
todo!("list");
},
& [b'"', ..] => {
todo!("string");
},
_ => {
todo!("shmbol");
}
}
}
}
impl FromStr for Value
{
type Err = ValueParseError;
fn from_str(s: &str) -> Result<Self, Self::Err> {
Self::parse_running(s).map(|(x, _)| x)
}
}
impl Value
{
pub fn try_as_symbol(&self) -> Result<&str, ValueTypeError>
{
match self {
Self::Symbol(s) => Ok(&s[..]),
_ => Err(ValueTypeError::Symbol),
}
}
pub fn try_as_string(&self) -> Result<&str, ValueTypeError>
{
match self {
Self::Symbol(s) |
Self::String(s) => Ok(&s[..]),
_ => Err(ValueTypeError::String),
}
}
pub fn try_as_list(&self) -> Result<&[Value], ValueTypeError>
{
match self {
Self::List(l) => Ok(&l[..]),
_ => Err(ValueTypeError::List),
}
}
pub fn as_symbol(&self) -> Option<&str>
{
match self {
Self::Symbol(s) => Some(&s[..]),
_ => None,
}
}
pub fn as_string(&self) -> Option<&str>
{
match self {
Self::Symbol(s) |
Self::String(s) => Some(&s[..]),
_ => None,
}
}
pub fn as_list(&self) -> Option<&[Value]>
{
match self {
Self::List(l) => Some(&l[..]),
_ => None,
}
}
}
/// Error when using `try_as_*` functions on `Value`.
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Copy)]
pub enum ValueTypeError
{
Symbol,
String,
List,
}
/// Error when parsing a `Value` from a stirng.
#[derive(Debug)]
pub struct ValueParseError(String);
impl error::Error for ValueParseError{}
impl fmt::Display for ValueParseError
{
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result
{
write!(f, "cannot parse {:?}", self.0)
}
}
impl error::Error for ValueTypeError{}
impl fmt::Display for ValueTypeError
{
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result
{
write!(f, "type error: expected ")?;
match self {
Self::Symbol => write!(f, "symbol"),
Self::String => write!(f, "string"),
Self::List => write!(f, "list"),
}
}
}

@ -1,13 +0,0 @@
//! Defined commands
use super::*;
use env::*;
use command::*;
/// Contains all operations
#[derive(Debug, Clone)]
pub struct Operations
{
}

@ -1,11 +1,9 @@
#![feature(try_blocks)] #![feature(never_type)]
#![allow(dead_code)] #![allow(dead_code)]
#[macro_use] extern crate pin_project; #[macro_use] extern crate pin_project;
#[macro_use] extern crate lazy_static; #[macro_use] extern crate lazy_static;
#[macro_use] extern crate cfg_if;
#[macro_use] extern crate ad_hoc_iter;
#[cfg(feature="inspect")] use serde::{Serialize, Deserialize}; #[cfg(feature="inspect")] use serde::{Serialize, Deserialize};
@ -31,7 +29,6 @@ use color_eyre::{
#[macro_use] mod ext; #[macro_use] mod ext;
pub use ext::prelude::*; pub use ext::prelude::*;
use std::sync::Arc;
mod bytes; mod bytes;
mod data; mod data;
@ -39,24 +36,46 @@ mod config;
mod state; mod state;
mod arg; mod arg;
mod work; mod work;
mod info;
#[cfg(feature="inspect")] mod serial; #[cfg(feature="inspect")] mod serial;
#[cfg(feature="defer-drop")] mod defer_drop; #[cfg(feature="defer-drop")] mod defer_drop;
#[cfg(feature="inspect")] async fn normal(cfg: config::Config) -> eyre::Result<()>
async fn write_graph(graph: Arc<data::HierarchicalINodeGraph>) -> eyre::Result<()>
{ {
let cfg = config::get_global(); let state = state::State::new(cfg
match cfg.serialise_output.as_ref().map(|ser_out| { .validate()
cfg_eprintln!(Verbose; cfg, "Writing graph to stream..."); .wrap_err(eyre!("Invalid config"))
.with_suggestion(|| "Try running `--help`")?);
let (graph, cfg) = tokio::select!{
x = work::work_on_all(state) => {x}
_ = tokio::signal::ctrl_c() => {
return Err(eyre!("Interrupt signalled, exiting"));
}
};
type BoxedWrite = Box<dyn tokio::io::AsyncWrite + Unpin + Send + Sync>; let cfg = cfg.make_global();
let graph = tokio::task::spawn_blocking(move || {
cfg_println!(Verbose; cfg, "Computing hierarchy...");
let mut graph = graph.into_hierarchical();
cfg_println!(Verbose; cfg, "Computing sizes...");
graph.compute_recursive_sizes();
graph
}).await.expect("Failed to compute hierarchy from graph");
#[cfg(debug_assertions)] cfg_eprintln!(Verbose; cfg, "{:?}", graph);
cfg_println!(Quiet; cfg, "Max size file: {:?}", graph.path_max_size_for(data::FsKind::File));
cfg_println!(Quiet; cfg, "Max size dir: {:?}", graph.path_max_size_for(data::FsKind::Directory));
cfg_println!(Quiet; cfg, "Max size all: {:?}", graph.path_max_size());
#[cfg(feature="inspect")]
match cfg.serialise_output.as_ref().map(|ser_out| {
type BoxedWrite = Box<dyn tokio::io::AsyncWrite + Unpin>;
use futures::FutureExt; use futures::FutureExt;
use config::OutputSerialisationMode; use config::OutputSerialisationMode;
let should_comp = ser_out.should_compress();
match ser_out { match ser_out {
OutputSerialisationMode::File(output_file) | OutputSerialisationMode::File(output_file) => {
OutputSerialisationMode::RawFile(output_file) => {
use tokio::fs::OpenOptions; use tokio::fs::OpenOptions;
(async move { (async move {
let stream = OpenOptions::new() let stream = OpenOptions::new()
@ -68,27 +87,25 @@ async fn write_graph(graph: Arc<data::HierarchicalINodeGraph>) -> eyre::Result<(
.with_section(|| format!("{:?}", output_file).header("File was"))?; .with_section(|| format!("{:?}", output_file).header("File was"))?;
Ok::<BoxedWrite, eyre::Report>(Box::new(stream)) Ok::<BoxedWrite, eyre::Report>(Box::new(stream))
}.boxed(), None, should_comp) }.boxed(), None)
}, },
OutputSerialisationMode::Stdout | OutputSerialisationMode::Stdout => (async move { Ok::<BoxedWrite, _>(Box::new(tokio::io::stdout())) }.boxed(), None),
OutputSerialisationMode::RawStdout => (async move { Ok::<BoxedWrite, _>(Box::new(tokio::io::stdout())) }.boxed(), Option::<&std::path::PathBuf>::None, should_comp),
#[cfg(feature="prealloc")] OutputSerialisationMode::PreallocFile(output_file) => { #[cfg(feature="prealloc")] OutputSerialisationMode::PreallocFile(output_file) => {
(async move { (async move {
Ok::<BoxedWrite, _>(Box::new(tokio::io::sink())) // we use a sink as a shim stream since it will never be used when tuple item `.1` is Some() Ok::<BoxedWrite, _>(Box::new(tokio::io::sink())) // we use a sink as a shim stream since it will never be used when tuple item `.1` is Some()
}.boxed(), Some(output_file), false) }.boxed(), Some(output_file))
}, },
} }
}) { }) {
// We use tuple item `.1` here to indicate if we're in normal write mode. // We use tuple item `.1` here to indicate if we're in normal write mode.
// None -> normal // None -> normal
// Some(path) -> prealloc // Some(path) -> prealloc
// `.2` indicates if we should compress while in normal write mode. Some((stream_fut, None)) => {
Some((stream_fut, None, compress)) => {
let stream = stream_fut.await?; let stream = stream_fut.await?;
serial::write_async(stream, graph.as_ref(), compress).await serial::write_async(stream, &graph, serial::compress::No).await
.wrap_err(eyre!("Failed to serialise graph to stream"))?; .wrap_err(eyre!("Failed to serialise graph to stream"))?;
}, },
#[cfg(feature="prealloc")] Some((_task_fut, Some(output_file), _)) => { #[cfg(feature="prealloc")] Some((_task_fut, Some(output_file))) => {
use tokio::fs::OpenOptions; use tokio::fs::OpenOptions;
let file = OpenOptions::new() let file = OpenOptions::new()
.write(true) .write(true)
@ -99,9 +116,8 @@ async fn write_graph(graph: Arc<data::HierarchicalINodeGraph>) -> eyre::Result<(
.wrap_err(eyre!("Failed to open file for mapping")) .wrap_err(eyre!("Failed to open file for mapping"))
.with_section(|| format!("{:?}", output_file).header("File was"))?; .with_section(|| format!("{:?}", output_file).header("File was"))?;
let mut file = file.into_std().await; let mut file = file.into_std().await;
cfg_eprintln!(Verbose; cfg, "Opened file for prealloc-write");
tokio::task::spawn_blocking(move || { tokio::task::spawn_blocking(move || {
serial::write_sync_map(&mut file, graph.as_ref()) serial::write_sync_map(&mut file, &graph)
}).await.wrap_err(eyre!("Prealloc panicked while dumping")) }).await.wrap_err(eyre!("Prealloc panicked while dumping"))
.with_section(|| format!("{:?}", output_file).header("File was"))? .with_section(|| format!("{:?}", output_file).header("File was"))?
.wrap_err(eyre!("Prealloc failed to dump graph to file")) .wrap_err(eyre!("Prealloc failed to dump graph to file"))
@ -109,99 +125,15 @@ async fn write_graph(graph: Arc<data::HierarchicalINodeGraph>) -> eyre::Result<(
}, },
_ => (), _ => (),
} }
Ok(())
}
async fn normal(cfg: config::Config) -> eyre::Result<()>
{
let state = state::State::new(cfg
.validate()
.wrap_err(eyre!("Invalid config"))
.with_suggestion(|| "Try running `--help`")?);
let (graph, cfg) = tokio::select!{
x = work::work_on_all(state) => {x}
_ = tokio::signal::ctrl_c() => {
return Err(eyre!("Interrupt signalled, exiting"));
}
};
let cfg = cfg.make_global();
let (treemap, graph) = tokio::task::spawn_blocking(move || {
cfg_println!(Verbose; cfg, "Computing hierarchy...");
// Create treemap
#[cfg(feature="treemap")]
let treemap = {
//Check config for if the treemap flag is present. If it is, get the width and height from there too.
if let Some(&(x,y)) = cfg.inspection.treemap.as_ref() {
Some(info::treemap(cfg, &graph, (
x as f64,
y as f64,
)))
} else {
None
}
};
#[cfg(not(feature="treemap"))]
let treemap = Option::<std::convert::Infallible>::None;
// Create recursive graph
let mut graph = graph.into_hierarchical();
cfg_println!(Verbose; cfg, "Computing sizes...");
graph.compute_recursive_sizes();
(treemap, graph)
}).await.expect("Failed to compute hierarchy from graph");
//#[cfg(debug_assertions)] cfg_eprintln!(Verbose; cfg, "{:?}", graph);
#[allow(unused_imports)] use futures::future::OptionFuture;
let (graph, writer) = {
cfg_if!{
if #[cfg(feature="inspect")] {
let graph = Arc::new(graph);
let (g, w) = if cfg.serialise_output.is_some() {
(Arc::clone(&graph), Some(tokio::spawn(write_graph(graph))))
} else {
(graph, None)
};
(g, OptionFuture::from(w))
} else {
(graph, ())//OptionFuture::from(Option::<futures::future::BoxFuture<'static, ()>>::None))
}
}
};
#[cfg(feature="treemap")]
if let Some(treemap) = treemap {
//TODO: Print treemap
todo!("{:?}",treemap);
}
info::print_basic_max_info(&cfg, &graph);
#[cfg(feature="inspect")]
match writer.await {
Some(Ok(Ok(_))) if cfg.serialise_output.is_some() => cfg_eprintln!(Verbose; cfg, "Written successfully"),
Some(Ok(error @ Err(_))) => return error.wrap_err(eyre!("Failed to write graph to output stream")),
Some(Err(_)) => cfg_eprintln!(Silent; cfg, "Panic while writing graph to stream"),
_ => (),
}
#[cfg(not(feature="inspect"))] drop(writer);
Ok(()) Ok(())
} }
async fn parse_mode() -> eyre::Result<()> async fn parse_mode() -> eyre::Result<()>
{ {
match arg::parse_args() match arg::parse_args().wrap_err(eyre!("Failed to parse args"))?
.wrap_err(eyre!("Failed to parse args"))
.with_suggestion(|| "Try running `--help`")?
{ {
arg::Mode::Normal(cfg) => { arg::Mode::Normal(cfg) => {
#[cfg(debug_assertions)] eprintln!("cfg: {:#?}", cfg);
normal(cfg).await normal(cfg).await
}, },
arg::Mode::Help => arg::help(), arg::Mode::Help => arg::help(),

@ -1,8 +1,6 @@
//! For serializing //! For serializing
use super::*; use super::*;
use tokio::prelude::*; use tokio::prelude::*;
use std::ops::{Deref, DerefMut};
use serde::de::DeserializeOwned;
use async_compression::tokio_02::write::{ use async_compression::tokio_02::write::{
BzEncoder, BzEncoder,
@ -13,141 +11,58 @@ type Compressor<T> = BzEncoder<T>;
type Decompressor<T> = BzDecoder<T>; type Decompressor<T> = BzDecoder<T>;
const DEFER_DROP_SIZE_FLOOR: usize = 1024 * 1024; // 1 MB const DEFER_DROP_SIZE_FLOOR: usize = 1024 * 1024; // 1 MB
const DESERIALISE_OBJECT_READ_LIMIT: usize = 1024 * 1024 * 1024 * 2; // 2GB
const BUFFER_SIZE: usize = 4096; pub trait Compression
#[derive(Debug)]
enum MaybeCompressor<'a, T>
{ {
Compressing(Compressor<&'a mut T>), type OutputStream: AsyncWrite + Unpin;
Decompressing(Decompressor<&'a mut T>), type InputStream: AsyncRead + Unpin;
Raw(&'a mut T),
}
/// Compress or decompress? fn create_output<W: AsyncWrite+ Unpin>(from: W) -> Result<Self::OutputStream, W>;
#[derive(Debug, Clone, PartialEq, Eq, Hash, Copy, PartialOrd, Ord)] fn create_input<W: AsyncRead+ Unpin>(from: W) -> Result<Self::InputStream, W>;
enum CompKind
{
Compress,
Decompress
} }
impl Default for CompKind pub mod compress
{ {
#[inline]
fn default() -> Self
{
Self::Compress
}
}
impl<'a, T> MaybeCompressor<'a, T> use super::*;
{ /// No compression.
/// What kind is this compressor set to #[derive(Debug)]
pub fn kind(&self) -> Option<CompKind> pub struct No;
{
Some(match self {
Self::Raw(_) => return None,
Self::Compressing(_) => CompKind::Compress,
Self::Decompressing(_) => CompKind::Decompress,
})
}
}
impl<'a, T> MaybeCompressor<'a, T> impl Compression for No
where T: AsyncWrite +Send + Unpin + 'a
{
pub fn new(raw: &'a mut T, compress: Option<CompKind>) -> Self
{ {
match compress { type OutputStream = DeadSink;
Some(CompKind::Compress) => Self::Compressing(Compressor::new(raw)), type InputStream = DeadSink;
Some(CompKind::Decompress) => Self::Decompressing(Decompressor::new(raw)),
None => Self::Raw(raw),
}
}
}
impl<'a, T> DerefMut for MaybeCompressor<'a, T> fn create_input<W: AsyncRead+ Unpin>(from: W) -> Result<Self::InputStream, W> {
where T: AsyncWrite + Send + Unpin + 'a Err(from)
{
fn deref_mut(&mut self) -> &mut Self::Target {
match self {
Self::Compressing(t) => t,
Self::Decompressing(t) => t,
Self::Raw(o) => o,
} }
} fn create_output<W: AsyncWrite+ Unpin>(from: W) -> Result<Self::OutputStream, W> {
} Err(from)
impl<'a, T> Deref for MaybeCompressor<'a, T>
where T: AsyncWrite + Unpin + Send + 'a
{
type Target = dyn AsyncWrite + Send + Unpin+ 'a;
fn deref(&self) -> &Self::Target {
match self {
Self::Compressing(t) => t,
Self::Decompressing(t) => t,
Self::Raw(o) => o,
} }
} }
}
async fn copy_with_limit<R,W>(mut from: R, mut to: W) -> io::Result<usize> #[derive(Debug)]
where R: AsyncRead + Unpin, pub struct Bz;
W: AsyncWrite + Unpin
{
let mut buffer = [0u8; BUFFER_SIZE];
let mut read;
let mut done=0; impl Compression for Bz
while {read = from.read(&mut buffer[..]).await?; read>0}
{ {
to.write_all(&buffer[..read]).await?; type OutputStream = Box<dyn AsyncWrite + Unpin>;
done+=read; type InputStream = Box<dyn AsyncRead + Unpin>;
if done > DESERIALISE_OBJECT_READ_LIMIT { fn create_input<W: AsyncRead+ Unpin>(from: W) -> Result<Self::InputStream, W> {
return Err(io::Error::new(io::ErrorKind::ConnectionAborted, eyre!("Exceeded limit, aborting.") panic!()
.with_section(|| DESERIALISE_OBJECT_READ_LIMIT.header("Object read size limit was")) }
.with_section(|| done.header("Currently read")))); fn create_output<W: AsyncWrite+ Unpin>(from: W) -> Result<Self::OutputStream, W> {
Ok(Box::new(super::Compressor::new(from)))
} }
} }
Ok(done)
} }
/// Deserialise an object from this stream asynchronously #[inline] fn _type_name<T: ?Sized>(_val: &T) -> &'static str {
/// std::any::type_name::<T>()
/// # Note
/// If the stream is compressed, `compressed` must be set to true or an error will be produced.
/// An autodetect feature may be added in the future
pub async fn read_async<T: DeserializeOwned + Send + 'static, R>(mut from: R, compressed: bool) -> eyre::Result<T>
where R: AsyncRead + Unpin + Send
{
let sect_type_name = || std::any::type_name::<T>().header("Type trying to deserialise was");
let sect_stream_type_name = || std::any::type_name::<R>().header("Stream type was");
let vec = {
let mut vec = Vec::new();
let mut writer = MaybeCompressor::new(&mut vec, compressed.then(|| CompKind::Decompress));
copy_with_limit(&mut from, writer.deref_mut()).await
.wrap_err(eyre!("Failed to copy stream into in-memory buffer"))
.with_section(sect_type_name.clone())
.with_section(sect_stream_type_name.clone())?;
writer.flush().await.wrap_err(eyre!("Failed to flush decompression stream"))?;
writer.shutdown().await.wrap_err(eyre!("Failed to shutdown decompression stream"))?;
vec
};
tokio::task::spawn_blocking(move || {
(serde_cbor::from_slice(&vec[..])
.wrap_err(eyre!("Failed to deseralised decompressed data"))
.with_section(sect_type_name.clone())
.with_section(sect_stream_type_name.clone()),
{drop!(vec vec);}).0
}).await.wrap_err(eyre!("Panic while deserialising decompressed data"))?
} }
/// Serialise this object asynchronously /// Serialise this object asynchronously
@ -155,11 +70,13 @@ where R: AsyncRead + Unpin + Send
/// # Note /// # Note
/// This compresses the output stream. /// This compresses the output stream.
/// It cannot be used by `prealloc` read/write functions, as they do not compress. /// It cannot be used by `prealloc` read/write functions, as they do not compress.
pub async fn write_async<T: Serialize, W>(mut to: W, item: &T, compress: bool) -> eyre::Result<()> pub async fn write_async<Compress: Compression>(mut to: impl AsyncWrite + Unpin, item: &impl Serialize, _comp: Compress) -> eyre::Result<()>
where W: AsyncWrite + Unpin + Send
{ {
let sect_type_name = || std::any::type_name::<T>().header("Type trying to serialise was"); let name_of_item = _type_name(item);
let sect_stream_type_name = || std::any::type_name::<W>().header("Stream type was"); let name_of_stream = _type_name(&to);
let sect_type_name = || name_of_item.header("Type trying to serialise was");
let sect_stream_type_name = || name_of_stream.header("Stream type was");
let vec = tokio::task::block_in_place(|| serde_cbor::to_vec(item)) let vec = tokio::task::block_in_place(|| serde_cbor::to_vec(item))
.wrap_err(eyre!("Failed to serialise item")) .wrap_err(eyre!("Failed to serialise item"))
@ -167,9 +84,9 @@ where W: AsyncWrite + Unpin + Send
.with_section(sect_type_name.clone())?; .with_section(sect_type_name.clone())?;
{ {
let mut stream = MaybeCompressor::new(&mut to, compress.then(|| CompKind::Compress)); let mut stream: EitherWrite<_, _> = Compress::create_output(&mut to).into();//Compressor::new(&mut to);
cfg_eprintln!(Verbose; config::get_global(), "Writing {} bytes of type {:?} to stream of type {:?}", vec.len(), std::any::type_name::<T>(), std::any::type_name::<W>()); cfg_eprintln!(Verbose; config::get_global(), "Writing {} bytes of type {:?} to stream of type {:?}", vec.len(), name_of_item, name_of_stream);
stream.write_all(&vec[..]) stream.write_all(&vec[..])
.await .await
@ -223,9 +140,6 @@ mod prealloc {
.with_section(sect_type_name.clone())?; .with_section(sect_type_name.clone())?;
let fd = file.as_raw_fd(); let fd = file.as_raw_fd();
cfg_eprintln!(Verbose; config::get_global(), "Writing (raw) {} bytes of type {:?} to fd {}", vec.len(), std::any::type_name::<T>(), fd);
unsafe { unsafe {
if libc::fallocate(fd, 0, 0, vec.len().try_into() if libc::fallocate(fd, 0, 0, vec.len().try_into()
.wrap_err(eyre!("Failed to cast buffer size to `off_t`")) .wrap_err(eyre!("Failed to cast buffer size to `off_t`"))
@ -272,7 +186,7 @@ mod prealloc {
pub fn read_prealloc<T: serde::de::DeserializeOwned>(file: &File) -> eyre::Result<T> pub fn read_prealloc<T: serde::de::DeserializeOwned>(file: &File) -> eyre::Result<T>
{ {
let map = unsafe { Mmap::map(file) } let map = unsafe { Mmap::map(file) }
.wrap_err(eyre!("Failed to map file for read")) .wrap_err(eyre!("Failed to map file for read + write"))
.with_section(|| file.as_raw_fd().header("fd was")) .with_section(|| file.as_raw_fd().header("fd was"))
.with_suggestion(|| "Do we have the premissions for both reading and writing of this file and fd?")?; .with_suggestion(|| "Do we have the premissions for both reading and writing of this file and fd?")?;

@ -1,4 +1,5 @@
use super::*; use super::*;
use std::sync::Arc;
use tokio::sync::{ use tokio::sync::{
Semaphore, Semaphore,

@ -88,9 +88,7 @@ fn walk(state: State, root: PathBuf, root_ino: INode) -> BoxFuture<'static, Hash
async move { async move {
{ {
let _guard = state.lock().enter().await; let _guard = state.lock().enter().await;
cfg_println!(state.config(), " -> {:?}", root);
cfg_println!(state.config(), " -> {:?}", root); //TODO: Flag to disable this?
match fs::read_dir(&root).await match fs::read_dir(&root).await
{ {
Ok(mut dir) => { Ok(mut dir) => {

Loading…
Cancel
Save