From ff898fc9b34d5f1a22463f56d1420e29aea8e6be Mon Sep 17 00:00:00 2001 From: Avril Date: Mon, 1 Apr 2024 15:24:16 +0100 Subject: [PATCH] Added feature `mapped-file`: Attempt to `mmap()` input where possible to avoid reads. (NOTE: This should be replaced with a cfg-target-unix check, for now it is a default feature that can be disabled.) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Started module `part`: Partitioning of memory areas (halve-then-`{r,m}emchr("\n")` pivot calculation) (parallelised via rayon + crossbeam-queue (+maybe tokio_uring?) if `threads` feature enabled. Fortune for reverse's current commit: Middle blessing − 中吉 --- Cargo.toml | 6 ++---- src/main.rs | 2 ++ src/part.rs | 3 +++ 3 files changed, 7 insertions(+), 4 deletions(-) create mode 100644 src/part.rs diff --git a/Cargo.toml b/Cargo.toml index 35d4339..ac9d24c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,7 @@ edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [features] -default = [] +default = ["mapped-file"] unstable = ["parking_lot?/nightly", "memchr/compiler_builtins", "crossbeam-queue?/nightly", "futures?/unstable", "color-eyre/track-caller"] @@ -32,9 +32,7 @@ threads-async = ["threads", "dep:futures", "dep:tokio-uring", "dep:tokio-stream" #XXX ["output-lines", "buffer-output", "ignore-output-errors", "ignore-invalid-args"] -#TODO: Branch repo, and begin refactor: -# -#TODO: Move these features to Clap runtime arg parse options; Major refactor, bump to 0.5.0 or to 1.0.0[r0] +#TODO: Move these features to Clap runtime arg parse options. #TODO: Add feature 'mapped-file': Attempt to map input file and process data backwards from end if possible. #TODO: Add feature 'threads': Use rayon to parallelise line-searching in buffer (or mapped file ^) (partition in half, move partition line to closest {r,}memchr('\n'); repeat N times recursively on each side (parallel) where N is number of logical CPU cores and/or to get the chunks below/in a certain size range. diff --git a/src/main.rs b/src/main.rs index 53b824c..144838a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,4 +1,6 @@ +mod part; + //#[inline] fn reverse(slice: &mut [T]) { diff --git a/src/part.rs b/src/part.rs new file mode 100644 index 0000000..d003187 --- /dev/null +++ b/src/part.rs @@ -0,0 +1,3 @@ +//! Partitioning even areas by delimitor byte. +use super::*; +