Compare commits
11 Commits
Author | SHA1 | Date |
---|---|---|
Avril | c054ac0174 | 3 years ago |
Avril | bd11a53947 | 3 years ago |
Avril | 24475646a1 | 3 years ago |
Avril | c5cf1106c3 | 3 years ago |
Avril | 204f511075 | 3 years ago |
Avril | 39607dc97c | 3 years ago |
Avril | 114472559e | 3 years ago |
Avril | 55ea30fcf8 | 3 years ago |
Avril | a4674cea6d | 3 years ago |
Avril | cad4decab5 | 3 years ago |
Avril | 5c889fed0d | 3 years ago |
@ -0,0 +1,134 @@
|
||||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
version = 3
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"wasi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lazy_static"
|
||||
version = "1.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.86"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b7282d924be3275cec7f6756ff4121987bc6481325397dde6ba3e7802b1a8b1c"
|
||||
|
||||
[[package]]
|
||||
name = "memmap"
|
||||
version = "0.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ppv-lite86"
|
||||
version = "0.2.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857"
|
||||
|
||||
[[package]]
|
||||
name = "rand"
|
||||
version = "0.8.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0ef9e7e66b4468674bfcb0c81af8b7fa0bb154fa9f28eb840da5c447baeb8d7e"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"rand_chacha",
|
||||
"rand_core",
|
||||
"rand_hc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_chacha"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e12735cf05c9e10bf21534da50a147b924d555dc7a547c42e6bb2d5b6017ae0d"
|
||||
dependencies = [
|
||||
"ppv-lite86",
|
||||
"rand_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_core"
|
||||
version = "0.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c026d7df8b298d90ccbbc5190bd04d85e159eaf5576caeacf8741da93ccbd2e5"
|
||||
dependencies = [
|
||||
"getrandom",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_hc"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3190ef7066a446f2e7f42e239d161e905420ccab01eb967c9eb27d21b2322a73"
|
||||
dependencies = [
|
||||
"rand_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "shuffle3rs"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"lazy_static",
|
||||
"memmap",
|
||||
"rand",
|
||||
"rand_chacha",
|
||||
"smallvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "smallvec"
|
||||
version = "1.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e"
|
||||
|
||||
[[package]]
|
||||
name = "wasi"
|
||||
version = "0.10.2+wasi-snapshot-preview1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"
|
||||
|
||||
[[package]]
|
||||
name = "winapi"
|
||||
version = "0.3.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
|
||||
dependencies = [
|
||||
"winapi-i686-pc-windows-gnu",
|
||||
"winapi-x86_64-pc-windows-gnu",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winapi-i686-pc-windows-gnu"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
|
||||
|
||||
[[package]]
|
||||
name = "winapi-x86_64-pc-windows-gnu"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
|
@ -0,0 +1,37 @@
|
||||
[package]
|
||||
name = "shuffle3rs"
|
||||
version = "0.1.0"
|
||||
authors = ["Avril <flanchan@cumallover.me>"]
|
||||
edition = "2018"
|
||||
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[profile.test]
|
||||
# Needed for tail call opt, to not cause stack overflow.
|
||||
opt-level = 1
|
||||
|
||||
[profile.dev]
|
||||
# Needed for tail call opt, to not cause stack overflow.
|
||||
opt-level = 1
|
||||
|
||||
[features]
|
||||
default = ["splash", "deferred-drop", "threads"]
|
||||
|
||||
# Print name and info when printing help info
|
||||
splash = []
|
||||
|
||||
# Process multiple inputs concurrently
|
||||
threads = []
|
||||
|
||||
# Move large objects to a seperate thread to be dropped.
|
||||
deferred-drop = []
|
||||
|
||||
[dependencies]
|
||||
cfg-if = "1.0.0"
|
||||
lazy_static = "1.4.0"
|
||||
memmap = "0.7.0"
|
||||
rand = "0.8.3"
|
||||
smallvec = {version = "1.6.1", features=["union"]}
|
||||
|
||||
[dev-dependencies]
|
||||
rand_chacha = "0.3.0"
|
@ -0,0 +1,122 @@
|
||||
use super::*;
|
||||
|
||||
use std::{fmt, error};
|
||||
use std::path::PathBuf;
|
||||
|
||||
/// The executable program name
|
||||
pub fn program_name() -> &'static str
|
||||
{
|
||||
lazy_static! {
|
||||
static ref NAME: String = std::env::args().next().unwrap();
|
||||
}
|
||||
&NAME[..]
|
||||
}
|
||||
|
||||
#[cfg(feature="splash")]
|
||||
#[inline] fn splash()
|
||||
{
|
||||
println!("shuffle3rs (v{}) - improved 3 pass binary shuffler (Rust ver)", env!("CARGO_PKG_VERSION"));
|
||||
println!(" written by {} with <3", env!("CARGO_PKG_AUTHORS"));
|
||||
println!(" licensed with GPL v3.0 or later\n");
|
||||
}
|
||||
|
||||
/// Print usage message
|
||||
pub fn usage()
|
||||
{
|
||||
#[cfg(feature="splash")] splash();
|
||||
|
||||
println!("Usage: {} -s <file>", program_name());
|
||||
println!("Usage: {} -u <file>", program_name());
|
||||
println!("\nOPTIONS:");
|
||||
println!(" -s\tShuffle file in place");
|
||||
println!(" -u\tUnshuffle file in place");
|
||||
}
|
||||
|
||||
/// Print usage message and then exit with error code `2`.
|
||||
#[inline] pub fn help() -> !
|
||||
{
|
||||
usage();
|
||||
|
||||
std::process::exit(2)
|
||||
}
|
||||
|
||||
pub enum Mode
|
||||
{
|
||||
Help,
|
||||
ShuffleInPlace(MaybeVec<PathBuf>),
|
||||
UnshuffleInPlace(MaybeVec<PathBuf>),
|
||||
}
|
||||
|
||||
/// Parse the `argv` of this process.
|
||||
#[inline] pub fn parse_args() -> Result<Mode, Error>
|
||||
{
|
||||
parse(std::env::args().skip(1))
|
||||
}
|
||||
|
||||
fn parse<I: IntoIterator<Item = String>>(args: I) -> Result<Mode, Error>
|
||||
{
|
||||
#[inline] fn validate_file(p: impl Into<PathBuf>) -> Result<PathBuf, Error>
|
||||
{
|
||||
let p = p.into();
|
||||
if p.is_file()
|
||||
{
|
||||
Ok(p)
|
||||
} else {
|
||||
Err(Error::FileNotFound(p))
|
||||
}
|
||||
}
|
||||
|
||||
let mut args = args.into_iter();
|
||||
|
||||
let farg = args.next();
|
||||
match farg
|
||||
{
|
||||
Some(yes) => {
|
||||
match yes.trim()
|
||||
{
|
||||
"-h" => Ok(Mode::Help),
|
||||
"-s" => {
|
||||
Ok(Mode::ShuffleInPlace(
|
||||
args
|
||||
.map(|file| validate_file(file)).collect::<Result<MaybeVec<_>, Error>>()
|
||||
.and_then(|v| if v.len() == 0 { Err(Error::InvalidUseArg("-s"))} else {Ok(v)})?))
|
||||
},
|
||||
"-u" => {
|
||||
Ok(Mode::UnshuffleInPlace(
|
||||
args
|
||||
.map(|file| validate_file(file)).collect::<Result<MaybeVec<_>, Error>>()
|
||||
.and_then(|v| if v.len() == 0 { Err(Error::InvalidUseArg("-u"))} else {Ok(v)})?))
|
||||
|
||||
},
|
||||
_ => Err(Error::UnknownOpt(yes))
|
||||
}
|
||||
}
|
||||
None => Err(Error::NoOpt),
|
||||
}
|
||||
}
|
||||
|
||||
/// Arg parsing error
|
||||
#[derive(Debug)]
|
||||
#[non_exhaustive]
|
||||
pub enum Error
|
||||
{
|
||||
InvalidUseArg(&'static str),
|
||||
NoOpt,
|
||||
|
||||
UnknownOpt(String),
|
||||
FileNotFound(PathBuf),
|
||||
}
|
||||
|
||||
impl error::Error for Error{}
|
||||
impl fmt::Display for Error
|
||||
{
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result
|
||||
{
|
||||
match self {
|
||||
Self::NoOpt => write!(f, "No options specified"),
|
||||
Self::UnknownOpt(opt) => write!(f, "Unknown option {:?}", opt),
|
||||
Self::FileNotFound(file) => write!(f, "File {:?} not found", file),
|
||||
Self::InvalidUseArg(string) => write!(f, "Argument {:?} expects a parameter", string),
|
||||
}
|
||||
}
|
||||
}
|
@ -0,0 +1,148 @@
|
||||
use super::*;
|
||||
|
||||
use std::{
|
||||
thread,
|
||||
time::Duration,
|
||||
sync::mpsc,
|
||||
any::Any,
|
||||
marker::{Send, Sync},
|
||||
};
|
||||
|
||||
pub type Defer = Box<dyn Any + Send + 'static>;
|
||||
|
||||
/// A deferred dropping handle.
|
||||
#[derive(Debug, Clone)]
|
||||
#[repr(transparent)]
|
||||
pub struct DeferredDropper(mpsc::Sender<Defer>);
|
||||
|
||||
impl DeferredDropper
|
||||
{
|
||||
/// Drop a Boxed item on the background thread
|
||||
///
|
||||
/// # Panics
|
||||
/// If the background thread has panicked.
|
||||
#[inline] pub fn drop_boxed<T: Any + Send + 'static>(&self, item: Box<T>)
|
||||
{
|
||||
self.0.send(item).unwrap();
|
||||
}
|
||||
/// Drop an item on the background thread
|
||||
///
|
||||
/// # Panics
|
||||
/// If the background thread has panicked.
|
||||
#[inline(always)] pub fn drop<T: Any +Send+ 'static>(&self, item: T)
|
||||
{
|
||||
self.drop_boxed(Box::new(item))
|
||||
}
|
||||
|
||||
/// Send this deferring drop
|
||||
///
|
||||
/// # Panics
|
||||
/// If the background thread has panicked.
|
||||
#[inline(always)] pub fn send(&self, item: impl Into<Defer>)
|
||||
{
|
||||
self.0.send(item.into()).unwrap()
|
||||
}
|
||||
}
|
||||
|
||||
/// Subscribe to the deferred dropper. Usually avoid doing this, and just use the `drop!` macro, or use the thread-local static dropper reference `HANDLE` directly.
|
||||
fn defer_drop_sub() -> DeferredDropper
|
||||
{
|
||||
use std::sync::Mutex; // Can we get rid of this somehow? I don't think we can. Either we mutex lock here, or we mutex lock on every (I think every, should look into that..) send. I think this is preferrable.
|
||||
#[repr(transparent)]
|
||||
struct Shim(Mutex<mpsc::Sender<Defer>>);
|
||||
|
||||
unsafe impl Sync for Shim{}
|
||||
|
||||
lazy_static! {
|
||||
static ref TX: Shim = {
|
||||
let (tx, rx) = mpsc::channel();
|
||||
thread::spawn(move || {
|
||||
for val in rx.into_iter()
|
||||
.lag(Duration::from_millis(10))
|
||||
{
|
||||
//let _ = thread::spawn(move || drop(val)).join(); // To catch panic?
|
||||
drop(val); // What if this panics?
|
||||
}
|
||||
});
|
||||
Shim(Mutex::new(tx))
|
||||
};
|
||||
}
|
||||
|
||||
DeferredDropper(TX.0.lock().unwrap().clone())
|
||||
}
|
||||
|
||||
thread_local! {
|
||||
pub static HANDLE: DeferredDropper = defer_drop_sub();
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests
|
||||
{
|
||||
#[test]
|
||||
fn mac()
|
||||
{
|
||||
use crate::*;
|
||||
let sub = super::defer_drop_sub();
|
||||
|
||||
let large_vec = vec![String::from("hello world"); 1000];
|
||||
|
||||
|
||||
drop!(in sub; large_vec.clone());
|
||||
drop!(large_vec);
|
||||
drop!(box Box::new("hello world?"));
|
||||
drop!(in sub; box Box::new("hello world?"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dropping_larges()
|
||||
{
|
||||
for joiner in std::iter::repeat_with(|| {
|
||||
let large_vec = vec![String::from("hello world"); 1000];
|
||||
let h = {
|
||||
let mut large_vec = large_vec.clone();
|
||||
std::thread::spawn(move || {
|
||||
large_vec.sort();
|
||||
drop!(large_vec);
|
||||
})
|
||||
};
|
||||
drop!(large_vec);
|
||||
h
|
||||
}).take(1000)
|
||||
{
|
||||
joiner.join().unwrap();
|
||||
}
|
||||
std::thread::sleep(std::time::Duration::from_millis(500));
|
||||
}
|
||||
#[test]
|
||||
fn dropping_vec()
|
||||
{
|
||||
let sub = super::defer_drop_sub();
|
||||
|
||||
let large_vec = vec![String::from("hello world"); 1000];
|
||||
|
||||
|
||||
sub.drop(large_vec.clone());
|
||||
sub.drop_boxed(Box::new(large_vec)) //we can't just send boxed slice because MUH SIZED???
|
||||
//FUCK THIS! J)QI EJOAIJAOIW
|
||||
/*
|
||||
unsafe {
|
||||
let raw = Box::into_raw(large_vec.into_boxed_slice());
|
||||
sub.send(Box::from_raw(raw as *mut (dyn std::any::Any + Send + 'static))).unwrap();
|
||||
}*/
|
||||
}
|
||||
#[test]
|
||||
fn clone_shim()
|
||||
{
|
||||
for joiner in std::iter::repeat_with(|| {
|
||||
std::thread::spawn(move || {
|
||||
let mut subs = Vec::new();
|
||||
for _ in 0..100 {
|
||||
subs.push(super::defer_drop_sub());
|
||||
}
|
||||
})
|
||||
}).take(1000)
|
||||
{
|
||||
joiner.join().unwrap();
|
||||
}
|
||||
}
|
||||
}
|
@ -0,0 +1,155 @@
|
||||
use std::time::Duration;
|
||||
use std::iter::{FusedIterator, DoubleEndedIterator};
|
||||
|
||||
use rand::Rng;
|
||||
use crate::shuffle;
|
||||
|
||||
pub type MaybeVec<T> = smallvec::SmallVec<[T; 1]>;
|
||||
|
||||
pub trait SliceElementExt<T>
|
||||
{
|
||||
fn random_element<R: Rng + ?Sized>(&self, rng: &mut R) -> &T;
|
||||
fn random_element_mut<R: Rng + ?Sized>(&mut self, rng: &mut R) -> &mut T;
|
||||
}
|
||||
|
||||
pub trait ShuffleExt<T>
|
||||
{
|
||||
fn shuffle<R: Rng + ?Sized>(&mut self, rng: &mut R);
|
||||
fn unshuffle<R: Rng + ?Sized>(&mut self, rng: &mut R);
|
||||
}
|
||||
|
||||
impl<T> SliceElementExt<T> for [T]
|
||||
{
|
||||
#[inline] fn random_element<R: Rng + ?Sized>(&self, rng: &mut R) -> &T {
|
||||
shuffle::element_in(self, rng)
|
||||
}
|
||||
#[inline] fn random_element_mut<R: Rng + ?Sized>(&mut self, rng: &mut R) -> &mut T {
|
||||
shuffle::element_in_mut(self, rng)
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> ShuffleExt<T> for [T]
|
||||
{
|
||||
#[inline] fn shuffle<R: Rng + ?Sized>(&mut self, rng: &mut R) {
|
||||
shuffle::shuffle(self, rng)
|
||||
}
|
||||
#[inline] fn unshuffle<R: Rng + ?Sized>(&mut self, rng: &mut R) {
|
||||
shuffle::unshuffle(self, rng)
|
||||
}
|
||||
}
|
||||
|
||||
/// A lagged iterator
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Lag<I: ?Sized, T>(Duration, I)
|
||||
where I: Iterator<Item=T>;
|
||||
|
||||
impl<I: ?Sized, T> Lag<I, T>
|
||||
where I: Iterator<Item=T>
|
||||
{
|
||||
/// Set the lag duration
|
||||
#[inline] pub fn set_duration(&mut self, dur: Duration)
|
||||
{
|
||||
self.0 = dur;
|
||||
}
|
||||
/// Get the lag duration
|
||||
#[inline] pub fn duration(&self) -> Duration
|
||||
{
|
||||
self.0
|
||||
}
|
||||
}
|
||||
impl<I, T> Lag<I, T>
|
||||
where I: Iterator<Item=T>
|
||||
{
|
||||
/// Consume into the inner iterator
|
||||
#[inline] pub fn into_inner(self) -> I
|
||||
{
|
||||
self.1
|
||||
}
|
||||
}
|
||||
|
||||
pub trait LagIterExt: Iterator
|
||||
{
|
||||
fn lag(self, dur: Duration) -> Lag<Self, Self::Item>;
|
||||
}
|
||||
|
||||
impl<I> LagIterExt for I
|
||||
where I: Iterator
|
||||
{
|
||||
#[inline] fn lag(self, dur: Duration) -> Lag<Self, Self::Item>
|
||||
{
|
||||
Lag(dur, self)
|
||||
}
|
||||
}
|
||||
|
||||
impl<I: ?Sized, T> Iterator for Lag<I, T>
|
||||
where I: Iterator<Item=T>
|
||||
{
|
||||
type Item = T;
|
||||
fn next(&mut self) -> Option<Self::Item>
|
||||
{
|
||||
std::thread::sleep(self.0);
|
||||
self.1.next()
|
||||
}
|
||||
fn size_hint(&self) -> (usize, Option<usize>) {
|
||||
self.1.size_hint()
|
||||
}
|
||||
}
|
||||
|
||||
impl<I: ?Sized, T> FusedIterator for Lag<I, T>
|
||||
where I: Iterator<Item=T> + FusedIterator{}
|
||||
impl<I: ?Sized, T> ExactSizeIterator for Lag<I, T>
|
||||
where I: Iterator<Item=T> + ExactSizeIterator{}
|
||||
impl<I: ?Sized, T> DoubleEndedIterator for Lag<I, T>
|
||||
where I: Iterator<Item=T> + DoubleEndedIterator
|
||||
{
|
||||
fn next_back(&mut self) -> Option<Self::Item>
|
||||
{
|
||||
std::thread::sleep(self.0);
|
||||
self.1.next()
|
||||
}
|
||||
}
|
||||
|
||||
cfg_if!{
|
||||
|
||||
if #[cfg(feature="deferred-drop")] {
|
||||
/// Drop this item on another thread
|
||||
#[macro_export] macro_rules! drop {
|
||||
(in $sub:ident; box $val:expr) => {
|
||||
$sub.drop_boxed($val)
|
||||
};
|
||||
(in $sub:ident; $val:expr) => {
|
||||
$sub.drop($val)
|
||||
};
|
||||
(box $val:expr) => {
|
||||
{
|
||||
$crate::defer_drop::HANDLE.with(move |sub| {
|
||||
sub.drop($val)
|
||||
});
|
||||
}
|
||||
};
|
||||
($val:expr) => {
|
||||
{
|
||||
$crate::defer_drop::HANDLE.with(move |sub| {
|
||||
sub.drop($val)
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
}
|
||||
} else {
|
||||
/// Drop this item on another thread
|
||||
#[macro_export] macro_rules! drop {
|
||||
(box $val:expr) => {
|
||||
{
|
||||
drop($val)
|
||||
}
|
||||
};
|
||||
($val:expr) => {
|
||||
{
|
||||
drop($val)
|
||||
}
|
||||
};
|
||||
|
||||
}
|
||||
}
|
||||
}
|
@ -0,0 +1,52 @@
|
||||
|
||||
#![allow(dead_code)]
|
||||
|
||||
#[macro_use] extern crate lazy_static;
|
||||
#[macro_use] extern crate cfg_if;
|
||||
|
||||
#[macro_use] mod ext; use ext::*;
|
||||
|
||||
#[cfg(feature="deferred-drop")] mod defer_drop;
|
||||
mod shuffle;
|
||||
mod arg;
|
||||
mod proc;
|
||||
mod rng;
|
||||
|
||||
use arg::Mode;
|
||||
|
||||
/// Handle a fatal error
|
||||
///
|
||||
/// # Method
|
||||
/// On `Err`, print the error message and then exit with error code provided
|
||||
#[inline] fn handle_err_fatal<T, E>(op: Result<T, E>, errcd: i32) -> T
|
||||
where E: std::error::Error
|
||||
{
|
||||
match op
|
||||
{
|
||||
Ok(v) => v,
|
||||
Err(err) => {
|
||||
eprintln!("Fatal error: {}", err);
|
||||
std::process::exit(errcd)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn main() {
|
||||
match arg::parse_args()
|
||||
{
|
||||
Ok(Mode::Help) => arg::usage(),
|
||||
Ok(Mode::ShuffleInPlace(file)) => {
|
||||
handle_err_fatal(proc::process_files_ip(file, proc::ModeKind::Shuffle), -1);
|
||||
},
|
||||
Ok(Mode::UnshuffleInPlace(file)) => {
|
||||
handle_err_fatal(proc::process_files_ip(file, proc::ModeKind::Unshuffle), -1);
|
||||
},
|
||||
Err(err) => {
|
||||
eprintln!("Error: {}", err);
|
||||
eprintln!("\nTry passing `-h`");
|
||||
std::process::exit(1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)] mod test;
|
@ -0,0 +1,86 @@
|
||||
//! Actual processing
|
||||
use super::*;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::io;
|
||||
use std::fs;
|
||||
|
||||
use memmap::MmapMut;
|
||||
|
||||
/// What kind of operation are we doing?
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash, Copy)]
|
||||
pub enum ModeKind
|
||||
{
|
||||
Shuffle,
|
||||
Unshuffle
|
||||
}
|
||||
|
||||
fn load_file_ip(file: impl AsRef<Path>) -> io::Result<memmap::MmapMut>
|
||||
{
|
||||
let file = fs::OpenOptions::new()
|
||||
.read(true)
|
||||
.write(true)
|
||||
.open(file)?;
|
||||
Ok(unsafe {
|
||||
MmapMut::map_mut(&file)?
|
||||
})
|
||||
}
|
||||
|
||||
pub fn shuffle_file_ip(file: impl AsRef<Path>) -> io::Result<()>
|
||||
{
|
||||
let mut file = load_file_ip(file)?;
|
||||
todo!("shuffling");
|
||||
file.flush()?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn unshuffle_file_ip(file: impl AsRef<Path>) -> io::Result<()>
|
||||
{
|
||||
let mut file = load_file_ip(file)?;
|
||||
todo!("unshuffling");
|
||||
file.flush()?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Process these files in place with this mode
|
||||
pub fn process_files_ip<I>(files: I, mode: ModeKind) -> io::Result<()>
|
||||
where I: IntoIterator<Item = PathBuf>
|
||||
{
|
||||
cfg_if! {
|
||||
if #[cfg(feature="threads")] {
|
||||
let handles: Vec<_> = files.into_iter().map(|file| {
|
||||
std::thread::spawn(move || {
|
||||
let res = match mode {
|
||||
ModeKind::Shuffle => shuffle_file_ip(&file),
|
||||
ModeKind::Unshuffle => unshuffle_file_ip(&file),
|
||||
};
|
||||
(res, file)
|
||||
})
|
||||
}).collect();
|
||||
|
||||
for h in handles
|
||||
{
|
||||
let (res, file) = h.join().expect("Worker thread panicked"); // TODO: How do we communicate which thread failed here? Should we even try?
|
||||
|
||||
res.map_err(move |x| {
|
||||
eprintln!("Failed to process {:?}", file);
|
||||
x
|
||||
})?;
|
||||
}
|
||||
Ok(())
|
||||
} else {
|
||||
for file in files.into_iter() {
|
||||
let res = match mode {
|
||||
ModeKind::Shuffle => shuffle_file_ip(&file),
|
||||
ModeKind::Unshuffle => unshuffle_file_ip(&file),
|
||||
};
|
||||
res.map_err(move |x| {
|
||||
eprintln!("Failed to process {:?}", file);
|
||||
x
|
||||
})?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
@ -0,0 +1,142 @@
|
||||
//! Contains the RNGs used
|
||||
use super::*;
|
||||
use std::{ptr, slice, mem::{self, MaybeUninit}};
|
||||
use std::ops::Drop;
|
||||
|
||||
/// A tuple packed into a type that can be used for SeedableRng::SEED.
|
||||
#[derive(Debug)]
|
||||
pub struct PackedTupleSeed<T>([MaybeUninit<T>; 2]);
|
||||
|
||||
impl<T: Eq> Eq for PackedTupleSeed<T>{}
|
||||
impl<T: PartialEq> PartialEq for PackedTupleSeed<T>
|
||||
{
|
||||
fn eq(&self, other: &Self) -> bool
|
||||
{
|
||||
self.first() == other.first() &&
|
||||
self.second() == other.second()
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> PackedTupleSeed<T>
|
||||
{
|
||||
pub const SIZE_BYTES: usize = mem::size_of::<T>() * 2;
|
||||
pub fn as_mut_array(&mut self) -> &mut [T; 2]
|
||||
{
|
||||
unsafe {&mut *(&mut self.0 as *mut [_; 2] as *mut [T; 2])}
|
||||
}
|
||||
pub fn as_array(&self) -> &[T; 2]
|
||||
{
|
||||
unsafe {& *(&self.0 as *const [_; 2] as *const [T; 2])}
|
||||
}
|
||||
pub fn second_mut(&mut self) -> &mut T
|
||||
{
|
||||
unsafe {&mut *self.0[1].as_mut_ptr()}
|
||||
}
|
||||
pub fn first_mut(&mut self) -> &mut T
|
||||
{
|
||||
unsafe {&mut *self.0[0].as_mut_ptr()}
|
||||
}
|
||||
pub fn second(&self) -> &T
|
||||
{
|
||||
unsafe {& *self.0[1].as_ptr()}
|
||||
}
|
||||
pub fn first(&self) -> &T
|
||||
{
|
||||
unsafe {& *self.0[0].as_ptr()}
|
||||
}
|
||||
|
||||
#[inline] pub fn new(a: T, b: T) -> Self {
|
||||
Self ([
|
||||
MaybeUninit::new(a),
|
||||
MaybeUninit::new(b),
|
||||
])
|
||||
}
|
||||
|
||||
#[inline] pub fn into_tuple(self) -> (T, T)
|
||||
{
|
||||
let ab = unsafe {
|
||||
(self.0[0].as_ptr().read(),
|
||||
self.0[1].as_ptr().read())
|
||||
};
|
||||
mem::forget(self);
|
||||
ab
|
||||
}
|
||||
|
||||
#[inline] pub fn into_second(self) -> T
|
||||
{
|
||||
self.into_tuple().1
|
||||
}
|
||||
#[inline] pub fn into_first(self) -> T
|
||||
{
|
||||
self.into_tuple().0
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> From<(T, T)> for PackedTupleSeed<T>
|
||||
{
|
||||
fn from((a,b): (T, T)) -> Self
|
||||
{
|
||||
Self::new(a, b)
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> From<PackedTupleSeed<T>> for (T, T)
|
||||
{
|
||||
fn from(from: PackedTupleSeed<T>) -> Self
|
||||
{
|
||||
from.into_tuple()
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: Clone> Clone for PackedTupleSeed<T>
|
||||
{
|
||||
fn clone(&self) -> Self {
|
||||
Self([
|
||||
MaybeUninit::new(self.first().clone()),
|
||||
MaybeUninit::new(self.second().clone()),
|
||||
])
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: Default> Default for PackedTupleSeed<T>
|
||||
{
|
||||
#[inline]
|
||||
fn default() -> Self
|
||||
{
|
||||
Self([
|
||||
MaybeUninit::new(Default::default()),
|
||||
MaybeUninit::new(Default::default()),
|
||||
])
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> AsMut<[u8]> for PackedTupleSeed<T>
|
||||
{
|
||||
fn as_mut(&mut self) -> &mut [u8]
|
||||
{
|
||||
unsafe {
|
||||
slice::from_raw_parts_mut(self as *mut Self as *mut u8, mem::size_of::<Self>())
|
||||
}
|
||||
}
|
||||
}
|
||||
impl<T> AsRef<[u8]> for PackedTupleSeed<T>
|
||||
{
|
||||
fn as_ref(&self) -> &[u8]
|
||||
{
|
||||
unsafe {
|
||||
slice::from_raw_parts(self as *const Self as *const u8, mem::size_of::<Self>())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> Drop for PackedTupleSeed<T>
|
||||
{
|
||||
fn drop(&mut self) {
|
||||
if mem::needs_drop::<T>() {
|
||||
unsafe {
|
||||
ptr::drop_in_place(self.0[0].as_mut_ptr());
|
||||
ptr::drop_in_place(self.0[1].as_mut_ptr());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
@ -0,0 +1,56 @@
|
||||
use rand::Rng;
|
||||
use std::mem::swap;
|
||||
|
||||
/// Get a random element in this slice.
|
||||
pub fn element_in<'a, T, R: Rng + ?Sized>(slice: &'a (impl AsRef<[T]> + ?Sized), with: &mut R) -> &'a T
|
||||
{
|
||||
let slice = slice.as_ref();
|
||||
&slice[with.gen_range(0..slice.len())]
|
||||
}
|
||||
|
||||
/// Get a random element in this slice.
|
||||
pub fn element_in_mut<'a, T, R: Rng + ?Sized>(slice: &'a mut (impl AsMut<[T]> + ?Sized), with: &mut R) -> &'a mut T
|
||||
{
|
||||
inside(slice.as_mut(), with)
|
||||
}
|
||||
|
||||
|
||||
#[inline(always)] fn inside<'a, T, R: Rng + ?Sized>(slice: &'a mut [T], with: &mut R) -> &'a mut T
|
||||
{
|
||||
&mut slice[with.gen_range(0..slice.len())]
|
||||
}
|
||||
|
||||
fn shuffle_slice<T, R: Rng + ?Sized>(slice: &mut [T], with: &mut R)
|
||||
{
|
||||
//XXX: Without optimisations this recursion causes stack overflow. Cannot work without tail call optimisation
|
||||
match slice
|
||||
{
|
||||
&mut [] | &mut [_] => {},
|
||||
&mut [ref mut inner @ .. , ref mut b] => {
|
||||
swap(b, inside(inner, with));
|
||||
shuffle(inner, with);
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn unshuffle_slice<T, R: Rng + ?Sized>(slice: &mut [T], with: &mut R)
|
||||
{
|
||||
let indecies: Vec<_> = (1..slice.len()).rev().map(|x| with.gen_range(0..x)).collect();
|
||||
|
||||
for (i, &idx) in (1..slice.len()).zip(indecies.iter().rev())
|
||||
{
|
||||
slice.swap(i, idx);
|
||||
}
|
||||
|
||||
drop!(indecies);
|
||||
}
|
||||
/// Shuffle this slice with this `Rng`.
|
||||
#[inline(always)] pub fn shuffle<T, R: Rng + ?Sized>(mut slice: impl AsMut<[T]>, with: &mut R)
|
||||
{
|
||||
shuffle_slice(slice.as_mut(), with)
|
||||
}
|
||||
|
||||
#[inline(always)] pub fn unshuffle<T, R: Rng + ?Sized>(mut slice: impl AsMut<[T]>, with: &mut R)
|
||||
{
|
||||
unshuffle_slice(slice.as_mut(), with)
|
||||
}
|
@ -0,0 +1,52 @@
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
pub fn shuffle_then_back()
|
||||
{
|
||||
let mut values: Vec<_> = (0..10i32).collect();
|
||||
let expected = values.clone();
|
||||
|
||||
let mut rng = {
|
||||
use rand::prelude::*;
|
||||
rand_chacha::ChaChaRng::from_seed([12; 32])
|
||||
};
|
||||
let mut nrng = rng.clone();
|
||||
|
||||
println!("Start: {:?}", values);
|
||||
|
||||
values.shuffle(&mut rng);
|
||||
|
||||
println!("Shuffled: {:?}", values);
|
||||
|
||||
values.unshuffle(&mut nrng);
|
||||
|
||||
println!("Unshuffled: {:?}", values);
|
||||
|
||||
assert_eq!(&values[..], &expected[..]);
|
||||
}
|
||||
|
||||
|
||||
#[test]
|
||||
pub fn shuffle_then_back_large()
|
||||
{
|
||||
let mut values: Vec<_> = (0..10000i32).collect();
|
||||
let expected = values.clone();
|
||||
|
||||
let mut rng = {
|
||||
use rand::prelude::*;
|
||||
rand_chacha::ChaChaRng::from_seed([0xfa; 32])
|
||||
};
|
||||
let mut nrng = rng.clone();
|
||||
|
||||
println!("Start (first 10): {:?}", &values[..10]);
|
||||
|
||||
values.shuffle(&mut rng);
|
||||
|
||||
println!("Shuffled (first 10): {:?}", &values[..10]);
|
||||
|
||||
values.unshuffle(&mut nrng);
|
||||
|
||||
println!("Unshuffled (first 10): {:?}", &values[..10]);
|
||||
|
||||
assert_eq!(&values[..], &expected[..]);
|
||||
}
|
Loading…
Reference in new issue