start new optional serialisation mode: prealloc

fuck_this_bullshit
Avril 4 years ago
parent 4ee524d859
commit 06c07371ee
Signed by: flanchan
GPG Key ID: 284488987C31F630

16
Cargo.lock generated

@ -122,6 +122,8 @@ dependencies = [
"color-eyre",
"futures",
"lazy_static",
"libc",
"memmap",
"num_cpus",
"once_cell",
"pin-project",
@ -311,9 +313,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]]
name = "libc"
version = "0.2.85"
version = "0.2.86"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ccac4b00700875e6a07c6cde370d44d32fa01c5a65cdd2fca6858c479d28bb3"
checksum = "b7282d924be3275cec7f6756ff4121987bc6481325397dde6ba3e7802b1a8b1c"
[[package]]
name = "log"
@ -330,6 +332,16 @@ version = "2.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525"
[[package]]
name = "memmap"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b"
dependencies = [
"libc",
"winapi 0.3.9",
]
[[package]]
name = "miniz_oxide"
version = "0.4.3"

@ -6,9 +6,18 @@ edition = "2018"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[profile.release]
opt-level = 3
lto = "fat"
codegen-units = 1
panic = "unwind"
[features]
default = ["splash", "inspect", "defer-drop"]
# Use `fallocate()` and memory mapping to save output data instead of normal `write()` syscalls
prealloc = ["inspect", "libc", "memmap"]
# Allow saving and loading of gathered data for later inspection
inspect = ["serde", "serde_cbor", "async-compression"]
@ -23,6 +32,8 @@ async-compression = {version = "0.3", features=["tokio-02", "bzip2"], optional=t
color-eyre = {version = "0.5.10", default-features=false}
futures = "0.3.12"
lazy_static = "1.4.0"
libc = {version = "0.2.86", optional = true}
memmap = {version = "0.7.0", optional = true}
num_cpus = "1.13.0"
once_cell = "1.5.2"
pin-project = "1.0.5"

@ -13,6 +13,10 @@ type Decompressor<T> = BzDecoder<T>;
const DEFER_DROP_SIZE_FLOOR: usize = 1024 * 1024; // 1 MB
/// Serialise this object asynchronously
///
/// # Note
/// This compresses the output stream.
/// It cannot be used by `prealloc` read/write functions, as they do not compress.
pub async fn write_async<T: Serialize, W>(mut to: W, item: &T) -> eyre::Result<()>
where W: AsyncWrite + Unpin
{
@ -57,3 +61,29 @@ where W: AsyncWrite + Unpin
}
flush_fut.await
}
#[cfg(feature="prealloc")]
mod prealloc {
use super::*;
use std::os::unix::prelude::*;
/// Write this object as-is to this file descriptor.
///
/// # Note
/// This does not compress like `write_aynsc()` does. It is just a 1-1 dump of the serialisation.
/// Therefore, `write_prealloc` cannot be used with `read_async()`, and other such things.
///
/// `fd` must be a valid *File* descriptor. fallocating and mapping stdout probably won't work
///
/// This is a completely synchronous operation. You should use it with `spawn_blocking` et al. to prevent task hangups.
pub fn write_prealloc<T: Serialize>(fd: impl AsRawFd, item: &T) -> eyre::Result<()>
{
//TODO: serialise `item` to vec.
//TODO: fallocate() `fd` to size of `vec`
//TODO: memmap() `fd`
//TODO: memcpy() vec to map
//TODO: flush map
//TODO: `drop!(vec)`
todo!()
}
}
#[cfg(feature="prealloc")] pub use prealloc::write_prealloc as write_sync_map;

Loading…
Cancel
Save