start new optional serialisation mode: prealloc

Cargo.lock

@@ -122,6 +122,8 @@ dependencies = [
  "color-eyre",
  "futures",
  "lazy_static",
+ "libc",
+ "memmap",
  "num_cpus",
  "once_cell",
  "pin-project",
@@ -311,9 +313,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 
 [[package]]
 name = "libc"
-version = "0.2.85"
+version = "0.2.86"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7ccac4b00700875e6a07c6cde370d44d32fa01c5a65cdd2fca6858c479d28bb3"
+checksum = "b7282d924be3275cec7f6756ff4121987bc6481325397dde6ba3e7802b1a8b1c"
 
 [[package]]
 name = "log"
@@ -330,6 +332,16 @@ version = "2.3.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525"
 
+[[package]]
+name = "memmap"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b"
+dependencies = [
+ "libc",
+ "winapi 0.3.9",
+]
+
 [[package]]
 name = "miniz_oxide"
 version = "0.4.3"

Cargo.toml

@@ -6,9 +6,18 @@ edition = "2018"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
+[profile.release]
+opt-level = 3
+lto = "fat"
+codegen-units = 1
+panic = "unwind"
+
 [features]
 default = ["splash", "inspect", "defer-drop"]
 
+# Use `fallocate()` and memory mapping to save output data instead of normal `write()` syscalls
+prealloc = ["inspect", "libc", "memmap"]
+
 # Allow saving and loading of gathered data for later inspection
 inspect = ["serde", "serde_cbor", "async-compression"]
 
@@ -23,6 +32,8 @@ async-compression = {version = "0.3", features=["tokio-02", "bzip2"], optional=t
 color-eyre = {version = "0.5.10", default-features=false}
 futures = "0.3.12"
 lazy_static = "1.4.0"
+libc = {version = "0.2.86", optional = true}
+memmap = {version = "0.7.0", optional = true}
 num_cpus = "1.13.0"
 once_cell = "1.5.2"
 pin-project = "1.0.5"

src/serial.rs

@@ -13,6 +13,10 @@ type Decompressor<T> = BzDecoder<T>;
 const DEFER_DROP_SIZE_FLOOR: usize = 1024 * 1024; // 1 MB
 
 /// Serialise this object asynchronously
+///
+/// # Note
+/// This compresses the output stream.
+/// It cannot be used by `prealloc` read/write functions, as they do not compress.
 pub async fn write_async<T: Serialize, W>(mut to: W, item: &T) -> eyre::Result<()>
 where W: AsyncWrite + Unpin
 {
@@ -57,3 +61,29 @@ where W: AsyncWrite + Unpin
     }
     flush_fut.await
 }
+
+#[cfg(feature="prealloc")] 
+mod prealloc {
+    use super::*;
+    use std::os::unix::prelude::*;
+    /// Write this object as-is to this file descriptor.
+    ///
+    /// # Note
+    /// This does not compress like `write_aynsc()` does. It is just a 1-1 dump of the serialisation.
+    /// Therefore, `write_prealloc` cannot be used with `read_async()`, and other such things.
+    ///
+    /// `fd` must be a valid *File* descriptor. fallocating and mapping stdout probably won't work
+    ///
+    /// This is a completely synchronous operation. You should use it with `spawn_blocking` et al. to prevent task hangups.
+    pub fn write_prealloc<T: Serialize>(fd: impl AsRawFd, item: &T) -> eyre::Result<()>
+    {
+	//TODO: serialise `item` to vec.
+	//TODO: fallocate() `fd` to size of `vec`
+	//TODO: memmap() `fd`
+	//TODO: memcpy() vec to map
+	//TODO: flush map
+	//TODO: `drop!(vec)`
+	todo!()
+    }
+}
+#[cfg(feature="prealloc")] pub use prealloc::write_prealloc as write_sync_map;