From 2583ae674ca89e7e0ecdfe288ad3fdfa702fb6fb Mon Sep 17 00:00:00 2001 From: Avril Date: Wed, 17 Feb 2021 03:23:25 +0000 Subject: [PATCH] prealloc: write and read --- Cargo.toml | 2 +- src/bytes.rs | 12 ++++++++ src/ext.rs | 28 ++++++++++++++++++ src/main.rs | 2 ++ src/serial.rs | 80 ++++++++++++++++++++++++++++++++++++++++++++------- 5 files changed, 112 insertions(+), 12 deletions(-) create mode 100644 src/bytes.rs diff --git a/Cargo.toml b/Cargo.toml index 7453c0e..b1f12c0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ codegen-units = 1 panic = "unwind" [features] -default = ["splash", "inspect", "defer-drop", "jemalloc"] +default = ["splash", "inspect", "defer-drop", "jemalloc", "prealloc"] # Use jemalloc as global allocator instead of system allocator. # May potentially cause some speedups and better memory profile on large runs. diff --git a/src/bytes.rs b/src/bytes.rs new file mode 100644 index 0000000..fa8f3a5 --- /dev/null +++ b/src/bytes.rs @@ -0,0 +1,12 @@ +//! Dealing with bytes and stuff + +/// Copy from `src` into `dst` and return the number of bytes copied. +/// +/// # Notes +/// The regions *must not* overlap. This is UB if they do. +#[inline] pub unsafe fn copy_nonoverlapping_unchecked(src: &[u8], dst: &mut [u8]) -> usize +{ + let len = std::cmp::min(dst.len(), src.len()); + std::ptr::copy_nonoverlapping(src.as_ptr(), dst.as_mut_ptr(), len); + len +} diff --git a/src/ext.rs b/src/ext.rs index 862f118..27aea52 100644 --- a/src/ext.rs +++ b/src/ext.rs @@ -11,6 +11,7 @@ use std::{ pin::Pin, task::{Poll, Context}, }; +use std::{fmt, error}; pub mod prelude { @@ -318,3 +319,30 @@ mod tests () } } + +/// Base type from macro `eyre_assert`. +#[derive(Debug)] +pub struct SoftAssertionFailedError; + +impl error::Error for SoftAssertionFailedError{} +impl fmt::Display for SoftAssertionFailedError +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result + { + write!(f, "Assertion failed") + } +} + + +/// A soft assertion that yields an `Err(eyre::Report)` if the condition fails. +#[macro_export] macro_rules! eyre_assert { + ($cond:expr $(; $message:literal)?) => { + if !$cond { + Err($crate::ext::SoftAssertionFailedError) + $(.wrap_err(eyre!($message)))? + .with_section(|| stringify!($cond).header("Expression was")) + } else { + Ok(()) + } + }; +} diff --git a/src/main.rs b/src/main.rs index a664c83..80ad764 100644 --- a/src/main.rs +++ b/src/main.rs @@ -13,6 +13,7 @@ use jemallocator::Jemalloc; #[global_allocator] static GLOBAL: Jemalloc = Jemalloc; +use std::convert::{TryFrom, TryInto}; use color_eyre::{ eyre::{ self, @@ -28,6 +29,7 @@ use color_eyre::{ #[macro_use] mod ext; pub use ext::prelude::*; +mod bytes; mod data; mod config; mod state; diff --git a/src/serial.rs b/src/serial.rs index a762422..364b4d6 100644 --- a/src/serial.rs +++ b/src/serial.rs @@ -66,24 +66,82 @@ where W: AsyncWrite + Unpin mod prealloc { use super::*; use std::os::unix::prelude::*; + use std::fs::File; + use memmap::{MmapMut, Mmap}; + /// Write this object as-is to this file descriptor. /// /// # Note /// This does not compress like `write_aynsc()` does. It is just a 1-1 dump of the serialisation. - /// Therefore, `write_prealloc` cannot be used with `read_async()`, and other such things. + /// Therefore, data written with `write_prealloc()` cannot be then read used with `read_async()`. /// - /// `fd` must be a valid *File* descriptor. fallocating and mapping stdout probably won't work + /// This is a completely synchronous operation. You should use it with `spawn_blocking` et al. to prevent task hangups. + pub fn write_prealloc(file: &mut File, item: &T) -> eyre::Result<()> + { + let sect_type_name = || std::any::type_name::().header("Type trying to serialise was"); + + let vec = tokio::task::block_in_place(|| serde_cbor::to_vec(item)) + .wrap_err(eyre!("Failed to serialise item")) + .with_section(sect_type_name.clone())?; + + let fd = file.as_raw_fd(); + unsafe { + if libc::fallocate(fd, 0, 0, vec.len().try_into() + .wrap_err(eyre!("Failed to cast buffer size to `off_t`")) + .with_section(|| vec.len().header("Buffer size was")) + .with_section(|| libc::off_t::MAX.to_string().header("Max value of `off_t` is")) + .with_warning(|| "Usually `off_t` is a signed 64 bit integer. Whereas the buffer's size is unsigned. On systems where `off_t` is 64 bits or higher, this should realistically never happen and probably indicates a bug.")?) < 0 { + // Error + Err(std::io::Error::last_os_error()) + } else { + Ok(()) + } + }.wrap_err("fallocate() failed") + .with_section(|| vec.len().header("Bytes to allocate was")) + .with_suggestion(|| "Make sure there is enough space for the fallocate() call") + .with_suggestion(|| "Make sure we are able to write to the file")?; + // fallocate() succeeded in allocating `vec.len()` bytes to map. + let mut map = unsafe { MmapMut::map_mut(file) } + .wrap_err(eyre!("Failed to map file for read + write")) + .with_section(|| fd.header("fd was")) + .with_suggestion(|| "Do we have the premissions for both reading and writing of this file and fd?")?; + + eyre_assert!(tokio::task::block_in_place(|| unsafe { + bytes::copy_nonoverlapping_unchecked(&vec[..], &mut map[..]) + }) == vec.len(); "Length mismatch") + .with_section(|| vec.len().header("Expected")) + .with_section(|| map.len().header("Got")) + .with_warning(|| "This should never happen, it indicates a bug")?; + + tokio::task::block_in_place(move || map.flush()) + .wrap_err(eyre!("Failed to flush map in place"))?; //map is dropped here + + drop!(vec vec); + Ok(()) + } + + /// Read this object as-is from this file descriptor. + /// + /// # Note + /// This does not decompress like `read_aynsc()` does. It is just a 1-1 read of the serialisation. + /// Therefore, `read_prealloc()` cannot be used with data written by `write_async()`. /// /// This is a completely synchronous operation. You should use it with `spawn_blocking` et al. to prevent task hangups. - pub fn write_prealloc(fd: impl AsRawFd, item: &T) -> eyre::Result<()> + // This must be `DeserializeOwned` because the lifetime it is bound to is that of the memory map created and destroyed in the function, not of the fd `file` itself. + pub fn read_prealloc(file: &File) -> eyre::Result { - //TODO: serialise `item` to vec. - //TODO: fallocate() `fd` to size of `vec` - //TODO: memmap() `fd` - //TODO: memcpy() vec to map - //TODO: flush map - //TODO: `drop!(vec)` - todo!() + let map = unsafe { Mmap::map(file) } + .wrap_err(eyre!("Failed to map file for read + write")) + .with_section(|| file.as_raw_fd().header("fd was")) + .with_suggestion(|| "Do we have the premissions for both reading and writing of this file and fd?")?; + + tokio::task:: + block_in_place(move || serde_cbor::from_slice(&map[..])) + .wrap_err(eyre!("Failed to deserialise from map")) + .with_note(|| "The prealloc read and write functions handle only *uncompressed* data. Make sure you're not feeding it compressed data (written with the non-prealloc read and write functions)") } } -#[cfg(feature="prealloc")] pub use prealloc::write_prealloc as write_sync_map; +#[cfg(feature="prealloc")] pub use prealloc::{ + write_prealloc as write_sync_map, + read_prealloc as read_sync_map, +};