prealloc: write and read

Cargo.toml

@@ -13,7 +13,7 @@ codegen-units = 1
 panic = "unwind"
 
 [features]
-default = ["splash", "inspect", "defer-drop", "jemalloc"]
+default = ["splash", "inspect", "defer-drop", "jemalloc", "prealloc"]
 
 # Use jemalloc as global allocator instead of system allocator.
 # May potentially cause some speedups and better memory profile on large runs.

src/bytes.rs

@@ -0,0 +1,12 @@
+//! Dealing with bytes and stuff
+
+/// Copy from `src` into `dst` and return the number of bytes copied.
+///
+/// # Notes
+/// The regions *must not* overlap. This is UB if they do.
+#[inline] pub unsafe fn copy_nonoverlapping_unchecked(src: &[u8], dst: &mut [u8]) -> usize
+{
+    let len = std::cmp::min(dst.len(), src.len());
+    std::ptr::copy_nonoverlapping(src.as_ptr(), dst.as_mut_ptr(), len);
+    len
+}

src/ext.rs

@@ -11,6 +11,7 @@ use std::{
     pin::Pin,
     task::{Poll, Context},
 };
+use std::{fmt, error};
 
 pub mod prelude
 {
@@ -318,3 +319,30 @@ mod tests
 	()
     }
 }
+
+/// Base type from macro `eyre_assert`.
+#[derive(Debug)]
+pub struct SoftAssertionFailedError;
+
+impl error::Error for SoftAssertionFailedError{}
+impl fmt::Display for SoftAssertionFailedError
+{
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result
+    {
+	write!(f, "Assertion failed")
+    }
+}
+
+
+/// A soft assertion that yields an `Err(eyre::Report)` if the condition fails.
+#[macro_export] macro_rules! eyre_assert {
+    ($cond:expr $(; $message:literal)?) => {
+	if !$cond {
+	    Err($crate::ext::SoftAssertionFailedError)
+		$(.wrap_err(eyre!($message)))?
+		.with_section(|| stringify!($cond).header("Expression was"))
+	} else {
+	    Ok(())
+	}
+    };
+}

src/main.rs

@@ -13,6 +13,7 @@ use jemallocator::Jemalloc;
 #[global_allocator]
 static GLOBAL: Jemalloc = Jemalloc;
 
+use std::convert::{TryFrom, TryInto};
 use color_eyre::{
     eyre::{
 	self,
@@ -28,6 +29,7 @@ use color_eyre::{
 #[macro_use] mod ext;
 pub use ext::prelude::*;
 
+mod bytes;
 mod data;
 mod config;
 mod state;

src/serial.rs

@@ -66,24 +66,82 @@ where W: AsyncWrite + Unpin
 mod prealloc {
     use super::*;
     use std::os::unix::prelude::*;
+    use std::fs::File;
+    use memmap::{MmapMut, Mmap};
+    
     /// Write this object as-is to this file descriptor.
     ///
     /// # Note
     /// This does not compress like `write_aynsc()` does. It is just a 1-1 dump of the serialisation.
-    /// Therefore, `write_prealloc` cannot be used with `read_async()`, and other such things.
+    /// Therefore, data written with `write_prealloc()` cannot be then read used with `read_async()`.
     ///
-    /// `fd` must be a valid *File* descriptor. fallocating and mapping stdout probably won't work
+    /// This is a completely synchronous operation. You should use it with `spawn_blocking` et al. to prevent task hangups.
+    pub fn write_prealloc<T: Serialize>(file: &mut File, item: &T) -> eyre::Result<()>
+    {	
+	let sect_type_name = || std::any::type_name::<T>().header("Type trying to serialise was");
+	
+	let vec = tokio::task::block_in_place(|| serde_cbor::to_vec(item))
+	    .wrap_err(eyre!("Failed to serialise item"))
+            .with_section(sect_type_name.clone())?;
+	
+	let fd = file.as_raw_fd();
+	unsafe {
+	    if libc::fallocate(fd, 0, 0, vec.len().try_into()
+			       .wrap_err(eyre!("Failed to cast buffer size to `off_t`"))
+			       .with_section(|| vec.len().header("Buffer size was"))
+			       .with_section(|| libc::off_t::MAX.to_string().header("Max value of `off_t` is"))
+			       .with_warning(|| "Usually `off_t` is a signed 64 bit integer. Whereas the buffer's size is unsigned. On systems where `off_t` is 64 bits or higher, this should realistically never happen and probably indicates a bug.")?) < 0 {
+		// Error
+		Err(std::io::Error::last_os_error())
+	    } else {
+		Ok(())
+	    }
+	}.wrap_err("fallocate() failed")
+	    .with_section(|| vec.len().header("Bytes to allocate was"))
+	    .with_suggestion(|| "Make sure there is enough space for the fallocate() call")
+	    .with_suggestion(|| "Make sure we are able to write to the file")?;
+	// fallocate() succeeded in allocating `vec.len()` bytes to map.
+	let mut map = unsafe { MmapMut::map_mut(file) }
+	.wrap_err(eyre!("Failed to map file for read + write"))
+	    .with_section(|| fd.header("fd was"))
+	    .with_suggestion(|| "Do we have the premissions for both reading and writing of this file and fd?")?;
+
+	eyre_assert!(tokio::task::block_in_place(|| unsafe {
+	    bytes::copy_nonoverlapping_unchecked(&vec[..], &mut map[..])
+	}) == vec.len(); "Length mismatch")
+	    .with_section(|| vec.len().header("Expected"))
+	    .with_section(|| map.len().header("Got"))
+	    .with_warning(|| "This should never happen, it indicates a bug")?;
+
+	tokio::task::block_in_place(move || map.flush())
+	    .wrap_err(eyre!("Failed to flush map in place"))?; //map is dropped here
+
+	drop!(vec vec);
+	Ok(())
+    }
+
+    /// Read this object as-is from this file descriptor.
+    ///
+    /// # Note
+    /// This does not decompress like `read_aynsc()` does. It is just a 1-1 read of the serialisation.
+    /// Therefore, `read_prealloc()` cannot be used with data written by `write_async()`.
     ///
     /// This is a completely synchronous operation. You should use it with `spawn_blocking` et al. to prevent task hangups.
-    pub fn write_prealloc<T: Serialize>(fd: impl AsRawFd, item: &T) -> eyre::Result<()>
+    // This must be `DeserializeOwned` because the lifetime it is bound to is that of the memory map created and destroyed in the function, not of the fd `file` itself.
+    pub fn read_prealloc<T: serde::de::DeserializeOwned>(file: &File) -> eyre::Result<T>
     {
-	//TODO: serialise `item` to vec.
+	let map = unsafe { Mmap::map(file) }
-	//TODO: fallocate() `fd` to size of `vec`
+	.wrap_err(eyre!("Failed to map file for read + write"))
-	//TODO: memmap() `fd`
+	    .with_section(|| file.as_raw_fd().header("fd was"))
-	//TODO: memcpy() vec to map
+	    .with_suggestion(|| "Do we have the premissions for both reading and writing of this file and fd?")?;
-	//TODO: flush map
+
-	//TODO: `drop!(vec)`
+	tokio::task::
-	todo!()
+	block_in_place(move || serde_cbor::from_slice(&map[..]))
+	    .wrap_err(eyre!("Failed to deserialise from map"))
+	    .with_note(|| "The prealloc read and write functions handle only *uncompressed* data. Make sure you're not feeding it compressed data (written with the non-prealloc read and write functions)")
     }
 }
-#[cfg(feature="prealloc")] pub use prealloc::write_prealloc as write_sync_map;
+#[cfg(feature="prealloc")] pub use prealloc::{
+    write_prealloc as write_sync_map,
+    read_prealloc as read_sync_map,
+};