Added TODOs for specific bugs (and impl related ideas for how to go about preventing/warning in the event of `mode-memfd` system OOM) ; moved some RawFd-related functions into a submodule

Fortune for collect's current commit: Small curse − 小凶

src/main.rs

@@ -6,7 +6,7 @@
 #[cfg(feature="memfile")] 
 #[macro_use] extern crate lazy_static;
 #[cfg(feature="memfile")]
-#[macro_use] extern crate stackalloc;
+extern crate stackalloc;
 
 /// Run this statement only if `tracing` is enabled
 macro_rules! if_trace {
@@ -65,6 +65,12 @@ macro_rules! function {
     }}
 }
 
+mod sys;
+use sys::{
+    try_get_size,
+    tell_file,
+};
+
 mod buffers;
 use buffers::prelude::*;
 
@@ -76,111 +82,6 @@ use bytes::{
     BufMut,
 };
 
-/* TODO: XXX: For colouring buffer::Perc
-#[derive(Debug)]
-struct StackStr<const MAXLEN: usize>(usize, std::mem::MaybeUninit<[u8; MAXLEN]>);
-
-impl<const SZ: usize> StackStr<SZ>
-{
-#[inline] 
-pub const fn new() -> Self
-{
-Self(0, std::mem::MaybeUninit::uninit())
-    }
-    
-    #[inline(always)] 
-    pub const unsafe fn slice_mut(&mut self) -> &mut [u8]
-    {
-	&mut self.1[self.0..]
-    }
-    #[inline] 
-    pub const fn slice(&self) -> &[u8]
-    {
-	&self.1[self.0..]
-    }
-    
-    #[inline] 
-    pub const unsafe fn as_str_unchecked(&self) -> &str
-    {
-	std::str::from_utf8_unchecked(&self.1[self.0..])
-    }
-
-    #[inline] 
-    pub const unsafe fn as_mut_str_unchecked(&mut self) -> &mut str
-    {
-	std::str::from_utf8_unchecked_mut(&mut self.1[..self.0])
-    }
-
-    #[inline]
-    #[cfg_attr(feature="logging", instrument(level="debug"))]
-    pub fn as_str(&self) -> &str
-    {
-	std::str::from_utf8(self.slice()).expect("Invalid string")
-    }
-
-    #[inline(always)]
-    const fn left(&self) -> usize {
-	SZ - self.0
-    }
-
-    #[inline(always)] 
-    pub fn write_bytes(&mut self, s: &[u8]) -> usize {
-	let b = &s[..std::cmp::min(match self.left() {
-	    0 => return 0,
-	    x => x,
-	}, s.len())];
-	unsafe { &mut self.slice_mut() [self.0..] }.copy_from_slice(b);
-	let v = b.len();
-	self.0 += v;
-	v
-    }
-}
-
-impl<const SZ: usize> std::fmt::Write for StackStr<SZ>
-{
-    #[inline] 
-    fn write_str(&mut self, s: &str) -> std::fmt::Result {
-	self.write_bytes(s.as_bytes());
-	Ok(())
-    }
-    #[inline] 
-    fn write_char(&mut self, c: char) -> std::fmt::Result {
-	let l = c.len_utf8();
-	if l > self.left() {
-	    return Ok(())
-	} 
-	self.write_bytes(c.encode_utf8(unsafe { &mut self.slice_mut() [self.0..] }));
-	self.0 += l;
-
-	Ok(())
-    }
-}
-*/
-
-#[cfg_attr(feature="logging", instrument(level="info", skip(reader), fields(reader = ?std::any::type_name::<R>())))]
-fn try_get_size<R: ?Sized>(reader: &R) -> Option<NonZeroUsize>
-where R: AsRawFd
-{
-    let fd = reader.as_raw_fd();
-    use libc::{
-	fstat64,
-	stat64,
-    };
-    if fd < 0 {
-	return None;
-    }
-    let mut st: MaybeUninit<stat64> = MaybeUninit::uninit();
-    unsafe {
-	match fstat64(fd, st.as_mut_ptr()) {
-	    0 => {
-		NonZeroUsize::new(st.assume_init().st_size as usize)
-	    },
-	    _ => None,
-	}
-    }
-}
-
-
 fn init() -> eyre::Result<()>
 {
     cfg_if!{ if #[cfg(feature="logging")] {
@@ -222,8 +123,21 @@ fn init() -> eyre::Result<()>
 
 #[cfg_attr(feature="logging", instrument(err))]
 #[inline]
-fn non_map_work() -> eyre::Result<()>
+fn feature_check() -> eyre::Result<()>
 {
+    if cfg!(feature="memfile") && cfg!(feature="mode-buffered") {
+	if_trace!(warn!("This is an incorrectly compiled binary! Compiled with `mode: buffered` and the `memfile` feature; `memfile` stragery will be used and the mode selection will be ignored."));
+    }
+
+    Ok(())
+}
+
+mod work {
+    use super::*;
+    #[cfg_attr(feature="logging", instrument(err))]
+    #[inline] 
+    pub(super) fn buffered() -> eyre::Result<()>
+    {
 	if_trace!(trace!("strategy: allocated buffer"));
 	
 	let (bytes, read) = {
@@ -254,13 +168,14 @@ fn non_map_work() -> eyre::Result<()>
 	}
 	
 	Ok(())
-}
+    }
 
-#[cfg_attr(feature="logging", instrument(err))]
+    #[cfg_attr(feature="logging", instrument(err))]
-#[inline]
+    #[inline]
-#[cfg(feature="memfile")] 
+    #[cfg(feature="memfile")]
-fn map_work() -> eyre::Result<()>
+    //TODO: We should establish a max memory threshold for this to prevent full system OOM: Output a warning message if it exceeeds, say, 70-80% of free memory (not including used by this program (TODO: How do we calculate this efficiently?)), and fail with an error if it exceeds 90% of memory... Or, instead of using free memory as basis of the requirement levels on the max size of the memory file, use max memory? Or just total free memory at the start of program? Or check free memory each time (slow!! probably not this one...). Umm... I think basing it off total memory would be best; perhaps make the percentage levels user-configurable at compile time (and allow the user to set the memory value as opposed to using the total system memory at runtime.) or runtime (compile-time preffered; use that crate that lets us use TOML config files at comptime (find it pretty easy by looking through ~/work's rust projects, I've used it before.))
-{        
+    pub(super) fn memfd() -> eyre::Result<()>
+    {
 	const DEFAULT_BUFFER_SIZE: fn () -> Option<std::num::NonZeroUsize> = || {
 	    cfg_if!{ 
 		if #[cfg(feature="memfile-preallocate")]  {
@@ -278,13 +193,6 @@ fn map_work() -> eyre::Result<()>
 
 	use std::borrow::Borrow;
 
-    #[inline(always)] 
-    fn tell_file<T>(file: &mut T) -> io::Result<u64>
-	where T: io::Seek + ?Sized
-    {
-	file.stream_position()
-    }
-
 	#[inline(always)] 
 	fn unwrap_int_string<T, E>(i: impl Borrow<Result<T, E>>) -> String
 	where T: std::fmt::Display,
@@ -319,7 +227,7 @@ fn map_work() -> eyre::Result<()>
 		use std::borrow::Cow;
 
 		let (read, sp, sl) = if cfg!(any(feature="memfile-preallocate", debug_assertions)) {
-		let sp = file.stream_position();
+		    let sp = file.stream_position(); //TODO: XXX: Is this really needed?
 		    let sl = memfile::stream_len(&file);
 		    
 		    if_trace!(trace!("Stream position after read: {:?}", sp));
@@ -411,20 +319,22 @@ fn map_work() -> eyre::Result<()>
 	}
 	
 	Ok(())
+    }
 }
 
 #[cfg_attr(feature="logging", instrument(err))]
 fn main() -> eyre::Result<()> {
     init()?;
+    feature_check()?;
     if_trace!(debug!("initialised"));
 
     cfg_if!{ 
 	if #[cfg(feature="memfile")] {
-	    map_work()
+	    work::memfd()
-		.wrap_err(eyre!("Operation failed").with_note(|| "With mapped memfd algorithm"))?;
+		.wrap_err("Operation failed").with_note(|| "Stragery was `memfd`")?;
 	} else {
-	    non_map_work()
+	    work::buffered()
-		.wrap_err(eyre!("Operation failed").with_note(|| "With alloc-buf (non-mapped) algorithm"))?;
+		.wrap_err("Operation failed").with_note(|| "Strategy was `buffered`")?;
 	}
     }
 

src/sys.rs

@@ -0,0 +1,40 @@
+//! System interactions
+//!
+//! Basic system interactions.
+use super::*;
+
+/// Attempt to get the size of any stream that is backed by a file-descriptor.
+///
+/// If one cannot be determined (or the fd is unsized), `None` is returned.
+#[cfg_attr(feature="logging", instrument(level="info", skip(reader), ret, fields(reader = std::any::type_name::<R>())))]
+#[inline]
+//TODO: XXX: What if the size of `reader` really *is* 0. We shouldn't use `NonZeroUsize` here, we should just use `usize`. I think `st_size` can be `-1` if `fstat64()` fails to find a size...
+pub fn try_get_size<R: ?Sized>(reader: &R) -> Option<NonZeroUsize>
+where R: AsRawFd
+{
+    let fd = reader.as_raw_fd();
+    use libc::{
+	fstat64,
+	stat64,
+    };
+    if fd < 0 {
+	return None;
+    }
+    let mut st: MaybeUninit<stat64> = MaybeUninit::uninit();
+    unsafe {
+	match fstat64(fd, st.as_mut_ptr()) {
+	    0 => {
+		NonZeroUsize::new(st.assume_init().st_size as usize)
+	    },
+	    _ => None,
+	}
+    }
+}
+
+/// Get the current stream position of any seekable stream.
+#[inline(always)] 
+pub fn tell_file<T>(file: &mut T) -> io::Result<u64>
+where T: io::Seek + ?Sized
+{
+    file.stream_position()
+}