From 0628853b432605a0bbdcbc34fe29387135711aa0 Mon Sep 17 00:00:00 2001 From: Avril Date: Mon, 25 Apr 2022 08:21:40 +0100 Subject: [PATCH] Working memfile implementation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fortune for collect's current commit: Future blessing − 末吉 --- Cargo.lock | 2 +- Cargo.toml | 2 +- src/main.rs | 70 ++++++++++++++++++++++++++++++++++++++++++++++++-- src/memfile.rs | 14 ++++++---- 4 files changed, 79 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8beefb0..5e65c26 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -84,7 +84,7 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "collect" -version = "0.1.0" +version = "0.2.0" dependencies = [ "bitflags", "bytes", diff --git a/Cargo.toml b/Cargo.toml index ed743dc..1022970 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,7 +6,7 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [features] -default = ["jemalloc", "logging", "tracing/release_max_level_warn"] #, "memfile" ] +default = ["jemalloc", "memfile", "logging", "tracing/release_max_level_warn"] #, "memfile" ] # TODO: mmap, memfd_create() ver memfile = ["bitflags", "lazy_static", "stackalloc"] diff --git a/src/main.rs b/src/main.rs index fa557bf..713a194 100644 --- a/src/main.rs +++ b/src/main.rs @@ -261,11 +261,77 @@ fn non_map_work() -> eyre::Result<()> #[cfg(feature="memfile")] fn map_work() -> eyre::Result<()> { + extern "C" { + fn getpagesize() -> libc::c_int; + } + /// 8 pages + const DEFAULT_BUFFER_SIZE: fn () -> Option = || { unsafe { std::num::NonZeroUsize::new((getpagesize() as usize) * 8) } }; if_trace!(trace!("strategy: mapped memory file")); - let file = memfile::create_memfile(Some("this is a test file"), 4096)?; - unimplemented!("Feature not yet implemented") + use std::borrow::Borrow; + + #[inline(always)] + fn tell_file(file: &mut T) -> io::Result + where T: io::Seek + ?Sized + { + file.stream_position() + } + + #[inline(always)] + fn unwrap_int_string(i: impl Borrow>) -> String + where T: std::fmt::Display, + E: std::fmt::Display + { + i.borrow().as_ref().map(ToString::to_string) + .unwrap_or_else(|e| format!("")) + } + + let (mut file, read) = { + let stdin = io::stdin(); + + let buffsz = try_get_size(&stdin).or_else(DEFAULT_BUFFER_SIZE); + if_trace!(trace!("Attempted determining input size: {:?}", buffsz)); + let mut file = memfile::create_memfile(Some("collect-buffer"), + buffsz.map(|x| x.get()).unwrap_or(0)) + .with_section(|| format!("{:?}", buffsz).header("Deduced input buffer size")) + .wrap_err(eyre!("Failed to create in-memory buffer"))?; + + let read = io::copy(&mut stdin.lock(), &mut file) + .with_section(|| format!("{:?}", file).header("Memory buffer file"))?; + + { + use io::*; + file.seek(SeekFrom::Start(0)) + .with_section(|| read.header("Actual read bytes")) + .wrap_err(eyre!("Failed to seek back to start of memory buffer file for output") + .with_section(|| unwrap_int_string(file.stream_position()).header("Memfile position")) + /*.with_section(|| file.stream_len().map(|x| x.to_string()) + .unwrap_or_else(|e| format!("")).header("Memfile full length"))*/)?; + } + + (file, usize::try_from(read) + .wrap_err(eyre!("Failed to convert read bytes to `usize`") + .with_section(|| read.header("Number of bytes was")) + .with_section(|| u128::abs_diff(read.into(), usize::MAX as u128).header("Difference between `read` and `usize::MAX` is")) + .with_suggestion(|| "It is likely you are running on a 32-bit ptr width machine and this input exceeds that of the maximum 32-bit unsigned integer value") + .with_note(|| usize::MAX.header("Maximum value of `usize`")))?) + }; + if_trace!(info!("collected {read} from stdin. starting write.")); + + let written = + io::copy(&mut file, &mut io::stdout().lock()) + .with_section(|| read.header("Bytes read from stdin")) + .with_section(|| unwrap_int_string(tell_file(&mut file)).header("Current buffer position")) + .wrap_err("Failed to write buffer to stdout")?; + if_trace!(info!("written {written} to stdout.")); + + if read != written as usize { + return Err(io::Error::new(io::ErrorKind::BrokenPipe, format!("read {read} bytes, but only wrote {written}"))) + .wrap_err("Writing failed: size mismatch"); + } + + Ok(()) } #[cfg_attr(feature="logging", instrument(err))] diff --git a/src/memfile.rs b/src/memfile.rs index f41559c..18a8bba 100644 --- a/src/memfile.rs +++ b/src/memfile.rs @@ -224,11 +224,15 @@ impl RawFile let fd = attempt_call!(-1, memfd_create(bname as *const _, MEMFD_CREATE_FLAGS), Create(name.map(str::to_owned), MEMFD_CREATE_FLAGS)) .map(Self::take_ownership_of_unchecked)?; // Ensures `fd` is dropped if any subsequent calls fail - - attempt_call!(-1 - , fallocate(fd.0.get(), 0, 0, len.try_into() - .map_err(|_| Allocate(None, len))?) - , Allocate(Some(fd.fileno().clone()), len))?; + + if len > 0 { + attempt_call!(-1 + , fallocate(fd.0.get(), 0, 0, len.try_into() + .map_err(|_| Allocate(None, len))?) + , Allocate(Some(fd.fileno().clone()), len))?; + } else { + if_trace!(trace!("No length provided, skipping fallocate() call")); + } Ok(fd) })