From ed957bcec855516a9f44add5b93021a78b092555 Mon Sep 17 00:00:00 2001 From: Avril Date: Sun, 24 Apr 2022 09:18:05 +0100 Subject: [PATCH] feature memfile: added `RawFile::open_mem()` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Needs testing, and the XXX comment about `memfd_create()` name lifetimes also needs testing. (If they do need to be static; `stackalloc` as a dependancy can be removed entirely, then: `DEFAULT_NAME` will be changed to a CString, function parameter `name` and `error::MemfileCreationStep::Create(name)` will be changed to `Option<&"static CStr>`. Fortune for collect's current commit: Future blessing − 末吉 --- Cargo.lock | 43 ++++++++++++++++ Cargo.toml | 5 +- src/main.rs | 15 ++++++ src/memfile.rs | 119 +++++++++++++++++++++++++++++++++++++------ src/memfile/error.rs | 84 +++++++++++++++++++++++++++++- src/memfile/fd.rs | 20 +++++++- src/memfile/map.rs | 22 ++++++++ 7 files changed, 289 insertions(+), 19 deletions(-) create mode 100644 src/memfile/map.rs diff --git a/Cargo.lock b/Cargo.lock index 524f8e6..8beefb0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -58,6 +58,12 @@ dependencies = [ "rustc-demangle", ] +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bytes" version = "1.1.0" @@ -80,14 +86,17 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" name = "collect" version = "0.1.0" dependencies = [ + "bitflags", "bytes", "cfg-if", "color-eyre", "jemallocator", "lazy_format", + "lazy_static", "libc", "memchr", "recolored", + "stackalloc", "tracing", "tracing-error", "tracing-subscriber", @@ -316,6 +325,30 @@ version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342" +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +dependencies = [ + "semver", +] + +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +dependencies = [ + "semver-parser", +] + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" + [[package]] name = "sharded-slab" version = "0.1.4" @@ -331,6 +364,16 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83" +[[package]] +name = "stackalloc" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4f5c9dd3feb8a4adc8eae861e5f48862a92f9a9f38cf8fc99b92fc6ec016121" +dependencies = [ + "cc", + "rustc_version", +] + [[package]] name = "syn" version = "1.0.91" diff --git a/Cargo.toml b/Cargo.toml index ab2aeea..1cfb1c8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,7 +9,7 @@ edition = "2021" default = ["jemalloc", "memfile", "logging", "tracing/release_max_level_warn"] # TODO: mmap, memfd_create() ver -memfile = [] +memfile = ["bitflags", "lazy_static"] # bytes: use `bytes` crate for collecting instead of `std::vec` @@ -51,3 +51,6 @@ color-eyre = { version = "0.6.1", default-features=false }#, features = ["captur recolored = { version = "1.9.3", optional = true } memchr = "2.4.1" lazy_format = "1.10.0" +bitflags = {version = "1.3.2", optional = true } +stackalloc = "1.1.1" +lazy_static = { version = "1.4.0", optional = true } diff --git a/src/main.rs b/src/main.rs index f5be30a..5857749 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,6 +3,9 @@ #[cfg(feature="logging")] #[macro_use] extern crate tracing; +#[macro_use] extern crate lazy_static; +#[macro_use] extern crate stackalloc; + /// Run this statement only if `tracing` is enabled macro_rules! if_trace { (? $expr:expr) => { @@ -48,6 +51,18 @@ use color_eyre::{ SectionExt, Help, }; +/// Get an `&'static str` of the current function name. +macro_rules! function { + () => {{ + fn f() {} + fn type_name_of(_: T) -> &'static str { + ::std::any::type_name::() + } + let name = type_name_of(f); + &name[..name.len() - 3] + }} +} + mod buffers; use buffers::prelude::*; diff --git a/src/memfile.rs b/src/memfile.rs index 24b01d8..9907f23 100644 --- a/src/memfile.rs +++ b/src/memfile.rs @@ -13,8 +13,12 @@ use std::{ }, }; -mod fd; +pub mod fd; pub mod error; +mod map; + +/// Flags passed to `memfd_create()` when used in this module +const MEMFD_CREATE_FLAGS: libc::c_uint = libc::MFD_CLOEXEC; #[derive(Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] #[repr(transparent)] @@ -24,21 +28,55 @@ impl RawFile { /// Get the raw fd for this raw file #[inline(always)] - pub const fn fileno(&self) -> RawFd + pub const fn fileno(&self) -> &fd::RawFileDescriptor { - self.0.get() + &self.0//.clone_const() } #[inline(always)] - pub fn into_fileno(self) -> RawFd + pub fn into_fileno(self) -> fd::RawFileDescriptor { - self.into_raw_fd() + // SAFETY: We know this is safe since we are just converting the released (valid) fd from `self` + unsafe { + fd::RawFileDescriptor::new_unchecked(self.into_raw_fd()) + } } #[inline(always)] - pub unsafe fn from_fileno(fd: RawFd) -> Self + pub unsafe fn from_fileno(fd: fd::RawFileDescriptor) -> Self { - Self::from_raw_fd(fd) + Self::from_raw_fd(fd.get()) + } + + #[inline(always)] + const fn take_ownership_of_unchecked(fd: RawFd) -> Self + { + //! **Internal**: Non-`unsafe` and `const` version of `take_ownership_of_raw_unchecked()` + //! : assumes `fd` is `>= 0` + //! + //! For use in `memfile` functions where `fd` has already been checked for validation (since `unsafe fn`s aren't first-class :/) + unsafe { + Self(fd::RawFileDescriptor::new_unchecked(fd)) + } + } + + #[inline] + pub fn take_ownership_of(fd: impl Into) -> Self + { + Self(fd.into()) + } + + #[inline] + pub fn take_ownership_of_raw(fd: impl Into) -> Result + { + let fd = fd.into(); + Ok(Self(fd.try_into().map_err(|_| fd)?)) + } + + #[inline] + pub unsafe fn take_ownership_of_raw_unchecked(fd: impl Into) -> Self + { + Self(fd::RawFileDescriptor::new_unchecked(fd.into())) } /// Attempt to link this instance's fd to another container over an fd @@ -51,7 +89,7 @@ impl RawFile where T: AsRawFd { if unsafe { - libc::dup2(self.fileno(), other.as_raw_fd()) + libc::dup2(self.0.get(), other.as_raw_fd()) } < 0 { Err(error::DuplicateError::new_dup2(self, other)) } else { @@ -91,12 +129,12 @@ impl RawFile } } - /// Consume into a managed file + /// Consume into another managed file type container #[inline(always)] - pub fn into_file(self) -> fs::File + pub fn into_file(self) -> T { unsafe { - fs::File::from_raw_fd(self.into_raw_fd()) + T::from_raw_fd(self.into_raw_fd()) } } @@ -106,14 +144,65 @@ impl RawFile { opt.borrow().open(path).map(Into::into) } + + /// Open a new in-memory (W+R) file with an optional name and a fixed size. + pub fn open_mem(name: Option<&str>, len: usize) -> Result + { + lazy_static! { + static ref DEFAULT_NAME: String = format!(concat!("", "{}", ":", line!(), "-", column!(), ">"), function!()); //TODO: If it turns out memfd_create() requires an `&'static str`; remove the use of stackalloc, and have this variable be a nul-terminated CString instead. + } + + use libc::{ + memfd_create, + fallocate, + }; + use error::MemfileCreationStep::*; + + let rname = name.unwrap_or(&DEFAULT_NAME); + + stackalloc::alloca_zeroed(rname.len()+1, move |bname| { //XXX: Isn't the whole point of making `name` `&'static` that I don't know if `memfd_create()` requires static-lifetime name strings? TODO: Check this + macro_rules! attempt_call + { + ($errcon:literal, $expr:expr, $step:expr) => { + match unsafe { + $expr + } { + $errcon => Err($step), + x => Ok(x) + } + } + } + + let bname = { + unsafe { + std::ptr::copy_nonoverlapping(rname.as_ptr(), bname.as_mut_ptr(), rname.len()); + } + debug_assert_eq!(bname[rname.len()], 0, "Copied name string not null-terminated?"); + bname.as_ptr() + }; + + let fd = attempt_call!(-1, memfd_create(bname as *const _, MEMFD_CREATE_FLAGS), Create(name.map(str::to_owned), MEMFD_CREATE_FLAGS)) + .map(Self::take_ownership_of_unchecked)?; // Ensures `fd` is dropped if any subsequent calls fail + + attempt_call!(-1 + , fallocate(fd.0.get(), 0, 0, len.try_into() + .map_err(|_| Allocate(fd.fileno().clone(), len))?) + , Allocate(fd.fileno().clone(), len))?; + + Ok(fd) + + }) + } } + + impl io::Write for RawFile { #[inline] fn write(&mut self, buf: &[u8]) -> io::Result { match unsafe { - libc::write(self.fileno(), buf.as_ptr() as *const _, buf.len()) + libc::write(self.0.get(), buf.as_ptr() as *const _, buf.len()) } { -1 => Err(io::Error::last_os_error()), wr => Ok(wr as usize) @@ -129,7 +218,7 @@ impl io::Write for RawFile fn write_vectored(&mut self, bufs: &[io::IoSlice<'_>]) -> io::Result { // SAFETY: IoSlice is guaranteed to be ABI-compatible with `struct iovec` match unsafe { - libc::writev(self.fileno(), bufs.as_ptr() as *const _, bufs.len() as i32) + libc::writev(self.0.get(), bufs.as_ptr() as *const _, bufs.len() as i32) } { -1 => Err(io::Error::last_os_error()), wr => Ok(wr as usize) @@ -142,7 +231,7 @@ impl io::Read for RawFile #[inline] fn read(&mut self, buf: &mut [u8]) -> io::Result { match unsafe { - libc::read(self.fileno(), buf.as_mut_ptr() as *mut _, buf.len()) + libc::read(self.0.get(), buf.as_mut_ptr() as *mut _, buf.len()) } { -1 => Err(io::Error::last_os_error()), wr => Ok(wr as usize) @@ -153,7 +242,7 @@ impl io::Read for RawFile fn read_vectored(&mut self, bufs: &mut [io::IoSliceMut<'_>]) -> io::Result { // SAFETY: IoSlice is guaranteed to be ABI-compatible with `struct iovec` match unsafe { - libc::readv(self.fileno(), bufs.as_mut_ptr() as *mut _, bufs.len() as i32) + libc::readv(self.0.get(), bufs.as_mut_ptr() as *mut _, bufs.len() as i32) } { -1 => Err(io::Error::last_os_error()), wr => Ok(wr as usize) diff --git a/src/memfile/error.rs b/src/memfile/error.rs index 5260a1f..662cfad 100644 --- a/src/memfile/error.rs +++ b/src/memfile/error.rs @@ -22,7 +22,7 @@ pub struct DuplicateError { impl DuplicateError { - #[inline(always)] + #[inline] pub fn new_dup(from: &T) -> Self { Self{ @@ -32,7 +32,7 @@ impl DuplicateError } } - #[inline(always)] + #[inline] pub fn new_dup2(from: &T, to: &U) -> Self { Self { @@ -110,3 +110,83 @@ impl fmt::Display for DuplicateError } } + +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum MemfileCreationStep +{ + /// `memfd_create()` call + Create(Option, libc::c_uint), + /// `fallocate()` call + Allocate(fd::RawFileDescriptor, usize), + /// `mmap()` call + Map { + addr: usize, + size: usize, + prot: map::MapProtection, + flags: libc::c_int, + fd: Option, + offset: libc::off_t, + }, +} + +#[derive(Debug)] +pub struct MemfileError +{ + step: MemfileCreationStep, + inner: io::Error, +} + +impl fmt::Display for MemfileCreationStep +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result + { + match self { + Self::Create(None, 0 | MEMFD_CREATE_FLAGS) => f.write_str("memfd_create()"), + Self::Create(None, flags) => write!(f, "memfd_create(, {flags})"), + Self::Create(Some(name), flag) => write!(f, "memfd_create({name}, {flag})"), + Self::Allocate(fd, size) => write!(f, "fallocate({fd}, 0, 0, {size})"), + Self::Map{ addr: 0, size, prot, flags, fd: Some(fd), offset } => write!(f, "mmap(NULL, {size}, {prot:?}, {flags}, {fd}, {offset})"), + Self::Map{ addr: 0, size, prot, flags, fd: None, offset } => write!(f, "mmap(NULL, {size}, {prot:?}, {flags}, -1, {offset})"), + Self::Map{ addr, size, prot, flags, fd: Some(fd), offset } => write!(f, "mmap(0x{addr:x}, {size}, {prot:?}, {flags}, {fd}, {offset})"), + Self::Map{ addr, size, prot, flags, fd: None, offset } => write!(f, "mmap(0x{addr:x}, {size}, {prot:?}, {flags}, -1, {offset})"), + } + } +} + +impl error::Error for MemfileError +{ + #[inline] + fn source(&self) -> Option<&(dyn error::Error + 'static)> { + Some(&self.inner) + } +} +impl fmt::Display for MemfileError +{ + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result + { + write!(f, "failed to create in-memory file: `{}` failed", self.step) + } +} + +impl MemfileError +{ + #[inline] + pub fn from_step(step: MemfileCreationStep) -> Self + { + Self { + step, + inner: io::Error::last_os_error() + } + } +} + +impl From for MemfileError +{ + #[inline] + fn from(from: MemfileCreationStep) -> Self + { + Self::from_step(from) + } +} + diff --git a/src/memfile/fd.rs b/src/memfile/fd.rs index 19f2b04..1973a5f 100644 --- a/src/memfile/fd.rs +++ b/src/memfile/fd.rs @@ -118,7 +118,7 @@ impl RawFileDescriptor pub const STDIN: Self = Self(unsafe { NonNegativeI32::new_unchecked(0) }); pub const STDOUT: Self = Self(unsafe { NonNegativeI32::new_unchecked(1) }); pub const STDERR: Self = Self(unsafe { NonNegativeI32::new_unchecked(2) }); - + #[inline(always)] pub fn try_new(fd: FileNo) -> Result { @@ -142,8 +142,26 @@ impl RawFileDescriptor { self.0.get() } + + #[inline(always)] + pub(super) const fn clone_const(&self) -> Self + { + //! **Internal**: `clone()` but useable in `memfile`-local `const fn`s + //! : since this type is essentially a `Copy` type, but without implicit copying. + Self(self.0) + } } +impl fmt::Display for RawFileDescriptor +{ + #[inline(always)] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result + { + write!(f, "{}", self.get()) + } +} + + impl PartialEq for RawFileDescriptor { #[inline] diff --git a/src/memfile/map.rs b/src/memfile/map.rs new file mode 100644 index 0000000..8300d88 --- /dev/null +++ b/src/memfile/map.rs @@ -0,0 +1,22 @@ +//! Memory mapping +use super::*; +use libc::{ + c_int, + + PROT_NONE, + PROT_READ, + PROT_WRITE, + PROT_EXEC, +}; + +//TODO: Make this a `bitflags` struct. +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Copy, Default)] +#[repr(i32)] +pub enum MapProtection +{ + #[default] + None = PROT_NONE, + Read = PROT_READ, + Write = PROT_WRITE, + Execute = PROT_EXEC, +}