From 65c297b228c1979da53c7c044f26dab80ba3a2f9 Mon Sep 17 00:00:00 2001 From: Avril Date: Sun, 24 Apr 2022 07:46:14 +0100 Subject: [PATCH] Started adding `memfile` feature: Use a `memfd_create()`d in-memory temporary file. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fortune for collect's current commit: Curse − 凶 --- Cargo.toml | 3 +- src/buffers.rs | 2 +- src/main.rs | 2 + src/memfile.rs | 224 +++++++++++++++++++++++++++++++++++++++++++ src/memfile/error.rs | 112 ++++++++++++++++++++++ src/memfile/fd.rs | 198 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 539 insertions(+), 2 deletions(-) create mode 100644 src/memfile.rs create mode 100644 src/memfile/error.rs create mode 100644 src/memfile/fd.rs diff --git a/Cargo.toml b/Cargo.toml index fcfff04..ab2aeea 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,9 +6,10 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [features] -default = ["jemalloc", "logging", "tracing/release_max_level_warn"] +default = ["jemalloc", "memfile", "logging", "tracing/release_max_level_warn"] # TODO: mmap, memfd_create() ver +memfile = [] # bytes: use `bytes` crate for collecting instead of `std::vec` diff --git a/src/buffers.rs b/src/buffers.rs index 554306e..620d08d 100644 --- a/src/buffers.rs +++ b/src/buffers.rs @@ -217,7 +217,7 @@ impl MutBuffer for bytes::BytesMut fn freeze(self) -> Self::Frozen { bytes::BytesMut::freeze(self) } - //TODO: XXX: Impl copy_from_slice() as is done in impl for Vec + //TODO: XXX: Impl copy_from_slice() as is done in impl for Vec? Or change how `.writer()` works for us to return the BytesMut writer which seems more efficient. /*#[instrument] fn copy_from_slice(&mut self, st: usize, buf: &[u8]) -> usize { diff --git a/src/main.rs b/src/main.rs index 3c7f97d..f5be30a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -51,6 +51,8 @@ use color_eyre::{ mod buffers; use buffers::prelude::*; +#[cfg(feature="memfile")] mod memfile; + #[cfg(feature="bytes")] use bytes::{ Buf, diff --git a/src/memfile.rs b/src/memfile.rs new file mode 100644 index 0000000..24b01d8 --- /dev/null +++ b/src/memfile.rs @@ -0,0 +1,224 @@ +//! Memory file handling +use super::*; +use std::os::unix::io::*; +use std::{ + mem, + ops, + fs, + io, + path::Path, + borrow::{ + Borrow, + BorrowMut, + }, +}; + +mod fd; +pub mod error; + +#[derive(Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +#[repr(transparent)] +pub struct RawFile(fd::RawFileDescriptor); + +impl RawFile +{ + /// Get the raw fd for this raw file + #[inline(always)] + pub const fn fileno(&self) -> RawFd + { + self.0.get() + } + + #[inline(always)] + pub fn into_fileno(self) -> RawFd + { + self.into_raw_fd() + } + + #[inline(always)] + pub unsafe fn from_fileno(fd: RawFd) -> Self + { + Self::from_raw_fd(fd) + } + + /// Attempt to link this instance's fd to another container over an fd + /// + /// This is a safe wrapper around `dup2()`, as `clone()` is a safe wrapper around `dup()`. + /// + /// # Note + /// If `T` is a buffered container (e.g. `std::fs::File`), make sure the buffer is flushed *before* calling this method on it, or the buffered data will be lost. + pub fn try_link<'o, T: ?Sized>(&self, other: &'o mut T) -> Result<&'o mut T, error::DuplicateError> + where T: AsRawFd + { + if unsafe { + libc::dup2(self.fileno(), other.as_raw_fd()) + } < 0 { + Err(error::DuplicateError::new_dup2(self, other)) + } else { + Ok(other) + } + } + + /// Consume a managed file into a raw file, attempting to synchronise it first. + /// + /// # Note + /// This method attempts to sync the file's data. + /// To also attempt to sync the file's metadata, set `metadata` to true. + /// + /// # Returns + /// If the sync should fail, the original file is returned, along with the error from the sync. + #[inline(always)] + pub fn try_from_file_synced(file: fs::File, metadata: bool) -> Result + { + match if metadata { + file.sync_all() + } else { + file.sync_data() + } { + Ok(()) => unsafe { + Ok(Self::from_raw_fd(file.into_raw_fd())) + }, + Err(ioe) => Err((file, ioe)) + } + } + + /// Consume a managed fd type into a raw file + #[inline(always)] + pub fn from_file(file: impl IntoRawFd) -> Self + { + unsafe { + Self::from_raw_fd(file.into_raw_fd()) + } + } + + /// Consume into a managed file + #[inline(always)] + pub fn into_file(self) -> fs::File + { + unsafe { + fs::File::from_raw_fd(self.into_raw_fd()) + } + } + + /// Attempt to open a new raw file with these options + #[inline] + pub fn open(path: impl AsRef, opt: impl Borrow) -> io::Result + { + opt.borrow().open(path).map(Into::into) + } +} + +impl io::Write for RawFile +{ + #[inline] + fn write(&mut self, buf: &[u8]) -> io::Result { + match unsafe { + libc::write(self.fileno(), buf.as_ptr() as *const _, buf.len()) + } { + -1 => Err(io::Error::last_os_error()), + wr => Ok(wr as usize) + } + } + #[inline] + fn flush(&mut self) -> io::Result<()> { + // Not buffered + Ok(()) + } + + #[inline] + fn write_vectored(&mut self, bufs: &[io::IoSlice<'_>]) -> io::Result { + // SAFETY: IoSlice is guaranteed to be ABI-compatible with `struct iovec` + match unsafe { + libc::writev(self.fileno(), bufs.as_ptr() as *const _, bufs.len() as i32) + } { + -1 => Err(io::Error::last_os_error()), + wr => Ok(wr as usize) + } + } +} + +impl io::Read for RawFile +{ + #[inline] + fn read(&mut self, buf: &mut [u8]) -> io::Result { + match unsafe { + libc::read(self.fileno(), buf.as_mut_ptr() as *mut _, buf.len()) + } { + -1 => Err(io::Error::last_os_error()), + wr => Ok(wr as usize) + } + } + + #[inline] + fn read_vectored(&mut self, bufs: &mut [io::IoSliceMut<'_>]) -> io::Result { + // SAFETY: IoSlice is guaranteed to be ABI-compatible with `struct iovec` + match unsafe { + libc::readv(self.fileno(), bufs.as_mut_ptr() as *mut _, bufs.len() as i32) + } { + -1 => Err(io::Error::last_os_error()), + wr => Ok(wr as usize) + } + } +} + +impl From for RawFile +{ + #[inline] + fn from(from: fs::File) -> Self + { + Self::from_file(from) + } +} + +impl From for fs::File +{ + #[inline] + fn from(from: RawFile) -> Self + { + from.into_file() + } +} + + +impl Clone for RawFile +{ + #[inline] + fn clone(&self) -> Self { + unsafe { Self::from_raw_fd(libc::dup(self.0.get())) } + } +} +impl ops::Drop for RawFile +{ + #[inline] + fn drop(&mut self) { + unsafe { + libc::close(self.0.get()); + } + } +} + +impl AsRawFd for RawFile +{ + #[inline] + fn as_raw_fd(&self) -> RawFd { + self.0.get() + } +} + +impl FromRawFd for RawFile +{ + #[inline] + unsafe fn from_raw_fd(fd: RawFd) -> Self { + Self(fd::RawFileDescriptor::new(fd)) + } +} + +impl IntoRawFd for RawFile +{ + #[inline] + fn into_raw_fd(self) -> RawFd { + let fd = self.0.get(); + mem::forget(self); // prevent close + fd + } +} diff --git a/src/memfile/error.rs b/src/memfile/error.rs new file mode 100644 index 0000000..5260a1f --- /dev/null +++ b/src/memfile/error.rs @@ -0,0 +1,112 @@ +//! Errors +use super::*; +use std::{fmt, error}; + +/// The kind of duplicate fd syscall that was attempted +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Copy)] +pub enum DuplicateKind +{ + /// A `dup()` call failed + Duplicate, + /// A `dup2(fd)` call failed + Link(RawFd), +} + +/// Error returned when duplicating a file descriptor fails +#[derive(Debug)] +pub struct DuplicateError { + pub(super) from: RawFd, + pub(super) to: DuplicateKind, + pub(super) inner: io::Error, +} + +impl DuplicateError +{ + #[inline(always)] + pub fn new_dup(from: &T) -> Self + { + Self{ + inner: io::Error::last_os_error(), + from: from.as_raw_fd(), + to: DuplicateKind::Duplicate, + } + } + + #[inline(always)] + pub fn new_dup2(from: &T, to: &U) -> Self + { + Self { + inner: io::Error::last_os_error(), + from: from.as_raw_fd(), + to: DuplicateKind::Link(to.as_raw_fd()), + } + } + + #[inline] + pub fn new(from: &T, kind: DuplicateKind, reason: impl Into) -> Self + { + Self { + from: from.as_raw_fd(), + to: kind, + inner: reason.into() + } + } + + #[inline(always)] + pub fn reason(&self) -> &io::Error + { + &self.inner + } + + #[inline(always)] + pub fn kind(&self) -> &DuplicateKind + { + &self.to + } + + #[inline(always)] + pub fn source_fileno(&self) -> RawFd + { + self.from + } +} + +impl fmt::Display for DuplicateKind +{ + #[inline(always)] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result + { + match self { + Self::Duplicate => f.write_str("dup()"), + Self::Link(fd) => write!(f, "dup2({fd})"), + } + } +} + +impl error::Error for DuplicateError +{ + #[inline] + fn source(&self) -> Option<&(dyn error::Error + 'static)> { + Some(&self.inner) + } +} + +impl std::borrow::Borrow for DuplicateError +{ + #[inline] + fn borrow(&self) -> &io::Error + { + self.reason() + } +} + + +impl fmt::Display for DuplicateError +{ + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result + { + write!(f, "failed to {} fd {}", self.to, self.from) + } +} + diff --git a/src/memfile/fd.rs b/src/memfile/fd.rs new file mode 100644 index 0000000..19f2b04 --- /dev/null +++ b/src/memfile/fd.rs @@ -0,0 +1,198 @@ +//! Managing raw `fd`s +use super::*; +use std::num::NonZeroU32; +use libc::{ + c_int, +}; +use std::{ + fmt, error +}; + +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Copy)] +#[repr(transparent)] +struct NonNegativeI32(NonZeroU32); + +impl NonNegativeI32 +{ + pub const MASK: u32 = c_int::MIN as u32; //0b10000000_00000000_00000000_00000000; + + #[inline(always)] + pub const fn new(from: i32) -> Option + { + if from < 0 { + None + } else { + Some(unsafe { + Self::new_unchecked(from) + }) + } + } + + #[inline(always)] + pub const unsafe fn new_unchecked(from: i32) -> Self + { + Self(NonZeroU32::new_unchecked( (from as u32) | Self::MASK )) + } + + #[inline(always)] + pub const fn get(self) -> i32 + { + (self.0.get() & (!Self::MASK)) as i32 + } +} + +impl PartialEq for NonNegativeI32 +{ + #[inline] + fn eq(&self, other: &i32) -> bool + { + self.get() == *other + } +} + +impl PartialOrd for NonNegativeI32 +{ + #[inline] + fn partial_cmp(&self, other: &i32) -> Option { + self.get().partial_cmp(other) + } +} + +impl Default for NonNegativeI32 +{ + #[inline(always)] + fn default() -> Self + { + unsafe { + Self::new_unchecked(0) + } + } +} + +impl From for i32 +{ + #[inline(always)] + fn from(from: NonNegativeI32) -> Self + { + from.get() + } +} + +impl TryFrom for NonNegativeI32 +{ + type Error = std::num::TryFromIntError; + + #[inline(always)] + fn try_from(from: i32) -> Result + { + NonZeroU32::try_from((!from as u32) & Self::MASK)?; + debug_assert!(from >= 0, "Bad check"); + unsafe { + Ok(Self::new_unchecked(from)) + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct BadFDError(()); + +impl error::Error for BadFDError{} +impl fmt::Display for BadFDError +{ + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result + { + f.write_str("invalid file descriptor") + } +} + + +pub type FileNo = RawFd; + +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +#[repr(transparent)] +pub struct RawFileDescriptor(NonNegativeI32); + +impl RawFileDescriptor +{ + pub const STDIN: Self = Self(unsafe { NonNegativeI32::new_unchecked(0) }); + pub const STDOUT: Self = Self(unsafe { NonNegativeI32::new_unchecked(1) }); + pub const STDERR: Self = Self(unsafe { NonNegativeI32::new_unchecked(2) }); + + #[inline(always)] + pub fn try_new(fd: FileNo) -> Result + { + NonNegativeI32::new(fd).ok_or(BadFDError(())).map(Self) + } + + #[inline] + pub fn new(fd: FileNo) -> Self + { + Self::try_new(fd).expect("Invalid fileno") + } + + #[inline(always)] + pub const unsafe fn new_unchecked(fd: FileNo) -> Self + { + Self(NonNegativeI32::new_unchecked(fd)) + } + + #[inline(always)] + pub const fn get(&self) -> FileNo + { + self.0.get() + } +} + +impl PartialEq for RawFileDescriptor +{ + #[inline] + fn eq(&self, other: &FileNo) -> bool + { + self.get() == *other + } +} + +impl PartialOrd for RawFileDescriptor +{ + #[inline] + fn partial_cmp(&self, other: &FileNo) -> Option { + self.get().partial_cmp(other) + } +} + +impl From for RawFileDescriptor +{ + #[inline(always)] + fn from(from: NonNegativeI32) -> Self + { + Self(from) + } +} + +impl TryFrom for RawFileDescriptor +{ + type Error = BadFDError; + + #[inline(always)] + fn try_from(from: FileNo) -> Result + { + Self::try_new(from) + } +} + +impl From for FileNo +{ + #[inline(always)] + fn from(from: RawFileDescriptor) -> Self + { + from.get() + } +} + +impl AsRawFd for RawFileDescriptor +{ + fn as_raw_fd(&self) -> RawFd { + self.get() + } +}