Control flow for mapped file accessors. (TODO: mapped_file::MemoryFile must implement io::Read+Write interfaces.)

Fortune for chacha20's current commit: Future small blessing − 末小吉
mmap-feature
Avril 2 years ago
parent e707df9a12
commit d9f4e729c2
Signed by: flanchan
GPG Key ID: 284488987C31F630

35
Cargo.lock generated

@ -38,9 +38,12 @@ version = "2.0.0"
dependencies = [
"base64",
"getrandom",
"lazy_static",
"libc",
"mapped-file",
"openssl",
"rustc_version",
"smallmap",
"smallvec",
]
@ -70,12 +73,35 @@ dependencies = [
"wasi",
]
[[package]]
name = "lazy_static"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]]
name = "libc"
version = "0.2.133"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c0f80d65747a3e43d1596c7c5492d95d5edddaabd45a7fcdb02b95f644164966"
[[package]]
name = "mapped-file"
version = "0.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b0228d78d6433a3ed6b3ac9b91aa6e4dac1094ac841fea554779b97fa4c111db"
dependencies = [
"lazy_static",
"libc",
"memchr",
]
[[package]]
name = "memchr"
version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
[[package]]
name = "once_cell"
version = "1.7.2"
@ -139,6 +165,15 @@ version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
[[package]]
name = "smallmap"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2df0ae8eb5af9e6e00be6ba531cc6860ba93d5226d6d1d4f80ead510a5d6296e"
dependencies = [
"rustc_version",
]
[[package]]
name = "smallvec"
version = "1.6.1"

@ -17,8 +17,11 @@ explicit_clear = []
[dependencies]
base64 = "0.13"
getrandom = "0.2"
lazy_static = "1.4.0"
libc = { version = "0.2.133", optional = true }
mapped-file = { version = "0.0.2", features = ["file"] }
openssl = "0.10"
smallmap = "1.4.0"
smallvec = {version = "1.6", features=["union"]}
[build-dependencies]

@ -1,8 +1,10 @@
#![cfg_attr(nightly, feature(asm))]
//#![cfg_attr(nightly, feature(asm))]
#![allow(dead_code)]
#[macro_use] extern crate lazy_static;
//extern crate test;
#[macro_use] mod ext; #[allow(unused_imports)] use ext::*;

@ -17,8 +17,16 @@ use std::{
use libc::{
mmap, munmap, madvise, MAP_FAILED,
};
use mapped_file::{
MappedFile,
Perm,
Flags,
file::memory::{
MemoryFile,
},
};
use openssl::symm::Crypter;
/*
#[derive(Debug)]
struct MapInner
{
@ -35,7 +43,7 @@ impl ops::Drop for MapInner
munmap(self.mem.as_ptr() as *mut _, self.size);
}
}
}
}*/
#[inline]
pub fn raw_file_size(fd: &(impl AsRawFd + ?Sized)) -> io::Result<u64>
@ -51,6 +59,17 @@ pub fn raw_file_size(fd: &(impl AsRawFd + ?Sized)) -> io::Result<u64>
}
}
#[inline]
fn try_truncate(fd: &(impl AsRawFd + ?Sized), to: u64) -> io::Result<()>
{
//use libc::ftruncate;
let to = to.try_into().map_err(|e| io::Error::new(io::ErrorKind::InvalidInput, e))?;
match unsafe { ftruncate(fd.as_raw_fd(), to) } {
0 => Ok(()),
_ => Err(io::Error::last_os_error()),
}
}
/*
#[derive(Debug)]
pub struct Mapped<T: ?Sized>
{
@ -79,7 +98,7 @@ impl<T: AsRawFd> Mapped<T>
let size = raw_file_size(&file)?;
Self::try_new_sized(file, size, rw)
}
}
}*/
/*
impl<T: AsRawFd> TryFrom<T> for Mapped<T>
{
@ -92,19 +111,178 @@ impl<T: AsRawFd> TryFrom<T> for Mapped<T>
}*/
fn process_mapped_files<T, U>(mode: &mut Crypter, input: &mut Mapped<T>, output: &mut Mapped<U>) -> io::Result<()>
where T: AsRawFd + ?Sized,
U: AsRawFd + ?Sized
fn process_mapped_files<T, U>(mode: &mut Crypter, input: &mut MappedFile<T>, output: &mut MappedFile<U>) -> io::Result<()>
where T: AsRawFd,
U: AsRawFd,
{
mode.update(&input[..], &mut output[..])?;
Ok(())
}
fn try_map_sized<T: AsRawFd>(file: T, perm: mapped_file::Perm, flags: impl mapped_file::MapFlags) -> Result<MappedFile<T>, T>
{
macro_rules! unwrap {
($res:expr) => {
match $res {
Ok(v) => v,
_ => return Err(file)
}
};
}
if let Ok(sz) = raw_file_size(&file) {
let sz = unwrap!(sz.try_into());
MappedFile::try_new(file, sz, perm, flags).map_err(|err| err.into_inner())
} else {
Err(file)
}
}
#[inline]
fn try_map_to<T: AsRawFd + ?Sized>(file: &T, file_size: usize) -> bool
{
todo!("mode.update(input.slice(), output.slice()); /* unneeded: mode.finalize(input.slice(), output.slice()) */ ");
let file_size = match file_size.try_into() {
Ok(v) => v,
_ => return false
};
if let Ok(stdout_size) = raw_file_size(file) {
if stdout_size >= file_size {
// Size already ok
return true;
}
}
// Grow file
unsafe {
libc::ftruncate(file.as_raw_fd(), file_size) == 0
}
}
#[inline]
fn translate_hugetlb_size(size: usize) -> mapped_file::hugetlb::HugePage
{
#[inline]
fn check_func(sizes_kb: &[usize]) -> Option<&usize>
{
sizes_kb.iter().skip_while(|&&kb| kb < 1024 * 1024).next()
.or_else(|| sizes_kb.iter().nth(1)
.or_else(|| sizes_kb.first()))
}
match size {
0..(1024*1024) => mapped_file::hugetlb::HugePage::Smallest,
(1024*1024)..(1024*1024*1024) => mapped_file::hugetlb::HugePage::Selected(check_func),
very_high => mapped_file::hugetlb::HugePage::Largest,
}
}
/// Create and map a temporary memory file of this size.
///
/// This function may optionally choose to huge hugepages if `size` is large enough
fn create_sized_temp_mapping(size: usize) -> io::Result<(MappedFile<MemoryFile>, bool)>
{
const HUGETLB_THRESH: usize = 1024 * 1024; // 1MB
let file = match size {
0 => MemoryFile::new(),
0..HUGETLB_THRESH => MemoryFile::with_size(size),
size => {
let hugetlb = translate_hugetlb_size(size);
let file = MemoryFile::with_size_hugetlb(size, hugetlb)?;
return MappedFile::new(file, size, Perm::ReadWrite, Flags::Shared.with_hugetlb(hugetlb)).map(|x| (x, true));
},
}?;
MappedFile::new(file, size, Perm::ReadWrite, Flags::Shared).map(|x| (x, false))
}
pub fn try_process(mut mode: impl BorrowMut<Crypter>) -> io::Result<io::Result<()>>
{
let mode = mode.borrow_mut();
let stdin = io::stdin().lock();
let stdout = io::stdout().lock();
let file_size = raw_file_size(&stdin)?;
macro_rules! attempt_mapping {
($file:expr, $perm:expr, $flags:expr) => {
{
let file = $file;
if try_map_to(&file, file_size) {
MappedFile::try_new(file, file_size, $perm, $flags).map_err(|e| e.into_inner())
} else {
return Err(io::Error::new(io::ErrorKind::InvalidData, concat!("Failed to truncate ", stringify!($file), " to size")));
}
}
};
($file:expr, $perm:expr) => {
attempt_mapping($file, $perm, Flags::default())
};
}
// Try map stdout
Ok(match attempt_mapping!(stdout, Perm::ReadWrite, Flags::Shared) {
Ok(mut mstdout) => {
let res = match try_map_sized(stdin, mapped_file::Perm::Readonly, mapped_file::Flags::Private)
{
Ok(mut mstdin) => {
// Stdin and stdout are mapped. (3)
process_mapped_files(&mut mode, &mut mstdin, &mut mstdout)
},
Err(_) => {
// Only stdout is mapped. (1)
let size = file_size as usize;
let is_huge = size >= 1024*1024;
if is_huge {
let (mapped_memfd, _) = create_sized_temp_mapping(size)?;
io::copy(&mut stdin, &mut &mapped_memfd[..]).and_then(move |_| mapped_memfd)
} else {
MemoryFile::with_size(size).or_else(|_| MemoryFile::new()).and_then(|mut memfd| {
let size = io::copy(&mut stdin, &mut memfd)?;
MappedFile::new(memfd, size as usize, Perm::ReadWrite, Flags::Shared)
})
}.and_then(move |mut mapped_memfd| {
process_mapped_files(mode, &mut mapped_memfd, &mut mstdout)
})
//todo!("Copy stdin into a memory-file, and then map that and process it to stdout (XXX: What if the file is too large, but we cannot tell the size of stdin?)")
}
};
if res.is_ok() {
mstdout.flush(mapped_file::Flush::Wait)
} else {
res
}
},
Err(mut stdout) => {
match try_map_sized(stdin, mapped_file::Perm::Readonly, mapped_file::Flags::Private)
{
Ok(mut mstdin) => {
// Only stdin is mapped. (2)
if cfg!(feature="sodiumoxide") {
todo!("XXX: When we switch to `sodiumoxide`, we won't *need* this, we can mutate the *private* stdin mapping directly.")
} else {
//TODO: XXX: When we switch to `sodiumoxide`, we won't *need* this, we can mutate the *private* stdin mapping directly.
//todo!("Create a memory file (possibly with hugetlb, depending on `file_size`), map that, process_mapped_files(...) into that memory file, then `io::copy(&mut &memfile[..], &stdout)`")
let (mapped_memfd, is_huge) = create_sized_temp_mapping(file_size as usize)?;
process_mapped_files(&mut mode, &mut mstdin, &mut mapped_memfd).and_then(move |_| {
if is_huge {
// Cannot use fd syscalls on `MFD_HUGELB`, copy into stdout from the mapped buffer itself.
io::copy(&mut &mapped_file[..], &mut stdout)
} else {
// Sync the contents into the memory file then use fd syscalls to copy into stdout.
mapped_memfd.flush(mapped_file::Flush::Wait).and_then(move |_| {
let mut memfd = mapped_memfd.into_inner();
io::copy(&mut memfd, &mut stdout)
})
}
}).map(|_| ())
}
},
Err(_) => {
// Neither are mapped. (0)
return Err(io::Error::new(io::ErrorKind::InvalidInput, "cannot map stdin or stdout"));
}
}
},
})
/*let mstdin = Mapped::try_new(input)?;
let mstdout = Mapped::try_new(output)?;*/ //TODO: if failed to map output, but input is successful (TODO: XXX: implement other way around), we can fall back to wrapping output in `Sink`.
todo!("Try to map the stdin and stdout streams, if that fails, return Err(last_os_err).");
todo!("return Ok(process_mapped_files(mode, mstdin, mstdout, key, iv))")
//todo!("Try to map the stdin and stdout streams, if that fails, return Err(last_os_err).");
//todo!("return Ok(process_mapped_files(mode, mstdin, mstdout, key, iv))")
}

Loading…
Cancel
Save