unmapping hopefully..

serial
Avril 4 years ago
parent 09508a9c5d
commit 18944e4838
Signed by: flanchan
GPG Key ID: 284488987C31F630

1
.gitignore vendored

@ -1,2 +1,3 @@
/target /target
/test
*~ *~

@ -1,5 +1,6 @@
#![allow(dead_code)] #![allow(dead_code)]
use std::convert::TryFrom;
#[macro_export] macro_rules! unwrap { #[macro_export] macro_rules! unwrap {
(return $($code:expr)?; $err:expr) => { (return $($code:expr)?; $err:expr) => {
@ -38,10 +39,23 @@ fn work(arg::Operation{output, inputs}: arg::Operation) -> Result<(), Box<dyn st
let output = open::output(output)?; let output = open::output(output)?;
// - open and stat all input files in order (`job::create_from_file`). // - open and stat all input files in order (`job::create_from_file`).
let inputs = open::input(inputs)?; let inputs = open::input(inputs)?;
// - `fallocate` the output file fd to the sum of all input file sizes if inputs.len() == 0 {
output.into_inner().set_len(0)?; // Just truncate the output and return
return Ok(());
}
let (jobs, size): (Vec<_>, _) = {
let mut off = 0;
((0..).zip(inputs.into_iter()).map(|(i, input)| job::create(i, input, &mut off)).collect(), off)
};
debug_assert!(u64::try_from(size).is_ok(), "Output file too large, would exceed unsigned 64bit integer");
// - `fallocate` the output file fd to the sum of all input file sizes
// - `mmap` the output file as writable // - `mmap` the output file as writable
// let output = output.complete(size)?;
// - create state
let state: state::State = {todo!()};
// - spawn the task thread pool // - spawn the task thread pool
// - move the output mapped file to the thread-safe refcounted `state::State`. // - move the output mapped file to the thread-safe refcounted `state::State`.
// - dispatch jobs to the pool with their fds, stats, and calculated output offsets; along with a reference to the output mapped file and a sender for the completion stream (`job::Prelude::start`) // - dispatch jobs to the pool with their fds, stats, and calculated output offsets; along with a reference to the output mapped file and a sender for the completion stream (`job::Prelude::start`)
@ -51,6 +65,12 @@ fn work(arg::Operation{output, inputs}: arg::Operation) -> Result<(), Box<dyn st
// - ensure all data was written. // - ensure all data was written.
// - truncate the output file to the correct size (sum of all the input sizes) if needed (we do not truncate before fallocate()ing it, so if a previous larger file existed there, it will have trailing garbage) // - truncate the output file to the correct size (sum of all the input sizes) if needed (we do not truncate before fallocate()ing it, so if a previous larger file existed there, it will have trailing garbage)
{
use std::io::Write;
let mut output = state.try_into_inner().unwrap().unmap();
output.set_len(u64::try_from(size).expect("File size is larger than u64::MAX"))?;
output.flush()?;
}
Ok(()) Ok(())
} }

@ -8,8 +8,8 @@ use memmap::MmapMut;
#[derive(Debug)] #[derive(Debug)]
pub struct MemoryMapMut pub struct MemoryMapMut
{ {
file: fs::File,
map: MmapMut, map: MmapMut,
file: fs::File,
} }
impl Drop for MemoryMapMut impl Drop for MemoryMapMut
@ -38,6 +38,21 @@ impl AsMut<[u8]> for MemoryMapMut
impl MemoryMapMut impl MemoryMapMut
{ {
/// Unmap this file and return the fd
pub fn unmap(self) -> fs::File
{
use std::{mem, ptr};
let mut this = mem::ManuallyDrop::new(self);
unsafe {
let map = (&mut this.map as *mut MmapMut).read();
let file = (&mut this.file as *mut fs::File).read();
let _ = map.flush();
drop(map);
file
}
}
#[inline] pub fn as_slice_mut(&mut self) -> &mut [u8] #[inline] pub fn as_slice_mut(&mut self) -> &mut [u8]
{ {
self.as_mut() self.as_mut()

@ -27,6 +27,12 @@ pub struct Output(File);
impl Output impl Output
{ {
/// Consume into a normal file.
#[inline] pub fn into_inner(self) -> File
{
self.0
}
/// Allocate this reserved output file `to` bytes, the map it and return. /// Allocate this reserved output file `to` bytes, the map it and return.
pub fn complete(self, to: usize) -> Result<map::MemoryMapMut, OutputError> pub fn complete(self, to: usize) -> Result<map::MemoryMapMut, OutputError>
{ {

Loading…
Cancel
Save