From 573845a66750a3ff05582664b6c613b8bf1b5979 Mon Sep 17 00:00:00 2001 From: Avril Date: Wed, 18 May 2022 01:23:43 +0100 Subject: [PATCH] memfile::hp: Added `get_masks()`: Returns an iterator over all `MAP_HUGE` masks found on system. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Added (default) feature-flag for `hugetlb`: Will use `memfile::hp` for `memfile` enabled feature. Added tests for both checked and unchecked `Mask` creation. Added feature `hugepage-checked-masks`: Use checked `Mask` creation in `get_masks{,_in}()`. Only useful for debugging or for ensuring no invalid values slip past mask creation. Disabled by default and offers not much of an improvement other than a bit more safety (integer arithmatic checking). Fortune for collect's current commit: Half curse − 半凶 --- Cargo.toml | 10 +++- src/memfile.rs | 2 +- src/memfile/hp.rs | 143 ++++++++++++++++++++++++++++++++++++++-------- 3 files changed, 128 insertions(+), 27 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 94c2773..8446ee7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,7 +24,7 @@ default = ["mode-memfile", "logging"] # Mode: default # Use physical-memory backed kernel file-descriptors. (see feature `memfile`.) -mode-memfile = ["memfile-preallocate"] #, "tracing/release_max_level_warn"] +mode-memfile = ["memfile-preallocate", "hugetlb"] #, "tracing/release_max_level_warn"] # Mode: alternative # Use non-physical memory allocated buffers. @@ -43,6 +43,14 @@ mode-buffered = ["jemalloc", "bytes"] # (This will very likely not happen unless you're specifically trying to make it happen, however.) memfile = ["bitflags", "lazy_static", "stackalloc"] +# Use `MAP_HUGE` masks when needed or asked for. +hugetlb = ["memfile"] + +# Check the conversion of kernel information into `MAP_HUGE` flags +# +# Usually only useful for debugging. +hugepage-checked-masks = ["hugetlb"] + # `memfile`: When unable to determine the size of the input, preallocate the buffer to a multiple of the system page-size before writing to it. This can save extra `ftruncate()` calls, but will also result in the buffer needing to be truncated to the correct size at the end if the sizes as not matched. # # *NOTE*: Requires `getpagesz()` to be available in libc. diff --git a/src/memfile.rs b/src/memfile.rs index d9f2797..709ad51 100644 --- a/src/memfile.rs +++ b/src/memfile.rs @@ -16,7 +16,7 @@ use std::{ pub mod fd; pub mod error; mod map; -//TODO: #[cfg(feature="hugetlb")] +#[cfg(feature="hugetlb")] mod hp; diff --git a/src/memfile/hp.rs b/src/memfile/hp.rs index 9b6f53d..f10ab6f 100644 --- a/src/memfile/hp.rs +++ b/src/memfile/hp.rs @@ -28,6 +28,67 @@ use libc::{ /// The contents of those subdirectories themselves are irrelevent for our purpose. pub const HUGEPAGE_SIZES_LOCATION: &'static str = "/sys/kernel/mm/hugepages"; +/// Should creation of `Mask`s from extracted kernel information be subject to integer conversion checks? +/// +/// This is `true` on debug builds or if the feature `hugepage-checked-masks` is enabled. +const CHECKED_MASK_CREATION: bool = if cfg!(feature="hugepage-checked-masks") || cfg!(debug_assertions) { true } else { false }; + + +/// Find all `Mask`s defined within this specific directory. +/// +/// This is usually only useful when passed `HUGEPAGE_SIZES_LOCATION` unless doing something funky with it. +/// For most use-cases, `get_masks()` should be fine. +#[cfg_attr(feature="logging", instrument(err, skip_all, fields(path = ?path.as_ref())))] +#[inline] +pub fn get_masks_in

(path: P) -> eyre::Result> + 'static> +where P: AsRef +{ + let path = path.as_ref(); + let root_path = { + let path = path.to_owned(); + move || path + }; + let root_path_section = { + let root_path = root_path.clone(); + move || + root_path().to_string_lossy().into_owned().header("Root path was") + }; + + + let dir = path.read_dir() + .wrap_err(eyre!("Failed to enumerate directory") + .with_section(root_path_section.clone()))?; + Ok(dir + .map(|x| x.map(|n| n.file_name())) + .map(|name| name.map(|name| (find_size_bytes(&name), name))) + .map(move |result| match result { + Ok((Some(ok), path)) => { + if CHECKED_MASK_CREATION { + Mask::new_checked(ok) + .wrap_err(eyre!("Failed to create mask from extracted bytes") + .with_section(|| ok.header("Bytes were")) + .with_section(move || format!("{path:?}").header("Checked path was")) + .with_section(root_path_section.clone())) + } else { + Ok(Mask::new(ok)) + } + }, + Ok((None, path)) => Err(eyre!("Failed to extract bytes from path")) + .with_section(move || format!("{path:?}").header("Checked path was")) + .with_section(root_path_section.clone()), + Err(e) => Err(e).wrap_err(eyre!("Failed to read path from which to extract bytes") + .with_section(root_path_section.clone())) + })) +} + +/// Find all `Mask`s on this system. +#[cfg_attr(feature="logging", instrument(level="trace"))] + #[inline] +pub fn get_masks() -> eyre::Result> + 'static> +{ + get_masks_in(HUGEPAGE_SIZES_LOCATION) +} + /// A huge-page mask that can be bitwise OR'd with `HUGETLB_MASK`. #[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Copy)] #[repr(transparent)] @@ -187,7 +248,7 @@ impl ops::BitOrAssign for Mask } } -//TODO: add test for `Mask::new{,_checked}()` above, and `.memfd_create_wrapper{,_flags}()` usage, too with some `MAP_HUGE_` constants as sizes +//TODO: add test for `Mask::new_checked()` above, and `.memfd_create_wrapper{,_flags}()` usage, too with some `MAP_HUGE_` constants as sizes /// Take a directory path and try to parse the hugepage size from it. /// @@ -290,22 +351,21 @@ fn find_size_bytes(path: impl AsRef) -> Option kmap_lookup(sz, k_chr) }) } -//TODO: add test for `find_size_bytes()` above #[cfg(test)] mod tests { use super::*; + #[inline] fn get_bytes<'a, P: 'a>(from: P) -> eyre::Result> +'a> - where P: AsRef + where P: AsRef { - use crate::*; let dir = from.as_ref().read_dir()?; Ok(dir - .map(|x| x.map(|n| n.file_name())) - .map(|name| name.map(|name| super::find_size_bytes(name))) - .map(|result| result.flatten())) + .map(|x| x.map(|n| n.file_name())) + .map(|name| name.map(|name| super::find_size_bytes(name))) + .map(|result| result.flatten())) } #[test] @@ -326,31 +386,64 @@ mod tests Ok(()) } - #[test] - fn find_map_huge_flags() -> eyre::Result<()> - { + mod map_huge { + use super::*; + /// Some `MAP_HUGE_` constants provided by libc. const CONSTANTS: &[c_int] = &[ libc::MAP_HUGE_1GB, libc::MAP_HUGE_1MB, libc::MAP_HUGE_2MB, ]; - eprintln!("Test array contains flags: {:#?}", CONSTANTS.iter().map(|x| format!("0x{x:X} (0b{x:b})")).collect::>()); - let mut ok = 0usize; - for bytes in get_bytes(super::HUGEPAGE_SIZES_LOCATION)? { - - let bytes = bytes?; - let flag = super::Mask::new(bytes); - if CONSTANTS.contains(&flag.raw()) { - println!("Found pre-set MAP_HUGE_ flag: {flag:X} ({flag:b}, {bytes} bytes)"); - ok +=1; + + #[inline] + fn find_constants_in(path: impl AsRef, checked: bool) -> eyre::Result + { + let mut ok = 0usize; + for bytes in get_bytes(path)? { + + let bytes = bytes?; + let flag = if checked { + super::Mask::new_checked(bytes) + .wrap_err(eyre!("Failed to create mask from bytes").with_section(|| bytes.header("Number of bytes was")))? + } else { + super::Mask::new(bytes) + }; + if CONSTANTS.contains(&flag.raw()) { + println!("Found pre-set MAP_HUGE_ flag: {flag:X} ({flag:b}, {bytes} bytes)"); + ok +=1; + } + } + Ok(ok) + } + + #[test] + fn find_map_huge_flags_checked() -> eyre::Result<()> + { + eprintln!("Test array contains flags: {:#?}", CONSTANTS.iter().map(|x| format!("0x{x:X} (0b{x:b})")).collect::>()); + let ok = find_constants_in(super::HUGEPAGE_SIZES_LOCATION, true).wrap_err("Failed to find constants (checked mask creation)")?; + if ok>0 { + println!("Found {ok} / {} of test flags set.", CONSTANTS.len()); + Ok(()) + } else { + println!("Found none of the test flags set..."); + Err(eyre!("Failed to find any matching map flags in test array of `MAP_HUGE_` flags: {:?}", CONSTANTS)) } } - if ok>0 { - println!("Found {ok} / {} of test flags set.", CONSTANTS.len()); - Ok(()) - } else { - println!("Found none of the test flags set..."); - Err(eyre!("Failed to find any matching map flags in test array of `MAP_HUGE_` flags: {:?}", CONSTANTS)) + + #[test] + fn find_map_huge_flags() -> eyre::Result<()> + { + eprintln!("Test array contains flags: {:#?}", CONSTANTS.iter().map(|x| format!("0x{x:X} (0b{x:b})")).collect::>()); + let ok = find_constants_in(super::HUGEPAGE_SIZES_LOCATION, false).wrap_err("Failed to find constants (unchecked mask creation)")?; + if ok>0 { + println!("Found {ok} / {} of test flags set.", CONSTANTS.len()); + Ok(()) + } else { + println!("Found none of the test flags set..."); + Err(eyre!("Failed to find any matching map flags in test array of `MAP_HUGE_` flags: {:?}", CONSTANTS)) + } } + + //TODO: test `get_masks()` } }