You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
318 lines
9.1 KiB
318 lines
9.1 KiB
//! Huge-page interface for `MappedFile<T>` and `MemoryFile`.
|
|
use super::*;
|
|
use std::{
|
|
mem,
|
|
hash,
|
|
num::NonZeroUsize,
|
|
fs,
|
|
path::{Path, PathBuf},
|
|
};
|
|
use libc::{
|
|
c_int,
|
|
MAP_HUGE_SHIFT,
|
|
};
|
|
|
|
/// Location in which the kernel exposes available huge-page sizes.
|
|
pub const HUGEPAGE_LOCATION: &'static str = "/sys/kernel/mm/hugepages/";
|
|
|
|
/// Represents a statically defined `MAP_HUGE_*` flag.
|
|
#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Copy)]
|
|
#[repr(transparent)]
|
|
pub struct MapHugeFlag(c_int);
|
|
|
|
impl Default for MapHugeFlag
|
|
{
|
|
#[inline]
|
|
fn default() -> Self {
|
|
Self(MAP_HUGE_SHIFT)
|
|
}
|
|
}
|
|
#[inline(always)]
|
|
const fn log2(n: usize) -> usize
|
|
{
|
|
usize::BITS as usize - n.leading_zeros() as usize - 1
|
|
}
|
|
|
|
impl MapHugeFlag
|
|
{
|
|
/// Create from a raw `MAP_HUGE_*` flag.
|
|
///
|
|
/// # Safety
|
|
/// The passed `flag` **must** be a valid bitmask representing a `MAP_HUGE_*` value **only**.
|
|
#[inline]
|
|
pub const unsafe fn from_mask_unchecked(flag: c_int) -> Self
|
|
{
|
|
Self(flag)
|
|
}
|
|
|
|
/// The kernel's default huge-page size.
|
|
pub const HUGE_DEFAULT: Self = Self(MAP_HUGE_SHIFT);
|
|
/// Predefined `MAP_HUGE_2MB` mask,
|
|
pub const HUGE_2MB: Self = Self(libc::MAP_HUGE_2MB);
|
|
/// Predefined `MAP_HUGE_1GB` mask,
|
|
pub const HUGE_1GB: Self = Self(libc::MAP_HUGE_1GB);
|
|
|
|
/// Calculate a `MAP_HUGE_*` flag from a size (in kB).
|
|
#[inline(always)]
|
|
pub const fn calculate(kilobytes: NonZeroUsize) -> Self
|
|
{
|
|
Self((log2(kilobytes.get()) << (MAP_HUGE_SHIFT as usize)) as c_int)
|
|
}
|
|
|
|
/// Attempt to calculate `MAP_HUGE_*` flag from a size (in kB).
|
|
#[inline]
|
|
pub const fn try_calculate(kilobytes: usize) -> Option<Self>
|
|
{
|
|
match kilobytes {
|
|
0 => None,
|
|
kilobytes => {
|
|
if let Some(shift) = log2(kilobytes).checked_shl(MAP_HUGE_SHIFT as u32) {
|
|
if shift <= c_int::MAX as usize {
|
|
return Some(Self(shift as c_int));
|
|
}
|
|
}
|
|
None
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Attempt to calculate `MAP_HUGE_*`, or use `HUGE_DEFAULT` on failure.
|
|
///
|
|
/// # Note
|
|
/// If `kilobytes` is `0`, or there is a calculation overflow, then `HUGE_DEFAULT` is returned.
|
|
#[inline]
|
|
pub const fn calculate_or_default(kilobytes: usize) -> Self
|
|
{
|
|
match Self::try_calculate(kilobytes) {
|
|
None => Self::HUGE_DEFAULT,
|
|
Some(x) => x,
|
|
}
|
|
}
|
|
|
|
/// Check if this is the smallest huge-page size the kernel supports.
|
|
#[inline]
|
|
pub const fn is_default(&self) -> bool
|
|
{
|
|
self.0 == Self::HUGE_DEFAULT.0
|
|
}
|
|
|
|
/// Get the `MAP_HUGE_*` mask.
|
|
#[inline(always)]
|
|
pub const fn get_mask(self) -> c_int
|
|
{
|
|
self.0
|
|
}
|
|
}
|
|
|
|
impl From<MapHugeFlag> for c_int
|
|
{
|
|
#[inline]
|
|
fn from(from: MapHugeFlag) -> Self
|
|
{
|
|
from.0
|
|
}
|
|
}
|
|
|
|
#[derive(Default, Clone, Copy)]
|
|
pub enum HugePage {
|
|
/// A staticly presented `MAP_HUGE_*` flag. See `MapHugeFlag` for details.
|
|
Static(MapHugeFlag),
|
|
/// A dynamically calculated `MAP_HUGE_*` flag from an arbitrary size *in kB*.
|
|
///
|
|
/// # Safety
|
|
/// The kernel must actually support huge-pages of this size.
|
|
///
|
|
/// If `kilobytes` is 0, or an overflow in calculation happens, then this is identical to `Smallest`.
|
|
Dynamic{ kilobytes: usize },
|
|
/// The smallest huge-page size on the system
|
|
#[default]
|
|
Smallest,
|
|
/// The largest huge-page size on the system
|
|
Largest,
|
|
/// Use a callback function to select the huge-page size (*in kB*) from an *ordered* (lowest to highest) enumeration of all available on the system.
|
|
//TODO: Remember to order the HUGEPAGE_LOCATION parsing results before passing them to this!
|
|
Selected(for<'r> fn (&'r [usize]) -> Option<&'r usize>),
|
|
}
|
|
|
|
impl hash::Hash for HugePage {
|
|
#[inline]
|
|
fn hash<H: hash::Hasher>(&self, state: &mut H) {
|
|
mem::discriminant(self).hash(state);
|
|
match self {
|
|
Self::Static(hpf) => hpf.hash(state),
|
|
Self::Dynamic { kilobytes } => kilobytes.hash(state),
|
|
Self::Selected(func) => ptr::hash(func as *const _, state),
|
|
_ => (),
|
|
};
|
|
}
|
|
}
|
|
|
|
impl fmt::Debug for HugePage
|
|
{
|
|
#[inline]
|
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result
|
|
{
|
|
f.debug_tuple("HugePage")
|
|
.field({
|
|
let v: &dyn fmt::Debug = match &self {
|
|
Self::Static(ref huge) => huge,
|
|
Self::Dynamic { ref kilobytes } => kilobytes,
|
|
Self::Smallest => &"<smallest>",
|
|
Self::Largest => &"<largest>",
|
|
Self::Selected(_) => &"<selector>",
|
|
};
|
|
v
|
|
})
|
|
.finish()
|
|
}
|
|
}
|
|
|
|
|
|
impl Eq for HugePage {}
|
|
impl PartialEq for HugePage
|
|
{
|
|
#[inline]
|
|
fn eq(&self, other: &Self) -> bool
|
|
{
|
|
match (self, other) {
|
|
(Self::Static(hpf), Self::Static(hpf2)) => hpf == hpf2,
|
|
(Self::Dynamic { kilobytes }, Self::Dynamic { kilobytes: kilobytes2 }) => kilobytes == kilobytes2,
|
|
(Self::Selected(func), Self::Selected(func2)) => ptr::eq(func, func2),
|
|
_ => mem::discriminant(self) == mem::discriminant(other),
|
|
}
|
|
}
|
|
}
|
|
|
|
impl HugePage
|
|
{
|
|
/// Compute the `MapHugeFlag` from this huge-page specification.
|
|
///
|
|
/// # Returns
|
|
/// * `None` - If there was an error in computing the correct flag.
|
|
/// * `Some` - If the computation was successful.
|
|
///
|
|
/// # Panics
|
|
/// In debug builds, if scanning the system for huge-pages fails after `SYSTEM_HUGEPAGES` has already failed.
|
|
#[inline] // This call is recursive, but also can be large for variant `Selected`, which we have factored out into a non-inline local function. All other variants are small enough for this to be okay.
|
|
pub fn compute_huge(self) -> Option<MapHugeFlag>
|
|
{
|
|
use HugePage::*;
|
|
match self {
|
|
Dynamic { kilobytes: 0 } |
|
|
Smallest |
|
|
Static(MapHugeFlag::HUGE_DEFAULT) => Some(MapHugeFlag::HUGE_DEFAULT),
|
|
Static(mask) => Some(mask),
|
|
Dynamic { kilobytes } => {
|
|
MapHugeFlag::try_calculate(kilobytes) //XXX: Should we use `calculate_or_default()` here?
|
|
},
|
|
Largest => Self::Selected(|sizes| sizes.iter().max()).compute_huge(),
|
|
Selected(func) => {
|
|
// Factored out into a non-`inline` function since it's the only one doing actual work, and allows the parent function to be `inline` without bloating to much
|
|
fn compute_selected(func: for<'r> fn (&'r [usize]) -> Option<&'r usize>) -> Option<MapHugeFlag>
|
|
{
|
|
use std::borrow::Cow;
|
|
let mask = match SYSTEM_HUGEPAGE_SIZES.as_ref() {
|
|
Ok(avail) => Cow::Borrowed(&avail[..]),
|
|
Err(_) => {
|
|
// Attempt to re-scan the system. Fail if scan fails.
|
|
#[cold]
|
|
fn rescan() -> io::Result<Vec<usize>>
|
|
{
|
|
scan_hugepages().and_then(|x| x.into_iter().collect())
|
|
}
|
|
let v = rescan();
|
|
let mut v = if cfg!(debug_assertions) {
|
|
v.expect("Failed to compute available hugetlb sizes")
|
|
} else {
|
|
v.ok()?
|
|
};
|
|
v.sort_unstable();
|
|
Cow::Owned(v)
|
|
},
|
|
};
|
|
|
|
match func(mask.as_ref()) {
|
|
Some(mask) => Dynamic { kilobytes: *mask }.compute_huge(),
|
|
None => Some(MapHugeFlag::HUGE_DEFAULT),
|
|
}
|
|
}
|
|
compute_selected(func)
|
|
},
|
|
}
|
|
}
|
|
}
|
|
|
|
lazy_static! {
|
|
/// A persistent invocation of `scan_hugepages()`.
|
|
pub(crate) static ref SYSTEM_HUGEPAGE_SIZES: io::Result<Vec<usize>> = {
|
|
let mut val: io::Result<Vec<usize>> = scan_hugepages().and_then(|x| x.into_iter().collect());
|
|
if let Ok(ref mut arr) = val.as_mut() {
|
|
arr.sort_unstable();
|
|
};
|
|
val
|
|
};
|
|
|
|
/// A list of all availble huge-page flags if enumeration of them is possible.
|
|
///
|
|
/// This is created from a persistent invocation of `scan_hugepages()`.
|
|
pub static ref SYSTEM_HUGEPAGES: io::Result<Vec<MapHugeFlag>> =
|
|
SYSTEM_HUGEPAGE_SIZES.as_ref()
|
|
.map_err(|err| io::Error::new(io::ErrorKind::InvalidInput, format!("SYSTEM_HUGEPAGES failed with error {err}")))
|
|
.map(|vec| vec.iter().map(|&size| MapHugeFlag::calculate_or_default(size)).collect());
|
|
}
|
|
|
|
/// Scan the system for available huge-page sizes (in kB).
|
|
///
|
|
/// # Returns
|
|
/// If reading the directory `HUGEPAGE_LOCATION` fails, then the error is returned.
|
|
/// Otherwise, an iterator over each item in this location, parsed for its size, is returned.
|
|
/// If reading an entry fails, an error is returned.
|
|
///
|
|
/// If an entry is not parsed correctly, then it is skipped.
|
|
pub fn scan_hugepages() -> io::Result<impl IntoIterator<Item=io::Result<usize>> + Send + Sync + 'static>
|
|
{
|
|
let path = Path::new(HUGEPAGE_LOCATION);
|
|
let dir = fs::read_dir(path)?;
|
|
|
|
#[derive(Debug)]
|
|
struct FilteredIterator(fs::ReadDir);
|
|
|
|
impl Iterator for FilteredIterator
|
|
{
|
|
type Item = io::Result<usize>;
|
|
fn next(&mut self) -> Option<Self::Item> {
|
|
loop {
|
|
break if let Some(next) = self.0.next() {
|
|
let path = match next {
|
|
Ok(next) => next.file_name(),
|
|
Err(err) => return Some(Err(err)),
|
|
};
|
|
let kbs = if let Some(dash) = memchr::memchr(b'-', path.as_bytes()) {
|
|
let name = &path.as_bytes()[(dash+1)..];
|
|
if let Some(k_loc) = memchr::memrchr(b'k', &name) {
|
|
&name[..k_loc]
|
|
} else {
|
|
continue
|
|
}
|
|
} else {
|
|
continue
|
|
};
|
|
let kb = if let Ok(kbs) = std::str::from_utf8(kbs) {
|
|
kbs.parse::<usize>().ok()
|
|
} else {
|
|
continue
|
|
};
|
|
match kb {
|
|
None => continue,
|
|
valid => valid.map(Ok)
|
|
}
|
|
} else {
|
|
None
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
Ok(FilteredIterator(dir))
|
|
}
|