data structures and layout

master
Avril 3 years ago
commit 48277f62ac
Signed by: flanchan
GPG Key ID: 284488987C31F630

2
.gitignore vendored

@ -0,0 +1,2 @@
/target
*~

627
Cargo.lock generated

@ -0,0 +1,627 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
[[package]]
name = "autocfg"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
[[package]]
name = "base64"
version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff"
[[package]]
name = "bitflags"
version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
[[package]]
name = "block-buffer"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4"
dependencies = [
"generic-array",
]
[[package]]
name = "build_const"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39092a32794787acd8525ee150305ff051b0aa6cc2abaf193924f5ab05425f39"
[[package]]
name = "bytes"
version = "0.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38"
[[package]]
name = "bytes"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e0dcbc35f504eb6fc275a6d20e4ebcda18cf50d40ba6fabff8c711fa16cb3b16"
[[package]]
name = "cc"
version = "1.0.66"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c0496836a84f8d0495758516b8621a622beb77c0fed418570e50764093ced48"
[[package]]
name = "cfg-if"
version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"
[[package]]
name = "cfg-if"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "cpuid-bool"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8aebca1129a03dc6dc2b127edd729435bbc4a37e1d5f4d7513165089ceb02634"
[[package]]
name = "crc"
version = "1.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d663548de7f5cca343f1e0a48d14dcfb0e9eb4e079ec58883b7251539fa10aeb"
dependencies = [
"build_const",
]
[[package]]
name = "crypto-mac"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "58bcd97a54c7ca5ce2f6eb16f6bede5b0ab5f0055fedc17d2f0b4466e21671ca"
dependencies = [
"generic-array",
"subtle",
]
[[package]]
name = "cryptohelpers"
version = "1.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1758ba574c79ae6db3ccf6623cacc5293c2c0a14de871a7b95d4286861cbd504"
dependencies = [
"crc",
"futures",
"getrandom",
"hex-literal",
"hmac",
"libc",
"openssl",
"pbkdf2",
"serde",
"serde_derive",
"sha2",
"tokio",
]
[[package]]
name = "digest"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066"
dependencies = [
"generic-array",
]
[[package]]
name = "foreign-types"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
dependencies = [
"foreign-types-shared",
]
[[package]]
name = "foreign-types-shared"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
[[package]]
name = "fs_extra"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2022715d62ab30faffd124d40b76f4134a550a87792276512b18d63272333394"
[[package]]
name = "futures"
version = "0.3.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b3b0c040a1fe6529d30b3c5944b280c7f0dcb2930d2c3062bca967b602583d0"
dependencies = [
"futures-channel",
"futures-core",
"futures-executor",
"futures-io",
"futures-sink",
"futures-task",
"futures-util",
]
[[package]]
name = "futures-channel"
version = "0.3.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4b7109687aa4e177ef6fe84553af6280ef2778bdb7783ba44c9dc3399110fe64"
dependencies = [
"futures-core",
"futures-sink",
]
[[package]]
name = "futures-core"
version = "0.3.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "847ce131b72ffb13b6109a221da9ad97a64cbe48feb1028356b836b47b8f1748"
[[package]]
name = "futures-executor"
version = "0.3.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4caa2b2b68b880003057c1dd49f1ed937e38f22fcf6c212188a121f08cf40a65"
dependencies = [
"futures-core",
"futures-task",
"futures-util",
]
[[package]]
name = "futures-io"
version = "0.3.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "611834ce18aaa1bd13c4b374f5d653e1027cf99b6b502584ff8c9a64413b30bb"
[[package]]
name = "futures-macro"
version = "0.3.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77408a692f1f97bcc61dc001d752e00643408fbc922e4d634c655df50d595556"
dependencies = [
"proc-macro-hack",
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "futures-sink"
version = "0.3.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f878195a49cee50e006b02b93cf7e0a95a38ac7b776b4c4d9cc1207cd20fcb3d"
[[package]]
name = "futures-task"
version = "0.3.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c554eb5bf48b2426c4771ab68c6b14468b6e76cc90996f528c3338d761a4d0d"
dependencies = [
"once_cell",
]
[[package]]
name = "futures-util"
version = "0.3.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d304cff4a7b99cfb7986f7d43fbe93d175e72e704a8860787cc95e9ffd85cbd2"
dependencies = [
"futures-channel",
"futures-core",
"futures-io",
"futures-macro",
"futures-sink",
"futures-task",
"memchr",
"pin-project",
"pin-utils",
"proc-macro-hack",
"proc-macro-nested",
"slab",
]
[[package]]
name = "generational-arena"
version = "0.2.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e1d3b771574f62d0548cee0ad9057857e9fc25d7a3335f140c84f6acd0bf601"
dependencies = [
"cfg-if 0.1.10",
]
[[package]]
name = "generic-array"
version = "0.14.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "501466ecc8a30d1d3b7fc9229b122b2ce8ed6e9d9223f1138d4babb253e51817"
dependencies = [
"typenum",
"version_check",
]
[[package]]
name = "getrandom"
version = "0.1.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc587bc0ec293155d5bfa6b9891ec18a1e330c234f896ea47fbada4cadbe47e6"
dependencies = [
"cfg-if 0.1.10",
"libc",
"wasi",
]
[[package]]
name = "hex-literal"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5af1f635ef1bc545d78392b136bfe1c9809e029023c84a3638a864a10b8819c8"
[[package]]
name = "hmac"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "deae6d9dbb35ec2c502d62b8f7b1c000a0822c3b0794ba36b3149c0a1c840dff"
dependencies = [
"crypto-mac",
"digest",
]
[[package]]
name = "jemalloc-sys"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0d3b9f3f5c9b31aa0f5ed3260385ac205db665baa41d49bb8338008ae94ede45"
dependencies = [
"cc",
"fs_extra",
"libc",
]
[[package]]
name = "jemallocator"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43ae63fcfc45e99ab3d1b29a46782ad679e98436c3169d15a167a1108a724b69"
dependencies = [
"jemalloc-sys",
"libc",
]
[[package]]
name = "lazy_static"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]]
name = "libc"
version = "0.2.81"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1482821306169ec4d07f6aca392a4681f66c75c9918aa49641a2595db64053cb"
[[package]]
name = "memchr"
version = "2.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525"
[[package]]
name = "memmap"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b"
dependencies = [
"libc",
"winapi",
]
[[package]]
name = "mtfse"
version = "0.1.0"
dependencies = [
"bytes 0.6.0",
"cryptohelpers",
"generational-arena",
"jemallocator",
"memmap",
"serde",
]
[[package]]
name = "once_cell"
version = "1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13bd41f508810a131401606d54ac32a467c97172d74ba7662562ebba5ad07fa0"
[[package]]
name = "opaque-debug"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
[[package]]
name = "openssl"
version = "0.10.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8d008f51b1acffa0d3450a68606e6a51c123012edaacb0f4e1426bd978869187"
dependencies = [
"bitflags",
"cfg-if 1.0.0",
"foreign-types",
"lazy_static",
"libc",
"openssl-sys",
]
[[package]]
name = "openssl-sys"
version = "0.9.59"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "de52d8eabd217311538a39bba130d7dea1f1e118010fee7a033d966845e7d5fe"
dependencies = [
"autocfg",
"cc",
"libc",
"pkg-config",
"vcpkg",
]
[[package]]
name = "pbkdf2"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7170d73bf11f39b4ce1809aabc95bf5c33564cdc16fc3200ddda17a5f6e5e48b"
dependencies = [
"base64",
"crypto-mac",
"hmac",
"rand",
"rand_core",
"sha2",
"subtle",
]
[[package]]
name = "pin-project"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ccc2237c2c489783abd8c4c80e5450fc0e98644555b1364da68cc29aa151ca7"
dependencies = [
"pin-project-internal",
]
[[package]]
name = "pin-project-internal"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8e8d2bf0b23038a4424865103a4df472855692821aab4e4f5c3312d461d9e5f"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "pin-project-lite"
version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c917123afa01924fc84bb20c4c03f004d9c38e5127e3c039bbf7f4b9c76a2f6b"
[[package]]
name = "pin-utils"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
[[package]]
name = "pkg-config"
version = "0.3.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c"
[[package]]
name = "ppv-lite86"
version = "0.2.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857"
[[package]]
name = "proc-macro-hack"
version = "0.5.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5"
[[package]]
name = "proc-macro-nested"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eba180dafb9038b050a4c280019bbedf9f2467b61e5d892dcad585bb57aadc5a"
[[package]]
name = "proc-macro2"
version = "1.0.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71"
dependencies = [
"unicode-xid",
]
[[package]]
name = "quote"
version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37"
dependencies = [
"proc-macro2",
]
[[package]]
name = "rand"
version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03"
dependencies = [
"rand_chacha",
"rand_core",
"rand_hc",
]
[[package]]
name = "rand_chacha"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402"
dependencies = [
"ppv-lite86",
"rand_core",
]
[[package]]
name = "rand_core"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19"
dependencies = [
"getrandom",
]
[[package]]
name = "rand_hc"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c"
dependencies = [
"rand_core",
]
[[package]]
name = "serde"
version = "1.0.118"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "06c64263859d87aa2eb554587e2d23183398d617427327cf2b3d0ed8c69e4800"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.118"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c84d3526699cd55261af4b941e4e725444df67aa4f9e6a3564f18030d12672df"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "sha2"
version = "0.9.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e7aab86fe2149bad8c507606bdb3f4ef5e7b2380eb92350f56122cca72a42a8"
dependencies = [
"block-buffer",
"cfg-if 1.0.0",
"cpuid-bool",
"digest",
"opaque-debug",
]
[[package]]
name = "slab"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8"
[[package]]
name = "subtle"
version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e81da0851ada1f3e9d4312c704aa4f8806f0f9d69faaf8df2f3464b4a9437c2"
[[package]]
name = "syn"
version = "1.0.54"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a2af957a63d6bd42255c359c93d9bfdb97076bd3b820897ce55ffbfbf107f44"
dependencies = [
"proc-macro2",
"quote",
"unicode-xid",
]
[[package]]
name = "tokio"
version = "0.2.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "099837d3464c16a808060bb3f02263b412f6fafcb5d01c533d309985fbeebe48"
dependencies = [
"bytes 0.5.6",
"memchr",
"pin-project-lite",
]
[[package]]
name = "typenum"
version = "1.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "373c8a200f9e67a0c95e62a4f52fbf80c23b4381c05a17845531982fa99e6b33"
[[package]]
name = "unicode-xid"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564"
[[package]]
name = "vcpkg"
version = "0.2.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b00bca6106a5e23f3eee943593759b7fcddb00554332e856d990c893966879fb"
[[package]]
name = "version_check"
version = "0.9.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed"
[[package]]
name = "wasi"
version = "0.9.0+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519"
[[package]]
name = "winapi"
version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
dependencies = [
"winapi-i686-pc-windows-gnu",
"winapi-x86_64-pc-windows-gnu",
]
[[package]]
name = "winapi-i686-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
[[package]]
name = "winapi-x86_64-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"

@ -0,0 +1,16 @@
[package]
name = "mtfse"
description = "Minimal Tagged File SErver"
version = "0.1.0"
authors = ["Avril <flanchan@cumallover.me>"]
edition = "2018"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
bytes = "0.6.0"
cryptohelpers = {version = "1.7", features = ["full", "async", "serde"]}
generational-arena = "0.2.8"
jemallocator = "0.3.2"
memmap = "0.7.0"
serde = {version = "1.0.118", features= ["derive"]}

@ -0,0 +1,42 @@
use super::*;
/// An open fd that has been memory mapped.
#[derive(Debug)]
pub struct OpenMMap
{
file: File,//note: this layout matters for destruction ordering.
map: Mmap,
}
/// Provides immutable caching of a file in a data entry.
#[derive(Debug)]
pub enum DataCacheState
{
/// There is no file cache for this item.
None,
/// The file is open, we have an fd.
Open(File),
/// The file is not open, but its whole contents have been loaded into memory.
Memory(Bytes), // load from file via `BytesMut` buffer, then call `.freeze()`
/// The file is open and memory mapped.
Mapped(OpenMMap),
}
impl Default for DataCacheState
{
#[inline]
fn default() -> Self
{
Self::None
}
}
impl DataCacheState
{
/// Drop the whole cache (if there is one).
#[inline] pub fn clear(&mut self)
{
*self = Self::None;
}
}

@ -0,0 +1,102 @@
use super::*;
use std::hash::{Hash, Hasher};
use std::borrow::Borrow;
#[derive(Debug, Serialize, Deserialize)] // Clone, PartialEq, Eq, Hash
pub struct Entry
{
name: String,
description: String,
/// The key used to encrypt the file, if any.
key: Option<aes::AesKey>,
/// The hash of the *unencrypted* data.
hash: sha256::Sha256Hash,
/// The tags for this entry.
///
/// # Note
/// This should be updated the same as the `Store`'s `tags` and `tag_mappings`. The duplication here is for lookup convenience / caching.
pub(super) tags: Vec<String>,
/// The *original* filename of the item, *not* the path to the real file for this item (see `location`).
filename: OsString,
/// Filename *relative* to the root of the store
location: PathBuf,
/// The state of caching this entry is currently in. This should not be saved as it has no meaning regarding the actual data itself.
#[serde(skip)]
cache: DataCacheState,
}
impl Entry
{
/// The sha256 hash of the data in this entry
pub fn hash(&self) -> &sha256::Sha256Hash
{
&self.hash
}
}
impl Entry
{
/// Consume and drop the cache. Only that useful as shim for mapping.
#[inline(always)] pub(super) fn with_no_cache(self) -> Self
{
Self {
cache: Default::default(),
..self
}
}
}
impl Borrow<sha256::Sha256Hash> for Entry
{
#[inline] fn borrow(&self) -> &sha256::Sha256Hash
{
&self.hash
}
}
impl Hash for Entry {
fn hash<H: Hasher>(&self, state: &mut H) {
self.hash.hash(state)
}
}
impl PartialEq for Entry
{
fn eq(&self, other: &Self) -> bool
{
macro_rules! eq {
($one:ident) => (self.$one == other.$one);
($($many:ident),*) => {
$(
eq!($many)
)&& *
};
}
eq!(name, description, key, tags, filename, location)
}
}
impl Eq for Entry{}
impl Clone for Entry
{
fn clone(&self) -> Self {
macro_rules! clone {
($($many:ident),*) => {
Self {
cache: Default::default(),
$($many: self.$many.clone()),*
}
}
}
clone!(name, description, key, tags, filename, location, hash)
}
}

@ -0,0 +1,98 @@
use super::*;
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
pub struct Freeze
{
metadata: StoreMetadata,
data: Vec<Entry>,
}
impl Freeze
{
pub(super) fn new_ref(from: &Store) -> Self
{
Self {
metadata: from.metadata.clone(),
data: from.data.iter().map(Entry::clone).collect(),
}
}
pub(super) fn new_moved(from: Store) -> Self
{
Self {
metadata: from.metadata,
data: from.data.into_iter().map(Entry::with_no_cache).collect(),
}
}
pub(super) fn create_new(&self) -> Store
{
let mut new = Store::with_capacity(self.metadata.clone(), self.data.len());
for entry in self.data.iter()
{
let hash = entry.hash().clone();
let hash_idx = new.data_hashes.insert(hash);
// insert the tags
for tag in entry.tags.iter()
{
if let Some(&ti) = new.tags.get(tag) {
// This tag has an entry already, append to it
new.tag_mappings.get_mut(ti).unwrap().push(hash_idx);
} else {
// This tag has no entry, create it
let ti = new.tag_mappings.insert(vec![hash_idx]);
new.tags.insert(tag.clone(), ti);
}
}
new.data.insert(entry.clone());
}
new
}
pub(super) fn into_new(self) -> Store
{
let Freeze { data, metadata } = self;
let mut new = Store::with_capacity(metadata, data.len());
for entry in data.into_iter()
{
let hash = entry.hash().clone();
let hash_idx = new.data_hashes.insert(hash);
// insert the tags
for tag in entry.tags.iter()
{
if let Some(&ti) = new.tags.get(tag) {
// This tag has an entry already, append to it
new.tag_mappings.get_mut(ti).unwrap().push(hash_idx);
} else {
// This tag has no entry, create it
let ti = new.tag_mappings.insert(vec![hash_idx]);
new.tags.insert(tag.clone(), ti);
}
}
new.data.insert(entry);
}
new
}
}
impl From<Store> for Freeze
{
#[inline] fn from(from: Store) -> Self
{
Self::new_moved(from)
}
}
impl From<Freeze> for Store
{
#[inline] fn from(from: Freeze) -> Self
{
from.into_new()
}
}

@ -0,0 +1,15 @@
use super::*;
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub struct StoreMetadata
{
pub name: String,
pub root: PathBuf,
}
impl StoreMetadata
{
}

@ -0,0 +1,118 @@
use super::*;
use generational_arena::{
Arena, Index as ArenaIndex,
};
use std::collections::BTreeMap;
use std::collections::HashSet;
use std::path::{Path, PathBuf};
use std::fs::File;
use std::ffi::OsString;
use memmap::Mmap;
use bytes::Bytes;
use cryptohelpers::{aes, sha256};
mod entry;
pub use entry::Entry;
mod cache;
pub use cache::DataCacheState;
mod metadata;
pub use metadata::StoreMetadata;
mod freeze;
pub use freeze::Freeze;
#[derive(Debug)]
pub struct Store
{
metadata: StoreMetadata,
data: HashSet<Entry>, // The entry sha256 hash is used as the `key` here, as `Entry` both hasshes to, and `Borrow`s to `Sha256Hash`.
data_hashes: Arena<sha256::Sha256Hash>, // used to lookup in `data`.
tag_mappings: Arena<Vec<ArenaIndex>>,
tags: BTreeMap<String, ArenaIndex>, // string (tags) -> index (tag_mappings) -> index (data_hashes) -> hash used for lookup (data)
}
// Comptime asserts (TODO: Unneeded. Remove.)
impl Store
{
fn _assert_tag_lookup_works(&self, tag: &str) -> &Entry
{
self.data.get(self.data_hashes.get(*self.tag_mappings.get(*self.tags.get(tag).unwrap()).unwrap().first().unwrap()).unwrap()).unwrap() // y-yes this also works...
}
fn _assert_lookup_internal_works(&self, data_idx: ArenaIndex) -> &Entry
{
self.data.get(self.data_hashes.get(data_idx).unwrap()).unwrap() // yes this works..
}
}
// Creating
impl Store
{
/// Create a new empty store with this metadata.
///
/// # Panics
/// If the root directory specified in `metadata` does not exist or is not a directory.
pub fn new(metadata: StoreMetadata) -> Self
{
assert!(metadata.root.exists() && metadata.root.is_dir(), "Metadata root {:?} passed to `new` not existant or not a directory", metadata.root);
Self {
metadata,
data: HashSet::new(),
data_hashes: Arena::new(),
tag_mappings: Arena::new(),
tags: BTreeMap::new(),
}
}
/// Create a new empty store with this metadata and initial storage capacity
///
/// # Panics
/// If the root directory specified in `metadata` does not exist or is not a directory.
pub fn with_capacity(metadata: StoreMetadata, cap: usize) -> Self
{
assert!(metadata.root.exists() && metadata.root.is_dir(), "Metadata root {:?} passed to `with_capacity` not existant or not a directory", metadata.root);
Self {
metadata,
data: HashSet::with_capacity(cap),
data_hashes: Arena::with_capacity(cap),
tag_mappings: Arena::with_capacity(cap),
tags: BTreeMap::new(),
}
}
}
// Freezing
impl Store
{
/// Create a snapshot of this store, cloning all data into a frozen and serialisable version of it.
/// # Notes
/// This method clones the entire store into the new `Freeze`. To avoid this, use `into_freeze` if the store is no longer used after the freeze.
#[inline] pub fn freeze(&self) -> Freeze
{
Freeze::new_ref(self)
}
/// Consume into a snapshot of this store, moving all data into a frozen and serializable version of it.
#[inline] pub fn into_freeze(self) -> Freeze
{
Freeze::new_moved(self)
}
/// Create a new store instance by cloning from a frozen snapshot of it.
/// # Notes
/// This method clones the entire `Freeze` into the new store. To avoid this, use `from_freeze` if the snapshot is no longer used after the unfreeze.
#[inline] pub fn unfreeze(freeze: &Freeze) -> Self
{
freeze.create_new()
}
/// Consume a store snapshot and move its entries into a new store.
#[inline] pub fn from_freeze(freeze: Freeze) -> Self
{
freeze.into_new()
}
}

@ -0,0 +1,15 @@
#![allow(dead_code)]
use serde::{Serialize, Deserialize};
use jemallocator::Jemalloc;
#[global_allocator]
static GLOBAL: Jemalloc = Jemalloc;
mod data;
fn main() {
println!("Hello, world!");
}
Loading…
Cancel
Save