From 48277f62acecca1ee76061492a5b4a34480014aa Mon Sep 17 00:00:00 2001 From: Avril Date: Fri, 18 Dec 2020 16:02:22 +0000 Subject: [PATCH] data structures and layout --- .gitignore | 2 + Cargo.lock | 627 +++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 16 ++ src/data/cache.rs | 42 +++ src/data/entry.rs | 102 +++++++ src/data/freeze.rs | 98 +++++++ src/data/metadata.rs | 15 ++ src/data/mod.rs | 118 ++++++++ src/main.rs | 15 ++ 9 files changed, 1035 insertions(+) create mode 100644 .gitignore create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100644 src/data/cache.rs create mode 100644 src/data/entry.rs create mode 100644 src/data/freeze.rs create mode 100644 src/data/metadata.rs create mode 100644 src/data/mod.rs create mode 100644 src/main.rs diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e2a3069 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +/target +*~ diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..41d76f7 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,627 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "autocfg" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" + +[[package]] +name = "base64" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff" + +[[package]] +name = "bitflags" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" + +[[package]] +name = "block-buffer" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" +dependencies = [ + "generic-array", +] + +[[package]] +name = "build_const" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39092a32794787acd8525ee150305ff051b0aa6cc2abaf193924f5ab05425f39" + +[[package]] +name = "bytes" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" + +[[package]] +name = "bytes" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0dcbc35f504eb6fc275a6d20e4ebcda18cf50d40ba6fabff8c711fa16cb3b16" + +[[package]] +name = "cc" +version = "1.0.66" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c0496836a84f8d0495758516b8621a622beb77c0fed418570e50764093ced48" + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "cpuid-bool" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8aebca1129a03dc6dc2b127edd729435bbc4a37e1d5f4d7513165089ceb02634" + +[[package]] +name = "crc" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d663548de7f5cca343f1e0a48d14dcfb0e9eb4e079ec58883b7251539fa10aeb" +dependencies = [ + "build_const", +] + +[[package]] +name = "crypto-mac" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58bcd97a54c7ca5ce2f6eb16f6bede5b0ab5f0055fedc17d2f0b4466e21671ca" +dependencies = [ + "generic-array", + "subtle", +] + +[[package]] +name = "cryptohelpers" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1758ba574c79ae6db3ccf6623cacc5293c2c0a14de871a7b95d4286861cbd504" +dependencies = [ + "crc", + "futures", + "getrandom", + "hex-literal", + "hmac", + "libc", + "openssl", + "pbkdf2", + "serde", + "serde_derive", + "sha2", + "tokio", +] + +[[package]] +name = "digest" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +dependencies = [ + "generic-array", +] + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "fs_extra" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2022715d62ab30faffd124d40b76f4134a550a87792276512b18d63272333394" + +[[package]] +name = "futures" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b3b0c040a1fe6529d30b3c5944b280c7f0dcb2930d2c3062bca967b602583d0" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b7109687aa4e177ef6fe84553af6280ef2778bdb7783ba44c9dc3399110fe64" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "847ce131b72ffb13b6109a221da9ad97a64cbe48feb1028356b836b47b8f1748" + +[[package]] +name = "futures-executor" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4caa2b2b68b880003057c1dd49f1ed937e38f22fcf6c212188a121f08cf40a65" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "611834ce18aaa1bd13c4b374f5d653e1027cf99b6b502584ff8c9a64413b30bb" + +[[package]] +name = "futures-macro" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77408a692f1f97bcc61dc001d752e00643408fbc922e4d634c655df50d595556" +dependencies = [ + "proc-macro-hack", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f878195a49cee50e006b02b93cf7e0a95a38ac7b776b4c4d9cc1207cd20fcb3d" + +[[package]] +name = "futures-task" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c554eb5bf48b2426c4771ab68c6b14468b6e76cc90996f528c3338d761a4d0d" +dependencies = [ + "once_cell", +] + +[[package]] +name = "futures-util" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d304cff4a7b99cfb7986f7d43fbe93d175e72e704a8860787cc95e9ffd85cbd2" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project", + "pin-utils", + "proc-macro-hack", + "proc-macro-nested", + "slab", +] + +[[package]] +name = "generational-arena" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d3b771574f62d0548cee0ad9057857e9fc25d7a3335f140c84f6acd0bf601" +dependencies = [ + "cfg-if 0.1.10", +] + +[[package]] +name = "generic-array" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "501466ecc8a30d1d3b7fc9229b122b2ce8ed6e9d9223f1138d4babb253e51817" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc587bc0ec293155d5bfa6b9891ec18a1e330c234f896ea47fbada4cadbe47e6" +dependencies = [ + "cfg-if 0.1.10", + "libc", + "wasi", +] + +[[package]] +name = "hex-literal" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5af1f635ef1bc545d78392b136bfe1c9809e029023c84a3638a864a10b8819c8" + +[[package]] +name = "hmac" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "deae6d9dbb35ec2c502d62b8f7b1c000a0822c3b0794ba36b3149c0a1c840dff" +dependencies = [ + "crypto-mac", + "digest", +] + +[[package]] +name = "jemalloc-sys" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d3b9f3f5c9b31aa0f5ed3260385ac205db665baa41d49bb8338008ae94ede45" +dependencies = [ + "cc", + "fs_extra", + "libc", +] + +[[package]] +name = "jemallocator" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43ae63fcfc45e99ab3d1b29a46782ad679e98436c3169d15a167a1108a724b69" +dependencies = [ + "jemalloc-sys", + "libc", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1482821306169ec4d07f6aca392a4681f66c75c9918aa49641a2595db64053cb" + +[[package]] +name = "memchr" +version = "2.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" + +[[package]] +name = "memmap" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "mtfse" +version = "0.1.0" +dependencies = [ + "bytes 0.6.0", + "cryptohelpers", + "generational-arena", + "jemallocator", + "memmap", + "serde", +] + +[[package]] +name = "once_cell" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13bd41f508810a131401606d54ac32a467c97172d74ba7662562ebba5ad07fa0" + +[[package]] +name = "opaque-debug" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" + +[[package]] +name = "openssl" +version = "0.10.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d008f51b1acffa0d3450a68606e6a51c123012edaacb0f4e1426bd978869187" +dependencies = [ + "bitflags", + "cfg-if 1.0.0", + "foreign-types", + "lazy_static", + "libc", + "openssl-sys", +] + +[[package]] +name = "openssl-sys" +version = "0.9.59" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de52d8eabd217311538a39bba130d7dea1f1e118010fee7a033d966845e7d5fe" +dependencies = [ + "autocfg", + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "pbkdf2" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7170d73bf11f39b4ce1809aabc95bf5c33564cdc16fc3200ddda17a5f6e5e48b" +dependencies = [ + "base64", + "crypto-mac", + "hmac", + "rand", + "rand_core", + "sha2", + "subtle", +] + +[[package]] +name = "pin-project" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ccc2237c2c489783abd8c4c80e5450fc0e98644555b1364da68cc29aa151ca7" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8e8d2bf0b23038a4424865103a4df472855692821aab4e4f5c3312d461d9e5f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c917123afa01924fc84bb20c4c03f004d9c38e5127e3c039bbf7f4b9c76a2f6b" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" + +[[package]] +name = "ppv-lite86" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" + +[[package]] +name = "proc-macro-hack" +version = "0.5.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" + +[[package]] +name = "proc-macro-nested" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eba180dafb9038b050a4c280019bbedf9f2467b61e5d892dcad585bb57aadc5a" + +[[package]] +name = "proc-macro2" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "quote" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "rand_chacha", + "rand_core", + "rand_hc", +] + +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core", +] + +[[package]] +name = "serde" +version = "1.0.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06c64263859d87aa2eb554587e2d23183398d617427327cf2b3d0ed8c69e4800" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c84d3526699cd55261af4b941e4e725444df67aa4f9e6a3564f18030d12672df" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "sha2" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e7aab86fe2149bad8c507606bdb3f4ef5e7b2380eb92350f56122cca72a42a8" +dependencies = [ + "block-buffer", + "cfg-if 1.0.0", + "cpuid-bool", + "digest", + "opaque-debug", +] + +[[package]] +name = "slab" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" + +[[package]] +name = "subtle" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e81da0851ada1f3e9d4312c704aa4f8806f0f9d69faaf8df2f3464b4a9437c2" + +[[package]] +name = "syn" +version = "1.0.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2af957a63d6bd42255c359c93d9bfdb97076bd3b820897ce55ffbfbf107f44" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "tokio" +version = "0.2.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "099837d3464c16a808060bb3f02263b412f6fafcb5d01c533d309985fbeebe48" +dependencies = [ + "bytes 0.5.6", + "memchr", + "pin-project-lite", +] + +[[package]] +name = "typenum" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "373c8a200f9e67a0c95e62a4f52fbf80c23b4381c05a17845531982fa99e6b33" + +[[package]] +name = "unicode-xid" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" + +[[package]] +name = "vcpkg" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b00bca6106a5e23f3eee943593759b7fcddb00554332e856d990c893966879fb" + +[[package]] +name = "version_check" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed" + +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..6bf16fb --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "mtfse" +description = "Minimal Tagged File SErver" +version = "0.1.0" +authors = ["Avril "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +bytes = "0.6.0" +cryptohelpers = {version = "1.7", features = ["full", "async", "serde"]} +generational-arena = "0.2.8" +jemallocator = "0.3.2" +memmap = "0.7.0" +serde = {version = "1.0.118", features= ["derive"]} diff --git a/src/data/cache.rs b/src/data/cache.rs new file mode 100644 index 0000000..4b974dd --- /dev/null +++ b/src/data/cache.rs @@ -0,0 +1,42 @@ +use super::*; + + +/// An open fd that has been memory mapped. +#[derive(Debug)] +pub struct OpenMMap +{ + file: File,//note: this layout matters for destruction ordering. + map: Mmap, +} + +/// Provides immutable caching of a file in a data entry. +#[derive(Debug)] +pub enum DataCacheState +{ + /// There is no file cache for this item. + None, + /// The file is open, we have an fd. + Open(File), + /// The file is not open, but its whole contents have been loaded into memory. + Memory(Bytes), // load from file via `BytesMut` buffer, then call `.freeze()` + /// The file is open and memory mapped. + Mapped(OpenMMap), +} + +impl Default for DataCacheState +{ + #[inline] + fn default() -> Self + { + Self::None + } +} + +impl DataCacheState +{ + /// Drop the whole cache (if there is one). + #[inline] pub fn clear(&mut self) + { + *self = Self::None; + } +} diff --git a/src/data/entry.rs b/src/data/entry.rs new file mode 100644 index 0000000..21a1e16 --- /dev/null +++ b/src/data/entry.rs @@ -0,0 +1,102 @@ +use super::*; +use std::hash::{Hash, Hasher}; +use std::borrow::Borrow; + + +#[derive(Debug, Serialize, Deserialize)] // Clone, PartialEq, Eq, Hash +pub struct Entry +{ + name: String, + description: String, + + /// The key used to encrypt the file, if any. + key: Option, + + /// The hash of the *unencrypted* data. + hash: sha256::Sha256Hash, + + /// The tags for this entry. + /// + /// # Note + /// This should be updated the same as the `Store`'s `tags` and `tag_mappings`. The duplication here is for lookup convenience / caching. + pub(super) tags: Vec, + + /// The *original* filename of the item, *not* the path to the real file for this item (see `location`). + filename: OsString, + + /// Filename *relative* to the root of the store + location: PathBuf, + + /// The state of caching this entry is currently in. This should not be saved as it has no meaning regarding the actual data itself. + #[serde(skip)] + cache: DataCacheState, +} + +impl Entry +{ + /// The sha256 hash of the data in this entry + pub fn hash(&self) -> &sha256::Sha256Hash + { + &self.hash + } +} + +impl Entry +{ + /// Consume and drop the cache. Only that useful as shim for mapping. + #[inline(always)] pub(super) fn with_no_cache(self) -> Self + { + Self { + cache: Default::default(), + ..self + } + } +} + +impl Borrow for Entry +{ + #[inline] fn borrow(&self) -> &sha256::Sha256Hash + { + &self.hash + } +} + + +impl Hash for Entry { + fn hash(&self, state: &mut H) { + self.hash.hash(state) + } +} + +impl PartialEq for Entry +{ + fn eq(&self, other: &Self) -> bool + { + macro_rules! eq { + ($one:ident) => (self.$one == other.$one); + ($($many:ident),*) => { + $( + eq!($many) + )&& * + }; + } + + eq!(name, description, key, tags, filename, location) + } +} +impl Eq for Entry{} + +impl Clone for Entry +{ + fn clone(&self) -> Self { + macro_rules! clone { + ($($many:ident),*) => { + Self { + cache: Default::default(), + $($many: self.$many.clone()),* + } + } + } + clone!(name, description, key, tags, filename, location, hash) + } +} diff --git a/src/data/freeze.rs b/src/data/freeze.rs new file mode 100644 index 0000000..17df4de --- /dev/null +++ b/src/data/freeze.rs @@ -0,0 +1,98 @@ +use super::*; + + +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct Freeze +{ + metadata: StoreMetadata, + data: Vec, +} + +impl Freeze +{ + pub(super) fn new_ref(from: &Store) -> Self + { + Self { + metadata: from.metadata.clone(), + data: from.data.iter().map(Entry::clone).collect(), + } + } + pub(super) fn new_moved(from: Store) -> Self + { + Self { + metadata: from.metadata, + data: from.data.into_iter().map(Entry::with_no_cache).collect(), + } + } + + pub(super) fn create_new(&self) -> Store + { + let mut new = Store::with_capacity(self.metadata.clone(), self.data.len()); + + for entry in self.data.iter() + { + let hash = entry.hash().clone(); + let hash_idx = new.data_hashes.insert(hash); + + // insert the tags + for tag in entry.tags.iter() + { + if let Some(&ti) = new.tags.get(tag) { + // This tag has an entry already, append to it + new.tag_mappings.get_mut(ti).unwrap().push(hash_idx); + } else { + // This tag has no entry, create it + let ti = new.tag_mappings.insert(vec![hash_idx]); + new.tags.insert(tag.clone(), ti); + } + } + + new.data.insert(entry.clone()); + } + new + + } + pub(super) fn into_new(self) -> Store + { + let Freeze { data, metadata } = self; + + let mut new = Store::with_capacity(metadata, data.len()); + for entry in data.into_iter() + { + let hash = entry.hash().clone(); + let hash_idx = new.data_hashes.insert(hash); + + // insert the tags + for tag in entry.tags.iter() + { + if let Some(&ti) = new.tags.get(tag) { + // This tag has an entry already, append to it + new.tag_mappings.get_mut(ti).unwrap().push(hash_idx); + } else { + // This tag has no entry, create it + let ti = new.tag_mappings.insert(vec![hash_idx]); + new.tags.insert(tag.clone(), ti); + } + } + + new.data.insert(entry); + } + new + } +} + +impl From for Freeze +{ + #[inline] fn from(from: Store) -> Self + { + Self::new_moved(from) + } +} + +impl From for Store +{ + #[inline] fn from(from: Freeze) -> Self + { + from.into_new() + } +} diff --git a/src/data/metadata.rs b/src/data/metadata.rs new file mode 100644 index 0000000..13a08dc --- /dev/null +++ b/src/data/metadata.rs @@ -0,0 +1,15 @@ +use super::*; + + +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct StoreMetadata +{ + pub name: String, + + pub root: PathBuf, +} + +impl StoreMetadata +{ + +} diff --git a/src/data/mod.rs b/src/data/mod.rs new file mode 100644 index 0000000..e175973 --- /dev/null +++ b/src/data/mod.rs @@ -0,0 +1,118 @@ +use super::*; +use generational_arena::{ + Arena, Index as ArenaIndex, +}; +use std::collections::BTreeMap; +use std::collections::HashSet; + +use std::path::{Path, PathBuf}; +use std::fs::File; +use std::ffi::OsString; +use memmap::Mmap; +use bytes::Bytes; +use cryptohelpers::{aes, sha256}; + +mod entry; +pub use entry::Entry; + +mod cache; +pub use cache::DataCacheState; + +mod metadata; +pub use metadata::StoreMetadata; + +mod freeze; +pub use freeze::Freeze; + +#[derive(Debug)] +pub struct Store +{ + metadata: StoreMetadata, + + data: HashSet, // The entry sha256 hash is used as the `key` here, as `Entry` both hasshes to, and `Borrow`s to `Sha256Hash`. + data_hashes: Arena, // used to lookup in `data`. + + tag_mappings: Arena>, + tags: BTreeMap, // string (tags) -> index (tag_mappings) -> index (data_hashes) -> hash used for lookup (data) +} + +// Comptime asserts (TODO: Unneeded. Remove.) +impl Store +{ + fn _assert_tag_lookup_works(&self, tag: &str) -> &Entry + { + self.data.get(self.data_hashes.get(*self.tag_mappings.get(*self.tags.get(tag).unwrap()).unwrap().first().unwrap()).unwrap()).unwrap() // y-yes this also works... + } + fn _assert_lookup_internal_works(&self, data_idx: ArenaIndex) -> &Entry + { + self.data.get(self.data_hashes.get(data_idx).unwrap()).unwrap() // yes this works.. + } +} + +// Creating +impl Store +{ + /// Create a new empty store with this metadata. + /// + /// # Panics + /// If the root directory specified in `metadata` does not exist or is not a directory. + pub fn new(metadata: StoreMetadata) -> Self + { + assert!(metadata.root.exists() && metadata.root.is_dir(), "Metadata root {:?} passed to `new` not existant or not a directory", metadata.root); + Self { + metadata, + data: HashSet::new(), + data_hashes: Arena::new(), + + tag_mappings: Arena::new(), + tags: BTreeMap::new(), + } + } + /// Create a new empty store with this metadata and initial storage capacity + /// + /// # Panics + /// If the root directory specified in `metadata` does not exist or is not a directory. + pub fn with_capacity(metadata: StoreMetadata, cap: usize) -> Self + { + assert!(metadata.root.exists() && metadata.root.is_dir(), "Metadata root {:?} passed to `with_capacity` not existant or not a directory", metadata.root); + Self { + metadata, + + data: HashSet::with_capacity(cap), + data_hashes: Arena::with_capacity(cap), + + tag_mappings: Arena::with_capacity(cap), + tags: BTreeMap::new(), + } + } +} + +// Freezing +impl Store +{ + /// Create a snapshot of this store, cloning all data into a frozen and serialisable version of it. + /// # Notes + /// This method clones the entire store into the new `Freeze`. To avoid this, use `into_freeze` if the store is no longer used after the freeze. + #[inline] pub fn freeze(&self) -> Freeze + { + Freeze::new_ref(self) + } + /// Consume into a snapshot of this store, moving all data into a frozen and serializable version of it. + #[inline] pub fn into_freeze(self) -> Freeze + { + Freeze::new_moved(self) + } + + /// Create a new store instance by cloning from a frozen snapshot of it. + /// # Notes + /// This method clones the entire `Freeze` into the new store. To avoid this, use `from_freeze` if the snapshot is no longer used after the unfreeze. + #[inline] pub fn unfreeze(freeze: &Freeze) -> Self + { + freeze.create_new() + } + /// Consume a store snapshot and move its entries into a new store. + #[inline] pub fn from_freeze(freeze: Freeze) -> Self + { + freeze.into_new() + } +} diff --git a/src/main.rs b/src/main.rs new file mode 100644 index 0000000..f19570f --- /dev/null +++ b/src/main.rs @@ -0,0 +1,15 @@ + +#![allow(dead_code)] + +use serde::{Serialize, Deserialize}; + +use jemallocator::Jemalloc; + +#[global_allocator] +static GLOBAL: Jemalloc = Jemalloc; + +mod data; + +fn main() { + println!("Hello, world!"); +}