From a515ac47ae7b7612a953803153ff483b08543c9c Mon Sep 17 00:00:00 2001 From: Avril Date: Tue, 18 Aug 2020 13:20:32 +0100 Subject: [PATCH] started database --- Cargo.lock | 47 +++++++++++++++ Cargo.toml | 1 + src/consts.rs | 4 ++ src/database.rs | 46 +++++++++++++++ src/hash.rs | 149 ++++++++++++++++++++++++++++++++++++++++++++++++ src/main.rs | 3 + src/util.rs | 39 +++++++++++++ 7 files changed, 289 insertions(+) create mode 100644 src/consts.rs create mode 100644 src/hash.rs create mode 100644 src/util.rs diff --git a/Cargo.lock b/Cargo.lock index 02aced3..0c20093 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6,6 +6,12 @@ version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d25d88fd6b8041580a654f9d0c581a047baee2b3efee13275f2fc392fc75034" +[[package]] +name = "autocfg" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d" + [[package]] name = "base64" version = "0.12.3" @@ -39,6 +45,17 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" +[[package]] +name = "chrono" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "942f72db697d8767c22d46a598e01f2d3b475501ea43d0db4f16d90259182d0b" +dependencies = [ + "num-integer", + "num-traits", + "time", +] + [[package]] name = "cpuid-bool" version = "0.1.2" @@ -222,6 +239,25 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "num-integer" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d59457e662d541ba17869cf51cf177c0b5f0cbf476c66bdc90bf1edac4f875b" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac267bcc07f48ee5f8935ab0d24f316fb722d7a1292e2913f0cc196b29ffd611" +dependencies = [ + "autocfg", +] + [[package]] name = "num_cpus" version = "1.13.0" @@ -320,6 +356,16 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "time" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438" +dependencies = [ + "libc", + "winapi 0.3.9", +] + [[package]] name = "tokio" version = "0.2.22" @@ -379,6 +425,7 @@ version = "0.1.0" dependencies = [ "base64", "cfg-if", + "chrono", "sha2", "tokio", ] diff --git a/Cargo.toml b/Cargo.toml index 9451ca9..b6c151f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,3 +21,4 @@ tokio = {version = "0.2", features=["full"]} base64 = {version = "0.12", optional = true} cfg-if = "0.1.10" sha2 = "0.9.1" +chrono = "0.4.13" diff --git a/src/consts.rs b/src/consts.rs new file mode 100644 index 0000000..818270f --- /dev/null +++ b/src/consts.rs @@ -0,0 +1,4 @@ +//! Consts + +/// Size for most buffers +pub const BUFFER_SIZE: usize = 4096; diff --git a/src/database.rs b/src/database.rs index c19fd65..ab0843a 100644 --- a/src/database.rs +++ b/src/database.rs @@ -1 +1,47 @@ //! Handles building the database, and exposes data structures for the database and ways to interact with them +use super::*; +use std::{ + collections::{ + HashMap, + HashSet, + }, + path::{ + PathBuf, + Path, + }, +}; +use chrono::{ + DateTime, + Utc, +}; + +#[derive(Debug, PartialEq, Eq)] +pub struct Database +{ + /// Key here is the actual location in /tmp/videl of the database and versioned files + database: HashMap, +} + +#[derive(Debug, PartialEq, Eq)] +pub struct Entry +{ + /// Versions are a hashset because we don't need to version files that are /entirely/ identical. (i.e. same timestamp and hash) + versions: HashSet, +} + +#[derive(Debug, PartialEq, Eq, Hash)] +pub struct Version +{ + date: DateTime, + hash: hash::Sha256Sum, + size: usize, +} + +impl Version +{ + /// Gets the lowermost name of the path for this specific version + pub fn get_pathname(&self) -> String + { + format!("{}", self.date.timestamp_nanos()) + } +} diff --git a/src/hash.rs b/src/hash.rs new file mode 100644 index 0000000..dfa3db0 --- /dev/null +++ b/src/hash.rs @@ -0,0 +1,149 @@ +//! Hashing abstraction +use super::*; +use sha2::{ + Sha256, Digest, +}; +use tokio::{ + prelude::*, + io::{ + AsyncRead, + }, +}; +use std::{ + io, + marker::Unpin, +}; + +pub const SHA256_SIZE: usize = 32; + +#[derive(Debug, Clone, PartialEq, Eq, Hash, Copy, Ord, PartialOrd)] +pub struct Sha256Sum([u8; SHA256_SIZE]); + +impl Default for Sha256Sum +{ + #[inline] + fn default() -> Self + { + Self([0; SHA256_SIZE]) + } +} + +impl Sha256Sum +{ + /// Create a new empty hash sum container + pub const fn empty() -> Self + { + Self([0; SHA256_SIZE]) + } + + pub const fn from_raw(from: [u8; SHA256_SIZE]) -> Self + { + Self(from) + } + + pub fn from_slice(from: &[u8]) -> Self + { + let mut output = [0u8; SHA256_SIZE]; + assert_eq!(util::copy_slice(&mut output, from), SHA256_SIZE); + Self(output) + } + + /// Compute SHA256 hash from a slice + pub fn compute_slice(from: T) -> Self + where T: AsRef<[u8]> + { + let mut hasher = Sha256::new(); + hasher.update(from.as_ref()); + + Self::from_slice(&hasher.finalize()[..]) + } + + /// Compute SHA256 from a stream + /// + /// # Notes + /// Will read until 0 bytes are returned from the `read()` call. + pub async fn compute_stream(from: &mut W) -> io::Result + where W: AsyncRead + Unpin + { + let mut read; + let mut buffer = [0u8; consts::BUFFER_SIZE]; + let mut hasher = Sha256::new(); + while {read = from.read(&mut buffer[..]).await?; read != 0} { + hasher.update(&buffer[..read]); + } + Ok(Self::from_slice(&hasher.finalize()[..])) + } + + /// Compute from a stream into this instance. Return the number of bytes successfully written into the instance. + pub async fn compute_into(&mut self, from: &mut W) -> io::Result + where W: AsyncRead + Unpin + { + let mut read; + let mut done=0; + let mut buffer = [0u8; consts::BUFFER_SIZE]; + let mut hasher = Sha256::new(); + while {read = from.read(&mut buffer[..]).await?; done += read; read !=0} { + hasher.update(&buffer[..read]); + } + util::copy_slice(self, hasher.finalize()); + Ok(done) + } +} + +impl AsRef<[u8]> for Sha256Sum +{ + #[inline] fn as_ref(&self) -> &[u8] + { + &self.0[..] + } +} + +impl AsMut<[u8]> for Sha256Sum +{ + #[inline] fn as_mut(&mut self) -> &mut [u8] + { + &mut self.0[..] + } +} + +impl From<[u8; SHA256_SIZE]> for Sha256Sum +{ + #[inline] fn from(from: [u8; SHA256_SIZE]) -> Self + { + Self(from) + } +} + +impl From for Sha256Sum +{ + fn from(from: Sha256) -> Self + { + Self::from_slice(&from.finalize()) + } +} + +/// Update a hasher from a stream async +pub async fn stream_async(hasher: &mut H, from: &mut W) -> io::Result +where H: Digest, + W: AsyncRead + Unpin +{ + let mut read; + let mut done=0; + let mut buffer = [0u8; consts::BUFFER_SIZE]; + while {read = from.read(&mut buffer[..]).await?; done += read; read !=0} { + hasher.update(&buffer[..read]); + } + Ok(done) +} + +impl std::fmt::Display for Sha256Sum +{ + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result + { + for byte in self.0.iter() + { + write!(f, "{:02x}", byte)?; + } + Ok(()) + } +} diff --git a/src/main.rs b/src/main.rs index 58f3dcb..daf5366 100644 --- a/src/main.rs +++ b/src/main.rs @@ -6,6 +6,9 @@ use cfg_if::cfg_if; mod ext; use ext::*; +mod consts; +mod util; +mod hash; mod metadata; mod resolve; mod database; diff --git a/src/util.rs b/src/util.rs new file mode 100644 index 0000000..3c697f0 --- /dev/null +++ b/src/util.rs @@ -0,0 +1,39 @@ +//! Utility functions +use std::{ + borrow::{ + Borrow, + ToOwned, + }, +}; + +/// Copy slice `src` into `dst` and return the number of elements copied. +#[inline] pub fn copy_slice(mut dst: V, src: W) -> usize +where V: AsMut<[T]>, + W: AsRef<[U]>, + U: ToOwned, + X: Borrow + Into +{ + + let mut i=0; + for (d, s) in dst.as_mut().iter_mut().zip(src.as_ref().iter()) + { + *d = s.to_owned().into(); + i+=1 + } + i +} + +#[cfg(test)] +mod tests +{ + #[test] + fn copy_slice() + { + let mut to = [0u8; 40]; + let from = [10u8; 37]; + + assert_eq!(super::copy_slice(&mut to, &from), 37); + + assert_eq!(from, &to[0..37]); + } +}