diff --git a/Cargo.lock b/Cargo.lock index df8e613..cdadb8d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -76,6 +76,16 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff" +[[package]] +name = "base65536" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04cb7e04ba4c08c961a722f1c13ebc20c5e3396423c7af00e1273584f45fa9b2" +dependencies = [ + "fnv", + "lazy_static", +] + [[package]] name = "bitflags" version = "1.2.1" @@ -258,12 +268,101 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" +[[package]] +name = "futures" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e05b85ec287aac0dc34db7d4a569323df697f9c55b99b15d6b4ef8cde49f613" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f366ad74c28cca6ba456d95e6422883cfb4b252a83bed929c83abfdbbf2967d5" +dependencies = [ + "futures-core", + "futures-sink", +] + [[package]] name = "futures-core" version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "59f5fff90fd5d971f936ad674802482ba441b6f09ba5e15fd8b39145582ca399" +[[package]] +name = "futures-executor" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10d6bb888be1153d3abeb9006b11b02cf5e9b209fda28693c31ae1e4e012e314" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de27142b013a8e869c14957e6d2edeef89e97c289e69d042ee3a49acd8b51789" + +[[package]] +name = "futures-macro" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0b5a30a4328ab5473878237c447333c093297bded83a4983d10f4deea240d39" +dependencies = [ + "proc-macro-hack", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f2032893cb734c7a05d85ce0cc8b8c4075278e93b24b66f9de99d6eb0fa8acc" + +[[package]] +name = "futures-task" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdb66b5f09e22019b1ab0830f7785bcea8e7a42148683f99214f73f8ec21a626" +dependencies = [ + "once_cell", +] + +[[package]] +name = "futures-util" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8764574ff08b701a084482c3c7031349104b07ac897393010494beaa18ce32c6" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project", + "pin-utils", + "proc-macro-hack", + "proc-macro-nested", + "slab", +] + [[package]] name = "generic-array" version = "0.14.4" @@ -291,6 +390,12 @@ version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aaf91faf136cb47367fa430cd46e37a788775e7fa104f8b4bcb3861dc389b724" +[[package]] +name = "half" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d36fab90f82edc3c747f9d438e06cf0a491055896f2a279638bb5beed6c40177" + [[package]] name = "hermit-abi" version = "0.1.15" @@ -589,6 +694,12 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282adbf10f2698a7a77f8e983a74b2d18176c19a7fd32a45446139ae7b02b715" +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + [[package]] name = "pkg-config" version = "0.3.18" @@ -601,6 +712,18 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c36fa947111f5c62a733b652544dd0016a43ce89619538a8ef92724a6f501a20" +[[package]] +name = "proc-macro-hack" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99c605b9a0adc77b7211c6b1f722dcb613d68d66859a44f3d485a6da332b0598" + +[[package]] +name = "proc-macro-nested" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eba180dafb9038b050a4c280019bbedf9f2467b61e5d892dcad585bb57aadc5a" + [[package]] name = "proc-macro2" version = "1.0.21" @@ -623,16 +746,21 @@ dependencies = [ name = "rae" version = "0.1.0" dependencies = [ + "base65536", "cfg-if", + "chrono", "color-eyre", "cryptohelpers", + "futures", "lazy_static", + "libc", + "pin-project", "recolored", "rustc_version", "serde", + "serde_cbor", "smallmap", "tokio", - "toml", "tracing", "tracing-error", "tracing-futures", @@ -765,6 +893,16 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde_cbor" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e18acfa2f90e8b735b2836ab8d538de304cbb6729a7360729ea5a895d15a622" +dependencies = [ + "half", + "serde", +] + [[package]] name = "serde_derive" version = "1.0.116" @@ -924,15 +1062,6 @@ dependencies = [ "syn", ] -[[package]] -name = "toml" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffc92d160b1eef40665be3a05630d003936a3bc7da7421277846c2613e92c71a" -dependencies = [ - "serde", -] - [[package]] name = "tracing" version = "0.1.19" diff --git a/Cargo.toml b/Cargo.toml index 9098377..0746da1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,20 +6,32 @@ edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[features] +default = ["short-ids"] + +short-ids = ["base65536"] +local-time = [] + [dependencies] color-eyre = "0.5.3" lazy_static = "1.4.0" -cryptohelpers = {version = "1.1.2", features=["full", "async", "serialise"]} +crypto = {package= "cryptohelpers", version = "1.1.2", features=["full", "async", "serialise"]} cfg-if = "0.1.10" tokio = {version = "0.2", features=["full"]} serde = {version ="1.0.116", features=["derive"]} -toml = "0.5.6" tracing = "0.1.19" tracing-subscriber = "0.2.12" tracing-futures = "0.2.4" tracing-error = "0.1.2" smallmap = "1.1.2" recolored = "1.9.3" +libc = "0.2.77" +base65536 = { version = "1.0.0", optional = true } +serde_cbor = "0.11.1" +futures = "0.3.5" +chrono = "0.4.15" +pin-project = "0.4.23" +#serde_json = "1.0.57" # serde not suitable for our text formatting :/ [build-dependencies] rustc_version = "0.2" diff --git a/src/bytes.rs b/src/bytes.rs new file mode 100644 index 0000000..06838ab --- /dev/null +++ b/src/bytes.rs @@ -0,0 +1,28 @@ +use libc::{ + c_void, +}; +/// Copy slice of bytes only +/// +/// # Notes +/// `dst` and `src` must not overlap. See [move_slice]. +pub fn copy_slice(dst: &mut [u8], src: &[u8]) -> usize +{ + let sz = std::cmp::min(dst.len(),src.len()); + unsafe { + libc::memcpy(&mut dst[0] as *mut u8 as *mut c_void, &src[0] as *const u8 as *const c_void, sz); + } + sz +} + +/// Move slice of bytes only +/// +/// # Notes +/// `dst` and `src` can overlap. +pub fn move_slice(dst: &mut [u8], src: &[u8]) -> usize +{ + let sz = std::cmp::min(dst.len(),src.len()); + unsafe { + libc::memmove(&mut dst[0] as *mut u8 as *mut c_void, &src[0] as *const u8 as *const c_void, sz); + } + sz +} diff --git a/src/config.rs b/src/config.rs index 8dcf77a..085fff9 100644 --- a/src/config.rs +++ b/src/config.rs @@ -2,7 +2,8 @@ use super::*; pub mod op { - + use super::*; + /// The crypt mode #[derive(Debug, PartialEq, Eq, Clone, Hash)] pub enum Mode { @@ -61,7 +62,7 @@ pub mod op { } } - #[derive(Debug ,PartialEq, Eq, Clone, Hash, Default)] + #[derive(Debug ,PartialEq, Eq, Clone, Hash, Default, Serialize, Deserialize)] pub struct KeyDescription { pub name: String, @@ -70,6 +71,7 @@ pub mod op { pub other: String, } + #[derive(Debug, Clone,PartialEq, Eq)] pub struct Rsa diff --git a/src/ext.rs b/src/ext.rs index bf5d776..887bc55 100644 --- a/src/ext.rs +++ b/src/ext.rs @@ -1,4 +1,14 @@ //! Extensions +use std::{ + fmt, + pin::Pin, + task::{Poll,Context,}, +}; +use tokio::{ + io::AsyncRead, + prelude::*, +}; +use futures::future::Future; pub trait JoinStrsExt: Sized { @@ -28,8 +38,8 @@ where I: Iterator, } /*macro_rules! typed_swap { - (@ [] $($reversed:tt)*) => { - fn swap(self) -> ($($reversed)*); +(@ [] $($reversed:tt)*) => { +fn swap(self) -> ($($reversed)*); }; (@ [$first:tt $($rest:tt)*] $($reversed:tt)*) => { typed_swap!{@ [$($rest)*] $first $($reversed)*} @@ -96,3 +106,184 @@ impl SwapTupleExt for (T,U) /*typed_swap!({A, B} {A, U, V} {T, U, V, W});*/ + + +const ASCII_MAP: [char; 256] = [ + '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', + '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', + '.', '!', '"', '#', '$', '%', '&', '\'', '(', ')', '*', '+', ',', '-', '.', '/', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', ':', ';', '<', '=', '>', '?', + '@', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', + 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', '[', '\\', ']', '^', '_', + '`', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', + 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '{', '|', '}', '~', '.', + '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', + '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', + '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', + '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', + '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', + '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', + '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', + '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', +]; + +pub struct HexStringIter<'a, I>(&'a I, bool); +pub struct HexView<'a, I>(&'a I); + +const SPLIT_EVERY: usize = 16; + +impl<'a, I: AsRef<[u8]>> fmt::Display for HexView<'a, I> +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result + { + use std::iter; + let mut abuf = ['\0'; SPLIT_EVERY]; + let mut last_n =0 ; + for (i, (n, &byte)) in (0..).zip(iter::repeat(0..SPLIT_EVERY).flatten().zip(self.0.as_ref().iter())) + { + if n== 0 { + write!(f,"0x{:016x}\t", i)?; + } + abuf[n] = ASCII_MAP[byte as usize]; + write!(f, "{:02x} ", byte)?; + if n==SPLIT_EVERY-1 { + write!(f, "\t\t")?; + for ch in abuf.iter().filter(|&x| *x!= '\0') + { + write!(f, "{}", ch)?; + } + writeln!(f)?; + abuf = ['\0'; SPLIT_EVERY]; + } + last_n = n; + } + if last_n != SPLIT_EVERY-1 + { + for _ in 0..(SPLIT_EVERY-last_n) + { + write!(f, " ")?; + } + write!(f, "\t\t")?; + for ch in abuf.iter().filter(|&x| *x!= '\0') + { + write!(f, "{}", ch)?; + } + writeln!(f)?; + } + Ok(()) + } +} + +impl<'a, I: AsRef<[u8]>> fmt::Display for HexStringIter<'a, I> +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result + { + if self.1 { + let mut iter = self.0.as_ref().iter(); + if let Some(byte) = iter.next() + { + write!(f, "{:02}", byte)?; + } else { + return Ok(()) + } + for byte in iter { + write!(f, " {:02x}", byte)?; + } + } else { + for byte in self.0.as_ref().iter() { + write!(f, "{:02x}", byte)?; + } + } + Ok(()) + } +} + +pub trait HexStringExt: Sized + AsRef<[u8]> +{ + fn fmt_view(&self) -> HexView<'_, Self>; + fn fmt_hex(&self) -> HexStringIter<'_, Self>; + fn to_hex_string(&self) -> String + { + let mut string = String::with_capacity(self.as_ref().len()*2); + use fmt::Write; + write!(&mut string, "{}", self.fmt_hex()).unwrap(); + string + } + fn to_broken_hex_string(&self) -> String + { + let fmt = HexStringIter( + self.fmt_hex().0, + true + ); + let mut string = String::with_capacity(self.as_ref().len()*3); + use fmt::Write; + write!(&mut string, "{}", fmt).unwrap(); + string + } +} + +impl> HexStringExt for T +{ + fn fmt_hex(&self) -> HexStringIter<'_, Self> + { + HexStringIter(&self, false) + } + + fn fmt_view(&self) -> HexView<'_, Self> + { + HexView(&self) + } +} + +#[cfg(test)] +mod tests +{ + use super::*; + fn format() + { + let bytes = b"hello world one two three \x142!"; + + panic!("\n{}\n", bytes.fmt_view()); + } +} + +#[pin_project] +pub struct ReadAllBytes<'a, T: AsyncRead+Unpin+?Sized>(#[pin] &'a mut T, Option); + +impl<'a, T: AsyncRead+Unpin+?Sized> Future for ReadAllBytes<'a, T> +{ + type Output = std::io::Result>; + fn poll(self: Pin<&mut Self>, ctx: &mut Context) -> Poll + { + let fut = async move { + let this = self.project(); + let mut output = Vec::with_capacity(4096*10); + let mut input = this.0; + let max = *this.1; + let mut buffer =[0u8; 4096]; + let mut read =0; + while {read = input.read(&mut buffer[..]).await?; read!=0} { + output.extend_from_slice(&buffer[..read]); + if let Some(max) = max { + if output.len() >=max { + return Err(std::io::Error::new(std::io::ErrorKind::Other, format!("Attempted to read more than allowed max {} bytes", max))); + } + } + } + Ok(output) + }; + tokio::pin!(fut); + fut.poll(ctx) + } +} + +pub trait ReadAllBytesExt: AsyncRead+Unpin +{ + /// Attempt to read the whole stream to a new `Vec`. + fn read_whole_stream(&mut self, max: Option) -> ReadAllBytes<'_, Self> + { + ReadAllBytes(self, max) + } +} + +impl ReadAllBytesExt for T{} diff --git a/src/format/key/mod.rs b/src/format/key/mod.rs new file mode 100644 index 0000000..c5c8d2a --- /dev/null +++ b/src/format/key/mod.rs @@ -0,0 +1,295 @@ +//! Keyiles format +use super::*; +use crypto::sha256::Sha256Hash; +use futures::{ + future::FutureExt, +}; +use std::{ + fmt, + error, + convert::{TryFrom, TryInto,}, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Ord,PartialOrd, Hash, Serialize, Deserialize)] +#[repr(u8)] +pub enum KeyHeaderKind +{ + Aes, + RsaPrivate, + RsaPublic, +} + +#[derive(Debug)] +pub struct ParsingError; + +impl error::Error for ParsingError{} + +impl fmt::Display for ParsingError +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result + { + write!(f, "failed to parse key header kind from byte") + } +} + + +impl TryFrom for KeyHeaderKind +{ + type Error = ParsingError; + + fn try_from(from: u8) -> Result + { + macro_rules! branches { + ($($num:path),*) => { + match from { + $( + x if x == $num as u8 => $num, + )* + _ => return Err(ParsingError), + } + } + } + + Ok(branches! { + Self::Aes, + Self::RsaPrivate, + Self::RsaPublic + }) + } +} + + +#[derive(Debug, PartialEq, Eq, Hash, Clone, Serialize, Deserialize)] +pub struct KeyHeader +{ + kind: KeyHeaderKind, + info: config::op::KeyDescription, + hash: Sha256Hash, + timestamp: i64, +} + +impl Header for KeyHeader +{ + const CHECK: u16 = CHECK_KEY; + fn hash(&self) -> Sha256Hash + { + let mut output = Vec::new(); + self.write_bytes(&mut output).now_or_never().unwrap().expect("Failed to write bytes to in-memory buffer"); + crypto::sha256::compute_slice(output) + } +} + +impl KeyHeader +{ + /// Create a new key header from these values + pub const fn new(kind: KeyHeaderKind, info: config::op::KeyDescription, hash: Sha256Hash, timestamp: i64) -> Self + { + Self { + kind, + info, + hash, + timestamp, + } + } + /// Create a new key header from these values with the current timestamp + pub fn new_now(kind: KeyHeaderKind, info: config::op::KeyDescription, hash: Sha256Hash) -> Self + { + Self { + timestamp: timestamp::now(), + ..Self::new(kind,info,hash,0) + } + } + + /// Write this superheader as text bytes to this stream + #[instrument(err, skip(out))] + pub async fn write_text(&self, out: &mut T) -> Result + { + let vec = serde_json::to_vec(self) + .wrap_err_with(|| eyre::eyre!("Failed to serialise self to JSON")) + .with_section(|| format!("{:?}", self).header("Self was"))?; + out.write_all(&vec[..]).await?; + Ok(vec.len()) + } + /// Read a superheader as text bytes from this stream + #[instrument(err, skip(input))] + pub async fn read_text(input: &mut T) -> Result + { + let whole = input.read_whole_stream(MAX_TEXT_SZ).await + .wrap_err_with(|| eyre::eyre!("Failed to read text stream into memory"))?; + + Ok(serde_json::from_slice(&whole[..]) + .wrap_err_with(|| eyre::eyre!("Failed to deserialise JSON to value")) + .with_section(move || String::from_utf8_lossy(&whole[..]).into_owned().header("Read text was"))?) + } + /// Write this key header as bytes to this stream + #[instrument(err, skip(out))] + pub async fn write_bytes(&self, out: &mut T) -> Result + { + out.write_u8(self.kind as u8).await?; + let desc = { + let buf = serde_cbor::to_vec(&self.info) + .wrap_err_with(|| eyre::eyre!("Failed to serialise key info into bytes")) + .with_section(|| format!("{:?}", self.info).header("Key info was"))?; + out.write_u64(buf.len().try_into() + .wrap_err_with(|| eyre::eyre!("Failed to represent key info size as 64 bit integer. Something is very wrong")) + .with_section(|| buf.len().to_string().header("Buffer size was"))?).await?; + out.write_all(&buf[..]).await?; + buf.len() + }; + out.write_all(self.hash.as_ref()).await?; + out.write_i64(self.timestamp).await?; + Ok(1 + std::mem::size_of::() + 8 + desc) + } + + /// Read a key header as bytes from this stream + #[instrument(err, skip(input))] + pub async fn read_bytes(input: &mut T) -> Result + { + let kind = { + let byte = input.read_u8().await?; + byte.try_into(). + with_section(|| [byte].to_hex_string().header("Invalid byte was"))? + }; + let info = { + let read = input.read_u64().await?; + let read = usize::try_from(read) + .wrap_err_with(|| eyre::eyre!("Read buffer size does not fit into usize (u64 -> u{}).", std::mem::size_of::()*8)) + .with_section(|| read.to_string().header("Read size was")) + .with_note(|| "This is a likely indication of file corruption")?; + let mut buffer = vec![0u8; read]; + input.read_exact(&mut buffer[..]).await?; + serde_cbor::from_reader(&buffer[..]) + .wrap_err_with(|| eyre::eyre!("Failed to deserialise key info from byte buffer")) + .with_section(|| read.to_string().header("Byte buffer's valid read size was")) + .with_section(|| format!("{}", buffer.fmt_view()).header("Byte buffer was"))? + }; + let hash = { + let mut hash = Sha256Hash::empty(); + input.read_exact(hash.as_mut()).await?; + hash + }; + let timestamp = input.read_i64().await?; + + Ok(Self { + kind, + info, + hash, + timestamp + }) + } +} + +#[cfg(test)] +mod tests +{ + use super::*; + #[test] + fn now_or_never() + { + let mut vec = Vec::new(); + use tokio::{ + prelude::*, + io::AsyncWrite, + }; + use futures::future::FutureExt; + async { + vec.write_all(&[0u8,1,2,3]).await.unwrap(); + vec.write_all(&[4u8,5,6,7]).await.unwrap(); + }.now_or_never().unwrap(); + + assert_eq!(&vec[..], &[0,1,2,3,4,5,6,7]); + } + + #[tokio::test] + async fn serde_with_super() -> Result<(), eyre::Report> + { + //color_eyre::install()?; + + let header = KeyHeader::new_now(KeyHeaderKind::Aes, Default::default(), Default::default()); + let mut ser = Vec::new(); + let superheader = SuperHeader::::new_for(&header); + println!("Writing: {:?} + {:?}", superheader, header); + let written = superheader.write_bytes(&mut ser).await? + + header.write_bytes(&mut ser).await?; + println!("Wrote {} bytes", written); + println!("{}\n", ser.fmt_view()); + + let mut read = &ser[..]; + let reads = SuperHeader::read_bytes(&mut read).await?; + println!("Read super: {:?}", reads); + let readheader = KeyHeader::read_bytes(&mut read).await?; + println!("Read real: {:?}", readheader); + + reads.verify_for(&header)?; + reads.verify_for(&readheader)?; + assert_eq!(readheader, header); + assert_eq!(reads, superheader); + + Ok(()) + } + #[tokio::test] + async fn serde() -> Result<(), eyre::Report> + { + let header = KeyHeader::new_now(KeyHeaderKind::Aes, Default::default(), Default::default()); + let mut ser = Vec::new(); + println!("Writing {:?}", header); + let val = header.write_bytes(&mut ser).await?; + println!("Wrote {} bytes:", val); + println!("{}\n", ser.fmt_view()); + + let reader = KeyHeader::read_bytes(&mut &ser[..]).await?; + println!("Read: {:?}", reader); + assert_eq!(reader, header); + + Ok(()) + } + + #[tokio::test] + async fn serde_text() -> Result<(), eyre::Report> + { + color_eyre::install()?; + + let header = KeyHeader::new_now(KeyHeaderKind::Aes, Default::default(), Default::default()); + let mut ser = Vec::new(); + println!("Writing {:?}", header); + let val = header.write_text(&mut ser).await?; + println!("Wrote {} bytes:", val); + println!("{}\n", ser.fmt_view()); + + let reader = KeyHeader::read_text(&mut &ser[..]).await?; + println!("Read: {:?}", reader); + assert_eq!(reader, header); + + Ok(()) + } + + #[tokio::test] + async fn serde_text_with_super() -> Result<(), eyre::Report> + { + //color_eyre::install()?; + + panic!("We're going to have to write our own text serialisation to get around the `read_whole_buffer` thingy..."); + + let header = KeyHeader::new_now(KeyHeaderKind::Aes, Default::default(), Default::default()); + let mut ser = Vec::new(); + let superheader = SuperHeader::::new_for(&header); + println!("Writing: {:?} + {:?}", superheader, header); + let written = superheader.write_text(&mut ser).await? + + header.write_text(&mut ser).await?; + println!("Wrote {} bytes", written); + println!("{}\n", ser.fmt_view()); + + let mut read = &ser[..]; + let reads = SuperHeader::read_text(&mut read).await?; + println!("Read super: {:?}", reads); + let readheader = KeyHeader::read_text(&mut read).await?; + println!("Read real: {:?}", readheader); + + reads.verify_for(&header)?; + reads.verify_for(&readheader)?; + assert_eq!(readheader, header); + assert_eq!(reads, superheader); + + Ok(()) + } +} diff --git a/src/format/mod.rs b/src/format/mod.rs new file mode 100644 index 0000000..97631a7 --- /dev/null +++ b/src/format/mod.rs @@ -0,0 +1,199 @@ +//! Handles file formats +use super::*; +use std::{ + error, + fmt, + marker::{ + PhantomData, + Unpin, + }, +}; +use tokio::{ + prelude::*, + io::{ + AsyncWrite, + AsyncRead, + }, +}; +use version::Version; + + +pub trait Header: fmt::Debug +{ + const CHECK: u16; + fn hash(&self) -> crypto::sha256::Sha256Hash; + +} + +pub const RAE_HEADER_BIT: [u8; 4] = *b"RAE0"; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +/// The header before all RAE files. +pub struct SuperHeader +{ + head: [u8; 4], + vers: Version, + chk: u16, + header_hash: crypto::sha256::Sha256Hash, + + _header: PhantomData, +} + +impl Default for SuperHeader +{ + #[inline] + fn default() -> Self + { + Self::new() + } +} + +const MAX_TEXT_SZ: Option = Some(1024 * 1024); + +impl SuperHeader +{ + /// Write this superheader as text bytes to this stream + #[instrument(err, skip(out))] + pub async fn write_text(&self, out: &mut T) -> Result + { + let vec = serde_json::to_vec(self) + .wrap_err_with(|| eyre::eyre!("Failed to serialise self to JSON")) + .with_section(|| format!("{:?}", self).header("Self was"))?; + out.write_all(&vec[..]).await?; + Ok(vec.len()) + } + /// Read a superheader as text bytes from this stream + #[instrument(err, skip(input))] + pub async fn read_text(input: &mut T) -> Result + { + let whole = input.read_whole_stream(MAX_TEXT_SZ).await + .wrap_err_with(|| eyre::eyre!("Failed to read text stream into memory"))?; + + Ok(serde_json::from_slice(&whole[..]) + .wrap_err_with(|| eyre::eyre!("Failed to deserialise JSON to value")) + .with_section(move || String::from_utf8_lossy(&whole[..]).into_owned().header("Read text was"))?) + } + + /// Write this superheader as bytes to this stream + #[instrument(err, skip(out))] + pub async fn write_bytes(&self, out: &mut T) -> Result + { + Ok({out.write_all(&self.head[..]).await?; self.head.len()} + + {out.write_all(self.vers.as_bytes()).await?; std::mem::size_of::()} + + {out.write_u16(self.chk).await?; 2} + + {out.write_all(self.header_hash.as_ref()).await?; std::mem::size_of::()}) + } + /// Read a superheader as bytes from this stream + #[instrument(err, skip(input))] + pub async fn read_bytes(input: &mut T) -> Result + { + let mut new = Self::new(); + input.read_exact(&mut new.head[..]).await?; + new.vers = { + let mut bytes = [0u8; std::mem::size_of::()]; + input.read_exact(&mut bytes[..]).await?; + Version::try_from_bytes(bytes) + .wrap_err_with(|| eyre::eyre!("Failed to decode version")) + .with_section(|| bytes.to_broken_hex_string().header("Bytes (hex) were"))? + }; + new.chk = input.read_u16().await?; + new.header_hash = { + let mut bytes= crypto::sha256::Sha256Hash::empty(); + input.read_exact(bytes.as_mut()).await?; + bytes + }; + if new.vers.should_warn(&CURRENT_VERSION) { + warn!("Header ({:?}) is read to have deprecated version {}, we are on version {}", new, new.vers, CURRENT_VERSION) + } + Ok(new) + } + + /// Create a new empty superheader + pub fn new() -> Self + { + Self{ + head: RAE_HEADER_BIT, + vers: CURRENT_VERSION, + chk: H::CHECK, + header_hash: Default::default(), + _header: PhantomData, + } + } + + /// Consume into a new instance with hash for `header`. + pub fn with_hash(self, header: &H) -> Self + { + + Self { + header_hash: header.hash(), + ..self + } + } + + /// Create a new superheader for + pub fn new_for(header: &H) -> Self + { + Self { + header_hash: header.hash(), + ..Self::new() + } + } + + /// Verify this empty superheader + pub fn verify(&self) -> Result<(), VerificationError> + { + macro_rules! check { + ($field:expr, $err:expr) => { + if !$field { + return Err($err); + } + } + } + check!(self.head == RAE_HEADER_BIT, VerificationError::BadHeaderBit); + check!(self.vers.is_compat(&CURRENT_VERSION), VerificationError::IncompatableVersion(self.vers)); + check!(self.chk == H::CHECK, VerificationError::BadCheckBit); + + Ok(()) + } + + /// Verify this not-empty header for `header`. + /// + /// # Notes + /// This also calls `self.verify()`. + pub fn verify_for(&self, header: &H) -> Result<(), VerificationError> + { + self.verify()?; + if self.header_hash == header.hash() { + Ok(()) + } else { + Err(VerificationError::BadHash) + } + } +} + +#[derive(Debug)] +#[non_exhaustive] +pub enum VerificationError +{ + BadHeaderBit, + IncompatableVersion(Version), + BadCheckBit, + BadHash, +} + +impl error::Error for VerificationError{} +impl fmt::Display for VerificationError +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result + { + match self { + Self::BadHeaderBit => write!(f, "bad header bit"), + Self::IncompatableVersion(vers) => write!(f, "contained incompatable version {}, we are on version {}", vers, CURRENT_VERSION), + Self::BadCheckBit => write!(f, "contained invalid check bit"), + Self::BadHash => write!(f, "contained invalid header hash"), + } + } +} + +mod key; +const CHECK_KEY: u16 = 0x0001; diff --git a/src/main.rs b/src/main.rs index e687d90..89eda4e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,6 +3,7 @@ #![allow(dead_code)] #[macro_use] extern crate tracing; +#[macro_use] extern crate pin_project; use std::{ convert::{TryFrom, TryInto}, }; @@ -17,8 +18,33 @@ use lazy_static::lazy_static; use serde::{Serialize, Deserialize}; use tracing_futures::Instrument; + +macro_rules! cfg_debug { + (if {$($if:tt)*} else {$($else:tt)*}) => { + { + cfg_if::cfg_if!{ + if #[cfg(debug_assertions)] { + $($if)* + } else { + $($else)* + } + } + } + }; +} + +macro_rules! static_assert { + ($expr:expr) => { + const _: [(); 1] = [(); ((!!$expr) as bool) as usize]; + }; + ($expr:expr, $lit:literal) => { + static_assert!{$expr} + } +} + pub const CURRENT_VERSION: version::Version = version::Version::new(0,0,0,version::Tag::Prerelease); +mod bytes; mod ext; use ext::*; mod version; @@ -26,6 +52,20 @@ mod config; mod resolve; mod args; +mod format; +mod work; + +pub mod timestamp +{ + use chrono::prelude::*; + /// Get the current timestamp + pub fn now() -> i64 + { + #[cfg(feature="local-time")] return Local::now().timestamp(); + #[cfg(not(feature="local-time"))] Utc::now().timestamp() + } +} + /*/// Dispatch params operations that can be handled at top level (i.e. `Help`) async fn dispatch_args() -> Result { @@ -35,19 +75,6 @@ match args::parse_args().await } }*/ -macro_rules! cfg_debug { - (if {$($if:tt)*} else {$($else:tt)*}) => { - { - cfg_if::cfg_if!{ - if #[cfg(debug_assertions)] { - $($if)* - } else { - $($else)* - } - } - } - }; -} fn install_tracing() { use tracing_error::ErrorLayer; diff --git a/src/version.rs b/src/version.rs index 717e132..7465229 100644 --- a/src/version.rs +++ b/src/version.rs @@ -8,7 +8,7 @@ use std::{ }; /// Represents other states of this container format version -#[derive(Debug, Clone, PartialEq, Eq, Hash, Ord, PartialOrd, Copy)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Ord, PartialOrd, Copy, Serialize, Deserialize)] #[repr(u8)] pub enum Tag { @@ -41,7 +41,7 @@ impl Tag $( x if x == $num as u8 => $num, )* - _ => return Err(ParsingError), + _ => return Err(ParsingError), } } } @@ -82,10 +82,18 @@ impl Default for Tag /// * Minor - If higher that current, fail parsing. /// * Bugfix - If higher than current, warn user. /// * Tag - Unused, but may represent things like prerelease or unstable -#[derive(Debug, Clone, PartialEq, Eq, Hash, Ord, PartialOrd, Copy, Default)] -#[repr(packed, C)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Ord, PartialOrd, Copy, Default, Serialize, Deserialize)] +#[repr(align(4))] pub struct Version(u8,u8,u8,Tag); + +static_assert!(std::mem::size_of::() == std::mem::size_of::(), + "Size of version != size of u32"); +static_assert!(std::mem::size_of::() == 4, + "Size of Version (u32) != 4"); +static_assert!(std::mem::align_of::() == std::mem::align_of::(), + "Align of version != align of u32 with `repr(align(4))`"); + impl fmt::Display for Version { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result @@ -114,6 +122,44 @@ impl Version Self(major,minor,bugfix,tag) } + /// Get binary representation (4 bytes) + pub fn as_bytes(&self) -> &[u8] + { + unsafe { + std::slice::from_raw_parts(self as *const Self as *const u8, std::mem::size_of::()) + } + } + + /// Try to create an instance from bytes + pub fn try_from_bytes(bytes: [u8; 4]) -> Result + { + Ok(Self::new(bytes[0], + bytes[1], + bytes[2], + Tag::try_from(bytes[3])?)) + } + + /// Uncheckedly create a version from bytes + pub unsafe fn from_bytes_unchecked(bytes: [u8; 4]) -> Self + { + let mut s = Self::default(); + assert_eq!(bytes::copy_slice(s.as_bytes_mut(), &bytes[..]), std::mem::size_of::()); + s + } + + /// Convert into 4 byte array + #[inline] pub fn into_bytes(self) -> [u8; 4] + { + [self.0, self.1, self.2, self.3 as u8] + } + + + /// Get mutable binary representation (4 bytes) + pub unsafe fn as_bytes_mut(&mut self) -> &mut [u8] + { + std::slice::from_raw_parts_mut(self as *mut Version as *mut u8, std::mem::size_of::()) + } + /// Encode as u32 pub fn as_u32(&self) -> u32 { @@ -142,9 +188,9 @@ impl Version } Ok(Self::new(index!(parts, 0), - index!(parts, 1), - index!(parts, 2), - Tag::try_from_u8(index!(parts, 3))?)) + index!(parts, 1), + index!(parts, 2), + Tag::try_from_u8(index!(parts, 3))?)) } } diff --git a/src/work/generate/aes.rs b/src/work/generate/aes.rs new file mode 100644 index 0000000..a8e58d4 --- /dev/null +++ b/src/work/generate/aes.rs @@ -0,0 +1,2 @@ +//! Generate aes key operations +use super::*; diff --git a/src/work/generate/mod.rs b/src/work/generate/mod.rs new file mode 100644 index 0000000..dd3ec4f --- /dev/null +++ b/src/work/generate/mod.rs @@ -0,0 +1,4 @@ +//! Generate key operations +use super::*; + +pub mod aes; diff --git a/src/work/mod.rs b/src/work/mod.rs new file mode 100644 index 0000000..6636d99 --- /dev/null +++ b/src/work/mod.rs @@ -0,0 +1,3 @@ +use super::*; + +pub mod generate;