diff --git a/Cargo.lock b/Cargo.lock index d236274..53d41b6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "ad-hoc-iter" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90a8dd76beceb5313687262230fcbaaf8d4e25c37541350cf0932e9adb8309c8" + [[package]] name = "addr2line" version = "0.16.0" @@ -880,6 +886,9 @@ name = "smallvec" version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" +dependencies = [ + "serde", +] [[package]] name = "subtle" @@ -953,6 +962,7 @@ dependencies = [ name = "transfer" version = "0.1.0" dependencies = [ + "ad-hoc-iter", "async-compression", "base64 0.13.0", "bytes 1.1.0", @@ -967,6 +977,7 @@ dependencies = [ "serde", "serde_cbor", "serde_json", + "smallvec", "tokio 1.12.0", ] diff --git a/Cargo.toml b/Cargo.toml index b6192b0..3ae550c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,11 +6,12 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +ad-hoc-iter = "0.2.3" async-compression = { version = "0.3.8", features = ["tokio", "brotli"] } base64 = "0.13.0" bytes = "1.1.0" color-eyre = { version = "0.5.11", default-features = false } -cryptohelpers = { version = "1.8.2", features = ["full"] } +cryptohelpers = { version = "1.8.2", features = ["sha256", "async", "rsa", "serde"] } futures = "0.3.17" getrandom = "0.2.3" lazy_static = "1.4.0" @@ -20,4 +21,5 @@ pretty_env_logger = "0.4.0" serde = { version = "1.0.130", features = ["derive"] } serde_cbor = "0.11.2" serde_json = "1.0.68" +smallvec = { version = "1.6.1", features = ["serde", "const_generics", "write"] } tokio = { version = "1.12.0", features = ["full"] } diff --git a/src/enc/mod.rs b/src/enc/mod.rs new file mode 100644 index 0000000..fefae3e --- /dev/null +++ b/src/enc/mod.rs @@ -0,0 +1,25 @@ +//! Encodings +use super::*; +use ext::*; +use std::{fmt, error}; +use bytes::{ + Buf, + Bytes, +}; +use std::io; +use tokio::io::{ + AsyncRead, AsyncWrite, + AsyncReadExt, AsyncWriteExt, +}; +use serde::{ + Serialize, + de::DeserializeOwned +}; +use cryptohelpers::{ + sha256, + rsa, +}; + +mod ser; +pub use ser::*; + diff --git a/src/enc.rs b/src/enc/ser.rs similarity index 55% rename from src/enc.rs rename to src/enc/ser.rs index f18fe3a..9aa17ac 100644 --- a/src/enc.rs +++ b/src/enc/ser.rs @@ -1,20 +1,6 @@ -//! Encodings +//! Data serialisation use super::*; -use ext::*; -use std::{fmt, error}; -use bytes::{ - Buf, - Bytes, -}; -use std::io; -use tokio::io::{ - AsyncRead, AsyncWrite, - AsyncReadExt, AsyncWriteExt, -}; -use serde::{ - Serialize, - de::DeserializeOwned -}; +use bytes::BufMut; #[derive(Debug, Clone, PartialEq, Eq, Hash, Copy, PartialOrd, Ord)] pub enum CompressionKind @@ -70,21 +56,135 @@ impl Default for SerialFormat } } -#[derive(Debug, Clone, PartialEq, Eq, Hash, Copy, Default)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Copy)] pub struct SendOpt { - pub comp: Option, - pub encrypt: Option, - pub format: SerialFormat, + comp: Option, + encrypt: Option, + format: SerialFormat, + hash: bool, + //pub sign: Option, //TODO: RSA private + public key types } ref_self!(SendOpt); +impl Default for SendOpt +{ + #[inline] + fn default() -> Self + { + Self::new() + } +} + + impl SendOpt { + pub const NORMAL: Self = Self::new(); + pub const CHECKED: Self = Self::new_checked(); + pub const COMPRESSED: Self = Self::new_compressed(); + + /// Add compression + pub const fn compress(self, k: CompressionKind) -> Self + { + Self { + comp: Some(k), + ..self + } + } + + /// Change the output format + /// + /// Default: **Binary** + /// + /// # Text format note + /// When using compression and/or encryption, the text format will end up unreadable anyway. + /// Likewise when using signing or hashing, a binary header is prepended to the message regardless of format. + /// + /// 2 ASCII whitespace characters are prepended to the message regardless of any other options (`\t`, ` |\n`). These are used to determine if the message is valid and if a header needs to be read from it. + /// Most external text-format parsing software should ignore these and be able to parse a non-headered message. + pub const fn format(self, format: SerialFormat) -> Self + { + Self { + format, + ..self + } + } + + /// Enable or disable hashing + /// + /// Default: *Disabled* + pub const fn hash(self, hash: bool) -> Self + { + Self { + hash, + ..self + } + } + + /// Add encryption with constant parameters + pub const fn encrypt(self, k: EncryptionKind) -> Self + { + Self { + encrypt: Some(k), + ..self + } + } + + /// Add default encryption with a randomly generated key and IV. + pub fn encrypt_cc20_gen(self) -> Self + { + self.encrypt(EncryptionKind::Chacha20(cha::keygen())) + } + + /// Normal options. + /// + /// Does not enable any features. + pub const fn new() -> Self + { + Self { + comp: None, + encrypt: None, + format: SerialFormat::Binary, + hash: false, + } + } + /// Normal options with data compression. + /// + /// Uses Brotli compression by default. + pub const fn new_compressed() -> Self + { + Self { + comp: Some(CompressionKind::Brotli), + ..Self::new() + } + } + /// Normal options with added integrity checks. + /// + /// Increases final size of object but provided data integrity and source validation. + //TODO: Give sig param + pub const fn new_checked() -> Self + { + Self { + hash: true, + //sig: ??? + ..Self::new() + } + } + /// Should a header be generated for this data? + #[inline(always)] fn needs_header(&self) -> bool + { + self.hash || /*self.sig*/ false + } + + #[inline] fn creates_header(&self) -> bool + { + self.needs_header() + } + /// Does the binary data of this format require special handling? /// /// True if encryption and/or compression are specified. - fn is_spec(&self) -> bool + #[inline(always)] fn is_spec(&self) -> bool { self.comp.is_some() || self.encrypt.is_some() } @@ -95,7 +195,7 @@ pub type RecvOpt = SendOpt; /// Default buffer size for encryption transform stream copying. pub const DEFAULT_BUFSIZE: usize = 4096; -async fn cha_copy(from: &mut F, to: &mut T, key: &key::Key, iv: &key::IV) -> io::Result<(usize, usize)> +pub(super) async fn cha_copy(from: &mut F, to: &mut T, key: &key::Key, iv: &key::IV) -> io::Result<(usize, usize)> where F: AsyncRead + Unpin + ?Sized, T: AsyncWrite + Unpin + ?Sized { @@ -121,10 +221,170 @@ where F: AsyncRead + Unpin + ?Sized, Ok((written, read)) } -async fn de_singleton_inner(buf: F, mut from: &[u8], how: &RecvOpt) -> Result +const H_SALT_SIZE: usize = 32; +#[derive(Debug, Clone, PartialEq, Eq, Hash, Default)] +struct FormatHeader +{ + hash: Option<(sha256::Sha256Hash, [u8; H_SALT_SIZE])>, + sig: Option, +} + +#[derive(Debug)] +pub enum HeaderValidationError +{ + Malformed, + Hash, + Signature, +} + +impl error::Error for HeaderValidationError{} +impl fmt::Display for HeaderValidationError +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result + { + match self { + Self::Malformed => write!(f, "header was malformed"), + Self::Hash => write!(f, "invalid hash"), + Self::Signature => write!(f, "signature could not be verified"), + } + } +} + + +impl FormatHeader +{ + pub const SIZE: usize = sha256::SIZE + H_SALT_SIZE + cryptohelpers::consts::RSA_SIG_SIZE + 2; + const fn empty_array() -> [u8; Self::SIZE] + { + [0u8; Self::SIZE] + } + fn gen_salt() -> [u8; H_SALT_SIZE] + { + let mut out = [0u8; H_SALT_SIZE]; + getrandom::getrandom(&mut out[..]).expect("rng fatal"); + out + } + fn generate(data: impl AsRef<[u8]>, opt: &SendOpt) -> Self + { + let hash = if opt.hash { + let salt = Self::gen_salt(); + Some((sha256::compute_slices(iter![data.as_ref(), &salt[..]]), salt)) + } else { + None + }; + let sig = if false /*let Some(sign_with) = opt.sign*/ { + unimplemented!() + } else { + None + }; + Self { + hash, + sig //TODO + } + } + fn validate(&self, data: impl AsRef<[u8]>, opt: &RecvOpt) -> Result<(), HeaderValidationError> + { + if opt.hash { + if !self.hash.as_ref().map(|(hash, salt)| &sha256::compute_slices(iter![data.as_ref(), &salt[..]]) == hash).unwrap_or(true) { + return Err(HeaderValidationError::Hash); + } + } + if /*opt.sig*/ false { + unimplemented!(); + //if let Some(verify_with) = opt.sig //XXX: How will this work? We will need to store **either** a private or public key in Send/RecvOpt and dynamically dispatch over it. + } + Ok(()) + } + fn to_buffer(&self, mut to: impl BufMut) + { + if let Some(hash) = &self.hash + { + to.put_u8(1); + to.put_slice(hash.0.as_ref()); + to.put_slice(hash.1.as_ref()); + } else { + to.put_u8(0); + to.put_bytes(0, sha256::SIZE + H_SALT_SIZE); + } + if let Some(sig) = &self.sig + { + to.put_u8(1); + to.put_slice(sig.as_ref()); + } else { + to.put_u8(0); + to.put_bytes(0, cryptohelpers::consts::RSA_SIG_SIZE); + } + } + fn from_buffer(mut from: impl Buf) -> Self + { + let hash = if from.get_u8() == 1 { + let mut hash = sha256::Sha256Hash::default(); + let mut salt = [0u8; H_SALT_SIZE]; + from.copy_to_slice(hash.as_mut()); + from.copy_to_slice(&mut salt[..]); + Some((hash,salt)) + } else { + from.advance(sha256::SIZE + H_SALT_SIZE); + None + }; + let sig = if from.get_u8() == 1 { + let mut sig = rsa::Signature::default(); + from.copy_to_slice(sig.as_mut()); + Some(sig) + } else { + from.advance(sha256::SIZE); + None + }; + Self { + hash, sig + } + } + #[inline] fn to_array(&self) -> [u8; Self::SIZE] + { + let mut ar = [0u8; Self::SIZE]; + self.to_buffer(&mut &mut ar[..]); + ar + } + #[inline] fn from_array(ar: [u8; Self::SIZE]) -> Self + { + Self::from_buffer(&ar[..]) + } +} + +const INFO_ASSERT_VALID: u8 = b'\t'; +const INFO_WITH_HEADER: u8 = b' '; +const INFO_NO_HEADER: u8 = b'\n'; + +/// If passing an externally generated message to be deserialised here, it must be prefixed with this regardless of its format. +/// +/// Operations that generate/require a message header will not work on these messages and if they are needed must be handled elsewhere by the user. (Hash and signature validation) +pub const BARE_MESSAGE_PREFIX: [u8; 2] = [INFO_ASSERT_VALID, INFO_NO_HEADER]; + +pub(super) async fn de_singleton_inner(buf: F, from: &[u8], how: &RecvOpt) -> Result where B: AsRef<[u8]> + AsyncWrite + Unpin + Default, F: FnOnce(&[u8]) -> B { + + // Read header + let mut header = FormatHeader::empty_array(); + if from.len() < 2 || from[0] != INFO_ASSERT_VALID { + return Err(TransformErrorKind::InvalidHeader(HeaderValidationError::Malformed)); + } + let (inf, mut from) = { + (&from[..2], &from[2..]) + }; + from = { + if inf[1] == INFO_WITH_HEADER { + if from.len() < FormatHeader::SIZE { + return Err(TransformErrorKind::InvalidHeader(HeaderValidationError::Malformed)); + } + let hf = &from[..FormatHeader::SIZE]; + header.copy_from_slice(hf); + &from[FormatHeader::SIZE..] + } else { + &from[..] + } + }; // Decompressor // The output is written to this (through writer) let mut is_spec = false; // This is set later. The value will sometimes differ from `how.is_spec()` depending on combinations of options. @@ -180,6 +440,9 @@ where B: AsRef<[u8]> + AsyncWrite + Unpin + Default, } }; // Deserialise + + FormatHeader::from_array(header).validate(from, how)?; + let v = match how.format { SerialFormat::Text => serde_json::from_slice(&from[..])?, SerialFormat::Binary => serde_cbor::from_slice(&from[..])?, @@ -188,14 +451,20 @@ where B: AsRef<[u8]> + AsyncWrite + Unpin + Default, Ok(v) } -async fn ser_singleton_inner(to: F, value: &T, how: impl AsRef) -> Result<(V, usize), TransformErrorKind> -where F: FnOnce(&Vec) -> V +pub(super) async fn ser_singleton_inner(to: F, value: &T, how: impl AsRef) -> Result<(V, usize), TransformErrorKind> +where F: FnOnce(&Vec) -> V, { let how = how.as_ref(); let ser = match how.format { SerialFormat::Text => serde_json::to_vec(value)?, SerialFormat::Binary => serde_cbor::to_vec(value)?, }; + let header = if how.needs_header() { + let header = FormatHeader::generate(&ser, how); + header.to_array() + } else { + FormatHeader::empty_array() + }; let mut a; let mut b; let reader: &mut (dyn AsyncRead + Unpin) = @@ -212,6 +481,12 @@ where F: FnOnce(&Vec) -> V &mut b }; let mut ser = to(&ser); + if how.needs_header() { + ser.write_all(&[INFO_ASSERT_VALID, INFO_WITH_HEADER]).await?; + ser.write_all(&header[..]).await?; + } else { + ser.write_all(&[INFO_ASSERT_VALID, INFO_NO_HEADER]).await?; + } let w= if let Some(enc) = &how.encrypt { let n = match enc { EncryptionKind::Chacha20((k, iv)) => { @@ -296,6 +571,8 @@ pub enum TransformErrorKind /// Misc. IO //TODO: Disambiguate when this happens into the two above cases. IO(io::Error), + /// The object header was invalid. + InvalidHeader(HeaderValidationError), } /// An error when sending / serialising an object. @@ -323,6 +600,7 @@ impl error::Error for RecvError Some(match &self.0.0 { TransformErrorKind::IO(io) => io, + TransformErrorKind::InvalidHeader(ih) => ih, _ => return None, }) } @@ -337,6 +615,7 @@ impl fmt::Display for RecvError TransformErrorKind::Compress => write!(f, "failed to decompress data"), TransformErrorKind::Encrypt => write!(f, "failed to decrypt data"), TransformErrorKind::IO(_) => write!(f, "i/o failure"), + TransformErrorKind::InvalidHeader(_) => write!(f, "invalid header"), } } } @@ -352,6 +631,7 @@ impl error::Error for SendError Some(match &self.0.0 { TransformErrorKind::IO(io) => io, + TransformErrorKind::InvalidHeader(ih) => ih, _ => return None, }) } @@ -367,6 +647,7 @@ impl fmt::Display for SendError TransformErrorKind::Compress => write!(f, "failed to compress data"), TransformErrorKind::Encrypt => write!(f, "failed to encrypt data"), TransformErrorKind::IO(_) => write!(f, "i/o failure"), + TransformErrorKind::InvalidHeader(_) => write!(f, "invalid header"), } } } @@ -379,6 +660,15 @@ impl From for TransformErrorKind } } +impl From for TransformErrorKind +{ + fn from(from: HeaderValidationError) -> Self + { + Self::InvalidHeader(from) + } +} + + impl From for TransformErrorKind { @@ -407,7 +697,7 @@ mod test let obj = String::from("Hello world"); let var = ser_singleton(&obj, &how).await?; - eprintln!("Ser: {}", var.hex()); + eprintln!("Ser ({} bytes): {}", var.len(), var.hex()); let des: String = de_singleton(&var, &how).await?; eprintln!("De: {:?}", des); assert_eq!(obj, des); @@ -434,6 +724,7 @@ mod test { ser_de_with(SendOpt { encrypt: Some(EncryptionKind::Chacha20(cha::keygen())), + //hash: true, ..Default::default() }).await } diff --git a/src/ext.rs b/src/ext.rs index bc00a91..d724c9e 100644 --- a/src/ext.rs +++ b/src/ext.rs @@ -152,10 +152,8 @@ impl + Clone> fmt::Display for HexStringIter } } -/* #[macro_export] macro_rules! prog1 { ($first:expr, $($rest:expr);+ $(;)?) => { ($first, $( $rest ),+).0 } } -*/ diff --git a/src/main.rs b/src/main.rs index 570dd40..8de7b04 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,6 +2,7 @@ #![allow(dead_code)] #[macro_use] extern crate log; +#[macro_use] extern crate ad_hoc_iter; #[macro_use] extern crate lazy_static; #[macro_use] extern crate serde;