enc::ser: Added message hashing option. Added message validation, optional message header for hash, salt and signature (if used.) Messages now contain 2 prefix (whitespace, to not change bare text-formatted messages semantics) characters used to determine if the message is valid and if there is a header to be read from it.

Fortune for transfer's current commit: Small blessing − 小吉

Cargo.lock

@@ -2,6 +2,12 @@
 # It is not intended for manual editing.
 version = 3
 
+[[package]]
+name = "ad-hoc-iter"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "90a8dd76beceb5313687262230fcbaaf8d4e25c37541350cf0932e9adb8309c8"
+
 [[package]]
 name = "addr2line"
 version = "0.16.0"
@@ -880,6 +886,9 @@ name = "smallvec"
 version = "1.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e"
+dependencies = [
+ "serde",
+]
 
 [[package]]
 name = "subtle"
@@ -953,6 +962,7 @@ dependencies = [
 name = "transfer"
 version = "0.1.0"
 dependencies = [
+ "ad-hoc-iter",
  "async-compression",
  "base64 0.13.0",
  "bytes 1.1.0",
@@ -967,6 +977,7 @@ dependencies = [
  "serde",
  "serde_cbor",
  "serde_json",
+ "smallvec",
  "tokio 1.12.0",
 ]
 

Cargo.toml

@@ -6,11 +6,12 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+ad-hoc-iter = "0.2.3"
 async-compression = { version = "0.3.8", features = ["tokio", "brotli"] }
 base64 = "0.13.0"
 bytes = "1.1.0"
 color-eyre = { version = "0.5.11", default-features = false }
-cryptohelpers = { version = "1.8.2", features = ["full"] }
+cryptohelpers = { version = "1.8.2", features = ["sha256", "async", "rsa", "serde"] }
 futures = "0.3.17"
 getrandom = "0.2.3"
 lazy_static = "1.4.0"
@@ -20,4 +21,5 @@ pretty_env_logger = "0.4.0"
 serde = { version = "1.0.130", features = ["derive"] }
 serde_cbor = "0.11.2"
 serde_json = "1.0.68"
+smallvec = { version = "1.6.1", features = ["serde", "const_generics", "write"] }
 tokio = { version = "1.12.0", features = ["full"] }

src/enc/mod.rs

@@ -0,0 +1,25 @@
+//! Encodings
+use super::*;
+use ext::*;
+use std::{fmt, error};
+use bytes::{
+    Buf,
+    Bytes,
+};
+use std::io;
+use tokio::io::{
+    AsyncRead, AsyncWrite,
+    AsyncReadExt, AsyncWriteExt,
+};
+use serde::{
+    Serialize,
+    de::DeserializeOwned
+};
+use cryptohelpers::{
+    sha256,
+    rsa,
+};
+
+mod ser;
+pub use ser::*;
+

src/enc/ser.rs

@@ -1,20 +1,6 @@
-//! Encodings
+//! Data serialisation
 use super::*;
-use ext::*;
+use bytes::BufMut;
-use std::{fmt, error};
-use bytes::{
-    Buf,
-    Bytes,
-};
-use std::io;
-use tokio::io::{
-    AsyncRead, AsyncWrite,
-    AsyncReadExt, AsyncWriteExt,
-};
-use serde::{
-    Serialize,
-    de::DeserializeOwned
-};
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash, Copy, PartialOrd, Ord)]
 pub enum CompressionKind
@@ -70,21 +56,135 @@ impl Default for SerialFormat
     }
 }
 
-#[derive(Debug, Clone, PartialEq, Eq, Hash, Copy, Default)]
+#[derive(Debug, Clone, PartialEq, Eq, Hash, Copy)]
 pub struct SendOpt
 {
-    pub comp: Option<CompressionKind>,
+    comp: Option<CompressionKind>,
-    pub encrypt: Option<EncryptionKind>,
+    encrypt: Option<EncryptionKind>,
-    pub format: SerialFormat,
+    format: SerialFormat,
+    hash: bool,
+    //pub sign: Option<???>, //TODO: RSA private + public key types
 }
 ref_self!(SendOpt);
 
+impl Default for SendOpt
+{
+    #[inline]
+    fn default() -> Self
+    {
+	Self::new()
+    }
+}
+
+
 impl SendOpt
 {
+    pub const NORMAL: Self = Self::new();
+    pub const CHECKED: Self = Self::new_checked();
+    pub const COMPRESSED: Self = Self::new_compressed();
+
+    /// Add compression
+    pub const fn compress(self, k: CompressionKind) -> Self
+    {
+	Self {
+	    comp: Some(k),
+	    ..self
+	}
+    }
+
+    /// Change the output format
+    ///
+    /// Default: **Binary**
+    ///
+    /// # Text format note
+    /// When using compression and/or encryption, the text format will end up unreadable anyway.
+    /// Likewise when using signing or hashing, a binary header is prepended to the message regardless of format.
+    ///
+    /// 2 ASCII whitespace characters are prepended to the message regardless of any other options (`\t`, ` |\n`). These are used to determine if the message is valid and if a header needs to be read from it.
+    /// Most external text-format parsing software should ignore these and be able to parse a non-headered message.
+    pub const fn format(self, format: SerialFormat) -> Self
+    {
+	Self {
+	    format,
+	    ..self
+	}
+    }
+
+    /// Enable or disable hashing
+    ///
+    /// Default: *Disabled*
+    pub const fn hash(self, hash: bool) -> Self
+    {
+	Self {
+	    hash,
+	    ..self
+	}
+    }
+
+    /// Add encryption with constant parameters
+    pub const fn encrypt(self, k: EncryptionKind) -> Self
+    {
+	Self {
+	    encrypt: Some(k),
+	    ..self
+	}
+    }
+
+    /// Add default encryption with a randomly generated key and IV.
+    pub fn encrypt_cc20_gen(self) -> Self
+    {
+	self.encrypt(EncryptionKind::Chacha20(cha::keygen()))
+    }
+    
+    /// Normal options.
+    ///
+    /// Does not enable any features.
+    pub const fn new() -> Self
+    {
+	Self {
+	    comp: None,
+	    encrypt: None,
+	    format: SerialFormat::Binary,
+	    hash: false,
+	}
+    }
+    /// Normal options with data compression.
+    ///
+    /// Uses Brotli compression by default.
+    pub const fn new_compressed() -> Self
+    {
+	Self {
+	    comp: Some(CompressionKind::Brotli),
+	    ..Self::new()
+	}
+    }
+    /// Normal options with added integrity checks.
+    ///
+    /// Increases final size of object but provided data integrity and source validation.
+    //TODO: Give sig param
+    pub const fn new_checked() -> Self
+    {
+	Self {
+	    hash: true,
+	    //sig: ???
+	    ..Self::new()
+	}
+    }
+    /// Should a header be generated for this data?
+    #[inline(always)] fn needs_header(&self) -> bool
+    {
+	self.hash || /*self.sig*/ false
+    }
+
+    #[inline] fn creates_header(&self) -> bool
+    {
+	self.needs_header()
+    }
+    
     /// Does the binary data of this format require special handling?
     ///
     /// True if encryption and/or compression are specified.
-    fn is_spec(&self) -> bool
+    #[inline(always)] fn is_spec(&self) -> bool
     {
 	self.comp.is_some() || self.encrypt.is_some()
     }
@@ -95,7 +195,7 @@ pub type RecvOpt = SendOpt;
 /// Default buffer size for encryption transform stream copying.
 pub const DEFAULT_BUFSIZE: usize = 4096;
 
-async fn cha_copy<F, T, const BUFSIZE: usize, const DECRYPT: bool>(from: &mut F, to: &mut T, key: &key::Key, iv: &key::IV) -> io::Result<(usize, usize)>
+pub(super) async fn cha_copy<F, T, const BUFSIZE: usize, const DECRYPT: bool>(from: &mut F, to: &mut T, key: &key::Key, iv: &key::IV) -> io::Result<(usize, usize)>
 where F: AsyncRead + Unpin + ?Sized,
       T: AsyncWrite + Unpin + ?Sized
 {
@@ -121,10 +221,170 @@ where F: AsyncRead + Unpin + ?Sized,
     Ok((written, read))
 }
 
-async fn de_singleton_inner<T: DeserializeOwned, B, F>(buf: F, mut from: &[u8], how: &RecvOpt) -> Result<T, TransformErrorKind>
+const H_SALT_SIZE: usize = 32;
+#[derive(Debug, Clone, PartialEq, Eq, Hash, Default)]
+struct FormatHeader
+{
+    hash: Option<(sha256::Sha256Hash, [u8; H_SALT_SIZE])>,
+    sig: Option<rsa::Signature>,
+}
+
+#[derive(Debug)]
+pub enum HeaderValidationError
+{
+    Malformed,
+    Hash,
+    Signature,
+}
+
+impl error::Error for HeaderValidationError{}
+impl fmt::Display for HeaderValidationError
+{
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result
+    {
+	match self {
+	    Self::Malformed => write!(f, "header was malformed"),
+	    Self::Hash => write!(f, "invalid hash"),
+	    Self::Signature => write!(f, "signature could not be verified"),
+	}
+    }
+}
+
+
+impl FormatHeader
+{
+    pub const SIZE: usize = sha256::SIZE + H_SALT_SIZE + cryptohelpers::consts::RSA_SIG_SIZE + 2;
+    const fn empty_array() -> [u8; Self::SIZE]
+    {
+	[0u8; Self::SIZE]
+    }
+    fn gen_salt() -> [u8; H_SALT_SIZE]
+    {
+	let mut out = [0u8; H_SALT_SIZE];
+	getrandom::getrandom(&mut out[..]).expect("rng fatal");
+	out
+    }
+    fn generate(data: impl AsRef<[u8]>, opt: &SendOpt) -> Self
+    {
+	let hash = if opt.hash {
+	    let salt = Self::gen_salt();
+	    Some((sha256::compute_slices(iter![data.as_ref(), &salt[..]]), salt))
+	} else {
+	    None
+	};
+	let sig = if false /*let Some(sign_with) = opt.sign*/ {
+	    unimplemented!()
+	} else {
+	    None
+	};
+	Self {
+	    hash,
+	    sig //TODO
+	}
+    }
+    fn validate(&self, data: impl AsRef<[u8]>, opt: &RecvOpt) -> Result<(), HeaderValidationError>
+    {
+	if opt.hash {
+	    if !self.hash.as_ref().map(|(hash, salt)| &sha256::compute_slices(iter![data.as_ref(), &salt[..]]) == hash).unwrap_or(true) {
+		return Err(HeaderValidationError::Hash);
+	    }
+	}
+	if /*opt.sig*/ false {
+	    unimplemented!();
+	    //if let Some(verify_with) = opt.sig //XXX: How will this work? We will need to store **either** a private or public key in Send/RecvOpt and dynamically dispatch over it.
+	}
+	Ok(())
+    }
+    fn to_buffer(&self, mut to: impl BufMut)
+    {
+	if let Some(hash) = &self.hash
+	{
+	    to.put_u8(1);
+	    to.put_slice(hash.0.as_ref());
+	    to.put_slice(hash.1.as_ref());
+	} else {
+	    to.put_u8(0);
+	    to.put_bytes(0, sha256::SIZE + H_SALT_SIZE);
+	}
+	if let Some(sig) = &self.sig
+	{
+	    to.put_u8(1);
+	    to.put_slice(sig.as_ref());
+	} else {
+	    to.put_u8(0);
+	    to.put_bytes(0, cryptohelpers::consts::RSA_SIG_SIZE);
+	}
+    }
+    fn from_buffer(mut from: impl Buf) -> Self
+    {
+	let hash = if from.get_u8() == 1 {
+	    let mut hash = sha256::Sha256Hash::default();
+	    let mut salt = [0u8; H_SALT_SIZE];
+	    from.copy_to_slice(hash.as_mut());
+	    from.copy_to_slice(&mut salt[..]);
+	    Some((hash,salt))
+	} else {
+	    from.advance(sha256::SIZE + H_SALT_SIZE);
+	    None
+	};
+	let sig = if from.get_u8() == 1 {
+	    let mut sig = rsa::Signature::default();
+	    from.copy_to_slice(sig.as_mut());
+	    Some(sig)
+	} else {
+	    from.advance(sha256::SIZE);
+	    None
+	};
+	Self {
+	    hash, sig
+	}
+    }
+    #[inline] fn to_array(&self) -> [u8; Self::SIZE]
+    {
+	let mut ar = [0u8; Self::SIZE];
+	self.to_buffer(&mut &mut ar[..]);
+	ar
+    }
+    #[inline] fn from_array(ar: [u8; Self::SIZE]) -> Self
+    {
+	Self::from_buffer(&ar[..])
+    }
+}
+
+const INFO_ASSERT_VALID: u8 = b'\t';
+const INFO_WITH_HEADER: u8 = b' ';
+const INFO_NO_HEADER: u8 = b'\n';
+
+/// If passing an externally generated message to be deserialised here, it must be prefixed with this regardless of its format.
+///
+/// Operations that generate/require a message header will not work on these messages and if they are needed must be handled elsewhere by the user. (Hash and signature validation)
+pub const BARE_MESSAGE_PREFIX: [u8; 2] = [INFO_ASSERT_VALID, INFO_NO_HEADER];
+
+pub(super) async fn de_singleton_inner<T: DeserializeOwned, B, F>(buf: F, from: &[u8], how: &RecvOpt) -> Result<T, TransformErrorKind>
 where B: AsRef<[u8]> + AsyncWrite + Unpin + Default,
       F: FnOnce(&[u8]) -> B
 {
+
+    // Read header
+    let mut header = FormatHeader::empty_array();
+    if from.len() < 2 || from[0] != INFO_ASSERT_VALID {
+	return Err(TransformErrorKind::InvalidHeader(HeaderValidationError::Malformed));
+    }
+    let (inf, mut from) = {
+	(&from[..2], &from[2..])
+    };
+    from = {
+	if inf[1] == INFO_WITH_HEADER {
+	    if from.len() < FormatHeader::SIZE {
+		return Err(TransformErrorKind::InvalidHeader(HeaderValidationError::Malformed));
+	    }
+	    let hf = &from[..FormatHeader::SIZE];
+	    header.copy_from_slice(hf);
+	    &from[FormatHeader::SIZE..]
+	} else {
+	    &from[..]
+	}
+    };
     // Decompressor
     // The output is written to this (through writer)
     let mut is_spec = false; // This is set later. The value will sometimes differ from `how.is_spec()` depending on combinations of options.
@@ -180,6 +440,9 @@ where B: AsRef<[u8]> + AsyncWrite + Unpin + Default,
 	}
     };
     // Deserialise
+    
+    FormatHeader::from_array(header).validate(from, how)?;
+    
     let v = match how.format {
 	SerialFormat::Text => serde_json::from_slice(&from[..])?,
 	SerialFormat::Binary => serde_cbor::from_slice(&from[..])?,
@@ -188,14 +451,20 @@ where B: AsRef<[u8]> + AsyncWrite + Unpin + Default,
     Ok(v)
 }
 
-async fn ser_singleton_inner<T: Serialize, V: AsyncWrite + Unpin, F>(to: F, value: &T, how: impl AsRef<SendOpt>) -> Result<(V, usize), TransformErrorKind>
+pub(super) async fn ser_singleton_inner<T: Serialize, V: AsyncWrite + Unpin, F>(to: F, value: &T, how: impl AsRef<SendOpt>) -> Result<(V, usize), TransformErrorKind>
-where F: FnOnce(&Vec<u8>) -> V
+where F: FnOnce(&Vec<u8>) -> V,
 {
     let how = how.as_ref();
     let ser = match how.format {
 	SerialFormat::Text => serde_json::to_vec(value)?,
 	SerialFormat::Binary => serde_cbor::to_vec(value)?,
     };
+    let header = if how.needs_header() {
+	let header = FormatHeader::generate(&ser, how);
+	header.to_array()
+    } else {
+	FormatHeader::empty_array()
+    };
     let mut a;
     let mut b;
     let reader: &mut (dyn AsyncRead + Unpin) = 
@@ -212,6 +481,12 @@ where F: FnOnce(&Vec<u8>) -> V
 	    &mut b
 	};
     let mut ser = to(&ser);
+    if how.needs_header() {
+	ser.write_all(&[INFO_ASSERT_VALID, INFO_WITH_HEADER]).await?;
+	ser.write_all(&header[..]).await?;
+    } else {
+	ser.write_all(&[INFO_ASSERT_VALID, INFO_NO_HEADER]).await?;
+    }
     let w= if let Some(enc) = &how.encrypt {
 	let n = match enc {
 	    EncryptionKind::Chacha20((k, iv)) => {
@@ -296,6 +571,8 @@ pub enum TransformErrorKind
     /// Misc. IO
     //TODO: Disambiguate when this happens into the two above cases.
     IO(io::Error),
+    /// The object header was invalid.
+    InvalidHeader(HeaderValidationError),
 }
 
 /// An error when sending / serialising an object.
@@ -323,6 +600,7 @@ impl error::Error for RecvError
 	Some(match &self.0.0
 	     {
 		 TransformErrorKind::IO(io) => io,
+		 TransformErrorKind::InvalidHeader(ih) => ih,
 		 _ => return None,
 	     })
     }
@@ -337,6 +615,7 @@ impl fmt::Display for RecvError
 	    TransformErrorKind::Compress => write!(f, "failed to decompress data"),
 	    TransformErrorKind::Encrypt => write!(f, "failed to decrypt data"),
 	    TransformErrorKind::IO(_) => write!(f, "i/o failure"),
+	    TransformErrorKind::InvalidHeader(_) => write!(f, "invalid header"),
 	}
     }
 }
@@ -352,6 +631,7 @@ impl error::Error for SendError
 	Some(match &self.0.0
 	     {
 		 TransformErrorKind::IO(io) => io,
+		 TransformErrorKind::InvalidHeader(ih) => ih,
 		 _ => return None,
 	     })
     }
@@ -367,6 +647,7 @@ impl fmt::Display for SendError
 	    TransformErrorKind::Compress => write!(f, "failed to compress data"),
 	    TransformErrorKind::Encrypt => write!(f, "failed to encrypt data"),
 	    TransformErrorKind::IO(_) => write!(f, "i/o failure"),
+	    TransformErrorKind::InvalidHeader(_) => write!(f, "invalid header"),
 	}
     }
 }
@@ -379,6 +660,15 @@ impl From<io::Error> for TransformErrorKind
     }
 }
 
+impl From<HeaderValidationError> for TransformErrorKind
+{
+    fn from(from: HeaderValidationError) -> Self
+    {
+	Self::InvalidHeader(from)
+    }
+}
+
+
 
 impl From<serde_cbor::Error> for TransformErrorKind
 {
@@ -407,7 +697,7 @@ mod test
 	let obj = String::from("Hello world");
 
 	let var = ser_singleton(&obj, &how).await?;
-	eprintln!("Ser: {}", var.hex());
+	eprintln!("Ser ({} bytes): {}", var.len(), var.hex());
 	let des: String = de_singleton(&var, &how).await?;
 	eprintln!("De: {:?}", des);
 	assert_eq!(obj, des);
@@ -434,6 +724,7 @@ mod test
     {
 	ser_de_with(SendOpt {
 	    encrypt: Some(EncryptionKind::Chacha20(cha::keygen())),
+	    //hash: true,
 	    ..Default::default()
 	}).await
     }

src/ext.rs

@@ -152,10 +152,8 @@ impl<I: Iterator<Item = u8> + Clone> fmt::Display for HexStringIter<I>
     }
 }
 
-/*
 #[macro_export] macro_rules! prog1 {
     ($first:expr, $($rest:expr);+ $(;)?) => {
 	($first, $( $rest ),+).0
     }
 }
-*/

src/main.rs

@@ -2,6 +2,7 @@
 #![allow(dead_code)]
 
 #[macro_use] extern crate log;
+#[macro_use] extern crate ad_hoc_iter;
 #[macro_use] extern crate lazy_static;
 #[macro_use] extern crate serde;