text format oke; begin passworded key headers (text)

Cargo.lock

@@ -206,9 +206,9 @@ dependencies = [
 
 [[package]]
 name = "cryptohelpers"
-version = "1.1.2"
+version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "825b8339215ceae0288ed0384ebe63fe717f5b0c77b102ff3c9f59aefeed9106"
+checksum = "1ef6d4c394fce0d9b42b9bd7242df5201b6c4996d1b9f63ac75c83ba9e6b05ce"
 dependencies = [
  "crc",
  "getrandom",
@@ -772,6 +772,7 @@ dependencies = [
  "cryptohelpers",
  "futures",
  "hex",
+ "hex-literal",
  "lazy_static",
  "libc",
  "pin-project",

Cargo.toml

@@ -15,7 +15,7 @@ local-time = []
 [dependencies]
 color-eyre = "0.5.3"
 lazy_static = "1.4.0"
-crypto = {package= "cryptohelpers", version = "1.1.2", features=["full", "async", "serialise"]}
+crypto = {package= "cryptohelpers", version = "1.2", features=["full", "async", "serialise"]}
 cfg-if = "0.1.10"
 tokio = {version = "0.2", features=["full"]}
 serde = {version ="1.0.116", features=["derive"]}
@@ -34,6 +34,7 @@ pin-project = "0.4.23"
 base64 = "0.12.3"
 hex = "0.4.2"
 async-trait = "0.1.40"
+hex-literal = "0.3.1"
 #serde_json = "1.0.57" # serde not suitable for our text formatting :/ maybe just use `cbor` -> base64 with text header? would be a PEM-like format. sounds good imo
 
 [build-dependencies]

src/ext.rs

@@ -1,6 +1,7 @@
 //! Extensions
 use std::{
     fmt,
+    error,
     pin::Pin,
     task::{Poll,Context,},
 };
@@ -234,19 +235,6 @@ impl<T: AsRef<[u8]>> HexStringExt for T
 	HexView(&self)
     }
 }
-
-#[cfg(test)]
-mod tests
-{
-    use super::*;
-    fn format()
-    {
-	let bytes = b"hello world one two three \x142!";
-
-	panic!("\n{}\n", bytes.fmt_view());
-    }
-}
-
 #[pin_project]
 pub struct ReadAllBytes<'a, T: AsyncRead+Unpin+?Sized>(#[pin] &'a mut T, Option<usize>);
 
@@ -287,3 +275,162 @@ pub trait ReadAllBytesExt: AsyncRead+Unpin
 }
 
 impl<T: AsyncRead+Unpin+?Sized> ReadAllBytesExt for T{}
+
+pub trait FromHexExt
+{
+    fn repl_with_hex<U: AsRef<[u8]>>(&mut self, input: U) -> Result<(), HexDecodeError>;
+}
+
+impl<T: AsMut<[u8]>+?Sized> FromHexExt for T
+{
+    fn repl_with_hex<U: AsRef<[u8]>>(&mut self, input: U) -> Result<(), HexDecodeError> {
+	let out = self.as_mut();
+	
+	#[inline] fn val(c: u8, idx: usize) -> Result<u8, HexDecodeError> {
+	    match c {
+		b'A'..=b'F' => Ok(c - b'A' + 10),
+		b'a'..=b'f' => Ok(c - b'a' + 10),
+		b'0'..=b'9' => Ok(c - b'0'),
+		_ => Err(HexDecodeError{
+		    chr: c as char,
+		    idx,
+		}),
+	    }
+	}
+	
+	for (i, (byte, digits)) in (0..).zip(out.iter_mut().zip(input.as_ref().chunks_exact(2)))
+	{
+	    *byte = val(digits[0], 2*i)? << 4 | val(digits[1], 2 * i + 1)?;
+	}
+
+	Ok(())
+    }
+}
+
+#[derive(Debug)]
+pub struct HexDecodeError {
+    idx: usize,
+    chr: char,
+}
+impl error::Error for HexDecodeError{}
+impl fmt::Display for HexDecodeError
+{
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result
+    {
+	write!(f, "Invalid hex at index {} (character was {:?})", self.idx, self.chr)
+    }
+}
+
+
+#[cfg(test)]
+mod tests
+{
+    use super::*;
+    fn format()
+    {
+	let bytes = b"hello world one two three \x142!";
+
+	panic!("\n{}\n", bytes.fmt_view());
+    }
+
+    #[test]
+    fn hex()
+    {
+	const INPUT_HEX: [u8; 32] = hex_literal::hex!("d0a2404173bac722b29282652f2c457b573261e3c8701b908bb0bd3ada3d7f2d");
+	const INPUT_STR: &str = "d0a2404173bac722b29282652f2c457b573261e3c8701b908bb0bd3ada3d7f2d";
+	let mut output = [0u8; 32];
+	output.repl_with_hex(INPUT_STR).expect("Failed!");
+
+	assert_eq!(&INPUT_HEX[..], &output[..]);
+    }
+
+    #[cfg(nightly)]
+    mod benchmarks
+    {
+	use super::*;
+	use test::{Bencher, black_box};
+
+	#[bench]
+	fn hex_via_val(b: &mut Bencher)
+	{
+	    fn repl_with_hex<U: AsRef<[u8]>>(out: &mut [u8], input: U) -> Result<(), HexDecodeError> {
+		
+		#[inline] fn val(c: u8, idx: usize) -> Result<u8, HexDecodeError> {
+		    match c {
+			b'A'..=b'F' => Ok(c - b'A' + 10),
+			b'a'..=b'f' => Ok(c - b'a' + 10),
+			b'0'..=b'9' => Ok(c - b'0'),
+			_ => Err(HexDecodeError{
+			    chr: c as char,
+			    idx,
+			}),
+		    }
+		}
+		
+		for (i, (byte, digits)) in (0..).zip(out.iter_mut().zip(input.as_ref().chunks_exact(2)))
+		{
+		    *byte = val(digits[0], 2*i)? << 4 | val(digits[1], 2 * i + 1)?;
+		}
+
+		Ok(())
+	    }
+	    
+	    const INPUT_HEX: [u8; 32] = hex_literal::hex!("d0a2404173bac722b29282652f2c457b573261e3c8701b908bb0bd3ada3d7f2d");
+	    const INPUT_STR: &str = "d0a2404173bac722b29282652f2c457b573261e3c8701b908bb0bd3ada3d7f2d";
+	    let mut output = [0u8; 32];
+
+	    b.iter(|| {
+		black_box(repl_with_hex(&mut output[..], INPUT_STR).unwrap());
+	    });
+	    
+	    assert_eq!(&INPUT_HEX[..], &output[..]);
+	}
+
+	#[bench]
+	fn hex_via_lazy(b: &mut Bencher)
+	{
+	    fn repl_with_hex<U: AsRef<[u8]>>(out: &mut [u8], input: U) -> Result<(), HexDecodeError> {
+
+		use smallmap::Map;
+		lazy_static::lazy_static! {
+		    static ref MAP: Map<u8, u8> = {
+			let mut map = Map::new();
+			for c in 0..=255u8
+			{
+			    map.insert(c, match c {
+				b'A'..=b'F' => c - b'A' + 10,
+				b'a'..=b'f' => c - b'a' + 10,
+				b'0'..=b'9' => c - b'0',
+				_ => continue,
+			    });
+			}
+			map
+		    };
+		}
+		
+		#[inline(always)] fn val(c: u8, idx: usize) -> Result<u8, HexDecodeError> {
+		    MAP.get(&c).copied()
+			.ok_or_else(|| HexDecodeError{idx, chr: c as char})
+		}
+
+		
+		for (i, (byte, digits)) in (0..).zip(out.iter_mut().zip(input.as_ref().chunks_exact(2)))
+		{
+		    *byte = val(digits[0], 2*i)? << 4 | val(digits[1], 2 * i + 1)?;
+		}
+
+		Ok(())
+	    }
+	    
+	    const INPUT_HEX: [u8; 32] = hex_literal::hex!("d0a2404173bac722b29282652f2c457b573261e3c8701b908bb0bd3ada3d7f2d");
+	    const INPUT_STR: &str = "d0a2404173bac722b29282652f2c457b573261e3c8701b908bb0bd3ada3d7f2d";
+	    let mut output = [0u8; 32];
+
+	    b.iter(|| {
+		black_box(repl_with_hex(&mut output[..], INPUT_STR).unwrap());
+	    });
+	    
+	    assert_eq!(&INPUT_HEX[..], &output[..]);
+	}
+    }
+}

src/format/key/mod.rs

@@ -9,6 +9,13 @@ use std::{
     error,
     convert::{TryFrom, TryInto,},
 };
+use crypto::{
+    password::{
+	Salt,
+	Password,
+	SALTSIZE
+    },
+};
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Ord,PartialOrd, Hash, Serialize, Deserialize)]
 #[repr(u8)]
@@ -80,6 +87,32 @@ impl Header for KeyHeader
     }
 }
 
+#[instrument]
+fn decode_salt(p: &str) -> Result<Salt, eyre::Report>
+{
+    trace!("Decoding salt");
+    let mut salt = Salt::none();
+    salt.repl_with_hex(p).wrap_err_with(|| eyre::eyre!("Failed to construct password salt from bytes"))?;
+    Ok(salt)
+}
+
+#[instrument]
+#[inline] fn encode_salt_to_string(salt: &Salt) -> String
+{
+    salt.to_hex_string()
+}
+
+#[instrument]
+fn encode_salt(salt: &Salt) -> Result<[u8; SALTSIZE*2], eyre::Report>
+{
+    let mut output = [0u8; SALTSIZE*2];
+
+    hex::encode_to_slice(salt.as_ref(), &mut output[..])
+        .wrap_err_with(|| eyre::eyre!("Failed to encode salt to {} hex char bytes", SALTSIZE *2))
+        .with_section(|| salt.to_hex_string().header("Salt was"))?;
+    Ok(output)
+}
+
 impl KeyHeader
 {
     /// Create a new key header from these values
@@ -109,7 +142,12 @@ impl KeyHeader
 	    .wrap_err_with(|| eyre::eyre!("Failed to serialise header to text"))
 	    .with_section(|| format!("{:?}", self).header("Header was"))?;
 
-	let mut written=0;
+	
+	let mut written={
+	    out.write_all(TEXT_NOPASS).await?;
+	    out.write_u8(b'\n').await?;
+	    TEXT_NOPASS.len() + 1
+	};
 	for bytes in text.as_bytes().chunks(16) {
 	    out.write_all(bytes).await?;
 	    out.write_u8(b'\n').await?;
@@ -123,19 +161,108 @@ impl KeyHeader
     pub async fn read_text<T: AsyncBufRead+Unpin+?Sized>(input: &mut T) -> Result<Self, eyre::Report>
     {
 	let (mut tx, mut rx) = mpsc::channel(1);
+
+	#[derive(Debug)]
+	enum SendError
+	{
+	    SendError,
+	    IO(std::io::Error),
+	}
+
+	impl std::error::Error for SendError
+	{
+	    fn source(&self) -> Option<&(dyn std::error::Error + 'static)>
+	    {
+		match &self {
+		    Self::IO(io) => Some(io),
+		    _ => None,
+		}
+	    }
+	}
+	impl fmt::Display for SendError
+	{
+	    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result
+	    {
+		write!(f, "line reading failed: ")?;
+		match self {
+		    Self::SendError => write!(f, "channel closed"),
+		    Self::IO(_) => write!(f, "io error"),
+		}
+	    }
+	}
+
+	impl From<std::io::Error> for SendError
+	{
+	    #[inline] fn from(from: std::io::Error) -> Self
+	    {
+		Self::IO(from)
+	    }
+	}
+
+	impl<T> From<tokio::sync::mpsc::error::SendError<T>> for SendError
+	{
+	    #[inline] fn from(_: tokio::sync::mpsc::error::SendError<T>) -> Self
+	    {
+		Self::SendError
+	    }
+	}
+
+	
 	let line_sender = async move { //this is actually kinda overkill for this...
 	    let mut buffer = String::new();
 	    while input.read_line(&mut buffer).await? != 0 {
 		tx.send(buffer.clone()).await?;
 		buffer.clear();
 	    }
-	    Ok::<(), eyre::Report>(())
+	    Ok::<(), SendError>(())
 	};
-	let line_reader = async {
+	let line_reader = async move {
+	    macro_rules! take_one {
+		($msg:literal $($tt:tt)*) => {
+		    loop {
+			if let Some(mut line) = rx.recv().await {
+			    if line.trim().len() == 0 {
+				continue;
+			    }
+			    if {
+				let bytes = line.as_bytes();
+				
+				if bytes.len() > 0 && bytes[bytes.len()-1] == b'\n' {
+				    true
+				} else {
+				    false
+				}
+			    } {
+				line.truncate(line.len()-1);
+			    }
+			    break line;
+			} else {
+			    return Err(eyre::eyre!(format!($msg $($tt)*))).wrap_err(eyre::eyre!("Failed to deserialise string"));
+			}
+		    }
+		}
+	    }
+
+
+	    trace!("Reading password");
+	    let password = {
+		let pass_part = take_one!("Failed to read password part");
+		trace!("Read password {}", pass_part);
+		if pass_part.as_bytes() == TEXT_NOPASS {
+		    None 
+		} else {
+		    Some(decode_salt(&pass_part[..])
+			 .with_section(move || pass_part.header("Password string part was"))?)
+		}
+	    };
+	    trace!("Decoded hex");
 	    let mut enc = String::new();
 	    let mut had_delim =false;
 	    while let Some(line) = rx.recv().await {
 		let line = line.trim();
+		if line.len() == 0 {
+		    continue;
+		}
 		if line == "---" {
 		    had_delim=true;
 		    break;
@@ -146,17 +273,32 @@ impl KeyHeader
 	    if !had_delim {
 		warn!("Buffer contained no end-of-entry delimiter");
 	    }
+	    
 	    //let  = take_one!("Expected header line");
 
-	    Ok::<Self, eyre::Report>(serialise::from_text(&enc[..])
+	    if let Some(salt) = password {
-				     .wrap_err_with(|| eyre::eyre!("Failed to deserialise string"))
+		todo!()
-				     .with_section(|| enc.header("Read string was"))?)
+	    } else {
+		Ok::<Self, eyre::Report>(serialise::from_text(&enc[..])
+					 .wrap_err_with(|| eyre::eyre!("Failed to deserialise string"))
+					 .with_section(|| enc.header("Read string was"))?)
+	    }
 	};
 	tokio::pin!(line_sender);
 	tokio::pin!(line_reader);
 	let (sres, rres) = tokio::join!(line_sender, line_reader);
-	sres?;
+	match sres {
-	Ok(rres?)
+	    Err(x @ SendError::IO(_)) => Err(x).with_note(|| "In line reader"),
+	    Err(s @ SendError::SendError) => {
+		rres
+		    .with_error(move || s)?;
+		warn!("Unreachable code entered");
+		Err(SendError::SendError)
+		    .with_note(|| "In line reader")
+		    .with_warning(|| "`sres` failed with `SendError` but `rres` completed successfully. This should not happen")
+	    },
+	    _ => Ok(rres?),
+	}
     }
     /// Write this key header as bytes to this stream
     #[instrument(err, skip(out))]

src/format/mod.rs

@@ -347,5 +347,7 @@ impl<H: Header+?Sized> serialise::TextSerialiseable for SuperHeader<H>
     }
 }
 
-mod key;
+/// Nopassword text constant
+const TEXT_NOPASS: &[u8; 16] = b"0000000000000000";
+pub mod key;
 const CHECK_KEY: u16 = 0x0001;

src/main.rs

@@ -2,12 +2,15 @@
 #![cfg_attr(nightly, feature(const_fn))] 
 #![cfg_attr(nightly, feature(const_fn_transmute))] 
 #![cfg_attr(nightly, feature(never_type))]
+#![cfg_attr(nightly, feature(test))] 
+
 
 #![allow(dead_code)]
 #![allow(unused_macros)]
 
 #[macro_use] extern crate tracing;
 #[macro_use] extern crate pin_project;
+#[cfg(nightly)] extern crate test;
 //#[macro_use] extern crate async_trait;
 use std::{
     convert::{TryFrom, TryInto},
@@ -137,6 +140,33 @@ fn install_tracing() {
         .with(ErrorLayer::default())
         .init();
 }
+async fn fuck() -> eyre::Result<()>
+{
+    use format::*;
+    use format::key::*;
+    let header = KeyHeader::new_now(KeyHeaderKind::Aes, Default::default(), Default::default());
+    let mut ser = Vec::new();
+    let superheader = SuperHeader::<KeyHeader>::new_for(&header);
+    println!("Writing: {:?} + {:?}", superheader, header);
+    let written = superheader.write_text(&mut ser).await?;
+    ser.extend(header.into_memory(serialise::Mode::Text)?); //header.write_text(&mut ser).await?;
+    println!("Wrote {} bytes", written);
+    println!("{}\n", ser.fmt_view());
+
+    let mut read = &ser[..];
+    let (reads, readn) = SuperHeader::from_memory(&mut read, serialise::Mode::Text)?; // SuperHeader::read_text(read).await?;
+    let mut read = &read[readn..];
+    println!("Read super: {:?}", reads);
+    let readheader = KeyHeader::read_text(&mut read).await?;
+    println!("Read real: {:?}", readheader);
+
+    reads.verify_for(&header)?;
+    reads.verify_for(&readheader)?;
+    assert_eq!(readheader, header);
+    assert_eq!(reads, superheader);
+
+    Ok(())
+}
 
 #[instrument]
 async fn work(op: config::Operation) -> Result<(), eyre::Report>
@@ -161,6 +191,8 @@ async fn main() -> Result<(), eyre::Report> {
     install_tracing();
     color_eyre::install()?;
 
+    fuck().await?;
+    return Ok(());
     trace!("Parsing args");
     let args = args::parse_args().await?;