generate hashes

TODO

@@ -0,0 +1 @@
+Validate DATA_HASHES[data_index] in c++ code

generator-native/Cargo.lock

@@ -1,5 +1,89 @@
 # This file is automatically @generated by Cargo.
 # It is not intended for manual editing.
+[[package]]
+name = "block-buffer"
+version = "0.7.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b"
+dependencies = [
+ "block-padding",
+ "byte-tools",
+ "byteorder",
+ "generic-array",
+]
+
+[[package]]
+name = "block-padding"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fa79dedbb091f449f1f39e53edf88d5dbe95f895dae6135a8d7b881fb5af73f5"
+dependencies = [
+ "byte-tools",
+]
+
+[[package]]
+name = "byte-tools"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7"
+
+[[package]]
+name = "byteorder"
+version = "1.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de"
+
+[[package]]
+name = "digest"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5"
+dependencies = [
+ "generic-array",
+]
+
+[[package]]
+name = "fake-simd"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed"
+
 [[package]]
 name = "generator-native"
 version = "0.2.0"
+dependencies = [
+ "sha2",
+]
+
+[[package]]
+name = "generic-array"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c68f0274ae0e023facc3c97b2e00f076be70e254bc851d972503b328db79b2ec"
+dependencies = [
+ "typenum",
+]
+
+[[package]]
+name = "opaque-debug"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c"
+
+[[package]]
+name = "sha2"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a256f46ea78a0c0d9ff00077504903ac881a1dafdc20da66545699e7776b3e69"
+dependencies = [
+ "block-buffer",
+ "digest",
+ "fake-simd",
+ "opaque-debug",
+]
+
+[[package]]
+name = "typenum"
+version = "1.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "373c8a200f9e67a0c95e62a4f52fbf80c23b4381c05a17845531982fa99e6b33"

generator-native/Cargo.toml

@@ -7,4 +7,8 @@ edition = "2018"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
+[features]
+hash = ["sha2"]
+
 [dependencies]
+sha2 = { version = "^0.8", optional = true }

generator-native/src/hash/mod.rs

@@ -0,0 +1,95 @@
+use sha2::{Sha256, Digest};
+
+#[cfg(test)]
+mod test
+{
+    use super::*;
+    #[test]
+    fn hash_iter_okay()
+    {
+	let hash_this = vec![vec![0,1,2,3,4],
+			     vec![5,6,7,8,9],
+			     vec![10,11,12,13,14],
+			     vec![15,16,17,18,19]];
+
+	let mut digest = Sha256::new();
+	let mut digest2 = Sha256::new();
+	for byte in hash_this.into_iter().into_hash_iter(&mut digest)
+	{
+	    digest2.input(&byte[..]);
+	}
+
+	assert_eq!(digest.result()[..], digest2.result()[..]);
+    }
+
+    #[test]
+    fn copy_slice_works()
+    {
+	let slice = [0xab, 0xad, 0xca, 0xfe];
+	let mut output = [0u8; 4];
+
+	assert_eq!(copy_slice(&mut output, &slice), 4);
+	assert_eq!(slice[..], output[..]);
+    }
+}
+
+pub struct HashingIter<'a, I, T>
+where I: Iterator<Item=T>,
+      T: AsRef<[u8]>
+{
+    iter: I,
+    digest: &'a mut Sha256,
+}
+
+
+impl<'a, I, T> Iterator for HashingIter<'a, I, T>
+where I: Iterator<Item=T>,
+      T: AsRef<[u8]>
+{
+    type Item = T;
+    fn next(&mut self) -> Option<Self::Item>
+    {
+	match self.iter.next() {
+	    Some(value) => {
+		self.digest.input(value.as_ref());
+		Some(value)
+	    },
+	    None => None,
+	}
+    }
+}
+
+pub trait HashingIterExt: Iterator + Sized
+where <Self as Iterator>::Item: AsRef<[u8]>
+{
+    fn into_hash_iter<'a>(self, hash: &'a mut Sha256) -> HashingIter<'a, Self, <Self as Iterator>::Item>;
+}
+
+impl<I, T> HashingIterExt for I
+where I: Iterator<Item=T>,
+      T: AsRef<[u8]>
+{
+    fn into_hash_iter<'a>(self, hash: &'a mut Sha256) -> HashingIter<'a, I, T>
+    {
+	HashingIter {
+	    iter: self,
+	    digest: hash,
+	}
+    }
+}
+
+pub fn copy_slice<T>(mut dst: impl AsMut<[T]>, src: impl AsRef<[T]>) -> usize
+    where T: Clone
+{
+    let dst = dst.as_mut();
+    let src = src.as_ref();
+
+    let mut i=0;
+    for (d, s) in dst.iter_mut().zip(src.iter())
+    {
+	*d = s.clone();
+	i+=1;
+    }
+
+    i
+}

generator-native/src/main.rs

@@ -1,5 +1,8 @@
 #![allow(dead_code)]
 
+#[cfg(feature = "hash")]
+extern crate sha2;
+
 use std::{
     fs::{
 	OpenOptions,
@@ -20,6 +23,9 @@ mod arg;
 
 use opt::Opt;
 
+#[cfg(feature = "hash")]
+mod hash;
+
 macro_rules! flush {
     () => {
 	std::io::stdout().flush().ok().expect("fatal: could not flush stdout")
@@ -56,9 +62,36 @@ fn usage() -> ! {
     std::process::exit(1)
 }
 
-fn write_file<From, To>(from: From, to: &mut To) -> io::Result<usize>
+#[cfg(feature = "hash")]
+type Sha256Hash = [u8; 32];
+#[cfg(not(feature = "hash"))]
+type Sha256Hash = ();
+
+fn write_file<From, To>(from: From, to: &mut To) -> io::Result<(Sha256Hash, usize)>
 where From: Read,
       To: Write + ?Sized
+{
+    #[cfg(feature = "hash")]
+    {
+	use sha2::{Sha256,Digest};
+	use hash::copy_slice;
+	let mut hash_output = [0u8; 32];
+	let mut count =0;
+	let mut digest = Sha256::new();
+	for buf in from.into_iter(8)
+	    .map(|byte| (digest.input(&[byte]), format!("0x{:02x},", byte)).1)
+	    .group_at(8)
+	    .map(|bytes| (count += bytes.len(), bytes).1)
+	    .map(|strs| format!("\t{}", strs.join(" ")))
+	{
+	    writeln!(to, "{}", buf)?;
+	}
+
+	copy_slice(&mut hash_output[..], &digest.result()[..]);
+
+	Ok((hash_output, count))
+    }
+    #[cfg(not(feature = "hash"))]
     {
 	let mut count =0;
 	for buf in from.into_iter(8)
@@ -70,7 +103,8 @@ where From: Read,
 	    writeln!(to, "{}", buf)?;
 	}
 
-    Ok(count)
+	Ok(((), count))
+    }
 }
 
 fn attempt_get_name<'a, P>(path: &'a P) -> Result<&'a str, &'static str>
@@ -89,6 +123,22 @@ where P: AsRef<Path> + ?Sized
     }
 }
 
+#[allow(unused_variables)]
+fn hash_str(hash: &Sha256Hash) -> String
+{
+    #[cfg(not(feature = "hash"))]
+    return "0,".repeat(32);
+    #[cfg(feature = "hash")]
+    {
+	let mut output = String::with_capacity(64);
+	for byte in hash.iter().map(|byte| format!("0x{:02x}, ", *byte))
+	{
+	    output.push_str(&byte);
+	}
+	output
+    }
+}
+
 fn main() -> Result<(), Box<dyn Error>>
 {
     match arg::parse()? {
@@ -126,6 +176,8 @@ fn main() -> Result<(), Box<dyn Error>>
 	    }
 
 	    let mut sizes = Vec::with_capacity(files.len());
+	    let mut hashes = Vec::with_capacity(files.len());
+	    
 	    writeln!(fp, "constexpr const unsigned char DATA[] = {{")?;
 
 	    for file in files.iter() {
@@ -136,8 +188,10 @@ fn main() -> Result<(), Box<dyn Error>>
 		    .open(file)?;
 
 		sizes.push(match write_file(file, &mut fp) {
-		    Ok(size) => {
+		    Ok((hash, size)) => {
+			hashes.push(hash);
 			println!(" OK");
+			
 			size
 		    },
 		    Err(error) => {
@@ -156,6 +210,17 @@ fn main() -> Result<(), Box<dyn Error>>
 	    }
 	    writeln!(fp, "\n}};")?;
 
+	    #[cfg(feature="hash")]
+	    {
+		println!("Adding hashes...");
+		writeln!(fp, "#define DATA_HASHED")?;
+	    }
+	    writeln!(fp, "constexpr const unsigned char DATA_HASHES[] = {{")?;
+	    for hash in hashes.into_iter() {
+		writeln!(fp, "\t{}", hash_str(&hash))?;
+	    }
+	    writeln!(fp, "}};")?;
+	    
 	    println!("Adding names...");
 
 	    writeln!(fp, "constexpr const char* const DATA_NAMES[DATA_COUNT] = {{")?;