more serialise shit

master
Avril 4 years ago
parent d708f41fb8
commit 38ffad2120
Signed by: flanchan
GPG Key ID: 284488987C31F630

12
Cargo.lock generated

@ -39,6 +39,17 @@ version = "0.4.7"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4d25d88fd6b8041580a654f9d0c581a047baee2b3efee13275f2fc392fc75034" checksum = "4d25d88fd6b8041580a654f9d0c581a047baee2b3efee13275f2fc392fc75034"
[[package]]
name = "async-trait"
version = "0.1.40"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "687c230d85c0a52504709705fc8a53e4a692b83a2184f03dae73e38e1e93a783"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]] [[package]]
name = "atty" name = "atty"
version = "0.2.14" version = "0.2.14"
@ -752,6 +763,7 @@ dependencies = [
name = "rae" name = "rae"
version = "0.1.0" version = "0.1.0"
dependencies = [ dependencies = [
"async-trait",
"base64", "base64",
"base65536", "base65536",
"cfg-if", "cfg-if",

@ -33,6 +33,7 @@ chrono = "0.4.15"
pin-project = "0.4.23" pin-project = "0.4.23"
base64 = "0.12.3" base64 = "0.12.3"
hex = "0.4.2" hex = "0.4.2"
async-trait = "0.1.40"
#serde_json = "1.0.57" # serde not suitable for our text formatting :/ maybe just use `cbor` -> base64 with text header? would be a PEM-like format. sounds good imo #serde_json = "1.0.57" # serde not suitable for our text formatting :/ maybe just use `cbor` -> base64 with text header? would be a PEM-like format. sounds good imo
[build-dependencies] [build-dependencies]

@ -216,6 +216,33 @@ impl KeyHeader
} }
} }
impl serialise::BinarySerialisable for KeyHeader
{
#[inline(always)] fn serialise_bytes<'a, 'b: 'a, T: AsyncWrite+Unpin+?Sized>(&'a self, out: &'b mut T) -> LocalBoxFuture<'a, Result<usize, eyre::Report>>
{
self.write_bytes(out).boxed_local()
}
#[inline(always)] fn deserialise_bytes<'b , T: AsyncRead+Unpin+?Sized>(input: &'b mut T) -> LocalBoxFuture<'b, Result<Self, eyre::Report>>
where Self: 'b
{
Self::read_bytes(input).boxed_local()
}
}
impl serialise::TextSerialiseable for KeyHeader
{
#[inline(always)] fn serialise_text<'a, 'b: 'a, T: AsyncWrite+Unpin+?Sized>(&'a self, out: &'b mut T) -> LocalBoxFuture<'a, Result<usize, eyre::Report>>
{
self.write_text(out).boxed_local()
}
#[inline(always)] fn deserialise_text<'a, T: AsyncBufRead+Unpin+?Sized>(input: &'a mut T) -> LocalBoxFuture<'a, Result<Self, eyre::Report>>
where Self: 'a
{
Self::read_text(input).boxed_local()
}
}
#[cfg(test)] #[cfg(test)]
mod tests mod tests
{ {
@ -247,15 +274,15 @@ mod tests
let mut ser = Vec::new(); let mut ser = Vec::new();
let superheader = SuperHeader::<KeyHeader>::new_for(&header); let superheader = SuperHeader::<KeyHeader>::new_for(&header);
println!("Writing: {:?} + {:?}", superheader, header); println!("Writing: {:?} + {:?}", superheader, header);
let written = superheader.write_bytes(&mut ser).await? + let written = superheader.serialise_bytes(&mut ser).await? +
header.write_bytes(&mut ser).await?; header.write_bytes(&mut ser).await?;
println!("Wrote {} bytes", written); println!("Wrote {} bytes", written);
println!("{}\n", ser.fmt_view()); println!("{}\n", ser.fmt_view());
let mut read = &ser[..]; let mut read = &ser[..];
let reads = SuperHeader::read_bytes(&mut read).await?; let reads = SuperHeader::deserialise_bytes(&mut read).await?;
println!("Read super: {:?}", reads); println!("Read super: {:?}", reads);
let readheader = KeyHeader::read_bytes(&mut read).await?; let readheader = KeyHeader::deserialise_bytes(&mut read).await?;
println!("Read real: {:?}", readheader); println!("Read real: {:?}", readheader);
reads.verify_for(&header)?; reads.verify_for(&header)?;
@ -307,16 +334,16 @@ mod tests
let mut ser = Vec::new(); let mut ser = Vec::new();
let superheader = SuperHeader::<KeyHeader>::new_for(&header); let superheader = SuperHeader::<KeyHeader>::new_for(&header);
println!("Writing: {:?} + {:?}", superheader, header); println!("Writing: {:?} + {:?}", superheader, header);
let written = superheader.write_text(&mut ser).await? + let written = superheader.write_text(&mut ser).await?;
header.write_text(&mut ser).await?; ser.extend(header.into_memory(serialise::Mode::Text)?); //header.write_text(&mut ser).await?;
println!("Wrote {} bytes", written); println!("Wrote {} bytes", written);
println!("{}\n", ser.fmt_view()); println!("{}\n", ser.fmt_view());
let mut read = &ser[..]; let mut read = &ser[..];
let read = &mut read; let (reads, readn) = SuperHeader::from_memory(&mut read, serialise::Mode::Text)?; // SuperHeader::read_text(read).await?;
let reads = SuperHeader::read_text(read).await?; let mut read = &read[readn..];
println!("Read super: {:?}", reads); println!("Read super: {:?}", reads);
let readheader = KeyHeader::read_text(read).await?; let readheader = KeyHeader::read_text(&mut read).await?;
println!("Read real: {:?}", readheader); println!("Read real: {:?}", readheader);
reads.verify_for(&header)?; reads.verify_for(&header)?;

@ -321,5 +321,31 @@ impl<H: Header+?Sized> fmt::Display for VerificationError<H>
} }
} }
impl<H: Header+?Sized> serialise::BinarySerialisable for SuperHeader<H>
{
#[inline(always)] fn serialise_bytes<'a, 'b: 'a, T: AsyncWrite+Unpin+?Sized>(&'a self, out: &'b mut T) -> LocalBoxFuture<'a, Result<usize, eyre::Report>>
{
self.write_bytes(out).boxed_local()
}
#[inline(always)] fn deserialise_bytes<'b , T: AsyncRead+Unpin+?Sized>(input: &'b mut T) -> LocalBoxFuture<'b, Result<Self, eyre::Report>>
where Self: 'b
{
Self::read_bytes(input).boxed_local()
}
}
impl<H: Header+?Sized> serialise::TextSerialiseable for SuperHeader<H>
{
#[inline(always)] fn serialise_text<'a, 'b: 'a, T: AsyncWrite+Unpin+?Sized>(&'a self, out: &'b mut T) -> LocalBoxFuture<'a, Result<usize, eyre::Report>>
{
self.write_text(out).boxed_local()
}
#[inline(always)] fn deserialise_text<'a, T: AsyncBufRead+Unpin+?Sized>(input: &'a mut T) -> LocalBoxFuture<'a, Result<Self, eyre::Report>>
where Self: 'a
{
Self::read_text(input).boxed_local()
}
}
mod key; mod key;
const CHECK_KEY: u16 = 0x0001; const CHECK_KEY: u16 = 0x0001;

@ -8,6 +8,7 @@
#[macro_use] extern crate tracing; #[macro_use] extern crate tracing;
#[macro_use] extern crate pin_project; #[macro_use] extern crate pin_project;
//#[macro_use] extern crate async_trait;
use std::{ use std::{
convert::{TryFrom, TryInto}, convert::{TryFrom, TryInto},
}; };
@ -21,6 +22,19 @@ use color_eyre::{
use lazy_static::lazy_static; use lazy_static::lazy_static;
use serde::{Serialize, Deserialize}; use serde::{Serialize, Deserialize};
use tracing_futures::Instrument; use tracing_futures::Instrument;
use futures::{
future::{
FutureExt as _,
BoxFuture,
LocalBoxFuture,
},
};
use serialise::{
TextSerialiseable as _,
BinarySerialisable as _,
Serialisable as _,
};
macro_rules! cfg_debug { macro_rules! cfg_debug {
(if {$($if:tt)*} else {$($else:tt)*}) => { (if {$($if:tt)*} else {$($else:tt)*}) => {
@ -95,77 +109,7 @@ pub mod timestamp
} }
} }
pub mod serialise pub mod serialise;
{
use super::*;
/// Serialise this object into base64 text
///
/// # TODO
/// Line wrapping
#[instrument(err, skip(value), fields(value_type=?std::any::type_name::<T>()))]
pub fn into_text<T: Serialize>(value: &T) -> Result<String, Error>
{
let bytes = serde_cbor::to_vec(value).map_err(Error::Serialise)?;
Ok(base64::encode(bytes))
}
#[derive(Debug)]
pub enum Error
{
Base64(base64::DecodeError),
Serialise(serde_cbor::Error),
Deserialise(serde_cbor::Error),
}
impl std::error::Error for Error
{
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
match &self {
Self::Base64(b) => Some(b),
Self::Serialise(b) => Some(b),
Self::Deserialise(b) => Some(b),
}
}
}
impl std::fmt::Display for Error
{
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result
{
match self {
Self::Base64(_) => write!(f, "base64 decode failed"),
Self::Serialise(_) => write!(f, "serialisation failed"),
Self::Deserialise(_) => write!(f, "deserialisation failed"),
}
}
}
/// Try to deserialise this object from text.
#[instrument(err, skip(string), fields(string = ?string.as_ref()))]
pub fn from_text<T: serde::de::DeserializeOwned>(string: impl AsRef<[u8]>) -> Result<T, Error>
{
let bytes = base64::decode(string).map_err(Error::Base64)?;
serde_cbor::from_reader(&bytes[..]).map_err(Error::Deserialise)
}
#[cfg(test)]
mod tests
{
use crate::*;
#[test]
fn test_ser() -> Result<(), eyre::Report>
{
let object = "hello world";
let enc = super::into_text(&String::from(object)).wrap_err(eyre::eyre!("failed to encode"))?;
println!("Enc: {:?}", enc);
let dec: String = super::from_text(enc).wrap_err(eyre::eyre!("failed to decode"))?;
assert_eq!(object,dec);
Ok(())
}
}
}
/*/// Dispatch params operations that can be handled at top level (i.e. `Help`) /*/// Dispatch params operations that can be handled at top level (i.e. `Help`)
async fn dispatch_args() -> Result<args::Operation, eyre::Report> async fn dispatch_args() -> Result<args::Operation, eyre::Report>

@ -0,0 +1,202 @@
//! Serialisation helpers & traits
use super::*;
use tokio::io::{AsyncWrite,AsyncBufRead,AsyncRead,};
use std::{
marker::Unpin,
fmt,
};
pub trait TextSerialiseable: Sized
{
fn serialise_text<'a, 'b: 'a, T: AsyncWrite+Unpin+?Sized>(&'a self, out: &'b mut T) -> LocalBoxFuture<'a, Result<usize, eyre::Report>>;
fn deserialise_text<'a, T: AsyncBufRead+Unpin+?Sized>(input: &'a mut T) -> LocalBoxFuture<'a, Result<Self, eyre::Report>>
where Self: 'a;
#[inline] fn size_hint(&self) -> (usize, Option<usize>)
{
(0,None)
}
}
pub trait BinarySerialisable: Sized
{
fn serialise_bytes<'a, 'b, T: AsyncWrite+Unpin+?Sized>(&'a self, out: &'b mut T) -> LocalBoxFuture<'a, Result<usize, eyre::Report>>
where 'b: 'a;
fn deserialise_bytes<'b , T: AsyncRead+Unpin+?Sized>(input: &'b mut T) -> LocalBoxFuture<'b, Result<Self, eyre::Report>>
where Self: 'b;
#[inline] fn size_hint(&self) -> (usize, Option<usize>)
{
(0,None)
}
}
pub trait Serialisable: BinarySerialisable + TextSerialiseable
{
/// Attempt to immediately serialise this object into memory.
///
/// Returns a newly allocated `Vec<u8>` containing the bytes.
/// Attempts to use the appropriate `size_hint` to allocate vector capacity if possible.
///
/// # Notes
/// This function will fail if any `await`ed future within the `serialise_*` method cannot complete immediately.
#[instrument(skip(self))]
fn into_memory(&self, mode: Mode) -> Result<Vec<u8>, eyre::Report>
{
let mut output = match mode.size_hint(self)
{
(0, None) | (_, Some(0)) => Vec::new(),
(_, Some(x)) | (x, None) => Vec::with_capacity(x),
};
trace!("Cap for membuf is {} bytes", output.capacity());
let note = || format!("While serialising {} into memory", std::any::type_name::<Self>());
let written = match mode {
Mode::Text => self.serialise_text(&mut output).now_or_never(),
Mode::Binary => self.serialise_bytes(&mut output).now_or_never(),
}
.ok_or_else(|| eyre::eyre!("Failed to complete {} write syncronously. A future returned `Poll::Pending`.", mode))
.with_note(note.clone())?
.wrap_err_with(|| eyre::eyre!("Failed to write {} into memory", mode))
.with_note(note)?;
if written != output.len()
{
error!("Function reported {} bytes written, but actually wrote {}", written, output.len());
}
Ok(output)
}
/// Attempt to immediately deserialise this object from memory
///
/// Returns the object and the number of bytes read from `buf`.
///
/// # Notes
/// This function will fail if any `await`ed future within the `deserialise_*` method cannot complete immediately.
#[instrument(skip(buf),fields(buf = ?buf.as_ref()))]
fn from_memory<T: AsRef<[u8]>>(buf: T, mode: Mode) -> Result<(Self, usize), eyre::Report>
{
let buf = buf.as_ref();
let mut reader = &buf[..];
trace!("Attempting read of up to {} bytes in {} mode to instantiate type {}", buf.len(), mode, std::any::type_name::<Self>());
let note = || format!("While deserialising {} from memory", std::any::type_name::<Self>());
let bytes = || format!("{}", buf.fmt_view()).header("Binary buffer was");
let text = || String::from_utf8_lossy(buf).into_owned().header("Text buffer was");
let value = match mode {
Mode::Binary => Self::deserialise_bytes(&mut reader).now_or_never(),
Mode::Text => Self::deserialise_text(&mut reader).now_or_never(),
}
.ok_or_else(|| eyre::eyre!("Failed to complete {} read syncronously. A future returned `Poll::Pending`.", mode))
.with_note(note.clone())?
.wrap_err_with(|| eyre::eyre!("Failed to read {} from memory", mode))
.with_note(note)
.with_section(bytes)
.with_section(text)?;
let diff = (&reader[0] as *const u8 as usize) - (&buf[0] as *const u8 as usize);
Ok((value, diff))
}
}
impl<T> Serialisable for T
where T: BinarySerialisable + TextSerialiseable{}
/// Serialisation mode for `Serialisable` types
#[derive(Debug, Clone, Copy)]
pub enum Mode
{
/// Serialise as text
Text,
/// Serialise as binary
Binary,
}
impl fmt::Display for Mode
{
#[inline] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result
{
match self {
Self::Text => write!(f, "text"),
Self::Binary => write!(f, "binary"),
}
}
}
impl Mode
{
#[inline(always)] fn size_hint<T: Serialisable>(&self, other: &T) -> (usize, Option<usize>)
{
match self {
Self::Text => TextSerialiseable::size_hint(other),
Self::Binary => BinarySerialisable::size_hint(other),
}
}
}
/// Serialise this object into base64 text
///
/// # TODO
/// Line wrapping
#[instrument(err, skip(value), fields(value_type=?std::any::type_name::<T>()))]
pub fn into_text<T: Serialize>(value: &T) -> Result<String, Error>
{
let bytes = serde_cbor::to_vec(value).map_err(Error::Serialise)?;
Ok(base64::encode(bytes))
}
#[derive(Debug)]
pub enum Error
{
Base64(base64::DecodeError),
Serialise(serde_cbor::Error),
Deserialise(serde_cbor::Error),
}
impl std::error::Error for Error
{
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
match &self {
Self::Base64(b) => Some(b),
Self::Serialise(b) => Some(b),
Self::Deserialise(b) => Some(b),
}
}
}
impl std::fmt::Display for Error
{
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result
{
match self {
Self::Base64(_) => write!(f, "base64 decode failed"),
Self::Serialise(_) => write!(f, "serialisation failed"),
Self::Deserialise(_) => write!(f, "deserialisation failed"),
}
}
}
/// Try to deserialise this object from text.
#[instrument(err, skip(string), fields(string = ?string.as_ref()))]
pub fn from_text<T: serde::de::DeserializeOwned>(string: impl AsRef<[u8]>) -> Result<T, Error>
{
let bytes = base64::decode(string).map_err(Error::Base64)?;
serde_cbor::from_reader(&bytes[..]).map_err(Error::Deserialise)
}
#[cfg(test)]
mod tests
{
use crate::*;
#[test]
fn test_ser() -> Result<(), eyre::Report>
{
let object = "hello world";
let enc = super::into_text(&String::from(object)).wrap_err(eyre::eyre!("failed to encode"))?;
println!("Enc: {:?}", enc);
let dec: String = super::from_text(enc).wrap_err(eyre::eyre!("failed to decode"))?;
assert_eq!(object,dec);
Ok(())
}
}
Loading…
Cancel
Save