From 93e1d335dc19250b20d8309826dca6637188e8c2 Mon Sep 17 00:00:00 2001 From: Avril Date: Thu, 8 Oct 2020 15:53:46 +0100 Subject: [PATCH] fix dumb shit --- Cargo.lock | 41 +++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 6 ++++-- src/save.rs | 32 +++++++++++++++++++++----------- 3 files changed, 66 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4d1af37..6719278 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -15,6 +15,19 @@ version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d25d88fd6b8041580a654f9d0c581a047baee2b3efee13275f2fc392fc75034" +[[package]] +name = "async-compression" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9021768bcce77296b64648cc7a7460e3df99979b97ed5c925c38d1cc83778d98" +dependencies = [ + "bzip2", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", +] + [[package]] name = "atty" version = "0.2.14" @@ -108,6 +121,27 @@ version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" +[[package]] +name = "bzip2" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42b7c3cbf0fa9c1b82308d57191728ca0256cb821220f4e2fd410a72ade26e3b" +dependencies = [ + "bzip2-sys", + "libc", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.9+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad3b39a260062fca31f7b0b12f207e8f2590a67d32ec7d59c20484b07ea7285e" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "cc" version = "1.0.60" @@ -607,6 +641,7 @@ dependencies = [ name = "markov" version = "0.3.3" dependencies = [ + "async-compression", "cfg-if 1.0.0", "futures", "hyper", @@ -833,6 +868,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkg-config" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d36492546b6af1463394d46f0c834346f31548646f6ba10849802c9c9a27ac33" + [[package]] name = "ppv-lite86" version = "0.2.9" diff --git a/Cargo.toml b/Cargo.toml index 2e7379e..2e31c38 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "markov" -version = "0.3.3" +version = "0.3.4" description = "Generate string of text from Markov chain fed by stdin" authors = ["Avril "] edition = "2018" @@ -8,7 +8,8 @@ edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [features] -compress-chain = ["lzzzz"] +default = ["compress-chain"] +compress-chain = ["async-compression"] [profile.release] opt-level = 3 @@ -28,3 +29,4 @@ serde_cbor = "0.11.1" lzzzz = {version = "0.2", features=["tokio-io"], optional=true} serde = {version ="1.0", features=["derive"]} toml = "0.5.6" +async-compression = {version = "0.3.5", features=["tokio-02", "bzip2"], optional=true} diff --git a/src/save.rs b/src/save.rs index 809cb5b..c7e3c41 100644 --- a/src/save.rs +++ b/src/save.rs @@ -23,18 +23,24 @@ use futures::{ }, }; #[cfg(feature="compress-chain")] -use lzzzz::{ - lz4f::{ - self, - AsyncWriteCompressor, - PreferencesBuilder, - AsyncReadDecompressor, +use async_compression::{ + tokio_02::{ + write::{ + BzEncoder, + BzDecoder, + }, }, }; const SAVE_INTERVAL: Option = Some(Duration::from_secs(2)); +#[cfg(feature="compress-chain")] +type Compressor = BzEncoder; +#[cfg(feature="compress-chain")] +type Decompressor = BzDecoder; + + pub async fn save_now(state: &State) -> io::Result<()> { let chain = state.chain().read().await; @@ -54,12 +60,11 @@ async fn save_now_to(chain: &Chain, to: impl AsRef) -> io::Result< let chain = serde_cbor::to_vec(chain).map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; { #[cfg(feature="compress-chain")] - let mut file = AsyncWriteCompressor::new(&mut file, PreferencesBuilder::new() - .compression_level(lz4f::CLEVEL_HIGH).build())?; + let mut file = Compressor::new(&mut file); file.write_all(&chain[..]).await?; - #[cfg(feature="compress-chain")] + #[cfg(feature="compress-chain")] file.flush().await?; - #[cfg(feature="compress-chain")] + #[cfg(feature="compress-chain")] file.shutdown().await?; } file.flush().await?; @@ -105,11 +110,16 @@ pub async fn load(from: impl AsRef) -> io::Result> let mut file = OpenOptions::new() .read(true) .open(from).await?; + #[allow(unused_mut)] let mut whole = Vec::new(); #[cfg(feature="compress-chain")] - let mut file = AsyncReadDecompressor::new(file)?; + let mut whole = Decompressor::new(whole); tokio::io::copy(&mut file, &mut whole).await?; whole.flush().await?; + #[cfg(feature="compress-chain")] + whole.shutdown().await?; + #[cfg(feature="compress-chain")] + let whole = whole.into_inner(); serde_cbor::from_slice(&whole[..]) .map_err(|e| io::Error::new(io::ErrorKind::InvalidInput, e)) }