post delta creation

legacy
Avril 4 years ago
parent 733970c55b
commit a84a3b3909
Signed by: flanchan
GPG Key ID: 284488987C31F630

146
Cargo.lock generated

@ -23,6 +23,12 @@ version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
[[package]]
name = "base64"
version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff"
[[package]]
name = "bitflags"
version = "1.2.1"
@ -38,6 +44,12 @@ dependencies = [
"generic-array",
]
[[package]]
name = "build_const"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39092a32794787acd8525ee150305ff051b0aa6cc2abaf193924f5ab05425f39"
[[package]]
name = "byteorder"
version = "1.3.4"
@ -50,6 +62,12 @@ version = "0.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38"
[[package]]
name = "cc"
version = "1.0.59"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "66120af515773fb005778dc07c261bd201ec8ce50bd6e7144c927753fe013381"
[[package]]
name = "cfg-if"
version = "0.1.10"
@ -74,19 +92,50 @@ version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8aebca1129a03dc6dc2b127edd729435bbc4a37e1d5f4d7513165089ceb02634"
[[package]]
name = "crc"
version = "1.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d663548de7f5cca343f1e0a48d14dcfb0e9eb4e079ec58883b7251539fa10aeb"
dependencies = [
"build_const",
]
[[package]]
name = "crypto-mac"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "58bcd97a54c7ca5ce2f6eb16f6bede5b0ab5f0055fedc17d2f0b4466e21671ca"
dependencies = [
"generic-array",
"subtle",
]
[[package]]
name = "cryptohelpers"
version = "1.1.1"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3aea2aded66cedf86364894060626fd5837b10b81d8a9592bc0bcde7a5a3af3d"
checksum = "825b8339215ceae0288ed0384ebe63fe717f5b0c77b102ff3c9f59aefeed9106"
dependencies = [
"crc",
"getrandom",
"hex-literal",
"hmac",
"libc",
"openssl",
"pbkdf2",
"serde",
"serde_derive",
"sha2",
"tokio",
]
[[package]]
name = "difference"
version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "524cbf6897b527295dff137cec09ecf3a05f4fddffd7dfcd1585403449e74198"
[[package]]
name = "digest"
version = "0.9.0"
@ -102,6 +151,21 @@ version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
[[package]]
name = "foreign-types"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
dependencies = [
"foreign-types-shared",
]
[[package]]
name = "foreign-types-shared"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
[[package]]
name = "fuchsia-zircon"
version = "0.3.3"
@ -249,6 +313,22 @@ dependencies = [
"libc",
]
[[package]]
name = "hex-literal"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5af1f635ef1bc545d78392b136bfe1c9809e029023c84a3638a864a10b8819c8"
[[package]]
name = "hmac"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "deae6d9dbb35ec2c502d62b8f7b1c000a0822c3b0794ba36b3149c0a1c840dff"
dependencies = [
"crypto-mac",
"digest",
]
[[package]]
name = "iovec"
version = "0.1.4"
@ -411,6 +491,48 @@ version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
[[package]]
name = "openssl"
version = "0.10.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8d575eff3665419f9b83678ff2815858ad9d11567e082f5ac1814baba4e2bcb4"
dependencies = [
"bitflags",
"cfg-if",
"foreign-types",
"lazy_static",
"libc",
"openssl-sys",
]
[[package]]
name = "openssl-sys"
version = "0.9.58"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a842db4709b604f0fe5d1170ae3565899be2ad3d9cbc72dedc789ac0511f78de"
dependencies = [
"autocfg",
"cc",
"libc",
"pkg-config",
"vcpkg",
]
[[package]]
name = "pbkdf2"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7170d73bf11f39b4ce1809aabc95bf5c33564cdc16fc3200ddda17a5f6e5e48b"
dependencies = [
"base64",
"crypto-mac",
"hmac",
"rand",
"rand_core",
"sha2",
"subtle",
]
[[package]]
name = "pin-project"
version = "0.4.23"
@ -443,6 +565,12 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
[[package]]
name = "pkg-config"
version = "0.3.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d36492546b6af1463394d46f0c834346f31548646f6ba10849802c9c9a27ac33"
[[package]]
name = "ppv-lite86"
version = "0.2.9"
@ -621,6 +749,12 @@ dependencies = [
"winapi 0.3.9",
]
[[package]]
name = "subtle"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "343f3f510c2915908f155e94f17220b19ccfacf2a64a2a5d8004f2c3e311e7fd"
[[package]]
name = "syn"
version = "1.0.40"
@ -700,6 +834,12 @@ dependencies = [
"serde",
]
[[package]]
name = "vcpkg"
version = "0.2.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6454029bf181f092ad1b853286f23e2c507d8e8194d01d92da4a55c274a5508c"
[[package]]
name = "version_check"
version = "0.9.2"
@ -770,9 +910,11 @@ dependencies = [
"byteorder",
"chrono",
"cryptohelpers",
"difference",
"futures",
"libc",
"once_cell",
"pin-project",
"rustc_version",
"serde",
"serde_cbor",

@ -12,18 +12,26 @@ default = ["experimental_inserter"]
# Use non-allocating array inserter
experimental_inserter = []
[profile.release]
opt-level = 3
lto = "fat"
codegen-units = 1
panic = "unwind"
[dependencies]
tokio = {version = "0.2", features=["full"]}
async-trait = "0.1.40"
async-trait = "0.1"
chrono = {version = "0.4.15", features=["serde"]}
uuid = {version = "0.8", features=["v4", "serde"]}
once_cell = "1.4.1"
crypto = {version = "1.1.1", package= "cryptohelpers", features= ["serialise", "async", "sha256"]}
once_cell = "1.4"
crypto = {version = "1.1.2", package= "cryptohelpers", features= ["serialise", "async", "sha256"]}
libc = "0.2.76"
byteorder = "1.3.4"
byteorder = "1.3"
serde_cbor = "0.11.1"
serde = {version = "1.0.115", features= ["derive"]}
futures = "0.3.5"
serde = {version = "1.0", features= ["derive"]}
futures = "0.3"
pin-project = "0.4.23"
difference = "2.0.0"
[build-dependencies]
rustc_version = "0.2"

@ -1,5 +1,81 @@
//! Extensions
use super::*;
use std::{
marker::PhantomData,
};
/// A trait for types that can insert objects at their end.
pub trait BackInserter<T>
{
/// Insert an object at the end of this container
fn push_back(&mut self, value: T);
}
impl<T> BackInserter<T> for Vec<T>
{
#[inline]
fn push_back(&mut self, value: T)
{
self.push(value)
}
}
/// Absracts a closure for `BackInserter<T>`.
pub struct BackInsertPass<T,F>(F, PhantomData<T>)
where F: FnMut(T);
impl<T,F: FnMut(T)> BackInsertPass<T,F>
{
/// Create a new instance with this closure
#[inline] pub fn new(func: F) -> Self
{
Self(func, PhantomData)
}
}
impl<T, F: FnMut(T)> BackInserter<T> for BackInsertPass<T,F>
{
#[inline] fn push_back(&mut self, value: T)
{
self.0(value)
}
}
/// A `BackInserter<T>` that will only add a max capacity of items before it starts dropping input to its `push_back` function.
pub struct CappedBackInserter<'a, T>(&'a mut T, usize, usize)
where T: BackInserter<T>;
impl<'a, T> CappedBackInserter<'a, T>
where T: BackInserter<T>
{
/// Create a new instance with this max capacity
#[inline] pub fn new(from: &'a mut T, cap: usize) -> Self
{
Self(from, 0, cap)
}
/// The number of elements pushed so far
#[inline] pub fn len(&self) -> usize {
self.1
}
/// The max number of elemnts allowed to be pushed
#[inline] pub fn cap(&self) -> usize {
self.2
}
}
impl<'a, T> BackInserter<T> for CappedBackInserter<'a, T>
where T: BackInserter<T>
{
#[inline] fn push_back(&mut self, value: T)
{
if self.1 < self.2 {
self.0.push_back(value);
self.1+=1;
}
}
}
pub trait VecExt<T>
{

@ -27,4 +27,14 @@ mod state;
fn main() {
/*
let mut vec = vec![vec![1, 0, 0],
vec![0, 0, 1]];
let span = vec![vec![0, 1, 0],
vec![1, 0, 1],
vec![0, 1, 0]];
for _ in 0..10000 {
vec.insert_exact(1, span.iter().cloned());
}*/
}

@ -1,11 +1,53 @@
//! Deltas and applying them
use super::*;
use difference::{
Changeset,
Difference,
};
const MAX_SINGLE_DELTA_SIZE: usize = 14;
const DELTA_BREAK: &str = "";
/// Infer all deltas needed to be sequentially applied to `orig` to transform it to `new`, return the number inserted into `output`.
pub fn infer_deltas<T: BackInserter<Delta> + ?Sized>(output: &mut T, orig: &str, new: &str) -> usize
{
let set = Changeset::new(orig, new, DELTA_BREAK);
let mut done=0;
let mut position =0;
for diff in set.diffs.into_iter() {
match diff {
Difference::Same(string) => {
position += string.len();
},
Difference::Rem(string) => {
output.push_back(Delta {
location: position,
kind: DeltaKind::RemoveAhead{span_len: string.len()},
});
done+=1;
},
Difference::Add(string) => {
let mut passer = BackInsertPass::new(|(span, span_len)| {
output.push_back(Delta {
location: position,
kind: DeltaKind::Insert {
span, span_len,
},
});
position+= usize::from(span_len);
done+=1;
});
delta_span_many(&mut passer, string.chars());
},
}
}
done
}
/// Create a delta span from an input iterator.
///
/// This function can take no more than 255 chars from the input. The number of chars inserted is also returned as `u8`.
/// This function can take no more than `min(255, MAX_SINGLE_DELTA_SIZE)` chars from the input. The number of chars inserted is also returned as `u8`.
pub(super) fn delta_span<I>(from: I) -> ([char; MAX_SINGLE_DELTA_SIZE], u8)
where I: IntoIterator<Item = char>
{
@ -21,11 +63,29 @@ where I: IntoIterator<Item = char>
(output, sz)
}
/// Create as many delta spans needed from an input iterator.
///
/// This function can take as many inputs as needed, and outputs the needed amount of spans into `into`, and then returns the number added.
pub(super) fn delta_span_many<T: BackInserter<([char; MAX_SINGLE_DELTA_SIZE], u8)> + ?Sized, I>(into: &mut T, from: I) -> usize
where I: IntoIterator<Item = char>
{
let mut iter = from.into_iter();
let mut added=0;
loop {
match delta_span(&mut iter) {
(_, 0) => break,
other => into.push_back(other),
}
added+=1;
}
added
}
/// Information about the delta to be applied in `Delta`.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Hash)]
pub enum DeltaKind
{
/// Append to the end of body. Equivilant to `Insert` with a location at `karada.scape.len()` (the end of the buffer). Might be removed idk
/// Append to the end of body. Equivilant to `Insert` with a location at `karada.scape.len()` (the end of the buffer).
Append{
span: [char; MAX_SINGLE_DELTA_SIZE],
span_len: u8,
@ -35,14 +95,18 @@ pub enum DeltaKind
span: [char; MAX_SINGLE_DELTA_SIZE],
span_len: u8,
},
/// Remove `span_len` chars ahead of `location`. (INCLUSIVE)
/// Remove `span_len` chars from `location`.
RemoveAhead{
span_len: usize,
},
/// Remove `span_len` chars behind this `location`. (EXCLUSIVE)
/// Remove `span_len` chars up to but not including `location`.
RemoveBehind{
span_len: usize,
},
/// Remove everything from `location` to the end.
Truncate,
/// Remove everything from 0 to `location`.
Shift,
/// Remove char at `location`
RemoveSingle,
/// Remove entire post body
@ -80,16 +144,21 @@ impl Delta
inserter.insert(self.location, span[0]);
} else if span.len() > 1 {
inserter.insert_exact(self.location, span.iter().copied());
/*// reserve the extra space
inserter.reserve(span.len());
// shift everything across, replacing with the new values
let splice: Vec<_> = inserter.splice(self.location.., span.iter().cloned()).collect();
// add tail back
inserter.extend(splice);*/
}
},
_ => unimplemented!(),
DeltaKind::RemoveAhead{span_len} => {
inserter.drain(self.location..(self.location+span_len));
},
DeltaKind::RemoveBehind{span_len} => {
let span_st = self.location.checked_sub(span_len).unwrap_or(0);
inserter.drain(span_st..self.location);
},
DeltaKind::RemoveSingle => {
inserter.remove(self.location);
},
DeltaKind::Clear => inserter.clear(),
DeltaKind::Truncate => inserter.truncate(self.location),
DeltaKind::Shift => ((),inserter.drain(..self.location)).0,
}
}
}
@ -99,6 +168,27 @@ mod tests
{
use super::*;
#[test]
fn infer_and_insert()
{
let orig = "the quick brown fox jumped over the lazy dog !!!!";
let new = "the quick brown dog jumped over the lazy fox twice";
let mut deltas = Vec::new();
infer_deltas(&mut deltas, orig, new);
println!("{:#?}", deltas);
let output: String = {
let mut scape: Vec<_> = orig.chars().collect();
for delta in deltas.into_iter() {
delta.insert(&mut scape);
}
scape.into_iter().collect()
};
assert_eq!(&output[..], &new[..]);
}
macro_rules! insert {
($expects:literal; $start:literal, $ins:literal, $where:expr) => {
{

@ -28,6 +28,16 @@ impl Karada
self.current_body.borrow().to_owned()
}
/// Create the deltas required to atomically transform current body into `new`.
///
/// Return the number of deltas added to `output`.
pub fn create_body_deltas<T: BackInserter<Delta> + ?Sized, U: AsRef<str>>(&self, output: &mut T, new: U) -> usize
{
let body = self.body();
let new = new.as_ref();
delta::infer_deltas(output, &body[..], new)
}
/// Consume this instance into a suspension
///
/// This will only acquire locks if needed, but since they might be needed, it must be awaited in case of `Kokoro` instances potentially owning the data.

@ -1,10 +1,37 @@
//! Worker that mutates `Kokoro`.
use super::*;
use std::{
pin::Pin,
task::{
Poll,
Context,
},
};
use tokio::{
sync::{
watch,
oneshot,
},
};
use pin_project::{
pin_project,
};
use futures::{
future::Future,
};
/// A handle on a spawned worker
#[pin_project]
pub struct WorkerHandle(#[pin] oneshot::Receiver<()>);
impl Future for WorkerHandle
{
type Output = ();
fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
let this = self.project();
this.0.poll(cx).map(|x| x.unwrap())
}
}
/// Handles working on `Karada` instances.
#[derive(Debug)]

Loading…
Cancel
Save