fix punctuation eating bug. requiring nightly for now...

serve
Avril 4 years ago
parent 51d72ec7bd
commit 5b56fcffd5
Signed by: flanchan
GPG Key ID: 284488987C31F630

2
Cargo.lock generated

@ -639,7 +639,7 @@ dependencies = [
[[package]] [[package]]
name = "markov" name = "markov"
version = "0.5.1" version = "0.5.2"
dependencies = [ dependencies = [
"async-compression", "async-compression",
"cfg-if 1.0.0", "cfg-if 1.0.0",

@ -1,6 +1,6 @@
[package] [package]
name = "markov" name = "markov"
version = "0.5.2" version = "0.5.3"
description = "Generate string of text from Markov chain fed by stdin" description = "Generate string of text from Markov chain fed by stdin"
authors = ["Avril <flanchan@cumallover.me>"] authors = ["Avril <flanchan@cumallover.me>"]
edition = "2018" edition = "2018"

@ -10,7 +10,7 @@ pub fn feed(chain: &mut Chain<String>, what: impl AsRef<str>, bounds: impl std::
cfg_if! { cfg_if! {
if #[cfg(feature="split-sentance")] { if #[cfg(feature="split-sentance")] {
let map = Sentance::new_iter(&what) //get each sentance in string let map = Sentance::new_iter(&what) //get each sentance in string
.map(|what| what.split_whitespace() // .words() here will remove the punctuation. .map(|what| what.words()
.filter(|word| !word.is_empty()) .filter(|word| !word.is_empty())
.map(|s| s.to_owned()).collect::<Vec<_>>()); .map(|s| s.to_owned()).collect::<Vec<_>>());
debug_assert!(!bounds.contains(&0), "Cannot allow 0 size feeds"); debug_assert!(!bounds.contains(&0), "Cannot allow 0 size feeds");
@ -24,7 +24,7 @@ pub fn feed(chain: &mut Chain<String>, what: impl AsRef<str>, bounds: impl std::
} }
} else { } else {
let map = Sentance::new_iter(&what) //get each sentance in string let map = Sentance::new_iter(&what) //get each sentance in string
.map(|what| what.split_whitespace() // .words() here will remove the punctuation. .map(|what| what.words()
.filter(|word| !word.is_empty())) .filter(|word| !word.is_empty()))
.flatten() // add all into one buffer .flatten() // add all into one buffer
.map(|s| s.to_owned()).collect::<Vec<_>>(); .map(|s| s.to_owned()).collect::<Vec<_>>();

@ -1,3 +1,5 @@
#![feature(split_inclusive)]
#![allow(dead_code)] #![allow(dead_code)]
#[macro_use] extern crate log; #[macro_use] extern crate log;

@ -71,7 +71,7 @@ impl Sentance
pub fn new_iter<'a>(from: &'a (impl AsRef<str> +'a + ?Sized)) -> impl Iterator<Item = &'a Self> pub fn new_iter<'a>(from: &'a (impl AsRef<str> +'a + ?Sized)) -> impl Iterator<Item = &'a Self>
{ {
let from = from.as_ref(); let from = from.as_ref();
from.split(is_sentance_boundary) from.split_inclusive(is_sentance_boundary)
.map(|x| new!(x.trim())) .map(|x| new!(x.trim()))
.filter(|x| !x.is_empty()) .filter(|x| !x.is_empty())
} }

@ -71,7 +71,8 @@ impl Word
pub fn new_iter<'a>(from: &'a (impl AsRef<Sentance> +?Sized+'a)) -> impl Iterator<Item = &'a Self> pub fn new_iter<'a>(from: &'a (impl AsRef<Sentance> +?Sized+'a)) -> impl Iterator<Item = &'a Self>
{ {
let from = from.as_ref(); let from = from.as_ref();
from.split(is_word_boundary) from.split_inclusive(is_word_boundary)
.map(|x| x.trim())
.filter(|x| !x.is_empty()) .filter(|x| !x.is_empty())
.map(|x| new!(x)) .map(|x| new!(x))
} }

Loading…
Cancel
Save