fix punctuation eating bug. requiring nightly for now...

Cargo.lock

@@ -639,7 +639,7 @@ dependencies = [
 
 [[package]]
 name = "markov"
-version = "0.5.1"
+version = "0.5.2"
 dependencies = [
  "async-compression",
  "cfg-if 1.0.0",

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "markov"
-version = "0.5.2"
+version = "0.5.3"
 description = "Generate string of text from Markov chain fed by stdin"
 authors = ["Avril <flanchan@cumallover.me>"]
 edition = "2018"

src/feed.rs

@@ -10,7 +10,7 @@ pub fn feed(chain: &mut Chain<String>, what: impl AsRef<str>, bounds: impl std::
     cfg_if! {
 	if #[cfg(feature="split-sentance")] { 
 	    let map = Sentance::new_iter(&what) //get each sentance in string
-		.map(|what| what.split_whitespace() // .words() here will remove the punctuation.
+		.map(|what| what.words()
 		     .filter(|word| !word.is_empty())
 		     .map(|s| s.to_owned()).collect::<Vec<_>>());
 	    debug_assert!(!bounds.contains(&0), "Cannot allow 0 size feeds");
@@ -24,7 +24,7 @@ pub fn feed(chain: &mut Chain<String>, what: impl AsRef<str>, bounds: impl std::
 	    }
 	} else {
 	    let map = Sentance::new_iter(&what) //get each sentance in string
-		.map(|what| what.split_whitespace() // .words() here will remove the punctuation.
+		.map(|what| what.words()
 		     .filter(|word| !word.is_empty()))
 		.flatten() // add all into one buffer
 		.map(|s| s.to_owned()).collect::<Vec<_>>();

src/main.rs

@@ -1,3 +1,5 @@
+#![feature(split_inclusive)]
+
 #![allow(dead_code)]
 
 #[macro_use] extern crate log;

src/sanitise/sentance.rs

@@ -71,7 +71,7 @@ impl Sentance
     pub fn new_iter<'a>(from: &'a (impl AsRef<str> +'a + ?Sized)) -> impl Iterator<Item = &'a Self>
     {
 	let from = from.as_ref();
-	from.split(is_sentance_boundary)
+	from.split_inclusive(is_sentance_boundary)
 	    .map(|x| new!(x.trim()))
 	    .filter(|x| !x.is_empty())
     }

src/sanitise/word.rs

@@ -71,7 +71,8 @@ impl Word
     pub fn new_iter<'a>(from: &'a (impl AsRef<Sentance> +?Sized+'a)) -> impl Iterator<Item = &'a Self>
     {
 	let from = from.as_ref();
-	from.split(is_word_boundary)
+	from.split_inclusive(is_word_boundary)
+	    .map(|x| x.trim())
 	    .filter(|x| !x.is_empty())
 	    .map(|x| new!(x))
     }