added outbound filter

serve
Avril 4 years ago
parent 2f9f4dbb3a
commit 988829c9b9
Signed by: flanchan
GPG Key ID: 284488987C31F630

@ -1,6 +1,6 @@
[package]
name = "markov"
version = "0.6.0"
version = "0.6.1"
description = "Generate string of text from Markov chain fed by stdin"
authors = ["Avril <flanchan@cumallover.me>"]
edition = "2018"

@ -6,4 +6,5 @@ max_gen_size = 256
trust_x_forwarded_for = false
[filter]
exclude = "<>)([]/"
inbound = "<>)([]/"
outbound = "*"

@ -32,18 +32,28 @@ pub struct Config
#[derive(Debug, Default, Clone, PartialEq, Eq, PartialOrd, Hash, Serialize, Deserialize)]
pub struct FilterConfig
{
exclude: String,
inbound: String,
#[serde(default)]
outbound: String,
}
impl FilterConfig
{
pub fn get_filter(&self) -> sanitise::filter::Filter
pub fn get_inbound_filter(&self) -> sanitise::filter::Filter
{
let filt: sanitise::filter::Filter = self.inbound.parse().unwrap();
if !filt.is_empty()
{
info!("Loaded inbound filter: {:?}", filt.iter().collect::<String>());
}
filt
}
pub fn get_outbound_filter(&self) -> sanitise::filter::Filter
{
let filt: sanitise::filter::Filter = self.exclude.parse().unwrap();
let filt: sanitise::filter::Filter = self.outbound.parse().unwrap();
if !filt.is_empty()
{
warn!("Loaded exclude filter: {:?}", filt.iter().collect::<String>());
info!("Loaded outbound filter: {:?}", filt.iter().collect::<String>());
}
filt
}

@ -1,6 +1,7 @@
//! Extensions
use std::{
iter,
ops::Range,
};
pub trait StringJoinExt: Sized
@ -25,3 +26,71 @@ where I: IntoIterator<Item=T>,
string
}
}
pub trait FindSliceBounds
{
type SliceType: ?Sized;
fn slice_bounds(&self, from: &Self::SliceType) -> Range<usize>;
}
impl<T: ?Sized + AsRef<str>> FindSliceBounds for T
{
type SliceType = str;
fn slice_bounds(&self, from: &Self::SliceType) -> Range<usize>{
let this = self.as_ref();
unsafe {
let sptr = from.as_ptr();
let eptr = sptr.add(from.len());
let ssptr = this.as_ptr();
let septr = ssptr.add(this.len());
let sptr = sptr as usize;
let ssptr = ssptr as usize;
let eptr = eptr as usize;
let septr = septr as usize;
assert!(sptr >= ssptr && sptr <= septr, "Start index of slice is outside the bounds of self");
assert!(eptr >= ssptr && eptr <= septr, "End index of slice is outside the bounds of self");
(sptr - ssptr)..(eptr - ssptr)
}
}
}
pub trait SliceInPlace
{
fn drain_inverse<R: std::ops::RangeBounds<usize>>(&mut self, slice: R);
}
impl SliceInPlace for String
{
fn drain_inverse<R: std::ops::RangeBounds<usize>>(&mut self, slice: R)
{
use std::ops::Bound;
match slice.end_bound() {
Bound::Excluded(&ex) => drop(self.drain(ex..)),
Bound::Included(&inc) => drop(self.drain(inc+1..)),
_ => (),
};
match slice.start_bound() {
Bound::Included(&ex) => drop(self.drain(..ex)),
Bound::Excluded(&ex) => drop(..ex+1),
_ => ()
};
}
}
pub trait TrimInPlace
{
fn trim_in_place(&mut self) -> &mut Self;
}
impl TrimInPlace for String
{
fn trim_in_place(&mut self) -> &mut Self {
let bounds = self.slice_bounds(self.trim());
self.drain_inverse(bounds);
self
}
}

@ -69,7 +69,7 @@ pub async fn full(who: &IpAddr, state: State, body: impl Unpin + Stream<Item = R
}
}
let buffer = std::str::from_utf8(&buffer[..]).map_err(|_| FillBodyError)?;
let buffer = state.filter().filter_cow(buffer);
let buffer = state.inbound_filter().filter_cow(buffer);
info!("{} -> {:?}", who, buffer);
let mut chain = state.chain().write().await;
cfg_if! {
@ -92,7 +92,7 @@ pub async fn full(who: &IpAddr, state: State, body: impl Unpin + Stream<Item = R
#[cfg(feature="hog-buffer")]
let mut chain = state.chain().write().await;
while let Some(line) = lines.next_line().await.map_err(|_| FillBodyError)? {
let line = state.filter().filter_cow(&line);
let line = state.inbound_filter().filter_cow(&line);
let line = line.trim();
if !line.is_empty() {
#[cfg(not(feature="hog-buffer"))]

@ -18,14 +18,15 @@ pub async fn body(state: State, num: Option<usize>, mut output: mpsc::Sender<Str
{
let chain = state.chain().read().await;
if !chain.is_empty() {
let filter = state.outbound_filter();
match num {
Some(num) if num < state.config().max_gen_size => {
//This could DoS `full_body` and writes, potentially.
for string in chain.str_iter_for(num) {
output.send(string).await.map_err(|e| GenBodyError(e.0))?;
output.send(filter.filter_owned(string)).await.map_err(|e| GenBodyError(e.0))?;
}
},
_ => output.send(chain.generate_str()).await.map_err(|e| GenBodyError(e.0))?,
_ => output.send(filter.filter_owned(chain.generate_str())).await.map_err(|e| GenBodyError(e.0))?,
}
}
Ok(())

@ -195,9 +195,14 @@ async fn main() {
async move {
let (tx, rx) = mpsc::channel(state.config().max_gen_size);
tokio::spawn(gen::body(state, num, tx));
Ok::<_, std::convert::Infallible>(Response::new(Body::wrap_stream(rx.map(move |x| {
info!("{} <- {:?}", host, x);
Ok::<_, std::convert::Infallible>(x)
Ok::<_, std::convert::Infallible>(Response::new(Body::wrap_stream(rx.filter_map(move |mut x| {
if x.trim_in_place().len() != 0 {
info!("{} <- {:?}", host, x);
x.push('\n');
Some(Ok::<_, std::convert::Infallible>(x))
} else {
None
}
}))))
}
})
@ -213,16 +218,14 @@ async fn main() {
async move {
let (tx, rx) = mpsc::channel(state.config().max_gen_size);
tokio::spawn(sentance::body(state, num, tx));
Ok::<_, std::convert::Infallible>(Response::new(Body::wrap_stream(rx.map(move |mut x| {
info!("{} (sentance) <- {:?}", host, x);
// match x.chars().last() {
// Some(chr) if sanitise::is_sentance_boundary(chr) => {
// x.push(' ');
// },
// _ => (),
// }
x.push(' ');
Ok::<_, std::convert::Infallible>(x)
Ok::<_, std::convert::Infallible>(Response::new(Body::wrap_stream(rx.filter_map(move |mut x| {
if x.trim_in_place().len() != 0 {
info!("{} (sentance) <- {:?}", host, x);
x.push(' ');
Some(Ok::<_, std::convert::Infallible>(x))
} else {
None
}
}))))
}
})

@ -133,7 +133,23 @@ impl Filter
self.0.get(&chr).is_some()
}
pub fn filter<'a, I: IntoIterator<Item=char>>(&'a self, from_iter: I) -> FilterIter<'a, I::IntoIter>
pub fn filter_owned(&self, input: impl Into<String>) -> String
{
let mut input = input.into();
self.filter(&mut input);
input
}
pub fn filter<'a>(&self, output: &'a mut String) -> &'a mut String
{
if self.is_empty() {
return output;
}
output.retain(|chr| !self.check(chr));
output
}
pub fn filter_iter<'a, I: IntoIterator<Item=char>>(&'a self, from_iter: I) -> FilterIter<'a, I::IntoIter>
where I::IntoIter: 'a
{
FilterIter(&self, from_iter.into_iter().fuse())

@ -16,9 +16,10 @@ pub async fn body(state: State, num: Option<usize>, mut output: mpsc::Sender<Str
};
debug!("Taking {:?} from {:?}" ,num, string);
let filter = state.outbound_filter();
for sen in sanitise::Sentance::new_iter(&string).take(num.unwrap_or(1))
{
output.send(sen.to_owned()).await.map_err(|e| gen::GenBodyError(e.0))?;
output.send(filter.filter_owned(sen.to_owned())).await.map_err(|e| gen::GenBodyError(e.0))?;
}
Ok(())
}

@ -11,7 +11,7 @@ use config::Config;
pub struct State
{
config: Arc<Config>, //to avoid cloning config
exclude: Arc<sanitise::filter::Filter>,
exclude: Arc<(sanitise::filter::Filter, sanitise::filter::Filter)>,
chain: Arc<RwLock<Chain<String>>>,
save: Arc<Notify>,
@ -21,16 +21,21 @@ pub struct State
impl State
{
pub fn filter(&self) -> &sanitise::filter::Filter
pub fn inbound_filter(&self) -> &sanitise::filter::Filter
{
&self.exclude
&self.exclude.0
}
pub fn outbound_filter(&self) -> &sanitise::filter::Filter
{
&self.exclude.1
}
pub fn new(config: Config, chain: Arc<RwLock<Chain<String>>>, save: Arc<Notify>) -> Self
{
let (shutdown, shutdown_recv) = watch::channel(false);
Self {
exclude: Arc::new(config.filter.get_filter()),
exclude: Arc::new((config.filter.get_inbound_filter(),
config.filter.get_outbound_filter())),
config: Arc::new(config),
chain,
save,

Loading…
Cancel
Save