Compare commits

..

1 Commits

Author SHA1 Message Date
Avril 11ad053ff1
trying to fix thingy
4 years ago

@ -1,24 +1,13 @@
[package]
name = "lolistealer"
version = "1.3.2"
version = "1.0.0"
authors = ["Avril <flanchan@cumallover.me>"]
edition = "2018"
license = "GPL-3.0-or-later"
homepage = "https://git.flanchan.moe/flanchan/lolistealer"
description = "Downloads cute images"
[package.metadata.ebuild]
iuse = "threads"
license = "GPL-3+"
homepage = "https://flanchan.moe"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[features]
default = ["async", "mem_find"]
async = ["tokio"]
mem_find = ["memchr"]
[profile.release]
opt-level = 3
@ -26,17 +15,10 @@ lto = "fat"
codegen-units = 1
[dependencies]
termprogress = "0.1.0"
termprogress = { path = "../termprogress" }
lazy_static = "1.4"
tokio = {version = "0.2", features= ["rt-threaded", "io-driver", "io-util", "macros", "fs"], optional=true}
reqwest = { version = "0.10.8", features = ["stream"] }
tokio = {version = "0.2", features= ["full"], optional=true}
reqwest = {version = "0.10", features= ["stream"]}
memmap = "0.7"
getrandom = "0.1"
base64 = "0.12"
memchr = {version = "2.3", optional = true}
[build-dependencies]
rustc_version = "0.2"
[target.'cfg(not(nightly))'.dependencies] # Fuck this. Just. Fuck. It.
linked-list = "0.0"
base64 = "0.12"

@ -1,5 +1,3 @@
Tags filtering before decode
implement output to dir
Remove Cargo.lock when termprogress is uploaded and Cargo.toml is properly formatted

@ -1,24 +0,0 @@
extern crate rustc_version;
use rustc_version::{version, version_meta, Channel};
fn main() {
// Assert we haven't travelled back in time
assert!(version().unwrap().major >= 1);
// Set cfg flags depending on release channel
match version_meta().unwrap().channel {
Channel::Stable => {
println!("cargo:rustc-cfg=stable");
}
Channel::Beta => {
println!("cargo:rustc-cfg=beta");
}
Channel::Nightly => {
println!("cargo:rustc-cfg=nightly");
}
Channel::Dev => {
println!("cargo:rustc-cfg=dev");
}
}
}

@ -17,24 +17,8 @@ lazy_static! {
/// Print usage then exit with code `1`
pub fn usage() -> !
{
println!(concat!("lolistealer version ", env!("CARGO_PKG_VERSION")));
println!(concat!(" written by ",env!("CARGO_PKG_AUTHORS")," with <3"));
println!(" licensed with GNU GPL 3.0 or later\n");
println!("Usage: {} [-q] [--rating <rating>] [--tags <filter expr>] [<outputs...>]", &PROGRAM_NAME[..]);
println!("Usage: {} [--rating <rating>] [--number <number>] <output dir>]", &PROGRAM_NAME[..]);
println!("Usage: {} --help", &PROGRAM_NAME[..]);
println!("Options:");
println!(" -q\t\t\tQuiet mode, do not output progress");
println!(" --rating <rating>\tEither `s`afe, `q`uestionable, or `e`xplicit");
println!(" --tags <filter expr>\tFilter downloaded images out by tags, see below for format.");
println!("Tags filter expreession:");
println!(" tag_name\tMust contain this tag");
println!(" -tag_name\tMust not contains this tag");
println!(" +tag_name\tMust contains at least one tag prepended with `+`");
println!("Note:");
println!(" Tag filter expreession is a single argument, tags must be seperated with spaces. e.g. 'tag1 -tag2 +tag3 +tag4'");
#[cfg(feature="async")]
println!("\n (compiled with threading support)");
std::process::exit(1)
}
@ -50,9 +34,7 @@ fn try_dir(path: impl AsRef<Path>) -> Result<config::OutputType, Error>
let path = path.as_ref();
if path.is_dir() {
Ok(config::OutputType::Directory(path.to_owned()))
} else if !path.exists() {
Ok(config::OutputType::File(path.to_owned()))
} else {
} else {
Err(Error::FileExists(path.to_owned()))
}
}
@ -73,8 +55,6 @@ where I: IntoIterator<Item=String>
let mut paths = Vec::new();
let mut one = String::default();
let mut reading = true;
let mut tags = Vec::new();
let mut verbose = Default::default();
macro_rules! take_one {
() => {
@ -95,9 +75,7 @@ where I: IntoIterator<Item=String>
match arg.to_lowercase().trim() {
"-" => reading = false,
"--help" => return Ok(Mode::Help),
"-q" => verbose = config::Verbosity::Silent,
"--rating" if take_one!() => rating = one.parse::<config::Rating>()?,
"--tags" if take_one!() => tags = tags::parse(&one),
_ => paths.push(try_dir(arg)?),
}
} else {
@ -109,13 +87,13 @@ where I: IntoIterator<Item=String>
return Err(Error::NoOutput);
}
Ok(Mode::Normal(config::Config{rating, output: paths, tags, verbose}))
Ok(Mode::Normal(config::Config{rating, output: paths}))
}
#[derive(Debug)]
pub enum Error {
UnknownRating(String),
FileExists(PathBuf),
BadPath(PathBuf),
NoOutput,
Internal(Box<dyn error::Error>),
@ -130,7 +108,7 @@ impl fmt::Display for Error
match self {
Error::NoOutput => write!(f, "need at least one output, try `{} --help`", &PROGRAM_NAME[..]),
Error::UnknownRating(rating) => write!(f, "{} is not a valid rating", rating),
Error::FileExists(path) => write!(f, "file already exists: {:?}", path),
Error::BadPath(path) => write!(f, "directory does not exist, or is file: {:?}", path),
Error::Internal(bx) => write!(f, "internal error: {}", bx),
_ => write!(f, "unknown error"),
}

@ -25,21 +25,6 @@ impl Rating
}
}
#[derive(Debug, PartialEq, Eq, Hash)]
pub enum Verbosity
{
Full,
Silent,
}
impl Default for Verbosity
{
fn default() -> Self
{
Self::Full
}
}
#[derive(Debug, PartialEq, Eq, Hash)]
pub enum OutputType
{
@ -50,9 +35,8 @@ pub enum OutputType
pub struct Config
{
pub rating: Rating,
pub output: Vec<OutputType>,
pub tags: Vec<tags::Tag>,
pub verbose: Verbosity,
pub number: u64,
pub output: OutputType,
}
impl Default for Rating
@ -69,8 +53,6 @@ impl Default for Config
Self {
rating: Rating::default(),
output: Vec::new(),
tags: Vec::new(),
verbose: Default::default(),
}
}
}

@ -6,7 +6,6 @@ use std::{
use super::{
tempfile,
loli,
tags,
};
#[derive(Debug)]
@ -17,9 +16,7 @@ pub enum Error
HTTP(reqwest::Error),
HTTPStatus(reqwest::StatusCode),
TempFile(tempfile::error::Error),
Tags(tags::OwnedError),
Loli(loli::error::Error),
ChildPanic
}
impl error::Error for Error
@ -44,8 +41,6 @@ impl fmt::Display for Error
Error::HTTP(http) => write!(f, "http internal error: {}", http),
Error::HTTPStatus(status) => write!(f, "response returned status code {}", status),
Error::Loli(loli) => write!(f, "loli interpretation: {}", loli),
Error::Tags(tags) => write!(f, "no match for tags: {}", tags),
Error::ChildPanic => write!(f, "child panic"),
_ => write!(f, "unknown error"),
}
}
@ -93,19 +88,3 @@ impl From<loli::error::DecodeError> for Error
Self::Loli(er.into())
}
}
impl From<tags::Error<'_>> for Error
{
fn from(er: tags::Error<'_>) -> Self
{
Self::Tags(er.into())
}
}
impl From<tags::OwnedError> for Error
{
fn from(er: tags::OwnedError) -> Self
{
Self::Tags(er)
}
}

@ -1,115 +0,0 @@
use std::{
mem,
};
struct Cons<T>
{
item: T,
next: Option<Box<Cons<T>>>,
}
/// A safe linked list, with remove() for non-nightly build.
pub struct LinkedList<T>
{
head: Option<Cons<T>>,
sz: usize,
}
pub struct Iter<'a, T>
{
current: Option<&'a Cons<T>>,
}
pub struct IterMut<'a, T>
{
current: Option<&'a mut Cons<T>>,
}
pub struct IntoIter<T>
{
current: Option<Box<Cons<T>>>,
}
impl<T> Default for LinkedList<T>
{
#[inline]
fn default() -> Self
{
Self::new()
}
}
impl<T> LinkedList<T>
{
pub const fn new() -> Self
{
Self { head: None, sz: 0 }
}
pub fn push(&mut self, value: T)
{
if let Some(head) = &mut self.head
{
head.next = Some(Box::new(mem::replace(head, Cons{ item: value, next: None })));
self.sz+=1;
} else {
self.head = Some(Cons{item: value, next: None});
self.sz=1;
}
}
#[inline]
pub fn len(&self) -> usize
{
self.sz
}
pub fn pop(&mut self) -> Option<T>
{
if let Some(_) = self.head
{
let old = mem::replace(&mut self.head, None).unwrap();
self.head = old.next.map(|x| *x);
self.sz-=1;
Some(old.item)
} else {
None
}
}
pub fn clear(&mut self)
{
self.head = None;
self.sz=0;
}
pub fn iter<'a>(&'a self) -> Iter<'a, T>
{
Iter{ current: self.head.as_ref() }
}
pub fn iter_mut<'a>(&'a mut self) -> IterMut<'a, T>
{
IterMut{ current: self.head.as_mut() }
}
pub fn remove(&mut self, at: usize) -> T
{
assert!(at < self.sz, "Cannot remove at element > than size");
todo!()
}
}
impl<'a, T> Iterator for Iter<'a, T>
{
type Item = &'a T;
fn next(&mut self) -> Option<Self::Item>
{
if self.current.is_some() {
let mut current = mem::replace(&mut self.current, None);
let mut nref = current.unwrap().next.as_ref().map(|x| x.as_ref())
} else {
None
}
}
}

@ -80,18 +80,19 @@ pub const fn data_size(base64: usize) -> usize
((4 * base64 / 3) + 3) & !3
}
use crate::{
search::{
find, find_back,
},
};
#[inline]
fn find(haystack: &[u8], needle: &[u8]) -> Option<usize> {
haystack.windows(needle.len()).position(|window| window == needle)
}
#[inline]
fn find_back(haystack: &[u8], needle: &[u8]) -> Option<usize> {
haystack.windows(needle.len()).rev().position(|window| window == needle).and_then(|v| Some(haystack.len() - v))
}
const MARKER_BASE64_BEGIN: &[u8] = b"base64,";
const MARKER_BASE64_END: &[u8] = b"' /><p"; //Search from end here with .rev()
const MARKER_TAGS_BEGIN: &[u8] = br"<p id='tags'>";
const MARKER_TAGS_END: &[u8] = br"</p><a id='about'";
/// Find the base64 page bounds in this array
pub(super) fn find_bounds(from: impl AsRef<[u8]>) -> Result<Range<usize>, error::DecodeError>
{
@ -101,36 +102,11 @@ pub(super) fn find_bounds(from: impl AsRef<[u8]>) -> Result<Range<usize>, error:
let start = start + MARKER_BASE64_BEGIN.len();
if let Some(end) = find_back(&from[start..], MARKER_BASE64_END) {//find_back(from, MARKER_BASE64_END) {
let end = end - MARKER_BASE64_END.len();
Ok(Range {
start,
end: end + start,
})
} else {
Err(error::DecodeError::Bounds(error::Bound::End))
}
} else {
Err(error::DecodeError::Bounds(error::Bound::Start))
}
}
/// Find the tag bounds in this array
// We should pass this a slice from `base64bounds.end..`
pub(super) fn find_tag_bounds(from: impl AsRef<[u8]>) -> Result<Range<usize>, error::DecodeError>
{
let from = from.as_ref();
if let Some(start) = find(from, MARKER_TAGS_BEGIN) {
let start = start + MARKER_TAGS_BEGIN.len();
if let Some(end) = find_back(&from[start..], MARKER_TAGS_END) {//find_back(from, MARKER_TAGS_END) {
let end = end - MARKER_TAGS_END.len();
Ok(Range {
return Ok(Range {
start,
end: end + start,
})
} else {
Err(error::DecodeError::Bounds(error::Bound::End))
});
}
} else {
Err(error::DecodeError::Bounds(error::Bound::Start))
}
Err(error::DecodeError::Bounds)
}

@ -5,14 +5,6 @@ use std::{
path::PathBuf,
};
/// Start or end bounds
#[derive(Debug)]
pub enum Bound
{
Start,
End,
}
#[derive(Debug)]
pub enum DecodeError
{
@ -23,7 +15,7 @@ pub enum DecodeError
/// Map contained invalid base64
Base64,
/// Couldn't find base64 bounds
Bounds(Bound),
Bounds,
/// Bad size
Size,
}
@ -46,8 +38,7 @@ impl fmt::Display for DecodeError
Self::Map(io, path) => write!(f, "mmap (file {:?}) failed: {}", path, io),
Self::Corrupt => write!(f, "data was corrupt (invalid utf-8)"),
Self::Base64 => write!(f, "data was corrupt (invalid base64)"),
Self::Bounds(Bound::Start) => write!(f, "couldn't find base64 bounds (start)"),
Self::Bounds(Bound::End) => write!(f, "couldn't find base64 bounds (end)"),
Self::Bounds => write!(f, "couldn't find base64 bounds"),
Self::Size => write!(f, "bad size"),
}
}

@ -77,12 +77,10 @@ impl BasedLoli
{
let bound = encoding::find_bounds(self.as_ref())?;
let image_type = self.try_get_type(bound.start)?;
let tags_range = encoding::find_tag_bounds(&self.map[bound.end..]).ok();
Ok(LoliBounds {
loli: self,
image_type,
range:bound,
tags_range,
})
}
@ -145,23 +143,15 @@ impl<'a> LoliBounds<'a>
image_type,
map: unsafe { MmapOptions::new().map_mut(&file).map_err(|e| error::DecodeError::Map(e, path.into()))? },
file: file,
tags: if let Some(range) = &self.tags_range {
let bytes = self.loli.bytes();
let range = &bytes[(self.range.end+range.start)..(self.range.end+range.end)];
if let Ok(string) = std::str::from_utf8(range) {
string.split(' ').map(|x| x.to_owned()).collect()
} else {
Vec::default()
}
} else { Vec::default() },
})
}
}
/// Attempt to decode to a child container of our owner
#[inline]
pub fn decode(&self, loli: &mut Loli) -> Result<usize, error::Error>
{
let bytes = self.loli.bytes();
decode(&bytes[self.range.clone()], loli)
}
}
@ -181,7 +171,6 @@ pub struct Loli
image_type: encoding::ImageType,
map: MmapMut,
file: File,
tags: Vec<String>,
}
impl AsMut<[u8]> for Loli
@ -197,14 +186,5 @@ pub struct LoliBounds<'a>
{
loli: &'a BasedLoli,
range: Range<usize>,
tags_range: Option<Range<usize>>,
image_type: encoding::ImageType,
}
impl Loli {
/// Get the tags for this loli
pub fn tags(&self) -> &[String] //TODO: Tags filter
{
&self.tags[..]
}
}

@ -1,8 +1,6 @@
#![cfg_attr(nightly, feature(linked_list_remove))]
#![cfg_attr(nightly, feature(test))]
#![feature(linked_list_remove)]
#![feature(label_break_value)]
#![allow(dead_code)]
#[cfg(all(nightly, test))] extern crate test;
use termprogress::{
ProgressBar,
@ -18,17 +16,12 @@ mod url;
mod error;
mod tempfile;
mod loli;
mod tags;
mod search;
#[cfg(feature="async")]
mod work_async;
#[cfg(not(feature="async"))]
mod work;
//mod list;
pub fn parse_args() -> Result<config::Config, args::Error>
{
match args::parse_args()? {
@ -44,17 +37,12 @@ async fn main() -> Result<(), Box<dyn std::error::Error>>
let conf = match parse_args() {
Ok(v) => v,
Err(e) => {
eprintln!("Failed to parse args: {}", e);
println!("Failed to parse args: {}", e);
std::process::exit(1)
},
};
if let Err(e) = work_async::work(conf).await {
eprintln!("Worker error: {}", e);
std::process::exit(1)
}
Ok(())
work_async::work(conf).await
}
#[cfg(not(feature="async"))]

@ -1,143 +0,0 @@
//! Searcher in slice
use memchr::{
memchr_iter,
memrchr_iter,
};
#[inline]
#[cfg(feature="mem_find")]
pub fn find(haystack: &[u8], needle: &[u8]) -> Option<usize>
{
if needle.len() < 1 {
return None;
}
for sz in memchr_iter(needle[0], haystack) {
if &haystack[sz..std::cmp::min(sz+needle.len(), haystack.len())] == needle {
return Some(sz);
}
}
None
}
#[inline]
#[cfg(feature="mem_find")]
pub fn find_back(haystack: &[u8], needle: &[u8]) -> Option<usize>
{
if needle.len() < 1 {
return None;
}
for sz in memrchr_iter(needle[0], haystack) {
if &haystack[sz..std::cmp::min(sz+needle.len(), haystack.len())] == needle {
return Some(sz+needle.len());
}
}
None
}
#[inline(always)]
#[cfg(not(feature="mem_find"))]
pub fn find(haystack: &[u8], needle: &[u8]) -> Option<usize>
{
wfind(haystack, needle)
}
#[inline(always)]
#[cfg(not(feature="mem_find"))]
pub fn find_back(haystack: &[u8], needle: &[u8]) -> Option<usize>
{
wfind_back(haystack, needle)
}
#[inline]
fn wfind(haystack: &[u8], needle: &[u8]) -> Option<usize> {
haystack.windows(needle.len()).position(|window| window == needle)
}
#[inline]
fn wfind_back(haystack: &[u8], needle: &[u8]) -> Option<usize> {
haystack.windows(needle.len()).rev().position(|window| window == needle).and_then(|v| Some(haystack.len() - v))
}
#[cfg(test)]
mod tests
{
use super::*;
const HAYSTACK: &str = r#"awn r9anw09e8tja\ł]erj\æ][jw3-9r8ja9w8jera9je\đ →je\¶}j g\}iejđ \→æje\¶iaj-wie ¶3-928H09Q82H39 -Ł ¶j@\}]ẻ¶\ }æ]ĸ«ø𳶲38r 9W³¶\ ³¶}→ E→ ÞÆØ→ŁẺLOLE!!!¶ØĸÆŁŁ¶ĸØÞ@ĸ³øþĸ}@→Ħ³¶Ø@ŁŁĸ}→²ĦE}Ħ¶ 30r EJ}AJ"#;
const NEEDLE: &str = "LOLE!!!";
#[test]
fn mfind()
{
let haystack = HAYSTACK.as_bytes();
let needle= NEEDLE.as_bytes();
assert_eq!(wfind(haystack, needle), find(haystack, needle));
}
#[test]
fn mfind_back()
{
let haystack = HAYSTACK.as_bytes();
let needle= NEEDLE.as_bytes();
assert_eq!(wfind_back(haystack, needle), find_back(haystack, needle));
}
#[cfg(nightly)]
mod benchmarks {
use super::{
NEEDLE, HAYSTACK,
};
use test::{
Bencher, black_box,
};
#[bench]
fn mfind(b: &mut Bencher)
{
let haystack = HAYSTACK.as_bytes();
let needle= NEEDLE.as_bytes();
b.iter(|| {
black_box(super::find(haystack,needle));
});
}
#[bench]
fn mfind_back(b: &mut Bencher)
{
let haystack = HAYSTACK.as_bytes();
let needle= NEEDLE.as_bytes();
b.iter(|| {
black_box(super::find_back(haystack,needle));
});
}
#[bench]
fn wfind(b: &mut Bencher)
{
let haystack = HAYSTACK.as_bytes();
let needle= NEEDLE.as_bytes();
b.iter(|| {
black_box(super::wfind(haystack,needle));
});
}
#[bench]
fn wfind_back(b: &mut Bencher)
{
let haystack = HAYSTACK.as_bytes();
let needle= NEEDLE.as_bytes();
b.iter(|| {
black_box(super::wfind_back(haystack,needle));
});
}
}
}

@ -1,328 +0,0 @@
use std::{
cmp::PartialEq,
fmt,
mem,
collections::HashSet,
str,
};
/// Tag matching rules
#[derive(Clone,Debug,PartialEq,Eq,Hash)]
pub enum Rule
{
/// This tag /must/ be present.
Required,
/// At least one tag marked `optional` must be present.
Optional,
/// This tag /must not/ be present.
Rejected,
}
/// Represents a tag
#[derive(Clone,Debug,PartialEq,Eq,Hash)]
pub struct Tag
{
rule: Rule,
strings: Vec<String>,
normal_idx: usize,
repr: RepresentationMode,
}
/// How should tags be interpreted?
#[derive(Debug,Clone,PartialEq,Eq,Hash)]
pub enum RepresentationMode
{
/// Ignores spaces, case, underscores, things in parenthesis, and special symbols.
Lenient,
/// Ignores spaces, case, and underscores.
Normal,
/// Ignores nothing
Strict,
}
impl Default for RepresentationMode
{
fn default() -> Self
{
Self::Strict
}
}
/// Remove consecutive whitespace, and normalise them to `' '`
fn remove_whitespace<T: IntoIterator<Item=char>>(input: T) -> String
{
let mut last = false;
input.into_iter()
.filter_map(|ch| {
if ch.is_whitespace() {
if last {
None
} else {
last = true;
Some(' ')
}
}
else {
last = false;
Some(ch)
}
})
.collect()
}
/// How big are the max output of `fuzz` likely to be?
const FUZZ_SIZE_HINT: usize = 7;
/// Creates representations
fn fuzz<S: Into<String>>(input: S, output: &mut Vec<String>, mode: &RepresentationMode) -> usize
{
let input = input.into();
match mode {
RepresentationMode::Strict => {
output.push(input);
0
},
RepresentationMode::Normal => {
output.push(input.to_lowercase());
{
let normal = remove_whitespace(input.chars());
output.push(normal.replace(" ", "_"));
output.push(normal.chars().filter(|ch| !ch.is_whitespace()).collect());
output.push(normal);
}
output.push(input);
output.len()-1
},
RepresentationMode::Lenient => {
const REMOVE: &[char] = &[
':',
'.',
',',
' ',
];
output.push(input.chars().filter(|ch| !REMOVE.contains(ch)).collect());
fuzz(input, output, mode)
},
}
}
impl Tag
{
/// Create a new tag representation
pub fn new<S: Into<String>>(tag: S, rule: Rule, mode: RepresentationMode) -> Self
{
let mut fz = Vec::with_capacity(FUZZ_SIZE_HINT);
let normal_idx = fuzz(tag, &mut fz, &mode);
Self {
rule,
strings: fz,
normal_idx,
repr: mode,
}
}
}
/// Returned when an empty tag is tried to be parsed
#[derive(Debug)]
pub struct ParseError;
impl std::error::Error for ParseError{}
impl fmt::Display for ParseError
{
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result
{
write!(f, "cannot have empty tag rule")
}
}
impl str::FromStr for Tag
{
type Err = ParseError;
fn from_str(s: &str) -> std::result::Result<Self, Self::Err>
{
let mut chars = s.chars();
let mut next = String::new();
let rule = match chars.next() {
Some('+') => Rule::Optional,
Some('-') => Rule::Rejected,
Some(x) => {
next.push(x);
Rule::Required
},
_ => return Err(ParseError),
};
let s = chars.as_str();
if s.len() == 0 {
return Err(ParseError);
}
Ok(Self::new(if next.len() > 0 {
next.push_str(s);
next
} else {s.into()}, rule, RepresentationMode::default()))
}
}
impl<T> PartialEq<T> for Tag
where T: AsRef<str>
{
fn eq(&self, other: &T) -> bool
{
let mut fz = Vec::with_capacity(FUZZ_SIZE_HINT);
fuzz(other.as_ref(), &mut fz, &self.repr);
for (i,j) in self.strings.iter().zip(fz.into_iter())
{
if i == j.as_str() {
return true;
}
}
false
}
}
impl From<Tag> for String
{
fn from(mut tag: Tag) -> Self
{
mem::replace(&mut tag.strings[tag.normal_idx], Default::default())
}
}
impl fmt::Display for Tag
{
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result
{
write!(f, "{}", &self.strings[self.normal_idx])
}
}
#[derive(Debug)]
pub enum Error<'a>
{
Required(&'a Tag),
NoOptional,
Rejected(&'a Tag),
}
#[derive(Debug)]
pub enum OwnedError
{
Required(Tag),
NoOptional,
Rejected(Tag),
}
pub type Result<'a> = std::result::Result<Vec<&'a Tag>, Error<'a>>;
/// Search a set of strings for a matching ruleset for `tags`. Returns the matched tags on success
pub fn search<'a, T, U, V>(matches: T, tags: U) -> Result<'a>
where T: IntoIterator<Item=V>,
U: IntoIterator<Item=&'a Tag>,
V: AsRef<str>,
{
let tags: Vec<&'a Tag> = tags.into_iter().collect();
let mut output = Vec::with_capacity(tags.len());
let mut matched = HashSet::new();
for matches in matches.into_iter()
{
for tag in tags.iter() {
if *tag == &matches {
matched.insert(tag);
break;
}
}
}
let mut needs_opt = false;
let mut has_opt = false;
for tag in tags.iter()
{
if matched.contains(tag) {
match tag.rule {
Rule::Optional => {has_opt = true; needs_opt = true;},
Rule::Rejected => return Err(Error::Rejected(tag)),
_ => (),
}
output.push(tag.clone());
} else if tag.rule == Rule::Optional {
needs_opt = true;
} else if tag.rule == Rule::Required {
return Err(Error::Required(tag));
}
}
if needs_opt && !has_opt {
return Err(Error::NoOptional);
}
Ok(output)
}
/// Parse a string of many tag rules, ignoring empty or invalid entries
#[inline]
pub fn parse<T>(string: T) -> Vec<Tag>
where T: AsRef<str>
{
let string = string.as_ref();
parse_iter(string.split(" "))
}
/// Parse a collection of tag rules, ignoring empty or invalid entries
pub fn parse_iter<T,I>(from: I) -> Vec<Tag>
where I: IntoIterator<Item=T>,
T: AsRef<str>
{
from.into_iter()
.filter_map(|s| {
if let Ok(tag) = s.as_ref().parse() {
Some(tag)
} else {
None
}
}).collect()
}
impl fmt::Display for Error<'_>
{
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result
{
match self {
Self::NoOptional => write!(f, "there were no optional tags, at least one is needed"),
Self::Rejected(tag) => write!(f, "a rejected tag was present: {}", tag),
Self::Required(tag) => write!(f, "a required tag was not present: {}", tag),
}
}
}
impl fmt::Display for OwnedError
{
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result
{
match self {
Self::NoOptional => write!(f, "there were no optional tags, at least one is needed"),
Self::Rejected(tag) => write!(f, "a rejected tag was present: {}", tag),
Self::Required(tag) => write!(f, "a required tag was not present: {}", tag),
}
}
}
impl std::error::Error for Error<'_>{}
impl std::error::Error for OwnedError{}
impl From<Error<'_>> for OwnedError
{
fn from(er: Error<'_>) -> Self
{
match er {
Error::NoOptional => Self::NoOptional,
Error::Rejected(t) => Self::Rejected(t.to_owned()),
Error::Required(t) => Self::Required(t.to_owned()),
}
}
}

@ -15,7 +15,7 @@ mod tasklist;
mod progress;
/// Decode a loli from path
pub async fn decode(from: impl AsRef<Path>, to: impl AsRef<Path>, tags: impl AsRef<[tags::Tag]>, progress: &mut progress::CommandSender) -> Result<loli::Loli, error::Error>
pub async fn decode(from: impl AsRef<Path>, to: impl AsRef<Path>, progress: &mut progress::CommandSender) -> Result<loli::Loli, error::Error>
{
prog_send!(progress.println("Mapping child"));
let base = loli::BasedLoli::map(from)?;
@ -26,15 +26,7 @@ pub async fn decode(from: impl AsRef<Path>, to: impl AsRef<Path>, tags: impl AsR
//Find extension
let mut decoded = bounds.create_child(to.as_ref().with_extension(bounds.image().ext()))?;
let tags=tags.as_ref();
if tags.len() > 0 {
let res = tags::search(decoded.tags().iter(), tags)?;
if res.len() > 0 {
prog_send!(progress.println(format!("Matched tags {}", res.into_iter().map(|x| format!("{}", x)).join(", "))));
}
}
prog_send!(progress.println("Decoding..."));
prog_send!(progress.println("Decoding..."));
let sz = bounds.decode(&mut decoded)?;
prog_send!(link progress.println(format!("Decode complete ({} bytes)", sz)));
@ -76,7 +68,6 @@ pub async fn perform(url: impl AsRef<str>, path: impl AsRef<Path>, progress: &mu
prog_send!(progress.bump(slice.len() as u64));
}
}
file.flush().await?;
if len.is_none() {
prog_send!(progress.bump(1));
}
@ -86,33 +77,20 @@ pub async fn perform(url: impl AsRef<str>, path: impl AsRef<Path>, progress: &mu
Ok(())
}
pub async fn work(conf: config::Config) -> Result<(), Box<dyn std::error::Error>>
{
let rating = conf.rating;
let mut children = Vec::with_capacity(conf.output.len());
let (mut prog_writer, prog) = match conf.verbose {
config::Verbosity::Full => {
let prog = progress::AsyncProgressCounter::<termprogress::progress::Bar>::new("Initialising...", 1);
let prog_writer = prog.writer();
let prog = prog.host();
(prog_writer, prog)
},
config::Verbosity::Silent => {
let prog = progress::AsyncProgressCounter::<progress::Silent>::new("Initialising...", 1);
let prog_writer = prog.writer();
let prog = prog.host();
(prog_writer, prog)
},
};
let prog = progress::AsyncProgressCounter::new("Initialising...", 1);
let mut prog_writer = prog.writer();
let prog = prog.host();
for path in conf.output.into_iter()
{
let url = url::parse(&rating);
let mut prog = prog_writer.clone_with(format!("-> {:?}", path));
let tags = conf.tags.clone();
children.push(tokio::task::spawn(async move {
prog.println(format!("Starting download ({})...", url)).await.expect("fatal");
@ -120,7 +98,7 @@ pub async fn work(conf: config::Config) -> Result<(), Box<dyn std::error::Error>
prog_send!(link unwind prog.push_task(&task));
let temp = tempfile::TempFile::new();
let return_value = loop {
let return_value = 'clean: {
match perform(&url, &temp, &mut prog).await {
Err(e) => prog_send!(link prog.println(format!("Failed downloading {} -> {:?}: {}", url, temp, e))),
Ok(_) => {
@ -128,19 +106,19 @@ pub async fn work(conf: config::Config) -> Result<(), Box<dyn std::error::Error>
config::OutputType::File(file) => file,
config::OutputType::Directory(dir) => unimplemented!(), //TODO: implement get hash to file
};
let loli = match decode(&temp, &path, &tags, &mut prog).await {
let loli = match decode(&temp, &path, &mut prog).await {
Ok(v) => v,
Err(e) => {
prog_send!(link prog.println(format!("Failed decoding: {}", e)));
break Some(e);
break 'clean false;
},
};
prog_send!(link prog.println(format!("{:?} Complete", loli)));
break None;
break 'clean true;
},
};
break Some(error::Error::Unknown);
false
};
prog_send!(link prog.pop_task(task));
@ -151,28 +129,21 @@ pub async fn work(conf: config::Config) -> Result<(), Box<dyn std::error::Error>
prog_send!(link prog_writer.println("Children working..."));
let mut done =0;
let total = children.len();
let mut failures = Vec::with_capacity(children.len());
for child in children.into_iter()
{
match child.await {
Ok(None) => done+=1,
Ok(Some(err)) => failures.push(err),
Ok(true) => done+=1,
Err(err) => {
prog_send!(try link unwind prog_writer.println(format!("Child panic: {}", err)));
failures.push(error::Error::ChildPanic);
},
_ => (),
}
}
prog_send!(link prog_writer.set_title(""));
prog_send!(try link prog_writer.kill());
prog.await.expect("mpsc fatal");
println!("Completed {} / {} lolis ({} failed).", done, total, total-done);
if failures.len() > 0 {
println!("Reasons for failure(s):");
for failure in failures.into_iter() {
eprintln!("\t{}", failure);
}
}
Ok(())
}

@ -22,7 +22,6 @@ use termprogress::{
enum CommandInternal
{
None,
PrintLine(String),
BumpMax(u64),
Bump(u64),
@ -48,8 +47,7 @@ impl std::ops::Drop for Command
}
}
pub struct AsyncProgressCounter<T>
where T: termprogress::ProgressBar + WithTitle + std::marker::Send + std::marker::Sync +'static,
pub struct AsyncProgressCounter
{
writer: Sender<Command>,
reader: Receiver<Command>,
@ -57,8 +55,6 @@ pub struct AsyncProgressCounter<T>
small: u64,
large: u64,
_phantom: std::marker::PhantomData<T>,
}
#[derive(Clone)]
@ -230,8 +226,7 @@ impl CommandCallback
}
}
impl<T> AsyncProgressCounter<T>
where T: termprogress::ProgressBar + WithTitle + std::marker::Send + std::marker::Sync +'static,
impl AsyncProgressCounter
{
/// Create a new `AsyncProgressCounter`
pub fn new(title: impl Into<String>, max: u64) -> Self
@ -244,8 +239,6 @@ impl<T> AsyncProgressCounter<T>
title: title.into(),
small: 0,
large: max,
_phantom: std::marker::PhantomData,
}
}
@ -264,7 +257,7 @@ impl<T> AsyncProgressCounter<T>
pub fn host(mut self) -> JoinHandle<()>
{
//let mut spin = spinner::Spin::with_title(&self.title[..], Default::default());
let mut bar = T::with_title(50, &self.title[..]);
let mut bar = termprogress::progress::Bar::with_title(50, &self.title[..]);
bar.update();
@ -280,8 +273,8 @@ impl<T> AsyncProgressCounter<T>
(small as f64) / (large as f64)
}
while let Some(mut com) = self.reader.recv().await {
match std::mem::replace(&mut com.internal, CommandInternal::None) {
while let Some(com) = self.reader.recv().await {
match &com.internal {
CommandInternal::PrintLine(line) => bar.println(&line[..]),
CommandInternal::BumpMax(by) => {
large += by;
@ -317,62 +310,10 @@ impl<T> AsyncProgressCounter<T>
CommandInternal::ClearTask(Some(title)) => {
task.clear();
bar.set_title(&title[..]);
self.title = title;
self.title = title.clone();
},
_ => {},
}
}
})
}
}
pub trait WithTitle: Sized + ProgressBar + Display
{
fn with_title(len: usize, string: impl AsRef<str>) -> Self;
fn update(&mut self);
fn complete(self);
}
impl WithTitle for termprogress::progress::Bar
{
fn with_title(len: usize, string: impl AsRef<str>) -> Self
{
Self::with_title(len, string)
}
fn update(&mut self)
{
self.update();
}
fn complete(self)
{
self.complete();
}
}
#[derive(Debug)]
pub struct Silent;
impl termprogress::Display for Silent
{
fn println(&self, _: &str){}
fn refresh(&self){}
fn blank(&self){}
fn get_title(&self) -> &str{""}
fn set_title(&mut self, _: &str){}
fn update_dimensions(&mut self, _:usize){}
}
impl termprogress::ProgressBar for Silent
{
fn set_progress(&mut self, _:f64){}
fn get_progress(&self) -> f64{0.0}
}
impl WithTitle for Silent
{
fn with_title(_: usize, _: impl AsRef<str>) -> Self{Self}
fn update(&mut self) {}
fn complete(self) {}
}

@ -1,20 +1,12 @@
use super::*;
#[cfg(nightly)]
use std::{
collections::LinkedList,
};
#[cfg(not(nightly))]
use linked_list::LinkedList;
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct TaskList(LinkedList<(usize, String)>, String, usize);
#[cfg(not(nightly))]
unsafe impl std::marker::Send for TaskList{}
#[cfg(not(nightly))]
unsafe impl std::marker::Sync for TaskList{}
fn find<T,F>(list: &LinkedList<T>, mut fun: F) -> Option<usize>
where F: FnMut(&T) -> bool
{

Loading…
Cancel
Save