You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

638 lines
19 KiB

using System;
using System.Collections.Generic;
using System.IO;
using System.Text;
using System.Threading;
using System.Threading.Channels;
using System.Threading.Tasks;
using System.Linq;
using napdump;
using Tools;
namespace ndview
{
class PageGenerator
{
public napdump.BoardInfo Board { get; }
public DirectoryInfo Images { get; }
public PageGenerator(napdump.BoardInfo board, DirectoryInfo images)
{
Board = board;
Images = images;
}
private async Task<string> ExtractImage(PostInfo post, DirectoryInfo imgOutput, bool encrypted, CancellationToken token)
{
if (post.ImageURL == null) throw new InvalidOperationException($"Post {post.PostNumber} does not have an image.");
var imageFn = Path.Join(Images.FullName, post.PostNumber.ToString());
if (File.Exists(imageFn))
{
var ifn = post.ImageURL.Split('/').Last();
await Task.Yield();
var ofn = Path.Join(imgOutput.FullName, ifn);
if (encrypted && Program.DeletedKey!=null)
{
Console.WriteLine($"Extracting encrypted image {post.PostNumber} -> {ofn}");
using (var inp = new FileStream(imageFn, FileMode.Open, FileAccess.Read))
{
using (var oup = new FileStream(ofn, FileMode.Create))
{
using (var dec = new encaes.AesEncryptor(inp) { KeepAlive = true })
{
dec.Key = Program.DeletedKey.Value;
await dec.Decrypt(oup, token);
}
}
}
}
else
{
Console.WriteLine($"Extracting image {post.PostNumber} -> {ofn}");
if (encrypted)
{
WriteYellowLine("Warning: Image for post "+post.PostNumber+" will be garbage: Cannot be decrypted.");
}
await CopyFileAsync(imageFn, ofn, token);
}
return ifn;
}
else throw new InvalidOperationException("Bad state: no image for " + post.PostNumber);
}
private static async Task CopyFileAsync(string from, string to, CancellationToken cancel = default)
{
using var ifs = new FileStream(from, FileMode.Open, FileAccess.Read);
using var ofs = new FileStream(to, FileMode.Create);
await ifs.CopyToAsync(ofs, cancel);
}
private async Task WritePost(HtmlGenerator index, PostInfo post, DirectoryInfo img, bool wasEnc, CancellationToken token)
{
bool postImgEnc;
if (postImgEnc = post.IsImageEncrypted)
{
if (Program.DeletedKey == null)
{
WriteYellowLine("Post " + post.PostNumber + " image is encrypted, not attempting to write.");
}
else
{
try
{
await post.DecryptImageAsync(Program.DeletedKey.Value, token);
}
catch (OperationCanceledException op) { throw op; }
catch (Exception ex)
{
WriteYellowLine("Failed to decrypt post " + post.PostNumber + " image, skipping: " + ex.Message);
return;
}
}
}
var imageExtractor = !post.IsImageEncrypted && post.ImageURL != null ? ExtractImage(post, img, postImgEnc || wasEnc, token) : Task.CompletedTask;
await using (await index.TagAsync("article", token, ("id", post.PostNumber.ToString())))
{
await WriteHeader(index, post, token);
if (post.ImageURL != null)
await WriteImageFigure(index, post, token);
await using (await index.TagAsync("blockquote", token))
{
await index.AppendHtml(post.Body);
}
}
await imageExtractor;
}
private static string HumanBytes(long byteCount)
{
string[] suf = { "B", "KB", "MB", "GB", "TB", "PB", "EB" }; //Longs run out around EB
if (byteCount == 0)
return "0" + suf[0];
long bytes = Math.Abs(byteCount);
int place = Convert.ToInt32(Math.Floor(Math.Log(bytes, 1024)));
double num = Math.Round(bytes / Math.Pow(1024, place), 1);
return (Math.Sign(byteCount) * num).ToString() + " " + suf[place];
}
private async Task WriteImageFigure(HtmlGenerator index, PostInfo thread, CancellationToken token)
{
if (thread.ImageURL == null) throw new InvalidOperationException($"Post {thread.PostNumber} has no image");
var ifn = thread.ImageURL.Split('/').Last();
await using (await index.TagAsync("figure", token))
{
await using (await index.TagAsync("figcaption"))
{
await using (await index.TagAsync("i"))
{
await index.Append($@"({HumanBytes(thread.ImageSize)} {thread.ImageDimensions.Width}x{thread.ImageDimensions.Height})");
await using (await index.TagAsync("a", token, ("rel", "nofollow"), ("title", thread.ImageFilename), ("download", thread.ImageFilename), ("href", $"i/{ifn}")))
{
await index.Append(thread.ImageFilename);
}
}
}
await index.TagSelfClosingAsync("img", token, ("src", $"i/{ifn}"), ("height", "250"), ("width", "250"));
}
}
private async Task WriteHeader(HtmlGenerator index, PostInfo thread, CancellationToken token)
{
await using (await index.TagAsync("header", token))
{
if (thread.Subject != null)
{
await using (await index.TagAsync("h3"))
{
await index.Append(thread.Subject, token);
}
await index.Append(" ", token);
}
await using (await index.TagAsync("b", token, ("class", "name")))
{
var emailTag = (thread.Email == null || thread.Email == "") ? null : await index.TagAsync("a", token, ("class", "email"), ("href", "mailto:" + thread.Email));
await index.Append(thread.Name, token);
if (thread.Tripcode != null)
{
await index.AppendHtml("<span class='tripcode'>", token);
await index.Append(thread.Tripcode, token);
await index.AppendHtml("</span> ", token);
}
if (thread.Capcode != null)
{
await index.AppendHtml("<span class='capcode'>", token);
await index.Append(thread.Capcode, token);
await index.AppendHtml("</span> ", token);
}
if (emailTag != null)
await emailTag.DisposeAsync();
}
await index.Append(" ", token);
await using (await index.TagAsync("time", token, ("title", thread.Timestamp.ToString())))
{
await index.Append(thread.Timestamp.ToShortDateString(), token);
}
await index.Append(" ", token);
await using (await index.TagAsync("a", token, ("href", "#" + thread.PostNumber)))
{
await index.Append(thread.PostNumber.ToString(), token);
}
}
}
private async Task WriteBody(HtmlGenerator index, PostInfo post, CancellationToken token)
{
await using (await index.TagAsync("blockquote", token))
{
await index.AppendHtml(post.Body);
}
}
private async Task WriteThread(HtmlGenerator index, ThreadInfo thread, DirectoryInfo img, bool wasEnc, CancellationToken token)
{
bool threadImgEnc;
if (threadImgEnc = thread.IsImageEncrypted)
{
if (Program.DeletedKey == null)
{
WriteYellowLine("Thread " + thread.PostNumber + " image is encrypted, not attempting to write.");
}
else
{
try
{
await thread.DecryptImageAsync(Program.DeletedKey.Value, token);
}
catch (OperationCanceledException op) { throw op; }
catch (Exception ex)
{
WriteYellowLine("Failed to decrypt thread " + thread.PostNumber + " image, skipping: " + ex.Message);
return;
}
}
}
var imgExtract = !thread.IsImageEncrypted && thread.ImageURL != null ? ExtractImage(thread, img, threadImgEnc || wasEnc, token) : Task.CompletedTask;
if (thread.ImageURL == null)
{
var ocol = Console.ForegroundColor;
Console.ForegroundColor = ConsoleColor.Yellow;
Console.WriteLine("Warning: Thread OP " + thread.PostNumber + " has no image.");
Console.ForegroundColor = ocol;
}
await using (await index.TagAsync("section", token, ("id", thread.PostNumber.ToString()), ("class", "thread")))
{
await WriteHeader(index, thread, token);
if (thread.ImageURL != null)
await WriteImageFigure(index, thread, token);
await WriteBody(index, thread, token);
await using (await index.TagAsync("small", token))
{
await index.Append($"{thread.Children.Count} replies", token);
}
//Write posts
await using (await index.TagAsync("div", token, ("class", "replies")))
foreach (var post in thread.Children.OrderBy(x => x.PostNumber))
{
bool pWasEnc;
if (pWasEnc = post.IsEncrypted)
{
if (Program.DeletedKey == null)
{
WriteYellowLine("Post encrypted, skipping.");
continue;
}
else
{
try
{
await post.DecryptPostAsync(Program.DeletedKey.Value, token);
}
catch (Exception ex)
{
WriteYellowLine("Failed to decrypt post, skipping: " + ex.Message);
continue;
}
}
}
await WritePost(index, post, img, pWasEnc, token);
}
}
await imgExtract;
}
public async Task GenerateFull(DirectoryInfo output, CancellationToken cancel)
{
await using var index = HtmlGenerator.CreateFile(Path.Combine(output.FullName, "index.html"), 10, cancel);
var imageOutputDir = new DirectoryInfo(Path.Combine(output.FullName, "i"));
imageOutputDir.Create();
index.Metas.Add(@"<meta charset='utf-8' />");
index.Metas.Add(@"<meta name='viewport' content='width=device-width, initial-scale=1.0' />");
await index.AppendHeaders(async (token) =>
{
await using (await index.TagAsync("title", token))
await index.Append($"Archive of `{Board.Title}'");
await using (await index.TagAsync("style", token))
await index.AppendHtml(Properties.Resources.css);
await using (await index.TagAsync("h1", token))
{
await index.Append(Board.Title, token);
}
await using (await index.TagAsync("div", cancel, ("class", "stat")))
{
await index.Append($"Showing {Board.Threads.Count} threads containing {Board.Threads.Select(x => x.Children.Count).Sum()} posts and {Board.Threads.Count + Board.Threads.Select(x => x.Children.Where(y => y.ImageURL != null).Count()).Sum()} images.", cancel);
await index.AppendHtml($"<br />Taken at <time>{Board.DumpTimestamp.ToString()}</time>.<br />Original: <a href='{Board.BoardURL}'>", cancel);
await index.Append(Board.BoardName, cancel);
await index.AppendHtml($"</a>", cancel);
}
await index.TagSelfClosingAsync("hr", cancel);
}, cancel);
await using (await index.TagAsync("body", cancel))
{
await using (await index.TagAsync("main", cancel))
{
foreach (var thread in Board.Threads.OrderByDescending(x => x.Children.Where(x => !(x.Email?.Equals("sage") ?? false)).LastOrDefault()?.PostNumber ?? x.PostNumber))
{
bool wasEnc;
if (wasEnc = thread.IsEncrypted)
{
if (Program.DeletedKey == null)
{
WriteYellowLine("Thread is encrypted, skipping.");
continue;
}
else
{
try
{
await thread.DecryptPostAsync(Program.DeletedKey.Value, cancel);
}
catch (OperationCanceledException op) { throw op; }
catch (Exception ex)
{
WriteYellowLine("Failed to decrypt thread, skipping: " + ex.Message);
continue;
}
}
}
await WriteThread(index, thread, imageOutputDir, wasEnc, cancel);
}
}
}
await index.AppendHtml(@$"<hr /><footer>Page generated on <time>{DateTime.Now.ToString()}</time> by <a href='https://public.flanchan.moe/#napdump'>ndview</a></footer>", cancel);
await using (await index.TagAsync("script", cancel))
await index.AppendHtml(Properties.Resources.script);
}
private static void WriteYellowLine(string value)
{
var kkey = Console.ForegroundColor;
Console.ForegroundColor = ConsoleColor.Yellow;
Console.WriteLine(value);
Console.ForegroundColor = kkey;
}
}
public readonly struct Safe<T>
{
public readonly T Value;
public Safe(T value) => Value = value;
public static explicit operator T(Safe<T> t) => t.Value;
}
public readonly struct HtmlAttribute
{
public readonly string Name, Value;
public HtmlAttribute(string nm, string vl)
{
Name = nm;
Value = vl.Replace("\'", "\\\'");
}
public override string ToString()
{
return $"{Name}='{Value}'";
}
public static implicit operator HtmlAttribute((string Name, string Value) from)
{
return new HtmlAttribute(from.Name, from.Value);
}
public void Deconstruct(out string name, out string value)
{
name = Name;
value = Value;
}
}
public class HtmlTag : IAsyncDisposable
{
public string Name { get; }
public HtmlAttribute[] Attributes { get; }
public HtmlGenerator Owner { get; }
public HtmlTag(HtmlGenerator super, string tname, params HtmlAttribute[] attrs)
{
Owner = super;
Name = tname;
Attributes = attrs;
}
public string AttributeString
=> Attributes.Length < 1 ? "" : (" " + string.Join(' ', Attributes.Select(x => x.ToString()).ToArray()));
public async Task SelfClose(CancellationToken token)
{
if (disposed) return;
disposed = true;
await Owner.AppendHtml($"<{Name}{AttributeString} />");
}
internal async Task Begin(CancellationToken token)
{
await Owner.AppendHtml($"<{Name}{AttributeString}>");
}
private bool disposed = false;
public async ValueTask DisposeAsync()
{
if (disposed) return;
disposed = true;
await Owner.AppendHtml($"</{Name}>");
}
}
public class HtmlGenerator : IDisposable, IAsyncDisposable
{
public HtmlTag Tag(string name, params HtmlAttribute[] attrs)
{
var t = new HtmlTag(this, name, attrs);
t.Begin(cancel.Token).Wait();
return t;
}
public async Task<HtmlTag> TagAsync(string name, params HtmlAttribute[] attrs)
{
var t = new HtmlTag(this, name, attrs);
await t.Begin(cancel.Token);
return t;
}
public async Task<HtmlTag> TagAsync(string name, CancellationToken token, params HtmlAttribute[] attrs)
{
using var link = CancellationTokenSource.CreateLinkedTokenSource(cancel.Token, token);
var t = new HtmlTag(this, name, attrs);
await t.Begin(link.Token);
return t;
}
public async Task<HtmlTag> TagAsync(string name, HtmlAttribute[] attrs, CancellationToken token = default)
{
using var link = CancellationTokenSource.CreateLinkedTokenSource(cancel.Token, token);
var t = new HtmlTag(this, name, attrs);
await t.Begin(link.Token);
return t;
}
public async Task<HtmlTag> TagAsync(string name, CancellationToken token = default)
{
using var link = CancellationTokenSource.CreateLinkedTokenSource(cancel.Token, token);
var t = new HtmlTag(this, name);
await t.Begin(link.Token);
return t;
}
public Task TagSelfClosingAsync(string name, params HtmlAttribute[] attrs)
=> TagSelfClosingAsync(name, default, attrs);
public async Task TagSelfClosingAsync(string name, CancellationToken cancel, params HtmlAttribute[] attrs)
{
using var link = CancellationTokenSource.CreateLinkedTokenSource(this.cancel.Token, cancel);
await using var t = new HtmlTag(this, name, attrs);
await t.SelfClose(cancel);
}
protected readonly Stream stream;
public StreamWriter Output { get; }
public bool OwnsStream { get; set; } = false;
protected readonly CancellationTokenSource cancel;
public List<string> Metas { get; } = new List<String>();
public Dispatcher OnFlush = new Dispatcher() { InSeries = true };
public Task AppendHeaders(CancellationToken token = default)
=> AppendHeaders((x) => Task.CompletedTask, token);
public async Task AppendHeaders(Func<CancellationToken, Task> head, CancellationToken token = default)
{
using var cancel = CancellationTokenSource.CreateLinkedTokenSource(this.cancel.Token, token);
await AppendHtml(@"<!DOCTYPE html><html><head>", cancel.Token);
await Task.WhenAll(Metas.Select(x => AppendHtml(x, cancel.Token)));
await head(cancel.Token);
token.ThrowIfCancellationRequested();
await AppendHtml(@"</head>", cancel.Token);
await OnFlush.HookAsync(async (c) =>
{
await AppendHtml("</html>", c);
}, token);
}
public static HtmlGenerator CreateFile(string file, int bufferLength = 0, CancellationToken globalToken = default)
{
var fs = new FileStream(file, FileMode.Create);
try
{
return new HtmlGenerator(fs, bufferLength, globalToken) { OwnsStream = true };
}
catch (Exception ex)
{
fs.Dispose();
throw ex;
}
}
public HtmlGenerator(Stream output, int bufferLength = 0, CancellationToken token = default)
{
stream = output;
cancel = CancellationTokenSource.CreateLinkedTokenSource(token);
Output = new StreamWriter(output);
var chan = bufferLength < 1 ? Channel.CreateUnbounded<object>() : Channel.CreateBounded<object>(bufferLength);
reader = chan.Reader;
writer = chan.Writer;
writeHook = Task.Run(async () =>
{
try
{
await foreach (var obj in reader.ReadAllAsync(cancel.Token))
{
if (obj is Safe<string> safe)
{
await Output.WriteAsync(safe.Value.AsMemory(), cancel.Token);
}
else
await Output.WriteAsync(System.Text.Encodings.Web.HtmlEncoder.Default.Encode(obj is string str ? str : obj?.ToString() ?? "").AsMemory(), cancel.Token);
}
}
catch (OperationCanceledException) { }
catch (Exception ex)
{
OnWriterError?.Invoke(ex);
CancelAllOperations();
}
});
}
public async ValueTask DisposeAsync()
{
await Flush();
await OnFlush.WaitForCurrentSignal;
Dispose();
}
public event Action<Exception> OnWriterError;
private readonly ChannelReader<object> reader;
protected readonly ChannelWriter<object> writer;
public void CancelAllOperations()
{
if (!cancel.IsCancellationRequested)
cancel.Cancel();
}
protected Task writeHook;
public Task Completion => writeHook;
protected bool flushed = false;
public async Task Flush(CancellationToken token = default)
{
if (flushed) return;
flushed = true;
await OnFlush.Signal(token);
writer.Complete();
await writeHook;
}
public async Task Append(string str, CancellationToken token = default)
{
using var cancel = CancellationTokenSource.CreateLinkedTokenSource(this.cancel.Token, token);
await writer.WriteAsync(str, cancel.Token);
}
public async Task Append(Safe<string> str, CancellationToken token = default)
{
using var cancel = CancellationTokenSource.CreateLinkedTokenSource(this.cancel.Token, token);
await writer.WriteAsync(str, cancel.Token);
}
public async Task AppendHtml(string str, CancellationToken token = default)
{
using var cancel = CancellationTokenSource.CreateLinkedTokenSource(this.cancel.Token, token);
await writer.WriteAsync(new Safe<string>(str), cancel.Token);
}
#region IDisposable Support
private bool disposedValue = false; // To detect redundant calls
protected virtual void Dispose(bool disposing)
{
if (!disposedValue)
{
if (disposing)
{
if (!cancel.IsCancellationRequested)
cancel.Cancel();
Output.Flush();
OnFlush.Dispose();
cancel.Dispose();
Output.Close();
if (OwnsStream) stream.Dispose();
}
disposedValue = true;
}
}
~HtmlGenerator()
{
Dispose(false);
}
public void Dispose()
{
Dispose(true);
GC.SuppressFinalize(this);
}
#endregion
}
}