using System; using System.Collections.Generic; using System.IO; using System.Text; using System.Threading; using System.Threading.Channels; using System.Threading.Tasks; using System.Linq; using napdump; using Tools; namespace ndview { class PageGenerator { public napdump.BoardInfo Board { get; } public DirectoryInfo Images { get; } private ThumbnailGenerator Thumbnailer {get;} = new ThumbnailGenerator(200); public PageGenerator(napdump.BoardInfo board, DirectoryInfo images) { Board = board; Images = images; } private DirectoryInfo ThumbnailDir(DirectoryInfo imageDir) { var exp = new DirectoryInfo(Path.Combine(imageDir.FullName, "thumb")); if(!exp.Exists) exp.Create(); return exp; } private async Task ExtractImage(PostInfo post, DirectoryInfo imgOutput, bool encrypted, CancellationToken token) { if (post.ImageURL == null) throw new InvalidOperationException($"Post {post.PostNumber} does not have an image."); var imageFn = Path.Join(Images.FullName, post.PostNumber.ToString()); if (File.Exists(imageFn)) { var ifn = post.ImageURL.Split('/').Last(); await Task.Yield(); var ofn = Path.Join(imgOutput.FullName, ifn); var thumbofn = Path.Join(ThumbnailDir(imgOutput).FullName, ifn); if (encrypted && Program.DeletedKey!=null) { Console.WriteLine($"Extracting encrypted image {post.PostNumber} -> {ofn}"); using (var inp = new FileStream(imageFn, FileMode.Open, FileAccess.Read)) { await using (var oup = new FileStream(ofn, FileMode.Create)) { using (var dec = new encaes.AesEncryptor(inp) { KeepAlive = true }) { dec.Key = Program.DeletedKey.Value; await dec.Decrypt(oup, token); } await oup.FlushAsync(); oup.Position=0; await using(var thumbo = new FileStream(thumbofn, FileMode.Create)) { await Thumbnailer.Thumbnail(oup, thumbo, token); } } } } else { Console.WriteLine($"Extracting image {post.PostNumber} -> {ofn}"); if (encrypted) { WriteYellowLine("Warning: Image for post "+post.PostNumber+" will be garbage: Cannot be decrypted."); } await CopyFileAsync(imageFn, ofn, token); await using(var inp = new FileStream(imageFn, FileMode.Open, FileAccess.Read)) { await using(var oup = new FileStream(thumbofn, FileMode.Create)) { await Thumbnailer.Thumbnail(inp, oup, token); } } } return ifn; } else throw new InvalidOperationException("Bad state: no image for " + post.PostNumber); } private static async Task CopyFileAsync(string from, string to, CancellationToken cancel = default) { using var ifs = new FileStream(from, FileMode.Open, FileAccess.Read); using var ofs = new FileStream(to, FileMode.Create); await ifs.CopyToAsync(ofs, cancel); } private async Task WritePost(HtmlGenerator index, PostInfo post, DirectoryInfo img, bool wasEnc, CancellationToken token) { bool postImgEnc; if (postImgEnc = post.IsImageEncrypted) { if (Program.DeletedKey == null) { WriteYellowLine("Post " + post.PostNumber + " image is encrypted, not attempting to write."); } else { try { await post.DecryptImageAsync(Program.DeletedKey.Value, token); } catch (OperationCanceledException op) { throw op; } catch (Exception ex) { WriteYellowLine("Failed to decrypt post " + post.PostNumber + " image, skipping: " + ex.Message); return; } } } var imageExtractor = !post.IsImageEncrypted && post.ImageURL != null ? ExtractImage(post, img, postImgEnc || wasEnc, token) : Task.CompletedTask; await using (await index.TagAsync("article", token, ("id", post.PostNumber.ToString()))) { await WriteHeader(index, post, token); if (post.ImageURL != null) await WriteImageFigure(index, post, token); await using (await index.TagAsync("blockquote", token)) { await index.AppendHtml(post.Body); } } await imageExtractor; } private static string HumanBytes(long byteCount) { string[] suf = { "B", "KB", "MB", "GB", "TB", "PB", "EB" }; //Longs run out around EB if (byteCount == 0) return "0" + suf[0]; long bytes = Math.Abs(byteCount); int place = Convert.ToInt32(Math.Floor(Math.Log(bytes, 1024))); double num = Math.Round(bytes / Math.Pow(1024, place), 1); return (Math.Sign(byteCount) * num).ToString() + " " + suf[place]; } private async Task WriteImageFigure(HtmlGenerator index, PostInfo thread, CancellationToken token) { if (thread.ImageURL == null) throw new InvalidOperationException($"Post {thread.PostNumber} has no image"); var ifn = thread.ImageURL.Split('/').Last(); await using (await index.TagAsync("figure", token)) { await using (await index.TagAsync("figcaption")) { await using (await index.TagAsync("i")) { await index.Append($@"({HumanBytes(thread.ImageSize)} {thread.ImageDimensions.Width}x{thread.ImageDimensions.Height})"); await using (await index.TagAsync("a", token, ("rel", "nofollow"), ("title", thread.ImageFilename), ("download", thread.ImageFilename), ("href", $"i/{ifn}"))) { await index.Append(thread.ImageFilename); } } } await index.TagSelfClosingAsync("img", token, ("src", $"i/thumb/{ifn}"), ("width", Thumbnailer.Width.ToString())); } } private async Task WriteHeader(HtmlGenerator index, PostInfo thread, CancellationToken token) { await using (await index.TagAsync("header", token)) { if (thread.Subject != null) { await using (await index.TagAsync("h3")) { await index.Append(thread.Subject, token); } await index.Append(" ", token); } await using (await index.TagAsync("b", token, ("class", "name"))) { var emailTag = (thread.Email == null || thread.Email == "") ? null : await index.TagAsync("a", token, ("class", "email"), ("href", "mailto:" + thread.Email)); await index.Append(thread.Name, token); if (thread.Tripcode != null) { await index.AppendHtml("", token); await index.Append(thread.Tripcode, token); await index.AppendHtml(" ", token); } if (thread.Capcode != null) { await index.AppendHtml("", token); await index.Append(thread.Capcode, token); await index.AppendHtml(" ", token); } if (emailTag != null) await emailTag.DisposeAsync(); } await index.Append(" ", token); await using (await index.TagAsync("time", token, ("title", thread.Timestamp.ToString()))) { await index.Append(thread.Timestamp.ToShortDateString(), token); } await index.Append(" ", token); await using (await index.TagAsync("a", token, ("href", "#" + thread.PostNumber))) { await index.Append(thread.PostNumber.ToString(), token); } } } private async Task WriteBody(HtmlGenerator index, PostInfo post, CancellationToken token) { await using (await index.TagAsync("blockquote", token)) { await index.AppendHtml(post.Body); } } private async Task WriteThread(HtmlGenerator index, ThreadInfo thread, DirectoryInfo img, bool wasEnc, CancellationToken token) { bool threadImgEnc; if (threadImgEnc = thread.IsImageEncrypted) { if (Program.DeletedKey == null) { WriteYellowLine("Thread " + thread.PostNumber + " image is encrypted, not attempting to write."); } else { try { await thread.DecryptImageAsync(Program.DeletedKey.Value, token); } catch (OperationCanceledException op) { throw op; } catch (Exception ex) { WriteYellowLine("Failed to decrypt thread " + thread.PostNumber + " image, skipping: " + ex.Message); return; } } } var imgExtract = !thread.IsImageEncrypted && thread.ImageURL != null ? ExtractImage(thread, img, threadImgEnc || wasEnc, token) : Task.CompletedTask; if (thread.ImageURL == null) { var ocol = Console.ForegroundColor; Console.ForegroundColor = ConsoleColor.Yellow; Console.WriteLine("Warning: Thread OP " + thread.PostNumber + " has no image."); Console.ForegroundColor = ocol; } await using (await index.TagAsync("section", token, ("id", thread.PostNumber.ToString()), ("class", "thread"))) { await index.AppendHtml($@""); await WriteHeader(index, thread, token); if (thread.ImageURL != null) await WriteImageFigure(index, thread, token); await WriteBody(index, thread, token); await using (await index.TagAsync("small", token)) { await index.Append($"{thread.Children.Count} replies", token); } //Write posts await using (await index.TagAsync("div", token, ("class", "replies"))) foreach (var post in thread.Children.OrderBy(x => x.PostNumber)) { bool pWasEnc; if (pWasEnc = post.IsEncrypted) { if (Program.DeletedKey == null) { WriteYellowLine("Post encrypted, skipping."); continue; } else { try { await post.DecryptPostAsync(Program.DeletedKey.Value, token); } catch (Exception ex) { WriteYellowLine("Failed to decrypt post, skipping: " + ex.Message); continue; } } } await WritePost(index, post, img, pWasEnc, token); } } await imgExtract; } public async Task GenerateFull(DirectoryInfo output, CancellationToken cancel) { await using var index = HtmlGenerator.CreateFile(Path.Combine(output.FullName, "index.html"), 10, cancel); var imageOutputDir = new DirectoryInfo(Path.Combine(output.FullName, "i")); imageOutputDir.Create(); index.Metas.Add(@""); index.Metas.Add(@""); await index.AppendHeaders(async (token) => { await using (await index.TagAsync("title", token)) await index.Append($"Archive of `{Board.Title}'"); await using (await index.TagAsync("style", token)) await index.AppendHtml(Properties.Resources.css); await using (await index.TagAsync("h1", token)) { await index.Append(Board.Title, token); } await using (await index.TagAsync("div", cancel, ("class", "stat"))) { await index.Append($"Showing {Board.Threads.Count} threads containing {Board.Threads.Select(x => x.Children.Count).Sum()} posts and {Board.Threads.Count + Board.Threads.Select(x => x.Children.Where(y => y.ImageURL != null).Count()).Sum()} images.", cancel); await index.AppendHtml($"
Taken at .
Original: ", cancel); await index.Append(Board.BoardName, cancel); await index.AppendHtml($"", cancel); } await index.TagSelfClosingAsync("hr", cancel); }, cancel); await using (await index.TagAsync("body", cancel)) { await using (await index.TagAsync("main", cancel)) { foreach (var thread in Board.Threads.OrderByDescending(x => x.Children.Where(x => !(x.Email?.Equals("sage") ?? false)).LastOrDefault()?.PostNumber ?? x.PostNumber)) { bool wasEnc; if (wasEnc = thread.IsEncrypted) { if (Program.DeletedKey == null) { WriteYellowLine("Thread is encrypted, skipping."); continue; } else { try { await thread.DecryptPostAsync(Program.DeletedKey.Value, cancel); } catch (OperationCanceledException op) { throw op; } catch (Exception ex) { WriteYellowLine("Failed to decrypt thread, skipping: " + ex.Message); continue; } } } await WriteThread(index, thread, imageOutputDir, wasEnc, cancel); } } } await index.AppendHtml(@$"
", cancel); await using (await index.TagAsync("script", cancel)) await index.AppendHtml(Properties.Resources.script); } private static void WriteYellowLine(string value) { var kkey = Console.ForegroundColor; Console.ForegroundColor = ConsoleColor.Yellow; Console.WriteLine(value); Console.ForegroundColor = kkey; } } public readonly struct Safe { public readonly T Value; public Safe(T value) => Value = value; public static explicit operator T(Safe t) => t.Value; } public readonly struct HtmlAttribute { public readonly string Name, Value; public HtmlAttribute(string nm, string vl) { Name = nm; Value = vl.Replace("\'", "\\\'"); } public override string ToString() { return $"{Name}='{Value}'"; } public static implicit operator HtmlAttribute((string Name, string Value) from) { return new HtmlAttribute(from.Name, from.Value); } public void Deconstruct(out string name, out string value) { name = Name; value = Value; } } public class HtmlTag : IAsyncDisposable { public string Name { get; } public HtmlAttribute[] Attributes { get; } public HtmlGenerator Owner { get; } public HtmlTag(HtmlGenerator super, string tname, params HtmlAttribute[] attrs) { Owner = super; Name = tname; Attributes = attrs; } public string AttributeString => Attributes.Length < 1 ? "" : (" " + string.Join(' ', Attributes.Select(x => x.ToString()).ToArray())); public async Task SelfClose(CancellationToken token) { if (disposed) return; disposed = true; await Owner.AppendHtml($"<{Name}{AttributeString} />"); } internal async Task Begin(CancellationToken token) { await Owner.AppendHtml($"<{Name}{AttributeString}>"); } private bool disposed = false; public async ValueTask DisposeAsync() { if (disposed) return; disposed = true; await Owner.AppendHtml($""); } } public class HtmlGenerator : IDisposable, IAsyncDisposable { public HtmlTag Tag(string name, params HtmlAttribute[] attrs) { var t = new HtmlTag(this, name, attrs); t.Begin(cancel.Token).Wait(); return t; } public async Task TagAsync(string name, params HtmlAttribute[] attrs) { var t = new HtmlTag(this, name, attrs); await t.Begin(cancel.Token); return t; } public async Task TagAsync(string name, CancellationToken token, params HtmlAttribute[] attrs) { using var link = CancellationTokenSource.CreateLinkedTokenSource(cancel.Token, token); var t = new HtmlTag(this, name, attrs); await t.Begin(link.Token); return t; } public async Task TagAsync(string name, HtmlAttribute[] attrs, CancellationToken token = default) { using var link = CancellationTokenSource.CreateLinkedTokenSource(cancel.Token, token); var t = new HtmlTag(this, name, attrs); await t.Begin(link.Token); return t; } public async Task TagAsync(string name, CancellationToken token = default) { using var link = CancellationTokenSource.CreateLinkedTokenSource(cancel.Token, token); var t = new HtmlTag(this, name); await t.Begin(link.Token); return t; } public Task TagSelfClosingAsync(string name, params HtmlAttribute[] attrs) => TagSelfClosingAsync(name, default, attrs); public async Task TagSelfClosingAsync(string name, CancellationToken cancel, params HtmlAttribute[] attrs) { using var link = CancellationTokenSource.CreateLinkedTokenSource(this.cancel.Token, cancel); await using var t = new HtmlTag(this, name, attrs); await t.SelfClose(cancel); } protected readonly Stream stream; public StreamWriter Output { get; } public bool OwnsStream { get; set; } = false; protected readonly CancellationTokenSource cancel; public List Metas { get; } = new List(); public Dispatcher OnFlush = new Dispatcher() { InSeries = true }; public Task AppendHeaders(CancellationToken token = default) => AppendHeaders((x) => Task.CompletedTask, token); public async Task AppendHeaders(Func head, CancellationToken token = default) { using var cancel = CancellationTokenSource.CreateLinkedTokenSource(this.cancel.Token, token); await AppendHtml(@"", cancel.Token); await Task.WhenAll(Metas.Select(x => AppendHtml(x, cancel.Token))); await head(cancel.Token); token.ThrowIfCancellationRequested(); await AppendHtml(@"", cancel.Token); await OnFlush.HookAsync(async (c) => { await AppendHtml("", c); }, token); } public static HtmlGenerator CreateFile(string file, int bufferLength = 0, CancellationToken globalToken = default) { var fs = new FileStream(file, FileMode.Create); try { return new HtmlGenerator(fs, bufferLength, globalToken) { OwnsStream = true }; } catch (Exception ex) { fs.Dispose(); throw ex; } } public HtmlGenerator(Stream output, int bufferLength = 0, CancellationToken token = default) { stream = output; cancel = CancellationTokenSource.CreateLinkedTokenSource(token); Output = new StreamWriter(output); var chan = bufferLength < 1 ? Channel.CreateUnbounded() : Channel.CreateBounded(bufferLength); reader = chan.Reader; writer = chan.Writer; writeHook = Task.Run(async () => { try { await foreach (var obj in reader.ReadAllAsync(cancel.Token)) { if (obj is Safe safe) { await Output.WriteAsync(safe.Value.AsMemory(), cancel.Token); } else await Output.WriteAsync(System.Text.Encodings.Web.HtmlEncoder.Default.Encode(obj is string str ? str : obj?.ToString() ?? "").AsMemory(), cancel.Token); } } catch (OperationCanceledException) { } catch (Exception ex) { OnWriterError?.Invoke(ex); CancelAllOperations(); } }); } public async ValueTask DisposeAsync() { await Flush(); await OnFlush.WaitForCurrentSignal; Dispose(); } public event Action OnWriterError; private readonly ChannelReader reader; protected readonly ChannelWriter writer; public void CancelAllOperations() { if (!cancel.IsCancellationRequested) cancel.Cancel(); } protected Task writeHook; public Task Completion => writeHook; protected bool flushed = false; public async Task Flush(CancellationToken token = default) { if (flushed) return; flushed = true; await OnFlush.Signal(token); writer.Complete(); await writeHook; } public async Task Append(string str, CancellationToken token = default) { using var cancel = CancellationTokenSource.CreateLinkedTokenSource(this.cancel.Token, token); await writer.WriteAsync(str, cancel.Token); } public async Task Append(Safe str, CancellationToken token = default) { using var cancel = CancellationTokenSource.CreateLinkedTokenSource(this.cancel.Token, token); await writer.WriteAsync(str, cancel.Token); } public async Task AppendHtml(string str, CancellationToken token = default) { using var cancel = CancellationTokenSource.CreateLinkedTokenSource(this.cancel.Token, token); await writer.WriteAsync(new Safe(str), cancel.Token); } #region IDisposable Support private bool disposedValue = false; // To detect redundant calls protected virtual void Dispose(bool disposing) { if (!disposedValue) { if (disposing) { if (!cancel.IsCancellationRequested) cancel.Cancel(); Output.Flush(); OnFlush.Dispose(); cancel.Dispose(); Output.Close(); if (OwnsStream) stream.Dispose(); } disposedValue = true; } } ~HtmlGenerator() { Dispose(false); } public void Dispose() { Dispose(true); GC.SuppressFinalize(this); } #endregion } }