From de451c05662892ac9e32918263b935fc1f7f92e9 Mon Sep 17 00:00:00 2001 From: Avril Date: Thu, 23 Apr 2020 18:41:05 +0100 Subject: [PATCH] Initial commit --- .gitignore | 360 +++++++++++++ create-binaries | 21 + napdump.sln | 43 ++ napdump/Dumper.cs | 344 +++++++++++++ napdump/Dumpers/Nineball.cs | 351 +++++++++++++ napdump/PostInfo.cs | 527 +++++++++++++++++++ napdump/Program.cs | 218 ++++++++ napdump/Serialisers.cs | 161 ++++++ napdump/napdump.csproj | 27 + ndimg/Program.cs | 312 ++++++++++++ ndimg/TempFile.cs | 100 ++++ ndimg/ndimg.csproj | 27 + ndpack/Program.cs | 262 ++++++++++ ndpack/TempFile.cs | 100 ++++ ndpack/ndpack.csproj | 27 + ndview/Dispatcher.cs | 152 ++++++ ndview/PageGenerator.cs | 638 ++++++++++++++++++++++++ ndview/Program.cs | 302 +++++++++++ ndview/Properties/Resources.Designer.cs | 81 +++ ndview/Properties/Resources.resx | 127 +++++ ndview/Properties/static/css/base.css | 64 +++ ndview/Properties/static/js/base.js | 4 + ndview/TempFile.cs | 87 ++++ ndview/ndview.csproj | 49 ++ 24 files changed, 4384 insertions(+) create mode 100644 .gitignore create mode 100644 create-binaries create mode 100644 napdump.sln create mode 100644 napdump/Dumper.cs create mode 100644 napdump/Dumpers/Nineball.cs create mode 100644 napdump/PostInfo.cs create mode 100644 napdump/Program.cs create mode 100644 napdump/Serialisers.cs create mode 100644 napdump/napdump.csproj create mode 100644 ndimg/Program.cs create mode 100644 ndimg/TempFile.cs create mode 100644 ndimg/ndimg.csproj create mode 100644 ndpack/Program.cs create mode 100644 ndpack/TempFile.cs create mode 100644 ndpack/ndpack.csproj create mode 100644 ndview/Dispatcher.cs create mode 100644 ndview/PageGenerator.cs create mode 100644 ndview/Program.cs create mode 100644 ndview/Properties/Resources.Designer.cs create mode 100644 ndview/Properties/Resources.resx create mode 100644 ndview/Properties/static/css/base.css create mode 100644 ndview/Properties/static/js/base.js create mode 100644 ndview/TempFile.cs create mode 100644 ndview/ndview.csproj diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..83ba081 --- /dev/null +++ b/.gitignore @@ -0,0 +1,360 @@ +## Ignore Visual Studio temporary files, build results, and +## files generated by popular Visual Studio add-ons. +## +## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore + +# User-specific files +*.rsuser +*.suo +*.user +*.userosscache +*.sln.docstates + +# User-specific files (MonoDevelop/Xamarin Studio) +*.userprefs + +# Mono auto generated files +mono_crash.* + +# Build results +[Dd]ebug/ +[Dd]ebugPublic/ +[Rr]elease/ +[Rr]eleases/ +x64/ +x86/ +[Ww][Ii][Nn]32/ +[Aa][Rr][Mm]/ +[Aa][Rr][Mm]64/ +bld/ +[Bb]in/ +[Oo]bj/ +[Ll]og/ +[Ll]ogs/ + +# Visual Studio 2015/2017 cache/options directory +.vs/ +# Uncomment if you have tasks that create the project's static files in wwwroot +#wwwroot/ + +# Visual Studio 2017 auto generated files +Generated\ Files/ + +# MSTest test Results +[Tt]est[Rr]esult*/ +[Bb]uild[Ll]og.* + +# NUnit +*.VisualState.xml +TestResult.xml +nunit-*.xml + +# Build Results of an ATL Project +[Dd]ebugPS/ +[Rr]eleasePS/ +dlldata.c + +# Benchmark Results +BenchmarkDotNet.Artifacts/ + +# .NET Core +project.lock.json +project.fragment.lock.json +artifacts/ + +# ASP.NET Scaffolding +ScaffoldingReadMe.txt + +# StyleCop +StyleCopReport.xml + +# Files built by Visual Studio +*_i.c +*_p.c +*_h.h +*.ilk +*.meta +*.obj +*.iobj +*.pch +*.pdb +*.ipdb +*.pgc +*.pgd +*.rsp +*.sbr +*.tlb +*.tli +*.tlh +*.tmp +*.tmp_proj +*_wpftmp.csproj +*.log +*.vspscc +*.vssscc +.builds +*.pidb +*.svclog +*.scc + +# Chutzpah Test files +_Chutzpah* + +# Visual C++ cache files +ipch/ +*.aps +*.ncb +*.opendb +*.opensdf +*.sdf +*.cachefile +*.VC.db +*.VC.VC.opendb + +# Visual Studio profiler +*.psess +*.vsp +*.vspx +*.sap + +# Visual Studio Trace Files +*.e2e + +# TFS 2012 Local Workspace +$tf/ + +# Guidance Automation Toolkit +*.gpState + +# ReSharper is a .NET coding add-in +_ReSharper*/ +*.[Rr]e[Ss]harper +*.DotSettings.user + +# TeamCity is a build add-in +_TeamCity* + +# DotCover is a Code Coverage Tool +*.dotCover + +# AxoCover is a Code Coverage Tool +.axoCover/* +!.axoCover/settings.json + +# Coverlet is a free, cross platform Code Coverage Tool +coverage*[.json, .xml, .info] + +# Visual Studio code coverage results +*.coverage +*.coveragexml + +# NCrunch +_NCrunch_* +.*crunch*.local.xml +nCrunchTemp_* + +# MightyMoose +*.mm.* +AutoTest.Net/ + +# Web workbench (sass) +.sass-cache/ + +# Installshield output folder +[Ee]xpress/ + +# DocProject is a documentation generator add-in +DocProject/buildhelp/ +DocProject/Help/*.HxT +DocProject/Help/*.HxC +DocProject/Help/*.hhc +DocProject/Help/*.hhk +DocProject/Help/*.hhp +DocProject/Help/Html2 +DocProject/Help/html + +# Click-Once directory +publish/ + +# Publish Web Output +*.[Pp]ublish.xml +*.azurePubxml +# Note: Comment the next line if you want to checkin your web deploy settings, +# but database connection strings (with potential passwords) will be unencrypted +*.pubxml +*.publishproj + +# Microsoft Azure Web App publish settings. Comment the next line if you want to +# checkin your Azure Web App publish settings, but sensitive information contained +# in these scripts will be unencrypted +PublishScripts/ + +# NuGet Packages +*.nupkg +# NuGet Symbol Packages +*.snupkg +# The packages folder can be ignored because of Package Restore +**/[Pp]ackages/* +# except build/, which is used as an MSBuild target. +!**/[Pp]ackages/build/ +# Uncomment if necessary however generally it will be regenerated when needed +#!**/[Pp]ackages/repositories.config +# NuGet v3's project.json files produces more ignorable files +*.nuget.props +*.nuget.targets + +# Microsoft Azure Build Output +csx/ +*.build.csdef + +# Microsoft Azure Emulator +ecf/ +rcf/ + +# Windows Store app package directories and files +AppPackages/ +BundleArtifacts/ +Package.StoreAssociation.xml +_pkginfo.txt +*.appx +*.appxbundle +*.appxupload + +# Visual Studio cache files +# files ending in .cache can be ignored +*.[Cc]ache +# but keep track of directories ending in .cache +!?*.[Cc]ache/ + +# Others +ClientBin/ +~$* +*~ +*.dbmdl +*.dbproj.schemaview +*.jfm +*.pfx +*.publishsettings +orleans.codegen.cs + +# Including strong name files can present a security risk +# (https://github.com/github/gitignore/pull/2483#issue-259490424) +#*.snk + +# Since there are multiple workflows, uncomment next line to ignore bower_components +# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) +#bower_components/ + +# RIA/Silverlight projects +Generated_Code/ + +# Backup & report files from converting an old project file +# to a newer Visual Studio version. Backup files are not needed, +# because we have git ;-) +_UpgradeReport_Files/ +Backup*/ +UpgradeLog*.XML +UpgradeLog*.htm +ServiceFabricBackup/ +*.rptproj.bak + +# SQL Server files +*.mdf +*.ldf +*.ndf + +# Business Intelligence projects +*.rdl.data +*.bim.layout +*.bim_*.settings +*.rptproj.rsuser +*- [Bb]ackup.rdl +*- [Bb]ackup ([0-9]).rdl +*- [Bb]ackup ([0-9][0-9]).rdl + +# Microsoft Fakes +FakesAssemblies/ + +# GhostDoc plugin setting file +*.GhostDoc.xml + +# Node.js Tools for Visual Studio +.ntvs_analysis.dat +node_modules/ + +# Visual Studio 6 build log +*.plg + +# Visual Studio 6 workspace options file +*.opt + +# Visual Studio 6 auto-generated workspace file (contains which files were open etc.) +*.vbw + +# Visual Studio LightSwitch build output +**/*.HTMLClient/GeneratedArtifacts +**/*.DesktopClient/GeneratedArtifacts +**/*.DesktopClient/ModelManifest.xml +**/*.Server/GeneratedArtifacts +**/*.Server/ModelManifest.xml +_Pvt_Extensions + +# Paket dependency manager +.paket/paket.exe +paket-files/ + +# FAKE - F# Make +.fake/ + +# CodeRush personal settings +.cr/personal + +# Python Tools for Visual Studio (PTVS) +__pycache__/ +*.pyc + +# Cake - Uncomment if you are using it +# tools/** +# !tools/packages.config + +# Tabs Studio +*.tss + +# Telerik's JustMock configuration file +*.jmconfig + +# BizTalk build output +*.btp.cs +*.btm.cs +*.odx.cs +*.xsd.cs + +# OpenCover UI analysis results +OpenCover/ + +# Azure Stream Analytics local run output +ASALocalRun/ + +# MSBuild Binary and Structured Log +*.binlog + +# NVidia Nsight GPU debugger configuration file +*.nvuser + +# MFractors (Xamarin productivity tool) working folder +.mfractor/ + +# Local History for Visual Studio +.localhistory/ + +# BeatPulse healthcheck temp database +healthchecksdb + +# Backup folder for Package Reference Convert tool in Visual Studio 2017 +MigrationBackup/ + +# Ionide (cross platform F# VS Code tools) working folder +.ionide/ + +# Fody - auto-generated XML schema +FodyWeavers.xsd diff --git a/create-binaries b/create-binaries new file mode 100644 index 0000000..b718489 --- /dev/null +++ b/create-binaries @@ -0,0 +1,21 @@ +#!/bin/bash + +cd $(dirname "$0") + +echo ">>> napdump" +sfexec-create-compress -s -e "dotnet %location/napdump.dll %args" - $(ls -1 | grep -E '\.(dll|json)$') >> /dev/null || exit 1 +mv sfexec napdump + +echo ">>> ndimg" +sfexec-create-compress -s -e "dotnet %location/ndimg.dll %args" - $(ls -1 | grep -E '\.(dll|json)$') >> /dev/null || exit 1 +mv sfexec ndimg + +echo ">>> ndpack" +sfexec-create-compress -s -e "dotnet %location/ndpack.dll %args" - $(ls -1 | grep -E '\.(dll|json)$') >> /dev/null || exit 1 +mv sfexec ndpack + +echo ">>> ndview" +sfexec-create-compress -s -e "dotnet %location/ndview.dll %args" - $(ls -1 | grep -E '\.(dll|json)$') >> /dev/null || exit 1 +mv sfexec ndview + +echo "Completed" diff --git a/napdump.sln b/napdump.sln new file mode 100644 index 0000000..df9106b --- /dev/null +++ b/napdump.sln @@ -0,0 +1,43 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.29609.76 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "napdump", "napdump\napdump.csproj", "{318F8582-6991-4CFF-84D6-47B7BBBFABBD}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ndimg", "ndimg\ndimg.csproj", "{51F1A04E-FB27-417E-B0B2-99D99C1442D0}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ndpack", "ndpack\ndpack.csproj", "{98E931DC-AB76-4830-BD60-0F511FBBC46B}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ndview", "ndview\ndview.csproj", "{E1FF4D09-77B2-4852-A52A-43D4BB60AC6E}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {318F8582-6991-4CFF-84D6-47B7BBBFABBD}.Debug|Any CPU.ActiveCfg = Release|Any CPU + {318F8582-6991-4CFF-84D6-47B7BBBFABBD}.Debug|Any CPU.Build.0 = Release|Any CPU + {318F8582-6991-4CFF-84D6-47B7BBBFABBD}.Release|Any CPU.ActiveCfg = Release|Any CPU + {318F8582-6991-4CFF-84D6-47B7BBBFABBD}.Release|Any CPU.Build.0 = Release|Any CPU + {51F1A04E-FB27-417E-B0B2-99D99C1442D0}.Debug|Any CPU.ActiveCfg = Release|Any CPU + {51F1A04E-FB27-417E-B0B2-99D99C1442D0}.Debug|Any CPU.Build.0 = Release|Any CPU + {51F1A04E-FB27-417E-B0B2-99D99C1442D0}.Release|Any CPU.ActiveCfg = Release|Any CPU + {51F1A04E-FB27-417E-B0B2-99D99C1442D0}.Release|Any CPU.Build.0 = Release|Any CPU + {98E931DC-AB76-4830-BD60-0F511FBBC46B}.Debug|Any CPU.ActiveCfg = Release|Any CPU + {98E931DC-AB76-4830-BD60-0F511FBBC46B}.Debug|Any CPU.Build.0 = Release|Any CPU + {98E931DC-AB76-4830-BD60-0F511FBBC46B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {98E931DC-AB76-4830-BD60-0F511FBBC46B}.Release|Any CPU.Build.0 = Release|Any CPU + {E1FF4D09-77B2-4852-A52A-43D4BB60AC6E}.Debug|Any CPU.ActiveCfg = Release|Any CPU + {E1FF4D09-77B2-4852-A52A-43D4BB60AC6E}.Debug|Any CPU.Build.0 = Release|Any CPU + {E1FF4D09-77B2-4852-A52A-43D4BB60AC6E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E1FF4D09-77B2-4852-A52A-43D4BB60AC6E}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {8BA0091A-D949-43B5-854F-4D7EFBDECFEC} + EndGlobalSection +EndGlobal diff --git a/napdump/Dumper.cs b/napdump/Dumper.cs new file mode 100644 index 0000000..df91549 --- /dev/null +++ b/napdump/Dumper.cs @@ -0,0 +1,344 @@ +using AngleSharp; +using AngleSharp.Dom; +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Runtime.CompilerServices; +using System.Text; +using System.Threading; +using System.Threading.Channels; +using System.Threading.Tasks; +using System.Linq; +using System.Runtime.Serialization; +using System.IO; + +using Tools; +using System.Text.RegularExpressions; +using Tools.Crypto; + +namespace napdump +{ + public readonly struct DumperConfig + { + public readonly int MaxThreads; + public readonly Dumper ShareContextWith; + public readonly (string Url, string Value)[] Cookies; + public readonly AESKey? EncryptDeleted; + + public DumperConfig(int maxThreads, Dumper ShareContext=null, (string,string)[] Cookies=null, AESKey? EncryptDeleted=null) + { + MaxThreads = maxThreads; + ShareContextWith = ShareContext; + this.Cookies = Cookies; + this.EncryptDeleted = EncryptDeleted; + } + + public bool Equals(in DumperConfig other) + { + return other.MaxThreads == this.MaxThreads && + ReferenceEquals(this.ShareContextWith, other.ShareContextWith) && + (ReferenceEquals(Cookies, other.Cookies) || (Cookies?.SequenceEqual(other.Cookies) ?? false)) && + (ReferenceEquals(EncryptDeleted, other.EncryptDeleted) || (EncryptDeleted?.Equals(other.EncryptDeleted ?? default) ?? false)); + } + + public override bool Equals(object obj) + { + return obj is DumperConfig conf && this.Equals(conf); + } + + public override int GetHashCode() + { + return MaxThreads.GetHashCode() ^ + (ShareContextWith?.GetHashCode() ?? 0) ^ + (Cookies?.Select(x => x.GetHashCode())?.Aggregate((x, y) => x ^ y) ?? 0) ^ + (EncryptDeleted?.GetHashCode() ?? 0); + } + + public static bool operator ==(DumperConfig left, DumperConfig right) + { + return left.Equals(right); + } + + public static bool operator !=(DumperConfig left, DumperConfig right) + { + return !(left == right); + } + } + public sealed class AsyncMutex : IDisposable + { + private readonly SemaphoreSlim sem; + + public AsyncMutex() + { + sem = new SemaphoreSlim(1, 1); + } + private AsyncMutex(SemaphoreSlim from) + { + sem = from; + } + + private class Lock : IDisposable + { + public AsyncMutex Parent { get; } + public Lock(AsyncMutex held) + { + Parent = held; + } + + public void Dispose() + { + Parent.sem.Release(); + } + } + + public IDisposable Aquire(int msTimeout, CancellationToken token = default) + { + sem.Wait(msTimeout, token); + return new Lock(this); + } + + public IDisposable Aquire(CancellationToken token) + { + sem.Wait(token); + return new Lock(this); + } + + public IDisposable Aquire() + { + sem.Wait(); + return new Lock(this); + } + + public async ValueTask AquireAsync(int msTimeout, CancellationToken token = default) + { + await sem.WaitAsync(msTimeout, token); + return new Lock(this); + } + + public async ValueTask AquireAsync(CancellationToken token = default) + { + await sem.WaitAsync(token); + return new Lock(this); + } + + public void Dispose() + { + sem.Dispose(); + } + + public static AsyncMutex Semaphore(int count, int max) + { + SemaphoreSlim sem = new SemaphoreSlim(count, max); + return new AsyncMutex(sem); + } + public static AsyncMutex Semaphore(int count) + => Semaphore(count, count); + } + + + public abstract class Dumper : IDisposable + { + public DumperConfig Config { get; } + + protected readonly AsyncMutex Pool; + protected readonly CancellationTokenSource globalCancel = new CancellationTokenSource(); + + protected Dumper(DumperConfig config) + { + Config = config; + + Pool = config.ShareContextWith?.Pool ?? AsyncMutex.Semaphore(config.MaxThreads); + } + + public void CancelAllOperations() + { + globalCancel.Cancel(); + } + + public async IAsyncEnumerable Parse(string boardUrl, Hooks hooks = default, [EnumeratorCancellation] CancellationToken token=default) + { + using var cancel = CancellationTokenSource.CreateLinkedTokenSource(globalCancel.Token, token); + + //var cataloguePage = await GetCataloguePage(boardUrl, cancel.Token); + var boardInfo = NewBoardInfo(); + boardInfo.BoardURL = boardUrl; + + if (hooks.PrintDebug) + Console.WriteLine($"({boardUrl}) gen PostInfo"); + + Channel threadGetters = Channel.CreateUnbounded(); + Channel completedThreads = Channel.CreateUnbounded(); + + Task completer = Task.Run(async () => + { + List getters = new List(); + if (hooks.PrintDebug) + Console.WriteLine($"({boardUrl}) gen getter"); + int gi = 0; + await foreach (var getter in threadGetters.Reader.ReadAllAsync(cancel.Token)) + { + getters.Add(getter); + if (hooks.PrintDebug) + Console.WriteLine($"({boardUrl}) add getter {gi++}"); + } + if (hooks.PrintDebug) + Console.WriteLine($"({boardUrl}) add getters {getters.Count}"); + await Task.WhenAll(getters); + if (hooks.PrintDebug) + Console.WriteLine($"({boardUrl}) getters complete"); + completedThreads.Writer.Complete(); + }); + + if (hooks.PrintDebug) + Console.WriteLine($"({boardUrl}) start completer"); + int ti = 0; + await foreach (var thread in GetThreads(boardInfo).WithCancellation(cancel.Token)) + { + //Thread got. + thread.BoardInfo = boardInfo; + + hooks.OnThreadRetrieved?.Invoke(thread); + ThreadRetrievedHook(thread); + await threadGetters.Writer.WriteAsync(Task.Run(async () => + { + try + { + await InternalGetPosts(thread, hooks, cancel.Token); + if (hooks.PrintDebug) Console.WriteLine($"({boardUrl}) writing to complete"); + await completedThreads.Writer.WriteAsync(thread, cancel.Token); + if (hooks.PrintDebug) Console.WriteLine($"({thread.BoardInfo.BoardURL}) written"); + }catch(Exception ex) + { + if (hooks.PrintDebug) + Console.WriteLine($"Whoops {thread.PostNumber} failed: {ex.Message}\n{ex.StackTrace}"); + hooks.OnThreadReadFailed?.Invoke(thread, ex); + ThreadReadFailedHook(thread, ex); + } + }), cancel.Token); + if (hooks.PrintDebug) + Console.WriteLine($"({boardUrl}) thread write ({ti++}) {thread.PostNumber}"); + } + threadGetters.Writer.Complete(); + + if (hooks.PrintDebug) + Console.WriteLine($"({boardUrl}) stop getter"); + + await foreach (var completedThread in completedThreads.Reader.ReadAllAsync(cancel.Token)) + { + boardInfo.AddChildThread(completedThread); + yield return completedThread; + } + if (hooks.PrintDebug) + Console.WriteLine($"({boardUrl}) completer complete"); + + await completer; + boardInfo.DumpTimestamp = DateTime.Now; + hooks.OnBoardRetrieved?.Invoke(boardInfo); + BoardRetrievedHook(boardInfo); + if (hooks.PrintDebug) + Console.WriteLine($"({boardUrl}) end"); + } + + public struct Hooks + { + public Action OnBoardRetrieved; + public Action OnThreadRetrieved; + public Action OnPostRetrieved; + public Action OnThreadReadFailed; + +#if DEBUG + public +#else + internal +#endif + bool PrintDebug; + } + + public event Action OnBoardRetrieved; + public event Action OnThreadRetrieved; + public event Action OnPostRetrieved; + public event Action OnThreadReadFailed; + + protected virtual void PostRetrievedHook(PostInfo post) => OnPostRetrieved?.Invoke(post); + protected virtual void ThreadRetrievedHook(ThreadInfo thread) => OnThreadRetrieved?.Invoke(thread); + protected virtual void BoardRetrievedHook(BoardInfo board) => OnBoardRetrieved?.Invoke(board); + protected virtual void ThreadReadFailedHook(ThreadInfo thread, Exception ex) => OnThreadReadFailed?.Invoke(thread, ex); + + private async Task InternalGetPosts(ThreadInfo thread, Hooks hooks, CancellationToken token = default) + { + using var cancel = CancellationTokenSource.CreateLinkedTokenSource(globalCancel.Token, token); + List posts = new List(); + if (hooks.PrintDebug) Console.WriteLine($"({thread.BoardInfo.BoardURL}) entering context"); + using (await Pool.AquireAsync(cancel.Token)) + { + //var threadPage = await GetThreadPage(thread, token); + if (hooks.PrintDebug) Console.WriteLine($" ctx_aqu ({thread.BoardInfo.BoardURL}) getting posts"); + await foreach (var post in GetPosts(thread).WithCancellation(cancel.Token)) + { + //Post got. + post.Parent = thread; + post.BoardInfo = thread.BoardInfo; + + cancel.Token.ThrowIfCancellationRequested(); + hooks.OnPostRetrieved?.Invoke(post); + PostRetrievedHook(post); + posts.Add(post); + } + if (hooks.PrintDebug) Console.WriteLine($" ctx_aqu ({thread.BoardInfo.BoardURL}) posts got"); + } + if (hooks.PrintDebug) Console.WriteLine($"({thread.BoardInfo.BoardURL}) adding children"); + thread.AddChildPosts(posts); + } + + /// + /// Run a block on this Dumper's thread pool. + /// + public async Task EnterContextAsync(CancellationToken token = default) + { + return await Pool.AquireAsync(token); + } + /// + /// Run a block on this Dumper's thread pool. + /// + public IDisposable EnterContext(CancellationToken token = default) + { + return Pool.Aquire(token); + } + + protected virtual BoardInfo NewBoardInfo() => new BoardInfo(); + protected abstract IAsyncEnumerable GetPosts(ThreadInfo thread, [EnumeratorCancellation] CancellationToken token = default); + protected abstract IAsyncEnumerable GetThreads(BoardInfo boardInfo, [EnumeratorCancellation] CancellationToken token = default); + +#region IDisposable Support + private bool disposedValue = false; + + protected virtual void Dispose(bool disposing) + { + if (!disposedValue) + { + if (disposing) + { + if (!globalCancel.IsCancellationRequested) globalCancel.Cancel(); + + if (Config.ShareContextWith == null) + Pool.Dispose(); + globalCancel.Dispose(); + } + + disposedValue = true; + } + } + + ~Dumper() + { + Dispose(false); + } + + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } +#endregion + } +} diff --git a/napdump/Dumpers/Nineball.cs b/napdump/Dumpers/Nineball.cs new file mode 100644 index 0000000..b4f1fbe --- /dev/null +++ b/napdump/Dumpers/Nineball.cs @@ -0,0 +1,351 @@ +using AngleSharp; +using AngleSharp.Dom; +using AngleSharp.Io; +using System; +using System.Collections.Generic; +using System.IO; +using System.Runtime.CompilerServices; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; +using System.Linq; + +namespace napdump.Dumpers +{ + class Nineball : Dumper + { + readonly IConfiguration browserConfig; + readonly IBrowsingContext context; + readonly ICookieProvider cookies; + public Nineball(DumperConfig config) : base(config) + { + browserConfig = Configuration.Default.WithDefaultCookies().WithDefaultLoader(); + cookies = browserConfig.Services.OfType().First(); + foreach (var c in config.Cookies ?? Array.Empty<(string Url, string Value)>()) + { + cookies.SetCookie(new Url(c.Url), c.Value); + } + context = BrowsingContext.New(browserConfig); + } + + private static readonly Regex reBoardName = new Regex(@"^(\/.*?\/)", RegexOptions.Compiled | RegexOptions.IgnoreCase); + private static readonly Regex reBoardNameW = new Regex(@"^\/(\w+)\/", RegexOptions.Compiled | RegexOptions.IgnoreCase); + protected async Task GetBoardInfo(BoardInfo bi, IDocument document, CancellationToken token) + { + await Task.Yield(); + token.ThrowIfCancellationRequested(); + bi.Title = document.QuerySelector("body > threads > h1").InnerHtml; + bi.BoardName = reBoardName.IsMatch(bi.Title) ? reBoardName.Match(bi.Title).Groups[1].Value : bi.Title; + bi.SafeName = reBoardNameW.IsMatch(bi.Title) ? reBoardNameW.Match(bi.Title).Groups[1].Value : "unbound"; + bi.Description = document.QuerySelector("#banner_info").TextContent; + bi.Tags = new[] { "meguca", "node", "liveboard" }; + } + private static readonly Regex reImageDim = new Regex(@"\((\d+) ([kmg]?b), (\d+)x(\d+)\)", RegexOptions.Compiled | RegexOptions.IgnoreCase); + private static bool TryParseImageDimInfo(string info, out long size, out int x, out int y) + { + //Console.WriteLine(info + " " + reImageDim.IsMatch(info)); + if(reImageDim.IsMatch(info)) + { + var groups = reImageDim.Match(info).Groups; + + if(long.TryParse(groups[1].Value, out var rawSize) && + int.TryParse(groups[3].Value, out x) && + int.TryParse(groups[4].Value, out y)) + { + long multiplier = 1; + switch (groups[2].Value.ToLower().Trim()) + { + case "b": + break; + case "kb": + multiplier = 1024; + break; + case "mb": + multiplier = 1024 * 1024; + break; + case "gb": + multiplier = 1024 * 1024 * 1024; + break; + default: + goto bad; + } + size = rawSize & multiplier; + return true; + } + + } + bad: + size = default; + x = y = default; + return false; + } + + private static readonly Regex reDateTime = new Regex(@"(\d\d) (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) (\d\d\d\d)\(\w+\)(\d\d):(\d\d)", RegexOptions.Compiled | RegexOptions.IgnoreCase); + private static bool TryParseDateTime(string htmlDateTime, out DateTime dt) + { + htmlDateTime = htmlDateTime.Trim(); + //Console.WriteLine(htmlDateTime + " " + reDateTime.IsMatch(htmlDateTime)); + if(reDateTime.IsMatch(htmlDateTime)) + { + var groups = reDateTime.Match(htmlDateTime).Groups; + + int day = int.Parse(groups[1].Value); + string month = groups[2].Value; + int year = int.Parse(groups[3].Value); + int hour = int.Parse(groups[4].Value); + int minute = int.Parse(groups[5].Value); + + try + { + dt = new DateTime(year, month switch + { + "Jan" => 1, + "Feb" => 2, + "Mar" => 3, + "Apr" => 4, + "May" => 5, + "Jun" => 6, + "Jul" => 7, + "Aug" => 8, + "Sep" => 9, + "Oct" => 10, + "Nov" => 11, + "Dec" => 12, + _ => throw new InvalidDataException(), + }, day, hour, minute, 0); + return true; + } + catch + { + dt = default; + return false; + } + } + dt = default; + return false; + } + + private static readonly Regex reImageDeleted = new Regex(@"^Image deleted by (\w+)$", RegexOptions.Compiled); + private static readonly Regex reImageSpoilered = new Regex(@"^Image spoilered by (\w+)$", RegexOptions.Compiled); + private static readonly Regex rePostDeleted = new Regex(@"^Post deleted by (\w+)$", RegexOptions.Compiled); + private static readonly Regex reUserBanned = new Regex(@"^User banned by (\w+)(?: for (.+))?$", RegexOptions.Compiled); + private static void getModlog(string nodeHtml, out Modlog log) + { + log = new Modlog(); + if (nodeHtml == null) return; + try + { + var split = nodeHtml.Split("
").Select(x => x.Trim()).Where(x=> x.Length>0); + + foreach (var line in split) + { + if (reImageDeleted.IsMatch(line)) + log.ImageDeleted = AdminInfo.Create(true, reImageDeleted.Match(line).Groups[1].Value); + + if (reImageSpoilered.IsMatch(line)) + log.ImageSpoilered = AdminInfo.Create(true, reImageSpoilered.Match(line).Groups[1].Value); + + if (rePostDeleted.IsMatch(line)) + log.PostDeleted = AdminInfo.Create(true, rePostDeleted.Match(line).Groups[1].Value); + + if (reUserBanned.IsMatch(line)) + { + var match = reUserBanned.Match(line).Groups; + log.UserBanned = AdminInfo.Create(true, match[1].Value); + if (match[2].Success) + log.BanMessage = AdminInfo.Create(match[2].Value, match[1].Value); + } + + } + } + catch(Exception ex) + { + Console.WriteLine("Modlog parsing error: "+ex.Message); + log = new Modlog(); + } + } + + private static readonly Regex reMailTo = new Regex(@"^mailto:", RegexOptions.Compiled); + protected override async IAsyncEnumerable GetPosts(ThreadInfo thread, [EnumeratorCancellation] CancellationToken token) + { + var document = await context.OpenAsync(thread.BoardInfo.BoardURL + thread.PostNumber, token); + + var section = document.QuerySelector("section"); + thread.Locked = section.ClassList.Contains("locked"); + + (string Name, string Tripcode, string Email, string Capcode) getTripcode(IElement header) + { + var bname = header.QuerySelector("b"); + + string name, trip, mail, cap; + name = trip = mail = cap = null; + + if(bname.FirstChild.NodeName.ToLower() == "a") + { + //Mail link + mail = bname.FirstElementChild.GetAttribute("href"); + if (reMailTo.IsMatch(mail)) + mail = reMailTo.Replace(mail, ""); + bname = bname.FirstElementChild; + } + + if(bname.ChildNodes.Length > 1 && bname.ChildNodes[1].NodeName=="CODE") + { + //Has tripcode & name + name = bname.FirstChild.TextContent; + trip = bname.ChildNodes[1].TextContent; + if (bname.ChildNodes.Length > 2) + cap = bname.ChildNodes[2].TextContent; + } + else if(bname.ChildNodes.Length>1) + { + name = bname.FirstChild.TextContent; + cap = bname.ChildNodes[1].TextContent; + } + else if(bname.FirstChild.NodeName.ToLower() == "code") + { + //Tripcode, no name. + trip = bname.FirstChild.TextContent; + } + else + { + //Name, no tripcode + name = bname.FirstChild.TextContent; + } + + return (name, trip, mail, cap); + } + + //Get thread's modlog. + getModlog(section.QuerySelector("b.modLog")?.InnerHtml, out var threadModlog); + thread.ModLog = threadModlog; + + //Get thread's info. + var imageInfo = section.QuerySelector("figure > figcaption > i"); + if (imageInfo != null) + { + string imageDimInfo = imageInfo.FirstChild.TextContent; + if (TryParseImageDimInfo(imageDimInfo, out var _imageSize, out var _x, out var _y)) + { + thread.ImageSize = _imageSize; + thread.ImageDimensions = (_x, _y); + + var imageNameInfo = imageInfo.QuerySelector("a"); + + thread.ImageURL = imageNameInfo.GetAttribute("href"); + thread.ImageFilename = imageNameInfo.GetAttribute("download"); + + if (TryParseDateTime(section.QuerySelector("header > time").FirstChild.TextContent, out var threadTimestamp)) + { + thread.Timestamp = threadTimestamp; + } + else + { + thread.Timestamp = default; + } + + (thread.Name, thread.Tripcode, thread.Email, thread.Capcode) = getTripcode(section.QuerySelector("header")); + } + else + { + thread.ImageDimensions = default; + thread.ImageFilename = null; + thread.ImageSize = 0; + thread.ImageURL = null; + } + } + thread.Body = section.QuerySelector("blockquote").InnerHtml; + thread.ThreadURL = document.Url; + thread.Subject = section.QuerySelector("header > h3")?.TextContent; + + //Get posts + foreach (var article in section.QuerySelectorAll("article")) + { + var post = new PostInfo() + { + Body = article.QuerySelector("blockquote").InnerHtml, + }; + + (post.Name, post.Tripcode, post.Email, post.Capcode) = getTripcode(article.QuerySelector("header")); + + if (TryParseDateTime(article.QuerySelector("header > time").TextContent, out var _time)) + post.Timestamp = _time; + else + post.Timestamp = default; + + if (ulong.TryParse(article.QuerySelector("header > nav > a[class=quote]").TextContent, out ulong _postNumber)) + post.PostNumber = _postNumber; + else + post.PostNumber = default; + + //Get modlog + getModlog(article.QuerySelector("b.modLog")?.InnerHtml, out var postModlog); + post.ModLog = postModlog; + + var figure = article.QuerySelector("figure > figcaption > i"); + if (figure != null) + { + //Has image + if (TryParseImageDimInfo(figure.FirstChild.TextContent, out var _imageSize, out var _x, out var _y)) + { + post.ImageDimensions = (_x, _y); + post.ImageSize = _imageSize; + + post.ImageURL = figure.QuerySelector("a").GetAttribute("href"); + post.ImageFilename = figure.QuerySelector("a").GetAttribute("download"); + + } + } + await EncryptIfRequired(post, token); + yield return post; + } + + await EncryptIfRequired(thread, token); + } + + private async Task EncryptIfRequired(PostInfo post, CancellationToken token) + { + try + { + if (Config.EncryptDeleted != null) + { + if (post.ModLog.ImageDeleted) + { + await post.EncryptImageAsync(Config.EncryptDeleted.Value, token); + } + if (post.ModLog.PostDeleted) + { + await post.EncryptPostAsync(Config.EncryptDeleted.Value, token); + } + } + }catch(Exception ex) + { + Console.WriteLine("Encryption for post "+post.PostNumber+" failed: " + ex.Message+"\n"+ex.StackTrace); + } + } + + protected override async IAsyncEnumerable GetThreads(BoardInfo boardInfo, [EnumeratorCancellation] CancellationToken token) + { + var document = await context.OpenAsync(boardInfo.BoardURL + "catalog", token); + + await GetBoardInfo(boardInfo, document, token); + + var threadLinks = document.QuerySelectorAll("#catalog > article > a[class=history]"); + foreach(var link in threadLinks) + { + if (link.HasAttribute("href")) + { + var href = link.GetAttribute("href"); + if (ulong.TryParse(href, out ulong postNumber)) + { + yield return new ThreadInfo() + { + PostNumber = postNumber, + }; + } + } + } + } + } +} diff --git a/napdump/PostInfo.cs b/napdump/PostInfo.cs new file mode 100644 index 0000000..cd3169e --- /dev/null +++ b/napdump/PostInfo.cs @@ -0,0 +1,527 @@ +using AngleSharp; +using AngleSharp.Dom; +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Runtime.CompilerServices; +using System.Text; +using System.Threading; +using System.Threading.Channels; +using System.Threading.Tasks; +using System.Linq; +using System.Runtime.Serialization; +using System.IO; + +using Tools; +using System.Text.RegularExpressions; +using System.Security.Cryptography; +using Tools.Crypto; +using System.Runtime.Serialization.Formatters.Binary; + +namespace napdump +{ + [Serializable] + public readonly struct AdminInfo + { + public readonly T Value; + public readonly string By; + public readonly bool Exists; + + public bool Equals(AdminInfo info) + => info.By == By && (ReferenceEquals(info.Value, Value) || (info.Value?.Equals(this.Value) ?? false)); + + public override int GetHashCode() + { + return (Value?.GetHashCode() ?? 0) ^ + (By?.GetHashCode() ?? 0); + } + + public override bool Equals(object obj) + { + return obj is AdminInfo t && Equals(t); + } + + internal AdminInfo(T value, string by) + { + Value = value; + By = by; + Exists = true; + } + + public override string ToString() + { + if (Exists) + return (Value?.ToString() ?? "null") + "(" + By + ")"; + else + return "None"; + } + + public static implicit operator T(AdminInfo info) + => info.Value; + + public static bool operator ==(AdminInfo left, AdminInfo right) + { + return left.Equals(right); + } + + public static bool operator !=(AdminInfo left, AdminInfo right) + { + return !(left == right); + } + } + public static class AdminInfo + { + public static AdminInfo Create(T value, string by) + => new AdminInfo(value, by); + + public static AdminInfo None() + => default; + } + + [Serializable] + public struct Modlog + { + public AdminInfo ImageDeleted; + public AdminInfo ImageSpoilered; + public AdminInfo PostDeleted; + public AdminInfo UserBanned; + public AdminInfo BanMessage; + + public override string ToString() + { + return $"{{ImageDeleted:{ImageDeleted}, ImageSpoilered:{ImageSpoilered}, PostDeleted:{PostDeleted}, UserBanned:{UserBanned}, BanMessage:'{BanMessage}'}}"; + } + + public bool Equals(in Modlog log) + { + return ImageDeleted.Equals(log.ImageDeleted) && + ImageSpoilered.Equals(log.ImageSpoilered) && + PostDeleted.Equals(log.PostDeleted) && + UserBanned.Equals(log.UserBanned) && + BanMessage.Equals(log.BanMessage); + } + + public override bool Equals(object obj) + { + return obj is Modlog log && this.Equals(log); + } + + public override int GetHashCode() + { + return ImageDeleted.GetHashCode() ^ + ImageSpoilered.GetHashCode() ^ + PostDeleted.GetHashCode() ^ + UserBanned.GetHashCode() ^ + BanMessage.GetHashCode(); + } + + public static bool operator ==(Modlog left, Modlog right) + { + return left.Equals(right); + } + + public static bool operator !=(Modlog left, Modlog right) + { + return !(left == right); + } + } + + [Serializable] + public class ThreadInfo : PostInfo + { + public override ThreadInfo Parent { get => null; internal set => throw new InvalidOperationException("Thread can have no parent thread"); } + + private readonly List children = new List(); + public IReadOnlyCollection Children => new ReadOnlyCollection(children); + public string ThreadURL { get; set; } + public bool Locked { get; set; } + + protected override void Blank() + { + base.Blank(); + + ThreadURL = default; + Locked = default; + children.Clear(); + } + protected override void CopyFrom(PostInfo other) + { + base.CopyFrom(other); + + if (other is ThreadInfo thread) + { + ThreadURL = thread.ThreadURL; + Locked = thread.Locked; + children.Clear(); + children.AddRange(thread.children); + } + else throw new InvalidOperationException("Not a thread."); + } + + internal void AddChildPost(PostInfo post) + { + children.Add(post); + } + internal void AddChildPosts(IEnumerable posts) + { + children.AddRange(posts); + } + + ~ThreadInfo() + { + children.Clear(); + } + + public override async Task DecryptPostAsync(AESKey with, CancellationToken cancel = default) + { + var post = await DecryptAsync(with, cancel) as ThreadInfo ?? throw new InvalidOperationException("Not a thread."); + CopyFrom(post); + } + public override async Task EncryptPostAsync(AESKey with, CancellationToken cancel = default) + { + var data = await EncryptAsync(with, cancel); + Blank(); + EncryptedData = data; + } + + public ThreadInfo() : base() { } + + protected override StringBuilder PropertyString() + { + var sb = base.PropertyString(); + if (!IsEncrypted) + { + AppendProp(sb, "Thread-Url", ThreadURL); + AppendProp(sb, "Locked", Locked ? Locked.ToString() : null); + if (children.Count > 0) + AppendProp(sb, "Children", $"({children.Count})[" + string.Join(',', Children.Select(x => x.PostNumber.ToString() + " ")).Trim() + "]"); + } + return sb; + } + } + [Serializable] + public struct ImageInfo + { + public string ImageURL { get; set; } + public string ImageFilename { get; set; } + public long ImageSize { get; set; } + public (int Width, int Height) ImageDimensions { get; set; } + + public bool Equals(in ImageInfo image) + { + return ImageURL == image.ImageURL && + ImageFilename == image.ImageFilename && + ImageSize == image.ImageSize && + ImageDimensions == image.ImageDimensions; + } + + public override bool Equals(object obj) + { + return obj is ImageInfo info && Equals(info); + } + + public override int GetHashCode() + { + return (ImageURL?.GetHashCode() ?? 0) ^ + (ImageFilename?.GetHashCode() ?? 0) ^ + (ImageSize.GetHashCode()) ^ + ImageDimensions.GetHashCode(); + } + + public static bool operator ==(ImageInfo left, ImageInfo right) + { + return left.Equals(right); + } + + public static bool operator !=(ImageInfo left, ImageInfo right) + { + return !(left == right); + } + } + [Serializable] + public class PostInfo +#if DEBUG && false + : ISerializable, IBinaryWritable, ITextWritable +#endif + { + [field: NonSerialized] + public BoardInfo BoardInfo { get; internal set; } + + [field: NonSerialized] + public virtual ThreadInfo Parent { get; internal set; } + + protected virtual void Blank() + { + Subject = default; + Body = default; + Name = default; + Tripcode = default; + Email = default; + Capcode = default; + Timestamp = default; + ImageField = default; + ModLog = default; + Extra = default; + EncryptedData = default; + EncryptedImageData = default; + } + + protected virtual void CopyFrom(PostInfo other) + { + PostNumber = other.PostNumber; + Subject = other.Subject; + Body = other.Body; + Name = other.Name; + Tripcode = other.Tripcode; + Email = other.Email; + Capcode = other.Capcode; + Timestamp = other.Timestamp; + ImageField = other.ImageField; + ModLog = other.ModLog; + Extra = other.Extra; + EncryptedData = other.EncryptedData; + EncryptedImageData = other.EncryptedImageData; + } + + public ulong PostNumber { get; set; } + public string Subject { get; set; } + public string Body { get; set; } + public string Name { get; set; } + public string Tripcode { get; set; } + public string Email { get; set; } + public string Capcode { get; set; } + public DateTime Timestamp { get; set; } + protected ImageInfo ImageField; + + public ImageInfo Image + { + get => IsImageEncrypted ? throw new InvalidOperationException("Image is encrypted.") : ImageField; + set => ImageField = IsImageEncrypted ? throw new InvalidOperationException("Image is encrypted.") : value; + } + public string ImageURL + { + get => IsImageEncrypted ? throw new InvalidOperationException("Image is encrypted.") : ImageField.ImageURL; + set => ImageField.ImageURL = IsImageEncrypted ? throw new InvalidOperationException("Image is encrypted.") : value; + } + public string ImageFilename + { + get => IsImageEncrypted ? throw new InvalidOperationException("Image is encrypted.") : ImageField.ImageFilename; + set => ImageField.ImageFilename = IsImageEncrypted ? throw new InvalidOperationException("Image is encrypted.") : value; + } + public long ImageSize + { + get => IsImageEncrypted ? throw new InvalidOperationException("Image is encrypted.") : ImageField.ImageSize; + set => ImageField.ImageSize = IsImageEncrypted ? throw new InvalidOperationException("Image is encrypted.") : value; + } + public (int Width, int Height) ImageDimensions + { + get => IsImageEncrypted ? throw new InvalidOperationException("Image is encrypted.") : ImageField.ImageDimensions; + set => ImageField.ImageDimensions = IsImageEncrypted ? throw new InvalidOperationException("Image is encrypted.") : value; + } + + public Modlog ModLog { get; set; } = default; + + public string Extra { get; set; } = ""; + + public bool IsEncrypted => EncryptedData != null; + protected byte[] EncryptedData { get; set; } = null; + protected byte[] EncryptedImageData { get; private set; } = null; + public bool IsImageEncrypted => EncryptedImageData != null; + + + public async Task EncryptImageAsync(AESKey with, CancellationToken token=default) + { + if (IsImageEncrypted) throw new InvalidOperationException("Already encrypted image."); + + using(var inp = new MemoryStream()) + { + var binf = new BinaryFormatter(); + binf.Serialize(inp, ImageField); + inp.Position = 0; + using (var enc = new encaes.AesEncryptor(inp) { KeepAlive = true }) + { + enc.Key = with; + using (var oup = new MemoryStream()) + { + await enc.Encrypt(oup, token); + oup.Position = 0; + EncryptedImageData = oup.ToArray(); + ImageField = default; + } + } + } + } + + public async Task DecryptImageAsync(AESKey with, CancellationToken token=default) + { + if (!IsImageEncrypted) throw new InvalidOperationException("Image is not encrypted."); + + using(var inp = new MemoryStream(EncryptedImageData)) + { + inp.Position = 0; + using (var dec = new encaes.AesEncryptor(inp) { KeepAlive = true }) + { + dec.Key = with; + using (var oup = new MemoryStream()) + { + await dec.Decrypt(oup, token); + oup.Position = 0; + + var binf = new BinaryFormatter(); + ImageField = (ImageInfo)binf.Deserialize(oup); + EncryptedImageData = null; + } + } + } + } + + protected async Task DecryptAsync(AESKey with, CancellationToken token=default) + { + if (!IsEncrypted) throw new InvalidOperationException("Not encrypted."); + + using (var inp = new MemoryStream()) + { + await inp.WriteAllAsync(EncryptedData, token); + inp.Position = 0; + using (var enc = new encaes.AesEncryptor(inp) { KeepAlive = true }) + { + enc.Key = with; + using (var oup = new MemoryStream()) + { + await enc.Decrypt(oup, token); + oup.Position = 0; + var binf = new BinaryFormatter(); + return binf.Deserialize(oup); + } + } + } + } + + protected async Task EncryptAsync(AESKey with, CancellationToken cancel = default) + { + using (var ms = new MemoryStream()) + { + var binf = new BinaryFormatter(); + await Task.Yield(); + binf.Serialize(ms, this); + ms.Position = 0; + using (var enc = new encaes.AesEncryptor(ms) { KeepAlive = true }) + { + enc.Key = with; + using (var op = new MemoryStream()) + { + await enc.Encrypt(op, cancel); + op.Position = 0; + return op.ToArray(); + } + } + } + } + + public virtual async Task DecryptPostAsync(AESKey with, CancellationToken cancel = default) + { + var post = await DecryptAsync(with, cancel) as PostInfo ?? throw new InvalidOperationException("Not a post."); + CopyFrom(post); + } + public virtual async Task EncryptPostAsync(AESKey with, CancellationToken cancel = default) + { + var data = await EncryptAsync(with, cancel); + Blank(); + EncryptedData = data; + } + + protected static StringBuilder AppendProp(StringBuilder to, string name, string value) + { + if (to == null) + { + to = new StringBuilder(); + to.Append($"[{name}={value}]: {{"); + } + else + { + if (value != null) + to.Append($"{name}={EnsureLength(value, 32)}, "); + else return to; + } + to.Append("\n\t"); + return to; + } + protected static string CompleteProp(StringBuilder from) + { + string vl = from.ToString().Trim(); + if (vl.EndsWith(",")) return vl.Substring(0, vl.Length - 1) + "}"; + else + return vl + "}"; + } + + protected static string EnsureLength(string str, int i) + { + if (str.Length < i) return str; + else + return str.Substring(0, i) + "(...)"; + } + + protected virtual StringBuilder PropertyString() + { + StringBuilder sb = AppendProp(null, "PostNumber", PostNumber.ToString()); + + AppendProp(sb, "Board", BoardInfo?.ToString()); + AppendProp(sb, "Parent", Parent?.ToString()); + AppendProp(sb, "Image-Encrypted", IsImageEncrypted ? "Yes" : null); + AppendProp(sb, "Encrypted", IsEncrypted ? "Yes" : null); + if (!IsEncrypted) + { + AppendProp(sb, "Name", Name); + AppendProp(sb, "Tripcode", Tripcode); + AppendProp(sb, "Email", Email); + AppendProp(sb, "Subject", Subject); + AppendProp(sb, "Image-URL", ImageField.ImageURL); + AppendProp(sb, "Image-Filename", ImageField.ImageFilename); + AppendProp(sb, "Image-Size", ImageField.ImageSize switch { default(long) => null, _ => ImageField.ImageSize.ToString() }); + AppendProp(sb, "Image-Dimensions", ImageField.ImageURL != null ? $"({ImageField.ImageDimensions.Width} . {ImageField.ImageDimensions.Height})" : null); + AppendProp(sb, "Timestamp", Timestamp.ToString()); + AppendProp(sb, "Body", Body.ToString()); + } + + return sb; + } + + public override sealed string ToString() + { + return CompleteProp(PropertyString()); + } + + } + + [Serializable] + public class BoardInfo + { + public string BoardURL { get; internal set; } + public string Title { get; set; } + public string Description { get; set; } + public string BoardName { get; set; } + public string SafeName { get; set; } + + public DateTime DumpTimestamp { get; set; } = DateTime.Now; + + private readonly List childThreads = new List(); + internal void AddChildThread(ThreadInfo threadInfo) + { + childThreads.Add(threadInfo); + } + public ReadOnlyCollection Threads => new ReadOnlyCollection(childThreads); + + ~BoardInfo() + { + childThreads.Clear(); + } + + public string[] Tags { get; set; } + + public override string ToString() + { + return $@"(BoardInfo){{Taken={DumpTimestamp.ToString()}, URL={BoardURL}, Title={Title}, Desc={Description}, Name={BoardName} ({SafeName}), Tags={string.Join(' ', Tags ?? new string[] { })}, Threads={childThreads.Count}}}"; + } + + } +} diff --git a/napdump/Program.cs b/napdump/Program.cs new file mode 100644 index 0000000..719b56b --- /dev/null +++ b/napdump/Program.cs @@ -0,0 +1,218 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using System.Threading.Channels; +using System.Linq; +using System.Collections.Generic; +using System.IO; +using Tools.Crypto; + +namespace napdump +{ + class Program + { + public static readonly CancellationTokenSource globalCancel = new CancellationTokenSource(); + + public const string NineballBaseUrl = "https://nineball.party"; + public static readonly string[] NineballBoards = new[] { "/nap/", "/srsbsn/", "/staff/"}; + + static volatile int totalThreadsDownloaded = 0; + static volatile int totalBoardsDownloaded = 0; + static volatile int totalPostsDownloaded = 0; + static async Task GrabBoard(Dumper dumper, string boardUrl, ChannelWriter onNewThread, CancellationToken token) + { + TaskCompletionSource get = new TaskCompletionSource(); + Dumper.Hooks hooks = new Dumper.Hooks() + { + OnBoardRetrieved = (bi) => + { + if (!token.IsCancellationRequested) + get.SetResult(bi); + }, + PrintDebug = false//boardUrl.EndsWith("srsbsn/"), + }; + + using var _reg_get_cancel = token.Register(() => get.SetException(new OperationCanceledException())); + + Console.WriteLine("\r [" + dumper.GetType().Name + "] Downloading " + boardUrl); + try + { + await foreach (var thread in dumper.Parse(boardUrl, hooks).WithCancellation(token)) + { + totalPostsDownloaded += thread.Children.Count + 1; + totalThreadsDownloaded += 1; + await onNewThread.WriteAsync(thread, token); + } + return await get.Task; + } + finally + { + Console.WriteLine("\r [" + dumper.GetType().Name + "] Complete " + boardUrl); + } + } + static async Task readOutputs(ChannelReader reader, CancellationToken token) + { + Dictionary numberPerBoard = new Dictionary(); + await foreach(var thread in reader.ReadAllAsync(token)) + { + string name = thread.BoardInfo.BoardName; + if (numberPerBoard.ContainsKey(name)) + numberPerBoard[name] += 1; + else numberPerBoard.Add(name, 1); + Console.Write($"\r"); + foreach(var kv in numberPerBoard) + { + Console.Write($"{kv.Key} - {kv.Value} "); + } + } + Console.WriteLine(); + } + static string getTimestamp(DateTime time) + { + return $"{time.Year}-{time.Month}-{time.Day}-{time.Hour}.{time.Minute}.{time.Second}.{time.Millisecond}"; + } + static async Task DumpBoardInfo(BoardInfo bi) + { + System.Runtime.Serialization.Formatters.Binary.BinaryFormatter binf = new System.Runtime.Serialization.Formatters.Binary.BinaryFormatter(); + + if (!Directory.Exists("dumps")) + Directory.CreateDirectory("dumps"); + var path = Path.Combine("dumps", $"{bi.SafeName ?? "unbound"}-{getTimestamp(DateTime.Now)}.board"); + using (var fs = new FileStream(path, FileMode.Create)) + { + await Task.Yield(); + binf.Serialize(fs, bi); + } + Console.WriteLine($"\r {bi.BoardName} -> {path}"); + } + + static async Task ParseArgs(string[] args) + { + string findarg(string name) + { + for(int i=0;i j != i && j != (i + 1)).ToArray(); + return r; + } + } + return null; + } + bool tryarg(string name, TryParser parser, out T value, T def = default) + { + var fa = findarg(name); + if(fa!=null) + { + if(! parser(fa, out value)) + { + value = def; + return false; + } + return true; + } + value = def; + return false; + } + static bool defp(string inp, out string op) + { + op = inp; + return true; + } + + string login; + string aesKeyfile; + int threads; + + tryarg("--login", defp, out login); + tryarg("--threads", int.TryParse, out threads, 3); + tryarg("--encrypt-deleted", defp, out aesKeyfile); + + return new DumperConfig(threads, Cookies: login == null ? null : new[] { (NineballBaseUrl, "a=" + login) }, EncryptDeleted: aesKeyfile == null ? (AESKey?)null : (await getKey(aesKeyfile))); + } + private static async Task getKey(string fn) + { + using (var fs = new FileStream(fn, FileMode.Open, FileAccess.Read)) + { + return await encaes.AesEncryptor.LoadKey(fs, async () => await encaes.AesEncryptor.ReadPassword("Keyfile is password protected: ")); + } + } + private delegate bool TryParser(string input, out T value); + + static async Task Main(string[] args) + { + if(args.Length==1 && args[0].ToLower()=="--help") + { + Console.WriteLine("napdump.exe [--login ] [--threads ] [--encrypt-deleted ]"); + return; + } + using var napDownloader = new Dumpers.Nineball(await ParseArgs(args)); + + Console.CancelKeyPress += (o,e) => + { + globalCancel.Cancel(); + e.Cancel = true; + }; + + List> downloaders = new List>(); + Channel threads = Channel.CreateUnbounded(); + Task outputReader = readOutputs(threads.Reader, globalCancel.Token); + try + { + foreach (var board in NineballBoards) + { + downloaders.Add(Task.Run(async () => + { + try + { + var bi = await GrabBoard(napDownloader, NineballBaseUrl + board, threads.Writer, globalCancel.Token); + totalBoardsDownloaded += 1; + try + { + await DumpBoardInfo(bi); + } + catch (Exception ex) + { + Console.WriteLine("Failed to dump board " + bi.BoardName + " to file: " + ex.Message); + + } + return bi; + } + catch (Exception ex) + { + Console.WriteLine("Failed to download board " + board + ": " + ex.Message); + return default(BoardInfo); + } + })); + } + + var boards = (await Task.WhenAll(downloaders)).Where(x => x != null); + + Console.WriteLine("\n\nDownloaded Boards:"); + foreach (var b in boards) + Console.WriteLine("\t" + b.ToString()); + } + catch (Exception ex) + { + Console.WriteLine("\n\nError: " + ex.Message); + + return; + } + finally + { + threads.Writer.Complete(); + try + { + await outputReader; + } + catch (OperationCanceledException) { } + } + + Console.WriteLine("Complete"); + Console.WriteLine($"Downloaded {totalBoardsDownloaded} boards, with {totalThreadsDownloaded} threads, containing {totalPostsDownloaded} posts."); + + } + } +} diff --git a/napdump/Serialisers.cs b/napdump/Serialisers.cs new file mode 100644 index 0000000..78e36d2 --- /dev/null +++ b/napdump/Serialisers.cs @@ -0,0 +1,161 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Runtime.Serialization; +using System.Runtime.Serialization.Formatters.Binary; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +using Tools; + +namespace napdump +{ + [Flags] + public enum WritingOption : uint + { + /// + /// Serialize dump IBInfos + /// + Dump = 1 << 0, + /// + /// Serialize as binary IBInfos + /// + Binary = 1 << 1, + /// + /// Serialize as text IBInfos + /// + Text = 1<<2, + /// + /// (flag) GZ compress IBInfos + /// + Gz = 1 << 3, + /// + /// (flag) File system serialise format + /// + Fs = 1 << 4, + /// + /// Archive serialise format + /// + Ar = 1 << 5, + } + + public static class BinWriter + { + public static WritingOption Options { get; private set; } + + public static void Initialise(WritingOption opt) + { + Options = opt; + } + public static async Task SaveOptions(Stream to) + { + await to.WriteValueUnmanagedAsync((uint)Options); + } + public static async Task LoadOptions(Stream from, CancellationToken token = default) + { + var opt = (WritingOption)await from.ReadValueUnmanagedAsync(token); + Initialise(opt); + return opt; + } + public static async Task WriteEntry(Stream to, T toWrite, CancellationToken token=default) where T : ISerializable, IBinaryWritable, ITextWritable + { + var Options = BinWriter.Options; + if (Options.HasFlag(WritingOption.Gz)) + to = new GZipStream(to, CompressionLevel.Optimal); + try + { + if (Options.HasFlag(WritingOption.Dump)) + { + var binf = new BinaryFormatter(); + await Task.Yield(); + token.ThrowIfCancellationRequested(); + binf.Serialize(to, toWrite); + } + else if (Options.HasFlag(WritingOption.Binary)) + { + await to.WriteStringAsync(toWrite.GetType().AssemblyQualifiedName, token); + await toWrite.WriteBinaryAsync(to, token); + } + else if (Options.HasFlag(WritingOption.Text)) + { + using (var textWriter = new StreamWriter(to)) + { + await textWriter.WriteLineAsync($"Type={toWrite.GetType().AssemblyQualifiedName}".AsMemory(), token); + await textWriter.WriteLineAsync(new ReadOnlyMemory(), token); + await toWrite.WriteTextAsync(textWriter, token); + } + } + } + finally + { + if (Options.HasFlag(WritingOption.Gz)) + await to.DisposeAsync(); + } + } + + public static async Task ReadEntry(Stream from, CancellationToken token = default) + { + var Options = BinWriter.Options; + if (Options.HasFlag(WritingOption.Gz)) + from = new GZipStream(from, CompressionMode.Decompress); + try + { + object value = default; + if (Options.HasFlag(WritingOption.Dump)) + { + var binf = new BinaryFormatter(); + await Task.Yield(); + token.ThrowIfCancellationRequested(); + value= binf.Deserialize(from); + } + else if (Options.HasFlag(WritingOption.Binary)) + { + string typeName = await from.ReadStringAsync(token); + Type type = Type.GetType(typeName); + value = Activator.CreateInstance(type); + + await ((value as IBinaryWritable)?.ReadBinaryAsync(from, token) ?? throw new InvalidDataException("Bad type definition: " + type.FullName + " does not implement IBinaryWritable")); + + } + else if (Options.HasFlag(WritingOption.Text)) + { + using (var reader = new StreamReader(from)) + { + var line0 = await reader.ReadLineAsync(); + token.ThrowIfCancellationRequested(); + var def = line0.Split(new char[] { '=' }, 2); + await reader.ReadLineAsync(); + + token.ThrowIfCancellationRequested(); + if (def.Length != 2||def[0].Trim().ToLower()!="type") throw new InvalidDataException("Bad type definition"); + + Type type = Type.GetType(def[1].Trim()); + value = Activator.CreateInstance(type); + + await ((value as ITextWritable)?.ReadTextAsync(reader, token) ?? throw new InvalidDataException("Bad type definition: "+type.FullName+" does not implement ITextWritable")); + } + } + return value; + } + finally + { + if (Options.HasFlag(WritingOption.Gz)) + await from.DisposeAsync(); + } + } + } + + public interface IBinaryWritable + { + ValueTask WriteBinaryAsync(Stream to, CancellationToken cancel); + ValueTask ReadBinaryAsync(Stream from, CancellationToken cancel); + } + public interface ITextWritable + { + ValueTask WriteTextAsync(StreamWriter to, CancellationToken cancel); + ValueTask ReadTextAsync(StreamReader from, CancellationToken cancel); + } +} + diff --git a/napdump/napdump.csproj b/napdump/napdump.csproj new file mode 100644 index 0000000..211e8a7 --- /dev/null +++ b/napdump/napdump.csproj @@ -0,0 +1,27 @@ + + + + Exe + netcoreapp3.1 + + + + + + + + + ..\..\encaes\encaes\bin\Release\netcoreapp3.1\encaes.dll + + + ..\..\extensions\excryptotools\bin\Release\netstandard2.0\exbintools.dll + + + ..\..\extensions\excryptotools\bin\Release\netstandard2.0\excryptotools.dll + + + ..\..\extensions\exstreamtools\bin\Release\netstandard2.0\exstreamtools.dll + + + + diff --git a/ndimg/Program.cs b/ndimg/Program.cs new file mode 100644 index 0000000..4e7b301 --- /dev/null +++ b/ndimg/Program.cs @@ -0,0 +1,312 @@ +using System; +using System.IO; +using System.Threading.Tasks; +using System.Threading; +using System.Threading.Channels; +using System.Web; +using napdump; +using System.Runtime.Serialization.Formatters.Binary; +using System.Net; +using System.Collections.Generic; +using System.Net.Http; +using Tools.Crypto; +using System.Linq; + +namespace ndimg +{ + class Program + { + const int MaxThreads = 3; + static async Task ReadDump(Stream from, CancellationToken token) + { + var binf = new BinaryFormatter(); + await Task.Yield(); + try + { + var bi = (BoardInfo)binf.Deserialize(from); + token.ThrowIfCancellationRequested(); + return bi; + }catch(Exception ex) + { + throw new InvalidOperationException("Could not load dump: " + ex.Message); + } + } + static readonly CancellationTokenSource cancel = new CancellationTokenSource(); + static readonly AsyncMutex semaphore = AsyncMutex.Semaphore(MaxThreads); + static async Task DownloadImage(string url, Stream to, CancellationToken token) + { + using var downloader = new WebClient(); + using (await semaphore.AquireAsync(token)) + { + await using var reader = await downloader.OpenReadTaskAsync(url); + token.ThrowIfCancellationRequested(); + await reader.CopyToAsync(to, token); + } + } + static async Task<(int Complete,int Failed)> Reader(ChannelReader chan, DirectoryInfo output, CancellationToken token) + { + List downloaders = new List(); + int compDownloads = 0; + await foreach(var post in chan.ReadAllAsync(token)) + { + if (post.ImageURL != null && post.ImageURL.Length > 0) + { + downloaders.Add(Task.Run(async () => + { + var ipath = Path.Combine(output.FullName, post.PostNumber.ToString()); + try + { + if ( DeletedKey!=null &&( post.ModLog.ImageDeleted || post.ModLog.PostDeleted)) + { + await using (var tf = new TempFile()) + { + await DownloadImage(post.ImageURL, tf.Stream, token); + + tf.Stream.Position = 0; + using (var enc = new encaes.AesEncryptor(tf.Stream)) + { + enc.KeepAlive = true; + enc.Key = DeletedKey.Value; + await using (var writeStream = new FileStream(ipath, FileMode.Create)) + { + await enc.Encrypt(writeStream, token); + compDownloads += 1; + Console.WriteLine($"{post.PostNumber} -> {ipath} (encrypted)"); + return; + } + } + } + } + else + { + await using (var writeStream = new FileStream(ipath, FileMode.Create)) + { + await DownloadImage(post.ImageURL, writeStream, token); + compDownloads += 1; + Console.WriteLine($"{post.PostNumber} -> {ipath}"); + return; + } + } + } + catch (OperationCanceledException) { } + catch (Exception ex) + { + Console.WriteLine($"Failed to download image for post {post.PostNumber} ({post.ImageURL}): {ex.Message}"); + } + if (File.Exists(ipath)) try + { + File.Delete(ipath); + } + catch (Exception ex) { Console.WriteLine("Warning: State corrupted in file " + ipath+": " + ex.Message); } + })); + } + } + Console.WriteLine("Waiting for downloaders..."); + await Task.WhenAll(downloaders); + return (compDownloads, compDownloads - downloaders.Count); + } + public static async Task readKeyFromFile(string fn, CancellationToken token) + { + using(var fs = new FileStream(fn,FileMode.Open,FileAccess.Read)) + { + return await encaes.AesEncryptor.LoadKey(fs, async () => await encaes.AesEncryptor.ReadPassword("Key is password protected: ", token), token); + } + } + public static AESKey? DeletedKey { get; private set; } = null; + static string isEncryptedA(ref string[] args) + { + for (int i = 0; i < args.Length - 1; i++) + { + if (args[i].ToLower() == "--deleted-key") + { + var ret = args[i + 1]; + args = args.Where((_, j) => j != i && j != (i + 1)).ToArray(); + return ret; + } + } + return null; + } + static async Task Main(string[] args) + { + if(args.Length <1) + { + Console.WriteLine("Usage: ndimg [--deleted-key ] [] "); + return; + } + + string encFn = isEncryptedA(ref args); + + if(!File.Exists(args[0])) + { + Console.WriteLine("Error: dump file " + args[0] + " does not exist."); + return; + } + + + string outputDir; + if (args.Length < 2) + { + var psplit = Path.GetFileName(args[0]).Split('.'); + + outputDir = Path.Join(Path.GetDirectoryName(args[0]), string.Join('.', psplit.AsMemory().Slice(0, psplit.Length - 1).ToArray())); + } + else + outputDir = args[1]; + + Console.WriteLine($"Downloading {args[0]} -> {outputDir}"); + + DirectoryInfo output; + if (!Directory.Exists(outputDir)) + { + output = Directory.CreateDirectory(outputDir); + } + else + output = new DirectoryInfo(outputDir); + + Console.CancelKeyPress += (o, e) => + { + if (!cancel.IsCancellationRequested) + { + cancel.Cancel(); + e.Cancel = true; + } + else + { + Console.WriteLine("Force exit"); + Environment.Exit(-1); + } + }; + + try + { + if (encFn != null) + { + try + { + DeletedKey= await readKeyFromFile(encFn, cancel.Token); + } + catch (Exception ex) + { + Console.WriteLine("Failed to read key from file " + encFn + ": " + ex.Message); + return; + } + } + Channel posts = Channel.CreateUnbounded(); + var reader = Reader(posts.Reader, output, cancel.Token); + await using (var stream = new FileStream(args[0], FileMode.Open, FileAccess.Read)) + { + var board = await ReadDump(stream, cancel.Token); + foreach (var thread in board.Threads) + { + cancel.Token.ThrowIfCancellationRequested(); + + if (thread.IsEncrypted) + { + if (DeletedKey == null) + { + Console.WriteLine("Thread is encrypted, skipping."); + continue; + } + else + { + try + { + await thread.DecryptPostAsync(DeletedKey.Value, cancel.Token); + } + catch (Exception ex) + { + Console.WriteLine("Failed to decrypt thread, skipping: " + ex.Message); + continue; + } + } + } + + if(thread.IsImageEncrypted) + { + if (DeletedKey == null) + { + Console.WriteLine("Thread image is encrypted, skipping."); + continue; + } + else + { + try + { + await thread.DecryptImageAsync(DeletedKey.Value, cancel.Token); + } + catch (Exception ex) + { + Console.WriteLine("Failed to decrypt thread image, skipping: " + ex.Message); + continue; + } + } + } + + await posts.Writer.WriteAsync(thread, cancel.Token); + foreach (var post in thread.Children) + { + if(post.IsEncrypted) + { + + if (DeletedKey == null) + { + Console.WriteLine("Post is encrypted, skipping."); + continue; + } + else + { + try + { + await post.DecryptPostAsync(DeletedKey.Value, cancel.Token); + } + catch (Exception ex) + { + Console.WriteLine("Failed to decrypt post, skipping: " + ex.Message); + continue; + } + } + } + if (post.IsImageEncrypted) + { + if (DeletedKey == null) + { + Console.WriteLine("Post image is encrypted, skipping."); + continue; + } + else + { + try + { + await post.DecryptImageAsync(DeletedKey.Value, cancel.Token); + } + catch (Exception ex) + { + Console.WriteLine("Failed to decrypt post image, skipping: " + ex.Message); + continue; + } + } + } + + await posts.Writer.WriteAsync(post, cancel.Token); + + } + } + posts.Writer.Complete(); + } + var comp = await reader; + Console.WriteLine($"Complete. ({comp.Complete} downloaded, {comp.Failed} failed)"); + } + catch (Exception ex) + { + if (!cancel.IsCancellationRequested) + cancel.Cancel(); + Console.WriteLine("Error downloading: " + ex.Message); + } + finally + { + semaphore.Dispose(); + cancel.Dispose(); + } + } + } +} diff --git a/ndimg/TempFile.cs b/ndimg/TempFile.cs new file mode 100644 index 0000000..a77e06c --- /dev/null +++ b/ndimg/TempFile.cs @@ -0,0 +1,100 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Threading.Tasks; + +namespace ndimg +{ + sealed class TempDirectory : IDisposable + { + public DirectoryInfo Directory { get; } + public TempDirectory(DirectoryInfo location) + { + Directory = new DirectoryInfo(Path.Combine(location.FullName, TempFile.GenerateLocation())); + Directory.Create(); + } + public TempDirectory() : this(new DirectoryInfo(TempFile.TempFileLocation)) { } + + public void Dispose() + { + Directory.Delete(true); + } + } + sealed class TempFile : IDisposable, IAsyncDisposable + { + public static string GenerateLocation() + { + var time = DateTime.Now; + return $"ndimg-{time.Year}-{time.Month}-{time.Day}-{time.Hour}.{time.Minute}.{time.Second}.{time.Millisecond}-{Guid.NewGuid().ToString()}"; + } + + public void Dispose() + { + if (File.Exists) + { + if (Stream != null) + Stream.Dispose(); + File.Delete(); + } + } + + public async ValueTask DisposeAsync() + { + if (File.Exists) + { + if(Stream!=null) + await Stream.DisposeAsync(); + File.Delete(); + } + } + + ~TempFile() + { + if (File.Exists) + { + try { File.Delete(); } catch { } + } + } + + public static string TempFileLocation { get; set; } + static TempFile() + { + TempFileLocation = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), GenerateLocation())).FullName; + + AppDomain.CurrentDomain.DomainUnload += (o, e) => + { + try + { + foreach (var file in new DirectoryInfo(TempFileLocation).GetFiles()) try + { + file.Delete(); + } + catch { } + Directory.Delete(TempFileLocation, true); + } + catch { } + }; + } + public FileInfo File { get; } + public Stream Stream { get; } + public TempFile(DirectoryInfo location) + { + var nam = Path.Combine(location.FullName, GenerateLocation() + ".tmp"); + File = new FileInfo(nam); + Stream = new FileStream(nam, FileMode.Create); + } + public TempFile() : this(new DirectoryInfo(TempFileLocation)) { } + + private TempFile(FileInfo fi) + { + File = fi; + Stream = null; + } + + public static TempFile WithNoStream() + { + return new TempFile(new FileInfo(Path.Combine(new DirectoryInfo(TempFileLocation).FullName, GenerateLocation() + ".tmp"))); + } + } +} diff --git a/ndimg/ndimg.csproj b/ndimg/ndimg.csproj new file mode 100644 index 0000000..6e7e89c --- /dev/null +++ b/ndimg/ndimg.csproj @@ -0,0 +1,27 @@ + + + + Exe + netcoreapp3.1 + + + + + + + + + ..\..\encaes\encaes\bin\Release\netcoreapp3.1\encaes.dll + + + ..\..\extensions\excryptotools\bin\Release\netstandard2.0\exbintools.dll + + + ..\..\extensions\excryptotools\bin\Release\netstandard2.0\excryptotools.dll + + + ..\..\extensions\exstreamtools\bin\Release\netstandard2.0\exstreamtools.dll + + + + diff --git a/ndpack/Program.cs b/ndpack/Program.cs new file mode 100644 index 0000000..994f1c0 --- /dev/null +++ b/ndpack/Program.cs @@ -0,0 +1,262 @@ +using ICSharpCode.SharpZipLib.Zip; +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading; +using System.Threading.Tasks; + +namespace ndpack +{ + class Program + { + static Task ReadPassword(string prompt = null, CancellationToken token = default) + { + TaskCompletionSource comp = new TaskCompletionSource(); + List pwd = new List(); + + bool set = false; + _ = Task.Run(() => + { + if (prompt != null) + Console.Write(prompt); + try + { + using var _c = token.Register(() => + { + if (!set) + { + set = true; + comp.SetException(new OperationCanceledException()); + } + }); + while (true) + { + ConsoleKeyInfo i = Console.ReadKey(true); + if (token.IsCancellationRequested) break; + if (i.Key == ConsoleKey.Enter) + { + Console.WriteLine(); + break; + } + else if (i.Key == ConsoleKey.Backspace) + { + if (pwd.Count > 0) + { + pwd.RemoveAt(pwd.Count - 1); + } + } + else if (i.KeyChar != '\u0000') + { + pwd.Add(i.KeyChar); + } + } + + if (!set) + { + set = true; + if (token.IsCancellationRequested) + comp.SetException(new OperationCanceledException()); + else + comp.SetResult(new string(pwd.ToArray())); + } + } + catch (Exception ex) + { + Console.WriteLine(); + pwd.Clear(); + if (!set) + { + set = true; + comp.SetException(ex); + } + } + }); + return comp.Task; + } + + static void usage() + { + Console.WriteLine("Usage: ndpack -c "); + Console.WriteLine("Usage: ndpack -e "); + Console.WriteLine("Usage: ndpack -u "); + Console.WriteLine("Usage: ndpack -d "); + } + static async Task CopyDirectory(string SourcePath, string DestinationPath) + { + Exception bad = null; + await Task.Run(() => + { + try + { + //Now Create all of the directories + foreach (string dirPath in Directory.GetDirectories(SourcePath, "*", + SearchOption.AllDirectories)) + Directory.CreateDirectory(dirPath.Replace(SourcePath, DestinationPath)); + + //Copy all the files & Replaces any files with the same name + foreach (string newPath in Directory.GetFiles(SourcePath, "*", + SearchOption.AllDirectories)) + File.Copy(newPath, newPath.Replace(SourcePath, DestinationPath), true); + }catch(Exception ex) + { + bad = ex; + } + }); + if (bad != null) throw bad; + } + static async Task create(string output, string dumpf, string dirf) + { + FastZip fs = new FastZip(); + using var tdir = new TempDirectory(); + await Task.Yield(); + Console.WriteLine("Copying dump..."); + File.Copy(dumpf, Path.Combine(tdir.Directory.FullName, Path.GetFileName(dumpf))); + Console.WriteLine($"Copying images..."); + var ipath = Path.Combine(tdir.Directory.FullName, new DirectoryInfo(dirf).Name); + Directory.CreateDirectory(ipath); + await CopyDirectory(dirf, ipath); + + await Task.Yield(); + fs.CreateZip(output, tdir.Directory.FullName, true, null, null); + } + static async Task unpack(string inputArchive, string outputDir) + { + if (!Directory.Exists(outputDir)) + Directory.CreateDirectory(outputDir); + + var fs = new FastZip(); + + await Task.Yield(); + fs.ExtractZip(inputArchive, outputDir, null); + } + static async Task decrypt(string inf, string of, string kf, Func> password) + { + using var infs = new FileStream(inf, FileMode.Open, FileAccess.Read); + using var enc = new encaes.AesEncryptor(infs); + using (var kfs = new FileStream(kf, FileMode.Open, FileAccess.Read)) + { + enc.Key = await encaes.AesEncryptor.LoadKey(kfs, () => + { + var tsk = password(); + tsk.Wait(); + if (tsk.IsFaulted) + throw tsk.Exception; + else return tsk.Result; + }); + } + + using var ofs = new FileStream(of, FileMode.Create); + await enc.Decrypt(ofs); + } + static async Task encrypt(string inf, string outf, string kf, Func> password) + { + using var infs = new FileStream(inf, FileMode.Open, FileAccess.Read); + using var enc = new encaes.AesEncryptor(infs); + using (var kfs = new FileStream(kf, FileMode.Open, FileAccess.Read)) + { + enc.Key = await encaes.AesEncryptor.LoadKey(kfs, () => + { + var tsk = password(); + tsk.Wait(); + if (tsk.IsFaulted) + throw tsk.Exception; + else return tsk.Result; + }); + } + using (var ofs = new FileStream(outf, FileMode.Create)) + { + await enc.Encrypt(ofs); + } + } + static async Task Main(string[] args) + { + if (args.Length < 3) + { + usage(); + return; + } + else + { + switch (args[0].ToLower()) + { + case "-c": + if (args.Length < 4) + { + usage(); + return; + } + try + { + await create(args[3], args[1], args[2]); + } + catch (Exception ex) + { + Console.WriteLine("Error creating: " + ex.Message); + } + break; + case "-e": + if(args.Length<5) + { + usage(); + return; + } + try + { + string realOutput = args[4]; + using var tf = TempFile.WithNoStream(); + await create(tf.File.FullName, args[1], args[2]); + + Console.WriteLine("Encrypting..."); + await encrypt(tf.File.FullName, realOutput, args[3], async () => + { + return await ReadPassword("Password: "); + }); + } + catch (Exception ex) + { + Console.WriteLine("Error creating: " + ex.Message); + } + break; + case "-u": + try + { + Console.WriteLine("Unpacking..."); + await unpack(args[1], args[2]); + } + catch (Exception ex) + { + Console.WriteLine("Error unpacking: " + ex.Message); + } + break; + case "-d": + if (args.Length < 4) + { + usage(); + return; + } + try + { + using var arcfs = TempFile.WithNoStream(); + + Console.WriteLine("Decrypting..."); + await decrypt(args[1], arcfs.File.FullName, args[2], async () => + { + return await ReadPassword("Password: "); + }); + + Console.WriteLine("Unpacking..."); + await unpack(arcfs.File.FullName, args[3]); + } + catch (Exception ex) + { + Console.WriteLine("Error unpacking: " + ex.Message); + } + break; + default: + usage(); + return; + } + } + } + } +} diff --git a/ndpack/TempFile.cs b/ndpack/TempFile.cs new file mode 100644 index 0000000..113d9d8 --- /dev/null +++ b/ndpack/TempFile.cs @@ -0,0 +1,100 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Threading.Tasks; + +namespace ndpack +{ + public sealed class TempDirectory : IDisposable + { + public DirectoryInfo Directory { get; } + public TempDirectory(DirectoryInfo location) + { + Directory = new DirectoryInfo(Path.Combine(location.FullName, TempFile.GenerateLocation())); + Directory.Create(); + } + public TempDirectory() : this(new DirectoryInfo(TempFile.TempFileLocation)) { } + + public void Dispose() + { + Directory.Delete(true); + } + } + public sealed class TempFile : IDisposable, IAsyncDisposable + { + public static string GenerateLocation() + { + var time = DateTime.Now; + return $"{time.Year}-{time.Month}-{time.Day}-{time.Hour}.{time.Minute}.{time.Second}.{time.Millisecond}-{Guid.NewGuid().ToString()}"; + } + + public void Dispose() + { + if (File.Exists) + { + if (Stream != null) + Stream.Dispose(); + File.Delete(); + } + } + + public async ValueTask DisposeAsync() + { + if (File.Exists) + { + if(Stream!=null) + await Stream.DisposeAsync(); + File.Delete(); + } + } + + ~TempFile() + { + if (File.Exists) + { + try { File.Delete(); } catch { } + } + } + + public static string TempFileLocation { get; set; } + static TempFile() + { + TempFileLocation = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), GenerateLocation())).FullName; + + AppDomain.CurrentDomain.DomainUnload += (o, e) => + { + try + { + foreach (var file in new DirectoryInfo(TempFileLocation).GetFiles()) try + { + file.Delete(); + } + catch { } + Directory.Delete(TempFileLocation, true); + } + catch { } + }; + } + public FileInfo File { get; } + public Stream Stream { get; } + public TempFile(DirectoryInfo location) + { + var nam = Path.Combine(location.FullName, GenerateLocation() + ".tmp"); + File = new FileInfo(nam); + Stream = new FileStream(nam, FileMode.Create); + } + public TempFile() : this(new DirectoryInfo(TempFileLocation)) { } + + private TempFile(FileInfo fi) + { + File = fi; + Stream = null; + } + + public static TempFile WithNoStream() + { + return new TempFile(new FileInfo(Path.Combine(new DirectoryInfo(TempFileLocation).FullName, GenerateLocation() + ".tmp"))); + } + } +} diff --git a/ndpack/ndpack.csproj b/ndpack/ndpack.csproj new file mode 100644 index 0000000..9e8334a --- /dev/null +++ b/ndpack/ndpack.csproj @@ -0,0 +1,27 @@ + + + + Exe + netcoreapp3.1 + + + + + + + + + ..\..\encaes\encaes\bin\Release\netcoreapp3.1\encaes.dll + + + ..\..\extensions\excryptotools\bin\Release\netstandard2.0\exbintools.dll + + + ..\..\extensions\excryptotools\bin\Release\netstandard2.0\excryptotools.dll + + + ..\..\extensions\exstreamtools\bin\Release\netstandard2.0\exstreamtools.dll + + + + diff --git a/ndview/Dispatcher.cs b/ndview/Dispatcher.cs new file mode 100644 index 0000000..f6e90b9 --- /dev/null +++ b/ndview/Dispatcher.cs @@ -0,0 +1,152 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using System.Threading.Channels; +using System.Collections.Generic; +using System.Linq; + +namespace ndview +{ + public class Dispatcher : IDisposable + { + protected napdump.AsyncMutex mutex = new napdump.AsyncMutex(); + protected Action Hooks = delegate { }; + public delegate Task AsyncHookFunction(AsyncHookFunction @this, CancellationToken token); + protected List> AsyncHooks = new List>(); + + public bool InSeries {get; set;} = false; + + public Dispatcher() + { + + } + + public void Dispose() + { + Hooks = delegate {}; + AsyncHooks.Clear(); + mutex.Dispose(); + sigMutex.Dispose(); + } + + public Task WaitForCurrentSignal + { + get + { + return Task.Run(async () => { + using var _m = await sigMutex.AquireAsync(); + }); + } + } + + public void Hook(Action run) + { + using(mutex.Aquire()) + Hooks += run; + } + + public Task HookAsync(Func run, CancellationToken token=default) + => HookAsync(run, false, token); + public async Task HookAsync(Func run, bool single, CancellationToken token=default) + { + using(await mutex.AquireAsync(token)) { + AsyncHooks.Add(run); + if(single) { + async Task remover(CancellationToken tok) + { + using var link = CancellationTokenSource.CreateLinkedTokenSource(tok,token); + await Task.WhenAll(RemoveHookAsync(run, link.Token), RemoveHookAsync(remover, link.Token)); + } + AsyncHooks.Add(remover); + } + } + } + + private readonly Dictionary, Func> ahnc = new Dictionary, Func>(); + public Task HookAsync(Func run, CancellationToken token=default) + => HookAsync(run, false, token); + public async Task HookAsync(Func run, bool single, CancellationToken token=default) + { + Func lam = (_) => run(); + using(await mutex.AquireAsync(token)) + { + ahnc.Add(run, lam); + AsyncHooks.Add(lam); + if(single) { + async Task remover(CancellationToken tok) + { + using var link = CancellationTokenSource.CreateLinkedTokenSource(tok,token); + await Task.WhenAll(RemoveHookAsync(run, link.Token), + RemoveHookAsync(remover, link.Token)); + } + AsyncHooks.Add(remover); + } + } + } + + public async Task RemoveHookAsync(Func hook, CancellationToken token=default) + { + using(await mutex.AquireAsync(token)) + AsyncHooks.Remove(hook); + } + + public async Task RemoveHookAsync(Func hook, CancellationToken token=default) + { + using(await mutex.AquireAsync(token)) + { + if(!ahnc.ContainsKey(hook)) return; + var run = ahnc[hook]; + AsyncHooks.Remove(run); + ahnc.Remove(hook); + } + } + + public async Task RemoveHookAsync(Action remove, CancellationToken token=default) + { + using(await mutex.AquireAsync(token)) + Hooks -= remove; + } + + protected napdump.AsyncMutex sigMutex = new napdump.AsyncMutex(); + public async Task Signal(CancellationToken token=default) + { + Action h; + Func[] ah; + using(await sigMutex.AquireAsync(token)) { + using(await mutex.AquireAsync(token)) + { + h = Hooks; + ah = AsyncHooks.ToArray(); + } + h?.Invoke(); + var sig = InSeries ? Task.Run(async () => { + foreach(var x in ah.Reverse()) + await x(token); + }) : Task.WhenAll(ah.Select(x=> x(token))); + TaskCompletionSource canceller = new TaskCompletionSource(); + using var _c = token.Register(()=> canceller.SetResult(true)); + + if( (await Task.WhenAny(sig, canceller.Task)) != sig) + throw new OperationCanceledException(); + } + } + + public async Task WaitForSignal(CancellationToken token=default) + { + TaskCompletionSource comp = new TaskCompletionSource(); + await HookAsync((tok)=> { + + if(tok.IsCancellationRequested || token.IsCancellationRequested) + comp.SetResult(false); + else + comp.SetResult(true); + return Task.CompletedTask; + }, true, token); + + + using var _c = token.Register(()=> comp.TrySetException(new OperationCanceledException())); + if(!await comp.Task) + throw new OperationCanceledException(); + } + } +} diff --git a/ndview/PageGenerator.cs b/ndview/PageGenerator.cs new file mode 100644 index 0000000..689ab82 --- /dev/null +++ b/ndview/PageGenerator.cs @@ -0,0 +1,638 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Threading; +using System.Threading.Channels; +using System.Threading.Tasks; +using System.Linq; + +using napdump; +using Tools; + +namespace ndview +{ + class PageGenerator + { + public napdump.BoardInfo Board { get; } + public DirectoryInfo Images { get; } + public PageGenerator(napdump.BoardInfo board, DirectoryInfo images) + { + Board = board; + Images = images; + } + + private async Task ExtractImage(PostInfo post, DirectoryInfo imgOutput, bool encrypted, CancellationToken token) + { + if (post.ImageURL == null) throw new InvalidOperationException($"Post {post.PostNumber} does not have an image."); + var imageFn = Path.Join(Images.FullName, post.PostNumber.ToString()); + if (File.Exists(imageFn)) + { + var ifn = post.ImageURL.Split('/').Last(); + await Task.Yield(); + var ofn = Path.Join(imgOutput.FullName, ifn); + if (encrypted && Program.DeletedKey!=null) + { + Console.WriteLine($"Extracting encrypted image {post.PostNumber} -> {ofn}"); + using (var inp = new FileStream(imageFn, FileMode.Open, FileAccess.Read)) + { + using (var oup = new FileStream(ofn, FileMode.Create)) + { + using (var dec = new encaes.AesEncryptor(inp) { KeepAlive = true }) + { + dec.Key = Program.DeletedKey.Value; + await dec.Decrypt(oup, token); + } + } + } + } + else + { + Console.WriteLine($"Extracting image {post.PostNumber} -> {ofn}"); + if (encrypted) + { + WriteYellowLine("Warning: Image for post "+post.PostNumber+" will be garbage: Cannot be decrypted."); + } + await CopyFileAsync(imageFn, ofn, token); + } + return ifn; + } + else throw new InvalidOperationException("Bad state: no image for " + post.PostNumber); + } + + private static async Task CopyFileAsync(string from, string to, CancellationToken cancel = default) + { + using var ifs = new FileStream(from, FileMode.Open, FileAccess.Read); + using var ofs = new FileStream(to, FileMode.Create); + + await ifs.CopyToAsync(ofs, cancel); + } + + private async Task WritePost(HtmlGenerator index, PostInfo post, DirectoryInfo img, bool wasEnc, CancellationToken token) + { + bool postImgEnc; + if (postImgEnc = post.IsImageEncrypted) + { + if (Program.DeletedKey == null) + { + WriteYellowLine("Post " + post.PostNumber + " image is encrypted, not attempting to write."); + } + else + { + try + { + await post.DecryptImageAsync(Program.DeletedKey.Value, token); + } + catch (OperationCanceledException op) { throw op; } + catch (Exception ex) + { + WriteYellowLine("Failed to decrypt post " + post.PostNumber + " image, skipping: " + ex.Message); + return; + } + } + } + + var imageExtractor = !post.IsImageEncrypted && post.ImageURL != null ? ExtractImage(post, img, postImgEnc || wasEnc, token) : Task.CompletedTask; + + await using (await index.TagAsync("article", token, ("id", post.PostNumber.ToString()))) + { + await WriteHeader(index, post, token); + if (post.ImageURL != null) + await WriteImageFigure(index, post, token); + + await using (await index.TagAsync("blockquote", token)) + { + await index.AppendHtml(post.Body); + } + } + + await imageExtractor; + } + + private static string HumanBytes(long byteCount) + { + string[] suf = { "B", "KB", "MB", "GB", "TB", "PB", "EB" }; //Longs run out around EB + if (byteCount == 0) + return "0" + suf[0]; + long bytes = Math.Abs(byteCount); + int place = Convert.ToInt32(Math.Floor(Math.Log(bytes, 1024))); + double num = Math.Round(bytes / Math.Pow(1024, place), 1); + return (Math.Sign(byteCount) * num).ToString() + " " + suf[place]; + } + + private async Task WriteImageFigure(HtmlGenerator index, PostInfo thread, CancellationToken token) + { + if (thread.ImageURL == null) throw new InvalidOperationException($"Post {thread.PostNumber} has no image"); + + var ifn = thread.ImageURL.Split('/').Last(); + + + await using (await index.TagAsync("figure", token)) + { + await using (await index.TagAsync("figcaption")) + { + await using (await index.TagAsync("i")) + { + await index.Append($@"({HumanBytes(thread.ImageSize)} {thread.ImageDimensions.Width}x{thread.ImageDimensions.Height})"); + await using (await index.TagAsync("a", token, ("rel", "nofollow"), ("title", thread.ImageFilename), ("download", thread.ImageFilename), ("href", $"i/{ifn}"))) + { + await index.Append(thread.ImageFilename); + } + } + } + + await index.TagSelfClosingAsync("img", token, ("src", $"i/{ifn}"), ("height", "250"), ("width", "250")); + } + + } + private async Task WriteHeader(HtmlGenerator index, PostInfo thread, CancellationToken token) + { + await using (await index.TagAsync("header", token)) + { + if (thread.Subject != null) + { + await using (await index.TagAsync("h3")) + { + await index.Append(thread.Subject, token); + } + await index.Append(" ", token); + } + await using (await index.TagAsync("b", token, ("class", "name"))) + { + + var emailTag = (thread.Email == null || thread.Email == "") ? null : await index.TagAsync("a", token, ("class", "email"), ("href", "mailto:" + thread.Email)); + await index.Append(thread.Name, token); + if (thread.Tripcode != null) + { + await index.AppendHtml("", token); + await index.Append(thread.Tripcode, token); + await index.AppendHtml(" ", token); + } + if (thread.Capcode != null) + { + await index.AppendHtml("", token); + await index.Append(thread.Capcode, token); + await index.AppendHtml(" ", token); + } + if (emailTag != null) + await emailTag.DisposeAsync(); + } + await index.Append(" ", token); + await using (await index.TagAsync("time", token, ("title", thread.Timestamp.ToString()))) + { + await index.Append(thread.Timestamp.ToShortDateString(), token); + } + await index.Append(" ", token); + await using (await index.TagAsync("a", token, ("href", "#" + thread.PostNumber))) + { + await index.Append(thread.PostNumber.ToString(), token); + } + } + } + + private async Task WriteBody(HtmlGenerator index, PostInfo post, CancellationToken token) + { + await using (await index.TagAsync("blockquote", token)) + { + await index.AppendHtml(post.Body); + } + } + + private async Task WriteThread(HtmlGenerator index, ThreadInfo thread, DirectoryInfo img, bool wasEnc, CancellationToken token) + { + bool threadImgEnc; + if (threadImgEnc = thread.IsImageEncrypted) + { + if (Program.DeletedKey == null) + { + WriteYellowLine("Thread " + thread.PostNumber + " image is encrypted, not attempting to write."); + } + else + { + try + { + await thread.DecryptImageAsync(Program.DeletedKey.Value, token); + } + catch (OperationCanceledException op) { throw op; } + catch (Exception ex) + { + WriteYellowLine("Failed to decrypt thread " + thread.PostNumber + " image, skipping: " + ex.Message); + return; + } + } + } + + var imgExtract = !thread.IsImageEncrypted && thread.ImageURL != null ? ExtractImage(thread, img, threadImgEnc || wasEnc, token) : Task.CompletedTask; + + if (thread.ImageURL == null) + { + var ocol = Console.ForegroundColor; + Console.ForegroundColor = ConsoleColor.Yellow; + Console.WriteLine("Warning: Thread OP " + thread.PostNumber + " has no image."); + Console.ForegroundColor = ocol; + } + + await using (await index.TagAsync("section", token, ("id", thread.PostNumber.ToString()), ("class", "thread"))) + { + await WriteHeader(index, thread, token); + if (thread.ImageURL != null) + await WriteImageFigure(index, thread, token); + + await WriteBody(index, thread, token); + await using (await index.TagAsync("small", token)) + { + await index.Append($"{thread.Children.Count} replies", token); + } + + //Write posts + await using (await index.TagAsync("div", token, ("class", "replies"))) + foreach (var post in thread.Children.OrderBy(x => x.PostNumber)) + { + bool pWasEnc; + if (pWasEnc = post.IsEncrypted) + { + if (Program.DeletedKey == null) + { + WriteYellowLine("Post encrypted, skipping."); + continue; + } + else + { + try + { + await post.DecryptPostAsync(Program.DeletedKey.Value, token); + } + catch (Exception ex) + { + WriteYellowLine("Failed to decrypt post, skipping: " + ex.Message); + continue; + } + } + } + + await WritePost(index, post, img, pWasEnc, token); + } + } + + await imgExtract; + } + + public async Task GenerateFull(DirectoryInfo output, CancellationToken cancel) + { + await using var index = HtmlGenerator.CreateFile(Path.Combine(output.FullName, "index.html"), 10, cancel); + var imageOutputDir = new DirectoryInfo(Path.Combine(output.FullName, "i")); + imageOutputDir.Create(); + + index.Metas.Add(@""); + index.Metas.Add(@""); + await index.AppendHeaders(async (token) => + { + await using (await index.TagAsync("title", token)) + await index.Append($"Archive of `{Board.Title}'"); + + await using (await index.TagAsync("style", token)) + await index.AppendHtml(Properties.Resources.css); + await using (await index.TagAsync("h1", token)) + { + await index.Append(Board.Title, token); + + } + await using (await index.TagAsync("div", cancel, ("class", "stat"))) + { + await index.Append($"Showing {Board.Threads.Count} threads containing {Board.Threads.Select(x => x.Children.Count).Sum()} posts and {Board.Threads.Count + Board.Threads.Select(x => x.Children.Where(y => y.ImageURL != null).Count()).Sum()} images.", cancel); + await index.AppendHtml($"
Taken at .
Original: ", cancel); + await index.Append(Board.BoardName, cancel); + await index.AppendHtml($"", cancel); + } + await index.TagSelfClosingAsync("hr", cancel); + }, cancel); + + + await using (await index.TagAsync("body", cancel)) + { + await using (await index.TagAsync("main", cancel)) + { + foreach (var thread in Board.Threads.OrderByDescending(x => x.Children.Where(x => !(x.Email?.Equals("sage") ?? false)).LastOrDefault()?.PostNumber ?? x.PostNumber)) + { + bool wasEnc; + if (wasEnc = thread.IsEncrypted) + { + if (Program.DeletedKey == null) + { + WriteYellowLine("Thread is encrypted, skipping."); + continue; + } + else + { + try + { + await thread.DecryptPostAsync(Program.DeletedKey.Value, cancel); + } + catch (OperationCanceledException op) { throw op; } + catch (Exception ex) + { + WriteYellowLine("Failed to decrypt thread, skipping: " + ex.Message); + continue; + } + } + } + + await WriteThread(index, thread, imageOutputDir, wasEnc, cancel); + } + } + } + + await index.AppendHtml(@$"
Page generated on by ndview
", cancel); + + await using (await index.TagAsync("script", cancel)) + await index.AppendHtml(Properties.Resources.script); + } + + private static void WriteYellowLine(string value) + { + var kkey = Console.ForegroundColor; + Console.ForegroundColor = ConsoleColor.Yellow; + Console.WriteLine(value); + Console.ForegroundColor = kkey; + } + } + public readonly struct Safe + { + public readonly T Value; + + public Safe(T value) => Value = value; + + public static explicit operator T(Safe t) => t.Value; + } + public readonly struct HtmlAttribute + { + public readonly string Name, Value; + public HtmlAttribute(string nm, string vl) + { + Name = nm; + Value = vl.Replace("\'", "\\\'"); + } + + public override string ToString() + { + return $"{Name}='{Value}'"; + } + + public static implicit operator HtmlAttribute((string Name, string Value) from) + { + return new HtmlAttribute(from.Name, from.Value); + } + + public void Deconstruct(out string name, out string value) + { + name = Name; + value = Value; + } + + } + public class HtmlTag : IAsyncDisposable + { + public string Name { get; } + public HtmlAttribute[] Attributes { get; } + public HtmlGenerator Owner { get; } + public HtmlTag(HtmlGenerator super, string tname, params HtmlAttribute[] attrs) + { + Owner = super; + Name = tname; + Attributes = attrs; + } + + public string AttributeString + => Attributes.Length < 1 ? "" : (" " + string.Join(' ', Attributes.Select(x => x.ToString()).ToArray())); + + public async Task SelfClose(CancellationToken token) + { + if (disposed) return; + disposed = true; + await Owner.AppendHtml($"<{Name}{AttributeString} />"); + } + + internal async Task Begin(CancellationToken token) + { + await Owner.AppendHtml($"<{Name}{AttributeString}>"); + } + + private bool disposed = false; + public async ValueTask DisposeAsync() + { + if (disposed) return; + disposed = true; + await Owner.AppendHtml($""); + } + } + public class HtmlGenerator : IDisposable, IAsyncDisposable + { + + public HtmlTag Tag(string name, params HtmlAttribute[] attrs) + { + var t = new HtmlTag(this, name, attrs); + t.Begin(cancel.Token).Wait(); + return t; + } + + public async Task TagAsync(string name, params HtmlAttribute[] attrs) + { + var t = new HtmlTag(this, name, attrs); + await t.Begin(cancel.Token); + return t; + } + public async Task TagAsync(string name, CancellationToken token, params HtmlAttribute[] attrs) + { + using var link = CancellationTokenSource.CreateLinkedTokenSource(cancel.Token, token); + var t = new HtmlTag(this, name, attrs); + await t.Begin(link.Token); + return t; + } + + public async Task TagAsync(string name, HtmlAttribute[] attrs, CancellationToken token = default) + { + using var link = CancellationTokenSource.CreateLinkedTokenSource(cancel.Token, token); + + var t = new HtmlTag(this, name, attrs); + await t.Begin(link.Token); + return t; + } + public async Task TagAsync(string name, CancellationToken token = default) + { + using var link = CancellationTokenSource.CreateLinkedTokenSource(cancel.Token, token); + + var t = new HtmlTag(this, name); + await t.Begin(link.Token); + return t; + } + public Task TagSelfClosingAsync(string name, params HtmlAttribute[] attrs) + => TagSelfClosingAsync(name, default, attrs); + public async Task TagSelfClosingAsync(string name, CancellationToken cancel, params HtmlAttribute[] attrs) + { + using var link = CancellationTokenSource.CreateLinkedTokenSource(this.cancel.Token, cancel); + + await using var t = new HtmlTag(this, name, attrs); + await t.SelfClose(cancel); + + } + + protected readonly Stream stream; + public StreamWriter Output { get; } + public bool OwnsStream { get; set; } = false; + protected readonly CancellationTokenSource cancel; + + public List Metas { get; } = new List(); + + public Dispatcher OnFlush = new Dispatcher() { InSeries = true }; + + public Task AppendHeaders(CancellationToken token = default) + => AppendHeaders((x) => Task.CompletedTask, token); + public async Task AppendHeaders(Func head, CancellationToken token = default) + { + using var cancel = CancellationTokenSource.CreateLinkedTokenSource(this.cancel.Token, token); + + await AppendHtml(@"", cancel.Token); + await Task.WhenAll(Metas.Select(x => AppendHtml(x, cancel.Token))); + await head(cancel.Token); + token.ThrowIfCancellationRequested(); + await AppendHtml(@"", cancel.Token); + + await OnFlush.HookAsync(async (c) => + { + await AppendHtml("", c); + }, token); + } + + public static HtmlGenerator CreateFile(string file, int bufferLength = 0, CancellationToken globalToken = default) + { + var fs = new FileStream(file, FileMode.Create); + try + { + return new HtmlGenerator(fs, bufferLength, globalToken) { OwnsStream = true }; + } + catch (Exception ex) + { + fs.Dispose(); + throw ex; + } + } + + public HtmlGenerator(Stream output, int bufferLength = 0, CancellationToken token = default) + { + stream = output; + cancel = CancellationTokenSource.CreateLinkedTokenSource(token); + Output = new StreamWriter(output); + + var chan = bufferLength < 1 ? Channel.CreateUnbounded() : Channel.CreateBounded(bufferLength); + reader = chan.Reader; + writer = chan.Writer; + + writeHook = Task.Run(async () => + { + try + { + await foreach (var obj in reader.ReadAllAsync(cancel.Token)) + { + if (obj is Safe safe) + { + await Output.WriteAsync(safe.Value.AsMemory(), cancel.Token); + } + else + await Output.WriteAsync(System.Text.Encodings.Web.HtmlEncoder.Default.Encode(obj is string str ? str : obj?.ToString() ?? "").AsMemory(), cancel.Token); + } + } + catch (OperationCanceledException) { } + catch (Exception ex) + { + OnWriterError?.Invoke(ex); + CancelAllOperations(); + } + }); + } + + public async ValueTask DisposeAsync() + { + await Flush(); + await OnFlush.WaitForCurrentSignal; + + Dispose(); + } + + public event Action OnWriterError; + + private readonly ChannelReader reader; + protected readonly ChannelWriter writer; + + public void CancelAllOperations() + { + if (!cancel.IsCancellationRequested) + cancel.Cancel(); + } + + protected Task writeHook; + public Task Completion => writeHook; + + protected bool flushed = false; + public async Task Flush(CancellationToken token = default) + { + if (flushed) return; + flushed = true; + await OnFlush.Signal(token); + writer.Complete(); + await writeHook; + } + + public async Task Append(string str, CancellationToken token = default) + { + using var cancel = CancellationTokenSource.CreateLinkedTokenSource(this.cancel.Token, token); + await writer.WriteAsync(str, cancel.Token); + } + public async Task Append(Safe str, CancellationToken token = default) + { + using var cancel = CancellationTokenSource.CreateLinkedTokenSource(this.cancel.Token, token); + await writer.WriteAsync(str, cancel.Token); + } + public async Task AppendHtml(string str, CancellationToken token = default) + { + using var cancel = CancellationTokenSource.CreateLinkedTokenSource(this.cancel.Token, token); + await writer.WriteAsync(new Safe(str), cancel.Token); + } + + #region IDisposable Support + private bool disposedValue = false; // To detect redundant calls + + protected virtual void Dispose(bool disposing) + { + if (!disposedValue) + { + if (disposing) + { + if (!cancel.IsCancellationRequested) + cancel.Cancel(); + Output.Flush(); + OnFlush.Dispose(); + cancel.Dispose(); + Output.Close(); + + + if (OwnsStream) stream.Dispose(); + } + + + disposedValue = true; + } + } + + ~HtmlGenerator() + { + Dispose(false); + } + + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + #endregion + } +} \ No newline at end of file diff --git a/ndview/Program.cs b/ndview/Program.cs new file mode 100644 index 0000000..a403f76 --- /dev/null +++ b/ndview/Program.cs @@ -0,0 +1,302 @@ +using System; +using System.IO; +using System.Threading.Tasks; +using System.Threading; +using System.Linq; +using ICSharpCode.SharpZipLib.Zip; +using System.Runtime.Serialization.Formatters.Binary; +using System.Security; +using System.Runtime.InteropServices; +using System.Text; +using System.Collections.Generic; +using Tools.Crypto; + +namespace ndview +{ + class Program + { + static void Usage() + { + Console.WriteLine("ndview [--deleted-key ] [] "); + } + + static async Task Decrypt(Stream from, Stream keyFile, Func getPasssword, CancellationToken token) + { + using (var enc = new encaes.AesEncryptor(from) { KeepAlive = true }) + { + enc.Key = await encaes.AesEncryptor.LoadKey(keyFile, getPasssword, token); + + var tempFile = new TempFile(); + try + { + await enc.Decrypt(tempFile.Stream, token); + return tempFile; + } + catch (Exception ex) + { + await tempFile.DisposeAsync(); + throw ex; + } + } + } + + static async Task Extract(Stream from, CancellationToken token) + { + var td = new TempDirectory(); + try + { + FastZip zip = new FastZip(); + await Task.Yield(); + token.ThrowIfCancellationRequested(); + zip.ExtractZip(from, td.Directory.FullName, FastZip.Overwrite.Always, (x) => true, null, null, false, false); + } + catch (Exception ex) + { + td.Dispose(); + throw ex; + } + return td; + } + + static (FileInfo BoardFile, DirectoryInfo ImagesDirectory) select(DirectoryInfo folder) + { + FileInfo bi = null; + DirectoryInfo di = null; + foreach (var fi in folder.GetFiles()) + { + if (fi.Name.EndsWith(".board")) + { + string nnm = string.Join('.', fi.Name.Split(".")[..^1]); + if (Directory.Exists(Path.Combine(folder.FullName, nnm))) + { + return (fi, new DirectoryInfo(Path.Combine(folder.FullName, nnm))); + } + } + bi = fi; + } + if (bi == null) throw new InvalidDataException("No board info found."); + di = folder.GetDirectories().FirstOrDefault(); + if (di == null) throw new InvalidDataException("No images dir found"); + + return (bi, di); + } + + static async Task generate(Stream bif, DirectoryInfo images, DirectoryInfo output, CancellationToken token) + { + var binf = new BinaryFormatter(); + await Task.Yield(); + token.ThrowIfCancellationRequested(); + var bi = (napdump.BoardInfo)binf.Deserialize(bif); + + if (!output.Exists) output.Create(); + + var gen = new PageGenerator(bi, images); + + Console.WriteLine($" starting {bi.BoardName} w/ {images.Name} -> {output.FullName}"); + await gen.GenerateFull(output, token); + } + + static Task ReadPassword(string prompt = null, CancellationToken token = default) + { + TaskCompletionSource comp = new TaskCompletionSource(); + List pwd = new List(); + + bool set = false; + _ = Task.Run(() => + { + if (prompt != null) + Console.Write(prompt); + try + { + using var _c = token.Register(() => + { + if (!set) + { + set = true; + comp.SetException(new OperationCanceledException()); + } + }); + while (true) + { + ConsoleKeyInfo i = Console.ReadKey(true); + if (token.IsCancellationRequested) break; + if (i.Key == ConsoleKey.Enter) + { + Console.WriteLine(); + break; + } + else if (i.Key == ConsoleKey.Backspace) + { + if (pwd.Count > 0) + { + pwd.RemoveAt(pwd.Count - 1); + } + } + else if (i.KeyChar != '\u0000') + { + pwd.Add(i.KeyChar); + } + } + + if (!set) + { + set = true; + if (token.IsCancellationRequested) + comp.SetException(new OperationCanceledException()); + else + comp.SetResult(new string(pwd.ToArray())); + } + } + catch (Exception ex) + { + Console.WriteLine(); + pwd.Clear(); + if (!set) + { + set = true; + comp.SetException(ex); + } + } + }); + return comp.Task; + } + + static readonly CancellationTokenSource cancel = new CancellationTokenSource(); + public static AESKey? DeletedKey { get; private set; } = null; + static async Task loadAesKey(string from) + { + using (var fs = new FileStream(from, FileMode.Open, FileAccess.Read)) + { + DeletedKey = await encaes.AesEncryptor.LoadKey(fs, async () => await encaes.AesEncryptor.ReadPassword("Key is password protected: ")); + } + } + static Task parseArgs(ref string[] args) + { + string delk = null; + for(int i=0;i j != i && j != (i + 1)).ToArray(); + } + } + + if (delk == null) return Task.CompletedTask; + else + { + return loadAesKey(delk); + } + } + static async Task Main(string[] args) + { + if (args.Length < 2) + { + Usage(); + return; + } + else + { + try + { + await parseArgs(ref args); + } + catch (Exception ex) + { + Console.WriteLine("Failed to load deleted key: " + ex.Message); + return; + } + if (File.Exists(args[0])) + { + Console.CancelKeyPress += (o, e) => + { + if (!cancel.IsCancellationRequested) + { + cancel.Cancel(); + e.Cancel = true; + } + }; + try + { + if (await encaes.AesEncryptor.IsEncryptedFile(args[0], cancel.Token)) + { + //decrypt & extract + if (args.Length < 3) { Usage(); return; } + + TempFile tf = null; + try + { + Console.WriteLine("Decrypting..."); + using (var inputFile = new FileStream(args[0], FileMode.Open, FileAccess.Read)) + { + using (var keyStream = new FileStream(args[1], FileMode.Open, FileAccess.Read)) + { + tf = await Decrypt(inputFile, keyStream, () => + { + var tsk = ReadPassword("Password: ", cancel.Token); + tsk.Wait(); + cancel.Token.ThrowIfCancellationRequested(); + if (tsk.IsFaulted) + throw tsk.Exception; + return tsk.Result; + }, cancel.Token); + } + } + + tf.Stream.Position = 0; + Console.WriteLine("Extracting..."); + using (var tempd = await Extract(tf.Stream, cancel.Token)) + { + Console.WriteLine("Selecting best matches"); + var (boardFile, imagesDir) = select(tempd.Directory); + + Console.WriteLine($"Begining generate for {boardFile.Name} with {imagesDir.Name}"); + using (var fs = new FileStream(boardFile.FullName, FileMode.Open, FileAccess.Read)) + await generate(fs, imagesDir, new DirectoryInfo(args[2]), cancel.Token); + Console.WriteLine("Complete"); + } + + } + finally + { + tf?.Dispose(); + } + } + else + { + //extract + TempDirectory tempd; + using (var inputFile = new FileStream(args[0], FileMode.Open, FileAccess.Read)) + { + Console.WriteLine("Extracting..."); + tempd = await Extract(inputFile, cancel.Token); + } + try + { + Console.WriteLine("Selecting best matches"); + var (boardFile, imagesDir) = select(tempd.Directory); + + Console.WriteLine($"Begining generate for {boardFile.Name} with {imagesDir.Name}"); + using (var fs = new FileStream(boardFile.FullName, FileMode.Open, FileAccess.Read)) + await generate(fs, imagesDir, new DirectoryInfo(args[1]), cancel.Token); + Console.WriteLine("Complete"); + } + finally + { + tempd.Dispose(); + } + } + } + catch (OperationCanceledException) { } + catch (Exception ex) + { + Console.WriteLine($"Error ({ex.GetType().Name}) in generation: {ex.Message}"); + } + } + else Console.WriteLine("Input archive must exist."); + } + cancel.Dispose(); + + } + } +} diff --git a/ndview/Properties/Resources.Designer.cs b/ndview/Properties/Resources.Designer.cs new file mode 100644 index 0000000..9f29195 --- /dev/null +++ b/ndview/Properties/Resources.Designer.cs @@ -0,0 +1,81 @@ +//------------------------------------------------------------------------------ +// +// This code was generated by a tool. +// Runtime Version:4.0.30319.42000 +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ + +namespace ndview.Properties { + using System; + + + /// + /// A strongly-typed resource class, for looking up localized strings, etc. + /// + // This class was auto-generated by the StronglyTypedResourceBuilder + // class via a tool like ResGen or Visual Studio. + // To add or remove a member, edit your .ResX file then rerun ResGen + // with the /str option, or rebuild your VS project. + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "16.0.0.0")] + [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] + [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()] + internal class Resources { + + private static global::System.Resources.ResourceManager resourceMan; + + private static global::System.Globalization.CultureInfo resourceCulture; + + [global::System.Diagnostics.CodeAnalysis.SuppressMessageAttribute("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] + internal Resources() { + } + + /// + /// Returns the cached ResourceManager instance used by this class. + /// + [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)] + internal static global::System.Resources.ResourceManager ResourceManager { + get { + if (object.ReferenceEquals(resourceMan, null)) { + global::System.Resources.ResourceManager temp = new global::System.Resources.ResourceManager("ndview.Properties.Resources", typeof(Resources).Assembly); + resourceMan = temp; + } + return resourceMan; + } + } + + /// + /// Overrides the current thread's CurrentUICulture property for all + /// resource lookups using this strongly typed resource class. + /// + [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)] + internal static global::System.Globalization.CultureInfo Culture { + get { + return resourceCulture; + } + set { + resourceCulture = value; + } + } + + /// + /// Looks up a localized string similar to . + /// + internal static string css { + get { + return ResourceManager.GetString("css", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to . + /// + internal static string script { + get { + return ResourceManager.GetString("script", resourceCulture); + } + } + } +} diff --git a/ndview/Properties/Resources.resx b/ndview/Properties/Resources.resx new file mode 100644 index 0000000..4ac4610 --- /dev/null +++ b/ndview/Properties/Resources.resx @@ -0,0 +1,127 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + + static\css\base.css;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;Windows-1252 + + + static\js\base.js;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;Windows-1252 + + \ No newline at end of file diff --git a/ndview/Properties/static/css/base.css b/ndview/Properties/static/css/base.css new file mode 100644 index 0000000..85b8984 --- /dev/null +++ b/ndview/Properties/static/css/base.css @@ -0,0 +1,64 @@ +footer { + font-style: italic; +} + +h3 { + font-weight: bold; + display: inline; + color: #0f0c5d; +} + +b.name { + color: #117743; + font-weight: bold; +} + +b.name > code { + font-family: inherit; + font-weight: 400; +} + +section { + background-color: #eef2ff; + padding: 5px; + margin-bottom: 10px; +} + +article { + background-color: #d6daf0; + padding-top: 5px; + margin: 2px; + margin-bottom: 5px; + padding-bottom: 2px; +} + +article > header { + padding-left: 5px; +} + +blockquote { + padding-left: 10px; +} + +figcaption { + margin-left: 5px; + margin-bottom: 2px; +} + +figcaption > i > a { + margin-left: 4px; +} + +.thread.hidden > div +{ + display: none; +} + + +.thread.hidden > small:after +{ + content: "(replies hidden)"; + display: block; + margin-left: 5px; + margin-top: 5px; +} \ No newline at end of file diff --git a/ndview/Properties/static/js/base.js b/ndview/Properties/static/js/base.js new file mode 100644 index 0000000..4882166 --- /dev/null +++ b/ndview/Properties/static/js/base.js @@ -0,0 +1,4 @@ +window.addEventListener('load', () => { + document.querySelectorAll(".thread").forEach(x=> x.classList.toggle("hidden")); + document.querySelectorAll(".script").forEach(x=> x.style=""); //unhide script-specific elements +}); \ No newline at end of file diff --git a/ndview/TempFile.cs b/ndview/TempFile.cs new file mode 100644 index 0000000..f1c85b2 --- /dev/null +++ b/ndview/TempFile.cs @@ -0,0 +1,87 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Threading.Tasks; + +namespace ndview +{ + public sealed class TempDirectory : IDisposable + { + public DirectoryInfo Directory { get; } + public TempDirectory(DirectoryInfo location) + { + Directory = new DirectoryInfo(Path.Combine(location.FullName, TempFile.GenerateLocation())); + Directory.Create(); + } + public TempDirectory() : this(new DirectoryInfo(TempFile.TempFileLocation)) { } + + public void Dispose() + { + Directory.Delete(true); + } + } + public sealed class TempFile : IDisposable, IAsyncDisposable + { + public static string GenerateLocation() + { + var time = DateTime.Now; + return $"{time.Year}-{time.Month}-{time.Day}-{time.Hour}.{time.Minute}.{time.Second}.{time.Millisecond}-{Guid.NewGuid().ToString()}"; + } + + public void Dispose() + { + if (File.Exists) + { + Stream.Dispose(); + File.Delete(); + } + } + + public async ValueTask DisposeAsync() + { + if (File.Exists) + { + await Stream.DisposeAsync(); + File.Delete(); + } + } + + ~TempFile() + { + if (File.Exists) + { + try { File.Delete(); } catch { } + } + } + + public static string TempFileLocation { get; set; } + static TempFile() + { + TempFileLocation = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), GenerateLocation())).FullName; + + AppDomain.CurrentDomain.DomainUnload += (o, e) => + { + try + { + foreach (var file in new DirectoryInfo(TempFileLocation).GetFiles()) try + { + file.Delete(); + } + catch { } + Directory.Delete(TempFileLocation, true); + } + catch { } + }; + } + private FileInfo File { get; } + public Stream Stream { get; } + public TempFile(DirectoryInfo location) + { + var nam = Path.Combine(location.FullName, GenerateLocation() + ".tmp"); + File = new FileInfo(nam); + Stream = new FileStream(nam, FileMode.Create); + } + public TempFile() : this(new DirectoryInfo(TempFileLocation)) { } + } +} diff --git a/ndview/ndview.csproj b/ndview/ndview.csproj new file mode 100644 index 0000000..4220abb --- /dev/null +++ b/ndview/ndview.csproj @@ -0,0 +1,49 @@ + + + + Exe + netcoreapp3.1 + + + + + + + + + + + + + + + + ..\..\encaes\encaes\bin\Release\netcoreapp3.1\encaes.dll + + + ..\..\extensions\excryptotools\bin\Release\netstandard2.0\exbintools.dll + + + ..\..\extensions\excryptotools\bin\Release\netstandard2.0\excryptotools.dll + + + ..\..\extensions\exstreamtools\bin\Release\netstandard2.0\exstreamtools.dll + + + + + + True + True + Resources.resx + + + + + + ResXFileCodeGenerator + Resources.Designer.cs + + + +