napdump/napdump/Program.cs

using System;
using System.Threading;
using System.Threading.Tasks;
using System.Threading.Channels;
using System.Linq;
using System.Collections.Generic;
using System.IO;
using Tools.Crypto;

namespace napdump
{
    class Program
    {
        public static readonly CancellationTokenSource globalCancel = new CancellationTokenSource();

        public const string NineballBaseUrl = "https://nineball.party";
        public static readonly string[] NineballBoards = new[] { "/nap/", "/srsbsn/", "/staff/"};

        static volatile int totalThreadsDownloaded = 0;
        static volatile int totalBoardsDownloaded = 0;
        static volatile int totalPostsDownloaded = 0;
        static async Task<BoardInfo> GrabBoard(Dumper dumper, string boardUrl, ChannelWriter<ThreadInfo> onNewThread, CancellationToken token)
        {
            TaskCompletionSource<BoardInfo> get = new TaskCompletionSource<BoardInfo>();
            Dumper.Hooks hooks = new Dumper.Hooks()
            {
                OnBoardRetrieved = (bi) =>
                {
                    if (!token.IsCancellationRequested)
                        get.SetResult(bi);
                },
                PrintDebug = false//boardUrl.EndsWith("srsbsn/"),
            };
        
            using var _reg_get_cancel = token.Register(() => get.SetException(new OperationCanceledException()));
            
            Console.WriteLine("\r [" + dumper.GetType().Name + "] Downloading " + boardUrl);
            try
            {
                await foreach (var thread in dumper.Parse(boardUrl, hooks).WithCancellation(token))
                {
                    totalPostsDownloaded += thread.Children.Count + 1;
                    totalThreadsDownloaded += 1;
                    await onNewThread.WriteAsync(thread, token);
                }
                return await get.Task;
            }
            finally
            {
                Console.WriteLine("\r [" + dumper.GetType().Name + "] Complete " + boardUrl);
            }
        }
        static async Task readOutputs(ChannelReader<ThreadInfo> reader, CancellationToken token)
        {
            Dictionary<string, int> numberPerBoard = new Dictionary<string, int>();
            await foreach(var thread in reader.ReadAllAsync(token))
            {
                string name = thread.BoardInfo.BoardName;
                if (numberPerBoard.ContainsKey(name))
                    numberPerBoard[name] += 1;
                else numberPerBoard.Add(name, 1);
                Console.Write($"\r");
                foreach(var kv in numberPerBoard)
                {
                    Console.Write($"{kv.Key} - {kv.Value} ");
                }
            }
            Console.WriteLine();
        }
        static string getTimestamp(DateTime time)
        {
            return $"{time.Year}-{time.Month}-{time.Day}-{time.Hour}.{time.Minute}.{time.Second}.{time.Millisecond}";
        }
        static async Task DumpBoardInfo(BoardInfo bi)
        {
            System.Runtime.Serialization.Formatters.Binary.BinaryFormatter binf = new System.Runtime.Serialization.Formatters.Binary.BinaryFormatter();

            if (!Directory.Exists("dumps"))
                Directory.CreateDirectory("dumps");
            var path = Path.Combine("dumps", $"{bi.SafeName ?? "unbound"}-{getTimestamp(DateTime.Now)}.board");
            using (var fs = new FileStream(path, FileMode.Create))
            {
                await Task.Yield();
                binf.Serialize(fs, bi);
            }
            Console.WriteLine($"\r {bi.BoardName} -> {path}");
        }

        static async Task<DumperConfig> ParseArgs(string[] args)
        {
            string findarg(string name)
            {
                for(int i=0;i<args.Length-1;i++)
                {
                    if (args[i].ToLower() == name)
                    {
                        var r = args[i + 1];
                        args = args.Where((_, j) => j != i && j != (i + 1)).ToArray();
                        return r;
                    }
                }
                return null;
            }
            bool tryarg<T>(string name, TryParser<T> parser, out T value, T def = default)
            {
                var fa = findarg(name);
                if(fa!=null)
                {
                    if(! parser(fa, out value))
                    {
                        value = def;
                        return false;
                    }
                    return true;
                }
                value = def;
                return false;
            }
            static bool defp(string inp, out string op)
            {
                op = inp;
                return true;
            }

            string login;
            string aesKeyfile;
            int threads;

            tryarg("--login", defp, out login);
            tryarg<int>("--threads", int.TryParse, out threads, 3);
            tryarg("--encrypt-deleted", defp, out aesKeyfile);

            return new DumperConfig(threads, Cookies: login == null ? null : new[] { (NineballBaseUrl, "a=" + login) }, EncryptDeleted: aesKeyfile == null ? (AESKey?)null : (await getKey(aesKeyfile)));
        }
        private static async Task<AESKey> getKey(string fn)
        {
            using (var fs = new FileStream(fn, FileMode.Open, FileAccess.Read))
            {
                return await encaes.AesEncryptor.LoadKey(fs, async () => await encaes.AesEncryptor.ReadPassword("Keyfile is password protected: "));
            }
        }
        private delegate bool TryParser<T>(string input, out T value);

        static async Task Main(string[] args)
        {
            if(args.Length==1 && args[0].ToLower()=="--help")
            {
                Console.WriteLine("napdump.exe [--login <login token `a'>] [--threads <concurrent downloader number>] [--encrypt-deleted <deleted key>]");
                return;
            }
            using var napDownloader = new Dumpers.Nineball(await ParseArgs(args));

            Console.CancelKeyPress += (o,e) =>
            {
                globalCancel.Cancel();
                e.Cancel = true;
            };

            List<Task<BoardInfo>> downloaders = new List<Task<BoardInfo>>();
            Channel<ThreadInfo> threads = Channel.CreateUnbounded<ThreadInfo>();
            Task outputReader = readOutputs(threads.Reader, globalCancel.Token);
            try
            {
                foreach (var board in NineballBoards)
                {
                    downloaders.Add(Task.Run(async () =>
                    {
                        try
                        {
                            var bi = await GrabBoard(napDownloader, NineballBaseUrl + board, threads.Writer, globalCancel.Token);
                            totalBoardsDownloaded += 1;
                            try
                            {
                                await DumpBoardInfo(bi);
                            }
                            catch (Exception ex)
                            {
                                Console.WriteLine("Failed to dump board " + bi.BoardName + " to file: " + ex.Message);
                                
                            }
                            return bi;
                        }
                        catch (Exception ex)
                        {
                            Console.WriteLine("Failed to download board " + board + ": " + ex.Message);
                            return default(BoardInfo);
                        }
                    }));
                }

                var boards = (await Task.WhenAll(downloaders)).Where(x => x != null);

                Console.WriteLine("\n\nDownloaded Boards:");
                foreach (var b in boards)
                    Console.WriteLine("\t" + b.ToString());
            }
            catch (Exception ex)
            {
                Console.WriteLine("\n\nError: " + ex.Message);

                return;
            }
            finally
            {
                threads.Writer.Complete();
                try
                {
                    await outputReader;
                }
                catch (OperationCanceledException) { }
            }

            Console.WriteLine("Complete");
            Console.WriteLine($"Downloaded {totalBoardsDownloaded} boards, with {totalThreadsDownloaded} threads, containing {totalPostsDownloaded} posts.");

        }
    }
}