using System; using System.Threading; using System.Threading.Tasks; using System.Threading.Channels; using System.Linq; using System.Collections.Generic; using System.IO; using Tools.Crypto; namespace napdump { class Program { public static readonly CancellationTokenSource globalCancel = new CancellationTokenSource(); public const string NineballBaseUrl = "https://nineball.party"; public static readonly string[] NineballBoards = new[] { "/nap/", "/srsbsn/", "/staff/"}; static volatile int totalThreadsDownloaded = 0; static volatile int totalBoardsDownloaded = 0; static volatile int totalPostsDownloaded = 0; static async Task GrabBoard(Dumper dumper, string boardUrl, ChannelWriter onNewThread, CancellationToken token) { TaskCompletionSource get = new TaskCompletionSource(); Dumper.Hooks hooks = new Dumper.Hooks() { OnBoardRetrieved = (bi) => { if (!token.IsCancellationRequested) get.SetResult(bi); }, PrintDebug = false//boardUrl.EndsWith("srsbsn/"), }; using var _reg_get_cancel = token.Register(() => get.SetException(new OperationCanceledException())); Console.WriteLine("\r [" + dumper.GetType().Name + "] Downloading " + boardUrl); try { await foreach (var thread in dumper.Parse(boardUrl, hooks).WithCancellation(token)) { totalPostsDownloaded += thread.Children.Count + 1; totalThreadsDownloaded += 1; await onNewThread.WriteAsync(thread, token); } return await get.Task; } finally { Console.WriteLine("\r [" + dumper.GetType().Name + "] Complete " + boardUrl); } } static async Task readOutputs(ChannelReader reader, CancellationToken token) { Dictionary numberPerBoard = new Dictionary(); await foreach(var thread in reader.ReadAllAsync(token)) { string name = thread.BoardInfo.BoardName; if (numberPerBoard.ContainsKey(name)) numberPerBoard[name] += 1; else numberPerBoard.Add(name, 1); Console.Write($"\r"); foreach(var kv in numberPerBoard) { Console.Write($"{kv.Key} - {kv.Value} "); } } Console.WriteLine(); } static string getTimestamp(DateTime time) { return $"{time.Year}-{time.Month}-{time.Day}-{time.Hour}.{time.Minute}.{time.Second}.{time.Millisecond}"; } static async Task DumpBoardInfo(BoardInfo bi) { System.Runtime.Serialization.Formatters.Binary.BinaryFormatter binf = new System.Runtime.Serialization.Formatters.Binary.BinaryFormatter(); if (!Directory.Exists("dumps")) Directory.CreateDirectory("dumps"); var path = Path.Combine("dumps", $"{bi.SafeName ?? "unbound"}-{getTimestamp(DateTime.Now)}.board"); using (var fs = new FileStream(path, FileMode.Create)) { await Task.Yield(); binf.Serialize(fs, bi); } Console.WriteLine($"\r {bi.BoardName} -> {path}"); } static async Task ParseArgs(string[] args) { string findarg(string name) { for(int i=0;i j != i && j != (i + 1)).ToArray(); return r; } } return null; } bool tryarg(string name, TryParser parser, out T value, T def = default) { var fa = findarg(name); if(fa!=null) { if(! parser(fa, out value)) { value = def; return false; } return true; } value = def; return false; } static bool defp(string inp, out string op) { op = inp; return true; } string login; string aesKeyfile; int threads; tryarg("--login", defp, out login); tryarg("--threads", int.TryParse, out threads, 3); tryarg("--encrypt-deleted", defp, out aesKeyfile); return new DumperConfig(threads, Cookies: login == null ? null : new[] { (NineballBaseUrl, "a=" + login) }, EncryptDeleted: aesKeyfile == null ? (AESKey?)null : (await getKey(aesKeyfile))); } private static async Task getKey(string fn) { using (var fs = new FileStream(fn, FileMode.Open, FileAccess.Read)) { return await encaes.AesEncryptor.LoadKey(fs, async () => await encaes.AesEncryptor.ReadPassword("Keyfile is password protected: ")); } } private delegate bool TryParser(string input, out T value); static async Task Main(string[] args) { if(args.Length==1 && args[0].ToLower()=="--help") { Console.WriteLine("napdump.exe [--login ] [--threads ] [--encrypt-deleted ]"); return; } using var napDownloader = new Dumpers.Nineball(await ParseArgs(args)); Console.CancelKeyPress += (o,e) => { globalCancel.Cancel(); e.Cancel = true; }; List> downloaders = new List>(); Channel threads = Channel.CreateUnbounded(); Task outputReader = readOutputs(threads.Reader, globalCancel.Token); try { foreach (var board in NineballBoards) { downloaders.Add(Task.Run(async () => { try { var bi = await GrabBoard(napDownloader, NineballBaseUrl + board, threads.Writer, globalCancel.Token); totalBoardsDownloaded += 1; try { await DumpBoardInfo(bi); } catch (Exception ex) { Console.WriteLine("Failed to dump board " + bi.BoardName + " to file: " + ex.Message); } return bi; } catch (Exception ex) { Console.WriteLine("Failed to download board " + board + ": " + ex.Message); return default(BoardInfo); } })); } var boards = (await Task.WhenAll(downloaders)).Where(x => x != null); Console.WriteLine("\n\nDownloaded Boards:"); foreach (var b in boards) Console.WriteLine("\t" + b.ToString()); } catch (Exception ex) { Console.WriteLine("\n\nError: " + ex.Message); return; } finally { threads.Writer.Complete(); try { await outputReader; } catch (OperationCanceledException) { } } Console.WriteLine("Complete"); Console.WriteLine($"Downloaded {totalBoardsDownloaded} boards, with {totalThreadsDownloaded} threads, containing {totalPostsDownloaded} posts."); } } }