diff --git a/README.md b/README.md index c2f2615..e58a97f 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,14 @@ The build and unstripped binary will be `shuffle3-debug`. Before switching between `release` and `debug` targets, remember to run `make clean`. To disable stripping of release build binaries, run with `make STRIP=: release` +### Compile-time flags +There are some build-time flags you can switch while building by appending to the `FEATURE_FLAGS` variable. +| Flag | Description | +|--------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `DEBUG` | Pretend we're building a debug release even though we're not. | +| `_FS_SPILL_BUFFER` | Spill buffers into a file if they grow over a threshold. Can cause massive slowdowns but prevent OOMs while unshuffling on systems with low available memory. See [shuffle3.h](./include/shuffle3.h) for more details | + + ## Gentoo ebuild There is a gentoo ebuild for this project in the overlay [test-overlay](https://git.flanchan.moe/birb/test-overlay). [direct link](https://git.flanchan.moe/birb/test-overlay/src/branch/master/app-misc/shuffle3/shuffle3-2.0.0.ebuild) diff --git a/include/fsvec.hpp b/include/fsvec.hpp index 3a18d6a..e399e1a 100644 --- a/include/fsvec.hpp +++ b/include/fsvec.hpp @@ -1,11 +1,15 @@ #pragma once #include +#include #include #include #include #include +#include + +#include template struct i_back_inserter @@ -16,6 +20,8 @@ struct i_back_inserter virtual T& back() =0; virtual const std::size_t size() const =0; + inline bool is_empty() const { return size()==0; } + virtual inline ~i_back_inserter() =default; }; @@ -90,49 +96,68 @@ struct file_vector : public i_back_inserter inline const std::size_t size() const override { return len; } private: file_back_buffer inserter; - std::size_t len; + std::size_t len=0; mutable std::vector current_back; // what an awful hack... }; -#define DEFAULT_SPILL_AT (1024 * 1024) -template + +template requires (Spill > 0) struct fixed_spill_vector : public i_back_inserter { constexpr const static std::size_t SPILL_AT = Spill; - inline fixed_spill_vector(){} + inline fixed_spill_vector() : mem(std::make_unique >()){} inline fixed_spill_vector(const fixed_spill_vector& c) = delete; inline fixed_spill_vector(fixed_spill_vector&& m) : mem(std::move(m.mem)), mem_fill_ptr(m.mem_fill_ptr), fil(std::move(m.fil)) {} + inline ~fixed_spill_vector() = default; - //TODO: Inserters/getters inline void push_back(T&& value) override { - + if(mem_is_full()) { + //D_dprintf("Inserting value into fs"); + fil.push_back(std::move(value)); + } else { + //D_dprintf("Inserting value into memory"); + (*mem)[++mem_fill_ptr] = value; + } } inline void pop_back() override { - + if(!size()) return; + + if(fil.size()) { + //D_dprintf("Popping from fs"); + fil.pop_back(); + } else { + //D_dprintf("Popping from memory %ld", mem_fill_ptr); + mem_fill_ptr -= 1; + } } inline const T& back() const override - { - + { + if (!size()) panic("back() (const) called on no elements"); + if(fil.size()) return fil.back(); + else return (*mem)[mem_fill_ptr]; } inline T& back() override { - + if (!size()) panic("back() called on no elements"); + if(fil.size()) return fil.back(); + else return (*mem)[mem_fill_ptr]; } inline const std::size_t size() const override { - + return fil.size() + (std::size_t)(mem_fill_ptr+1); } private: - std::array mem; - std::size_t mem_fill_ptr; + inline bool mem_is_full() const { return mem_fill_ptr >= (ssize_t)(Spill-1); } + ssize_t mem_fill_ptr=-1; + std::unique_ptr> mem; file_vector fil; }; diff --git a/include/shuffle.hpp b/include/shuffle.hpp index ba651ad..4833f24 100644 --- a/include/shuffle.hpp +++ b/include/shuffle.hpp @@ -29,8 +29,7 @@ namespace rng { if(!span.size()) return; #ifdef _FS_SPILL_BUFFER -#warning "_FS_SPILL_BUFFER is still experimental" - file_vector rng_values(span.size()); + fixed_spill_vector rng_values;//(span.size()); //TODO: dynamic_spill_vector #else std::vector rng_values(span.size()); #endif diff --git a/include/shuffle3.h b/include/shuffle3.h index f3ad0b2..0a927c5 100644 --- a/include/shuffle3.h +++ b/include/shuffle3.h @@ -18,7 +18,10 @@ extern "C" { //#define _FS_SPILL_BUFFER /* Use a file-backed buffer when unshuffling in cases of too high memory usage. Will cause massive slowdowns but can stop OOMs when unshuffling large files */ -// +/// When to spill a file-backed buffer onto the fs (only used when `_FS_SPILL_BUFFER` is enabled). +#define FSV_DEFAULT_SPILL_AT ((1024 * 1024) * 10) //10MB + +//** Globals *// extern const char* _prog_name; diff --git a/src/fsvec.cpp b/src/fsvec.cpp index 2a0173e..16dedec 100644 --- a/src/fsvec.cpp +++ b/src/fsvec.cpp @@ -83,7 +83,7 @@ bool FB::pop_n(size_t len) extern "C" void _fb_run_tests() { - file_vector test; + fixed_spill_vector test; int r0,r1=0; for(int i=0;i<10;i++) { D_dprintf("push: %d", (10-i));