_FS_SPILL_BUFFER

README.md

@@ -75,6 +75,14 @@ The build and unstripped binary will be `shuffle3-debug`.
 Before switching between `release` and `debug` targets, remember to run `make clean`.
 To disable stripping of release build binaries, run with `make STRIP=: release`
 
+### Compile-time flags
+There are some build-time flags you can switch while building by appending to the `FEATURE_FLAGS` variable.
+| Flag               | Description                                                                                                                                                                                                           |
+|--------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `DEBUG`            | Pretend we're building a debug release even though we're not.                                                                                                                                                         |
+| `_FS_SPILL_BUFFER` | Spill buffers into a file if they grow over a threshold. Can cause massive slowdowns but prevent OOMs while unshuffling on systems with low available memory. See [shuffle3.h](./include/shuffle3.h) for more details |
+
+
 ## Gentoo ebuild
 There is a gentoo ebuild for this project in the overlay [test-overlay](https://git.flanchan.moe/birb/test-overlay). 
 [direct link](https://git.flanchan.moe/birb/test-overlay/src/branch/master/app-misc/shuffle3/shuffle3-2.0.0.ebuild)

include/fsvec.hpp

@@ -1,11 +1,15 @@
 #pragma once
 
 #include <cstdint>
+#include <cstddef>
 #include <memory>
 #include <utility>
 #include <vector>
 
 #include <panic.h>
+#include <debug.h>
+
+#include <shuffle3.h>
 
 template<typename T>
 struct i_back_inserter
@@ -16,6 +20,8 @@ struct i_back_inserter
 	virtual T& back() =0;
 	virtual const std::size_t size() const =0;
 
+	inline bool is_empty() const { return size()==0; }
+
 	virtual inline ~i_back_inserter() =default;
 };
 
@@ -90,49 +96,68 @@ struct file_vector : public i_back_inserter<T>
 	inline const std::size_t size() const override { return len; }
 private:
 	file_back_buffer inserter;
-	std::size_t len;
+	std::size_t len=0;
 	mutable std::vector<unsigned char> current_back; // what an awful hack...
 };
 
-#define DEFAULT_SPILL_AT (1024 * 1024)
+
-template<typename T, std::size_t Spill = DEFAULT_SPILL_AT >
+template<typename T, std::size_t Spill = FSV_DEFAULT_SPILL_AT >
 	requires (Spill > 0)
 struct fixed_spill_vector : public i_back_inserter<T>
 {
 	constexpr const static std::size_t SPILL_AT = Spill;
 
-	inline fixed_spill_vector(){}
+	inline fixed_spill_vector() : mem(std::make_unique<std::array<T, Spill> >()){}
 	inline fixed_spill_vector(const fixed_spill_vector<T>& c) = delete;
 	inline fixed_spill_vector(fixed_spill_vector<T>&& m)
 		: mem(std::move(m.mem)),
 		  mem_fill_ptr(m.mem_fill_ptr),
 		  fil(std::move(m.fil))
 	{}
+	inline ~fixed_spill_vector() = default;
 
-	//TODO: Inserters/getters
 	inline void push_back(T&& value) override
 	{
-
+		if(mem_is_full()) { 
+			//D_dprintf("Inserting value into fs");
+			fil.push_back(std::move(value));
+		} else { 
+			//D_dprintf("Inserting value into memory");
+			(*mem)[++mem_fill_ptr] = value;
+		}
 	}
 	inline void pop_back() override
 	{
-
+		if(!size()) return;
+
+		if(fil.size()) {
+			//D_dprintf("Popping from fs");
+			fil.pop_back();
+		} else {
+			//D_dprintf("Popping from memory %ld", mem_fill_ptr);
+			mem_fill_ptr -= 1;
+		}
 	}
 	inline const T& back() const override
-	{
+	{	
-
+		if (!size()) panic("back() (const) called on no elements");
+		if(fil.size()) return fil.back();
+		else return (*mem)[mem_fill_ptr];
 	}
 	inline T& back() override
 	{
-
+		if (!size()) panic("back() called on no elements");
+		if(fil.size()) return fil.back();
+		else return (*mem)[mem_fill_ptr];
 	}
 	inline const std::size_t size() const override
 	{
-
+		return fil.size() + (std::size_t)(mem_fill_ptr+1);
 	}
 private:
-	std::array<T, Spill> mem;
+	inline bool mem_is_full() const { return mem_fill_ptr >= (ssize_t)(Spill-1); }
-	std::size_t mem_fill_ptr;
+	ssize_t mem_fill_ptr=-1;
 
+	std::unique_ptr<std::array<T, Spill>> mem;
 	file_vector<T> fil;
 };

include/shuffle.hpp

@@ -29,8 +29,7 @@ namespace rng {
 		if(!span.size()) return;
 
 #ifdef _FS_SPILL_BUFFER
-#warning "_FS_SPILL_BUFFER is still experimental"
+		fixed_spill_vector<std::size_t> rng_values;//(span.size()); //TODO: dynamic_spill_vector
-		file_vector<std::size_t> rng_values(span.size());
 #else
 		std::vector<std::size_t> rng_values(span.size());
 #endif

include/shuffle3.h

@@ -18,7 +18,10 @@ extern "C" {
 
 //#define _FS_SPILL_BUFFER /* Use a file-backed buffer when unshuffling in cases of too high memory usage. Will cause massive slowdowns but can stop OOMs when unshuffling large files */
 
-//
+/// When to spill a file-backed buffer onto the fs (only used when `_FS_SPILL_BUFFER` is enabled).
+#define FSV_DEFAULT_SPILL_AT ((1024 * 1024) * 10) //10MB 
+
+//** Globals *//
 
 extern const char* _prog_name;
 

src/fsvec.cpp

@@ -83,7 +83,7 @@ bool FB::pop_n(size_t len)
 
 extern "C" void _fb_run_tests()
 {
-	file_vector<int> test;
+	fixed_spill_vector<int, 8> test;
 	int r0,r1=0;
 	for(int i=0;i<10;i++) {
 		D_dprintf("push: %d", (10-i));