_FS_SPILL_BUFFER

progress
Avril 4 years ago
parent 7d34ed2496
commit 6b2b9a4a9d
Signed by: flanchan
GPG Key ID: 284488987C31F630

@ -75,6 +75,14 @@ The build and unstripped binary will be `shuffle3-debug`.
Before switching between `release` and `debug` targets, remember to run `make clean`.
To disable stripping of release build binaries, run with `make STRIP=: release`
### Compile-time flags
There are some build-time flags you can switch while building by appending to the `FEATURE_FLAGS` variable.
| Flag | Description |
|--------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `DEBUG` | Pretend we're building a debug release even though we're not. |
| `_FS_SPILL_BUFFER` | Spill buffers into a file if they grow over a threshold. Can cause massive slowdowns but prevent OOMs while unshuffling on systems with low available memory. See [shuffle3.h](./include/shuffle3.h) for more details |
## Gentoo ebuild
There is a gentoo ebuild for this project in the overlay [test-overlay](https://git.flanchan.moe/birb/test-overlay).
[direct link](https://git.flanchan.moe/birb/test-overlay/src/branch/master/app-misc/shuffle3/shuffle3-2.0.0.ebuild)

@ -1,11 +1,15 @@
#pragma once
#include <cstdint>
#include <cstddef>
#include <memory>
#include <utility>
#include <vector>
#include <panic.h>
#include <debug.h>
#include <shuffle3.h>
template<typename T>
struct i_back_inserter
@ -16,6 +20,8 @@ struct i_back_inserter
virtual T& back() =0;
virtual const std::size_t size() const =0;
inline bool is_empty() const { return size()==0; }
virtual inline ~i_back_inserter() =default;
};
@ -90,49 +96,68 @@ struct file_vector : public i_back_inserter<T>
inline const std::size_t size() const override { return len; }
private:
file_back_buffer inserter;
std::size_t len;
std::size_t len=0;
mutable std::vector<unsigned char> current_back; // what an awful hack...
};
#define DEFAULT_SPILL_AT (1024 * 1024)
template<typename T, std::size_t Spill = DEFAULT_SPILL_AT >
template<typename T, std::size_t Spill = FSV_DEFAULT_SPILL_AT >
requires (Spill > 0)
struct fixed_spill_vector : public i_back_inserter<T>
{
constexpr const static std::size_t SPILL_AT = Spill;
inline fixed_spill_vector(){}
inline fixed_spill_vector() : mem(std::make_unique<std::array<T, Spill> >()){}
inline fixed_spill_vector(const fixed_spill_vector<T>& c) = delete;
inline fixed_spill_vector(fixed_spill_vector<T>&& m)
: mem(std::move(m.mem)),
mem_fill_ptr(m.mem_fill_ptr),
fil(std::move(m.fil))
{}
inline ~fixed_spill_vector() = default;
//TODO: Inserters/getters
inline void push_back(T&& value) override
{
if(mem_is_full()) {
//D_dprintf("Inserting value into fs");
fil.push_back(std::move(value));
} else {
//D_dprintf("Inserting value into memory");
(*mem)[++mem_fill_ptr] = value;
}
}
inline void pop_back() override
{
if(!size()) return;
if(fil.size()) {
//D_dprintf("Popping from fs");
fil.pop_back();
} else {
//D_dprintf("Popping from memory %ld", mem_fill_ptr);
mem_fill_ptr -= 1;
}
}
inline const T& back() const override
{
if (!size()) panic("back() (const) called on no elements");
if(fil.size()) return fil.back();
else return (*mem)[mem_fill_ptr];
}
inline T& back() override
{
if (!size()) panic("back() called on no elements");
if(fil.size()) return fil.back();
else return (*mem)[mem_fill_ptr];
}
inline const std::size_t size() const override
{
return fil.size() + (std::size_t)(mem_fill_ptr+1);
}
private:
std::array<T, Spill> mem;
std::size_t mem_fill_ptr;
inline bool mem_is_full() const { return mem_fill_ptr >= (ssize_t)(Spill-1); }
ssize_t mem_fill_ptr=-1;
std::unique_ptr<std::array<T, Spill>> mem;
file_vector<T> fil;
};

@ -29,8 +29,7 @@ namespace rng {
if(!span.size()) return;
#ifdef _FS_SPILL_BUFFER
#warning "_FS_SPILL_BUFFER is still experimental"
file_vector<std::size_t> rng_values(span.size());
fixed_spill_vector<std::size_t> rng_values;//(span.size()); //TODO: dynamic_spill_vector
#else
std::vector<std::size_t> rng_values(span.size());
#endif

@ -18,7 +18,10 @@ extern "C" {
//#define _FS_SPILL_BUFFER /* Use a file-backed buffer when unshuffling in cases of too high memory usage. Will cause massive slowdowns but can stop OOMs when unshuffling large files */
//
/// When to spill a file-backed buffer onto the fs (only used when `_FS_SPILL_BUFFER` is enabled).
#define FSV_DEFAULT_SPILL_AT ((1024 * 1024) * 10) //10MB
//** Globals *//
extern const char* _prog_name;

@ -83,7 +83,7 @@ bool FB::pop_n(size_t len)
extern "C" void _fb_run_tests()
{
file_vector<int> test;
fixed_spill_vector<int, 8> test;
int r0,r1=0;
for(int i=0;i<10;i++) {
D_dprintf("push: %d", (10-i));

Loading…
Cancel
Save