allocation bugfix

progress v2.1.2
Avril 4 years ago
parent ed9c43e910
commit fb2ecf7bec
Signed by: flanchan
GPG Key ID: 284488987C31F630

@ -80,10 +80,11 @@ To disable stripping of release build binaries, run with `make STRIP=: release`
### Compile-time flags
There are some build-time flags you can switch while building by appending to the `FEATURE_FLAGS` variable.
| Flag | Description |
|--------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `DEBUG` | Pretend we're building a debug release even though we're not. |
| `_FS_SPILL_BUFFER` | Spill buffers into a file if they grow over a threshold. Can cause massive slowdowns but prevent OOMs while unshuffling on systems with low available memory. See [shuffle3.h](./include/shuffle3.h) for more details |
| Flag | Description |
|------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `DEBUG` | Pretend we're building a debug release even though we're not. |
| `_FS_SPILL_BUFFER` | Spill buffers into a file if they grow over a threshold. Can cause massive slowdowns but prevent OOMs while unshuffling on systems with low available memory. See [shuffle3.h](./include/shuffle3.h) for more details |
| `_FS_SPILL_BUFFER=DYN` | Same as above except allocates memory dynamically. Might be faster. |
## Gentoo ebuild

@ -107,7 +107,9 @@ struct fixed_spill_vector : public i_back_inserter<T>
{
constexpr const static std::size_t SPILL_AT = Spill;
inline fixed_spill_vector() : mem(std::make_unique<std::array<T, Spill> >()){}
inline fixed_spill_vector() : mem(std::make_unique<std::array<T, Spill> >()){
D_dprintf("alloc cap (static): %lu", Spill);
}
inline fixed_spill_vector(const fixed_spill_vector<T>& c) = delete;
inline fixed_spill_vector(fixed_spill_vector<T>&& m)
: mem(std::move(m.mem)),
@ -161,3 +163,50 @@ private:
std::unique_ptr<std::array<T, Spill>> mem;
file_vector<T> fil;
};
template<typename T>
struct dynamic_spill_vector : public i_back_inserter<T>
{
inline dynamic_spill_vector() : dynamic_spill_vector(FSV_DEFAULT_SPILL_AT){}
inline dynamic_spill_vector(std::size_t cap) : dynamic_spill_vector(cap, cap){}
inline dynamic_spill_vector(std::size_t cap, std::size_t spill) : _spill_at(spill), mem(std::vector<T>()), fil(file_vector<T>(cap)) {
mem.reserve(cap);
D_dprintf("alloc cap %lu (sz %lu == 0?), spill %lu", cap, mem.size(), spill_at());
}
inline dynamic_spill_vector(const dynamic_spill_vector<T>& c) = delete;
inline dynamic_spill_vector(dynamic_spill_vector<T>&& m) :
_spill_at(m._spill_at),
mem(std::move(m.mem)),
fil(std::move(m.fil)){}
inline void push_back(T&& value) override
{
if(size()>=spill_at()) {
D_dprintf("Spilling: sz %lu, spl: %lu", size(), spill_at());
fil.push_back(std::move(value));
}
else mem.push_back(std::move(value));
}
inline void pop_back() override
{
if(fil.size()) fil.pop_back();
else mem.pop_back();
}
inline const T& back() const override
{
if(fil.size()) return fil.back();
else return mem.back();
}
inline T& back() override
{
if(fil.size()) return fil.back();
else return mem.back();
}
inline const std::size_t size() const override { return mem.size() + fil.size(); }
inline const std::size_t spill_at() const { return _spill_at; }
private:
std::size_t _spill_at;
std::vector<T> mem;
file_vector<T> fil;
};

@ -23,15 +23,31 @@ namespace rng {
std::cout << " OK" << std::endl;
}
namespace {
extern "C" int _can_allocate(std::size_t bytes);
template<typename T>
inline bool can_allocate(std::size_t len)
{
return !!_can_allocate(len*sizeof(T));
}
}
template<typename T, typename R>
inline void unshuffle(R& rng, span<T> span)
{
if(!span.size()) return;
#ifdef _FS_SPILL_BUFFER
fixed_spill_vector<std::size_t> rng_values;//(span.size()); //TODO: dynamic_spill_vector
#if defined(_FS_SPILL_BUFFER) && _FS_SPILL_BUFFER == DYN
dynamic_spill_vector<std::size_t> rng_values =
can_allocate<std::size_t>(span.size()) //Is there any way we can not waste this malloc() when it's valid?
? dynamic_spill_vector<std::size_t> (span.size())
: dynamic_spill_vector<std::size_t> (FSV_DEFAULT_SPILL_AT);
#elif defined(_FS_SPILL_BUFFER)
fixed_spill_vector<std::size_t> rng_values;
#else
std::vector<std::size_t> rng_values(span.size());
std::vector<std::size_t> rng_values;
rng_values.reserve(span.size());
#endif
std::cout << " -> unshuffling " << span.size() << " objects...";

@ -21,6 +21,16 @@ extern "C" {
/// When to spill a file-backed buffer onto the fs (only used when `_FS_SPILL_BUFFER` is enabled).
#define FSV_DEFAULT_SPILL_AT ((1024 * 1024) * 10) //10MB
/*
#ifdef _FS_SPILL_BUFFER
#define DYN 1
#if _FS_SPILL_BUFFER == DYN
#undef _FS_SPILL_BUFFER
#define _FS_SPILL_BUFFER DYN
#endif
#undef DYN
#endif
*/
//** Globals *//
extern const char* _prog_name;

@ -0,0 +1,12 @@
#include <stdint.h>
#include <stddef.h>
#include <stdlib.h>
int _can_allocate(size_t bytes)
{
void* shim = malloc(bytes);
if(shim) {
free(shim);
return 1;
} else return 0;
}
Loading…
Cancel
Save