in-place un/shuffling works

lean
Avril 4 years ago
parent fba298744a
commit 5d6d60e5c4
Signed by: flanchan
GPG Key ID: 284488987C31F630

@ -16,7 +16,7 @@ CXX_OPT_FLAGS?= $(OPT_FLAGS) -felide-constructors
CFLAGS += $(COMMON_FLAGS) --std=gnu11
CXXFLAGS += $(COMMON_FLAGS) --std=gnu++20 -fno-exceptions
LDFLAGS +=
LDFLAGS += -lfmt
RELEASE_CFLAGS?= -O3 -flto $(OPT_FLAGS)
RELEASE_CXXFLAGS?= -O3 -flto $(CXX_OPT_FLAGS)

@ -24,6 +24,7 @@ void* map_and_then(const char* file, map_cb callback, void* user);
#ifdef __cplusplus
}
#include <panic.h>
#include "reinterpret.h"
#include <cstdint>
namespace mm {
struct mmap {
@ -52,6 +53,9 @@ namespace mm {
}
}
inline const span<const unsigned char> as_span() const { return span(as_ptr(), size()); }
inline span<unsigned char> as_span() { return span(as_ptr(), size()); }
inline const std::uint8_t* as_ptr() const { return (const std::uint8_t*)inner.ptr; }
inline std::uint8_t* as_ptr() { return (std::uint8_t*)inner.ptr; }

@ -0,0 +1,36 @@
#pragma once
#include <rng.h>
#include <reinterpret.h>
#include <algorithm>
#include <fmt/format.h>
namespace rng {
template<typename T, typename R>
inline void shuffle(R& rng, span<T> span)
{
fmt::print(" -> shuffling {} objects...", span.size());
for(std::size_t i=span.size()-1;i>0;i--)
{
auto j = rng.next_long(i);
std::swap(span[i], span[j]);
}
fmt::print(" OK\n");
}
template<typename T, typename R>
inline void unshuffle(R& rng, span<T> span)
{
std::vector<std::size_t> rng_values;
fmt::print(" -> unshuffling {} objects...", span.size());
for(std::size_t i=span.size()-1;i>0;i--)
rng_values.push_back(rng.next_long(i));
for(std::size_t i=1;i<span.size();i++) {
std::swap(span[i], span[rng_values.back()]);
rng_values.pop_back();
}
fmt::print(" OK\n");
}
}

@ -45,14 +45,18 @@ int main(int argc, char** argv)
{
struct prog_args args = {.argc = argc, .argv = argv};
rng_test();
//rng_test();
rng_t r = rng_new(RNG_INIT(RNG_KIND_FRNG, frng = { { 1.0, 2.0 } }));
rng_test_spec(r);
rng_free(r);
if( argv[1] ) {
map_and_then(argv[1], &map_callback, &args);
//map_and_then(argv[1], &map_callback, &args);
return do_work((work_args_t) {
.op = OP_SHUFFLE_IP,
.data.op_shuffle_ip.file = argv[1],
});
}
return 0;

@ -16,7 +16,7 @@ int open_and_map(const char* file, mmap_t* restrict ptr)
{
int fd;
struct stat st;
if ((fd = open(file, O_RDONLY, FILEMODE)) < 0) {
if ((fd = open(file, O_RDWR, FILEMODE)) < 0) {
perror("Failed to open file");
return 0;
}
@ -29,7 +29,7 @@ int open_and_map(const char* file, mmap_t* restrict ptr)
register struct mmap map = { .fd = fd, .ptr = NULL, .len = st.st_size };
if ((map.ptr = mmap(NULL, map.len, PROT_READ, MAP_SHARED,fd, 0)) == MAP_FAILED) {
if ((map.ptr = mmap(NULL, map.len, PROT_READ | PROT_WRITE, MAP_SHARED,fd, 0)) == MAP_FAILED) {
perror("mmap() failed");
close(fd);
return 0;

@ -1,6 +1,42 @@
#include <tuple>
#include <functional>
#include <cfloat>
#include <fmt/format.h>
#include <shuffle3.h>
#include <work.h>
#include <panic.h>
#include <map.h>
#include <reinterpret.h>
#include <shuffle.hpp>
#include <work.h>
template<typename T, typename Fn>
std::tuple<T, T> minmax_t(const span<T>& array, Fn keep)
{
T lowest;
T highest;
for(std::size_t i=0;i<array.size();i++)
{
if(!keep(array[i])) continue;
if(!i) lowest = highest = array[i];
else {
auto item = array[i];
if(item < lowest) lowest = item;
if(item > highest) highest = item;
}
}
fmt::print("MMX {}, {}\n", lowest, highest);
return {lowest, highest};
}
template<typename T>
inline std::tuple<T, T> minmax_t(const span<T>& array)
{
return minmax_t(array, [](T _val) { return true; });
}
namespace work
{
@ -8,6 +44,35 @@ namespace work
template<bool unshuffle>
int xshuffle_ip(const char* file)
{
mm::mmap map(file);
if constexpr(unshuffle)
{
auto [byte_l, byte_h] = minmax_t(map.as_span().reinterpret<std::int8_t>());
rng::drng drng((std::int32_t) ((0xfffa << 16) | (byte_l<<7) | byte_h ));
rng::unshuffle(drng, map.as_span());
auto [float_l, float_h] = minmax_t(map.as_span().reinterpret<float>(), [](float f) -> bool { return !( (f!=f) || f < -FLT_MAX || f > FLT_MAX); });
rng::frng frng(float_l, float_h);
rng::unshuffle(frng, map.as_span().reinterpret<float>());
auto [long_l, long_h] = minmax_t(map.as_span().reinterpret<std::int64_t>());
rng::xoroshiro128plus xorng(*(const std::uint64_t*)&long_l, *(const std::uint64_t*)&long_h);
rng::unshuffle(xorng, map.as_span().reinterpret<std::int64_t>());
} else {
auto [long_l, long_h] = minmax_t(map.as_span().reinterpret<std::int64_t>());
rng::xoroshiro128plus xorng(*(const std::uint64_t*)&long_l, *(const std::uint64_t*)&long_h);
rng::shuffle(xorng, map.as_span().reinterpret<std::int64_t>());
auto [float_l, float_h] = minmax_t(map.as_span().reinterpret<float>(), [](float f) -> bool { return !( (f!=f) || f < -FLT_MAX || f > FLT_MAX); });
rng::frng frng(float_l, float_h);
rng::shuffle(frng, map.as_span().reinterpret<float>());
auto [byte_l, byte_h] = minmax_t(map.as_span().reinterpret<std::int8_t>());
rng::drng drng((std::int32_t) ((0xfffa << 16) | (byte_l<<7) | byte_h ));
rng::shuffle(drng, map.as_span());
}
return 0;
}
@ -15,6 +80,14 @@ namespace work
template<bool unshuffle>
int xshuffle_op(const char* ifile, const char* ofile, bool is_buffered)
{
if constexpr(unshuffle)
{
} else {
}
panic("Unimplemented");
return 0;
}

Loading…
Cancel
Save