Update to 2.0!

progress v2.0.0
Avril 3 years ago
parent 07cd0c252d
commit d5dd86ed24
Signed by: flanchan
GPG Key ID: 284488987C31F630

@ -1,50 +1,73 @@
SRC = $(wildcard src/*.c)
INCLUDE = include/
SRC_C = $(wildcard src/*.c)
SRC_CXX = $(wildcard src/*.cpp)
SRC_CXX+= $(wildcard src/rng/*.cpp)
INCLUDE = include
PROJECT=shuffle3
BUILD=build
COMMON_FLAGS = -Wall -pedantic $(addprefix -I,$(INCLUDE)) -fno-strict-aliasing
OPT_FLAGS?= -march=native -fgraphite -fopenmp -floop-parallelize-all -ftree-parallelize-loops=4 \
-floop-interchange -ftree-loop-distribution -floop-strip-mine -floop-block
CFLAGS+= $(addprefix -I,$(INCLUDE)) -Wall -pedantic --std=gnu11
LDFLAGS+= -lm
-floop-interchange -ftree-loop-distribution -floop-strip-mine -floop-block \
-fno-stack-check
RELEASE_CFLAGS?= -O3 -flto $(OPT_FLAGS)
RELEASE_LDFLAGS?= -O3 -flto
CXX_OPT_FLAGS?= $(OPT_FLAGS) -felide-constructors
DEBUG_CFLAGS?= -g -O0
DEBUG_LDFLAGS?= -O0
CFLAGS += $(COMMON_FLAGS) --std=gnu11
CXXFLAGS += $(COMMON_FLAGS) --std=gnu++20 -fno-exceptions
LDFLAGS += -lfmt
STRIP=strip
OBJ = $(addprefix obj/,$(SRC:.c=.o))
RELEASE_CFLAGS?= -O3 -flto $(OPT_FLAGS)
RELEASE_CXXFLAGS?= -O3 -flto $(CXX_OPT_FLAGS)
RELEASE_LDFLAGS?= -O3 -flto
DEBUG_CFLAGS?= -O0 -g -DDEBUG
DEBUG_CXXFLAGS?= $(DEBUG_CFLAGS)
DEBUG_LDFLAGS?=
# Objects
OBJ_C = $(addprefix obj/c/,$(SRC_C:.c=.o))
OBJ_CXX = $(addprefix obj/cxx/,$(SRC_CXX:.cpp=.o))
OBJ = $(OBJ_C) $(OBJ_CXX)
# Phonies
.PHONY: release
release: | dirs $(BUILD)/$(PROJECT)-release
release: | dirs $(PROJECT)-release
.PHONY: debug
debug: | dirs $(BUILD)/$(PROJECT)-debug
debug: | dirs $(PROJECT)-debug
# Targets
dirs:
@mkdir -p obj/src
@mkdir -p $(BUILD)
@mkdir -p obj/c{,xx}/src
@mkdir -p obj/cxx/src/rng
obj/%.o: %.c
obj/c/%.o: %.c
$(CC) -c $< $(CFLAGS) -o $@ $(LDFLAGS)
$(BUILD)/$(PROJECT)-release: CFLAGS+= $(RELEASE_CFLAGS)
$(BUILD)/$(PROJECT)-release: LDFLAGS+= $(RELEASE_LDFLAGS)
$(BUILD)/$(PROJECT)-release: $(OBJ)
$(CC) $^ $(CFLAGS) -o $@ $(LDFLAGS)
obj/cxx/%.o: %.cpp
$(CXX) -c $< $(CXXFLAGS) -o $@ $(LDFLAGS)
$(PROJECT)-release: CFLAGS+= $(RELEASE_CFLAGS)
$(PROJECT)-release: CXXFLAGS += $(RELEASE_CXXFLAGS)
$(PROJECT)-release: LDFLAGS += $(RELEASE_LDFLAGS)
$(PROJECT)-release: $(OBJ)
$(CXX) $^ $(CXXFLAGS) -o $@ $(LDFLAGS)
$(STRIP) $@
$(BUILD)/$(PROJECT)-debug: CFLAGS+= $(DEBUG_CFLAGS)
$(BUILD)/$(PROJECT)-debug: LDFLAGS+= $(DEBUG_LDFLAGS)
$(BUILD)/$(PROJECT)-debug: $(OBJ)
$(CC) $^ $(CFLAGS) -o $@ $(LDFLAGS)
$(PROJECT)-debug: CFLAGS+= $(DEBUG_CFLAGS)
$(PROJECT)-debug: CXXFLAGS += $(DEBUG_CXXFLAGS)
$(PROJECT)-debug: LDFLAGS += $(DEBUG_LDFLAGS)
$(PROJECT)-debug: $(OBJ)
$(CXX) $^ $(CXXFLAGS) -o $@ $(LDFLAGS)
clean:
rm -rf obj
rm -f $(BUILD)/*
rm -f $(PROJECT)-{release,debug}

@ -1,4 +1,4 @@
# shuffle3 - 3 stage byte shuffler
# `shuffle3-lean` - Improved 3 stage byte shuffler
Deterministically and reversably shuffle a file's bytes around.
@ -17,24 +17,25 @@ $ shuffle3 -u file
```
## Other options
Run with `--help` for more options.
Run with `-h` for more options.
# Building
Run `make` to build the normal binary. It will output to `shuffle3-release`.
## Release target
The `release` (default) target uses the variables `RELEASE_CFLAGS` and `RELEASE_LDFLAGS` to specify opitimisations. These can be set by you if you wish.
The `release` (default) target uses the variables `RELEASE_CFLAGS`, `RELEASE_CXXFLAGS` and `RELEASE_LDFLAGS` to specify opitimisations, as well as the `OPT_FLAGS` variable. These can be set by you if you wish.
### Note
The default `RELEASE_CFLAGS` contains the flag `-march=native`. This may be underisable for you, in which case set the variable or modify the makefile to remove it.
The default `OPT_FLAGS` contains the flag `-march=native`. This may be underisable for you, in which case set the variable or modify the makefile to remove it.
## Debug target
To build with debug information, run `make debug`. Extra debug flags can be provided with the `DEBUG_CFLAGS` and `DEBUG_LDFLAGS` variables which have default values in the Makefile.
To build with debug information, run `make debug`. Extra debug flags can be provided with the `DEBUG_CFLAGS`, `DEBUG_CXXFLAGS` and `DEBUG_LDFLAGS` variables which have default values in the Makefile.
The build and unstripped binary will be `shuffle3-debug`.
## Notes
Before switching between `release` and `debug` targets, remember to run `make clean`.
To disable stripping of release build binaries, run with `make STRIP=: release`
# License
GPL'd with <3

@ -1 +0,0 @@
Complete rewrite using `mmap` et al. Maybe in Rust of C++ idk

@ -1,25 +1,59 @@
#ifndef _RNG_H
#define _RNG_H
typedef struct rng_algo *RNG; //RNG algorithm reference
#define rng_reinterpret(rng, value, type) (*((type*)rng_next(rng, &value, sizeof(type))))
#define rng_reinterpret_new(rng, value, type) (*((type*)rng_next(rng, malloc(sizeof(type)), sizeof(type))))
#define rng_reinterpret_stackalloc(rng, value, type) rng_reinterpret(rng, alloca(sizeof(type)), type)
#define RNG_LEXBIND(rng, type, name) type name = rng_reinterpret(rng, name, type)
double rng_next_double(RNG algo);
int rng_next_int_bounded(RNG algo, int min, int max);
int rng_next_int(RNG algo, int max);
void* rng_next(RNG algo, void* data, int len);
int rng_chance(RNG algo, double d);
void rng_free(RNG algo);
void rng_seed(RNG algo, void* seed);
RNG rng_new(RNG (*instantiate)(void));
#define RNG_IMPL_DEFINITION(name) RNG __rng_impl_ ## name(void)
#define RNG_ALGO(name) &__rng_impl_ ## name
#define RNG_NEW(name) rng_new(RNG_ALGO(name))
#include "shuffle3.h"
#ifdef __cplusplus
extern "C" {
#endif
enum rng_kind {
RNG_KIND_FRNG,
RNG_KIND_DRNG,
RNG_KIND_XORNG,
};
typedef struct rng_init
{
enum rng_kind kind;
union {
struct {
double state[2];
} frng;
struct {
int32_t state;
} drng;
struct {
uint64_t state[2];
} xorng;
} init;
} rng_init_opt;
typedef struct rng_impl* _UNIQUE rng_t;
rng_t rng_new(rng_init_opt kind);
#define RNG_INIT(_kind,...) ((rng_init_opt){.kind=(_kind), .init.__VA_ARGS__ })
void rng_free(rng_t ptr);
// Tests
extern void rng_test();
extern void rng_test_spec(rng_t rng);
#ifdef __cplusplus
}
// RNG interfaces
#include <rng/frng.hpp>
#include <rng/drng.hpp>
#include <rng/xoroshiro128plus.hpp>
namespace rng {
void test_algo(RNG&& rng);
template<class R,
typename std::enable_if<std::is_base_of<RNG, R>::value>::value __fuck>
inline void test_algo(R&& rng) { test_algo(static_cast<RNG&&>(rng)); }
}
#endif
#endif /* _RNG_H */

@ -1,73 +0,0 @@
SRC_C = $(wildcard src/*.c)
SRC_CXX = $(wildcard src/*.cpp)
SRC_CXX+= $(wildcard src/rng/*.cpp)
INCLUDE = include
PROJECT=shuffle3
COMMON_FLAGS = -Wall -pedantic $(addprefix -I,$(INCLUDE)) -fno-strict-aliasing
OPT_FLAGS?= -march=native -fgraphite -fopenmp -floop-parallelize-all -ftree-parallelize-loops=4 \
-floop-interchange -ftree-loop-distribution -floop-strip-mine -floop-block \
-fno-stack-check
CXX_OPT_FLAGS?= $(OPT_FLAGS) -felide-constructors
CFLAGS += $(COMMON_FLAGS) --std=gnu11
CXXFLAGS += $(COMMON_FLAGS) --std=gnu++20 -fno-exceptions
LDFLAGS += -lfmt
STRIP=strip
RELEASE_CFLAGS?= -O3 -flto $(OPT_FLAGS)
RELEASE_CXXFLAGS?= -O3 -flto $(CXX_OPT_FLAGS)
RELEASE_LDFLAGS?= -O3 -flto
DEBUG_CFLAGS?= -O0 -g -DDEBUG
DEBUG_CXXFLAGS?= $(DEBUG_CFLAGS)
DEBUG_LDFLAGS?=
# Objects
OBJ_C = $(addprefix obj/c/,$(SRC_C:.c=.o))
OBJ_CXX = $(addprefix obj/cxx/,$(SRC_CXX:.cpp=.o))
OBJ = $(OBJ_C) $(OBJ_CXX)
# Phonies
.PHONY: release
release: | dirs $(PROJECT)-release
.PHONY: debug
debug: | dirs $(PROJECT)-debug
# Targets
dirs:
@mkdir -p obj/c{,xx}/src
@mkdir -p obj/cxx/src/rng
obj/c/%.o: %.c
$(CC) -c $< $(CFLAGS) -o $@ $(LDFLAGS)
obj/cxx/%.o: %.cpp
$(CXX) -c $< $(CXXFLAGS) -o $@ $(LDFLAGS)
$(PROJECT)-release: CFLAGS+= $(RELEASE_CFLAGS)
$(PROJECT)-release: CXXFLAGS += $(RELEASE_CXXFLAGS)
$(PROJECT)-release: LDFLAGS += $(RELEASE_LDFLAGS)
$(PROJECT)-release: $(OBJ)
$(CXX) $^ $(CXXFLAGS) -o $@ $(LDFLAGS)
$(STRIP) $@
$(PROJECT)-debug: CFLAGS+= $(DEBUG_CFLAGS)
$(PROJECT)-debug: CXXFLAGS += $(DEBUG_CXXFLAGS)
$(PROJECT)-debug: LDFLAGS += $(DEBUG_LDFLAGS)
$(PROJECT)-debug: $(OBJ)
$(CXX) $^ $(CXXFLAGS) -o $@ $(LDFLAGS)
clean:
rm -rf obj
rm -f $(PROJECT)-{release,debug}

@ -1,59 +0,0 @@
#ifndef _RNG_H
#define _RNG_H
#include "shuffle3.h"
#ifdef __cplusplus
extern "C" {
#endif
enum rng_kind {
RNG_KIND_FRNG,
RNG_KIND_DRNG,
RNG_KIND_XORNG,
};
typedef struct rng_init
{
enum rng_kind kind;
union {
struct {
double state[2];
} frng;
struct {
int32_t state;
} drng;
struct {
uint64_t state[2];
} xorng;
} init;
} rng_init_opt;
typedef struct rng_impl* _UNIQUE rng_t;
rng_t rng_new(rng_init_opt kind);
#define RNG_INIT(_kind,...) ((rng_init_opt){.kind=(_kind), .init.__VA_ARGS__ })
void rng_free(rng_t ptr);
// Tests
extern void rng_test();
extern void rng_test_spec(rng_t rng);
#ifdef __cplusplus
}
// RNG interfaces
#include <rng/frng.hpp>
#include <rng/drng.hpp>
#include <rng/xoroshiro128plus.hpp>
namespace rng {
void test_algo(RNG&& rng);
template<class R,
typename std::enable_if<std::is_base_of<RNG, R>::value>::value __fuck>
inline void test_algo(R&& rng) { test_algo(static_cast<RNG&&>(rng)); }
}
#endif
#endif /* _RNG_H */

@ -1,78 +0,0 @@
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <shuffle3.h>
#include <panic.h>
#include <reinterpret.h>
#include <map.h>
#include <rng.h>
#include <work.h>
#include <debug.h>
#define noreturn __attribute__((noreturn)) void
_Static_assert(sizeof(float)==sizeof(uint32_t), "float is not 32 bits");
const char* _prog_name;
noreturn help_then_exit()
{
fprintf(stderr, "Try passing `-h`\n");
exit(1);
}
void usage()
{
printf( "shuffle3 - 3 pass binary shuffler\n"
"Usage: %s -s <file>\n"
"Usage: %s -u <file>\n", _prog_name, _prog_name);
printf("\nOPTIONS:\n"
" -s\tShuffle file in place\n"
" -u\tUnshuffle file in place\n");
}
int main(int argc, char** argv)
{
_prog_name = argv[0];
work_args_t parsed;
if( !argv[1] || *(argv[1]) != '-') help_then_exit();
D_dprintf("Parsing `%c'", argv[1][1]);
switch(argv[1][1])
{
case 's':
parsed.op = OP_SHUFFLE_IP;
if(!(parsed.data.op_shuffle_ip.file = argv[2]))
{
fprintf(stderr, "Error: -s expected file argument.\n");
return 1;
}
D_dprintf("parsed.op = %d", OP_SHUFFLE_IP);
break;
case 'u':
parsed.op = OP_UNSHUFFLE_IP;
if(!(parsed.data.op_unshuffle_ip.file = argv[2]))
{
fprintf(stderr, "Error: -u expected file argument.\n");
return 1;
}
D_dprintf("parsed.op = %d", OP_UNSHUFFLE_IP);
break;
case 'h':
usage();
return 0;
default:
fprintf(stderr, "Error: unknown argument `%s'\n\n", argv[1]);
help_then_exit();
panic("Unreachable");
}
return do_work(parsed);
}

@ -0,0 +1,50 @@
SRC = $(wildcard src/*.c)
INCLUDE = include/
PROJECT=shuffle3
BUILD=build
OPT_FLAGS?= -march=native -fgraphite -fopenmp -floop-parallelize-all -ftree-parallelize-loops=4 \
-floop-interchange -ftree-loop-distribution -floop-strip-mine -floop-block
CFLAGS+= $(addprefix -I,$(INCLUDE)) -Wall -pedantic --std=gnu11
LDFLAGS+= -lm
RELEASE_CFLAGS?= -O3 -flto $(OPT_FLAGS)
RELEASE_LDFLAGS?= -O3 -flto
DEBUG_CFLAGS?= -g -O0
DEBUG_LDFLAGS?= -O0
STRIP=strip
OBJ = $(addprefix obj/,$(SRC:.c=.o))
.PHONY: release
release: | dirs $(BUILD)/$(PROJECT)-release
.PHONY: debug
debug: | dirs $(BUILD)/$(PROJECT)-debug
dirs:
@mkdir -p obj/src
@mkdir -p $(BUILD)
obj/%.o: %.c
$(CC) -c $< $(CFLAGS) -o $@ $(LDFLAGS)
$(BUILD)/$(PROJECT)-release: CFLAGS+= $(RELEASE_CFLAGS)
$(BUILD)/$(PROJECT)-release: LDFLAGS+= $(RELEASE_LDFLAGS)
$(BUILD)/$(PROJECT)-release: $(OBJ)
$(CC) $^ $(CFLAGS) -o $@ $(LDFLAGS)
$(STRIP) $@
$(BUILD)/$(PROJECT)-debug: CFLAGS+= $(DEBUG_CFLAGS)
$(BUILD)/$(PROJECT)-debug: LDFLAGS+= $(DEBUG_LDFLAGS)
$(BUILD)/$(PROJECT)-debug: $(OBJ)
$(CC) $^ $(CFLAGS) -o $@ $(LDFLAGS)
clean:
rm -rf obj
rm -f $(BUILD)/*

@ -0,0 +1,25 @@
#ifndef _RNG_H
#define _RNG_H
typedef struct rng_algo *RNG; //RNG algorithm reference
#define rng_reinterpret(rng, value, type) (*((type*)rng_next(rng, &value, sizeof(type))))
#define rng_reinterpret_new(rng, value, type) (*((type*)rng_next(rng, malloc(sizeof(type)), sizeof(type))))
#define rng_reinterpret_stackalloc(rng, value, type) rng_reinterpret(rng, alloca(sizeof(type)), type)
#define RNG_LEXBIND(rng, type, name) type name = rng_reinterpret(rng, name, type)
double rng_next_double(RNG algo);
int rng_next_int_bounded(RNG algo, int min, int max);
int rng_next_int(RNG algo, int max);
void* rng_next(RNG algo, void* data, int len);
int rng_chance(RNG algo, double d);
void rng_free(RNG algo);
void rng_seed(RNG algo, void* seed);
RNG rng_new(RNG (*instantiate)(void));
#define RNG_IMPL_DEFINITION(name) RNG __rng_impl_ ## name(void)
#define RNG_ALGO(name) &__rng_impl_ ## name
#define RNG_NEW(name) rng_new(RNG_ALGO(name))
#endif /* _RNG_H */

@ -0,0 +1,373 @@
#include <stdio.h>
#include <float.h>
#include <stdint.h>
#include <string.h>
#include <assert.h>
#include <rng.h>
#include <rng_algos.h>
#include <ptr_store.h>
#include <array.h>
_Static_assert(sizeof(float)==4, "Float must be 32 bits.");
int surpress_out=0;
/*void test_destructor(object* ptr)
{
printf("die\n");
}*/
void _rng_object_destructor(object* obj)
{
rng_free(obj->state);
}
RNG rng_object(S_LEXENV, RNG rng)
{
object proto = OBJ_PROTO;
proto.state = rng;
proto.destructor = &_rng_object_destructor;
return snew_obj(proto)->state;
}
void shuffle(RNG with, array_t data)
{
if(!surpress_out) printf(" -> shuffling %d objects...", (int)ar_size(data));
for(int i=ar_size(data)-1;i>0;i--)
{
int j = rng_next_int(with, i);
ar_swap(data, i, j);
}
if(!surpress_out) printf(" Okay\n");
}
void unshuffle(RNG with, array_t data)
{
int rng_values[ar_size(data)-1];
int k=0;
if(!surpress_out) printf(" -> unshuffling %d objects...", (int)ar_size(data));
for(int i=ar_size(data)-1;i>0;i--)
rng_values[k++] = rng_next_int(with, i);
for(int i=1;i<ar_size(data);i++)
{
ar_swap(data, i, rng_values[--k]);
}
if(!surpress_out) printf(" Okay\n");
}
void minmax_int64_ts(int64_t* min, int64_t* max, const array_t data)
{
for(register int i=0;i<ar_size(data);i++)
{
if(ar_get_v(data, int64_t, i) > *max) *max = ar_get_v(data, int64_t, i);
if(ar_get_v(data, int64_t, i) < *min) *min = ar_get_v(data, int64_t, i);
}
}
int valid_float(float f)
{
return !( (f!=f) || (f< -FLT_MAX || f> FLT_MAX));
}
void minmax_floats(float* min, float* max, const array_t data)
{
for(register int i=0;i<ar_size(data);i++)
{
if(!valid_float(ar_get_v(data,float,i))) continue;
if(ar_get_v(data, float, i) > *max) *max = ar_get_v(data, float, i);
if(ar_get_v(data, float, i) < *min) *min = ar_get_v(data, float, i);
}
}
void minmax_sbytes(int8_t* min, int8_t* max, const array_t data)
{
for(register int i=0;i<ar_size(data);i++)
{
if(ar_get_v(data, int8_t, i) > *max) *max = ar_get_v(data, int8_t, i);
if(ar_get_v(data, int8_t, i) < *min) *min = ar_get_v(data, int8_t, i);
}
}
void print_array(const array_t data)
{
printf("---%d elements---\n", (int)ar_size(data));
for(register int i=0;i<ar_size(data);i++)
printf("%d ", (int)ar_get_v(data, unsigned char, i));
printf("\n---\n");
}
void unshuffle3(S_LEXENV, array_t data)
{
if(ar_type(data)!=sizeof(int8_t))
ar_reinterpret(data, sizeof(int8_t));
RNG xoro = S_LEXCALL(rng_object, rng_new(RNG_ALGO(xoroshiro128plus)));
RNG frng = S_LEXCALL(rng_object, rng_new(RNG_ALGO(frng)));
RNG drng = S_LEXCALL(rng_object, rng_new(RNG_ALGO(drng)));
int8_t bmax = INT8_MIN;
int8_t bmin = INT8_MAX;
minmax_sbytes(&bmin, &bmax, data);
uint32_t seed = (0xfffa << 16) | (bmin<<8) | bmax;
rng_seed(drng, &seed);
unshuffle(drng, data);
float fmin = (float)DBL_MAX;
float fmax = (float)-DBL_MAX;
ar_reinterpret(data, sizeof(float));
minmax_floats(&fmin, &fmax, data);
double fseed[2];
fseed[0] = fmin;
fseed[1] = fmax;
rng_seed(frng, fseed);
unshuffle(frng, data);
int64_t min = INT64_MAX;
int64_t max = INT64_MIN;
ar_reinterpret(data, sizeof(int64_t));
minmax_int64_ts(&min, &max, data);
uint64_t xseed[2];
xseed[0] = min;
xseed[1] = max;
rng_seed(xoro, xseed);
unshuffle(xoro, data);
}
void shuffle3(S_LEXENV, array_t data)
{
if(ar_type(data)!=sizeof(int64_t))
ar_reinterpret(data, sizeof(int64_t));
RNG xoro = S_LEXCALL(rng_object, rng_new(RNG_ALGO(xoroshiro128plus)));
RNG frng = S_LEXCALL(rng_object, rng_new(RNG_ALGO(frng)));
RNG drng = S_LEXCALL(rng_object, rng_new(RNG_ALGO(drng)));
int64_t min = INT64_MAX;
int64_t max = INT64_MIN;
float fmin = (float)DBL_MAX;
float fmax = (float)-DBL_MAX;
int8_t bmax = INT8_MIN;
int8_t bmin = INT8_MAX;
minmax_int64_ts(&min, &max, data);
uint64_t xseed[2];
xseed[0] = min;
xseed[1] = max;
rng_seed(xoro, xseed);
shuffle(xoro, data);
ar_reinterpret(data, sizeof(float));
minmax_floats(&fmin, &fmax, data);
double fseed[2];
fseed[0] = fmin;
fseed[1] = fmax;
rng_seed(frng, fseed);
shuffle(frng, data);
ar_reinterpret(data,sizeof(int8_t));
minmax_sbytes(&bmin, &bmax, data);
uint32_t seed = (0xfffa << 16) | (bmin<<8) | bmax;
rng_seed(drng, &seed);
shuffle(drng, data);
}
void print_usage(char** argv)
{
printf("Usage: %s -[b]s <file> [<outfile>]\nUsage: %s -[b]u <file> [<outfile>]\n", argv[0], argv[0]);
printf("Usage: %s -S <string>|-\nUsage: %s -U <string>|-\n", argv[0], argv[0]);
printf(" <file>\t\tFile to shuffle\n");
printf(" <outfile>\tOutput file (only valid for buffered mode)\n");
printf(" <string>\tThe string to shuffle (for -(S|U)). If `-`, read from stdin\n");
printf("\ts\tShuffle\n");
printf("\tu\tReverse shuffle\n");
printf("\tS\tShuffle string\n");
printf("\tU\tReverse shuffle string\n");
printf("\n\tb\tUse buffered mode instead of shuffling in place (must specify output file)\n");
}
array_t read_whole_file(S_NAMED_LEXENV(base), FILE* fp)
{
fseek(fp,0,SEEK_END);
int64_t sz = ftell(fp);
fseek(fp,0,SEEK_SET);
array_t ar;
MANAGED({
void* buf = smalloc(sz);
register size_t read;
register size_t full=0;
while ((read=fread(buf, 1, sz-full, fp))>0)
full+=read;
assert(full == sz);
ar = ar_create_memory_from(base, buf, 1, sz);
});
return ar;
}
#define BUFSIZE 512
#define BUFMIN 128
char* read_stdin()
{
char* input, *p;
int len,remain,n,size;
size = BUFSIZE;
input = malloc(size);
memset(input,0,size);
len=0;
remain = size;
while(!feof(stdin))
{
if(remain<= BUFMIN)
{
remain+=size;
size *=2;
p = realloc(input, size);
if(p==NULL) {
free(input);
return NULL;
}
input = p;
}
fgets(input+len, remain, stdin);
n+=strlen(input+len);
len+=n;
remain-=n;
}
return input;
}
int string_shuffle(char* string, int un)
{
if(!string)
{
char* ptr = read_stdin();
if(!ptr) {
printf("! read from stdin failed\n");
return -1;
}
int ret = string_shuffle(ptr, un);
free(ptr);
return ret;
}
else if(strcmp(string, "-") == 0)
{
return string_shuffle(NULL, un);
}
else {
surpress_out=1;
MANAGED({
array_t ar = ar_create_memory_from(LEXENV, string, 1, strlen(string));
if(un) unshuffle3(LEXENV, ar);
else shuffle3(LEXENV, ar);
ar_reinterpret(ar, sizeof(char));
for(int i=0;i<ar_size(ar);i++)
{
printf("%c", ar_get_v(ar, char, i));
}
});
return 0;
}
}
int main(int argc, char** argv)
{
int is_buffered=0;
int i=1;
int is_unshuffling =0;
if(!argv[1] || !argv[2] || argv[1][0] != '-')
{
print_usage(argv);
return 1;
}
else {
do_switch:
switch(argv[1][i])
{
case 'b':
if(is_buffered)
{
print_usage(argv);
return 1;
}
is_buffered = 1;
i+=1;
goto do_switch;
case 'u':
is_unshuffling=1;
break;
case 's':
break;
case 'S':
return string_shuffle(argv[2], 0);
case 'U':
return string_shuffle(argv[2], 1);
default:
print_usage(argv);
return 1;
}
}
if(is_buffered && !argv[3])
{
printf("Buffered option requires an output file\n");
return 1;
}
MANAGED_RETURNABLE(int ,{
array_t array;
if(is_buffered)
{
FILE* infile = fopen(argv[2], "rb");
if(!infile)
{
printf("! could not open file for reading\n");
MANAGED_RETURN(1);
}
array = read_whole_file(LEXENV, infile);
fclose(infile);
printf(" buffered file (%ld bytes)\n", ar_size(array));
}
else {
FILE* infile = fopen(argv[2], "r+b");
if(!infile)
{
printf("! could not open file for reading+writing\n");
MANAGED_RETURN(1);
}
array = ar_create_file(LEXENV, infile, 1, 1);
}
//print_array(array);
if(is_unshuffling) unshuffle3(LEXENV, array);
else shuffle3(LEXENV, array);
if(is_buffered)
{
FILE* outfile = fopen(argv[3], "wb");
if(!outfile)
{
printf("! could not open outfile for writing\n");
MANAGED_RETURN(1);
}
void* wbuf = smalloc(ar_full_size(array));
if(!ar_ndump(array, wbuf, ar_full_size(array), 0, ar_size(array)))
printf("W memory dump failed, continuing anyway\n");
fwrite(wbuf, 1, ar_full_size(array), outfile);
fclose(outfile);
printf(" write completed (%d bytes)\n", (int)ar_size(array));
}
});
return 0;
}

@ -1,373 +1,80 @@
#include <stdlib.h>
#include <stdio.h>
#include <float.h>
#include <stdint.h>
#include <string.h>
#include <assert.h>
#include <shuffle3.h>
#include <panic.h>
#include <reinterpret.h>
#include <map.h>
#include <rng.h>
#include <rng_algos.h>
#include <ptr_store.h>
#include <array.h>
#include <work.h>
_Static_assert(sizeof(float)==4, "Float must be 32 bits.");
#include <debug.h>
int surpress_out=0;
#define noreturn __attribute__((noreturn)) void
/*void test_destructor(object* ptr)
{
printf("die\n");
}*/
void _rng_object_destructor(object* obj)
{
rng_free(obj->state);
}
RNG rng_object(S_LEXENV, RNG rng)
{
object proto = OBJ_PROTO;
proto.state = rng;
proto.destructor = &_rng_object_destructor;
_Static_assert(sizeof(float)==sizeof(uint32_t), "float is not 32 bits");
return snew_obj(proto)->state;
}
const char* _prog_name;
void shuffle(RNG with, array_t data)
noreturn help_then_exit()
{
if(!surpress_out) printf(" -> shuffling %d objects...", (int)ar_size(data));
for(int i=ar_size(data)-1;i>0;i--)
{
int j = rng_next_int(with, i);
ar_swap(data, i, j);
}
if(!surpress_out) printf(" Okay\n");
fprintf(stderr, "Try passing `-h`\n");
exit(1);
}
void unshuffle(RNG with, array_t data)
void usage()
{
int rng_values[ar_size(data)-1];
int k=0;
if(!surpress_out) printf(" -> unshuffling %d objects...", (int)ar_size(data));
for(int i=ar_size(data)-1;i>0;i--)
rng_values[k++] = rng_next_int(with, i);
for(int i=1;i<ar_size(data);i++)
{
ar_swap(data, i, rng_values[--k]);
}
if(!surpress_out) printf(" Okay\n");
}
void minmax_int64_ts(int64_t* min, int64_t* max, const array_t data)
{
for(register int i=0;i<ar_size(data);i++)
{
if(ar_get_v(data, int64_t, i) > *max) *max = ar_get_v(data, int64_t, i);
if(ar_get_v(data, int64_t, i) < *min) *min = ar_get_v(data, int64_t, i);
}
}
int valid_float(float f)
{
return !( (f!=f) || (f< -FLT_MAX || f> FLT_MAX));
}
void minmax_floats(float* min, float* max, const array_t data)
{
for(register int i=0;i<ar_size(data);i++)
{
if(!valid_float(ar_get_v(data,float,i))) continue;
if(ar_get_v(data, float, i) > *max) *max = ar_get_v(data, float, i);
if(ar_get_v(data, float, i) < *min) *min = ar_get_v(data, float, i);
}
}
void minmax_sbytes(int8_t* min, int8_t* max, const array_t data)
{
for(register int i=0;i<ar_size(data);i++)
{
if(ar_get_v(data, int8_t, i) > *max) *max = ar_get_v(data, int8_t, i);
if(ar_get_v(data, int8_t, i) < *min) *min = ar_get_v(data, int8_t, i);
}
}
void print_array(const array_t data)
{
printf("---%d elements---\n", (int)ar_size(data));
for(register int i=0;i<ar_size(data);i++)
printf("%d ", (int)ar_get_v(data, unsigned char, i));
printf("\n---\n");
}
void unshuffle3(S_LEXENV, array_t data)
{
if(ar_type(data)!=sizeof(int8_t))
ar_reinterpret(data, sizeof(int8_t));
RNG xoro = S_LEXCALL(rng_object, rng_new(RNG_ALGO(xoroshiro128plus)));
RNG frng = S_LEXCALL(rng_object, rng_new(RNG_ALGO(frng)));
RNG drng = S_LEXCALL(rng_object, rng_new(RNG_ALGO(drng)));
int8_t bmax = INT8_MIN;
int8_t bmin = INT8_MAX;
minmax_sbytes(&bmin, &bmax, data);
uint32_t seed = (0xfffa << 16) | (bmin<<8) | bmax;
rng_seed(drng, &seed);
unshuffle(drng, data);
float fmin = (float)DBL_MAX;
float fmax = (float)-DBL_MAX;
ar_reinterpret(data, sizeof(float));
minmax_floats(&fmin, &fmax, data);
double fseed[2];
fseed[0] = fmin;
fseed[1] = fmax;
rng_seed(frng, fseed);
unshuffle(frng, data);
int64_t min = INT64_MAX;
int64_t max = INT64_MIN;
ar_reinterpret(data, sizeof(int64_t));
minmax_int64_ts(&min, &max, data);
uint64_t xseed[2];
xseed[0] = min;
xseed[1] = max;
rng_seed(xoro, xseed);
unshuffle(xoro, data);
}
void shuffle3(S_LEXENV, array_t data)
{
if(ar_type(data)!=sizeof(int64_t))
ar_reinterpret(data, sizeof(int64_t));
RNG xoro = S_LEXCALL(rng_object, rng_new(RNG_ALGO(xoroshiro128plus)));
RNG frng = S_LEXCALL(rng_object, rng_new(RNG_ALGO(frng)));
RNG drng = S_LEXCALL(rng_object, rng_new(RNG_ALGO(drng)));
int64_t min = INT64_MAX;
int64_t max = INT64_MIN;
float fmin = (float)DBL_MAX;
float fmax = (float)-DBL_MAX;
int8_t bmax = INT8_MIN;
int8_t bmin = INT8_MAX;
minmax_int64_ts(&min, &max, data);
uint64_t xseed[2];
xseed[0] = min;
xseed[1] = max;
rng_seed(xoro, xseed);
shuffle(xoro, data);
ar_reinterpret(data, sizeof(float));
minmax_floats(&fmin, &fmax, data);
double fseed[2];
fseed[0] = fmin;
fseed[1] = fmax;
rng_seed(frng, fseed);
shuffle(frng, data);
ar_reinterpret(data,sizeof(int8_t));
minmax_sbytes(&bmin, &bmax, data);
uint32_t seed = (0xfffa << 16) | (bmin<<8) | bmax;
rng_seed(drng, &seed);
shuffle(drng, data);
printf( "shuffle3-lean - improved 3 pass binary shuffler\n"
" written by Avril <flanchan@cumallover.me> (flanchan.moe) with <3\n"
" licensed with GPL v3.0 or later\n\n"
"Usage: %s -s <file>\n"
"Usage: %s -u <file>\n", _prog_name, _prog_name);
printf("\nOPTIONS:\n"
" -s\tShuffle file in place\n"
" -u\tUnshuffle file in place\n");
}
void print_usage(char** argv)
int main(int argc, char** argv)
{
printf("Usage: %s -[b]s <file> [<outfile>]\nUsage: %s -[b]u <file> [<outfile>]\n", argv[0], argv[0]);
printf("Usage: %s -S <string>|-\nUsage: %s -U <string>|-\n", argv[0], argv[0]);
printf(" <file>\t\tFile to shuffle\n");
printf(" <outfile>\tOutput file (only valid for buffered mode)\n");
printf(" <string>\tThe string to shuffle (for -(S|U)). If `-`, read from stdin\n");
printf("\ts\tShuffle\n");
printf("\tu\tReverse shuffle\n");
printf("\tS\tShuffle string\n");
printf("\tU\tReverse shuffle string\n");
printf("\n\tb\tUse buffered mode instead of shuffling in place (must specify output file)\n");
}
_prog_name = argv[0];
array_t read_whole_file(S_NAMED_LEXENV(base), FILE* fp)
{
fseek(fp,0,SEEK_END);
int64_t sz = ftell(fp);
fseek(fp,0,SEEK_SET);
work_args_t parsed;
array_t ar;
MANAGED({
void* buf = smalloc(sz);
register size_t read;
register size_t full=0;
while ((read=fread(buf, 1, sz-full, fp))>0)
full+=read;
assert(full == sz);
if( !argv[1] || *(argv[1]) != '-') help_then_exit();
ar = ar_create_memory_from(base, buf, 1, sz);
});
return ar;
}
#define BUFSIZE 512
#define BUFMIN 128
char* read_stdin()
{
char* input, *p;
int len,remain,n,size;
size = BUFSIZE;
input = malloc(size);
memset(input,0,size);
len=0;
remain = size;
while(!feof(stdin))
D_dprintf("Parsing `%c'", argv[1][1]);
switch(argv[1][1])
{
if(remain<= BUFMIN)
{
remain+=size;
size *=2;
p = realloc(input, size);
if(p==NULL) {
free(input);
return NULL;
}
input = p;
}
fgets(input+len, remain, stdin);
n+=strlen(input+len);
len+=n;
remain-=n;
}
return input;
}
int string_shuffle(char* string, int un)
{
if(!string)
{
char* ptr = read_stdin();
if(!ptr) {
printf("! read from stdin failed\n");
return -1;
}
int ret = string_shuffle(ptr, un);
free(ptr);
return ret;
}
else if(strcmp(string, "-") == 0)
{
return string_shuffle(NULL, un);
}
else {
surpress_out=1;
MANAGED({
array_t ar = ar_create_memory_from(LEXENV, string, 1, strlen(string));
if(un) unshuffle3(LEXENV, ar);
else shuffle3(LEXENV, ar);
ar_reinterpret(ar, sizeof(char));
for(int i=0;i<ar_size(ar);i++)
case 's':
parsed.op = OP_SHUFFLE_IP;
if(!(parsed.data.op_shuffle_ip.file = argv[2]))
{
printf("%c", ar_get_v(ar, char, i));
}
});
return 0;
}
}
int main(int argc, char** argv)
{
int is_buffered=0;
int i=1;
int is_unshuffling =0;
if(!argv[1] || !argv[2] || argv[1][0] != '-')
{
print_usage(argv);
return 1;
}
else {
do_switch:
switch(argv[1][i])
{
case 'b':
if(is_buffered)
{
print_usage(argv);
return 1;
}
is_buffered = 1;
i+=1;
goto do_switch;
case 'u':
is_unshuffling=1;
break;
case 's':
break;
case 'S':
return string_shuffle(argv[2], 0);
case 'U':
return string_shuffle(argv[2], 1);
default:
print_usage(argv);
fprintf(stderr, "Error: -s expected file argument.\n");
return 1;
}
}
if(is_buffered && !argv[3])
{
printf("Buffered option requires an output file\n");
return 1;
}
MANAGED_RETURNABLE(int ,{
array_t array;
if(is_buffered)
{
FILE* infile = fopen(argv[2], "rb");
if(!infile)
{
printf("! could not open file for reading\n");
MANAGED_RETURN(1);
}
array = read_whole_file(LEXENV, infile);
fclose(infile);
printf(" buffered file (%ld bytes)\n", ar_size(array));
}
else {
FILE* infile = fopen(argv[2], "r+b");
if(!infile)
D_dprintf("parsed.op = %d", OP_SHUFFLE_IP);
break;
case 'u':
parsed.op = OP_UNSHUFFLE_IP;
if(!(parsed.data.op_unshuffle_ip.file = argv[2]))
{
printf("! could not open file for reading+writing\n");
MANAGED_RETURN(1);
fprintf(stderr, "Error: -u expected file argument.\n");
return 1;
}
array = ar_create_file(LEXENV, infile, 1, 1);
}
//print_array(array);
D_dprintf("parsed.op = %d", OP_UNSHUFFLE_IP);
break;
case 'h':
usage();
return 0;
default:
fprintf(stderr, "Error: unknown argument `%s'\n\n", argv[1]);
help_then_exit();
panic("Unreachable");
}
if(is_unshuffling) unshuffle3(LEXENV, array);
else shuffle3(LEXENV, array);
if(is_buffered)
{
FILE* outfile = fopen(argv[3], "wb");
if(!outfile)
{
printf("! could not open outfile for writing\n");
MANAGED_RETURN(1);
}
void* wbuf = smalloc(ar_full_size(array));
if(!ar_ndump(array, wbuf, ar_full_size(array), 0, ar_size(array)))
printf("W memory dump failed, continuing anyway\n");
fwrite(wbuf, 1, ar_full_size(array), outfile);
fclose(outfile);
printf(" write completed (%d bytes)\n", (int)ar_size(array));
}
});
return 0;
return do_work(parsed);
}

Loading…
Cancel
Save