diff --git a/README.md b/README.md index 86dc30f..88144ea 100644 --- a/README.md +++ b/README.md @@ -2,3 +2,9 @@ Maximum Entropy Compressor - Using Probability Hashes w/Hash-Chaining to compress "impossible" data sets. Copyright © 2021 by Brett Kuntz. All rights reserved. + +# TEST SOFTWARE + +# TEST SOFTWARE + +# TEST SOFTWARE \ No newline at end of file diff --git a/compress.c b/compress.c index c72d5e3..12603f5 100755 --- a/compress.c +++ b/compress.c @@ -25,7 +25,11 @@ si main(si argc, s8 ** argv) return EXIT_FAILURE; } - if (argc != 3) return EXIT_FAILURE; + if (argc != 3) + { + printf("param error\n"); + return EXIT_FAILURE; + } const ui CUTOFF = atol(argv[1]); const ul SAMPLES = atol(argv[2]); diff --git a/compress3.c b/compress3.c index 6bb0b1d..9df3354 100755 --- a/compress3.c +++ b/compress3.c @@ -1,7 +1,7 @@ //---------------------------------------------------------------------------------------------------------------------- // Copyright © 2021 by Brett Kuntz. All rights reserved. //---------------------------------------------------------------------------------------------------------------------- -#include "compress.h" +#include "compress3.h" //---------------------------------------------------------------------------------------------------------------------- //#include si main(si argc, s8 ** argv) diff --git a/compress3.h b/compress3.h index ab323a3..b3fb092 100755 --- a/compress3.h +++ b/compress3.h @@ -42,18 +42,10 @@ static const u64 BLAKE_IV = UINT64_C(0xA54FF53A5F1D36F1); //---------------------------------------------------------------------------------------------------------------------- static void check(void); static u64 find_hash(ui * const restrict, u8 * const restrict, u8 const * const restrict, u64 * const restrict, u64 * const restrict, u8 const * const restrict, const u64, const ui); -static u64 find_p_hash(ui * const restrict, u8 * const restrict, u8 const * const restrict, u64 * const restrict, u64 * const restrict, u8 const * const restrict, const u64, const u8, const ui); static u64 find_p_hash2(ui * const restrict, u8 * const restrict, u8 const * const restrict, u64 * const restrict, u64 * const restrict, u8 const * const restrict, const u64, const ui, const ui); static void hash(u8 * const restrict, u64 const * const restrict, u64 * const, u64 const * const restrict, u8 const * const restrict); -static void p_hash(u8 * const restrict, u64 const * const restrict, u64 * const, u64 const * const restrict, const u8, u8 const * const restrict); static void p_hash2(u8 * const restrict, u64 const * const restrict, u64 * const, u64 const * const restrict, const ui, u8 const * const restrict); static si get_hash_score(void const * const); static void blake2b(u64 * const restrict, u64 const * const restrict); static u64 tick(void); -static void set_bit(void * const restrict, const ui, const ui); -static ui get_bit(void const * const restrict, const ui); -static u16 rng_word(u64 * const, u64 const * const restrict, ui * const restrict); -static u16 rng(u64 * const restrict, u64 const * const restrict, const u16, ui * const restrict); -static void print_bytes(s8 const * const restrict, void const * const, const ui); -static void print_population(s8 const * const restrict, void const * const); //---------------------------------------------------------------------------------------------------------------------- \ No newline at end of file diff --git a/compress4.c b/compress4.c new file mode 100755 index 0000000..5f65190 --- /dev/null +++ b/compress4.c @@ -0,0 +1,380 @@ +//---------------------------------------------------------------------------------------------------------------------- +// Copyright © 2021 by Brett Kuntz. All rights reserved. +//---------------------------------------------------------------------------------------------------------------------- +#include "compress4.h" +#define CUTS_LENGTH 5 +static ui CHAIN_CUTS[CUTS_LENGTH] = { 37, 23, 17, 14, 11 }; +//---------------------------------------------------------------------------------------------------------------------- +//#include +si main(si argc, s8 ** argv) +{ + /*if (SetPriorityClass(GetCurrentProcess(), IDLE_PRIORITY_CLASS)) + { + printf("Low priority set\n"); + }*/ + + if (sem_init(&csoutput, 1, 1) == -1) + { + printf("sem_init error\n"); + return EXIT_FAILURE; + } + + if ((global_total = mmap(0, sizeof(u64), PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0)) == MAP_FAILED) + { + printf("mmap error\n"); + return EXIT_FAILURE; + } + + /*if (argc != 3) + { + printf("param error\n"); + return EXIT_FAILURE; + } + + const ui CUTOFF = atol(argv[1]); + const ul SAMPLES = atol(argv[2]);*/ + + const ui CUTOFF = 36; + const ul SAMPLES = 10000; + + // Start + printf("Starting\n"); + + const ui threads = get_nprocs(); + + SAMPLES_PER_TEST = round((r64)SAMPLES / threads); + + printf("CUTOFF we're testing: %u\n", CUTOFF); + printf("SAMPLES: %lu\n", SAMPLES); + printf("threads: %u\n", threads); + printf("SAMPLES per thread: %lu\n", (ul)SAMPLES_PER_TEST); + fflush(0); + + for (ui t=0;t best_distance) + { + best_n = n; + best_distance = distance; + } + + for (ui i=0;i<16;i++) + { + m[i] += BLAKE_IV; + } + } + + memcpy(m, input_iv, 128); + + for (ui i=0;i<16;i++) + { + m[i] += BLAKE_IV * best_n; + } + + p_hash(output_block, RO_IV, v, m, cutoff, input_block); + + si temp_distance = get_hash_score(output_block); + + if (temp_distance < 0) + { + for (ui i=0;i<128;i++) + { + output_block[i] = ~output_block[i]; + } + + temp_distance = -temp_distance; + } + + if (temp_distance != best_distance) + { + printf("ERROR: temp_distance [%d] != best_distance [%u]\nHash confirmation failed!!\n", temp_distance, best_distance); + fflush(0); + exit(EXIT_FAILURE); + } + + if (distance) + { + *distance = temp_distance; + } + + return best_n; +} +//---------------------------------------------------------------------------------------------------------------------- +static void p_hash(u8 * const restrict block, u64 const * const restrict RO_IV, u64 * const v, u64 const * const restrict m, const u8 cutoff, u8 const * const restrict input_block) +{ + memcpy(v, RO_IV, 128); + + blake2b(v, m); + + u8 const * vp = (u8 *)v; + u8 const * const vl = &vp[128]; + + for (ui i=0;i<128;i++) + { + if (vp == vl) + { + vp = (u8 *)v; + blake2b(v, m); + } + + u8 byte = 0; + + for (u8 b=1;b;b<<=1,vp++) + { + if (*vp < cutoff) + { + byte |= b; + } + } + + block[i] = byte ^ input_block[i]; + } +} +//---------------------------------------------------------------------------------------------------------------------- +static u64 find_hash(ui * const restrict distance, u8 * const restrict output_block, u8 const * const restrict input_block, u64 * const restrict v, u64 * const restrict m, u8 const * const restrict input_iv, const u64 block_n, const ui limit) +{ + u64 best_n = 0; + ui best_distance = 0; + const u64 total_n = (u64)1 << (limit - 1); + + u64 RO_IV[16]; + memcpy(RO_IV, input_iv, 128); + + for (ui i=0;i<16;i++) + { + RO_IV[i] += BLAKE_IV * block_n; + } + + memcpy(m, input_iv, 128); + + for (u64 n=0;n best_distance) + { + best_n = n; + best_distance = distance; + } + + for (ui i=0;i<16;i++) + { + m[i] += BLAKE_IV; + } + } + + memcpy(m, input_iv, 128); + + for (ui i=0;i<16;i++) + { + m[i] += BLAKE_IV * best_n; + } + + hash(output_block, RO_IV, v, m, input_block); + + si temp_distance = get_hash_score(output_block); + + if (temp_distance < 0) + { + for (ui i=0;i<128;i++) + { + output_block[i] = ~output_block[i]; + } + + temp_distance = -temp_distance; + } + + if (temp_distance != best_distance) + { + printf("ERROR: temp_distance [%d] != best_distance [%u]\nHash confirmation failed!!\n", temp_distance, best_distance); + fflush(0); + exit(EXIT_FAILURE); + } + + if (distance) + { + *distance = temp_distance; + } + + return best_n; +} +//---------------------------------------------------------------------------------------------------------------------- +static void hash(u8 * const restrict block, u64 const * const restrict RO_IV, u64 * const v, u64 const * const restrict m, u8 const * const restrict input_block) +{ + u8 const * const restrict vp = (u8 *)v; + + memcpy(v, RO_IV, 128); + + blake2b(v, m); + + for (ui i=0;i<128;i++) + { + block[i] = vp[i] ^ input_block[i]; + } +} +//---------------------------------------------------------------------------------------------------------------------- +static si get_hash_score(void const * const block) +{ + u8 const * const restrict vp = (u8 *)block; + + si population = 0; + + for (ui i=0;i<16;i++) + { + u64 temp; + memcpy(&temp, &vp[i * 8], 8); + population += __builtin_popcountl(temp); + } + + return 512 - population; +} +//---------------------------------------------------------------------------------------------------------------------- +static void blake2b(u64 * const restrict v, u64 const * const restrict m) +{ + #define G(x, y, a, b, c, d) \ + do { \ + a = a + b + m[x]; \ + d = ((d ^ a) >> 32) | ((d ^ a) << 32); \ + c = c + d; \ + b = ((b ^ c) >> 24) | ((b ^ c) << 40); \ + a = a + b + m[y]; \ + d = ((d ^ a) >> 16) | ((d ^ a) << 48); \ + c = c + d; \ + b = ((b ^ c) >> 63) | ((b ^ c) << 1); \ + } while (0) + + G(13, 11, v[ 0], v[ 4], v[ 8], v[12]); + G( 7, 14, v[ 1], v[ 5], v[ 9], v[13]); + G(12, 1, v[ 2], v[ 6], v[10], v[14]); + G( 3, 9, v[ 3], v[ 7], v[11], v[15]); + G( 5, 0, v[ 0], v[ 5], v[10], v[15]); + G(15, 4, v[ 1], v[ 6], v[11], v[12]); + G( 8, 6, v[ 2], v[ 7], v[ 8], v[13]); + G( 2, 10, v[ 3], v[ 4], v[ 9], v[14]); + + #undef G +} +//---------------------------------------------------------------------------------------------------------------------- +static u64 tick(void) +{ + struct timespec now; + clock_gettime(CLOCK_MONOTONIC, &now); + return ((u64)now.tv_sec * 1000) + ((u64)now.tv_nsec / 1000000); +} +//---------------------------------------------------------------------------------------------------------------------- \ No newline at end of file diff --git a/compress4.h b/compress4.h new file mode 100755 index 0000000..5f71d37 --- /dev/null +++ b/compress4.h @@ -0,0 +1,50 @@ +//---------------------------------------------------------------------------------------------------------------------- +// Copyright © 2021 by Brett Kuntz. All rights reserved. +//--NOV-26-2018--------------------------------------------------------------------------------------------------------- +#include +#include +static_assert(CHAR_BIT == 8, "This code requires [char] to be exactly 8 bits."); +static_assert(sizeof(long) == 8, "This code requires [long] to be exactly 8 bytes."); // __builtin_popcountl +//---------------------------------------------------------------------------------------------------------------------- +#include +typedef unsigned char u8 ; typedef char s8 ; +typedef uint16_t u16 ; typedef int16_t s16 ; +typedef uint32_t u32 ; typedef int32_t s32 ; +typedef uint64_t u64 ; typedef int64_t s64 ; +typedef __uint128_t u128 ; typedef __int128_t s128 ; +typedef unsigned int ui ; typedef int si ; +typedef unsigned long ul ; typedef long sl ; +typedef unsigned long long ull ; typedef long long sll ; +typedef float r32 ; typedef double r64 ; +//---------------------------------------------------------------------------------------------------------------------- +#define halt do { fflush(0); while (1) sleep(-1); } while (0) +#define cwait do { fflush(0); sleep(1); do errno = 0, wait(0); while (errno != ECHILD); sleep(1); } while(0) +//---------------------------------------------------------------------------------------------------------------------- +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +//---------------------------------------------------------------------------------------------------------------------- +static u64 * global_total, SAMPLES_PER_TEST; +static sem_t csoutput; +static const u64 BLAKE_IV = UINT64_C(0xA54FF53A5F1D36F1); +//---------------------------------------------------------------------------------------------------------------------- +static void check(void); +static u64 find_hash(ui * const restrict, u8 * const restrict, u8 const * const restrict, u64 * const restrict, u64 * const restrict, u8 const * const restrict, const u64, const ui); +static u64 find_p_hash(ui * const restrict, u8 * const restrict, u8 const * const restrict, u64 * const restrict, u64 * const restrict, u8 const * const restrict, const u64, const u8, const ui); +static void hash(u8 * const restrict, u64 const * const restrict, u64 * const, u64 const * const restrict, u8 const * const restrict); +static void p_hash(u8 * const restrict, u64 const * const restrict, u64 * const, u64 const * const restrict, const u8, u8 const * const restrict); +static si get_hash_score(void const * const); +static void blake2b(u64 * const restrict, u64 const * const restrict); +static u64 tick(void); +//---------------------------------------------------------------------------------------------------------------------- \ No newline at end of file diff --git a/make.sh b/make.sh index 530a987..6398154 100755 --- a/make.sh +++ b/make.sh @@ -2,5 +2,5 @@ # Copyright © 2021 by Brett Kuntz. All rights reserved. -#clang compress.c -o compute -pthread -lm -Wall -Werror -Wfatal-errors -Wno-unused-function -pipe -O3 -flto -fomit-frame-pointer -march=native -mtune=native -clang compress.c -o compute.exe -pthread -lm -Wall -Werror -Wfatal-errors -Wno-unused-function -pipe -O3 -fomit-frame-pointer -march=native -mtune=native \ No newline at end of file +clang compress.c -o compute -pthread -lm -Wall -Werror -Wfatal-errors -Wno-unused-function -pipe -O3 -flto -fomit-frame-pointer -march=native -mtune=native +#clang compress4.c -o compute4.exe -pthread -lm -Wall -Werror -Wfatal-errors -Wno-unused-function -pipe -O3 -fomit-frame-pointer -march=native -mtune=native \ No newline at end of file