From 2c6992e1dc00a0519a6adc8f4b692e0bfa19ca6c Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 27 Oct 2015 12:18:00 +0100 Subject: [PATCH] blocksize & T --- lib/zstd_static.h | 16 ++++---- lib/zstdhc.c | 2 +- lib/zstdhc_static.h | 10 +++++ programs/bench.c | 6 +-- programs/paramgrill.c | 96 +++++++++++++++++++++++++++---------------- 5 files changed, 82 insertions(+), 48 deletions(-) diff --git a/lib/zstd_static.h b/lib/zstd_static.h index 76315c3a..147dcd0c 100644 --- a/lib/zstd_static.h +++ b/lib/zstd_static.h @@ -94,11 +94,11 @@ MEM_STATIC unsigned ZSTD_NbCommonBytes (register size_t val) { if (MEM_64bits()) { -# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT) +# if defined(_MSC_VER) && defined(_WIN64) unsigned long r = 0; _BitScanForward64( &r, (U64)val ); return (int)(r>>3); -# elif defined(__GNUC__) && (__GNUC__ >= 3) && !defined(LZ4_FORCE_SW_BITCOUNT) +# elif defined(__GNUC__) && (__GNUC__ >= 3) return (__builtin_ctzll((U64)val) >> 3); # else static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; @@ -107,11 +107,11 @@ MEM_STATIC unsigned ZSTD_NbCommonBytes (register size_t val) } else /* 32 bits */ { -# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) +# if defined(_MSC_VER) unsigned long r; _BitScanForward( &r, (U32)val ); return (int)(r>>3); -# elif defined(__GNUC__) && (__GNUC__ >= 3) && !defined(LZ4_FORCE_SW_BITCOUNT) +# elif defined(__GNUC__) && (__GNUC__ >= 3) return (__builtin_ctz((U32)val) >> 3); # else static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; @@ -123,11 +123,11 @@ MEM_STATIC unsigned ZSTD_NbCommonBytes (register size_t val) { if (MEM_32bits()) { -# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT) +# if defined(_MSC_VER) && defined(_WIN64) unsigned long r = 0; _BitScanReverse64( &r, val ); return (unsigned)(r>>3); -# elif defined(__GNUC__) && (__GNUC__ >= 3) && !defined(LZ4_FORCE_SW_BITCOUNT) +# elif defined(__GNUC__) && (__GNUC__ >= 3) return (__builtin_clzll(val) >> 3); # else unsigned r; @@ -140,11 +140,11 @@ MEM_STATIC unsigned ZSTD_NbCommonBytes (register size_t val) } else /* 32 bits */ { -# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) +# if defined(_MSC_VER) unsigned long r = 0; _BitScanReverse( &r, (unsigned long)val ); return (unsigned)(r>>3); -# elif defined(__GNUC__) && (__GNUC__ >= 3) && !defined(LZ4_FORCE_SW_BITCOUNT) +# elif defined(__GNUC__) && (__GNUC__ >= 3) return (__builtin_clz((U32)val) >> 3); # else unsigned r; diff --git a/lib/zstdhc.c b/lib/zstdhc.c index 869e4355..5a79b9f8 100644 --- a/lib/zstdhc.c +++ b/lib/zstdhc.c @@ -305,7 +305,7 @@ static size_t ZSTD_HC_compressBlock(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstS seqStorePtr->lit += lastLLSize; } - /* Finale compression stage */ + /* Final compression stage */ return ZSTD_compressSequences((BYTE*)dst, maxDstSize, seqStorePtr, srcSize); } diff --git a/lib/zstdhc_static.h b/lib/zstdhc_static.h index 06d8c290..786d6188 100644 --- a/lib/zstdhc_static.h +++ b/lib/zstdhc_static.h @@ -53,6 +53,16 @@ typedef struct U32 searchLog; /* nb of searches : larger == more compression, slower*/ } ZSTD_HC_parameters; +/* parameters boundaries */ +#define ZSTD_HC_WINDOWLOG_MAX 26 +#define ZSTD_HC_WINDOWLOG_MIN 17 +#define ZSTD_HC_CHAINLOG_MAX ZSTD_HC_WINDOWLOG_MAX +#define ZSTD_HC_CHAINLOG_MIN 4 +#define ZSTD_HC_HASHLOG_MAX 28 +#define ZSTD_HC_HASHLOG_MIN 4 +#define ZSTD_HC_SEARCHLOG_MAX (ZSTD_HC_CHAINLOG_MAX-1) +#define ZSTD_HC_SEARCHLOG_MIN 1 + /* ************************************* * Functions diff --git a/programs/bench.c b/programs/bench.c index aed0a446..f52210b9 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -175,7 +175,6 @@ static U32 BMK_rand(U32* src) return rand32 >> 9; } - #define BMK_RAND15BITS ( BMK_rand(&seed) & 0x7FFF) #define BMK_RANDLENGTH ((BMK_rand(&seed) & 3) ? (BMK_rand(&seed) % 15) : (BMK_rand(&seed) % 510) + 15) #define BMK_RANDCHAR (BYTE)((BMK_rand(&seed) & 63) + '0') @@ -230,7 +229,6 @@ typedef struct size_t resSize; } blockParam_t; - typedef size_t (*compressor_t) (void* dst, size_t maxDstSize, const void* src, size_t srcSize, unsigned compressionLevel); static size_t local_compress_fast (void* dst, size_t maxDstSize, const void* src, size_t srcSize, unsigned compressionLevel) @@ -249,7 +247,7 @@ static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, i const size_t maxCompressedSize = (size_t)nbBlocks * ZSTD_compressBound(blockSize); void* const compressedBuffer = malloc(maxCompressedSize); void* const resultBuffer = malloc(srcSize); - const compressor_t compressor = (cLevel <= 1) ? local_compress_fast : ZSTD_HC_compress; + const compressor_t compressor = (cLevel == 0) ? local_compress_fast : ZSTD_HC_compress; U64 crcOrig; /* Memory allocation & restrictions */ @@ -328,7 +326,7 @@ static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, i ratio = (double)cSize / (double)srcSize*100.; DISPLAY("%1i-%-14.14s : %9i -> %9i (%5.2f%%),%7.1f MB/s\r", loopNb, fileName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000.); -#if 1 +#if 0 /* Decompression */ memset(resultBuffer, 0xD6, srcSize); diff --git a/programs/paramgrill.c b/programs/paramgrill.c index 3d030570..93ba0b48 100644 --- a/programs/paramgrill.c +++ b/programs/paramgrill.c @@ -100,10 +100,10 @@ #define COMPRESSIBILITY_DEFAULT 0.50 static const size_t sampleSize = 10000000; -static const int g_grillDuration = 30000000; /* about 8.5 hours */ +static const int g_grillDuration = 50000000; /* about 13 hours */ static const int g_maxParamTime = 15000; /* 15 sec */ static const int g_maxVariationTime = 60000; /* 60 sec */ -static const int g_maxNbVariations = 32; +static const int g_maxNbVariations = 64; /************************************** * Macros @@ -119,6 +119,7 @@ static double g_compressibility = COMPRESSIBILITY_DEFAULT; static U32 g_blockSize = 0; static U32 g_rand = 1; static U32 g_singleRun = 0; +static U32 g_target = 0; static ZSTD_HC_parameters g_params = { 0, 0, 0, 0 }; void BMK_SetNbIterations(int nbLoops) @@ -462,24 +463,18 @@ static int BMK_seed(winnerInfo_t* winners, ZSTD_HC_parameters params, return better; } -#define WINDOWLOG_MAX 26 -#define WINDOWLOG_MIN 17 -#define CHAINLOG_MAX WINDOWLOG_MAX -#define CHAINLOG_MIN 4 -#define HASHLOG_MAX 28 -#define HASHLOG_MIN 4 -#define SEARCHLOG_MAX (CHAINLOG_MAX-1) -#define SEARCHLOG_MIN 1 - #define MAX(a,b) ( (a) > (b) ? (a) : (b) ) -static BYTE g_alreadyTested[WINDOWLOG_MAX+1-WINDOWLOG_MIN] - [CHAINLOG_MAX+1-CHAINLOG_MIN] - [HASHLOG_MAX+1-HASHLOG_MIN] - [SEARCHLOG_MAX+1-SEARCHLOG_MIN] = {}; /* init to zero */ +static BYTE g_alreadyTested[ZSTD_HC_WINDOWLOG_MAX+1-ZSTD_HC_WINDOWLOG_MIN] + [ZSTD_HC_CHAINLOG_MAX+1-ZSTD_HC_CHAINLOG_MIN] + [ZSTD_HC_HASHLOG_MAX+1-ZSTD_HC_HASHLOG_MIN] + [ZSTD_HC_SEARCHLOG_MAX+1-ZSTD_HC_SEARCHLOG_MIN] = {}; /* init to zero */ #define NB_TESTS_PLAYED(p) \ - g_alreadyTested[p.windowLog-WINDOWLOG_MIN][p.chainLog-CHAINLOG_MIN][p.hashLog-HASHLOG_MIN][p.searchLog-SEARCHLOG_MIN] + g_alreadyTested[p.windowLog-ZSTD_HC_WINDOWLOG_MIN] \ + [p.chainLog-ZSTD_HC_CHAINLOG_MIN] \ + [p.hashLog-ZSTD_HC_HASHLOG_MIN] \ + [p.searchLog-ZSTD_HC_SEARCHLOG_MIN] static void playAround(FILE* f, winnerInfo_t* winners, @@ -520,14 +515,14 @@ static void playAround(FILE* f, winnerInfo_t* winners, } /* validate new conf */ - if (p.windowLog > WINDOWLOG_MAX) continue; - if (p.windowLog < MAX(WINDOWLOG_MIN, p.chainLog)) continue; + if (p.windowLog > ZSTD_HC_WINDOWLOG_MAX) continue; + if (p.windowLog < MAX(ZSTD_HC_WINDOWLOG_MIN, p.chainLog)) continue; if (p.chainLog > p.windowLog) continue; - if (p.chainLog < CHAINLOG_MIN) continue; - if (p.hashLog > HASHLOG_MAX) continue; - if (p.hashLog < HASHLOG_MIN) continue; + if (p.chainLog < ZSTD_HC_CHAINLOG_MIN) continue; + if (p.hashLog > ZSTD_HC_HASHLOG_MAX) continue; + if (p.hashLog < ZSTD_HC_HASHLOG_MIN) continue; if (p.searchLog > p.chainLog) continue; - if (p.searchLog < SEARCHLOG_MIN) continue; + if (p.searchLog < ZSTD_HC_SEARCHLOG_MIN) continue; /* exclude faster if already played params */ if (FUZ_rand(&g_rand) & ((1 << NB_TESTS_PLAYED(p))-1)) continue; @@ -554,10 +549,10 @@ static void BMK_selectRandomStart( { /* totally random entry */ ZSTD_HC_parameters p; - p.chainLog = FUZ_rand(&g_rand) % (CHAINLOG_MAX+1 - CHAINLOG_MIN) + CHAINLOG_MIN; - p.hashLog = FUZ_rand(&g_rand) % (HASHLOG_MAX+1 - HASHLOG_MIN) + HASHLOG_MIN; - p.searchLog = FUZ_rand(&g_rand) % (SEARCHLOG_MAX+1 - SEARCHLOG_MIN) + SEARCHLOG_MIN; - p.windowLog = FUZ_rand(&g_rand) % (WINDOWLOG_MAX+1 - WINDOWLOG_MIN) + WINDOWLOG_MIN; + p.chainLog = FUZ_rand(&g_rand) % (ZSTD_HC_CHAINLOG_MAX+1 - ZSTD_HC_CHAINLOG_MIN) + ZSTD_HC_CHAINLOG_MIN; + p.hashLog = FUZ_rand(&g_rand) % (ZSTD_HC_HASHLOG_MAX+1 - ZSTD_HC_HASHLOG_MIN) + ZSTD_HC_HASHLOG_MIN; + p.searchLog = FUZ_rand(&g_rand) % (ZSTD_HC_SEARCHLOG_MAX+1 - ZSTD_HC_SEARCHLOG_MIN) + ZSTD_HC_SEARCHLOG_MIN; + p.windowLog = FUZ_rand(&g_rand) % (ZSTD_HC_WINDOWLOG_MAX+1 - ZSTD_HC_WINDOWLOG_MIN) + ZSTD_HC_WINDOWLOG_MIN; playAround(f, winners, p, srcBuffer, srcSize, ctx); } else @@ -589,24 +584,29 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize) f = fopen(rfName, "w"); if (f==NULL) { DISPLAY("error opening %s \n", rfName); exit(1); } - /* baseline config for level 9 */ - params.windowLog = 19; - params.chainLog = 19; - params.hashLog = 19; - params.searchLog = 9; - BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, params); + if (g_target) + g_cSpeedTarget[9] = g_target * 1000; + else + { + /* baseline config for level 9 */ + params.windowLog = 19; + params.chainLog = 19; + params.hashLog = 19; + params.searchLog = 9; + BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, params); + g_cSpeedTarget[9] = (testResult.cSpeed * 15) >> 4; + } - /* establish speed objectives (relative to current platform) */ - g_cSpeedTarget[9] = (testResult.cSpeed * 15) >> 4; + /* establish speed objectives (relative to level9) */ g_cSpeedTarget[1] = g_cSpeedTarget[9] << 4; g_cSpeedTarget[0] = (g_cSpeedTarget[1] * 181) >> 7; /* sqrt2 */ for (i=2; i> 8; /* populate initial solution */ - BMK_seed(winners, params, srcBuffer, srcSize, ctx); for (i=1; i<=ZSTD_HC_MAX_CLEVEL; i++) BMK_seed(winners, seedParams[i], srcBuffer, srcSize, ctx); + BMK_seed(winners, params, srcBuffer, srcSize, ctx); BMK_printWinners(f, winners, srcSize); /* start tests */ @@ -622,6 +622,7 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize) /* end summary */ BMK_printWinners(f, winners, srcSize); + DISPLAY("grillParams operations completed \n"); /* clean up*/ fclose(f); @@ -822,6 +823,31 @@ int main(int argc, char** argv) break; } + /* target level9 objective, in MB/s */ + case 'T': + argument++; + g_target = 0; + while ((*argument >= '0') && (*argument <= '9')) + { + g_target *= 10; + g_target += *argument - '0'; + argument++; + } + break; + + /* cut input into blocks */ + case 'B': + { + g_blockSize = 0; + argument++; + while ((*argument >='0') && (*argument <='9')) + g_blockSize *= 10, g_blockSize += *argument++ - '0'; + if (*argument=='K') g_blockSize<<=10, argument++; /* allows using KB notation */ + if (*argument=='M') g_blockSize<<=20, argument++; + if (*argument=='B') argument++; + } + break; + /* Unknown command */ default : return badusage(exename); }