From 81e49e6654ee48dd3fbed11f16dcdc72a777c1cb Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 28 Oct 2015 15:56:48 +0100 Subject: [PATCH] fine tune & messages --- NEWS | 2 +- lib/zstdhc_static.h | 9 +++- programs/paramgrill.c | 97 +++++++++++++++++++++++++++++++------------ 3 files changed, 79 insertions(+), 29 deletions(-) diff --git a/NEWS b/NEWS index 8388dfce..4c4c3001 100644 --- a/NEWS +++ b/NEWS @@ -1,7 +1,7 @@ v0.2.2 Fix : Visual Studio 2013 & 2015 release compilation, by Christophe Chevalier -v0.2.2 +v0.2.1 Fix : Read errors, advanced fuzzer tests, by Hanno Böck v0.2.0 diff --git a/lib/zstdhc_static.h b/lib/zstdhc_static.h index b1618ada..5bfc408c 100644 --- a/lib/zstdhc_static.h +++ b/lib/zstdhc_static.h @@ -78,7 +78,7 @@ size_t ZSTD_HC_compress_advanced (ZSTD_HC_CCtx* ctx, /* ************************************* * Pre-defined compression levels ***************************************/ -#define ZSTD_HC_MAX_CLEVEL 20 +#define ZSTD_HC_MAX_CLEVEL 25 static const ZSTD_HC_parameters ZSTD_HC_defaultParameters[ZSTD_HC_MAX_CLEVEL+1] = { /* W, C, H, S */ { 18, 4, 12, 1 }, /* 0 - should never be used */ @@ -101,7 +101,12 @@ static const ZSTD_HC_parameters ZSTD_HC_defaultParameters[ZSTD_HC_MAX_CLEVEL+1] { 24, 24, 23, 11 }, /*17 */ { 24, 24, 23, 12 }, /*18 */ { 24, 24, 23, 13 }, /*19 */ - { 25, 25, 25, 22 }, /*20 - ultra-slow */ + { 24, 24, 23, 13 }, /*20 */ + { 24, 24, 23, 13 }, /*21 */ + { 24, 24, 23, 13 }, /*22 */ + { 24, 24, 23, 13 }, /*23 */ + { 24, 24, 23, 13 }, /*24 */ + { 24, 24, 23, 13 }, /*25 */ }; diff --git a/programs/paramgrill.c b/programs/paramgrill.c index afc81d4e..8ac9a2c2 100644 --- a/programs/paramgrill.c +++ b/programs/paramgrill.c @@ -464,7 +464,7 @@ static void BMK_printWinners(FILE* f, const winnerInfo_t* winners, size_t srcSiz } -static int BMK_seed(winnerInfo_t* winners, ZSTD_HC_parameters params, +static int BMK_seed(winnerInfo_t* winners, const ZSTD_HC_parameters params, const void* srcBuffer, size_t srcSize, ZSTD_HC_CCtx* ctx) { @@ -474,47 +474,91 @@ static int BMK_seed(winnerInfo_t* winners, ZSTD_HC_parameters params, BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, params); - for (cLevel = 0; cLevel <= ZSTD_HC_MAX_CLEVEL; cLevel++) + for (cLevel = 1; cLevel <= ZSTD_HC_MAX_CLEVEL; cLevel++) { - if ( (testResult.cSpeed > g_cSpeedTarget[cLevel]) && (winners[cLevel].result.cSize==0) ) - better = 1; /* first solution for this cLevel */ + if (testResult.cSpeed < g_cSpeedTarget[cLevel]) + continue; /* not fast enough for this level */ + if (winners[cLevel].result.cSize==0) + { + /* first solution for this cLevel */ + winners[cLevel].result = testResult; + winners[cLevel].params = params; + BMK_printWinner(stdout, cLevel, testResult, params, srcSize); + better = 1; + continue; + } - if ( (testResult.cSpeed > g_cSpeedTarget[cLevel]) && ((double)testResult.cSize <= (double)winners[cLevel].result.cSize * 1.001 )) + if ((double)testResult.cSize <= ((double)winners[cLevel].result.cSize * (1. + (0.01 / cLevel))) ) { /* Validate solution is "good enough" */ - double W_ratioNote = log ( (double)srcSize / testResult.cSize); - double O_ratioNote = log ( (double)srcSize / winners[cLevel].result.cSize); + double W_ratio = (double)srcSize / testResult.cSize; + double O_ratio = (double)srcSize / winners[cLevel].result.cSize; + double W_ratioNote = log (W_ratio); + double O_ratioNote = log (O_ratio); size_t W_DMemUsed = (1 << params.windowLog) + (16 KB); size_t O_DMemUsed = (1 << winners[cLevel].params.windowLog) + (16 KB); - double W_DMemUsed_note = W_ratioNote * ( 18 + 2*cLevel) - log((double)W_DMemUsed); - double O_DMemUsed_note = O_ratioNote * ( 18 + 2*cLevel) - log((double)O_DMemUsed); + double W_DMemUsed_note = W_ratioNote * ( 25 + 8*cLevel) - log((double)W_DMemUsed); + double O_DMemUsed_note = O_ratioNote * ( 25 + 8*cLevel) - log((double)O_DMemUsed); size_t W_CMemUsed = (1 << params.windowLog) + 4 * (1 << params.hashLog) + 4 * (1 << params.chainLog); size_t O_CMemUsed = (1 << winners[cLevel].params.windowLog) + 4 * (1 << winners[cLevel].params.hashLog) + 4 * (1 << winners[cLevel].params.chainLog); - double W_CMemUsed_note = W_ratioNote * ( 35 + 5*cLevel) - log((double)W_CMemUsed); - double O_CMemUsed_note = O_ratioNote * ( 35 + 5*cLevel) - log((double)O_CMemUsed); + double W_CMemUsed_note = W_ratioNote * ( 50 + 13*cLevel) - log((double)W_CMemUsed); + double O_CMemUsed_note = O_ratioNote * ( 50 + 13*cLevel) - log((double)O_CMemUsed); - double W_CSpeed_note = W_ratioNote * ( 20 + 3*cLevel) + log((double)testResult.cSpeed); - double O_CSpeed_note = O_ratioNote * ( 20 + 3*cLevel) + log((double)winners[cLevel].result.cSpeed); + double W_CSpeed_note = W_ratioNote * ( 30 + 8*cLevel) + log((double)testResult.cSpeed); + double O_CSpeed_note = O_ratioNote * ( 30 + 8*cLevel) + log((double)winners[cLevel].result.cSpeed); - double W_DSpeed_note = W_ratioNote * ( 10 + cLevel) + log((double)testResult.dSpeed); - double O_DSpeed_note = O_ratioNote * ( 10 + cLevel) + log((double)winners[cLevel].result.dSpeed); + double W_DSpeed_note = W_ratioNote * ( 20 + 2*cLevel) + log((double)testResult.dSpeed); + double O_DSpeed_note = O_ratioNote * ( 20 + 2*cLevel) + log((double)winners[cLevel].result.dSpeed); - if (W_DMemUsed_note < O_DMemUsed_note) continue; /* uses too much Decompression memory for too little benefit */ - if (W_CMemUsed_note < O_CMemUsed_note) continue; /* uses too much memory for compression for too little benefit */ - if (W_CSpeed_note < O_CSpeed_note ) continue; /* too large compression speed difference for the compression benefit */ - if (W_DSpeed_note < O_DSpeed_note ) continue; /* too large decompression speed difference for the compression benefit */ + if (W_DMemUsed_note < O_DMemUsed_note) + { + /* uses too much Decompression memory for too little benefit */ + if (W_ratio > O_ratio) + DISPLAY ("Decompression Memory : %5.3f @ %4.1f MB vs %5.3f @ %4.1f MB : not enough for level %i\n", + W_ratio, (double)(W_DMemUsed) / 1024 / 1024, + O_ratio, (double)(O_DMemUsed) / 1024 / 1024, cLevel); + continue; + } + if (W_CMemUsed_note < O_CMemUsed_note) + { + /* uses too much memory for compression for too little benefit */ + if (W_ratio > O_ratio) + DISPLAY ("Compression Memory : %5.3f @ %4.1f MB vs %5.3f @ %4.1f MB : not enough for level %i\n", + W_ratio, (double)(W_CMemUsed) / 1024 / 1024, + O_ratio, (double)(O_CMemUsed) / 1024 / 1024, cLevel); + continue; + } + if (W_CSpeed_note < O_CSpeed_note ) + { + /* too large compression speed difference for the compression benefit */ + if (W_ratio > O_ratio) + DISPLAY ("Compression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n", + W_ratio, (double)(testResult.cSpeed) / 1000., + O_ratio, (double)(winners[cLevel].result.cSpeed) / 1000., cLevel); + continue; + } + if (W_DSpeed_note < O_DSpeed_note ) + { + /* too large decompression speed difference for the compression benefit */ + if (W_ratio > O_ratio) + DISPLAY ("Decompression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n", + W_ratio, (double)(testResult.dSpeed) / 1000., + O_ratio, (double)(winners[cLevel].result.dSpeed) / 1000., cLevel); + continue; + } + + if (W_ratio < O_ratio) + DISPLAY("Solution %4.3f selected over %4.3f at level %i, due to better secondary statistics \n", W_ratio, O_ratio, cLevel); + + winners[cLevel].result = testResult; + winners[cLevel].params = params; + BMK_printWinner(stdout, cLevel, testResult, params, srcSize); better = 1; } - if (better) - { - winners[cLevel].result = testResult; - winners[cLevel].params = params; - BMK_printWinner(stdout, cLevel, testResult, params, srcSize); - } } return better; @@ -665,7 +709,8 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize) /* populate initial solution */ { - const int maxSeeds = g_noSeed ? 1 : ZSTD_HC_MAX_CLEVEL; + const int maxSeeds = g_noSeed ? 1 : 13; + //const int maxSeeds = g_noSeed ? 1 : ZSTD_HC_MAX_CLEVEL; //BMK_seed(winners, params, srcBuffer, srcSize, ctx); for (i=1; i<=maxSeeds; i++) {