facebook
/
zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fixed paramgrill

dev
Yann Collet 2016-03-30 20:42:19 +02:00
parent dd6466a211
commit 51d500425f
2 changed files with 82 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
lib/zstd_compress.c
View File

@ -196,9 +196,12 @@ void ZSTD_adjustCParams(ZSTD_compressionParameters* params, U64 srcSize, size_t
} }
size_t ZSTD_sizeofCCtx(ZSTD_parameters params) /* hidden interface, for paramagrill */ size_t ZSTD_sizeofCCtx(ZSTD_compressionParameters cParams) /* hidden interface, for paramagrill */
{ {
ZSTD_CCtx* zc = ZSTD_createCCtx(); ZSTD_CCtx* zc = ZSTD_createCCtx();
ZSTD_parameters params;
params.cParams = cParams;
params.fParams.contentSizeFlag = 1;
ZSTD_compressBegin_advanced(zc, NULL, 0, params, 0); ZSTD_compressBegin_advanced(zc, NULL, 0, params, 0);
{ size_t const ccsize = sizeof(*zc) + zc->workSpaceSize; { size_t const ccsize = sizeof(*zc) + zc->workSpaceSize;
ZSTD_freeCCtx(zc); ZSTD_freeCCtx(zc);

148
programs/paramgrill.c
View File

@ -127,7 +127,7 @@ static U32 g_rand = 1;
static U32 g_singleRun = 0; static U32 g_singleRun = 0;
static U32 g_target = 0; static U32 g_target = 0;
static U32 g_noSeed = 0; static U32 g_noSeed = 0;
static ZSTD_parameters g_params = { 0, 0, 0, 0, 0, 0, 0, ZSTD_greedy }; static ZSTD_compressionParameters g_params = { 0, 0, 0, 0, 0, 0, ZSTD_greedy };
void BMK_SetNbIterations(int nbLoops) void BMK_SetNbIterations(int nbLoops)
{ {
@ -252,7 +252,7 @@ typedef struct
static size_t BMK_benchParam(BMK_result_t* resultPtr, static size_t BMK_benchParam(BMK_result_t* resultPtr,
const void* srcBuffer, size_t srcSize, const void* srcBuffer, size_t srcSize,
ZSTD_CCtx* ctx, ZSTD_CCtx* ctx,
const ZSTD_parameters params) const ZSTD_compressionParameters cParams)
{ {
const size_t blockSize = g_blockSize ? g_blockSize : srcSize; const size_t blockSize = g_blockSize ? g_blockSize : srcSize;
const U32 nbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize); const U32 nbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize);
@ -260,13 +260,14 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
const size_t maxCompressedSize = (size_t)nbBlocks * ZSTD_compressBound(blockSize); const size_t maxCompressedSize = (size_t)nbBlocks * ZSTD_compressBound(blockSize);
void* const compressedBuffer = malloc(maxCompressedSize); void* const compressedBuffer = malloc(maxCompressedSize);
void* const resultBuffer = malloc(srcSize); void* const resultBuffer = malloc(srcSize);
U32 Wlog = params.windowLog; ZSTD_parameters params;
U32 Clog = params.contentLog; U32 Wlog = cParams.windowLog;
U32 Hlog = params.hashLog; U32 Clog = cParams.contentLog;
U32 Slog = params.searchLog; U32 Hlog = cParams.hashLog;
U32 Slength = params.searchLength; U32 Slog = cParams.searchLog;
U32 Tlength = params.targetLength; U32 Slength = cParams.searchLength;
ZSTD_strategy strat = params.strategy; U32 Tlength = cParams.targetLength;
ZSTD_strategy strat = cParams.strategy;
char name[30] = { 0 }; char name[30] = { 0 };
U64 crcOrig; U64 crcOrig;
@ -316,6 +317,8 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
const int startTime =BMK_GetMilliStart(); const int startTime =BMK_GetMilliStart();
DISPLAY("\r%79s\r", ""); DISPLAY("\r%79s\r", "");
params.cParams = cParams;
params.fParams.contentSizeFlag = 0;
for (loopNb = 1; loopNb <= g_nbIterations; loopNb++) { for (loopNb = 1; loopNb <= g_nbIterations; loopNb++) {
int nbLoops; int nbLoops;
int milliTime; int milliTime;
@ -408,11 +411,11 @@ const char* g_stratName[] = { "ZSTD_fast ",
"ZSTD_btlazy2", "ZSTD_btlazy2",
"ZSTD_btopt " }; "ZSTD_btopt " };
static void BMK_printWinner(FILE* f, U32 cLevel, BMK_result_t result, ZSTD_parameters params, size_t srcSize) static void BMK_printWinner(FILE* f, U32 cLevel, BMK_result_t result, ZSTD_compressionParameters params, size_t srcSize)
{ {
DISPLAY("\r%79s\r", ""); DISPLAY("\r%79s\r", "");
fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u,%3u, %s }, ", fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ",
0, params.windowLog, params.contentLog, params.hashLog, params.searchLog, params.searchLength, params.windowLog, params.contentLog, params.hashLog, params.searchLog, params.searchLength,
params.targetLength, g_stratName[(U32)(params.strategy)]); params.targetLength, g_stratName[(U32)(params.strategy)]);
fprintf(f, fprintf(f,
"/* level %2u */ /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n", "/* level %2u */ /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n",
@ -424,7 +427,7 @@ static U32 g_cSpeedTarget[NB_LEVELS_TRACKED] = { 0 }; /* NB_LEVELS_TRACKED : c
typedef struct { typedef struct {
BMK_result_t result; BMK_result_t result;
ZSTD_parameters params; ZSTD_compressionParameters params;
} winnerInfo_t; } winnerInfo_t;
static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSize) static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSize)
@ -432,7 +435,7 @@ static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSi
unsigned cLevel; unsigned cLevel;
fprintf(f, "\n /* Proposed configurations : */ \n"); fprintf(f, "\n /* Proposed configurations : */ \n");
fprintf(f, " /* l, W, C, H, S, L, T, strat */ \n"); fprintf(f, " /* W, C, H, S, L, T, strat */ \n");
for (cLevel=0; cLevel <= ZSTD_maxCLevel(); cLevel++) for (cLevel=0; cLevel <= ZSTD_maxCLevel(); cLevel++)
BMK_printWinner(f, cLevel, winners[cLevel].result, winners[cLevel].params, srcSize); BMK_printWinner(f, cLevel, winners[cLevel].result, winners[cLevel].params, srcSize);
@ -447,9 +450,9 @@ static void BMK_printWinners(FILE* f, const winnerInfo_t* winners, size_t srcSiz
BMK_printWinners2(stdout, winners, srcSize); BMK_printWinners2(stdout, winners, srcSize);
} }
size_t ZSTD_sizeofCCtx(ZSTD_parameters params); /* hidden interface, declared here */ size_t ZSTD_sizeofCCtx(ZSTD_compressionParameters params); /* hidden interface, declared here */
static int BMK_seed(winnerInfo_t* winners, const ZSTD_parameters params, static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters params,
const void* srcBuffer, size_t srcSize, const void* srcBuffer, size_t srcSize,
ZSTD_CCtx* ctx) ZSTD_CCtx* ctx)
{ {
@ -542,7 +545,7 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_parameters params,
/* nullified useless params, to ensure count stats */ /* nullified useless params, to ensure count stats */
static ZSTD_parameters* sanitizeParams(ZSTD_parameters params) static ZSTD_compressionParameters* sanitizeParams(ZSTD_compressionParameters params)
{ {
g_params = params; g_params = params;
if (params.strategy == ZSTD_fast) if (params.strategy == ZSTD_fast)
@ -553,44 +556,50 @@ static ZSTD_parameters* sanitizeParams(ZSTD_parameters params)
} }
static void paramVariation(ZSTD_parameters* p) static void paramVariation(ZSTD_compressionParameters* ptr)
{ {
U32 nbChanges = (FUZ_rand(&g_rand) & 3) + 1; ZSTD_compressionParameters p;
for (; nbChanges; nbChanges--) { U32 validated = 0;
const U32 changeID = FUZ_rand(&g_rand) % 14; while (!validated) {
switch(changeID) U32 nbChanges = (FUZ_rand(&g_rand) & 3) + 1;
{ p = *ptr;
case 0: for ( ; nbChanges ; nbChanges--) {
p->contentLog++; break; const U32 changeID = FUZ_rand(&g_rand) % 14;
case 1: switch(changeID)
p->contentLog--; break; {
case 2: case 0:
p->hashLog++; break; p.contentLog++; break;
case 3: case 1:
p->hashLog--; break; p.contentLog--; break;
case 4: case 2:
p->searchLog++; break; p.hashLog++; break;
case 5: case 3:
p->searchLog--; break; p.hashLog--; break;
case 6: case 4:
p->windowLog++; break; p.searchLog++; break;
case 7: case 5:
p->windowLog--; break; p.searchLog--; break;
case 8: case 6:
p->searchLength++; break; p.windowLog++; break;
case 9: case 7:
p->searchLength--; break; p.windowLog--; break;
case 10: case 8:
p->strategy = (ZSTD_strategy)(((U32)p->strategy)+1); break; p.searchLength++; break;
case 11: case 9:
p->strategy = (ZSTD_strategy)(((U32)p->strategy)-1); break; p.searchLength--; break;
case 12: case 10:
p->targetLength *= 1 + ((double)(FUZ_rand(&g_rand)&255)) / 256.; break; p.strategy = (ZSTD_strategy)(((U32)p.strategy)+1); break;
case 13: case 11:
p->targetLength /= 1 + ((double)(FUZ_rand(&g_rand)&255)) / 256.; break; p.strategy = (ZSTD_strategy)(((U32)p.strategy)-1); break;
case 12:
p.targetLength *= 1 + ((double)(FUZ_rand(&g_rand)&255)) / 256.; break;
case 13:
p.targetLength /= 1 + ((double)(FUZ_rand(&g_rand)&255)) / 256.; break;
}
} }
validated = !ZSTD_isError(ZSTD_checkCParams(p));
} }
ZSTD_validateParams(p); *ptr = p;
} }
@ -606,7 +615,7 @@ static BYTE g_alreadyTested[PARAMTABLESIZE] = {0}; /* init to zero */
#define MAX(a,b) ( (a) > (b) ? (a) : (b) ) #define MAX(a,b) ( (a) > (b) ? (a) : (b) )
static void playAround(FILE* f, winnerInfo_t* winners, static void playAround(FILE* f, winnerInfo_t* winners,
ZSTD_parameters params, ZSTD_compressionParameters params,
const void* srcBuffer, size_t srcSize, const void* srcBuffer, size_t srcSize,
ZSTD_CCtx* ctx) ZSTD_CCtx* ctx)
{ {
@ -614,7 +623,7 @@ static void playAround(FILE* f, winnerInfo_t* winners,
const int startTime = BMK_GetMilliStart(); const int startTime = BMK_GetMilliStart();
while (BMK_GetMilliSpan(startTime) < g_maxVariationTime) { while (BMK_GetMilliSpan(startTime) < g_maxVariationTime) {
ZSTD_parameters p = params; ZSTD_compressionParameters p = params;
if (nbVariations++ > g_maxNbVariations) break; if (nbVariations++ > g_maxNbVariations) break;
paramVariation(&p); paramVariation(&p);
@ -635,10 +644,12 @@ static void playAround(FILE* f, winnerInfo_t* winners,
} }
static void potentialRandomParams(ZSTD_parameters* p, U32 inverseChance) static void potentialRandomParams(ZSTD_compressionParameters* p, U32 inverseChance)
{ {
U32 chance = (FUZ_rand(&g_rand) % (inverseChance+1)); U32 chance = (FUZ_rand(&g_rand) % (inverseChance+1));
if (!chance) { U32 validated = 0;
if (!chance)
while (!validated) {
/* totally random entry */ /* totally random entry */
p->contentLog = FUZ_rand(&g_rand) % (ZSTD_CONTENTLOG_MAX+1 - ZSTD_CONTENTLOG_MIN) + ZSTD_CONTENTLOG_MIN; p->contentLog = FUZ_rand(&g_rand) % (ZSTD_CONTENTLOG_MAX+1 - ZSTD_CONTENTLOG_MIN) + ZSTD_CONTENTLOG_MIN;
p->hashLog = FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN) + ZSTD_HASHLOG_MIN; p->hashLog = FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN) + ZSTD_HASHLOG_MIN;
@ -647,7 +658,7 @@ static void potentialRandomParams(ZSTD_parameters* p, U32 inverseChance)
p->searchLength=FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN) + ZSTD_SEARCHLENGTH_MIN; p->searchLength=FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN) + ZSTD_SEARCHLENGTH_MIN;
p->targetLength=FUZ_rand(&g_rand) % (ZSTD_TARGETLENGTH_MAX+1 - ZSTD_TARGETLENGTH_MIN) + ZSTD_TARGETLENGTH_MIN; p->targetLength=FUZ_rand(&g_rand) % (ZSTD_TARGETLENGTH_MAX+1 - ZSTD_TARGETLENGTH_MIN) + ZSTD_TARGETLENGTH_MIN;
p->strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btopt +1)); p->strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btopt +1));
ZSTD_validateParams(p); validated = !ZSTD_isError(ZSTD_checkCParams(*p));
} }
} }
@ -659,10 +670,9 @@ static void BMK_selectRandomStart(
U32 id = (FUZ_rand(&g_rand) % (ZSTD_maxCLevel()+1)); U32 id = (FUZ_rand(&g_rand) % (ZSTD_maxCLevel()+1));
if ((id==0) || (winners[id].params.windowLog==0)) { if ((id==0) || (winners[id].params.windowLog==0)) {
/* totally random entry */ /* totally random entry */
ZSTD_parameters p; ZSTD_compressionParameters p;
potentialRandomParams(&p, 1); potentialRandomParams(&p, 1);
p.srcSize = srcSize; ZSTD_adjustCParams(&p, srcSize, 0);
ZSTD_validateParams(&p);
playAround(f, winners, p, srcBuffer, srcSize, ctx); playAround(f, winners, p, srcBuffer, srcSize, ctx);
} }
else else
@ -673,7 +683,7 @@ static void BMK_selectRandomStart(
static void BMK_benchMem(void* srcBuffer, size_t srcSize) static void BMK_benchMem(void* srcBuffer, size_t srcSize)
{ {
ZSTD_CCtx* ctx = ZSTD_createCCtx(); ZSTD_CCtx* ctx = ZSTD_createCCtx();
ZSTD_parameters params; ZSTD_compressionParameters params;
winnerInfo_t winners[NB_LEVELS_TRACKED]; winnerInfo_t winners[NB_LEVELS_TRACKED];
int i; int i;
unsigned u; unsigned u;
@ -683,8 +693,7 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize)
if (g_singleRun) { if (g_singleRun) {
BMK_result_t testResult; BMK_result_t testResult;
g_params.srcSize = blockSize; ZSTD_adjustCParams(&g_params, srcSize, 0);
ZSTD_validateParams(&g_params);
BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, g_params); BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, g_params);
DISPLAY("\n"); DISPLAY("\n");
return; return;
@ -700,7 +709,7 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize)
else { else {
/* baseline config for level 1 */ /* baseline config for level 1 */
BMK_result_t testResult; BMK_result_t testResult;
params = ZSTD_getParams(1, blockSize); params = ZSTD_getCParams(1, blockSize, 0);
BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, params); BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, params);
g_cSpeedTarget[1] = (testResult.cSpeed * 31) >> 5; g_cSpeedTarget[1] = (testResult.cSpeed * 31) >> 5;
} }
@ -713,8 +722,7 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize)
{ {
const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel();
for (i=1; i<=maxSeeds; i++) { for (i=1; i<=maxSeeds; i++) {
params = ZSTD_getParams(i, blockSize); params = ZSTD_getCParams(i, blockSize, 0);
ZSTD_validateParams(&params);
BMK_seed(winners, params, srcBuffer, srcSize, ctx); BMK_seed(winners, params, srcBuffer, srcSize, ctx);
} }
} }
@ -865,7 +873,7 @@ int optimizeForSize(char* inFileName)
{ {
ZSTD_CCtx* ctx = ZSTD_createCCtx(); ZSTD_CCtx* ctx = ZSTD_createCCtx();
ZSTD_parameters params; ZSTD_compressionParameters params;
winnerInfo_t winner; winnerInfo_t winner;
BMK_result_t candidate; BMK_result_t candidate;
const size_t blockSize = g_blockSize ? g_blockSize : benchedSize; const size_t blockSize = g_blockSize ? g_blockSize : benchedSize;
@ -879,7 +887,7 @@ int optimizeForSize(char* inFileName)
{ {
const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel();
for (i=1; i<=maxSeeds; i++) { for (i=1; i<=maxSeeds; i++) {
params = ZSTD_getParams(i, blockSize); params = ZSTD_getCParams(i, blockSize, 0);
BMK_benchParam(&candidate, origBuff, benchedSize, ctx, params); BMK_benchParam(&candidate, origBuff, benchedSize, ctx, params);
if ( (candidate.cSize < winner.result.cSize) if ( (candidate.cSize < winner.result.cSize)
||((candidate.cSize == winner.result.cSize) && (candidate.cSpeed > winner.result.cSpeed)) ) ||((candidate.cSize == winner.result.cSize) && (candidate.cSpeed > winner.result.cSpeed)) )
@ -1029,7 +1037,7 @@ int main(int argc, char** argv)
case 'S': case 'S':
g_singleRun = 1; g_singleRun = 1;
argument++; argument++;
g_params = ZSTD_getParams(2, g_blockSize); g_params = ZSTD_getCParams(2, g_blockSize, 0);
for ( ; ; ) { for ( ; ; ) {
switch(*argument) switch(*argument)
{ {
@ -1080,7 +1088,7 @@ int main(int argc, char** argv)
argument++; argument++;
while ((*argument>= '0') && (*argument<='9')) while ((*argument>= '0') && (*argument<='9'))
cLevel *= 10, cLevel += *argument++ - '0'; cLevel *= 10, cLevel += *argument++ - '0';
g_params = ZSTD_getParams(cLevel, g_blockSize); g_params = ZSTD_getCParams(cLevel, g_blockSize, 0);
continue; continue;
} }
default : ; default : ;