From 3f2d024dca7723b4235618d99ce755287e530472 Mon Sep 17 00:00:00 2001 From: George Lu Date: Thu, 9 Aug 2018 10:42:35 -0700 Subject: [PATCH] forceAttachDict --- tests/paramgrill.c | 746 ++++++++++++++++++++++++--------------------- 1 file changed, 397 insertions(+), 349 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index af445918..aab46ede 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -66,22 +66,10 @@ static const int g_maxNbVariations = 64; #define CUSTOM_LEVEL 99 #define BASE_CLEVEL 1 -/* indices for each of the variables */ -typedef enum { - wlog_ind = 0, - clog_ind = 1, - hlog_ind = 2, - slog_ind = 3, - slen_ind = 4, - tlen_ind = 5, - strt_ind = 6 -} varInds_t; - -#define NUM_PARAMS 7 -/* just don't use strategy as a param. */ - #undef ZSTD_WINDOWLOG_MAX #define ZSTD_WINDOWLOG_MAX 27 //no long range stuff for now. +#define FADT_MIN 0 +#define FADT_MAX ((U32)-1) #define ZSTD_TARGETLENGTH_MIN 0 #define ZSTD_TARGETLENGTH_MAX 999 @@ -93,13 +81,146 @@ typedef enum { #define SLEN_RANGE (ZSTD_SEARCHLENGTH_MAX - ZSTD_SEARCHLENGTH_MIN + 1) #define TLEN_RANGE 17 #define STRT_RANGE (ZSTD_btultra - ZSTD_fast + 1) -/* TLEN_RANGE picked manually */ +#define FADT_RANGE 3 #define CHECKTIME(r) { if(BMK_timeSpan(g_time) > g_timeLimit_s) { DEBUGOUTPUT("Time Limit Reached\n"); return r; } } #define CHECKTIMEGT(ret, val, _gototag) {if(BMK_timeSpan(g_time) > g_timeLimit_s) { DEBUGOUTPUT("Time Limit Reached\n"); ret = val; goto _gototag; } } -static const int rangetable[NUM_PARAMS] = { WLOG_RANGE, CLOG_RANGE, HLOG_RANGE, SLOG_RANGE, SLEN_RANGE, TLEN_RANGE, STRT_RANGE }; +#define PARAM_UNSET ((U32)-2) /* can't be -1 b/c fadt */ + +/*-************************************ +* Setup for Adding new params +**************************************/ + +/* indices for each of the variables */ +typedef enum { + wlog_ind = 0, + clog_ind = 1, + hlog_ind = 2, + slog_ind = 3, + slen_ind = 4, + tlen_ind = 5, + strt_ind = 6, + fadt_ind = 7, /* forceAttachDict */ + NUM_PARAMS = 8 +} varInds_t; + +/* maximum value of parameters */ +static const U32 mintable[NUM_PARAMS] = + { ZSTD_WINDOWLOG_MIN, ZSTD_CHAINLOG_MIN, ZSTD_HASHLOG_MIN, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLENGTH_MIN, ZSTD_TARGETLENGTH_MIN, ZSTD_fast, FADT_MIN }; + +/* minimum value of parameters */ +static const U32 maxtable[NUM_PARAMS] = + { ZSTD_WINDOWLOG_MAX, ZSTD_CHAINLOG_MAX, ZSTD_HASHLOG_MAX, ZSTD_SEARCHLOG_MAX, ZSTD_SEARCHLENGTH_MAX, ZSTD_TARGETLENGTH_MAX, ZSTD_btultra, FADT_MAX }; + +/* # of values parameters can take on */ +static const U32 rangetable[NUM_PARAMS] = + { WLOG_RANGE, CLOG_RANGE, HLOG_RANGE, SLOG_RANGE, SLEN_RANGE, TLEN_RANGE, STRT_RANGE, FADT_RANGE }; + +static const ZSTD_cParameter cctxSetParamTable[NUM_PARAMS] = + { ZSTD_p_windowLog, ZSTD_p_chainLog, ZSTD_p_hashLog, ZSTD_p_searchLog, ZSTD_p_minMatch, ZSTD_p_targetLength, ZSTD_p_compressionStrategy, ZSTD_p_forceAttachDict }; + +static const char* g_paramNames[NUM_PARAMS] = + { "windowLog", "chainLog", "hashLog","searchLog", "searchLength", "targetLength", "strategy", "forceAttachDict"}; + + static const U32 tlen_table[TLEN_RANGE] = { 0, 1, 2, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 256, 512, 999 }; +/* maps value from 0 to rangetable[param] - 1 to valid paramvalue */ +static U32 rangeMap(varInds_t param, U32 ind) { + ind = MIN(ind, rangetable[param] - 1); + switch(param) { + case tlen_ind: + return tlen_table[ind]; + case fadt_ind: /* 0, 1, 2 -> -1, 0, 1 */ + return ind - 1; + case wlog_ind: /* using default: triggers -Wswitch-enum */ + case clog_ind: + case hlog_ind: + case slog_ind: + case slen_ind: + case strt_ind: + return mintable[param] + ind; + case NUM_PARAMS: + return (U32)-1; + } + return 0; /* should never happen, stop compiler warnings */ +} + +/* inverse of rangeMap */ +static U32 invRangeMap(varInds_t param, U32 value) { + value = MIN(MAX(mintable[param], value), maxtable[param]); + switch(param) { + case tlen_ind: /* bin search */ + { + int lo = 0; + int hi = TLEN_RANGE; + while(lo < hi) { + int mid = (lo + hi) / 2; + if(tlen_table[mid] < value) { + lo = mid + 1; + } if(tlen_table[mid] == value) { + return mid; + } else { + hi = mid; + } + } + return lo; + } + case fadt_ind: + return value + 1; + case wlog_ind: + case clog_ind: + case hlog_ind: + case slog_ind: + case slen_ind: + case strt_ind: + return value - mintable[param]; + case NUM_PARAMS: + return (U32)-1; + } + return 0; /* should never happen, stop compiler warnings */ +} + +typedef struct { + U32 vals[NUM_PARAMS]; +} paramValues_t; + +//TODO: unset -> 0? +static ZSTD_compressionParameters pvalsToCParams(paramValues_t p) { + ZSTD_compressionParameters c; + c.windowLog = p.vals[wlog_ind]; + c.chainLog = p.vals[clog_ind]; + c.hashLog = p.vals[hlog_ind]; + c.searchLog = p.vals[slog_ind]; + c.searchLength = p.vals[slen_ind]; + c.targetLength = p.vals[tlen_ind]; + c.strategy = p.vals[strt_ind]; + /* no forceAttachDict */ + return c; +} + +/* 0 = auto for fadt */ +static paramValues_t cParamsToPVals(ZSTD_compressionParameters c) { + paramValues_t p; + p.vals[wlog_ind] = c.windowLog; + p.vals[clog_ind] = c.chainLog; + p.vals[hlog_ind] = c.hashLog; + p.vals[slog_ind] = c.searchLog; + p.vals[slen_ind] = c.searchLength; + p.vals[tlen_ind] = c.targetLength; + p.vals[strt_ind] = c.strategy; + p.vals[fadt_ind] = 0; + return p; +} + +/* equivalent of ZSTD_adjustCParams for paramValues_t */ +static paramValues_t adjustParams(paramValues_t p, size_t maxBlockSize, size_t dictSize) { + U32 fval = p.vals[fadt_ind]; + p = cParamsToPVals(ZSTD_adjustCParams(pvalsToCParams(p), maxBlockSize, dictSize)); + p.vals[fadt_ind] = fval; + return p; +} + /*-************************************ * Benchmark Parameters/Global Variables **************************************/ @@ -115,15 +236,20 @@ static U32 g_singleRun = 0; static U32 g_optimizer = 0; static U32 g_target = 0; static U32 g_noSeed = 0; -static ZSTD_compressionParameters g_params; /* Initialized at the beginning of main w/ emptyParams() function */ +static paramValues_t g_params; /* Initialized at the beginning of main w/ emptyParams() function */ static UTIL_time_t g_time; /* to be used to compare solution finding speeds to compare to original */ typedef struct { BMK_result_t result; - ZSTD_compressionParameters params; + paramValues_t params; } winnerInfo_t; +typedef struct { + BMK_result_t result; + ZSTD_compressionParameters params; +} oldWinnerInfo_t; + typedef struct { U32 cSpeed; /* bytes / sec */ U32 dSpeed; @@ -242,8 +368,9 @@ static void findClockGranularity(void) { DEBUGOUTPUT("Granularity: %llu\n", (unsigned long long)g_clockGranularity); } +/* allows zeros */ #define CLAMPCHECK(val,min,max) { \ - if (val && (((val)<(min)) | ((val)>(max)))) { \ + if (((val)<(min)) | ((val)>(max))) { \ DISPLAY("INVALID PARAMETER CONSTRAINTS\n"); \ return 0; \ } } @@ -251,36 +378,53 @@ static void findClockGranularity(void) { /* Like ZSTD_checkCParams() but allows 0's */ /* no check on targetLen? */ -static int cParamValid(ZSTD_compressionParameters paramTarget) { - CLAMPCHECK(paramTarget.hashLog, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX); - CLAMPCHECK(paramTarget.searchLog, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX); - CLAMPCHECK(paramTarget.searchLength, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX); - CLAMPCHECK(paramTarget.windowLog, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX); - CLAMPCHECK(paramTarget.chainLog, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX); - if(paramTarget.targetLength > ZSTD_TARGETLENGTH_MAX) { - DISPLAY("INVALID PARAMETER CONSTRAINTS\n"); - return 0; - } - if(paramTarget.strategy > ZSTD_btultra) { - DISPLAY("INVALID PARAMETER CONSTRAINTS\n"); - return 0; +static int paramValid(paramValues_t paramTarget) { + U32 i; + for(i = 0; i < NUM_PARAMS; i++) { + CLAMPCHECK(paramTarget.vals[i], mintable[i], maxtable[i]); } + //TODO: Strategy could be valid at 0 before, is that right? return 1; } -static void cParamZeroMin(ZSTD_compressionParameters* paramTarget) { - paramTarget->windowLog = paramTarget->windowLog ? paramTarget->windowLog : ZSTD_WINDOWLOG_MIN; - paramTarget->searchLog = paramTarget->searchLog ? paramTarget->searchLog : ZSTD_SEARCHLOG_MIN; - paramTarget->chainLog = paramTarget->chainLog ? paramTarget->chainLog : ZSTD_CHAINLOG_MIN; - paramTarget->hashLog = paramTarget->hashLog ? paramTarget->hashLog : ZSTD_HASHLOG_MIN; - paramTarget->searchLength = paramTarget->searchLength ? paramTarget->searchLength : ZSTD_SEARCHLENGTH_MIN; - paramTarget->targetLength = paramTarget->targetLength ? paramTarget->targetLength : 0; +//TODO: doesn't affect strategy? +static paramValues_t cParamUnsetMin(paramValues_t paramTarget) { + varInds_t i; + for(i = 0; i < NUM_PARAMS; i++) { + if(paramTarget.vals[i] == PARAM_UNSET) { + paramTarget.vals[i] = mintable[i]; + } + } + return paramTarget; } -static void BMK_translateAdvancedParams(const ZSTD_compressionParameters params) -{ - DISPLAY("--zstd=windowLog=%u,chainLog=%u,hashLog=%u,searchLog=%u,searchLength=%u,targetLength=%u,strategy=%u \n", - params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, params.targetLength, (U32)(params.strategy)); +static void BMK_translateAdvancedParams(FILE* f, const paramValues_t params) { + U32 i; + fprintf(f,"--zstd="); + for(i = 0; i < NUM_PARAMS; i++) { + fprintf(f,"%s", g_paramNames[i]); + fprintf(f,"=%u", params.vals[i]); + if(i != NUM_PARAMS - 1) { + fprintf(f, ","); + } + } + fprintf(f, "\n"); +} + + +static const char* g_stratName[ZSTD_btultra+1] = { + "(none) ", "ZSTD_fast ", "ZSTD_dfast ", + "ZSTD_greedy ", "ZSTD_lazy ", "ZSTD_lazy2 ", + "ZSTD_btlazy2 ", "ZSTD_btopt ", "ZSTD_btultra "}; + +static void BMK_displayOneResult(FILE* f, winnerInfo_t res, size_t srcSize) { + res.params = cParamUnsetMin(res.params); + fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u,%3d, %s}, ", + res.params.vals[wlog_ind], res.params.vals[clog_ind], res.params.vals[hlog_ind], res.params.vals[slog_ind], res.params.vals[slen_ind], + res.params.vals[tlen_ind], (int)res.params.vals[fadt_ind], g_stratName[res.params.vals[strt_ind]]); + fprintf(f, + " /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n", + (double)srcSize / res.result.cSize, (double)res.result.cSpeed / (1 MB), (double)res.result.dSpeed / (1 MB)); } /* checks results are feasible */ @@ -343,17 +487,16 @@ static constraint_t relaxTarget(constraint_t target) { * Bench functions *********************************************************/ -const char* g_stratName[ZSTD_btultra+1] = { - "(none) ", "ZSTD_fast ", "ZSTD_dfast ", - "ZSTD_greedy ", "ZSTD_lazy ", "ZSTD_lazy2 ", - "ZSTD_btlazy2 ", "ZSTD_btopt ", "ZSTD_btultra "}; - -static ZSTD_compressionParameters emptyParams(void) { - ZSTD_compressionParameters p = { 0, 0, 0, 0, 0, 0, (ZSTD_strategy)0 }; +static paramValues_t emptyParams(void) { + U32 i; + paramValues_t p; + for(i = 0; i < NUM_PARAMS; i++) { + p.vals[i] = PARAM_UNSET; + } return p; } -static winnerInfo_t initWinnerInfo(ZSTD_compressionParameters p) { +static winnerInfo_t initWinnerInfo(paramValues_t p) { winnerInfo_t w1; w1.result.cSpeed = 0.; w1.result.dSpeed = 0.; @@ -389,27 +532,16 @@ typedef struct { static void BMK_initCCtx(ZSTD_CCtx* ctx, const void* dictBuffer, const size_t dictBufferSize, const int cLevel, - const ZSTD_compressionParameters* comprParams, const BMK_advancedParams_t* adv) { + const paramValues_t* comprParams) { + varInds_t i; ZSTD_CCtx_reset(ctx); ZSTD_CCtx_resetParameters(ctx); - if (adv->nbWorkers==1) { - ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbWorkers, 0); - } else { - ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbWorkers, adv->nbWorkers); - } ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionLevel, cLevel); - ZSTD_CCtx_setParameter(ctx, ZSTD_p_enableLongDistanceMatching, adv->ldmFlag); - ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmMinMatch, adv->ldmMinMatch); - ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmHashLog, adv->ldmHashLog); - ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmBucketSizeLog, adv->ldmBucketSizeLog); - ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmHashEveryLog, adv->ldmHashEveryLog); - ZSTD_CCtx_setParameter(ctx, ZSTD_p_windowLog, comprParams->windowLog); - ZSTD_CCtx_setParameter(ctx, ZSTD_p_hashLog, comprParams->hashLog); - ZSTD_CCtx_setParameter(ctx, ZSTD_p_chainLog, comprParams->chainLog); - ZSTD_CCtx_setParameter(ctx, ZSTD_p_searchLog, comprParams->searchLog); - ZSTD_CCtx_setParameter(ctx, ZSTD_p_minMatch, comprParams->searchLength); - ZSTD_CCtx_setParameter(ctx, ZSTD_p_targetLength, comprParams->targetLength); - ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionStrategy, comprParams->strategy); + + for(i = 0; i < NUM_PARAMS; i++) { + if(comprParams->vals[i] != PARAM_UNSET) + ZSTD_CCtx_setParameter(ctx, cctxSetParamTable[i], comprParams->vals[i]); + } ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize); } @@ -424,13 +556,12 @@ typedef struct { const void* dictBuffer; size_t dictBufferSize; int cLevel; - const ZSTD_compressionParameters* comprParams; - const BMK_advancedParams_t* adv; + const paramValues_t* comprParams; } BMK_initCCtxArgs; static size_t local_initCCtx(void* payload) { const BMK_initCCtxArgs* ag = (const BMK_initCCtxArgs*)payload; - BMK_initCCtx(ag->ctx, ag->dictBuffer, ag->dictBufferSize, ag->cLevel, ag->comprParams, ag->adv); + BMK_initCCtx(ag->ctx, ag->dictBuffer, ag->dictBufferSize, ag->cLevel, ag->comprParams); return 0; } @@ -507,10 +638,7 @@ static size_t local_defaultDecompress( * From bench.c End *********************************************************/ -static void freeBuffers(const buffers_t b) { - if(b.srcPtrs != NULL) { - free(b.srcBuffer); - } +static void freeNonSrcBuffers(const buffers_t b) { free(b.srcPtrs); free(b.srcSizes); @@ -528,6 +656,13 @@ static void freeBuffers(const buffers_t b) { free(b.resSizes); } +static void freeBuffers(const buffers_t b) { + if(b.srcPtrs != NULL) { + free(b.srcBuffer); + } + freeNonSrcBuffers(b); +} + /* srcBuffer will be freed by freeBuffers now */ static int createBuffersFromMemory(buffers_t* buff, void * srcBuffer, size_t nbFiles, const size_t* fileSizes) @@ -559,7 +694,7 @@ static int createBuffersFromMemory(buffers_t* buff, void * srcBuffer, size_t nbF if(!buff->srcPtrs || !buff->srcSizes || !buff->dstPtrs || !buff->dstCapacities || !buff->dstSizes || !buff->resPtrs || !buff->resSizes) { DISPLAY("alloc error\n"); - freeBuffers(*buff); + freeNonSrcBuffers(*buff); return 1; } @@ -570,7 +705,7 @@ static int createBuffersFromMemory(buffers_t* buff, void * srcBuffer, size_t nbF if(!buff->dstPtrs[0] || !buff->resPtrs[0]) { DISPLAY("alloc error\n"); - freeBuffers(*buff); + freeNonSrcBuffers(*buff); return 1; } @@ -611,12 +746,20 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab size_t totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, (U32)nbFiles); size_t benchedSize = MIN(BMK_findMaxMem(totalSizeToLoad * 3) / 3, totalSizeToLoad); size_t* fileSizes = calloc(sizeof(size_t), nbFiles); - void* srcBuffer = malloc(benchedSize); + void* srcBuffer = NULL; int ret = 0; + if(!totalSizeToLoad || !benchedSize) { + ret = 1; + DISPLAY("Nothing to Bench\n"); + goto _cleanUp; + } + + srcBuffer = malloc(benchedSize); if(!fileSizes || !srcBuffer) { - return 1; + ret = 1; + goto _cleanUp; } for(n = 0; n < nbFiles; n++) { @@ -633,10 +776,9 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab f = fopen(fileNamesTable[n], "rb"); if (f==NULL) { DISPLAY("impossible to open file %s\n", fileNamesTable[n]); - free(fileSizes); - free(srcBuffer); fclose(f); - return 10; + ret = 10; + goto _cleanUp; } DISPLAY("Loading %s... \r", fileNamesTable[n]); @@ -645,21 +787,22 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab { char* buffer = (char*)(srcBuffer); size_t const readSize = fread((buffer)+pos, 1, (size_t)fileSize, f); - + fclose(f); if (readSize != (size_t)fileSize) { /* should we accept partial read? */ DISPLAY("could not read %s", fileNamesTable[n]); - free(fileSizes); - free(srcBuffer); - return 1; + ret = 1; + goto _cleanUp; } fileSizes[n] = readSize; pos += readSize; } - fclose(f); } ret = createBuffersFromMemory(buff, srcBuffer, nbFiles, fileSizes); + +_cleanUp: + if(ret) { free(srcBuffer); } free(fileSizes); return ret; } @@ -717,7 +860,7 @@ static int createContexts(contexts_t* ctx, const char* dictFileName) { /* if in decodeOnly, then srcPtr's will be compressed blocks, and uncompressedBlocks will be written to dstPtrs? */ /* dictionary nullable, nothing else though. */ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t ctx, - const int cLevel, const ZSTD_compressionParameters* comprParams, + const int cLevel, const paramValues_t* comprParams, const BMK_mode_t mode, const BMK_loopMode_t loopMode, const unsigned nbSeconds) { U32 i; @@ -736,11 +879,6 @@ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t ZSTD_CCtx* cctx = ctx.cctx; ZSTD_DCtx* dctx = ctx.dctx; - BMK_advancedParams_t adv = BMK_initAdvancedParams(); - adv.mode = mode; - adv.loopMode = loopMode; - adv.nbSeconds = nbSeconds; - /* warmimg up memory */ /* can't do this if decode only */ for(i = 0; i < buf.nbBlocks; i++) { @@ -762,7 +900,6 @@ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t cctxprep.dictBufferSize = dictBufferSize; cctxprep.cLevel = cLevel; cctxprep.comprParams = comprParams; - cctxprep.adv = &adv; dctxprep.dctx = dctx; dctxprep.dictBuffer = dictBuffer; dctxprep.dictBufferSize = dictBufferSize; @@ -852,13 +989,13 @@ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t } } /* Bench */ - results.result.cMem = (1 << (comprParams->windowLog)) + ZSTD_sizeof_CCtx(cctx); + results.result.cMem = (1 << (comprParams->vals[wlog_ind])) + ZSTD_sizeof_CCtx(cctx); return results; } static int BMK_benchParam(BMK_result_t* resultPtr, buffers_t buf, contexts_t ctx, - const ZSTD_compressionParameters cParams) { + const paramValues_t cParams) { BMK_return_t res = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_both, BMK_timeMode, 3); *resultPtr = res.result; return res.error; @@ -986,23 +1123,21 @@ static int insertWinner(winnerInfo_t w, constraint_t targetConstraints) { /* Writes to f the results of a parameter benchmark */ /* when used with --optimize, will only print results better than previously discovered */ -static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result, const ZSTD_compressionParameters params, const size_t srcSize) +static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result, const paramValues_t params, const size_t srcSize) { char lvlstr[15] = "Custom Level"; + winnerInfo_t w; + w.params = params; + w.result = result; fprintf(f, "\r%79s\r", ""); - fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ", - params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, - params.targetLength, g_stratName[(U32)(params.strategy)]); - if(cLevel != CUSTOM_LEVEL) { snprintf(lvlstr, 15, " Level %2u ", cLevel); } - fprintf(f, - "/* %s */ /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */", - lvlstr, (double)srcSize / result.cSize, (double)result.cSpeed / (1 MB), (double)result.dSpeed / (1 MB)); + fprintf(f, "/* %s */ ", lvlstr); + BMK_displayOneResult(f, w, srcSize); if(TIMED) { const U64 time = UTIL_clockSpanNano(g_time); @@ -1012,11 +1147,10 @@ static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result fprintf(f, "\n"); } -static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t result, const ZSTD_compressionParameters params, const constraint_t targetConstraints, const size_t srcSize) +static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t result, const paramValues_t params, const constraint_t targetConstraints, const size_t srcSize) { /* global winner used for constraints */ - static winnerInfo_t g_winner = { { 0, 0, (size_t)-1, (size_t)-1 } , { 0, 0, 0, 0, 0, 0, ZSTD_fast } }; - + static winnerInfo_t g_winner = { { (size_t)-1LL, 0, 0, (size_t)-1LL }, { { PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET } } }; if(DEBUG || compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { if(DEBUG && compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { DISPLAY("New Winner: \n"); @@ -1025,7 +1159,7 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res BMK_printWinner(f, cLevel, result, params, srcSize); if(compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { - BMK_translateAdvancedParams(params); + BMK_translateAdvancedParams(f, params); g_winner.result = result; g_winner.params = params; } @@ -1046,13 +1180,7 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res fprintf(f, "================================\n"); for(n = g_winners; n != NULL; n = n->next) { fprintf(f, "\r%79s\r", ""); - - fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ", - n->res.params.windowLog, n->res.params.chainLog, n->res.params.hashLog, n->res.params.searchLog, n->res.params.searchLength, - n->res.params.targetLength, g_stratName[(U32)(n->res.params.strategy)]); - fprintf(f, - " /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n", - (double)srcSize / n->res.result.cSize, (double)n->res.result.cSpeed / (1 MB), (double)n->res.result.dSpeed / (1 MB)); + BMK_displayOneResult(f, n->res, srcSize); } fprintf(f, "================================\n"); fprintf(f, "Level Bounds: R: > %.3f AND C: < %.1f MB/s \n\n", @@ -1060,27 +1188,15 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res fprintf(f, "Overall Winner: \n"); - fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ", - g_winner.params.windowLog, g_winner.params.chainLog, g_winner.params.hashLog, g_winner.params.searchLog, g_winner.params.searchLength, - g_winner.params.targetLength, g_stratName[(U32)(g_winner.params.strategy)]); - fprintf(f, - " /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n", - (double)srcSize / g_winner.result.cSize, (double)g_winner.result.cSpeed / (1 MB), (double)g_winner.result.dSpeed / (1 MB)); - - BMK_translateAdvancedParams(g_winner.params); - - fprintf(f, "Latest BMK: \n"); - fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ", - params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, - params.targetLength, g_stratName[(U32)(params.strategy)]); - fprintf(f, - " /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n", - (double)srcSize / result.cSize, (double)result.cSpeed / (1 MB), (double)result.dSpeed / (1 MB)); + BMK_displayOneResult(f, g_winner, srcSize); + BMK_translateAdvancedParams(f, g_winner.params); + fprintf(f, "Latest BMK: \n");\ + BMK_displayOneResult(f, w, srcSize); } } -static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSize) +static void BMK_printWinners2(FILE* f, const oldWinnerInfo_t* winners, size_t srcSize) { int cLevel; @@ -1088,11 +1204,11 @@ static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSi fprintf(f, " /* W, C, H, S, L, T, strat */ \n"); for (cLevel=0; cLevel <= NB_LEVELS_TRACKED; cLevel++) - BMK_printWinner(f, cLevel, winners[cLevel].result, winners[cLevel].params, srcSize); + BMK_printWinner(f, cLevel, winners[cLevel].result, cParamsToPVals(winners[cLevel].params), srcSize); } -static void BMK_printWinners(FILE* f, const winnerInfo_t* winners, size_t srcSize) +static void BMK_printWinners(FILE* f, const oldWinnerInfo_t* winners, size_t srcSize) { fseek(f, 0, SEEK_SET); BMK_printWinners2(f, winners, srcSize); @@ -1129,14 +1245,14 @@ static void BMK_init_level_constraints(int bytePerSec_level1) } } } -static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters params, +static int BMK_seed(oldWinnerInfo_t* winners, const ZSTD_compressionParameters params, buffers_t buf, contexts_t ctx) { BMK_result_t testResult; int better = 0; int cLevel; - BMK_benchParam(&testResult, buf, ctx, params); + BMK_benchParam(&testResult, buf, ctx, cParamsToPVals(params)); for (cLevel = 1; cLevel <= NB_LEVELS_TRACKED; cLevel++) { @@ -1152,7 +1268,7 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para /* first solution for this cLevel */ winners[cLevel].result = testResult; winners[cLevel].params = params; - BMK_printWinner(stdout, cLevel, testResult, params, buf.srcSize); + BMK_printWinner(stdout, cLevel, testResult, cParamsToPVals(params), buf.srcSize); better = 1; continue; } @@ -1217,7 +1333,7 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para winners[cLevel].result = testResult; winners[cLevel].params = params; - BMK_printWinner(stdout, cLevel, testResult, params, buf.srcSize); + BMK_printWinner(stdout, cLevel, testResult, cParamsToPVals(params), buf.srcSize); better = 1; } } @@ -1234,20 +1350,21 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para /* no point in windowLog < chainLog (no point 2x chainLog for bt) */ /* now with built in bounds-checking */ /* no longer does anything with sanitizeVarArray + clampcheck */ -static ZSTD_compressionParameters sanitizeParams(ZSTD_compressionParameters params) +static paramValues_t sanitizeParams(paramValues_t params) { - if (params.strategy == ZSTD_fast) - params.chainLog = 0, params.searchLog = 0; - if (params.strategy == ZSTD_dfast) - params.searchLog = 0; - if (params.strategy != ZSTD_btopt && params.strategy != ZSTD_btultra && params.strategy != ZSTD_fast) - params.targetLength = 0; + if (params.vals[strt_ind] == ZSTD_fast) + params.vals[clog_ind] = 0, params.vals[slog_ind] = 0; + if (params.vals[strt_ind] == ZSTD_dfast) + params.vals[slog_ind] = 0; + if (params.vals[strt_ind] != ZSTD_btopt && params.vals[strt_ind] != ZSTD_btultra && params.vals[strt_ind] != ZSTD_fast) + params.vals[tlen_ind] = 0; return params; } -/* new length */ +/* return: new length */ /* keep old array, will need if iter over strategy. */ +/* prunes useless params */ static int sanitizeVarArray(varInds_t* varNew, const int varLength, const varInds_t* varArray, const ZSTD_strategy strat) { int i, j = 0; for(i = 0; i < varLength; i++) { @@ -1264,80 +1381,31 @@ static int sanitizeVarArray(varInds_t* varNew, const int varLength, const varInd } /* res should be NUM_PARAMS size */ -/* constructs varArray from ZSTD_compressionParameters style parameter */ -static int variableParams(const ZSTD_compressionParameters paramConstraints, varInds_t* res) { +/* constructs varArray from paramValues_t style parameter */ +/* pass in using dict. */ +static int variableParams(const paramValues_t paramConstraints, varInds_t* res, const int usingDictionary) { + varInds_t i; int j = 0; - if(!paramConstraints.windowLog) { - res[j] = wlog_ind; - j++; - } - if(!paramConstraints.chainLog) { - res[j] = clog_ind; - j++; - } - if(!paramConstraints.hashLog) { - res[j] = hlog_ind; - j++; - } - if(!paramConstraints.searchLog) { - res[j] = slog_ind; - j++; - } - if(!paramConstraints.searchLength) { - res[j] = slen_ind; - j++; - } - if(!paramConstraints.targetLength) { - res[j] = tlen_ind; - j++; - } - if(!paramConstraints.strategy) { - res[j] = strt_ind; - j++; - } - return j; -} - -/* bin-search on tlen_table for correct index. */ -static int tlen_inv(U32 x) { - int lo = 0; - int hi = TLEN_RANGE; - while(lo < hi) { - int mid = (lo + hi) / 2; - if(tlen_table[mid] < x) { - lo = mid + 1; - } if(tlen_table[mid] == x) { - return mid; - } else { - hi = mid; + for(i = 0; i < NUM_PARAMS; i++) { + if(paramConstraints.vals[i] == PARAM_UNSET) { + if(i == fadt_ind && !usingDictionary) continue; /* don't use fadt if no dictionary */ + res[j] = i; j++; } } - return lo; + return j; } /* amt will probably always be \pm 1? */ /* slight change from old paramVariation, targetLength can only take on powers of 2 now (999 ~= 1024?) */ /* take max/min bounds into account as well? */ -static void paramVaryOnce(const varInds_t paramIndex, const int amt, ZSTD_compressionParameters* ptr) { - switch(paramIndex) - { - case wlog_ind: ptr->windowLog += amt; break; - case clog_ind: ptr->chainLog += amt; break; - case hlog_ind: ptr->hashLog += amt; break; - case slog_ind: ptr->searchLog += amt; break; - case slen_ind: ptr->searchLength += amt; break; - case tlen_ind: - ptr->targetLength = tlen_table[MAX(0, MIN(TLEN_RANGE - 1, tlen_inv(ptr->targetLength) + amt))]; - break; - case strt_ind: ptr->strategy += amt; break; - default: break; - } +static void paramVaryOnce(const varInds_t paramIndex, const int amt, paramValues_t* ptr) { + ptr->vals[paramIndex] = rangeMap(paramIndex, invRangeMap(paramIndex, ptr->vals[paramIndex]) + amt); //TODO: bounds check. } /* varies ptr by nbChanges respecting varyParams*/ -static void paramVariation(ZSTD_compressionParameters* ptr, const varInds_t* varyParams, const int varyLen, const U32 nbChanges) +static void paramVariation(paramValues_t* ptr, const varInds_t* varyParams, const int varyLen, const U32 nbChanges) { - ZSTD_compressionParameters p; + paramValues_t p; U32 validated = 0; while (!validated) { U32 i; @@ -1346,7 +1414,7 @@ static void paramVariation(ZSTD_compressionParameters* ptr, const varInds_t* var const U32 changeID = FUZ_rand(&g_rand) % (varyLen << 1); paramVaryOnce(varyParams[changeID >> 1], ((changeID & 1) << 1) - 1, &p); } - validated = !ZSTD_isError(ZSTD_checkCParams(p)) && p.strategy > 0; + validated = paramValid(p); } *ptr = p; } @@ -1364,48 +1432,25 @@ static size_t memoTableLen(const varInds_t* varyParams, const int varyLen) { } /* returns unique index in memotable of compression parameters */ -static unsigned memoTableInd(const ZSTD_compressionParameters* ptr, const varInds_t* varyParams, const int varyLen) { +static unsigned memoTableInd(const paramValues_t* ptr, const varInds_t* varyParams, const int varyLen) { int i; unsigned ind = 0; for(i = 0; i < varyLen; i++) { - switch(varyParams[i]) { - case wlog_ind: ind *= WLOG_RANGE; ind += ptr->windowLog - - ZSTD_WINDOWLOG_MIN ; break; - case clog_ind: ind *= CLOG_RANGE; ind += ptr->chainLog - - ZSTD_CHAINLOG_MIN ; break; - case hlog_ind: ind *= HLOG_RANGE; ind += ptr->hashLog - - ZSTD_HASHLOG_MIN ; break; - case slog_ind: ind *= SLOG_RANGE; ind += ptr->searchLog - - ZSTD_SEARCHLOG_MIN ; break; - case slen_ind: ind *= SLEN_RANGE; ind += ptr->searchLength - - ZSTD_SEARCHLENGTH_MIN; break; - case tlen_ind: ind *= TLEN_RANGE; ind += tlen_inv(ptr->targetLength) - - ZSTD_TARGETLENGTH_MIN; break; - case strt_ind: break; - } + varInds_t v = varyParams[i]; + if(v == strt_ind) continue; /* exclude strategy from memotable */ + ind *= rangetable[v]; ind += invRangeMap(v, ptr->vals[v]); } return ind; } /* inverse of above function (from index to parameters) */ -static void memoTableIndInv(ZSTD_compressionParameters* ptr, const varInds_t* varyParams, const int varyLen, size_t ind) { +static void memoTableIndInv(paramValues_t* ptr, const varInds_t* varyParams, const int varyLen, size_t ind) { int i; for(i = varyLen - 1; i >= 0; i--) { - switch(varyParams[i]) { - case wlog_ind: ptr->windowLog = ind % WLOG_RANGE + ZSTD_WINDOWLOG_MIN; - ind /= WLOG_RANGE; break; - case clog_ind: ptr->chainLog = ind % CLOG_RANGE + ZSTD_CHAINLOG_MIN; - ind /= CLOG_RANGE; break; - case hlog_ind: ptr->hashLog = ind % HLOG_RANGE + ZSTD_HASHLOG_MIN; - ind /= HLOG_RANGE; break; - case slog_ind: ptr->searchLog = ind % SLOG_RANGE + ZSTD_SEARCHLOG_MIN; - ind /= SLOG_RANGE; break; - case slen_ind: ptr->searchLength = ind % SLEN_RANGE + ZSTD_SEARCHLENGTH_MIN; - ind /= SLEN_RANGE; break; - case tlen_ind: ptr->targetLength = tlen_table[(ind % TLEN_RANGE)]; - ind /= TLEN_RANGE; break; - case strt_ind: break; - } + varInds_t v = varyParams[i]; + if(v == strt_ind) continue; + ptr->vals[v] = rangeMap(v, ind % rangetable[v]); + ind /= rangetable[v]; } } @@ -1414,36 +1459,36 @@ static void memoTableIndInv(ZSTD_compressionParameters* ptr, const varInds_t* va * redundant / obviously non-optimal parameter configurations (e.g. wlog - 1 larger) * than srcSize, clog > wlog, ... */ -static void initMemoTable(U8* memoTable, ZSTD_compressionParameters paramConstraints, const constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { +static void initMemoTable(U8* memoTable, paramValues_t paramConstraints, const constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { size_t i; size_t arrayLen = memoTableLen(varyParams, varyLen); - int cwFixed = !paramConstraints.chainLog || !paramConstraints.windowLog; - int scFixed = !paramConstraints.searchLog || !paramConstraints.chainLog; - int whFixed = !paramConstraints.windowLog || !paramConstraints.hashLog; - int wFixed = !paramConstraints.windowLog; + int cwFixed = paramConstraints.vals[clog_ind] == PARAM_UNSET || paramConstraints.vals[wlog_ind] == PARAM_UNSET; + int scFixed = paramConstraints.vals[slog_ind] == PARAM_UNSET || paramConstraints.vals[clog_ind] == PARAM_UNSET; + int whFixed = paramConstraints.vals[wlog_ind] == PARAM_UNSET || paramConstraints.vals[hlog_ind] == PARAM_UNSET; + int wFixed = paramConstraints.vals[wlog_ind] == PARAM_UNSET; int j = 0; assert(memoTable != NULL); memset(memoTable, 0, arrayLen); - cParamZeroMin(¶mConstraints); + paramConstraints = cParamUnsetMin(paramConstraints); for(i = 0; i < arrayLen; i++) { memoTableIndInv(¶mConstraints, varyParams, varyLen, i); - if(ZSTD_estimateCStreamSize_usingCParams(paramConstraints) > (size_t)target.cMem) { + if(ZSTD_estimateCStreamSize_usingCParams(pvalsToCParams(paramConstraints)) > (size_t)target.cMem) { memoTable[i] = 255; j++; } - if(wFixed && (1ULL << (paramConstraints.windowLog - 1)) > srcSize) { + if(wFixed && (1ULL << (paramConstraints.vals[wlog_ind] - 1)) >= srcSize && paramConstraints.vals[wlog_ind] != mintable[wlog_ind]) { memoTable[i] = 255; } /* nil out parameter sets equivalent to others. */ - if(cwFixed/* at most least 1 param fixed. */) { - if(paramConstraints.strategy == ZSTD_btlazy2 || paramConstraints.strategy == ZSTD_btopt || paramConstraints.strategy == ZSTD_btultra) { - if(paramConstraints.chainLog > paramConstraints.windowLog + 1) { + if(cwFixed) { + if(paramConstraints.vals[strt_ind] == ZSTD_btlazy2 || paramConstraints.vals[strt_ind] == ZSTD_btopt || paramConstraints.vals[strt_ind] == ZSTD_btultra) { + if(paramConstraints.vals[clog_ind] > paramConstraints.vals[wlog_ind]+ 1) { if(memoTable[i] != 255) { j++; } memoTable[i] = 255; } } else { - if(paramConstraints.chainLog > paramConstraints.windowLog) { + if(paramConstraints.vals[clog_ind] > paramConstraints.vals[wlog_ind]) { if(memoTable[i] != 255) { j++; } memoTable[i] = 255; } @@ -1451,14 +1496,14 @@ static void initMemoTable(U8* memoTable, ZSTD_compressionParameters paramConstra } if(scFixed) { - if(paramConstraints.searchLog > paramConstraints.chainLog) { + if(paramConstraints.vals[slog_ind] > paramConstraints.vals[clog_ind]) { if(memoTable[i] != 255) { j++; } memoTable[i] = 255; } } if(whFixed) { - if(paramConstraints.hashLog > paramConstraints.windowLog + 1) { + if(paramConstraints.vals[hlog_ind] > paramConstraints.vals[wlog_ind] + 1) { if(memoTable[i] != 255) { j++; } memoTable[i] = 255; } @@ -1466,7 +1511,7 @@ static void initMemoTable(U8* memoTable, ZSTD_compressionParameters paramConstra } DEBUGOUTPUT("%d / %d Invalid\n", j, (int)i); if((int)i == j) { - DEBUGOUTPUT("!!!Strategy %d totally infeasible\n", (int)paramConstraints.strategy) + DEBUGOUTPUT("!!!Strategy %d totally infeasible\n", (int)paramConstraints.vals[strt_ind]); } } @@ -1482,7 +1527,7 @@ static void freeMemoTableArray(U8** mtAll) { /* inits memotables for all (including mallocs), all strategies */ /* takes unsanitized varyParams */ -static U8** createMemoTableArray(ZSTD_compressionParameters paramConstraints, constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { +static U8** createMemoTableArray(paramValues_t paramConstraints, constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { varInds_t varNew[NUM_PARAMS]; U8** mtAll = (U8**)calloc(sizeof(U8*),(ZSTD_btultra + 1)); int i; @@ -1503,14 +1548,13 @@ static U8** createMemoTableArray(ZSTD_compressionParameters paramConstraints, co return mtAll; } -static ZSTD_compressionParameters overwriteParams(ZSTD_compressionParameters base, ZSTD_compressionParameters mask) { - base.windowLog = mask.windowLog ? mask.windowLog : base.windowLog; - base.chainLog = mask.chainLog ? mask.chainLog : base.chainLog; - base.hashLog = mask.hashLog ? mask.hashLog : base.hashLog; - base.searchLog = mask.searchLog ? mask.searchLog : base.searchLog; - base.searchLength = mask.searchLength ? mask.searchLength : base.searchLength; - base.targetLength = mask.targetLength ? mask.targetLength : base.targetLength; - base.strategy = mask.strategy ? mask.strategy : base.strategy; +static paramValues_t overwriteParams(paramValues_t base, paramValues_t mask) { + U32 i; + for(i = 0; i < NUM_PARAMS; i++) { + if(mask.vals[i] != PARAM_UNSET) { + base.vals[i] = mask.vals[i]; + } + } return base; } @@ -1524,38 +1568,41 @@ static BYTE g_alreadyTested[PARAMTABLESIZE] = {0}; /* init to zero */ g_alreadyTested[(XXH64(((void*)&sanitizeParams(p), sizeof(p), 0) >> 3) & PARAMTABLEMASK] */ static BYTE* NB_TESTS_PLAYED(ZSTD_compressionParameters p) { - ZSTD_compressionParameters p2 = sanitizeParams(p); + ZSTD_compressionParameters p2 = pvalsToCParams(sanitizeParams(cParamsToPVals(p))); return &g_alreadyTested[(XXH64((void*)&p2, sizeof(p2), 0) >> 3) & PARAMTABLEMASK]; } -static void playAround(FILE* f, winnerInfo_t* winners, +static void playAround(FILE* f, oldWinnerInfo_t* winners, ZSTD_compressionParameters params, buffers_t buf, contexts_t ctx) { int nbVariations = 0; UTIL_time_t const clockStart = UTIL_getTime(); - const U32 unconstrained[NUM_PARAMS] = { 0, 1, 2, 3, 4, 5, 6 }; + const U32 unconstrained[NUM_PARAMS] = { 0, 1, 2, 3, 4, 5, 6 }; /* no fadt */ while (UTIL_clockSpanMicro(clockStart) < g_maxVariationTime) { - ZSTD_compressionParameters p = params; + paramValues_t p = cParamsToPVals(params); + ZSTD_compressionParameters p2; BYTE* b; if (nbVariations++ > g_maxNbVariations) break; paramVariation(&p, unconstrained, NUM_PARAMS, 4); + p2 = pvalsToCParams(p); + /* exclude faster if already played params */ - if (FUZ_rand(&g_rand) & ((1 << *NB_TESTS_PLAYED(p))-1)) + if (FUZ_rand(&g_rand) & ((1 << *NB_TESTS_PLAYED(p2))-1)) continue; /* test */ - b = NB_TESTS_PLAYED(p); + b = NB_TESTS_PLAYED(p2); (*b)++; - if (!BMK_seed(winners, p, buf, ctx)) continue; + if (!BMK_seed(winners, p2, buf, ctx)) continue; /* improvement found => search more */ BMK_printWinners(f, winners, buf.srcSize); - playAround(f, winners, p, buf, ctx); + playAround(f, winners, p2, buf, ctx); } } @@ -1587,7 +1634,7 @@ static ZSTD_compressionParameters randomParams(void) } /* Sets pc to random unmeasured set of parameters */ -static void randomConstrainedParams(ZSTD_compressionParameters* pc, varInds_t* varArray, int varLen, U8* memoTable) +static void randomConstrainedParams(paramValues_t* pc, varInds_t* varArray, int varLen, U8* memoTable) { size_t tries = memoTableLen(varArray, varLen); const size_t maxSize = memoTableLen(varArray, varLen); @@ -1601,7 +1648,7 @@ static void randomConstrainedParams(ZSTD_compressionParameters* pc, varInds_t* v } static void BMK_selectRandomStart( - FILE* f, winnerInfo_t* winners, + FILE* f, oldWinnerInfo_t* winners, buffers_t buf, contexts_t ctx) { U32 const id = FUZ_rand(&g_rand) % (NB_LEVELS_TRACKED+1); @@ -1617,7 +1664,7 @@ static void BMK_selectRandomStart( static void BMK_benchFullTable(buffers_t buf, contexts_t ctx, const size_t maxBlockSize) { ZSTD_compressionParameters params; - winnerInfo_t winners[NB_LEVELS_TRACKED+1]; + oldWinnerInfo_t winners[NB_LEVELS_TRACKED+1]; const char* const rfName = "grillResults.txt"; FILE* const f = fopen(rfName, "w"); @@ -1632,7 +1679,7 @@ static void BMK_benchFullTable(buffers_t buf, contexts_t ctx, const size_t maxBl /* baseline config for level 1 */ ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, maxBlockSize, ctx.dictSize); BMK_result_t testResult; - BMK_benchParam(&testResult, buf, ctx, l1params); + BMK_benchParam(&testResult, buf, ctx, cParamsToPVals(l1params)); BMK_init_level_constraints((int)((testResult.cSpeed * 31) / 32)); } @@ -1751,7 +1798,7 @@ int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileNam DISPLAY("using %d Files : \n", nbFiles); } - g_params = ZSTD_adjustCParams(overwriteParams(ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize), g_params), maxBlockSize, ctx.dictSize); + g_params = adjustParams(overwriteParams(cParamsToPVals(ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize)), g_params), maxBlockSize, ctx.dictSize); if(g_singleRun) { ret = benchOnce(buf, ctx); @@ -1769,7 +1816,7 @@ int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileNam #define HIGH_VARIANCE 100.0 static int allBench(BMK_result_t* resultPtr, const buffers_t buf, const contexts_t ctx, - const ZSTD_compressionParameters cParams, + const paramValues_t cParams, const constraint_t target, BMK_result_t* winnerResult, int feas) { BMK_return_t benchres; @@ -1870,7 +1917,7 @@ static int allBench(BMK_result_t* resultPtr, /* Memoized benchmarking, won't benchmark anything which has already been benchmarked before. */ static int benchMemo(BMK_result_t* resultPtr, const buffers_t buf, const contexts_t ctx, - const ZSTD_compressionParameters cParams, + const paramValues_t cParams, const constraint_t target, BMK_result_t* winnerResult, U8* const memoTable, const varInds_t* varyParams, const int varyLen, const int feas) { @@ -1914,13 +1961,13 @@ static winnerInfo_t climbOnce(const constraint_t target, const varInds_t* varArray, const int varLen, ZSTD_strategy strat, U8** memoTableArray, buffers_t buf, contexts_t ctx, - const ZSTD_compressionParameters init) { + const paramValues_t init) { /* * cparam - currently considered 'center' * candidate - params to benchmark/results * winner - best option found so far. */ - ZSTD_compressionParameters cparam = init; + paramValues_t cparam = init; winnerInfo_t candidateInfo, winnerInfo; int better = 1; int feas = 0; @@ -1948,10 +1995,10 @@ static winnerInfo_t climbOnce(const constraint_t target, candidateInfo.params = cparam; paramVaryOnce(varArray[i], offset, &candidateInfo.params); - if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params)) && candidateInfo.params.strategy > 0) { + if(paramValid(candidateInfo.params)) { int res; - if(strat != candidateInfo.params.strategy) { /* maybe only try strategy switching after exhausting non-switching solutions? */ - strat = candidateInfo.params.strategy; + if(strat != candidateInfo.params.vals[strt_ind]) { /* maybe only try strategy switching after exhausting non-switching solutions? */ + strat = candidateInfo.params.vals[strt_ind]; varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat); } res = benchMemo(&candidateInfo.result, buf, ctx, @@ -1981,8 +2028,8 @@ static winnerInfo_t climbOnce(const constraint_t target, /* param error checking already done here */ paramVariation(&candidateInfo.params, varArray, varLen, dist); - if(strat != candidateInfo.params.strategy) { - strat = candidateInfo.params.strategy; + if(strat != candidateInfo.params.vals[strt_ind]) { + strat = candidateInfo.params.vals[strt_ind]; varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat); } @@ -2030,7 +2077,7 @@ static winnerInfo_t climbOnce(const constraint_t target, */ static winnerInfo_t optimizeFixedStrategy( const buffers_t buf, const contexts_t ctx, - const constraint_t target, ZSTD_compressionParameters paramTarget, + const constraint_t target, paramValues_t paramTarget, const ZSTD_strategy strat, const varInds_t* varArray, const int varLen, U8** memoTableArray, const int tries) { @@ -2038,14 +2085,14 @@ static winnerInfo_t optimizeFixedStrategy( varInds_t varNew[NUM_PARAMS]; int varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat); - ZSTD_compressionParameters init; + paramValues_t init; winnerInfo_t winnerInfo, candidateInfo; winnerInfo = initWinnerInfo(emptyParams()); /* so climb is given the right fixed strategy */ - paramTarget.strategy = strat; + paramTarget.vals[strt_ind] = strat; /* to pass ZSTD_checkCParams */ - cParamZeroMin(¶mTarget); + paramTarget = cParamUnsetMin(paramTarget); init = paramTarget; @@ -2112,25 +2159,23 @@ static int nextStrategy(const int currentStrategy, const int bestStrategy) { static int g_maxTries = 3; #define TRY_DECAY 1 -static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget, int cLevel) +static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, paramValues_t paramTarget, int cLevelOpt, int cLevelRun) { varInds_t varArray [NUM_PARAMS]; int ret = 0; - const int varLen = variableParams(paramTarget, varArray); + const int varLen = variableParams(paramTarget, varArray, dictFileName != NULL); winnerInfo_t winner = initWinnerInfo(emptyParams()); U8** allMT = NULL; - size_t k; - size_t maxBlockSize = 0; + paramValues_t paramBase = cParamUnsetMin(paramTarget); + size_t k, maxBlockSize = 0; contexts_t ctx; buffers_t buf; g_time = UTIL_getTime(); - /* Init */ - if(!cParamValid(paramTarget)) { + if(!paramValid(paramBase)) { return 1; } - /* load dictionary*/ if(createBuffers(&buf, fileNamesTable, nbFiles)) { DISPLAY("unable to load files\n"); return 1; @@ -2154,23 +2199,23 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } /* if strategy is fixed, only init that part of memotable */ - if(paramTarget.strategy) { + if(paramTarget.vals[strt_ind] != PARAM_UNSET) { varInds_t varNew[NUM_PARAMS]; - int varLenNew = sanitizeVarArray(varNew, varLen, varArray, paramTarget.strategy); + int varLenNew = sanitizeVarArray(varNew, varLen, varArray, paramTarget.vals[strt_ind]); allMT = (U8**)calloc(sizeof(U8*), (ZSTD_btultra + 1)); if(allMT == NULL) { ret = 57; goto _cleanUp; } - allMT[paramTarget.strategy] = malloc(sizeof(U8) * memoTableLen(varNew, varLenNew)); + allMT[paramTarget.vals[strt_ind]] = malloc(sizeof(U8) * memoTableLen(varNew, varLenNew)); - if(allMT[paramTarget.strategy] == NULL) { + if(allMT[paramTarget.vals[strt_ind]] == NULL) { ret = 58; goto _cleanUp; } - initMemoTable(allMT[paramTarget.strategy], paramTarget, target, varNew, varLenNew, maxBlockSize); + initMemoTable(allMT[paramTarget.vals[strt_ind]], paramTarget, target, varNew, varLenNew, maxBlockSize); } else { allMT = createMemoTableArray(paramTarget, target, varArray, varLen, maxBlockSize); } @@ -2180,7 +2225,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ ret = 2; goto _cleanUp; } - + /* default strictness = Maximum for */ if(g_strictness == DEFAULT_STRICTNESS) { if(g_optmode) { @@ -2199,7 +2244,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ /* use level'ing mode instead of normal target mode */ /* Should lvl be parameter-masked here? */ if(g_optmode) { - winner.params = ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize); + winner.params = cParamsToPVals(ZSTD_getCParams(cLevelOpt, maxBlockSize, ctx.dictSize)); if(BMK_benchParam(&winner.result, buf, ctx, winner.params)) { ret = 3; goto _cleanUp; @@ -2213,13 +2258,13 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ target.cSpeed = (U32)g_lvltarget.cSpeed; target.dSpeed = (U32)g_lvltarget.dSpeed; //See if this is reasonable. - BMK_printWinnerOpt(stdout, cLevel, winner.result, winner.params, target, buf.srcSize); + BMK_printWinnerOpt(stdout, cLevelOpt, winner.result, winner.params, target, buf.srcSize); } /* Don't want it to return anything worse than the best known result */ if(g_singleRun) { BMK_result_t res; - g_params = ZSTD_adjustCParams(maskParams(ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize), g_params), maxBlockSize, ctx.dictSize); + g_params = adjustParams(overwriteParams(cParamsToPVals(ZSTD_getCParams(cLevelRun, maxBlockSize, ctx.dictSize)), g_params), maxBlockSize, ctx.dictSize); if(BMK_benchParam(&res, buf, ctx, g_params)) { ret = 45; goto _cleanUp; @@ -2246,19 +2291,19 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ findClockGranularity(); { - ZSTD_compressionParameters CParams; + paramValues_t CParams; /* find best solution from default params */ { /* strategy selection */ const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); DEBUGOUTPUT("Strategy Selection\n"); - if(paramTarget.strategy == 0) { + if(paramTarget.vals[strt_ind] == PARAM_UNSET) { BMK_result_t candidate; int i; for (i=1; i<=maxSeeds; i++) { int ec; - CParams = overwriteParams(ZSTD_getCParams(i, maxBlockSize, ctx.dictSize), paramTarget); + CParams = overwriteParams(cParamsToPVals(ZSTD_getCParams(i, maxBlockSize, ctx.dictSize)), paramTarget); ec = BMK_benchParam(&candidate, buf, ctx, CParams); BMK_printWinnerOpt(stdout, i, candidate, CParams, target, buf.srcSize); @@ -2271,17 +2316,18 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ /* if the current params are too slow, just stop. */ if(target.cSpeed > candidate.cSpeed * 3 / 2) { break; } } + + BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winner.result, winner.params, target, buf.srcSize); + BMK_translateAdvancedParams(stdout, winner.params); } } - BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winner.result, winner.params, target, buf.srcSize); - BMK_translateAdvancedParams(winner.params); DEBUGOUTPUT("Real Opt\n"); /* start 'real' tests */ { - int bestStrategy = (int)winner.params.strategy; - if(paramTarget.strategy == 0) { - int st = (int)winner.params.strategy; + int bestStrategy = (int)winner.params.vals[strt_ind]; + if(paramTarget.vals[strt_ind] == PARAM_UNSET) { + int st = bestStrategy; int tries = g_maxTries; { @@ -2298,7 +2344,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ winnerInfo_t wc; DEBUGOUTPUT("StrategySwitch: %s\n", g_stratName[st]); - wc = optimizeFixedStrategy(buf, ctx, target, paramTarget, + wc = optimizeFixedStrategy(buf, ctx, target, paramBase, st, varArray, varLen, allMT, tries); if(compareResultLT(winner.result, wc.result, target, buf.srcSize)) { @@ -2312,7 +2358,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ CHECKTIMEGT(ret, 0, _cleanUp); } } else { - winner = optimizeFixedStrategy(buf, ctx, target, paramTarget, paramTarget.strategy, + winner = optimizeFixedStrategy(buf, ctx, target, paramBase, paramTarget.vals[strt_ind], varArray, varLen, allMT, g_maxTries); } @@ -2326,7 +2372,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } /* end summary */ BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winner.result, winner.params, target, buf.srcSize); - BMK_translateAdvancedParams(winner.params); + BMK_translateAdvancedParams(stdout, winner.params); DISPLAY("grillParams size - optimizer completed \n"); } @@ -2350,7 +2396,9 @@ static void errorOut(const char* msg) static unsigned readU32FromChar(const char** stringPtr) { const char errorMsg[] = "error: numeric value too large"; + unsigned sign = 1; unsigned result = 0; + if(**stringPtr == '-') { sign = (unsigned)-1; (*stringPtr)++; } while ((**stringPtr >='0') && (**stringPtr <='9')) { unsigned const max = (((unsigned)(-1)) / 10) - 1; if (result > max) errorOut(errorMsg); @@ -2368,7 +2416,7 @@ static unsigned readU32FromChar(const char** stringPtr) if (**stringPtr=='i') (*stringPtr)++; if (**stringPtr=='B') (*stringPtr)++; } - return result; + return result * sign; } static int usage(const char* exename) @@ -2408,13 +2456,14 @@ static int badusage(const char* exename) #define PARSE_SUB_ARGS(stringLong, stringShort, variable) { if (longCommandWArg(&argument, stringLong) || longCommandWArg(&argument, stringShort)) { variable = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } } #define PARSE_CPARAMS(variable) \ { \ - PARSE_SUB_ARGS("windowLog=", "wlog=", variable.windowLog); \ - PARSE_SUB_ARGS("chainLog=" , "clog=", variable.chainLog); \ - PARSE_SUB_ARGS("hashLog=", "hlog=", variable.hashLog); \ - PARSE_SUB_ARGS("searchLog=" , "slog=", variable.searchLog); \ - PARSE_SUB_ARGS("searchLength=", "slen=", variable.searchLength); \ - PARSE_SUB_ARGS("targetLength=" , "tlen=", variable.targetLength); \ - PARSE_SUB_ARGS("strategy=", "strat=", variable.strategy); \ + PARSE_SUB_ARGS("windowLog=", "wlog=", variable.vals[wlog_ind]); \ + PARSE_SUB_ARGS("chainLog=" , "clog=", variable.vals[clog_ind]); \ + PARSE_SUB_ARGS("hashLog=", "hlog=", variable.vals[hlog_ind]); \ + PARSE_SUB_ARGS("searchLog=" , "slog=", variable.vals[slog_ind]); \ + PARSE_SUB_ARGS("searchLength=", "slen=", variable.vals[slen_ind]); \ + PARSE_SUB_ARGS("targetLength=" , "tlen=", variable.vals[tlen_ind]); \ + PARSE_SUB_ARGS("strategy=", "strat=", variable.vals[strt_ind]); \ + PARSE_SUB_ARGS("forceAttachDict=", "fad=" , variable.vals[fadt_ind]); \ } int main(int argc, const char** argv) @@ -2426,12 +2475,11 @@ int main(int argc, const char** argv) const char* input_filename = NULL; const char* dictFileName = NULL; U32 main_pause = 0; - int cLevel = 0; + int cLevelOpt = 0, cLevelRun = 0; int seperateFiles = 0; - constraint_t target = { 0, 0, (U32)-1 }; - ZSTD_compressionParameters paramTarget = emptyParams(); + paramValues_t paramTarget = emptyParams(); g_params = emptyParams(); assert(argc>=1); /* for exename */ @@ -2441,9 +2489,7 @@ int main(int argc, const char** argv) for(i=1; i