From 8c918edd3a87a006200f704b9119434c5f3250f3 Mon Sep 17 00:00:00 2001 From: George Lu Date: Fri, 10 Aug 2018 16:14:12 -0700 Subject: [PATCH] MAke it easier to add params Make memoTable size limited --- tests/paramgrill.c | 499 ++++++++++++++++++++++++--------------------- 1 file changed, 272 insertions(+), 227 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 4324678a..c2979cb8 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -86,7 +86,15 @@ static const int g_maxNbVariations = 64; #define CHECKTIME(r) { if(BMK_timeSpan(g_time) > g_timeLimit_s) { DEBUGOUTPUT("Time Limit Reached\n"); return r; } } #define CHECKTIMEGT(ret, val, _gototag) {if(BMK_timeSpan(g_time) > g_timeLimit_s) { DEBUGOUTPUT("Time Limit Reached\n"); ret = val; goto _gototag; } } -#define PARAM_UNSET ((U32)-2) /* can't be -1 b/c fadt */ +#define PARAM_UNSET ((U32)-2) /* can't be -1 b/c fadt uses -1 */ + +static const char* g_stratName[ZSTD_btultra+1] = { + "(none) ", "ZSTD_fast ", "ZSTD_dfast ", + "ZSTD_greedy ", "ZSTD_lazy ", "ZSTD_lazy2 ", + "ZSTD_btlazy2 ", "ZSTD_btopt ", "ZSTD_btultra "}; + + +static const U32 tlen_table[TLEN_RANGE] = { 0, 1, 2, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 256, 512, 999 }; /*-************************************ * Setup for Adding new params @@ -105,6 +113,14 @@ typedef enum { NUM_PARAMS = 8 } varInds_t; +typedef struct { + U32 vals[NUM_PARAMS]; +} paramValues_t; + +/* list of parameters */ +static const varInds_t paramTable[NUM_PARAMS] = + { wlog_ind, clog_ind, hlog_ind, slog_ind, slen_ind, tlen_ind, strt_ind, fadt_ind }; + /* maximum value of parameters */ static const U32 mintable[NUM_PARAMS] = { ZSTD_WINDOWLOG_MIN, ZSTD_CHAINLOG_MIN, ZSTD_HASHLOG_MIN, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLENGTH_MIN, ZSTD_TARGETLENGTH_MIN, ZSTD_fast, FADT_MIN }; @@ -117,15 +133,19 @@ static const U32 maxtable[NUM_PARAMS] = static const U32 rangetable[NUM_PARAMS] = { WLOG_RANGE, CLOG_RANGE, HLOG_RANGE, SLOG_RANGE, SLEN_RANGE, TLEN_RANGE, STRT_RANGE, FADT_RANGE }; +/* ZSTD_cctxSetParameter() index to set */ static const ZSTD_cParameter cctxSetParamTable[NUM_PARAMS] = { ZSTD_p_windowLog, ZSTD_p_chainLog, ZSTD_p_hashLog, ZSTD_p_searchLog, ZSTD_p_minMatch, ZSTD_p_targetLength, ZSTD_p_compressionStrategy, ZSTD_p_forceAttachDict }; +/* names of parameters */ static const char* g_paramNames[NUM_PARAMS] = - { "windowLog", "chainLog", "hashLog","searchLog", "searchLength", "targetLength", "strategy", "forceAttachDict"}; + { "windowLog", "chainLog", "hashLog","searchLog", "searchLength", "targetLength", "strategy", "forceAttachDict" }; +/* shortened names of parameters */ +static const char* g_shortParamNames[NUM_PARAMS] = + { "wlog", "clog", "hlog","slog", "slen", "tlen", "strt", "fadt" }; -static const U32 tlen_table[TLEN_RANGE] = { 0, 1, 2, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 256, 512, 999 }; -/* maps value from 0 to rangetable[param] - 1 to valid paramvalue */ +/* maps value from { 0 to rangetable[param] - 1 } to valid paramvalues */ static U32 rangeMap(varInds_t param, U32 ind) { ind = MIN(ind, rangetable[param] - 1); switch(param) { @@ -141,6 +161,7 @@ static U32 rangeMap(varInds_t param, U32 ind) { case strt_ind: return mintable[param] + ind; case NUM_PARAMS: + DISPLAY("Error, not a valid param\n "); return (U32)-1; } return 0; /* should never happen, stop compiler warnings */ @@ -176,49 +197,26 @@ static U32 invRangeMap(varInds_t param, U32 value) { case strt_ind: return value - mintable[param]; case NUM_PARAMS: + DISPLAY("Error, not a valid param\n "); return (U32)-1; } return 0; /* should never happen, stop compiler warnings */ } -typedef struct { - U32 vals[NUM_PARAMS]; -} paramValues_t; - -static ZSTD_compressionParameters pvalsToCParams(paramValues_t p) { - ZSTD_compressionParameters c; - memset(&c, 0, sizeof(ZSTD_compressionParameters)); - c.windowLog = p.vals[wlog_ind]; - c.chainLog = p.vals[clog_ind]; - c.hashLog = p.vals[hlog_ind]; - c.searchLog = p.vals[slog_ind]; - c.searchLength = p.vals[slen_ind]; - c.targetLength = p.vals[tlen_ind]; - c.strategy = p.vals[strt_ind]; - /* no forceAttachDict */ - return c; -} - -/* 0 = auto for fadt */ -static paramValues_t cParamsToPVals(ZSTD_compressionParameters c) { - paramValues_t p; - p.vals[wlog_ind] = c.windowLog; - p.vals[clog_ind] = c.chainLog; - p.vals[hlog_ind] = c.hashLog; - p.vals[slog_ind] = c.searchLog; - p.vals[slen_ind] = c.searchLength; - p.vals[tlen_ind] = c.targetLength; - p.vals[strt_ind] = c.strategy; - p.vals[fadt_ind] = 0; - return p; -} - -/* equivalent of ZSTD_adjustCParams for paramValues_t */ -static paramValues_t adjustParams(paramValues_t p, size_t maxBlockSize, size_t dictSize) { - U32 fval = p.vals[fadt_ind]; - p = cParamsToPVals(ZSTD_adjustCParams(pvalsToCParams(p), maxBlockSize, dictSize)); - p.vals[fadt_ind] = fval; - return p; +/* display of params */ +static void displayParamVal(FILE* f, varInds_t param, U32 value, int width) { + switch(param) { + case fadt_ind: if(width) { fprintf(f, "%*d", width, (int)value); } else { fprintf(f, "%d", (int)value); } break; + case strt_ind: if(width) { fprintf(f, "%*s", width, g_stratName[value]); } else { fprintf(f, "%s", g_stratName[value]); } break; + case wlog_ind: + case clog_ind: + case hlog_ind: + case slog_ind: + case slen_ind: + case tlen_ind: if(width) { fprintf(f, "%*u", width, value); } else { fprintf(f, "%u", value); } break; + case NUM_PARAMS: + DISPLAY("Error, not a valid param\n "); break; + } } /*-************************************ @@ -238,7 +236,7 @@ static U32 g_target = 0; static U32 g_noSeed = 0; static paramValues_t g_params; /* Initialized at the beginning of main w/ emptyParams() function */ static UTIL_time_t g_time; /* to be used to compare solution finding speeds to compare to original */ - +static U32 g_memoLimit = (U32)-1; //32 MB typedef struct { BMK_result_t result; @@ -293,6 +291,51 @@ void BMK_SetNbIterations(int nbLoops) * Private functions *********************************************************/ +static ZSTD_compressionParameters pvalsToCParams(paramValues_t p) { + ZSTD_compressionParameters c; + memset(&c, 0, sizeof(ZSTD_compressionParameters)); + c.windowLog = p.vals[wlog_ind]; + c.chainLog = p.vals[clog_ind]; + c.hashLog = p.vals[hlog_ind]; + c.searchLog = p.vals[slog_ind]; + c.searchLength = p.vals[slen_ind]; + c.targetLength = p.vals[tlen_ind]; + c.strategy = p.vals[strt_ind]; + /* no forceAttachDict */ + return c; +} + +static paramValues_t cParamsToPVals(ZSTD_compressionParameters c) { + paramValues_t p; + varInds_t i; + p.vals[wlog_ind] = c.windowLog; + p.vals[clog_ind] = c.chainLog; + p.vals[hlog_ind] = c.hashLog; + p.vals[slog_ind] = c.searchLog; + p.vals[slen_ind] = c.searchLength; + p.vals[tlen_ind] = c.targetLength; + p.vals[strt_ind] = c.strategy; + + /* set all other params to their minimum value */ + for(i = strt_ind + 1; i < NUM_PARAMS; i++) { + p.vals[i] = mintable[i]; + } + return p; +} + +/* equivalent of ZSTD_adjustCParams for paramValues_t */ +static paramValues_t adjustParams(paramValues_t p, size_t maxBlockSize, size_t dictSize) { + paramValues_t ot = p; + varInds_t i; + p = cParamsToPVals(ZSTD_adjustCParams(pvalsToCParams(p), maxBlockSize, dictSize)); + + /* retain value of all other parameters */ + for(i = strt_ind + 1; i < NUM_PARAMS; i++) { + p.vals[i] = ot.vals[i]; + } + return p; +} + /* accuracy in seconds only, span can be multiple years */ static U32 BMK_timeSpan(UTIL_time_t tStart) { return (U32)(UTIL_clockSpanMicro(tStart) / 1000000ULL); } @@ -375,9 +418,6 @@ static void findClockGranularity(void) { return 0; \ } } - -/* Like ZSTD_checkCParams() but allows 0's */ -/* no check on targetLen? */ static int paramValid(paramValues_t paramTarget) { U32 i; for(i = 0; i < NUM_PARAMS; i++) { @@ -400,8 +440,11 @@ static void BMK_translateAdvancedParams(FILE* f, const paramValues_t params) { U32 i; fprintf(f,"--zstd="); for(i = 0; i < NUM_PARAMS; i++) { - fprintf(f,"%s", g_paramNames[i]); - fprintf(f,"=%u", params.vals[i]); + fprintf(f,"%s=", g_paramNames[i]); + + if(i == strt_ind) { fprintf(f,"%u", params.vals[i]); } + else { displayParamVal(f, i, params.vals[i], 0); } + if(i != NUM_PARAMS - 1) { fprintf(f, ","); } @@ -409,19 +452,16 @@ static void BMK_translateAdvancedParams(FILE* f, const paramValues_t params) { fprintf(f, "\n"); } - -static const char* g_stratName[ZSTD_btultra+1] = { - "(none) ", "ZSTD_fast ", "ZSTD_dfast ", - "ZSTD_greedy ", "ZSTD_lazy ", "ZSTD_lazy2 ", - "ZSTD_btlazy2 ", "ZSTD_btopt ", "ZSTD_btultra "}; - static void BMK_displayOneResult(FILE* f, winnerInfo_t res, size_t srcSize) { + varInds_t v; res.params = cParamUnsetMin(res.params); - fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u,%3d, %s}, ", - res.params.vals[wlog_ind], res.params.vals[clog_ind], res.params.vals[hlog_ind], res.params.vals[slog_ind], res.params.vals[slen_ind], - res.params.vals[tlen_ind], (int)res.params.vals[fadt_ind], g_stratName[res.params.vals[strt_ind]]); - fprintf(f, - " /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n", + fprintf(f," {"); + for(v = 0; v < NUM_PARAMS; v++) { + if(v != 0) { fprintf(f, ","); } + displayParamVal(f, v, res.params.vals[v], 3); + } + + fprintf(f, "}, /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n", (double)srcSize / res.result.cSize, (double)res.result.cSpeed / (1 MB), (double)res.result.dSpeed / (1 MB)); } @@ -494,7 +534,7 @@ static paramValues_t emptyParams(void) { return p; } -static winnerInfo_t initWinnerInfo(paramValues_t p) { +static winnerInfo_t initWinnerInfo(const paramValues_t p) { winnerInfo_t w1; w1.result.cSpeed = 0.; w1.result.dSpeed = 0.; @@ -515,6 +555,7 @@ typedef struct { void** resPtrs; size_t* resSizes; size_t nbBlocks; + size_t maxBlockSize; } buffers_t; typedef struct { @@ -528,27 +569,6 @@ typedef struct { * From bench.c *********************************************************/ -static void BMK_initCCtx(ZSTD_CCtx* ctx, - const void* dictBuffer, const size_t dictBufferSize, const int cLevel, - const paramValues_t* comprParams) { - varInds_t i; - ZSTD_CCtx_reset(ctx); - ZSTD_CCtx_resetParameters(ctx); - ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionLevel, cLevel); - - for(i = 0; i < NUM_PARAMS; i++) { - if(comprParams->vals[i] != PARAM_UNSET) - ZSTD_CCtx_setParameter(ctx, cctxSetParamTable[i], comprParams->vals[i]); - } - ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize); -} - -static void BMK_initDCtx(ZSTD_DCtx* dctx, - const void* dictBuffer, const size_t dictBufferSize) { - ZSTD_DCtx_reset(dctx); - ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictBufferSize); -} - typedef struct { ZSTD_CCtx* ctx; const void* dictBuffer; @@ -559,7 +579,17 @@ typedef struct { static size_t local_initCCtx(void* payload) { const BMK_initCCtxArgs* ag = (const BMK_initCCtxArgs*)payload; - BMK_initCCtx(ag->ctx, ag->dictBuffer, ag->dictBufferSize, ag->cLevel, ag->comprParams); + varInds_t i; + ZSTD_CCtx_reset(ag->ctx); + ZSTD_CCtx_resetParameters(ag->ctx); + ZSTD_CCtx_setParameter(ag->ctx, ZSTD_p_compressionLevel, ag->cLevel); + + for(i = 0; i < NUM_PARAMS; i++) { + if(ag->comprParams->vals[i] != PARAM_UNSET) + ZSTD_CCtx_setParameter(ag->ctx, cctxSetParamTable[i], ag->comprParams->vals[i]); + } + ZSTD_CCtx_loadDictionary(ag->ctx, ag->dictBuffer, ag->dictBufferSize); + return 0; } @@ -571,7 +601,8 @@ typedef struct { static size_t local_initDCtx(void* payload) { const BMK_initDCtxArgs* ag = (const BMK_initDCtxArgs*)payload; - BMK_initDCtx(ag->dctx, ag->dictBuffer, ag->dictBufferSize); + ZSTD_DCtx_reset(ag->dctx); + ZSTD_DCtx_loadDictionary(ag->dctx, ag->dictBuffer, ag->dictBufferSize); return 0; } @@ -636,6 +667,72 @@ static size_t local_defaultDecompress( * From bench.c End *********************************************************/ +static void optimizerAdjustInput(paramValues_t* pc, const size_t maxBlockSize) { + varInds_t v; + for(v = 0; v < NUM_PARAMS; v++) { + if(pc->vals[v] != PARAM_UNSET) { + U32 newval = MIN(MAX(pc->vals[v], mintable[v]), maxtable[v]); + if(newval != pc->vals[v]) { + pc->vals[v] = newval; + DISPLAY("Warning: parameter %s not in valid range, adjusting to ", g_paramNames[v]); displayParamVal(stderr, v, newval, 0); DISPLAY("\n"); + } + } + } + + if(pc->vals[wlog_ind] != PARAM_UNSET) { + + U32 sshb = maxBlockSize > 1 ? ZSTD_highbit32((U32)(maxBlockSize-1)) + 1 : 1; + /* edge case of highBit not working for 0 */ + + if(maxBlockSize < (1ULL << 31) && sshb + 1 < pc->vals[wlog_ind]) { + U32 adjust = MAX(mintable[wlog_ind], sshb); + if(adjust != pc->vals[wlog_ind]) { + pc->vals[wlog_ind] = adjust; + DISPLAY("Warning: windowLog larger than src/block size, adjusted to %u\n", pc->vals[wlog_ind]); + } + } + } + + if(pc->vals[wlog_ind] != PARAM_UNSET && pc->vals[clog_ind] != PARAM_UNSET) { + U32 maxclog; + if(pc->vals[strt_ind] == PARAM_UNSET || pc->vals[strt_ind] >= (U32)ZSTD_btlazy2) { + maxclog = pc->vals[wlog_ind] + 1; + } else { + maxclog = pc->vals[wlog_ind]; + } + + if(pc->vals[clog_ind] > maxclog) { + pc->vals[clog_ind] = maxclog; + DISPLAY("Warning: chainlog too much larger than windowLog size, adjusted to %u\n", pc->vals[clog_ind]); + } + } + + if(pc->vals[wlog_ind] != PARAM_UNSET && pc->vals[hlog_ind] != PARAM_UNSET) { + if(pc->vals[wlog_ind] + 1 < pc->vals[hlog_ind]) { + pc->vals[hlog_ind] = pc->vals[wlog_ind] + 1; + DISPLAY("Warning: hashlog too much larger than windowLog size, adjusted to %u\n", pc->vals[hlog_ind]); + } + } + + if(pc->vals[slog_ind] != PARAM_UNSET && pc->vals[clog_ind] != PARAM_UNSET) { + if(pc->vals[slog_ind] > pc->vals[clog_ind]) { + pc->vals[clog_ind] = pc->vals[slog_ind]; + DISPLAY("Warning: searchLog larger than chainLog, adjusted to %u\n", pc->vals[slog_ind]); + } + } +} + +/* what about low something like clog vs hlog in lvl 1? */ +static int redundantParams(const paramValues_t paramValues, const constraint_t target, const size_t srcSize) { + return + (ZSTD_estimateCStreamSize_usingCParams(pvalsToCParams(paramValues)) > (size_t)target.cMem) /* Uses too much memory */ + || ((1ULL << (paramValues.vals[wlog_ind] - 1)) >= srcSize && paramValues.vals[wlog_ind] != mintable[wlog_ind]) /* wlog too much bigger than src size */ + || (paramValues.vals[clog_ind] > (paramValues.vals[wlog_ind] + (paramValues.vals[strt_ind] > ZSTD_btlazy2))) /* chainLog larger than windowLog*/ + || (paramValues.vals[slog_ind] > paramValues.vals[clog_ind]) /* searchLog larger than chainLog */ + || (paramValues.vals[hlog_ind] > paramValues.vals[wlog_ind] + 1); /* hashLog larger than windowLog + 1 */ + +} + static void freeNonSrcBuffers(const buffers_t b) { free(b.srcPtrs); free(b.srcSizes); @@ -722,6 +819,7 @@ static int createBuffersFromMemory(buffers_t* buff, void * srcBuffer, size_t nbF buff->dstCapacities[0] = ZSTD_compressBound(buff->srcSizes[0]); buff->dstSizes[0] = buff->dstCapacities[0]; buff->resSizes[0] = buff->srcSizes[0]; + buff->maxBlockSize = buff->srcSizes[0]; for(n = 1; n < blockNb; n++) { buff->dstPtrs[n] = ((char*)buff->dstPtrs[n-1]) + buff->dstCapacities[n-1]; @@ -729,6 +827,8 @@ static int createBuffersFromMemory(buffers_t* buff, void * srcBuffer, size_t nbF buff->dstCapacities[n] = ZSTD_compressBound(buff->srcSizes[n]); buff->dstSizes[n] = buff->dstCapacities[n]; buff->resSizes[n] = buff->srcSizes[n]; + + buff->maxBlockSize = MAX(buff->maxBlockSize, buff->srcSizes[n]); } buff->nbBlocks = blockNb; @@ -786,7 +886,7 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab char* buffer = (char*)(srcBuffer); size_t const readSize = fread((buffer)+pos, 1, (size_t)fileSize, f); fclose(f); - if (readSize != (size_t)fileSize) { /* should we accept partial read? */ + if (readSize != (size_t)fileSize) { DISPLAY("could not read %s", fileNamesTable[n]); ret = 1; goto _cleanUp; @@ -992,7 +1092,7 @@ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t } static int BMK_benchParam(BMK_result_t* resultPtr, - buffers_t buf, contexts_t ctx, + const buffers_t buf, const contexts_t ctx, const paramValues_t cParams) { BMK_return_t res = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_both, BMK_timeMode, 3); *resultPtr = res.result; @@ -1009,7 +1109,7 @@ static int BMK_benchParam(BMK_result_t* resultPtr, #define SPEED_RESULT 4 #define SIZE_RESULT 5 /* maybe have epsilon-eq to limit table size? */ -static int speedSizeCompare(BMK_result_t r1, BMK_result_t r2) { +static int speedSizeCompare(const BMK_result_t r1, const BMK_result_t r2) { if(r1.cSpeed < r2.cSpeed) { if(r1.cSize >= r2.cSize) { return BETTER_RESULT; @@ -1025,7 +1125,7 @@ static int speedSizeCompare(BMK_result_t r1, BMK_result_t r2) { /* 0 for insertion, 1 for no insert */ /* maintain invariant speedSizeCompare(n, n->next) = SPEED_RESULT */ -static int insertWinner(winnerInfo_t w, constraint_t targetConstraints) { +static int insertWinner(const winnerInfo_t w, const constraint_t targetConstraints) { BMK_result_t r = w.result; winner_ll_node* cur_node = g_winners; /* first node to insert */ @@ -1194,7 +1294,7 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res } } -static void BMK_printWinners2(FILE* f, const oldWinnerInfo_t* winners, size_t srcSize) +static void BMK_printWinners2(FILE* f, const oldWinnerInfo_t* winners, const size_t srcSize) { int cLevel; @@ -1206,7 +1306,7 @@ static void BMK_printWinners2(FILE* f, const oldWinnerInfo_t* winners, size_t sr } -static void BMK_printWinners(FILE* f, const oldWinnerInfo_t* winners, size_t srcSize) +static void BMK_printWinners(FILE* f, const oldWinnerInfo_t* winners, const size_t srcSize) { fseek(f, 0, SEEK_SET); BMK_printWinners2(f, winners, srcSize); @@ -1244,7 +1344,7 @@ static void BMK_init_level_constraints(int bytePerSec_level1) } static int BMK_seed(oldWinnerInfo_t* winners, const ZSTD_compressionParameters params, - buffers_t buf, contexts_t ctx) + const buffers_t buf, const contexts_t ctx) { BMK_result_t testResult; int better = 0; @@ -1422,9 +1522,8 @@ static size_t memoTableLen(const varInds_t* varyParams, const int varyLen) { size_t arrayLen = 1; int i; for(i = 0; i < varyLen; i++) { - if(varyParams[i] != strt_ind) { - arrayLen *= rangetable[varyParams[i]]; - } + if(varyParams[i] == strt_ind) continue; /* strategy separated by table */ + arrayLen *= rangetable[varyParams[i]]; } return arrayLen; } @@ -1452,13 +1551,13 @@ static void memoTableIndInv(paramValues_t* ptr, const varInds_t* varyParams, con ind /= rangetable[v]; */ switch(varyParams[i]) { - case wlog_ind: ptr->vals[wlog_ind] = ind % rangetable[wlog_ind] + mintable[wlog_ind]; ind /= rangetable[wlog_ind]; break; - case clog_ind: ptr->vals[clog_ind] = ind % rangetable[clog_ind] + mintable[clog_ind]; ind /= rangetable[clog_ind]; break; - case hlog_ind: ptr->vals[hlog_ind] = ind % rangetable[hlog_ind] + mintable[hlog_ind]; ind /= rangetable[hlog_ind]; break; - case slog_ind: ptr->vals[slog_ind] = ind % rangetable[slog_ind] + mintable[slog_ind]; ind /= rangetable[slog_ind]; break; - case slen_ind: ptr->vals[slen_ind] = ind % rangetable[slen_ind] + mintable[slen_ind]; ind /= rangetable[slen_ind]; break; - case tlen_ind: ptr->vals[tlen_ind] = tlen_table[(ind % rangetable[tlen_ind])]; ind /= rangetable[tlen_ind]; break; - case fadt_ind: ptr->vals[fadt_ind] = ind % rangetable[fadt_ind] - 1; ind /= rangetable[fadt_ind]; break; + case wlog_ind: ptr->vals[wlog_ind] = ind % rangetable[wlog_ind] + mintable[wlog_ind]; ind /= rangetable[wlog_ind]; break; + case clog_ind: ptr->vals[clog_ind] = ind % rangetable[clog_ind] + mintable[clog_ind]; ind /= rangetable[clog_ind]; break; + case hlog_ind: ptr->vals[hlog_ind] = ind % rangetable[hlog_ind] + mintable[hlog_ind]; ind /= rangetable[hlog_ind]; break; + case slog_ind: ptr->vals[slog_ind] = ind % rangetable[slog_ind] + mintable[slog_ind]; ind /= rangetable[slog_ind]; break; + case slen_ind: ptr->vals[slen_ind] = ind % rangetable[slen_ind] + mintable[slen_ind]; ind /= rangetable[slen_ind]; break; + case tlen_ind: ptr->vals[tlen_ind] = tlen_table[(ind % rangetable[tlen_ind])]; ind /= rangetable[tlen_ind]; break; + case fadt_ind: ptr->vals[fadt_ind] = ind % rangetable[fadt_ind] - 1; ind /= rangetable[fadt_ind]; break; case strt_ind: case NUM_PARAMS: break; } @@ -1473,58 +1572,19 @@ static void memoTableIndInv(paramValues_t* ptr, const varInds_t* varyParams, con static void initMemoTable(U8* memoTable, paramValues_t paramConstraints, const constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { size_t i; size_t arrayLen = memoTableLen(varyParams, varyLen); - int cwFixed = paramConstraints.vals[clog_ind] == PARAM_UNSET || paramConstraints.vals[wlog_ind] == PARAM_UNSET; - int scFixed = paramConstraints.vals[slog_ind] == PARAM_UNSET || paramConstraints.vals[clog_ind] == PARAM_UNSET; - int whFixed = paramConstraints.vals[wlog_ind] == PARAM_UNSET || paramConstraints.vals[hlog_ind] == PARAM_UNSET; - int wFixed = paramConstraints.vals[wlog_ind] == PARAM_UNSET; int j = 0; assert(memoTable != NULL); memset(memoTable, 0, arrayLen); paramConstraints = cParamUnsetMin(paramConstraints); - for(i = 0; i < arrayLen; i++) { memoTableIndInv(¶mConstraints, varyParams, varyLen, i); - if(ZSTD_estimateCStreamSize_usingCParams(pvalsToCParams(paramConstraints)) > (size_t)target.cMem) { - memoTable[i] = 255; - j++; - } - if(wFixed && (1ULL << (paramConstraints.vals[wlog_ind] - 1)) >= srcSize && paramConstraints.vals[wlog_ind] != mintable[wlog_ind]) { - memoTable[i] = 255; - } - /* nil out parameter sets equivalent to others. */ - if(cwFixed) { - if(paramConstraints.vals[strt_ind] == ZSTD_btlazy2 || paramConstraints.vals[strt_ind] == ZSTD_btopt || paramConstraints.vals[strt_ind] == ZSTD_btultra) { - if(paramConstraints.vals[clog_ind] > paramConstraints.vals[wlog_ind]+ 1) { - if(memoTable[i] != 255) { j++; } - memoTable[i] = 255; - } - } else { - if(paramConstraints.vals[clog_ind] > paramConstraints.vals[wlog_ind]) { - if(memoTable[i] != 255) { j++; } - memoTable[i] = 255; - } - } - } - - if(scFixed) { - if(paramConstraints.vals[slog_ind] > paramConstraints.vals[clog_ind]) { - if(memoTable[i] != 255) { j++; } - memoTable[i] = 255; - } - } - - if(whFixed) { - if(paramConstraints.vals[hlog_ind] > paramConstraints.vals[wlog_ind] + 1) { - if(memoTable[i] != 255) { j++; } - memoTable[i] = 255; - } + if(redundantParams(paramConstraints, target, srcSize)) { + memoTable[i] = 255; j++; } } - DEBUGOUTPUT("%d / %d Invalid\n", j, (int)i); - if((int)i == j) { - DEBUGOUTPUT("!!!Strategy %d totally infeasible\n", (int)paramConstraints.vals[strt_ind]); - } + + DEBUGOUTPUT("%d / %d Invalid\n", j, (int)arrayLen); } /* frees all allocated memotables */ @@ -1539,7 +1599,7 @@ static void freeMemoTableArray(U8** mtAll) { /* inits memotables for all (including mallocs), all strategies */ /* takes unsanitized varyParams */ -static U8** createMemoTableArray(paramValues_t paramConstraints, constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { +static U8** createMemoTableArray(paramValues_t paramConstraints, const constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { varInds_t varNew[NUM_PARAMS]; U8** mtAll = (U8**)calloc(sizeof(U8*),(ZSTD_btultra + 1)); int i; @@ -1549,18 +1609,21 @@ static U8** createMemoTableArray(paramValues_t paramConstraints, constraint_t ta for(i = 1; i <= (int)ZSTD_btultra; i++) { const int varLenNew = sanitizeVarArray(varNew, varyLen, varyParams, i); - mtAll[i] = malloc(sizeof(U8) * memoTableLen(varNew, varLenNew)); + size_t mtl = memoTableLen(varNew, varLenNew); + if(mtl > g_memoLimit) { mtAll[i] = NULL; continue; } + mtAll[i] = malloc(sizeof(U8) * mtl); if(mtAll[i] == NULL) { freeMemoTableArray(mtAll); return NULL; } + paramConstraints.vals[strt_ind] = i; initMemoTable(mtAll[i], paramConstraints, target, varNew, varLenNew, srcSize); } return mtAll; } -static paramValues_t overwriteParams(paramValues_t base, paramValues_t mask) { +static paramValues_t overwriteParams(paramValues_t base, const paramValues_t mask) { U32 i; for(i = 0; i < NUM_PARAMS; i++) { if(mask.vals[i] != PARAM_UNSET) { @@ -1586,7 +1649,7 @@ static BYTE* NB_TESTS_PLAYED(ZSTD_compressionParameters p) { static void playAround(FILE* f, oldWinnerInfo_t* winners, ZSTD_compressionParameters params, - buffers_t buf, contexts_t ctx) + const buffers_t buf, const contexts_t ctx) { int nbVariations = 0; UTIL_time_t const clockStart = UTIL_getTime(); @@ -1619,49 +1682,35 @@ static void playAround(FILE* f, oldWinnerInfo_t* winners, } +/* Sets pc to random unmeasured set of parameters */ +/* Doesn't do strategy! */ +static void randomConstrainedParams(paramValues_t* pc, const varInds_t* varArray, const int varLen, const U8* memoTable) +{ + size_t j; + for(j = 0; j < MIN(g_memoLimit, memoTableLen(varArray, varLen)); j++) { + int i; + for(i = 0; i < NUM_PARAMS; i++) { + varInds_t v = varArray[i]; + if(v == strt_ind) continue; //don't do strategy (dependent on MT) + pc->vals[v] = rangeMap(v, FUZ_rand(&g_rand) % rangetable[v]); + } + + if(memoTable == NULL || memoTable[memoTableInd(pc, varArray, varLen)]) { break; } + } +} + /* Completely random parameter selection */ static ZSTD_compressionParameters randomParams(void) { - ZSTD_compressionParameters p; - U32 validated = 0; - while (!validated) { - /* totally random entry */ - p.chainLog = (FUZ_rand(&g_rand) % (ZSTD_CHAINLOG_MAX+1 - ZSTD_CHAINLOG_MIN)) - + ZSTD_CHAINLOG_MIN; - p.hashLog = (FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN)) - + ZSTD_HASHLOG_MIN; - p.searchLog = (FUZ_rand(&g_rand) % (ZSTD_SEARCHLOG_MAX+1 - ZSTD_SEARCHLOG_MIN)) - + ZSTD_SEARCHLOG_MIN; - p.windowLog = (FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN)) - + ZSTD_WINDOWLOG_MIN; - p.searchLength=(FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN)) - + ZSTD_SEARCHLENGTH_MIN; - p.targetLength=(FUZ_rand(&g_rand) % (512)); - - p.strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btultra +1)); - - validated = !ZSTD_isError(ZSTD_checkCParams(p)); - } - return p; -} - -/* Sets pc to random unmeasured set of parameters */ -static void randomConstrainedParams(paramValues_t* pc, varInds_t* varArray, int varLen, U8* memoTable) -{ - size_t tries = memoTableLen(varArray, varLen); - const size_t maxSize = memoTableLen(varArray, varLen); - size_t ind; - do { - ind = (FUZ_rand(&g_rand)) % maxSize; - tries--; - } while(memoTable[ind] > 0 && tries > 0); - - memoTableIndInv(pc, varArray, varLen, (unsigned)ind); + paramValues_t p; + p.vals[strt_ind] = rangeMap(strt_ind, rangeMap(strt_ind, FUZ_rand(&g_rand) % rangetable[strt_ind])); + randomConstrainedParams(&p, paramTable, NUM_PARAMS, NULL); + return pvalsToCParams(p); } static void BMK_selectRandomStart( FILE* f, oldWinnerInfo_t* winners, - buffers_t buf, contexts_t ctx) + const buffers_t buf, const contexts_t ctx) { U32 const id = FUZ_rand(&g_rand) % (NB_LEVELS_TRACKED+1); if ((id==0) || (winners[id].params.windowLog==0)) { @@ -1673,7 +1722,7 @@ static void BMK_selectRandomStart( } } -static void BMK_benchFullTable(buffers_t buf, contexts_t ctx, const size_t maxBlockSize) +static void BMK_benchFullTable(const buffers_t buf, const contexts_t ctx) { ZSTD_compressionParameters params; oldWinnerInfo_t winners[NB_LEVELS_TRACKED+1]; @@ -1689,7 +1738,7 @@ static void BMK_benchFullTable(buffers_t buf, contexts_t ctx, const size_t maxBl BMK_init_level_constraints(g_target * (1 MB)); } else { /* baseline config for level 1 */ - ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, maxBlockSize, ctx.dictSize); + ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, buf.maxBlockSize, ctx.dictSize); BMK_result_t testResult; BMK_benchParam(&testResult, buf, ctx, cParamsToPVals(l1params)); BMK_init_level_constraints((int)((testResult.cSpeed * 31) / 32)); @@ -1699,7 +1748,7 @@ static void BMK_benchFullTable(buffers_t buf, contexts_t ctx, const size_t maxBl { const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); int i; for (i=0; i<=maxSeeds; i++) { - params = ZSTD_getCParams(i, maxBlockSize, 0); + params = ZSTD_getCParams(i, buf.maxBlockSize, 0); BMK_seed(winners, params, buf, ctx); } } BMK_printWinners(f, winners, buf.srcSize); @@ -1719,7 +1768,7 @@ static void BMK_benchFullTable(buffers_t buf, contexts_t ctx, const size_t maxBl fclose(f); } -static int benchOnce(buffers_t buf, contexts_t ctx) { +static int benchOnce(const buffers_t buf, const contexts_t ctx) { BMK_result_t testResult; if(BMK_benchParam(&testResult, buf, ctx, g_params)) { @@ -1736,7 +1785,6 @@ static int benchSample(void) { const char* const name = "Sample 10MB"; size_t const benchedSize = 10 MB; - U32 blockSize = g_blockSize ? g_blockSize : benchedSize; void* srcBuffer = malloc(benchedSize); int ret = 0; @@ -1769,7 +1817,7 @@ static int benchSample(void) if(g_singleRun) { ret = benchOnce(buf, ctx); } else { - BMK_benchFullTable(buf, ctx, MIN(blockSize, benchedSize)); + BMK_benchFullTable(buf, ctx); } freeBuffers(buf); @@ -1785,7 +1833,6 @@ int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileNam { buffers_t buf; contexts_t ctx; - size_t maxBlockSize = 0, i; int ret = 0; if(createBuffers(&buf, fileNamesTable, nbFiles)) { @@ -1799,10 +1846,6 @@ int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileNam return 2; } - for(i = 0; i < buf.nbBlocks; i++) { - maxBlockSize = MAX(maxBlockSize, buf.srcSizes[i]); - } - DISPLAY("\r%79s\r", ""); if(nbFiles == 1) { DISPLAY("using %s : \n", fileNamesTable[0]); @@ -1810,12 +1853,12 @@ int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileNam DISPLAY("using %d Files : \n", nbFiles); } - g_params = adjustParams(overwriteParams(cParamsToPVals(ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize)), g_params), maxBlockSize, ctx.dictSize); + g_params = adjustParams(overwriteParams(cParamsToPVals(ZSTD_getCParams(cLevel, buf.maxBlockSize, ctx.dictSize)), g_params), buf.maxBlockSize, ctx.dictSize); if(g_singleRun) { ret = benchOnce(buf, ctx); } else { - BMK_benchFullTable(buf, ctx, maxBlockSize); + BMK_benchFullTable(buf, ctx); } freeBuffers(buf); @@ -1937,7 +1980,7 @@ static int benchMemo(BMK_result_t* resultPtr, size_t memind = memoTableInd(&cParams, varyParams, varyLen); int res; - if(memoTable[memind] >= INFEASIBLE_THRESHOLD) { return WORSE_RESULT; } + if((memoTable == NULL && redundantParams(cParams, target, buf.maxBlockSize)) || ((memoTable != NULL) && memoTable[memind] >= INFEASIBLE_THRESHOLD)) { return WORSE_RESULT; } res = allBench(resultPtr, buf, ctx, cParams, target, winnerResult, feas); @@ -1947,7 +1990,7 @@ static int benchMemo(BMK_result_t* resultPtr, } BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, *resultPtr, cParams, target, buf.srcSize); - if(res == BETTER_RESULT || feas) { + if(memoTable != NULL && (res == BETTER_RESULT || feas)) { memoTable[memind] = 255; } return res; @@ -1972,7 +2015,7 @@ static int benchMemo(BMK_result_t* resultPtr, static winnerInfo_t climbOnce(const constraint_t target, const varInds_t* varArray, const int varLen, ZSTD_strategy strat, U8** memoTableArray, - buffers_t buf, contexts_t ctx, + const buffers_t buf, const contexts_t ctx, const paramValues_t init) { /* * cparam - currently considered 'center' @@ -2103,7 +2146,6 @@ static winnerInfo_t optimizeFixedStrategy( /* so climb is given the right fixed strategy */ paramTarget.vals[strt_ind] = strat; /* to pass ZSTD_checkCParams */ - paramTarget = cParamUnsetMin(paramTarget); init = paramTarget; @@ -2178,16 +2220,11 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ const int varLen = variableParams(paramTarget, varArray, dictFileName != NULL); winnerInfo_t winner = initWinnerInfo(emptyParams()); U8** allMT = NULL; - paramValues_t paramBase = cParamUnsetMin(paramTarget); - size_t k, maxBlockSize = 0; + paramValues_t paramBase; contexts_t ctx; buffers_t buf; g_time = UTIL_getTime(); - if(!paramValid(paramBase)) { - return 1; - } - if(createBuffers(&buf, fileNamesTable, nbFiles)) { DISPLAY("unable to load files\n"); return 1; @@ -2205,10 +2242,9 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ DISPLAY("Loading %lu Files... \r", (unsigned long)nbFiles); } - - for(k = 0; k < buf.nbBlocks; k++) { - maxBlockSize = MAX(buf.srcSizes[k], maxBlockSize); - } + /* sanitize paramTarget */ + optimizerAdjustInput(¶mTarget, buf.maxBlockSize); + paramBase = cParamUnsetMin(paramTarget); /* if strategy is fixed, only init that part of memotable */ if(paramTarget.vals[strt_ind] != PARAM_UNSET) { @@ -2227,9 +2263,9 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ goto _cleanUp; } - initMemoTable(allMT[paramTarget.vals[strt_ind]], paramTarget, target, varNew, varLenNew, maxBlockSize); + initMemoTable(allMT[paramTarget.vals[strt_ind]], paramTarget, target, varNew, varLenNew, buf.maxBlockSize); } else { - allMT = createMemoTableArray(paramTarget, target, varArray, varLen, maxBlockSize); + allMT = createMemoTableArray(paramTarget, target, varArray, varLen, buf.maxBlockSize); } if(!allMT) { @@ -2256,7 +2292,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ /* use level'ing mode instead of normal target mode */ /* Should lvl be parameter-masked here? */ if(g_optmode) { - winner.params = cParamsToPVals(ZSTD_getCParams(cLevelOpt, maxBlockSize, ctx.dictSize)); + winner.params = cParamsToPVals(ZSTD_getCParams(cLevelOpt, buf.maxBlockSize, ctx.dictSize)); if(BMK_benchParam(&winner.result, buf, ctx, winner.params)) { ret = 3; goto _cleanUp; @@ -2276,7 +2312,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ /* Don't want it to return anything worse than the best known result */ if(g_singleRun) { BMK_result_t res; - g_params = adjustParams(overwriteParams(cParamsToPVals(ZSTD_getCParams(cLevelRun, maxBlockSize, ctx.dictSize)), g_params), maxBlockSize, ctx.dictSize); + g_params = adjustParams(overwriteParams(cParamsToPVals(ZSTD_getCParams(cLevelRun, buf.maxBlockSize, ctx.dictSize)), g_params), buf.maxBlockSize, ctx.dictSize); if(BMK_benchParam(&res, buf, ctx, g_params)) { ret = 45; goto _cleanUp; @@ -2315,7 +2351,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ int i; for (i=1; i<=maxSeeds; i++) { int ec; - CParams = overwriteParams(cParamsToPVals(ZSTD_getCParams(i, maxBlockSize, ctx.dictSize)), paramTarget); + CParams = overwriteParams(cParamsToPVals(ZSTD_getCParams(i, buf.maxBlockSize, ctx.dictSize)), paramTarget); ec = BMK_benchParam(&candidate, buf, ctx, CParams); BMK_printWinnerOpt(stdout, i, candidate, CParams, target, buf.srcSize); @@ -2466,16 +2502,24 @@ static int badusage(const char* exename) } #define PARSE_SUB_ARGS(stringLong, stringShort, variable) { if (longCommandWArg(&argument, stringLong) || longCommandWArg(&argument, stringShort)) { variable = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } } -#define PARSE_CPARAMS(variable) \ -{ \ - PARSE_SUB_ARGS("windowLog=", "wlog=", variable.vals[wlog_ind]); \ - PARSE_SUB_ARGS("chainLog=" , "clog=", variable.vals[clog_ind]); \ - PARSE_SUB_ARGS("hashLog=", "hlog=", variable.vals[hlog_ind]); \ - PARSE_SUB_ARGS("searchLog=" , "slog=", variable.vals[slog_ind]); \ - PARSE_SUB_ARGS("searchLength=", "slen=", variable.vals[slen_ind]); \ - PARSE_SUB_ARGS("targetLength=" , "tlen=", variable.vals[tlen_ind]); \ - PARSE_SUB_ARGS("strategy=", "strat=", variable.vals[strt_ind]); \ - PARSE_SUB_ARGS("forceAttachDict=", "fad=" , variable.vals[fadt_ind]); \ +/* 1 if successful parse, 0 otherwise */ +static int parse_params(const char** argptr, paramValues_t* pv) { + int matched = 0; + const char* argOrig = *argptr; + varInds_t v; + for(v = 0; v < NUM_PARAMS; v++) { + if(longCommandWArg(argptr,g_shortParamNames[v]) || longCommandWArg(argptr, g_paramNames[v])) { + if(**argptr == '=') { + (*argptr)++; + pv->vals[v] = readU32FromChar(argptr); + matched = 1; + break; + } + } + /* reset and try again */ + *argptr = argOrig; + } + return matched; } int main(int argc, const char** argv) @@ -2509,7 +2553,7 @@ int main(int argc, const char** argv) if (longCommandWArg(&argument, "--optimize=")) { g_optimizer = 1; for ( ; ;) { - PARSE_CPARAMS(paramTarget); + if(parse_params(&argument, ¶mTarget)) { if(argument[0] == ',') { argument++; continue; } else break; } PARSE_SUB_ARGS("compressionSpeed=" , "cSpeed=", target.cSpeed); PARSE_SUB_ARGS("decompressionSpeed=", "dSpeed=", target.dSpeed); PARSE_SUB_ARGS("compressionMemory=" , "cMem=", target.cMem); @@ -2517,6 +2561,7 @@ int main(int argc, const char** argv) PARSE_SUB_ARGS("preferSpeed=", "prfSpd=", g_speedMultiplier); PARSE_SUB_ARGS("preferRatio=", "prfRto=", g_ratioMultiplier); PARSE_SUB_ARGS("maxTries=", "tries=", g_maxTries); + PARSE_SUB_ARGS("memoLimit=", "memo=", g_memoLimit); if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevelOpt = readU32FromChar(&argument); g_optmode = 1; if (argument[0]==',') { argument++; continue; } else break; } DISPLAY("invalid optimization parameter \n"); @@ -2532,7 +2577,7 @@ int main(int argc, const char** argv) /* Decode command (note : aggregated commands are allowed) */ g_singleRun = 1; for ( ; ;) { - PARSE_CPARAMS(g_params) + if(parse_params(&argument, &g_params)) { if(argument[0] == ',') { argument++; continue; } else break; } if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevelRun = readU32FromChar(&argument); g_params = emptyParams(); if (argument[0]==',') { argument++; continue; } else break; } DISPLAY("invalid compression parameter \n");