facebook/zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Renames, Documentation Updates

Browse Source
This commit is contained in:
George Lu 2018-07-27 16:49:33 -07:00
parent a884b76bc2
commit 43b4971ca8
2 changed files with 58 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
tests/README.md
View File

@ -107,15 +107,19 @@ Full list of arguments
L# - level L# - level
--zstd= : Single run, parameter selection syntax same as zstdcli --zstd= : Single run, parameter selection syntax same as zstdcli
--optimize= : find parameters to maximize compression ratio given parameters --optimize= : find parameters to maximize compression ratio given parameters
Can use all --zstd= commands to constrain the type of solution found in addition to the following constraints Can use all --zstd= commands to constrain the type of solution found in addition to the following constraints
cSpeed= - Minimum compression speed cSpeed= : Minimum compression speed
dSpeed= - Minimum decompression speed dSpeed= : Minimum decompression speed
cMem= - compression memory cMem= : Maximum compression memory
lvl= - Searches for solutions which are strictly better than that compression lvl in ratio and cSpeed, lvl= : Searches for solutions which are strictly better than that compression lvl in ratio and cSpeed,
stc= - When invoked with lvl=, represents slack in ratio/cSpeed allowed for a solution to be considered stc= : When invoked with lvl=, represents percentage slack in ratio/cSpeed allowed for a solution to be considered (Default 99%)
- In normal operation, represents slack in strategy selection in choosing the default parameters : In normal operation, represents percentage slack in choosing viable starting strategy selection in choosing the default parameters
prefer[Speed/Ratio]= - Only affects lvl= invocations. Defines value placed on compression speed or ratio (Lower value will begin with stronger strategies) (Default 90%)
when determining overall winner (default 1 for both). preferSpeed= / preferRatio=
: Only affects lvl = invocations. Defines value placed on compression speed or ratio
when determining overall winner (default 1 for both, higher = more valued).
tries= : Maximum number of random restarts on a single strategy before switching (Default 5)
Higher values will make optimizer run longer, more chances to find better solution.
--optimize= : same as -O with more verbose syntax --optimize= : same as -O with more verbose syntax
-P# : generated sample compressibility -P# : generated sample compressibility
-t# : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours )) -t# : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours ))

84
tests/paramgrill.c
View File

@ -125,15 +125,13 @@ typedef struct {
U32 cMem; /* bytes */ U32 cMem; /* bytes */
} constraint_t; } constraint_t;
static constraint_t g_targetConstraints; typedef struct winner_ll_node winner_ll_node;
struct winner_ll_node {
typedef struct ll_node ll_node;
struct ll_node {
winnerInfo_t res; winnerInfo_t res;
ll_node* next; winner_ll_node* next;
}; };
static ll_node* g_winners; /* linked list sorted ascending by cSize & cSpeed */ static winner_ll_node* g_winners; /* linked list sorted ascending by cSize & cSpeed */
static BMK_result_t g_lvltarget; static BMK_result_t g_lvltarget;
static int g_optmode = 0; static int g_optmode = 0;
@ -323,8 +321,8 @@ static int compareResultLT(const BMK_result_t result1, const BMK_result_t result
static constraint_t relaxTarget(constraint_t target) { static constraint_t relaxTarget(constraint_t target) {
target.cMem = (U32)-1; target.cMem = (U32)-1;
target.cSpeed *= ((double)(g_strictness) / 100); target.cSpeed *= ((double)g_strictness) / 100;
target.dSpeed *= ((double)(g_strictness) / 100); target.dSpeed *= ((double)g_strictness) / 100;
return target; return target;
} }
@ -855,20 +853,19 @@ static int speedSizeCompare(BMK_result_t r1, BMK_result_t r2) {
return SPEED_RESULT; /* r2 is faster but not smaller */ return SPEED_RESULT; /* r2 is faster but not smaller */
} }
} }
/* assumes candidate is already strictly better than old winner. */
/* 0 for success, 1 for no insert */ /* 0 for insertion, 1 for no insert */
/* indicate whether inserted as well? */
/* maintain invariant speedSizeCompare(n, n->next) = SPEED_RESULT */ /* maintain invariant speedSizeCompare(n, n->next) = SPEED_RESULT */
static int insertWinner(winnerInfo_t w) { static int insertWinner(winnerInfo_t w, constraint_t targetConstraints) {
BMK_result_t r = w.result; BMK_result_t r = w.result;
ll_node* cur_node = g_winners; winner_ll_node* cur_node = g_winners;
/* first node to insert */ /* first node to insert */
if(!feasible(r, g_targetConstraints)) { if(!feasible(r, targetConstraints)) {
return 1; return 1;
} }
if(g_winners == NULL) { if(g_winners == NULL) {
ll_node* first_node = malloc(sizeof(ll_node)); winner_ll_node* first_node = malloc(sizeof(winner_ll_node));
if(first_node == NULL) { if(first_node == NULL) {
return 1; return 1;
} }
@ -886,7 +883,7 @@ static int insertWinner(winnerInfo_t w) {
} }
case WORSE_RESULT: case WORSE_RESULT:
{ {
ll_node* tmp; winner_ll_node* tmp;
cur_node->res = cur_node->next->res; cur_node->res = cur_node->next->res;
tmp = cur_node->next; tmp = cur_node->next;
cur_node->next = cur_node->next->next; cur_node->next = cur_node->next->next;
@ -900,7 +897,7 @@ static int insertWinner(winnerInfo_t w) {
} }
case SIZE_RESULT: /* insert after first size result, then return */ case SIZE_RESULT: /* insert after first size result, then return */
{ {
ll_node* newnode = malloc(sizeof(ll_node)); winner_ll_node* newnode = malloc(sizeof(winner_ll_node));
if(newnode == NULL) { if(newnode == NULL) {
return 1; return 1;
} }
@ -927,7 +924,7 @@ static int insertWinner(winnerInfo_t w) {
} }
case SPEED_RESULT: case SPEED_RESULT:
{ {
ll_node* newnode = malloc(sizeof(ll_node)); winner_ll_node* newnode = malloc(sizeof(winner_ll_node));
if(newnode == NULL) { if(newnode == NULL) {
return 1; return 1;
} }
@ -938,7 +935,7 @@ static int insertWinner(winnerInfo_t w) {
} }
case SIZE_RESULT: /* insert before first size result, then return */ case SIZE_RESULT: /* insert before first size result, then return */
{ {
ll_node* newnode = malloc(sizeof(ll_node)); winner_ll_node* newnode = malloc(sizeof(winner_ll_node));
if(newnode == NULL) { if(newnode == NULL) {
return 1; return 1;
} }
@ -961,7 +958,7 @@ static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result
const U64 time = UTIL_clockSpanNano(g_time); const U64 time = UTIL_clockSpanNano(g_time);
const U64 minutes = time / (60ULL * TIMELOOP_NANOSEC); const U64 minutes = time / (60ULL * TIMELOOP_NANOSEC);
DISPLAY("\r%79s\r", ""); fprintf(f, "\r%79s\r", "");
fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ", fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ",
params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength,
@ -1001,10 +998,10 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res
//prints out tradeoff table if using lvl //prints out tradeoff table if using lvl
if(g_optmode && g_optimizer) { if(g_optmode && g_optimizer) {
winnerInfo_t w; winnerInfo_t w;
ll_node* n; winner_ll_node* n;
w.result = result; w.result = result;
w.params = params; w.params = params;
insertWinner(w); insertWinner(w, targetConstraints);
if(!DEBUG) { fprintf(f, "\033c"); } if(!DEBUG) { fprintf(f, "\033c"); }
fprintf(f, "\n"); fprintf(f, "\n");
@ -1012,7 +1009,7 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res
/* the table */ /* the table */
fprintf(f, "================================\n"); fprintf(f, "================================\n");
for(n = g_winners; n != NULL; n = n->next) { for(n = g_winners; n != NULL; n = n->next) {
DISPLAY("\r%79s\r", ""); fprintf(f, "\r%79s\r", "");
fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ", fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ",
n->res.params.windowLog, n->res.params.chainLog, n->res.params.hashLog, n->res.params.searchLog, n->res.params.searchLength, n->res.params.windowLog, n->res.params.chainLog, n->res.params.hashLog, n->res.params.searchLog, n->res.params.searchLength,
@ -1045,6 +1042,12 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res
(double)srcSize / result.cSize, (double)result.cSpeed / (1 MB), (double)result.dSpeed / (1 MB)); (double)srcSize / result.cSize, (double)result.cSpeed / (1 MB), (double)result.dSpeed / (1 MB));
} }
#if 0
if(BMK_timeSpan(g_time) > g_grillDuration_s) {
exit(0);
}
#endif
} }
static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSize) static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSize)
@ -2097,7 +2100,7 @@ static ZSTD_compressionParameters maskParams(ZSTD_compressionParameters base, ZS
* cLevel - compression level to exceed (all solutions must be > lvl in cSpeed + ratio) * cLevel - compression level to exceed (all solutions must be > lvl in cSpeed + ratio)
*/ */
#define MAX_TRIES 5 static int g_maxTries = 5;
#define TRY_DECAY 1 #define TRY_DECAY 1
static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget, int cLevel) static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget, int cLevel)
@ -2185,20 +2188,21 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
} }
/* use level'ing mode instead of normal target mode */ /* use level'ing mode instead of normal target mode */
/* Should lvl be parameter-masked here? */
if(g_optmode) { if(g_optmode) {
winner.params = ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize); winner.params = ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize);
if(BMK_benchParam(&winner.result, buf, ctx, winner.params)) { if(BMK_benchParam(&winner.result, buf, ctx, winner.params)) {
ret = 3; ret = 3;
goto _cleanUp; goto _cleanUp;
} }
target.cSpeed = (U32)winner.result.cSpeed;
g_targetConstraints = target;
g_lvltarget = winner.result; g_lvltarget = winner.result;
g_lvltarget.cSpeed *= ((double)(g_strictness) / 100); g_lvltarget.cSpeed *= ((double)g_strictness) / 100;
g_lvltarget.cSize /= ((double)(g_strictness) / 100); g_lvltarget.dSpeed *= ((double)g_strictness) / 100;
g_lvltarget.cSize /= ((double)g_strictness) / 100;
target.cSpeed = (U32)g_lvltarget.cSpeed;
target.dSpeed = (U32)g_lvltarget.dSpeed; //See if this is worth
BMK_printWinnerOpt(stdout, cLevel, winner.result, winner.params, target, buf.srcSize); BMK_printWinnerOpt(stdout, cLevel, winner.result, winner.params, target, buf.srcSize);
} }
@ -2241,7 +2245,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
} }
/* if the current params are too slow, just stop. */ /* if the current params are too slow, just stop. */
if(target.cSpeed > candidate.cSpeed * 2) { break; } if(target.cSpeed > candidate.cSpeed * 3 / 2) { break; }
} }
} }
} }
@ -2254,7 +2258,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
int bestStrategy = (int)winner.params.strategy; int bestStrategy = (int)winner.params.strategy;
if(paramTarget.strategy == 0) { if(paramTarget.strategy == 0) {
int st = (int)winner.params.strategy; int st = (int)winner.params.strategy;
int tries = MAX_TRIES; int tries = g_maxTries;
{ {
/* one iterations of hill climbing with the level-defined parameters. */ /* one iterations of hill climbing with the level-defined parameters. */
@ -2269,12 +2273,13 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
while(st && tries > 0) { while(st && tries > 0) {
winnerInfo_t wc; winnerInfo_t wc;
DEBUGOUTPUT("StrategySwitch: %s\n", g_stratName[st]); DEBUGOUTPUT("StrategySwitch: %s\n", g_stratName[st]);
wc = optimizeFixedStrategy(buf, ctx, target, paramTarget, wc = optimizeFixedStrategy(buf, ctx, target, paramTarget,
st, varArray, varLen, allMT[st], tries); st, varArray, varLen, allMT[st], tries);
if(compareResultLT(winner.result, wc.result, target, buf.srcSize)) { if(compareResultLT(winner.result, wc.result, target, buf.srcSize)) {
winner = wc; winner = wc;
tries = MAX_TRIES; tries = g_maxTries;
bestStrategy = st; bestStrategy = st;
} else { } else {
st = nextStrategy(st, bestStrategy); st = nextStrategy(st, bestStrategy);
@ -2283,7 +2288,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
} }
} else { } else {
winner = optimizeFixedStrategy(buf, ctx, target, paramTarget, paramTarget.strategy, winner = optimizeFixedStrategy(buf, ctx, target, paramTarget, paramTarget.strategy,
varArray, varLen, allMT[paramTarget.strategy], 10); varArray, varLen, allMT[paramTarget.strategy], g_maxTries);
} }
} }
@ -2402,17 +2407,17 @@ int main(int argc, const char** argv)
ZSTD_compressionParameters paramTarget = { 0, 0, 0, 0, 0, 0, 0 }; ZSTD_compressionParameters paramTarget = { 0, 0, 0, 0, 0, 0, 0 };
assert(argc>=1); /* for exename */
g_time = UTIL_getTime(); g_time = UTIL_getTime();
assert(argc>=1); /* for exename */
/* Welcome message */ /* Welcome message */
DISPLAY(WELCOME_MESSAGE); DISPLAY(WELCOME_MESSAGE);
for(i=1; i<argc; i++) { for(i=1; i<argc; i++) {
const char* argument = argv[i]; const char* argument = argv[i];
DISPLAY("%d: ", i); DEBUGOUTPUT("%d: ", i);
DISPLAY("%s\n", argument); DEBUGOUTPUT("%s\n", argument);
assert(argument != NULL); assert(argument != NULL);
@ -2429,6 +2434,7 @@ int main(int argc, const char** argv)
PARSE_SUB_ARGS("strict=", "stc=", g_strictness); PARSE_SUB_ARGS("strict=", "stc=", g_strictness);
PARSE_SUB_ARGS("preferSpeed=", "prfSpd=", g_speedMultiplier); PARSE_SUB_ARGS("preferSpeed=", "prfSpd=", g_speedMultiplier);
PARSE_SUB_ARGS("preferRatio=", "prfRto=", g_ratioMultiplier); PARSE_SUB_ARGS("preferRatio=", "prfRto=", g_ratioMultiplier);
PARSE_SUB_ARGS("maxTries", "tries", g_maxTries);
DISPLAY("invalid optimization parameter \n"); DISPLAY("invalid optimization parameter \n");
return 1; return 1;