From 486e586eed5d30f6c4ec079bb403d6c15004bb9d Mon Sep 17 00:00:00 2001
From: George Lu <gclu@fb.com>
Date: Mon, 13 Aug 2018 16:13:46 -0700
Subject: [PATCH 01/15] Revert "Default lvl 1"

This reverts commit 0cc75d6ee02efe9da7901848ce588a7f9f61b4f9.
---
 tests/paramgrill.c | 27 +++++++++++++--------------
 1 file changed, 13 insertions(+), 14 deletions(-)

diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index d7165021..bcb6bcae 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -65,7 +65,6 @@ static const int g_maxNbVariations = 64;
 #define MIN(a,b)   ( (a) < (b) ? (a) : (b) )
 #define MAX(a,b)   ( (a) > (b) ? (a) : (b) )
 #define CUSTOM_LEVEL 99
-#define BASE_CLEVEL 1
 
 /* indices for each of the variables */
 typedef enum {
@@ -303,7 +302,7 @@ BMK_benchParam1(BMK_result_t* resultPtr,
                const void* srcBuffer, size_t srcSize,
                const ZSTD_compressionParameters cParams) {
 
-    BMK_return_t res = BMK_benchMem(srcBuffer,srcSize, &srcSize, 1, BASE_CLEVEL, &cParams, NULL, 0, 0, "File");
+    BMK_return_t res = BMK_benchMem(srcBuffer,srcSize, &srcSize, 1, 0, &cParams, NULL, 0, 0, "File");
     *resultPtr = res.result;
     return res.error;
 }
@@ -793,16 +792,16 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para
                 /* too large compression speed difference for the compression benefit */
                 if (W_ratio > O_ratio)
                 DISPLAY ("Compression Speed : %5.3f @ %4.1f MB/s  vs  %5.3f @ %4.1f MB/s   : not enough for level %i\n",
-                         W_ratio, (double)testResult.cSpeed / (1 MB),
-                         O_ratio, (double)winners[cLevel].result.cSpeed / (1 MB),   cLevel);
+                         W_ratio, (double)testResult.cSpeed / 1000000,
+                         O_ratio, (double)winners[cLevel].result.cSpeed / 1000000.,   cLevel);
                 continue;
             }
             if (W_DSpeed_note   < O_DSpeed_note  ) {
                 /* too large decompression speed difference for the compression benefit */
                 if (W_ratio > O_ratio)
                 DISPLAY ("Decompression Speed : %5.3f @ %4.1f MB/s  vs  %5.3f @ %4.1f MB/s   : not enough for level %i\n",
-                         W_ratio, (double)testResult.dSpeed / (1 MB),
-                         O_ratio, (double)winners[cLevel].result.dSpeed / (1 MB),   cLevel);
+                         W_ratio, (double)testResult.dSpeed / 1000000.,
+                         O_ratio, (double)winners[cLevel].result.dSpeed / 1000000.,   cLevel);
                 continue;
             }
 
@@ -1194,7 +1193,7 @@ static void BMK_benchOnce(const void* srcBuffer, size_t srcSize)
     g_params = ZSTD_adjustCParams(g_params, srcSize, 0);
     BMK_benchParam1(&testResult, srcBuffer, srcSize, g_params);
     DISPLAY("Compression Ratio: %.3f  Compress Speed: %.1f MB/s Decompress Speed: %.1f MB/s\n", (double)srcSize / testResult.cSize, 
-        (double)testResult.cSpeed / (1 MB), (double)testResult.dSpeed / (1 MB));
+        (double)testResult.cSpeed / 1000000, (double)testResult.dSpeed / 1000000);
     return;
 }
 
@@ -1212,7 +1211,7 @@ static void BMK_benchFullTable(const void* srcBuffer, size_t srcSize)
     if (f==NULL) { DISPLAY("error opening %s \n", rfName); exit(1); }
 
     if (g_target) {
-        BMK_init_level_constraints(g_target*(1 MB));
+        BMK_init_level_constraints(g_target*1000000);
     } else {
         /* baseline config for level 1 */
         ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, blockSize, 0);
@@ -1257,7 +1256,7 @@ static void BMK_benchMemInit(const void* srcBuffer, size_t srcSize)
 static int benchSample(void)
 {
     const char* const name = "Sample 10MB";
-    size_t const benchedSize = (10 MB);
+    size_t const benchedSize = 10000000;
 
     void* origBuff = malloc(benchedSize);
     if (!origBuff) { perror("not enough memory"); return 12; }
@@ -1355,7 +1354,7 @@ static int allBench(BMK_result_t* resultPtr,
     double winnerRS;
 
     /* initial benchmarking, gives exact ratio and memory, warms up future runs */
-    benchres = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_both, BMK_iterMode, 1);
+    benchres = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_both, BMK_iterMode, 1);
 
     winnerRS = resultScore(*winnerResult, buf.srcSize, target);
     DEBUGOUTPUT("WinnerScore: %f\n ", winnerRS);
@@ -1390,14 +1389,14 @@ static int allBench(BMK_result_t* resultPtr,
 
     /* second run, if first run is too short, gives approximate cSpeed + dSpeed */
     if(loopDurationC < TIMELOOP_NANOSEC / 10) {
-        BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_compressOnly, BMK_iterMode, 1);
+        BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_compressOnly, BMK_iterMode, 1);
         if(benchres2.error) {
             return ERROR_RESULT;
         }
         benchres = benchres2;
     }
     if(loopDurationD < TIMELOOP_NANOSEC / 10) {
-        BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_decodeOnly, BMK_iterMode, 1);
+        BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_decodeOnly, BMK_iterMode, 1);
         if(benchres2.error) {
             return ERROR_RESULT;
         }
@@ -1420,7 +1419,7 @@ static int allBench(BMK_result_t* resultPtr,
 
     /* Final full run if estimates are unclear */
     if(loopDurationC < TIMELOOP_NANOSEC) {
-        BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_compressOnly, BMK_timeMode, 1);
+        BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_compressOnly, BMK_timeMode, 1);
         if(benchres2.error) {
             return ERROR_RESULT;
         }
@@ -1428,7 +1427,7 @@ static int allBench(BMK_result_t* resultPtr,
     } 
 
     if(loopDurationD < TIMELOOP_NANOSEC) {
-        BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_decodeOnly, BMK_timeMode, 1);
+        BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_decodeOnly, BMK_timeMode, 1);
         if(benchres2.error) {
             return ERROR_RESULT;
         }

From 0cea75402478ded21d2300544bcb538f4d3e3786 Mon Sep 17 00:00:00 2001
From: George Lu <gclu@fb.com>
Date: Mon, 13 Aug 2018 16:15:34 -0700
Subject: [PATCH 02/15] Revert "Reorder declaration"

This reverts commit 3ac2c22485ab5508f47e3eab642b787af0e68b5f.
---
 tests/paramgrill.c | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index bcb6bcae..e530af73 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -2256,9 +2256,8 @@ int main(int argc, const char** argv)
                             g_params.strategy = (ZSTD_strategy)readU32FromChar(&argument);
                             continue;
                         case 'L':
-                            {   int cLevel;
-                                argument++;
-                                cLevel = readU32FromChar(&argument);
+                            {   argument++;
+                                int const cLevel = readU32FromChar(&argument);
                                 g_params = ZSTD_getCParams(cLevel, g_blockSize, 0);
                                 continue;
                             }

From 13611249a5d1426e4902fe32b1cf913228f246fa Mon Sep 17 00:00:00 2001
From: George Lu <gclu@fb.com>
Date: Tue, 24 Jul 2018 17:26:21 -0700
Subject: [PATCH 03/15] Table

Compiling
+Euclidean Metric
---
 tests/paramgrill.c | 186 +++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 179 insertions(+), 7 deletions(-)

diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index e530af73..5c654d99 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -279,6 +279,29 @@ static int compareResultLT(const BMK_result_t result1, const BMK_result_t result
 
 }
 
+/* calculates normalized euclidean distance of result1 if it is in the first quadrant relative to lvlRes */
+static double resultDistLvl(const BMK_result_t result1, const BMK_result_t lvlRes) {
+    double normalizedCSpeedGain1 = result1.cSpeed / lvlRes.cSpeed - 1;
+    double normalizedRatioGain1 = lvlRes.cSize / result1.cSize - 1;
+    if(normalizedRatioGain1 < 0 || normalizedRatioGain1 < 0) {
+        return 0.0;
+    }
+    return normalizedRatioGain1 * normalizedRatioGain1 + normalizedCSpeedGain1 * normalizedCSpeedGain1;
+}
+
+static int lvlFeasible(const BMK_result_t result, const BMK_result_t lvlRes) {
+    return lvlRes.cSpeed < result.cSpeed && lvlRes.cSize > result.cSize;
+}
+
+/* redefines feasibility for lvl mode */
+static int compareResultLT2(const BMK_result_t result1, const BMK_result_t result2, const BMK_result_t lvltarget, size_t srcSize) {
+    constraint_t target = { (U32)lvltarget.cSpeed, 0, (U32)-1 };
+    if(lvlFeasible(result1, lvltarget) && lvlFeasible(result2, lvltarget)) {
+        return resultDistLvl(result1, lvltarget) < resultDistLvl(result2, lvltarget);
+    }
+    return lvlFeasible(result2, lvltarget) || (!lvlFeasible(result1, lvltarget) && (resultScore(result1, srcSize, target) < resultScore(result2, srcSize, target)));
+}
+
 /* factor sort of arbitrary */
 static constraint_t relaxTarget(constraint_t target) {
     target.cMem = (U32)-1;
@@ -629,6 +652,132 @@ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t
 }
 
 
+typedef struct ll_node ll_node;
+struct ll_node {
+    winnerInfo_t res;
+    ll_node* next;
+};
+
+static ll_node* g_winners; /* linked list sorted ascending by cSize & cSpeed */
+static BMK_result_t g_lvltarget;
+
+/* comparison function: */
+/* strictly better, strictly worse, equal, speed-side adv, size-side adv */
+//Maybe use compress_only for benchmark first run?
+#define WORSE_RESULT 0
+#define BETTER_RESULT 1
+#define ERROR_RESULT 2
+
+#define SPEED_RESULT 4
+#define SIZE_RESULT 5
+static int speedSizeCompare(BMK_result_t r1, BMK_result_t r2) {
+    if(r1.cSpeed > r2.cSpeed) {
+        if(r1.cSize <= r2.cSize) {
+            return WORSE_RESULT;
+        }
+        return SIZE_RESULT; /* r2 is smaller but not faster. */
+    } else {
+        if(r1.cSize >= r2.cSize) {
+            return BETTER_RESULT;
+        }
+        return SPEED_RESULT; /* r2 is faster but not smaller */
+    }
+}
+/* assumes candidate is already strictly better than old winner. */
+/* 0 for success, 1 for no insert */
+/* indicate whether inserted as well? */
+/* maintain invariant speedSizeCompare(n, n->next) = SPEED_RESULT */
+static int insertWinner(winnerInfo_t w) {
+    BMK_result_t r = w.result;
+    ll_node* cur_node = g_winners;
+    /* first node to insert */
+    if(!lvlFeasible(r, g_lvltarget)) {
+        return 1;
+    }
+
+    if(g_winners == NULL) {
+        ll_node* first_node = malloc(sizeof(ll_node));
+        if(first_node == NULL) {
+            return 1;
+        }
+        first_node->next = NULL;
+        first_node->res = w;
+        g_winners = first_node;
+        return 0;
+    }
+
+    while(cur_node->next != NULL) {
+        switch(speedSizeCompare(r, cur_node->res.result)) {
+            case BETTER_RESULT:
+            {
+                return 1; /* never insert if better */
+            }
+            case WORSE_RESULT:
+            {
+                ll_node* tmp;
+                cur_node->res = cur_node->next->res;
+                tmp = cur_node->next;
+                cur_node->next = cur_node->next->next;
+                free(tmp);
+                break; 
+            }
+            case SPEED_RESULT:
+                cur_node = cur_node->next;
+            case SIZE_RESULT: /* insert after first size result, then return */
+            {
+                ll_node* newnode = malloc(sizeof(ll_node));
+                if(newnode == NULL) {
+                    return 1;
+                }
+                newnode->res = cur_node->res;
+                cur_node->res = w;
+                newnode->next = cur_node->next;
+                cur_node->next = newnode;
+                return 0;
+            }
+        } 
+
+    }
+
+    //assert(cur_node->next == NULL)
+    switch(speedSizeCompare(r, cur_node->res.result)) {
+        case BETTER_RESULT:
+        {
+            return 1; /* never insert if better */
+        }
+        case WORSE_RESULT:
+        {
+            cur_node->res = w;
+            return 0;
+        }
+        case SPEED_RESULT:
+        {
+            ll_node* newnode = malloc(sizeof(ll_node));
+            if(newnode == NULL) {
+                return 1;
+            }
+            newnode->res = w;
+            newnode->next = NULL;
+            cur_node->next = newnode;
+            return 0;
+        }
+        case SIZE_RESULT: /* insert before first size result, then return */
+        {
+            ll_node* newnode = malloc(sizeof(ll_node));
+            if(newnode == NULL) {
+                return 1;
+            }
+            newnode->res = cur_node->res;
+            cur_node->res = w;
+            newnode->next = cur_node->next;
+            cur_node->next = newnode;
+            return 0;
+        }
+        default: 
+            return 1;
+    } 
+}
+
 static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result, const ZSTD_compressionParameters params, const size_t srcSize)
 {
     char lvlstr[15] = "Custom Level";
@@ -658,6 +807,29 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res
     /* global winner used for constraints */
     static winnerInfo_t g_winner = { { 0, 0, (size_t)-1, (size_t)-1 } , { 0, 0, 0, 0, 0, 0, ZSTD_fast } }; 
     
+    /* print lvl if optmode */
+    if(g_lvltarget.cSize != 0) {
+        winnerInfo_t w;
+        ll_node* n;
+        int i;
+        w.result = result;
+        w.params = params;
+        i = insertWinner(w);
+        if(i) return;
+
+        fprintf(f, "\033c"); 
+        for(n = g_winners; n != NULL; n = n->next) {
+             DISPLAY("\r%79s\r", "");
+             fprintf(f,"    {%3u,%3u,%3u,%3u,%3u,%3u, %s },  ",
+                params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength,
+                params.targetLength, g_stratName[(U32)(params.strategy)]);
+            fprintf(f,
+            "   /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n",
+            (double)srcSize / result.cSize, result.cSpeed / (1 << 20), result.dSpeed / (1 << 20));
+        }
+        return;
+    }
+
     if(DEBUG || compareResultLT(g_winner.result, result, targetConstraints, srcSize)) {
         if(DEBUG && compareResultLT(g_winner.result, result, targetConstraints, srcSize)) {
             DISPLAY("New Winner: \n");
@@ -1334,12 +1506,6 @@ int benchFiles(const char** fileNamesTable, int nbFiles)
     return 0;
 }
 
-
-
-#define WORSE_RESULT 0
-#define BETTER_RESULT 1
-#define ERROR_RESULT 2
-
 /* Benchmarking which stops when we are sufficiently sure the solution is infeasible / worse than the winner */
 #define VARIANCE 1.1 
 static int allBench(BMK_result_t* resultPtr,
@@ -1939,6 +2105,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
         goto _cleanUp;
     }
  
+    /* use level'ing mode instead of normal target mode */
     if(cLevel) {
         winner.params = ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize);
         if(BMK_benchParam(&winner.result, buf, ctx, winner.params)) {
@@ -1947,6 +2114,11 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
         }
 
         target.cSpeed = (U32)winner.result.cSpeed;
+
+        g_targetConstraints = target;
+        
+        g_lvltarget = winner.result;
+
         BMK_printWinnerOpt(stdout, cLevel, winner.result, winner.params, target, buf.srcSize);
     }
 
@@ -2137,8 +2309,8 @@ int main(int argc, const char** argv)
     U32 main_pause = 0;
     int optimizerCLevel = 0;
 
-
     constraint_t target = { 0, 0, (U32)-1 }; 
+
     ZSTD_compressionParameters paramTarget = { 0, 0, 0, 0, 0, 0, 0 };
 
     assert(argc>=1);   /* for exename */

From 5f4502fc0775f10175e759a7987289a80d9dbd92 Mon Sep 17 00:00:00 2001
From: George Lu <gclu@fb.com>
Date: Tue, 24 Jul 2018 17:55:17 -0700
Subject: [PATCH 04/15] New climb

feas part 2 uses euclidean metric
---
 tests/paramgrill.c | 31 ++++++++++++++++++++++---------
 1 file changed, 22 insertions(+), 9 deletions(-)

diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index 5c654d99..ffb5c1f7 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -1518,6 +1518,7 @@ static int allBench(BMK_result_t* resultPtr,
     U64 loopDurationC = 0, loopDurationD = 0;
     double uncertaintyConstantC, uncertaintyConstantD;
     double winnerRS;
+    int lvlmode = g_lvltarget.cSize != 0;
 
     /* initial benchmarking, gives exact ratio and memory, warms up future runs */
     benchres = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_both, BMK_iterMode, 1);
@@ -1548,9 +1549,11 @@ static int allBench(BMK_result_t* resultPtr,
         uncertaintyConstantD = 3;
     }
 
+    if(!lvlmode) {
     /* anything with worse ratio in feas is definitely worse, discard */
-    if(feas && benchres.result.cSize < winnerResult->cSize) {
-        return WORSE_RESULT;
+        if(feas && benchres.result.cSize < winnerResult->cSize) {
+            return WORSE_RESULT;
+        }
     }
 
     /* second run, if first run is too short, gives approximate cSpeed + dSpeed */
@@ -1578,8 +1581,9 @@ static int allBench(BMK_result_t* resultPtr,
 
     /* disregard infeasible results in feas mode */
     /* disregard if resultMax < winner in infeas mode */
-    if((feas && !feasible(resultMax, target)) || 
-      (!feas && (winnerRS > resultScore(resultMax, buf.srcSize, target)))) {
+    if((feas && (!lvlmode && !feasible(resultMax, target))) ||
+      (!feas && ((!lvlmode && winnerRS > resultScore(resultMax, buf.srcSize, target)) || 
+        (lvlmode && resultDistLvl(*winnerResult, g_lvltarget) > resultDistLvl(resultMax, g_lvltarget))))) {
         return WORSE_RESULT;
     }
 
@@ -1604,11 +1608,20 @@ static int allBench(BMK_result_t* resultPtr,
 
     /* compare by resultScore when in infeas */
     /* compare by compareResultLT when in feas */
-    if((!feas && (resultScore(benchres.result, buf.srcSize, target) > resultScore(*winnerResult, buf.srcSize, target))) || 
-       (feas && (compareResultLT(*winnerResult, benchres.result, target, buf.srcSize))) )  { 
-        return BETTER_RESULT; 
-    } else { 
-        return WORSE_RESULT; 
+    if(!lvlmode) {
+        if((!feas && (resultScore(benchres.result, buf.srcSize, target) > resultScore(*winnerResult, buf.srcSize, target))) || 
+           (feas && (compareResultLT(*winnerResult, benchres.result, target, buf.srcSize))) )  { 
+            return BETTER_RESULT; 
+        } else { 
+            return WORSE_RESULT; 
+        }
+    } else {
+        if((feas && (compareResultLT2(*winnerResult, benchres.result, g_lvltarget, buf.srcSize))) || 
+            (!feas && (resultScore(benchres.result, buf.srcSize, target) > resultScore(*winnerResult, buf.srcSize, target)))) {
+            return BETTER_RESULT;
+        } else {
+            return WORSE_RESULT;
+        }
     }
 
 }

From f67d040c39af10f68d7b6c61b06e72e1273551ff Mon Sep 17 00:00:00 2001
From: George Lu <gclu@fb.com>
Date: Wed, 25 Jul 2018 11:37:20 -0700
Subject: [PATCH 05/15] Bugfixes, style changes

Complete euclidean distance climb
---
 tests/paramgrill.c | 191 +++++++++++++++++++++++++--------------------
 1 file changed, 108 insertions(+), 83 deletions(-)

diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index ffb5c1f7..0dbe6653 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -112,6 +112,36 @@ static U32 g_noSeed = 0;
 static ZSTD_compressionParameters g_params = { 0, 0, 0, 0, 0, 0, ZSTD_greedy };
 static UTIL_time_t g_time; /* to be used to compare solution finding speeds to compare to original */
 
+
+typedef struct {
+    BMK_result_t result;
+    ZSTD_compressionParameters params;
+} winnerInfo_t;
+
+/* global winner used for display. */
+//Should be totally 0 initialized? 
+static winnerInfo_t g_winner = { { 0, 0, (size_t)-1, (size_t)-1 } , { 0, 0, 0, 0, 0, 0, ZSTD_fast } }; 
+
+typedef struct {
+    U32 cSpeed; /* bytes / sec */
+    U32 dSpeed;
+    U32 cMem;    /* bytes */    
+} constraint_t;
+
+static constraint_t g_targetConstraints; 
+
+typedef struct ll_node ll_node;
+struct ll_node {
+    winnerInfo_t res;
+    ll_node* next;
+};
+
+static ll_node* g_winners; /* linked list sorted ascending by cSize & cSpeed */
+static BMK_result_t g_lvltarget;
+
+/* range 0 - 99 */
+static U32 g_strictness = 99;
+
 void BMK_SetNbIterations(int nbLoops)
 {
     g_nbIterations = nbLoops;
@@ -197,12 +227,6 @@ static void findClockGranularity(void) {
     DEBUGOUTPUT("Granularity: %llu\n", (unsigned long long)g_clockGranularity);
 }
 
-typedef struct {
-    U32 cSpeed; /* bytes / sec */
-    U32 dSpeed;
-    U32 cMem;    /* bytes */    
-} constraint_t;
-
 #define CLAMPCHECK(val,min,max) {                     \
     if (val && (((val)<(min)) | ((val)>(max)))) {     \
         DISPLAY("INVALID PARAMETER CONSTRAINTS\n");   \
@@ -246,7 +270,7 @@ static void BMK_translateAdvancedParams(const ZSTD_compressionParameters params)
 
 /* checks results are feasible */
 static int feasible(const BMK_result_t results, const constraint_t target) {
-    return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.cMem);
+    return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.cMem) && (!g_lvltarget.cSize || results.cSize <= g_lvltarget.cSize);
 }
 
 /* hill climbing value for part 1 */
@@ -269,44 +293,33 @@ static double resultScore(const BMK_result_t res, const size_t srcSize, const co
     return ret;
 }
 
-/* return true if r2 strictly better than r1 */ 
-static int compareResultLT(const BMK_result_t result1, const BMK_result_t result2, const constraint_t target, size_t srcSize) {
-    if(feasible(result1, target) && feasible(result2, target)) {
-        return (result1.cSize > result2.cSize) || (result1.cSize == result2.cSize && result2.cSpeed > result1.cSpeed)
-        || (result1.cSize == result2.cSize && result2.cSpeed == result1.cSpeed && result2.dSpeed > result1.dSpeed);
-    }
-    return feasible(result2, target) || (!feasible(result1, target) && (resultScore(result1, srcSize, target) < resultScore(result2, srcSize, target)));
-
-}
-
 /* calculates normalized euclidean distance of result1 if it is in the first quadrant relative to lvlRes */
 static double resultDistLvl(const BMK_result_t result1, const BMK_result_t lvlRes) {
-    double normalizedCSpeedGain1 = result1.cSpeed / lvlRes.cSpeed - 1;
-    double normalizedRatioGain1 = lvlRes.cSize / result1.cSize - 1;
-    if(normalizedRatioGain1 < 0 || normalizedRatioGain1 < 0) {
+    double normalizedCSpeedGain1 = (result1.cSpeed / lvlRes.cSpeed) - 1;
+    double normalizedRatioGain1 = ((double)lvlRes.cSize / result1.cSize) - 1;
+    if(normalizedRatioGain1 < 0 || normalizedCSpeedGain1 < 0) {
         return 0.0;
     }
     return normalizedRatioGain1 * normalizedRatioGain1 + normalizedCSpeedGain1 * normalizedCSpeedGain1;
 }
 
-static int lvlFeasible(const BMK_result_t result, const BMK_result_t lvlRes) {
-    return lvlRes.cSpeed < result.cSpeed && lvlRes.cSize > result.cSize;
-}
-
-/* redefines feasibility for lvl mode */
-static int compareResultLT2(const BMK_result_t result1, const BMK_result_t result2, const BMK_result_t lvltarget, size_t srcSize) {
-    constraint_t target = { (U32)lvltarget.cSpeed, 0, (U32)-1 };
-    if(lvlFeasible(result1, lvltarget) && lvlFeasible(result2, lvltarget)) {
-        return resultDistLvl(result1, lvltarget) < resultDistLvl(result2, lvltarget);
+/* return true if r2 strictly better than r1 */ 
+static int compareResultLT(const BMK_result_t result1, const BMK_result_t result2, const constraint_t target, size_t srcSize) {
+    if(feasible(result1, target) && feasible(result2, target)) {
+        if(g_lvltarget.cSize == 0) {
+            return (result1.cSize > result2.cSize) || (result1.cSize == result2.cSize && result2.cSpeed > result1.cSpeed)
+            || (result1.cSize == result2.cSize && result2.cSpeed == result1.cSpeed && result2.dSpeed > result1.dSpeed);
+        } else {
+            return resultDistLvl(result1, g_lvltarget) < resultDistLvl(result2, g_lvltarget);
+        }
     }
-    return lvlFeasible(result2, lvltarget) || (!lvlFeasible(result1, lvltarget) && (resultScore(result1, srcSize, target) < resultScore(result2, srcSize, target)));
+    return feasible(result2, target) || (!feasible(result1, target) && (resultScore(result1, srcSize, target) < resultScore(result2, srcSize, target)));
 }
 
-/* factor sort of arbitrary */
 static constraint_t relaxTarget(constraint_t target) {
     target.cMem = (U32)-1;
-    target.cSpeed *= 0.9; 
-    target.dSpeed *= 0.9;
+    target.cSpeed *= ((double)(g_strictness + 1) / 100); 
+    target.dSpeed *= ((double)(g_strictness + 1) / 100);
     return target;
 }
 
@@ -330,11 +343,6 @@ BMK_benchParam1(BMK_result_t* resultPtr,
     return res.error;
 }
 
-typedef struct {
-    BMK_result_t result;
-    ZSTD_compressionParameters params;
-} winnerInfo_t;
-
 static ZSTD_compressionParameters emptyParams(void) {
     ZSTD_compressionParameters p = { 0, 0, 0, 0, 0, 0, (ZSTD_strategy)0 };
     return p;
@@ -651,16 +659,6 @@ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t
     return results;
 }
 
-
-typedef struct ll_node ll_node;
-struct ll_node {
-    winnerInfo_t res;
-    ll_node* next;
-};
-
-static ll_node* g_winners; /* linked list sorted ascending by cSize & cSpeed */
-static BMK_result_t g_lvltarget;
-
 /* comparison function: */
 /* strictly better, strictly worse, equal, speed-side adv, size-side adv */
 //Maybe use compress_only for benchmark first run?
@@ -691,7 +689,7 @@ static int insertWinner(winnerInfo_t w) {
     BMK_result_t r = w.result;
     ll_node* cur_node = g_winners;
     /* first node to insert */
-    if(!lvlFeasible(r, g_lvltarget)) {
+    if(!feasible(r, g_targetConstraints)) {
         return 1;
     }
 
@@ -722,7 +720,10 @@ static int insertWinner(winnerInfo_t w) {
                 break; 
             }
             case SPEED_RESULT:
+            {
                 cur_node = cur_node->next;
+                break;
+            }
             case SIZE_RESULT: /* insert after first size result, then return */
             {
                 ll_node* newnode = malloc(sizeof(ll_node));
@@ -843,6 +844,34 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res
             g_winner.params = params;
         }
     }  
+
+    //prints out tradeoff table if using lvl
+    if(g_lvltarget.cSize != 0) {
+        winnerInfo_t w;
+        ll_node* n;
+        int i;
+        w.result = result;
+        w.params = params;
+        i = insertWinner(w);
+        if(i) return;
+
+        if(!DEBUG) { fprintf(f, "\033c"); }
+        fprintf(f, "\n"); 
+
+        /* the table */
+        fprintf(f, "================================\n");
+        for(n = g_winners; n != NULL; n = n->next) {
+            DISPLAY("\r%79s\r", "");
+
+            fprintf(f,"    {%3u,%3u,%3u,%3u,%3u,%3u, %s },  ",
+                n->res.params.windowLog, n->res.params.chainLog, n->res.params.hashLog, n->res.params.searchLog, n->res.params.searchLength,
+                n->res.params.targetLength, g_stratName[(U32)(n->res.params.strategy)]);
+            fprintf(f,
+            "   /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n",
+            (double)srcSize / n->res.result.cSize, n->res.result.cSpeed / (1 << 20), n->res.result.dSpeed / (1 << 20));
+        }
+        fprintf(f, "================================\n");
+    }
 }
 
 static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSize)
@@ -1518,7 +1547,6 @@ static int allBench(BMK_result_t* resultPtr,
     U64 loopDurationC = 0, loopDurationD = 0;
     double uncertaintyConstantC, uncertaintyConstantD;
     double winnerRS;
-    int lvlmode = g_lvltarget.cSize != 0;
 
     /* initial benchmarking, gives exact ratio and memory, warms up future runs */
     benchres = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_both, BMK_iterMode, 1);
@@ -1549,11 +1577,9 @@ static int allBench(BMK_result_t* resultPtr,
         uncertaintyConstantD = 3;
     }
 
-    if(!lvlmode) {
     /* anything with worse ratio in feas is definitely worse, discard */
-        if(feas && benchres.result.cSize < winnerResult->cSize) {
-            return WORSE_RESULT;
-        }
+    if(feas && benchres.result.cSize < winnerResult->cSize && g_lvltarget.cSize == 0) {
+        return WORSE_RESULT;
     }
 
     /* second run, if first run is too short, gives approximate cSpeed + dSpeed */
@@ -1581,9 +1607,8 @@ static int allBench(BMK_result_t* resultPtr,
 
     /* disregard infeasible results in feas mode */
     /* disregard if resultMax < winner in infeas mode */
-    if((feas && (!lvlmode && !feasible(resultMax, target))) ||
-      (!feas && ((!lvlmode && winnerRS > resultScore(resultMax, buf.srcSize, target)) || 
-        (lvlmode && resultDistLvl(*winnerResult, g_lvltarget) > resultDistLvl(resultMax, g_lvltarget))))) {
+    if((feas && !feasible(resultMax, target)) ||
+      (!feas && (winnerRS > resultScore(resultMax, buf.srcSize, target)))) {
         return WORSE_RESULT;
     }
 
@@ -1608,22 +1633,12 @@ static int allBench(BMK_result_t* resultPtr,
 
     /* compare by resultScore when in infeas */
     /* compare by compareResultLT when in feas */
-    if(!lvlmode) {
-        if((!feas && (resultScore(benchres.result, buf.srcSize, target) > resultScore(*winnerResult, buf.srcSize, target))) || 
-           (feas && (compareResultLT(*winnerResult, benchres.result, target, buf.srcSize))) )  { 
-            return BETTER_RESULT; 
-        } else { 
-            return WORSE_RESULT; 
-        }
-    } else {
-        if((feas && (compareResultLT2(*winnerResult, benchres.result, g_lvltarget, buf.srcSize))) || 
-            (!feas && (resultScore(benchres.result, buf.srcSize, target) > resultScore(*winnerResult, buf.srcSize, target)))) {
-            return BETTER_RESULT;
-        } else {
-            return WORSE_RESULT;
-        }
+    if((!feas && (resultScore(benchres.result, buf.srcSize, target) > resultScore(*winnerResult, buf.srcSize, target))) || 
+       (feas && (compareResultLT(*winnerResult, benchres.result, target, buf.srcSize))) )  { 
+        return BETTER_RESULT; 
+    } else { 
+        return WORSE_RESULT; 
     }
-
 }
 
 #define INFEASIBLE_THRESHOLD 200
@@ -1655,6 +1670,7 @@ static int benchMemo(BMK_result_t* resultPtr,
     return res;
 }
 
+
 /* One iteration of hill climbing. Specifically, it first tries all 
  * valid parameter configurations w/ manhattan distance 1 and picks the best one
  * failing that, it progressively tries candidates further and further away (up to #dim + 2)
@@ -1667,6 +1683,10 @@ static int benchMemo(BMK_result_t* resultPtr,
  * Phase 2 optimizes in accordance with what the original function sets out to maximize, with
  * all feasible solutions valued over all infeasible solutions.
  */
+
+/* sanitize all params here. 
+ * all generation after random should be sanitized. (maybe sanitize random)
+ */
 static winnerInfo_t climbOnce(const constraint_t target, 
                 const varInds_t* varArray, const int varLen, 
                 U8* const memoTable,
@@ -1800,6 +1820,7 @@ static winnerInfo_t optimizeFixedStrategy(
             BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, target, buf.srcSize);
             i = 0;
         }
+
         i++;
     }
     return winnerInfo;
@@ -2129,8 +2150,10 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
         target.cSpeed = (U32)winner.result.cSpeed;
 
         g_targetConstraints = target;
-        
-        g_lvltarget = winner.result;
+
+        g_lvltarget = winner.result; 
+        g_lvltarget.cSpeed *= ((double)(g_strictness + 1) / 100);
+        g_lvltarget.cSize /= ((double)(g_strictness + 1) / 100);
 
         BMK_printWinnerOpt(stdout, cLevel, winner.result, winner.params, target, buf.srcSize);
     }
@@ -2299,15 +2322,16 @@ static int badusage(const char* exename)
 }
 
 #define PARSE_SUB_ARGS(stringLong, stringShort, variable) { if (longCommandWArg(&argument, stringLong) || longCommandWArg(&argument, stringShort)) { variable = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } }
-#define PARSE_CPARAMS(variable)                                          \
-{                                                                        \
-    PARSE_SUB_ARGS("windowLog=",     "wlog=",  variable.windowLog);      \
-    PARSE_SUB_ARGS("chainLog=" ,     "clog=",  variable.chainLog);       \
-    PARSE_SUB_ARGS("hashLog=",       "hlog=",  variable.hashLog);        \
-    PARSE_SUB_ARGS("searchLog=" ,    "slog=",  variable.searchLog);      \
-    PARSE_SUB_ARGS("searchLength=",  "slen=",  variable.searchLength);   \
-    PARSE_SUB_ARGS("targetLength=" , "tlen=",  variable.targetLength);   \
-    PARSE_SUB_ARGS("strategy=",      "strat=", variable.strategy);       \
+#define PARSE_CPARAMS(variable)                                            \
+{                                                                          \
+    PARSE_SUB_ARGS("windowLog=",       "wlog=",  variable.vals[wlog_ind]); \
+    PARSE_SUB_ARGS("chainLog=" ,       "clog=",  variable.vals[clog_ind]); \
+    PARSE_SUB_ARGS("hashLog=",         "hlog=",  variable.vals[hlog_ind]); \
+    PARSE_SUB_ARGS("searchLog=" ,      "slog=",  variable.vals[slog_ind]); \
+    PARSE_SUB_ARGS("searchLength=",    "slen=",  variable.vals[slen_ind]); \
+    PARSE_SUB_ARGS("targetLength=" ,   "tlen=",  variable.vals[tlen_ind]); \
+    PARSE_SUB_ARGS("strategy=",        "strat=", variable.vals[strt_ind]); \
+    PARSE_SUB_ARGS("forceAttachDict=", "fad="  , variable.vals[strt_ind]); \
 }
 
 int main(int argc, const char** argv)
@@ -2350,6 +2374,7 @@ int main(int argc, const char** argv)
                 PARSE_SUB_ARGS("decompressionSpeed=", "dSpeed=", target.dSpeed); 
                 PARSE_SUB_ARGS("compressionMemory=" , "cMem=", target.cMem);
                 PARSE_SUB_ARGS("level=", "lvl=", optimizerCLevel);
+                PARSE_SUB_ARGS("strict=", "stc=", g_strictness);
                 DISPLAY("invalid optimization parameter \n");
                 return 1;
             }

From 2bdfe6ca71d2171c02468214759080bc1d3e7e3a Mon Sep 17 00:00:00 2001
From: George Lu <gclu@fb.com>
Date: Wed, 25 Jul 2018 11:55:09 -0700
Subject: [PATCH 06/15] Better Display

---
 tests/paramgrill.c | 34 ++++++++++++++++++++++++++++------
 1 file changed, 28 insertions(+), 6 deletions(-)

diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index 0dbe6653..fa5a5390 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -293,7 +293,7 @@ static double resultScore(const BMK_result_t res, const size_t srcSize, const co
     return ret;
 }
 
-/* calculates normalized euclidean distance of result1 if it is in the first quadrant relative to lvlRes */
+/* calculates normalized squared euclidean distance of result1 if it is in the first quadrant relative to lvlRes */
 static double resultDistLvl(const BMK_result_t result1, const BMK_result_t lvlRes) {
     double normalizedCSpeedGain1 = (result1.cSpeed / lvlRes.cSpeed) - 1;
     double normalizedRatioGain1 = ((double)lvlRes.cSize / result1.cSize) - 1;
@@ -668,6 +668,7 @@ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t
 
 #define SPEED_RESULT 4
 #define SIZE_RESULT 5
+/* maybe have epsilon-eq to limit table size? */
 static int speedSizeCompare(BMK_result_t r1, BMK_result_t r2) {
     if(r1.cSpeed > r2.cSpeed) {
         if(r1.cSize <= r2.cSize) {
@@ -853,11 +854,11 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res
         w.result = result;
         w.params = params;
         i = insertWinner(w);
-        if(i) return;
+        //if(i) return;
 
         if(!DEBUG) { fprintf(f, "\033c"); }
         fprintf(f, "\n"); 
-
+        
         /* the table */
         fprintf(f, "================================\n");
         for(n = g_winners; n != NULL; n = n->next) {
@@ -871,6 +872,27 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res
             (double)srcSize / n->res.result.cSize, n->res.result.cSpeed / (1 << 20), n->res.result.dSpeed / (1 << 20));
         }
         fprintf(f, "================================\n");
+        fprintf(f, "Level Bounds: R: > %.3f AND C: < %.1f MB/s \n\n",
+            (double)srcSize / g_lvltarget.cSize, g_lvltarget.cSpeed / (1 << 20));
+
+
+        fprintf(f, "Overall Winner: \n");
+        fprintf(f,"    {%3u,%3u,%3u,%3u,%3u,%3u, %s },  ",
+            g_winner.params.windowLog, g_winner.params.chainLog, g_winner.params.hashLog, g_winner.params.searchLog, g_winner.params.searchLength,
+            g_winner.params.targetLength, g_stratName[(U32)(g_winner.params.strategy)]);
+        fprintf(f,
+        "   /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n",
+        (double)srcSize / g_winner.result.cSize, g_winner.result.cSpeed / (1 << 20), g_winner.result.dSpeed / (1 << 20));
+
+
+        fprintf(f, "Latest BMK: \n");
+        fprintf(f,"    {%3u,%3u,%3u,%3u,%3u,%3u, %s },  ",
+            params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength,
+            params.targetLength, g_stratName[(U32)(params.strategy)]);
+        fprintf(f,
+            "   /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n",
+            (double)srcSize / result.cSize, result.cSpeed / (1 << 20), result.dSpeed / (1 << 20));
+
     }
 }
 
@@ -1707,7 +1729,7 @@ static winnerInfo_t climbOnce(const constraint_t target,
 
     {
         winnerInfo_t bestFeasible1 = initWinnerInfo(cparam);
-        DISPLAY("Climb Part 1\n");
+        DEBUGOUTPUT("Climb Part 1\n");
         while(better) {
 
             int i, dist, offset;
@@ -1774,7 +1796,7 @@ static winnerInfo_t climbOnce(const constraint_t target,
                 feas = 1;
                 better = 1;
                 winnerInfo = bestFeasible1; /* note with change, bestFeasible may not necessarily be feasible, but if one has been benchmarked, it will be. */
-                DISPLAY("Climb Part 2\n");
+                DEBUGOUTPUT("Climb Part 2\n");
             }
         }
         winnerInfo = bestFeasible1;
@@ -2218,9 +2240,9 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
                 }
 
                 while(st && tries) {
+                    DEBUGOUTPUT("StrategySwitch: %s\n", g_stratName[st]);
                     winnerInfo_t wc = optimizeFixedStrategy(buf, ctx, target, paramTarget, 
                         st, varArray, varLen, allMT[st], tries);
-                    DEBUGOUTPUT("StratNum %d\n", st);
                     if(compareResultLT(winner.result, wc.result, target, buf.srcSize)) {
                         winner = wc;
                     }

From 3a2e95eba45a28afa57afae00dac15dce29959e9 Mon Sep 17 00:00:00 2001
From: George Lu <gclu@fb.com>
Date: Thu, 26 Jul 2018 16:45:00 -0700
Subject: [PATCH 07/15] Perf improvements

try decay
strategy selection skipping
---
 tests/paramgrill.c | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index fa5a5390..0db8510e 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -2079,7 +2079,9 @@ static ZSTD_compressionParameters maskParams(ZSTD_compressionParameters base, ZS
     return base;
 }
 
+/* experiment with playing with this and decay value */
 #define MAX_TRIES 8
+#define TRY_DECAY 3
 /* main fn called when using --optimize */
 /* Does strategy selection by benchmarking default compression levels
  * then optimizes by strategy, starting with the best one and moving 
@@ -2092,6 +2094,7 @@ static ZSTD_compressionParameters maskParams(ZSTD_compressionParameters base, ZS
  * paramTarget - parameter constraints (i.e. restriction search space to where strategy = ZSTD_fast)
  * cLevel - compression level to exceed (all solutions must be > lvl in cSpeed + ratio)
  */
+
 static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget, int cLevel)
 {
     varInds_t varArray [NUM_PARAMS];
@@ -2202,7 +2205,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
             /* strategy selection */
             const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel();
             DEBUGOUTPUT("Strategy Selection\n");
-            if(paramTarget.strategy == 0) { /* no variable based constraints */  
+            if(paramTarget.strategy == 0) {
                 BMK_result_t candidate;
                 int i;
                 for (i=1; i<=maxSeeds; i++) {
@@ -2216,6 +2219,11 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
                         winner.result = candidate;
                         winner.params = CParams;
                     }
+
+                    /* if the current params are too slow, just stop. */
+                    if(target.cSpeed != 0 && target.cSpeed > winner.result.cSpeed / 2) {
+                        break;
+                    }
                 }
             }
         }
@@ -2248,7 +2256,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
                     }
 
                     st = nextStrategy(st, bestStrategy);
-                    tries--;
+                    tries -= TRY_DECAY;
                 }
             } else {
                 winner = optimizeFixedStrategy(buf, ctx, target, paramTarget, paramTarget.strategy, 

From 8ff0de15e4484b656dc199e118b063b3665cb2dd Mon Sep 17 00:00:00 2001
From: George Lu <gclu@fb.com>
Date: Fri, 27 Jul 2018 08:20:31 -0700
Subject: [PATCH 08/15] Generalize, macro magic numbers

---
 tests/README.md    |  3 +++
 tests/paramgrill.c | 54 +++++++++++++++++++++++++++++++---------------
 2 files changed, 40 insertions(+), 17 deletions(-)

diff --git a/tests/README.md b/tests/README.md
index 2f0026fd..04eb5094 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -112,6 +112,9 @@ Full list of arguments
     dSpeed= - Minimum decompression speed
     cMem= - compression memory
     lvl= - Automatically sets compression speed constraint to the speed of that level
+    stc= - In lvl mode, represents slack in ratio/cSpeed allowed for a solution to be considered
+         - In normal operation, represents slack in strategy selection in choosing the default parameters
+ --optimize=  : same as -O with more verbose syntax 
  -P#          : generated sample compressibility 
  -t#          : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours )) 
  -v           : Prints Benchmarking output
diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index 0db8510e..fca6b200 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -139,8 +139,9 @@ struct ll_node {
 static ll_node* g_winners; /* linked list sorted ascending by cSize & cSpeed */
 static BMK_result_t g_lvltarget;
 
-/* range 0 - 99 */
-static U32 g_strictness = 99;
+/* range 0 - 99, measure of how strict  */
+#define DEFAULT_STRICTNESS 99999
+static U32 g_strictness = DEFAULT_STRICTNESS;
 
 void BMK_SetNbIterations(int nbLoops)
 {
@@ -318,8 +319,8 @@ static int compareResultLT(const BMK_result_t result1, const BMK_result_t result
 
 static constraint_t relaxTarget(constraint_t target) {
     target.cMem = (U32)-1;
-    target.cSpeed *= ((double)(g_strictness + 1) / 100); 
-    target.dSpeed *= ((double)(g_strictness + 1) / 100);
+    target.cSpeed *= ((double)(g_strictness) / 100); 
+    target.dSpeed *= ((double)(g_strictness) / 100);
     return target;
 }
 
@@ -2080,8 +2081,7 @@ static ZSTD_compressionParameters maskParams(ZSTD_compressionParameters base, ZS
 }
 
 /* experiment with playing with this and decay value */
-#define MAX_TRIES 8
-#define TRY_DECAY 3
+
 /* main fn called when using --optimize */
 /* Does strategy selection by benchmarking default compression levels
  * then optimizes by strategy, starting with the best one and moving 
@@ -2095,6 +2095,9 @@ static ZSTD_compressionParameters maskParams(ZSTD_compressionParameters base, ZS
  * cLevel - compression level to exceed (all solutions must be > lvl in cSpeed + ratio)
  */
 
+#define MAX_TRIES 3
+#define TRY_DECAY 1
+
 static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget, int cLevel)
 {
     varInds_t varArray [NUM_PARAMS];
@@ -2164,6 +2167,21 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
         goto _cleanUp;
     }
  
+    /* default strictness = Maximum for */
+    if(g_strictness == DEFAULT_STRICTNESS) {
+        if(cLevel) {
+            g_strictness = 99;
+        } else {
+            g_strictness = 90;
+        }
+    } else {
+        if(0 >= g_strictness || g_strictness > 100) {
+            DISPLAY("Strictness Outside of Bounds\n");
+            ret = 4;
+            goto _cleanUp;
+        }
+    }
+
     /* use level'ing mode instead of normal target mode */
     if(cLevel) {
         winner.params = ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize);
@@ -2177,8 +2195,8 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
         g_targetConstraints = target;
 
         g_lvltarget = winner.result; 
-        g_lvltarget.cSpeed *= ((double)(g_strictness + 1) / 100);
-        g_lvltarget.cSize /= ((double)(g_strictness + 1) / 100);
+        g_lvltarget.cSpeed *= ((double)(g_strictness) / 100);
+        g_lvltarget.cSize /= ((double)(g_strictness) / 100);
 
         BMK_printWinnerOpt(stdout, cLevel, winner.result, winner.params, target, buf.srcSize);
     }
@@ -2199,6 +2217,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
 
     {   
         varInds_t varNew[NUM_PARAMS];
+        ZSTD_compressionParameters CParams;
 
         /* find best solution from default params */
         {
@@ -2210,8 +2229,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
                 int i;
                 for (i=1; i<=maxSeeds; i++) {
                     int ec;
-                    ZSTD_compressionParameters CParams = ZSTD_getCParams(i, maxBlockSize, ctx.dictSize);
-                    CParams = maskParams(CParams, paramTarget);
+                    CParams = maskParams(ZSTD_getCParams(i, maxBlockSize, ctx.dictSize), paramTarget);
                     ec = BMK_benchParam(&candidate, buf, ctx, CParams);
                     BMK_printWinnerOpt(stdout, i, candidate, CParams, target, buf.srcSize);
 
@@ -2221,9 +2239,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
                     }
 
                     /* if the current params are too slow, just stop. */
-                    if(target.cSpeed != 0 && target.cSpeed > winner.result.cSpeed / 2) {
-                        break;
-                    }
+                    if(target.cSpeed > candidate.cSpeed * 2) { break; }
                 }
             }
         }
@@ -2239,6 +2255,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
                 int tries = MAX_TRIES;
 
                 { 
+                    /* one iterations of hill climbing with the level-defined parameters. */
                     int varLenNew = sanitizeVarArray(varNew, varLen, varArray, st);
                     winnerInfo_t w1 = climbOnce(target, varNew, varLenNew, allMT[st], 
                         buf, ctx, winner.params);
@@ -2247,16 +2264,19 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
                     }
                 }
 
-                while(st && tries) {
+                while(st && tries > 0) {
                     DEBUGOUTPUT("StrategySwitch: %s\n", g_stratName[st]);
                     winnerInfo_t wc = optimizeFixedStrategy(buf, ctx, target, paramTarget, 
                         st, varArray, varLen, allMT[st], tries);
+
                     if(compareResultLT(winner.result, wc.result, target, buf.srcSize)) {
                         winner = wc;
+                        tries = MAX_TRIES;
+                        bestStrategy = st;
+                    } else {
+                        st = nextStrategy(st, bestStrategy);
+                        tries -= TRY_DECAY;
                     }
-
-                    st = nextStrategy(st, bestStrategy);
-                    tries -= TRY_DECAY;
                 }
             } else {
                 winner = optimizeFixedStrategy(buf, ctx, target, paramTarget, paramTarget.strategy, 

From b3544217b77bcff66b7bf754a4f7b745a7d0493a Mon Sep 17 00:00:00 2001
From: George Lu <gclu@fb.com>
Date: Fri, 27 Jul 2018 11:47:14 -0700
Subject: [PATCH 09/15] Cleanup

---
 tests/README.md    |  6 ++++--
 tests/paramgrill.c | 30 ++++++++++++++++++++----------
 2 files changed, 24 insertions(+), 12 deletions(-)

diff --git a/tests/README.md b/tests/README.md
index 04eb5094..1410ca97 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -111,9 +111,11 @@ Full list of arguments
     cSpeed= - Minimum compression speed
     dSpeed= - Minimum decompression speed
     cMem= - compression memory
-    lvl= - Automatically sets compression speed constraint to the speed of that level
-    stc= - In lvl mode, represents slack in ratio/cSpeed allowed for a solution to be considered
+    lvl= - Searches for solutions which are strictly better than that compression lvl in ratio and cSpeed, 
+    stc= - When invoked with lvl=, represents slack in ratio/cSpeed allowed for a solution to be considered
          - In normal operation, represents slack in strategy selection in choosing the default parameters
+    prefer[Speed/Ratio]= - Only affects lvl= invocations. Defines value placed on compression speed or ratio
+          when determining overall winner (default 1 for both).
  --optimize=  : same as -O with more verbose syntax 
  -P#          : generated sample compressibility 
  -t#          : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours )) 
diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index fca6b200..188b0493 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -123,7 +123,7 @@ typedef struct {
 static winnerInfo_t g_winner = { { 0, 0, (size_t)-1, (size_t)-1 } , { 0, 0, 0, 0, 0, 0, ZSTD_fast } }; 
 
 typedef struct {
-    U32 cSpeed; /* bytes / sec */
+    U32 cSpeed;  /* bytes / sec */
     U32 dSpeed;
     U32 cMem;    /* bytes */    
 } constraint_t;
@@ -138,6 +138,12 @@ struct ll_node {
 
 static ll_node* g_winners; /* linked list sorted ascending by cSize & cSpeed */
 static BMK_result_t g_lvltarget;
+static int g_optmode = 0;
+
+static U32 g_speedMultiplier = 1;
+static U32 g_ratioMultiplier = 1;
+
+/* g_mode? */
 
 /* range 0 - 99, measure of how strict  */
 #define DEFAULT_STRICTNESS 99999
@@ -271,7 +277,7 @@ static void BMK_translateAdvancedParams(const ZSTD_compressionParameters params)
 
 /* checks results are feasible */
 static int feasible(const BMK_result_t results, const constraint_t target) {
-    return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.cMem) && (!g_lvltarget.cSize || results.cSize <= g_lvltarget.cSize);
+    return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.cMem) && (!g_optmode || results.cSize <= g_lvltarget.cSize);
 }
 
 /* hill climbing value for part 1 */
@@ -291,6 +297,7 @@ static double resultScore(const BMK_result_t res, const size_t srcSize, const co
 
     ret = (MIN(1, cs) + MIN(1, ds)  + MIN(1, cm))*r1 + rt * rtr + 
          (MAX(0, log(cs))+ MAX(0, log(ds))+ MAX(0, log(cm))) * r2;
+
     return ret;
 }
 
@@ -301,17 +308,17 @@ static double resultDistLvl(const BMK_result_t result1, const BMK_result_t lvlRe
     if(normalizedRatioGain1 < 0 || normalizedCSpeedGain1 < 0) {
         return 0.0;
     }
-    return normalizedRatioGain1 * normalizedRatioGain1 + normalizedCSpeedGain1 * normalizedCSpeedGain1;
+    return normalizedRatioGain1 * g_ratioMultiplier + normalizedCSpeedGain1 * g_speedMultiplier;
 }
 
 /* return true if r2 strictly better than r1 */ 
 static int compareResultLT(const BMK_result_t result1, const BMK_result_t result2, const constraint_t target, size_t srcSize) {
     if(feasible(result1, target) && feasible(result2, target)) {
-        if(g_lvltarget.cSize == 0) {
+        if(g_optmode) {
+            return resultDistLvl(result1, g_lvltarget) < resultDistLvl(result2, g_lvltarget);
+        } else {
             return (result1.cSize > result2.cSize) || (result1.cSize == result2.cSize && result2.cSpeed > result1.cSpeed)
             || (result1.cSize == result2.cSize && result2.cSpeed == result1.cSpeed && result2.dSpeed > result1.dSpeed);
-        } else {
-            return resultDistLvl(result1, g_lvltarget) < resultDistLvl(result2, g_lvltarget);
         }
     }
     return feasible(result2, target) || (!feasible(result1, target) && (resultScore(result1, srcSize, target) < resultScore(result2, srcSize, target)));
@@ -848,7 +855,7 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res
     }  
 
     //prints out tradeoff table if using lvl
-    if(g_lvltarget.cSize != 0) {
+    if(g_optmode) {
         winnerInfo_t w;
         ll_node* n;
         int i;
@@ -1601,7 +1608,7 @@ static int allBench(BMK_result_t* resultPtr,
     }
 
     /* anything with worse ratio in feas is definitely worse, discard */
-    if(feas && benchres.result.cSize < winnerResult->cSize && g_lvltarget.cSize == 0) {
+    if(feas && benchres.result.cSize < winnerResult->cSize && !g_optmode) {
         return WORSE_RESULT;
     }
 
@@ -2169,7 +2176,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
  
     /* default strictness = Maximum for */
     if(g_strictness == DEFAULT_STRICTNESS) {
-        if(cLevel) {
+        if(g_optmode) {
             g_strictness = 99;
         } else {
             g_strictness = 90;
@@ -2183,7 +2190,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
     }
 
     /* use level'ing mode instead of normal target mode */
-    if(cLevel) {
+    if(g_optmode) {
         winner.params = ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize);
         if(BMK_benchParam(&winner.result, buf, ctx, winner.params)) {
             ret = 3;
@@ -2425,6 +2432,9 @@ int main(int argc, const char** argv)
                 PARSE_SUB_ARGS("compressionMemory=" , "cMem=", target.cMem);
                 PARSE_SUB_ARGS("level=", "lvl=", optimizerCLevel);
                 PARSE_SUB_ARGS("strict=", "stc=", g_strictness);
+                PARSE_SUB_ARGS("preferSpeed=", "prfSpd=", g_speedMultiplier);
+                PARSE_SUB_ARGS("preferRatio=", "prfRto=", g_ratioMultiplier);
+
                 DISPLAY("invalid optimization parameter \n");
                 return 1;
             }

From a884b76bc2fedd086c36b16e52b0dc43d075ca28 Mon Sep 17 00:00:00 2001
From: George Lu <gclu@fb.com>
Date: Fri, 27 Jul 2018 14:19:55 -0700
Subject: [PATCH 10/15] Style Changes

Add single run dictionaries
Change MB to be consistent 1 << 20 rather than 1,000,000
---
 tests/paramgrill.c | 481 ++++++++++++++++++++++-----------------------
 1 file changed, 238 insertions(+), 243 deletions(-)

diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index 188b0493..8df72eb0 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -107,6 +107,7 @@ static double g_compressibility = COMPRESSIBILITY_DEFAULT;
 static U32 g_blockSize = 0;
 static U32 g_rand = 1;
 static U32 g_singleRun = 0;
+static U32 g_optimizer = 0;
 static U32 g_target = 0;
 static U32 g_noSeed = 0;
 static ZSTD_compressionParameters g_params = { 0, 0, 0, 0, 0, 0, ZSTD_greedy };
@@ -118,10 +119,6 @@ typedef struct {
     ZSTD_compressionParameters params;
 } winnerInfo_t;
 
-/* global winner used for display. */
-//Should be totally 0 initialized? 
-static winnerInfo_t g_winner = { { 0, 0, (size_t)-1, (size_t)-1 } , { 0, 0, 0, 0, 0, 0, ZSTD_fast } }; 
-
 typedef struct {
     U32 cSpeed;  /* bytes / sec */
     U32 dSpeed;
@@ -386,8 +383,7 @@ typedef struct {
     ZSTD_DCtx* dctx;
 } contexts_t;
 
-static int
-BMK_benchParam(BMK_result_t* resultPtr,
+static int BMK_benchParam(BMK_result_t* resultPtr,
                 const buffers_t buf, const contexts_t ctx,
                 const ZSTD_compressionParameters cParams) {
     BMK_return_t res = BMK_benchMem(buf.srcPtrs[0], buf.srcSize, buf.srcSizes, (unsigned)buf.nbBlocks, 0, &cParams, ctx.dictBuffer, ctx.dictSize, 0, "Files");
@@ -520,6 +516,175 @@ static size_t local_defaultDecompress(
 *  From Paramgrill End
 *********************************************************/
 
+static void freeBuffers(const buffers_t b) {
+    if(b.srcPtrs != NULL) {
+        free(b.srcBuffer);
+    }
+    free(b.srcPtrs);
+    free(b.srcSizes);
+
+    if(b.dstPtrs != NULL) {
+        free(b.dstPtrs[0]);
+    }
+    free(b.dstPtrs);
+    free(b.dstCapacities);
+    free(b.dstSizes);
+
+    if(b.resPtrs != NULL) {
+        free(b.resPtrs[0]);
+    }
+    free(b.resPtrs);
+}
+
+/* allocates buffer's arguments. returns success / failuere */
+static int createBuffers(buffers_t* buff, const char* const * const fileNamesTable, 
+                          const size_t nbFiles)
+{
+    size_t pos = 0;
+    size_t n;
+    U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, (U32)nbFiles);
+    size_t benchedSize = MIN(BMK_findMaxMem(totalSizeToLoad * 3) / 3, totalSizeToLoad);
+    const size_t blockSize = g_blockSize ? g_blockSize : totalSizeToLoad; //(largest fileSize or total fileSize)
+    U32 const maxNbBlocks = (U32) ((totalSizeToLoad + (blockSize-1)) / blockSize) + (U32)nbFiles;
+    U32 blockNb = 0;
+
+    buff->srcPtrs = (const void**)calloc(maxNbBlocks, sizeof(void*));
+    buff->srcSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
+
+    buff->dstPtrs = (void**)calloc(maxNbBlocks, sizeof(void*));
+    buff->dstCapacities = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
+    buff->dstSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
+
+    buff->resPtrs = (void**)calloc(maxNbBlocks, sizeof(void*));
+    buff->resSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); 
+
+    if(!buff->srcPtrs || !buff->srcSizes || !buff->dstPtrs || !buff->dstCapacities || !buff->dstSizes || !buff->resPtrs || !buff->resSizes) {
+        DISPLAY("alloc error\n");
+        freeBuffers(*buff);
+        return 1;
+    }
+
+    buff->srcBuffer = malloc(benchedSize);
+    buff->srcPtrs[0] = (const void*)buff->srcBuffer;
+    buff->dstPtrs[0] = malloc(ZSTD_compressBound(benchedSize) + (maxNbBlocks * 1024));
+    buff->resPtrs[0] = malloc(benchedSize);
+
+    if(!buff->srcPtrs[0] || !buff->dstPtrs[0] || !buff->resPtrs[0]) {
+        DISPLAY("alloc error\n");
+        freeBuffers(*buff);
+        return 1;
+    }
+
+    for(n = 0; n < nbFiles; n++) {
+        FILE* f;
+        U64 fileSize = UTIL_getFileSize(fileNamesTable[n]);
+        if (UTIL_isDirectory(fileNamesTable[n])) {
+            DISPLAY("Ignoring %s directory...       \n", fileNamesTable[n]);
+            continue;
+        }
+        if (fileSize == UTIL_FILESIZE_UNKNOWN) {
+            DISPLAY("Cannot evaluate size of %s, ignoring ... \n", fileNamesTable[n]);
+            continue;
+        }
+        f = fopen(fileNamesTable[n], "rb");
+        if (f==NULL) {
+            DISPLAY("impossible to open file %s\n", fileNamesTable[n]);
+            freeBuffers(*buff);
+            fclose(f);
+            return 10;
+        }
+
+        DISPLAY("Loading %s...       \r", fileNamesTable[n]);
+
+        if (fileSize + pos > benchedSize) fileSize = benchedSize - pos, n=nbFiles;   /* buffer too small - stop after this file */
+        {
+            char* buffer = (char*)(buff->srcBuffer); 
+            size_t const readSize = fread((buffer)+pos, 1, (size_t)fileSize, f);
+            size_t blocked = 0;
+            while(blocked < readSize) {
+                buff->srcPtrs[blockNb] = (const void*)((buffer) + (pos + blocked));
+                buff->srcSizes[blockNb] = blockSize;
+                blocked += blockSize;
+                blockNb++;
+            }
+            if(readSize > 0) { buff->srcSizes[blockNb - 1] = ((readSize - 1) % blockSize) + 1; }
+
+            if (readSize != (size_t)fileSize) {
+                DISPLAY("could not read %s", fileNamesTable[n]);
+                freeBuffers(*buff);
+                fclose(f);
+                return 1;
+            }
+
+            pos += readSize;
+
+        }
+        fclose(f);
+    }
+
+    buff->dstCapacities[0] = ZSTD_compressBound(buff->srcSizes[0]);
+    buff->dstSizes[0] = buff->dstCapacities[0];
+    buff->resSizes[0] = buff->srcSizes[0];
+
+    for(n = 1; n < blockNb; n++) {
+        buff->dstPtrs[n] = ((char*)buff->dstPtrs[n-1]) + buff->dstCapacities[n-1];
+        buff->resPtrs[n] = ((char*)buff->resPtrs[n-1]) + buff->resSizes[n-1];
+        buff->dstCapacities[n] = ZSTD_compressBound(buff->srcSizes[n]);
+        buff->dstSizes[n] = buff->dstCapacities[n];
+        buff->resSizes[n] = buff->srcSizes[n];
+    }
+    buff->srcSize = pos;
+    buff->nbBlocks = blockNb;
+
+    if (pos == 0) { DISPLAY("\nno data to bench\n"); return 1; }
+
+    return 0;
+}
+
+static void freeContexts(const contexts_t ctx) {
+    free(ctx.dictBuffer);
+    ZSTD_freeCCtx(ctx.cctx);
+    ZSTD_freeDCtx(ctx.dctx);
+}
+
+static int createContexts(contexts_t* ctx, const char* dictFileName) {
+    FILE* f;
+    size_t readSize;
+    ctx->cctx = ZSTD_createCCtx();
+    ctx->dctx = ZSTD_createDCtx();
+    if(dictFileName == NULL) {
+        ctx->dictSize = 0;
+        ctx->dictBuffer = NULL;
+        return 0;
+    }
+    ctx->dictSize = UTIL_getFileSize(dictFileName);
+    ctx->dictBuffer = malloc(ctx->dictSize);
+
+    f = fopen(dictFileName, "rb");
+    
+    if(!f) {
+        DISPLAY("unable to open file\n");
+        fclose(f);
+        freeContexts(*ctx);
+        return 1;
+    }
+    
+    if(ctx->dictSize > 64 MB || !(ctx->dictBuffer)) {
+        DISPLAY("dictionary too large\n");
+        fclose(f);
+        freeContexts(*ctx);
+        return 1;
+    }
+    readSize = fread(ctx->dictBuffer, 1, ctx->dictSize, f);
+    if(readSize != ctx->dictSize) {
+        DISPLAY("unable to read file\n");
+        fclose(f);
+        freeContexts(*ctx);
+        return 1;
+    }
+    return 0;
+}
+
 /* Replicate functionality of benchMemAdvanced, but with pre-split src / dst buffers */
 /* The purpose is so that sufficient information is returned so that a decompression call to benchMemInvertible is possible */
 /* BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); */
@@ -788,6 +953,8 @@ static int insertWinner(winnerInfo_t w) {
     } 
 }
 
+/* Writes to f the results of a parameter benchmark */
+/* when used with --optimize, will only print results better than previously discovered */
 static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result, const ZSTD_compressionParameters params, const size_t srcSize)
 {
     char lvlstr[15] = "Custom Level";
@@ -806,7 +973,7 @@ static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result
 
     fprintf(f,
         "/* %s */   /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */",
-        lvlstr, (double)srcSize / result.cSize, (double)result.cSpeed / (1 << 20), (double)result.dSpeed / (1 << 20));
+        lvlstr, (double)srcSize / result.cSize, (double)result.cSpeed / (1 MB), (double)result.dSpeed / (1 MB));
 
     if(TIMED) { fprintf(f, " - %1lu:%2lu:%05.2f", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC); }
     fprintf(f, "\n"); 
@@ -816,29 +983,6 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res
 {
     /* global winner used for constraints */
     static winnerInfo_t g_winner = { { 0, 0, (size_t)-1, (size_t)-1 } , { 0, 0, 0, 0, 0, 0, ZSTD_fast } }; 
-    
-    /* print lvl if optmode */
-    if(g_lvltarget.cSize != 0) {
-        winnerInfo_t w;
-        ll_node* n;
-        int i;
-        w.result = result;
-        w.params = params;
-        i = insertWinner(w);
-        if(i) return;
-
-        fprintf(f, "\033c"); 
-        for(n = g_winners; n != NULL; n = n->next) {
-             DISPLAY("\r%79s\r", "");
-             fprintf(f,"    {%3u,%3u,%3u,%3u,%3u,%3u, %s },  ",
-                params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength,
-                params.targetLength, g_stratName[(U32)(params.strategy)]);
-            fprintf(f,
-            "   /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n",
-            (double)srcSize / result.cSize, result.cSpeed / (1 << 20), result.dSpeed / (1 << 20));
-        }
-        return;
-    }
 
     if(DEBUG || compareResultLT(g_winner.result, result, targetConstraints, srcSize)) {
         if(DEBUG && compareResultLT(g_winner.result, result, targetConstraints, srcSize)) {
@@ -855,14 +999,12 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res
     }  
 
     //prints out tradeoff table if using lvl
-    if(g_optmode) {
+    if(g_optmode && g_optimizer) {
         winnerInfo_t w;
         ll_node* n;
-        int i;
         w.result = result;
         w.params = params;
-        i = insertWinner(w);
-        //if(i) return;
+        insertWinner(w);
 
         if(!DEBUG) { fprintf(f, "\033c"); }
         fprintf(f, "\n"); 
@@ -877,11 +1019,11 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res
                 n->res.params.targetLength, g_stratName[(U32)(n->res.params.strategy)]);
             fprintf(f,
             "   /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n",
-            (double)srcSize / n->res.result.cSize, n->res.result.cSpeed / (1 << 20), n->res.result.dSpeed / (1 << 20));
+            (double)srcSize / n->res.result.cSize, (double)n->res.result.cSpeed / (1 MB), (double)n->res.result.dSpeed / (1 MB));
         }
         fprintf(f, "================================\n");
         fprintf(f, "Level Bounds: R: > %.3f AND C: < %.1f MB/s \n\n",
-            (double)srcSize / g_lvltarget.cSize, g_lvltarget.cSpeed / (1 << 20));
+            (double)srcSize / g_lvltarget.cSize, (double)g_lvltarget.cSpeed / (1 MB));
 
 
         fprintf(f, "Overall Winner: \n");
@@ -890,8 +1032,9 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res
             g_winner.params.targetLength, g_stratName[(U32)(g_winner.params.strategy)]);
         fprintf(f,
         "   /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n",
-        (double)srcSize / g_winner.result.cSize, g_winner.result.cSpeed / (1 << 20), g_winner.result.dSpeed / (1 << 20));
+        (double)srcSize / g_winner.result.cSize, (double)g_winner.result.cSpeed / (1 MB), (double)g_winner.result.dSpeed / (1 MB));
 
+        BMK_translateAdvancedParams(g_winner.params);
 
         fprintf(f, "Latest BMK: \n");
         fprintf(f,"    {%3u,%3u,%3u,%3u,%3u,%3u, %s },  ",
@@ -899,7 +1042,7 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res
             params.targetLength, g_stratName[(U32)(params.strategy)]);
         fprintf(f,
             "   /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n",
-            (double)srcSize / result.cSize, result.cSpeed / (1 << 20), result.dSpeed / (1 << 20));
+            (double)srcSize / result.cSize, (double)result.cSpeed / (1 MB), (double)result.dSpeed / (1 MB));
 
     }
 }
@@ -1023,16 +1166,16 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para
                 /* too large compression speed difference for the compression benefit */
                 if (W_ratio > O_ratio)
                 DISPLAY ("Compression Speed : %5.3f @ %4.1f MB/s  vs  %5.3f @ %4.1f MB/s   : not enough for level %i\n",
-                         W_ratio, (double)testResult.cSpeed / 1000000,
-                         O_ratio, (double)winners[cLevel].result.cSpeed / 1000000.,   cLevel);
+                         W_ratio, (double)testResult.cSpeed / (1 MB),
+                         O_ratio, (double)winners[cLevel].result.cSpeed / (1 MB),   cLevel);
                 continue;
             }
             if (W_DSpeed_note   < O_DSpeed_note  ) {
                 /* too large decompression speed difference for the compression benefit */
                 if (W_ratio > O_ratio)
                 DISPLAY ("Decompression Speed : %5.3f @ %4.1f MB/s  vs  %5.3f @ %4.1f MB/s   : not enough for level %i\n",
-                         W_ratio, (double)testResult.dSpeed / 1000000.,
-                         O_ratio, (double)winners[cLevel].result.dSpeed / 1000000.,   cLevel);
+                         W_ratio, (double)testResult.dSpeed / (1 MB),
+                         O_ratio, (double)winners[cLevel].result.dSpeed / (1 MB),   cLevel);
                 continue;
             }
 
@@ -1417,7 +1560,6 @@ static void BMK_selectRandomStart(
     }
 }
 
-
 static void BMK_benchOnce(const void* srcBuffer, size_t srcSize)
 {
     BMK_result_t testResult;
@@ -1442,7 +1584,7 @@ static void BMK_benchFullTable(const void* srcBuffer, size_t srcSize)
     if (f==NULL) { DISPLAY("error opening %s \n", rfName); exit(1); }
 
     if (g_target) {
-        BMK_init_level_constraints(g_target*1000000);
+        BMK_init_level_constraints(g_target * (1 MB));
     } else {
         /* baseline config for level 1 */
         ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, blockSize, 0);
@@ -1487,7 +1629,7 @@ static void BMK_benchMemInit(const void* srcBuffer, size_t srcSize)
 static int benchSample(void)
 {
     const char* const name = "Sample 10MB";
-    size_t const benchedSize = 10000000;
+    size_t const benchedSize = 10 MB;
 
     void* origBuff = malloc(benchedSize);
     if (!origBuff) { perror("not enough memory"); return 12; }
@@ -1505,13 +1647,56 @@ static int benchSample(void)
 }
 
 
+static int benchOnce(const char** fileNamesTable, int nbFiles, const char* dictFileName) {
+    buffers_t buf;
+    contexts_t ctx;
+    BMK_result_t testResult;
+    size_t maxBlockSize = 0, i;
+
+    if(createBuffers(&buf, fileNamesTable, nbFiles)) {
+        DISPLAY("unable to load files\n");
+        return 1;
+    }
+
+    if(createContexts(&ctx, dictFileName)) {
+        DISPLAY("unable to load dictionary\n");
+        freeBuffers(buf);
+        return 2;
+    }
+
+    for(i = 0; i < buf.nbBlocks; i++) {
+        maxBlockSize = MAX(maxBlockSize, buf.srcSizes[i]);
+    }
+
+    g_params = ZSTD_adjustCParams(g_params, maxBlockSize, 0);
+
+    if(BMK_benchParam(&testResult, buf, ctx, g_params)) {
+        DISPLAY("Error during benchmarking\n");
+        freeBuffers(buf);
+        freeContexts(ctx);
+        return 3;
+    }
+
+    DISPLAY("Compression Ratio: %.3f  Compress Speed: %.1f MB/s Decompress Speed: %.1f MB/s\n", (double)buf.srcSize / testResult.cSize, 
+        (double)testResult.cSpeed / (1 MB), (double)testResult.dSpeed / (1 MB));
+
+    freeBuffers(buf);
+    freeContexts(ctx);
+    return 0;
+}
+
 /* benchFiles() :
  * note: while this function takes a table of filenames,
  * in practice, only the first filename will be used */
-int benchFiles(const char** fileNamesTable, int nbFiles)
+//TODO: dictionaries still not supported in fullTable mode
+int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileName)
 {
     int fileIdx=0;
 
+    if(g_singleRun) {
+        return benchOnce(fileNamesTable, nbFiles, dictFileName);
+    }
+
     /* Loop for each file */
     while (fileIdx<nbFiles) {
         const char* const inFileName = fileNamesTable[fileIdx++];
@@ -1856,196 +2041,6 @@ static winnerInfo_t optimizeFixedStrategy(
     return winnerInfo;
 }
 
-static void freeBuffers(const buffers_t b) {
-    if(b.srcPtrs != NULL) {
-        free(b.srcBuffer);
-    }
-    free(b.srcPtrs);
-    free(b.srcSizes);
-
-    if(b.dstPtrs != NULL) {
-        free(b.dstPtrs[0]);
-    }
-    free(b.dstPtrs);
-    free(b.dstCapacities);
-    free(b.dstSizes);
-
-    if(b.resPtrs != NULL) {
-        free(b.resPtrs[0]);
-    }
-    free(b.resPtrs);
-}
-
-/* allocates buffer's arguments. returns 0 = success / 1 = failuere */
-static int createBuffers(buffers_t* const buff, const char* const * const fileNamesTable, 
-                          const size_t nbFiles)
-{
-    size_t pos = 0;
-    size_t n;
-    U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, (U32)nbFiles);
-    size_t benchedSize = MIN(BMK_findMaxMem(totalSizeToLoad * 3) / 3, totalSizeToLoad);
-    const size_t blockSize = g_blockSize ? g_blockSize : totalSizeToLoad; //(largest fileSize or total fileSize)
-    U32 const maxNbBlocks = (U32) ((totalSizeToLoad + (blockSize-1)) / blockSize) + (U32)nbFiles;
-    U32 blockNb = 0;
-
-    memset(buff, 0, sizeof(buffers_t));
-
-    buff->srcPtrs = (const void**)calloc(maxNbBlocks, sizeof(void*));
-    buff->srcSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
-
-    buff->dstPtrs = (void**)calloc(maxNbBlocks, sizeof(void*));
-    buff->dstCapacities = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
-    buff->dstSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
-
-    buff->resPtrs = (void**)calloc(maxNbBlocks, sizeof(void*));
-    buff->resSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); 
-
-    if(!buff->srcPtrs || !buff->srcSizes || !buff->dstPtrs || !buff->dstCapacities || !buff->dstSizes || !buff->resPtrs || !buff->resSizes) {
-        DISPLAY("alloc error\n");
-        freeBuffers(*buff);
-        return 1;
-    }
-
-    buff->srcBuffer = malloc(benchedSize);
-    buff->srcPtrs[0] = (const void*)buff->srcBuffer;
-    buff->dstPtrs[0] = malloc(ZSTD_compressBound(benchedSize) + (maxNbBlocks * 1024));
-    buff->resPtrs[0] = malloc(benchedSize);
-
-    if(!buff->srcPtrs[0] || !buff->dstPtrs[0] || !buff->resPtrs[0]) {
-        DISPLAY("alloc error\n");
-        freeBuffers(*buff);
-        return 1;
-    }
-
-    for(n = 0; n < nbFiles; n++) {
-        FILE* f;
-        U64 fileSize = UTIL_getFileSize(fileNamesTable[n]);
-        if (UTIL_isDirectory(fileNamesTable[n])) {
-            DISPLAY("Ignoring %s directory...       \n", fileNamesTable[n]);
-            continue;
-        }
-        if (fileSize == UTIL_FILESIZE_UNKNOWN) {
-            DISPLAY("Cannot evaluate size of %s, ignoring ... \n", fileNamesTable[n]);
-            continue;
-        }
-        f = fopen(fileNamesTable[n], "rb");
-        if (f==NULL) {
-            DISPLAY("impossible to open file %s\n", fileNamesTable[n]);
-            freeBuffers(*buff);
-            fclose(f);
-            return 10;
-        }
-
-        DISPLAY("Loading %s...       \r", fileNamesTable[n]);
-
-        if (fileSize + pos > benchedSize) fileSize = benchedSize - pos, n = nbFiles;   /* buffer too small - stop after this file */
-        {
-            char* buffer = (char*)(buff->srcBuffer); 
-            size_t const readSize = fread(((buffer)+pos), 1, (size_t)fileSize, f);
-            size_t blocked = 0;
-            while(blocked < readSize) {
-                buff->srcPtrs[blockNb] = (const void*)((buffer) + (pos + blocked));
-                buff->srcSizes[blockNb] = blockSize;
-                blocked += blockSize;
-                blockNb++;
-            }
-            if(readSize > 0) { buff->srcSizes[blockNb - 1] = ((readSize - 1) % blockSize) + 1; }
-
-            if (readSize != (size_t)fileSize) {
-                DISPLAY("could not read %s", fileNamesTable[n]);
-                freeBuffers(*buff);
-                fclose(f);
-                return 1;
-            }
-
-            pos += readSize;
-
-        }
-        fclose(f);
-    }
-
-    buff->dstCapacities[0] = ZSTD_compressBound(buff->srcSizes[0]);
-    buff->dstSizes[0] = buff->dstCapacities[0];
-    buff->resSizes[0] = buff->srcSizes[0];
-
-    for(n = 1; n < blockNb; n++) {
-        buff->dstPtrs[n] = ((char*)buff->dstPtrs[n-1]) + buff->dstCapacities[n-1];
-        buff->resPtrs[n] = ((char*)buff->resPtrs[n-1]) + buff->resSizes[n-1];
-        buff->dstCapacities[n] = ZSTD_compressBound(buff->srcSizes[n]);
-        buff->dstSizes[n] = buff->dstCapacities[n];
-        buff->resSizes[n] = buff->srcSizes[n];
-    }
-    buff->srcSize = pos;
-    buff->nbBlocks = blockNb;
-
-    if (pos == 0) { DISPLAY("\nno data to bench\n"); return 1; }
-
-    return 0;
-}
-
-static void freeContexts(const contexts_t ctx) {
-    free(ctx.dictBuffer);
-    ZSTD_freeCCtx(ctx.cctx);
-    ZSTD_freeDCtx(ctx.dctx);
-}
-
-/* Creates struct holding contexts and dictionary buffers. returns 0 on success, 1 on failure. */
-static int createContexts(contexts_t* const ctx, const char* dictFileName) {
-    FILE* f;
-    size_t readSize;
-    U64 dictSize;
-    ctx->cctx = ZSTD_createCCtx();
-    ctx->dctx = ZSTD_createDCtx();
-    ctx->dictSize = 0;
-    ctx->dictBuffer = NULL;
-
-    if(!ctx->cctx || !ctx->dctx) {
-        DISPLAY("context allocation error\n");
-        freeContexts(*ctx);
-        return 1;
-    }
-
-    if(dictFileName == NULL) {
-        return 0;
-    }
-
-    dictSize = UTIL_getFileSize(dictFileName);
-
-    if(dictSize == UTIL_FILESIZE_UNKNOWN) {
-        DISPLAY("Unable to get dictionary size\n");
-        freeContexts(*ctx);
-        return 1;
-    } else {
-        ctx->dictSize = (size_t)dictSize;
-    }
-
-    ctx->dictBuffer = malloc(ctx->dictSize);
-
-    f = fopen(dictFileName, "rb");
-    
-    if(!f) {
-        DISPLAY("unable to open file\n");
-        fclose(f);
-        freeContexts(*ctx);
-        return 1;
-    }
-    
-    if(ctx->dictSize > 64 MB || !(ctx->dictBuffer)) {
-        DISPLAY("dictionary too large\n");
-        fclose(f);
-        freeContexts(*ctx);
-        return 1;
-    }
-    readSize = fread(ctx->dictBuffer, 1, ctx->dictSize, f);
-    if(readSize != ctx->dictSize) {
-        DISPLAY("unable to read file\n");
-        fclose(f);
-        freeContexts(*ctx);
-        return 1;
-    }
-    return 0;
-}
-
 /* goes best, best-1, best+1, best-2, ... */
 /* return 0 if nothing remaining */
 static int nextStrategy(const int currentStrategy, const int bestStrategy) {
@@ -2102,7 +2097,7 @@ static ZSTD_compressionParameters maskParams(ZSTD_compressionParameters base, ZS
  * cLevel - compression level to exceed (all solutions must be > lvl in cSpeed + ratio)
  */
 
-#define MAX_TRIES 3
+#define MAX_TRIES 5
 #define TRY_DECAY 1
 
 static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget, int cLevel)
@@ -2272,8 +2267,9 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
                 }
 
                 while(st && tries > 0) {
+                    winnerInfo_t wc;
                     DEBUGOUTPUT("StrategySwitch: %s\n", g_stratName[st]);
-                    winnerInfo_t wc = optimizeFixedStrategy(buf, ctx, target, paramTarget, 
+                    wc = optimizeFixedStrategy(buf, ctx, target, paramTarget, 
                         st, varArray, varLen, allMT[st], tries);
 
                     if(compareResultLT(winner.result, wc.result, target, buf.srcSize)) {
@@ -2399,7 +2395,6 @@ int main(int argc, const char** argv)
     const char* exename=argv[0];
     const char* input_filename = NULL;
     const char* dictFileName = NULL;
-    U32 optimizer = 0;
     U32 main_pause = 0;
     int optimizerCLevel = 0;
 
@@ -2424,7 +2419,7 @@ int main(int argc, const char** argv)
         if(!strcmp(argument,"--no-seed")) { g_noSeed = 1; continue; }
 
         if (longCommandWArg(&argument, "--optimize=")) {
-            optimizer = 1;
+            g_optimizer = 1;
             for ( ; ;) {
                 PARSE_CPARAMS(paramTarget);
                 PARSE_SUB_ARGS("compressionSpeed=" ,  "cSpeed=", target.cSpeed);
@@ -2580,17 +2575,17 @@ int main(int argc, const char** argv)
         if (!input_filename) { input_filename=argument; filenamesStart=i; continue; }
     }
     if (filenamesStart==0) {
-        if (optimizer) {
+        if (g_optimizer) {
             DISPLAY("Optimizer Expects File\n");
             return 1;
         } else {
             result = benchSample();
         }
     } else {
-        if (optimizer) {
+        if (g_optimizer) {
             result = optimizeForSize(argv+filenamesStart, argc-filenamesStart, dictFileName, target, paramTarget, optimizerCLevel);
         } else {
-            result = benchFiles(argv+filenamesStart, argc-filenamesStart);
+            result = benchFiles(argv+filenamesStart, argc-filenamesStart, dictFileName);
     }   }
 
     if (main_pause) { int unused; printf("press enter...\n"); unused = getchar(); (void)unused; }

From 43b4971ca8910ccedebb6c23a6daa89069df6a17 Mon Sep 17 00:00:00 2001
From: George Lu <gclu@fb.com>
Date: Fri, 27 Jul 2018 16:49:33 -0700
Subject: [PATCH 11/15] Renames, Documentation Updates

---
 tests/README.md    | 22 +++++++-----
 tests/paramgrill.c | 84 +++++++++++++++++++++++++---------------------
 2 files changed, 58 insertions(+), 48 deletions(-)

diff --git a/tests/README.md b/tests/README.md
index 1410ca97..946f890b 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -107,15 +107,19 @@ Full list of arguments
     L# - level
  --zstd=      : Single run, parameter selection syntax same as zstdcli
  --optimize=  : find parameters to maximize compression ratio given parameters
-    Can use all --zstd= commands to constrain the type of solution found in addition to the following constraints
-    cSpeed= - Minimum compression speed
-    dSpeed= - Minimum decompression speed
-    cMem= - compression memory
-    lvl= - Searches for solutions which are strictly better than that compression lvl in ratio and cSpeed, 
-    stc= - When invoked with lvl=, represents slack in ratio/cSpeed allowed for a solution to be considered
-         - In normal operation, represents slack in strategy selection in choosing the default parameters
-    prefer[Speed/Ratio]= - Only affects lvl= invocations. Defines value placed on compression speed or ratio
-          when determining overall winner (default 1 for both).
+                Can use all --zstd= commands to constrain the type of solution found in addition to the following constraints
+    cSpeed=   : Minimum compression speed
+    dSpeed=   : Minimum decompression speed
+    cMem=     : Maximum compression memory
+    lvl=      : Searches for solutions which are strictly better than that compression lvl in ratio and cSpeed, 
+    stc=      : When invoked with lvl=, represents percentage slack in ratio/cSpeed allowed for a solution to be considered (Default 99%)
+              : In normal operation, represents percentage slack in choosing viable starting strategy selection in choosing the default parameters
+                (Lower value will begin with stronger strategies) (Default 90%)
+    preferSpeed= / preferRatio=
+              : Only affects lvl = invocations. Defines value placed on compression speed or ratio
+                when determining overall winner (default 1 for both, higher = more valued).
+    tries=    : Maximum number of random restarts on a single strategy before switching (Default 5)
+                Higher values will make optimizer run longer, more chances to find better solution.
  --optimize=  : same as -O with more verbose syntax 
  -P#          : generated sample compressibility 
  -t#          : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours )) 
diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index 8df72eb0..7e86c2e9 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -125,15 +125,13 @@ typedef struct {
     U32 cMem;    /* bytes */    
 } constraint_t;
 
-static constraint_t g_targetConstraints; 
-
-typedef struct ll_node ll_node;
-struct ll_node {
+typedef struct winner_ll_node winner_ll_node;
+struct winner_ll_node {
     winnerInfo_t res;
-    ll_node* next;
+    winner_ll_node* next;
 };
 
-static ll_node* g_winners; /* linked list sorted ascending by cSize & cSpeed */
+static winner_ll_node* g_winners; /* linked list sorted ascending by cSize & cSpeed */
 static BMK_result_t g_lvltarget;
 static int g_optmode = 0;
 
@@ -323,8 +321,8 @@ static int compareResultLT(const BMK_result_t result1, const BMK_result_t result
 
 static constraint_t relaxTarget(constraint_t target) {
     target.cMem = (U32)-1;
-    target.cSpeed *= ((double)(g_strictness) / 100); 
-    target.dSpeed *= ((double)(g_strictness) / 100);
+    target.cSpeed *= ((double)g_strictness) / 100; 
+    target.dSpeed *= ((double)g_strictness) / 100;
     return target;
 }
 
@@ -855,20 +853,19 @@ static int speedSizeCompare(BMK_result_t r1, BMK_result_t r2) {
         return SPEED_RESULT; /* r2 is faster but not smaller */
     }
 }
-/* assumes candidate is already strictly better than old winner. */
-/* 0 for success, 1 for no insert */
-/* indicate whether inserted as well? */
+
+/* 0 for insertion, 1 for no insert */
 /* maintain invariant speedSizeCompare(n, n->next) = SPEED_RESULT */
-static int insertWinner(winnerInfo_t w) {
+static int insertWinner(winnerInfo_t w, constraint_t targetConstraints) {
     BMK_result_t r = w.result;
-    ll_node* cur_node = g_winners;
+    winner_ll_node* cur_node = g_winners;
     /* first node to insert */
-    if(!feasible(r, g_targetConstraints)) {
+    if(!feasible(r, targetConstraints)) {
         return 1;
     }
 
     if(g_winners == NULL) {
-        ll_node* first_node = malloc(sizeof(ll_node));
+        winner_ll_node* first_node = malloc(sizeof(winner_ll_node));
         if(first_node == NULL) {
             return 1;
         }
@@ -886,7 +883,7 @@ static int insertWinner(winnerInfo_t w) {
             }
             case WORSE_RESULT:
             {
-                ll_node* tmp;
+                winner_ll_node* tmp;
                 cur_node->res = cur_node->next->res;
                 tmp = cur_node->next;
                 cur_node->next = cur_node->next->next;
@@ -900,7 +897,7 @@ static int insertWinner(winnerInfo_t w) {
             }
             case SIZE_RESULT: /* insert after first size result, then return */
             {
-                ll_node* newnode = malloc(sizeof(ll_node));
+                winner_ll_node* newnode = malloc(sizeof(winner_ll_node));
                 if(newnode == NULL) {
                     return 1;
                 }
@@ -927,7 +924,7 @@ static int insertWinner(winnerInfo_t w) {
         }
         case SPEED_RESULT:
         {
-            ll_node* newnode = malloc(sizeof(ll_node));
+            winner_ll_node* newnode = malloc(sizeof(winner_ll_node));
             if(newnode == NULL) {
                 return 1;
             }
@@ -938,7 +935,7 @@ static int insertWinner(winnerInfo_t w) {
         }
         case SIZE_RESULT: /* insert before first size result, then return */
         {
-            ll_node* newnode = malloc(sizeof(ll_node));
+            winner_ll_node* newnode = malloc(sizeof(winner_ll_node));
             if(newnode == NULL) {
                 return 1;
             }
@@ -961,7 +958,7 @@ static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result
     const U64 time = UTIL_clockSpanNano(g_time);
     const U64 minutes = time / (60ULL * TIMELOOP_NANOSEC);
 
-    DISPLAY("\r%79s\r", "");
+    fprintf(f, "\r%79s\r", "");
 
     fprintf(f,"    {%3u,%3u,%3u,%3u,%3u,%3u, %s },  ",
         params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength,
@@ -1001,10 +998,10 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res
     //prints out tradeoff table if using lvl
     if(g_optmode && g_optimizer) {
         winnerInfo_t w;
-        ll_node* n;
+        winner_ll_node* n;
         w.result = result;
         w.params = params;
-        insertWinner(w);
+        insertWinner(w, targetConstraints);
 
         if(!DEBUG) { fprintf(f, "\033c"); }
         fprintf(f, "\n"); 
@@ -1012,7 +1009,7 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res
         /* the table */
         fprintf(f, "================================\n");
         for(n = g_winners; n != NULL; n = n->next) {
-            DISPLAY("\r%79s\r", "");
+            fprintf(f, "\r%79s\r", "");
 
             fprintf(f,"    {%3u,%3u,%3u,%3u,%3u,%3u, %s },  ",
                 n->res.params.windowLog, n->res.params.chainLog, n->res.params.hashLog, n->res.params.searchLog, n->res.params.searchLength,
@@ -1045,6 +1042,12 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res
             (double)srcSize / result.cSize, (double)result.cSpeed / (1 MB), (double)result.dSpeed / (1 MB));
 
     }
+
+#if 0
+    if(BMK_timeSpan(g_time) > g_grillDuration_s) {
+        exit(0);
+    }
+#endif
 }
 
 static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSize)
@@ -2097,7 +2100,7 @@ static ZSTD_compressionParameters maskParams(ZSTD_compressionParameters base, ZS
  * cLevel - compression level to exceed (all solutions must be > lvl in cSpeed + ratio)
  */
 
-#define MAX_TRIES 5
+static int g_maxTries = 5;
 #define TRY_DECAY 1
 
 static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget, int cLevel)
@@ -2185,20 +2188,21 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
     }
 
     /* use level'ing mode instead of normal target mode */
+    /* Should lvl be parameter-masked here? */
     if(g_optmode) {
         winner.params = ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize);
         if(BMK_benchParam(&winner.result, buf, ctx, winner.params)) {
             ret = 3;
             goto _cleanUp;
         }
-
-        target.cSpeed = (U32)winner.result.cSpeed;
-
-        g_targetConstraints = target;
-
+ 
         g_lvltarget = winner.result; 
-        g_lvltarget.cSpeed *= ((double)(g_strictness) / 100);
-        g_lvltarget.cSize /= ((double)(g_strictness) / 100);
+        g_lvltarget.cSpeed *= ((double)g_strictness) / 100;
+        g_lvltarget.dSpeed *= ((double)g_strictness) / 100;
+        g_lvltarget.cSize /= ((double)g_strictness) / 100;
+
+        target.cSpeed = (U32)g_lvltarget.cSpeed;  
+        target.dSpeed = (U32)g_lvltarget.dSpeed; //See if this is worth
 
         BMK_printWinnerOpt(stdout, cLevel, winner.result, winner.params, target, buf.srcSize);
     }
@@ -2241,7 +2245,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
                     }
 
                     /* if the current params are too slow, just stop. */
-                    if(target.cSpeed > candidate.cSpeed * 2) { break; }
+                    if(target.cSpeed > candidate.cSpeed * 3 / 2) { break; }
                 }
             }
         }
@@ -2254,7 +2258,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
             int bestStrategy = (int)winner.params.strategy;
             if(paramTarget.strategy == 0) {
                 int st = (int)winner.params.strategy;
-                int tries = MAX_TRIES;
+                int tries = g_maxTries;
 
                 { 
                     /* one iterations of hill climbing with the level-defined parameters. */
@@ -2269,12 +2273,13 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
                 while(st && tries > 0) {
                     winnerInfo_t wc;
                     DEBUGOUTPUT("StrategySwitch: %s\n", g_stratName[st]);
+                    
                     wc = optimizeFixedStrategy(buf, ctx, target, paramTarget, 
                         st, varArray, varLen, allMT[st], tries);
 
                     if(compareResultLT(winner.result, wc.result, target, buf.srcSize)) {
                         winner = wc;
-                        tries = MAX_TRIES;
+                        tries = g_maxTries;
                         bestStrategy = st;
                     } else {
                         st = nextStrategy(st, bestStrategy);
@@ -2283,7 +2288,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
                 }
             } else {
                 winner = optimizeFixedStrategy(buf, ctx, target, paramTarget, paramTarget.strategy, 
-                    varArray, varLen, allMT[paramTarget.strategy], 10);
+                    varArray, varLen, allMT[paramTarget.strategy], g_maxTries);
             }
 
         }
@@ -2402,17 +2407,17 @@ int main(int argc, const char** argv)
 
     ZSTD_compressionParameters paramTarget = { 0, 0, 0, 0, 0, 0, 0 };
 
-    assert(argc>=1);   /* for exename */
-
     g_time = UTIL_getTime();
 
+    assert(argc>=1);   /* for exename */
+
     /* Welcome message */
     DISPLAY(WELCOME_MESSAGE);
 
     for(i=1; i<argc; i++) {
         const char* argument = argv[i];
-        DISPLAY("%d: ", i);
-        DISPLAY("%s\n", argument);
+        DEBUGOUTPUT("%d: ", i);
+        DEBUGOUTPUT("%s\n", argument);
 
         assert(argument != NULL);
 
@@ -2429,6 +2434,7 @@ int main(int argc, const char** argv)
                 PARSE_SUB_ARGS("strict=", "stc=", g_strictness);
                 PARSE_SUB_ARGS("preferSpeed=", "prfSpd=", g_speedMultiplier);
                 PARSE_SUB_ARGS("preferRatio=", "prfRto=", g_ratioMultiplier);
+                PARSE_SUB_ARGS("maxTries", "tries", g_maxTries);
 
                 DISPLAY("invalid optimization parameter \n");
                 return 1;

From d4730a4f66746e6fe1473f06eb2d9bb49c831b83 Mon Sep 17 00:00:00 2001
From: George Lu <gclu@fb.com>
Date: Mon, 30 Jul 2018 17:42:46 -0700
Subject: [PATCH 12/15] Update fulltable to use same interface

Add seperateFiles flag
---
 tests/README.md    |   1 +
 tests/paramgrill.c | 389 ++++++++++++++++++++++++---------------------
 2 files changed, 207 insertions(+), 183 deletions(-)

diff --git a/tests/README.md b/tests/README.md
index 946f890b..73691693 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -125,6 +125,7 @@ Full list of arguments
  -t#          : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours )) 
  -v           : Prints Benchmarking output
  -D           : Next argument dictionary file
+ -s           : Benchmark all files separately
 
 ```
  Any inputs afterwards are treated as files to benchmark.
diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index 7e86c2e9..cba97ebb 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -96,7 +96,7 @@ typedef enum {
 static const int rangetable[NUM_PARAMS] = { WLOG_RANGE, CLOG_RANGE, HLOG_RANGE, SLOG_RANGE, SLEN_RANGE, TLEN_RANGE };
 static const U32 tlen_table[TLEN_RANGE] = { 0, 1, 2, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 256, 512, 999 };
 /*-************************************
-*  Benchmark Parameters
+*  Benchmark Parameters/Global Variables
 **************************************/
 
 typedef BYTE U8;
@@ -110,7 +110,7 @@ static U32 g_singleRun = 0;
 static U32 g_optimizer = 0;
 static U32 g_target = 0;
 static U32 g_noSeed = 0;
-static ZSTD_compressionParameters g_params = { 0, 0, 0, 0, 0, 0, ZSTD_greedy };
+static ZSTD_compressionParameters g_params;
 static UTIL_time_t g_time; /* to be used to compare solution finding speeds to compare to original */
 
 
@@ -150,6 +150,14 @@ void BMK_SetNbIterations(int nbLoops)
     DISPLAY("- %u iterations -\n", g_nbIterations);
 }
 
+/*
+ * Additional Global Variables (Defined Above Use)
+ * g_stratName
+ * g_level_constraint
+ * g_alreadyTested
+ * g_maxTries
+ */
+
 /*-*******************************************************
 *  Private functions
 *********************************************************/
@@ -335,17 +343,6 @@ const char* g_stratName[ZSTD_btultra+1] = {
                 "ZSTD_greedy  ", "ZSTD_lazy    ", "ZSTD_lazy2   ",
                 "ZSTD_btlazy2 ", "ZSTD_btopt   ", "ZSTD_btultra "};
 
-/* benchParam but only takes in one input buffer. */
-static int
-BMK_benchParam1(BMK_result_t* resultPtr,
-               const void* srcBuffer, size_t srcSize,
-               const ZSTD_compressionParameters cParams) {
-
-    BMK_return_t res = BMK_benchMem(srcBuffer,srcSize, &srcSize, 1, 0, &cParams, NULL, 0, 0, "File");
-    *resultPtr = res.result;
-    return res.error;
-}
-
 static ZSTD_compressionParameters emptyParams(void) {
     ZSTD_compressionParameters p = { 0, 0, 0, 0, 0, 0, (ZSTD_strategy)0 };
     return p;
@@ -381,14 +378,6 @@ typedef struct {
     ZSTD_DCtx* dctx;
 } contexts_t;
 
-static int BMK_benchParam(BMK_result_t* resultPtr,
-                const buffers_t buf, const contexts_t ctx,
-                const ZSTD_compressionParameters cParams) {
-    BMK_return_t res = BMK_benchMem(buf.srcPtrs[0], buf.srcSize, buf.srcSizes, (unsigned)buf.nbBlocks, 0, &cParams, ctx.dictBuffer, ctx.dictSize, 0, "Files");
-    *resultPtr = res.result;
-    return res.error;
-}
-
 /*-*******************************************************
 *  From Paramgrill
 *********************************************************/
@@ -542,7 +531,7 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab
     size_t n;
     U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, (U32)nbFiles);
     size_t benchedSize = MIN(BMK_findMaxMem(totalSizeToLoad * 3) / 3, totalSizeToLoad);
-    const size_t blockSize = g_blockSize ? g_blockSize : totalSizeToLoad; //(largest fileSize or total fileSize)
+    const size_t blockSize = g_blockSize ? g_blockSize : totalSizeToLoad;
     U32 const maxNbBlocks = (U32) ((totalSizeToLoad + (blockSize-1)) / blockSize) + (U32)nbFiles;
     U32 blockNb = 0;
 
@@ -830,6 +819,14 @@ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t
     return results;
 }
 
+static int BMK_benchParam(BMK_result_t* resultPtr,
+                buffers_t buf, contexts_t ctx,
+                const ZSTD_compressionParameters cParams) {
+    BMK_return_t res = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_both, BMK_timeMode, 3);
+    *resultPtr = res.result;
+    return res.error;
+}
+
 /* comparison function: */
 /* strictly better, strictly worse, equal, speed-side adv, size-side adv */
 //Maybe use compress_only for benchmark first run?
@@ -911,7 +908,7 @@ static int insertWinner(winnerInfo_t w, constraint_t targetConstraints) {
 
     }
 
-    //assert(cur_node->next == NULL)
+    assert(cur_node->next == NULL);
     switch(speedSizeCompare(r, cur_node->res.result)) {
         case BETTER_RESULT:
         {
@@ -995,7 +992,7 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res
         }
     }  
 
-    //prints out tradeoff table if using lvl
+    //prints out tradeoff table if using lvloptimize
     if(g_optmode && g_optimizer) {
         winnerInfo_t w;
         winner_ll_node* n;
@@ -1042,12 +1039,6 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res
             (double)srcSize / result.cSize, (double)result.cSpeed / (1 MB), (double)result.dSpeed / (1 MB));
 
     }
-
-#if 0
-    if(BMK_timeSpan(g_time) > g_grillDuration_s) {
-        exit(0);
-    }
-#endif
 }
 
 static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSize)
@@ -1099,14 +1090,14 @@ static void BMK_init_level_constraints(int bytePerSec_level1)
     }   }
 }
 
-static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters params,
-              const void* srcBuffer, size_t srcSize)
+static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters params, 
+                    buffers_t buf, contexts_t ctx)
 {
     BMK_result_t testResult;
     int better = 0;
     int cLevel;
 
-    BMK_benchParam1(&testResult, srcBuffer, srcSize, params);
+    BMK_benchParam(&testResult, buf, ctx, params);
 
 
     for (cLevel = 1; cLevel <= NB_LEVELS_TRACKED; cLevel++) {
@@ -1122,15 +1113,15 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para
             /* first solution for this cLevel */
             winners[cLevel].result = testResult;
             winners[cLevel].params = params;
-            BMK_printWinner(stdout, cLevel, testResult, params, srcSize);
+            BMK_printWinner(stdout, cLevel, testResult, params, buf.srcSize);
             better = 1;
             continue;
         }
 
         if ((double)testResult.cSize <= ((double)winners[cLevel].result.cSize * (1. + (0.02 / cLevel))) ) {
             /* Validate solution is "good enough" */
-            double W_ratio = (double)srcSize / testResult.cSize;
-            double O_ratio = (double)srcSize / winners[cLevel].result.cSize;
+            double W_ratio = (double)buf.srcSize / testResult.cSize;
+            double O_ratio = (double)buf.srcSize / winners[cLevel].result.cSize;
             double W_ratioNote = log (W_ratio);
             double O_ratioNote = log (O_ratio);
             size_t W_DMemUsed = (1 << params.windowLog) + (16 KB);
@@ -1187,7 +1178,7 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para
 
             winners[cLevel].result = testResult;
             winners[cLevel].params = params;
-            BMK_printWinner(stdout, cLevel, testResult, params, srcSize);
+            BMK_printWinner(stdout, cLevel, testResult, params, buf.srcSize);
 
             better = 1;
     }   }
@@ -1315,6 +1306,26 @@ static void paramVariation(ZSTD_compressionParameters* ptr, const varInds_t* var
     *ptr = p;
 }
 
+/* maybe put strategy back in */
+static void paramVariationWithStrategy(ZSTD_compressionParameters* ptr, const varInds_t* varyParams, const int varyLen, const U32 nbChanges)
+{
+    ZSTD_compressionParameters p;
+    U32 validated = 0;
+    while (!validated) {
+        U32 i;
+        p = *ptr;
+        for (i = 0 ; i < nbChanges ; i++) {
+            const U32 changeID = FUZ_rand(&g_rand) % ((varyLen + 1) << 1);
+            if(changeID < (U32)(varyLen << 1)) {
+                paramVaryOnce(varyParams[changeID >> 1], ((changeID & 1) << 1) - 1, &p);
+            } else {
+                p.strategy += ((FUZ_rand(&g_rand) % 2) << 1) - 1; /* +/- 1 */
+            }
+        }
+        validated = !ZSTD_isError(ZSTD_checkCParams(p));
+    }
+    *ptr = p;
+}
 /* length of memo table given free variables */
 static size_t memoTableLen(const varInds_t* varyParams, const int varyLen) {
     size_t arrayLen = 1;
@@ -1463,6 +1474,17 @@ static U8** createMemoTableArray(ZSTD_compressionParameters paramConstraints, co
     return mtAll;
 }
 
+static ZSTD_compressionParameters maskParams(ZSTD_compressionParameters base, ZSTD_compressionParameters mask) {
+    base.windowLog = mask.windowLog ? mask.windowLog : base.windowLog;
+    base.chainLog = mask.chainLog ? mask.chainLog : base.chainLog;
+    base.hashLog = mask.hashLog ? mask.hashLog : base.hashLog;
+    base.searchLog = mask.searchLog ? mask.searchLog : base.searchLog;
+    base.searchLength = mask.searchLength ? mask.searchLength : base.searchLength;
+    base.targetLength = mask.targetLength ? mask.targetLength : base.targetLength;
+    base.strategy = mask.strategy ? mask.strategy : base.strategy;
+    return base;
+}
+
 #define PARAMTABLELOG   25
 #define PARAMTABLESIZE (1<<PARAMTABLELOG)
 #define PARAMTABLEMASK (PARAMTABLESIZE-1)
@@ -1479,7 +1501,7 @@ static BYTE* NB_TESTS_PLAYED(ZSTD_compressionParameters p) {
 
 static void playAround(FILE* f, winnerInfo_t* winners,
                        ZSTD_compressionParameters params,
-                       const void* srcBuffer, size_t srcSize)
+                       buffers_t buf, contexts_t ctx)
 {
     int nbVariations = 0;
     UTIL_time_t const clockStart = UTIL_getTime();
@@ -1491,7 +1513,7 @@ static void playAround(FILE* f, winnerInfo_t* winners,
         BYTE* b;
 
         if (nbVariations++ > g_maxNbVariations) break;
-        paramVariation(&p, unconstrained, 7, 4);
+        paramVariationWithStrategy(&p, unconstrained, NUM_PARAMS, 4);
 
         /* exclude faster if already played params */
         if (FUZ_rand(&g_rand) & ((1 << *NB_TESTS_PLAYED(p))-1))
@@ -1500,11 +1522,11 @@ static void playAround(FILE* f, winnerInfo_t* winners,
         /* test */
         b = NB_TESTS_PLAYED(p);
         (*b)++;
-        if (!BMK_seed(winners, p, srcBuffer, srcSize)) continue;
+        if (!BMK_seed(winners, p, buf, ctx)) continue;
 
         /* improvement found => search more */
-        BMK_printWinners(f, winners, srcSize);
-        playAround(f, winners, p, srcBuffer, srcSize);
+        BMK_printWinners(f, winners, buf.srcSize);
+        playAround(f, winners, p, buf, ctx);
     }
 
 }
@@ -1551,35 +1573,24 @@ static void randomConstrainedParams(ZSTD_compressionParameters* pc, varInds_t* v
 
 static void BMK_selectRandomStart(
                        FILE* f, winnerInfo_t* winners,
-                       const void* srcBuffer, size_t srcSize)
+                       buffers_t buf, contexts_t ctx)
 {
     U32 const id = FUZ_rand(&g_rand) % (NB_LEVELS_TRACKED+1);
     if ((id==0) || (winners[id].params.windowLog==0)) {
         /* use some random entry */
-        ZSTD_compressionParameters const p = ZSTD_adjustCParams(randomParams(), srcSize, 0);
-        playAround(f, winners, p, srcBuffer, srcSize);
+        ZSTD_compressionParameters const p = ZSTD_adjustCParams(randomParams(), buf.srcSize, 0);
+        playAround(f, winners, p, buf, ctx);
     } else {
-        playAround(f, winners, winners[id].params, srcBuffer, srcSize);
+        playAround(f, winners, winners[id].params, buf, ctx);
     }
 }
 
-static void BMK_benchOnce(const void* srcBuffer, size_t srcSize)
-{
-    BMK_result_t testResult;
-    g_params = ZSTD_adjustCParams(g_params, srcSize, 0);
-    BMK_benchParam1(&testResult, srcBuffer, srcSize, g_params);
-    DISPLAY("Compression Ratio: %.3f  Compress Speed: %.1f MB/s Decompress Speed: %.1f MB/s\n", (double)srcSize / testResult.cSize, 
-        (double)testResult.cSpeed / 1000000, (double)testResult.dSpeed / 1000000);
-    return;
-}
-
-static void BMK_benchFullTable(const void* srcBuffer, size_t srcSize)
+static void BMK_benchFullTable(buffers_t buf, contexts_t ctx, const size_t maxBlockSize) 
 {
     ZSTD_compressionParameters params;
     winnerInfo_t winners[NB_LEVELS_TRACKED+1];
     const char* const rfName = "grillResults.txt";
     FILE* const f = fopen(rfName, "w");
-    const size_t blockSize = g_blockSize ? g_blockSize : srcSize;   /* cut by block or not ? */
 
     /* init */
     assert(g_singleRun==0);
@@ -1590,9 +1601,9 @@ static void BMK_benchFullTable(const void* srcBuffer, size_t srcSize)
         BMK_init_level_constraints(g_target * (1 MB));
     } else {
         /* baseline config for level 1 */
-        ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, blockSize, 0);
+        ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, maxBlockSize, ctx.dictSize); //is dictionary ever even useful here? 
         BMK_result_t testResult;
-        BMK_benchParam1(&testResult, srcBuffer, srcSize, l1params);
+        BMK_benchParam(&testResult, buf, ctx, l1params);
         BMK_init_level_constraints((int)((testResult.cSpeed * 31) / 32));
     }
 
@@ -1600,61 +1611,124 @@ static void BMK_benchFullTable(const void* srcBuffer, size_t srcSize)
     {   const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel();
         int i;
         for (i=0; i<=maxSeeds; i++) {
-            params = ZSTD_getCParams(i, blockSize, 0);
-            BMK_seed(winners, params, srcBuffer, srcSize);
+            params = ZSTD_getCParams(i, maxBlockSize, 0);
+            BMK_seed(winners, params, buf, ctx);
     }   }
-    BMK_printWinners(f, winners, srcSize);
+    BMK_printWinners(f, winners, buf.srcSize);
 
     /* start tests */
     {   const time_t grillStart = time(NULL);
         do {
-            BMK_selectRandomStart(f, winners, srcBuffer, srcSize);
+            BMK_selectRandomStart(f, winners, buf, ctx);
         } while (BMK_timeSpan(grillStart) < g_grillDuration_s);
     }
 
     /* end summary */
-    BMK_printWinners(f, winners, srcSize);
+    BMK_printWinners(f, winners, buf.srcSize);
     DISPLAY("grillParams operations completed \n");
 
     /* clean up*/
     fclose(f);
 }
 
-static void BMK_benchMemInit(const void* srcBuffer, size_t srcSize)
-{
-    if (g_singleRun)
-        return BMK_benchOnce(srcBuffer, srcSize);
-    else
-        return BMK_benchFullTable(srcBuffer, srcSize);
-}
-
-
 static int benchSample(void)
 {
     const char* const name = "Sample 10MB";
     size_t const benchedSize = 10 MB;
+    U32 blockSize = g_blockSize ? g_blockSize : benchedSize;
+    U32 const maxNbBlocks = (U32) ((benchedSize + (blockSize-1)) / blockSize) + 1;
+    size_t splitSize = 0;
 
-    void* origBuff = malloc(benchedSize);
-    if (!origBuff) { perror("not enough memory"); return 12; }
+    buffers_t buf;
+    contexts_t ctx;
 
-    /* Fill buffer */
-    RDG_genBuffer(origBuff, benchedSize, g_compressibility, 0.0, 0);
+    buf.srcPtrs = (const void**)calloc(maxNbBlocks, sizeof(void*));
+    buf.dstPtrs = (void**)calloc(maxNbBlocks, sizeof(void*));
+    buf.resPtrs = (void**)calloc(maxNbBlocks, sizeof(void*));
+    buf.srcSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
+    buf.dstSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
+    buf.dstCapacities = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
+    buf.resSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
+    buf.srcSize = benchedSize;
+
+    if(!buf.srcPtrs || !buf.dstPtrs || !buf.resPtrs || !buf.srcSizes || !buf.dstSizes || !buf.dstCapacities || !buf.resSizes) {
+        DISPLAY("Allocation Error\n");
+        freeBuffers(buf);
+        return 1;
+    } 
+
+    buf.srcBuffer = malloc(benchedSize);
+    buf.srcPtrs[0] = (const void*)buf.srcBuffer;
+    buf.dstPtrs[0] = malloc(ZSTD_compressBound(benchedSize) + 1024 * maxNbBlocks);
+    buf.resPtrs[0] = malloc(benchedSize);
+
+    if(!buf.srcPtrs[0] || !buf.dstPtrs[0] || !buf.resPtrs[0]) {
+        DISPLAY("Allocation Error\n");
+        freeBuffers(buf);
+        return 1;
+    }
+
+
+    splitSize = MIN(benchedSize, blockSize);
+    buf.srcSizes[0] = splitSize;
+    buf.dstCapacities[0] = ZSTD_compressBound(splitSize);
+    buf.resSizes[0] = splitSize;
+
+    for(buf.nbBlocks = 1; splitSize < benchedSize; buf.nbBlocks++) {
+        const size_t i = buf.nbBlocks;
+        const size_t nextBlockSize = MIN(benchedSize - splitSize, blockSize);
+        buf.srcSizes[i] = nextBlockSize;
+        buf.dstCapacities[i] = ZSTD_compressBound(nextBlockSize);
+        buf.resSizes[i] = nextBlockSize;
+        buf.srcPtrs[i] = (const void*)(((const char*)buf.srcPtrs[i-1]) + buf.srcSizes[i-1]);
+        buf.dstPtrs[i] = (void*)(((char*)buf.dstPtrs[i-1]) + buf.dstSizes[i-1]);
+        buf.resPtrs[i] = (void*)(((char*)buf.resPtrs[i-1]) + buf.resSizes[i-1]);
+        splitSize += nextBlockSize;
+    }
+
+    if(createContexts(&ctx, NULL)) {
+        DISPLAY("Context Creation Error\n");
+        freeBuffers(buf);
+        return 1;
+    }
+
+    RDG_genBuffer(buf.srcBuffer, benchedSize, g_compressibility, 0.0, 0);
 
     /* bench */
     DISPLAY("\r%79s\r", "");
     DISPLAY("using %s %i%%: \n", name, (int)(g_compressibility*100));
-    BMK_benchMemInit(origBuff, benchedSize);
 
-    free(origBuff);
+    BMK_benchFullTable(buf, ctx, MIN(blockSize, benchedSize));
+
+    freeBuffers(buf);
+    freeContexts(ctx);
+
     return 0;
 }
 
 
-static int benchOnce(const char** fileNamesTable, int nbFiles, const char* dictFileName) {
+static int benchOnce(buffers_t buf, contexts_t ctx) {
+    BMK_result_t testResult;
+
+    if(BMK_benchParam(&testResult, buf, ctx, g_params)) {
+        DISPLAY("Error during benchmarking\n");
+        return 1;
+    }
+
+    DISPLAY("Compression Ratio: %.3f  Compress Speed: %.1f MB/s Decompress Speed: %.1f MB/s\n", (double)buf.srcSize / testResult.cSize, 
+        (double)testResult.cSpeed / (1 MB), (double)testResult.dSpeed / (1 MB));
+    return 0;
+}
+
+/* benchFiles() :
+ * note: while this function takes a table of filenames,
+ * in practice, only the first filename will be used */
+int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileName, int cLevel)
+{
     buffers_t buf;
     contexts_t ctx;
-    BMK_result_t testResult;
     size_t maxBlockSize = 0, i;
+    int ret = 0;
 
     if(createBuffers(&buf, fileNamesTable, nbFiles)) {
         DISPLAY("unable to load files\n");
@@ -1671,86 +1745,24 @@ static int benchOnce(const char** fileNamesTable, int nbFiles, const char* dictF
         maxBlockSize = MAX(maxBlockSize, buf.srcSizes[i]);
     }
 
-    g_params = ZSTD_adjustCParams(g_params, maxBlockSize, 0);
-
-    if(BMK_benchParam(&testResult, buf, ctx, g_params)) {
-        DISPLAY("Error during benchmarking\n");
-        freeBuffers(buf);
-        freeContexts(ctx);
-        return 3;
+    DISPLAY("\r%79s\r", "");
+    if(nbFiles == 1) {
+        DISPLAY("using %s : \n", fileNamesTable[0]);
+    } else {
+        DISPLAY("using %d Files : \n", nbFiles);
     }
 
-    DISPLAY("Compression Ratio: %.3f  Compress Speed: %.1f MB/s Decompress Speed: %.1f MB/s\n", (double)buf.srcSize / testResult.cSize, 
-        (double)testResult.cSpeed / (1 MB), (double)testResult.dSpeed / (1 MB));
+    g_params = ZSTD_adjustCParams(maskParams(ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize), g_params), maxBlockSize, ctx.dictSize);
+
+    if(g_singleRun) {
+        ret = benchOnce(buf, ctx);
+    } else {
+        BMK_benchFullTable(buf, ctx, maxBlockSize);
+    }
 
     freeBuffers(buf);
     freeContexts(ctx);
-    return 0;
-}
-
-/* benchFiles() :
- * note: while this function takes a table of filenames,
- * in practice, only the first filename will be used */
-//TODO: dictionaries still not supported in fullTable mode
-int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileName)
-{
-    int fileIdx=0;
-
-    if(g_singleRun) {
-        return benchOnce(fileNamesTable, nbFiles, dictFileName);
-    }
-
-    /* Loop for each file */
-    while (fileIdx<nbFiles) {
-        const char* const inFileName = fileNamesTable[fileIdx++];
-        FILE* const inFile = fopen( inFileName, "rb" );
-        U64 const inFileSize = UTIL_getFileSize(inFileName);
-        size_t benchedSize;
-        void* origBuff;
-
-        /* Check file existence */
-        if (inFile==NULL) {
-            DISPLAY( "Pb opening %s\n", inFileName);
-            return 11;
-        }
-        if (inFileSize == UTIL_FILESIZE_UNKNOWN) {
-            DISPLAY("Pb evaluating size of %s \n", inFileName);
-            fclose(inFile);
-            return 11;
-        }
-
-        /* Memory allocation */
-        benchedSize = BMK_findMaxMem(inFileSize*3) / 3;
-        if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize;
-        if (benchedSize < inFileSize)
-            DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20));
-        origBuff = malloc(benchedSize);
-        if (origBuff==NULL) {
-            DISPLAY("\nError: not enough memory!\n");
-            fclose(inFile);
-            return 12;
-        }
-
-        /* Fill input buffer */
-        DISPLAY("Loading %s...       \r", inFileName);
-        {   size_t const readSize = fread(origBuff, 1, benchedSize, inFile);
-            fclose(inFile);
-            if(readSize != benchedSize) {
-                DISPLAY("\nError: problem reading file '%s' !!    \n", inFileName);
-                free(origBuff);
-                return 13;
-        }   }
-
-        /* bench */
-        DISPLAY("\r%79s\r", "");
-        DISPLAY("using %s : \n", inFileName);
-        BMK_benchMemInit(origBuff, benchedSize);
-
-        /* clean */
-        free(origBuff);
-    }
-
-    return 0;
+    return ret;
 }
 
 /* Benchmarking which stops when we are sufficiently sure the solution is infeasible / worse than the winner */
@@ -2074,17 +2086,6 @@ static int nextStrategy(const int currentStrategy, const int bestStrategy) {
     }
 }
 
-static ZSTD_compressionParameters maskParams(ZSTD_compressionParameters base, ZSTD_compressionParameters mask) {
-    base.windowLog = mask.windowLog ? mask.windowLog : base.windowLog;
-    base.chainLog = mask.chainLog ? mask.chainLog : base.chainLog;
-    base.hashLog = mask.hashLog ? mask.hashLog : base.hashLog;
-    base.searchLog = mask.searchLog ? mask.searchLog : base.searchLog;
-    base.searchLength = mask.searchLength ? mask.searchLength : base.searchLength;
-    base.targetLength = mask.targetLength ? mask.targetLength : base.targetLength;
-    base.strategy = mask.strategy ? mask.strategy : base.strategy;
-    return base;
-}
-
 /* experiment with playing with this and decay value */
 
 /* main fn called when using --optimize */
@@ -2115,6 +2116,8 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
     contexts_t ctx;
     buffers_t buf;
 
+    g_time = UTIL_getTime();
+
     /* Init */
     if(!cParamValid(paramTarget)) {
         return 1;
@@ -2202,7 +2205,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
         g_lvltarget.cSize /= ((double)g_strictness) / 100;
 
         target.cSpeed = (U32)g_lvltarget.cSpeed;  
-        target.dSpeed = (U32)g_lvltarget.dSpeed; //See if this is worth
+        target.dSpeed = (U32)g_lvltarget.dSpeed; //See if this is reasonable. 
 
         BMK_printWinnerOpt(stdout, cLevel, winner.result, winner.params, target, buf.srcSize);
     }
@@ -2214,6 +2217,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
     } else {
         DISPLAY("optimizing for %lu Files", (unsigned long)nbFiles);
     }
+
     if(target.cSpeed != 0) { DISPLAY(" - limit compression speed %u MB/s", target.cSpeed >> 20); }
     if(target.dSpeed != 0) { DISPLAY(" - limit decompression speed %u MB/s", target.dSpeed >> 20); }
     if(target.cMem != (U32)-1) { DISPLAY(" - limit memory %u MB", target.cMem >> 20); }
@@ -2369,6 +2373,7 @@ static int usage_advanced(void)
     DISPLAY( " -t#          : Caps runtime of operation in seconds (default : %u seconds (%.1f hours)) \n", (U32)g_grillDuration_s, g_grillDuration_s / 3600);
     DISPLAY( " -v           : Prints Benchmarking output\n");
     DISPLAY( " -D           : Next argument dictionary file\n");
+    DISPLAY( " -s           : Seperate Files\n");
     return 0;
 }
 
@@ -2401,13 +2406,13 @@ int main(int argc, const char** argv)
     const char* input_filename = NULL;
     const char* dictFileName = NULL;
     U32 main_pause = 0;
-    int optimizerCLevel = 0;
+    int cLevel = 0;
+    int seperateFiles = 0;
 
     constraint_t target = { 0, 0, (U32)-1 }; 
 
-    ZSTD_compressionParameters paramTarget = { 0, 0, 0, 0, 0, 0, 0 };
-
-    g_time = UTIL_getTime();
+    ZSTD_compressionParameters paramTarget = emptyParams();
+    g_params = emptyParams();
 
     assert(argc>=1);   /* for exename */
 
@@ -2430,11 +2435,11 @@ int main(int argc, const char** argv)
                 PARSE_SUB_ARGS("compressionSpeed=" ,  "cSpeed=", target.cSpeed);
                 PARSE_SUB_ARGS("decompressionSpeed=", "dSpeed=", target.dSpeed); 
                 PARSE_SUB_ARGS("compressionMemory=" , "cMem=", target.cMem);
-                PARSE_SUB_ARGS("level=", "lvl=", optimizerCLevel);
+                PARSE_SUB_ARGS("level=", "lvl=", cLevel);
                 PARSE_SUB_ARGS("strict=", "stc=", g_strictness);
                 PARSE_SUB_ARGS("preferSpeed=", "prfSpd=", g_speedMultiplier);
                 PARSE_SUB_ARGS("preferRatio=", "prfRto=", g_ratioMultiplier);
-                PARSE_SUB_ARGS("maxTries", "tries", g_maxTries);
+                PARSE_SUB_ARGS("maxTries=", "tries=", g_maxTries);
 
                 DISPLAY("invalid optimization parameter \n");
                 return 1;
@@ -2448,10 +2453,11 @@ int main(int argc, const char** argv)
         } else if (longCommandWArg(&argument, "--zstd=")) {
         /* Decode command (note : aggregated commands are allowed) */
             g_singleRun = 1;
-            g_params = ZSTD_getCParams(2, g_blockSize, 0);
+            cLevel = 2;
             for ( ; ;) {
                 PARSE_CPARAMS(g_params)
-                if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { g_params = ZSTD_getCParams(readU32FromChar(&argument), g_blockSize, 0); if (argument[0]==',') { argument++; continue; } else break; }
+                if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevel = readU32FromChar(&argument); g_params = emptyParams(); if (argument[0]==',') { argument++; continue; } else break; }
+
                 DISPLAY("invalid compression parameter \n");
                 return 1;
             }
@@ -2528,8 +2534,8 @@ int main(int argc, const char** argv)
                             continue;
                         case 'L':
                             {   argument++;
-                                int const cLevel = readU32FromChar(&argument);
-                                g_params = ZSTD_getCParams(cLevel, g_blockSize, 0);
+                                cLevel = readU32FromChar(&argument);
+                                g_params = emptyParams();
                                 continue;
                             }
                         default : ;
@@ -2558,6 +2564,10 @@ int main(int argc, const char** argv)
                     g_grillDuration_s = (double)readU32FromChar(&argument);
                     break;
 
+                case 's':
+                    seperateFiles = 1;
+                    break;
+
                 /* load dictionary file (only applicable for optimizer rn) */
                 case 'D':
                     if(i == argc - 1) { /* last argument, return error. */ 
@@ -2588,11 +2598,24 @@ int main(int argc, const char** argv)
             result = benchSample();
         }
     } else {
-        if (g_optimizer) {
-            result = optimizeForSize(argv+filenamesStart, argc-filenamesStart, dictFileName, target, paramTarget, optimizerCLevel);
+        if(seperateFiles) {
+            for(i = 0; i < argc - filenamesStart; i++) {
+                if (g_optimizer) {
+                    result = optimizeForSize(argv+filenamesStart + i, 1, dictFileName, target, paramTarget, cLevel);
+                    if(result) { DISPLAY("Error on File %d", i); return result; }
+                } else {
+                    result = benchFiles(argv+filenamesStart + i, 1, dictFileName, cLevel);
+                    if(result) { DISPLAY("Error on File %d", i); return result; }
+                }
+            }
         } else {
-            result = benchFiles(argv+filenamesStart, argc-filenamesStart, dictFileName);
-    }   }
+            if (g_optimizer) {
+                result = optimizeForSize(argv+filenamesStart, argc-filenamesStart, dictFileName, target, paramTarget, cLevel);
+            } else {
+                result = benchFiles(argv+filenamesStart, argc-filenamesStart, dictFileName, cLevel);
+            }
+        }   
+    }
 
     if (main_pause) { int unused; printf("press enter...\n"); unused = getchar(); (void)unused; }
 

From 3b36fe5c68952dd03556ab04ff50dcbe0f1c9ec4 Mon Sep 17 00:00:00 2001
From: George Lu <gclu@fb.com>
Date: Tue, 31 Jul 2018 11:13:44 -0700
Subject: [PATCH 13/15] strategy switching

---
 tests/paramgrill.c | 124 +++++++++++++++++++++++----------------------
 1 file changed, 64 insertions(+), 60 deletions(-)

diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index cba97ebb..955032e8 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -73,10 +73,11 @@ typedef enum {
     hlog_ind = 2,
     slog_ind = 3,
     slen_ind = 4,
-    tlen_ind = 5
+    tlen_ind = 5,
+    strt_ind = 6
 } varInds_t;
 
-#define NUM_PARAMS 6
+#define NUM_PARAMS 7
 /* just don't use strategy as a param. */ 
 
 #undef ZSTD_WINDOWLOG_MAX
@@ -91,9 +92,10 @@ typedef enum {
 #define SLOG_RANGE (ZSTD_SEARCHLOG_MAX - ZSTD_SEARCHLOG_MIN + 1)
 #define SLEN_RANGE (ZSTD_SEARCHLENGTH_MAX - ZSTD_SEARCHLENGTH_MIN + 1)
 #define TLEN_RANGE 17
+#define STRT_RANGE (ZSTD_btultra - ZSTD_fast + 1)
 /* TLEN_RANGE picked manually */
 
-static const int rangetable[NUM_PARAMS] = { WLOG_RANGE, CLOG_RANGE, HLOG_RANGE, SLOG_RANGE, SLEN_RANGE, TLEN_RANGE };
+static const int rangetable[NUM_PARAMS] = { WLOG_RANGE, CLOG_RANGE, HLOG_RANGE, SLOG_RANGE, SLEN_RANGE, TLEN_RANGE, STRT_RANGE };
 static const U32 tlen_table[TLEN_RANGE] = { 0, 1, 2, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 256, 512, 999 };
 /*-************************************
 *  Benchmark Parameters/Global Variables
@@ -1213,8 +1215,10 @@ static int sanitizeVarArray(varInds_t* varNew, const int varLength, const varInd
     int i, j = 0;
     for(i = 0; i < varLength; i++) {
         if( !((varArray[i] == clog_ind && strat == ZSTD_fast)
+            || (varArray[i] == slog_ind && strat == ZSTD_fast)
             || (varArray[i] == slog_ind && strat == ZSTD_dfast) 
-            || (varArray[i] == tlen_ind && strat != ZSTD_btopt && strat != ZSTD_btultra && strat != ZSTD_fast))) {
+            || (varArray[i] == tlen_ind && strat != ZSTD_btopt && strat != ZSTD_btultra && strat != ZSTD_fast)
+            /* || varArray[i] == strt_ind */ )) {
             varNew[j] = varArray[i];
             j++;
         }
@@ -1251,6 +1255,10 @@ static int variableParams(const ZSTD_compressionParameters paramConstraints, var
         res[j] = tlen_ind;
         j++;
     }
+    if(!paramConstraints.strategy) {
+        res[j] = strt_ind;
+        j++;
+    }
     return j;
 }
 
@@ -1285,6 +1293,7 @@ static void paramVaryOnce(const varInds_t paramIndex, const int amt, ZSTD_compre
         case tlen_ind: 
             ptr->targetLength = tlen_table[MAX(0, MIN(TLEN_RANGE - 1, tlen_inv(ptr->targetLength) + amt))];
             break;
+        case strt_ind: ptr->strategy     += amt; break;
         default: break;
     }
 }
@@ -1301,37 +1310,19 @@ static void paramVariation(ZSTD_compressionParameters* ptr, const varInds_t* var
             const U32 changeID = FUZ_rand(&g_rand) % (varyLen << 1);
             paramVaryOnce(varyParams[changeID >> 1], ((changeID & 1) << 1) - 1, &p);
         }
-        validated = !ZSTD_isError(ZSTD_checkCParams(p));
+        validated = !ZSTD_isError(ZSTD_checkCParams(p)) && p.strategy > 0;
     }
     *ptr = p;
 }
 
-/* maybe put strategy back in */
-static void paramVariationWithStrategy(ZSTD_compressionParameters* ptr, const varInds_t* varyParams, const int varyLen, const U32 nbChanges)
-{
-    ZSTD_compressionParameters p;
-    U32 validated = 0;
-    while (!validated) {
-        U32 i;
-        p = *ptr;
-        for (i = 0 ; i < nbChanges ; i++) {
-            const U32 changeID = FUZ_rand(&g_rand) % ((varyLen + 1) << 1);
-            if(changeID < (U32)(varyLen << 1)) {
-                paramVaryOnce(varyParams[changeID >> 1], ((changeID & 1) << 1) - 1, &p);
-            } else {
-                p.strategy += ((FUZ_rand(&g_rand) % 2) << 1) - 1; /* +/- 1 */
-            }
-        }
-        validated = !ZSTD_isError(ZSTD_checkCParams(p));
-    }
-    *ptr = p;
-}
 /* length of memo table given free variables */
 static size_t memoTableLen(const varInds_t* varyParams, const int varyLen) {
     size_t arrayLen = 1;
     int i;
     for(i = 0; i < varyLen; i++) {
-        arrayLen *= rangetable[varyParams[i]];
+        if(varyParams[i] != strt_ind) {
+            arrayLen *= rangetable[varyParams[i]];
+        }
     }
     return arrayLen;
 }
@@ -1354,6 +1345,7 @@ static unsigned memoTableInd(const ZSTD_compressionParameters* ptr, const varInd
                 - ZSTD_SEARCHLENGTH_MIN; break;
             case tlen_ind: ind *= TLEN_RANGE; ind += tlen_inv(ptr->targetLength) 
                 - ZSTD_TARGETLENGTH_MIN; break;
+            case strt_ind: break;
         }
     }
     return ind;
@@ -1376,6 +1368,7 @@ static void memoTableIndInv(ZSTD_compressionParameters* ptr, const varInds_t* va
                 ind /= SLEN_RANGE; break;
             case tlen_ind: ptr->targetLength = tlen_table[(ind % TLEN_RANGE)];
                 ind /= TLEN_RANGE; break;
+            case strt_ind: break;
         }
     }
 }
@@ -1505,7 +1498,7 @@ static void playAround(FILE* f, winnerInfo_t* winners,
 {
     int nbVariations = 0;
     UTIL_time_t const clockStart = UTIL_getTime();
-    const U32 unconstrained[NUM_PARAMS] = { 0, 1, 2, 3, 4, 5 };
+    const U32 unconstrained[NUM_PARAMS] = { 0, 1, 2, 3, 4, 5, 6 };
 
 
     while (UTIL_clockSpanMicro(clockStart) < g_maxVariationTime) {
@@ -1513,7 +1506,7 @@ static void playAround(FILE* f, winnerInfo_t* winners,
         BYTE* b;
 
         if (nbVariations++ > g_maxNbVariations) break;
-        paramVariationWithStrategy(&p, unconstrained, NUM_PARAMS, 4);
+        paramVariation(&p, unconstrained, NUM_PARAMS, 4);
 
         /* exclude faster if already played params */
         if (FUZ_rand(&g_rand) & ((1 << *NB_TESTS_PLAYED(p))-1))
@@ -1715,8 +1708,7 @@ static int benchOnce(buffers_t buf, contexts_t ctx) {
         return 1;
     }
 
-    DISPLAY("Compression Ratio: %.3f  Compress Speed: %.1f MB/s Decompress Speed: %.1f MB/s\n", (double)buf.srcSize / testResult.cSize, 
-        (double)testResult.cSpeed / (1 MB), (double)testResult.dSpeed / (1 MB));
+    BMK_printWinner(stdout, CUSTOM_LEVEL, testResult, g_params, buf.srcSize);
     return 0;
 }
 
@@ -1766,7 +1758,7 @@ int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileNam
 }
 
 /* Benchmarking which stops when we are sufficiently sure the solution is infeasible / worse than the winner */
-#define VARIANCE 1.1 
+#define VARIANCE 1.2 
 static int allBench(BMK_result_t* resultPtr,
                 const buffers_t buf, const contexts_t ctx,
                 const ZSTD_compressionParameters cParams,
@@ -1918,9 +1910,9 @@ static int benchMemo(BMK_result_t* resultPtr,
  * all generation after random should be sanitized. (maybe sanitize random)
  */
 static winnerInfo_t climbOnce(const constraint_t target, 
-                const varInds_t* varArray, const int varLen, 
-                U8* const memoTable,
-                const buffers_t buf, const contexts_t ctx,
+                const varInds_t* varArray, const int varLen, ZSTD_strategy strat,
+                U8** memoTableArray, 
+                buffers_t buf, contexts_t ctx,
                 const ZSTD_compressionParameters init) {
     /* 
      * cparam - currently considered 'center'
@@ -1931,6 +1923,8 @@ static winnerInfo_t climbOnce(const constraint_t target,
     winnerInfo_t candidateInfo, winnerInfo;
     int better = 1;
     int feas = 0;
+    varInds_t varNew[NUM_PARAMS];
+    int varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat);
 
     winnerInfo = initWinnerInfo(init);
     candidateInfo = winnerInfo;
@@ -1951,15 +1945,20 @@ static winnerInfo_t climbOnce(const constraint_t target,
                 for(offset = -1; offset <= 1; offset += 2) {
                     candidateInfo.params = cparam;
                     paramVaryOnce(varArray[i], offset, &candidateInfo.params); 
-                    candidateInfo.params = sanitizeParams(candidateInfo.params);
-                    if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) {
-                        int res = benchMemo(&candidateInfo.result,
+                    
+                    if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params)) && candidateInfo.params.strategy > 0) {
+                        int res;
+                        if(strat != candidateInfo.params.strategy) { /* maybe only try strategy switching after exhausting non-switching solutions? */
+                            strat = candidateInfo.params.strategy;
+                            varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat);
+                        }
+                        res = benchMemo(&candidateInfo.result,
                             buf, ctx,
-                            candidateInfo.params, target, &winnerInfo.result, memoTable,
-                            varArray, varLen, feas);
+                            sanitizeParams(candidateInfo.params), target, &winnerInfo.result, memoTableArray[strat],
+                            varNew, varLenNew, feas);
                         if(res == BETTER_RESULT) { /* synonymous with better when called w/ infeasibleBM */
                             winnerInfo = candidateInfo;
-                            BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, target, buf.srcSize);
+                            BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, sanitizeParams(winnerInfo.params), target, buf.srcSize);
                             better = 1;
                             if(compareResultLT(bestFeasible1.result, winnerInfo.result, target, buf.srcSize)) {
                                 bestFeasible1 = winnerInfo;
@@ -1974,22 +1973,29 @@ static winnerInfo_t climbOnce(const constraint_t target,
             }
 
             for(dist = 2; dist < varLen + 2; dist++) { /* varLen is # dimensions */
-                for(i = 0; i < 2 * varLen + 2; i++) {
+                for(i = 0; i < (1 << varLen) / varLen + 2; i++) {
                     int res;
                     candidateInfo.params = cparam;
                     /* param error checking already done here */
                     paramVariation(&candidateInfo.params, varArray, varLen, dist);
+
+                    if(strat != candidateInfo.params.strategy) {
+                        strat = candidateInfo.params.strategy;
+                        varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat);
+                    }
+
                     res = benchMemo(&candidateInfo.result,
                         buf, ctx, 
-                        candidateInfo.params, target, &winnerInfo.result, memoTable,
-                        varArray, varLen, feas);
+                        sanitizeParams(candidateInfo.params), target, &winnerInfo.result, memoTableArray[strat],
+                        varNew, varLenNew, feas);
                     if(res == BETTER_RESULT) { /* synonymous with better in this case*/
                         winnerInfo = candidateInfo;
-                        BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, target, buf.srcSize);
+                        BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, sanitizeParams(winnerInfo.params), target, buf.srcSize);
                         better = 1;
                         if(compareResultLT(bestFeasible1.result, winnerInfo.result, target, buf.srcSize)) {
                             bestFeasible1 = winnerInfo;
                         }
+                        break;
                     }
 
                 }
@@ -2026,10 +2032,11 @@ static winnerInfo_t optimizeFixedStrategy(
     const constraint_t target, ZSTD_compressionParameters paramTarget,
     const ZSTD_strategy strat, 
     const varInds_t* varArray, const int varLen,
-    U8* const memoTable, const int tries) {
+    U8** memoTableArray, const int tries) {
     int i = 0;
     varInds_t varNew[NUM_PARAMS];
     int varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat);
+
     ZSTD_compressionParameters init;
     winnerInfo_t winnerInfo, candidateInfo; 
     winnerInfo = initWinnerInfo(emptyParams());
@@ -2043,8 +2050,8 @@ static winnerInfo_t optimizeFixedStrategy(
 
     while(i < tries) {
         DEBUGOUTPUT("Restart\n"); 
-        randomConstrainedParams(&init, varNew, varLenNew, memoTable);
-        candidateInfo = climbOnce(target, varNew, varLenNew, memoTable, buf, ctx, init);
+        randomConstrainedParams(&init, varNew, varLenNew, memoTableArray[strat]);
+        candidateInfo = climbOnce(target, varArray, varLen, strat, memoTableArray, buf, ctx, init);
         if(compareResultLT(winnerInfo.result, candidateInfo.result, target, buf.srcSize)) {
             winnerInfo = candidateInfo;
             BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, target, buf.srcSize);
@@ -2226,7 +2233,6 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
     findClockGranularity();
 
     {   
-        varInds_t varNew[NUM_PARAMS];
         ZSTD_compressionParameters CParams;
 
         /* find best solution from default params */
@@ -2266,8 +2272,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
 
                 { 
                     /* one iterations of hill climbing with the level-defined parameters. */
-                    int varLenNew = sanitizeVarArray(varNew, varLen, varArray, st);
-                    winnerInfo_t w1 = climbOnce(target, varNew, varLenNew, allMT[st], 
+                    winnerInfo_t w1 = climbOnce(target, varArray, varLen, st, allMT, 
                         buf, ctx, winner.params);
                     if(compareResultLT(winner.result, w1.result, target, buf.srcSize)) {
                         winner = w1;
@@ -2279,7 +2284,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
                     DEBUGOUTPUT("StrategySwitch: %s\n", g_stratName[st]);
                     
                     wc = optimizeFixedStrategy(buf, ctx, target, paramTarget, 
-                        st, varArray, varLen, allMT[st], tries);
+                        st, varArray, varLen, allMT, tries);
 
                     if(compareResultLT(winner.result, wc.result, target, buf.srcSize)) {
                         winner = wc;
@@ -2292,7 +2297,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
                 }
             } else {
                 winner = optimizeFixedStrategy(buf, ctx, target, paramTarget, paramTarget.strategy, 
-                    varArray, varLen, allMT[paramTarget.strategy], g_maxTries);
+                    varArray, varLen, allMT, g_maxTries);
             }
 
         }
@@ -2387,14 +2392,13 @@ static int badusage(const char* exename)
 #define PARSE_SUB_ARGS(stringLong, stringShort, variable) { if (longCommandWArg(&argument, stringLong) || longCommandWArg(&argument, stringShort)) { variable = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } }
 #define PARSE_CPARAMS(variable)                                            \
 {                                                                          \
-    PARSE_SUB_ARGS("windowLog=",       "wlog=",  variable.vals[wlog_ind]); \
-    PARSE_SUB_ARGS("chainLog=" ,       "clog=",  variable.vals[clog_ind]); \
-    PARSE_SUB_ARGS("hashLog=",         "hlog=",  variable.vals[hlog_ind]); \
-    PARSE_SUB_ARGS("searchLog=" ,      "slog=",  variable.vals[slog_ind]); \
-    PARSE_SUB_ARGS("searchLength=",    "slen=",  variable.vals[slen_ind]); \
-    PARSE_SUB_ARGS("targetLength=" ,   "tlen=",  variable.vals[tlen_ind]); \
-    PARSE_SUB_ARGS("strategy=",        "strat=", variable.vals[strt_ind]); \
-    PARSE_SUB_ARGS("forceAttachDict=", "fad="  , variable.vals[strt_ind]); \
+    PARSE_SUB_ARGS("windowLog=",       "wlog=",  variable.windowLog);      \
+    PARSE_SUB_ARGS("chainLog=" ,       "clog=",  variable.chainLog);       \
+    PARSE_SUB_ARGS("hashLog=",         "hlog=",  variable.hashLog);        \
+    PARSE_SUB_ARGS("searchLog=" ,      "slog=",  variable.searchLog);      \
+    PARSE_SUB_ARGS("searchLength=",    "slen=",  variable.searchLength);   \
+    PARSE_SUB_ARGS("targetLength=" ,   "tlen=",  variable.targetLength);   \
+    PARSE_SUB_ARGS("strategy=",        "strat=", variable.strategy);       \
 }
 
 int main(int argc, const char** argv)

From 614aaa3ae152aa13c3b17ae01a39f151be6c1c56 Mon Sep 17 00:00:00 2001
From: George Lu <gclu@fb.com>
Date: Mon, 13 Aug 2018 16:38:51 -0700
Subject: [PATCH 14/15] rebase clevel

---
 tests/paramgrill.c | 63 ++++++++++++++++++++++++----------------------
 1 file changed, 33 insertions(+), 30 deletions(-)

diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index 955032e8..58f2e13b 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -65,6 +65,7 @@ static const int g_maxNbVariations = 64;
 #define MIN(a,b)   ( (a) < (b) ? (a) : (b) )
 #define MAX(a,b)   ( (a) > (b) ? (a) : (b) )
 #define CUSTOM_LEVEL 99
+#define BASE_CLEVEL 1
 
 /* indices for each of the variables */
 typedef enum {
@@ -112,7 +113,7 @@ static U32 g_singleRun = 0;
 static U32 g_optimizer = 0;
 static U32 g_target = 0;
 static U32 g_noSeed = 0;
-static ZSTD_compressionParameters g_params;
+static ZSTD_compressionParameters g_params; /* Initialized at the beginning of main w/ emptyParams() function */
 static UTIL_time_t g_time; /* to be used to compare solution finding speeds to compare to original */
 
 
@@ -523,6 +524,7 @@ static void freeBuffers(const buffers_t b) {
         free(b.resPtrs[0]);
     }
     free(b.resPtrs);
+    free(b.resSizes);
 }
 
 /* allocates buffer's arguments. returns success / failuere */
@@ -532,7 +534,7 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab
     size_t pos = 0;
     size_t n;
     U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, (U32)nbFiles);
-    size_t benchedSize = MIN(BMK_findMaxMem(totalSizeToLoad * 3) / 3, totalSizeToLoad);
+    const size_t benchedSize = MIN(BMK_findMaxMem(totalSizeToLoad * 3) / 3, totalSizeToLoad);
     const size_t blockSize = g_blockSize ? g_blockSize : totalSizeToLoad;
     U32 const maxNbBlocks = (U32) ((totalSizeToLoad + (blockSize-1)) / blockSize) + (U32)nbFiles;
     U32 blockNb = 0;
@@ -671,6 +673,7 @@ static int createContexts(contexts_t* ctx, const char* dictFileName) {
         freeContexts(*ctx);
         return 1;
     }
+    fclose(f);
     return 0;
 }
 
@@ -824,7 +827,7 @@ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t
 static int BMK_benchParam(BMK_result_t* resultPtr,
                 buffers_t buf, contexts_t ctx,
                 const ZSTD_compressionParameters cParams) {
-    BMK_return_t res = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_both, BMK_timeMode, 3);
+    BMK_return_t res = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_both, BMK_timeMode, 3);
     *resultPtr = res.result;
     return res.error;
 }
@@ -840,16 +843,16 @@ static int BMK_benchParam(BMK_result_t* resultPtr,
 #define SIZE_RESULT 5
 /* maybe have epsilon-eq to limit table size? */
 static int speedSizeCompare(BMK_result_t r1, BMK_result_t r2) {
-    if(r1.cSpeed > r2.cSpeed) {
-        if(r1.cSize <= r2.cSize) {
-            return WORSE_RESULT;
-        }
-        return SIZE_RESULT; /* r2 is smaller but not faster. */
-    } else {
+    if(r1.cSpeed < r2.cSpeed) {
         if(r1.cSize >= r2.cSize) {
             return BETTER_RESULT;
         }
-        return SPEED_RESULT; /* r2 is faster but not smaller */
+        return SPEED_RESULT; /* r2 is smaller but not faster. */
+    } else {
+        if(r1.cSize <= r2.cSize) {
+            return WORSE_RESULT;
+        }
+        return SIZE_RESULT; /* r2 is faster but not smaller */
     }
 }
 
@@ -875,12 +878,12 @@ static int insertWinner(winnerInfo_t w, constraint_t targetConstraints) {
     }
 
     while(cur_node->next != NULL) {
-        switch(speedSizeCompare(r, cur_node->res.result)) {
-            case BETTER_RESULT:
+        switch(speedSizeCompare(cur_node->res.result, r)) {
+            case WORSE_RESULT:
             {
                 return 1; /* never insert if better */
             }
-            case WORSE_RESULT:
+            case BETTER_RESULT:
             {
                 winner_ll_node* tmp;
                 cur_node->res = cur_node->next->res;
@@ -889,12 +892,12 @@ static int insertWinner(winnerInfo_t w, constraint_t targetConstraints) {
                 free(tmp);
                 break; 
             }
-            case SPEED_RESULT:
+            case SIZE_RESULT:
             {
                 cur_node = cur_node->next;
                 break;
             }
-            case SIZE_RESULT: /* insert after first size result, then return */
+            case SPEED_RESULT: /* insert after first size result, then return */
             {
                 winner_ll_node* newnode = malloc(sizeof(winner_ll_node));
                 if(newnode == NULL) {
@@ -911,17 +914,17 @@ static int insertWinner(winnerInfo_t w, constraint_t targetConstraints) {
     }
 
     assert(cur_node->next == NULL);
-    switch(speedSizeCompare(r, cur_node->res.result)) {
-        case BETTER_RESULT:
+    switch(speedSizeCompare(cur_node->res.result, r)) {
+        case WORSE_RESULT:
         {
             return 1; /* never insert if better */
         }
-        case WORSE_RESULT:
+        case BETTER_RESULT:
         {
             cur_node->res = w;
             return 0;
         }
-        case SPEED_RESULT:
+        case SIZE_RESULT:
         {
             winner_ll_node* newnode = malloc(sizeof(winner_ll_node));
             if(newnode == NULL) {
@@ -932,7 +935,7 @@ static int insertWinner(winnerInfo_t w, constraint_t targetConstraints) {
             cur_node->next = newnode;
             return 0;
         }
-        case SIZE_RESULT: /* insert before first size result, then return */
+        case SPEED_RESULT: /* insert before first size result, then return */
         {
             winner_ll_node* newnode = malloc(sizeof(winner_ll_node));
             if(newnode == NULL) {
@@ -1448,7 +1451,7 @@ static void freeMemoTableArray(U8** mtAll) {
 /* takes unsanitized varyParams */
 static U8** createMemoTableArray(ZSTD_compressionParameters paramConstraints, constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) {
     varInds_t varNew[NUM_PARAMS];
-    U8** mtAll = calloc(sizeof(U8*),(ZSTD_btultra + 1));
+    U8** mtAll = (U8**)calloc(sizeof(U8*),(ZSTD_btultra + 1));
     int i;
     if(mtAll == NULL) {
         return NULL;
@@ -1467,7 +1470,7 @@ static U8** createMemoTableArray(ZSTD_compressionParameters paramConstraints, co
     return mtAll;
 }
 
-static ZSTD_compressionParameters maskParams(ZSTD_compressionParameters base, ZSTD_compressionParameters mask) {
+static ZSTD_compressionParameters overwriteParams(ZSTD_compressionParameters base, ZSTD_compressionParameters mask) {
     base.windowLog = mask.windowLog ? mask.windowLog : base.windowLog;
     base.chainLog = mask.chainLog ? mask.chainLog : base.chainLog;
     base.hashLog = mask.hashLog ? mask.hashLog : base.hashLog;
@@ -1744,7 +1747,7 @@ int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileNam
         DISPLAY("using %d Files : \n", nbFiles);
     }
 
-    g_params = ZSTD_adjustCParams(maskParams(ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize), g_params), maxBlockSize, ctx.dictSize);
+    g_params = ZSTD_adjustCParams(overwriteParams(ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize), g_params), maxBlockSize, ctx.dictSize);
 
     if(g_singleRun) {
         ret = benchOnce(buf, ctx);
@@ -1771,7 +1774,7 @@ static int allBench(BMK_result_t* resultPtr,
     double winnerRS;
 
     /* initial benchmarking, gives exact ratio and memory, warms up future runs */
-    benchres = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_both, BMK_iterMode, 1);
+    benchres = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_both, BMK_iterMode, 1);
 
     winnerRS = resultScore(*winnerResult, buf.srcSize, target);
     DEBUGOUTPUT("WinnerScore: %f\n ", winnerRS);
@@ -1806,14 +1809,14 @@ static int allBench(BMK_result_t* resultPtr,
 
     /* second run, if first run is too short, gives approximate cSpeed + dSpeed */
     if(loopDurationC < TIMELOOP_NANOSEC / 10) {
-        BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_compressOnly, BMK_iterMode, 1);
+        BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_compressOnly, BMK_iterMode, 1);
         if(benchres2.error) {
             return ERROR_RESULT;
         }
         benchres = benchres2;
     }
     if(loopDurationD < TIMELOOP_NANOSEC / 10) {
-        BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_decodeOnly, BMK_iterMode, 1);
+        BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_decodeOnly, BMK_iterMode, 1);
         if(benchres2.error) {
             return ERROR_RESULT;
         }
@@ -1836,7 +1839,7 @@ static int allBench(BMK_result_t* resultPtr,
 
     /* Final full run if estimates are unclear */
     if(loopDurationC < TIMELOOP_NANOSEC) {
-        BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_compressOnly, BMK_timeMode, 1);
+        BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_compressOnly, BMK_timeMode, 1);
         if(benchres2.error) {
             return ERROR_RESULT;
         }
@@ -1844,7 +1847,7 @@ static int allBench(BMK_result_t* resultPtr,
     } 
 
     if(loopDurationD < TIMELOOP_NANOSEC) {
-        BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_decodeOnly, BMK_timeMode, 1);
+        BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_decodeOnly, BMK_timeMode, 1);
         if(benchres2.error) {
             return ERROR_RESULT;
         }
@@ -2245,7 +2248,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
                 int i;
                 for (i=1; i<=maxSeeds; i++) {
                     int ec;
-                    CParams = maskParams(ZSTD_getCParams(i, maxBlockSize, ctx.dictSize), paramTarget);
+                    CParams = overwriteParams(ZSTD_getCParams(i, maxBlockSize, ctx.dictSize), paramTarget);
                     ec = BMK_benchParam(&candidate, buf, ctx, CParams);
                     BMK_printWinnerOpt(stdout, i, candidate, CParams, target, buf.srcSize);
 
@@ -2439,11 +2442,11 @@ int main(int argc, const char** argv)
                 PARSE_SUB_ARGS("compressionSpeed=" ,  "cSpeed=", target.cSpeed);
                 PARSE_SUB_ARGS("decompressionSpeed=", "dSpeed=", target.dSpeed); 
                 PARSE_SUB_ARGS("compressionMemory=" , "cMem=", target.cMem);
-                PARSE_SUB_ARGS("level=", "lvl=", cLevel);
                 PARSE_SUB_ARGS("strict=", "stc=", g_strictness);
                 PARSE_SUB_ARGS("preferSpeed=", "prfSpd=", g_speedMultiplier);
                 PARSE_SUB_ARGS("preferRatio=", "prfRto=", g_ratioMultiplier);
                 PARSE_SUB_ARGS("maxTries=", "tries=", g_maxTries);
+                if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevel = readU32FromChar(&argument); g_optmode = 1; if (argument[0]==',') { argument++; continue; } else break; }
 
                 DISPLAY("invalid optimization parameter \n");
                 return 1;

From 76acba025da5ead78532bfd0dff4de6af67a6506 Mon Sep 17 00:00:00 2001
From: George Lu <gclu@fb.com>
Date: Tue, 14 Aug 2018 11:57:15 -0700
Subject: [PATCH 15/15] scan-build

---
 tests/paramgrill.c | 32 +++++++++++++++++++++++---------
 1 file changed, 23 insertions(+), 9 deletions(-)

diff --git a/tests/paramgrill.c b/tests/paramgrill.c
index 58f2e13b..7e85bf83 100644
--- a/tests/paramgrill.c
+++ b/tests/paramgrill.c
@@ -176,14 +176,14 @@ static size_t BMK_findMaxMem(U64 requiredMem)
     requiredMem = (((requiredMem >> 26) + 1) << 26);
     if (requiredMem > maxMemory) requiredMem = maxMemory;
 
-    requiredMem += 2*step;
-    while (!testmem) {
-        requiredMem -= step;
+    requiredMem += 2 * step;
+    while (!testmem && requiredMem > 0) {
         testmem = malloc ((size_t)requiredMem);
+        requiredMem -= step;
     }
 
     free (testmem);
-    return (size_t) (requiredMem - step);
+    return (size_t) requiredMem;
 }
 
 
@@ -536,9 +536,14 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab
     U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, (U32)nbFiles);
     const size_t benchedSize = MIN(BMK_findMaxMem(totalSizeToLoad * 3) / 3, totalSizeToLoad);
     const size_t blockSize = g_blockSize ? g_blockSize : totalSizeToLoad;
-    U32 const maxNbBlocks = (U32) ((totalSizeToLoad + (blockSize-1)) / blockSize) + (U32)nbFiles;
+    U32 const maxNbBlocks = (U32) ((totalSizeToLoad + (blockSize-1)) / MAX(blockSize, 1)) + (U32)nbFiles;
     U32 blockNb = 0;
 
+    if(!totalSizeToLoad || !benchedSize) {
+        DISPLAY("Nothing to Bench\n");
+        return 1;
+    }
+    
     buff->srcPtrs = (const void**)calloc(maxNbBlocks, sizeof(void*));
     buff->srcSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
 
@@ -555,6 +560,7 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab
         return 1;
     }
 
+
     buff->srcBuffer = malloc(benchedSize);
     buff->srcPtrs[0] = (const void*)buff->srcBuffer;
     buff->dstPtrs[0] = malloc(ZSTD_compressBound(benchedSize) + (maxNbBlocks * 1024));
@@ -613,6 +619,12 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab
         fclose(f);
     }
 
+    if(!blockNb) {
+        DISPLAY("Failed to load any files\n");
+        freeBuffers(*buff);
+        return 1;
+    }
+
     buff->dstCapacities[0] = ZSTD_compressBound(buff->srcSizes[0]);
     buff->dstSizes[0] = buff->dstCapacities[0];
     buff->resSizes[0] = buff->srcSizes[0];
@@ -957,8 +969,6 @@ static int insertWinner(winnerInfo_t w, constraint_t targetConstraints) {
 static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result, const ZSTD_compressionParameters params, const size_t srcSize)
 {
     char lvlstr[15] = "Custom Level";
-    const U64 time = UTIL_clockSpanNano(g_time);
-    const U64 minutes = time / (60ULL * TIMELOOP_NANOSEC);
 
     fprintf(f, "\r%79s\r", "");
 
@@ -974,7 +984,11 @@ static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result
         "/* %s */   /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */",
         lvlstr, (double)srcSize / result.cSize, (double)result.cSpeed / (1 MB), (double)result.dSpeed / (1 MB));
 
-    if(TIMED) { fprintf(f, " - %1lu:%2lu:%05.2f", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC); }
+    if(TIMED) {
+        const U64 time = UTIL_clockSpanNano(g_time);
+        const U64 minutes = time / (60ULL * TIMELOOP_NANOSEC);
+        fprintf(f, " - %1lu:%2lu:%05.2f", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC); 
+    }
     fprintf(f, "\n"); 
 }
 
@@ -2160,7 +2174,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_
     if(paramTarget.strategy) {
         varInds_t varNew[NUM_PARAMS];
         int varLenNew = sanitizeVarArray(varNew, varLen, varArray, paramTarget.strategy);
-        allMT = calloc(sizeof(U8), (ZSTD_btultra + 1));
+        allMT = (U8**)calloc(sizeof(U8*), (ZSTD_btultra + 1));
         if(allMT == NULL) {
             ret = 57;
             goto _cleanUp;