improved memory size evaluation by paramgrill

lib/zstd_compress.c

@@ -129,7 +129,7 @@ size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx)
     return 0;   /* reserved as a potential error code in the future */
 }
 
-seqStore_t ZSTD_copySeqStore(const ZSTD_CCtx* ctx)
+seqStore_t ZSTD_copySeqStore(const ZSTD_CCtx* ctx)   /* hidden interface */
 {
     return ctx->seqStore;
 }
@@ -170,11 +170,26 @@ void ZSTD_validateParams(ZSTD_parameters* params)
 }
 
 
+size_t ZSTD_sizeofCCtx(ZSTD_parameters params)   /* hidden interface, for paramagrill */
+{   /* copy / pasted from ZSTD_resetCCtx_advanced */
+    const size_t blockSize = MIN(BLOCKSIZE, (size_t)1 << params.windowLog);
+    const U32    contentLog = (params.strategy == ZSTD_fast) ? 1 : params.contentLog;
+    const U32    divider = (params.searchLength==3) ? 3 : 4;
+    const size_t maxNbSeq = blockSize / divider;
+    const size_t tokenSpace = blockSize + 8*maxNbSeq;
+    const size_t tableSpace = ((1 << contentLog) + (1 << params.hashLog) + (1 << params.hashLog3)) * sizeof(U32);
+    const size_t optSpace   = ((1<<MLbits) + (1<<LLbits) + (1<<Offbits) + (1<<Litbits))*sizeof(U32) + (ZSTD_OPT_NUM+1)*(sizeof(ZSTD_match_t) + sizeof(ZSTD_optimal_t));
+    const size_t neededSpace = tableSpace + (256*sizeof(U32)) /* huffTable */ + tokenSpace
+                           + ((params.strategy == ZSTD_btopt) ? optSpace : 0);
+
+    return sizeof(ZSTD_CCtx) + neededSpace;
+}
+
+
 static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
                                        ZSTD_parameters params)
 {   /* note : params considered validated here */
     const size_t blockSize = MIN(BLOCKSIZE, (size_t)1 << params.windowLog);
-    /* reserve table memory */
     const U32    contentLog = (params.strategy == ZSTD_fast) ? 1 : params.contentLog;
     const U32    divider = (params.searchLength==3) ? 3 : 4;
     const size_t maxNbSeq = blockSize / divider;

programs/.gitignore

@@ -6,6 +6,7 @@ fullbench32
 fuzzer
 fuzzer32
 datagen
+paramgrill
 
 # Object files
 *.o
@@ -30,5 +31,6 @@ datagen
 *.suo
 *.user
 
-# Default dictionary name
+# Default result files
 dictionary
+grillResults.txt

programs/paramgrill.c

@@ -447,6 +447,7 @@ static void BMK_printWinners(FILE* f, const winnerInfo_t* winners, size_t srcSiz
     BMK_printWinners2(stdout, winners, srcSize);
 }
 
+size_t ZSTD_sizeofCCtx(ZSTD_parameters params);   /* hidden interface, declared here */
 
 static int BMK_seed(winnerInfo_t* winners, const ZSTD_parameters params,
               const void* srcBuffer, size_t srcSize,
@@ -481,10 +482,8 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_parameters params,
             double W_DMemUsed_note = W_ratioNote * ( 40 + 9*cLevel) - log((double)W_DMemUsed);
             double O_DMemUsed_note = O_ratioNote * ( 40 + 9*cLevel) - log((double)O_DMemUsed);
 
-            size_t W_CMemUsed = (1 << params.windowLog) + 4 * (1 << params.hashLog) +
-                                ((params.strategy==ZSTD_fast) ? 0 : 4 * (1 << params.contentLog));
-            size_t O_CMemUsed = (1 << winners[cLevel].params.windowLog) + 4 * (1 << winners[cLevel].params.hashLog) +
-                                ((winners[cLevel].params.strategy==ZSTD_fast) ? 0 :  4 * (1 << winners[cLevel].params.contentLog));
+            size_t W_CMemUsed = (1 << params.windowLog) + ZSTD_sizeofCCtx(params);
+            size_t O_CMemUsed = (1 << winners[cLevel].params.windowLog) + ZSTD_sizeofCCtx(winners[cLevel].params);
             double W_CMemUsed_note = W_ratioNote * ( 50 + 13*cLevel) - log((double)W_CMemUsed);
             double O_CMemUsed_note = O_ratioNote * ( 50 + 13*cLevel) - log((double)O_CMemUsed);