[bench] Extending largeNbDicts to compression (#2089)

* adding cdict_collection_t

* adding shuffleCDictionaries()

* adding compressInstructions

* adding compress()

* integrating compression into bench()

* copy paste error fix

* static analyzer uninit value complaint fix

* changing to control

* removing assert

* changing to control

* moving memcpy to seperate function

* fixing static analyzer complaint

* another hacky solution attempt

* Copying createbuffer logic

contrib/largeNbDicts/largeNbDicts.c

@@ -334,6 +334,42 @@ createBufferCollection_fromSliceCollectionSizes(slice_collection_t sc)
     return result;
 }
 
+static buffer_collection_t
+createBufferCollection_fromSliceCollection(slice_collection_t sc)
+{
+    size_t const bufferSize = sliceCollection_totalCapacity(sc);
+
+    buffer_t buffer = createBuffer(bufferSize);
+    CONTROL(buffer.ptr != NULL);
+
+    size_t const nbSlices = sc.nbSlices;
+    void** const slices = (void**)malloc(nbSlices * sizeof(*slices));
+    CONTROL(slices != NULL);
+
+    size_t* const capacities = (size_t*)malloc(nbSlices * sizeof(*capacities));
+    CONTROL(capacities != NULL);
+
+    char* const ptr = (char*)buffer.ptr;
+    size_t pos = 0;
+    for (size_t n=0; n < nbSlices; n++) {
+        capacities[n] = sc.capacities[n];
+        slices[n] = ptr + pos;
+        pos += capacities[n];
+    }
+
+    for (size_t i = 0; i < nbSlices; i++) {
+        memcpy(slices[i], sc.slicePtrs[i], sc.capacities[i]);
+        capacities[i] = sc.capacities[i];
+    }
+
+    buffer_collection_t result;
+    result.buffer = buffer;
+    result.slices.nbSlices = nbSlices;
+    result.slices.capacities = capacities;
+    result.slices.slicePtrs = slices;
+
+    return result;
+}
 
 /* @return : kBuffNull if any error */
 static buffer_collection_t
@@ -397,6 +433,36 @@ typedef struct {
     size_t nbDDict;
 } ddict_collection_t;
 
+typedef struct {
+    ZSTD_CDict** cdicts;
+    size_t nbCDict;
+} cdict_collection_t;
+
+static const cdict_collection_t kNullCDictCollection = { NULL, 0 };
+
+static void freeCDictCollection(cdict_collection_t cdictc)
+{
+    for (size_t dictNb=0; dictNb < cdictc.nbCDict; dictNb++) {
+        ZSTD_freeCDict(cdictc.cdicts[dictNb]);
+    }
+    free(cdictc.cdicts);
+}
+
+/* returns .buffers=NULL if operation fails */
+static cdict_collection_t createCDictCollection(const void* dictBuffer, size_t dictSize, size_t nbCDict, int cLevel)
+{
+    ZSTD_CDict** const cdicts = malloc(nbCDict * sizeof(ZSTD_CDict*));
+    if (cdicts==NULL) return kNullCDictCollection;
+    for (size_t dictNb=0; dictNb < nbCDict; dictNb++) {
+        cdicts[dictNb] = ZSTD_createCDict(dictBuffer, dictSize, cLevel);
+        CONTROL(cdicts[dictNb] != NULL);
+    }
+    cdict_collection_t cdictc;
+    cdictc.cdicts = cdicts;
+    cdictc.nbCDict = nbCDict;
+    return cdictc;
+}
+
 static const ddict_collection_t kNullDDictCollection = { NULL, 0 };
 
 static void freeDDictCollection(ddict_collection_t ddictc)
@@ -425,7 +491,26 @@ static ddict_collection_t createDDictCollection(const void* dictBuffer, size_t d
 
 
 /* mess with addresses, so that linear scanning dictionaries != linear address scanning */
-void shuffleDictionaries(ddict_collection_t dicts)
+void shuffleCDictionaries(cdict_collection_t dicts)
+{
+    size_t const nbDicts = dicts.nbCDict;
+    for (size_t r=0; r<nbDicts; r++) {
+        size_t const d = (size_t)rand() % nbDicts;
+        ZSTD_CDict* tmpd = dicts.cdicts[d];
+        dicts.cdicts[d] = dicts.cdicts[r];
+        dicts.cdicts[r] = tmpd;
+    }
+    for (size_t r=0; r<nbDicts; r++) {
+        size_t const d1 = (size_t)rand() % nbDicts;
+        size_t const d2 = (size_t)rand() % nbDicts;
+        ZSTD_CDict* tmpd = dicts.cdicts[d1];
+        dicts.cdicts[d1] = dicts.cdicts[d2];
+        dicts.cdicts[d2] = tmpd;
+    }
+}
+
+/* mess with addresses, so that linear scanning dictionaries != linear address scanning */
+void shuffleDDictionaries(ddict_collection_t dicts)
 {
     size_t const nbDicts = dicts.nbDDict;
     for (size_t r=0; r<nbDicts; r++) {
@@ -485,6 +570,29 @@ static size_t compressBlocks(size_t* cSizes,   /* optional (can be NULL). If pre
 
 /* ---  Benchmark  --- */
 
+typedef struct {
+    ZSTD_CCtx* cctx;
+    size_t nbDicts;
+    size_t dictNb;
+    cdict_collection_t dictionaries;
+} compressInstructions;
+
+compressInstructions createCompressInstructions(cdict_collection_t dictionaries)
+{
+    compressInstructions ci;
+    ci.cctx = ZSTD_createCCtx();
+    CONTROL(ci.cctx != NULL);
+    ci.nbDicts = dictionaries.nbCDict;
+    ci.dictNb = 0;
+    ci.dictionaries = dictionaries;
+    return ci;
+}
+
+void freeCompressInstructions(compressInstructions ci)
+{
+    ZSTD_freeCCtx(ci.cctx);
+}
+
 typedef struct {
     ZSTD_DCtx* dctx;
     size_t nbDicts;
@@ -508,6 +616,23 @@ void freeDecompressInstructions(decompressInstructions di)
     ZSTD_freeDCtx(di.dctx);
 }
 
+/* benched function */
+size_t compress(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* payload)
+{
+    compressInstructions* const ci = (compressInstructions*) payload;
+    (void)dstCapacity;
+
+    ZSTD_compress_usingCDict(ci->cctx,
+                    dst, srcSize,
+                    src, srcSize,
+                    ci->dictionaries.cdicts[ci->dictNb]);
+
+    ci->dictNb = ci->dictNb + 1;
+    if (ci->dictNb >= ci->nbDicts) ci->dictNb = 0;
+
+    return srcSize;
+}
+
 /* benched function */
 size_t decompress(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* payload)
 {
@@ -527,8 +652,9 @@ size_t decompress(const void* src, size_t srcSize, void* dst, size_t dstCapacity
 
 static int benchMem(slice_collection_t dstBlocks,
                     slice_collection_t srcBlocks,
-                    ddict_collection_t dictionaries,
+                    ddict_collection_t ddictionaries,
-                    unsigned nbRounds)
+                    cdict_collection_t cdictionaries,
+                    unsigned nbRounds, int benchCompression)
 {
     assert(dstBlocks.nbSlices == srcBlocks.nbSlices);
 
@@ -539,10 +665,13 @@ static int benchMem(slice_collection_t dstBlocks,
 
     BMK_timedFnState_t* const benchState =
             BMK_createTimedFnState(total_time_ms, ms_per_round);
-    decompressInstructions di = createDecompressInstructions(dictionaries);
+
+    decompressInstructions di = createDecompressInstructions(ddictionaries);
+    compressInstructions ci = createCompressInstructions(cdictionaries);
+    void* payload = benchCompression ? (void*)&ci : (void*)&di;
     BMK_benchParams_t const bp = {
-        .benchFn = decompress,
+        .benchFn = benchCompression ? compress : decompress,
-        .benchPayload = &di,
+        .benchPayload = payload,
         .initFn = NULL,
         .initPayload = NULL,
         .errorFn = ZSTD_isError,
@@ -562,15 +691,20 @@ static int benchMem(slice_collection_t dstBlocks,
         double const dTime_ns = result.nanoSecPerRun;
         double const dTime_sec = (double)dTime_ns / 1000000000;
         size_t const srcSize = result.sumOfReturn;
-        double const dSpeed_MBps = (double)srcSize / dTime_sec / (1 MB);
+        double const speed_MBps = (double)srcSize / dTime_sec / (1 MB);
-        if (dSpeed_MBps > bestSpeed) bestSpeed = dSpeed_MBps;
+        if (speed_MBps > bestSpeed) bestSpeed = speed_MBps;
+        if (benchCompression)
+            DISPLAY("Compression Speed : %.1f MB/s \r", bestSpeed);
+        else
             DISPLAY("Decompression Speed : %.1f MB/s \r", bestSpeed);
+
         fflush(stdout);
         if (BMK_isCompleted_TimedFn(benchState)) break;
     }
     DISPLAY("\n");
 
     freeDecompressInstructions(di);
+    freeCompressInstructions(ci);
     BMK_freeTimedFnState(benchState);
 
     return 0;   /* success */
@@ -586,7 +720,7 @@ int bench(const char** fileNameTable, unsigned nbFiles,
           const char* dictionary,
           size_t blockSize, int clevel,
           unsigned nbDictMax, unsigned nbBlocks,
-          unsigned nbRounds)
+          unsigned nbRounds, int benchCompression)
 {
     int result = 0;
 
@@ -662,25 +796,47 @@ int bench(const char** fileNameTable, unsigned nbFiles,
     /* now dstSlices contain the real compressed size of each block, instead of the maximum capacity */
     shrinkSizes(dstSlices, cSizes);
 
-    size_t const dictMem = ZSTD_estimateDDictSize(dictBuffer.size, ZSTD_dlm_byCopy);
     unsigned const nbDicts = nbDictMax ? nbDictMax : nbBlocks;
+
+    cdict_collection_t const cdictionaries = createCDictCollection(dictBuffer.ptr, dictBuffer.size, nbDicts, clevel);
+    CONTROL(cdictionaries.cdicts != NULL);
+
+    ddict_collection_t const ddictionaries = createDDictCollection(dictBuffer.ptr, dictBuffer.size, nbDicts);
+    CONTROL(ddictionaries.ddicts != NULL);
+
+    if (benchCompression) {
+        size_t const dictMem = ZSTD_estimateCDictSize(dictBuffer.size, ZSTD_dlm_byCopy);
         size_t const allDictMem = dictMem * nbDicts;
         DISPLAYLEVEL(3, "generating %u dictionaries, using %.1f MB of memory \n",
                         nbDicts, (double)allDictMem / (1 MB));
 
-    ddict_collection_t const dictionaries = createDDictCollection(dictBuffer.ptr, dictBuffer.size, nbDicts);
+        shuffleCDictionaries(cdictionaries);
-    CONTROL(dictionaries.ddicts != NULL);
 
-    shuffleDictionaries(dictionaries);
+        buffer_collection_t resultCollection = createBufferCollection_fromSliceCollection(srcSlices);
+        CONTROL(resultCollection.buffer.ptr != NULL);
+
+        result = benchMem(dstSlices, resultCollection.slices, ddictionaries, cdictionaries, nbRounds, benchCompression);
+
+        freeBufferCollection(resultCollection);
+    } else {
+        size_t const dictMem = ZSTD_estimateDDictSize(dictBuffer.size, ZSTD_dlm_byCopy);
+        size_t const allDictMem = dictMem * nbDicts;
+        DISPLAYLEVEL(3, "generating %u dictionaries, using %.1f MB of memory \n",
+                        nbDicts, (double)allDictMem / (1 MB));
+
+        shuffleDDictionaries(ddictionaries);
 
         buffer_collection_t resultCollection = createBufferCollection_fromSliceCollectionSizes(srcSlices);
         CONTROL(resultCollection.buffer.ptr != NULL);
 
-    result = benchMem(resultCollection.slices, dstSlices, dictionaries, nbRounds);
+        result = benchMem(resultCollection.slices, dstSlices, ddictionaries, cdictionaries, nbRounds, benchCompression);
+
+        freeBufferCollection(resultCollection);
+    }
 
     /* free all heap objects in reverse order */
-    freeBufferCollection(resultCollection);
+    freeCDictCollection(cdictionaries);
-    freeDDictCollection(dictionaries);
+    freeDDictCollection(ddictionaries);
     free(cSizes);
     ZSTD_freeCDict(cdict);
     freeBuffer(dictBuffer);
@@ -744,6 +900,8 @@ int usage(const char* exeName)
     DISPLAY (" %s [Options] filename(s) \n", exeName);
     DISPLAY (" \n");
     DISPLAY ("Options : \n");
+    DISPLAY ("-z          : benchmark compression (default) \n");
+    DISPLAY ("-d          : benchmark decompression \n");
     DISPLAY ("-r          : recursively load all files in subdirectories (default: off) \n");
     DISPLAY ("-B#         : split input into blocks of size # (default: no split) \n");
     DISPLAY ("-#          : use compression level # (default: %u) \n", CLEVEL_DEFAULT);
@@ -765,6 +923,7 @@ int bad_usage(const char* exeName)
 int main (int argc, const char** argv)
 {
     int recursiveMode = 0;
+    int benchCompression = 1;
     unsigned nbRounds = BENCH_TIME_DEFAULT_S;
     const char* const exeName = argv[0];
 
@@ -783,6 +942,8 @@ int main (int argc, const char** argv)
     for (int argNb = 1; argNb < argc ; argNb++) {
         const char* argument = argv[argNb];
         if (!strcmp(argument, "-h")) { free(nameTable); return usage(exeName); }
+        if (!strcmp(argument, "-d")) { benchCompression = 0; continue; }
+        if (!strcmp(argument, "-z")) { benchCompression = 1; continue; }
         if (!strcmp(argument, "-r")) { recursiveMode = 1; continue; }
         if (!strcmp(argument, "-D")) { argNb++; assert(argNb < argc); dictionary = argv[argNb]; continue; }
         if (longCommandWArg(&argument, "-i")) { nbRounds = readU32FromChar(&argument); continue; }
@@ -809,7 +970,7 @@ int main (int argc, const char** argv)
         nameTable = NULL;  /* UTIL_createFileNamesTable() takes ownership of nameTable */
     }
 
-    int result = bench(filenameTable->fileNames, (unsigned)filenameTable->tableSize, dictionary, blockSize, cLevel, nbDicts, nbBlocks, nbRounds);
+    int result = bench(filenameTable->fileNames, (unsigned)filenameTable->tableSize, dictionary, blockSize, cLevel, nbDicts, nbBlocks, nbRounds, benchCompression);
 
     UTIL_freeFileNamesTable(filenameTable);
     free(nameTable);