From 0301ef5d0463f53e88d589233b5f16c8d57e7295 Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Mon, 4 May 2020 12:42:22 -0500 Subject: [PATCH] [bench] Extending largeNbDicts to compression (#2089) * adding cdict_collection_t * adding shuffleCDictionaries() * adding compressInstructions * adding compress() * integrating compression into bench() * copy paste error fix * static analyzer uninit value complaint fix * changing to control * removing assert * changing to control * moving memcpy to seperate function * fixing static analyzer complaint * another hacky solution attempt * Copying createbuffer logic --- contrib/largeNbDicts/largeNbDicts.c | 207 ++++++++++++++++++++++++---- 1 file changed, 184 insertions(+), 23 deletions(-) diff --git a/contrib/largeNbDicts/largeNbDicts.c b/contrib/largeNbDicts/largeNbDicts.c index 121cf2f9..c84d48b2 100644 --- a/contrib/largeNbDicts/largeNbDicts.c +++ b/contrib/largeNbDicts/largeNbDicts.c @@ -334,6 +334,42 @@ createBufferCollection_fromSliceCollectionSizes(slice_collection_t sc) return result; } +static buffer_collection_t +createBufferCollection_fromSliceCollection(slice_collection_t sc) +{ + size_t const bufferSize = sliceCollection_totalCapacity(sc); + + buffer_t buffer = createBuffer(bufferSize); + CONTROL(buffer.ptr != NULL); + + size_t const nbSlices = sc.nbSlices; + void** const slices = (void**)malloc(nbSlices * sizeof(*slices)); + CONTROL(slices != NULL); + + size_t* const capacities = (size_t*)malloc(nbSlices * sizeof(*capacities)); + CONTROL(capacities != NULL); + + char* const ptr = (char*)buffer.ptr; + size_t pos = 0; + for (size_t n=0; n < nbSlices; n++) { + capacities[n] = sc.capacities[n]; + slices[n] = ptr + pos; + pos += capacities[n]; + } + + for (size_t i = 0; i < nbSlices; i++) { + memcpy(slices[i], sc.slicePtrs[i], sc.capacities[i]); + capacities[i] = sc.capacities[i]; + } + + buffer_collection_t result; + result.buffer = buffer; + result.slices.nbSlices = nbSlices; + result.slices.capacities = capacities; + result.slices.slicePtrs = slices; + + return result; +} /* @return : kBuffNull if any error */ static buffer_collection_t @@ -397,6 +433,36 @@ typedef struct { size_t nbDDict; } ddict_collection_t; +typedef struct { + ZSTD_CDict** cdicts; + size_t nbCDict; +} cdict_collection_t; + +static const cdict_collection_t kNullCDictCollection = { NULL, 0 }; + +static void freeCDictCollection(cdict_collection_t cdictc) +{ + for (size_t dictNb=0; dictNb < cdictc.nbCDict; dictNb++) { + ZSTD_freeCDict(cdictc.cdicts[dictNb]); + } + free(cdictc.cdicts); +} + +/* returns .buffers=NULL if operation fails */ +static cdict_collection_t createCDictCollection(const void* dictBuffer, size_t dictSize, size_t nbCDict, int cLevel) +{ + ZSTD_CDict** const cdicts = malloc(nbCDict * sizeof(ZSTD_CDict*)); + if (cdicts==NULL) return kNullCDictCollection; + for (size_t dictNb=0; dictNb < nbCDict; dictNb++) { + cdicts[dictNb] = ZSTD_createCDict(dictBuffer, dictSize, cLevel); + CONTROL(cdicts[dictNb] != NULL); + } + cdict_collection_t cdictc; + cdictc.cdicts = cdicts; + cdictc.nbCDict = nbCDict; + return cdictc; +} + static const ddict_collection_t kNullDDictCollection = { NULL, 0 }; static void freeDDictCollection(ddict_collection_t ddictc) @@ -425,7 +491,26 @@ static ddict_collection_t createDDictCollection(const void* dictBuffer, size_t d /* mess with addresses, so that linear scanning dictionaries != linear address scanning */ -void shuffleDictionaries(ddict_collection_t dicts) +void shuffleCDictionaries(cdict_collection_t dicts) +{ + size_t const nbDicts = dicts.nbCDict; + for (size_t r=0; rcctx, + dst, srcSize, + src, srcSize, + ci->dictionaries.cdicts[ci->dictNb]); + + ci->dictNb = ci->dictNb + 1; + if (ci->dictNb >= ci->nbDicts) ci->dictNb = 0; + + return srcSize; +} + /* benched function */ size_t decompress(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* payload) { @@ -527,8 +652,9 @@ size_t decompress(const void* src, size_t srcSize, void* dst, size_t dstCapacity static int benchMem(slice_collection_t dstBlocks, slice_collection_t srcBlocks, - ddict_collection_t dictionaries, - unsigned nbRounds) + ddict_collection_t ddictionaries, + cdict_collection_t cdictionaries, + unsigned nbRounds, int benchCompression) { assert(dstBlocks.nbSlices == srcBlocks.nbSlices); @@ -539,10 +665,13 @@ static int benchMem(slice_collection_t dstBlocks, BMK_timedFnState_t* const benchState = BMK_createTimedFnState(total_time_ms, ms_per_round); - decompressInstructions di = createDecompressInstructions(dictionaries); + + decompressInstructions di = createDecompressInstructions(ddictionaries); + compressInstructions ci = createCompressInstructions(cdictionaries); + void* payload = benchCompression ? (void*)&ci : (void*)&di; BMK_benchParams_t const bp = { - .benchFn = decompress, - .benchPayload = &di, + .benchFn = benchCompression ? compress : decompress, + .benchPayload = payload, .initFn = NULL, .initPayload = NULL, .errorFn = ZSTD_isError, @@ -562,15 +691,20 @@ static int benchMem(slice_collection_t dstBlocks, double const dTime_ns = result.nanoSecPerRun; double const dTime_sec = (double)dTime_ns / 1000000000; size_t const srcSize = result.sumOfReturn; - double const dSpeed_MBps = (double)srcSize / dTime_sec / (1 MB); - if (dSpeed_MBps > bestSpeed) bestSpeed = dSpeed_MBps; - DISPLAY("Decompression Speed : %.1f MB/s \r", bestSpeed); + double const speed_MBps = (double)srcSize / dTime_sec / (1 MB); + if (speed_MBps > bestSpeed) bestSpeed = speed_MBps; + if (benchCompression) + DISPLAY("Compression Speed : %.1f MB/s \r", bestSpeed); + else + DISPLAY("Decompression Speed : %.1f MB/s \r", bestSpeed); + fflush(stdout); if (BMK_isCompleted_TimedFn(benchState)) break; } DISPLAY("\n"); freeDecompressInstructions(di); + freeCompressInstructions(ci); BMK_freeTimedFnState(benchState); return 0; /* success */ @@ -586,7 +720,7 @@ int bench(const char** fileNameTable, unsigned nbFiles, const char* dictionary, size_t blockSize, int clevel, unsigned nbDictMax, unsigned nbBlocks, - unsigned nbRounds) + unsigned nbRounds, int benchCompression) { int result = 0; @@ -662,25 +796,47 @@ int bench(const char** fileNameTable, unsigned nbFiles, /* now dstSlices contain the real compressed size of each block, instead of the maximum capacity */ shrinkSizes(dstSlices, cSizes); - size_t const dictMem = ZSTD_estimateDDictSize(dictBuffer.size, ZSTD_dlm_byCopy); unsigned const nbDicts = nbDictMax ? nbDictMax : nbBlocks; - size_t const allDictMem = dictMem * nbDicts; - DISPLAYLEVEL(3, "generating %u dictionaries, using %.1f MB of memory \n", - nbDicts, (double)allDictMem / (1 MB)); - ddict_collection_t const dictionaries = createDDictCollection(dictBuffer.ptr, dictBuffer.size, nbDicts); - CONTROL(dictionaries.ddicts != NULL); + cdict_collection_t const cdictionaries = createCDictCollection(dictBuffer.ptr, dictBuffer.size, nbDicts, clevel); + CONTROL(cdictionaries.cdicts != NULL); - shuffleDictionaries(dictionaries); + ddict_collection_t const ddictionaries = createDDictCollection(dictBuffer.ptr, dictBuffer.size, nbDicts); + CONTROL(ddictionaries.ddicts != NULL); - buffer_collection_t resultCollection = createBufferCollection_fromSliceCollectionSizes(srcSlices); - CONTROL(resultCollection.buffer.ptr != NULL); + if (benchCompression) { + size_t const dictMem = ZSTD_estimateCDictSize(dictBuffer.size, ZSTD_dlm_byCopy); + size_t const allDictMem = dictMem * nbDicts; + DISPLAYLEVEL(3, "generating %u dictionaries, using %.1f MB of memory \n", + nbDicts, (double)allDictMem / (1 MB)); - result = benchMem(resultCollection.slices, dstSlices, dictionaries, nbRounds); + shuffleCDictionaries(cdictionaries); + + buffer_collection_t resultCollection = createBufferCollection_fromSliceCollection(srcSlices); + CONTROL(resultCollection.buffer.ptr != NULL); + + result = benchMem(dstSlices, resultCollection.slices, ddictionaries, cdictionaries, nbRounds, benchCompression); + + freeBufferCollection(resultCollection); + } else { + size_t const dictMem = ZSTD_estimateDDictSize(dictBuffer.size, ZSTD_dlm_byCopy); + size_t const allDictMem = dictMem * nbDicts; + DISPLAYLEVEL(3, "generating %u dictionaries, using %.1f MB of memory \n", + nbDicts, (double)allDictMem / (1 MB)); + + shuffleDDictionaries(ddictionaries); + + buffer_collection_t resultCollection = createBufferCollection_fromSliceCollectionSizes(srcSlices); + CONTROL(resultCollection.buffer.ptr != NULL); + + result = benchMem(resultCollection.slices, dstSlices, ddictionaries, cdictionaries, nbRounds, benchCompression); + + freeBufferCollection(resultCollection); + } /* free all heap objects in reverse order */ - freeBufferCollection(resultCollection); - freeDDictCollection(dictionaries); + freeCDictCollection(cdictionaries); + freeDDictCollection(ddictionaries); free(cSizes); ZSTD_freeCDict(cdict); freeBuffer(dictBuffer); @@ -744,6 +900,8 @@ int usage(const char* exeName) DISPLAY (" %s [Options] filename(s) \n", exeName); DISPLAY (" \n"); DISPLAY ("Options : \n"); + DISPLAY ("-z : benchmark compression (default) \n"); + DISPLAY ("-d : benchmark decompression \n"); DISPLAY ("-r : recursively load all files in subdirectories (default: off) \n"); DISPLAY ("-B# : split input into blocks of size # (default: no split) \n"); DISPLAY ("-# : use compression level # (default: %u) \n", CLEVEL_DEFAULT); @@ -765,6 +923,7 @@ int bad_usage(const char* exeName) int main (int argc, const char** argv) { int recursiveMode = 0; + int benchCompression = 1; unsigned nbRounds = BENCH_TIME_DEFAULT_S; const char* const exeName = argv[0]; @@ -783,6 +942,8 @@ int main (int argc, const char** argv) for (int argNb = 1; argNb < argc ; argNb++) { const char* argument = argv[argNb]; if (!strcmp(argument, "-h")) { free(nameTable); return usage(exeName); } + if (!strcmp(argument, "-d")) { benchCompression = 0; continue; } + if (!strcmp(argument, "-z")) { benchCompression = 1; continue; } if (!strcmp(argument, "-r")) { recursiveMode = 1; continue; } if (!strcmp(argument, "-D")) { argNb++; assert(argNb < argc); dictionary = argv[argNb]; continue; } if (longCommandWArg(&argument, "-i")) { nbRounds = readU32FromChar(&argument); continue; } @@ -809,7 +970,7 @@ int main (int argc, const char** argv) nameTable = NULL; /* UTIL_createFileNamesTable() takes ownership of nameTable */ } - int result = bench(filenameTable->fileNames, (unsigned)filenameTable->tableSize, dictionary, blockSize, cLevel, nbDicts, nbBlocks, nbRounds); + int result = bench(filenameTable->fileNames, (unsigned)filenameTable->tableSize, dictionary, blockSize, cLevel, nbDicts, nbBlocks, nbRounds, benchCompression); UTIL_freeFileNamesTable(filenameTable); free(nameTable);