facebook/zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[bench] Extending largeNbDicts to compression (#2089)

Browse Source 
* adding cdict_collection_t

* adding shuffleCDictionaries()

* adding compressInstructions

* adding compress()

* integrating compression into bench()

* copy paste error fix

* static analyzer uninit value complaint fix

* changing to control

* removing assert

* changing to control

* moving memcpy to seperate function

* fixing static analyzer complaint

* another hacky solution attempt

* Copying createbuffer logic
This commit is contained in:
Bimba Shrestha 2020-05-04 12:42:22 -05:00 committed by GitHub
parent 7e9aabd652
commit 0301ef5d04
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 184 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

195
contrib/largeNbDicts/largeNbDicts.c
View File

@ -334,6 +334,42 @@ createBufferCollection_fromSliceCollectionSizes(slice_collection_t sc)
return result;
}
static buffer_collection_t
createBufferCollection_fromSliceCollection(slice_collection_t sc)
{
size_t const bufferSize = sliceCollection_totalCapacity(sc);
buffer_t buffer = createBuffer(bufferSize);
CONTROL(buffer.ptr != NULL);
size_t const nbSlices = sc.nbSlices;
void** const slices = (void**)malloc(nbSlices * sizeof(*slices));
CONTROL(slices != NULL);
size_t* const capacities = (size_t*)malloc(nbSlices * sizeof(*capacities));
CONTROL(capacities != NULL);
char* const ptr = (char*)buffer.ptr;
size_t pos = 0;
for (size_t n=0; n < nbSlices; n++) {
capacities[n] = sc.capacities[n];
slices[n] = ptr + pos;
pos += capacities[n];
}
for (size_t i = 0; i < nbSlices; i++) {
memcpy(slices[i], sc.slicePtrs[i], sc.capacities[i]);
capacities[i] = sc.capacities[i];
}
buffer_collection_t result;
result.buffer = buffer;
result.slices.nbSlices = nbSlices;
result.slices.capacities = capacities;
result.slices.slicePtrs = slices;
return result;
}
/* @return : kBuffNull if any error */
static buffer_collection_t
@ -397,6 +433,36 @@ typedef struct {
size_t nbDDict;
} ddict_collection_t;
typedef struct {
ZSTD_CDict** cdicts;
size_t nbCDict;
} cdict_collection_t;
static const cdict_collection_t kNullCDictCollection = { NULL, 0 };
static void freeCDictCollection(cdict_collection_t cdictc)
{
for (size_t dictNb=0; dictNb < cdictc.nbCDict; dictNb++) {
ZSTD_freeCDict(cdictc.cdicts[dictNb]);
}
free(cdictc.cdicts);
}
/* returns .buffers=NULL if operation fails */
static cdict_collection_t createCDictCollection(const void* dictBuffer, size_t dictSize, size_t nbCDict, int cLevel)
{
ZSTD_CDict** const cdicts = malloc(nbCDict * sizeof(ZSTD_CDict*));
if (cdicts==NULL) return kNullCDictCollection;
for (size_t dictNb=0; dictNb < nbCDict; dictNb++) {
cdicts[dictNb] = ZSTD_createCDict(dictBuffer, dictSize, cLevel);
CONTROL(cdicts[dictNb] != NULL);
}
cdict_collection_t cdictc;
cdictc.cdicts = cdicts;
cdictc.nbCDict = nbCDict;
return cdictc;
}
static const ddict_collection_t kNullDDictCollection = { NULL, 0 };
static void freeDDictCollection(ddict_collection_t ddictc)
@ -425,7 +491,26 @@ static ddict_collection_t createDDictCollection(const void* dictBuffer, size_t d
/* mess with addresses, so that linear scanning dictionaries != linear address scanning */
void shuffleDictionaries(ddict_collection_t dicts)
void shuffleCDictionaries(cdict_collection_t dicts)
{
size_t const nbDicts = dicts.nbCDict;
for (size_t r=0; r<nbDicts; r++) {
size_t const d = (size_t)rand() % nbDicts;
ZSTD_CDict* tmpd = dicts.cdicts[d];
dicts.cdicts[d] = dicts.cdicts[r];
dicts.cdicts[r] = tmpd;
}
for (size_t r=0; r<nbDicts; r++) {
size_t const d1 = (size_t)rand() % nbDicts;
size_t const d2 = (size_t)rand() % nbDicts;
ZSTD_CDict* tmpd = dicts.cdicts[d1];
dicts.cdicts[d1] = dicts.cdicts[d2];
dicts.cdicts[d2] = tmpd;
}
}
/* mess with addresses, so that linear scanning dictionaries != linear address scanning */
void shuffleDDictionaries(ddict_collection_t dicts)
{
size_t const nbDicts = dicts.nbDDict;
for (size_t r=0; r<nbDicts; r++) {
@ -485,6 +570,29 @@ static size_t compressBlocks(size_t* cSizes, /* optional (can be NULL). If pre
/* --- Benchmark --- */
typedef struct {
ZSTD_CCtx* cctx;
size_t nbDicts;
size_t dictNb;
cdict_collection_t dictionaries;
} compressInstructions;
compressInstructions createCompressInstructions(cdict_collection_t dictionaries)
{
compressInstructions ci;
ci.cctx = ZSTD_createCCtx();
CONTROL(ci.cctx != NULL);
ci.nbDicts = dictionaries.nbCDict;
ci.dictNb = 0;
ci.dictionaries = dictionaries;
return ci;
}
void freeCompressInstructions(compressInstructions ci)
{
ZSTD_freeCCtx(ci.cctx);
}
typedef struct {
ZSTD_DCtx* dctx;
size_t nbDicts;
@ -508,6 +616,23 @@ void freeDecompressInstructions(decompressInstructions di)
ZSTD_freeDCtx(di.dctx);
}
/* benched function */
size_t compress(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* payload)
{
compressInstructions* const ci = (compressInstructions*) payload;
(void)dstCapacity;
ZSTD_compress_usingCDict(ci->cctx,
dst, srcSize,
src, srcSize,
ci->dictionaries.cdicts[ci->dictNb]);
ci->dictNb = ci->dictNb + 1;
if (ci->dictNb >= ci->nbDicts) ci->dictNb = 0;
return srcSize;
}
/* benched function */
size_t decompress(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* payload)
{
@ -527,8 +652,9 @@ size_t decompress(const void* src, size_t srcSize, void* dst, size_t dstCapacity
static int benchMem(slice_collection_t dstBlocks,
slice_collection_t srcBlocks,
ddict_collection_t dictionaries,
unsigned nbRounds)
ddict_collection_t ddictionaries,
cdict_collection_t cdictionaries,
unsigned nbRounds, int benchCompression)
{
assert(dstBlocks.nbSlices == srcBlocks.nbSlices);
@ -539,10 +665,13 @@ static int benchMem(slice_collection_t dstBlocks,
BMK_timedFnState_t* const benchState =
BMK_createTimedFnState(total_time_ms, ms_per_round);
decompressInstructions di = createDecompressInstructions(dictionaries);
decompressInstructions di = createDecompressInstructions(ddictionaries);
compressInstructions ci = createCompressInstructions(cdictionaries);
void* payload = benchCompression ? (void*)&ci : (void*)&di;
BMK_benchParams_t const bp = {
.benchFn = decompress,
.benchPayload = &di,
.benchFn = benchCompression ? compress : decompress,
.benchPayload = payload,
.initFn = NULL,
.initPayload = NULL,
.errorFn = ZSTD_isError,
@ -562,15 +691,20 @@ static int benchMem(slice_collection_t dstBlocks,
double const dTime_ns = result.nanoSecPerRun;
double const dTime_sec = (double)dTime_ns / 1000000000;
size_t const srcSize = result.sumOfReturn;
double const dSpeed_MBps = (double)srcSize / dTime_sec / (1 MB);
if (dSpeed_MBps > bestSpeed) bestSpeed = dSpeed_MBps;
double const speed_MBps = (double)srcSize / dTime_sec / (1 MB);
if (speed_MBps > bestSpeed) bestSpeed = speed_MBps;
if (benchCompression)
DISPLAY("Compression Speed : %.1f MB/s \r", bestSpeed);
else
DISPLAY("Decompression Speed : %.1f MB/s \r", bestSpeed);
fflush(stdout);
if (BMK_isCompleted_TimedFn(benchState)) break;
}
DISPLAY("\n");
freeDecompressInstructions(di);
freeCompressInstructions(ci);
BMK_freeTimedFnState(benchState);
return 0; /* success */
@ -586,7 +720,7 @@ int bench(const char** fileNameTable, unsigned nbFiles,
const char* dictionary,
size_t blockSize, int clevel,
unsigned nbDictMax, unsigned nbBlocks,
unsigned nbRounds)
unsigned nbRounds, int benchCompression)
{
int result = 0;
@ -662,25 +796,47 @@ int bench(const char** fileNameTable, unsigned nbFiles,
/* now dstSlices contain the real compressed size of each block, instead of the maximum capacity */
shrinkSizes(dstSlices, cSizes);
size_t const dictMem = ZSTD_estimateDDictSize(dictBuffer.size, ZSTD_dlm_byCopy);
unsigned const nbDicts = nbDictMax ? nbDictMax : nbBlocks;
cdict_collection_t const cdictionaries = createCDictCollection(dictBuffer.ptr, dictBuffer.size, nbDicts, clevel);
CONTROL(cdictionaries.cdicts != NULL);
ddict_collection_t const ddictionaries = createDDictCollection(dictBuffer.ptr, dictBuffer.size, nbDicts);
CONTROL(ddictionaries.ddicts != NULL);
if (benchCompression) {
size_t const dictMem = ZSTD_estimateCDictSize(dictBuffer.size, ZSTD_dlm_byCopy);
size_t const allDictMem = dictMem * nbDicts;
DISPLAYLEVEL(3, "generating %u dictionaries, using %.1f MB of memory \n",
nbDicts, (double)allDictMem / (1 MB));
ddict_collection_t const dictionaries = createDDictCollection(dictBuffer.ptr, dictBuffer.size, nbDicts);
CONTROL(dictionaries.ddicts != NULL);
shuffleCDictionaries(cdictionaries);
shuffleDictionaries(dictionaries);
buffer_collection_t resultCollection = createBufferCollection_fromSliceCollection(srcSlices);
CONTROL(resultCollection.buffer.ptr != NULL);
result = benchMem(dstSlices, resultCollection.slices, ddictionaries, cdictionaries, nbRounds, benchCompression);
freeBufferCollection(resultCollection);
} else {
size_t const dictMem = ZSTD_estimateDDictSize(dictBuffer.size, ZSTD_dlm_byCopy);
size_t const allDictMem = dictMem * nbDicts;
DISPLAYLEVEL(3, "generating %u dictionaries, using %.1f MB of memory \n",
nbDicts, (double)allDictMem / (1 MB));
shuffleDDictionaries(ddictionaries);
buffer_collection_t resultCollection = createBufferCollection_fromSliceCollectionSizes(srcSlices);
CONTROL(resultCollection.buffer.ptr != NULL);
result = benchMem(resultCollection.slices, dstSlices, dictionaries, nbRounds);
result = benchMem(resultCollection.slices, dstSlices, ddictionaries, cdictionaries, nbRounds, benchCompression);
freeBufferCollection(resultCollection);
}
/* free all heap objects in reverse order */
freeBufferCollection(resultCollection);
freeDDictCollection(dictionaries);
freeCDictCollection(cdictionaries);
freeDDictCollection(ddictionaries);
free(cSizes);
ZSTD_freeCDict(cdict);
freeBuffer(dictBuffer);
@ -744,6 +900,8 @@ int usage(const char* exeName)
DISPLAY (" %s [Options] filename(s) \n", exeName);
DISPLAY (" \n");
DISPLAY ("Options : \n");
DISPLAY ("-z : benchmark compression (default) \n");
DISPLAY ("-d : benchmark decompression \n");
DISPLAY ("-r : recursively load all files in subdirectories (default: off) \n");
DISPLAY ("-B# : split input into blocks of size # (default: no split) \n");
DISPLAY ("-# : use compression level # (default: %u) \n", CLEVEL_DEFAULT);
@ -765,6 +923,7 @@ int bad_usage(const char* exeName)
int main (int argc, const char** argv)
{
int recursiveMode = 0;
int benchCompression = 1;
unsigned nbRounds = BENCH_TIME_DEFAULT_S;
const char* const exeName = argv[0];
@ -783,6 +942,8 @@ int main (int argc, const char** argv)
for (int argNb = 1; argNb < argc ; argNb++) {
const char* argument = argv[argNb];
if (!strcmp(argument, "-h")) { free(nameTable); return usage(exeName); }
if (!strcmp(argument, "-d")) { benchCompression = 0; continue; }
if (!strcmp(argument, "-z")) { benchCompression = 1; continue; }
if (!strcmp(argument, "-r")) { recursiveMode = 1; continue; }
if (!strcmp(argument, "-D")) { argNb++; assert(argNb < argc); dictionary = argv[argNb]; continue; }
if (longCommandWArg(&argument, "-i")) { nbRounds = readU32FromChar(&argument); continue; }
@ -809,7 +970,7 @@ int main (int argc, const char** argv)
nameTable = NULL; /* UTIL_createFileNamesTable() takes ownership of nameTable */
}
int result = bench(filenameTable->fileNames, (unsigned)filenameTable->tableSize, dictionary, blockSize, cLevel, nbDicts, nbBlocks, nbRounds);
int result = bench(filenameTable->fileNames, (unsigned)filenameTable->tableSize, dictionary, blockSize, cLevel, nbDicts, nbBlocks, nbRounds, benchCompression);
UTIL_freeFileNamesTable(filenameTable);
free(nameTable);