From 678272515515afc6fe99975a8c716ce27907fc7e Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 26 Aug 2018 19:29:12 -0700 Subject: [PATCH] first sketch for largeNbDicts test program --- contrib/largeNbDicts/Makefile | 30 +++ contrib/largeNbDicts/largeNbDicts | Bin 0 -> 13626 bytes contrib/largeNbDicts/largeNbDicts.c | 289 ++++++++++++++++++++++++++++ lib/Makefile | 2 +- lib/dictBuilder/zdict.h | 3 +- programs/bench.h | 2 +- 6 files changed, 323 insertions(+), 3 deletions(-) create mode 100644 contrib/largeNbDicts/Makefile create mode 100755 contrib/largeNbDicts/largeNbDicts create mode 100644 contrib/largeNbDicts/largeNbDicts.c diff --git a/contrib/largeNbDicts/Makefile b/contrib/largeNbDicts/Makefile new file mode 100644 index 00000000..082f0102 --- /dev/null +++ b/contrib/largeNbDicts/Makefile @@ -0,0 +1,30 @@ +# ################################################################ +# Copyright (c) 2018-present, Yann Collet, Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under both the BSD-style license (found in the +# LICENSE file in the root directory of this source tree) and the GPLv2 (found +# in the COPYING file in the root directory of this source tree). +# ################################################################ + + +CPPFLAGS+= -I../../lib -I../../lib/common -I../../lib/dictBuilder -I../../programs + +CFLAGS ?= -O3 +DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ + -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ + -Wstrict-prototypes -Wundef -Wpointer-arith -Wformat-security \ + -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \ + -Wredundant-decls +CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) + + +default: largeNbDicts + +largeNbDicts: LDFLAGS += -lzstd +largeNbDicts: largeNbDicts.c + $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ + + +clean: + $(RM) largeNbDicts diff --git a/contrib/largeNbDicts/largeNbDicts b/contrib/largeNbDicts/largeNbDicts new file mode 100755 index 0000000000000000000000000000000000000000..40416f050fa594cb085444ebb1ca433629d59318 GIT binary patch literal 13626 zcmeHOeQX@X6(8GiF|_0y%7;<;SWt`{L@&M)LlWZDm-t{Wtb>W2k3z!c&bJpIJ>OZo zdy0byGT0|&d%f00rHVw6D58ZTT3l3UYC`G~Ck_Y&*91rjAE6Df$A|cANcr&f_h#qL z=Szb6N7cV}(s4t;`D1>J@tkce7{iRS<4Ko!silTCRYtD1*MMaMSE)1v<_NM(Kduq29@6~j4 zDz~>&*c%ZB2vgjPDSPVd}BOCLu4k_n@#> zC7dE$T4wkPg{r0!n`4QH8cRfz9M9a}hc9&eYZpccm)j_{Pv>MIHc(Yt{hj_2jpW$j z2b9FXywHI~q3y;Q=9RFjZc0u^?d0}qs52F5dMrWO&pE`1qN<6wYHaBVC*x|bsUPQG zjpXtpF$yfXA}Uas>VZKXRl6j>}?Zc2$$NU^`0qoL?~{~Jyn7$uMM;YJB6~b zEN;M5me8d_Av*|nv0RMj0Pli^Vp9X!P}Jf+0k>m7zmKtvkkek)q1wPizr38W7U;hT zz8$r+(3DiaT`DZ&yD`4LOh#}vs)YJ2)cGkxpC6Bf=SQ~0BfwO?gG%$?f9TP%2d-ay z|DBKTcyq~$yB?kkSuH9%js7kzAd;Y9W&JdY;TDz_nxzGmbhN@*RTQHNF>|sj6rZ2$ zHRs=Am=W#>oFU%p^TMTAn0vXfbo}>@;8GUG!*g`%*kYJkk9V0Eg5G#)p^~?U5lxEz zJb@{?ZgC^vM!=1L8v!>0ZUo#2xDjw8;6}iWfExif0&WEUKM`0hTW|W;`q%hZ`#U@2 zf%ocU>l4}9Iw5BQU<3Ppr<+T(PF){C>~iyD$;VuAQf>!Gb zaO%7Ra>YU7Yl7DL12qEF5~vlRjzFCN5&=no1_BKNG!keOU?G8p0P?_7HS)mTW2v9Q zX;TVj7aRy?J6~W!vQ;Tt7w{f?(F48mPp96I7aiNYLY_B%t=0O~v1_eczdS~(QLF~V z+E}AleYNtySlx~O_5Kb1oBZmA-Ey}15G^^|beNOVZIF>HR%!nV`6j4$I_keV>X%4; z7pd=nIvS07Jr@oFGB$I6Z-1|Ww#52E$$WPb5e#Ow(j>pfaNnqGy=<@M)*qXg%I9V4 zbvpu>9`&TxBW2|IQ~ChJ_3T5⊀t=>WuXw%Gd$(c63C-seE9WFBkG=kvjVsk~?K z8+>@v7|sq&_B;|wIALvRsNRK}iuIZODwpA09WkX{l(E8TP^^y?>)-Y;RLY_c&DnH7 zehU!r9S!)7C|2GcAeHrNT7_agcF-Y*L12fL>>2zGY(;~f#NMbUTrE?~JnO65IlXoC zW6uw&JHaC|Pl60r*h^th&ZNnNY_-nCfll)fWwK;kwv@RwvNdb2oPDwntJ_5xFZ;eP zp~<94fQ$5?j5rS7Nh{8*-sZPn5}|(p6a1O_Q~Xy?*awgj{RVWw?+saG4}gkl}FXLFzKAt~;R&zFb|QY>v$vHl@jZTl3QX#06GXpPa9d<-@{ zYHki@+UUTpg8f~T(;T5~w5Onbg1B5QZG~s;Xw);$3Fd>$)(}ul+4!t+#Zbe9VI7jB!4`QjME~cvL}IA!qw}rDBAChoI3j*0gwIOhpC1u zutgL03Je{Al+K}Iy=Bh=M!+XlWS4wJ&e?vlZSBD`&og*7T*1LLFZUf1-48*62hgju zW9Dsgez&sd3;myp^{$+`%ASBy9@t$I%s-d;G&e~HG}|1e6P#`O87(4v`prNi zPtYP((coTJ8Igxk%OeZ+)*EL)E$m`9-=aGnAB%F>Ya{yfulZ`)NhmeUua-qYf z(B$iP@phEAPw{q)w|jZJkGIe8_F3NU=PljaeElSE>3-+yU*zpU-oDJ+SI{ci^V>lD z9r^lZ(4lVlz;bV4#PNSo?^Z zsAR-$f!^zjN^Q%~DXb;Bx*59gnH;YSuhZR-E^3xmu2vMrT{dafSfV#&qBWCJeM$;D z-ok88s85)Nbc?Bx0};a{*9%4fzjhMtFU=7_G-Am_NZ)c9d2CXQ%;T&(%Ra#%%)CiU zXnKe{3@?lJVw!Q8lrnhqL zqXFt+5$)Dij9Q+I$2GnL=2*FrY^@A$D24)WkEUN!6<${n#dCtc0ZUo#2xDjw8;6}iWfExif0{?Xc z=3HF2aSnQ)(W26a@R4N7>}A*MY^a4kt>vG_t6_ZIjkMDc8P)6Nw5T_<2A6lLrXGqV zuGW)1A}uwUO%`js< zNJ()`2l7y)Z7FD@x~3Iv$)(bwrS^g}W&|kdMdR1;<%SGH(@j-36C~RRS+pyjH24hN zgp$1)SM?I;#W(Y2lzR3Nq|_OK{t5^2QQJ7!tjA0ahY0qB;z)Q=*X;xiGi1&UuDq@_ zaK!=$S%r!(qGzc+-Ky57b)`%VzZ?irxP7H5zosx-+=FnlHI&AwV#D~3xsqq$lu$(F zDJFJ(6&v!B?n*TM^zkvu@TG^V?me}gXMa2ucyEG2d786PJz$3Mih=M&0@t47;0A%2 z$nSUsF8sRUDqxJGp}ed@TZ*G)xUUTVs0`m%h951%PnBWHaAQ86)B?+1E5o@m{81S` zR)#Aa=Zp4g%JA$mOfN9-SL}a58K$IfslFQXA@#<;wx?IIr=p&QN?*y-x8|p#dQfSH z_C;ZxD_@cPf&W;zX%Vq8CRPCr>Rsk0*Hqoapik zS;@68l{nYbksN>39r^SnpT?!*3Kttsu#ziX@;octYjLBCcUjzGA{tYp&b=*M*qJx= JxThgD;Xm-ofn)#x literal 0 HcmV?d00001 diff --git a/contrib/largeNbDicts/largeNbDicts.c b/contrib/largeNbDicts/largeNbDicts.c new file mode 100644 index 00000000..749d9660 --- /dev/null +++ b/contrib/largeNbDicts/largeNbDicts.c @@ -0,0 +1,289 @@ +/* + * Copyright (c) 2018-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* largeNbDicts + * This is a benchmark test tool + * dedicated to the specific case of dictionary decompression + * using a very large nb of dictionaries + * thus generating many cache-misses. + * It's created in a bid to investigate performance and find optimizations. */ + + +/*--- Dependencies ---*/ + +#include /* size_t */ +#include /* malloc, free */ +#include /* printf */ +#include /* assert */ + +#include "util.h" +#define ZSTD_STATIC_LINKING_ONLY +#include "zstd.h" +#include "zdict.h" + + +/*--- Constants --- */ + +#define KB *(1<<10) +#define MB *(1<<20) + +#define BLOCKSIZE (4 KB) +#define DICTSIZE (4 KB) +#define COMP_LEVEL 3 + +#define DISPLAY_LEVEL_DEFAULT 3 + + +/*--- Display Macros ---*/ + +#define DISPLAY(...) fprintf(stdout, __VA_ARGS__) +#define DISPLAYLEVEL(l, ...) { if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } } +static int g_displayLevel = DISPLAY_LEVEL_DEFAULT; /* 0 : no display, 1: errors, 2 : + result + interaction + warnings, 3 : + progression, 4 : + information */ + + +/*--- buffer_t ---*/ + +typedef struct { + void* ptr; + size_t size; + size_t capacity; +} buffer_t; + +static const buffer_t kBuffNull = { NULL, 0, 0 }; + + +static buffer_t fillBuffer_fromHandle(buffer_t buff, FILE* f) +{ + size_t const readSize = fread(buff.ptr, 1, buff.capacity, f); + buff.size = readSize; + return buff; +} + +static void freeBuffer(buffer_t buff) +{ + free(buff.ptr); +} + +/* @return : kBuffNull if any error */ +static buffer_t createBuffer_fromHandle(FILE* f, size_t bufferSize) +{ + void* const buffer = malloc(bufferSize); + if (buffer==NULL) return kBuffNull; + + { buffer_t buff = { buffer, 0, bufferSize }; + buff = fillBuffer_fromHandle(buff, f); + if (buff.size != buff.capacity) { + freeBuffer(buff); + return kBuffNull; + } + return buff; + } +} + +/* @return : kBuffNull if any error */ +static buffer_t createBuffer_fromFile(const char* fileName) +{ + U64 const fileSize = UTIL_getFileSize(fileName); + size_t const bufferSize = (size_t) fileSize; + + if (fileSize == UTIL_FILESIZE_UNKNOWN) return kBuffNull; + assert((U64)bufferSize == fileSize); /* check overflow */ + + { buffer_t buff; + FILE* const f = fopen(fileName, "rb"); + if (f == NULL) return kBuffNull; + + buff = createBuffer_fromHandle(f, bufferSize); + fclose(f); /* do nothing specific if fclose() fails */ + return buff; + } +} + + +/*--- buffer_collection_t ---*/ + +typedef struct { + void** buffers; + size_t* capacities; + size_t nbBuffers; +} buffer_collection_t; + +static const buffer_collection_t kNullCollection = { NULL, NULL, 0 }; + +static void freeCollection(buffer_collection_t collection) +{ + free(collection.buffers); + free(collection.capacities); +} + +/* returns .buffers=NULL if operation fails */ +buffer_collection_t splitBuffer(buffer_t srcBuffer, size_t blockSize) +{ + size_t const nbBlocks = (srcBuffer.size + (blockSize-1)) / blockSize; + + void** const buffers = malloc(nbBlocks * sizeof(void*)); + size_t* const capacities = malloc(nbBlocks * sizeof(size_t*)); + if ((buffers==NULL) || capacities==NULL) { + free(buffers); + free(capacities); + return kNullCollection; + } + + char* newBlockPtr = (char*)srcBuffer.ptr; + char* const srcEnd = newBlockPtr + srcBuffer.size; + assert(nbBlocks >= 1); + for (size_t blockNb = 0; blockNb < nbBlocks-1; blockNb++) { + buffers[blockNb] = newBlockPtr; + capacities[blockNb] = blockSize; + newBlockPtr += blockSize; + } + + /* last block */ + assert(newBlockPtr <= srcEnd); + size_t const lastBlockSize = (srcEnd - newBlockPtr); + buffers[nbBlocks-1] = newBlockPtr; + capacities[nbBlocks-1] = lastBlockSize; + + buffer_collection_t result; + result.buffers = buffers; + result.capacities = capacities; + result.nbBuffers = nbBlocks; + return result; +} + + + +/*--- ddict_collection_t ---*/ + +typedef struct { + ZSTD_DDict** ddicts; + size_t nbDDict; +} ddict_collection_t; + +static const ddict_collection_t kNullDDictCollection = { NULL, 0 }; + +static void freeDDictCollection(ddict_collection_t ddictc) +{ + for (size_t dictNb=0; dictNb < ddictc.nbDDict; dictNb++) { + ZSTD_freeDDict(ddictc.ddicts[dictNb]); + } + free(ddictc.ddicts); +} + +/* returns .buffers=NULL if operation fails */ +static ddict_collection_t createDDictCollection(const void* dictBuffer, size_t dictSize, size_t nbDDict) +{ + ZSTD_DDict** const ddicts = malloc(nbDDict * sizeof(ZSTD_DDict*)); + if (ddicts==NULL) return kNullDDictCollection; + for (size_t dictNb=0; dictNb < nbDDict; dictNb++) { + ddicts[dictNb] = ZSTD_createDDict(dictBuffer, dictSize); + assert(ddicts[dictNb] != NULL); + } + ddict_collection_t ddictc; + ddictc.ddicts = ddicts; + ddictc.nbDDict = nbDDict; + return ddictc; +} + + + +/*--- Benchmark --- */ + + +/* bench() : + * @return : 0 is success, 1+ otherwise */ +int bench(const char* fileName) +{ + int result = 0; + + DISPLAYLEVEL(3, "loading %s... \n", fileName); + buffer_t const srcBuffer = createBuffer_fromFile(fileName); + if (srcBuffer.ptr == NULL) { + DISPLAYLEVEL(1," error reading file %s \n", fileName); + return 1; + } + DISPLAYLEVEL(3, "created src buffer of size %.1f MB \n", + (double)(srcBuffer.size) / (1 MB)); + + buffer_collection_t const srcBlockBuffers = splitBuffer(srcBuffer, BLOCKSIZE); + assert(srcBlockBuffers.buffers != NULL); + unsigned const nbBlocks = (unsigned)srcBlockBuffers.nbBuffers; + DISPLAYLEVEL(3, "splitting input into %u blocks of max size %u bytes \n", + nbBlocks, BLOCKSIZE); + + size_t const dstBlockSize = ZSTD_compressBound(BLOCKSIZE); + size_t const dstBufferCapacity = nbBlocks * dstBlockSize; + void* const dstPtr = malloc(dstBufferCapacity); + assert(dstPtr != NULL); + buffer_t dstBuffer; + dstBuffer.ptr = dstPtr; + dstBuffer.capacity = dstBufferCapacity; + dstBuffer.size = dstBufferCapacity; + + buffer_collection_t const dstBlockBuffers = splitBuffer(dstBuffer, dstBlockSize); + assert(dstBlockBuffers.buffers != NULL); + + DISPLAYLEVEL(3, "creating dictionary, of target size %u bytes \n", DICTSIZE); + void* const dictBuffer = malloc(DICTSIZE); + if (dictBuffer == NULL) { result = 1; goto _cleanup; } + + size_t const dictSize = ZDICT_trainFromBuffer(dictBuffer, DICTSIZE, + srcBuffer.ptr, + srcBlockBuffers.capacities, + nbBlocks); + if (ZSTD_isError(dictSize)) { + DISPLAYLEVEL(1, "error creating dictionary \n"); + result = 1; + goto _cleanup; + } + + size_t const dictMem = ZSTD_estimateDDictSize(dictSize, ZSTD_dlm_byCopy); + size_t const allDictMem = dictMem * nbBlocks; + DISPLAYLEVEL(3, "generating %u dictionaries, using %.1f MB of memory \n", + nbBlocks, (double)allDictMem / (1 MB)); + + ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, COMP_LEVEL); + do { + ddict_collection_t const dictionaries = createDDictCollection(dictBuffer, dictSize, nbBlocks); + assert(dictionaries.ddicts != NULL); + + freeDDictCollection(dictionaries); + } while(0); + ZSTD_freeCDict(cdict); + +_cleanup: + free(dictBuffer); + freeCollection(dstBlockBuffers); + freeBuffer(dstBuffer); + freeCollection(srcBlockBuffers); + freeBuffer(srcBuffer); + + return result; +} + + + + +/*--- Command Line ---*/ + +int bad_usage(const char* exeName) +{ + DISPLAY (" bad usage : \n"); + DISPLAY (" %s filename \n", exeName); + return 1; +} + +int main (int argc, const char** argv) +{ + const char* const exeName = argv[0]; + + if (argc != 2) return bad_usage(exeName); + return bench(argv[1]); +} diff --git a/lib/Makefile b/lib/Makefile index 01689c6d..cf8e45b0 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -23,7 +23,7 @@ ifeq ($(OS),Windows_NT) # MinGW assumed CPPFLAGS += -D__USE_MINGW_ANSI_STDIO # compatibility with %zu formatting endif CFLAGS ?= -O3 -DEBUGFLAGS = -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ +DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ -Wstrict-prototypes -Wundef -Wpointer-arith -Wformat-security \ -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \ diff --git a/lib/dictBuilder/zdict.h b/lib/dictBuilder/zdict.h index 4094669d..3b3a6527 100644 --- a/lib/dictBuilder/zdict.h +++ b/lib/dictBuilder/zdict.h @@ -52,7 +52,8 @@ extern "C" { * It's recommended that total size of all samples be about ~x100 times the target size of dictionary. */ ZDICTLIB_API size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity, - const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples); + const void* samplesBuffer, + const size_t* samplesSizes, unsigned nbSamples); /*====== Helper functions ======*/ diff --git a/programs/bench.h b/programs/bench.h index f6a53fc6..184cc3e0 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -233,7 +233,7 @@ typedef size_t (*BMK_initFn_t)(void* initPayload); * srcSizes - an array of the sizes of above buffers * dstBuffers - an array of buffers to be written into by benchFn * dstCapacities - an array of the capacities of above buffers - * blockResults - store the return value of benchFn for each block. Optional. Use NULL if this result is not requested. + * blockResults - Optional: store the return value of benchFn for each block. Use NULL if this result is not requested. * nbLoops - defines number of times benchFn is run. * @return: a variant, which express either an error, or can generate a valid BMK_runTime_t result. * Use BMK_isSuccessful_runOutcome() to check if function was successful.