From 678272515515afc6fe99975a8c716ce27907fc7e Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 26 Aug 2018 19:29:12 -0700 Subject: [PATCH 01/15] first sketch for largeNbDicts test program --- contrib/largeNbDicts/Makefile | 30 +++ contrib/largeNbDicts/largeNbDicts | Bin 0 -> 13626 bytes contrib/largeNbDicts/largeNbDicts.c | 289 ++++++++++++++++++++++++++++ lib/Makefile | 2 +- lib/dictBuilder/zdict.h | 3 +- programs/bench.h | 2 +- 6 files changed, 323 insertions(+), 3 deletions(-) create mode 100644 contrib/largeNbDicts/Makefile create mode 100755 contrib/largeNbDicts/largeNbDicts create mode 100644 contrib/largeNbDicts/largeNbDicts.c diff --git a/contrib/largeNbDicts/Makefile b/contrib/largeNbDicts/Makefile new file mode 100644 index 00000000..082f0102 --- /dev/null +++ b/contrib/largeNbDicts/Makefile @@ -0,0 +1,30 @@ +# ################################################################ +# Copyright (c) 2018-present, Yann Collet, Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under both the BSD-style license (found in the +# LICENSE file in the root directory of this source tree) and the GPLv2 (found +# in the COPYING file in the root directory of this source tree). +# ################################################################ + + +CPPFLAGS+= -I../../lib -I../../lib/common -I../../lib/dictBuilder -I../../programs + +CFLAGS ?= -O3 +DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ + -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ + -Wstrict-prototypes -Wundef -Wpointer-arith -Wformat-security \ + -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \ + -Wredundant-decls +CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) + + +default: largeNbDicts + +largeNbDicts: LDFLAGS += -lzstd +largeNbDicts: largeNbDicts.c + $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ + + +clean: + $(RM) largeNbDicts diff --git a/contrib/largeNbDicts/largeNbDicts b/contrib/largeNbDicts/largeNbDicts new file mode 100755 index 0000000000000000000000000000000000000000..40416f050fa594cb085444ebb1ca433629d59318 GIT binary patch literal 13626 zcmeHOeQX@X6(8GiF|_0y%7;<;SWt`{L@&M)LlWZDm-t{Wtb>W2k3z!c&bJpIJ>OZo zdy0byGT0|&d%f00rHVw6D58ZTT3l3UYC`G~Ck_Y&*91rjAE6Df$A|cANcr&f_h#qL z=Szb6N7cV}(s4t;`D1>J@tkce7{iRS<4Ko!silTCRYtD1*MMaMSE)1v<_NM(Kduq29@6~j4 zDz~>&*c%ZB2vgjPDSPVd}BOCLu4k_n@#> zC7dE$T4wkPg{r0!n`4QH8cRfz9M9a}hc9&eYZpccm)j_{Pv>MIHc(Yt{hj_2jpW$j z2b9FXywHI~q3y;Q=9RFjZc0u^?d0}qs52F5dMrWO&pE`1qN<6wYHaBVC*x|bsUPQG zjpXtpF$yfXA}Uas>VZKXRl6j>}?Zc2$$NU^`0qoL?~{~Jyn7$uMM;YJB6~b zEN;M5me8d_Av*|nv0RMj0Pli^Vp9X!P}Jf+0k>m7zmKtvkkek)q1wPizr38W7U;hT zz8$r+(3DiaT`DZ&yD`4LOh#}vs)YJ2)cGkxpC6Bf=SQ~0BfwO?gG%$?f9TP%2d-ay z|DBKTcyq~$yB?kkSuH9%js7kzAd;Y9W&JdY;TDz_nxzGmbhN@*RTQHNF>|sj6rZ2$ zHRs=Am=W#>oFU%p^TMTAn0vXfbo}>@;8GUG!*g`%*kYJkk9V0Eg5G#)p^~?U5lxEz zJb@{?ZgC^vM!=1L8v!>0ZUo#2xDjw8;6}iWfExif0&WEUKM`0hTW|W;`q%hZ`#U@2 zf%ocU>l4}9Iw5BQU<3Ppr<+T(PF){C>~iyD$;VuAQf>!Gb zaO%7Ra>YU7Yl7DL12qEF5~vlRjzFCN5&=no1_BKNG!keOU?G8p0P?_7HS)mTW2v9Q zX;TVj7aRy?J6~W!vQ;Tt7w{f?(F48mPp96I7aiNYLY_B%t=0O~v1_eczdS~(QLF~V z+E}AleYNtySlx~O_5Kb1oBZmA-Ey}15G^^|beNOVZIF>HR%!nV`6j4$I_keV>X%4; z7pd=nIvS07Jr@oFGB$I6Z-1|Ww#52E$$WPb5e#Ow(j>pfaNnqGy=<@M)*qXg%I9V4 zbvpu>9`&TxBW2|IQ~ChJ_3T5⊀t=>WuXw%Gd$(c63C-seE9WFBkG=kvjVsk~?K z8+>@v7|sq&_B;|wIALvRsNRK}iuIZODwpA09WkX{l(E8TP^^y?>)-Y;RLY_c&DnH7 zehU!r9S!)7C|2GcAeHrNT7_agcF-Y*L12fL>>2zGY(;~f#NMbUTrE?~JnO65IlXoC zW6uw&JHaC|Pl60r*h^th&ZNnNY_-nCfll)fWwK;kwv@RwvNdb2oPDwntJ_5xFZ;eP zp~<94fQ$5?j5rS7Nh{8*-sZPn5}|(p6a1O_Q~Xy?*awgj{RVWw?+saG4}gkl}FXLFzKAt~;R&zFb|QY>v$vHl@jZTl3QX#06GXpPa9d<-@{ zYHki@+UUTpg8f~T(;T5~w5Onbg1B5QZG~s;Xw);$3Fd>$)(}ul+4!t+#Zbe9VI7jB!4`QjME~cvL}IA!qw}rDBAChoI3j*0gwIOhpC1u zutgL03Je{Al+K}Iy=Bh=M!+XlWS4wJ&e?vlZSBD`&og*7T*1LLFZUf1-48*62hgju zW9Dsgez&sd3;myp^{$+`%ASBy9@t$I%s-d;G&e~HG}|1e6P#`O87(4v`prNi zPtYP((coTJ8Igxk%OeZ+)*EL)E$m`9-=aGnAB%F>Ya{yfulZ`)NhmeUua-qYf z(B$iP@phEAPw{q)w|jZJkGIe8_F3NU=PljaeElSE>3-+yU*zpU-oDJ+SI{ci^V>lD z9r^lZ(4lVlz;bV4#PNSo?^Z zsAR-$f!^zjN^Q%~DXb;Bx*59gnH;YSuhZR-E^3xmu2vMrT{dafSfV#&qBWCJeM$;D z-ok88s85)Nbc?Bx0};a{*9%4fzjhMtFU=7_G-Am_NZ)c9d2CXQ%;T&(%Ra#%%)CiU zXnKe{3@?lJVw!Q8lrnhqL zqXFt+5$)Dij9Q+I$2GnL=2*FrY^@A$D24)WkEUN!6<${n#dCtc0ZUo#2xDjw8;6}iWfExif0{?Xc z=3HF2aSnQ)(W26a@R4N7>}A*MY^a4kt>vG_t6_ZIjkMDc8P)6Nw5T_<2A6lLrXGqV zuGW)1A}uwUO%`js< zNJ()`2l7y)Z7FD@x~3Iv$)(bwrS^g}W&|kdMdR1;<%SGH(@j-36C~RRS+pyjH24hN zgp$1)SM?I;#W(Y2lzR3Nq|_OK{t5^2QQJ7!tjA0ahY0qB;z)Q=*X;xiGi1&UuDq@_ zaK!=$S%r!(qGzc+-Ky57b)`%VzZ?irxP7H5zosx-+=FnlHI&AwV#D~3xsqq$lu$(F zDJFJ(6&v!B?n*TM^zkvu@TG^V?me}gXMa2ucyEG2d786PJz$3Mih=M&0@t47;0A%2 z$nSUsF8sRUDqxJGp}ed@TZ*G)xUUTVs0`m%h951%PnBWHaAQ86)B?+1E5o@m{81S` zR)#Aa=Zp4g%JA$mOfN9-SL}a58K$IfslFQXA@#<;wx?IIr=p&QN?*y-x8|p#dQfSH z_C;ZxD_@cPf&W;zX%Vq8CRPCr>Rsk0*Hqoapik zS;@68l{nYbksN>39r^SnpT?!*3Kttsu#ziX@;octYjLBCcUjzGA{tYp&b=*M*qJx= JxThgD;Xm-ofn)#x literal 0 HcmV?d00001 diff --git a/contrib/largeNbDicts/largeNbDicts.c b/contrib/largeNbDicts/largeNbDicts.c new file mode 100644 index 00000000..749d9660 --- /dev/null +++ b/contrib/largeNbDicts/largeNbDicts.c @@ -0,0 +1,289 @@ +/* + * Copyright (c) 2018-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* largeNbDicts + * This is a benchmark test tool + * dedicated to the specific case of dictionary decompression + * using a very large nb of dictionaries + * thus generating many cache-misses. + * It's created in a bid to investigate performance and find optimizations. */ + + +/*--- Dependencies ---*/ + +#include /* size_t */ +#include /* malloc, free */ +#include /* printf */ +#include /* assert */ + +#include "util.h" +#define ZSTD_STATIC_LINKING_ONLY +#include "zstd.h" +#include "zdict.h" + + +/*--- Constants --- */ + +#define KB *(1<<10) +#define MB *(1<<20) + +#define BLOCKSIZE (4 KB) +#define DICTSIZE (4 KB) +#define COMP_LEVEL 3 + +#define DISPLAY_LEVEL_DEFAULT 3 + + +/*--- Display Macros ---*/ + +#define DISPLAY(...) fprintf(stdout, __VA_ARGS__) +#define DISPLAYLEVEL(l, ...) { if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } } +static int g_displayLevel = DISPLAY_LEVEL_DEFAULT; /* 0 : no display, 1: errors, 2 : + result + interaction + warnings, 3 : + progression, 4 : + information */ + + +/*--- buffer_t ---*/ + +typedef struct { + void* ptr; + size_t size; + size_t capacity; +} buffer_t; + +static const buffer_t kBuffNull = { NULL, 0, 0 }; + + +static buffer_t fillBuffer_fromHandle(buffer_t buff, FILE* f) +{ + size_t const readSize = fread(buff.ptr, 1, buff.capacity, f); + buff.size = readSize; + return buff; +} + +static void freeBuffer(buffer_t buff) +{ + free(buff.ptr); +} + +/* @return : kBuffNull if any error */ +static buffer_t createBuffer_fromHandle(FILE* f, size_t bufferSize) +{ + void* const buffer = malloc(bufferSize); + if (buffer==NULL) return kBuffNull; + + { buffer_t buff = { buffer, 0, bufferSize }; + buff = fillBuffer_fromHandle(buff, f); + if (buff.size != buff.capacity) { + freeBuffer(buff); + return kBuffNull; + } + return buff; + } +} + +/* @return : kBuffNull if any error */ +static buffer_t createBuffer_fromFile(const char* fileName) +{ + U64 const fileSize = UTIL_getFileSize(fileName); + size_t const bufferSize = (size_t) fileSize; + + if (fileSize == UTIL_FILESIZE_UNKNOWN) return kBuffNull; + assert((U64)bufferSize == fileSize); /* check overflow */ + + { buffer_t buff; + FILE* const f = fopen(fileName, "rb"); + if (f == NULL) return kBuffNull; + + buff = createBuffer_fromHandle(f, bufferSize); + fclose(f); /* do nothing specific if fclose() fails */ + return buff; + } +} + + +/*--- buffer_collection_t ---*/ + +typedef struct { + void** buffers; + size_t* capacities; + size_t nbBuffers; +} buffer_collection_t; + +static const buffer_collection_t kNullCollection = { NULL, NULL, 0 }; + +static void freeCollection(buffer_collection_t collection) +{ + free(collection.buffers); + free(collection.capacities); +} + +/* returns .buffers=NULL if operation fails */ +buffer_collection_t splitBuffer(buffer_t srcBuffer, size_t blockSize) +{ + size_t const nbBlocks = (srcBuffer.size + (blockSize-1)) / blockSize; + + void** const buffers = malloc(nbBlocks * sizeof(void*)); + size_t* const capacities = malloc(nbBlocks * sizeof(size_t*)); + if ((buffers==NULL) || capacities==NULL) { + free(buffers); + free(capacities); + return kNullCollection; + } + + char* newBlockPtr = (char*)srcBuffer.ptr; + char* const srcEnd = newBlockPtr + srcBuffer.size; + assert(nbBlocks >= 1); + for (size_t blockNb = 0; blockNb < nbBlocks-1; blockNb++) { + buffers[blockNb] = newBlockPtr; + capacities[blockNb] = blockSize; + newBlockPtr += blockSize; + } + + /* last block */ + assert(newBlockPtr <= srcEnd); + size_t const lastBlockSize = (srcEnd - newBlockPtr); + buffers[nbBlocks-1] = newBlockPtr; + capacities[nbBlocks-1] = lastBlockSize; + + buffer_collection_t result; + result.buffers = buffers; + result.capacities = capacities; + result.nbBuffers = nbBlocks; + return result; +} + + + +/*--- ddict_collection_t ---*/ + +typedef struct { + ZSTD_DDict** ddicts; + size_t nbDDict; +} ddict_collection_t; + +static const ddict_collection_t kNullDDictCollection = { NULL, 0 }; + +static void freeDDictCollection(ddict_collection_t ddictc) +{ + for (size_t dictNb=0; dictNb < ddictc.nbDDict; dictNb++) { + ZSTD_freeDDict(ddictc.ddicts[dictNb]); + } + free(ddictc.ddicts); +} + +/* returns .buffers=NULL if operation fails */ +static ddict_collection_t createDDictCollection(const void* dictBuffer, size_t dictSize, size_t nbDDict) +{ + ZSTD_DDict** const ddicts = malloc(nbDDict * sizeof(ZSTD_DDict*)); + if (ddicts==NULL) return kNullDDictCollection; + for (size_t dictNb=0; dictNb < nbDDict; dictNb++) { + ddicts[dictNb] = ZSTD_createDDict(dictBuffer, dictSize); + assert(ddicts[dictNb] != NULL); + } + ddict_collection_t ddictc; + ddictc.ddicts = ddicts; + ddictc.nbDDict = nbDDict; + return ddictc; +} + + + +/*--- Benchmark --- */ + + +/* bench() : + * @return : 0 is success, 1+ otherwise */ +int bench(const char* fileName) +{ + int result = 0; + + DISPLAYLEVEL(3, "loading %s... \n", fileName); + buffer_t const srcBuffer = createBuffer_fromFile(fileName); + if (srcBuffer.ptr == NULL) { + DISPLAYLEVEL(1," error reading file %s \n", fileName); + return 1; + } + DISPLAYLEVEL(3, "created src buffer of size %.1f MB \n", + (double)(srcBuffer.size) / (1 MB)); + + buffer_collection_t const srcBlockBuffers = splitBuffer(srcBuffer, BLOCKSIZE); + assert(srcBlockBuffers.buffers != NULL); + unsigned const nbBlocks = (unsigned)srcBlockBuffers.nbBuffers; + DISPLAYLEVEL(3, "splitting input into %u blocks of max size %u bytes \n", + nbBlocks, BLOCKSIZE); + + size_t const dstBlockSize = ZSTD_compressBound(BLOCKSIZE); + size_t const dstBufferCapacity = nbBlocks * dstBlockSize; + void* const dstPtr = malloc(dstBufferCapacity); + assert(dstPtr != NULL); + buffer_t dstBuffer; + dstBuffer.ptr = dstPtr; + dstBuffer.capacity = dstBufferCapacity; + dstBuffer.size = dstBufferCapacity; + + buffer_collection_t const dstBlockBuffers = splitBuffer(dstBuffer, dstBlockSize); + assert(dstBlockBuffers.buffers != NULL); + + DISPLAYLEVEL(3, "creating dictionary, of target size %u bytes \n", DICTSIZE); + void* const dictBuffer = malloc(DICTSIZE); + if (dictBuffer == NULL) { result = 1; goto _cleanup; } + + size_t const dictSize = ZDICT_trainFromBuffer(dictBuffer, DICTSIZE, + srcBuffer.ptr, + srcBlockBuffers.capacities, + nbBlocks); + if (ZSTD_isError(dictSize)) { + DISPLAYLEVEL(1, "error creating dictionary \n"); + result = 1; + goto _cleanup; + } + + size_t const dictMem = ZSTD_estimateDDictSize(dictSize, ZSTD_dlm_byCopy); + size_t const allDictMem = dictMem * nbBlocks; + DISPLAYLEVEL(3, "generating %u dictionaries, using %.1f MB of memory \n", + nbBlocks, (double)allDictMem / (1 MB)); + + ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, COMP_LEVEL); + do { + ddict_collection_t const dictionaries = createDDictCollection(dictBuffer, dictSize, nbBlocks); + assert(dictionaries.ddicts != NULL); + + freeDDictCollection(dictionaries); + } while(0); + ZSTD_freeCDict(cdict); + +_cleanup: + free(dictBuffer); + freeCollection(dstBlockBuffers); + freeBuffer(dstBuffer); + freeCollection(srcBlockBuffers); + freeBuffer(srcBuffer); + + return result; +} + + + + +/*--- Command Line ---*/ + +int bad_usage(const char* exeName) +{ + DISPLAY (" bad usage : \n"); + DISPLAY (" %s filename \n", exeName); + return 1; +} + +int main (int argc, const char** argv) +{ + const char* const exeName = argv[0]; + + if (argc != 2) return bad_usage(exeName); + return bench(argv[1]); +} diff --git a/lib/Makefile b/lib/Makefile index 01689c6d..cf8e45b0 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -23,7 +23,7 @@ ifeq ($(OS),Windows_NT) # MinGW assumed CPPFLAGS += -D__USE_MINGW_ANSI_STDIO # compatibility with %zu formatting endif CFLAGS ?= -O3 -DEBUGFLAGS = -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ +DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ -Wstrict-prototypes -Wundef -Wpointer-arith -Wformat-security \ -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \ diff --git a/lib/dictBuilder/zdict.h b/lib/dictBuilder/zdict.h index 4094669d..3b3a6527 100644 --- a/lib/dictBuilder/zdict.h +++ b/lib/dictBuilder/zdict.h @@ -52,7 +52,8 @@ extern "C" { * It's recommended that total size of all samples be about ~x100 times the target size of dictionary. */ ZDICTLIB_API size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity, - const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples); + const void* samplesBuffer, + const size_t* samplesSizes, unsigned nbSamples); /*====== Helper functions ======*/ diff --git a/programs/bench.h b/programs/bench.h index f6a53fc6..184cc3e0 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -233,7 +233,7 @@ typedef size_t (*BMK_initFn_t)(void* initPayload); * srcSizes - an array of the sizes of above buffers * dstBuffers - an array of buffers to be written into by benchFn * dstCapacities - an array of the capacities of above buffers - * blockResults - store the return value of benchFn for each block. Optional. Use NULL if this result is not requested. + * blockResults - Optional: store the return value of benchFn for each block. Use NULL if this result is not requested. * nbLoops - defines number of times benchFn is run. * @return: a variant, which express either an error, or can generate a valid BMK_runTime_t result. * Use BMK_isSuccessful_runOutcome() to check if function was successful. From 274b60e6e6571ccce5982e36ac82eabe606c1364 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 27 Aug 2018 17:08:44 -0700 Subject: [PATCH 02/15] largeNbDicts can compress and compare dict vs noDict --- contrib/largeNbDicts/Makefile | 2 + contrib/largeNbDicts/largeNbDicts | Bin 13626 -> 14034 bytes contrib/largeNbDicts/largeNbDicts.c | 144 +++++++++++++++++++++------- 3 files changed, 114 insertions(+), 32 deletions(-) diff --git a/contrib/largeNbDicts/Makefile b/contrib/largeNbDicts/Makefile index 082f0102..026d76f1 100644 --- a/contrib/largeNbDicts/Makefile +++ b/contrib/largeNbDicts/Makefile @@ -21,6 +21,8 @@ CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) default: largeNbDicts +all : largeNbDicts + largeNbDicts: LDFLAGS += -lzstd largeNbDicts: largeNbDicts.c $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ diff --git a/contrib/largeNbDicts/largeNbDicts b/contrib/largeNbDicts/largeNbDicts index 40416f050fa594cb085444ebb1ca433629d59318..c057a2b78aa551a2de831b6e304f8747a6ea3d0f 100755 GIT binary patch literal 14034 zcmeHOe{fXCecwY;tzye32#)2{hDRzTqOk>sEnB$A(IE-m;U_GN6gwF4xI5jUo1E@6 zckhB^j3X>v%j@%{nlyNrq;8e8PKo0-YU3nW2}BOq$TPL$V3^_%+thKKM`gw;OfbQe z>*u?>_Y=%yCi(Mr^mf1B{p^0f-~H})_r2Tq^s@`6FI~f!ql~fPIgGJtTw@iCC7A+> zv72zIxOBbKy{3I*`~CM(ZDDd0vRC9tPCXRpdRO~{U4>0}f2FWas%Nl`!5sD{64rGi zFlgl3UMA0p!2lk)R<;3Oo@-9ba6YT+aUcA0&Zim*?b!-&t=LLx}s$BB?h*Tdec$*WO>yuGqL@%i#-yfqS|g8U}dspeUC zZ+n|3v&NPc`3*d!qcUJg4&0qhpTs`En;;|KRHCfUqKqvAzYiU37h{8{r@16M29`61 zsbvcq8Jhx5%dZm`C2B(#sU(><;`*qxJ(hp;aZxA$CidezNNiG;a>2R z5cb@T(<|1Lz4qJYhxYnz^7P%h{nNFmuciib>F+Nx9MvFUWW!|rEV(Z)k(L%*gpuXt zW#l7=)ExDD!%L$B#?pu5MlUx6#t_}$YJf>0AGfk8zyI=guqZSAVL2LY7>XN#{)Sbu z3tBVwDO_aj7I>4&C6XuamQu+BB@dK5Q1U>@10@fXJW%q$AMSx})%>;l0r%(J-R`ap z>d1RFs(Ds5cf6`v?I7EywUkkob{)~oS@nan9`mr4s(pd9YUXe32|VI@%om7!ij#*r zlBbO|9m#i$Rh|`ZIv;DqGozYnuU0LlT05py$6?L9q^4hSj?O@YC-uN3^M6PF`xk)Xhg#$8n-7J?5sPm6E9@rdl#J#MDSeAx4o*Ju&r?Sw_q<$utqu1V$bCX{9>y z;`zki!e~SBr0zKCNp&4#W2!kvH5c+0FQNrnrC&d7DCq=NO4W6 z<}d66kfNC%sOFpYk04}bQ=18!8t)-B^jPkivcHZdKOx%wHYkJv&Hd^FRHK>iYv$XU z`8&6H++GA!MQ`&hyAxz`D$&wmHh9dy0@L7ZfhJS_eBf59KH-N+;dSTh?&MRE`I$+do7w%&QA)`^H_B_=#0m(oTr{FTJ7DAEu@AFQ2@WZLL} zTkUP&$tTGZhH`*@SD}$+o`J{gwScr0vqly5YX#HpI_El}nQ40=7jkoLhiTMSQdT|l z3D4oSS~wiiK>#PBoPO{zb>|@(EIE}&)D`0a9AfaKmR<`j6%(X!@iGnSjhpd10%{Pu(&+c)M_hLv%D5&C}_fC%x?XV z8$uP>r>3|eZWy9kPcB297)A*4pQ9LaO(Cjn{QMRnI|j;Q9#YMo3Ag!%X8y|K{O%j% z@i#a?Gvyqe4{4gYe~@eF*@ec#ShXmHg}mPC7|u7Z32im=UHe-XxN*dC!`{VDXjq9| z^~rjwOYN`ZEzhY|_g>X}U9;LJ?DZGX+4?>q9m#Xf(N}q=o@ve8F{W8hRil2A4nPn= zq~jeN3PR>SuHYcrYS#W15^n8QsmKI^wF4W?CDrU6SFMgIXqr~7o(Yfnx?N4~GM}ey zck~;i%&Is|3(9H=s?NwuG>XTyrHq-n)g(>AWVG{p_?6F`E%@kL< zV-G2|a*9dm*ZHVOsmFZFIjRc13DS!dVf|%R_5D)y#w)8HhUy7NoB6gq4~kvq=r-{N zjuSMoAS4~^B+eqHGNQ9F%vtfnI9JHKQP6Uw!tHZAhK3QY zqB%)beFfa)f6>@pqbW}{&+_`lKNql#Vsbu4YCkLBp9{ELz~%7PdxVnKgb!z+rHcuaR4R3|P!>!Tl1?L%2gzNOw$m z%wwvVn5Ji=jukIElM0F5h<+aPMZR1cP=Qc!j?#_Db)wysR!4SJGoz;{9%vW1<2Y6d z*5Lvwr&Q}YD)(tt`Egp@R!6mc0n)HTQB>aqCAe)G9gg3~D4U$=&g|#tcU5`r=ioOU zW8ch4JPZ;N-8qT-X3Ai&W}e5LxV|uy4holkz&%)1NiMxeKHCK$@c2Iq__%;iftx%> za7e&O0Y4|;zasb)!D|H^6!6ys{0{`bNAPC=^YzB(<8vGM{i*RjDEC;`L+b_1-~l=j zokT-s(`0z8dl2!n0K|>!0suOv#md;+b1>tt?EK<+z&xv2w~wS9kJf3{_R4nGJ0s~i zM)yd%Jh4EvwpT;i^N?mAgN)>q(V|+nyh7$$*TR6ILIeI0ZLrkQCCUxJ-P|5Ro%_dR z!A!O5+)TM@#cOEw51mh^+su>MTk}}D?#qyyuA{e3>AJ7*{Hr{Fg6Dg9{xzQeJAG>Af0O4l4e7dX@q91OpXT{9JpVVIPw<>pZ@TVTp8o(jq9b)n1qk8$(7g4=LwG*|GsBWU@=F{gVYBf;dQpc6ZKC- zEd&Lv*|e+`XZY<|mJ?J*DN{djPkbO8GFBymLA=8Xdt+Mz>wIk?zY%Znvxsk1IO_jG zT)C%Naj{5X8)rL>n6k22iO2lykzN*#dVBGTN$EwOP&DF=4JmbTWiIo_0$!sKxK)V; z6@ye5N<8#%K&eY8z9A!!Z5|-SLI@*y!YXBPv$C#R)7avT8@k%`P`o`Bi^l4y!3IcK zk~N_&-q6qh)xJQ)-^XB|=-+^MI9VCotUxbWulV?&$tYn>gDa?XtbzdQN&^@D;|-!= zJS&8|ObJB>5(aW3$_FAm1!4W(!OMo%8#l;pnUR3Ks)F3$3yr7xtPp=!)bNJM1?!@1 z5?IFk(f)y0ARZ^zdkrNV_+lVT_H7FpeNos;BIG>Xt=x|(iZ<6ZEDtJ5eP%Y6(ClSp zle(N#crGDC=O*v>lejey3B*K1KvA|&C=kC@ima3VCl?0#qX?wAO!0Y1qjzgSxtmyo zRxlI}M7;e0<)Pc!a&m@x0N>WeV!o^gMC`I+Un8jbyvLPh9>fI`kp-FGZwzK#T^M;H zI7IA;cxog6w?@O^0Cz6yrSawBMX(Q1+&;VN> zV`DA!X)XUWUiabaZX5&B`&1UInb)FkZ1c2s=|;>OimZ-B`^Cu#@R&j%9mi$Pn^r{= zI1L0+UyQW28iNEZD+Y9al+;RRCxka;aXLB-WV*E!RcD&3h4uUhhQxGc>H~2j)DJ-% zq0<^dAIzY^SRjy9y(PakE2%G2D~}wh{{VNNHy#hf3_WH3Uoz=ZSN!XCz;Z53ND+GcKp2n)fE2zLQXci=)vBu~A_^ZCId-YVjsD&l`o#7`ITzbN9*6!CN#m#?2*L+0~}V9578(OWkB zW%(6Fyt{~3i}>|L{DVdO<{~~=#7BzwM~e8-A|7vE89qW_IC|v&5r8i&SQRdMcZDgT zZxa|ofnhq?$8gbJL~~C6A%KwJ|J6Xpn>g-cbS6qKWES9Bhzp5-2KC35ui$?>E?*}) z4xMGF>*8I4e%WHd%EU_1^H@a-_aMr>cA=JG92 z)aN$89G~AWv*F=Rk+;?5-8VA(o!sd3pxohfNXFJiMF6?j%rd)J;r0cN?^~DMeWbI+ Wy#|t^{i|{0&vJJf{-%Wo#s33;RQ9z1 literal 13626 zcmeHOeQX@X6(8GiF|_0y%7;<;SWt`{L@&M)LlWZDm-t{Wtb>W2k3z!c&bJpIJ>OZo zdy0byGT0|&d%f00rHVw6D58ZTT3l3UYC`G~Ck_Y&*91rjAE6Df$A|cANcr&f_h#qL z=Szb6N7cV}(s4t;`D1>J@tkce7{iRS<4Ko!silTCRYtD1*MMaMSE)1v<_NM(Kduq29@6~j4 zDz~>&*c%ZB2vgjPDSPVd}BOCLu4k_n@#> zC7dE$T4wkPg{r0!n`4QH8cRfz9M9a}hc9&eYZpccm)j_{Pv>MIHc(Yt{hj_2jpW$j z2b9FXywHI~q3y;Q=9RFjZc0u^?d0}qs52F5dMrWO&pE`1qN<6wYHaBVC*x|bsUPQG zjpXtpF$yfXA}Uas>VZKXRl6j>}?Zc2$$NU^`0qoL?~{~Jyn7$uMM;YJB6~b zEN;M5me8d_Av*|nv0RMj0Pli^Vp9X!P}Jf+0k>m7zmKtvkkek)q1wPizr38W7U;hT zz8$r+(3DiaT`DZ&yD`4LOh#}vs)YJ2)cGkxpC6Bf=SQ~0BfwO?gG%$?f9TP%2d-ay z|DBKTcyq~$yB?kkSuH9%js7kzAd;Y9W&JdY;TDz_nxzGmbhN@*RTQHNF>|sj6rZ2$ zHRs=Am=W#>oFU%p^TMTAn0vXfbo}>@;8GUG!*g`%*kYJkk9V0Eg5G#)p^~?U5lxEz zJb@{?ZgC^vM!=1L8v!>0ZUo#2xDjw8;6}iWfExif0&WEUKM`0hTW|W;`q%hZ`#U@2 zf%ocU>l4}9Iw5BQU<3Ppr<+T(PF){C>~iyD$;VuAQf>!Gb zaO%7Ra>YU7Yl7DL12qEF5~vlRjzFCN5&=no1_BKNG!keOU?G8p0P?_7HS)mTW2v9Q zX;TVj7aRy?J6~W!vQ;Tt7w{f?(F48mPp96I7aiNYLY_B%t=0O~v1_eczdS~(QLF~V z+E}AleYNtySlx~O_5Kb1oBZmA-Ey}15G^^|beNOVZIF>HR%!nV`6j4$I_keV>X%4; z7pd=nIvS07Jr@oFGB$I6Z-1|Ww#52E$$WPb5e#Ow(j>pfaNnqGy=<@M)*qXg%I9V4 zbvpu>9`&TxBW2|IQ~ChJ_3T5⊀t=>WuXw%Gd$(c63C-seE9WFBkG=kvjVsk~?K z8+>@v7|sq&_B;|wIALvRsNRK}iuIZODwpA09WkX{l(E8TP^^y?>)-Y;RLY_c&DnH7 zehU!r9S!)7C|2GcAeHrNT7_agcF-Y*L12fL>>2zGY(;~f#NMbUTrE?~JnO65IlXoC zW6uw&JHaC|Pl60r*h^th&ZNnNY_-nCfll)fWwK;kwv@RwvNdb2oPDwntJ_5xFZ;eP zp~<94fQ$5?j5rS7Nh{8*-sZPn5}|(p6a1O_Q~Xy?*awgj{RVWw?+saG4}gkl}FXLFzKAt~;R&zFb|QY>v$vHl@jZTl3QX#06GXpPa9d<-@{ zYHki@+UUTpg8f~T(;T5~w5Onbg1B5QZG~s;Xw);$3Fd>$)(}ul+4!t+#Zbe9VI7jB!4`QjME~cvL}IA!qw}rDBAChoI3j*0gwIOhpC1u zutgL03Je{Al+K}Iy=Bh=M!+XlWS4wJ&e?vlZSBD`&og*7T*1LLFZUf1-48*62hgju zW9Dsgez&sd3;myp^{$+`%ASBy9@t$I%s-d;G&e~HG}|1e6P#`O87(4v`prNi zPtYP((coTJ8Igxk%OeZ+)*EL)E$m`9-=aGnAB%F>Ya{yfulZ`)NhmeUua-qYf z(B$iP@phEAPw{q)w|jZJkGIe8_F3NU=PljaeElSE>3-+yU*zpU-oDJ+SI{ci^V>lD z9r^lZ(4lVlz;bV4#PNSo?^Z zsAR-$f!^zjN^Q%~DXb;Bx*59gnH;YSuhZR-E^3xmu2vMrT{dafSfV#&qBWCJeM$;D z-ok88s85)Nbc?Bx0};a{*9%4fzjhMtFU=7_G-Am_NZ)c9d2CXQ%;T&(%Ra#%%)CiU zXnKe{3@?lJVw!Q8lrnhqL zqXFt+5$)Dij9Q+I$2GnL=2*FrY^@A$D24)WkEUN!6<${n#dCtc0ZUo#2xDjw8;6}iWfExif0{?Xc z=3HF2aSnQ)(W26a@R4N7>}A*MY^a4kt>vG_t6_ZIjkMDc8P)6Nw5T_<2A6lLrXGqV zuGW)1A}uwUO%`js< zNJ()`2l7y)Z7FD@x~3Iv$)(bwrS^g}W&|kdMdR1;<%SGH(@j-36C~RRS+pyjH24hN zgp$1)SM?I;#W(Y2lzR3Nq|_OK{t5^2QQJ7!tjA0ahY0qB;z)Q=*X;xiGi1&UuDq@_ zaK!=$S%r!(qGzc+-Ky57b)`%VzZ?irxP7H5zosx-+=FnlHI&AwV#D~3xsqq$lu$(F zDJFJ(6&v!B?n*TM^zkvu@TG^V?me}gXMa2ucyEG2d786PJz$3Mih=M&0@t47;0A%2 z$nSUsF8sRUDqxJGp}ed@TZ*G)xUUTVs0`m%h951%PnBWHaAQ86)B?+1E5o@m{81S` zR)#Aa=Zp4g%JA$mOfN9-SL}a58K$IfslFQXA@#<;wx?IIr=p&QN?*y-x8|p#dQfSH z_C;ZxD_@cPf&W;zX%Vq8CRPCr>Rsk0*Hqoapik zS;@68l{nYbksN>39r^SnpT?!*3Kttsu#ziX@;octYjLBCcUjzGA{tYp&b=*M*qJx= JxThgD;Xm-ofn)#x diff --git a/contrib/largeNbDicts/largeNbDicts.c b/contrib/largeNbDicts/largeNbDicts.c index 749d9660..536e45ff 100644 --- a/contrib/largeNbDicts/largeNbDicts.c +++ b/contrib/largeNbDicts/largeNbDicts.c @@ -159,6 +159,33 @@ buffer_collection_t splitBuffer(buffer_t srcBuffer, size_t blockSize) } +/*--- dictionary creation ---*/ + +buffer_t createDictionary(const char* dictionary, + const void* srcBuffer, size_t* srcBlockSizes, unsigned nbBlocks) +{ + if (dictionary) { + DISPLAYLEVEL(3, "loading dictionary %s \n", dictionary); + return createBuffer_fromFile(dictionary); + } else { + DISPLAYLEVEL(3, "creating dictionary, of target size %u bytes \n", DICTSIZE); + void* const dictBuffer = malloc(DICTSIZE); + assert(dictBuffer != NULL); + + size_t const dictSize = ZDICT_trainFromBuffer(dictBuffer, DICTSIZE, + srcBuffer, + srcBlockSizes, + nbBlocks); + assert(!ZSTD_isError(dictSize)); + + buffer_t result; + result.ptr = dictBuffer; + result.capacity = DICTSIZE; + result.size = dictSize; + return result; + } +} + /*--- ddict_collection_t ---*/ @@ -181,6 +208,7 @@ static void freeDDictCollection(ddict_collection_t ddictc) static ddict_collection_t createDDictCollection(const void* dictBuffer, size_t dictSize, size_t nbDDict) { ZSTD_DDict** const ddicts = malloc(nbDDict * sizeof(ZSTD_DDict*)); + assert(ddicts != NULL); if (ddicts==NULL) return kNullDDictCollection; for (size_t dictNb=0; dictNb < nbDDict; dictNb++) { ddicts[dictNb] = ZSTD_createDDict(dictBuffer, dictSize); @@ -193,29 +221,64 @@ static ddict_collection_t createDDictCollection(const void* dictBuffer, size_t d } +/* --- Compression --- */ -/*--- Benchmark --- */ +/* compressBlocks() : + * @return : total compressed size of all blocks, + * or 0 if error. + */ +static size_t compressBlocks(buffer_collection_t dstBlockBuffers, buffer_collection_t srcBlockBuffers, ZSTD_CDict* cdict, int cLevel) +{ + size_t const nbBlocks = srcBlockBuffers.nbBuffers; + assert(dstBlockBuffers.nbBuffers == srcBlockBuffers.nbBuffers); + + ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + assert(cctx != NULL); + + size_t totalCSize = 0; + for (size_t blockNb=0; blockNb < nbBlocks; blockNb++) { + size_t cBlockSize; + if (cdict == NULL) { + cBlockSize = ZSTD_compressCCtx(cctx, + dstBlockBuffers.buffers[blockNb], dstBlockBuffers.capacities[blockNb], + srcBlockBuffers.buffers[blockNb], srcBlockBuffers.capacities[blockNb], + cLevel); + assert(!ZSTD_isError(cBlockSize)); + } else { + cBlockSize = ZSTD_compress_usingCDict(cctx, + dstBlockBuffers.buffers[blockNb], dstBlockBuffers.capacities[blockNb], + srcBlockBuffers.buffers[blockNb], srcBlockBuffers.capacities[blockNb], + cdict); + assert(!ZSTD_isError(cBlockSize)); + } + totalCSize += cBlockSize; + } + return totalCSize; +} + + +/* --- Benchmark --- */ /* bench() : + * fileName : file to load for benchmarking purpose + * dictionary : optional (can be NULL), file to load as dictionary, + * if none provided : will be calculated on the fly by the program. * @return : 0 is success, 1+ otherwise */ -int bench(const char* fileName) +int bench(const char* fileName, const char* dictionary) { int result = 0; DISPLAYLEVEL(3, "loading %s... \n", fileName); buffer_t const srcBuffer = createBuffer_fromFile(fileName); - if (srcBuffer.ptr == NULL) { - DISPLAYLEVEL(1," error reading file %s \n", fileName); - return 1; - } + assert(srcBuffer.ptr != NULL); DISPLAYLEVEL(3, "created src buffer of size %.1f MB \n", (double)(srcBuffer.size) / (1 MB)); buffer_collection_t const srcBlockBuffers = splitBuffer(srcBuffer, BLOCKSIZE); assert(srcBlockBuffers.buffers != NULL); unsigned const nbBlocks = (unsigned)srcBlockBuffers.nbBuffers; - DISPLAYLEVEL(3, "splitting input into %u blocks of max size %u bytes \n", + DISPLAYLEVEL(3, "split input into %u blocks of max size %u bytes \n", nbBlocks, BLOCKSIZE); size_t const dstBlockSize = ZSTD_compressBound(BLOCKSIZE); @@ -230,36 +293,44 @@ int bench(const char* fileName) buffer_collection_t const dstBlockBuffers = splitBuffer(dstBuffer, dstBlockSize); assert(dstBlockBuffers.buffers != NULL); - DISPLAYLEVEL(3, "creating dictionary, of target size %u bytes \n", DICTSIZE); - void* const dictBuffer = malloc(DICTSIZE); - if (dictBuffer == NULL) { result = 1; goto _cleanup; } + /* dictionary determination */ + buffer_t const dictBuffer = createDictionary(dictionary, + srcBuffer.ptr, + srcBlockBuffers.capacities, nbBlocks); + assert(dictBuffer.ptr != NULL); - size_t const dictSize = ZDICT_trainFromBuffer(dictBuffer, DICTSIZE, - srcBuffer.ptr, - srcBlockBuffers.capacities, - nbBlocks); - if (ZSTD_isError(dictSize)) { - DISPLAYLEVEL(1, "error creating dictionary \n"); - result = 1; - goto _cleanup; - } + ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer.ptr, dictBuffer.size, COMP_LEVEL); + assert(cdict != NULL); - size_t const dictMem = ZSTD_estimateDDictSize(dictSize, ZSTD_dlm_byCopy); + size_t const cTotalSizeNoDict = compressBlocks(dstBlockBuffers, srcBlockBuffers, NULL, COMP_LEVEL); + assert(cTotalSizeNoDict != 0); + DISPLAYLEVEL(3, "compressing at level %u without dictionary : Ratio=%.2f (%u bytes) \n", + COMP_LEVEL, + (double)srcBuffer.size / cTotalSizeNoDict, (unsigned)cTotalSizeNoDict); + + size_t const cTotalSize = compressBlocks(dstBlockBuffers, srcBlockBuffers, cdict, COMP_LEVEL); + assert(cTotalSize != 0); + DISPLAYLEVEL(3, "compressed using a %u bytes dictionary : Ratio=%.2f (%u bytes) \n", + (unsigned)dictBuffer.size, + (double)srcBuffer.size / cTotalSize, (unsigned)cTotalSize); + + size_t const dictMem = ZSTD_estimateDDictSize(dictBuffer.size, ZSTD_dlm_byCopy); size_t const allDictMem = dictMem * nbBlocks; DISPLAYLEVEL(3, "generating %u dictionaries, using %.1f MB of memory \n", nbBlocks, (double)allDictMem / (1 MB)); - ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, COMP_LEVEL); - do { - ddict_collection_t const dictionaries = createDDictCollection(dictBuffer, dictSize, nbBlocks); - assert(dictionaries.ddicts != NULL); + ddict_collection_t const dictionaries = createDDictCollection(dictBuffer.ptr, dictBuffer.size, nbBlocks); + assert(dictionaries.ddicts != NULL); - freeDDictCollection(dictionaries); - } while(0); + + + //result = benchMem(srcBlockBuffers, dstBlockBuffers, dictionaries);; + + + + freeDDictCollection(dictionaries); ZSTD_freeCDict(cdict); - -_cleanup: - free(dictBuffer); + freeBuffer(dictBuffer); freeCollection(dstBlockBuffers); freeBuffer(dstBuffer); freeCollection(srcBlockBuffers); @@ -276,7 +347,7 @@ _cleanup: int bad_usage(const char* exeName) { DISPLAY (" bad usage : \n"); - DISPLAY (" %s filename \n", exeName); + DISPLAY (" %s filename [-D dictionary] \n", exeName); return 1; } @@ -284,6 +355,15 @@ int main (int argc, const char** argv) { const char* const exeName = argv[0]; - if (argc != 2) return bad_usage(exeName); - return bench(argv[1]); + if (argc < 2) return bad_usage(exeName); + const char* const fileName = argv[1]; + + const char* dictionary = NULL; + if (argc > 2) { + if (argc != 4) return bad_usage(exeName); + if (strcmp(argv[2], "-D")) return bad_usage(exeName); + dictionary = argv[3]; + } + + return bench(fileName, dictionary); } From 0c66a44d1bcc0b7eae7f8ef52d6008541abdb7b1 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 28 Aug 2018 15:47:07 -0700 Subject: [PATCH 03/15] first working test program measures : - compression ratio with / without dictionary - create one dictionary per block - memory budget for dictionaries - decompression speed, using one different dictionary per block current limitations : - only one file - 4K blocks only - automatic dictionary built with 4K size dictionary can be selected on command line, with -D --- contrib/largeNbDicts/.gitignore | 2 + contrib/largeNbDicts/Makefile | 15 ++- contrib/largeNbDicts/largeNbDicts | Bin 14034 -> 0 bytes contrib/largeNbDicts/largeNbDicts.c | 156 ++++++++++++++++++++++++++-- programs/bench.c | 4 +- 5 files changed, 162 insertions(+), 15 deletions(-) create mode 100644 contrib/largeNbDicts/.gitignore delete mode 100755 contrib/largeNbDicts/largeNbDicts diff --git a/contrib/largeNbDicts/.gitignore b/contrib/largeNbDicts/.gitignore new file mode 100644 index 00000000..e77c4e49 --- /dev/null +++ b/contrib/largeNbDicts/.gitignore @@ -0,0 +1,2 @@ +# build artifacts +largeNbDicts diff --git a/contrib/largeNbDicts/Makefile b/contrib/largeNbDicts/Makefile index 026d76f1..f4b060ae 100644 --- a/contrib/largeNbDicts/Makefile +++ b/contrib/largeNbDicts/Makefile @@ -7,8 +7,10 @@ # in the COPYING file in the root directory of this source tree). # ################################################################ +PROGDIR = ../../programs +LIBDIR = ../../lib -CPPFLAGS+= -I../../lib -I../../lib/common -I../../lib/dictBuilder -I../../programs +CPPFLAGS+= -I$(LIBDIR) -I$(LIBDIR)/common -I$(LIBDIR)/dictBuilder -I$(PROGDIR) CFLAGS ?= -O3 DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ @@ -24,9 +26,18 @@ default: largeNbDicts all : largeNbDicts largeNbDicts: LDFLAGS += -lzstd -largeNbDicts: largeNbDicts.c +largeNbDicts: bench.o datagen.o xxhash.o largeNbDicts.c $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ +bench.o : $(PROGDIR)/bench.c + $(CC) $(CPPFLAGS) $(CFLAGS) $^ -c + +datagen.o: $(PROGDIR)/datagen.c + $(CC) $(CPPFLAGS) $(CFLAGS) $^ -c + +xxhash.o : $(LIBDIR)/common/xxhash.c + $(CC) $(CPPFLAGS) $(CFLAGS) $^ -c clean: + $(RM) *.o $(RM) largeNbDicts diff --git a/contrib/largeNbDicts/largeNbDicts b/contrib/largeNbDicts/largeNbDicts deleted file mode 100755 index c057a2b78aa551a2de831b6e304f8747a6ea3d0f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 14034 zcmeHOe{fXCecwY;tzye32#)2{hDRzTqOk>sEnB$A(IE-m;U_GN6gwF4xI5jUo1E@6 zckhB^j3X>v%j@%{nlyNrq;8e8PKo0-YU3nW2}BOq$TPL$V3^_%+thKKM`gw;OfbQe z>*u?>_Y=%yCi(Mr^mf1B{p^0f-~H})_r2Tq^s@`6FI~f!ql~fPIgGJtTw@iCC7A+> zv72zIxOBbKy{3I*`~CM(ZDDd0vRC9tPCXRpdRO~{U4>0}f2FWas%Nl`!5sD{64rGi zFlgl3UMA0p!2lk)R<;3Oo@-9ba6YT+aUcA0&Zim*?b!-&t=LLx}s$BB?h*Tdec$*WO>yuGqL@%i#-yfqS|g8U}dspeUC zZ+n|3v&NPc`3*d!qcUJg4&0qhpTs`En;;|KRHCfUqKqvAzYiU37h{8{r@16M29`61 zsbvcq8Jhx5%dZm`C2B(#sU(><;`*qxJ(hp;aZxA$CidezNNiG;a>2R z5cb@T(<|1Lz4qJYhxYnz^7P%h{nNFmuciib>F+Nx9MvFUWW!|rEV(Z)k(L%*gpuXt zW#l7=)ExDD!%L$B#?pu5MlUx6#t_}$YJf>0AGfk8zyI=guqZSAVL2LY7>XN#{)Sbu z3tBVwDO_aj7I>4&C6XuamQu+BB@dK5Q1U>@10@fXJW%q$AMSx})%>;l0r%(J-R`ap z>d1RFs(Ds5cf6`v?I7EywUkkob{)~oS@nan9`mr4s(pd9YUXe32|VI@%om7!ij#*r zlBbO|9m#i$Rh|`ZIv;DqGozYnuU0LlT05py$6?L9q^4hSj?O@YC-uN3^M6PF`xk)Xhg#$8n-7J?5sPm6E9@rdl#J#MDSeAx4o*Ju&r?Sw_q<$utqu1V$bCX{9>y z;`zki!e~SBr0zKCNp&4#W2!kvH5c+0FQNrnrC&d7DCq=NO4W6 z<}d66kfNC%sOFpYk04}bQ=18!8t)-B^jPkivcHZdKOx%wHYkJv&Hd^FRHK>iYv$XU z`8&6H++GA!MQ`&hyAxz`D$&wmHh9dy0@L7ZfhJS_eBf59KH-N+;dSTh?&MRE`I$+do7w%&QA)`^H_B_=#0m(oTr{FTJ7DAEu@AFQ2@WZLL} zTkUP&$tTGZhH`*@SD}$+o`J{gwScr0vqly5YX#HpI_El}nQ40=7jkoLhiTMSQdT|l z3D4oSS~wiiK>#PBoPO{zb>|@(EIE}&)D`0a9AfaKmR<`j6%(X!@iGnSjhpd10%{Pu(&+c)M_hLv%D5&C}_fC%x?XV z8$uP>r>3|eZWy9kPcB297)A*4pQ9LaO(Cjn{QMRnI|j;Q9#YMo3Ag!%X8y|K{O%j% z@i#a?Gvyqe4{4gYe~@eF*@ec#ShXmHg}mPC7|u7Z32im=UHe-XxN*dC!`{VDXjq9| z^~rjwOYN`ZEzhY|_g>X}U9;LJ?DZGX+4?>q9m#Xf(N}q=o@ve8F{W8hRil2A4nPn= zq~jeN3PR>SuHYcrYS#W15^n8QsmKI^wF4W?CDrU6SFMgIXqr~7o(Yfnx?N4~GM}ey zck~;i%&Is|3(9H=s?NwuG>XTyrHq-n)g(>AWVG{p_?6F`E%@kL< zV-G2|a*9dm*ZHVOsmFZFIjRc13DS!dVf|%R_5D)y#w)8HhUy7NoB6gq4~kvq=r-{N zjuSMoAS4~^B+eqHGNQ9F%vtfnI9JHKQP6Uw!tHZAhK3QY zqB%)beFfa)f6>@pqbW}{&+_`lKNql#Vsbu4YCkLBp9{ELz~%7PdxVnKgb!z+rHcuaR4R3|P!>!Tl1?L%2gzNOw$m z%wwvVn5Ji=jukIElM0F5h<+aPMZR1cP=Qc!j?#_Db)wysR!4SJGoz;{9%vW1<2Y6d z*5Lvwr&Q}YD)(tt`Egp@R!6mc0n)HTQB>aqCAe)G9gg3~D4U$=&g|#tcU5`r=ioOU zW8ch4JPZ;N-8qT-X3Ai&W}e5LxV|uy4holkz&%)1NiMxeKHCK$@c2Iq__%;iftx%> za7e&O0Y4|;zasb)!D|H^6!6ys{0{`bNAPC=^YzB(<8vGM{i*RjDEC;`L+b_1-~l=j zokT-s(`0z8dl2!n0K|>!0suOv#md;+b1>tt?EK<+z&xv2w~wS9kJf3{_R4nGJ0s~i zM)yd%Jh4EvwpT;i^N?mAgN)>q(V|+nyh7$$*TR6ILIeI0ZLrkQCCUxJ-P|5Ro%_dR z!A!O5+)TM@#cOEw51mh^+su>MTk}}D?#qyyuA{e3>AJ7*{Hr{Fg6Dg9{xzQeJAG>Af0O4l4e7dX@q91OpXT{9JpVVIPw<>pZ@TVTp8o(jq9b)n1qk8$(7g4=LwG*|GsBWU@=F{gVYBf;dQpc6ZKC- zEd&Lv*|e+`XZY<|mJ?J*DN{djPkbO8GFBymLA=8Xdt+Mz>wIk?zY%Znvxsk1IO_jG zT)C%Naj{5X8)rL>n6k22iO2lykzN*#dVBGTN$EwOP&DF=4JmbTWiIo_0$!sKxK)V; z6@ye5N<8#%K&eY8z9A!!Z5|-SLI@*y!YXBPv$C#R)7avT8@k%`P`o`Bi^l4y!3IcK zk~N_&-q6qh)xJQ)-^XB|=-+^MI9VCotUxbWulV?&$tYn>gDa?XtbzdQN&^@D;|-!= zJS&8|ObJB>5(aW3$_FAm1!4W(!OMo%8#l;pnUR3Ks)F3$3yr7xtPp=!)bNJM1?!@1 z5?IFk(f)y0ARZ^zdkrNV_+lVT_H7FpeNos;BIG>Xt=x|(iZ<6ZEDtJ5eP%Y6(ClSp zle(N#crGDC=O*v>lejey3B*K1KvA|&C=kC@ima3VCl?0#qX?wAO!0Y1qjzgSxtmyo zRxlI}M7;e0<)Pc!a&m@x0N>WeV!o^gMC`I+Un8jbyvLPh9>fI`kp-FGZwzK#T^M;H zI7IA;cxog6w?@O^0Cz6yrSawBMX(Q1+&;VN> zV`DA!X)XUWUiabaZX5&B`&1UInb)FkZ1c2s=|;>OimZ-B`^Cu#@R&j%9mi$Pn^r{= zI1L0+UyQW28iNEZD+Y9al+;RRCxka;aXLB-WV*E!RcD&3h4uUhhQxGc>H~2j)DJ-% zq0<^dAIzY^SRjy9y(PakE2%G2D~}wh{{VNNHy#hf3_WH3Uoz=ZSN!XCz;Z53ND+GcKp2n)fE2zLQXci=)vBu~A_^ZCId-YVjsD&l`o#7`ITzbN9*6!CN#m#?2*L+0~}V9578(OWkB zW%(6Fyt{~3i}>|L{DVdO<{~~=#7BzwM~e8-A|7vE89qW_IC|v&5r8i&SQRdMcZDgT zZxa|ofnhq?$8gbJL~~C6A%KwJ|J6Xpn>g-cbS6qKWES9Bhzp5-2KC35ui$?>E?*}) z4xMGF>*8I4e%WHd%EU_1^H@a-_aMr>cA=JG92 z)aN$89G~AWv*F=Rk+;?5-8VA(o!sd3pxohfNXFJiMF6?j%rd)J;r0cN?^~DMeWbI+ Wy#|t^{i|{0&vJJf{-%Wo#s33;RQ9z1 diff --git a/contrib/largeNbDicts/largeNbDicts.c b/contrib/largeNbDicts/largeNbDicts.c index 536e45ff..16876601 100644 --- a/contrib/largeNbDicts/largeNbDicts.c +++ b/contrib/largeNbDicts/largeNbDicts.c @@ -12,7 +12,7 @@ * This is a benchmark test tool * dedicated to the specific case of dictionary decompression * using a very large nb of dictionaries - * thus generating many cache-misses. + * thus suffering latency from lots of cache misses. * It's created in a bid to investigate performance and find optimizations. */ @@ -24,6 +24,7 @@ #include /* assert */ #include "util.h" +#include "bench.h" #define ZSTD_STATIC_LINKING_ONLY #include "zstd.h" #include "zdict.h" @@ -158,6 +159,17 @@ buffer_collection_t splitBuffer(buffer_t srcBuffer, size_t blockSize) return result; } +/* shrinkSizes() : + * update sizes in buffer collection */ +void shrinkSizes(buffer_collection_t collection, + const size_t* sizes) /* presumed same size as collection */ +{ + size_t const nbBlocks = collection.nbBuffers; + for (size_t blockNb = 0; blockNb < nbBlocks; blockNb++) { + assert(sizes[blockNb] <= collection.capacities[blockNb]); + collection.capacities[blockNb] = sizes[blockNb]; + } +} /*--- dictionary creation ---*/ @@ -221,13 +233,30 @@ static ddict_collection_t createDDictCollection(const void* dictBuffer, size_t d } +/* mess with adresses, so that linear scanning dictionaries != linear address scanning */ +void shuffleDictionaries(ddict_collection_t dicts) +{ + size_t const nbDicts = dicts.nbDDict; + for (size_t r=0; rdctx, + dst, dstCapacity, + src, srcSize, + di->dictionaries.ddicts[di->blockNb]); + + di->blockNb = di->blockNb + 1; + if (di->blockNb >= di->nbBlocks) di->blockNb = 0; + + return result; +} + + +#define BENCH_TIME_DEFAULT_MS 6000 +#define RUN_TIME_DEFAULT_MS 1000 + +static int benchMem(buffer_collection_t dstBlocks, + buffer_collection_t srcBlocks, + ddict_collection_t dictionaries) +{ + assert(dstBlocks.nbBuffers == srcBlocks.nbBuffers); + assert(dstBlocks.nbBuffers == dictionaries.nbDDict); + + double bestSpeed = 0.; + + BMK_timedFnState_t* const benchState = + BMK_createTimedFnState(BENCH_TIME_DEFAULT_MS, RUN_TIME_DEFAULT_MS); + decompressInstructions di = createDecompressInstructions(dictionaries); + + for (;;) { + BMK_runOutcome_t const outcome = BMK_benchTimedFn(benchState, + decompress, &di, + NULL, NULL, + dstBlocks.nbBuffers, + (const void* const *)srcBlocks.buffers, srcBlocks.capacities, + dstBlocks.buffers, dstBlocks.capacities, + NULL); + + assert(BMK_isSuccessful_runOutcome(outcome)); + BMK_runTime_t const result = BMK_extract_runTime(outcome); + U64 const dTime_ns = result.nanoSecPerRun; + double const dTime_sec = (double)dTime_ns / 1000000000; + size_t const srcSize = result.sumOfReturn; + double const dSpeed_MBps = (double)srcSize / dTime_sec / (1 MB); + if (dSpeed_MBps > bestSpeed) bestSpeed = dSpeed_MBps; + DISPLAY("Decompression Speed : %.1f MB/s \r", bestSpeed); + if (BMK_isCompleted_TimedFn(benchState)) break; + } + DISPLAY("\n"); + + freeDecompressInstructions(di); + BMK_freeTimedFnState(benchState); + + return 0; /* success */ +} + /* bench() : * fileName : file to load for benchmarking purpose @@ -272,8 +387,9 @@ int bench(const char* fileName, const char* dictionary) DISPLAYLEVEL(3, "loading %s... \n", fileName); buffer_t const srcBuffer = createBuffer_fromFile(fileName); assert(srcBuffer.ptr != NULL); + size_t const srcSize = srcBuffer.size; DISPLAYLEVEL(3, "created src buffer of size %.1f MB \n", - (double)(srcBuffer.size) / (1 MB)); + (double)srcSize / (1 MB)); buffer_collection_t const srcBlockBuffers = splitBuffer(srcBuffer, BLOCKSIZE); assert(srcBlockBuffers.buffers != NULL); @@ -302,17 +418,20 @@ int bench(const char* fileName, const char* dictionary) ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer.ptr, dictBuffer.size, COMP_LEVEL); assert(cdict != NULL); - size_t const cTotalSizeNoDict = compressBlocks(dstBlockBuffers, srcBlockBuffers, NULL, COMP_LEVEL); + size_t const cTotalSizeNoDict = compressBlocks(NULL, dstBlockBuffers, srcBlockBuffers, NULL, COMP_LEVEL); assert(cTotalSizeNoDict != 0); DISPLAYLEVEL(3, "compressing at level %u without dictionary : Ratio=%.2f (%u bytes) \n", COMP_LEVEL, - (double)srcBuffer.size / cTotalSizeNoDict, (unsigned)cTotalSizeNoDict); + (double)srcSize / cTotalSizeNoDict, (unsigned)cTotalSizeNoDict); - size_t const cTotalSize = compressBlocks(dstBlockBuffers, srcBlockBuffers, cdict, COMP_LEVEL); + size_t* const cSizes = malloc(nbBlocks * sizeof(size_t)); + assert(cSizes != NULL); + + size_t const cTotalSize = compressBlocks(cSizes, dstBlockBuffers, srcBlockBuffers, cdict, COMP_LEVEL); assert(cTotalSize != 0); DISPLAYLEVEL(3, "compressed using a %u bytes dictionary : Ratio=%.2f (%u bytes) \n", (unsigned)dictBuffer.size, - (double)srcBuffer.size / cTotalSize, (unsigned)cTotalSize); + (double)srcSize / cTotalSize, (unsigned)cTotalSize); size_t const dictMem = ZSTD_estimateDDictSize(dictBuffer.size, ZSTD_dlm_byCopy); size_t const allDictMem = dictMem * nbBlocks; @@ -322,13 +441,28 @@ int bench(const char* fileName, const char* dictionary) ddict_collection_t const dictionaries = createDDictCollection(dictBuffer.ptr, dictBuffer.size, nbBlocks); assert(dictionaries.ddicts != NULL); + shuffleDictionaries(dictionaries); + // for (size_t u = 0; u < dictionaries.nbDDict; u++) DISPLAY("dict address : %p \n", dictionaries.ddicts[u]); /* check dictionary addresses */ + void* const resultPtr = malloc(srcSize); + assert(resultPtr != NULL); + buffer_t resultBuffer; + resultBuffer.ptr = resultPtr; + resultBuffer.capacity = srcSize; + resultBuffer.size = srcSize; - //result = benchMem(srcBlockBuffers, dstBlockBuffers, dictionaries);; + buffer_collection_t const resultBlockBuffers = splitBuffer(resultBuffer, BLOCKSIZE); + assert(resultBlockBuffers.buffers != NULL); + shrinkSizes(dstBlockBuffers, cSizes); + result = benchMem(resultBlockBuffers, dstBlockBuffers, dictionaries); + /* free all heap objects in reverse order */ + freeCollection(resultBlockBuffers); + free(resultPtr); freeDDictCollection(dictionaries); + free(cSizes); ZSTD_freeCDict(cdict); freeBuffer(dictBuffer); freeCollection(dstBlockBuffers); @@ -342,7 +476,7 @@ int bench(const char* fileName, const char* dictionary) -/*--- Command Line ---*/ +/* --- Command Line --- */ int bad_usage(const char* exeName) { diff --git a/programs/bench.c b/programs/bench.c index b3a8222d..5ff9afac 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -253,7 +253,7 @@ static size_t local_defaultCompress( /* `addArgs` is the context */ static size_t local_defaultDecompress( const void* srcBuffer, size_t srcSize, - void* dstBuffer, size_t dstSize, + void* dstBuffer, size_t dstCapacity, void* addArgs) { size_t moreToFlush = 1; @@ -261,7 +261,7 @@ static size_t local_defaultDecompress( ZSTD_inBuffer in; ZSTD_outBuffer out; in.src = srcBuffer; in.size = srcSize; in.pos = 0; - out.dst = dstBuffer; out.size = dstSize; out.pos = 0; + out.dst = dstBuffer; out.size = dstCapacity; out.pos = 0; while (moreToFlush) { if(out.pos == out.size) { return (size_t)-ZSTD_error_dstSize_tooSmall; From 6c398df24118427bb802bb9b504a77e629c63831 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 28 Aug 2018 18:05:31 -0700 Subject: [PATCH 04/15] level, block size and nb dicts can be set on command line --- contrib/largeNbDicts/largeNbDicts.c | 111 +++++++++++++++++++++------- 1 file changed, 83 insertions(+), 28 deletions(-) diff --git a/contrib/largeNbDicts/largeNbDicts.c b/contrib/largeNbDicts/largeNbDicts.c index 16876601..4c05e24e 100644 --- a/contrib/largeNbDicts/largeNbDicts.c +++ b/contrib/largeNbDicts/largeNbDicts.c @@ -35,9 +35,9 @@ #define KB *(1<<10) #define MB *(1<<20) -#define BLOCKSIZE (4 KB) +#define BLOCKSIZE_DEFAULT (4 KB) #define DICTSIZE (4 KB) -#define COMP_LEVEL 3 +#define CLEVEL_DEFAULT 3 #define DISPLAY_LEVEL_DEFAULT 3 @@ -293,8 +293,8 @@ typedef size_t (*BMK_initFn_t)(void* initPayload); typedef struct { ZSTD_DCtx* dctx; - size_t nbBlocks; - size_t blockNb; + size_t nbDicts; + size_t dictNb; ddict_collection_t dictionaries; } decompressInstructions; @@ -303,8 +303,8 @@ decompressInstructions createDecompressInstructions(ddict_collection_t dictionar decompressInstructions di; di.dctx = ZSTD_createDCtx(); assert(di.dctx != NULL); - di.nbBlocks = dictionaries.nbDDict; - di.blockNb = 0; + di.nbDicts = dictionaries.nbDDict; + di.dictNb = 0; di.dictionaries = dictionaries; return di; } @@ -322,10 +322,10 @@ size_t decompress(const void* src, size_t srcSize, void* dst, size_t dstCapacity size_t const result = ZSTD_decompress_usingDDict(di->dctx, dst, dstCapacity, src, srcSize, - di->dictionaries.ddicts[di->blockNb]); + di->dictionaries.ddicts[di->dictNb]); - di->blockNb = di->blockNb + 1; - if (di->blockNb >= di->nbBlocks) di->blockNb = 0; + di->dictNb = di->dictNb + 1; + if (di->dictNb >= di->nbDicts) di->dictNb = 0; return result; } @@ -339,7 +339,6 @@ static int benchMem(buffer_collection_t dstBlocks, ddict_collection_t dictionaries) { assert(dstBlocks.nbBuffers == srcBlocks.nbBuffers); - assert(dstBlocks.nbBuffers == dictionaries.nbDDict); double bestSpeed = 0.; @@ -364,6 +363,7 @@ static int benchMem(buffer_collection_t dstBlocks, double const dSpeed_MBps = (double)srcSize / dTime_sec / (1 MB); if (dSpeed_MBps > bestSpeed) bestSpeed = dSpeed_MBps; DISPLAY("Decompression Speed : %.1f MB/s \r", bestSpeed); + fflush(stdout); if (BMK_isCompleted_TimedFn(benchState)) break; } DISPLAY("\n"); @@ -380,7 +380,8 @@ static int benchMem(buffer_collection_t dstBlocks, * dictionary : optional (can be NULL), file to load as dictionary, * if none provided : will be calculated on the fly by the program. * @return : 0 is success, 1+ otherwise */ -int bench(const char* fileName, const char* dictionary) +int bench(const char* fileName, const char* dictionary, + size_t blockSize, int clevel, unsigned nbDictMax) { int result = 0; @@ -391,13 +392,13 @@ int bench(const char* fileName, const char* dictionary) DISPLAYLEVEL(3, "created src buffer of size %.1f MB \n", (double)srcSize / (1 MB)); - buffer_collection_t const srcBlockBuffers = splitBuffer(srcBuffer, BLOCKSIZE); + buffer_collection_t const srcBlockBuffers = splitBuffer(srcBuffer, blockSize); assert(srcBlockBuffers.buffers != NULL); unsigned const nbBlocks = (unsigned)srcBlockBuffers.nbBuffers; DISPLAYLEVEL(3, "split input into %u blocks of max size %u bytes \n", - nbBlocks, BLOCKSIZE); + nbBlocks, (unsigned)blockSize); - size_t const dstBlockSize = ZSTD_compressBound(BLOCKSIZE); + size_t const dstBlockSize = ZSTD_compressBound(blockSize); size_t const dstBufferCapacity = nbBlocks * dstBlockSize; void* const dstPtr = malloc(dstBufferCapacity); assert(dstPtr != NULL); @@ -415,30 +416,31 @@ int bench(const char* fileName, const char* dictionary) srcBlockBuffers.capacities, nbBlocks); assert(dictBuffer.ptr != NULL); - ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer.ptr, dictBuffer.size, COMP_LEVEL); + ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer.ptr, dictBuffer.size, clevel); assert(cdict != NULL); - size_t const cTotalSizeNoDict = compressBlocks(NULL, dstBlockBuffers, srcBlockBuffers, NULL, COMP_LEVEL); + size_t const cTotalSizeNoDict = compressBlocks(NULL, dstBlockBuffers, srcBlockBuffers, NULL, clevel); assert(cTotalSizeNoDict != 0); DISPLAYLEVEL(3, "compressing at level %u without dictionary : Ratio=%.2f (%u bytes) \n", - COMP_LEVEL, + clevel, (double)srcSize / cTotalSizeNoDict, (unsigned)cTotalSizeNoDict); size_t* const cSizes = malloc(nbBlocks * sizeof(size_t)); assert(cSizes != NULL); - size_t const cTotalSize = compressBlocks(cSizes, dstBlockBuffers, srcBlockBuffers, cdict, COMP_LEVEL); + size_t const cTotalSize = compressBlocks(cSizes, dstBlockBuffers, srcBlockBuffers, cdict, clevel); assert(cTotalSize != 0); DISPLAYLEVEL(3, "compressed using a %u bytes dictionary : Ratio=%.2f (%u bytes) \n", (unsigned)dictBuffer.size, (double)srcSize / cTotalSize, (unsigned)cTotalSize); size_t const dictMem = ZSTD_estimateDDictSize(dictBuffer.size, ZSTD_dlm_byCopy); - size_t const allDictMem = dictMem * nbBlocks; + unsigned const nbDicts = nbDictMax ? nbDictMax : nbBlocks; + size_t const allDictMem = dictMem * nbDicts; DISPLAYLEVEL(3, "generating %u dictionaries, using %.1f MB of memory \n", - nbBlocks, (double)allDictMem / (1 MB)); + nbDicts, (double)allDictMem / (1 MB)); - ddict_collection_t const dictionaries = createDDictCollection(dictBuffer.ptr, dictBuffer.size, nbBlocks); + ddict_collection_t const dictionaries = createDDictCollection(dictBuffer.ptr, dictBuffer.size, nbDicts); assert(dictionaries.ddicts != NULL); shuffleDictionaries(dictionaries); @@ -451,7 +453,7 @@ int bench(const char* fileName, const char* dictionary) resultBuffer.capacity = srcSize; resultBuffer.size = srcSize; - buffer_collection_t const resultBlockBuffers = splitBuffer(resultBuffer, BLOCKSIZE); + buffer_collection_t const resultBlockBuffers = splitBuffer(resultBuffer, blockSize); assert(resultBlockBuffers.buffers != NULL); shrinkSizes(dstBlockBuffers, cSizes); @@ -478,26 +480,79 @@ int bench(const char* fileName, const char* dictionary) /* --- Command Line --- */ +/*! readU32FromChar() : + * @return : unsigned integer value read from input in `char` format. + * allows and interprets K, KB, KiB, M, MB and MiB suffix. + * Will also modify `*stringPtr`, advancing it to position where it stopped reading. + * Note : function will exit() program if digit sequence overflows */ +static unsigned readU32FromChar(const char** stringPtr) +{ + unsigned result = 0; + while ((**stringPtr >='0') && (**stringPtr <='9')) { + unsigned const max = (((unsigned)(-1)) / 10) - 1; + assert(result <= max); /* check overflow */ + result *= 10, result += **stringPtr - '0', (*stringPtr)++ ; + } + if ((**stringPtr=='K') || (**stringPtr=='M')) { + unsigned const maxK = ((unsigned)(-1)) >> 10; + assert(result <= maxK); /* check overflow */ + result <<= 10; + if (**stringPtr=='M') { + assert(result <= maxK); /* check overflow */ + result <<= 10; + } + (*stringPtr)++; /* skip `K` or `M` */ + if (**stringPtr=='i') (*stringPtr)++; + if (**stringPtr=='B') (*stringPtr)++; + } + return result; +} + +/** longCommandWArg() : + * check if *stringPtr is the same as longCommand. + * If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand. + * @return 0 and doesn't modify *stringPtr otherwise. + */ +static unsigned longCommandWArg(const char** stringPtr, const char* longCommand) +{ + size_t const comSize = strlen(longCommand); + int const result = !strncmp(*stringPtr, longCommand, comSize); + if (result) *stringPtr += comSize; + return result; +} + + int bad_usage(const char* exeName) { DISPLAY (" bad usage : \n"); - DISPLAY (" %s filename [-D dictionary] \n", exeName); + DISPLAY (" %s filename [Options] \n", exeName); + DISPLAY ("Options : \n"); + DISPLAY ("--clevel=# : use compression level # (default: %u) \n", CLEVEL_DEFAULT); + DISPLAY ("--blockSize=# : cut input into blocks of size # (default: %u) \n", BLOCKSIZE_DEFAULT); + DISPLAY ("--dictionary=# : use # as a dictionary (default: create one) \n"); + DISPLAY ("--nbDicts=# : set nb of dictionaries to # (default: one per block) \n"); return 1; } int main (int argc, const char** argv) { const char* const exeName = argv[0]; + int cLevel = CLEVEL_DEFAULT; + size_t blockSize = BLOCKSIZE_DEFAULT; + size_t nbDicts = 0; if (argc < 2) return bad_usage(exeName); const char* const fileName = argv[1]; const char* dictionary = NULL; - if (argc > 2) { - if (argc != 4) return bad_usage(exeName); - if (strcmp(argv[2], "-D")) return bad_usage(exeName); - dictionary = argv[3]; + for (int argNb = 2; argNb < argc ; argNb++) { + const char* argument = argv[argNb]; + if (longCommandWArg(&argument, "--clevel=")) { cLevel = readU32FromChar(&argument); continue; } + if (longCommandWArg(&argument, "--blockSize=")) { blockSize = readU32FromChar(&argument); continue; } + if (longCommandWArg(&argument, "--dictionary=")) { dictionary = argument; continue; } + if (longCommandWArg(&argument, "--nbDicts=")) { nbDicts = readU32FromChar(&argument); continue; } + return bad_usage(exeName); } - return bench(fileName, dictionary); + return bench(fileName, dictionary, blockSize, cLevel, nbDicts); } From 6444c50035c466580ecbd90b5596288204a7d363 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 28 Aug 2018 18:13:46 -0700 Subject: [PATCH 05/15] increases randomness of ddict ptrs --- contrib/largeNbDicts/largeNbDicts.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/contrib/largeNbDicts/largeNbDicts.c b/contrib/largeNbDicts/largeNbDicts.c index 4c05e24e..19336249 100644 --- a/contrib/largeNbDicts/largeNbDicts.c +++ b/contrib/largeNbDicts/largeNbDicts.c @@ -237,6 +237,12 @@ static ddict_collection_t createDDictCollection(const void* dictBuffer, size_t d void shuffleDictionaries(ddict_collection_t dicts) { size_t const nbDicts = dicts.nbDDict; + for (size_t r=0; r 0); + size_t* const fileSizes = (size_t*)calloc(nbFiles, sizeof(*fileSizes)); + assert(fileSizes != NULL); + + /* Load input buffer */ + int const errorCode = loadFiles(srcBuffer, loadedSize, + fileSizes, + fileNamesTable, nbFiles); + assert(errorCode == 0); + + void** sliceTable = (void**)malloc(nbFiles * sizeof(*sliceTable)); + assert(sliceTable != NULL); + + char* const ptr = (char*)srcBuffer; + size_t pos = 0; + unsigned fileNb = 0; + for ( ; (pos < loadedSize) && (fileNb < nbFiles); fileNb++) { + sliceTable[fileNb] = ptr + pos; + pos += fileSizes[fileNb]; + } + assert(pos == loadedSize); + assert(fileNb == nbFiles); + + + buffer_t buffer; + buffer.ptr = srcBuffer; + buffer.capacity = loadedSize; + buffer.size = loadedSize; + + slice_collection_t slices; + slices.slicePtrs = sliceTable; + slices.capacities = fileSizes; + slices.nbSlices = nbFiles; + + buffer_collection_t bc; + bc.buffer = buffer; + bc.slices = slices; + return bc; +} + + + + /*--- ddict_collection_t ---*/ typedef struct { @@ -260,12 +438,12 @@ void shuffleDictionaries(ddict_collection_t dicts) * or 0 if error. */ static size_t compressBlocks(size_t* cSizes, /* optional (can be NULL). If present, must contain at least nbBlocks fields */ - buffer_collection_t dstBlockBuffers, - buffer_collection_t srcBlockBuffers, + slice_collection_t dstBlockBuffers, + slice_collection_t srcBlockBuffers, ZSTD_CDict* cdict, int cLevel) { - size_t const nbBlocks = srcBlockBuffers.nbBuffers; - assert(dstBlockBuffers.nbBuffers == srcBlockBuffers.nbBuffers); + size_t const nbBlocks = srcBlockBuffers.nbSlices; + assert(dstBlockBuffers.nbSlices == srcBlockBuffers.nbSlices); ZSTD_CCtx* const cctx = ZSTD_createCCtx(); assert(cctx != NULL); @@ -275,16 +453,16 @@ static size_t compressBlocks(size_t* cSizes, /* optional (can be NULL). If pre size_t cBlockSize; if (cdict == NULL) { cBlockSize = ZSTD_compressCCtx(cctx, - dstBlockBuffers.buffers[blockNb], dstBlockBuffers.capacities[blockNb], - srcBlockBuffers.buffers[blockNb], srcBlockBuffers.capacities[blockNb], + dstBlockBuffers.slicePtrs[blockNb], dstBlockBuffers.capacities[blockNb], + srcBlockBuffers.slicePtrs[blockNb], srcBlockBuffers.capacities[blockNb], cLevel); } else { cBlockSize = ZSTD_compress_usingCDict(cctx, - dstBlockBuffers.buffers[blockNb], dstBlockBuffers.capacities[blockNb], - srcBlockBuffers.buffers[blockNb], srcBlockBuffers.capacities[blockNb], + dstBlockBuffers.slicePtrs[blockNb], dstBlockBuffers.capacities[blockNb], + srcBlockBuffers.slicePtrs[blockNb], srcBlockBuffers.capacities[blockNb], cdict); } - assert(!ZSTD_isError(cBlockSize)); + CONTROL(!ZSTD_isError(cBlockSize)); if (cSizes) cSizes[blockNb] = cBlockSize; totalCSize += cBlockSize; } @@ -337,31 +515,32 @@ size_t decompress(const void* src, size_t srcSize, void* dst, size_t dstCapacity } -#define BENCH_TIME_DEFAULT_MS 6000 -#define RUN_TIME_DEFAULT_MS 1000 - -static int benchMem(buffer_collection_t dstBlocks, - buffer_collection_t srcBlocks, - ddict_collection_t dictionaries) +static int benchMem(slice_collection_t dstBlocks, + slice_collection_t srcBlocks, + ddict_collection_t dictionaries, + int nbRounds) { - assert(dstBlocks.nbBuffers == srcBlocks.nbBuffers); + assert(dstBlocks.nbSlices == srcBlocks.nbSlices); + + unsigned const ms_per_round = RUN_TIME_DEFAULT_MS; + unsigned const total_time_ms = nbRounds * ms_per_round; double bestSpeed = 0.; BMK_timedFnState_t* const benchState = - BMK_createTimedFnState(BENCH_TIME_DEFAULT_MS, RUN_TIME_DEFAULT_MS); + BMK_createTimedFnState(total_time_ms, ms_per_round); decompressInstructions di = createDecompressInstructions(dictionaries); for (;;) { BMK_runOutcome_t const outcome = BMK_benchTimedFn(benchState, decompress, &di, NULL, NULL, - dstBlocks.nbBuffers, - (const void* const *)srcBlocks.buffers, srcBlocks.capacities, - dstBlocks.buffers, dstBlocks.capacities, + dstBlocks.nbSlices, + (const void* const *)srcBlocks.slicePtrs, srcBlocks.capacities, + dstBlocks.slicePtrs, dstBlocks.capacities, NULL); + CONTROL(BMK_isSuccessful_runOutcome(outcome)); - assert(BMK_isSuccessful_runOutcome(outcome)); BMK_runTime_t const result = BMK_extract_runTime(outcome); U64 const dTime_ns = result.nanoSecPerRun; double const dTime_sec = (double)dTime_ns / 1000000000; @@ -381,65 +560,87 @@ static int benchMem(buffer_collection_t dstBlocks, } -/* bench() : - * fileName : file to load for benchmarking purpose - * dictionary : optional (can be NULL), file to load as dictionary, +/*! bench() : + * fileName : file to load for benchmarking purpose + * dictionary : optional (can be NULL), file to load as dictionary, * if none provided : will be calculated on the fly by the program. * @return : 0 is success, 1+ otherwise */ -int bench(const char* fileName, const char* dictionary, - size_t blockSize, int clevel, unsigned nbDictMax) +int bench(const char** fileNameTable, unsigned nbFiles, + const char* dictionary, + size_t blockSize, int clevel, unsigned nbDictMax, int nbRounds) { int result = 0; - DISPLAYLEVEL(3, "loading %s... \n", fileName); - buffer_t const srcBuffer = createBuffer_fromFile(fileName); - assert(srcBuffer.ptr != NULL); + DISPLAYLEVEL(3, "loading %u files... \n", nbFiles); + buffer_collection_t const srcs = createBufferCollection_fromFiles(fileNameTable, nbFiles); + CONTROL(srcs.buffer.ptr != NULL); + buffer_t srcBuffer = srcs.buffer; size_t const srcSize = srcBuffer.size; DISPLAYLEVEL(3, "created src buffer of size %.1f MB \n", (double)srcSize / (1 MB)); - buffer_collection_t const srcBlockBuffers = splitBuffer(srcBuffer, blockSize); - assert(srcBlockBuffers.buffers != NULL); - unsigned const nbBlocks = (unsigned)srcBlockBuffers.nbBuffers; - DISPLAYLEVEL(3, "split input into %u blocks of max size %u bytes \n", - nbBlocks, (unsigned)blockSize); + slice_collection_t const srcSlices = splitSlices(srcs.slices, blockSize); + unsigned const nbBlocks = (unsigned)(srcSlices.nbSlices); + DISPLAYLEVEL(3, "split input into %u blocks ", nbBlocks); + if (blockSize) + DISPLAYLEVEL(3, "of max size %u bytes ", (unsigned)blockSize); + DISPLAYLEVEL(3, "\n"); - size_t const dstBlockSize = ZSTD_compressBound(blockSize); - size_t const dstBufferCapacity = nbBlocks * dstBlockSize; - void* const dstPtr = malloc(dstBufferCapacity); - assert(dstPtr != NULL); - buffer_t dstBuffer; - dstBuffer.ptr = dstPtr; - dstBuffer.capacity = dstBufferCapacity; - dstBuffer.size = dstBufferCapacity; - buffer_collection_t const dstBlockBuffers = splitBuffer(dstBuffer, dstBlockSize); - assert(dstBlockBuffers.buffers != NULL); + size_t* const dstCapacities = malloc(nbBlocks * sizeof(*dstCapacities)); + CONTROL(dstCapacities != NULL); + size_t dstBufferCapacity = 0; + for (size_t bnb=0; bnb bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n; /* buffer too small - stop after this file */ - { size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f); - if (readSize != (size_t)fileSize) EXM_THROW_INT(11, "could not read %s", fileNamesTable[n]); - pos += readSize; } + { size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f); + if (readSize != (size_t)fileSize) EXM_THROW_INT(11, "could not read %s", fileNamesTable[n]); + pos += readSize; + } fileSizes[n] = (size_t)fileSize; totalSize += (size_t)fileSize; fclose(f); diff --git a/programs/util.h b/programs/util.h index 4392a5bd..76000d99 100644 --- a/programs/util.h +++ b/programs/util.h @@ -526,7 +526,10 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ * After finishing usage of the list the structures should be freed with UTIL_freeFileList(params: return value, allocatedBuffer) * In case of error UTIL_createFileList returns NULL and UTIL_freeFileList should not be called. */ -UTIL_STATIC const char** UTIL_createFileList(const char **inputNames, unsigned inputNamesNb, char** allocatedBuffer, unsigned* allocatedNamesNb, int followLinks) +UTIL_STATIC const char** +UTIL_createFileList(const char **inputNames, unsigned inputNamesNb, + char** allocatedBuffer, unsigned* allocatedNamesNb, + int followLinks) { size_t pos; unsigned i, nbFiles; From 39ef91a599c4b68a724e37e55c49db930ff3305a Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 30 Aug 2018 14:59:10 -0700 Subject: [PATCH 09/15] -std=c99 for largeNbDicts --- Makefile | 1 + contrib/largeNbDicts/Makefile | 1 + 2 files changed, 2 insertions(+) diff --git a/Makefile b/Makefile index 45ab20fd..03a26cd7 100644 --- a/Makefile +++ b/Makefile @@ -104,6 +104,7 @@ clean: @$(MAKE) -C contrib/pzstd $@ > $(VOID) @$(MAKE) -C contrib/seekable_format/examples $@ > $(VOID) @$(MAKE) -C contrib/adaptive-compression $@ > $(VOID) + @$(MAKE) -C contrib/largeNbDicts $@ > $(VOID) @$(RM) zstd$(EXT) zstdmt$(EXT) tmp* @$(RM) -r lz4 @echo Cleaning completed diff --git a/contrib/largeNbDicts/Makefile b/contrib/largeNbDicts/Makefile index cf529399..0514e3d8 100644 --- a/contrib/largeNbDicts/Makefile +++ b/contrib/largeNbDicts/Makefile @@ -15,6 +15,7 @@ LIBZSTD = $(LIBDIR)/libzstd.a CPPFLAGS+= -I$(LIBDIR) -I$(LIBDIR)/common -I$(LIBDIR)/dictBuilder -I$(PROGDIR) CFLAGS ?= -O3 +CFLAGS += -std=c99 DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ -Wstrict-aliasing=1 -Wswitch-enum \ -Wstrict-prototypes -Wundef -Wpointer-arith -Wformat-security \ From 39c55a118f778c6120cdfca40689a7411ce9e6a4 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 30 Aug 2018 15:54:14 -0700 Subject: [PATCH 10/15] fixed minor compatibility issues with older compilers --- contrib/largeNbDicts/Makefile | 2 +- contrib/largeNbDicts/largeNbDicts.c | 5 ++--- programs/util.h | 10 ++++++---- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/contrib/largeNbDicts/Makefile b/contrib/largeNbDicts/Makefile index 0514e3d8..05b54fd3 100644 --- a/contrib/largeNbDicts/Makefile +++ b/contrib/largeNbDicts/Makefile @@ -15,7 +15,7 @@ LIBZSTD = $(LIBDIR)/libzstd.a CPPFLAGS+= -I$(LIBDIR) -I$(LIBDIR)/common -I$(LIBDIR)/dictBuilder -I$(PROGDIR) CFLAGS ?= -O3 -CFLAGS += -std=c99 +CFLAGS += -std=gnu99 DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ -Wstrict-aliasing=1 -Wswitch-enum \ -Wstrict-prototypes -Wundef -Wpointer-arith -Wformat-security \ diff --git a/contrib/largeNbDicts/largeNbDicts.c b/contrib/largeNbDicts/largeNbDicts.c index 5b982bd4..0c5f89a8 100644 --- a/contrib/largeNbDicts/largeNbDicts.c +++ b/contrib/largeNbDicts/largeNbDicts.c @@ -50,6 +50,8 @@ /*--- Macros ---*/ #define CONTROL(c) assert(c) +#undef MIN +#define MIN(a,b) ((a) < (b) ? (a) : (b)) /*--- Display Macros ---*/ @@ -472,9 +474,6 @@ static size_t compressBlocks(size_t* cSizes, /* optional (can be NULL). If pre /* --- Benchmark --- */ -typedef size_t (*BMK_benchFn_t)(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* customPayload); -typedef size_t (*BMK_initFn_t)(void* initPayload); - typedef struct { ZSTD_DCtx* dctx; size_t nbDicts; diff --git a/programs/util.h b/programs/util.h index 76000d99..88d04840 100644 --- a/programs/util.h +++ b/programs/util.h @@ -319,15 +319,17 @@ UTIL_STATIC U32 UTIL_isDirectory(const char* infilename) UTIL_STATIC U32 UTIL_isLink(const char* infilename) { -#if defined(_WIN32) - /* no symlinks on windows */ - (void)infilename; -#else +/* macro guards, as defined in : https://linux.die.net/man/2/lstat */ +#if defined(_BSD_SOURCE) \ + || (defined(_XOPEN_SOURCE) && (_XOPEN_SOURCE >= 500)) \ + || (defined(_XOPEN_SOURCE) && defined(_XOPEN_SOURCE_EXTENDED)) \ + || (defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 200112L)) int r; stat_t statbuf; r = lstat(infilename, &statbuf); if (!r && S_ISLNK(statbuf.st_mode)) return 1; #endif + (void)infilename; return 0; } From f76253bb702bdaa6227e9285111f8c08fb4e9830 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 30 Aug 2018 16:24:44 -0700 Subject: [PATCH 11/15] minor : createDictionaryBuffer() can create dictionaries of different sizes --- contrib/largeNbDicts/largeNbDicts.c | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/contrib/largeNbDicts/largeNbDicts.c b/contrib/largeNbDicts/largeNbDicts.c index 0c5f89a8..70013cdd 100644 --- a/contrib/largeNbDicts/largeNbDicts.c +++ b/contrib/largeNbDicts/largeNbDicts.c @@ -131,25 +131,28 @@ static buffer_t createBuffer_fromFile(const char* fileName) static buffer_t createDictionaryBuffer(const char* dictionaryName, const void* srcBuffer, - const size_t* srcBlockSizes, unsigned nbBlocks) + const size_t* srcBlockSizes, unsigned nbBlocks, + size_t requestedDictSize) { if (dictionaryName) { DISPLAYLEVEL(3, "loading dictionary %s \n", dictionaryName); return createBuffer_fromFile(dictionaryName); - } else { - DISPLAYLEVEL(3, "creating dictionary, of target size %u bytes \n", DICTSIZE); - void* const dictBuffer = malloc(DICTSIZE); - assert(dictBuffer != NULL); - size_t const dictSize = ZDICT_trainFromBuffer(dictBuffer, DICTSIZE, - srcBuffer, - srcBlockSizes, - nbBlocks); - assert(!ZSTD_isError(dictSize)); + } else { + + DISPLAYLEVEL(3, "creating dictionary, of target size %u bytes \n", + (unsigned)requestedDictSize); + void* const dictBuffer = malloc(requestedDictSize); + CONTROL(dictBuffer != NULL); + + size_t const dictSize = ZDICT_trainFromBuffer(dictBuffer, requestedDictSize, + srcBuffer, + srcBlockSizes, nbBlocks); + CONTROL(!ZSTD_isError(dictSize)); buffer_t result; result.ptr = dictBuffer; - result.capacity = DICTSIZE; + result.capacity = requestedDictSize; result.size = dictSize; return result; } @@ -616,7 +619,8 @@ int bench(const char** fileNameTable, unsigned nbFiles, /* dictionary determination */ buffer_t const dictBuffer = createDictionaryBuffer(dictionary, srcBuffer.ptr, - srcSlices.capacities, nbBlocks); + srcSlices.capacities, nbBlocks, + DICTSIZE); CONTROL(dictBuffer.ptr != NULL); ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer.ptr, dictBuffer.size, clevel); From 0ff67511e64fb81880544ffaa4d03af7125e3de2 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 30 Aug 2018 16:43:28 -0700 Subject: [PATCH 12/15] fixed link order for old compilers --- contrib/largeNbDicts/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/largeNbDicts/Makefile b/contrib/largeNbDicts/Makefile index 05b54fd3..624140fa 100644 --- a/contrib/largeNbDicts/Makefile +++ b/contrib/largeNbDicts/Makefile @@ -28,7 +28,7 @@ default: largeNbDicts all : largeNbDicts -largeNbDicts: bench.o datagen.o xxhash.o $(LIBZSTD) largeNbDicts.c +largeNbDicts: bench.o datagen.o xxhash.o largeNbDicts.c $(LIBZSTD) $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ .PHONY: $(LIBZSTD) From 11b8b8c10016902e216878223db8951c3c65441a Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 31 Aug 2018 10:01:06 -0700 Subject: [PATCH 13/15] silenced false-positive scan-build warning --- contrib/largeNbDicts/largeNbDicts.c | 39 +++++++++++++++-------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/contrib/largeNbDicts/largeNbDicts.c b/contrib/largeNbDicts/largeNbDicts.c index 70013cdd..943a644d 100644 --- a/contrib/largeNbDicts/largeNbDicts.c +++ b/contrib/largeNbDicts/largeNbDicts.c @@ -74,6 +74,7 @@ static const buffer_t kBuffNull = { NULL, 0, 0 }; /* @return : kBuffNull if any error */ static buffer_t createBuffer(size_t capacity) { + assert(capacity > 0); void* const ptr = malloc(capacity); if (ptr==NULL) return kBuffNull; @@ -96,18 +97,6 @@ static void fillBuffer_fromHandle(buffer_t* buff, FILE* f) buff->size = readSize; } -/* @return : kBuffNull if any error */ -static buffer_t createBuffer_fromHandle(FILE* f, size_t bufferSize) -{ - buffer_t buff = createBuffer(bufferSize); - if (buff.ptr == NULL) return kBuffNull; - fillBuffer_fromHandle(&buff, f); - if (buff.size != buff.capacity) { - freeBuffer(buff); - return kBuffNull; - } - return buff; -} /* @return : kBuffNull if any error */ static buffer_t createBuffer_fromFile(const char* fileName) @@ -118,16 +107,22 @@ static buffer_t createBuffer_fromFile(const char* fileName) if (fileSize == UTIL_FILESIZE_UNKNOWN) return kBuffNull; assert((U64)bufferSize == fileSize); /* check overflow */ - { buffer_t buff; - FILE* const f = fopen(fileName, "rb"); + { FILE* const f = fopen(fileName, "rb"); if (f == NULL) return kBuffNull; - buff = createBuffer_fromHandle(f, bufferSize); + buffer_t buff = createBuffer(bufferSize); + CONTROL(buff.ptr != NULL); + + fillBuffer_fromHandle(&buff, f); + CONTROL(buff.size == buff.capacity); + fclose(f); /* do nothing specific if fclose() fails */ return buff; } } + +/* @return : kBuffNull if any error */ static buffer_t createDictionaryBuffer(const char* dictionaryName, const void* srcBuffer, @@ -136,7 +131,7 @@ createDictionaryBuffer(const char* dictionaryName, { if (dictionaryName) { DISPLAYLEVEL(3, "loading dictionary %s \n", dictionaryName); - return createBuffer_fromFile(dictionaryName); + return createBuffer_fromFile(dictionaryName); /* note : result might be kBuffNull */ } else { @@ -172,11 +167,11 @@ static int loadFiles(void* buffer, size_t bufferSize, for (unsigned n=0; n 0); void* const srcBuffer = malloc(loadedSize); assert(srcBuffer != NULL); @@ -777,5 +773,10 @@ int main (int argc, const char** argv) filenameTable = UTIL_createFileList(nameTable, nameIdx, &buffer_containing_filenames, &nbFiles, 1 /* follow_links */); } - return bench(filenameTable, nbFiles, dictionary, blockSize, cLevel, nbDicts, nbRounds); + int result = bench(filenameTable, nbFiles, dictionary, blockSize, cLevel, nbDicts, nbRounds); + + free(buffer_containing_filenames); + free(nameTable); + + return result; } From 1d487d587f3a8964e274ecd5062150c24deea98e Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 4 Sep 2018 14:57:45 -0700 Subject: [PATCH 14/15] updated documentation --- contrib/largeNbDicts/README.md | 22 ++++++++++++++-------- contrib/largeNbDicts/largeNbDicts.c | 26 ++++++++++++++++++-------- 2 files changed, 32 insertions(+), 16 deletions(-) diff --git a/contrib/largeNbDicts/README.md b/contrib/largeNbDicts/README.md index 7eba229a..f29bcdfe 100644 --- a/contrib/largeNbDicts/README.md +++ b/contrib/largeNbDicts/README.md @@ -3,17 +3,23 @@ largeNbDicts `largeNbDicts` is a benchmark test tool dedicated to the specific scenario of -dictionary decompression using a very large number of dictionaries, -which suffers from increased latency due to cache misses. -It's created in a bid to investigate performance for this scenario, +dictionary decompression using a very large number of dictionaries. +When dictionaries are constantly changing, they are always "cold", +suffering from increased latency due to cache misses. + +The tool is created in a bid to investigate performance for this scenario, and experiment mitigation techniques. Command line : ``` -$ largeNbDicts filename [Options] +largeNbDicts [Options] filename(s) + Options : ---clevel=# : use compression level # (default: 3) ---blockSize=# : cut input into blocks of size # (default: 4096) ---dictionary=# : use # as a dictionary (default: create one) ---nbDicts=# : set nb of dictionaries to # (default: one per block) +-r : recursively load all files in subdirectories (default: off) +-B# : split input into blocks of size # (default: no split) +-# : use compression level # (default: 3) +-D # : use # as a dictionary (default: create one) +-i# : nb benchmark rounds (default: 6) +--nbDicts=# : set nb of dictionaries to # (default: one per block) +-h : help (this text) ``` diff --git a/contrib/largeNbDicts/largeNbDicts.c b/contrib/largeNbDicts/largeNbDicts.c index 943a644d..21fbd2aa 100644 --- a/contrib/largeNbDicts/largeNbDicts.c +++ b/contrib/largeNbDicts/largeNbDicts.c @@ -716,17 +716,26 @@ static unsigned longCommandWArg(const char** stringPtr, const char* longCommand) } +int usage(const char* exeName) +{ + DISPLAY (" \n"); + DISPLAY (" %s [Options] filename(s) \n", exeName); + DISPLAY (" \n"); + DISPLAY ("Options : \n"); + DISPLAY ("-r : recursively load all files in subdirectories (default: off) \n"); + DISPLAY ("-B# : split input into blocks of size # (default: no split) \n"); + DISPLAY ("-# : use compression level # (default: %u) \n", CLEVEL_DEFAULT); + DISPLAY ("-D # : use # as a dictionary (default: create one) \n"); + DISPLAY ("-i# : nb benchmark rounds (default: %u) \n", BENCH_TIME_DEFAULT_S); + DISPLAY ("--nbDicts=# : create # dictionaries for bench (default: one per block) \n"); + DISPLAY ("-h : help (this text) \n"); + return 0; +} + int bad_usage(const char* exeName) { DISPLAY (" bad usage : \n"); - DISPLAY (" %s filename [Options] \n", exeName); - DISPLAY ("Options : \n"); - DISPLAY ("-r : recursively load all files in subdirectories (default: off) \n"); - DISPLAY ("-B# : split input into blocks of size # (default: no split) \n"); - DISPLAY ("-# : use compression level # (default: %u) \n", CLEVEL_DEFAULT); - DISPLAY ("-D # : use # as a dictionary (default: create one) \n"); - DISPLAY ("-i# : nb benchmark rounds (default: %u) \n", BENCH_TIME_DEFAULT_S); - DISPLAY ("--nbDicts=# : create # dictionaries for bench (default: one per block) \n"); + usage(exeName); return 1; } @@ -749,6 +758,7 @@ int main (int argc, const char** argv) for (int argNb = 1; argNb < argc ; argNb++) { const char* argument = argv[argNb]; + if (!strcmp(argument, "-h")) { return usage(exeName); } if (!strcmp(argument, "-r")) { recursiveMode = 1; continue; } if (!strcmp(argument, "-D")) { argNb++; assert(argNb < argc); dictionary = argv[argNb]; continue; } if (longCommandWArg(&argument, "-i")) { nbRounds = readU32FromChar(&argument); continue; } From c57a856d64174a199852481b73bbe20820bd3590 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 5 Sep 2018 14:33:51 -0700 Subject: [PATCH 15/15] fixed minor static analyzer warning --- contrib/largeNbDicts/largeNbDicts.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/largeNbDicts/largeNbDicts.c b/contrib/largeNbDicts/largeNbDicts.c index 21fbd2aa..d094065a 100644 --- a/contrib/largeNbDicts/largeNbDicts.c +++ b/contrib/largeNbDicts/largeNbDicts.c @@ -758,7 +758,7 @@ int main (int argc, const char** argv) for (int argNb = 1; argNb < argc ; argNb++) { const char* argument = argv[argNb]; - if (!strcmp(argument, "-h")) { return usage(exeName); } + if (!strcmp(argument, "-h")) { free(nameTable); return usage(exeName); } if (!strcmp(argument, "-r")) { recursiveMode = 1; continue; } if (!strcmp(argument, "-D")) { argNb++; assert(argNb < argc); dictionary = argv[argNb]; continue; } if (longCommandWArg(&argument, "-i")) { nbRounds = readU32FromChar(&argument); continue; }