From 77c137b3ae4d3c961952acf659b4fa515dffb1db Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 14 Sep 2017 15:12:57 -0700 Subject: [PATCH 1/6] minor comment refactor --- lib/dictBuilder/zdict.c | 6 +++--- programs/dibio.c | 48 ++++++++++++++++++++++------------------- 2 files changed, 29 insertions(+), 25 deletions(-) diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c index b76e7695..1bb8b068 100644 --- a/lib/dictBuilder/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -375,7 +375,7 @@ static int isIncluded(const void* in, const void* container, size_t length) return u==length; } -/*! ZDICT_checkMerge +/*! ZDICT_tryMerge() : check if dictItem can be merged, do it if possible @return : id of destination elt, 0 if not merged */ @@ -440,8 +440,8 @@ static U32 ZDICT_tryMerge(dictItem* table, dictItem elt, U32 eltNbToSkip, const static void ZDICT_removeDictItem(dictItem* table, U32 id) { - /* convention : first element is nb of elts */ - U32 const max = table->pos; + /* convention : table[0].pos stores nb of elts */ + U32 const max = table[0].pos; U32 u; if (!id) return; /* protection, should never happen */ for (u=id; u> 5; } +/* DiB_shuffle() : + * shuffle a table of file names in a semi-random way + * It improves dictionary quality by reducing "locality" impact, so if sample set is very large, + * it will load random elements from it, instead of just the first ones. */ static void DiB_shuffle(const char** fileNamesTable, unsigned nbFiles) { - /* Initialize the pseudorandom number generator */ - U32 seed = 0xFD2FB528; - unsigned i; - for (i = nbFiles - 1; i > 0; --i) { - unsigned const j = DiB_rand(&seed) % (i + 1); - const char* tmp = fileNamesTable[j]; - fileNamesTable[j] = fileNamesTable[i]; - fileNamesTable[i] = tmp; - } + U32 seed = 0xFD2FB528; + unsigned i; + for (i = nbFiles - 1; i > 0; --i) { + unsigned const j = DiB_rand(&seed) % (i + 1); + const char* const tmp = fileNamesTable[j]; + fileNamesTable[j] = fileNamesTable[i]; + fileNamesTable[i] = tmp; + } } @@ -162,7 +167,7 @@ static size_t DiB_findMaxMem(unsigned long long requiredMem) requiredMem = (((requiredMem >> 23) + 1) << 23); requiredMem += step; - if (requiredMem > maxMemory) requiredMem = maxMemory; + if (requiredMem > g_maxMemory) requiredMem = g_maxMemory; while (!testmem) { testmem = malloc((size_t)requiredMem); @@ -203,7 +208,7 @@ static void DiB_saveDict(const char* dictFileName, static int g_tooLargeSamples = 0; -static U64 DiB_getTotalCappedFileSize(const char** fileNamesTable, unsigned nbFiles) +static U64 DiB_totalCappedFileSize(const char** fileNamesTable, unsigned nbFiles) { U64 total = 0; unsigned n; @@ -236,7 +241,7 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize, { void* const dictBuffer = malloc(maxDictSize); size_t* const fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t)); - unsigned long long const totalSizeToLoad = DiB_getTotalCappedFileSize(fileNamesTable, nbFiles); + unsigned long long const totalSizeToLoad = DiB_totalCappedFileSize(fileNamesTable, nbFiles); size_t const memMult = params ? MEMMULT : COVER_MEMMULT; size_t const maxMem = DiB_findMaxMem(totalSizeToLoad * memMult) / memMult; size_t benchedSize = (size_t) MIN ((unsigned long long)maxMem, totalSizeToLoad); @@ -246,8 +251,9 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize, /* Checks */ if (params) g_displayLevel = params->zParams.notificationLevel; else if (coverParams) g_displayLevel = coverParams->zParams.notificationLevel; - else EXM_THROW(13, "Neither dictionary algorith selected"); /* should not happen */ - if ((!fileSizes) || (!srcBuffer) || (!dictBuffer)) EXM_THROW(12, "not enough memory for DiB_trainFiles"); /* should not happen */ + else EXM_THROW(13, "Neither dictionary algorithm selected"); /* should not happen */ + if ((!fileSizes) || (!srcBuffer) || (!dictBuffer)) + EXM_THROW(12, "not enough memory for DiB_trainFiles"); /* should not happen */ if (g_tooLargeSamples) { DISPLAYLEVEL(2, "! Warning : some samples are very large \n"); DISPLAYLEVEL(2, "! Note that dictionary is only useful for small files or beginning of large files. \n"); @@ -270,8 +276,7 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize, DiB_shuffle(fileNamesTable, nbFiles); nbFiles = DiB_loadFiles(srcBuffer, &benchedSize, fileSizes, fileNamesTable, nbFiles); - { - size_t dictSize; + { size_t dictSize; if (params) { DiB_fillNoise((char*)srcBuffer + benchedSize, NOISELENGTH); /* guard band, for end of buffer condition */ dictSize = ZDICT_trainFromBuffer_unsafe_legacy(dictBuffer, maxDictSize, @@ -285,9 +290,8 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize, DISPLAYLEVEL(2, "k=%u\nd=%u\nsteps=%u\n", coverParams->k, coverParams->d, coverParams->steps); } } else { - dictSize = - ZDICT_trainFromBuffer_cover(dictBuffer, maxDictSize, srcBuffer, - fileSizes, nbFiles, *coverParams); + dictSize = ZDICT_trainFromBuffer_cover(dictBuffer, maxDictSize, srcBuffer, + fileSizes, nbFiles, *coverParams); } if (ZDICT_isError(dictSize)) { DISPLAYLEVEL(1, "dictionary training failed : %s \n", ZDICT_getErrorName(dictSize)); /* should not happen */ From 086b9597d9f53444fb43c6aa23db609078e7e5e9 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 14 Sep 2017 16:45:10 -0700 Subject: [PATCH 2/6] added ability to split input files for dictionary training using command -B# This is the same behavior as benchmark module, which can also split input into arbitrary size blocks, using -B#. --- doc/zstd_manual.html | 29 +++++++++ programs/dibio.c | 139 +++++++++++++++++++++++++------------------ programs/dibio.h | 2 +- programs/zstd.1.md | 4 +- programs/zstdcli.c | 4 +- 5 files changed, 116 insertions(+), 62 deletions(-) diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html index 1c298726..e7d86170 100644 --- a/doc/zstd_manual.html +++ b/doc/zstd_manual.html @@ -845,6 +845,35 @@ size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long /* advanced parameters - may not remain available after API update */ ZSTD_p_forceMaxWindow=1100, /* Force back-reference distances to remain < windowSize, * even when referencing into Dictionary content (default:0) */ + ZSTD_p_enableLongDistanceMatching=1200, /* Enable long distance matching. + * This parameter is designed to improve the compression + * ratio for large inputs with long distance matches. + * This increases the memory usage as well as window size. + * Note: setting this parameter sets all the LDM parameters + * as well as ZSTD_p_windowLog. It should be set after + * ZSTD_p_compressionLevel and before ZSTD_p_windowLog and + * other LDM parameters. Setting the compression level + * after this parameter overrides the window log, though LDM + * will remain enabled until explicitly disabled. */ + ZSTD_p_ldmHashLog, /* Size of the table for long distance matching, as a power of 2. + * Larger values increase memory usage and compression ratio, but decrease + * compression speed. + * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX + * (default: 20). */ + ZSTD_p_ldmMinMatch, /* Minimum size of searched matches for long distance matcher. + * Larger/too small values usually decrease compression ratio. + * Must be clamped between ZSTD_LDM_MINMATCH_MIN + * and ZSTD_LDM_MINMATCH_MAX (default: 64). */ + ZSTD_p_ldmBucketSizeLog, /* Log size of each bucket in the LDM hash table for collision resolution. + * Larger values usually improve collision resolution but may decrease + * compression speed. + * The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX (default: 3). */ + ZSTD_p_ldmHashEveryLog, /* Frequency of inserting/looking up entries in the LDM hash table. + * The default is MAX(0, (windowLog - ldmHashLog)) to + * optimize hash table usage. + * Larger values improve compression speed. Deviating far from the + * default value will likely result in a decrease in compression ratio. + * Must be clamped between 0 and ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN. */ } ZSTD_cParameter;
diff --git a/programs/dibio.c b/programs/dibio.c index 79f27291..d7b7601d 100644 --- a/programs/dibio.c +++ b/programs/dibio.c @@ -53,13 +53,12 @@ static const size_t g_maxMemory = (sizeof(size_t) == 4) ? (2 GB - 64 MB) : ((siz * Console display ***************************************/ #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) -#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } -static int g_displayLevel = 0; /* 0 : no display; 1: errors; 2: default; 4: full information */ +#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } -#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \ - if ((DIB_clockSpan(g_time) > refreshRate) || (g_displayLevel>=4)) \ +#define DISPLAYUPDATE(l, ...) if (displayLevel>=l) { \ + if ((DIB_clockSpan(g_time) > refreshRate) || (displayLevel>=4)) \ { g_time = clock(); DISPLAY(__VA_ARGS__); \ - if (g_displayLevel>=4) fflush(stderr); } } + if (displayLevel>=4) fflush(stderr); } } static const clock_t refreshRate = CLOCKS_PER_SEC * 2 / 10; static clock_t g_time = 0; @@ -76,9 +75,9 @@ static clock_t DIB_clockSpan(clock_t nPrevious) { return clock() - nPrevious; } #define EXM_THROW(error, ...) \ { \ DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \ - DISPLAYLEVEL(1, "Error %i : ", error); \ - DISPLAYLEVEL(1, __VA_ARGS__); \ - DISPLAYLEVEL(1, "\n"); \ + DISPLAY("Error %i : ", error); \ + DISPLAY(__VA_ARGS__); \ + DISPLAY("\n"); \ exit(error); \ } @@ -102,30 +101,42 @@ const char* DiB_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCo * @return : nb of files effectively loaded into `buffer` * *bufferSizePtr is modified, it provides the amount data loaded within buffer */ static unsigned DiB_loadFiles(void* buffer, size_t* bufferSizePtr, - size_t* fileSizes, - const char** fileNamesTable, unsigned nbFiles) + size_t* chunkSizes, + const char** fileNamesTable, unsigned nbFiles, size_t targetChunkSize, + unsigned displayLevel) { char* const buff = (char*)buffer; size_t pos = 0; - unsigned n; + unsigned nbLoadedChunks = 0, fileIndex; - for (n=0; n *bufferSizePtr-pos) break; - { FILE* const f = fopen(fileName, "rb"); - if (f==NULL) EXM_THROW(10, "zstd: dictBuilder: %s %s ", fileName, strerror(errno)); - DISPLAYUPDATE(2, "Loading %s... \r", fileName); - { size_t const readSize = fread(buff+pos, 1, fileSize, f); - if (readSize != fileSize) EXM_THROW(11, "Pb reading %s", fileName); - pos += readSize; } - fileSizes[n] = fileSize; - fclose(f); - } } + unsigned long long remainingToLoad = fs64; + U32 const nbChunks = targetChunkSize ? (U32)((fs64 + (targetChunkSize-1)) / targetChunkSize) : 1; + U64 const chunkSize = targetChunkSize ? MIN(targetChunkSize, fs64) : fs64; + size_t const maxChunkSize = MIN(chunkSize, SAMPLESIZE_MAX); + U32 cnb; + FILE* const f = fopen(fileName, "rb"); + if (f==NULL) EXM_THROW(10, "zstd: dictBuilder: %s %s ", fileName, strerror(errno)); + DISPLAYUPDATE(2, "Loading %s... \r", fileName); + for (cnb=0; cnb *bufferSizePtr-pos) break; + { size_t const readSize = fread(buff+pos, 1, toLoad, f); + if (readSize != toLoad) EXM_THROW(11, "Pb reading %s", fileName); + pos += readSize; + chunkSizes[nbLoadedChunks++] = toLoad; + remainingToLoad -= targetChunkSize; + if (toLoad < targetChunkSize) { + fseek(f, (targetChunkSize - toLoad), SEEK_CUR); + } } } + fclose(f); + } DISPLAYLEVEL(2, "\r%79s\r", ""); *bufferSizePtr = pos; - return n; + DISPLAYLEVEL(4, "loaded : %u KB \n", (U32)(pos >> 10)) + return nbLoadedChunks; } #define DiB_rotl32(x,r) ((x << r) | (x >> (32 - r))) @@ -207,18 +218,28 @@ static void DiB_saveDict(const char* dictFileName, } -static int g_tooLargeSamples = 0; -static U64 DiB_totalCappedFileSize(const char** fileNamesTable, unsigned nbFiles) +typedef struct { + U64 totalSizeToLoad; + unsigned oneSampleTooLarge; + unsigned nbChunks; +} fileStats; + +static fileStats DiB_fileStats(const char** fileNamesTable, unsigned nbFiles, size_t chunkSize, unsigned displayLevel) { - U64 total = 0; + fileStats fs; unsigned n; + memset(&fs, 0, sizeof(fs)); for (n=0; n 2*SAMPLESIZE_MAX); + U32 const nbChunks = (U32)(chunkSize ? (fileSize + (chunkSize-1)) / chunkSize : 1); + U64 const chunkToLoad = chunkSize ? MIN(chunkSize, fileSize) : fileSize; + size_t const cappedChunkSize = MIN(chunkToLoad, SAMPLESIZE_MAX); + fs.totalSizeToLoad += cappedChunkSize * nbChunks; + fs.oneSampleTooLarge |= (chunkSize > 2*SAMPLESIZE_MAX); + fs.nbChunks += nbChunks; } - return total; + DISPLAYLEVEL(4, "Preparing to load : %u KB \n", (U32)(fs.totalSizeToLoad >> 10)); + return fs; } @@ -235,63 +256,65 @@ size_t ZDICT_trainFromBuffer_unsafe_legacy(void* dictBuffer, size_t dictBufferCa int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize, - const char** fileNamesTable, unsigned nbFiles, + const char** fileNamesTable, unsigned nbFiles, size_t chunkSize, ZDICT_legacy_params_t *params, ZDICT_cover_params_t *coverParams, int optimizeCover) { + unsigned displayLevel = params ? params->zParams.notificationLevel : + coverParams ? coverParams->zParams.notificationLevel : + 0; /* should never happen */ void* const dictBuffer = malloc(maxDictSize); - size_t* const fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t)); - unsigned long long const totalSizeToLoad = DiB_totalCappedFileSize(fileNamesTable, nbFiles); + fileStats const fs = DiB_fileStats(fileNamesTable, nbFiles, chunkSize, displayLevel); + size_t* const chunkSizes = (size_t*)malloc(fs.nbChunks * sizeof(size_t)); size_t const memMult = params ? MEMMULT : COVER_MEMMULT; - size_t const maxMem = DiB_findMaxMem(totalSizeToLoad * memMult) / memMult; - size_t benchedSize = (size_t) MIN ((unsigned long long)maxMem, totalSizeToLoad); - void* const srcBuffer = malloc(benchedSize+NOISELENGTH); + size_t const maxMem = DiB_findMaxMem(fs.totalSizeToLoad * memMult) / memMult; + size_t loadedSize = (size_t) MIN ((unsigned long long)maxMem, fs.totalSizeToLoad); + void* const srcBuffer = malloc(loadedSize+NOISELENGTH); int result = 0; /* Checks */ - if (params) g_displayLevel = params->zParams.notificationLevel; - else if (coverParams) g_displayLevel = coverParams->zParams.notificationLevel; - else EXM_THROW(13, "Neither dictionary algorithm selected"); /* should not happen */ - if ((!fileSizes) || (!srcBuffer) || (!dictBuffer)) + if ((!chunkSizes) || (!srcBuffer) || (!dictBuffer)) EXM_THROW(12, "not enough memory for DiB_trainFiles"); /* should not happen */ - if (g_tooLargeSamples) { - DISPLAYLEVEL(2, "! Warning : some samples are very large \n"); - DISPLAYLEVEL(2, "! Note that dictionary is only useful for small files or beginning of large files. \n"); - DISPLAYLEVEL(2, "! As a consequence, only the first %u bytes of each file are loaded \n", SAMPLESIZE_MAX); + if (fs.oneSampleTooLarge) { + DISPLAYLEVEL(2, "! Warning : some sample(s) are very large \n"); + DISPLAYLEVEL(2, "! Note that dictionary is only useful for small samples. \n"); + DISPLAYLEVEL(2, "! As a consequence, only the first %u bytes of each sample are loaded \n", SAMPLESIZE_MAX); } - if ((nbFiles < 5) || (totalSizeToLoad < 9 * (unsigned long long)maxDictSize)) { + if (fs.nbChunks < 5) { DISPLAYLEVEL(2, "! Warning : nb of samples too low for proper processing ! \n"); DISPLAYLEVEL(2, "! Please provide _one file per sample_. \n"); - DISPLAYLEVEL(2, "! Do not concatenate samples together into a single file, \n"); - DISPLAYLEVEL(2, "! as dictBuilder will be unable to find the beginning of each sample, \n"); - DISPLAYLEVEL(2, "! resulting in poor dictionary quality. \n"); + EXM_THROW(14, "nb of samples too low"); /* we now clearly forbid this case */ + } + if (fs.totalSizeToLoad < (unsigned long long)(8 * maxDictSize)) { + DISPLAYLEVEL(2, "! Warning : data size of samples too small for target dictionary size \n"); + DISPLAYLEVEL(2, "! Samples should be about 100x larger than target dictionary size \n"); } /* init */ - if (benchedSize < totalSizeToLoad) - DISPLAYLEVEL(1, "Not enough memory; training on %u MB only...\n", (unsigned)(benchedSize >> 20)); + if (loadedSize < fs.totalSizeToLoad) + DISPLAYLEVEL(1, "Not enough memory; training on %u MB only...\n", (unsigned)(loadedSize >> 20)); /* Load input buffer */ DISPLAYLEVEL(3, "Shuffling input files\n"); DiB_shuffle(fileNamesTable, nbFiles); - nbFiles = DiB_loadFiles(srcBuffer, &benchedSize, fileSizes, fileNamesTable, nbFiles); + nbFiles = DiB_loadFiles(srcBuffer, &loadedSize, chunkSizes, fileNamesTable, nbFiles, chunkSize, displayLevel); { size_t dictSize; if (params) { - DiB_fillNoise((char*)srcBuffer + benchedSize, NOISELENGTH); /* guard band, for end of buffer condition */ + DiB_fillNoise((char*)srcBuffer + loadedSize, NOISELENGTH); /* guard band, for end of buffer condition */ dictSize = ZDICT_trainFromBuffer_unsafe_legacy(dictBuffer, maxDictSize, - srcBuffer, fileSizes, nbFiles, + srcBuffer, chunkSizes, fs.nbChunks, *params); } else if (optimizeCover) { dictSize = ZDICT_optimizeTrainFromBuffer_cover(dictBuffer, maxDictSize, - srcBuffer, fileSizes, nbFiles, + srcBuffer, chunkSizes, fs.nbChunks, coverParams); if (!ZDICT_isError(dictSize)) { DISPLAYLEVEL(2, "k=%u\nd=%u\nsteps=%u\n", coverParams->k, coverParams->d, coverParams->steps); } } else { dictSize = ZDICT_trainFromBuffer_cover(dictBuffer, maxDictSize, srcBuffer, - fileSizes, nbFiles, *coverParams); + chunkSizes, fs.nbChunks, *coverParams); } if (ZDICT_isError(dictSize)) { DISPLAYLEVEL(1, "dictionary training failed : %s \n", ZDICT_getErrorName(dictSize)); /* should not happen */ @@ -306,7 +329,7 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize, /* clean up */ _cleanup: free(srcBuffer); + free(chunkSizes); free(dictBuffer); - free(fileSizes); return result; } diff --git a/programs/dibio.h b/programs/dibio.h index ac242449..499e3036 100644 --- a/programs/dibio.h +++ b/programs/dibio.h @@ -32,7 +32,7 @@ @return : 0 == ok. Any other : error. */ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize, - const char** fileNamesTable, unsigned nbFiles, + const char** fileNamesTable, unsigned nbFiles, size_t chunkSize, ZDICT_legacy_params_t *params, ZDICT_cover_params_t *coverParams, int optimizeCover); diff --git a/programs/zstd.1.md b/programs/zstd.1.md index 2fcedde3..e446422b 100644 --- a/programs/zstd.1.md +++ b/programs/zstd.1.md @@ -184,6 +184,8 @@ Typical gains range from 10% (at 64KB) to x5 better (at <1KB). Dictionary saved into `file` (default name: dictionary). * `--maxdict=#`: Limit dictionary to specified size (default: 112640). +* `-B#`: + Split input files in blocks of size # (default: no split) * `--dictID=#`: A dictionary ID is a locally unique ID that a decoder can use to verify it is using the right dictionary. @@ -373,7 +375,7 @@ The list of available _options_: default value will likely result in a decrease in compression ratio. The default value is `wlog - ldmhlog`. - + ### -B#: Select the size of each compression job. This parameter is available only when multi-threading is enabled. diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 607287c9..78adcb6c 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -759,13 +759,13 @@ int main(int argCount, const char* argv[]) int const optimize = !coverParams.k || !coverParams.d; coverParams.nbThreads = nbThreads; coverParams.zParams = zParams; - operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, NULL, &coverParams, optimize); + operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, NULL, &coverParams, optimize); } else { ZDICT_legacy_params_t dictParams; memset(&dictParams, 0, sizeof(dictParams)); dictParams.selectivityLevel = dictSelect; dictParams.zParams = zParams; - operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, &dictParams, NULL, 0); + operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, &dictParams, NULL, 0); } #endif goto _end; From a9694231ca4ebf2391b2100675d0d7a2639680d9 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 15 Sep 2017 10:16:26 -0700 Subject: [PATCH 3/6] fixed minor conversion warning --- programs/dibio.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/dibio.c b/programs/dibio.c index d7b7601d..5ea384c6 100644 --- a/programs/dibio.c +++ b/programs/dibio.c @@ -129,7 +129,7 @@ static unsigned DiB_loadFiles(void* buffer, size_t* bufferSizePtr, chunkSizes[nbLoadedChunks++] = toLoad; remainingToLoad -= targetChunkSize; if (toLoad < targetChunkSize) { - fseek(f, (targetChunkSize - toLoad), SEEK_CUR); + fseek(f, (long)(targetChunkSize - toLoad), SEEK_CUR); } } } fclose(f); } From 25a60488dd3371881058f401185a61436373de1d Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 15 Sep 2017 11:55:13 -0700 Subject: [PATCH 4/6] fixed 64-to-32 conversion warnings --- programs/dibio.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/programs/dibio.c b/programs/dibio.c index 5ea384c6..6e86e846 100644 --- a/programs/dibio.c +++ b/programs/dibio.c @@ -115,13 +115,13 @@ static unsigned DiB_loadFiles(void* buffer, size_t* bufferSizePtr, unsigned long long remainingToLoad = fs64; U32 const nbChunks = targetChunkSize ? (U32)((fs64 + (targetChunkSize-1)) / targetChunkSize) : 1; U64 const chunkSize = targetChunkSize ? MIN(targetChunkSize, fs64) : fs64; - size_t const maxChunkSize = MIN(chunkSize, SAMPLESIZE_MAX); + size_t const maxChunkSize = (size_t)MIN(chunkSize, SAMPLESIZE_MAX); U32 cnb; FILE* const f = fopen(fileName, "rb"); if (f==NULL) EXM_THROW(10, "zstd: dictBuilder: %s %s ", fileName, strerror(errno)); DISPLAYUPDATE(2, "Loading %s... \r", fileName); for (cnb=0; cnb *bufferSizePtr-pos) break; { size_t const readSize = fread(buff+pos, 1, toLoad, f); if (readSize != toLoad) EXM_THROW(11, "Pb reading %s", fileName); @@ -233,7 +233,7 @@ static fileStats DiB_fileStats(const char** fileNamesTable, unsigned nbFiles, si U64 const fileSize = UTIL_getFileSize(fileNamesTable[n]); U32 const nbChunks = (U32)(chunkSize ? (fileSize + (chunkSize-1)) / chunkSize : 1); U64 const chunkToLoad = chunkSize ? MIN(chunkSize, fileSize) : fileSize; - size_t const cappedChunkSize = MIN(chunkToLoad, SAMPLESIZE_MAX); + size_t const cappedChunkSize = (size_t)MIN(chunkToLoad, SAMPLESIZE_MAX); fs.totalSizeToLoad += cappedChunkSize * nbChunks; fs.oneSampleTooLarge |= (chunkSize > 2*SAMPLESIZE_MAX); fs.nbChunks += nbChunks; From c68d17f2da0af7c3c98bcebb0187ff344a2a817f Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 15 Sep 2017 15:31:31 -0700 Subject: [PATCH 5/6] ensures that sampleSizes table is large enough as recommended by @terrelln --- programs/dibio.c | 54 ++++++++++++++++++++++++++++++------------------ 1 file changed, 34 insertions(+), 20 deletions(-) diff --git a/programs/dibio.c b/programs/dibio.c index 6e86e846..66f7d152 100644 --- a/programs/dibio.c +++ b/programs/dibio.c @@ -97,11 +97,16 @@ const char* DiB_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCo * File related operations **********************************************************/ /** DiB_loadFiles() : - * load files listed in fileNamesTable into buffer, even if buffer is too small. - * @return : nb of files effectively loaded into `buffer` - * *bufferSizePtr is modified, it provides the amount data loaded within buffer */ + * load samples from files listed in fileNamesTable into buffer. + * works even if buffer is too small to load all samples. + * Also provides the size of each sample into sampleSizes table + * which must be sized correctly, using DiB_fileStats(). + * @return : nb of samples effectively loaded into `buffer` + * *bufferSizePtr is modified, it provides the amount data loaded within buffer. + * sampleSizes is filled with the size of each sample. + */ static unsigned DiB_loadFiles(void* buffer, size_t* bufferSizePtr, - size_t* chunkSizes, + size_t* sampleSizes, unsigned sstSize, const char** fileNamesTable, unsigned nbFiles, size_t targetChunkSize, unsigned displayLevel) { @@ -126,8 +131,12 @@ static unsigned DiB_loadFiles(void* buffer, size_t* bufferSizePtr, { size_t const readSize = fread(buff+pos, 1, toLoad, f); if (readSize != toLoad) EXM_THROW(11, "Pb reading %s", fileName); pos += readSize; - chunkSizes[nbLoadedChunks++] = toLoad; + sampleSizes[nbLoadedChunks++] = toLoad; remainingToLoad -= targetChunkSize; + if (nbLoadedChunks == sstSize) { /* no more space left in sampleSizes table */ + fileIndex = nbFiles; /* stop there */ + break; + } if (toLoad < targetChunkSize) { fseek(f, (long)(targetChunkSize - toLoad), SEEK_CUR); } } } @@ -221,9 +230,14 @@ static void DiB_saveDict(const char* dictFileName, typedef struct { U64 totalSizeToLoad; unsigned oneSampleTooLarge; - unsigned nbChunks; + unsigned nbSamples; } fileStats; +/*! DiB_fileStats() : + * Given a list of files, and a chunkSize (0 == no chunk, whole files) + * provides the amount of data to be loaded and the resulting nb of samples. + * This is useful primarily for allocation purpose => sample buffer, and sample sizes table. + */ static fileStats DiB_fileStats(const char** fileNamesTable, unsigned nbFiles, size_t chunkSize, unsigned displayLevel) { fileStats fs; @@ -231,12 +245,12 @@ static fileStats DiB_fileStats(const char** fileNamesTable, unsigned nbFiles, si memset(&fs, 0, sizeof(fs)); for (n=0; n 2*SAMPLESIZE_MAX); - fs.nbChunks += nbChunks; + fs.nbSamples += nbSamples; } DISPLAYLEVEL(4, "Preparing to load : %u KB \n", (U32)(fs.totalSizeToLoad >> 10)); return fs; @@ -260,12 +274,12 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize, ZDICT_legacy_params_t *params, ZDICT_cover_params_t *coverParams, int optimizeCover) { - unsigned displayLevel = params ? params->zParams.notificationLevel : - coverParams ? coverParams->zParams.notificationLevel : - 0; /* should never happen */ + unsigned const displayLevel = params ? params->zParams.notificationLevel : + coverParams ? coverParams->zParams.notificationLevel : + 0; /* should never happen */ void* const dictBuffer = malloc(maxDictSize); fileStats const fs = DiB_fileStats(fileNamesTable, nbFiles, chunkSize, displayLevel); - size_t* const chunkSizes = (size_t*)malloc(fs.nbChunks * sizeof(size_t)); + size_t* const sampleSizes = (size_t*)malloc(fs.nbSamples * sizeof(size_t)); size_t const memMult = params ? MEMMULT : COVER_MEMMULT; size_t const maxMem = DiB_findMaxMem(fs.totalSizeToLoad * memMult) / memMult; size_t loadedSize = (size_t) MIN ((unsigned long long)maxMem, fs.totalSizeToLoad); @@ -273,14 +287,14 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize, int result = 0; /* Checks */ - if ((!chunkSizes) || (!srcBuffer) || (!dictBuffer)) + if ((!sampleSizes) || (!srcBuffer) || (!dictBuffer)) EXM_THROW(12, "not enough memory for DiB_trainFiles"); /* should not happen */ if (fs.oneSampleTooLarge) { DISPLAYLEVEL(2, "! Warning : some sample(s) are very large \n"); DISPLAYLEVEL(2, "! Note that dictionary is only useful for small samples. \n"); DISPLAYLEVEL(2, "! As a consequence, only the first %u bytes of each sample are loaded \n", SAMPLESIZE_MAX); } - if (fs.nbChunks < 5) { + if (fs.nbSamples < 5) { DISPLAYLEVEL(2, "! Warning : nb of samples too low for proper processing ! \n"); DISPLAYLEVEL(2, "! Please provide _one file per sample_. \n"); EXM_THROW(14, "nb of samples too low"); /* we now clearly forbid this case */ @@ -297,24 +311,24 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize, /* Load input buffer */ DISPLAYLEVEL(3, "Shuffling input files\n"); DiB_shuffle(fileNamesTable, nbFiles); - nbFiles = DiB_loadFiles(srcBuffer, &loadedSize, chunkSizes, fileNamesTable, nbFiles, chunkSize, displayLevel); + nbFiles = DiB_loadFiles(srcBuffer, &loadedSize, sampleSizes, fs.nbSamples, fileNamesTable, nbFiles, chunkSize, displayLevel); { size_t dictSize; if (params) { DiB_fillNoise((char*)srcBuffer + loadedSize, NOISELENGTH); /* guard band, for end of buffer condition */ dictSize = ZDICT_trainFromBuffer_unsafe_legacy(dictBuffer, maxDictSize, - srcBuffer, chunkSizes, fs.nbChunks, + srcBuffer, sampleSizes, fs.nbSamples, *params); } else if (optimizeCover) { dictSize = ZDICT_optimizeTrainFromBuffer_cover(dictBuffer, maxDictSize, - srcBuffer, chunkSizes, fs.nbChunks, + srcBuffer, sampleSizes, fs.nbSamples, coverParams); if (!ZDICT_isError(dictSize)) { DISPLAYLEVEL(2, "k=%u\nd=%u\nsteps=%u\n", coverParams->k, coverParams->d, coverParams->steps); } } else { dictSize = ZDICT_trainFromBuffer_cover(dictBuffer, maxDictSize, srcBuffer, - chunkSizes, fs.nbChunks, *coverParams); + sampleSizes, fs.nbSamples, *coverParams); } if (ZDICT_isError(dictSize)) { DISPLAYLEVEL(1, "dictionary training failed : %s \n", ZDICT_getErrorName(dictSize)); /* should not happen */ @@ -329,7 +343,7 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize, /* clean up */ _cleanup: free(srcBuffer); - free(chunkSizes); + free(sampleSizes); free(dictBuffer); return result; } From 172205579918c485e56daddd7a457935dfcb1a05 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 15 Sep 2017 16:23:50 -0700 Subject: [PATCH 6/6] add comment on using -B# to split input file for dictionary training --- programs/dibio.c | 1 + 1 file changed, 1 insertion(+) diff --git a/programs/dibio.c b/programs/dibio.c index 66f7d152..2cb2a426 100644 --- a/programs/dibio.c +++ b/programs/dibio.c @@ -297,6 +297,7 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize, if (fs.nbSamples < 5) { DISPLAYLEVEL(2, "! Warning : nb of samples too low for proper processing ! \n"); DISPLAYLEVEL(2, "! Please provide _one file per sample_. \n"); + DISPLAYLEVEL(2, "! Alternatively, split files into fixed-size blocks representative of samples, with -B# \n"); EXM_THROW(14, "nb of samples too low"); /* we now clearly forbid this case */ } if (fs.totalSizeToLoad < (unsigned long long)(8 * maxDictSize)) {