From 671f28d1e50865bac0662297eb0d2372a705385e Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 13 Dec 2016 12:18:07 +0100 Subject: [PATCH 01/17] added parseCompressionParameters --- programs/zstdcli.c | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 7ffb0bb6..9bc96e19 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -204,6 +204,38 @@ static unsigned longCommandWArg(const char** stringPtr, const char* longCommand) return result; } + +/** parseCompressionParameters() : + * reads compression parameters from *stringPtr (e.g. "--zstd=wlog=23,clog=23,hlog=22,slog=6,slen=3,tlen=48,strat=6") into *params + * @return 1 means that compression parameters were correct + * @return 0 in case of malformed parameters + */ +static unsigned parseCompressionParameters(const char* stringPtr, ZSTD_compressionParameters* params) +{ + for ( ; ;) { + DISPLAY("arg=%s\n", stringPtr); + if (longCommandWArg(&stringPtr, "windowLog=") || longCommandWArg(&stringPtr, "wlog=")) { params->windowLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } + if (longCommandWArg(&stringPtr, "chainLog=") || longCommandWArg(&stringPtr, "clog=")) { params->chainLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } + if (longCommandWArg(&stringPtr, "hashLog=") || longCommandWArg(&stringPtr, "hlog=")) { params->hashLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } + if (longCommandWArg(&stringPtr, "searchLog=") || longCommandWArg(&stringPtr, "slog=")) { params->searchLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } + if (longCommandWArg(&stringPtr, "searchLength=") || longCommandWArg(&stringPtr, "slen=")) { params->searchLength = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } + if (longCommandWArg(&stringPtr, "targetLength=") || longCommandWArg(&stringPtr, "tlen=")) { params->targetLength = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } + if (longCommandWArg(&stringPtr, "strategy=") || longCommandWArg(&stringPtr, "strat=")) { params->strategy = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } + return 0; + } + + if (stringPtr[0] != 0) return 0; /* check the end of string */ + DISPLAYLEVEL(4, "windowLog=%d\n", params->windowLog); + DISPLAYLEVEL(4, "chainLog=%d\n", params->chainLog); + DISPLAYLEVEL(4, "hashLog=%d\n", params->hashLog); + DISPLAYLEVEL(4, "searchLog=%d\n", params->searchLog); + DISPLAYLEVEL(4, "searchLength=%d\n", params->searchLength); + DISPLAYLEVEL(4, "targetLength=%d\n", params->targetLength); + DISPLAYLEVEL(4, "strategy=%d\n", params->strategy); + return 1; +} + + typedef enum { zom_compress, zom_decompress, zom_test, zom_bench, zom_train } zstd_operation_mode; #define CLEAN_RETURN(i) { operationResult = (i); goto _end; } @@ -222,6 +254,7 @@ int main(int argCount, const char* argv[]) ultra=0, lastCommand = 0; zstd_operation_mode operation = zom_compress; + ZSTD_compressionParameters compressionParams; int cLevel = ZSTDCLI_CLEVEL_DEFAULT; int cLevelLast = 1; unsigned recursive = 0; @@ -258,6 +291,7 @@ int main(int argCount, const char* argv[]) /* preset behaviors */ if (!strcmp(programName, ZSTD_UNZSTD)) operation=zom_decompress; if (!strcmp(programName, ZSTD_CAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; displayLevel=1; } + memset(&compressionParams, 0, sizeof(compressionParams)); /* command switches */ for (argNb=1; argNbwindowLog) zparams.cParams.windowLog = comprParams->windowLog; + if (comprParams->chainLog) zparams.cParams.chainLog = comprParams->chainLog; + if (comprParams->hashLog) zparams.cParams.hashLog = comprParams->hashLog; + if (comprParams->searchLog) zparams.cParams.searchLog = comprParams->searchLog; + if (comprParams->searchLength) zparams.cParams.searchLength = comprParams->searchLength; + if (comprParams->targetLength) zparams.cParams.targetLength = comprParams->targetLength; + if (comprParams->strategy) zparams.cParams.strategy = comprParams->strategy; do { U32 blockNb; size_t rSize; @@ -247,9 +255,9 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize, cdict); } else { - rSize = ZSTD_compressCCtx (ctx, + rSize = ZSTD_compress_advanced (ctx, blockTable[blockNb].cPtr, blockTable[blockNb].cRoom, - blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize, cLevel); + blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize, NULL, 0, zparams); } if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_compress_usingCDict() failed : %s", ZSTD_getErrorName(rSize)); blockTable[blockNb].cSize = rSize; @@ -387,7 +395,8 @@ static size_t BMK_findMaxMem(U64 requiredMem) static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, const char* displayName, int cLevel, int cLevelLast, const size_t* fileSizes, unsigned nbFiles, - const void* dictBuffer, size_t dictBufferSize) + const void* dictBuffer, size_t dictBufferSize, + ZSTD_compressionParameters *compressionParams) { int l; @@ -406,7 +415,7 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, BMK_benchMem(srcBuffer, benchedSize, displayName, l, fileSizes, nbFiles, - dictBuffer, dictBufferSize); + dictBuffer, dictBufferSize, compressionParams); } } @@ -443,8 +452,8 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize, if (totalSize == 0) EXM_THROW(12, "no data to bench"); } -static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, - const char* dictFileName, int cLevel, int cLevelLast) +static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName, + int cLevel, int cLevelLast, ZSTD_compressionParameters *compressionParams) { void* srcBuffer; size_t benchedSize; @@ -483,7 +492,7 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, BMK_benchCLevel(srcBuffer, benchedSize, displayName, cLevel, cLevelLast, fileSizes, nbFiles, - dictBuffer, dictBufferSize); + dictBuffer, dictBufferSize, compressionParams); } /* clean up */ @@ -493,7 +502,7 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, } -static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility) +static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility, ZSTD_compressionParameters* compressionParams) { char name[20] = {0}; size_t benchedSize = 10000000; @@ -507,15 +516,15 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility /* Bench */ snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100)); - BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1, NULL, 0); + BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1, NULL, 0, compressionParams); /* clean up */ free(srcBuffer); } -int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, - const char* dictFileName, int cLevel, int cLevelLast) +int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName, + int cLevel, int cLevelLast, ZSTD_compressionParameters* compressionParams) { double const compressibility = (double)g_compressibilityDefault / 100; @@ -525,8 +534,8 @@ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, if (cLevelLast > cLevel) DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast); if (nbFiles == 0) - BMK_syntheticTest(cLevel, cLevelLast, compressibility); + BMK_syntheticTest(cLevel, cLevelLast, compressibility, compressionParams); else - BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast); + BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast, compressionParams); return 0; } diff --git a/programs/bench.h b/programs/bench.h index 7009dc2f..314f3465 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -12,9 +12,11 @@ #define BENCH_H_121279284357 #include /* size_t */ +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressionParameters */ +#include "zstd.h" /* ZSTD_compressionParameters */ -int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, - const char* dictFileName, int cLevel, int cLevelLast); +int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,const char* dictFileName, + int cLevel, int cLevelLast, ZSTD_compressionParameters* compressionParams); /* Set Parameters */ void BMK_SetNbSeconds(unsigned nbLoops); diff --git a/programs/zstdcli.c b/programs/zstdcli.c index e11f6484..fb873a86 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -214,7 +214,6 @@ static unsigned longCommandWArg(const char** stringPtr, const char* longCommand) static unsigned parseCompressionParameters(const char* stringPtr, ZSTD_compressionParameters* params) { for ( ; ;) { - DISPLAY("arg=%s\n", stringPtr); if (longCommandWArg(&stringPtr, "windowLog=") || longCommandWArg(&stringPtr, "wlog=")) { params->windowLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } if (longCommandWArg(&stringPtr, "chainLog=") || longCommandWArg(&stringPtr, "clog=")) { params->chainLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } if (longCommandWArg(&stringPtr, "hashLog=") || longCommandWArg(&stringPtr, "hlog=")) { params->hashLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } @@ -226,13 +225,8 @@ static unsigned parseCompressionParameters(const char* stringPtr, ZSTD_compressi } if (stringPtr[0] != 0) return 0; /* check the end of string */ - DISPLAYLEVEL(4, "windowLog=%d\n", params->windowLog); - DISPLAYLEVEL(4, "chainLog=%d\n", params->chainLog); - DISPLAYLEVEL(4, "hashLog=%d\n", params->hashLog); - DISPLAYLEVEL(4, "searchLog=%d\n", params->searchLog); - DISPLAYLEVEL(4, "searchLength=%d\n", params->searchLength); - DISPLAYLEVEL(4, "targetLength=%d\n", params->targetLength); - DISPLAYLEVEL(4, "strategy=%d\n", params->strategy); + DISPLAYLEVEL(4, "windowLog=%d\nchainLog=%d\nhashLog=%d\nsearchLog=%d\n", params->windowLog, params->chainLog, params->hashLog, params->searchLog); + DISPLAYLEVEL(4, "searchLength=%d\ntargetLength=%d\nstrategy=%d\n", params->searchLength, params->targetLength, params->strategy); return 1; } @@ -523,7 +517,7 @@ int main(int argCount, const char* argv[]) if (operation==zom_bench) { #ifndef ZSTD_NOBENCH BMK_setNotificationLevel(displayLevel); - BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast); + BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast, &compressionParams); #endif goto _end; } From 8349d675e01faee2e1b33582175f4aaea71342bd Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 13 Dec 2016 13:24:59 +0100 Subject: [PATCH 03/17] fileio.c: support advanced compression parameters --- programs/fileio.c | 19 ++++++++++++++----- programs/fileio.h | 9 +++++++-- programs/zstdcli.c | 4 ++-- 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index a493e97c..597422ae 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -247,7 +247,8 @@ typedef struct { FILE* srcFile; } cRess_t; -static cRess_t FIO_createCResources(const char* dictFileName, int cLevel, U64 srcSize) +static cRess_t FIO_createCResources(const char* dictFileName, int cLevel, + U64 srcSize, ZSTD_compressionParameters* comprParams) { cRess_t ress; memset(&ress, 0, sizeof(ress)); @@ -268,6 +269,13 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel, U64 sr params.fParams.contentSizeFlag = 1; params.fParams.checksumFlag = g_checksumFlag; params.fParams.noDictIDFlag = !g_dictIDFlag; + if (comprParams->windowLog) params.cParams.windowLog = comprParams->windowLog; + if (comprParams->chainLog) params.cParams.chainLog = comprParams->chainLog; + if (comprParams->hashLog) params.cParams.hashLog = comprParams->hashLog; + if (comprParams->searchLog) params.cParams.searchLog = comprParams->searchLog; + if (comprParams->searchLength) params.cParams.searchLength = comprParams->searchLength; + if (comprParams->targetLength) params.cParams.targetLength = comprParams->targetLength; + if (comprParams->strategy) params.cParams.strategy = comprParams->strategy; { size_t const errorCode = ZSTD_initCStream_advanced(ress.cctx, dictBuffer, dictBuffSize, params, srcSize); if (ZSTD_isError(errorCode)) EXM_THROW(33, "Error initializing CStream : %s", ZSTD_getErrorName(errorCode)); } } @@ -401,12 +409,12 @@ static int FIO_compressFilename_dstFile(cRess_t ress, int FIO_compressFilename(const char* dstFileName, const char* srcFileName, - const char* dictFileName, int compressionLevel) + const char* dictFileName, int compressionLevel, ZSTD_compressionParameters* comprParams) { clock_t const start = clock(); U64 const srcSize = UTIL_getFileSize(srcFileName); - cRess_t const ress = FIO_createCResources(dictFileName, compressionLevel, srcSize); + cRess_t const ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, comprParams); int const result = FIO_compressFilename_dstFile(ress, dstFileName, srcFileName); double const seconds = (double)(clock() - start) / CLOCKS_PER_SEC; @@ -419,14 +427,15 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName, int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFiles, const char* suffix, - const char* dictFileName, int compressionLevel) + const char* dictFileName, int compressionLevel, + ZSTD_compressionParameters* comprParams) { int missed_files = 0; size_t dfnSize = FNSPACE; char* dstFileName = (char*)malloc(FNSPACE); size_t const suffixSize = suffix ? strlen(suffix) : 0; U64 const srcSize = (nbFiles != 1) ? 0 : UTIL_getFileSize(inFileNamesTable[0]) ; - cRess_t ress = FIO_createCResources(dictFileName, compressionLevel, srcSize); + cRess_t ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, comprParams); /* init */ if (dstFileName==NULL) EXM_THROW(27, "FIO_compressMultipleFilenames : allocation error for dstFileName"); diff --git a/programs/fileio.h b/programs/fileio.h index 06e0be37..b7165833 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -11,6 +11,9 @@ #ifndef FILEIO_H_23981798732 #define FILEIO_H_23981798732 +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressionParameters */ +#include "zstd.h" /* ZSTD_compressionParameters */ + #if defined (__cplusplus) extern "C" { #endif @@ -44,7 +47,8 @@ void FIO_setMemLimit(unsigned memLimit); ***************************************/ /** FIO_compressFilename() : @return : 0 == ok; 1 == pb with src file. */ -int FIO_compressFilename (const char* outfilename, const char* infilename, const char* dictFileName, int compressionLevel); +int FIO_compressFilename (const char* outfilename, const char* infilename, const char* dictFileName, + int compressionLevel, ZSTD_compressionParameters* comprParams); /** FIO_decompressFilename() : @return : 0 == ok; 1 == pb with src file. */ @@ -58,7 +62,8 @@ int FIO_decompressFilename (const char* outfilename, const char* infilename, con @return : nb of missing files */ int FIO_compressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles, const char* suffix, - const char* dictFileName, int compressionLevel); + const char* dictFileName, int compressionLevel, + ZSTD_compressionParameters* comprParams); /** FIO_decompressMultipleFilenames() : @return : nb of missing or skipped files */ diff --git a/programs/zstdcli.c b/programs/zstdcli.c index fb873a86..f2a6c71f 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -569,9 +569,9 @@ int main(int argCount, const char* argv[]) if (operation==zom_compress) { #ifndef ZSTD_NOCOMPRESS if ((filenameIdx==1) && outFileName) - operationResult = FIO_compressFilename(outFileName, filenameTable[0], dictFileName, cLevel); + operationResult = FIO_compressFilename(outFileName, filenameTable[0], dictFileName, cLevel, &compressionParams); else - operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, outFileName ? outFileName : ZSTD_EXTENSION, dictFileName, cLevel); + operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, outFileName ? outFileName : ZSTD_EXTENSION, dictFileName, cLevel, &compressionParams); #else DISPLAY("Compression not supported\n"); #endif From 98ef0f98df98402f94a72c85b59559e2634f5336 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 13 Dec 2016 14:52:21 +0100 Subject: [PATCH 04/17] fixed conversion warning --- programs/zstdcli.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index f2a6c71f..68400f19 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -220,7 +220,7 @@ static unsigned parseCompressionParameters(const char* stringPtr, ZSTD_compressi if (longCommandWArg(&stringPtr, "searchLog=") || longCommandWArg(&stringPtr, "slog=")) { params->searchLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } if (longCommandWArg(&stringPtr, "searchLength=") || longCommandWArg(&stringPtr, "slen=")) { params->searchLength = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } if (longCommandWArg(&stringPtr, "targetLength=") || longCommandWArg(&stringPtr, "tlen=")) { params->targetLength = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } - if (longCommandWArg(&stringPtr, "strategy=") || longCommandWArg(&stringPtr, "strat=")) { params->strategy = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } + if (longCommandWArg(&stringPtr, "strategy=") || longCommandWArg(&stringPtr, "strat=")) { params->strategy = (ZSTD_strategy)readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } return 0; } From c71e552b2e91c1f95a998887edb9eb8fe5c3f4ce Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 13 Dec 2016 20:04:32 +0100 Subject: [PATCH 05/17] fixed "strategy" in advanced compression parameters --- programs/bench.c | 2 +- programs/fileio.c | 2 +- programs/zstdcli.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 6203af6b..6c2383a8 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -244,7 +244,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, if (comprParams->searchLog) zparams.cParams.searchLog = comprParams->searchLog; if (comprParams->searchLength) zparams.cParams.searchLength = comprParams->searchLength; if (comprParams->targetLength) zparams.cParams.targetLength = comprParams->targetLength; - if (comprParams->strategy) zparams.cParams.strategy = comprParams->strategy; + if (comprParams->strategy) zparams.cParams.strategy = (ZSTD_strategy)(comprParams->strategy - 1); do { U32 blockNb; size_t rSize; diff --git a/programs/fileio.c b/programs/fileio.c index 597422ae..555b5614 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -275,7 +275,7 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel, if (comprParams->searchLog) params.cParams.searchLog = comprParams->searchLog; if (comprParams->searchLength) params.cParams.searchLength = comprParams->searchLength; if (comprParams->targetLength) params.cParams.targetLength = comprParams->targetLength; - if (comprParams->strategy) params.cParams.strategy = comprParams->strategy; + if (comprParams->strategy) params.cParams.strategy = (ZSTD_strategy)(comprParams->strategy - 1); { size_t const errorCode = ZSTD_initCStream_advanced(ress.cctx, dictBuffer, dictBuffSize, params, srcSize); if (ZSTD_isError(errorCode)) EXM_THROW(33, "Error initializing CStream : %s", ZSTD_getErrorName(errorCode)); } } diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 68400f19..c1eb7b73 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -220,7 +220,7 @@ static unsigned parseCompressionParameters(const char* stringPtr, ZSTD_compressi if (longCommandWArg(&stringPtr, "searchLog=") || longCommandWArg(&stringPtr, "slog=")) { params->searchLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } if (longCommandWArg(&stringPtr, "searchLength=") || longCommandWArg(&stringPtr, "slen=")) { params->searchLength = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } if (longCommandWArg(&stringPtr, "targetLength=") || longCommandWArg(&stringPtr, "tlen=")) { params->targetLength = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } - if (longCommandWArg(&stringPtr, "strategy=") || longCommandWArg(&stringPtr, "strat=")) { params->strategy = (ZSTD_strategy)readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } + if (longCommandWArg(&stringPtr, "strategy=") || longCommandWArg(&stringPtr, "strat=")) { params->strategy = (ZSTD_strategy)(1 + readU32FromChar(&stringPtr)); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } return 0; } From 25314428c975b66891f3414836e117bd4093a4bc Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 14 Dec 2016 16:10:13 +0100 Subject: [PATCH 06/17] zstd.1: added advanced compression options --- programs/zstd.1 | 109 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) diff --git a/programs/zstd.1 b/programs/zstd.1 index 63b60d1e..9b10b187 100644 --- a/programs/zstd.1 +++ b/programs/zstd.1 @@ -264,6 +264,115 @@ and weight typically 100x the target dictionary size (for example, 10 MB for a 1 cut file into independent blocks of size # (default: no block) +.SH ADVANCED COMPRESSION OPTIONS +.TP +.B \--zstd[=\fIoptions\fR] +.PD +\fBzstd\fR provides 22 predefined compression levels. The selected or default predefined compression level can be changed with advanced compression options. +The \fIoptions\fR are provided as a comma-separated list. You may specify only the \fIoptions\fR you want to change and the rest will be taken from the selected or default compression level. +The list of available \fIoptions\fR: +.RS + +.TP +.BI strategy= strat +.PD 0 +.TP +.BI strat= strat +.PD +Specify a strategy used by a match finder. +.IP "" +There are 8 strategies numbered from 0 to 7, from faster to stronger: +0=ZSTD_fast, 1=ZSTD_dfast, 2=ZSTD_greedy, 3=ZSTD_lazy, 4=ZSTD_lazy2, 5=ZSTD_btlazy2, 6=ZSTD_btopt, 7=ZSTD_btopt2. +.IP "" + +.TP +.BI windowLog= wlog +.PD 0 +.TP +.BI wlog= wlog +.PD +Specify the maximum number of bits for a match distance. +.IP "" +The higher number of bits increases the chance to find a match what usually improves compression ratio. +It also increases memory requirements for compressor and decompressor. +.IP "" +The minimum \fIwlog\fR is 10 (1 KiB) and the maximum is 25 (32 MiB) for 32-bit compilation and 27 (128 MiB) for 64-bit compilation. +.IP "" + +.TP +.BI hashLog= hlog +.PD 0 +.TP +.BI hlog= hlog +.PD +Specify the maximum number of bits for a hash table. +.IP "" +The bigger hash table causes less collisions what usually make compression faster but requires more memory during compression. +.IP "" +The minimum \fIhlog\fR is 6 (64 B) and the maximum is 25 (32 MiB) for 32-bit compilation and 27 (128 MiB) for 64-bit compilation. + +.TP +.BI chainLog= clog +.PD 0 +.TP +.BI clog= clog +.PD +Specify the maximum number of bits for a hash chain or a binary tree. +.IP "" +The higher number of bits increases the chance to find a match what usually improves compression ratio. +It also slows down compression speed and increases memory requirements for compression. +This option is ignored for the ZSTD_fast strategy. +.IP "" +The minimum \fIclog\fR is 6 (64 B) and the maximum is 26 (64 MiB) for 32-bit compilation and 28 (256 MiB) for 64-bit compilation. +.IP "" + +.TP +.BI searchLog= slog +.PD 0 +.TP +.BI slog= slog +.PD +Specify the maximum number of searches in a hash chain or a binary tree using logarithmic scale. +.IP "" +The bigger number of searches increases the chance to find a match what usually improves compression ratio but decreases compression speed. +.IP "" +The minimum \fIslog\fR is 1 and the maximum is 24 for 32-bit compilation and 26 for 64-bit compilation. +.IP "" + +.TP +.BI searchLength= slen +.PD 0 +.TP +.BI slen= slen +.PD +Specify the minimum searched length of a match in a hash table. +.IP "" +The bigger search length usually decreases compression ratio but improves decompression speed. +.IP "" +The minimum \fIslen\fR is 3 and the maximum is 7. +.IP "" + +.TP +.BI targetLength= tlen +.PD 0 +.TP +.BI tlen= tlen +.PD +Specify the minimum match length that causes a match finder to interrupt searching of better matches. +.IP "" +The bigger minimum match length usually improves compression ratio but decreases compression speed. +This option is used only with ZSTD_btopt and ZSTD_btopt2 strategies. +.IP "" +The minimum \fItlen\fR is 4 and the maximum is 999. +.IP "" + +.PP +.B An example +.br +The following parameters sets advanced compression options to predefined level 19 for files bigger than 256 KB: +.IP "" +\fB--zstd=\fRwindowLog=23,chainLog=23,hashLog=22,searchLog=6,searchLength=3,targetLength=48,strategy=6 + .SH BUGS Report bugs at:- https://github.com/facebook/zstd/issues From 9b4fa0ddf722cc154e6f2274abaa346e62a4c2b4 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 14 Dec 2016 16:50:00 +0100 Subject: [PATCH 07/17] playTests.sh: added Advanced compression parameters --- tests/playTests.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/playTests.sh b/tests/playTests.sh index abde72c8..c5bc577d 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -101,6 +101,15 @@ $ZSTD -f tmp && die "tmp not present : should have failed" ls tmp.zst && die "tmp.zst should not be created" +$ECHO "\n**** Advanced compression parameters **** " +$ZSTD --zstd=windowLog=21, && die "wrong parameters not detected!" +$ZSTD --zstd=windowLok=21 && die "wrong parameters not detected!" +$ZSTD --zstd=windowLog=21,slog= && die "wrong parameters not detected!" +roundTripTest -g512K +roundTripTest -g512K " --zstd=windowLog=23,chainLog=23,hashLog=22,searchLog=6,searchLength=3,targetLength=48,strategy=6" +roundTripTest -g512K 19 + + $ECHO "\n**** Pass-Through mode **** " $ECHO "Hello world 1!" | $ZSTD -df $ECHO "Hello world 2!" | $ZSTD -dcf From ab5ed6fa7fa7f1f917c19894e9c444dc8a1a2e48 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 14 Dec 2016 17:10:38 +0100 Subject: [PATCH 08/17] improved playTests.sh --- tests/playTests.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/playTests.sh b/tests/playTests.sh index c5bc577d..6f7891fb 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -106,6 +106,8 @@ $ZSTD --zstd=windowLog=21, && die "wrong parameters not detected!" $ZSTD --zstd=windowLok=21 && die "wrong parameters not detected!" $ZSTD --zstd=windowLog=21,slog= && die "wrong parameters not detected!" roundTripTest -g512K +roundTripTest -g512K " --zstd=slen=3,tlen=48,strat=6" +roundTripTest -g512K " --zstd=strat=6,wlog=23,clog=23,hlog=22,slog=6" roundTripTest -g512K " --zstd=windowLog=23,chainLog=23,hashLog=22,searchLog=6,searchLength=3,targetLength=48,strategy=6" roundTripTest -g512K 19 From 24a4236111d28b354b3712de942ae03b3ef8c161 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 14 Dec 2016 18:07:31 +0100 Subject: [PATCH 09/17] improved playTests.sh (2) --- tests/playTests.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/playTests.sh b/tests/playTests.sh index 6f7891fb..b98acfb9 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -27,7 +27,6 @@ case "$OS" in Windows*) isWindows=true ECHO="echo -e" - INTOVOID="nul" ;; esac @@ -102,9 +101,10 @@ ls tmp.zst && die "tmp.zst should not be created" $ECHO "\n**** Advanced compression parameters **** " -$ZSTD --zstd=windowLog=21, && die "wrong parameters not detected!" -$ZSTD --zstd=windowLok=21 && die "wrong parameters not detected!" -$ZSTD --zstd=windowLog=21,slog= && die "wrong parameters not detected!" +$ECHO "Hello world!" | $ZSTD --zstd=windowLog=21, - -o tmp.zst && die "wrong parameters not detected!" +$ECHO "Hello world!" | $ZSTD --zstd=windowLo=21 - -o tmp.zst && die "wrong parameters not detected!" +$ECHO "Hello world!" | $ZSTD --zstd=windowLog=21,slog= - -o tmp.zst && die "wrong parameters not detected!" +ls tmp.zst && die "tmp.zst should not be created" roundTripTest -g512K roundTripTest -g512K " --zstd=slen=3,tlen=48,strat=6" roundTripTest -g512K " --zstd=strat=6,wlog=23,clog=23,hlog=22,slog=6" From f9a56668a697885d9b7b217c9ef377085f23a87b Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 14 Dec 2016 18:43:06 +0100 Subject: [PATCH 10/17] improved playTests.sh (3 --- tests/playTests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/playTests.sh b/tests/playTests.sh index b98acfb9..89ff45f3 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -103,7 +103,7 @@ ls tmp.zst && die "tmp.zst should not be created" $ECHO "\n**** Advanced compression parameters **** " $ECHO "Hello world!" | $ZSTD --zstd=windowLog=21, - -o tmp.zst && die "wrong parameters not detected!" $ECHO "Hello world!" | $ZSTD --zstd=windowLo=21 - -o tmp.zst && die "wrong parameters not detected!" -$ECHO "Hello world!" | $ZSTD --zstd=windowLog=21,slog= - -o tmp.zst && die "wrong parameters not detected!" +$ECHO "Hello world!" | $ZSTD --zstd=windowLog=21,slog - -o tmp.zst && die "wrong parameters not detected!" ls tmp.zst && die "tmp.zst should not be created" roundTripTest -g512K roundTripTest -g512K " --zstd=slen=3,tlen=48,strat=6" From 8de46ab51ab7c2c54734c7fe9f2c1cfbaa3e3bc3 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Fri, 16 Dec 2016 13:27:30 -0800 Subject: [PATCH 11/17] Export all API functions --- build/VS2005/zstdlib/zstdlib.vcproj | 4 +++ build/VS2010/libzstd-dll/libzstd-dll.vcxproj | 1 + build/VS2010/libzstd/libzstd.vcxproj | 1 + build/cmake/lib/CMakeLists.txt | 1 + lib/common/zstd_common.c | 4 --- lib/common/zstd_errors.h | 18 ++++++++++++-- lib/deprecated/zbuff_common.c | 26 ++++++++++++++++++++ lib/dictBuilder/zdict.h | 23 +++++++++-------- lib/zstd.h | 21 +++++++++------- 9 files changed, 74 insertions(+), 25 deletions(-) create mode 100644 lib/deprecated/zbuff_common.c diff --git a/build/VS2005/zstdlib/zstdlib.vcproj b/build/VS2005/zstdlib/zstdlib.vcproj index 1b78986b..d95212b3 100644 --- a/build/VS2005/zstdlib/zstdlib.vcproj +++ b/build/VS2005/zstdlib/zstdlib.vcproj @@ -359,6 +359,10 @@ RelativePath="..\..\..\lib\common\xxhash.c" > + + diff --git a/build/VS2010/libzstd-dll/libzstd-dll.vcxproj b/build/VS2010/libzstd-dll/libzstd-dll.vcxproj index f8527197..f1ea5c82 100644 --- a/build/VS2010/libzstd-dll/libzstd-dll.vcxproj +++ b/build/VS2010/libzstd-dll/libzstd-dll.vcxproj @@ -29,6 +29,7 @@ + diff --git a/build/VS2010/libzstd/libzstd.vcxproj b/build/VS2010/libzstd/libzstd.vcxproj index f5f30d73..228e83da 100644 --- a/build/VS2010/libzstd/libzstd.vcxproj +++ b/build/VS2010/libzstd/libzstd.vcxproj @@ -29,6 +29,7 @@ + diff --git a/build/cmake/lib/CMakeLists.txt b/build/cmake/lib/CMakeLists.txt index abcf4ffe..65942b41 100644 --- a/build/cmake/lib/CMakeLists.txt +++ b/build/cmake/lib/CMakeLists.txt @@ -69,6 +69,7 @@ SET(Sources ${LIBRARY_DIR}/decompress/zstd_decompress.c ${LIBRARY_DIR}/dictBuilder/divsufsort.c ${LIBRARY_DIR}/dictBuilder/zdict.c + ${LIBRARY_DIR}/deprecated/zbuff_common.c ${LIBRARY_DIR}/deprecated/zbuff_compress.c ${LIBRARY_DIR}/deprecated/zbuff_decompress.c) diff --git a/lib/common/zstd_common.c b/lib/common/zstd_common.c index f30128c7..749b2870 100644 --- a/lib/common/zstd_common.c +++ b/lib/common/zstd_common.c @@ -43,10 +43,6 @@ ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); } * provides error code string from enum */ const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorName(code); } -/* --- ZBUFF Error Management (deprecated) --- */ -unsigned ZBUFF_isError(size_t errorCode) { return ERR_isError(errorCode); } -const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); } - /*=************************************************************** * Custom allocator diff --git a/lib/common/zstd_errors.h b/lib/common/zstd_errors.h index 50dc4f72..949dbd0f 100644 --- a/lib/common/zstd_errors.h +++ b/lib/common/zstd_errors.h @@ -18,6 +18,20 @@ extern "C" { #include /* size_t */ +/* ===== ZSTDERRORLIB_API : control library symbols visibility ===== */ +#if defined(__GNUC__) && (__GNUC__ >= 4) +# define ZSTDERRORLIB_VISIBILITY __attribute__ ((visibility ("default"))) +#else +# define ZSTDERRORLIB_VISIBILITY +#endif +#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZSTDERRORLIB_API __declspec(dllexport) ZSTDERRORLIB_VISIBILITY +#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) +# define ZSTDERRORLIB_API __declspec(dllimport) ZSTDERRORLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +#else +# define ZSTDERRORLIB_API ZSTDERRORLIB_VISIBILITY +#endif + /*-**************************************** * error codes list ******************************************/ @@ -49,8 +63,8 @@ typedef enum { /*! ZSTD_getErrorCode() : convert a `size_t` function result into a `ZSTD_ErrorCode` enum type, which can be used to compare directly with enum list published into "error_public.h" */ -ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult); -const char* ZSTD_getErrorString(ZSTD_ErrorCode code); +ZSTDERRORLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult); +ZSTDERRORLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code); #if defined (__cplusplus) diff --git a/lib/deprecated/zbuff_common.c b/lib/deprecated/zbuff_common.c new file mode 100644 index 00000000..9fff6eb2 --- /dev/null +++ b/lib/deprecated/zbuff_common.c @@ -0,0 +1,26 @@ +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + +/*-************************************* +* Dependencies +***************************************/ +#include "error_private.h" +#include "zbuff.h" + +/*-**************************************** +* ZBUFF Error Management (deprecated) +******************************************/ + +/*! ZBUFF_isError() : +* tells if a return value is an error code */ +unsigned ZBUFF_isError(size_t errorCode) { return ERR_isError(errorCode); } +/*! ZBUFF_getErrorName() : +* provides error code string from function result (useful for debugging) */ +const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); } + diff --git a/lib/dictBuilder/zdict.h b/lib/dictBuilder/zdict.h index 642a4351..d6cf1839 100644 --- a/lib/dictBuilder/zdict.h +++ b/lib/dictBuilder/zdict.h @@ -19,15 +19,18 @@ extern "C" { #include /* size_t */ -/*====== Export for Windows ======*/ -/*! -* ZSTD_DLL_EXPORT : -* Enable exporting of functions when building a Windows DLL -*/ -#if defined(_WIN32) && defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) -# define ZDICTLIB_API __declspec(dllexport) +/* ===== ZDICTLIB_API : control library symbols visibility ===== */ +#if defined(__GNUC__) && (__GNUC__ >= 4) +# define ZDICTLIB_VISIBILITY __attribute__ ((visibility ("default"))) #else -# define ZDICTLIB_API +# define ZDICTLIB_VISIBILITY +#endif +#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZDICTLIB_API __declspec(dllexport) ZDICTLIB_VISIBILITY +#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) +# define ZDICTLIB_API __declspec(dllimport) ZDICTLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +#else +# define ZDICTLIB_API ZDICTLIB_VISIBILITY #endif @@ -79,7 +82,7 @@ typedef struct { or an error code, which can be tested by ZDICT_isError(). note : ZDICT_trainFromBuffer_advanced() will send notifications into stderr if instructed to, using notificationLevel>0. */ -size_t ZDICT_trainFromBuffer_advanced(void* dictBuffer, size_t dictBufferCapacity, +ZDICTLIB_API size_t ZDICT_trainFromBuffer_advanced(void* dictBuffer, size_t dictBufferCapacity, const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, ZDICT_params_t parameters); @@ -97,7 +100,7 @@ size_t ZDICT_trainFromBuffer_advanced(void* dictBuffer, size_t dictBufferCapacit starting from its beginning. @return : size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`). */ -size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity, +ZDICTLIB_API size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity, const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples); diff --git a/lib/zstd.h b/lib/zstd.h index b02e16b4..64e782b5 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -20,13 +20,16 @@ extern "C" { /* ===== ZSTDLIB_API : control library symbols visibility ===== */ #if defined(__GNUC__) && (__GNUC__ >= 4) -# define ZSTDLIB_API __attribute__ ((visibility ("default"))) -#elif defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) -# define ZSTDLIB_API __declspec(dllexport) -#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) -# define ZSTDLIB_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +# define ZSTDLIB_VISIBILITY __attribute__ ((visibility ("default"))) #else -# define ZSTDLIB_API +# define ZSTDLIB_VISIBILITY +#endif +#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZSTDLIB_API __declspec(dllexport) ZSTDLIB_VISIBILITY +#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) +# define ZSTDLIB_API __declspec(dllimport) ZSTDLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +#else +# define ZSTDLIB_API ZSTDLIB_VISIBILITY #endif @@ -463,13 +466,13 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); * Provides the dictID stored within dictionary. * if @return == 0, the dictionary is not conformant with Zstandard specification. * It can still be loaded, but as a content-only dictionary. */ -unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize); +ZSTDLIB_API unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize); /*! ZSTD_getDictID_fromDDict() : * Provides the dictID of the dictionary loaded into `ddict`. * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ -unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict); +ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict); /*! ZSTD_getDictID_fromFrame() : * Provides the dictID required to decompressed the frame stored within `src`. @@ -481,7 +484,7 @@ unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict); * - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`). * - This is not a Zstandard frame. * When identifying the exact failure cause, it's possible to used ZSTD_getFrameParams(), which will provide a more precise error code. */ -unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize); +ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize); /******************************************************************** From 61e62c014f59757ae4af814ef3a096d060fe4781 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Fri, 16 Dec 2016 13:29:23 -0800 Subject: [PATCH 12/17] Test that all API symbols are exported --- .travis.yml | 2 +- appveyor.yml | 3 +- tests/.gitignore | 1 + tests/Makefile | 19 ++++++- tests/symbols.c | 144 +++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 164 insertions(+), 5 deletions(-) create mode 100644 tests/symbols.c diff --git a/.travis.yml b/.travis.yml index 148a98f1..92c07718 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,7 @@ matrix: os: linux sudo: false - - env: Ubu=12.04cont Cmd="make zlibwrapper && make clean && make -C tests test-zstd-nolegacy && make clean && make cmaketest && make clean && make -C contrib/pzstd googletest pzstd tests check && make -C contrib/pzstd clean" + - env: Ubu=12.04cont Cmd="make zlibwrapper && make clean && make -C tests test-symbols && make clean && make -C tests test-zstd-nolegacy && make clean && make cmaketest && make clean && make -C contrib/pzstd googletest pzstd tests check && make -C contrib/pzstd clean" os: linux sudo: false language: cpp diff --git a/appveyor.yml b/appveyor.yml index 27c83c04..1af9da71 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -63,7 +63,8 @@ build_script: make -v && cc -v && ECHO make %MAKE_PARAMS% && - make %MAKE_PARAMS% + make %MAKE_PARAMS% && + make -C tests test-symbols ) - if [%COMPILER%]==[gcc] if [%PLATFORM%]==[mingw64] ( COPY programs\zstd.exe bin\zstd.exe && diff --git a/tests/.gitignore b/tests/.gitignore index b558ac25..59a7a1bd 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -12,6 +12,7 @@ paramgrill paramgrill32 roundTripCrash longmatch +symbols # Tmp test directory zstdtest diff --git a/tests/Makefile b/tests/Makefile index d7d25026..b27d1cc6 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -28,7 +28,7 @@ PYTHON ?= python3 TESTARTEFACT := versionsTest namespaceTest -CPPFLAGS= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/dictBuilder -I$(PRGDIR) +CPPFLAGS= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR) CFLAGS ?= -O3 CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 \ -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef @@ -127,6 +127,15 @@ roundTripCrash : $(ZSTD_FILES) roundTripCrash.c longmatch : $(ZSTD_FILES) longmatch.c $(CC) $(FLAGS) $^ -o $@$(EXT) +symbols : symbols.c + $(MAKE) -C $(ZSTDDIR) libzstd +ifneq (,$(filter Windows%,$(OS))) + cp $(ZSTDDIR)/dll/libzstd.dll . + $(CC) $(FLAGS) $^ -o $@$(EXT) -DZSTD_DLL_IMPORT=1 libzstd.dll +else + $(CC) $(FLAGS) $^ -o $@$(EXT) -Wl,-rpath=$(ZSTDDIR) $(ZSTDDIR)/libzstd.so +endif + namespaceTest: if $(CC) namespaceTest.c ../lib/common/xxhash.c -o $@ ; then echo compilation should fail; exit 1 ; fi $(RM) $@ @@ -142,8 +151,9 @@ clean: fullbench$(EXT) fullbench32$(EXT) \ fullbench-lib$(EXT) fullbench-dll$(EXT) \ fuzzer$(EXT) fuzzer32$(EXT) zbufftest$(EXT) zbufftest32$(EXT) \ - zstreamtest$(EXT) zstreamtest32$(EXT) \ - datagen$(EXT) paramgrill$(EXT) roundTripCrash$(EXT) longmatch$(EXT) + zstreamtest$(EXT) zstreamtest32$(EXT) \ + datagen$(EXT) paramgrill$(EXT) roundTripCrash$(EXT) longmatch$(EXT) \ + symbols$(EXT) @echo Cleaning completed @@ -243,4 +253,7 @@ test-zstream32: zstreamtest32 test-longmatch: longmatch $(QEMU_SYS) ./longmatch +test-symbols: symbols + $(QEMU_SYS) ./symbols + endif diff --git a/tests/symbols.c b/tests/symbols.c new file mode 100644 index 00000000..8d03df2f --- /dev/null +++ b/tests/symbols.c @@ -0,0 +1,144 @@ +#include +#include "zstd_errors.h" +#define ZSTD_STATIC_LINKING_ONLY +#include "zstd.h" +#define ZBUFF_DISABLE_DEPRECATE_WARNINGS +#define ZBUFF_STATIC_LINKING_ONLY +#include "zbuff.h" +#define ZDICT_STATIC_LINKING_ONLY +#include "zdict.h" + +static const void *symbols[] = { +/* zstd.h */ + &ZSTD_versionNumber, + &ZSTD_compress, + &ZSTD_decompress, + &ZSTD_getDecompressedSize, + &ZSTD_maxCLevel, + &ZSTD_compressBound, + &ZSTD_isError, + &ZSTD_getErrorName, + &ZSTD_createCCtx, + &ZSTD_freeCCtx, + &ZSTD_compressCCtx, + &ZSTD_createDCtx, + &ZSTD_freeDCtx, + &ZSTD_decompressDCtx, + &ZSTD_compress_usingDict, + &ZSTD_decompress_usingDict, + &ZSTD_createCDict, + &ZSTD_freeCDict, + &ZSTD_compress_usingCDict, + &ZSTD_createDDict, + &ZSTD_freeDDict, + &ZSTD_decompress_usingDDict, + &ZSTD_createCStream, + &ZSTD_freeCStream, + &ZSTD_initCStream, + &ZSTD_compressStream, + &ZSTD_flushStream, + &ZSTD_endStream, + &ZSTD_CStreamInSize, + &ZSTD_CStreamOutSize, + &ZSTD_createDStream, + &ZSTD_freeDStream, + &ZSTD_initDStream, + &ZSTD_decompressStream, + &ZSTD_DStreamInSize, + &ZSTD_DStreamOutSize, +/* zstd.h: advanced functions */ + &ZSTD_estimateCCtxSize, + &ZSTD_createCCtx_advanced, + &ZSTD_sizeof_CCtx, + &ZSTD_createCDict_advanced, + &ZSTD_sizeof_CDict, + &ZSTD_getCParams, + &ZSTD_getParams, + &ZSTD_checkCParams, + &ZSTD_adjustCParams, + &ZSTD_compress_advanced, + &ZSTD_isFrame, + &ZSTD_estimateDCtxSize, + &ZSTD_createDCtx_advanced, + &ZSTD_sizeof_DCtx, + &ZSTD_sizeof_DDict, + &ZSTD_getDictID_fromDict, + &ZSTD_getDictID_fromDDict, + &ZSTD_getDictID_fromFrame, + &ZSTD_createCStream_advanced, + &ZSTD_initCStream_srcSize, + &ZSTD_initCStream_usingDict, + &ZSTD_initCStream_advanced, + &ZSTD_initCStream_usingCDict, + &ZSTD_resetCStream, + &ZSTD_sizeof_CStream, + &ZSTD_createDStream_advanced, + &ZSTD_initDStream_usingDict, + &ZSTD_setDStreamParameter, + &ZSTD_initDStream_usingDDict, + &ZSTD_resetDStream, + &ZSTD_sizeof_DStream, + &ZSTD_compressBegin, + &ZSTD_compressBegin_usingDict, + &ZSTD_compressBegin_advanced, + &ZSTD_copyCCtx, + &ZSTD_compressContinue, + &ZSTD_compressEnd, + &ZSTD_getFrameParams, + &ZSTD_decompressBegin, + &ZSTD_decompressBegin_usingDict, + &ZSTD_copyDCtx, + &ZSTD_nextSrcSizeToDecompress, + &ZSTD_decompressContinue, + &ZSTD_nextInputType, + &ZSTD_getBlockSizeMax, + &ZSTD_compressBlock, + &ZSTD_decompressBlock, + &ZSTD_insertBlock, +/* zstd_errors.h */ + &ZSTD_getErrorCode, + &ZSTD_getErrorString, +/* zbuff.h */ + &ZBUFF_createCCtx, + &ZBUFF_freeCCtx, + &ZBUFF_compressInit, + &ZBUFF_compressInitDictionary, + &ZBUFF_compressContinue, + &ZBUFF_compressFlush, + &ZBUFF_compressEnd, + &ZBUFF_createDCtx, + &ZBUFF_freeDCtx, + &ZBUFF_decompressInit, + &ZBUFF_decompressInitDictionary, + &ZBUFF_decompressContinue, + &ZBUFF_isError, + &ZBUFF_getErrorName, + &ZBUFF_recommendedCInSize, + &ZBUFF_recommendedCOutSize, + &ZBUFF_recommendedDInSize, + &ZBUFF_recommendedDOutSize, +/* zbuff.h: advanced functions */ + &ZBUFF_createCCtx_advanced, + &ZBUFF_createDCtx_advanced, + &ZBUFF_compressInit_advanced, +/* zdict.h */ + &ZDICT_trainFromBuffer, + &ZDICT_getDictID, + &ZDICT_isError, + &ZDICT_getErrorName, +/* zdict.h: advanced functions */ + &ZDICT_trainFromBuffer_advanced, + &ZDICT_addEntropyTablesFromBuffer, + NULL, +}; + +int main(int argc, const char** argv) { + const void **symbol; + (void)argc; + (void)argv; + + for (symbol = symbols; *symbol != NULL; ++symbol) { + printf("%p\n", *symbol); + } + return 0; +} From d46ecb58a5d537fad672fdc15b3f91da8e0d17be Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 17 Dec 2016 16:28:12 +0100 Subject: [PATCH 13/17] added dll compilation tests --- .travis.yml | 22 ++++++++++++++++++---- NEWS | 3 +++ lib/dictBuilder/zdict.c | 5 ++--- lib/zstd.h | 2 +- programs/dibio.c | 2 +- tests/.gitignore | 2 ++ tests/Makefile | 27 ++++++++++++++++++++------- 7 files changed, 47 insertions(+), 16 deletions(-) diff --git a/.travis.yml b/.travis.yml index 92c07718..ef1e67e3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -69,7 +69,7 @@ matrix: # Ubuntu 14.04 LTS Server Edition 64 bit - - env: Ubu=14.04 Cmd="make armtest && make clean && make aarch64test" + - env: Ubu=14.04 Cmd="make armtest" dist: trusty sudo: required addons: @@ -78,7 +78,17 @@ matrix: - qemu-system-arm - qemu-user-static - gcc-arm-linux-gnueabi - - libc6-dev-armel-cross + - libc6-dev-armel-cross + + # Ubuntu 14.04 LTS Server Edition 64 bit + - env: Ubu=14.04 Cmd="make aarch64test" + dist: trusty + sudo: required + addons: + apt: + packages: + - qemu-system-arm + - qemu-user-static - gcc-aarch64-linux-gnu - libc6-dev-arm64-cross @@ -91,6 +101,7 @@ matrix: - qemu-system-ppc - qemu-user-static - gcc-powerpc-linux-gnu + - libc6-dev-armel-cross - env: Ubu=14.04 Cmd='make -C lib all && CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make -C tests valgrindTest' os: linux @@ -101,7 +112,10 @@ matrix: packages: - valgrind - - env: Ubu=14.04 Cmd="make gpptest && make clean && make gnu90test && make clean && make c99test && make clean && make gnu99test && make clean && make clangtest && make clean && make -C contrib/pzstd googletest32 && make -C contrib/pzstd all32 && make -C contrib/pzstd check && make -C contrib/pzstd clean" + - env: Ubu=14.04 Cmd="make gpptest && make clean && make gnu90test && make clean \ + && make c99test && make clean && make gnu99test && make clean \ + && make clangtest && make clean && make -C contrib/pzstd googletest32 \ + && make -C contrib/pzstd all32 && make -C contrib/pzstd check && make -C contrib/pzstd clean" os: linux dist: trusty sudo: required @@ -127,7 +141,7 @@ matrix: - libc6-dev-i386 - gcc-multilib - - env: Ubu=14.04 Cmd="make gcc5test && make clean && make gcc6test" + - env: Ubu=14.04 Cmd="make gcc5test && make clean && make gcc6test && make clean && make -C tests dll" os: linux dist: trusty sudo: required diff --git a/NEWS b/NEWS index 9b000163..0fc90aff 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,6 @@ +v1.1.3 +API : fix : all symbols properly exposed in libzstd, by Nick Terrell + v1.1.2 API : streaming : decompression : changed : automatic implicit reset when chain-decoding new frames without init API : experimental : added : dictID retrieval functions, and ZSTD_initCStream_srcSize() diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c index ea50b54a..b539c09f 100644 --- a/lib/dictBuilder/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -36,12 +36,11 @@ #include /* clock */ #include "mem.h" /* read */ -#include "error_private.h" #include "fse.h" /* FSE_normalizeCount, FSE_writeNCount */ #define HUF_STATIC_LINKING_ONLY -#include "huf.h" +#include "huf.h" /* HUF_buildCTable, HUF_writeCTable */ #include "zstd_internal.h" /* includes zstd.h */ -#include "xxhash.h" +#include "xxhash.h" /* XXH64 */ #include "divsufsort.h" #ifndef ZDICT_STATIC_LINKING_ONLY # define ZDICT_STATIC_LINKING_ONLY diff --git a/lib/zstd.h b/lib/zstd.h index 64e782b5..7eda6987 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -56,7 +56,7 @@ extern "C" { /*------ Version ------*/ #define ZSTD_VERSION_MAJOR 1 #define ZSTD_VERSION_MINOR 1 -#define ZSTD_VERSION_RELEASE 2 +#define ZSTD_VERSION_RELEASE 3 #define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE #define ZSTD_QUOTE(str) #str diff --git a/programs/dibio.c b/programs/dibio.c index a793669c..6fce405b 100644 --- a/programs/dibio.c +++ b/programs/dibio.c @@ -183,7 +183,7 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize, size_t* const fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t)); unsigned long long const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles); size_t const maxMem = DiB_findMaxMem(totalSizeToLoad * MEMMULT) / MEMMULT; - size_t benchedSize = MIN (maxMem, (size_t)totalSizeToLoad); + size_t benchedSize = (size_t) MIN ((unsigned long long)maxMem, totalSizeToLoad); void* const srcBuffer = malloc(benchedSize+NOISELENGTH); int result = 0; diff --git a/tests/.gitignore b/tests/.gitignore index 59a7a1bd..586c17b6 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -3,10 +3,12 @@ fullbench fullbench32 fuzzer fuzzer32 +fuzzer-dll zbufftest zbufftest32 zstreamtest zstreamtest32 +zstreamtest-dll datagen paramgrill paramgrill32 diff --git a/tests/Makefile b/tests/Makefile index b27d1cc6..6e37d160 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -28,12 +28,12 @@ PYTHON ?= python3 TESTARTEFACT := versionsTest namespaceTest -CPPFLAGS= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR) -CFLAGS ?= -O3 -CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 \ +CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR) +CFLAGS ?= -O3 +CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 \ -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef -CFLAGS += $(MOREFLAGS) -FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) +CFLAGS += $(MOREFLAGS) +FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) ZSTDCOMMON_FILES := $(ZSTDDIR)/common/*.c @@ -65,6 +65,8 @@ all: fullbench fuzzer zstreamtest paramgrill datagen zbufftest all32: fullbench32 fuzzer32 zstreamtest32 zbufftest32 +dll: fuzzer-dll zstreamtest-dll + zstd: @@ -93,11 +95,16 @@ fullbench-dll: $(PRGDIR)/datagen.c fullbench.c $(MAKE) -C $(ZSTDDIR) libzstd $(CC) $(FLAGS) $^ -o $@$(EXT) -DZSTD_DLL_IMPORT=1 $(ZSTDDIR)/dll/libzstd.dll -fuzzer : $(ZSTD_FILES) $(ZDICT_FILES) $(PRGDIR)/datagen.c fuzzer.c +fuzzer : $(ZSTD_FILES) $(ZDICT_FILES) $(PRGDIR)/datagen.c fuzzer.c $(CC) $(FLAGS) $^ -o $@$(EXT) fuzzer32 : $(ZSTD_FILES) $(ZDICT_FILES) $(PRGDIR)/datagen.c fuzzer.c - $(CC) -m32 $(FLAGS) $^ -o $@$(EXT) + $(CC) -m32 $(FLAGS) $^ -o $@$(EXT) + +fuzzer-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd +fuzzer-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/datagen.c fuzzer.c + $(MAKE) -C $(ZSTDDIR) libzstd + $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@$(EXT) zbufftest : CPPFLAGS += -I$(ZSTDDIR)/deprecated zbufftest : CFLAGS += -Wno-deprecated-declarations # required to silence deprecation warnings @@ -115,6 +122,11 @@ zstreamtest : $(ZSTD_FILES) $(PRGDIR)/datagen.c zstreamtest.c zstreamtest32 : $(ZSTD_FILES) $(PRGDIR)/datagen.c zstreamtest.c $(CC) -m32 $(FLAGS) $^ -o $@$(EXT) +zstreamtest-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd +zstreamtest-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/datagen.c zstreamtest.c + $(MAKE) -C $(ZSTDDIR) libzstd + $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@$(EXT) + paramgrill : $(ZSTD_FILES) $(PRGDIR)/datagen.c paramgrill.c $(CC) $(FLAGS) $^ -lm -o $@$(EXT) @@ -151,6 +163,7 @@ clean: fullbench$(EXT) fullbench32$(EXT) \ fullbench-lib$(EXT) fullbench-dll$(EXT) \ fuzzer$(EXT) fuzzer32$(EXT) zbufftest$(EXT) zbufftest32$(EXT) \ + fuzzer-dll$(EXT) zstreamtest-dll$(EXT) \ zstreamtest$(EXT) zstreamtest32$(EXT) \ datagen$(EXT) paramgrill$(EXT) roundTripCrash$(EXT) longmatch$(EXT) \ symbols$(EXT) From 31ff2a23be05bd01f8dfebb2015eaec34b9aa1a3 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 17 Dec 2016 19:10:10 +0100 Subject: [PATCH 14/17] fix Travis long test list; added zbufftest-dll --- .travis.yml | 6 +++--- tests/Makefile | 13 ++++++++++--- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index ef1e67e3..6bf99f1b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -112,9 +112,9 @@ matrix: packages: - valgrind - - env: Ubu=14.04 Cmd="make gpptest && make clean && make gnu90test && make clean \ - && make c99test && make clean && make gnu99test && make clean \ - && make clangtest && make clean && make -C contrib/pzstd googletest32 \ + - env: Ubu=14.04 Cmd="make gpptest && make clean && make gnu90test && make clean + && make c99test && make clean && make gnu99test && make clean + && make clangtest && make clean && make -C contrib/pzstd googletest32 && make -C contrib/pzstd all32 && make -C contrib/pzstd check && make -C contrib/pzstd clean" os: linux dist: trusty diff --git a/tests/Makefile b/tests/Makefile index 6e37d160..abf89f2f 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -57,7 +57,7 @@ ZSTREAM_TESTTIME = -T2mn FUZZERTEST= -T5mn ZSTDRTTEST= --test-large-data -.PHONY: default all all32 clean test test32 test-all namespaceTest versionsTest +.PHONY: default all all32 dll clean test test32 test-all namespaceTest versionsTest default: fullbench @@ -65,7 +65,7 @@ all: fullbench fuzzer zstreamtest paramgrill datagen zbufftest all32: fullbench32 fuzzer32 zstreamtest32 zbufftest32 -dll: fuzzer-dll zstreamtest-dll +dll: fuzzer-dll zstreamtest-dll zbufftest-dll @@ -116,6 +116,13 @@ zbufftest32 : CFLAGS += -Wno-deprecated-declarations -m32 zbufftest32 : $(ZSTD_FILES) $(ZBUFF_FILES) $(PRGDIR)/datagen.c zbufftest.c $(CC) $(FLAGS) $^ -o $@$(EXT) +zbufftest-dll : CPPFLAGS += -I$(ZSTDDIR)/deprecated +zbufftest-dll : CFLAGS += -Wno-deprecated-declarations # required to silence deprecation warnings +zbufftest-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd +zbufftest-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/datagen.c zbufftest.c + $(MAKE) -C $(ZSTDDIR) libzstd + $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@$(EXT) + zstreamtest : $(ZSTD_FILES) $(PRGDIR)/datagen.c zstreamtest.c $(CC) $(FLAGS) $^ -o $@$(EXT) @@ -163,7 +170,7 @@ clean: fullbench$(EXT) fullbench32$(EXT) \ fullbench-lib$(EXT) fullbench-dll$(EXT) \ fuzzer$(EXT) fuzzer32$(EXT) zbufftest$(EXT) zbufftest32$(EXT) \ - fuzzer-dll$(EXT) zstreamtest-dll$(EXT) \ + fuzzer-dll$(EXT) zstreamtest-dll$(EXT) zbufftest-dll$(EXT)\ zstreamtest$(EXT) zstreamtest32$(EXT) \ datagen$(EXT) paramgrill$(EXT) roundTripCrash$(EXT) longmatch$(EXT) \ symbols$(EXT) From 36321d2e329900d09c57c5ee6a4080234f8e333d Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 17 Dec 2016 19:37:55 +0100 Subject: [PATCH 15/17] updated NEWS --- NEWS | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS b/NEWS index 0fc90aff..9a341781 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,5 @@ v1.1.3 +cli : new : commands for advanced parameters, by Przemyslaw Skibinski API : fix : all symbols properly exposed in libzstd, by Nick Terrell v1.1.2 From 1496c3dc47fc0cf849fb82d7e6b13a1c19fdd34a Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 18 Dec 2016 11:58:23 +0100 Subject: [PATCH 16/17] Fix : size estimation when some samples are very large --- lib/dictBuilder/zdict.c | 2 +- programs/dibio.c | 29 +++++++++++++++++++++++++---- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c index b539c09f..921e3788 100644 --- a/lib/dictBuilder/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -569,7 +569,7 @@ static void ZDICT_countEStats(EStats_ress_t esr, ZSTD_parameters params, if (ZSTD_isError(errorCode)) { DISPLAYLEVEL(1, "warning : ZSTD_copyCCtx failed \n"); return; } } cSize = ZSTD_compressBlock(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_ABSOLUTEMAX, src, srcSize); - if (ZSTD_isError(cSize)) { DISPLAYLEVEL(1, "warning : could not compress sample size %u \n", (U32)srcSize); return; } + if (ZSTD_isError(cSize)) { DISPLAYLEVEL(3, "warning : could not compress sample size %u \n", (U32)srcSize); return; } if (cSize) { /* if == 0; block is not compressible */ const seqStore_t* seqStorePtr = ZSTD_getSeqStore(esr.zc); diff --git a/programs/dibio.c b/programs/dibio.c index 6fce405b..fa4241b1 100644 --- a/programs/dibio.c +++ b/programs/dibio.c @@ -31,7 +31,8 @@ #define MB *(1 <<20) #define GB *(1U<<30) -#define MEMMULT 11 +#define SAMPLESIZE_MAX (128 KB) +#define MEMMULT 11 /* rough estimation : memory cost to analyze 1 byte of sample */ static const size_t maxMemory = (sizeof(size_t) == 4) ? (2 GB - 64 MB) : ((size_t)(512 MB) << sizeof(size_t)); #define NOISELENGTH 32 @@ -97,7 +98,7 @@ static unsigned DiB_loadFiles(void* buffer, size_t* bufferSizePtr, for (n=0; n *bufferSizePtr-pos) break; { FILE* const f = fopen(fileName, "rb"); if (f==NULL) EXM_THROW(10, "zstd: dictBuilder: %s %s ", fileName, strerror(errno)); @@ -163,6 +164,21 @@ static void DiB_saveDict(const char* dictFileName, } +static int g_tooLargeSamples = 0; +static U64 DiB_getTotalCappedFileSize(const char** fileNamesTable, unsigned nbFiles) +{ + U64 total = 0; + unsigned n; + for (n=0; n 2*SAMPLESIZE_MAX); + } + return total; +} + + /*! ZDICT_trainFromBuffer_unsafe() : Strictly Internal use only !! Same as ZDICT_trainFromBuffer_advanced(), but does not control `samplesBuffer`. @@ -181,7 +197,7 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize, { void* const dictBuffer = malloc(maxDictSize); size_t* const fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t)); - unsigned long long const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles); + unsigned long long const totalSizeToLoad = DiB_getTotalCappedFileSize(fileNamesTable, nbFiles); size_t const maxMem = DiB_findMaxMem(totalSizeToLoad * MEMMULT) / MEMMULT; size_t benchedSize = (size_t) MIN ((unsigned long long)maxMem, totalSizeToLoad); void* const srcBuffer = malloc(benchedSize+NOISELENGTH); @@ -190,7 +206,12 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize, /* Checks */ if ((!fileSizes) || (!srcBuffer) || (!dictBuffer)) EXM_THROW(12, "not enough memory for DiB_trainFiles"); /* should not happen */ g_displayLevel = params.notificationLevel; - if (nbFiles < 5) { + if (g_tooLargeSamples) { + DISPLAYLEVEL(2, "! Warning : some samples are very large \n"); + DISPLAYLEVEL(2, "! Note that dictionary is only useful for small files or beginning of large files. \n"); + DISPLAYLEVEL(2, "! As a consequence, only the first %u bytes of each file are loaded \n", SAMPLESIZE_MAX); + } + if ((nbFiles < 5) || (totalSizeToLoad < 9 * (unsigned long long)maxDictSize)) { DISPLAYLEVEL(2, "! Warning : nb of samples too low for proper processing ! \n"); DISPLAYLEVEL(2, "! Please provide _one file per sample_. \n"); DISPLAYLEVEL(2, "! Do not concatenate samples together into a single file, \n"); From d564faa3c6222c9c485f83160c3f2be3b4a82495 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 18 Dec 2016 21:39:15 +0100 Subject: [PATCH 17/17] fix : ZSTD_initCStream_srcSize() correctly set srcSize in frame header --- lib/compress/zstd_compress.c | 5 +++-- tests/zstreamtest.c | 8 ++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 3d10fbd9..7fd73baa 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2900,7 +2900,7 @@ size_t ZSTD_CStreamOutSize(void) { return ZSTD_compressBound(ZSTD_BLOCKSIZE_ABSO size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize) { - if (zcs->inBuffSize==0) return ERROR(stage_wrong); /* zcs has not been init at least once */ + if (zcs->inBuffSize==0) return ERROR(stage_wrong); /* zcs has not been init at least once => can't reset */ if (zcs->cdict) CHECK_F(ZSTD_compressBegin_usingCDict(zcs->cctx, zcs->cdict, pledgedSrcSize)) else CHECK_F(ZSTD_compressBegin_advanced(zcs->cctx, NULL, 0, zcs->params, pledgedSrcSize)); @@ -2967,7 +2967,8 @@ size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t di size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize) { - ZSTD_parameters const params = ZSTD_getParams(compressionLevel, pledgedSrcSize, 0); + ZSTD_parameters params = ZSTD_getParams(compressionLevel, pledgedSrcSize, 0); + if (pledgedSrcSize) params.fParams.contentSizeFlag = 1; return ZSTD_initCStream_advanced(zcs, NULL, 0, params, pledgedSrcSize); } diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index 7783fe11..ce619308 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -248,7 +248,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo /* _srcSize compression test */ DISPLAYLEVEL(4, "test%3i : compress_srcSize %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); ZSTD_initCStream_srcSize(zc, 1, CNBufferSize); - outBuff.dst = (char*)(compressedBuffer)+cSize; + outBuff.dst = (char*)(compressedBuffer); outBuff.size = compressedBufferSize; outBuff.pos = 0; inBuff.src = CNBuffer; @@ -259,12 +259,16 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */ { size_t const r = ZSTD_endStream(zc, &outBuff); if (r != 0) goto _output_error; } /* error, or some data not flushed */ + { unsigned long long origSize = ZSTD_getDecompressedSize(outBuff.dst, outBuff.pos); + DISPLAY("outBuff.pos : %u \n", (U32)outBuff.pos); + DISPLAY("origSize = %u \n", (U32)origSize); + if ((size_t)origSize != CNBufferSize) goto _output_error; } /* exact original size must be present */ DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100); /* wrong _srcSize compression test */ DISPLAYLEVEL(4, "test%3i : wrong srcSize : %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH-1); ZSTD_initCStream_srcSize(zc, 1, CNBufferSize-1); - outBuff.dst = (char*)(compressedBuffer)+cSize; + outBuff.dst = (char*)(compressedBuffer); outBuff.size = compressedBufferSize; outBuff.pos = 0; inBuff.src = CNBuffer;