ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize,
int compressionLevel);
- When compressing multiple messages / blocks using the same dictionary, it's recommended to load it only once.
- ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup cost.
+
When compressing multiple messages or blocks using the same dictionary,
+ it's recommended to digest the dictionary only once, since it's a costly operation.
+ ZSTD_createCDict() will create a state from digesting a dictionary.
+ The resulting state can be used for future compression operations with very limited startup cost.
ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
- `dictBuffer` can be released after ZSTD_CDict creation, because its content is copied within CDict.
- Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate `dictBuffer` content.
- Note : A ZSTD_CDict can be created from an empty dictBuffer, but it is inefficient when used to compress small data.
+ @dictBuffer can be released after ZSTD_CDict creation, because its content is copied within CDict.
+ Note 1 : Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate @dictBuffer content.
+ Note 2 : A ZSTD_CDict can be created from an empty @dictBuffer,
+ in which case the only thing that it transports is the @compressionLevel.
+ This can be useful in a pipeline featuring ZSTD_compress_usingCDict() exclusively,
+ expecting a ZSTD_CDict parameter with any data, including those without a known dictionary.
size_t ZSTD_freeCDict(ZSTD_CDict* CDict);
@@ -947,7 +952,7 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
* to evolve and should be considered only in the context of extremely
* advanced performance tuning.
*
- * Zstd currently supports the use of a CDict in two ways:
+ * Zstd currently supports the use of a CDict in three ways:
*
* - The contents of the CDict can be copied into the working context. This
* means that the compression can search both the dictionary and input
@@ -963,6 +968,12 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
* working context's tables can be reused). For small inputs, this can be
* faster than copying the CDict's tables.
*
+ * - The CDict's tables are not used at all, and instead we use the working
+ * context alone to reload the dictionary and use params based on the source
+ * size. See ZSTD_compress_insertDictionary() and ZSTD_compress_usingDict().
+ * This method is effective when the dictionary sizes are very small relative
+ * to the input size, and the input size is fairly large to begin with.
+ *
* Zstd has a simple internal heuristic that selects which strategy to use
* at the beginning of a compression. However, if experimentation shows that
* Zstd is making poor choices, it is possible to override that choice with
@@ -970,7 +981,8 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
*/
ZSTD_dictDefaultAttach = 0, /* Use the default heuristic. */
ZSTD_dictForceAttach = 1, /* Never copy the dictionary. */
- ZSTD_dictForceCopy = 2 /* Always copy the dictionary. */
+ ZSTD_dictForceCopy = 2, /* Always copy the dictionary. */
+ ZSTD_dictForceLoad = 3 /* Always reload the dictionary */
} ZSTD_dictAttachPref_e;
typedef enum {
diff --git a/lib/Makefile b/lib/Makefile
index 87a396c5..273ceb90 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -244,8 +244,6 @@ libzstd.pc:
libzstd.pc: libzstd.pc.in
@echo creating pkgconfig
@sed -e 's|@PREFIX@|$(PREFIX)|' \
- -e 's|@LIBDIR@|$(LIBDIR)|' \
- -e 's|@INCLUDEDIR@|$(INCLUDEDIR)|' \
-e 's|@VERSION@|$(VERSION)|' \
$< >$@
diff --git a/lib/README.md b/lib/README.md
index 792729b1..0062c0d6 100644
--- a/lib/README.md
+++ b/lib/README.md
@@ -27,10 +27,10 @@ Enabling multithreading requires 2 conditions :
Both conditions are automatically applied when invoking `make lib-mt` target.
When linking a POSIX program with a multithreaded version of `libzstd`,
-note that it's necessary to request the `-pthread` flag during link stage.
+note that it's necessary to invoke the `-pthread` flag during link stage.
Multithreading capabilities are exposed
-via the [advanced API defined in `lib/zstd.h`](https://github.com/facebook/zstd/blob/v1.3.8/lib/zstd.h#L592).
+via the [advanced API defined in `lib/zstd.h`](https://github.com/facebook/zstd/blob/v1.4.3/lib/zstd.h#L351).
#### API
@@ -112,6 +112,17 @@ The file structure is designed to make this selection manually achievable for an
will expose the deprecated `ZSTDMT` API exposed by `zstdmt_compress.h` in
the shared library, which is now hidden by default.
+- The build macro `DYNAMIC_BMI2` can be set to 1 or 0 in order to generate binaries
+ which can detect at runtime the presence of BMI2 instructions, and use them only if present.
+ These instructions contribute to better performance, notably on the decoder side.
+ By default, this feature is automatically enabled on detecting
+ the right instruction set (x64) and compiler (clang or gcc >= 5).
+ It's obviously disabled for different cpus,
+ or when BMI2 instruction set is _required_ by the compiler command line
+ (in this case, only the BMI2 code path is generated).
+ Setting this macro will either force to generate the BMI2 dispatcher (1)
+ or prevent it (0). It overrides automatic detection.
+
#### Windows : using MinGW+MSYS to create DLL
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index ded25d8d..35346b92 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -50,6 +50,7 @@ struct ZSTD_CDict_s {
ZSTD_compressedBlockState_t cBlockState;
ZSTD_customMem customMem;
U32 dictID;
+ int compressionLevel; /* 0 indicates that advanced API was used to select CDict params */
}; /* typedef'd to ZSTD_CDict within "zstd.h" */
ZSTD_CCtx* ZSTD_createCCtx(void)
@@ -387,7 +388,7 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param)
case ZSTD_c_forceAttachDict:
ZSTD_STATIC_ASSERT(ZSTD_dictDefaultAttach < ZSTD_dictForceCopy);
bounds.lowerBound = ZSTD_dictDefaultAttach;
- bounds.upperBound = ZSTD_dictForceCopy; /* note : how to ensure at compile time that this is the highest value enum ? */
+ bounds.upperBound = ZSTD_dictForceLoad; /* note : how to ensure at compile time that this is the highest value enum ? */
return bounds;
case ZSTD_c_literalCompressionMode:
@@ -2770,7 +2771,7 @@ static size_t ZSTD_checkDictNCount(short* normalizedCounter, unsigned dictMaxSym
/*! ZSTD_loadZstdDictionary() :
* @return : dictID, or an error code
* assumptions : magic number supposed already checked
- * dictSize supposed > 8
+ * dictSize supposed >= 8
*/
static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs,
ZSTD_matchState_t* ms,
@@ -2787,7 +2788,7 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs,
size_t dictID;
ZSTD_STATIC_ASSERT(HUF_WORKSPACE_SIZE >= (1< 8);
+ assert(dictSize >= 8);
assert(MEM_readLE32(dictPtr) == ZSTD_MAGIC_DICTIONARY);
dictPtr += 4; /* skip magic number */
@@ -2889,7 +2890,10 @@ ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs,
void* workspace)
{
DEBUGLOG(4, "ZSTD_compress_insertDictionary (dictSize=%u)", (U32)dictSize);
- if ((dict==NULL) || (dictSize<=8)) return 0;
+ if ((dict==NULL) || (dictSize<8)) {
+ RETURN_ERROR_IF(dictContentType == ZSTD_dct_fullDict, dictionary_wrong);
+ return 0;
+ }
ZSTD_reset_compressedBlockState(bs);
@@ -2912,6 +2916,9 @@ ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs,
bs, ms, ws, params, dict, dictSize, dtlm, workspace);
}
+#define ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF (128 KB)
+#define ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER (6)
+
/*! ZSTD_compressBegin_internal() :
* @return : 0, or an error code */
static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
@@ -2926,17 +2933,27 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
/* params are supposed to be fully validated at this point */
assert(!ZSTD_isError(ZSTD_checkCParams(params->cParams)));
assert(!((dict) && (cdict))); /* either dict or cdict, not both */
-
- if (cdict && cdict->dictContentSize>0) {
+ if ( (cdict)
+ && (cdict->dictContentSize > 0)
+ && ( pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF
+ || pledgedSrcSize < cdict->dictContentSize * ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER
+ || pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN
+ || cdict->compressionLevel == 0)
+ && (params->attachDictPref != ZSTD_dictForceLoad) ) {
return ZSTD_resetCCtx_usingCDict(cctx, cdict, params, pledgedSrcSize, zbuff);
}
FORWARD_IF_ERROR( ZSTD_resetCCtx_internal(cctx, *params, pledgedSrcSize,
ZSTDcrp_makeClean, zbuff) );
- { size_t const dictID = ZSTD_compress_insertDictionary(
- cctx->blockState.prevCBlock, &cctx->blockState.matchState,
- &cctx->workspace, params, dict, dictSize, dictContentType, dtlm,
- cctx->entropyWorkspace);
+ { size_t const dictID = cdict ?
+ ZSTD_compress_insertDictionary(
+ cctx->blockState.prevCBlock, &cctx->blockState.matchState,
+ &cctx->workspace, params, cdict->dictContent, cdict->dictContentSize,
+ dictContentType, dtlm, cctx->entropyWorkspace)
+ : ZSTD_compress_insertDictionary(
+ cctx->blockState.prevCBlock, &cctx->blockState.matchState,
+ &cctx->workspace, params, dict, dictSize,
+ dictContentType, dtlm, cctx->entropyWorkspace);
FORWARD_IF_ERROR(dictID);
assert(dictID <= UINT_MAX);
cctx->dictID = (U32)dictID;
@@ -3205,7 +3222,7 @@ static size_t ZSTD_initCDict_internal(
ZSTDirp_reset,
ZSTD_resetTarget_CDict));
/* (Maybe) load the dictionary
- * Skips loading the dictionary if it is <= 8 bytes.
+ * Skips loading the dictionary if it is < 8 bytes.
*/
{ ZSTD_CCtx_params params;
memset(¶ms, 0, sizeof(params));
@@ -3254,6 +3271,8 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize,
assert(cdict != NULL);
ZSTD_cwksp_move(&cdict->workspace, &ws);
cdict->customMem = customMem;
+ cdict->compressionLevel = 0; /* signals advanced API usage */
+
if (ZSTD_isError( ZSTD_initCDict_internal(cdict,
dictBuffer, dictSize,
dictLoadMethod, dictContentType,
@@ -3269,9 +3288,12 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize,
ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel)
{
ZSTD_compressionParameters cParams = ZSTD_getCParams(compressionLevel, 0, dictSize);
- return ZSTD_createCDict_advanced(dict, dictSize,
- ZSTD_dlm_byCopy, ZSTD_dct_auto,
- cParams, ZSTD_defaultCMem);
+ ZSTD_CDict* cdict = ZSTD_createCDict_advanced(dict, dictSize,
+ ZSTD_dlm_byCopy, ZSTD_dct_auto,
+ cParams, ZSTD_defaultCMem);
+ if (cdict)
+ cdict->compressionLevel = compressionLevel == 0 ? ZSTD_CLEVEL_DEFAULT : compressionLevel;
+ return cdict;
}
ZSTD_CDict* ZSTD_createCDict_byReference(const void* dict, size_t dictSize, int compressionLevel)
@@ -3361,7 +3383,15 @@ size_t ZSTD_compressBegin_usingCDict_advanced(
DEBUGLOG(4, "ZSTD_compressBegin_usingCDict_advanced");
RETURN_ERROR_IF(cdict==NULL, dictionary_wrong);
{ ZSTD_CCtx_params params = cctx->requestedParams;
- params.cParams = ZSTD_getCParamsFromCDict(cdict);
+ params.cParams = ( pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF
+ || pledgedSrcSize < cdict->dictContentSize * ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER
+ || pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN
+ || cdict->compressionLevel == 0 )
+ && (params.attachDictPref != ZSTD_dictForceLoad) ?
+ ZSTD_getCParamsFromCDict(cdict)
+ : ZSTD_getCParams(cdict->compressionLevel,
+ pledgedSrcSize,
+ cdict->dictContentSize);
/* Increase window log to fit the entire dictionary and source if the
* source size is known. Limit the increase to 19, which is the
* window log for compression level 1 with the largest source size.
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index ca47a667..dd4591b7 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -1096,7 +1096,7 @@ ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy,
size_t const dictContentSize = (size_t)(dictEnd - (dictPtr+12));
for (i=0; i<3; i++) {
U32 const rep = MEM_readLE32(dictPtr); dictPtr += 4;
- RETURN_ERROR_IF(rep==0 || rep >= dictContentSize,
+ RETURN_ERROR_IF(rep==0 || rep > dictContentSize,
dictionary_corrupted);
entropy->rep[i] = rep;
} }
@@ -1265,7 +1265,7 @@ size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx,
{
RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong);
ZSTD_clearDict(dctx);
- if (dict && dictSize >= 8) {
+ if (dict && dictSize != 0) {
dctx->ddictLocal = ZSTD_createDDict_advanced(dict, dictSize, dictLoadMethod, dictContentType, dctx->customMem);
RETURN_ERROR_IF(dctx->ddictLocal == NULL, memory_allocation);
dctx->ddict = dctx->ddictLocal;
diff --git a/lib/decompress/zstd_decompress_block.c b/lib/decompress/zstd_decompress_block.c
index cbb66c8d..767e5f9a 100644
--- a/lib/decompress/zstd_decompress_block.c
+++ b/lib/decompress/zstd_decompress_block.c
@@ -617,7 +617,7 @@ static void ZSTD_safecopy(BYTE* op, BYTE* const oend_w, BYTE const* ip, ptrdiff_
ptrdiff_t const diff = op - ip;
BYTE* const oend = op + length;
- assert((ovtype == ZSTD_no_overlap && (diff <= -8 || diff >= 8)) ||
+ assert((ovtype == ZSTD_no_overlap && (diff <= -8 || diff >= 8 || op >= oend_w)) ||
(ovtype == ZSTD_overlap_src_before_dst && diff >= 0));
if (length < 8) {
diff --git a/lib/deprecated/zbuff.h b/lib/deprecated/zbuff.h
index a93115da..04183eab 100644
--- a/lib/deprecated/zbuff.h
+++ b/lib/deprecated/zbuff.h
@@ -36,16 +36,17 @@ extern "C" {
*****************************************************************/
/* Deprecation warnings */
/* Should these warnings be a problem,
- it is generally possible to disable them,
- typically with -Wno-deprecated-declarations for gcc
- or _CRT_SECURE_NO_WARNINGS in Visual.
- Otherwise, it's also possible to define ZBUFF_DISABLE_DEPRECATE_WARNINGS */
+ * it is generally possible to disable them,
+ * typically with -Wno-deprecated-declarations for gcc
+ * or _CRT_SECURE_NO_WARNINGS in Visual.
+ * Otherwise, it's also possible to define ZBUFF_DISABLE_DEPRECATE_WARNINGS
+ */
#ifdef ZBUFF_DISABLE_DEPRECATE_WARNINGS
# define ZBUFF_DEPRECATED(message) ZSTDLIB_API /* disable deprecation warnings */
#else
# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
# define ZBUFF_DEPRECATED(message) [[deprecated(message)]] ZSTDLIB_API
-# elif (defined(__GNUC__) && (__GNUC__ >= 5)) || defined(__clang__)
+# elif (defined(GNUC) && (GNUC > 4 || (GNUC == 4 && GNUC_MINOR >= 5))) || defined(__clang__)
# define ZBUFF_DEPRECATED(message) ZSTDLIB_API __attribute__((deprecated(message)))
# elif defined(__GNUC__) && (__GNUC__ >= 3)
# define ZBUFF_DEPRECATED(message) ZSTDLIB_API __attribute__((deprecated))
diff --git a/lib/libzstd.pc.in b/lib/libzstd.pc.in
index 1d07b91f..e7880be4 100644
--- a/lib/libzstd.pc.in
+++ b/lib/libzstd.pc.in
@@ -3,8 +3,9 @@
# BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
prefix=@PREFIX@
-libdir=@LIBDIR@
-includedir=@INCLUDEDIR@
+exec_prefix=${prefix}
+includedir=${prefix}/include
+libdir=${exec_prefix}/lib
Name: zstd
Description: fast lossless compression algorithm library
diff --git a/lib/zstd.h b/lib/zstd.h
index b8669e66..72080ea8 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -808,12 +808,17 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
typedef struct ZSTD_CDict_s ZSTD_CDict;
/*! ZSTD_createCDict() :
- * When compressing multiple messages / blocks using the same dictionary, it's recommended to load it only once.
- * ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup cost.
+ * When compressing multiple messages or blocks using the same dictionary,
+ * it's recommended to digest the dictionary only once, since it's a costly operation.
+ * ZSTD_createCDict() will create a state from digesting a dictionary.
+ * The resulting state can be used for future compression operations with very limited startup cost.
* ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
- * `dictBuffer` can be released after ZSTD_CDict creation, because its content is copied within CDict.
- * Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate `dictBuffer` content.
- * Note : A ZSTD_CDict can be created from an empty dictBuffer, but it is inefficient when used to compress small data. */
+ * @dictBuffer can be released after ZSTD_CDict creation, because its content is copied within CDict.
+ * Note 1 : Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate @dictBuffer content.
+ * Note 2 : A ZSTD_CDict can be created from an empty @dictBuffer,
+ * in which case the only thing that it transports is the @compressionLevel.
+ * This can be useful in a pipeline featuring ZSTD_compress_usingCDict() exclusively,
+ * expecting a ZSTD_CDict parameter with any data, including those without a known dictionary. */
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize,
int compressionLevel);
@@ -1152,7 +1157,7 @@ typedef enum {
* to evolve and should be considered only in the context of extremely
* advanced performance tuning.
*
- * Zstd currently supports the use of a CDict in two ways:
+ * Zstd currently supports the use of a CDict in three ways:
*
* - The contents of the CDict can be copied into the working context. This
* means that the compression can search both the dictionary and input
@@ -1168,6 +1173,12 @@ typedef enum {
* working context's tables can be reused). For small inputs, this can be
* faster than copying the CDict's tables.
*
+ * - The CDict's tables are not used at all, and instead we use the working
+ * context alone to reload the dictionary and use params based on the source
+ * size. See ZSTD_compress_insertDictionary() and ZSTD_compress_usingDict().
+ * This method is effective when the dictionary sizes are very small relative
+ * to the input size, and the input size is fairly large to begin with.
+ *
* Zstd has a simple internal heuristic that selects which strategy to use
* at the beginning of a compression. However, if experimentation shows that
* Zstd is making poor choices, it is possible to override that choice with
@@ -1175,7 +1186,8 @@ typedef enum {
*/
ZSTD_dictDefaultAttach = 0, /* Use the default heuristic. */
ZSTD_dictForceAttach = 1, /* Never copy the dictionary. */
- ZSTD_dictForceCopy = 2 /* Always copy the dictionary. */
+ ZSTD_dictForceCopy = 2, /* Always copy the dictionary. */
+ ZSTD_dictForceLoad = 3 /* Always reload the dictionary */
} ZSTD_dictAttachPref_e;
typedef enum {
diff --git a/programs/fileio.c b/programs/fileio.c
index 16d06287..98337672 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -319,6 +319,8 @@ struct FIO_prefs_s {
/* Computation resources preferences */
unsigned memLimit;
int nbWorkers;
+
+ int excludeCompressedFiles;
};
@@ -359,6 +361,7 @@ FIO_prefs_t* FIO_createPreferences(void)
ret->srcSizeHint = 0;
ret->testMode = 0;
ret->literalCompressionMode = ZSTD_lcm_auto;
+ ret->excludeCompressedFiles = 0;
return ret;
}
@@ -402,6 +405,8 @@ void FIO_setNbWorkers(FIO_prefs_t* const prefs, int nbWorkers) {
prefs->nbWorkers = nbWorkers;
}
+void FIO_setExcludeCompressedFile(FIO_prefs_t* const prefs, int excludeCompressedFiles) { prefs->excludeCompressedFiles = excludeCompressedFiles; }
+
void FIO_setBlockSize(FIO_prefs_t* const prefs, int blockSize) {
if (blockSize && prefs->nbWorkers==0)
DISPLAYLEVEL(2, "Setting block size is useless in single-thread mode \n");
@@ -520,7 +525,11 @@ static FILE* FIO_openSrcFile(const char* srcFileName)
return NULL;
}
- if (!UTIL_isRegularFile(srcFileName)) {
+ if (!UTIL_isRegularFile(srcFileName)
+#ifndef _MSC_VER
+ && !UTIL_isFIFO(srcFileName)
+#endif /* _MSC_VER */
+ ) {
DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n",
srcFileName);
return NULL;
@@ -1425,6 +1434,21 @@ static int FIO_compressFilename_dstFile(FIO_prefs_t* const prefs,
return result;
}
+/* List used to compare file extensions (used with --exclude-compressed flag)
+* Different from the suffixList and should only apply to ZSTD compress operationResult
+*/
+static const char *compressedFileExtensions[] = {
+ ZSTD_EXTENSION,
+ TZSTD_EXTENSION,
+ GZ_EXTENSION,
+ TGZ_EXTENSION,
+ LZMA_EXTENSION,
+ XZ_EXTENSION,
+ TXZ_EXTENSION,
+ LZ4_EXTENSION,
+ TLZ4_EXTENSION,
+ NULL
+};
/*! FIO_compressFilename_srcFile() :
* @return : 0 : compression completed correctly,
@@ -1451,6 +1475,15 @@ FIO_compressFilename_srcFile(FIO_prefs_t* const prefs,
return 1;
}
+ /* Check if "srcFile" is compressed. Only done if --exclude-compressed flag is used
+ * YES => ZSTD will skip compression of the file and will return 0.
+ * NO => ZSTD will resume with compress operation.
+ */
+ if (prefs->excludeCompressedFiles == 1 && UTIL_isCompressedFile(srcFileName, compressedFileExtensions)) {
+ DISPLAYLEVEL(4, "File is already compressed : %s \n", srcFileName);
+ return 0;
+ }
+
ress.srcFile = FIO_openSrcFile(srcFileName);
if (ress.srcFile == NULL) return 1; /* srcFile could not be opened */
@@ -1496,17 +1529,17 @@ FIO_determineCompressedName(const char* srcFileName, const char* outDirName, con
static char* dstFileNameBuffer = NULL; /* using static allocation : this function cannot be multi-threaded */
char* outDirFilename = NULL;
size_t sfnSize = strlen(srcFileName);
- size_t const suffixSize = strlen(suffix);
+ size_t const srcSuffixLen = strlen(suffix);
if (outDirName) {
- outDirFilename = FIO_createFilename_fromOutDir(srcFileName, outDirName, suffixSize);
+ outDirFilename = FIO_createFilename_fromOutDir(srcFileName, outDirName, srcSuffixLen);
sfnSize = strlen(outDirFilename);
assert(outDirFilename != NULL);
}
- if (dfnbCapacity <= sfnSize+suffixSize+1) {
+ if (dfnbCapacity <= sfnSize+srcSuffixLen+1) {
/* resize buffer for dstName */
free(dstFileNameBuffer);
- dfnbCapacity = sfnSize + suffixSize + 30;
+ dfnbCapacity = sfnSize + srcSuffixLen + 30;
dstFileNameBuffer = (char*)malloc(dfnbCapacity);
if (!dstFileNameBuffer) {
EXM_THROW(30, "zstd: %s", strerror(errno));
@@ -1520,7 +1553,7 @@ FIO_determineCompressedName(const char* srcFileName, const char* outDirName, con
} else {
memcpy(dstFileNameBuffer, srcFileName, sfnSize);
}
- memcpy(dstFileNameBuffer+sfnSize, suffix, suffixSize+1 /* Include terminating null */);
+ memcpy(dstFileNameBuffer+sfnSize, suffix, srcSuffixLen+1 /* Include terminating null */);
return dstFileNameBuffer;
}
@@ -2287,6 +2320,37 @@ int FIO_decompressFilename(FIO_prefs_t* const prefs,
return decodingError;
}
+static const char *suffixList[] = {
+ ZSTD_EXTENSION,
+ TZSTD_EXTENSION,
+#ifdef ZSTD_GZDECOMPRESS
+ GZ_EXTENSION,
+ TGZ_EXTENSION,
+#endif
+#ifdef ZSTD_LZMADECOMPRESS
+ LZMA_EXTENSION,
+ XZ_EXTENSION,
+ TXZ_EXTENSION,
+#endif
+#ifdef ZSTD_LZ4DECOMPRESS
+ LZ4_EXTENSION,
+ TLZ4_EXTENSION,
+#endif
+ NULL
+};
+
+static const char *suffixListStr =
+ ZSTD_EXTENSION "/" TZSTD_EXTENSION
+#ifdef ZSTD_GZDECOMPRESS
+ "/" GZ_EXTENSION "/" TGZ_EXTENSION
+#endif
+#ifdef ZSTD_LZMADECOMPRESS
+ "/" LZMA_EXTENSION "/" XZ_EXTENSION "/" TXZ_EXTENSION
+#endif
+#ifdef ZSTD_LZ4DECOMPRESS
+ "/" LZ4_EXTENSION "/" TLZ4_EXTENSION
+#endif
+;
/* FIO_determineDstName() :
* create a destination filename from a srcFileName.
@@ -2297,71 +2361,78 @@ FIO_determineDstName(const char* srcFileName, const char* outDirName)
{
static size_t dfnbCapacity = 0;
static char* dstFileNameBuffer = NULL; /* using static allocation : this function cannot be multi-threaded */
+ size_t dstFileNameEndPos;
char* outDirFilename = NULL;
+ const char* dstSuffix = "";
+ size_t dstSuffixLen = 0;
+
size_t sfnSize = strlen(srcFileName);
- size_t suffixSize;
- const char* const suffixPtr = strrchr(srcFileName, '.');
- if (suffixPtr == NULL) {
- DISPLAYLEVEL(1, "zstd: %s: unknown suffix -- ignored \n",
- srcFileName);
+ size_t srcSuffixLen;
+ const char* const srcSuffix = strrchr(srcFileName, '.');
+ if (srcSuffix == NULL) {
+ DISPLAYLEVEL(1,
+ "zstd: %s: unknown suffix (%s expected). "
+ "Can't derive the output file name. "
+ "Specify it with -o dstFileName. Ignoring.\n",
+ srcFileName, suffixListStr);
return NULL;
}
- suffixSize = strlen(suffixPtr);
+ srcSuffixLen = strlen(srcSuffix);
- /* check suffix is authorized */
- if (sfnSize <= suffixSize
- || ( strcmp(suffixPtr, ZSTD_EXTENSION)
- #ifdef ZSTD_GZDECOMPRESS
- && strcmp(suffixPtr, GZ_EXTENSION)
- #endif
- #ifdef ZSTD_LZMADECOMPRESS
- && strcmp(suffixPtr, XZ_EXTENSION)
- && strcmp(suffixPtr, LZMA_EXTENSION)
- #endif
- #ifdef ZSTD_LZ4DECOMPRESS
- && strcmp(suffixPtr, LZ4_EXTENSION)
- #endif
- ) ) {
- const char* suffixlist = ZSTD_EXTENSION
- #ifdef ZSTD_GZDECOMPRESS
- "/" GZ_EXTENSION
- #endif
- #ifdef ZSTD_LZMADECOMPRESS
- "/" XZ_EXTENSION "/" LZMA_EXTENSION
- #endif
- #ifdef ZSTD_LZ4DECOMPRESS
- "/" LZ4_EXTENSION
- #endif
- ;
- DISPLAYLEVEL(1, "zstd: %s: unknown suffix (%s expected) -- ignored \n",
- srcFileName, suffixlist);
- return NULL;
+ {
+ const char** matchedSuffixPtr;
+ for (matchedSuffixPtr = suffixList; *matchedSuffixPtr != NULL; matchedSuffixPtr++) {
+ if (!strcmp(*matchedSuffixPtr, srcSuffix)) {
+ break;
+ }
+ }
+
+ /* check suffix is authorized */
+ if (sfnSize <= srcSuffixLen || *matchedSuffixPtr == NULL) {
+ DISPLAYLEVEL(1,
+ "zstd: %s: unknown suffix (%s expected). "
+ "Can't derive the output file name. "
+ "Specify it with -o dstFileName. Ignoring.\n",
+ srcFileName, suffixListStr);
+ return NULL;
+ }
+
+ if ((*matchedSuffixPtr)[1] == 't') {
+ dstSuffix = ".tar";
+ dstSuffixLen = strlen(dstSuffix);
+ }
}
+
if (outDirName) {
outDirFilename = FIO_createFilename_fromOutDir(srcFileName, outDirName, 0);
sfnSize = strlen(outDirFilename);
assert(outDirFilename != NULL);
}
- if (dfnbCapacity+suffixSize <= sfnSize+1) {
+ if (dfnbCapacity+srcSuffixLen <= sfnSize+1+dstSuffixLen) {
/* allocate enough space to write dstFilename into it */
free(dstFileNameBuffer);
dfnbCapacity = sfnSize + 20;
dstFileNameBuffer = (char*)malloc(dfnbCapacity);
if (dstFileNameBuffer==NULL)
- EXM_THROW(74, "%s : not enough memory for dstFileName", strerror(errno));
+ EXM_THROW(74, "%s : not enough memory for dstFileName",
+ strerror(errno));
}
/* return dst name == src name truncated from suffix */
assert(dstFileNameBuffer != NULL);
+ dstFileNameEndPos = sfnSize - srcSuffixLen;
if (outDirFilename) {
- memcpy(dstFileNameBuffer, outDirFilename, sfnSize - suffixSize);
+ memcpy(dstFileNameBuffer, outDirFilename, dstFileNameEndPos);
free(outDirFilename);
} else {
- memcpy(dstFileNameBuffer, srcFileName, sfnSize - suffixSize);
+ memcpy(dstFileNameBuffer, srcFileName, dstFileNameEndPos);
}
- dstFileNameBuffer[sfnSize-suffixSize] = '\0';
+
+ /* The short tar extensions tzst, tgz, txz and tlz4 files should have "tar"
+ * extension on decompression. Also writes terminating null. */
+ strcpy(dstFileNameBuffer + dstFileNameEndPos, dstSuffix);
return dstFileNameBuffer;
/* note : dstFileNameBuffer memory is not going to be free */
diff --git a/programs/fileio.h b/programs/fileio.h
index 4b0143be..a7da089f 100644
--- a/programs/fileio.h
+++ b/programs/fileio.h
@@ -30,11 +30,23 @@ extern "C" {
#else
# define nulmark "/dev/null"
#endif
+
+/**
+ * We test whether the extension we found starts with 't', and if so, we append
+ * ".tar" to the end of the output name.
+ */
#define LZMA_EXTENSION ".lzma"
#define XZ_EXTENSION ".xz"
+#define TXZ_EXTENSION ".txz"
+
#define GZ_EXTENSION ".gz"
+#define TGZ_EXTENSION ".tgz"
+
#define ZSTD_EXTENSION ".zst"
+#define TZSTD_EXTENSION ".tzst"
+
#define LZ4_EXTENSION ".lz4"
+#define TLZ4_EXTENSION ".tlz4"
/*-*************************************
@@ -81,6 +93,7 @@ void FIO_setLiteralCompressionMode(
void FIO_setNoProgress(unsigned noProgress);
void FIO_setNotificationLevel(int level);
+void FIO_setExcludeCompressedFile(FIO_prefs_t* const prefs, int excludeCompressedFiles);
/*-*************************************
* Single File functions
diff --git a/programs/util.c b/programs/util.c
index e1aa447a..5d15450d 100644
--- a/programs/util.c
+++ b/programs/util.c
@@ -126,6 +126,21 @@ int UTIL_isSameFile(const char* fName1, const char* fName2)
#endif
}
+#ifndef _MSC_VER
+/* Using this to distinguish named pipes */
+U32 UTIL_isFIFO(const char* infilename)
+{
+/* macro guards, as defined in : https://linux.die.net/man/2/lstat */
+#if PLATFORM_POSIX_VERSION >= 200112L
+ stat_t statbuf;
+ int r = UTIL_getFileStat(infilename, &statbuf);
+ if (!r && S_ISFIFO(statbuf.st_mode)) return 1;
+#endif
+ (void)infilename;
+ return 0;
+}
+#endif
+
U32 UTIL_isLink(const char* infilename)
{
/* macro guards, as defined in : https://linux.die.net/man/2/lstat */
@@ -313,6 +328,27 @@ int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char
#endif /* #ifdef _WIN32 */
+int UTIL_isCompressedFile(const char *inputName, const char *extensionList[])
+{
+ const char* ext = UTIL_getFileExtension(inputName);
+ while(*extensionList!=NULL)
+ {
+ const int isCompressedExtension = strcmp(ext,*extensionList);
+ if(isCompressedExtension==0)
+ return 1;
+ ++extensionList;
+ }
+ return 0;
+}
+
+/*Utility function to get file extension from file */
+const char* UTIL_getFileExtension(const char* infilename)
+{
+ const char* extension = strrchr(infilename, '.');
+ if(!extension || extension==infilename) return "";
+ return extension;
+}
+
/*
* UTIL_createFileList - takes a list of files and directories (params: inputNames, inputNamesNb), scans directories,
* and returns a new list of files (params: return value, allocatedBuffer, allocatedNamesNb).
diff --git a/programs/util.h b/programs/util.h
index c73f7e9b..1f524f29 100644
--- a/programs/util.h
+++ b/programs/util.h
@@ -40,7 +40,6 @@ extern "C" {
#include /* clock_t, clock, CLOCKS_PER_SEC, nanosleep */
#include "mem.h" /* U32, U64 */
-
/*-************************************************************
* Avoid fseek()'s 2GiB barrier with MSVC, macOS, *BSD, MinGW
***************************************************************/
@@ -135,7 +134,12 @@ U32 UTIL_isDirectory(const char* infilename);
int UTIL_getFileStat(const char* infilename, stat_t* statbuf);
int UTIL_isSameFile(const char* file1, const char* file2);
int UTIL_compareStr(const void *p1, const void *p2);
+int UTIL_isCompressedFile(const char* infilename, const char *extensionList[]);
+const char* UTIL_getFileExtension(const char* infilename);
+#ifndef _MSC_VER
+U32 UTIL_isFIFO(const char* infilename);
+#endif
U32 UTIL_isLink(const char* infilename);
#define UTIL_FILESIZE_UNKNOWN ((U64)(-1))
U64 UTIL_getFileSize(const char* infilename);
diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index 803c5037..cd6b40bc 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -136,6 +136,7 @@ static int usage_advanced(const char* programName)
DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n");
DISPLAY( " -c : force write to standard output, even if it is the console\n");
DISPLAY( " -l : print information about zstd compressed files \n");
+ DISPLAY( "--exclude-compressed: only compress files that are not previously compressed \n");
#ifndef ZSTD_NOCOMPRESS
DISPLAY( "--ultra : enable levels beyond %i, up to %i (requires more memory)\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel());
DISPLAY( "--long[=#]: enable long distance matching with given window log (default: %u)\n", g_defaultMaxWindowLog);
@@ -708,7 +709,7 @@ int main(int argCount, const char* argv[])
if (!strcmp(argument, "--compress-literals")) { literalCompressionMode = ZSTD_lcm_huffman; continue; }
if (!strcmp(argument, "--no-compress-literals")) { literalCompressionMode = ZSTD_lcm_uncompressed; continue; }
if (!strcmp(argument, "--no-progress")) { FIO_setNoProgress(1); continue; }
-
+ if (!strcmp(argument, "--exclude-compressed")) { FIO_setExcludeCompressedFile(prefs, 1); continue; }
/* long commands with arguments */
#ifndef ZSTD_NODICT
if (longCommandWArg(&argument, "--train-cover")) {
@@ -1004,7 +1005,11 @@ int main(int argCount, const char* argv[])
if (!followLinks) {
unsigned u;
for (u=0, fileNamesNb=0; u
+#include
+#include
+#include
+#include "fuzz_helpers.h"
+#include "zstd_helpers.h"
+#include "fuzz_data_producer.h"
+
+/**
+ * Compresses the data and returns the compressed size or an error.
+ */
+static size_t compress(void* compressed, size_t compressedCapacity,
+ void const* source, size_t sourceSize,
+ void const* dict, size_t dictSize,
+ ZSTD_dictLoadMethod_e dictLoadMethod,
+ ZSTD_dictContentType_e dictContentType)
+{
+ ZSTD_CCtx* cctx = ZSTD_createCCtx();
+ FUZZ_ZASSERT(ZSTD_CCtx_loadDictionary_advanced(
+ cctx, dict, dictSize, dictLoadMethod, dictContentType));
+ size_t const compressedSize = ZSTD_compress2(
+ cctx, compressed, compressedCapacity, source, sourceSize);
+ ZSTD_freeCCtx(cctx);
+ return compressedSize;
+}
+
+static size_t decompress(void* result, size_t resultCapacity,
+ void const* compressed, size_t compressedSize,
+ void const* dict, size_t dictSize,
+ ZSTD_dictLoadMethod_e dictLoadMethod,
+ ZSTD_dictContentType_e dictContentType)
+{
+ ZSTD_DCtx* dctx = ZSTD_createDCtx();
+ FUZZ_ZASSERT(ZSTD_DCtx_loadDictionary_advanced(
+ dctx, dict, dictSize, dictLoadMethod, dictContentType));
+ size_t const resultSize = ZSTD_decompressDCtx(
+ dctx, result, resultCapacity, compressed, compressedSize);
+ FUZZ_ZASSERT(resultSize);
+ ZSTD_freeDCtx(dctx);
+ return resultSize;
+}
+
+int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
+{
+ FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size);
+ ZSTD_dictLoadMethod_e const dlm =
+ size = FUZZ_dataProducer_uint32Range(producer, 0, 1);
+ ZSTD_dictContentType_e const dct =
+ FUZZ_dataProducer_uint32Range(producer, 0, 2);
+ size = FUZZ_dataProducer_remainingBytes(producer);
+
+ DEBUGLOG(2, "Dict load method %d", dlm);
+ DEBUGLOG(2, "Dict content type %d", dct);
+ DEBUGLOG(2, "Dict size %u", (unsigned)size);
+
+ void* const rBuf = malloc(size);
+ FUZZ_ASSERT(rBuf);
+ size_t const cBufSize = ZSTD_compressBound(size);
+ void* const cBuf = malloc(cBufSize);
+ FUZZ_ASSERT(cBuf);
+
+ size_t const cSize =
+ compress(cBuf, cBufSize, src, size, src, size, dlm, dct);
+ /* compression failing is okay */
+ if (ZSTD_isError(cSize)) {
+ FUZZ_ASSERT_MSG(dct != ZSTD_dct_rawContent, "Raw must always succeed!");
+ goto out;
+ }
+ size_t const rSize =
+ decompress(rBuf, size, cBuf, cSize, src, size, dlm, dct);
+ FUZZ_ASSERT_MSG(rSize == size, "Incorrect regenerated size");
+ FUZZ_ASSERT_MSG(!memcmp(src, rBuf, size), "Corruption!");
+
+out:
+ free(cBuf);
+ free(rBuf);
+ FUZZ_dataProducer_free(producer);
+ return 0;
+}
diff --git a/tests/fuzz/fuzz.py b/tests/fuzz/fuzz.py
index 9df68df0..87f115af 100755
--- a/tests/fuzz/fuzz.py
+++ b/tests/fuzz/fuzz.py
@@ -27,6 +27,7 @@ def abs_join(a, *p):
class InputType(object):
RAW_DATA = 1
COMPRESSED_DATA = 2
+ DICTIONARY_DATA = 3
class FrameType(object):
@@ -54,6 +55,7 @@ TARGET_INFO = {
'dictionary_decompress': TargetInfo(InputType.COMPRESSED_DATA),
'zstd_frame_info': TargetInfo(InputType.COMPRESSED_DATA),
'simple_compress': TargetInfo(InputType.RAW_DATA),
+ 'dictionary_loader': TargetInfo(InputType.DICTIONARY_DATA),
}
TARGETS = list(TARGET_INFO.keys())
ALL_TARGETS = TARGETS + ['all']
@@ -73,6 +75,7 @@ LIB_FUZZING_ENGINE = os.environ.get('LIB_FUZZING_ENGINE', 'libregression.a')
AFL_FUZZ = os.environ.get('AFL_FUZZ', 'afl-fuzz')
DECODECORPUS = os.environ.get('DECODECORPUS',
abs_join(FUZZ_DIR, '..', 'decodecorpus'))
+ZSTD = os.environ.get('ZSTD', abs_join(FUZZ_DIR, '..', '..', 'zstd'))
# Sanitizer environment variables
MSAN_EXTRA_CPPFLAGS = os.environ.get('MSAN_EXTRA_CPPFLAGS', '')
@@ -673,6 +676,11 @@ def gen_parser(args):
default=DECODECORPUS,
help="decodecorpus binary (default: $DECODECORPUS='{}')".format(
DECODECORPUS))
+ parser.add_argument(
+ '--zstd',
+ type=str,
+ default=ZSTD,
+ help="zstd binary (default: $ZSTD='{}')".format(ZSTD))
parser.add_argument(
'--fuzz-rng-seed-size',
type=int,
@@ -707,46 +715,66 @@ def gen(args):
return 1
seed = create(args.seed)
- with tmpdir() as compressed:
- with tmpdir() as decompressed:
- cmd = [
- args.decodecorpus,
- '-n{}'.format(args.number),
- '-p{}/'.format(compressed),
- '-o{}'.format(decompressed),
+ with tmpdir() as compressed, tmpdir() as decompressed, tmpdir() as dict:
+ info = TARGET_INFO[args.TARGET]
+
+ if info.input_type == InputType.DICTIONARY_DATA:
+ number = max(args.number, 1000)
+ else:
+ number = args.number
+ cmd = [
+ args.decodecorpus,
+ '-n{}'.format(args.number),
+ '-p{}/'.format(compressed),
+ '-o{}'.format(decompressed),
+ ]
+
+ if info.frame_type == FrameType.BLOCK:
+ cmd += [
+ '--gen-blocks',
+ '--max-block-size-log={}'.format(min(args.max_size_log, 17))
]
+ else:
+ cmd += ['--max-content-size-log={}'.format(args.max_size_log)]
- info = TARGET_INFO[args.TARGET]
- if info.frame_type == FrameType.BLOCK:
- cmd += [
- '--gen-blocks',
- '--max-block-size-log={}'.format(min(args.max_size_log, 17))
+ print(' '.join(cmd))
+ subprocess.check_call(cmd)
+
+ if info.input_type == InputType.RAW_DATA:
+ print('using decompressed data in {}'.format(decompressed))
+ samples = decompressed
+ elif info.input_type == InputType.COMPRESSED_DATA:
+ print('using compressed data in {}'.format(compressed))
+ samples = compressed
+ else:
+ assert info.input_type == InputType.DICTIONARY_DATA
+ print('making dictionary data from {}'.format(decompressed))
+ samples = dict
+ min_dict_size_log = 9
+ max_dict_size_log = max(min_dict_size_log + 1, args.max_size_log)
+ for dict_size_log in range(min_dict_size_log, max_dict_size_log):
+ dict_size = 1 << dict_size_log
+ cmd = [
+ args.zstd,
+ '--train',
+ '-r', decompressed,
+ '--maxdict={}'.format(dict_size),
+ '-o', abs_join(dict, '{}.zstd-dict'.format(dict_size))
]
- else:
- cmd += ['--max-content-size-log={}'.format(args.max_size_log)]
+ print(' '.join(cmd))
+ subprocess.check_call(cmd)
- print(' '.join(cmd))
- subprocess.check_call(cmd)
-
- if info.input_type == InputType.RAW_DATA:
- print('using decompressed data in {}'.format(decompressed))
- samples = decompressed
- else:
- assert info.input_type == InputType.COMPRESSED_DATA
- print('using compressed data in {}'.format(compressed))
- samples = compressed
-
- # Copy the samples over and prepend the RNG seeds
- for name in os.listdir(samples):
- samplename = abs_join(samples, name)
- outname = abs_join(seed, name)
- with open(samplename, 'rb') as sample:
- with open(outname, 'wb') as out:
- CHUNK_SIZE = 131072
+ # Copy the samples over and prepend the RNG seeds
+ for name in os.listdir(samples):
+ samplename = abs_join(samples, name)
+ outname = abs_join(seed, name)
+ with open(samplename, 'rb') as sample:
+ with open(outname, 'wb') as out:
+ CHUNK_SIZE = 131072
+ chunk = sample.read(CHUNK_SIZE)
+ while len(chunk) > 0:
+ out.write(chunk)
chunk = sample.read(CHUNK_SIZE)
- while len(chunk) > 0:
- out.write(chunk)
- chunk = sample.read(CHUNK_SIZE)
return 0
diff --git a/tests/playTests.sh b/tests/playTests.sh
index 796a5bde..c09e0c21 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -215,6 +215,37 @@ $ZSTD tmp -c --compress-literals -19 | $ZSTD -t
$ZSTD -b --fast=1 -i0e1 tmp --compress-literals
$ZSTD -b --fast=1 -i0e1 tmp --no-compress-literals
+println "test: --exclude-compressed flag"
+rm -rf precompressedFilterTestDir
+mkdir -p precompressedFilterTestDir
+./datagen $size > precompressedFilterTestDir/input.5
+./datagen $size > precompressedFilterTestDir/input.6
+$ZSTD --exclude-compressed --long --rm -r precompressedFilterTestDir
+sleep 5
+./datagen $size > precompressedFilterTestDir/input.7
+./datagen $size > precompressedFilterTestDir/input.8
+$ZSTD --exclude-compressed --long --rm -r precompressedFilterTestDir
+test ! -f precompressedFilterTestDir/input.5.zst.zst
+test ! -f precompressedFilterTestDir/input.6.zst.zst
+file1timestamp=`date -r precompressedFilterTestDir/input.5.zst +%s`
+file2timestamp=`date -r precompressedFilterTestDir/input.7.zst +%s`
+if [[ $file2timestamp -ge $file1timestamp ]]; then
+ println "Test is successful. input.5.zst is precompressed and therefore not compressed/modified again."
+else
+ println "Test is not successful"
+fi
+#File Extension check.
+./datagen $size > precompressedFilterTestDir/input.zstbar
+$ZSTD --exclude-compressed --long --rm -r precompressedFilterTestDir
+#ZSTD should compress input.zstbar
+test -f precompressedFilterTestDir/input.zstbar.zst
+#Check without the --exclude-compressed flag
+$ZSTD --long --rm -r precompressedFilterTestDir
+#Files should get compressed again without the --exclude-compressed flag.
+test -f precompressedFilterTestDir/input.5.zst.zst
+test -f precompressedFilterTestDir/input.6.zst.zst
+println "Test completed"
+
println "test : file removal"
$ZSTD -f --rm tmp
test ! -f tmp # tmp should no longer be present
@@ -845,6 +876,46 @@ if [ $LZ4MODE -ne 1 ]; then
grep ".lz4" tmplg > $INTOVOID && die "Unsupported suffix listed"
fi
+println "\n===> tar extension tests "
+
+rm -f tmp tmp.tar tmp.tzst tmp.tgz tmp.txz tmp.tlz4
+
+./datagen > tmp
+tar cf tmp.tar tmp
+$ZSTD tmp.tar -o tmp.tzst
+rm tmp.tar
+$ZSTD -d tmp.tzst
+[ -e tmp.tar ] || die ".tzst failed to decompress to .tar!"
+rm -f tmp.tar tmp.tzst
+
+if [ $GZIPMODE -eq 1 ]; then
+ tar czf tmp.tgz tmp
+ $ZSTD -d tmp.tgz
+ [ -e tmp.tar ] || die ".tgz failed to decompress to .tar!"
+ rm -f tmp.tar tmp.tgz
+fi
+
+if [ $LZMAMODE -eq 1 ]; then
+ tar c tmp | $ZSTD --format=xz > tmp.txz
+ $ZSTD -d tmp.txz
+ [ -e tmp.tar ] || die ".txz failed to decompress to .tar!"
+ rm -f tmp.tar tmp.txz
+fi
+
+if [ $LZ4MODE -eq 1 ]; then
+ tar c tmp | $ZSTD --format=lz4 > tmp.tlz4
+ $ZSTD -d tmp.tlz4
+ [ -e tmp.tar ] || die ".tlz4 failed to decompress to .tar!"
+ rm -f tmp.tar tmp.tlz4
+fi
+
+touch tmp.t tmp.tz tmp.tzs
+! $ZSTD -d tmp.t
+! $ZSTD -d tmp.tz
+! $ZSTD -d tmp.tzs
+
+exit
+
println "\n===> zstd round-trip tests "
roundTripTest
@@ -1099,4 +1170,18 @@ test -f dictionary
rm -f tmp* dictionary
+if [ "$isWindows" = false ] ; then
+
+println "\n===> zstd fifo named pipe test "
+head -c 10 /dev/zero > tmp_original
+mkfifo named_pipe
+head -c 10 /dev/zero > named_pipe &
+$ZSTD named_pipe -o tmp_compressed
+$ZSTD -d -o tmp_decompressed tmp_compressed
+$DIFF -s tmp_original tmp_decompressed
+rm -rf tmp*
+rm -rf named_pipe
+
+fi
+
rm -f tmp*
diff --git a/tests/regression/method.c b/tests/regression/method.c
index 1e84021c..b74f5481 100644
--- a/tests/regression/method.c
+++ b/tests/regression/method.c
@@ -444,6 +444,8 @@ static int init_cstream(
ZSTD_parameters const params = config_get_zstd_params(config, 0, 0);
ZSTD_CDict* dict = NULL;
if (cdict) {
+ if (!config->use_dictionary)
+ return 1;
*cdict = ZSTD_createCDict_advanced(
state->dictionary.data,
state->dictionary.size,
@@ -459,14 +461,18 @@ static int init_cstream(
} else {
zret = ZSTD_initCStream_advanced(
zcs,
- state->dictionary.data,
- state->dictionary.size,
+ config->use_dictionary ? state->dictionary.data : NULL,
+ config->use_dictionary ? state->dictionary.size : 0,
params,
ZSTD_CONTENTSIZE_UNKNOWN);
}
} else {
int const level = config_get_level(config);
+ if (level == CONFIG_NO_LEVEL)
+ return 1;
if (cdict) {
+ if (!config->use_dictionary)
+ return 1;
*cdict = ZSTD_createCDict(
state->dictionary.data,
state->dictionary.size,
@@ -477,7 +483,10 @@ static int init_cstream(
zret = ZSTD_initCStream_usingCDict(zcs, *cdict);
} else if (config->use_dictionary) {
zret = ZSTD_initCStream_usingDict(
- zcs, state->dictionary.data, state->dictionary.size, level);
+ zcs,
+ state->dictionary.data,
+ state->dictionary.size,
+ level);
} else {
zret = ZSTD_initCStream(zcs, level);
}
@@ -506,9 +515,17 @@ static result_t old_streaming_compress_internal(
result = result_error(result_error_compression_error);
goto out;
}
+ if (!advanced && config_get_level(config) == CONFIG_NO_LEVEL) {
+ result = result_error(result_error_skip);
+ goto out;
+ }
+ if (cdict && !config->use_dictionary) {
+ result = result_error(result_error_skip);
+ goto out;
+ }
if (init_cstream(state, zcs, config, advanced, cdict ? &cd : NULL)) {
- result = result_error(result_error_compression_error);
- goto out;
+ result = result_error(result_error_compression_error);
+ goto out;
}
result_data_t data = {.total_size = 0};
@@ -629,21 +646,21 @@ method_t const old_streaming = {
method_t const old_streaming_advanced = {
.name = "old streaming advanced",
.create = buffer_state_create,
- .compress = old_streaming_compress,
+ .compress = old_streaming_compress_advanced,
.destroy = buffer_state_destroy,
};
method_t const old_streaming_cdict = {
.name = "old streaming cdcit",
.create = buffer_state_create,
- .compress = old_streaming_compress,
+ .compress = old_streaming_compress_cdict,
.destroy = buffer_state_destroy,
};
method_t const old_streaming_advanced_cdict = {
.name = "old streaming advanced cdict",
.create = buffer_state_create,
- .compress = old_streaming_compress,
+ .compress = old_streaming_compress_cdict_advanced,
.destroy = buffer_state_destroy,
};
diff --git a/tests/regression/results.csv b/tests/regression/results.csv
index ba1295c5..a0e1566a 100644
--- a/tests/regression/results.csv
+++ b/tests/regression/results.csv
@@ -461,17 +461,9 @@ silesia, level 13, old stre
silesia, level 16, old streaming, 4377389
silesia, level 19, old streaming, 4293262
silesia, no source size, old streaming, 4849455
-silesia, long distance mode, old streaming, 12000408
-silesia, multithreaded, old streaming, 12000408
-silesia, multithreaded long distance mode, old streaming, 12000408
-silesia, small window log, old streaming, 12000408
-silesia, small hash log, old streaming, 12000408
-silesia, small chain log, old streaming, 12000408
-silesia, explicit params, old streaming, 12000408
silesia, uncompressed literals, old streaming, 4849491
silesia, uncompressed literals optimal, old streaming, 4293262
silesia, huffman literals, old streaming, 6183385
-silesia, multithreaded with advanced params, old streaming, 12000408
silesia.tar, level -5, old streaming, 6982738
silesia.tar, level -3, old streaming, 6641264
silesia.tar, level -1, old streaming, 6190789
@@ -487,17 +479,9 @@ silesia.tar, level 13, old stre
silesia.tar, level 16, old streaming, 4381284
silesia.tar, level 19, old streaming, 4281511
silesia.tar, no source size, old streaming, 4861372
-silesia.tar, long distance mode, old streaming, 12022046
-silesia.tar, multithreaded, old streaming, 12022046
-silesia.tar, multithreaded long distance mode, old streaming, 12022046
-silesia.tar, small window log, old streaming, 12022046
-silesia.tar, small hash log, old streaming, 12022046
-silesia.tar, small chain log, old streaming, 12022046
-silesia.tar, explicit params, old streaming, 12022046
silesia.tar, uncompressed literals, old streaming, 4861376
silesia.tar, uncompressed literals optimal, old streaming, 4281511
silesia.tar, huffman literals, old streaming, 6190789
-silesia.tar, multithreaded with advanced params, old streaming, 12022046
github, level -5, old streaming, 205285
github, level -5 with dict, old streaming, 46718
github, level -3, old streaming, 190643
@@ -527,17 +511,9 @@ github, level 16 with dict, old stre
github, level 19, old streaming, 133717
github, level 19 with dict, old streaming, 37576
github, no source size, old streaming, 140631
-github, long distance mode, old streaming, 412933
-github, multithreaded, old streaming, 412933
-github, multithreaded long distance mode, old streaming, 412933
-github, small window log, old streaming, 412933
-github, small hash log, old streaming, 412933
-github, small chain log, old streaming, 412933
-github, explicit params, old streaming, 412933
github, uncompressed literals, old streaming, 136311
github, uncompressed literals optimal, old streaming, 133717
github, huffman literals, old streaming, 175568
-github, multithreaded with advanced params, old streaming, 412933
silesia, level -5, old streaming advanced, 6882466
silesia, level -3, old streaming advanced, 6568358
silesia, level -1, old streaming advanced, 6183385
@@ -553,17 +529,17 @@ silesia, level 13, old stre
silesia, level 16, old streaming advanced, 4377389
silesia, level 19, old streaming advanced, 4293262
silesia, no source size, old streaming advanced, 4849455
-silesia, long distance mode, old streaming advanced, 12000408
-silesia, multithreaded, old streaming advanced, 12000408
-silesia, multithreaded long distance mode, old streaming advanced, 12000408
-silesia, small window log, old streaming advanced, 12000408
-silesia, small hash log, old streaming advanced, 12000408
-silesia, small chain log, old streaming advanced, 12000408
-silesia, explicit params, old streaming advanced, 12000408
+silesia, long distance mode, old streaming advanced, 4849491
+silesia, multithreaded, old streaming advanced, 4849491
+silesia, multithreaded long distance mode, old streaming advanced, 4849491
+silesia, small window log, old streaming advanced, 7123534
+silesia, small hash log, old streaming advanced, 6554898
+silesia, small chain log, old streaming advanced, 4931093
+silesia, explicit params, old streaming advanced, 4797048
silesia, uncompressed literals, old streaming advanced, 4849491
silesia, uncompressed literals optimal, old streaming advanced, 4293262
silesia, huffman literals, old streaming advanced, 6183385
-silesia, multithreaded with advanced params, old streaming advanced, 12000408
+silesia, multithreaded with advanced params, old streaming advanced, 4849491
silesia.tar, level -5, old streaming advanced, 6982738
silesia.tar, level -3, old streaming advanced, 6641264
silesia.tar, level -1, old streaming advanced, 6190789
@@ -579,238 +555,82 @@ silesia.tar, level 13, old stre
silesia.tar, level 16, old streaming advanced, 4381284
silesia.tar, level 19, old streaming advanced, 4281511
silesia.tar, no source size, old streaming advanced, 4861372
-silesia.tar, long distance mode, old streaming advanced, 12022046
-silesia.tar, multithreaded, old streaming advanced, 12022046
-silesia.tar, multithreaded long distance mode, old streaming advanced, 12022046
-silesia.tar, small window log, old streaming advanced, 12022046
-silesia.tar, small hash log, old streaming advanced, 12022046
-silesia.tar, small chain log, old streaming advanced, 12022046
-silesia.tar, explicit params, old streaming advanced, 12022046
+silesia.tar, long distance mode, old streaming advanced, 4861376
+silesia.tar, multithreaded, old streaming advanced, 4861376
+silesia.tar, multithreaded long distance mode, old streaming advanced, 4861376
+silesia.tar, small window log, old streaming advanced, 7127552
+silesia.tar, small hash log, old streaming advanced, 6587834
+silesia.tar, small chain log, old streaming advanced, 4943271
+silesia.tar, explicit params, old streaming advanced, 4808570
silesia.tar, uncompressed literals, old streaming advanced, 4861376
silesia.tar, uncompressed literals optimal, old streaming advanced, 4281511
silesia.tar, huffman literals, old streaming advanced, 6190789
-silesia.tar, multithreaded with advanced params, old streaming advanced, 12022046
-github, level -5, old streaming advanced, 205285
-github, level -5 with dict, old streaming advanced, 46718
-github, level -3, old streaming advanced, 190643
-github, level -3 with dict, old streaming advanced, 45395
-github, level -1, old streaming advanced, 175568
-github, level -1 with dict, old streaming advanced, 43170
-github, level 0, old streaming advanced, 136311
-github, level 0 with dict, old streaming advanced, 41148
-github, level 1, old streaming advanced, 142450
-github, level 1 with dict, old streaming advanced, 41682
-github, level 3, old streaming advanced, 136311
-github, level 3 with dict, old streaming advanced, 41148
-github, level 4, old streaming advanced, 136144
-github, level 4 with dict, old streaming advanced, 41251
-github, level 5, old streaming advanced, 135106
-github, level 5 with dict, old streaming advanced, 38938
-github, level 6, old streaming advanced, 135108
-github, level 6 with dict, old streaming advanced, 38632
-github, level 7, old streaming advanced, 135108
-github, level 7 with dict, old streaming advanced, 38766
-github, level 9, old streaming advanced, 135108
-github, level 9 with dict, old streaming advanced, 39326
-github, level 13, old streaming advanced, 133717
-github, level 13 with dict, old streaming advanced, 39716
-github, level 16, old streaming advanced, 133717
-github, level 16 with dict, old streaming advanced, 37577
+silesia.tar, multithreaded with advanced params, old streaming advanced, 4861376
+github, level -5, old streaming advanced, 216734
+github, level -5 with dict, old streaming advanced, 49562
+github, level -3, old streaming advanced, 192160
+github, level -3 with dict, old streaming advanced, 44956
+github, level -1, old streaming advanced, 181108
+github, level -1 with dict, old streaming advanced, 42383
+github, level 0, old streaming advanced, 141090
+github, level 0 with dict, old streaming advanced, 41113
+github, level 1, old streaming advanced, 143682
+github, level 1 with dict, old streaming advanced, 42430
+github, level 3, old streaming advanced, 141090
+github, level 3 with dict, old streaming advanced, 41113
+github, level 4, old streaming advanced, 141090
+github, level 4 with dict, old streaming advanced, 41084
+github, level 5, old streaming advanced, 139391
+github, level 5 with dict, old streaming advanced, 39159
+github, level 6, old streaming advanced, 139394
+github, level 6 with dict, old streaming advanced, 38749
+github, level 7, old streaming advanced, 138675
+github, level 7 with dict, old streaming advanced, 38746
+github, level 9, old streaming advanced, 138675
+github, level 9 with dict, old streaming advanced, 38987
+github, level 13, old streaming advanced, 138675
+github, level 13 with dict, old streaming advanced, 39724
+github, level 16, old streaming advanced, 138675
+github, level 16 with dict, old streaming advanced, 40771
github, level 19, old streaming advanced, 133717
github, level 19 with dict, old streaming advanced, 37576
github, no source size, old streaming advanced, 140631
-github, long distance mode, old streaming advanced, 412933
-github, multithreaded, old streaming advanced, 412933
-github, multithreaded long distance mode, old streaming advanced, 412933
-github, small window log, old streaming advanced, 412933
-github, small hash log, old streaming advanced, 412933
-github, small chain log, old streaming advanced, 412933
-github, explicit params, old streaming advanced, 412933
-github, uncompressed literals, old streaming advanced, 136311
+github, long distance mode, old streaming advanced, 141090
+github, multithreaded, old streaming advanced, 141090
+github, multithreaded long distance mode, old streaming advanced, 141090
+github, small window log, old streaming advanced, 141090
+github, small hash log, old streaming advanced, 141578
+github, small chain log, old streaming advanced, 139258
+github, explicit params, old streaming advanced, 140930
+github, uncompressed literals, old streaming advanced, 141090
github, uncompressed literals optimal, old streaming advanced, 133717
-github, huffman literals, old streaming advanced, 175568
-github, multithreaded with advanced params, old streaming advanced, 412933
-silesia, level -5, old streaming cdcit, 6882466
-silesia, level -3, old streaming cdcit, 6568358
-silesia, level -1, old streaming cdcit, 6183385
-silesia, level 0, old streaming cdcit, 4849491
-silesia, level 1, old streaming cdcit, 5314109
-silesia, level 3, old streaming cdcit, 4849491
-silesia, level 4, old streaming cdcit, 4786913
-silesia, level 5, old streaming cdcit, 4710178
-silesia, level 6, old streaming cdcit, 4659996
-silesia, level 7, old streaming cdcit, 4596234
-silesia, level 9, old streaming cdcit, 4543862
-silesia, level 13, old streaming cdcit, 4482073
-silesia, level 16, old streaming cdcit, 4377389
-silesia, level 19, old streaming cdcit, 4293262
-silesia, no source size, old streaming cdcit, 4849455
-silesia, long distance mode, old streaming cdcit, 12000408
-silesia, multithreaded, old streaming cdcit, 12000408
-silesia, multithreaded long distance mode, old streaming cdcit, 12000408
-silesia, small window log, old streaming cdcit, 12000408
-silesia, small hash log, old streaming cdcit, 12000408
-silesia, small chain log, old streaming cdcit, 12000408
-silesia, explicit params, old streaming cdcit, 12000408
-silesia, uncompressed literals, old streaming cdcit, 4849491
-silesia, uncompressed literals optimal, old streaming cdcit, 4293262
-silesia, huffman literals, old streaming cdcit, 6183385
-silesia, multithreaded with advanced params, old streaming cdcit, 12000408
-silesia.tar, level -5, old streaming cdcit, 6982738
-silesia.tar, level -3, old streaming cdcit, 6641264
-silesia.tar, level -1, old streaming cdcit, 6190789
-silesia.tar, level 0, old streaming cdcit, 4861376
-silesia.tar, level 1, old streaming cdcit, 5336879
-silesia.tar, level 3, old streaming cdcit, 4861376
-silesia.tar, level 4, old streaming cdcit, 4799583
-silesia.tar, level 5, old streaming cdcit, 4722276
-silesia.tar, level 6, old streaming cdcit, 4672240
-silesia.tar, level 7, old streaming cdcit, 4606657
-silesia.tar, level 9, old streaming cdcit, 4554106
-silesia.tar, level 13, old streaming cdcit, 4491707
-silesia.tar, level 16, old streaming cdcit, 4381284
-silesia.tar, level 19, old streaming cdcit, 4281511
-silesia.tar, no source size, old streaming cdcit, 4861372
-silesia.tar, long distance mode, old streaming cdcit, 12022046
-silesia.tar, multithreaded, old streaming cdcit, 12022046
-silesia.tar, multithreaded long distance mode, old streaming cdcit, 12022046
-silesia.tar, small window log, old streaming cdcit, 12022046
-silesia.tar, small hash log, old streaming cdcit, 12022046
-silesia.tar, small chain log, old streaming cdcit, 12022046
-silesia.tar, explicit params, old streaming cdcit, 12022046
-silesia.tar, uncompressed literals, old streaming cdcit, 4861376
-silesia.tar, uncompressed literals optimal, old streaming cdcit, 4281511
-silesia.tar, huffman literals, old streaming cdcit, 6190789
-silesia.tar, multithreaded with advanced params, old streaming cdcit, 12022046
-github, level -5, old streaming cdcit, 205285
+github, huffman literals, old streaming advanced, 181108
+github, multithreaded with advanced params, old streaming advanced, 141090
github, level -5 with dict, old streaming cdcit, 46718
-github, level -3, old streaming cdcit, 190643
github, level -3 with dict, old streaming cdcit, 45395
-github, level -1, old streaming cdcit, 175568
github, level -1 with dict, old streaming cdcit, 43170
-github, level 0, old streaming cdcit, 136311
github, level 0 with dict, old streaming cdcit, 41148
-github, level 1, old streaming cdcit, 142450
github, level 1 with dict, old streaming cdcit, 41682
-github, level 3, old streaming cdcit, 136311
github, level 3 with dict, old streaming cdcit, 41148
-github, level 4, old streaming cdcit, 136144
github, level 4 with dict, old streaming cdcit, 41251
-github, level 5, old streaming cdcit, 135106
github, level 5 with dict, old streaming cdcit, 38938
-github, level 6, old streaming cdcit, 135108
github, level 6 with dict, old streaming cdcit, 38632
-github, level 7, old streaming cdcit, 135108
github, level 7 with dict, old streaming cdcit, 38766
-github, level 9, old streaming cdcit, 135108
github, level 9 with dict, old streaming cdcit, 39326
-github, level 13, old streaming cdcit, 133717
github, level 13 with dict, old streaming cdcit, 39716
-github, level 16, old streaming cdcit, 133717
github, level 16 with dict, old streaming cdcit, 37577
-github, level 19, old streaming cdcit, 133717
github, level 19 with dict, old streaming cdcit, 37576
-github, no source size, old streaming cdcit, 140631
-github, long distance mode, old streaming cdcit, 412933
-github, multithreaded, old streaming cdcit, 412933
-github, multithreaded long distance mode, old streaming cdcit, 412933
-github, small window log, old streaming cdcit, 412933
-github, small hash log, old streaming cdcit, 412933
-github, small chain log, old streaming cdcit, 412933
-github, explicit params, old streaming cdcit, 412933
-github, uncompressed literals, old streaming cdcit, 136311
-github, uncompressed literals optimal, old streaming cdcit, 133717
-github, huffman literals, old streaming cdcit, 175568
-github, multithreaded with advanced params, old streaming cdcit, 412933
-silesia, level -5, old streaming advanced cdict, 6882466
-silesia, level -3, old streaming advanced cdict, 6568358
-silesia, level -1, old streaming advanced cdict, 6183385
-silesia, level 0, old streaming advanced cdict, 4849491
-silesia, level 1, old streaming advanced cdict, 5314109
-silesia, level 3, old streaming advanced cdict, 4849491
-silesia, level 4, old streaming advanced cdict, 4786913
-silesia, level 5, old streaming advanced cdict, 4710178
-silesia, level 6, old streaming advanced cdict, 4659996
-silesia, level 7, old streaming advanced cdict, 4596234
-silesia, level 9, old streaming advanced cdict, 4543862
-silesia, level 13, old streaming advanced cdict, 4482073
-silesia, level 16, old streaming advanced cdict, 4377389
-silesia, level 19, old streaming advanced cdict, 4293262
-silesia, no source size, old streaming advanced cdict, 4849455
-silesia, long distance mode, old streaming advanced cdict, 12000408
-silesia, multithreaded, old streaming advanced cdict, 12000408
-silesia, multithreaded long distance mode, old streaming advanced cdict, 12000408
-silesia, small window log, old streaming advanced cdict, 12000408
-silesia, small hash log, old streaming advanced cdict, 12000408
-silesia, small chain log, old streaming advanced cdict, 12000408
-silesia, explicit params, old streaming advanced cdict, 12000408
-silesia, uncompressed literals, old streaming advanced cdict, 4849491
-silesia, uncompressed literals optimal, old streaming advanced cdict, 4293262
-silesia, huffman literals, old streaming advanced cdict, 6183385
-silesia, multithreaded with advanced params, old streaming advanced cdict, 12000408
-silesia.tar, level -5, old streaming advanced cdict, 6982738
-silesia.tar, level -3, old streaming advanced cdict, 6641264
-silesia.tar, level -1, old streaming advanced cdict, 6190789
-silesia.tar, level 0, old streaming advanced cdict, 4861376
-silesia.tar, level 1, old streaming advanced cdict, 5336879
-silesia.tar, level 3, old streaming advanced cdict, 4861376
-silesia.tar, level 4, old streaming advanced cdict, 4799583
-silesia.tar, level 5, old streaming advanced cdict, 4722276
-silesia.tar, level 6, old streaming advanced cdict, 4672240
-silesia.tar, level 7, old streaming advanced cdict, 4606657
-silesia.tar, level 9, old streaming advanced cdict, 4554106
-silesia.tar, level 13, old streaming advanced cdict, 4491707
-silesia.tar, level 16, old streaming advanced cdict, 4381284
-silesia.tar, level 19, old streaming advanced cdict, 4281511
-silesia.tar, no source size, old streaming advanced cdict, 4861372
-silesia.tar, long distance mode, old streaming advanced cdict, 12022046
-silesia.tar, multithreaded, old streaming advanced cdict, 12022046
-silesia.tar, multithreaded long distance mode, old streaming advanced cdict, 12022046
-silesia.tar, small window log, old streaming advanced cdict, 12022046
-silesia.tar, small hash log, old streaming advanced cdict, 12022046
-silesia.tar, small chain log, old streaming advanced cdict, 12022046
-silesia.tar, explicit params, old streaming advanced cdict, 12022046
-silesia.tar, uncompressed literals, old streaming advanced cdict, 4861376
-silesia.tar, uncompressed literals optimal, old streaming advanced cdict, 4281511
-silesia.tar, huffman literals, old streaming advanced cdict, 6190789
-silesia.tar, multithreaded with advanced params, old streaming advanced cdict, 12022046
-github, level -5, old streaming advanced cdict, 205285
-github, level -5 with dict, old streaming advanced cdict, 46718
-github, level -3, old streaming advanced cdict, 190643
-github, level -3 with dict, old streaming advanced cdict, 45395
-github, level -1, old streaming advanced cdict, 175568
-github, level -1 with dict, old streaming advanced cdict, 43170
-github, level 0, old streaming advanced cdict, 136311
-github, level 0 with dict, old streaming advanced cdict, 41148
-github, level 1, old streaming advanced cdict, 142450
-github, level 1 with dict, old streaming advanced cdict, 41682
-github, level 3, old streaming advanced cdict, 136311
-github, level 3 with dict, old streaming advanced cdict, 41148
-github, level 4, old streaming advanced cdict, 136144
-github, level 4 with dict, old streaming advanced cdict, 41251
-github, level 5, old streaming advanced cdict, 135106
-github, level 5 with dict, old streaming advanced cdict, 38938
-github, level 6, old streaming advanced cdict, 135108
-github, level 6 with dict, old streaming advanced cdict, 38632
-github, level 7, old streaming advanced cdict, 135108
-github, level 7 with dict, old streaming advanced cdict, 38766
-github, level 9, old streaming advanced cdict, 135108
-github, level 9 with dict, old streaming advanced cdict, 39326
-github, level 13, old streaming advanced cdict, 133717
-github, level 13 with dict, old streaming advanced cdict, 39716
-github, level 16, old streaming advanced cdict, 133717
-github, level 16 with dict, old streaming advanced cdict, 37577
-github, level 19, old streaming advanced cdict, 133717
+github, level -5 with dict, old streaming advanced cdict, 49562
+github, level -3 with dict, old streaming advanced cdict, 44956
+github, level -1 with dict, old streaming advanced cdict, 42383
+github, level 0 with dict, old streaming advanced cdict, 41113
+github, level 1 with dict, old streaming advanced cdict, 42430
+github, level 3 with dict, old streaming advanced cdict, 41113
+github, level 4 with dict, old streaming advanced cdict, 41084
+github, level 5 with dict, old streaming advanced cdict, 39158
+github, level 6 with dict, old streaming advanced cdict, 38748
+github, level 7 with dict, old streaming advanced cdict, 38744
+github, level 9 with dict, old streaming advanced cdict, 38986
+github, level 13 with dict, old streaming advanced cdict, 39724
+github, level 16 with dict, old streaming advanced cdict, 40771
github, level 19 with dict, old streaming advanced cdict, 37576
-github, no source size, old streaming advanced cdict, 140631
-github, long distance mode, old streaming advanced cdict, 412933
-github, multithreaded, old streaming advanced cdict, 412933
-github, multithreaded long distance mode, old streaming advanced cdict, 412933
-github, small window log, old streaming advanced cdict, 412933
-github, small hash log, old streaming advanced cdict, 412933
-github, small chain log, old streaming advanced cdict, 412933
-github, explicit params, old streaming advanced cdict, 412933
-github, uncompressed literals, old streaming advanced cdict, 136311
-github, uncompressed literals optimal, old streaming advanced cdict, 133717
-github, huffman literals, old streaming advanced cdict, 175568
-github, multithreaded with advanced params, old streaming advanced cdict, 412933
diff --git a/tests/zbufftest.c b/tests/zbufftest.c
index 94414826..8a4a2790 100644
--- a/tests/zbufftest.c
+++ b/tests/zbufftest.c
@@ -81,7 +81,7 @@ static U64 g_clockTime = 0;
@return : a 27 bits random value, from a 32-bits `seed`.
`seed` is also modified */
# define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
-unsigned int FUZ_rand(unsigned int* seedPtr)
+static unsigned int FUZ_rand(unsigned int* seedPtr)
{
U32 rand32 = *seedPtr;
rand32 *= prime1;
@@ -467,7 +467,7 @@ _output_error:
/*-*******************************************************
* Command line
*********************************************************/
-int FUZ_usage(const char* programName)
+static int FUZ_usage(const char* programName)
{
DISPLAY( "Usage :\n");
DISPLAY( " %s [args]\n", programName);
diff --git a/zlibWrapper/Makefile b/zlibWrapper/Makefile
index f63291c9..6addb743 100644
--- a/zlibWrapper/Makefile
+++ b/zlibWrapper/Makefile
@@ -103,6 +103,7 @@ zstd_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/z
zstdTurnedOn_zlibwrapper.o: CPPFLAGS += -DZWRAP_USE_ZSTD=1
zstdTurnedOn_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.h
+ $(CC) $(CPPFLAGS) $(CFLAGS) $< -c -o $@
$(ZSTDLIBDIR)/libzstd.a:
$(MAKE) -C $(ZSTDLIBDIR) libzstd.a
diff --git a/zlibWrapper/gzread.c b/zlibWrapper/gzread.c
index bcac9700..359d1788 100644
--- a/zlibWrapper/gzread.c
+++ b/zlibWrapper/gzread.c
@@ -8,6 +8,14 @@
#include "gzguts.h"
+/* fix for Visual Studio, which doesn't support ssize_t type.
+ * see https://github.com/facebook/zstd/issues/1800#issuecomment-545945050 */
+#if defined(_MSC_VER) && !defined(ssize_t)
+# include
+ typedef SSIZE_T ssize_t;
+#endif
+
+
/* Local functions */
local int gz_load OF((gz_statep, unsigned char *, unsigned, unsigned *));
local int gz_avail OF((gz_statep));