From a4ca246ecae023469a6c93c0c274f2bc97d7103f Mon Sep 17 00:00:00 2001 From: Sergey Ponomarev Date: Sat, 14 Sep 2019 20:15:35 +0300 Subject: [PATCH 01/15] build/cmake/README.md: improve --- build/cmake/README.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/build/cmake/README.md b/build/cmake/README.md index 854389ad..73b30dc7 100644 --- a/build/cmake/README.md +++ b/build/cmake/README.md @@ -5,9 +5,9 @@ use case sensitivity that matches modern (ie. cmake version 2.6 and above) conventions of using lower-case for commands, and upper-case for variables. -# How to build +## How to build -As cmake doesn't support command like `cmake clean`, it's recommanded to perform a "out of source build". +As cmake doesn't support command like `cmake clean`, it's recommended to perform a "out of source build". To do this, you can create a new directory and build in it: ```sh cd build/cmake @@ -16,7 +16,7 @@ cd builddir cmake .. make ``` -Then you can clean all cmake caches by simpily delete the new directory: +Then you can clean all cmake caches by simply delete the new directory: ```sh rm -rf build/cmake/builddir ``` @@ -34,19 +34,19 @@ cd build/cmake/builddir cmake -LH .. ``` -Bool options can be set to ON/OFF with -D\[option\]=\[ON/OFF\]. You can configure cmake options like this: +Bool options can be set to `ON/OFF` with `-D[option]=[ON/OFF]`. You can configure cmake options like this: ```sh cd build/cmake/builddir cmake -DZSTD_BUILD_TESTS=ON -DZSTD_LEGACY_SUPPORT=ON .. make ``` -## referring +### referring [Looking for a 'cmake clean' command to clear up CMake output](https://stackoverflow.com/questions/9680420/looking-for-a-cmake-clean-command-to-clear-up-cmake-output) -# CMake Style Recommendations +## CMake Style Recommendations -## Indent all code correctly, i.e. the body of +### Indent all code correctly, i.e. the body of * if/else/endif * foreach/endforeach @@ -57,7 +57,7 @@ make Use spaces for indenting, 2, 3 or 4 spaces preferably. Use the same amount of spaces for indenting as is used in the rest of the file. Do not use tabs. -## Upper/lower casing +### Upper/lower casing Most important: use consistent upper- or lowercasing within one file ! @@ -77,7 +77,7 @@ Add_Executable(hello hello.c) aDd_ExEcUtAbLe(blub blub.c) ``` -## End commands +### End commands To make the code easier to read, use empty commands for endforeach(), endif(), endfunction(), endmacro() and endwhile(). Also, use empty else() commands. @@ -99,6 +99,6 @@ if(BARVAR) endif(BARVAR) ``` -## Other resources for best practices +### Other resources for best practices -`https://cmake.org/cmake/help/latest/manual/cmake-developer.7.html#modules` +https://cmake.org/cmake/help/latest/manual/cmake-developer.7.html#modules From b804dd3e5bd3397fa5bbb3a5313496e9156d4fb5 Mon Sep 17 00:00:00 2001 From: Sergey Ponomarev Date: Sat, 14 Sep 2019 21:14:43 +0300 Subject: [PATCH 02/15] #754 move sufixlist upper and improve error message on missing suffix --- programs/fileio.c | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 8a45563d..e5fb1aad 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -2169,12 +2169,24 @@ FIO_determineDstName(const char* srcFileName) static size_t dfnbCapacity = 0; static char* dstFileNameBuffer = NULL; /* using static allocation : this function cannot be multi-threaded */ + const char* suffixlist = ZSTD_EXTENSION + #ifdef ZSTD_GZDECOMPRESS + "/" GZ_EXTENSION + #endif + #ifdef ZSTD_LZMADECOMPRESS + "/" XZ_EXTENSION "/" LZMA_EXTENSION + #endif + #ifdef ZSTD_LZ4DECOMPRESS + "/" LZ4_EXTENSION + #endif + ; + size_t const sfnSize = strlen(srcFileName); size_t suffixSize; const char* const suffixPtr = strrchr(srcFileName, '.'); if (suffixPtr == NULL) { - DISPLAYLEVEL(1, "zstd: %s: unknown suffix -- ignored \n", - srcFileName); + DISPLAYLEVEL(1, "zstd: %s: missing suffix (%s expected) -- ignored \n", + srcFileName, suffixlist); return NULL; } suffixSize = strlen(suffixPtr); @@ -2193,17 +2205,6 @@ FIO_determineDstName(const char* srcFileName) && strcmp(suffixPtr, LZ4_EXTENSION) #endif ) ) { - const char* suffixlist = ZSTD_EXTENSION - #ifdef ZSTD_GZDECOMPRESS - "/" GZ_EXTENSION - #endif - #ifdef ZSTD_LZMADECOMPRESS - "/" XZ_EXTENSION "/" LZMA_EXTENSION - #endif - #ifdef ZSTD_LZ4DECOMPRESS - "/" LZ4_EXTENSION - #endif - ; DISPLAYLEVEL(1, "zstd: %s: unknown suffix (%s expected) -- ignored \n", srcFileName, suffixlist); return NULL; From 8cc815a941d2b96a867a4c3d89af3b4cd574d0e8 Mon Sep 17 00:00:00 2001 From: Sergey Ponomarev Date: Sat, 14 Sep 2019 21:15:24 +0300 Subject: [PATCH 03/15] #754 sufixlist->SUFFIX_LIST --- programs/fileio.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index e5fb1aad..7ad54055 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -2169,7 +2169,7 @@ FIO_determineDstName(const char* srcFileName) static size_t dfnbCapacity = 0; static char* dstFileNameBuffer = NULL; /* using static allocation : this function cannot be multi-threaded */ - const char* suffixlist = ZSTD_EXTENSION + const char* SUFFIX_LIST = ZSTD_EXTENSION #ifdef ZSTD_GZDECOMPRESS "/" GZ_EXTENSION #endif @@ -2186,7 +2186,7 @@ FIO_determineDstName(const char* srcFileName) const char* const suffixPtr = strrchr(srcFileName, '.'); if (suffixPtr == NULL) { DISPLAYLEVEL(1, "zstd: %s: missing suffix (%s expected) -- ignored \n", - srcFileName, suffixlist); + srcFileName, SUFFIX_LIST); return NULL; } suffixSize = strlen(suffixPtr); @@ -2206,7 +2206,7 @@ FIO_determineDstName(const char* srcFileName) #endif ) ) { DISPLAYLEVEL(1, "zstd: %s: unknown suffix (%s expected) -- ignored \n", - srcFileName, suffixlist); + srcFileName, SUFFIX_LIST); return NULL; } From 7d9cd22e2145f68cb04b87d76907366726ab0bc3 Mon Sep 17 00:00:00 2001 From: Sergey Ponomarev Date: Sat, 14 Sep 2019 21:23:47 +0300 Subject: [PATCH 04/15] #754 Add a hint about -o option --- programs/fileio.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 7ad54055..edf58fee 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -2185,7 +2185,7 @@ FIO_determineDstName(const char* srcFileName) size_t suffixSize; const char* const suffixPtr = strrchr(srcFileName, '.'); if (suffixPtr == NULL) { - DISPLAYLEVEL(1, "zstd: %s: missing suffix (%s expected) -- ignored \n", + DISPLAYLEVEL(1, "zstd: %s: missing suffix (%s expected). Can't derive the output file name so specify it with -o dstFileName. -- ignored \n", srcFileName, SUFFIX_LIST); return NULL; } @@ -2205,7 +2205,7 @@ FIO_determineDstName(const char* srcFileName) && strcmp(suffixPtr, LZ4_EXTENSION) #endif ) ) { - DISPLAYLEVEL(1, "zstd: %s: unknown suffix (%s expected) -- ignored \n", + DISPLAYLEVEL(1, "zstd: %s: unknown suffix (%s expected). Can't derive the output file name so specify it with -o dstFileName. -- ignored \n", srcFileName, SUFFIX_LIST); return NULL; } From a101721f4e2f3cfa0296b5b61147f272e77c8b68 Mon Sep 17 00:00:00 2001 From: Sergey Ponomarev Date: Sat, 14 Sep 2019 21:26:27 +0300 Subject: [PATCH 05/15] Use one strstr() call instead of chain of strcmp() --- programs/fileio.c | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index edf58fee..8816115b 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -2193,18 +2193,7 @@ FIO_determineDstName(const char* srcFileName) /* check suffix is authorized */ if (sfnSize <= suffixSize - || ( strcmp(suffixPtr, ZSTD_EXTENSION) - #ifdef ZSTD_GZDECOMPRESS - && strcmp(suffixPtr, GZ_EXTENSION) - #endif - #ifdef ZSTD_LZMADECOMPRESS - && strcmp(suffixPtr, XZ_EXTENSION) - && strcmp(suffixPtr, LZMA_EXTENSION) - #endif - #ifdef ZSTD_LZ4DECOMPRESS - && strcmp(suffixPtr, LZ4_EXTENSION) - #endif - ) ) { + || (strstr(SUFFIX_LIST, suffixPtr) == NULL)) { DISPLAYLEVEL(1, "zstd: %s: unknown suffix (%s expected). Can't derive the output file name so specify it with -o dstFileName. -- ignored \n", srcFileName, SUFFIX_LIST); return NULL; From 59f369a6da86e83948084270a06ad72368f079cc Mon Sep 17 00:00:00 2001 From: Sergey Ponomarev Date: Sat, 14 Sep 2019 21:30:15 +0300 Subject: [PATCH 06/15] Add short tar's extensions .tgz (.tar.gz), .txz (.tar.xz), .tzst (.tar.zst) --- programs/fileio.c | 8 ++++---- programs/fileio.h | 4 ++++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 8816115b..f7e3b234 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -2169,15 +2169,15 @@ FIO_determineDstName(const char* srcFileName) static size_t dfnbCapacity = 0; static char* dstFileNameBuffer = NULL; /* using static allocation : this function cannot be multi-threaded */ - const char* SUFFIX_LIST = ZSTD_EXTENSION + const char* SUFFIX_LIST = ZSTD_EXTENSION "/" TZSTD_EXTENSION #ifdef ZSTD_GZDECOMPRESS - "/" GZ_EXTENSION + "/" GZ_EXTENSION "/" TGZ_EXTENSION #endif #ifdef ZSTD_LZMADECOMPRESS - "/" XZ_EXTENSION "/" LZMA_EXTENSION + "/" XZ_EXTENSION "/" LZMA_EXTENSION "/" TXZ_EXTENSION #endif #ifdef ZSTD_LZ4DECOMPRESS - "/" LZ4_EXTENSION + "/" LZ4_EXTENSION "/" TLZ4_EXTENSION #endif ; diff --git a/programs/fileio.h b/programs/fileio.h index 096d90b5..ebd2ffbe 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -32,9 +32,13 @@ extern "C" { #endif #define LZMA_EXTENSION ".lzma" #define XZ_EXTENSION ".xz" +#define TXZ_EXTENSION ".txz" #define GZ_EXTENSION ".gz" +#define TGZ_EXTENSION ".tgz" #define ZSTD_EXTENSION ".zst" +#define TZSTD_EXTENSION ".tzst" #define LZ4_EXTENSION ".lz4" +#define TLZ4_EXTENSION ".tlz4" /*-************************************* From bfb4d830b299feb14da189b775700b5fa41950d5 Mon Sep 17 00:00:00 2001 From: Sergey Ponomarev Date: Wed, 18 Sep 2019 09:21:00 +0300 Subject: [PATCH 07/15] FIO_determineDstName: extract dstFileNameEndPos variable --- programs/fileio.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index f7e3b234..96170b14 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -2210,8 +2210,9 @@ FIO_determineDstName(const char* srcFileName) /* return dst name == src name truncated from suffix */ assert(dstFileNameBuffer != NULL); - memcpy(dstFileNameBuffer, srcFileName, sfnSize - suffixSize); - dstFileNameBuffer[sfnSize-suffixSize] = '\0'; + size_t dstFileNameEndPos = sfnSize - suffixSize; + memcpy(dstFileNameBuffer, srcFileName, dstFileNameEndPos); + dstFileNameBuffer[dstFileNameEndPos] = '\0'; return dstFileNameBuffer; /* note : dstFileNameBuffer memory is not going to be free */ From dafe796e39492a180dacec35b4a4c963dcd88a37 Mon Sep 17 00:00:00 2001 From: Sergey Ponomarev Date: Wed, 18 Sep 2019 09:23:10 +0300 Subject: [PATCH 08/15] #1790 short tar's extensions tgz, txz, tlz4m .tzst should be decompressed with .tar suffix --- programs/fileio.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/programs/fileio.c b/programs/fileio.c index 96170b14..5aaad0e9 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -2212,6 +2212,15 @@ FIO_determineDstName(const char* srcFileName) assert(dstFileNameBuffer != NULL); size_t dstFileNameEndPos = sfnSize - suffixSize; memcpy(dstFileNameBuffer, srcFileName, dstFileNameEndPos); + /* The short tar extensions tzst, tgz, txz and tlz4 files should have "tar" extension on decompression + * To check that the file is one of them we can check that it starts with "t" + */ + if (suffixPtr[1] == 't') { + dstFileNameBuffer[dstFileNameEndPos++] = '.'; + dstFileNameBuffer[dstFileNameEndPos++] = 't'; + dstFileNameBuffer[dstFileNameEndPos++] = 'a'; + dstFileNameBuffer[dstFileNameEndPos++] = 'r'; + } dstFileNameBuffer[dstFileNameEndPos] = '\0'; return dstFileNameBuffer; From 42a22af78b068bd934123ae7e1d6eee1544282f6 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 24 Oct 2019 15:19:05 -0700 Subject: [PATCH 09/15] fix zlibWrapper for Visual Studio As per https://github.com/facebook/zstd/issues/1800#issuecomment-545945050, Visual does not support `ssize_t` type, which is an issue for `gzread.c`. Added a work around, suggested by @bluenlive Note : I have not been able to confirm the problem, so this is a blind fix. This seems safe outside of Visual, since it is gated by _MSC_VER macro. --- zlibWrapper/gzread.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/zlibWrapper/gzread.c b/zlibWrapper/gzread.c index bcac9700..359d1788 100644 --- a/zlibWrapper/gzread.c +++ b/zlibWrapper/gzread.c @@ -8,6 +8,14 @@ #include "gzguts.h" +/* fix for Visual Studio, which doesn't support ssize_t type. + * see https://github.com/facebook/zstd/issues/1800#issuecomment-545945050 */ +#if defined(_MSC_VER) && !defined(ssize_t) +# include + typedef SSIZE_T ssize_t; +#endif + + /* Local functions */ local int gz_load OF((gz_statep, unsigned char *, unsigned, unsigned *)); local int gz_avail OF((gz_statep)); From 91c3f545cc51fbcd8d5131ee06a944125911ac02 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 24 Oct 2019 20:18:57 -0400 Subject: [PATCH 10/15] Fix Build; Refactor --- programs/fileio.c | 122 ++++++++++++++++++++++++++++++---------------- programs/fileio.h | 15 ++++-- 2 files changed, 91 insertions(+), 46 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 07503f05..c2c5618e 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1496,17 +1496,17 @@ FIO_determineCompressedName(const char* srcFileName, const char* outDirName, con static char* dstFileNameBuffer = NULL; /* using static allocation : this function cannot be multi-threaded */ char* outDirFilename = NULL; size_t sfnSize = strlen(srcFileName); - size_t const suffixSize = strlen(suffix); + size_t const srcSuffixLen = strlen(suffix); if (outDirName) { - outDirFilename = FIO_createFilename_fromOutDir(srcFileName, outDirName, suffixSize); + outDirFilename = FIO_createFilename_fromOutDir(srcFileName, outDirName, srcSuffixLen); sfnSize = strlen(outDirFilename); assert(outDirFilename != NULL); } - if (dfnbCapacity <= sfnSize+suffixSize+1) { + if (dfnbCapacity <= sfnSize+srcSuffixLen+1) { /* resize buffer for dstName */ free(dstFileNameBuffer); - dfnbCapacity = sfnSize + suffixSize + 30; + dfnbCapacity = sfnSize + srcSuffixLen + 30; dstFileNameBuffer = (char*)malloc(dfnbCapacity); if (!dstFileNameBuffer) { EXM_THROW(30, "zstd: %s", strerror(errno)); @@ -1520,7 +1520,7 @@ FIO_determineCompressedName(const char* srcFileName, const char* outDirName, con } else { memcpy(dstFileNameBuffer, srcFileName, sfnSize); } - memcpy(dstFileNameBuffer+sfnSize, suffix, suffixSize+1 /* Include terminating null */); + memcpy(dstFileNameBuffer+sfnSize, suffix, srcSuffixLen+1 /* Include terminating null */); return dstFileNameBuffer; } @@ -2287,6 +2287,37 @@ int FIO_decompressFilename(FIO_prefs_t* const prefs, return decodingError; } +static const char *suffixList[] = { + ZSTD_EXTENSION, + TZSTD_EXTENSION, +#ifdef ZSTD_GZDECOMPRESS + GZ_EXTENSION, + TGZ_EXTENSION, +#endif +#ifdef ZSTD_LZMADECOMPRESS + LZMA_EXTENSION, + XZ_EXTENSION, + TXZ_EXTENSION, +#endif +#ifdef ZSTD_LZ4DECOMPRESS + LZ4_EXTENSION, + TLZ4_EXTENSION, +#endif + NULL +}; + +static const char *suffixListStr = + ZSTD_EXTENSION "/" TZSTD_EXTENSION +#ifdef ZSTD_GZDECOMPRESS + "/" GZ_EXTENSION "/" TGZ_EXTENSION +#endif +#ifdef ZSTD_LZMADECOMPRESS + "/" LZMA_EXTENSION "/" XZ_EXTENSION "/" TXZ_EXTENSION +#endif +#ifdef ZSTD_LZ4DECOMPRESS + "/" LZ4_EXTENSION "/" TLZ4_EXTENSION +#endif +; /* FIO_determineDstName() : * create a destination filename from a srcFileName. @@ -2297,72 +2328,79 @@ FIO_determineDstName(const char* srcFileName, const char* outDirName) { static size_t dfnbCapacity = 0; static char* dstFileNameBuffer = NULL; /* using static allocation : this function cannot be multi-threaded */ + size_t dstFileNameEndPos; char* outDirFilename = NULL; - - const char* SUFFIX_LIST = ZSTD_EXTENSION "/" TZSTD_EXTENSION - #ifdef ZSTD_GZDECOMPRESS - "/" GZ_EXTENSION "/" TGZ_EXTENSION - #endif - #ifdef ZSTD_LZMADECOMPRESS - "/" XZ_EXTENSION "/" LZMA_EXTENSION "/" TXZ_EXTENSION - #endif - #ifdef ZSTD_LZ4DECOMPRESS - "/" LZ4_EXTENSION "/" TLZ4_EXTENSION - #endif - ; + const char* dstSuffix = ""; + size_t dstSuffixLen = 0; size_t sfnSize = strlen(srcFileName); - size_t suffixSize; - const char* const suffixPtr = strrchr(srcFileName, '.'); - if (suffixPtr == NULL) { - DISPLAYLEVEL(1, "zstd: %s: missing suffix (%s expected). Can't derive the output file name so specify it with -o dstFileName. -- ignored \n", - srcFileName, SUFFIX_LIST); + size_t srcSuffixLen; + const char* const srcSuffix = strrchr(srcFileName, '.'); + if (srcSuffix == NULL) { + DISPLAYLEVEL(1, + "zstd: %s: unknown suffix (%s expected). " + "Can't derive the output file name. " + "Specify it with -o dstFileName. Ignoring.\n", + srcFileName, suffixListStr); return NULL; } - suffixSize = strlen(suffixPtr); + srcSuffixLen = strlen(srcSuffix); - /* check suffix is authorized */ - if (sfnSize <= suffixSize - || (strstr(SUFFIX_LIST, suffixPtr) == NULL)) { - DISPLAYLEVEL(1, "zstd: %s: unknown suffix (%s expected). Can't derive the output file name so specify it with -o dstFileName. -- ignored \n", - srcFileName, SUFFIX_LIST); - return NULL; + { + const char** matchedSuffixPtr; + for (matchedSuffixPtr = suffixList; *matchedSuffixPtr != NULL; matchedSuffixPtr++) { + if (!strcmp(*matchedSuffixPtr, srcSuffix)) { + break; + } + } + + /* check suffix is authorized */ + if (sfnSize <= srcSuffixLen || *matchedSuffixPtr == NULL) { + DISPLAYLEVEL(1, + "zstd: %s: unknown suffix (%s expected). " + "Can't derive the output file name. " + "Specify it with -o dstFileName. Ignoring.\n", + srcFileName, suffixListStr); + return NULL; + } + + if ((*matchedSuffixPtr)[1] == 't') { + dstSuffix = ".tar"; + dstSuffixLen = strlen(dstSuffix); + } } + if (outDirName) { outDirFilename = FIO_createFilename_fromOutDir(srcFileName, outDirName, 0); sfnSize = strlen(outDirFilename); assert(outDirFilename != NULL); } - if (dfnbCapacity+suffixSize <= sfnSize+1) { + if (dfnbCapacity+srcSuffixLen <= sfnSize+1+dstSuffixLen) { /* allocate enough space to write dstFilename into it */ free(dstFileNameBuffer); dfnbCapacity = sfnSize + 20; dstFileNameBuffer = (char*)malloc(dfnbCapacity); if (dstFileNameBuffer==NULL) - EXM_THROW(74, "%s : not enough memory for dstFileName", strerror(errno)); + EXM_THROW(74, "%s : not enough memory for dstFileName", + strerror(errno)); } /* return dst name == src name truncated from suffix */ assert(dstFileNameBuffer != NULL); - size_t dstFileNameEndPos = sfnSize - suffixSize; + dstFileNameEndPos = sfnSize - srcSuffixLen; if (outDirFilename) { memcpy(dstFileNameBuffer, outDirFilename, dstFileNameEndPos); free(outDirFilename); } else { memcpy(dstFileNameBuffer, srcFileName, dstFileNameEndPos); } - /* The short tar extensions tzst, tgz, txz and tlz4 files should have "tar" extension on decompression - * To check that the file is one of them we can check that it starts with "t" - */ - if (suffixPtr[1] == 't') { - dstFileNameBuffer[dstFileNameEndPos++] = '.'; - dstFileNameBuffer[dstFileNameEndPos++] = 't'; - dstFileNameBuffer[dstFileNameEndPos++] = 'a'; - dstFileNameBuffer[dstFileNameEndPos++] = 'r'; - } - dstFileNameBuffer[dstFileNameEndPos] = '\0'; + + /* The short tar extensions tzst, tgz, txz and tlz4 files should have "tar" + * extension on decompression. Also writes terminating null. */ + strcpy(dstFileNameBuffer + dstFileNameEndPos, dstSuffix); + dstFileNameEndPos += dstSuffixLen; return dstFileNameBuffer; /* note : dstFileNameBuffer memory is not going to be free */ diff --git a/programs/fileio.h b/programs/fileio.h index 42e2274a..39eb544a 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -30,15 +30,22 @@ extern "C" { #else # define nulmark "/dev/null" #endif + +/* We test whether the extension we found starts with 't', and if so, we append + * ".tar" to the end of the output name. + */ #define LZMA_EXTENSION ".lzma" #define XZ_EXTENSION ".xz" -#define TXZ_EXTENSION ".txz" +#define TXZ_EXTENSION ".txz" + #define GZ_EXTENSION ".gz" -#define TGZ_EXTENSION ".tgz" +#define TGZ_EXTENSION ".tgz" + #define ZSTD_EXTENSION ".zst" -#define TZSTD_EXTENSION ".tzst" +#define TZSTD_EXTENSION ".tzst" + #define LZ4_EXTENSION ".lz4" -#define TLZ4_EXTENSION ".tlz4" +#define TLZ4_EXTENSION ".tlz4" /*-************************************* From 24499036ba88dbead8cca53a6075c9229ef959f9 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 24 Oct 2019 20:19:10 -0400 Subject: [PATCH 11/15] Add Tests --- tests/playTests.sh | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/tests/playTests.sh b/tests/playTests.sh index f68ee81a..9d9c8e3d 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -845,6 +845,46 @@ if [ $LZ4MODE -ne 1 ]; then grep ".lz4" tmplg > $INTOVOID && die "Unsupported suffix listed" fi +println "\n===> tar extension tests " + +rm -f tmp tmp.tar tmp.tzst tmp.tgz tmp.txz tmp.tlz4 + +./datagen > tmp +tar cf tmp.tar tmp +$ZSTD tmp.tar -o tmp.tzst +rm tmp.tar +$ZSTD -d tmp.tzst +[ -e tmp.tar ] || die ".tzst failed to decompress to .tar!" +rm -f tmp.tar tmp.tzst + +if [ $GZIPMODE -eq 1 ]; then + tar czf tmp.tgz tmp + $ZSTD -d tmp.tgz + [ -e tmp.tar ] || die ".tgz failed to decompress to .tar!" + rm -f tmp.tar tmp.tgz +fi + +if [ $LZMAMODE -eq 1 ]; then + tar c tmp | xz > tmp.txz + $ZSTD -d tmp.txz + [ -e tmp.tar ] || die ".txz failed to decompress to .tar!" + rm -f tmp.tar tmp.txz +fi + +if [ $LZ4MODE -eq 1 ]; then + tar c tmp | lz4 > tmp.tlz4 + $ZSTD -d tmp.tlz4 + [ -e tmp.tar ] || die ".tlz4 failed to decompress to .tar!" + rm -f tmp.tar tmp.tlz4 +fi + +touch tmp.t tmp.tz tmp.tzs +! $ZSTD -d tmp.t +! $ZSTD -d tmp.tz +! $ZSTD -d tmp.tzs + +exit + println "\n===> zstd round-trip tests " roundTripTest From 4eccc82875d6e5ca8a621fc80acd363a5efb9d3f Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Fri, 25 Oct 2019 13:58:58 -0400 Subject: [PATCH 12/15] Minor Fixes --- programs/fileio.c | 1 - programs/fileio.h | 3 ++- tests/playTests.sh | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index c2c5618e..828878c6 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -2400,7 +2400,6 @@ FIO_determineDstName(const char* srcFileName, const char* outDirName) /* The short tar extensions tzst, tgz, txz and tlz4 files should have "tar" * extension on decompression. Also writes terminating null. */ strcpy(dstFileNameBuffer + dstFileNameEndPos, dstSuffix); - dstFileNameEndPos += dstSuffixLen; return dstFileNameBuffer; /* note : dstFileNameBuffer memory is not going to be free */ diff --git a/programs/fileio.h b/programs/fileio.h index 39eb544a..af2c5d9d 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -31,7 +31,8 @@ extern "C" { # define nulmark "/dev/null" #endif -/* We test whether the extension we found starts with 't', and if so, we append +/** + * We test whether the extension we found starts with 't', and if so, we append * ".tar" to the end of the output name. */ #define LZMA_EXTENSION ".lzma" diff --git a/tests/playTests.sh b/tests/playTests.sh index 9d9c8e3d..c1da1650 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -865,14 +865,14 @@ if [ $GZIPMODE -eq 1 ]; then fi if [ $LZMAMODE -eq 1 ]; then - tar c tmp | xz > tmp.txz + tar c tmp | $ZSTD --format=xz > tmp.txz $ZSTD -d tmp.txz [ -e tmp.tar ] || die ".txz failed to decompress to .tar!" rm -f tmp.tar tmp.txz fi if [ $LZ4MODE -eq 1 ]; then - tar c tmp | lz4 > tmp.tlz4 + tar c tmp | $ZSTD --format=lz4 > tmp.tlz4 $ZSTD -d tmp.tlz4 [ -e tmp.tar ] || die ".tlz4 failed to decompress to .tar!" rm -f tmp.tar tmp.tlz4 From 74bd76c3ffb3837a3fefc055edbc347620447236 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Fri, 25 Oct 2019 15:05:20 -0400 Subject: [PATCH 13/15] In pkg-config File, Derive Lib and Include Dir from Prefix at Use-Time Addresses #1794. Instead of deriving the lib dir and include dir at build-time, let's do it like everyone else does at pkg-config run-time. This has the disadvantage that we can no longer override LIBDIR and INCLUDEDIR in the Makefile and have that reflected in the .pc file. --- build/cmake/lib/CMakeLists.txt | 3 +-- lib/Makefile | 2 -- lib/libzstd.pc.in | 5 +++-- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/build/cmake/lib/CMakeLists.txt b/build/cmake/lib/CMakeLists.txt index 77b389ca..7adca875 100644 --- a/build/cmake/lib/CMakeLists.txt +++ b/build/cmake/lib/CMakeLists.txt @@ -134,11 +134,10 @@ if (UNIX) # pkg-config set(PREFIX "${CMAKE_INSTALL_PREFIX}") set(LIBDIR "${CMAKE_INSTALL_FULL_LIBDIR}") - set(INCLUDEDIR "${CMAKE_INSTALL_FULL_INCLUDEDIR}") set(VERSION "${zstd_VERSION_MAJOR}.${zstd_VERSION_MINOR}.${zstd_VERSION_PATCH}") add_custom_target(libzstd.pc ALL ${CMAKE_COMMAND} -DIN="${LIBRARY_DIR}/libzstd.pc.in" -DOUT="libzstd.pc" - -DPREFIX="${PREFIX}" -DLIBDIR="${LIBDIR}" -DINCLUDEDIR="${INCLUDEDIR}" -DVERSION="${VERSION}" + -DPREFIX="${PREFIX}" -DVERSION="${VERSION}" -P "${CMAKE_CURRENT_SOURCE_DIR}/pkgconfig.cmake" COMMENT "Creating pkg-config file") diff --git a/lib/Makefile b/lib/Makefile index 87a396c5..273ceb90 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -244,8 +244,6 @@ libzstd.pc: libzstd.pc: libzstd.pc.in @echo creating pkgconfig @sed -e 's|@PREFIX@|$(PREFIX)|' \ - -e 's|@LIBDIR@|$(LIBDIR)|' \ - -e 's|@INCLUDEDIR@|$(INCLUDEDIR)|' \ -e 's|@VERSION@|$(VERSION)|' \ $< >$@ diff --git a/lib/libzstd.pc.in b/lib/libzstd.pc.in index 1d07b91f..e7880be4 100644 --- a/lib/libzstd.pc.in +++ b/lib/libzstd.pc.in @@ -3,8 +3,9 @@ # BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) prefix=@PREFIX@ -libdir=@LIBDIR@ -includedir=@INCLUDEDIR@ +exec_prefix=${prefix} +includedir=${prefix}/include +libdir=${exec_prefix}/lib Name: zstd Description: fast lossless compression algorithm library From 74065da4c5ea14a9755982e5ca0a72bf2d4f10d5 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 28 Oct 2019 11:15:41 -0700 Subject: [PATCH 14/15] updated API inline doc and manual regarding ZSTD_CDict created without a dictBuffer. --- doc/zstd_manual.html | 26 +++++++++++++++++++------- lib/zstd.h | 17 +++++++++++------ 2 files changed, 30 insertions(+), 13 deletions(-) diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html index 0021eec2..43c5555b 100644 --- a/doc/zstd_manual.html +++ b/doc/zstd_manual.html @@ -692,12 +692,17 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds); 
ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize,
                              int compressionLevel);
-
  When compressing multiple messages / blocks using the same dictionary, it's recommended to load it only once.
-  ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup cost.
+
  When compressing multiple messages or blocks using the same dictionary,
+  it's recommended to digest the dictionary only once, since it's a costly operation.
+  ZSTD_createCDict() will create a state from digesting a dictionary.
+  The resulting state can be used for future compression operations with very limited startup cost.
   ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
- `dictBuffer` can be released after ZSTD_CDict creation, because its content is copied within CDict.
-  Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate `dictBuffer` content.
-  Note : A ZSTD_CDict can be created from an empty dictBuffer, but it is inefficient when used to compress small data. 
+ @dictBuffer can be released after ZSTD_CDict creation, because its content is copied within CDict.
+  Note 1 : Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate @dictBuffer content.
+  Note 2 : A ZSTD_CDict can be created from an empty @dictBuffer,
+      in which case the only thing that it transports is the @compressionLevel.
+      This can be useful in a pipeline featuring ZSTD_compress_usingCDict() exclusively,
+      expecting a ZSTD_CDict parameter with any data, including those without a known dictionary. 
 

size_t      ZSTD_freeCDict(ZSTD_CDict* CDict);
@@ -947,7 +952,7 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
      * to evolve and should be considered only in the context of extremely
      * advanced performance tuning.
      *
-     * Zstd currently supports the use of a CDict in two ways:
+     * Zstd currently supports the use of a CDict in three ways:
      *
      * - The contents of the CDict can be copied into the working context. This
      *   means that the compression can search both the dictionary and input
@@ -963,6 +968,12 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
      *   working context's tables can be reused). For small inputs, this can be
      *   faster than copying the CDict's tables.
      *
+     * - The CDict's tables are not used at all, and instead we use the working
+     *   context alone to reload the dictionary and use params based on the source
+     *   size. See ZSTD_compress_insertDictionary() and ZSTD_compress_usingDict().
+     *   This method is effective when the dictionary sizes are very small relative
+     *   to the input size, and the input size is fairly large to begin with.
+     *
      * Zstd has a simple internal heuristic that selects which strategy to use
      * at the beginning of a compression. However, if experimentation shows that
      * Zstd is making poor choices, it is possible to override that choice with
@@ -970,7 +981,8 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
      */
     ZSTD_dictDefaultAttach = 0, /* Use the default heuristic. */
     ZSTD_dictForceAttach   = 1, /* Never copy the dictionary. */
-    ZSTD_dictForceCopy     = 2  /* Always copy the dictionary. */
+    ZSTD_dictForceCopy     = 2, /* Always copy the dictionary. */
+    ZSTD_dictForceLoad     = 3  /* Always reload the dictionary */
 } ZSTD_dictAttachPref_e;
 

typedef enum {
diff --git a/lib/zstd.h b/lib/zstd.h
index 24d23ff8..72080ea8 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -808,12 +808,17 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
 typedef struct ZSTD_CDict_s ZSTD_CDict;
 
 /*! ZSTD_createCDict() :
- *  When compressing multiple messages / blocks using the same dictionary, it's recommended to load it only once.
- *  ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup cost.
+ *  When compressing multiple messages or blocks using the same dictionary,
+ *  it's recommended to digest the dictionary only once, since it's a costly operation.
+ *  ZSTD_createCDict() will create a state from digesting a dictionary.
+ *  The resulting state can be used for future compression operations with very limited startup cost.
  *  ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
- * `dictBuffer` can be released after ZSTD_CDict creation, because its content is copied within CDict.
- *  Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate `dictBuffer` content.
- *  Note : A ZSTD_CDict can be created from an empty dictBuffer, but it is inefficient when used to compress small data. */
+ * @dictBuffer can be released after ZSTD_CDict creation, because its content is copied within CDict.
+ *  Note 1 : Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate @dictBuffer content.
+ *  Note 2 : A ZSTD_CDict can be created from an empty @dictBuffer,
+ *      in which case the only thing that it transports is the @compressionLevel.
+ *      This can be useful in a pipeline featuring ZSTD_compress_usingCDict() exclusively,
+ *      expecting a ZSTD_CDict parameter with any data, including those without a known dictionary. */
 ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize,
                                          int compressionLevel);
 
@@ -1167,7 +1172,7 @@ typedef enum {
      *   tables. However, this model incurs no start-up cost (as long as the
      *   working context's tables can be reused). For small inputs, this can be
      *   faster than copying the CDict's tables.
-     * 
+     *
      * - The CDict's tables are not used at all, and instead we use the working
      *   context alone to reload the dictionary and use params based on the source
      *   size. See ZSTD_compress_insertDictionary() and ZSTD_compress_usingDict().

From faddd2aa1dd85bdb51e1eaa55f77b76cbde2d9e5 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 28 Oct 2019 12:57:05 -0700
Subject: [PATCH 15/15] updated CHANGELOG for v1.4.4

---
 CHANGELOG | 28 ++++++++++++++++++++++++++--
 1 file changed, 26 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG b/CHANGELOG
index ae54896a..44600267 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,8 +1,32 @@
+v1.4.4
+perf: Improved decompression speed, by > 10%, by @terrelln
+perf: Better compression speed when re-using a context, by @felixhandte
+perf: Fix compression ratio when compressing large files with small dictionary, by @senhuang42
+perf: zstd reference encoder can generate RLE blocks, by @bimbashrestha
+perf: minor generic speed optimization, by @davidbolvansky
+api: new ability to extract sequences from the parser for analysis, by @bimbashrestha
+api: fixed decoding of magic-less frames, by @terrelln
+api: fixed ZSTD_initCStream_advanced() performance with fast modes, reported by @QrczakMK
+cli: Named pipes support, by @bimbashrestha
+cli: short tar's extension support, by @stokito
+cli: command --output-dir-flat= , generates target files into requested directory, by @senhuang42
+cli: commands --stream-size=# and --size-hint=#, by @nmagerko
+cli: faster `-t` test mode
+cli: improved some error messages, by @vangyzen
+cli: rare deadlock condition within dictionary builder, by @terrelln
+build: single-file decoder with emscripten compilation script, by @cwoffenden
+build: fixed zlibWrapper compilation on Visual Studio, reported by @bluenlive
+build: fixed deprecation warning for certain gcc version, reported by @jasonma163
+build: fix compilation on old gcc versions, by @cemeyer
+build: improved installation directories for cmake script, by Dmitri Shubin
+pack: modified pkgconfig, for better integration into openwrt, requested by @neheb
+misc: Improved documentation : ZSTD_CLEVEL, DYNAMIC_BMI2, ZSTD_CDict, function deprecation, zstd format
+misc: fixed educational decoder : accept larger literals section, and removed UNALIGNED() macro
+
 v1.4.3
 bug: Fix Dictionary Compression Ratio Regression by @cyan4973 (#1709)
-bug: Fix Buffer Overflow in v0.3 Decompression by @felixhandte (#1722)
+bug: Fix Buffer Overflow in legacy v0.3 decompression by @felixhandte (#1722)
 build: Add support for IAR C/C++ Compiler for Arm by @joseph0918 (#1705)
-misc: Add NULL pointer check in util.c by @leeyoung624 (#1706)
 
 v1.4.2
 bug: Fix bug in zstd-0.5 decoder by @terrelln (#1696)