From cfd4dc299a9ed70c0e2b25b22ac5e973ccf5bdb1 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 8 Feb 2017 15:17:55 +0100 Subject: [PATCH 01/11] add "--format=gzip" option --- programs/Makefile | 2 +- programs/fileio.h | 7 +++++++ programs/zstdcli.c | 4 ++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/programs/Makefile b/programs/Makefile index b189224f..c9602963 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -72,7 +72,7 @@ ifeq ($(HAVE_ZLIB), 1) TEMP := $(shell rm have_zlib$(EXT)) endif ifeq ($(HAVE_ZLIB), 1) -ZLIBCPP = -DZSTD_GZDECOMPRESS +ZLIBCPP = -DZSTD_GZCOMPRESS -DZSTD_GZDECOMPRESS ZLIBLD = -lz endif diff --git a/programs/fileio.h b/programs/fileio.h index daff0312..2b627557 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -31,9 +31,16 @@ extern "C" { #endif +/*-************************************* +* Types +***************************************/ +typedef enum { FIO_zstdCompression, FIO_gzipCompression } FIO_compresionType_t; + + /*-************************************* * Parameters ***************************************/ +void FIO_setCompresionType(FIO_compresionType_t compresionType); void FIO_overwriteMode(void); void FIO_setNotificationLevel(unsigned level); void FIO_setSparseWrite(unsigned sparse); /**< 0: no sparse; 1: disable on stdout; 2: always enabled */ diff --git a/programs/zstdcli.c b/programs/zstdcli.c index a7dbda31..651255b0 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -123,6 +123,9 @@ static int usage_advanced(const char* programName) DISPLAY( " -T# : use # threads for compression (default:1) \n"); DISPLAY( " -B# : select size of independent sections (default:0==automatic) \n"); #endif +#ifdef ZSTD_GZCOMPRESS + DISPLAY( "--format=gzip : output .gz files \n"); +#endif #endif #ifndef ZSTD_NODECOMPRESS DISPLAY( "--test : test compressed file integrity \n"); @@ -359,6 +362,7 @@ int main(int argCount, const char* argv[]) if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(0); continue; } if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(1); continue; } if (!strcmp(argument, "--priority=rt")) { setRealTimePrio = 1; continue; } + if (!strcmp(argument, "--format=gzip")) { FIO_setCompresionType(FIO_gzipCompression); continue; } /* long commands with arguments */ #ifndef ZSTD_NODICT From 02018c83cfaff4289907e30ae217cf41e1d47df1 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 8 Feb 2017 16:54:23 +0100 Subject: [PATCH 02/11] added FIO_compressGzFrame --- programs/fileio.c | 89 ++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 80 insertions(+), 9 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 087bf950..0f6e0067 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -38,7 +38,7 @@ #ifdef ZSTD_MULTITHREAD # include "zstdmt_compress.h" #endif -#ifdef ZSTD_GZDECOMPRESS +#if defined(ZSTD_GZCOMPRESS) || defined(ZSTD_GZDECOMPRESS) # include # if !defined(z_const) # define z_const @@ -95,6 +95,8 @@ static clock_t g_time = 0; /*-************************************* * Local Parameters - Not thread safe ***************************************/ +static FIO_compresionType_t g_compresionType = FIO_zstdCompression; +void FIO_setCompresionType(FIO_compresionType_t compresionType) { g_compresionType = compresionType; } static U32 g_overwrite = 0; void FIO_overwriteMode(void) { g_overwrite=1; } static U32 g_sparseFileSupport = 1; /* 0 : no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */ @@ -335,13 +337,75 @@ static void FIO_freeCResources(cRess_t ress) } +#ifdef ZSTD_GZCOMPRESS +static unsigned long long FIO_compressGzFrame(cRess_t* ress, const char* srcFileName, U64 const srcFileSize, int compressionLevel, U64* readsize) +{ + unsigned long long inFileSize = 0, outFileSize = 0; + z_stream strm; + + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + strm.next_in = 0; + strm.avail_in = Z_NULL; + if (deflateInit2(&strm, compressionLevel, Z_DEFLATED, 15 /* maxWindowLogSize */ + 16 /* gzip only */, 8, Z_DEFAULT_STRATEGY) != Z_OK) + EXM_THROW(70, "deflateInit2 error"); /* see http://www.zlib.net/manual.html */ + + strm.next_out = (Bytef*)ress->dstBuffer; + strm.avail_out = (uInt)ress->dstBufferSize; + + while (1) { + int ret; + if (strm.avail_in == 0) { + size_t const inSize = fread(ress->srcBuffer, 1, ress->srcBufferSize, ress->srcFile); + if (inSize == 0) break; + inFileSize += inSize; + strm.next_in = (z_const unsigned char*)ress->srcBuffer; + strm.avail_in = (uInt)inSize; + } + ret = deflate(&strm, Z_NO_FLUSH); + if (ret != Z_OK) EXM_THROW(71, "zstd: %s: deflate error %d \n", srcFileName, ret); + { size_t const decompBytes = ress->dstBufferSize - strm.avail_out; + if (decompBytes) { + if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) EXM_THROW(73, "Write error : cannot write to output file"); + outFileSize += decompBytes; + strm.next_out = (Bytef*)ress->dstBuffer; + strm.avail_out = (uInt)ress->dstBufferSize; + } + } + if (!srcFileSize) DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%", (U32)(inFileSize>>20), (double)outFileSize/inFileSize*100) + else DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%", (U32)(inFileSize>>20), (U32)(srcFileSize>>20), (double)outFileSize/inFileSize*100); + } + + while (1) { + int ret = deflate(&strm, Z_FINISH); + if (ret != Z_OK && ret != Z_STREAM_END) EXM_THROW(75, "zstd: %s: deflate error %d \n", srcFileName, ret); + { size_t const decompBytes = ress->dstBufferSize - strm.avail_out; + if (decompBytes) { + if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) EXM_THROW(77, "Write error : cannot write to output file"); + outFileSize += decompBytes; + strm.next_out = (Bytef*)ress->dstBuffer; + strm.avail_out = (uInt)ress->dstBufferSize; + } + } + if (ret == Z_STREAM_END) break; + } + + deflateEnd(&strm); + *readsize = inFileSize; + + return outFileSize; +} +#endif + + /*! FIO_compressFilename_internal() : * same as FIO_compressFilename_extRess(), with `ress.desFile` already opened. * @return : 0 : compression completed correctly, * 1 : missing or pb opening srcFileName */ static int FIO_compressFilename_internal(cRess_t ress, - const char* dstFileName, const char* srcFileName) + const char* dstFileName, const char* srcFileName, int compressionLevel) { FILE* const srcFile = ress.srcFile; FILE* const dstFile = ress.dstFile; @@ -349,6 +413,12 @@ static int FIO_compressFilename_internal(cRess_t ress, U64 compressedfilesize = 0; U64 const fileSize = UTIL_getFileSize(srcFileName); + if (g_compresionType) { + compressedfilesize = FIO_compressGzFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize); + // printf("g_compresionType=%d compressionLevel=%d compressedfilesize=%d\n", g_compresionType, compressionLevel, (int)compressedfilesize); + goto finish; + } + /* init */ #ifdef ZSTD_MULTITHREAD { size_t const resetError = ZSTDMT_resetCStream(ress.cctx, fileSize); @@ -406,6 +476,7 @@ static int FIO_compressFilename_internal(cRess_t ress, } } +finish: /* Status */ DISPLAYLEVEL(2, "\r%79s\r", ""); DISPLAYLEVEL(2,"%-20s :%6.2f%% (%6llu => %6llu bytes, %s) \n", srcFileName, @@ -423,7 +494,7 @@ static int FIO_compressFilename_internal(cRess_t ress, * 1 : missing or pb opening srcFileName */ static int FIO_compressFilename_srcFile(cRess_t ress, - const char* dstFileName, const char* srcFileName) + const char* dstFileName, const char* srcFileName, int compressionLevel) { int result; @@ -436,7 +507,7 @@ static int FIO_compressFilename_srcFile(cRess_t ress, ress.srcFile = FIO_openSrcFile(srcFileName); if (!ress.srcFile) return 1; /* srcFile could not be opened */ - result = FIO_compressFilename_internal(ress, dstFileName, srcFileName); + result = FIO_compressFilename_internal(ress, dstFileName, srcFileName, compressionLevel); fclose(ress.srcFile); if (g_removeSrcFile && !result) { if (remove(srcFileName)) EXM_THROW(1, "zstd: %s: %s", srcFileName, strerror(errno)); } /* remove source file : --rm */ @@ -449,7 +520,7 @@ static int FIO_compressFilename_srcFile(cRess_t ress, * 1 : pb */ static int FIO_compressFilename_dstFile(cRess_t ress, - const char* dstFileName, const char* srcFileName) + const char* dstFileName, const char* srcFileName, int compressionLevel) { int result; stat_t statbuf; @@ -459,7 +530,7 @@ static int FIO_compressFilename_dstFile(cRess_t ress, if (ress.dstFile==NULL) return 1; /* could not open dstFileName */ if (strcmp (srcFileName, stdinmark) && UTIL_getFileStat(srcFileName, &statbuf)) stat_result = 1; - result = FIO_compressFilename_srcFile(ress, dstFileName, srcFileName); + result = FIO_compressFilename_srcFile(ress, dstFileName, srcFileName, compressionLevel); if (fclose(ress.dstFile)) { DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno)); result=1; } /* error closing dstFile */ if (result!=0) { if (remove(dstFileName)) EXM_THROW(1, "zstd: %s: %s", dstFileName, strerror(errno)); } /* remove operation artefact */ @@ -475,7 +546,7 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName, U64 const srcSize = UTIL_getFileSize(srcFileName); cRess_t const ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, comprParams); - int const result = FIO_compressFilename_dstFile(ress, dstFileName, srcFileName); + int const result = FIO_compressFilename_dstFile(ress, dstFileName, srcFileName, compressionLevel); double const seconds = (double)(clock() - start) / CLOCKS_PER_SEC; DISPLAYLEVEL(4, "Completed in %.2f sec \n", seconds); @@ -507,7 +578,7 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile ress.dstFile = stdout; SET_BINARY_MODE(stdout); for (u=0; u Date: Wed, 8 Feb 2017 17:37:14 +0100 Subject: [PATCH 03/11] .gz suffix for gzip compressed files --- programs/fileio.c | 24 +++++++++++++++--------- programs/zstdcli.c | 6 ++++-- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 0f6e0067..6b5f80ff 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -342,20 +342,21 @@ static unsigned long long FIO_compressGzFrame(cRess_t* ress, const char* srcFile { unsigned long long inFileSize = 0, outFileSize = 0; z_stream strm; + int ret; strm.zalloc = Z_NULL; strm.zfree = Z_NULL; strm.opaque = Z_NULL; + + if (deflateInit2(&strm, compressionLevel, Z_DEFLATED, 15 /* maxWindowLogSize */ + 16 /* gzip only */, 8, Z_DEFAULT_STRATEGY) != Z_OK) + EXM_THROW(71, "zstd: %s: deflateInit2 error %d \n", srcFileName, ret); /* see http://www.zlib.net/manual.html */ + strm.next_in = 0; strm.avail_in = Z_NULL; - if (deflateInit2(&strm, compressionLevel, Z_DEFLATED, 15 /* maxWindowLogSize */ + 16 /* gzip only */, 8, Z_DEFAULT_STRATEGY) != Z_OK) - EXM_THROW(70, "deflateInit2 error"); /* see http://www.zlib.net/manual.html */ - strm.next_out = (Bytef*)ress->dstBuffer; strm.avail_out = (uInt)ress->dstBufferSize; while (1) { - int ret; if (strm.avail_in == 0) { size_t const inSize = fread(ress->srcBuffer, 1, ress->srcBufferSize, ress->srcFile); if (inSize == 0) break; @@ -364,7 +365,7 @@ static unsigned long long FIO_compressGzFrame(cRess_t* ress, const char* srcFile strm.avail_in = (uInt)inSize; } ret = deflate(&strm, Z_NO_FLUSH); - if (ret != Z_OK) EXM_THROW(71, "zstd: %s: deflate error %d \n", srcFileName, ret); + if (ret != Z_OK) EXM_THROW(72, "zstd: %s: deflate error %d \n", srcFileName, ret); { size_t const decompBytes = ress->dstBufferSize - strm.avail_out; if (decompBytes) { if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) EXM_THROW(73, "Write error : cannot write to output file"); @@ -378,20 +379,21 @@ static unsigned long long FIO_compressGzFrame(cRess_t* ress, const char* srcFile } while (1) { - int ret = deflate(&strm, Z_FINISH); - if (ret != Z_OK && ret != Z_STREAM_END) EXM_THROW(75, "zstd: %s: deflate error %d \n", srcFileName, ret); + ret = deflate(&strm, Z_FINISH); { size_t const decompBytes = ress->dstBufferSize - strm.avail_out; if (decompBytes) { - if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) EXM_THROW(77, "Write error : cannot write to output file"); + if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) EXM_THROW(75, "Write error : cannot write to output file"); outFileSize += decompBytes; strm.next_out = (Bytef*)ress->dstBuffer; strm.avail_out = (uInt)ress->dstBufferSize; } } if (ret == Z_STREAM_END) break; + if (ret != Z_BUF_ERROR) EXM_THROW(77, "zstd: %s: deflate error %d \n", srcFileName, ret); } - deflateEnd(&strm); + ret = deflateEnd(&strm); + if (ret != Z_OK) EXM_THROW(79, "zstd: %s: deflateEnd error %d \n", srcFileName, ret); *readsize = inFileSize; return outFileSize; @@ -414,9 +416,13 @@ static int FIO_compressFilename_internal(cRess_t ress, U64 const fileSize = UTIL_getFileSize(srcFileName); if (g_compresionType) { +#ifdef ZSTD_GZCOMPRESS compressedfilesize = FIO_compressGzFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize); // printf("g_compresionType=%d compressionLevel=%d compressedfilesize=%d\n", g_compresionType, compressionLevel, (int)compressedfilesize); goto finish; +#else + EXM_THROW(20, "zstd: %s: file cannot be compressed as gzip (zstd compiled without ZSTD_GZCOMPRESS) -- ignored \n", srcFileName); +#endif } /* init */ diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 651255b0..08e169ed 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -49,6 +49,7 @@ #define AUTHOR "Yann Collet" #define WELCOME_MESSAGE "*** %s %i-bits %s, by %s ***\n", COMPRESSOR_NAME, (int)(sizeof(size_t)*8), ZSTD_VERSION, AUTHOR +#define GZ_EXTENSION ".gz" #define ZSTD_EXTENSION ".zst" #define ZSTD_CAT "zstdcat" #define ZSTD_UNZSTD "unzstd" @@ -286,6 +287,7 @@ int main(int argCount, const char* argv[]) const char* programName = argv[0]; const char* outFileName = NULL; const char* dictFileName = NULL; + const char* suffix = ZSTD_EXTENSION; unsigned maxDictSize = g_defaultMaxDictSize; unsigned dictID = 0; int dictCLevel = g_defaultDictCLevel; @@ -362,7 +364,7 @@ int main(int argCount, const char* argv[]) if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(0); continue; } if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(1); continue; } if (!strcmp(argument, "--priority=rt")) { setRealTimePrio = 1; continue; } - if (!strcmp(argument, "--format=gzip")) { FIO_setCompresionType(FIO_gzipCompression); continue; } + if (!strcmp(argument, "--format=gzip")) { suffix = GZ_EXTENSION; FIO_setCompresionType(FIO_gzipCompression); continue; } /* long commands with arguments */ #ifndef ZSTD_NODICT @@ -647,7 +649,7 @@ int main(int argCount, const char* argv[]) if ((filenameIdx==1) && outFileName) operationResult = FIO_compressFilename(outFileName, filenameTable[0], dictFileName, cLevel, &compressionParams); else - operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, outFileName ? outFileName : ZSTD_EXTENSION, dictFileName, cLevel, &compressionParams); + operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, outFileName ? outFileName : suffix, dictFileName, cLevel, &compressionParams); #else DISPLAY("Compression not supported\n"); #endif From 4f9eaa7bb35e57e0e294c8a397410d86e656f432 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 8 Feb 2017 18:08:09 +0100 Subject: [PATCH 04/11] fixed gcc warnings --- programs/fileio.c | 3 ++- programs/zstdcli.c | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 6b5f80ff..43012777 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -419,10 +419,11 @@ static int FIO_compressFilename_internal(cRess_t ress, #ifdef ZSTD_GZCOMPRESS compressedfilesize = FIO_compressGzFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize); // printf("g_compresionType=%d compressionLevel=%d compressedfilesize=%d\n", g_compresionType, compressionLevel, (int)compressedfilesize); - goto finish; #else + (void)compressionLevel; EXM_THROW(20, "zstd: %s: file cannot be compressed as gzip (zstd compiled without ZSTD_GZCOMPRESS) -- ignored \n", srcFileName); #endif + goto finish; } /* init */ diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 08e169ed..30aa0b94 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -125,7 +125,7 @@ static int usage_advanced(const char* programName) DISPLAY( " -B# : select size of independent sections (default:0==automatic) \n"); #endif #ifdef ZSTD_GZCOMPRESS - DISPLAY( "--format=gzip : output .gz files \n"); + DISPLAY( "--format=gzip : compress files to the .gz format \n"); #endif #endif #ifndef ZSTD_NODECOMPRESS From 64f7221958084b10f9f9c8b86639a96a73b106a2 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Mon, 13 Feb 2017 21:00:41 +0100 Subject: [PATCH 05/11] limit zlib compression level to Z_BEST_COMPRESSION --- programs/fileio.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/programs/fileio.c b/programs/fileio.c index 43012777..9f1560b4 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -344,6 +344,8 @@ static unsigned long long FIO_compressGzFrame(cRess_t* ress, const char* srcFile z_stream strm; int ret; + if (compressionLevel > Z_BEST_COMPRESSION) compressionLevel = Z_BEST_COMPRESSION; + strm.zalloc = Z_NULL; strm.zfree = Z_NULL; strm.opaque = Z_NULL; From 1a195b3b7a4cb9efaa26e7d9c5c64173e65ba8e8 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Mon, 13 Feb 2017 22:56:31 +0100 Subject: [PATCH 06/11] fixed unitialized variable warning --- programs/fileio.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index ca83db6f..b8066a7a 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -376,8 +376,8 @@ static unsigned long long FIO_compressGzFrame(cRess_t* ress, const char* srcFile strm.zfree = Z_NULL; strm.opaque = Z_NULL; - if (deflateInit2(&strm, compressionLevel, Z_DEFLATED, 15 /* maxWindowLogSize */ + 16 /* gzip only */, 8, Z_DEFAULT_STRATEGY) != Z_OK) - EXM_THROW(71, "zstd: %s: deflateInit2 error %d \n", srcFileName, ret); /* see http://www.zlib.net/manual.html */ + ret = deflateInit2(&strm, compressionLevel, Z_DEFLATED, 15 /* maxWindowLogSize */ + 16 /* gzip only */, 8, Z_DEFAULT_STRATEGY); + if (ret != Z_OK) EXM_THROW(71, "zstd: %s: deflateInit2 error %d \n", srcFileName, ret); /* see http://www.zlib.net/manual.html */ strm.next_in = 0; strm.avail_in = Z_NULL; From 08e6a88a97f8a472a8e89aa1ea49b6b8669ff595 Mon Sep 17 00:00:00 2001 From: ds77 Date: Tue, 14 Feb 2017 00:14:24 +0100 Subject: [PATCH 07/11] avoid empty translation unit warning without #pragma --- lib/common/threading.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/common/threading.c b/lib/common/threading.c index b56e594b..32d58796 100644 --- a/lib/common/threading.c +++ b/lib/common/threading.c @@ -15,10 +15,11 @@ * This file will hold wrapper for systems, which do not support pthreads */ -/* ====== Compiler specifics ====== */ -#if defined(_MSC_VER) -# pragma warning(disable : 4206) /* disable: C4206: translation unit is empty (when ZSTD_MULTITHREAD is not defined) */ -#endif +/* When ZSTD_MULTITHREAD is not defined, this file would become an empty translation unit. +* Include some ISO C header code to prevent this and portably avoid related warnings. +* (Visual C++: C4206 / GCC: -Wpedantic / Clang: -Wempty-translation-unit) +*/ +#include #if defined(ZSTD_MULTITHREAD) && defined(_WIN32) From 2bb6fc2a944d30d0ec3ec18d3db0fc462cf06ccf Mon Sep 17 00:00:00 2001 From: zefanxu2 Date: Mon, 13 Feb 2017 21:12:59 -0600 Subject: [PATCH 08/11] fix memory leak --- examples/simple_compression.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/simple_compression.c b/examples/simple_compression.c index deb0bbfc..2aab48f4 100644 --- a/examples/simple_compression.c +++ b/examples/simple_compression.c @@ -127,6 +127,6 @@ int main(int argc, const char** argv) const char* const outFilename = createOutFilename_orDie(inFilename); compress_orDie(inFilename, outFilename); - + free(outFilename); return 0; } From 98509a70acc58c71464181c3564f9bfd9cc319ed Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 14 Feb 2017 09:23:32 +0100 Subject: [PATCH 09/11] fixed function name --- programs/fileio.c | 11 +++++------ programs/fileio.h | 4 ++-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index b8066a7a..e905948b 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -121,8 +121,8 @@ static clock_t g_time = 0; /*-************************************* * Local Parameters - Not thread safe ***************************************/ -static FIO_compresionType_t g_compresionType = FIO_zstdCompression; -void FIO_setCompresionType(FIO_compresionType_t compresionType) { g_compresionType = compresionType; } +static FIO_compressionType_t g_compressionType = FIO_zstdCompression; +void FIO_setCompressionType(FIO_compressionType_t compressionType) { g_compressionType = compressionType; } static U32 g_overwrite = 0; void FIO_overwriteMode(void) { g_overwrite=1; } static U32 g_sparseFileSupport = 1; /* 0 : no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */ @@ -376,8 +376,8 @@ static unsigned long long FIO_compressGzFrame(cRess_t* ress, const char* srcFile strm.zfree = Z_NULL; strm.opaque = Z_NULL; - ret = deflateInit2(&strm, compressionLevel, Z_DEFLATED, 15 /* maxWindowLogSize */ + 16 /* gzip only */, 8, Z_DEFAULT_STRATEGY); - if (ret != Z_OK) EXM_THROW(71, "zstd: %s: deflateInit2 error %d \n", srcFileName, ret); /* see http://www.zlib.net/manual.html */ + ret = deflateInit2(&strm, compressionLevel, Z_DEFLATED, 15 /* maxWindowLogSize */ + 16 /* gzip only */, 8, Z_DEFAULT_STRATEGY); /* see http://www.zlib.net/manual.html */ + if (ret != Z_OK) EXM_THROW(71, "zstd: %s: deflateInit2 error %d \n", srcFileName, ret); strm.next_in = 0; strm.avail_in = Z_NULL; @@ -443,10 +443,9 @@ static int FIO_compressFilename_internal(cRess_t ress, U64 compressedfilesize = 0; U64 const fileSize = UTIL_getFileSize(srcFileName); - if (g_compresionType) { + if (g_compressionType) { #ifdef ZSTD_GZCOMPRESS compressedfilesize = FIO_compressGzFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize); - // printf("g_compresionType=%d compressionLevel=%d compressedfilesize=%d\n", g_compresionType, compressionLevel, (int)compressedfilesize); #else (void)compressionLevel; EXM_THROW(20, "zstd: %s: file cannot be compressed as gzip (zstd compiled without ZSTD_GZCOMPRESS) -- ignored \n", srcFileName); diff --git a/programs/fileio.h b/programs/fileio.h index 2b627557..e828b385 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -34,13 +34,13 @@ extern "C" { /*-************************************* * Types ***************************************/ -typedef enum { FIO_zstdCompression, FIO_gzipCompression } FIO_compresionType_t; +typedef enum { FIO_zstdCompression, FIO_gzipCompression } FIO_compressionType_t; /*-************************************* * Parameters ***************************************/ -void FIO_setCompresionType(FIO_compresionType_t compresionType); +void FIO_setCompressionType(FIO_compressionType_t compressionType); void FIO_overwriteMode(void); void FIO_setNotificationLevel(unsigned level); void FIO_setSparseWrite(unsigned sparse); /**< 0: no sparse; 1: disable on stdout; 2: always enabled */ From 970419535ff739200acd0957d9abb48acf000c80 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 14 Feb 2017 09:47:29 +0100 Subject: [PATCH 10/11] fixed function name (2) --- programs/zstdcli.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 30aa0b94..33c0cd8d 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -364,7 +364,7 @@ int main(int argCount, const char* argv[]) if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(0); continue; } if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(1); continue; } if (!strcmp(argument, "--priority=rt")) { setRealTimePrio = 1; continue; } - if (!strcmp(argument, "--format=gzip")) { suffix = GZ_EXTENSION; FIO_setCompresionType(FIO_gzipCompression); continue; } + if (!strcmp(argument, "--format=gzip")) { suffix = GZ_EXTENSION; FIO_setCompressionType(FIO_gzipCompression); continue; } /* long commands with arguments */ #ifndef ZSTD_NODICT From c09d16ba8c9328c8c6d0955b291470caabff0d83 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 14 Feb 2017 10:45:19 -0800 Subject: [PATCH 11/11] preset behavior for gzip, gunzip and gzcat when zstd is called through a link named gzip, gunzip or gzcat, provides the same behavior as the related program. gzip compresses using --format=gz both gzip and gunzip enable --rm by default --- NEWS | 5 +++-- programs/zstdcli.c | 8 +++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/NEWS b/NEWS index 09f1d010..a710da8b 100644 --- a/NEWS +++ b/NEWS @@ -1,8 +1,9 @@ v1.1.4 -cli : new : advanced comnmand --priority=rt, by Przemyslaw Skibinski +cli : new : can compress in *.gz format, using --format=gzip command, by Przemyslaw Skibinski +cli : new : advanced benchmark command --priority=rt cli : fix : write on sparse-enabled file systems in 32-bits mode, by @ds77 API : new : ZSTD_getFrameCompressedSize(), ZSTD_getFrameContentSize(), ZSTD_findDecompressedSize(), by Sean Purcell -API : change : ZSTD_compress*() with srcSize==0 create an empty-frame of known size, by Sean Purcell +API : change : ZSTD_compress*() with srcSize==0 create an empty-frame of known size doc : new : educational decoder, by Sean Purcell v1.1.3 diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 33c0cd8d..58811191 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -51,8 +51,11 @@ #define GZ_EXTENSION ".gz" #define ZSTD_EXTENSION ".zst" -#define ZSTD_CAT "zstdcat" #define ZSTD_UNZSTD "unzstd" +#define ZSTD_CAT "zstdcat" +#define ZSTD_GZ "gzip" +#define ZSTD_GUNZIP "gunzip" +#define ZSTD_GZCAT "gzcat" #define KB *(1 <<10) #define MB *(1 <<20) @@ -319,6 +322,9 @@ int main(int argCount, const char* argv[]) /* preset behaviors */ if (!strcmp(programName, ZSTD_UNZSTD)) operation=zom_decompress; if (!strcmp(programName, ZSTD_CAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; displayLevel=1; } + if (!strcmp(programName, ZSTD_GZ)) { suffix = GZ_EXTENSION; FIO_setCompressionType(FIO_gzipCompression); FIO_setRemoveSrcFile(1); } /* behave like gzip */ + if (!strcmp(programName, ZSTD_GUNZIP)) { operation=zom_decompress; FIO_setRemoveSrcFile(1); } /* behave like gunzip */ + if (!strcmp(programName, ZSTD_GZCAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; displayLevel=1; } /* behave like gzcat */ memset(&compressionParams, 0, sizeof(compressionParams)); /* command switches */