diff --git a/NEWS b/NEWS index 09f1d010..a710da8b 100644 --- a/NEWS +++ b/NEWS @@ -1,8 +1,9 @@ v1.1.4 -cli : new : advanced comnmand --priority=rt, by Przemyslaw Skibinski +cli : new : can compress in *.gz format, using --format=gzip command, by Przemyslaw Skibinski +cli : new : advanced benchmark command --priority=rt cli : fix : write on sparse-enabled file systems in 32-bits mode, by @ds77 API : new : ZSTD_getFrameCompressedSize(), ZSTD_getFrameContentSize(), ZSTD_findDecompressedSize(), by Sean Purcell -API : change : ZSTD_compress*() with srcSize==0 create an empty-frame of known size, by Sean Purcell +API : change : ZSTD_compress*() with srcSize==0 create an empty-frame of known size doc : new : educational decoder, by Sean Purcell v1.1.3 diff --git a/examples/simple_compression.c b/examples/simple_compression.c index deb0bbfc..2aab48f4 100644 --- a/examples/simple_compression.c +++ b/examples/simple_compression.c @@ -127,6 +127,6 @@ int main(int argc, const char** argv) const char* const outFilename = createOutFilename_orDie(inFilename); compress_orDie(inFilename, outFilename); - + free(outFilename); return 0; } diff --git a/lib/common/threading.c b/lib/common/threading.c index b56e594b..32d58796 100644 --- a/lib/common/threading.c +++ b/lib/common/threading.c @@ -15,10 +15,11 @@ * This file will hold wrapper for systems, which do not support pthreads */ -/* ====== Compiler specifics ====== */ -#if defined(_MSC_VER) -# pragma warning(disable : 4206) /* disable: C4206: translation unit is empty (when ZSTD_MULTITHREAD is not defined) */ -#endif +/* When ZSTD_MULTITHREAD is not defined, this file would become an empty translation unit. +* Include some ISO C header code to prevent this and portably avoid related warnings. +* (Visual C++: C4206 / GCC: -Wpedantic / Clang: -Wempty-translation-unit) +*/ +#include #if defined(ZSTD_MULTITHREAD) && defined(_WIN32) diff --git a/programs/Makefile b/programs/Makefile index 862ba35d..efe68443 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -70,7 +70,7 @@ VOID = /dev/null HAVE_ZLIB := $(shell printf '\#include \nint main(){}' | $(CC) -o have_zlib -x c - -lz 2> $(VOID) && echo 1 || echo 0) ifeq ($(HAVE_ZLIB), 1) TEMP := $(shell rm have_zlib$(EXT)) -ZLIBCPP = -DZSTD_GZDECOMPRESS +ZLIBCPP = -DZSTD_GZCOMPRESS -DZSTD_GZDECOMPRESS ZLIBLD = -lz endif diff --git a/programs/fileio.c b/programs/fileio.c index b384f3d1..3cbd83ae 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -38,7 +38,7 @@ #ifdef ZSTD_MULTITHREAD # include "zstdmt_compress.h" #endif -#ifdef ZSTD_GZDECOMPRESS +#if defined(ZSTD_GZCOMPRESS) || defined(ZSTD_GZDECOMPRESS) # include # if !defined(z_const) # define z_const @@ -126,6 +126,8 @@ static clock_t g_time = 0; /*-************************************* * Local Parameters - Not thread safe ***************************************/ +static FIO_compressionType_t g_compressionType = FIO_zstdCompression; +void FIO_setCompressionType(FIO_compressionType_t compressionType) { g_compressionType = compressionType; } static U32 g_overwrite = 0; void FIO_overwriteMode(void) { g_overwrite=1; } static U32 g_sparseFileSupport = 1; /* 0 : no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */ @@ -366,13 +368,79 @@ static void FIO_freeCResources(cRess_t ress) } +#ifdef ZSTD_GZCOMPRESS +static unsigned long long FIO_compressGzFrame(cRess_t* ress, const char* srcFileName, U64 const srcFileSize, int compressionLevel, U64* readsize) +{ + unsigned long long inFileSize = 0, outFileSize = 0; + z_stream strm; + int ret; + + if (compressionLevel > Z_BEST_COMPRESSION) compressionLevel = Z_BEST_COMPRESSION; + + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + + ret = deflateInit2(&strm, compressionLevel, Z_DEFLATED, 15 /* maxWindowLogSize */ + 16 /* gzip only */, 8, Z_DEFAULT_STRATEGY); /* see http://www.zlib.net/manual.html */ + if (ret != Z_OK) EXM_THROW(71, "zstd: %s: deflateInit2 error %d \n", srcFileName, ret); + + strm.next_in = 0; + strm.avail_in = Z_NULL; + strm.next_out = (Bytef*)ress->dstBuffer; + strm.avail_out = (uInt)ress->dstBufferSize; + + while (1) { + if (strm.avail_in == 0) { + size_t const inSize = fread(ress->srcBuffer, 1, ress->srcBufferSize, ress->srcFile); + if (inSize == 0) break; + inFileSize += inSize; + strm.next_in = (z_const unsigned char*)ress->srcBuffer; + strm.avail_in = (uInt)inSize; + } + ret = deflate(&strm, Z_NO_FLUSH); + if (ret != Z_OK) EXM_THROW(72, "zstd: %s: deflate error %d \n", srcFileName, ret); + { size_t const decompBytes = ress->dstBufferSize - strm.avail_out; + if (decompBytes) { + if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) EXM_THROW(73, "Write error : cannot write to output file"); + outFileSize += decompBytes; + strm.next_out = (Bytef*)ress->dstBuffer; + strm.avail_out = (uInt)ress->dstBufferSize; + } + } + if (!srcFileSize) DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%", (U32)(inFileSize>>20), (double)outFileSize/inFileSize*100) + else DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%", (U32)(inFileSize>>20), (U32)(srcFileSize>>20), (double)outFileSize/inFileSize*100); + } + + while (1) { + ret = deflate(&strm, Z_FINISH); + { size_t const decompBytes = ress->dstBufferSize - strm.avail_out; + if (decompBytes) { + if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) EXM_THROW(75, "Write error : cannot write to output file"); + outFileSize += decompBytes; + strm.next_out = (Bytef*)ress->dstBuffer; + strm.avail_out = (uInt)ress->dstBufferSize; + } + } + if (ret == Z_STREAM_END) break; + if (ret != Z_BUF_ERROR) EXM_THROW(77, "zstd: %s: deflate error %d \n", srcFileName, ret); + } + + ret = deflateEnd(&strm); + if (ret != Z_OK) EXM_THROW(79, "zstd: %s: deflateEnd error %d \n", srcFileName, ret); + *readsize = inFileSize; + + return outFileSize; +} +#endif + + /*! FIO_compressFilename_internal() : * same as FIO_compressFilename_extRess(), with `ress.desFile` already opened. * @return : 0 : compression completed correctly, * 1 : missing or pb opening srcFileName */ static int FIO_compressFilename_internal(cRess_t ress, - const char* dstFileName, const char* srcFileName) + const char* dstFileName, const char* srcFileName, int compressionLevel) { FILE* const srcFile = ress.srcFile; FILE* const dstFile = ress.dstFile; @@ -380,6 +448,16 @@ static int FIO_compressFilename_internal(cRess_t ress, U64 compressedfilesize = 0; U64 const fileSize = UTIL_getFileSize(srcFileName); + if (g_compressionType) { +#ifdef ZSTD_GZCOMPRESS + compressedfilesize = FIO_compressGzFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize); +#else + (void)compressionLevel; + EXM_THROW(20, "zstd: %s: file cannot be compressed as gzip (zstd compiled without ZSTD_GZCOMPRESS) -- ignored \n", srcFileName); +#endif + goto finish; + } + /* init */ #ifdef ZSTD_MULTITHREAD { size_t const resetError = ZSTDMT_resetCStream(ress.cctx, fileSize); @@ -437,6 +515,7 @@ static int FIO_compressFilename_internal(cRess_t ress, } } +finish: /* Status */ DISPLAYLEVEL(2, "\r%79s\r", ""); DISPLAYLEVEL(2,"%-20s :%6.2f%% (%6llu => %6llu bytes, %s) \n", srcFileName, @@ -454,7 +533,7 @@ static int FIO_compressFilename_internal(cRess_t ress, * 1 : missing or pb opening srcFileName */ static int FIO_compressFilename_srcFile(cRess_t ress, - const char* dstFileName, const char* srcFileName) + const char* dstFileName, const char* srcFileName, int compressionLevel) { int result; @@ -467,7 +546,7 @@ static int FIO_compressFilename_srcFile(cRess_t ress, ress.srcFile = FIO_openSrcFile(srcFileName); if (!ress.srcFile) return 1; /* srcFile could not be opened */ - result = FIO_compressFilename_internal(ress, dstFileName, srcFileName); + result = FIO_compressFilename_internal(ress, dstFileName, srcFileName, compressionLevel); fclose(ress.srcFile); if (g_removeSrcFile && !result) { if (remove(srcFileName)) EXM_THROW(1, "zstd: %s: %s", srcFileName, strerror(errno)); } /* remove source file : --rm */ @@ -480,7 +559,7 @@ static int FIO_compressFilename_srcFile(cRess_t ress, * 1 : pb */ static int FIO_compressFilename_dstFile(cRess_t ress, - const char* dstFileName, const char* srcFileName) + const char* dstFileName, const char* srcFileName, int compressionLevel) { int result; stat_t statbuf; @@ -490,7 +569,7 @@ static int FIO_compressFilename_dstFile(cRess_t ress, if (ress.dstFile==NULL) return 1; /* could not open dstFileName */ if (strcmp (srcFileName, stdinmark) && UTIL_getFileStat(srcFileName, &statbuf)) stat_result = 1; - result = FIO_compressFilename_srcFile(ress, dstFileName, srcFileName); + result = FIO_compressFilename_srcFile(ress, dstFileName, srcFileName, compressionLevel); if (fclose(ress.dstFile)) { DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno)); result=1; } /* error closing dstFile */ if (result!=0) { if (remove(dstFileName)) EXM_THROW(1, "zstd: %s: %s", dstFileName, strerror(errno)); } /* remove operation artefact */ @@ -507,7 +586,7 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName, int const regFile = UTIL_isRegFile(srcFileName); cRess_t const ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, regFile, comprParams); - int const result = FIO_compressFilename_dstFile(ress, dstFileName, srcFileName); + int const result = FIO_compressFilename_dstFile(ress, dstFileName, srcFileName, compressionLevel); double const seconds = (double)(clock() - start) / CLOCKS_PER_SEC; DISPLAYLEVEL(4, "Completed in %.2f sec \n", seconds); @@ -540,7 +619,7 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile ress.dstFile = stdout; SET_BINARY_MODE(stdout); for (u=0; u