diff --git a/.travis.yml b/.travis.yml index 11651964..ff6ab0f7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -67,12 +67,18 @@ matrix: - os: linux dist: trusty sudo: required - env: PLATFORM="Ubuntu 14.04" CMD="make gpptest && make clean && make gnu90test && make clean && make c99test && make clean && make gnu99test && make clean && make clangtest" + install: + - export CXX="g++-4.8" CC="gcc-4.8" + env: PLATFORM="Ubuntu 14.04" CMD="make gpptest && make clean && make gnu90test && make clean && make c99test && make clean && make gnu99test && make clean && make clangtest && make clean && make -C contrib/pzstd pzstd32 && make -C contrib/pzstd googletest32 && make -C contrib/pzstd test32 && make -C contrib/pzstd clean" addons: apt: packages: - libc6-dev-i386 - - g++-multilib + - g++-multilib + - gcc-4.8 + - gcc-4.8-multilib + - g++-4.8 + - g++-4.8-multilib - os: linux dist: trusty sudo: required diff --git a/appveyor.yml b/appveyor.yml index 8f4e4504..6345c7b3 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -52,6 +52,8 @@ build_script: ECHO *** && ECHO make -C contrib\pzstd pzstd && make -C contrib\pzstd pzstd && + make -C contrib\pzstd googletest-mingw64 && + make -C contrib\pzstd test && make -C contrib\pzstd clean ) - if [%COMPILER%]==[gcc] ( diff --git a/build/VS2005/zstd/zstd.vcproj b/build/VS2005/zstd/zstd.vcproj index ff45e393..68c3578e 100644 --- a/build/VS2005/zstd/zstd.vcproj +++ b/build/VS2005/zstd/zstd.vcproj @@ -44,7 +44,7 @@ Name="VCCLCompilerTool" Optimization="0" AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder" - PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE" + PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE" MinimalRebuild="true" BasicRuntimeChecks="3" RuntimeLibrary="3" @@ -121,7 +121,7 @@ EnableIntrinsicFunctions="true" OmitFramePointers="true" AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder" - PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE" + PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE" RuntimeLibrary="0" EnableFunctionLevelLinking="true" UsePrecompiledHeader="0" @@ -196,7 +196,7 @@ Name="VCCLCompilerTool" Optimization="0" AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder" - PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE" + PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE" MinimalRebuild="true" BasicRuntimeChecks="3" RuntimeLibrary="3" @@ -274,7 +274,7 @@ EnableIntrinsicFunctions="true" OmitFramePointers="true" AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder" - PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE" + PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE" RuntimeLibrary="0" EnableFunctionLevelLinking="true" UsePrecompiledHeader="0" diff --git a/build/VS2005/zstdlib/zstdlib.vcproj b/build/VS2005/zstdlib/zstdlib.vcproj index 2313c87a..7ea3d9b7 100644 --- a/build/VS2005/zstdlib/zstdlib.vcproj +++ b/build/VS2005/zstdlib/zstdlib.vcproj @@ -44,7 +44,7 @@ Name="VCCLCompilerTool" Optimization="0" AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder" - PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE" + PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE" MinimalRebuild="true" BasicRuntimeChecks="3" RuntimeLibrary="3" @@ -120,7 +120,7 @@ EnableIntrinsicFunctions="true" OmitFramePointers="true" AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder" - PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE" + PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE" RuntimeLibrary="0" EnableFunctionLevelLinking="true" UsePrecompiledHeader="0" @@ -194,7 +194,7 @@ Name="VCCLCompilerTool" Optimization="0" AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder" - PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE" + PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE" MinimalRebuild="true" BasicRuntimeChecks="3" RuntimeLibrary="3" @@ -271,7 +271,7 @@ EnableIntrinsicFunctions="true" OmitFramePointers="true" AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder" - PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE" + PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE" RuntimeLibrary="0" EnableFunctionLevelLinking="true" UsePrecompiledHeader="0" @@ -379,6 +379,34 @@ RelativePath="..\..\..\lib\decompress\zstd_decompress.c" > + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Level4 Disabled - WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) true false @@ -165,7 +165,7 @@ Level4 Disabled - WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) true false @@ -183,7 +183,7 @@ MaxSpeed true true - WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) false false MultiThreaded @@ -204,7 +204,7 @@ MaxSpeed true true - WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) false false MultiThreaded diff --git a/build/VS2010/zstdlib/zstdlib.vcxproj b/build/VS2010/zstdlib/zstdlib.vcxproj index 232fdf44..b97808dd 100644 --- a/build/VS2010/zstdlib/zstdlib.vcxproj +++ b/build/VS2010/zstdlib/zstdlib.vcxproj @@ -32,6 +32,13 @@ + + + + + + + @@ -40,6 +47,14 @@ + + + + + + + + @@ -126,7 +141,7 @@ Level4 Disabled - ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) true EnableFastChecks MultiThreadedDebugDLL @@ -146,7 +161,7 @@ Level4 Disabled - ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) true EnableFastChecks MultiThreadedDebugDLL @@ -166,7 +181,7 @@ MaxSpeed true true - ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) false MultiThreaded ProgramDatabase @@ -188,7 +203,7 @@ MaxSpeed true true - ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) false false MultiThreaded diff --git a/contrib/pzstd/Makefile b/contrib/pzstd/Makefile index d71cf5b3..e30be0be 100644 --- a/contrib/pzstd/Makefile +++ b/contrib/pzstd/Makefile @@ -30,7 +30,7 @@ else EXT = endif -.PHONY: default all test clean +.PHONY: default all test clean test32 googletest googletest32 default: pzstd @@ -41,7 +41,6 @@ libzstd.a: $(ZSTD_FILES) $(MAKE) -C $(ZSTDDIR) libzstd @cp $(ZSTDDIR)/libzstd.a . - Pzstd.o: Pzstd.h Pzstd.cpp ErrorHolder.h utils/*.h $(CXX) $(FLAGS) -c Pzstd.cpp -o $@ @@ -54,23 +53,66 @@ Options.o: Options.h Options.cpp main.o: main.cpp *.h utils/*.h $(CXX) $(FLAGS) -c main.cpp -o $@ -pzstd: Pzstd.o SkippableFrame.o Options.o main.o libzstd.a +pzstd: Pzstd.o SkippableFrame.o Options.o main.o libzstd.a $(CXX) $(FLAGS) $^ -o $@$(EXT) -lpthread +libzstd32.a: $(ZSTD_FILES) + $(MAKE) -C $(ZSTDDIR) libzstd MOREFLAGS="-m32" + @cp $(ZSTDDIR)/libzstd.a libzstd32.a + +Pzstd32.o: Pzstd.h Pzstd.cpp ErrorHolder.h utils/*.h + $(CXX) -m32 $(FLAGS) -c Pzstd.cpp -o $@ + +SkippableFrame32.o: SkippableFrame.h SkippableFrame.cpp utils/*.h + $(CXX) -m32 $(FLAGS) -c SkippableFrame.cpp -o $@ + +Options32.o: Options.h Options.cpp + $(CXX) -m32 $(FLAGS) -c Options.cpp -o $@ + +main32.o: main.cpp *.h utils/*.h + $(CXX) -m32 $(FLAGS) -c main.cpp -o $@ + +pzstd32: Pzstd32.o SkippableFrame32.o Options32.o main32.o libzstd32.a + $(CXX) -m32 $(FLAGS) $^ -o $@$(EXT) -lpthread + googletest: + @$(RM) -rf googletest @git clone https://github.com/google/googletest @mkdir -p googletest/build @cd googletest/build && cmake .. && make -test: libzstd.a Pzstd.o Options.o SkippableFrame.o +googletest32: + @$(RM) -rf googletest + @git clone https://github.com/google/googletest + @mkdir -p googletest/build + @cd googletest/build && cmake .. -DCMAKE_CXX_FLAGS=-m32 && make + +googletest-mingw64: + $(RM) -rf googletest + git clone https://github.com/google/googletest + mkdir -p googletest/build + cd googletest/build && cmake -G "MSYS Makefiles" .. && $(MAKE) + +test: + $(MAKE) libzstd.a + $(MAKE) pzstd MOREFLAGS="-Wall -Wextra -pedantic -Werror" $(MAKE) -C utils/test clean - $(MAKE) -C utils/test test + $(MAKE) -C utils/test test MOREFLAGS="-Wall -Wextra -pedantic -Werror" $(MAKE) -C test clean - $(MAKE) -C test test + $(MAKE) -C test test MOREFLAGS="-Wall -Wextra -pedantic -Werror" + +test32: + $(MAKE) libzstd.a MOREFLAGS="-m32" + $(MAKE) pzstd MOREFLAGS="-m32 -Wall -Wextra -pedantic -Werror" + $(MAKE) -C utils/test clean + $(MAKE) -C utils/test test MOREFLAGS="-m32 -Wall -Wextra -pedantic -Werror" + $(MAKE) -C test clean + $(MAKE) -C test test MOREFLAGS="-m32 -Wall -Wextra -pedantic -Werror" + clean: $(MAKE) -C $(ZSTDDIR) clean $(MAKE) -C utils/test clean $(MAKE) -C test clean - @$(RM) -rf libzstd.a *.o pzstd$(EXT) + @$(RM) -rf libzstd.a *.o pzstd$(EXT) pzstd32$(EXT) @echo Cleaning completed diff --git a/contrib/pzstd/Options.cpp b/contrib/pzstd/Options.cpp index 122f4fb3..5562ee18 100644 --- a/contrib/pzstd/Options.cpp +++ b/contrib/pzstd/Options.cpp @@ -7,182 +7,419 @@ * of patent rights can be found in the PATENTS file in the same directory. */ #include "Options.h" +#include "utils/ScopeGuard.h" +#include +#include #include #include #include +#include +#include + +#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || \ + defined(__CYGWIN__) +#include /* _isatty */ +#define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) +#else +#if defined(_POSIX_C_SOURCE) || defined(_XOPEN_SOURCE) || \ + defined(_POSIX_SOURCE) || \ + (defined(__APPLE__) && \ + defined( \ + __MACH__)) /* https://sourceforge.net/p/predef/wiki/OperatingSystems/ \ + */ +#include /* isatty */ +#define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) +#else +#define IS_CONSOLE(stdStream) 0 +#endif +#endif namespace pzstd { namespace { -unsigned parseUnsigned(const char* arg) { +unsigned defaultNumThreads() { +#ifdef PZSTD_NUM_THREADS + return PZSTD_NUM_THREADS; +#else + return std::thread::hardware_concurrency(); +#endif +} + +unsigned parseUnsigned(const char **arg) { unsigned result = 0; - while (*arg >= '0' && *arg <= '9') { + while (**arg >= '0' && **arg <= '9') { result *= 10; - result += *arg - '0'; - ++arg; + result += **arg - '0'; + ++(*arg); } return result; } -const std::string zstdExtension = ".zst"; -constexpr unsigned defaultCompressionLevel = 3; -constexpr unsigned maxNonUltraCompressionLevel = 19; +const char *getArgument(const char *options, const char **argv, int &i, + int argc) { + if (options[1] != 0) { + return options + 1; + } + ++i; + if (i == argc) { + std::fprintf(stderr, "Option -%c requires an argument, but none provided\n", + *options); + return nullptr; + } + return argv[i]; +} + +const std::string kZstdExtension = ".zst"; +constexpr char kStdIn[] = "-"; +constexpr char kStdOut[] = "-"; +constexpr unsigned kDefaultCompressionLevel = 3; +constexpr unsigned kMaxNonUltraCompressionLevel = 19; + +#ifdef _WIN32 +const char nullOutput[] = "nul"; +#else +const char nullOutput[] = "/dev/null"; +#endif + +void notSupported(const char *option) { + std::fprintf(stderr, "Operation not supported: %s\n", option); +} void usage() { std::fprintf(stderr, "Usage:\n"); - std::fprintf(stderr, "\tpzstd [args] FILE\n"); + std::fprintf(stderr, " pzstd [args] [FILE(s)]\n"); std::fprintf(stderr, "Parallel ZSTD options:\n"); - std::fprintf(stderr, "\t-n/--num-threads #: Number of threads to spawn\n"); - std::fprintf(stderr, "\t-p/--pzstd-headers: Write pzstd headers to enable parallel decompression\n"); + std::fprintf(stderr, " -p, --processes # : number of threads to use for (de)compression (default:%d)\n", defaultNumThreads()); std::fprintf(stderr, "ZSTD options:\n"); - std::fprintf(stderr, "\t-u/--ultra : enable levels beyond %i, up to %i (requires more memory)\n", maxNonUltraCompressionLevel, ZSTD_maxCLevel()); - std::fprintf(stderr, "\t-h/--help : display help and exit\n"); - std::fprintf(stderr, "\t-V/--version : display version number and exit\n"); - std::fprintf(stderr, "\t-d/--decompress : decompression\n"); - std::fprintf(stderr, "\t-f/--force : overwrite output\n"); - std::fprintf(stderr, "\t-o/--output file : result stored into `file`\n"); - std::fprintf(stderr, "\t-c/--stdout : write output to standard output\n"); - std::fprintf(stderr, "\t-# : # compression level (1-%d, default:%d)\n", maxNonUltraCompressionLevel, defaultCompressionLevel); + std::fprintf(stderr, " -# : # compression level (1-%d, default:%d)\n", kMaxNonUltraCompressionLevel, kDefaultCompressionLevel); + std::fprintf(stderr, " -d, --decompress : decompression\n"); + std::fprintf(stderr, " -o file : result stored into `file` (only if 1 input file)\n"); + std::fprintf(stderr, " -f, --force : overwrite output without prompting\n"); + std::fprintf(stderr, " --rm : remove source file(s) after successful (de)compression\n"); + std::fprintf(stderr, " -k, --keep : preserve source file(s) (default)\n"); + std::fprintf(stderr, " -h, --help : display help and exit\n"); + std::fprintf(stderr, " -V, --version : display version number and exit\n"); + std::fprintf(stderr, " -v, --verbose : verbose mode; specify multiple times to increase log level (default:2)\n"); + std::fprintf(stderr, " -q, --quiet : suppress warnings; specify twice to suppress errors too\n"); + std::fprintf(stderr, " -c, --stdout : force wrtie to standard output, even if it is the console\n"); +#ifdef UTIL_HAS_CREATEFILELIST + std::fprintf(stderr, " -r : operate recursively on directories\n"); +#endif + std::fprintf(stderr, " --ultra : enable levels beyond %i, up to %i (requires more memory)\n", kMaxNonUltraCompressionLevel, ZSTD_maxCLevel()); + std::fprintf(stderr, " -C, --check : integrity check (default)\n"); + std::fprintf(stderr, " --no-check : no integrity check\n"); + std::fprintf(stderr, " -t, --test : test compressed file integrity\n"); + std::fprintf(stderr, " -- : all arguments after \"--\" are treated as files\n"); } } // anonymous namespace Options::Options() - : numThreads(0), - maxWindowLog(23), - compressionLevel(defaultCompressionLevel), - decompress(false), - overwrite(false), - pzstdHeaders(false) {} + : numThreads(defaultNumThreads()), maxWindowLog(23), + compressionLevel(kDefaultCompressionLevel), decompress(false), + overwrite(false), keepSource(true), writeMode(WriteMode::Auto), + checksum(true), verbosity(2) {} -bool Options::parse(int argc, const char** argv) { +Options::Status Options::parse(int argc, const char **argv) { + bool test = false; + bool recursive = false; bool ultra = false; + bool forceStdout = false; + // Local copy of input files, which are pointers into argv. + std::vector localInputFiles; for (int i = 1; i < argc; ++i) { - const char* arg = argv[i]; - // Arguments with a short option - char option = 0; - if (!std::strcmp(arg, "--num-threads")) { - option = 'n'; - } else if (!std::strcmp(arg, "--pzstd-headers")) { - option = 'p'; - } else if (!std::strcmp(arg, "--ultra")) { - option = 'u'; - } else if (!std::strcmp(arg, "--version")) { - option = 'V'; - } else if (!std::strcmp(arg, "--help")) { - option = 'h'; - } else if (!std::strcmp(arg, "--decompress")) { - option = 'd'; - } else if (!std::strcmp(arg, "--force")) { - option = 'f'; - } else if (!std::strcmp(arg, "--output")) { - option = 'o'; - } else if (!std::strcmp(arg, "--stdout")) { - option = 'c'; - }else if (arg[0] == '-' && arg[1] != 0) { - // Parse the compression level or short option - if (arg[1] >= '0' && arg[1] <= '9') { - compressionLevel = parseUnsigned(arg + 1); - continue; - } - option = arg[1]; - } else if (inputFile.empty()) { - inputFile = arg; + const char *arg = argv[i]; + // Protect against empty arguments + if (arg[0] == 0) { continue; - } else { - std::fprintf(stderr, "Invalid argument: %s.\n", arg); - return false; } - - switch (option) { - case 'n': - if (++i == argc) { - std::fprintf(stderr, "Invalid argument: -n requires an argument.\n"); - return false; - } - numThreads = parseUnsigned(argv[i]); - if (numThreads == 0) { - std::fprintf(stderr, "Invalid argument: # of threads must be > 0.\n"); - return false; - } - break; - case 'p': - pzstdHeaders = true; - break; - case 'u': + // Everything after "--" is an input file + if (!std::strcmp(arg, "--")) { + ++i; + std::copy(argv + i, argv + argc, std::back_inserter(localInputFiles)); + break; + } + // Long arguments that don't have a short option + { + bool isLongOption = true; + if (!std::strcmp(arg, "--rm")) { + keepSource = false; + } else if (!std::strcmp(arg, "--ultra")) { ultra = true; maxWindowLog = 0; - break; - case 'V': - std::fprintf(stderr, "ZSTD version: %s.\n", ZSTD_VERSION_STRING); - return false; + } else if (!std::strcmp(arg, "--no-check")) { + checksum = false; + } else if (!std::strcmp(arg, "--sparse")) { + writeMode = WriteMode::Sparse; + notSupported("Sparse mode"); + return Status::Failure; + } else if (!std::strcmp(arg, "--no-sparse")) { + writeMode = WriteMode::Regular; + notSupported("Sparse mode"); + return Status::Failure; + } else if (!std::strcmp(arg, "--dictID")) { + notSupported(arg); + return Status::Failure; + } else if (!std::strcmp(arg, "--no-dictID")) { + notSupported(arg); + return Status::Failure; + } else { + isLongOption = false; + } + if (isLongOption) { + continue; + } + } + // Arguments with a short option simply set their short option. + const char *options = nullptr; + if (!std::strcmp(arg, "--processes")) { + options = "p"; + } else if (!std::strcmp(arg, "--version")) { + options = "V"; + } else if (!std::strcmp(arg, "--help")) { + options = "h"; + } else if (!std::strcmp(arg, "--decompress")) { + options = "d"; + } else if (!std::strcmp(arg, "--force")) { + options = "f"; + } else if (!std::strcmp(arg, "--stdout")) { + options = "c"; + } else if (!std::strcmp(arg, "--keep")) { + options = "k"; + } else if (!std::strcmp(arg, "--verbose")) { + options = "v"; + } else if (!std::strcmp(arg, "--quiet")) { + options = "q"; + } else if (!std::strcmp(arg, "--check")) { + options = "C"; + } else if (!std::strcmp(arg, "--test")) { + options = "t"; + } else if (arg[0] == '-' && arg[1] != 0) { + options = arg + 1; + } else { + localInputFiles.emplace_back(arg); + continue; + } + assert(options != nullptr); + + bool finished = false; + while (!finished && *options != 0) { + // Parse the compression level + if (*options >= '0' && *options <= '9') { + compressionLevel = parseUnsigned(&options); + continue; + } + + switch (*options) { case 'h': + case 'H': usage(); - return false; + return Status::Message; + case 'V': + std::fprintf(stderr, "PZSTD version: %s.\n", ZSTD_VERSION_STRING); + return Status::Message; + case 'p': { + finished = true; + const char *optionArgument = getArgument(options, argv, i, argc); + if (optionArgument == nullptr) { + return Status::Failure; + } + if (*optionArgument < '0' || *optionArgument > '9') { + std::fprintf(stderr, "Option -p expects a number, but %s provided\n", + optionArgument); + return Status::Failure; + } + numThreads = parseUnsigned(&optionArgument); + if (*optionArgument != 0) { + std::fprintf(stderr, + "Option -p expects a number, but %u%s provided\n", + numThreads, optionArgument); + return Status::Failure; + } + break; + } + case 'o': { + finished = true; + const char *optionArgument = getArgument(options, argv, i, argc); + if (optionArgument == nullptr) { + return Status::Failure; + } + outputFile = optionArgument; + break; + } + case 'C': + checksum = true; + break; + case 'k': + keepSource = true; + break; case 'd': decompress = true; break; case 'f': overwrite = true; + forceStdout = true; break; - case 'o': - if (++i == argc) { - std::fprintf(stderr, "Invalid argument: -o requires an argument.\n"); - return false; - } - outputFile = argv[i]; + case 't': + test = true; + decompress = true; break; +#ifdef UTIL_HAS_CREATEFILELIST + case 'r': + recursive = true; + break; +#endif case 'c': - outputFile = '-'; + outputFile = kStdOut; + forceStdout = true; break; + case 'v': + ++verbosity; + break; + case 'q': + --verbosity; + // Ignore them for now + break; + // Unsupported options from Zstd + case 'D': + case 's': + notSupported("Zstd dictionaries."); + return Status::Failure; + case 'b': + case 'e': + case 'i': + case 'B': + notSupported("Zstd benchmarking options."); + return Status::Failure; default: - std::fprintf(stderr, "Invalid argument: %s.\n", arg); - return false; - } - } - // Determine input file if not specified - if (inputFile.empty()) { - inputFile = "-"; - } - // Determine output file if not specified - if (outputFile.empty()) { - if (inputFile == "-") { - outputFile = "-"; - } else { - // Attempt to add/remove zstd extension from the input file - if (decompress) { - int stemSize = inputFile.size() - zstdExtension.size(); - if (stemSize > 0 && inputFile.substr(stemSize) == zstdExtension) { - outputFile = inputFile.substr(0, stemSize); - } else { - std::fprintf( - stderr, "Invalid argument: Unable to determine output file.\n"); - return false; - } - } else { - outputFile = inputFile + zstdExtension; + std::fprintf(stderr, "Invalid argument: %s\n", arg); + return Status::Failure; } + if (!finished) { + ++options; + } + } // while (*options != 0); + } // for (int i = 1; i < argc; ++i); + + // Input file defaults to standard input if not provided. + if (localInputFiles.empty()) { + localInputFiles.emplace_back(kStdIn); + } + + // Check validity of input files + if (localInputFiles.size() > 1) { + const auto it = std::find(localInputFiles.begin(), localInputFiles.end(), + std::string{kStdIn}); + if (it != localInputFiles.end()) { + std::fprintf( + stderr, + "Cannot specify standard input when handling multiple files\n"); + return Status::Failure; } } + if (localInputFiles.size() > 1 || recursive) { + if (!outputFile.empty() && outputFile != nullOutput) { + std::fprintf( + stderr, + "Cannot specify an output file when handling multiple inputs\n"); + return Status::Failure; + } + } + + // Translate input files/directories into files to (de)compress + if (recursive) { + char *scratchBuffer = nullptr; + unsigned numFiles = 0; + const char **files = + UTIL_createFileList(localInputFiles.data(), localInputFiles.size(), + &scratchBuffer, &numFiles); + if (files == nullptr) { + std::fprintf(stderr, "Error traversing directories\n"); + return Status::Failure; + } + auto guard = + makeScopeGuard([&] { UTIL_freeFileList(files, scratchBuffer); }); + if (numFiles == 0) { + std::fprintf(stderr, "No files found\n"); + return Status::Failure; + } + inputFiles.resize(numFiles); + std::copy(files, files + numFiles, inputFiles.begin()); + } else { + inputFiles.resize(localInputFiles.size()); + std::copy(localInputFiles.begin(), localInputFiles.end(), + inputFiles.begin()); + } + localInputFiles.clear(); + assert(!inputFiles.empty()); + + // If reading from standard input, default to standard output + if (inputFiles[0] == kStdIn && outputFile.empty()) { + assert(inputFiles.size() == 1); + outputFile = "-"; + } + + if (inputFiles[0] == kStdIn && IS_CONSOLE(stdin)) { + assert(inputFiles.size() == 1); + std::fprintf(stderr, "Cannot read input from interactive console\n"); + return Status::Failure; + } + if (outputFile == "-" && IS_CONSOLE(stdout) && !(forceStdout && decompress)) { + std::fprintf(stderr, "Will not write to console stdout unless -c or -f is " + "specified and decompressing\n"); + return Status::Failure; + } + // Check compression level { - unsigned maxCLevel = ultra ? ZSTD_maxCLevel() : maxNonUltraCompressionLevel; - if (compressionLevel > maxCLevel) { - std::fprintf( - stderr, "Invalid compression level %u.\n", compressionLevel); - return false; + unsigned maxCLevel = + ultra ? ZSTD_maxCLevel() : kMaxNonUltraCompressionLevel; + if (compressionLevel > maxCLevel || compressionLevel == 0) { + std::fprintf(stderr, "Invalid compression level %u.\n", compressionLevel); + return Status::Failure; } } + // Check that numThreads is set if (numThreads == 0) { - numThreads = std::thread::hardware_concurrency(); - if (numThreads == 0) { - std::fprintf(stderr, "Invalid arguments: # of threads not specified " - "and unable to determine hardware concurrency.\n"); - return false; - } + std::fprintf(stderr, "Invalid arguments: # of threads not specified " + "and unable to determine hardware concurrency.\n"); + return Status::Failure; + } + + // Modify verbosity + // If we are piping input and output, turn off interaction + if (inputFiles[0] == kStdIn && outputFile == kStdOut && verbosity == 2) { + verbosity = 1; + } + // If we are in multi-file mode, turn off interaction + if (inputFiles.size() > 1 && verbosity == 2) { + verbosity = 1; + } + + // Set options for test mode + if (test) { + outputFile = nullOutput; + keepSource = true; + } + return Status::Success; +} + +std::string Options::getOutputFile(const std::string &inputFile) const { + if (!outputFile.empty()) { + return outputFile; + } + // Attempt to add/remove zstd extension from the input file + if (decompress) { + int stemSize = inputFile.size() - kZstdExtension.size(); + if (stemSize > 0 && inputFile.substr(stemSize) == kZstdExtension) { + return inputFile.substr(0, stemSize); + } else { + return ""; + } + } else { + return inputFile + kZstdExtension; } - return true; } } diff --git a/contrib/pzstd/Options.h b/contrib/pzstd/Options.h index 47c5f78a..97c3885e 100644 --- a/contrib/pzstd/Options.h +++ b/contrib/pzstd/Options.h @@ -14,47 +14,55 @@ #include #include +#include namespace pzstd { struct Options { + enum class WriteMode { Regular, Auto, Sparse }; + unsigned numThreads; unsigned maxWindowLog; unsigned compressionLevel; bool decompress; - std::string inputFile; + std::vector inputFiles; std::string outputFile; bool overwrite; - bool pzstdHeaders; + bool keepSource; + WriteMode writeMode; + bool checksum; + int verbosity; + + enum class Status { + Success, // Successfully parsed options + Failure, // Failure to parse options + Message // Options specified to print a message (e.g. "-h") + }; Options(); - Options( - unsigned numThreads, - unsigned maxWindowLog, - unsigned compressionLevel, - bool decompress, - const std::string& inputFile, - const std::string& outputFile, - bool overwrite, - bool pzstdHeaders) - : numThreads(numThreads), - maxWindowLog(maxWindowLog), - compressionLevel(compressionLevel), - decompress(decompress), - inputFile(inputFile), - outputFile(outputFile), - overwrite(overwrite), - pzstdHeaders(pzstdHeaders) {} + Options(unsigned numThreads, unsigned maxWindowLog, unsigned compressionLevel, + bool decompress, std::vector inputFiles, + std::string outputFile, bool overwrite, bool keepSource, + WriteMode writeMode, bool checksum, int verbosity) + : numThreads(numThreads), maxWindowLog(maxWindowLog), + compressionLevel(compressionLevel), decompress(decompress), + inputFiles(std::move(inputFiles)), outputFile(std::move(outputFile)), + overwrite(overwrite), keepSource(keepSource), writeMode(writeMode), + checksum(checksum), verbosity(verbosity) {} - bool parse(int argc, const char** argv); + Status parse(int argc, const char **argv); ZSTD_parameters determineParameters() const { ZSTD_parameters params = ZSTD_getParams(compressionLevel, 0, 0); + params.fParams.contentSizeFlag = 1; + params.fParams.checksumFlag = checksum; if (maxWindowLog != 0 && params.cParams.windowLog > maxWindowLog) { params.cParams.windowLog = maxWindowLog; params.cParams = ZSTD_adjustCParams(params.cParams, 0, 0); } return params; } + + std::string getOutputFile(const std::string &inputFile) const; }; } diff --git a/contrib/pzstd/Pzstd.cpp b/contrib/pzstd/Pzstd.cpp index 87c4c202..ccd4f626 100644 --- a/contrib/pzstd/Pzstd.cpp +++ b/contrib/pzstd/Pzstd.cpp @@ -19,6 +19,15 @@ #include #include +#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) +# include /* _O_BINARY */ +# include /* _setmode, _isatty */ +# define SET_BINARY_MODE(file) { if (_setmode(_fileno(file), _O_BINARY) == -1) perror("Cannot set _O_BINARY"); } +#else +# include /* isatty */ +# define SET_BINARY_MODE(file) +#endif + namespace pzstd { namespace { @@ -31,43 +40,27 @@ const std::string nullOutput = "/dev/null"; using std::size_t; -size_t pzstdMain(const Options& options, ErrorHolder& errorHolder) { - // Open the input file and attempt to determine its size - FILE* inputFd = stdin; - std::uintmax_t inputSize = 0; - if (options.inputFile != "-") { - inputFd = std::fopen(options.inputFile.c_str(), "rb"); - if (!errorHolder.check(inputFd != nullptr, "Failed to open input file")) { - return 0; - } - std::error_code ec; - inputSize = file_size(options.inputFile, ec); - if (ec) { - inputSize = 0; - } +static std::uintmax_t fileSizeOrZero(const std::string &file) { + if (file == "-") { + return 0; } - auto closeInputGuard = makeScopeGuard([&] { std::fclose(inputFd); }); - - // Check if the output file exists and then open it - FILE* outputFd = stdout; - if (options.outputFile != "-") { - if (!options.overwrite && options.outputFile != nullOutput) { - outputFd = std::fopen(options.outputFile.c_str(), "rb"); - if (!errorHolder.check(outputFd == nullptr, "Output file exists")) { - return 0; - } - } - outputFd = std::fopen(options.outputFile.c_str(), "wb"); - if (!errorHolder.check( - outputFd != nullptr, "Failed to open output file")) { - return 0; - } + std::error_code ec; + auto size = file_size(file, ec); + if (ec) { + size = 0; } - auto closeOutputGuard = makeScopeGuard([&] { std::fclose(outputFd); }); + return size; +} +static size_t handleOneInput(const Options &options, + const std::string &inputFile, + FILE* inputFd, + FILE* outputFd, + ErrorHolder &errorHolder) { + auto inputSize = fileSizeOrZero(inputFile); // WorkQueue outlives ThreadPool so in the case of error we are certain // we don't accidently try to call push() on it after it is destroyed. - WorkQueue> outs{2 * options.numThreads}; + WorkQueue> outs{options.numThreads + 1}; size_t bytesWritten; { // Initialize the thread pool with numThreads + 1 @@ -89,21 +82,136 @@ size_t pzstdMain(const Options& options, ErrorHolder& errorHolder) { options.determineParameters()); }); // Start writing - bytesWritten = - writeFile(errorHolder, outs, outputFd, options.pzstdHeaders); + bytesWritten = writeFile(errorHolder, outs, outputFd, options.decompress); } else { // Add a job that reads the input and starts all the decompression jobs executor.add([&errorHolder, &outs, &executor, inputFd] { asyncDecompressFrames(errorHolder, outs, executor, inputFd); }); // Start writing - bytesWritten = writeFile( - errorHolder, outs, outputFd, /* writeSkippableFrames */ false); + bytesWritten = writeFile(errorHolder, outs, outputFd, options.decompress); } } return bytesWritten; } +static FILE *openInputFile(const std::string &inputFile, + ErrorHolder &errorHolder) { + if (inputFile == "-") { + SET_BINARY_MODE(stdin); + return stdin; + } + // Check if input file is a directory + { + std::error_code ec; + if (is_directory(inputFile, ec)) { + errorHolder.setError("Output file is a directory -- ignored"); + return nullptr; + } + } + auto inputFd = std::fopen(inputFile.c_str(), "rb"); + if (!errorHolder.check(inputFd != nullptr, "Failed to open input file")) { + return nullptr; + } + return inputFd; +} + +static FILE *openOutputFile(const Options &options, + const std::string &outputFile, + ErrorHolder &errorHolder) { + if (outputFile == "-") { + SET_BINARY_MODE(stdout); + return stdout; + } + // Check if the output file exists and then open it + if (!options.overwrite && outputFile != nullOutput) { + auto outputFd = std::fopen(outputFile.c_str(), "rb"); + if (outputFd != nullptr) { + std::fclose(outputFd); + if (options.verbosity <= 1) { + errorHolder.setError("Output file exists"); + return nullptr; + } + std::fprintf( + stderr, + "pzstd: %s already exists; do you wish to overwrite (y/n) ? ", + outputFile.c_str()); + int c = getchar(); + if (c != 'y' && c != 'Y') { + errorHolder.setError("Not overwritten"); + return nullptr; + } + } + } + auto outputFd = std::fopen(outputFile.c_str(), "wb"); + if (!errorHolder.check( + outputFd != nullptr, "Failed to open output file")) { + return 0; + } + return outputFd; +} + +int pzstdMain(const Options &options) { + int returnCode = 0; + for (const auto& input : options.inputFiles) { + // Setup the error holder + ErrorHolder errorHolder; + auto printErrorGuard = makeScopeGuard([&] { + if (errorHolder.hasError()) { + returnCode = 1; + if (options.verbosity > 0) { + std::fprintf(stderr, "pzstd: %s: %s.\n", input.c_str(), + errorHolder.getError().c_str()); + } + } else { + + } + }); + // Open the input file + auto inputFd = openInputFile(input, errorHolder); + if (inputFd == nullptr) { + continue; + } + auto closeInputGuard = makeScopeGuard([&] { std::fclose(inputFd); }); + // Open the output file + auto outputFile = options.getOutputFile(input); + if (!errorHolder.check(outputFile != "", + "Input file does not have extension .zst")) { + continue; + } + auto outputFd = openOutputFile(options, outputFile, errorHolder); + if (outputFd == nullptr) { + continue; + } + auto closeOutputGuard = makeScopeGuard([&] { std::fclose(outputFd); }); + // (de)compress the file + handleOneInput(options, input, inputFd, outputFd, errorHolder); + if (errorHolder.hasError()) { + continue; + } + // Delete the input file if necessary + if (!options.keepSource) { + // Be sure that we are done and have written everything before we delete + if (!errorHolder.check(std::fclose(inputFd) == 0, + "Failed to close input file")) { + continue; + } + closeInputGuard.dismiss(); + if (!errorHolder.check(std::fclose(outputFd) == 0, + "Failed to close output file")) { + continue; + } + closeOutputGuard.dismiss(); + if (std::remove(input.c_str()) != 0) { + errorHolder.setError("Failed to remove input file"); + continue; + } + } + } + // Returns 1 if any of the files failed to (de)compress. + return returnCode; +} + /// Construct a `ZSTD_inBuffer` that points to the data in `buffer`. static ZSTD_inBuffer makeZstdInBuffer(const Buffer& buffer) { return ZSTD_inBuffer{buffer.data(), buffer.size(), 0}; @@ -224,10 +332,9 @@ static size_t calculateStep( size_t step = size_t{1} << (params.cParams.windowLog + 2); // If file size is known, see if a smaller step will spread work more evenly if (size != 0) { - const std::uintmax_t newStep = size / std::uintmax_t{numThreads}; - if (newStep != 0 && - newStep <= std::uintmax_t{std::numeric_limits::max()}) { - step = std::min(step, size_t{newStep}); + const std::uintmax_t newStep = size / numThreads; + if (newStep != 0 && newStep <= std::numeric_limits::max()) { + step = std::min(step, static_cast(newStep)); } } return step; @@ -451,12 +558,12 @@ size_t writeFile( ErrorHolder& errorHolder, WorkQueue>& outs, FILE* outputFd, - bool writeSkippableFrames) { + bool decompress) { size_t bytesWritten = 0; std::shared_ptr out; // Grab the output queue for each decompression job (in order). while (outs.pop(out) && !errorHolder.hasError()) { - if (writeSkippableFrames) { + if (!decompress) { // If we are compressing and want to write skippable frames we can't // start writing before compression is done because we need to know the // compressed size. diff --git a/contrib/pzstd/Pzstd.h b/contrib/pzstd/Pzstd.h index 51d15846..0c21d135 100644 --- a/contrib/pzstd/Pzstd.h +++ b/contrib/pzstd/Pzstd.h @@ -28,11 +28,9 @@ namespace pzstd { * An error occurred if `errorHandler.hasError()`. * * @param options The pzstd options to use for (de)compression - * @param errorHolder Used to report errors and coordinate early shutdown - * if an error occured - * @returns The number of bytes written. + * @returns 0 upon success and non-zero on failure. */ -std::size_t pzstdMain(const Options& options, ErrorHolder& errorHolder); +int pzstdMain(const Options& options); /** * Streams input from `fd`, breaks input up into chunks, and compresses each @@ -79,16 +77,16 @@ void asyncDecompressFrames( * Streams input in from each queue in `outs` in order, and writes the data to * `outputFd`. * - * @param errorHolder Used to report errors and coordinate early exit - * @param outs A queue of output queues, one for each - * (de)compression job. - * @param outputFd The file descriptor to write to - * @param writeSkippableFrames Should we write pzstd headers? - * @returns The number of bytes written + * @param errorHolder Used to report errors and coordinate early exit + * @param outs A queue of output queues, one for each + * (de)compression job. + * @param outputFd The file descriptor to write to + * @param decompress Are we decompressing? + * @returns The number of bytes written */ std::size_t writeFile( ErrorHolder& errorHolder, WorkQueue>& outs, FILE* outputFd, - bool writeSkippableFrames); + bool decompress); } diff --git a/contrib/pzstd/main.cpp b/contrib/pzstd/main.cpp index 7ff2cef7..279cbfb5 100644 --- a/contrib/pzstd/main.cpp +++ b/contrib/pzstd/main.cpp @@ -19,16 +19,14 @@ using namespace pzstd; int main(int argc, const char** argv) { Options options; - if (!options.parse(argc, argv)) { + switch (options.parse(argc, argv)) { + case Options::Status::Failure: return 1; + case Options::Status::Message: + return 0; + default: + break; } - ErrorHolder errorHolder; - pzstdMain(options, errorHolder); - - if (errorHolder.hasError()) { - std::fprintf(stderr, "Error: %s.\n", errorHolder.getError().c_str()); - return 1; - } - return 0; + return pzstdMain(options); } diff --git a/contrib/pzstd/test/Makefile b/contrib/pzstd/test/Makefile index 5fd167d1..4f6ba999 100644 --- a/contrib/pzstd/test/Makefile +++ b/contrib/pzstd/test/Makefile @@ -21,19 +21,19 @@ ZSTDDIR = ../../../lib # Set GTEST_INC and GTEST_LIB to work with your install of gtest GTEST_INC ?= -isystem $(PZSTDDIR)/googletest/googletest/include GTEST_LIB ?= -L $(PZSTDDIR)/googletest/build/googlemock/gtest - -CPPFLAGS = -I$(PZSTDDIR) $(GTEST_INC) $(GTEST_LIB) -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(PROGDIR) -I. +GTEST_FLAGS = $(GTEST_INC) $(GTEST_LIB) +CPPFLAGS = -I$(PZSTDDIR) -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(PROGDIR) -I. CXXFLAGS ?= -O3 -CXXFLAGS += -std=c++11 +CXXFLAGS += -std=c++11 -Wno-deprecated-declarations CXXFLAGS += $(MOREFLAGS) FLAGS = $(CPPFLAGS) $(CXXFLAGS) $(LDFLAGS) datagen.o: $(PROGDIR)/datagen.* - $(CXX) $(FLAGS) $(PROGDIR)/datagen.c -c -o $@ + $(CC) $(CPPFLAGS) -O3 $(MOREFLAGS) $(LDFLAGS) -Wno-long-long -Wno-variadic-macros $(PROGDIR)/datagen.c -c -o $@ %: %.cpp *.h datagen.o - $(CXX) $(FLAGS) $@.cpp datagen.o $(PZSTDDIR)/Pzstd.o $(PZSTDDIR)/SkippableFrame.o $(PZSTDDIR)/Options.o $(PZSTDDIR)/libzstd.a -o $@$(EXT) -lgtest -lgtest_main -lpthread + $(CXX) $(FLAGS) $@.cpp datagen.o $(PZSTDDIR)/Pzstd.o $(PZSTDDIR)/SkippableFrame.o $(PZSTDDIR)/Options.o $(PZSTDDIR)/libzstd.a -o $@$(EXT) $(GTEST_FLAGS) -lgtest -lgtest_main -lpthread .PHONY: test clean diff --git a/contrib/pzstd/test/OptionsTest.cpp b/contrib/pzstd/test/OptionsTest.cpp index b87358c0..e7d4b2b3 100644 --- a/contrib/pzstd/test/OptionsTest.cpp +++ b/contrib/pzstd/test/OptionsTest.cpp @@ -8,172 +8,535 @@ */ #include "Options.h" -#include #include +#include using namespace pzstd; namespace pzstd { -bool operator==(const Options& lhs, const Options& rhs) { +bool operator==(const Options &lhs, const Options &rhs) { return lhs.numThreads == rhs.numThreads && - lhs.maxWindowLog == rhs.maxWindowLog && - lhs.compressionLevel == rhs.compressionLevel && - lhs.decompress == rhs.decompress && lhs.inputFile == rhs.inputFile && - lhs.outputFile == rhs.outputFile && lhs.overwrite == rhs.overwrite && - lhs.pzstdHeaders == rhs.pzstdHeaders; + lhs.maxWindowLog == rhs.maxWindowLog && + lhs.compressionLevel == rhs.compressionLevel && + lhs.decompress == rhs.decompress && lhs.inputFiles == rhs.inputFiles && + lhs.outputFile == rhs.outputFile && lhs.overwrite == rhs.overwrite && + lhs.keepSource == rhs.keepSource && lhs.writeMode == rhs.writeMode && + lhs.checksum == rhs.checksum && lhs.verbosity == rhs.verbosity; } + +std::ostream &operator<<(std::ostream &out, const Options &opt) { + out << "{"; + { + out << "\n\t" + << "numThreads: " << opt.numThreads; + out << ",\n\t" + << "maxWindowLog: " << opt.maxWindowLog; + out << ",\n\t" + << "compressionLevel: " << opt.compressionLevel; + out << ",\n\t" + << "decompress: " << opt.decompress; + out << ",\n\t" + << "inputFiles: {"; + { + bool first = true; + for (const auto &file : opt.inputFiles) { + if (!first) { + out << ","; + } + first = false; + out << "\n\t\t" << file; + } + } + out << "\n\t}"; + out << ",\n\t" + << "outputFile: " << opt.outputFile; + out << ",\n\t" + << "overwrite: " << opt.overwrite; + out << ",\n\t" + << "keepSource: " << opt.keepSource; + out << ",\n\t" + << "writeMode: " << static_cast(opt.writeMode); + out << ",\n\t" + << "checksum: " << opt.checksum; + out << ",\n\t" + << "verbosity: " << opt.verbosity; + } + out << "\n}"; + return out; +} +} + +namespace { +#ifdef _WIN32 +const char nullOutput[] = "nul"; +#else +const char nullOutput[] = "/dev/null"; +#endif + +constexpr auto autoMode = Options::WriteMode::Auto; +} // anonymous namespace + +#define EXPECT_SUCCESS(...) EXPECT_EQ(Options::Status::Success, __VA_ARGS__) +#define EXPECT_FAILURE(...) EXPECT_EQ(Options::Status::Failure, __VA_ARGS__) +#define EXPECT_MESSAGE(...) EXPECT_EQ(Options::Status::Message, __VA_ARGS__) + +template +std::array makeArray(Args... args) { + return {{nullptr, args...}}; } TEST(Options, ValidInputs) { { Options options; - std::array args = { - {nullptr, "--num-threads", "5", "-o", "-", "-f"}}; - EXPECT_TRUE(options.parse(args.size(), args.data())); - Options expected = {5, 23, 3, false, "-", "-", true, false}; + auto args = makeArray("--processes", "5", "-o", "x", "y", "-f"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + Options expected = {5, 23, 3, false, {"y"}, "x", + true, true, autoMode, true, 2}; EXPECT_EQ(expected, options); } { Options options; - std::array args = { - {nullptr, "-n", "1", "input", "-19", "-p"}}; - EXPECT_TRUE(options.parse(args.size(), args.data())); - Options expected = {1, 23, 19, false, "input", "input.zst", false, true}; + auto args = makeArray("-p", "1", "input", "-19"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + Options expected = {1, 23, 19, false, {"input"}, "", + false, true, autoMode, true, 2}; EXPECT_EQ(expected, options); } { Options options; - std::array args = {{nullptr, - "--ultra", - "-22", - "-n", - "1", - "--output", - "x", - "-d", - "x.zst", - "-f"}}; - EXPECT_TRUE(options.parse(args.size(), args.data())); - Options expected = {1, 0, 22, true, "x.zst", "x", true, false}; + auto args = + makeArray("--ultra", "-22", "-p", "1", "-o", "x", "-d", "x.zst", "-f"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + Options expected = {1, 0, 22, true, {"x.zst"}, "x", + true, true, autoMode, true, 2}; EXPECT_EQ(expected, options); } { Options options; - std::array args = {{nullptr, - "--num-threads", - "100", - "hello.zst", - "--decompress", - "--force"}}; - EXPECT_TRUE(options.parse(args.size(), args.data())); - Options expected = {100, 23, 3, true, "hello.zst", "hello", true, false}; + auto args = makeArray("--processes", "100", "hello.zst", "--decompress", + "--force"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + Options expected = {100, 23, 3, true, {"hello.zst"}, "", true, + true, autoMode, true, 2}; EXPECT_EQ(expected, options); } { Options options; - std::array args = {{nullptr, "-", "-n", "1", "-c"}}; - EXPECT_TRUE(options.parse(args.size(), args.data())); - Options expected = {1, 23, 3, false, "-", "-", false, false}; + auto args = makeArray("x", "-dp", "1", "-c"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + Options expected = {1, 23, 3, true, {"x"}, "-", + false, true, autoMode, true, 2}; EXPECT_EQ(expected, options); } { Options options; - std::array args = {{nullptr, "-", "-n", "1", "--stdout"}}; - EXPECT_TRUE(options.parse(args.size(), args.data())); - Options expected = {1, 23, 3, false, "-", "-", false, false}; + auto args = makeArray("x", "-dp", "1", "--stdout"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + Options expected = {1, 23, 3, true, {"x"}, "-", + false, true, autoMode, true, 2}; EXPECT_EQ(expected, options); } { Options options; - std::array args = {{nullptr, - "-n", - "1", - "-", - "-5", - "-o", - "-", - "-u", - "-d", - "--pzstd-headers"}}; - EXPECT_TRUE(options.parse(args.size(), args.data())); - Options expected = {1, 0, 5, true, "-", "-", false, true}; + auto args = makeArray("-p", "1", "x", "-5", "-fo", "-", "--ultra", "-d"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + Options expected = {1, 0, 5, true, {"x"}, "-", + true, true, autoMode, true, 2}; + EXPECT_EQ(expected, options); } { Options options; - std::array args = { - {nullptr, "silesia.tar", "-o", "silesia.tar.pzstd", "-n", "2"}}; - EXPECT_TRUE(options.parse(args.size(), args.data())); - Options expected = { - 2, 23, 3, false, "silesia.tar", "silesia.tar.pzstd", false, false}; + auto args = makeArray("silesia.tar", "-o", "silesia.tar.pzstd", "-p", "2"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + Options expected = {2, + 23, + 3, + false, + {"silesia.tar"}, + "silesia.tar.pzstd", + false, + true, + autoMode, + true, + 2}; + EXPECT_EQ(expected, options); } { Options options; - std::array args = {{nullptr, "-n", "1"}}; - EXPECT_TRUE(options.parse(args.size(), args.data())); + auto args = makeArray("x", "-p", "1"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); } { Options options; - std::array args = {{nullptr, "-", "-n", "1"}}; - EXPECT_TRUE(options.parse(args.size(), args.data())); + auto args = makeArray("x", "-p", "1"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + } +} + +TEST(Options, GetOutputFile) { + { + Options options; + auto args = makeArray("x"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ("x.zst", options.getOutputFile(options.inputFiles[0])); + } + { + Options options; + auto args = makeArray("-o-"); + EXPECT_FAILURE(options.parse(args.size(), args.data())); + EXPECT_EQ("-", options.getOutputFile(options.inputFiles[0])); + } + { + Options options; + auto args = makeArray("x", "y", "-o", nullOutput); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ(nullOutput, options.getOutputFile(options.inputFiles[0])); + } + { + Options options; + auto args = makeArray("x.zst", "-do", nullOutput); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ(nullOutput, options.getOutputFile(options.inputFiles[0])); + } + { + Options options; + auto args = makeArray("x.zst", "-d"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ("x", options.getOutputFile(options.inputFiles[0])); + } + { + Options options; + auto args = makeArray("xzst", "-d"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ("", options.getOutputFile(options.inputFiles[0])); + } + { + Options options; + auto args = makeArray("xzst", "-doxx"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ("xx", options.getOutputFile(options.inputFiles[0])); + } +} + +TEST(Options, MultipleFiles) { + { + Options options; + auto args = makeArray("x", "y", "z"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + Options expected; + expected.inputFiles = {"x", "y", "z"}; + expected.verbosity = 1; + EXPECT_EQ(expected, options); + } + { + Options options; + auto args = makeArray("x", "y", "z", "-o", nullOutput); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + Options expected; + expected.inputFiles = {"x", "y", "z"}; + expected.outputFile = nullOutput; + expected.verbosity = 1; + EXPECT_EQ(expected, options); + } + { + Options options; + auto args = makeArray("x", "y", "-o-"); + EXPECT_FAILURE(options.parse(args.size(), args.data())); + } + { + Options options; + auto args = makeArray("x", "y", "-o", "file"); + EXPECT_FAILURE(options.parse(args.size(), args.data())); + } + { + Options options; + auto args = makeArray("-qqvd12qp4", "-f", "x", "--", "--rm", "-c"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + Options expected = {4, 23, 12, true, {"x", "--rm", "-c"}, + "", true, true, autoMode, true, + 0}; + EXPECT_EQ(expected, options); } } TEST(Options, NumThreads) { { Options options; - std::array args = {{nullptr, "-o", "-"}}; - EXPECT_TRUE(options.parse(args.size(), args.data())); + auto args = makeArray("x", "-dfo", "-"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); } { Options options; - std::array args = {{nullptr, "-n", "0", "-o", "-"}}; - EXPECT_FALSE(options.parse(args.size(), args.data())); + auto args = makeArray("x", "-p", "0", "-fo", "-"); + EXPECT_FAILURE(options.parse(args.size(), args.data())); } { Options options; - std::array args = {{nullptr, "-n", "-o", "-"}}; - EXPECT_FALSE(options.parse(args.size(), args.data())); + auto args = makeArray("-f", "-p", "-o", "-"); + EXPECT_FAILURE(options.parse(args.size(), args.data())); } } TEST(Options, BadCompressionLevel) { { Options options; - std::array args = {{nullptr, "x", "-20"}}; - EXPECT_FALSE(options.parse(args.size(), args.data())); + auto args = makeArray("x", "-20"); + EXPECT_FAILURE(options.parse(args.size(), args.data())); } { Options options; - std::array args = {{nullptr, "x", "-u", "-23"}}; - EXPECT_FALSE(options.parse(args.size(), args.data())); + auto args = makeArray("x", "--ultra", "-23"); + EXPECT_FAILURE(options.parse(args.size(), args.data())); + } + { + Options options; + auto args = makeArray("x", "--1"); // negative 1? + EXPECT_FAILURE(options.parse(args.size(), args.data())); } } TEST(Options, InvalidOption) { { Options options; - std::array args = {{nullptr, "x", "-x"}}; - EXPECT_FALSE(options.parse(args.size(), args.data())); + auto args = makeArray("x", "-x"); + EXPECT_FAILURE(options.parse(args.size(), args.data())); } } TEST(Options, BadOutputFile) { { Options options; - std::array args = {{nullptr, "notzst", "-d", "-n", "1"}}; - EXPECT_FALSE(options.parse(args.size(), args.data())); + auto args = makeArray("notzst", "-d", "-p", "1"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ("", options.getOutputFile(options.inputFiles.front())); + } +} + +TEST(Options, BadOptionsWithArguments) { + { + Options options; + auto args = makeArray("x", "-pf"); + EXPECT_FAILURE(options.parse(args.size(), args.data())); + } + { + Options options; + auto args = makeArray("x", "-p", "10f"); + EXPECT_FAILURE(options.parse(args.size(), args.data())); + } + { + Options options; + auto args = makeArray("x", "-p"); + EXPECT_FAILURE(options.parse(args.size(), args.data())); + } + { + Options options; + auto args = makeArray("x", "-o"); + EXPECT_FAILURE(options.parse(args.size(), args.data())); + } + { + Options options; + auto args = makeArray("x", "-o"); + EXPECT_FAILURE(options.parse(args.size(), args.data())); + } +} + +TEST(Options, KeepSource) { + { + Options options; + auto args = makeArray("x", "--rm", "-k"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ(true, options.keepSource); + } + { + Options options; + auto args = makeArray("x", "--rm", "--keep"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ(true, options.keepSource); + } + { + Options options; + auto args = makeArray("x"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ(true, options.keepSource); + } + { + Options options; + auto args = makeArray("x", "--rm"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ(false, options.keepSource); + } +} + +TEST(Options, Verbosity) { + { + Options options; + auto args = makeArray("x"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ(2, options.verbosity); + } + { + Options options; + auto args = makeArray("--quiet", "-qq", "x"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ(-1, options.verbosity); + } + { + Options options; + auto args = makeArray("x", "y"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ(1, options.verbosity); + } + { + Options options; + auto args = makeArray("--", "x", "y"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ(1, options.verbosity); + } + { + Options options; + auto args = makeArray("-qv", "x", "y"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ(1, options.verbosity); + } + { + Options options; + auto args = makeArray("-v", "x", "y"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ(3, options.verbosity); + } + { + Options options; + auto args = makeArray("-v", "x"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ(3, options.verbosity); + } +} + +TEST(Options, TestMode) { + { + Options options; + auto args = makeArray("x", "-t"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ(true, options.keepSource); + EXPECT_EQ(true, options.decompress); + EXPECT_EQ(nullOutput, options.outputFile); + } + { + Options options; + auto args = makeArray("x", "--test", "--rm", "-ohello"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ(true, options.keepSource); + EXPECT_EQ(true, options.decompress); + EXPECT_EQ(nullOutput, options.outputFile); + } +} + +TEST(Options, Checksum) { + { + Options options; + auto args = makeArray("x.zst", "--no-check", "-Cd"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ(true, options.checksum); + } + { + Options options; + auto args = makeArray("x"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ(true, options.checksum); + } + { + Options options; + auto args = makeArray("x", "--no-check", "--check"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ(true, options.checksum); + } + { + Options options; + auto args = makeArray("x", "--no-check"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ(false, options.checksum); + } +} + +TEST(Options, InputFiles) { + { + Options options; + auto args = makeArray("-cd"); + options.parse(args.size(), args.data()); + EXPECT_EQ(1, options.inputFiles.size()); + EXPECT_EQ("-", options.inputFiles[0]); + EXPECT_EQ("-", options.outputFile); + } + { + Options options; + auto args = makeArray(); + options.parse(args.size(), args.data()); + EXPECT_EQ(1, options.inputFiles.size()); + EXPECT_EQ("-", options.inputFiles[0]); + EXPECT_EQ("-", options.outputFile); + } + { + Options options; + auto args = makeArray("-d"); + options.parse(args.size(), args.data()); + EXPECT_EQ(1, options.inputFiles.size()); + EXPECT_EQ("-", options.inputFiles[0]); + EXPECT_EQ("-", options.outputFile); + } + { + Options options; + auto args = makeArray("x", "-"); + EXPECT_FAILURE(options.parse(args.size(), args.data())); + } +} + +TEST(Options, InvalidOptions) { + { + Options options; + auto args = makeArray("-ibasdf"); + EXPECT_FAILURE(options.parse(args.size(), args.data())); + } + { + Options options; + auto args = makeArray("- "); + EXPECT_FAILURE(options.parse(args.size(), args.data())); + } + { + Options options; + auto args = makeArray("-n15"); + EXPECT_FAILURE(options.parse(args.size(), args.data())); + } + { + Options options; + auto args = makeArray("-0", "x"); + EXPECT_FAILURE(options.parse(args.size(), args.data())); } } TEST(Options, Extras) { { Options options; - std::array args = {{nullptr, "-h"}}; - EXPECT_FALSE(options.parse(args.size(), args.data())); + auto args = makeArray("-h"); + EXPECT_MESSAGE(options.parse(args.size(), args.data())); } { Options options; - std::array args = {{nullptr, "-V"}}; - EXPECT_FALSE(options.parse(args.size(), args.data())); + auto args = makeArray("-H"); + EXPECT_MESSAGE(options.parse(args.size(), args.data())); + } + { + Options options; + auto args = makeArray("-V"); + EXPECT_MESSAGE(options.parse(args.size(), args.data())); + } + { + Options options; + auto args = makeArray("--help"); + EXPECT_MESSAGE(options.parse(args.size(), args.data())); + } + { + Options options; + auto args = makeArray("--version"); + EXPECT_MESSAGE(options.parse(args.size(), args.data())); } } diff --git a/contrib/pzstd/test/PzstdTest.cpp b/contrib/pzstd/test/PzstdTest.cpp index 9d1256fa..64bcf9ca 100644 --- a/contrib/pzstd/test/PzstdTest.cpp +++ b/contrib/pzstd/test/PzstdTest.cpp @@ -6,14 +6,16 @@ * LICENSE file in the root directory of this source tree. An additional grant * of patent rights can be found in the PATENTS file in the same directory. */ -#include "datagen.h" #include "Pzstd.h" +extern "C" { +#include "datagen.h" +} #include "test/RoundTrip.h" #include "utils/ScopeGuard.h" -#include #include #include +#include #include #include @@ -25,11 +27,14 @@ TEST(Pzstd, SmallSizes) { std::fprintf(stderr, "Pzstd.SmallSizes seed: %u\n", seed); std::mt19937 gen(seed); - for (unsigned len = 1; len < 1028; ++len) { + for (unsigned len = 1; len < 256; ++len) { + if (len % 16 == 0) { + std::fprintf(stderr, "%u / 16\n", len / 16); + } std::string inputFile = std::tmpnam(nullptr); auto guard = makeScopeGuard([&] { std::remove(inputFile.c_str()); }); { - static uint8_t buf[1028]; + static uint8_t buf[256]; RDG_genBuffer(buf, len, 0.5, 0.0, gen()); auto fd = std::fopen(inputFile.c_str(), "wb"); auto written = std::fwrite(buf, 1, len, fd); @@ -37,19 +42,16 @@ TEST(Pzstd, SmallSizes) { ASSERT_EQ(written, len); } for (unsigned headers = 0; headers <= 1; ++headers) { - for (unsigned numThreads = 1; numThreads <= 4; numThreads *= 2) { - for (unsigned level = 1; level <= 8; level *= 8) { + for (unsigned numThreads = 1; numThreads <= 2; ++numThreads) { + for (unsigned level = 1; level <= 4; level *= 4) { auto errorGuard = makeScopeGuard([&] { - guard.dismiss(); - std::fprintf(stderr, "file: %s\n", inputFile.c_str()); std::fprintf(stderr, "pzstd headers: %u\n", headers); std::fprintf(stderr, "# threads: %u\n", numThreads); std::fprintf(stderr, "compression level: %u\n", level); }); Options options; - options.pzstdHeaders = headers; options.overwrite = true; - options.inputFile = inputFile; + options.inputFiles = {inputFile}; options.numThreads = numThreads; options.compressionLevel = level; ASSERT_TRUE(roundTrip(options)); @@ -80,17 +82,14 @@ TEST(Pzstd, LargeSizes) { for (unsigned numThreads = 1; numThreads <= 16; numThreads *= 4) { for (unsigned level = 1; level <= 4; level *= 2) { auto errorGuard = makeScopeGuard([&] { - guard.dismiss(); - std::fprintf(stderr, "file: %s\n", inputFile.c_str()); std::fprintf(stderr, "pzstd headers: %u\n", headers); std::fprintf(stderr, "# threads: %u\n", numThreads); std::fprintf(stderr, "compression level: %u\n", level); }); Options options; - options.pzstdHeaders = headers; options.overwrite = true; - options.inputFile = inputFile; - options.numThreads = numThreads; + options.inputFiles = {inputFile}; + options.numThreads = std::min(numThreads, options.numThreads); options.compressionLevel = level; ASSERT_TRUE(roundTrip(options)); errorGuard.dismiss(); @@ -100,6 +99,40 @@ TEST(Pzstd, LargeSizes) { } } +TEST(Pzstd, ExtremelyLargeSize) { + unsigned seed = std::random_device{}(); + std::fprintf(stderr, "Pzstd.ExtremelyLargeSize seed: %u\n", seed); + std::mt19937 gen(seed); + + std::string inputFile = std::tmpnam(nullptr); + auto guard = makeScopeGuard([&] { std::remove(inputFile.c_str()); }); + + { + // Write 4GB + 64 MB + constexpr size_t kLength = 1 << 26; + std::unique_ptr buf(new uint8_t[kLength]); + auto fd = std::fopen(inputFile.c_str(), "wb"); + auto closeGuard = makeScopeGuard([&] { std::fclose(fd); }); + for (size_t i = 0; i < (1 << 6) + 1; ++i) { + RDG_genBuffer(buf.get(), kLength, 0.5, 0.0, gen()); + auto written = std::fwrite(buf.get(), 1, kLength, fd); + if (written != kLength) { + std::fprintf(stderr, "Failed to write file, skipping test\n"); + return; + } + } + } + + Options options; + options.overwrite = true; + options.inputFiles = {inputFile}; + options.compressionLevel = 1; + if (options.numThreads == 0) { + options.numThreads = 1; + } + ASSERT_TRUE(roundTrip(options)); +} + TEST(Pzstd, ExtremelyCompressible) { std::string inputFile = std::tmpnam(nullptr); auto guard = makeScopeGuard([&] { std::remove(inputFile.c_str()); }); @@ -112,9 +145,8 @@ TEST(Pzstd, ExtremelyCompressible) { ASSERT_EQ(written, 10000); } Options options; - options.pzstdHeaders = false; options.overwrite = true; - options.inputFile = inputFile; + options.inputFiles = {inputFile}; options.numThreads = 1; options.compressionLevel = 1; ASSERT_TRUE(roundTrip(options)); diff --git a/contrib/pzstd/test/RoundTrip.h b/contrib/pzstd/test/RoundTrip.h index 829c95ca..8b908845 100644 --- a/contrib/pzstd/test/RoundTrip.h +++ b/contrib/pzstd/test/RoundTrip.h @@ -55,7 +55,10 @@ inline bool check(std::string source, std::string decompressed) { } inline bool roundTrip(Options& options) { - std::string source = options.inputFile; + if (options.inputFiles.size() != 1) { + return false; + } + std::string source = options.inputFiles.front(); std::string compressedFile = std::tmpnam(nullptr); std::string decompressedFile = std::tmpnam(nullptr); auto guard = makeScopeGuard([&] { @@ -66,21 +69,15 @@ inline bool roundTrip(Options& options) { { options.outputFile = compressedFile; options.decompress = false; - ErrorHolder errorHolder; - pzstdMain(options, errorHolder); - if (errorHolder.hasError()) { - errorHolder.getError(); + if (pzstdMain(options) != 0) { return false; } } { options.decompress = true; - options.inputFile = compressedFile; + options.inputFiles.front() = compressedFile; options.outputFile = decompressedFile; - ErrorHolder errorHolder; - pzstdMain(options, errorHolder); - if (errorHolder.hasError()) { - errorHolder.getError(); + if (pzstdMain(options) != 0) { return false; } } diff --git a/contrib/pzstd/test/RoundTripTest.cpp b/contrib/pzstd/test/RoundTripTest.cpp index 01c1c811..ed2ea770 100644 --- a/contrib/pzstd/test/RoundTripTest.cpp +++ b/contrib/pzstd/test/RoundTripTest.cpp @@ -6,7 +6,9 @@ * LICENSE file in the root directory of this source tree. An additional grant * of patent rights can be found in the PATENTS file in the same directory. */ +extern "C" { #include "datagen.h" +} #include "Options.h" #include "test/RoundTrip.h" #include "utils/ScopeGuard.h" @@ -46,14 +48,12 @@ string generateInputFile(Generator& gen) { template Options generateOptions(Generator& gen, const string& inputFile) { Options options; - options.inputFile = inputFile; + options.inputFiles = {inputFile}; options.overwrite = true; - std::bernoulli_distribution pzstdHeaders{0.75}; std::uniform_int_distribution numThreads{1, 32}; std::uniform_int_distribution compressionLevel{1, 10}; - options.pzstdHeaders = pzstdHeaders(gen); options.numThreads = numThreads(gen); options.compressionLevel = compressionLevel(gen); @@ -61,7 +61,7 @@ Options generateOptions(Generator& gen, const string& inputFile) { } } -int main(int argc, char** argv) { +int main() { std::mt19937 gen(std::random_device{}()); auto newlineGuard = makeScopeGuard([] { std::fprintf(stderr, "\n"); }); @@ -77,8 +77,6 @@ int main(int argc, char** argv) { std::fprintf(stderr, "numThreads: %u\n", options.numThreads); std::fprintf(stderr, "level: %u\n", options.compressionLevel); std::fprintf(stderr, "decompress? %u\n", (unsigned)options.decompress); - std::fprintf( - stderr, "pzstd headers? %u\n", (unsigned)options.pzstdHeaders); std::fprintf(stderr, "file: %s\n", inputFile.c_str()); return 1; } diff --git a/contrib/pzstd/utils/FileSystem.h b/contrib/pzstd/utils/FileSystem.h index 979c82b7..7d597047 100644 --- a/contrib/pzstd/utils/FileSystem.h +++ b/contrib/pzstd/utils/FileSystem.h @@ -21,10 +21,11 @@ namespace pzstd { +// using file_status = ... causes gcc to emit a false positive warning #if defined(_MSC_VER) -using file_status = struct ::_stat64; +typedef struct ::_stat64 file_status; #else -using file_status = struct ::stat; +typedef struct ::stat file_status; #endif /// http://en.cppreference.com/w/cpp/filesystem/status @@ -59,6 +60,22 @@ inline bool is_regular_file(StringPiece path, std::error_code& ec) noexcept { return is_regular_file(status(path, ec)); } +/// http://en.cppreference.com/w/cpp/filesystem/is_directory +inline bool is_directory(file_status status) noexcept { +#if defined(S_ISDIR) + return S_ISDIR(status.st_mode); +#elif !defined(S_ISDIR) && defined(S_IFMT) && defined(S_IFDIR) + return (status.st_mode & S_IFMT) == S_IFDIR; +#else + static_assert(false, "NO POSIX stat() support."); +#endif +} + +/// http://en.cppreference.com/w/cpp/filesystem/is_directory +inline bool is_directory(StringPiece path, std::error_code& ec) noexcept { + return is_directory(status(path, ec)); +} + /// http://en.cppreference.com/w/cpp/filesystem/file_size inline std::uintmax_t file_size( StringPiece path, diff --git a/contrib/pzstd/utils/test/ThreadPoolTest.cpp b/contrib/pzstd/utils/test/ThreadPoolTest.cpp index 9b9868cb..1d857aae 100644 --- a/contrib/pzstd/utils/test/ThreadPoolTest.cpp +++ b/contrib/pzstd/utils/test/ThreadPoolTest.cpp @@ -20,12 +20,12 @@ TEST(ThreadPool, Ordering) { { ThreadPool executor(1); - for (int i = 0; i < 100; ++i) { + for (int i = 0; i < 10; ++i) { executor.add([ &results, i ] { results.push_back(i); }); } } - for (int i = 0; i < 100; ++i) { + for (int i = 0; i < 10; ++i) { EXPECT_EQ(i, results[i]); } } @@ -35,7 +35,7 @@ TEST(ThreadPool, AllJobsFinished) { std::atomic start{false}; { ThreadPool executor(5); - for (int i = 0; i < 1000; ++i) { + for (int i = 0; i < 10; ++i) { executor.add([ &numFinished, &start ] { while (!start.load()) { // spin @@ -45,7 +45,7 @@ TEST(ThreadPool, AllJobsFinished) { } start.store(true); } - EXPECT_EQ(1000, numFinished.load()); + EXPECT_EQ(10, numFinished.load()); } TEST(ThreadPool, AddJobWhileJoining) { diff --git a/contrib/pzstd/utils/test/WorkQueueTest.cpp b/contrib/pzstd/utils/test/WorkQueueTest.cpp index 84d8573c..ebf375a8 100644 --- a/contrib/pzstd/utils/test/WorkQueueTest.cpp +++ b/contrib/pzstd/utils/test/WorkQueueTest.cpp @@ -89,14 +89,14 @@ TEST(WorkQueue, SPSC) { TEST(WorkQueue, SPMC) { WorkQueue queue; - std::vector results(10000, -1); + std::vector results(50, -1); std::mutex mutex; std::vector threads; - for (int i = 0; i < 100; ++i) { + for (int i = 0; i < 5; ++i) { threads.emplace_back(Popper{&queue, results.data(), &mutex}); } - for (int i = 0; i < 10000; ++i) { + for (int i = 0; i < 50; ++i) { queue.push(i); } queue.finish(); @@ -105,24 +105,24 @@ TEST(WorkQueue, SPMC) { thread.join(); } - for (int i = 0; i < 10000; ++i) { + for (int i = 0; i < 50; ++i) { EXPECT_EQ(i, results[i]); } } TEST(WorkQueue, MPMC) { WorkQueue queue; - std::vector results(10000, -1); + std::vector results(100, -1); std::mutex mutex; std::vector popperThreads; - for (int i = 0; i < 100; ++i) { + for (int i = 0; i < 4; ++i) { popperThreads.emplace_back(Popper{&queue, results.data(), &mutex}); } std::vector pusherThreads; - for (int i = 0; i < 10; ++i) { - auto min = i * 1000; - auto max = (i + 1) * 1000; + for (int i = 0; i < 2; ++i) { + auto min = i * 50; + auto max = (i + 1) * 50; pusherThreads.emplace_back( [ &queue, min, max ] { for (int i = min; i < max; ++i) { @@ -140,7 +140,7 @@ TEST(WorkQueue, MPMC) { thread.join(); } - for (int i = 0; i < 10000; ++i) { + for (int i = 0; i < 100; ++i) { EXPECT_EQ(i, results[i]); } } @@ -197,16 +197,16 @@ TEST(WorkQueue, SetMaxSize) { } TEST(WorkQueue, BoundedSizeMPMC) { - WorkQueue queue(100); - std::vector results(10000, -1); + WorkQueue queue(10); + std::vector results(200, -1); std::mutex mutex; std::vector popperThreads; - for (int i = 0; i < 10; ++i) { + for (int i = 0; i < 4; ++i) { popperThreads.emplace_back(Popper{&queue, results.data(), &mutex}); } std::vector pusherThreads; - for (int i = 0; i < 100; ++i) { + for (int i = 0; i < 2; ++i) { auto min = i * 100; auto max = (i + 1) * 100; pusherThreads.emplace_back( @@ -226,7 +226,7 @@ TEST(WorkQueue, BoundedSizeMPMC) { thread.join(); } - for (int i = 0; i < 10000; ++i) { + for (int i = 0; i < 200; ++i) { EXPECT_EQ(i, results[i]); } } diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c index 5a9bc40e..d9e89f80 100644 --- a/lib/legacy/zstd_v06.c +++ b/lib/legacy/zstd_v06.c @@ -326,7 +326,7 @@ extern "C" { * It avoids reloading the dictionary each time. * `preparedDCtx` must have been properly initialized using ZSTDv06_decompressBegin_usingDict(). * Requires 2 contexts : 1 for reference (preparedDCtx), which will not be modified, and 1 to run the decompression operation (dctx) */ -ZSTDLIB_API size_t ZSTDv06_decompress_usingPreparedDCtx( +ZSTDLIBv06_API size_t ZSTDv06_decompress_usingPreparedDCtx( ZSTDv06_DCtx* dctx, const ZSTDv06_DCtx* preparedDCtx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); @@ -337,7 +337,7 @@ ZSTDLIB_API size_t ZSTDv06_decompress_usingPreparedDCtx( static const size_t ZSTDv06_frameHeaderSize_min = 5; static const size_t ZSTDv06_frameHeaderSize_max = ZSTDv06_FRAMEHEADERSIZE_MAX; -ZSTDLIB_API size_t ZSTDv06_decompressBegin(ZSTDv06_DCtx* dctx); +ZSTDLIBv06_API size_t ZSTDv06_decompressBegin(ZSTDv06_DCtx* dctx); /* Streaming decompression, direct mode (bufferless) @@ -396,7 +396,7 @@ ZSTDLIB_API size_t ZSTDv06_decompressBegin(ZSTDv06_DCtx* dctx); */ #define ZSTDv06_BLOCKSIZE_MAX (128 * 1024) /* define, for static allocation */ -ZSTDLIB_API size_t ZSTDv06_decompressBlock(ZSTDv06_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTDLIBv06_API size_t ZSTDv06_decompressBlock(ZSTDv06_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); diff --git a/lib/legacy/zstd_v06.h b/lib/legacy/zstd_v06.h index bcc6efbc..14040abd 100644 --- a/lib/legacy/zstd_v06.h +++ b/lib/legacy/zstd_v06.h @@ -14,23 +14,19 @@ extern "C" { #endif -/*-************************************* -* Dependencies -***************************************/ +/*====== Dependency ======*/ #include /* size_t */ -/*-*************************************************************** -* Export parameters -*****************************************************************/ +/*====== Export for Windows ======*/ /*! * ZSTDv06_DLL_EXPORT : * Enable exporting of functions when building a Windows DLL */ #if defined(_WIN32) && defined(ZSTDv06_DLL_EXPORT) && (ZSTDv06_DLL_EXPORT==1) -# define ZSTDLIB_API __declspec(dllexport) +# define ZSTDLIBv06_API __declspec(dllexport) #else -# define ZSTDLIB_API +# define ZSTDLIBv06_API #endif @@ -42,18 +38,18 @@ extern "C" { `dstCapacity` must be large enough, equal or larger than originalSize. @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), or an errorCode if it fails (which can be tested using ZSTDv06_isError()) */ -ZSTDLIB_API size_t ZSTDv06_decompress( void* dst, size_t dstCapacity, - const void* src, size_t compressedSize); +ZSTDLIBv06_API size_t ZSTDv06_decompress( void* dst, size_t dstCapacity, + const void* src, size_t compressedSize); /* ************************************* * Helper functions ***************************************/ -ZSTDLIB_API size_t ZSTDv06_compressBound(size_t srcSize); /*!< maximum compressed size (worst case scenario) */ +ZSTDLIBv06_API size_t ZSTDv06_compressBound(size_t srcSize); /*!< maximum compressed size (worst case scenario) */ /* Error Management */ -ZSTDLIB_API unsigned ZSTDv06_isError(size_t code); /*!< tells if a `size_t` function result is an error code */ -ZSTDLIB_API const char* ZSTDv06_getErrorName(size_t code); /*!< provides readable string for an error code */ +ZSTDLIBv06_API unsigned ZSTDv06_isError(size_t code); /*!< tells if a `size_t` function result is an error code */ +ZSTDLIBv06_API const char* ZSTDv06_getErrorName(size_t code); /*!< provides readable string for an error code */ /* ************************************* @@ -61,12 +57,12 @@ ZSTDLIB_API const char* ZSTDv06_getErrorName(size_t code); /*!< provides rea ***************************************/ /** Decompression context */ typedef struct ZSTDv06_DCtx_s ZSTDv06_DCtx; -ZSTDLIB_API ZSTDv06_DCtx* ZSTDv06_createDCtx(void); -ZSTDLIB_API size_t ZSTDv06_freeDCtx(ZSTDv06_DCtx* dctx); /*!< @return : errorCode */ +ZSTDLIBv06_API ZSTDv06_DCtx* ZSTDv06_createDCtx(void); +ZSTDLIBv06_API size_t ZSTDv06_freeDCtx(ZSTDv06_DCtx* dctx); /*!< @return : errorCode */ /** ZSTDv06_decompressDCtx() : * Same as ZSTDv06_decompress(), but requires an already allocated ZSTDv06_DCtx (see ZSTDv06_createDCtx()) */ -ZSTDLIB_API size_t ZSTDv06_decompressDCtx(ZSTDv06_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTDLIBv06_API size_t ZSTDv06_decompressDCtx(ZSTDv06_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); /*-*********************** @@ -76,10 +72,10 @@ ZSTDLIB_API size_t ZSTDv06_decompressDCtx(ZSTDv06_DCtx* ctx, void* dst, size_t d * Decompression using a pre-defined Dictionary content (see dictBuilder). * Dictionary must be identical to the one used during compression, otherwise regenerated data will be corrupted. * Note : dict can be NULL, in which case, it's equivalent to ZSTDv06_decompressDCtx() */ -ZSTDLIB_API size_t ZSTDv06_decompress_usingDict(ZSTDv06_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const void* dict,size_t dictSize); +ZSTDLIBv06_API size_t ZSTDv06_decompress_usingDict(ZSTDv06_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize); /*-************************ @@ -88,12 +84,12 @@ ZSTDLIB_API size_t ZSTDv06_decompress_usingDict(ZSTDv06_DCtx* dctx, struct ZSTDv06_frameParams_s { unsigned long long frameContentSize; unsigned windowLog; }; typedef struct ZSTDv06_frameParams_s ZSTDv06_frameParams; -ZSTDLIB_API size_t ZSTDv06_getFrameParams(ZSTDv06_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input */ -ZSTDLIB_API size_t ZSTDv06_decompressBegin_usingDict(ZSTDv06_DCtx* dctx, const void* dict, size_t dictSize); -ZSTDLIB_API void ZSTDv06_copyDCtx(ZSTDv06_DCtx* dctx, const ZSTDv06_DCtx* preparedDCtx); +ZSTDLIBv06_API size_t ZSTDv06_getFrameParams(ZSTDv06_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input */ +ZSTDLIBv06_API size_t ZSTDv06_decompressBegin_usingDict(ZSTDv06_DCtx* dctx, const void* dict, size_t dictSize); +ZSTDLIBv06_API void ZSTDv06_copyDCtx(ZSTDv06_DCtx* dctx, const ZSTDv06_DCtx* preparedDCtx); -ZSTDLIB_API size_t ZSTDv06_nextSrcSizeToDecompress(ZSTDv06_DCtx* dctx); -ZSTDLIB_API size_t ZSTDv06_decompressContinue(ZSTDv06_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTDLIBv06_API size_t ZSTDv06_nextSrcSizeToDecompress(ZSTDv06_DCtx* dctx); +ZSTDLIBv06_API size_t ZSTDv06_decompressContinue(ZSTDv06_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); @@ -102,15 +98,15 @@ ZSTDLIB_API size_t ZSTDv06_decompressContinue(ZSTDv06_DCtx* dctx, void* dst, siz ***************************************/ typedef struct ZBUFFv06_DCtx_s ZBUFFv06_DCtx; -ZSTDLIB_API ZBUFFv06_DCtx* ZBUFFv06_createDCtx(void); -ZSTDLIB_API size_t ZBUFFv06_freeDCtx(ZBUFFv06_DCtx* dctx); +ZSTDLIBv06_API ZBUFFv06_DCtx* ZBUFFv06_createDCtx(void); +ZSTDLIBv06_API size_t ZBUFFv06_freeDCtx(ZBUFFv06_DCtx* dctx); -ZSTDLIB_API size_t ZBUFFv06_decompressInit(ZBUFFv06_DCtx* dctx); -ZSTDLIB_API size_t ZBUFFv06_decompressInitDictionary(ZBUFFv06_DCtx* dctx, const void* dict, size_t dictSize); +ZSTDLIBv06_API size_t ZBUFFv06_decompressInit(ZBUFFv06_DCtx* dctx); +ZSTDLIBv06_API size_t ZBUFFv06_decompressInitDictionary(ZBUFFv06_DCtx* dctx, const void* dict, size_t dictSize); -ZSTDLIB_API size_t ZBUFFv06_decompressContinue(ZBUFFv06_DCtx* dctx, - void* dst, size_t* dstCapacityPtr, - const void* src, size_t* srcSizePtr); +ZSTDLIBv06_API size_t ZBUFFv06_decompressContinue(ZBUFFv06_DCtx* dctx, + void* dst, size_t* dstCapacityPtr, + const void* src, size_t* srcSizePtr); /*-*************************************************************************** * Streaming decompression howto @@ -140,13 +136,13 @@ ZSTDLIB_API size_t ZBUFFv06_decompressContinue(ZBUFFv06_DCtx* dctx, /* ************************************* * Tool functions ***************************************/ -ZSTDLIB_API unsigned ZBUFFv06_isError(size_t errorCode); -ZSTDLIB_API const char* ZBUFFv06_getErrorName(size_t errorCode); +ZSTDLIBv06_API unsigned ZBUFFv06_isError(size_t errorCode); +ZSTDLIBv06_API const char* ZBUFFv06_getErrorName(size_t errorCode); /** Functions below provide recommended buffer sizes for Compression or Decompression operations. * These sizes are just hints, they tend to offer better latency */ -ZSTDLIB_API size_t ZBUFFv06_recommendedDInSize(void); -ZSTDLIB_API size_t ZBUFFv06_recommendedDOutSize(void); +ZSTDLIBv06_API size_t ZBUFFv06_recommendedDInSize(void); +ZSTDLIBv06_API size_t ZBUFFv06_recommendedDOutSize(void); /*-************************************* diff --git a/lib/legacy/zstd_v07.c b/lib/legacy/zstd_v07.c index dac71aeb..f4c8073f 100644 --- a/lib/legacy/zstd_v07.c +++ b/lib/legacy/zstd_v07.c @@ -68,27 +68,27 @@ typedef struct { ZSTDv07_allocFunction customAlloc; ZSTDv07_freeFunction customF /*! ZSTDv07_estimateDCtxSize() : * Gives the potential amount of memory allocated to create a ZSTDv07_DCtx */ -ZSTDLIB_API size_t ZSTDv07_estimateDCtxSize(void); +ZSTDLIBv07_API size_t ZSTDv07_estimateDCtxSize(void); /*! ZSTDv07_createDCtx_advanced() : * Create a ZSTD decompression context using external alloc and free functions */ -ZSTDLIB_API ZSTDv07_DCtx* ZSTDv07_createDCtx_advanced(ZSTDv07_customMem customMem); +ZSTDLIBv07_API ZSTDv07_DCtx* ZSTDv07_createDCtx_advanced(ZSTDv07_customMem customMem); /*! ZSTDv07_sizeofDCtx() : * Gives the amount of memory used by a given ZSTDv07_DCtx */ -ZSTDLIB_API size_t ZSTDv07_sizeofDCtx(const ZSTDv07_DCtx* dctx); +ZSTDLIBv07_API size_t ZSTDv07_sizeofDCtx(const ZSTDv07_DCtx* dctx); /* ****************************************************************** * Buffer-less streaming functions (synchronous mode) ********************************************************************/ -ZSTDLIB_API size_t ZSTDv07_decompressBegin(ZSTDv07_DCtx* dctx); -ZSTDLIB_API size_t ZSTDv07_decompressBegin_usingDict(ZSTDv07_DCtx* dctx, const void* dict, size_t dictSize); -ZSTDLIB_API void ZSTDv07_copyDCtx(ZSTDv07_DCtx* dctx, const ZSTDv07_DCtx* preparedDCtx); +ZSTDLIBv07_API size_t ZSTDv07_decompressBegin(ZSTDv07_DCtx* dctx); +ZSTDLIBv07_API size_t ZSTDv07_decompressBegin_usingDict(ZSTDv07_DCtx* dctx, const void* dict, size_t dictSize); +ZSTDLIBv07_API void ZSTDv07_copyDCtx(ZSTDv07_DCtx* dctx, const ZSTDv07_DCtx* preparedDCtx); -ZSTDLIB_API size_t ZSTDv07_nextSrcSizeToDecompress(ZSTDv07_DCtx* dctx); -ZSTDLIB_API size_t ZSTDv07_decompressContinue(ZSTDv07_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTDLIBv07_API size_t ZSTDv07_nextSrcSizeToDecompress(ZSTDv07_DCtx* dctx); +ZSTDLIBv07_API size_t ZSTDv07_decompressContinue(ZSTDv07_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); /* Buffer-less streaming decompression (synchronous mode) @@ -169,8 +169,8 @@ ZSTDLIB_API size_t ZSTDv07_decompressContinue(ZSTDv07_DCtx* dctx, void* dst, siz */ #define ZSTDv07_BLOCKSIZE_ABSOLUTEMAX (128 * 1024) /* define, for static allocation */ -ZSTDLIB_API size_t ZSTDv07_decompressBlock(ZSTDv07_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); -ZSTDLIB_API size_t ZSTDv07_insertBlock(ZSTDv07_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert block into `dctx` history. Useful for uncompressed blocks */ +ZSTDLIBv07_API size_t ZSTDv07_decompressBlock(ZSTDv07_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTDLIBv07_API size_t ZSTDv07_insertBlock(ZSTDv07_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert block into `dctx` history. Useful for uncompressed blocks */ #endif /* ZSTDv07_STATIC_LINKING_ONLY */ @@ -650,8 +650,8 @@ MEM_STATIC size_t BITv07_readBitsFast(BITv07_DStream_t* bitD, U32 nbBits) if status == unfinished, internal register is filled with >= (sizeof(bitD->bitContainer)*8 - 7) bits */ MEM_STATIC BITv07_DStream_status BITv07_reloadDStream(BITv07_DStream_t* bitD) { - if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should not happen => corruption detected */ - return BITv07_DStream_overflow; + if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should not happen => corruption detected */ + return BITv07_DStream_overflow; if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer)) { bitD->ptr -= bitD->bitsConsumed >> 3; @@ -3831,7 +3831,7 @@ size_t ZSTDv07_decompressBlock(ZSTDv07_DCtx* dctx, /** ZSTDv07_insertBlock() : insert `src` block into `dctx` history. Useful to track uncompressed blocks. */ -ZSTDLIB_API size_t ZSTDv07_insertBlock(ZSTDv07_DCtx* dctx, const void* blockStart, size_t blockSize) +ZSTDLIBv07_API size_t ZSTDv07_insertBlock(ZSTDv07_DCtx* dctx, const void* blockStart, size_t blockSize) { ZSTDv07_checkContinuity(dctx, blockStart); dctx->previousDstEnd = (const char*)blockStart + blockSize; @@ -4233,7 +4233,7 @@ size_t ZSTDv07_freeDDict(ZSTDv07_DDict* ddict) /*! ZSTDv07_decompress_usingDDict() : * Decompression using a pre-digested Dictionary * Use dictionary without significant overhead. */ -ZSTDLIB_API size_t ZSTDv07_decompress_usingDDict(ZSTDv07_DCtx* dctx, +ZSTDLIBv07_API size_t ZSTDv07_decompress_usingDDict(ZSTDv07_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const ZSTDv07_DDict* ddict) @@ -4320,7 +4320,7 @@ struct ZBUFFv07_DCtx_s { ZSTDv07_customMem customMem; }; /* typedef'd to ZBUFFv07_DCtx within "zstd_buffered.h" */ -ZSTDLIB_API ZBUFFv07_DCtx* ZBUFFv07_createDCtx_advanced(ZSTDv07_customMem customMem); +ZSTDLIBv07_API ZBUFFv07_DCtx* ZBUFFv07_createDCtx_advanced(ZSTDv07_customMem customMem); ZBUFFv07_DCtx* ZBUFFv07_createDCtx(void) { diff --git a/lib/legacy/zstd_v07.h b/lib/legacy/zstd_v07.h index d1fbc083..30725dcf 100644 --- a/lib/legacy/zstd_v07.h +++ b/lib/legacy/zstd_v07.h @@ -24,13 +24,12 @@ extern "C" { * Enable exporting of functions when building a Windows DLL */ #if defined(_WIN32) && defined(ZSTDv07_DLL_EXPORT) && (ZSTDv07_DLL_EXPORT==1) -# define ZSTDLIB_API __declspec(dllexport) +# define ZSTDLIBv07_API __declspec(dllexport) #else -# define ZSTDLIB_API +# define ZSTDLIBv07_API #endif - /* ************************************* * Simple API ***************************************/ @@ -46,12 +45,12 @@ unsigned long long ZSTDv07_getDecompressedSize(const void* src, size_t srcSize); `dstCapacity` must be equal or larger than originalSize. @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), or an errorCode if it fails (which can be tested using ZSTDv07_isError()) */ -ZSTDLIB_API size_t ZSTDv07_decompress( void* dst, size_t dstCapacity, - const void* src, size_t compressedSize); +ZSTDLIBv07_API size_t ZSTDv07_decompress( void* dst, size_t dstCapacity, + const void* src, size_t compressedSize); /*====== Helper functions ======*/ -ZSTDLIB_API unsigned ZSTDv07_isError(size_t code); /*!< tells if a `size_t` function result is an error code */ -ZSTDLIB_API const char* ZSTDv07_getErrorName(size_t code); /*!< provides readable string from an error code */ +ZSTDLIBv07_API unsigned ZSTDv07_isError(size_t code); /*!< tells if a `size_t` function result is an error code */ +ZSTDLIBv07_API const char* ZSTDv07_getErrorName(size_t code); /*!< provides readable string from an error code */ /*-************************************* @@ -59,12 +58,12 @@ ZSTDLIB_API const char* ZSTDv07_getErrorName(size_t code); /*!< provides rea ***************************************/ /** Decompression context */ typedef struct ZSTDv07_DCtx_s ZSTDv07_DCtx; -ZSTDLIB_API ZSTDv07_DCtx* ZSTDv07_createDCtx(void); -ZSTDLIB_API size_t ZSTDv07_freeDCtx(ZSTDv07_DCtx* dctx); /*!< @return : errorCode */ +ZSTDLIBv07_API ZSTDv07_DCtx* ZSTDv07_createDCtx(void); +ZSTDLIBv07_API size_t ZSTDv07_freeDCtx(ZSTDv07_DCtx* dctx); /*!< @return : errorCode */ /** ZSTDv07_decompressDCtx() : * Same as ZSTDv07_decompress(), requires an allocated ZSTDv07_DCtx (see ZSTDv07_createDCtx()) */ -ZSTDLIB_API size_t ZSTDv07_decompressDCtx(ZSTDv07_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTDLIBv07_API size_t ZSTDv07_decompressDCtx(ZSTDv07_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); /*-************************ @@ -74,10 +73,10 @@ ZSTDLIB_API size_t ZSTDv07_decompressDCtx(ZSTDv07_DCtx* ctx, void* dst, size_t d * Decompression using a pre-defined Dictionary content (see dictBuilder). * Dictionary must be identical to the one used during compression. * Note : This function load the dictionary, resulting in a significant startup time */ -ZSTDLIB_API size_t ZSTDv07_decompress_usingDict(ZSTDv07_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const void* dict,size_t dictSize); +ZSTDLIBv07_API size_t ZSTDv07_decompress_usingDict(ZSTDv07_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize); /*-************************** @@ -87,16 +86,16 @@ ZSTDLIB_API size_t ZSTDv07_decompress_usingDict(ZSTDv07_DCtx* dctx, * Create a digested dictionary, ready to start decompression operation without startup delay. * `dict` can be released after creation */ typedef struct ZSTDv07_DDict_s ZSTDv07_DDict; -ZSTDLIB_API ZSTDv07_DDict* ZSTDv07_createDDict(const void* dict, size_t dictSize); -ZSTDLIB_API size_t ZSTDv07_freeDDict(ZSTDv07_DDict* ddict); +ZSTDLIBv07_API ZSTDv07_DDict* ZSTDv07_createDDict(const void* dict, size_t dictSize); +ZSTDLIBv07_API size_t ZSTDv07_freeDDict(ZSTDv07_DDict* ddict); /*! ZSTDv07_decompress_usingDDict() : * Decompression using a pre-digested Dictionary * Faster startup than ZSTDv07_decompress_usingDict(), recommended when same dictionary is used multiple times. */ -ZSTDLIB_API size_t ZSTDv07_decompress_usingDDict(ZSTDv07_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const ZSTDv07_DDict* ddict); +ZSTDLIBv07_API size_t ZSTDv07_decompress_usingDDict(ZSTDv07_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTDv07_DDict* ddict); typedef struct { unsigned long long frameContentSize; @@ -105,7 +104,7 @@ typedef struct { unsigned checksumFlag; } ZSTDv07_frameParams; -ZSTDLIB_API size_t ZSTDv07_getFrameParams(ZSTDv07_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input */ +ZSTDLIBv07_API size_t ZSTDv07_getFrameParams(ZSTDv07_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input */ @@ -114,13 +113,13 @@ ZSTDLIB_API size_t ZSTDv07_getFrameParams(ZSTDv07_frameParams* fparamsPtr, const * Streaming functions ***************************************/ typedef struct ZBUFFv07_DCtx_s ZBUFFv07_DCtx; -ZSTDLIB_API ZBUFFv07_DCtx* ZBUFFv07_createDCtx(void); -ZSTDLIB_API size_t ZBUFFv07_freeDCtx(ZBUFFv07_DCtx* dctx); +ZSTDLIBv07_API ZBUFFv07_DCtx* ZBUFFv07_createDCtx(void); +ZSTDLIBv07_API size_t ZBUFFv07_freeDCtx(ZBUFFv07_DCtx* dctx); -ZSTDLIB_API size_t ZBUFFv07_decompressInit(ZBUFFv07_DCtx* dctx); -ZSTDLIB_API size_t ZBUFFv07_decompressInitDictionary(ZBUFFv07_DCtx* dctx, const void* dict, size_t dictSize); +ZSTDLIBv07_API size_t ZBUFFv07_decompressInit(ZBUFFv07_DCtx* dctx); +ZSTDLIBv07_API size_t ZBUFFv07_decompressInitDictionary(ZBUFFv07_DCtx* dctx, const void* dict, size_t dictSize); -ZSTDLIB_API size_t ZBUFFv07_decompressContinue(ZBUFFv07_DCtx* dctx, +ZSTDLIBv07_API size_t ZBUFFv07_decompressContinue(ZBUFFv07_DCtx* dctx, void* dst, size_t* dstCapacityPtr, const void* src, size_t* srcSizePtr); @@ -152,13 +151,13 @@ ZSTDLIB_API size_t ZBUFFv07_decompressContinue(ZBUFFv07_DCtx* dctx, /* ************************************* * Tool functions ***************************************/ -ZSTDLIB_API unsigned ZBUFFv07_isError(size_t errorCode); -ZSTDLIB_API const char* ZBUFFv07_getErrorName(size_t errorCode); +ZSTDLIBv07_API unsigned ZBUFFv07_isError(size_t errorCode); +ZSTDLIBv07_API const char* ZBUFFv07_getErrorName(size_t errorCode); /** Functions below provide recommended buffer sizes for Compression or Decompression operations. * These sizes are just hints, they tend to offer better latency */ -ZSTDLIB_API size_t ZBUFFv07_recommendedDInSize(void); -ZSTDLIB_API size_t ZBUFFv07_recommendedDOutSize(void); +ZSTDLIBv07_API size_t ZBUFFv07_recommendedDInSize(void); +ZSTDLIBv07_API size_t ZBUFFv07_recommendedDOutSize(void); /*-************************************* diff --git a/programs/datagen.h b/programs/datagen.h index 55f9d828..094056b6 100644 --- a/programs/datagen.h +++ b/programs/datagen.h @@ -6,7 +6,8 @@ * LICENSE file in the root directory of this source tree. An additional grant * of patent rights can be found in the PATENTS file in the same directory. */ - +#ifndef DATAGEN_H +#define DATAGEN_H #include /* size_t */ @@ -22,3 +23,5 @@ void RDG_genBuffer(void* buffer, size_t size, double matchProba, double litProba RDG_genStdout Same as RDG_genBuffer, but generates data into stdout */ + +#endif diff --git a/programs/util.h b/programs/util.h index 9d28c82d..aabebe96 100644 --- a/programs/util.h +++ b/programs/util.h @@ -31,12 +31,12 @@ extern "C" { /* Unix Large Files support (>4GB) */ -#if !defined(__LP64__) /* No point defining Large file for 64 bit */ -# define _FILE_OFFSET_BITS 64 /* turn off_t into a 64-bit type for ftello, fseeko */ -# if defined(__sun__) /* Sun Solaris 32-bits requires specific definitions */ -# define _LARGEFILE_SOURCE /* fseeko, ftello */ -# else -# define _LARGEFILE64_SOURCE /* off64_t, fseeko64, ftello64 */ +#if !defined(__LP64__) /* No point defining Large file for 64 bit */ +# define _FILE_OFFSET_BITS 64 /* turn off_t into a 64-bit type for ftello, fseeko */ +# if defined(__sun__) && !defined(_LARGEFILE_SOURCE) /* Sun Solaris 32-bits requires specific definitions */ +# define _LARGEFILE_SOURCE /* fseeko, ftello */ +# elif !defined(_LARGEFILE64_SOURCE) +# define _LARGEFILE64_SOURCE /* off64_t, fseeko64, ftello64 */ # endif #endif