diff --git a/Makefile b/Makefile index 8522d9d5..55c24530 100644 --- a/Makefile +++ b/Makefile @@ -122,8 +122,8 @@ contrib: lib $(MAKE) -C contrib/seekable_format/examples all $(MAKE) -C contrib/seekable_format/tests test $(MAKE) -C contrib/largeNbDicts all - cd contrib/single_file_libs/ ; ./build_decoder_test.sh - cd contrib/single_file_libs/ ; ./build_library_test.sh + cd build/single_file_libs/ ; ./build_decoder_test.sh + cd build/single_file_libs/ ; ./build_library_test.sh .PHONY: cleanTabs cleanTabs: diff --git a/TESTING.md b/TESTING.md index b851d1c8..32b133b6 100644 --- a/TESTING.md +++ b/TESTING.md @@ -40,5 +40,4 @@ They consist of the following tests: - Versions test (ensuring `zstd` can decode files from all previous versions) - `pzstd` with asan and tsan, as well as in 32-bits mode - Testing `zstd` with legacy mode off -- Testing `zbuff` (old streaming API) - Entire test suite and make install on macOS diff --git a/appveyor.yml b/appveyor.yml index 80d5e65b..551a7b44 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -52,6 +52,15 @@ PLATFORM: "Win32" CONFIGURATION: "Release" + - COMPILER: "clang-cl" + HOST: "cmake-visual" + PLATFORM: "x64" + CONFIGURATION: "Release" + CMAKE_GENERATOR: "Visual Studio 15 2017" + CMAKE_GENERATOR_PLATFORM: "x64" + CMAKE_GENERATOR_TOOLSET: "LLVM" + APPVEYOR_BUILD_WORKER_IMAGE: "Visual Studio 2017" + install: - ECHO Installing %COMPILER% %PLATFORM% %CONFIGURATION% - SET PATH_ORIGINAL=%PATH% @@ -154,6 +163,15 @@ COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe && COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe tests\ ) + - if [%HOST%]==[cmake-visual] ( + ECHO *** && + ECHO *** Building %CMAKE_GENERATOR% ^(%CMAKE_GENERATOR_TOOLSET%^) %PLATFORM%\%CONFIGURATION% && + PUSHD build\cmake && + cmake -DBUILD_TESTING=ON . && + cmake --build . --config %CONFIGURATION% -j4 && + POPD && + ECHO *** + ) test_script: - ECHO Testing %COMPILER% %PLATFORM% %CONFIGURATION% @@ -223,6 +241,15 @@ PLATFORM: "Win32" CONFIGURATION: "Release" + - COMPILER: "clang-cl" + HOST: "cmake-visual" + PLATFORM: "x64" + CONFIGURATION: "Release" + CMAKE_GENERATOR: "Visual Studio 15 2017" + CMAKE_GENERATOR_PLATFORM: "x64" + CMAKE_GENERATOR_TOOLSET: "LLVM" + APPVEYOR_BUILD_WORKER_IMAGE: "Visual Studio 2017" + install: - ECHO Installing %COMPILER% %PLATFORM% %CONFIGURATION% - SET PATH_ORIGINAL=%PATH% @@ -281,6 +308,15 @@ COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe && COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe tests\ ) + - if [%HOST%]==[cmake-visual] ( + ECHO *** && + ECHO *** Building %CMAKE_GENERATOR% ^(%CMAKE_GENERATOR_TOOLSET%^) %PLATFORM%\%CONFIGURATION% && + PUSHD build\cmake && + cmake -DBUILD_TESTING=ON . && + cmake --build . --config %CONFIGURATION% -j4 && + POPD && + ECHO *** + ) test_script: diff --git a/build/VS2008/fullbench/fullbench.vcproj b/build/VS2008/fullbench/fullbench.vcproj index 5e349dce..5a30c0b0 100644 --- a/build/VS2008/fullbench/fullbench.vcproj +++ b/build/VS2008/fullbench/fullbench.vcproj @@ -45,7 +45,7 @@ Name="VCCLCompilerTool" Optimization="0" AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\programs\legacy" - PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE" + PreprocessorDefinitions="ZSTD_DISABLE_DEPRECATE_WARNINGS=1;WIN32;_DEBUG;_CONSOLE" MinimalRebuild="true" BasicRuntimeChecks="3" RuntimeLibrary="3" @@ -121,7 +121,7 @@ EnableIntrinsicFunctions="true" OmitFramePointers="true" AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\programs\legacy" - PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE" + PreprocessorDefinitions="ZSTD_DISABLE_DEPRECATE_WARNINGS=1;WIN32;NDEBUG;_CONSOLE" RuntimeLibrary="0" EnableFunctionLevelLinking="true" UsePrecompiledHeader="0" @@ -195,7 +195,7 @@ Name="VCCLCompilerTool" Optimization="0" AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\programs\legacy" - PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE" + PreprocessorDefinitions="ZSTD_DISABLE_DEPRECATE_WARNINGS=1;WIN32;_DEBUG;_CONSOLE" MinimalRebuild="true" BasicRuntimeChecks="3" RuntimeLibrary="3" @@ -272,7 +272,7 @@ EnableIntrinsicFunctions="true" OmitFramePointers="true" AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\programs\legacy" - PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE" + PreprocessorDefinitions="ZSTD_DISABLE_DEPRECATE_WARNINGS=1;WIN32;NDEBUG;_CONSOLE" RuntimeLibrary="0" EnableFunctionLevelLinking="true" UsePrecompiledHeader="0" diff --git a/build/VS2008/fuzzer/fuzzer.vcproj b/build/VS2008/fuzzer/fuzzer.vcproj index 32f28468..664e60cb 100644 --- a/build/VS2008/fuzzer/fuzzer.vcproj +++ b/build/VS2008/fuzzer/fuzzer.vcproj @@ -45,7 +45,7 @@ Name="VCCLCompilerTool" Optimization="0" AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\compress;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs" - PreprocessorDefinitions="ZSTD_MULTITHREAD=1;WIN32;_DEBUG;_CONSOLE" + PreprocessorDefinitions="ZSTD_DISABLE_DEPRECATE_WARNINGS=1;ZSTD_MULTITHREAD=1;WIN32;_DEBUG;_CONSOLE" MinimalRebuild="true" BasicRuntimeChecks="3" RuntimeLibrary="3" @@ -121,7 +121,7 @@ EnableIntrinsicFunctions="true" OmitFramePointers="true" AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\compress;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs" - PreprocessorDefinitions="ZSTD_MULTITHREAD=1;WIN32;NDEBUG;_CONSOLE" + PreprocessorDefinitions="ZSTD_DISABLE_DEPRECATE_WARNINGS=1;ZSTD_MULTITHREAD=1;WIN32;NDEBUG;_CONSOLE" RuntimeLibrary="0" EnableFunctionLevelLinking="true" UsePrecompiledHeader="0" @@ -195,7 +195,7 @@ Name="VCCLCompilerTool" Optimization="0" AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\compress;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs" - PreprocessorDefinitions="ZSTD_MULTITHREAD=1;WIN32;_DEBUG;_CONSOLE" + PreprocessorDefinitions="ZSTD_DISABLE_DEPRECATE_WARNINGS=1;ZSTD_MULTITHREAD=1;WIN32;_DEBUG;_CONSOLE" MinimalRebuild="true" BasicRuntimeChecks="3" RuntimeLibrary="3" @@ -272,7 +272,7 @@ EnableIntrinsicFunctions="true" OmitFramePointers="true" AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\compress;$(SolutionDir)..\..\programs" - PreprocessorDefinitions="ZSTD_MULTITHREAD=1;WIN32;NDEBUG;_CONSOLE" + PreprocessorDefinitions="ZSTD_DISABLE_DEPRECATE_WARNINGS=1;ZSTD_MULTITHREAD=1;WIN32;NDEBUG;_CONSOLE" RuntimeLibrary="0" EnableFunctionLevelLinking="true" UsePrecompiledHeader="0" diff --git a/build/VS2010/fullbench-dll/fullbench-dll.vcxproj b/build/VS2010/fullbench-dll/fullbench-dll.vcxproj index befdc044..74336c2d 100644 --- a/build/VS2010/fullbench-dll/fullbench-dll.vcxproj +++ b/build/VS2010/fullbench-dll/fullbench-dll.vcxproj @@ -90,7 +90,7 @@ Level4 Disabled - WIN32;_DEBUG;_CONSOLE;ZSTD_DLL_IMPORT=1;%(PreprocessorDefinitions) + ZSTD_DISABLE_DEPRECATE_WARNINGS=1;WIN32;_DEBUG;_CONSOLE;ZSTD_DLL_IMPORT=1;%(PreprocessorDefinitions) true false @@ -108,7 +108,7 @@ Level4 Disabled - WIN32;_DEBUG;_CONSOLE;ZSTD_DLL_IMPORT=1;%(PreprocessorDefinitions) + ZSTD_DISABLE_DEPRECATE_WARNINGS=1;WIN32;_DEBUG;_CONSOLE;ZSTD_DLL_IMPORT=1;%(PreprocessorDefinitions) true false @@ -127,7 +127,7 @@ MaxSpeed true true - WIN32;_DEBUG;_CONSOLE;ZSTD_DLL_IMPORT=1;%(PreprocessorDefinitions) + ZSTD_DISABLE_DEPRECATE_WARNINGS=1;WIN32;_DEBUG;_CONSOLE;ZSTD_DLL_IMPORT=1;%(PreprocessorDefinitions) false false MultiThreaded @@ -150,7 +150,7 @@ MaxSpeed true true - WIN32;_DEBUG;_CONSOLE;ZSTD_DLL_IMPORT=1;%(PreprocessorDefinitions) + ZSTD_DISABLE_DEPRECATE_WARNINGS=1;WIN32;_DEBUG;_CONSOLE;ZSTD_DLL_IMPORT=1;%(PreprocessorDefinitions) false false MultiThreaded diff --git a/build/VS2010/fullbench/fullbench.vcxproj b/build/VS2010/fullbench/fullbench.vcxproj index 2e0a042b..e2d7b0de 100644 --- a/build/VS2010/fullbench/fullbench.vcxproj +++ b/build/VS2010/fullbench/fullbench.vcxproj @@ -90,7 +90,7 @@ Level4 Disabled - WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + ZSTD_DISABLE_DEPRECATE_WARNINGS=1;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) true false @@ -105,7 +105,7 @@ Level4 Disabled - WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + ZSTD_DISABLE_DEPRECATE_WARNINGS=1;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) true false @@ -122,7 +122,7 @@ MaxSpeed true true - WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + ZSTD_DISABLE_DEPRECATE_WARNINGS=1;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) false false MultiThreaded @@ -142,7 +142,7 @@ MaxSpeed true true - WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + ZSTD_DISABLE_DEPRECATE_WARNINGS=1;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) false false MultiThreaded diff --git a/build/VS2010/fuzzer/fuzzer.vcxproj b/build/VS2010/fuzzer/fuzzer.vcxproj index 91974ec7..c1d31cf5 100644 --- a/build/VS2010/fuzzer/fuzzer.vcxproj +++ b/build/VS2010/fuzzer/fuzzer.vcxproj @@ -90,7 +90,7 @@ Level4 Disabled - ZSTD_MULTITHREAD=1;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + ZSTD_DISABLE_DEPRECATE_WARNINGS=1;ZSTD_MULTITHREAD=1;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) true false @@ -105,7 +105,7 @@ Level4 Disabled - ZSTD_MULTITHREAD=1;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + ZSTD_DISABLE_DEPRECATE_WARNINGS=1;ZSTD_MULTITHREAD=1;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) true false @@ -122,7 +122,7 @@ MaxSpeed true true - ZSTD_MULTITHREAD=1;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + ZSTD_DISABLE_DEPRECATE_WARNINGS=1;ZSTD_MULTITHREAD=1;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) false false MultiThreaded @@ -142,7 +142,7 @@ MaxSpeed true true - ZSTD_MULTITHREAD=1;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + ZSTD_DISABLE_DEPRECATE_WARNINGS=1;ZSTD_MULTITHREAD=1;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) false false MultiThreaded diff --git a/build/cmake/CMakeModules/AddZstdCompilationFlags.cmake b/build/cmake/CMakeModules/AddZstdCompilationFlags.cmake index 62389717..e23b9d60 100644 --- a/build/cmake/CMakeModules/AddZstdCompilationFlags.cmake +++ b/build/cmake/CMakeModules/AddZstdCompilationFlags.cmake @@ -26,7 +26,12 @@ macro(ADD_ZSTD_COMPILATION_FLAGS) EnableCompilerFlag("-std=c++11" false true) #Set c99 by default EnableCompilerFlag("-std=c99" true false) - EnableCompilerFlag("-Wall" true true) + if (CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND MSVC) + # clang-cl normally maps -Wall to -Weverything. + EnableCompilerFlag("/clang:-Wall" true true) + else () + EnableCompilerFlag("-Wall" true true) + endif () EnableCompilerFlag("-Wextra" true true) EnableCompilerFlag("-Wundef" true true) EnableCompilerFlag("-Wshadow" true true) diff --git a/build/cmake/lib/CMakeLists.txt b/build/cmake/lib/CMakeLists.txt index d58c652a..5f756652 100644 --- a/build/cmake/lib/CMakeLists.txt +++ b/build/cmake/lib/CMakeLists.txt @@ -24,28 +24,24 @@ file(GLOB CommonSources ${LIBRARY_DIR}/common/*.c) file(GLOB CompressSources ${LIBRARY_DIR}/compress/*.c) file(GLOB DecompressSources ${LIBRARY_DIR}/decompress/*.c) file(GLOB DictBuilderSources ${LIBRARY_DIR}/dictBuilder/*.c) -file(GLOB DeprecatedSources ${LIBRARY_DIR}/deprecated/*.c) set(Sources ${CommonSources} ${CompressSources} ${DecompressSources} - ${DictBuilderSources} - ${DeprecatedSources}) + ${DictBuilderSources}) file(GLOB CommonHeaders ${LIBRARY_DIR}/common/*.h) file(GLOB CompressHeaders ${LIBRARY_DIR}/compress/*.h) file(GLOB DecompressHeaders ${LIBRARY_DIR}/decompress/*.h) file(GLOB DictBuilderHeaders ${LIBRARY_DIR}/dictBuilder/*.h) -file(GLOB DeprecatedHeaders ${LIBRARY_DIR}/deprecated/*.h) set(Headers ${LIBRARY_DIR}/zstd.h ${CommonHeaders} ${CompressHeaders} ${DecompressHeaders} - ${DictBuilderHeaders} - ${DeprecatedHeaders}) + ${DictBuilderHeaders}) if (ZSTD_LEGACY_SUPPORT) set(LIBRARY_LEGACY_DIR ${LIBRARY_DIR}/legacy) diff --git a/build/cmake/tests/CMakeLists.txt b/build/cmake/tests/CMakeLists.txt index fe095eb6..27acd41b 100644 --- a/build/cmake/tests/CMakeLists.txt +++ b/build/cmake/tests/CMakeLists.txt @@ -57,6 +57,7 @@ target_link_libraries(datagen libzstd_static) # fullbench # add_executable(fullbench ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${TESTS_DIR}/fullbench.c) +set_property(TARGET fullbench APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_DISABLE_DEPRECATE_WARNINGS") target_link_libraries(fullbench libzstd_static) add_test(NAME fullbench COMMAND fullbench ${ZSTD_FULLBENCH_FLAGS}) @@ -64,6 +65,7 @@ add_test(NAME fullbench COMMAND fullbench ${ZSTD_FULLBENCH_FLAGS}) # fuzzer # add_executable(fuzzer ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/fuzzer.c) +set_property(TARGET fuzzer APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_DISABLE_DEPRECATE_WARNINGS") target_link_libraries(fuzzer libzstd_static) AddTestFlagsOption(ZSTD_FUZZER_FLAGS "$ENV{FUZZERTEST} $ENV{FUZZER_FLAGS}" "Semicolon-separated list of flags to pass to the fuzzer test (see `fuzzer -h` for usage)") @@ -76,6 +78,7 @@ add_test(NAME fuzzer COMMAND fuzzer ${ZSTD_FUZZER_FLAGS}) # zstreamtest # add_executable(zstreamtest ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/seqgen.c ${TESTS_DIR}/zstreamtest.c) +set_property(TARGET zstreamtest APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_DISABLE_DEPRECATE_WARNINGS") target_link_libraries(zstreamtest libzstd_static) AddTestFlagsOption(ZSTD_ZSTREAM_FLAGS "$ENV{ZSTREAM_TESTTIME} $ENV{FUZZER_FLAGS}" "Semicolon-separated list of flags to pass to the zstreamtest test (see `zstreamtest -h` for usage)") diff --git a/build/meson/lib/meson.build b/build/meson/lib/meson.build index 2cbd39cf..5cc9fee8 100644 --- a/build/meson/lib/meson.build +++ b/build/meson/lib/meson.build @@ -14,8 +14,7 @@ libzstd_includes = [include_directories(join_paths(zstd_rootdir,'lib'), join_paths(zstd_rootdir, 'lib/common'), join_paths(zstd_rootdir, 'lib/compress'), join_paths(zstd_rootdir, 'lib/decompress'), - join_paths(zstd_rootdir, 'lib/dictBuilder'), - join_paths(zstd_rootdir, 'lib/deprecated'))] + join_paths(zstd_rootdir, 'lib/dictBuilder'))] libzstd_sources = [join_paths(zstd_rootdir, 'lib/common/entropy_common.c'), join_paths(zstd_rootdir, 'lib/common/fse_decompress.c'), diff --git a/contrib/single_file_libs/.gitignore b/build/single_file_libs/.gitignore similarity index 100% rename from contrib/single_file_libs/.gitignore rename to build/single_file_libs/.gitignore diff --git a/contrib/single_file_libs/README.md b/build/single_file_libs/README.md similarity index 96% rename from contrib/single_file_libs/README.md rename to build/single_file_libs/README.md index d88e8fc6..1705b769 100644 --- a/contrib/single_file_libs/README.md +++ b/build/single_file_libs/README.md @@ -11,7 +11,7 @@ This is the most common use case. The decompression library is small, adding, fo Create `zstddeclib.c` from the Zstd source using: ``` -cd zstd/contrib/single_file_libs +cd zstd/build/single_file_libs ./combine.sh -r ../../lib -o zstddeclib.c zstddeclib-in.c ``` Then add the resulting file to your project (see the [example files](examples)). @@ -25,7 +25,7 @@ The same tool can amalgamate the entire Zstd library for ease of adding both com Create `zstd.c` from the Zstd source using: ``` -cd zstd/contrib/single_file_libs +cd zstd/build/single_file_libs ./combine.sh -r ../../lib -o zstd.c zstd-in.c ``` It's possible to create a compressor-only library but since the decompressor is so small in comparison this doesn't bring much of a gain (but for the curious, simply remove the files in the _decompress_ section at the end of `zstd-in.c`). diff --git a/contrib/single_file_libs/build_decoder_test.sh b/build/single_file_libs/build_decoder_test.sh similarity index 100% rename from contrib/single_file_libs/build_decoder_test.sh rename to build/single_file_libs/build_decoder_test.sh diff --git a/contrib/single_file_libs/build_library_test.sh b/build/single_file_libs/build_library_test.sh similarity index 100% rename from contrib/single_file_libs/build_library_test.sh rename to build/single_file_libs/build_library_test.sh diff --git a/contrib/single_file_libs/combine.sh b/build/single_file_libs/combine.sh similarity index 100% rename from contrib/single_file_libs/combine.sh rename to build/single_file_libs/combine.sh diff --git a/contrib/single_file_libs/create_single_file_decoder.sh b/build/single_file_libs/create_single_file_decoder.sh similarity index 100% rename from contrib/single_file_libs/create_single_file_decoder.sh rename to build/single_file_libs/create_single_file_decoder.sh diff --git a/contrib/single_file_libs/create_single_file_library.sh b/build/single_file_libs/create_single_file_library.sh similarity index 100% rename from contrib/single_file_libs/create_single_file_library.sh rename to build/single_file_libs/create_single_file_library.sh diff --git a/contrib/single_file_libs/examples/README.md b/build/single_file_libs/examples/README.md similarity index 100% rename from contrib/single_file_libs/examples/README.md rename to build/single_file_libs/examples/README.md diff --git a/contrib/single_file_libs/examples/emscripten.c b/build/single_file_libs/examples/emscripten.c similarity index 100% rename from contrib/single_file_libs/examples/emscripten.c rename to build/single_file_libs/examples/emscripten.c diff --git a/contrib/single_file_libs/examples/roundtrip.c b/build/single_file_libs/examples/roundtrip.c similarity index 100% rename from contrib/single_file_libs/examples/roundtrip.c rename to build/single_file_libs/examples/roundtrip.c diff --git a/contrib/single_file_libs/examples/shell.html b/build/single_file_libs/examples/shell.html similarity index 100% rename from contrib/single_file_libs/examples/shell.html rename to build/single_file_libs/examples/shell.html diff --git a/contrib/single_file_libs/examples/simple.c b/build/single_file_libs/examples/simple.c similarity index 100% rename from contrib/single_file_libs/examples/simple.c rename to build/single_file_libs/examples/simple.c diff --git a/contrib/single_file_libs/examples/testcard-dxt1.inl b/build/single_file_libs/examples/testcard-dxt1.inl similarity index 100% rename from contrib/single_file_libs/examples/testcard-dxt1.inl rename to build/single_file_libs/examples/testcard-dxt1.inl diff --git a/contrib/single_file_libs/examples/testcard-zstd.inl b/build/single_file_libs/examples/testcard-zstd.inl similarity index 100% rename from contrib/single_file_libs/examples/testcard-zstd.inl rename to build/single_file_libs/examples/testcard-zstd.inl diff --git a/contrib/single_file_libs/examples/testcard.png b/build/single_file_libs/examples/testcard.png similarity index 100% rename from contrib/single_file_libs/examples/testcard.png rename to build/single_file_libs/examples/testcard.png diff --git a/contrib/single_file_libs/zstd-in.c b/build/single_file_libs/zstd-in.c similarity index 100% rename from contrib/single_file_libs/zstd-in.c rename to build/single_file_libs/zstd-in.c diff --git a/contrib/single_file_libs/zstddeclib-in.c b/build/single_file_libs/zstddeclib-in.c similarity index 100% rename from contrib/single_file_libs/zstddeclib-in.c rename to build/single_file_libs/zstddeclib-in.c diff --git a/contrib/VS2005/zstdlib/zstdlib.vcproj b/contrib/VS2005/zstdlib/zstdlib.vcproj index e028bd67..67ddd2db 100644 --- a/contrib/VS2005/zstdlib/zstdlib.vcproj +++ b/contrib/VS2005/zstdlib/zstdlib.vcproj @@ -473,10 +473,6 @@ RelativePath="..\..\..\lib\common\xxhash.h" > - - diff --git a/contrib/linux-kernel/test/Makefile b/contrib/linux-kernel/test/Makefile index 9c77f067..2908839f 100644 --- a/contrib/linux-kernel/test/Makefile +++ b/contrib/linux-kernel/test/Makefile @@ -11,7 +11,7 @@ LINUX := ../linux LINUX_ZSTDLIB := $(LINUX)/lib/zstd -CPPFLAGS += -I$(LINUX)/include -I$(LINUX_ZSTDLIB) -Iinclude -DNDEBUG +CPPFLAGS += -I$(LINUX)/include -I$(LINUX_ZSTDLIB) -Iinclude -DNDEBUG -Wno-deprecated-declarations # Don't poison the workspace, it currently doesn't work with static allocation and workspace reuse CPPFLAGS += -DZSTD_ASAN_DONT_POISON_WORKSPACE diff --git a/contrib/pzstd/Makefile b/contrib/pzstd/Makefile index 8d2b1932..25265e79 100644 --- a/contrib/pzstd/Makefile +++ b/contrib/pzstd/Makefile @@ -30,6 +30,9 @@ CXXFLAGS ?= -O3 -Wall -Wextra -pedantic CPPFLAGS ?= LDFLAGS ?= +# PZstd uses legacy APIs +CFLAGS += -Wno-deprecated-declarations + # Include flags PZSTD_INC = -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(PROGDIR) -I. GTEST_INC = -isystem googletest/googletest/include diff --git a/contrib/seekable_format/examples/Makefile b/contrib/seekable_format/examples/Makefile index 543780f7..9df6b75f 100644 --- a/contrib/seekable_format/examples/Makefile +++ b/contrib/seekable_format/examples/Makefile @@ -13,7 +13,7 @@ ZSTDLIB_PATH = ../../../lib ZSTDLIB_NAME = libzstd.a ZSTDLIB = $(ZSTDLIB_PATH)/$(ZSTDLIB_NAME) -CPPFLAGS += -I../ -I../../../lib -I../../../lib/common +CPPFLAGS += -DXXH_NAMESPACE=ZSTD_ -I../ -I../../../lib -I../../../lib/common CFLAGS ?= -O3 CFLAGS += -g diff --git a/contrib/seekable_format/examples/parallel_compression.c b/contrib/seekable_format/examples/parallel_compression.c index 69644d2b..4118b0ad 100644 --- a/contrib/seekable_format/examples/parallel_compression.c +++ b/contrib/seekable_format/examples/parallel_compression.c @@ -21,7 +21,6 @@ # define SLEEP(x) usleep(x * 1000) #endif -#define XXH_NAMESPACE ZSTD_ #include "xxhash.h" #include "pool.h" // use zstd thread pool for demo diff --git a/contrib/seekable_format/tests/Makefile b/contrib/seekable_format/tests/Makefile index 15eadb40..d51deb3e 100644 --- a/contrib/seekable_format/tests/Makefile +++ b/contrib/seekable_format/tests/Makefile @@ -13,7 +13,7 @@ ZSTDLIB_PATH = ../../../lib ZSTDLIB_NAME = libzstd.a ZSTDLIB = $(ZSTDLIB_PATH)/$(ZSTDLIB_NAME) -CPPFLAGS += -I../ -I$(ZSTDLIB_PATH) -I$(ZSTDLIB_PATH)/common +CPPFLAGS += -DXXH_NAMESPACE=ZSTD_ -I../ -I$(ZSTDLIB_PATH) -I$(ZSTDLIB_PATH)/common CFLAGS ?= -O3 CFLAGS += -g -Wall -Wextra -Wcast-qual -Wcast-align -Wconversion \ diff --git a/contrib/seekable_format/zstdseek_compress.c b/contrib/seekable_format/zstdseek_compress.c index d92917a6..242bd2ac 100644 --- a/contrib/seekable_format/zstdseek_compress.c +++ b/contrib/seekable_format/zstdseek_compress.c @@ -12,7 +12,6 @@ #include #define XXH_STATIC_LINKING_ONLY -#define XXH_NAMESPACE ZSTD_ #include "xxhash.h" #define ZSTD_STATIC_LINKING_ONLY @@ -83,7 +82,7 @@ static size_t ZSTD_seekable_frameLog_freeVec(ZSTD_frameLog* fl) ZSTD_frameLog* ZSTD_seekable_createFrameLog(int checksumFlag) { - ZSTD_frameLog* const fl = malloc(sizeof(ZSTD_frameLog)); + ZSTD_frameLog* const fl = (ZSTD_frameLog*)malloc(sizeof(ZSTD_frameLog)); if (fl == NULL) return NULL; if (ZSTD_isError(ZSTD_seekable_frameLog_allocVec(fl))) { @@ -108,7 +107,7 @@ size_t ZSTD_seekable_freeFrameLog(ZSTD_frameLog* fl) ZSTD_seekable_CStream* ZSTD_seekable_createCStream(void) { - ZSTD_seekable_CStream* const zcs = malloc(sizeof(ZSTD_seekable_CStream)); + ZSTD_seekable_CStream* const zcs = (ZSTD_seekable_CStream*)malloc(sizeof(ZSTD_seekable_CStream)); if (zcs == NULL) return NULL; memset(zcs, 0, sizeof(*zcs)); @@ -177,7 +176,7 @@ size_t ZSTD_seekable_logFrame(ZSTD_frameLog* fl, if (fl->size == fl->capacity) { /* exponential size increase for constant amortized runtime */ size_t const newCapacity = fl->capacity * 2; - framelogEntry_t* const newEntries = realloc(fl->entries, + framelogEntry_t* const newEntries = (framelogEntry_t*)realloc(fl->entries, sizeof(framelogEntry_t) * newCapacity); if (newEntries == NULL) return ERROR(memory_allocation); diff --git a/contrib/seekable_format/zstdseek_decompress.c b/contrib/seekable_format/zstdseek_decompress.c index ecf816c1..5eed0249 100644 --- a/contrib/seekable_format/zstdseek_decompress.c +++ b/contrib/seekable_format/zstdseek_decompress.c @@ -60,7 +60,6 @@ #include #define XXH_STATIC_LINKING_ONLY -#define XXH_NAMESPACE ZSTD_ #include "xxhash.h" #define ZSTD_STATIC_LINKING_ONLY @@ -176,7 +175,7 @@ struct ZSTD_seekable_s { ZSTD_seekable* ZSTD_seekable_create(void) { - ZSTD_seekable* const zs = malloc(sizeof(ZSTD_seekable)); + ZSTD_seekable* const zs = (ZSTD_seekable*)malloc(sizeof(ZSTD_seekable)); if (zs == NULL) return NULL; /* also initializes stage to zsds_init */ @@ -202,7 +201,7 @@ size_t ZSTD_seekable_free(ZSTD_seekable* zs) ZSTD_seekTable* ZSTD_seekTable_create_fromSeekable(const ZSTD_seekable* zs) { - ZSTD_seekTable* const st = malloc(sizeof(ZSTD_seekTable)); + ZSTD_seekTable* const st = (ZSTD_seekTable*)malloc(sizeof(ZSTD_seekTable)); if (st==NULL) return NULL; st->checksumFlag = zs->seekTable.checksumFlag; diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html index 4371156e..f961d4c3 100644 --- a/doc/zstd_manual.html +++ b/doc/zstd_manual.html @@ -1611,6 +1611,10 @@ ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + Note: ZSTD_resetCStream() interprets pledgedSrcSize == 0 as ZSTD_CONTENTSIZE_UNKNOWN, but + ZSTD_CCtx_setPledgedSrcSize() does not do the same, so ZSTD_CONTENTSIZE_UNKNOWN must be + explicitly specified. + start a new frame, using same parameters from previous frame. This is typically useful to skip dictionary loading stage, since it will re-use it in-place. Note that zcs must be init at least once before using ZSTD_resetCStream(). diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 6e18b68e..15a5c7c8 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -559,6 +559,11 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param) bounds.upperBound = (int)ZSTD_urm_enableRowMatchFinder; return bounds; + case ZSTD_c_deterministicRefPrefix: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + default: bounds.error = ERROR(parameter_unsupported); return bounds; @@ -622,6 +627,7 @@ static int ZSTD_isUpdateAuthorized(ZSTD_cParameter param) case ZSTD_c_validateSequences: case ZSTD_c_splitBlocks: case ZSTD_c_useRowMatchFinder: + case ZSTD_c_deterministicRefPrefix: default: return 0; } @@ -676,6 +682,7 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value) case ZSTD_c_validateSequences: case ZSTD_c_splitBlocks: case ZSTD_c_useRowMatchFinder: + case ZSTD_c_deterministicRefPrefix: break; default: RETURN_ERROR(parameter_unsupported, "unknown parameter"); @@ -897,6 +904,11 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams, CCtxParams->useRowMatchFinder = (ZSTD_useRowMatchFinderMode_e)value; return CCtxParams->useRowMatchFinder; + case ZSTD_c_deterministicRefPrefix: + BOUNDCHECK(ZSTD_c_deterministicRefPrefix, value); + CCtxParams->deterministicRefPrefix = !!value; + return CCtxParams->deterministicRefPrefix; + default: RETURN_ERROR(parameter_unsupported, "unknown parameter"); } } @@ -1026,6 +1038,9 @@ size_t ZSTD_CCtxParams_getParameter( case ZSTD_c_useRowMatchFinder : *value = (int)CCtxParams->useRowMatchFinder; break; + case ZSTD_c_deterministicRefPrefix: + *value = (int)CCtxParams->deterministicRefPrefix; + break; default: RETURN_ERROR(parameter_unsupported, "unknown parameter"); } return 0; @@ -1802,7 +1817,7 @@ static int ZSTD_dictTooBig(size_t const loadedDictSize) * note : `params` are assumed fully validated at this stage. */ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, - ZSTD_CCtx_params params, + ZSTD_CCtx_params const* params, U64 const pledgedSrcSize, size_t const loadedDictSize, ZSTD_compResetPolicy_e const crp, @@ -1810,30 +1825,36 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, { ZSTD_cwksp* const ws = &zc->workspace; DEBUGLOG(4, "ZSTD_resetCCtx_internal: pledgedSrcSize=%u, wlog=%u, useRowMatchFinder=%d", - (U32)pledgedSrcSize, params.cParams.windowLog, (int)params.useRowMatchFinder); - assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); + (U32)pledgedSrcSize, params->cParams.windowLog, (int)params->useRowMatchFinder); + assert(!ZSTD_isError(ZSTD_checkCParams(params->cParams))); zc->isFirstBlock = 1; - assert(params.useRowMatchFinder != ZSTD_urm_auto); - if (params.ldmParams.enableLdm) { + /* Set applied params early so we can modify them for LDM, + * and point params at the applied params. + */ + zc->appliedParams = *params; + params = &zc->appliedParams; + + assert(params->useRowMatchFinder != ZSTD_urm_auto); + if (params->ldmParams.enableLdm) { /* Adjust long distance matching parameters */ - ZSTD_ldm_adjustParameters(¶ms.ldmParams, ¶ms.cParams); - assert(params.ldmParams.hashLog >= params.ldmParams.bucketSizeLog); - assert(params.ldmParams.hashRateLog < 32); + ZSTD_ldm_adjustParameters(&zc->appliedParams.ldmParams, ¶ms->cParams); + assert(params->ldmParams.hashLog >= params->ldmParams.bucketSizeLog); + assert(params->ldmParams.hashRateLog < 32); } - { size_t const windowSize = MAX(1, (size_t)MIN(((U64)1 << params.cParams.windowLog), pledgedSrcSize)); + { size_t const windowSize = MAX(1, (size_t)MIN(((U64)1 << params->cParams.windowLog), pledgedSrcSize)); size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize); - U32 const divider = (params.cParams.minMatch==3) ? 3 : 4; + U32 const divider = (params->cParams.minMatch==3) ? 3 : 4; size_t const maxNbSeq = blockSize / divider; - size_t const buffOutSize = (zbuff == ZSTDb_buffered && params.outBufferMode == ZSTD_bm_buffered) + size_t const buffOutSize = (zbuff == ZSTDb_buffered && params->outBufferMode == ZSTD_bm_buffered) ? ZSTD_compressBound(blockSize) + 1 : 0; - size_t const buffInSize = (zbuff == ZSTDb_buffered && params.inBufferMode == ZSTD_bm_buffered) + size_t const buffInSize = (zbuff == ZSTDb_buffered && params->inBufferMode == ZSTD_bm_buffered) ? windowSize + blockSize : 0; - size_t const maxNbLdmSeq = ZSTD_ldm_getMaxNbSeq(params.ldmParams, blockSize); + size_t const maxNbLdmSeq = ZSTD_ldm_getMaxNbSeq(params->ldmParams, blockSize); int const indexTooClose = ZSTD_indexTooCloseToMax(zc->blockState.matchState.window); int const dictTooBig = ZSTD_dictTooBig(loadedDictSize); @@ -1842,7 +1863,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, size_t const neededSpace = ZSTD_estimateCCtxSize_usingCCtxParams_internal( - ¶ms.cParams, ¶ms.ldmParams, zc->staticSize != 0, params.useRowMatchFinder, + ¶ms->cParams, ¶ms->ldmParams, zc->staticSize != 0, params->useRowMatchFinder, buffInSize, buffOutSize, pledgedSrcSize); int resizeWorkspace; @@ -1885,8 +1906,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, ZSTD_cwksp_clear(ws); /* init params */ - zc->appliedParams = params; - zc->blockState.matchState.cParams = params.cParams; + zc->blockState.matchState.cParams = params->cParams; zc->pledgedSrcSizePlusOne = pledgedSrcSize+1; zc->consumedSrcSize = 0; zc->producedCSize = 0; @@ -1917,11 +1937,11 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, zc->outBuff = (char*)ZSTD_cwksp_reserve_buffer(ws, buffOutSize); /* ldm bucketOffsets table */ - if (params.ldmParams.enableLdm) { + if (params->ldmParams.enableLdm) { /* TODO: avoid memset? */ size_t const numBuckets = - ((size_t)1) << (params.ldmParams.hashLog - - params.ldmParams.bucketSizeLog); + ((size_t)1) << (params->ldmParams.hashLog - + params->ldmParams.bucketSizeLog); zc->ldmState.bucketOffsets = ZSTD_cwksp_reserve_buffer(ws, numBuckets); ZSTD_memset(zc->ldmState.bucketOffsets, 0, numBuckets); } @@ -1937,16 +1957,16 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, FORWARD_IF_ERROR(ZSTD_reset_matchState( &zc->blockState.matchState, ws, - ¶ms.cParams, - params.useRowMatchFinder, + ¶ms->cParams, + params->useRowMatchFinder, crp, needsIndexReset, ZSTD_resetTarget_CCtx), ""); /* ldm hash table */ - if (params.ldmParams.enableLdm) { + if (params->ldmParams.enableLdm) { /* TODO: avoid memset? */ - size_t const ldmHSize = ((size_t)1) << params.ldmParams.hashLog; + size_t const ldmHSize = ((size_t)1) << params->ldmParams.hashLog; zc->ldmState.hashTable = (ldmEntry_t*)ZSTD_cwksp_reserve_aligned(ws, ldmHSize * sizeof(ldmEntry_t)); ZSTD_memset(zc->ldmState.hashTable, 0, ldmHSize * sizeof(ldmEntry_t)); zc->ldmSequences = (rawSeq*)ZSTD_cwksp_reserve_aligned(ws, maxNbLdmSeq * sizeof(rawSeq)); @@ -2014,7 +2034,8 @@ ZSTD_resetCCtx_byAttachingCDict(ZSTD_CCtx* cctx, U64 pledgedSrcSize, ZSTD_buffered_policy_e zbuff) { - DEBUGLOG(4, "ZSTD_resetCCtx_byAttachingCDict() pledgedSrcSize=%zu", pledgedSrcSize); + DEBUGLOG(4, "ZSTD_resetCCtx_byAttachingCDict() pledgedSrcSize=%llu", + (unsigned long long)pledgedSrcSize); { ZSTD_compressionParameters adjusted_cdict_cParams = cdict->matchState.cParams; unsigned const windowLog = params.cParams.windowLog; @@ -2031,7 +2052,7 @@ ZSTD_resetCCtx_byAttachingCDict(ZSTD_CCtx* cctx, cdict->dictContentSize, ZSTD_cpm_attachDict); params.cParams.windowLog = windowLog; params.useRowMatchFinder = cdict->useRowMatchFinder; /* cdict overrides */ - FORWARD_IF_ERROR(ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, + FORWARD_IF_ERROR(ZSTD_resetCCtx_internal(cctx, ¶ms, pledgedSrcSize, /* loadedDictSize */ 0, ZSTDcrp_makeClean, zbuff), ""); assert(cctx->appliedParams.cParams.strategy == adjusted_cdict_cParams.strategy); @@ -2076,7 +2097,8 @@ static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx, const ZSTD_compressionParameters *cdict_cParams = &cdict->matchState.cParams; assert(!cdict->matchState.dedicatedDictSearch); - DEBUGLOG(4, "ZSTD_resetCCtx_byCopyingCDict() pledgedSrcSize=%zu", pledgedSrcSize); + DEBUGLOG(4, "ZSTD_resetCCtx_byCopyingCDict() pledgedSrcSize=%llu", + (unsigned long long)pledgedSrcSize); { unsigned const windowLog = params.cParams.windowLog; assert(windowLog != 0); @@ -2084,7 +2106,7 @@ static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx, params.cParams = *cdict_cParams; params.cParams.windowLog = windowLog; params.useRowMatchFinder = cdict->useRowMatchFinder; - FORWARD_IF_ERROR(ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, + FORWARD_IF_ERROR(ZSTD_resetCCtx_internal(cctx, ¶ms, pledgedSrcSize, /* loadedDictSize */ 0, ZSTDcrp_leaveDirty, zbuff), ""); assert(cctx->appliedParams.cParams.strategy == cdict_cParams->strategy); @@ -2190,7 +2212,7 @@ static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx, assert(srcCCtx->appliedParams.useRowMatchFinder != ZSTD_urm_auto); params.useRowMatchFinder = srcCCtx->appliedParams.useRowMatchFinder; params.fParams = fParams; - ZSTD_resetCCtx_internal(dstCCtx, params, pledgedSrcSize, + ZSTD_resetCCtx_internal(dstCCtx, ¶ms, pledgedSrcSize, /* loadedDictSize */ 0, ZSTDcrp_leaveDirty, zbuff); assert(dstCCtx->appliedParams.cParams.windowLog == srcCCtx->appliedParams.cParams.windowLog); @@ -4045,11 +4067,12 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx, if (!srcSize) return fhSize; /* do not generate an empty block if no input */ - if (!ZSTD_window_update(&ms->window, src, srcSize)) { + if (!ZSTD_window_update(&ms->window, src, srcSize, ms->forceNonContiguous)) { + ms->forceNonContiguous = 0; ms->nextToUpdate = ms->window.dictLimit; } if (cctx->appliedParams.ldmParams.enableLdm) { - ZSTD_window_update(&cctx->ldmState.window, src, srcSize); + ZSTD_window_update(&cctx->ldmState.window, src, srcSize, /* forceNonContiguous */ 0); } if (!frame) { @@ -4141,11 +4164,12 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms, } DEBUGLOG(4, "ZSTD_loadDictionaryContent(): useRowMatchFinder=%d", (int)params->useRowMatchFinder); - ZSTD_window_update(&ms->window, src, srcSize); + ZSTD_window_update(&ms->window, src, srcSize, /* forceNonContiguous */ 0); ms->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ms->window.base); + ms->forceNonContiguous = params->deterministicRefPrefix; if (loadLdmDict) { - ZSTD_window_update(&ls->window, src, srcSize); + ZSTD_window_update(&ls->window, src, srcSize, /* forceNonContiguous */ 0); ls->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ls->window.base); } @@ -4422,7 +4446,7 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, return ZSTD_resetCCtx_usingCDict(cctx, cdict, params, pledgedSrcSize, zbuff); } - FORWARD_IF_ERROR( ZSTD_resetCCtx_internal(cctx, *params, pledgedSrcSize, + FORWARD_IF_ERROR( ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, dictContentSize, ZSTDcrp_makeClean, zbuff) , ""); { size_t const dictID = cdict ? @@ -4591,15 +4615,14 @@ size_t ZSTD_compress_advanced (ZSTD_CCtx* cctx, const void* dict,size_t dictSize, ZSTD_parameters params) { - ZSTD_CCtx_params cctxParams; DEBUGLOG(4, "ZSTD_compress_advanced"); FORWARD_IF_ERROR(ZSTD_checkCParams(params.cParams), ""); - ZSTD_CCtxParams_init_internal(&cctxParams, ¶ms, ZSTD_NO_CLEVEL); + ZSTD_CCtxParams_init_internal(&cctx->simpleApiParams, ¶ms, ZSTD_NO_CLEVEL); return ZSTD_compress_advanced_internal(cctx, dst, dstCapacity, src, srcSize, dict, dictSize, - &cctxParams); + &cctx->simpleApiParams); } /* Internal */ @@ -4623,14 +4646,13 @@ size_t ZSTD_compress_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel) { - ZSTD_CCtx_params cctxParams; { ZSTD_parameters const params = ZSTD_getParams_internal(compressionLevel, srcSize, dict ? dictSize : 0, ZSTD_cpm_noAttachDict); assert(params.fParams.contentSizeFlag == 1); - ZSTD_CCtxParams_init_internal(&cctxParams, ¶ms, (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT: compressionLevel); + ZSTD_CCtxParams_init_internal(&cctx->simpleApiParams, ¶ms, (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT: compressionLevel); } DEBUGLOG(4, "ZSTD_compress_usingDict (srcSize=%u)", (unsigned)srcSize); - return ZSTD_compress_advanced_internal(cctx, dst, dstCapacity, src, srcSize, dict, dictSize, &cctxParams); + return ZSTD_compress_advanced_internal(cctx, dst, dstCapacity, src, srcSize, dict, dictSize, &cctx->simpleApiParams); } size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx, @@ -4963,15 +4985,15 @@ unsigned ZSTD_getDictID_fromCDict(const ZSTD_CDict* cdict) return cdict->dictID; } - -/* ZSTD_compressBegin_usingCDict_advanced() : - * cdict must be != NULL */ -size_t ZSTD_compressBegin_usingCDict_advanced( +/* ZSTD_compressBegin_usingCDict_internal() : + * Implementation of various ZSTD_compressBegin_usingCDict* functions. + */ +static size_t ZSTD_compressBegin_usingCDict_internal( ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize) { ZSTD_CCtx_params cctxParams; - DEBUGLOG(4, "ZSTD_compressBegin_usingCDict_advanced"); + DEBUGLOG(4, "ZSTD_compressBegin_usingCDict_internal"); RETURN_ERROR_IF(cdict==NULL, dictionary_wrong, "NULL pointer!"); /* Initialize the cctxParams from the cdict */ { @@ -5003,23 +5025,46 @@ size_t ZSTD_compressBegin_usingCDict_advanced( ZSTDb_not_buffered); } + +/* ZSTD_compressBegin_usingCDict_advanced() : + * This function is DEPRECATED. + * cdict must be != NULL */ +size_t ZSTD_compressBegin_usingCDict_advanced( + ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, + ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize) +{ + return ZSTD_compressBegin_usingCDict_internal(cctx, cdict, fParams, pledgedSrcSize); +} + /* ZSTD_compressBegin_usingCDict() : - * pledgedSrcSize=0 means "unknown" - * if pledgedSrcSize>0, it will enable contentSizeFlag */ + * cdict must be != NULL */ size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict) { ZSTD_frameParameters const fParams = { 0 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ }; - DEBUGLOG(4, "ZSTD_compressBegin_usingCDict : dictIDFlag == %u", !fParams.noDictIDFlag); - return ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, ZSTD_CONTENTSIZE_UNKNOWN); + return ZSTD_compressBegin_usingCDict_internal(cctx, cdict, fParams, ZSTD_CONTENTSIZE_UNKNOWN); } +/*! ZSTD_compress_usingCDict_internal(): + * Implementation of various ZSTD_compress_usingCDict* functions. + */ +static size_t ZSTD_compress_usingCDict_internal(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict, ZSTD_frameParameters fParams) +{ + FORWARD_IF_ERROR(ZSTD_compressBegin_usingCDict_internal(cctx, cdict, fParams, srcSize), ""); /* will check if cdict != NULL */ + return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize); +} + +/*! ZSTD_compress_usingCDict_advanced(): + * This function is DEPRECATED. + */ size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams) { - FORWARD_IF_ERROR(ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, srcSize), ""); /* will check if cdict != NULL */ - return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize); + return ZSTD_compress_usingCDict_internal(cctx, dst, dstCapacity, src, srcSize, cdict, fParams); } /*! ZSTD_compress_usingCDict() : @@ -5033,7 +5078,7 @@ size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict) { ZSTD_frameParameters const fParams = { 1 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ }; - return ZSTD_compress_usingCDict_advanced(cctx, dst, dstCapacity, src, srcSize, cdict, fParams); + return ZSTD_compress_usingCDict_internal(cctx, dst, dstCapacity, src, srcSize, cdict, fParams); } diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index c0b01794..03974ae3 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -219,6 +219,8 @@ struct ZSTD_matchState_t { U32* hashTable3; U32* chainTable; + U32 forceNonContiguous; /* Non-zero if we should force non-contiguous load for the next window update. */ + int dedicatedDictSearch; /* Indicates whether this matchState is using the * dedicated dictionary search structure. */ @@ -317,6 +319,9 @@ struct ZSTD_CCtx_params_s { /* Param for deciding whether to use row-based matchfinder */ ZSTD_useRowMatchFinderMode_e useRowMatchFinder; + /* Always load a dictionary in ext-dict mode (not prefix mode)? */ + int deterministicRefPrefix; + /* Internal use, for createCCtxParams() and freeCCtxParams() only */ ZSTD_customMem customMem; }; /* typedef'd to ZSTD_CCtx_params within "zstd.h" */ @@ -340,6 +345,7 @@ struct ZSTD_CCtx_s { int bmi2; /* == 1 if the CPU supports BMI2 and 0 otherwise. CPU support is determined dynamically once per context lifetime. */ ZSTD_CCtx_params requestedParams; ZSTD_CCtx_params appliedParams; + ZSTD_CCtx_params simpleApiParams; /* Param storage used by the simple API - not sticky. Must only be used in top-level simple API functions for storage. */ U32 dictID; size_t dictContentSize; @@ -1138,7 +1144,8 @@ MEM_STATIC void ZSTD_window_init(ZSTD_window_t* window) { * Returns non-zero if the segment is contiguous. */ MEM_STATIC U32 ZSTD_window_update(ZSTD_window_t* window, - void const* src, size_t srcSize) + void const* src, size_t srcSize, + int forceNonContiguous) { BYTE const* const ip = (BYTE const*)src; U32 contiguous = 1; @@ -1148,7 +1155,7 @@ MEM_STATIC U32 ZSTD_window_update(ZSTD_window_t* window, assert(window->base != NULL); assert(window->dictBase != NULL); /* Check if blocks follow each other */ - if (src != window->nextSrc) { + if (src != window->nextSrc || forceNonContiguous) { /* not contiguous */ size_t const distanceFromBase = (size_t)(window->nextSrc - window->base); DEBUGLOG(5, "Non contiguous blocks, new segment starts at %u", window->dictLimit); diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c index cfd75cc4..22aa3e12 100644 --- a/lib/compress/zstdmt_compress.c +++ b/lib/compress/zstdmt_compress.c @@ -512,7 +512,7 @@ ZSTDMT_serialState_reset(serialState_t* serialState, if (dictSize > 0) { if (dictContentType == ZSTD_dct_rawContent) { BYTE const* const dictEnd = (const BYTE*)dict + dictSize; - ZSTD_window_update(&serialState->ldmState.window, dict, dictSize); + ZSTD_window_update(&serialState->ldmState.window, dict, dictSize, /* forceNonContiguous */ 0); ZSTD_ldm_fillHashTable(&serialState->ldmState, (const BYTE*)dict, dictEnd, ¶ms.ldmParams); serialState->ldmState.loadedDictEnd = params.forceWindow ? 0 : (U32)(dictEnd - serialState->ldmState.window.base); } else { @@ -569,7 +569,7 @@ static void ZSTDMT_serialState_update(serialState_t* serialState, assert(seqStore.seq != NULL && seqStore.pos == 0 && seqStore.size == 0 && seqStore.capacity > 0); assert(src.size <= serialState->params.jobSize); - ZSTD_window_update(&serialState->ldmState.window, src.start, src.size); + ZSTD_window_update(&serialState->ldmState.window, src.start, src.size, /* forceNonContiguous */ 0); error = ZSTD_ldm_generateSequences( &serialState->ldmState, &seqStore, &serialState->params.ldmParams, src.start, src.size); @@ -695,6 +695,10 @@ static void ZSTDMT_compressionJob(void* jobDescription) { size_t const forceWindowError = ZSTD_CCtxParams_setParameter(&jobParams, ZSTD_c_forceMaxWindow, !job->firstJob); if (ZSTD_isError(forceWindowError)) JOB_ERROR(forceWindowError); } + if (!job->firstJob) { + size_t const err = ZSTD_CCtxParams_setParameter(&jobParams, ZSTD_c_deterministicRefPrefix, 0); + if (ZSTD_isError(err)) JOB_ERROR(err); + } { size_t const initError = ZSTD_compressBegin_advanced_internal(cctx, job->prefix.start, job->prefix.size, ZSTD_dct_rawContent, /* load dictionary in "content-only" mode (no header analysis) */ ZSTD_dtlm_fast, @@ -750,6 +754,12 @@ static void ZSTDMT_compressionJob(void* jobDescription) if (ZSTD_isError(cSize)) JOB_ERROR(cSize); lastCBlockSize = cSize; } } + if (!job->firstJob) { + /* Double check that we don't have an ext-dict, because then our + * repcode invalidation doesn't work. + */ + assert(!ZSTD_window_hasExtDict(cctx->blockState.matchState.window)); + } ZSTD_CCtx_trace(cctx, 0); _endJob: diff --git a/lib/zdict.h b/lib/zdict.h index 4b8589f2..75b05dbf 100644 --- a/lib/zdict.h +++ b/lib/zdict.h @@ -36,6 +36,145 @@ extern "C" { # define ZDICTLIB_API ZDICTLIB_VISIBILITY #endif +/******************************************************************************* + * Zstd dictionary builder + * + * FAQ + * === + * Why should I use a dictionary? + * ------------------------------ + * + * Zstd can use dictionaries to improve compression ratio of small data. + * Traditionally small files don't compress well because there is very little + * repetion in a single sample, since it is small. But, if you are compressing + * many similar files, like a bunch of JSON records that share the same + * structure, you can train a dictionary on ahead of time on some samples of + * these files. Then, zstd can use the dictionary to find repetitions that are + * present across samples. This can vastly improve compression ratio. + * + * When is a dictionary useful? + * ---------------------------- + * + * Dictionaries are useful when compressing many small files that are similar. + * The larger a file is, the less benefit a dictionary will have. Generally, + * we don't expect dictionary compression to be effective past 100KB. And the + * smaller a file is, the more we would expect the dictionary to help. + * + * How do I use a dictionary? + * -------------------------- + * + * Simply pass the dictionary to the zstd compressor with + * `ZSTD_CCtx_loadDictionary()`. The same dictionary must then be passed to + * the decompressor, using `ZSTD_DCtx_loadDictionary()`. There are other + * more advanced functions that allow selecting some options, see zstd.h for + * complete documentation. + * + * What is a zstd dictionary? + * -------------------------- + * + * A zstd dictionary has two pieces: Its header, and its content. The header + * contains a magic number, the dictionary ID, and entropy tables. These + * entropy tables allow zstd to save on header costs in the compressed file, + * which really matters for small data. The content is just bytes, which are + * repeated content that is common across many samples. + * + * What is a raw content dictionary? + * --------------------------------- + * + * A raw content dictionary is just bytes. It doesn't have a zstd dictionary + * header, a dictionary ID, or entropy tables. Any buffer is a valid raw + * content dictionary. + * + * How do I train a dictionary? + * ---------------------------- + * + * Gather samples from your use case. These samples should be similar to each + * other. If you have several use cases, you could try to train one dictionary + * per use case. + * + * Pass those samples to `ZDICT_trainFromBuffer()` and that will train your + * dictionary. There are a few advanced versions of this function, but this + * is a great starting point. If you want to further tune your dictionary + * you could try `ZDICT_optimizeTrainFromBuffer_cover()`. If that is too slow + * you can try `ZDICT_optimizeTrainFromBuffer_fastCover()`. + * + * If the dictionary training function fails, that is likely because you + * either passed too few samples, or a dictionary would not be effective + * for your data. Look at the messages that the dictionary trainer printed, + * if it doesn't say too few samples, then a dictionary would not be effective. + * + * How large should my dictionary be? + * ---------------------------------- + * + * A reasonable dictionary size, the `dictBufferCapacity`, is about 100KB. + * The zstd CLI defaults to a 110KB dictionary. You likely don't need a + * dictionary larger than that. But, most use cases can get away with a + * smaller dictionary. The advanced dictionary builders can automatically + * shrink the dictionary for you, and select a the smallest size that + * doesn't hurt compression ratio too much. See the `shrinkDict` parameter. + * A smaller dictionary can save memory, and potentially speed up + * compression. + * + * How many samples should I provide to the dictionary builder? + * ------------------------------------------------------------ + * + * We generally recommend passing ~100x the size of the dictionary + * in samples. A few thousand should suffice. Having too few samples + * can hurt the dictionaries effectiveness. Having more samples will + * only improve the dictionaries effectiveness. But having too many + * samples can slow down the dictionary builder. + * + * How do I determine if a dictionary will be effective? + * ----------------------------------------------------- + * + * Simply train a dictionary and try it out. You can use zstd's built in + * benchmarking tool to test the dictionary effectiveness. + * + * # Benchmark levels 1-3 without a dictionary + * zstd -b1e3 -r /path/to/my/files + * # Benchmark levels 1-3 with a dictioanry + * zstd -b1e3 -r /path/to/my/files -D /path/to/my/dictionary + * + * When should I retrain a dictionary? + * ----------------------------------- + * + * You should retrain a dictionary when its effectiveness drops. Dictionary + * effectiveness drops as the data you are compressing changes. Generally, we do + * expect dictionaries to "decay" over time, as your data changes, but the rate + * at which they decay depends on your use case. Internally, we regularly + * retrain dictionaries, and if the new dictionary performs significantly + * better than the old dictionary, we will ship the new dictionary. + * + * I have a raw content dictionary, how do I turn it into a zstd dictionary? + * ------------------------------------------------------------------------- + * + * If you have a raw content dictionary, e.g. by manually constructing it, or + * using a third-party dictionary builder, you can turn it into a zstd + * dictionary by using `ZDICT_finalizeDictionary()`. You'll also have to + * provide some samples of the data. It will add the zstd header to the + * raw content, which contains a dictionary ID and entropy tables, which + * will improve compression ratio, and allow zstd to write the dictionary ID + * into the frame, if you so choose. + * + * Do I have to use zstd's dictionary builder? + * ------------------------------------------- + * + * No! You can construct dictionary content however you please, it is just + * bytes. It will always be valid as a raw content dictionary. If you want + * a zstd dictionary, which can improve compression ratio, use + * `ZDICT_finalizeDictionary()`. + * + * What is the attack surface of a zstd dictionary? + * ------------------------------------------------ + * + * Zstd is heavily fuzz tested, including loading fuzzed dictionaries, so + * zstd should never crash, or access out-of-bounds memory no matter what + * the dictionary is. However, if an attacker can control the dictionary + * during decompression, they can cause zstd to generate arbitrary bytes, + * just like if they controlled the compressed data. + * + ******************************************************************************/ + /*! ZDICT_trainFromBuffer(): * Train a dictionary from an array of samples. @@ -64,7 +203,14 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCap typedef struct { int compressionLevel; /*< optimize for a specific zstd compression level; 0 means default */ unsigned notificationLevel; /*< Write log to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */ - unsigned dictID; /*< force dictID value; 0 means auto mode (32-bits random value) */ + unsigned dictID; /*< force dictID value; 0 means auto mode (32-bits random value) + * NOTE: The zstd format reserves some dictionary IDs for future use. + * You may use them in private settings, but be warned that they + * may be used by zstd in a public dictionary registry in the future. + * These dictionary IDs are: + * - low range : <= 32767 + * - high range : >= (2^31) + */ } ZDICT_params_t; /*! ZDICT_finalizeDictionary(): diff --git a/lib/zstd.h b/lib/zstd.h index 8f0ba268..fbc4a98d 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -449,7 +449,8 @@ typedef enum { ZSTD_c_experimentalParam11=1008, ZSTD_c_experimentalParam12=1009, ZSTD_c_experimentalParam13=1010, - ZSTD_c_experimentalParam14=1011 + ZSTD_c_experimentalParam14=1011, + ZSTD_c_experimentalParam15=1012 } ZSTD_cParameter; typedef struct { @@ -1085,6 +1086,28 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); #if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY) #define ZSTD_H_ZSTD_STATIC_LINKING_ONLY +/* Deprecation warnings : + * Should these warnings be a problem, it is generally possible to disable them, + * typically with -Wno-deprecated-declarations for gcc or _CRT_SECURE_NO_WARNINGS in Visual. + * Otherwise, it's also possible to define ZSTD_DISABLE_DEPRECATE_WARNINGS. + */ +#ifdef ZSTD_DISABLE_DEPRECATE_WARNINGS +# define ZSTD_DEPRECATED(message) ZSTDLIB_API /* disable deprecation warnings */ +#else +# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ +# define ZSTD_DEPRECATED(message) [[deprecated(message)]] ZSTDLIB_API +# elif (defined(GNUC) && (GNUC > 4 || (GNUC == 4 && GNUC_MINOR >= 5))) || defined(__clang__) +# define ZSTD_DEPRECATED(message) ZSTDLIB_API __attribute__((deprecated(message))) +# elif defined(__GNUC__) && (__GNUC__ >= 3) +# define ZSTD_DEPRECATED(message) ZSTDLIB_API __attribute__((deprecated)) +# elif defined(_MSC_VER) +# define ZSTD_DEPRECATED(message) ZSTDLIB_API __declspec(deprecated(message)) +# else +# pragma message("WARNING: You need to implement ZSTD_DEPRECATED for this compiler") +# define ZSTD_DEPRECATED(message) ZSTDLIB_API +# endif +#endif /* ZSTD_DISABLE_DEPRECATE_WARNINGS */ + /**************************************************************************************** * experimental API (static linking only) **************************************************************************************** @@ -1620,18 +1643,20 @@ ZSTDLIB_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParame /*! ZSTD_compress_advanced() : * Note : this function is now DEPRECATED. * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_setParameter() and other parameter setters. - * This prototype will be marked as deprecated and generate compilation warning on reaching v1.5.x */ -ZSTDLIB_API size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx, + * This prototype will generate compilation warnings. */ +ZSTD_DEPRECATED("use ZSTD_compress2") +size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const void* dict,size_t dictSize, ZSTD_parameters params); /*! ZSTD_compress_usingCDict_advanced() : - * Note : this function is now REDUNDANT. + * Note : this function is now DEPRECATED. * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_loadDictionary() and other parameter setters. - * This prototype will be marked as deprecated and generate compilation warning in some future version */ -ZSTDLIB_API size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx, + * This prototype will generate compilation warnings. */ +ZSTD_DEPRECATED("use ZSTD_compress2 with ZSTD_CCtx_loadDictionary") +size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const ZSTD_CDict* cdict, @@ -1859,6 +1884,26 @@ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* pre */ #define ZSTD_c_useRowMatchFinder ZSTD_c_experimentalParam14 +/* ZSTD_c_deterministicRefPrefix + * Default is 0 == disabled. Set to 1 to enable. + * + * Zstd produces different results for prefix compression when the prefix is + * directly adjacent to the data about to be compressed vs. when it isn't. + * This is because zstd detects that the two buffers are contiguous and it can + * use a more efficient match finding algorithm. However, this produces different + * results than when the two buffers are non-contiguous. This flag forces zstd + * to always load the prefix in non-contiguous mode, even if it happens to be + * adjacent to the data, to guarantee determinism. + * + * If you really care about determinism when using a dictionary or prefix, + * like when doing delta compression, you should select this option. It comes + * at a speed penalty of about ~2.5% if the dictionary and data happened to be + * contiguous, and is free if they weren't contiguous. We don't expect that + * intentionally making the dictionary and data contiguous will be worth the + * cost to memcpy() the data. + */ +#define ZSTD_c_deterministicRefPrefix ZSTD_c_experimentalParam15 + /*! ZSTD_CCtx_getParameter() : * Get the requested compression parameter value, selected by enum ZSTD_cParameter, * and store it into int* value. @@ -2070,11 +2115,13 @@ ZSTDLIB_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param /*! ZSTD_DCtx_setFormat() : + * This function is REDUNDANT. Prefer ZSTD_DCtx_setParameter(). * Instruct the decoder context about what kind of data to decode next. * This instruction is mandatory to decode data without a fully-formed header, * such ZSTD_f_zstd1_magicless for example. * @return : 0, or an error code (which can be tested using ZSTD_isError()). */ -ZSTDLIB_API size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format); +ZSTD_DEPRECATED("use ZSTD_DCtx_setParameter() instead") +size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format); /*! ZSTD_decompressStream_simpleArgs() : * Same as ZSTD_decompressStream(), @@ -2098,7 +2145,7 @@ ZSTDLIB_API size_t ZSTD_decompressStream_simpleArgs ( /*===== Advanced Streaming compression functions =====*/ /*! ZSTD_initCStream_srcSize() : - * This function is deprecated, and equivalent to: + * This function is DEPRECATED, and equivalent to: * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any) * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); @@ -2107,15 +2154,15 @@ ZSTDLIB_API size_t ZSTD_decompressStream_simpleArgs ( * pledgedSrcSize must be correct. If it is not known at init time, use * ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs, * "0" also disables frame content size field. It may be enabled in the future. - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x + * This prototype will generate compilation warnings. */ -ZSTDLIB_API size_t -ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, +ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); /*! ZSTD_initCStream_usingDict() : - * This function is deprecated, and is equivalent to: + * This function is DEPRECATED, and is equivalent to: * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize); @@ -2124,15 +2171,15 @@ ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, * dict == NULL or dictSize < 8, in which case no dict is used. * Note: dict is loaded with ZSTD_dct_auto (treated as a full zstd dictionary if * it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy. - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x + * This prototype will generate compilation warnings. */ -ZSTDLIB_API size_t -ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, +ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /*! ZSTD_initCStream_advanced() : - * This function is deprecated, and is approximately equivalent to: + * This function is DEPRECATED, and is approximately equivalent to: * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); * // Pseudocode: Set each zstd parameter and leave the rest as-is. * for ((param, value) : params) { @@ -2144,23 +2191,24 @@ ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, * dict is loaded with ZSTD_dct_auto and ZSTD_dlm_byCopy. * pledgedSrcSize must be correct. * If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x + * This prototype will generate compilation warnings. */ -ZSTDLIB_API size_t -ZSTD_initCStream_advanced(ZSTD_CStream* zcs, +ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /*! ZSTD_initCStream_usingCDict() : - * This function is deprecated, and equivalent to: + * This function is DEPRECATED, and equivalent to: * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); * ZSTD_CCtx_refCDict(zcs, cdict); - * + * * note : cdict will just be referenced, and must outlive compression session - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x + * This prototype will generate compilation warnings. */ -ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); +ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions") +size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); /*! ZSTD_initCStream_usingCDict_advanced() : * This function is DEPRECATED, and is approximately equivalent to: @@ -2175,18 +2223,21 @@ ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDi * same as ZSTD_initCStream_usingCDict(), with control over frame parameters. * pledgedSrcSize must be correct. If srcSize is not known at init time, use * value ZSTD_CONTENTSIZE_UNKNOWN. - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x + * This prototype will generate compilation warnings. */ -ZSTDLIB_API size_t -ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, +ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions") +size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize); /*! ZSTD_resetCStream() : - * This function is deprecated, and is equivalent to: + * This function is DEPRECATED, and is equivalent to: * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + * Note: ZSTD_resetCStream() interprets pledgedSrcSize == 0 as ZSTD_CONTENTSIZE_UNKNOWN, but + * ZSTD_CCtx_setPledgedSrcSize() does not do the same, so ZSTD_CONTENTSIZE_UNKNOWN must be + * explicitly specified. * * start a new frame, using same parameters from previous frame. * This is typically useful to skip dictionary loading stage, since it will re-use it in-place. @@ -2196,9 +2247,10 @@ ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, * For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs, * but it will change to mean "empty" in future version, so use macro ZSTD_CONTENTSIZE_UNKNOWN instead. * @return : 0, or an error code (which can be tested using ZSTD_isError()) - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x + * This prototype will generate compilation warnings. */ -ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); +ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); typedef struct { @@ -2285,8 +2337,7 @@ ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); ZSTD_CCtx object can be re-used multiple times within successive compression operations. Start by initializing a context. - Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression, - or ZSTD_compressBegin_advanced(), for finer parameter control. + Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression. It's also possible to duplicate a reference context which has already been initialized, using ZSTD_copyCCtx() Then, consume your input using ZSTD_compressContinue(). @@ -2311,15 +2362,17 @@ ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); /*===== Buffer-less streaming compression functions =====*/ ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel); ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); -ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */ ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /**< note: fails if cdict==NULL */ -ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */ ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */ ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); - +/* The ZSTD_compressBegin_advanced() and ZSTD_compressBegin_usingCDict_advanced() are now DEPRECATED and will generate a compiler warning */ +ZSTD_DEPRECATED("use advanced API to access custom parameters") +size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */ +ZSTD_DEPRECATED("use advanced API to access custom parameters") +size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */ /** Buffer-less streaming decompression (synchronous mode) diff --git a/programs/benchzstd.c b/programs/benchzstd.c index ccf705d2..49c03490 100644 --- a/programs/benchzstd.c +++ b/programs/benchzstd.c @@ -67,18 +67,10 @@ static const size_t maxMemory = (sizeof(size_t)==4) ? /* ************************************* * console display ***************************************/ -#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define DISPLAY(...) { fprintf(stderr, __VA_ARGS__); fflush(NULL); } #define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */ -static const U64 g_refreshRate = SEC_TO_MICRO / 6; -static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; - -#define DISPLAYUPDATE(l, ...) { if (displayLevel>=l) { \ - if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (displayLevel>=4)) \ - { g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \ - if (displayLevel>=4) fflush(stderr); } } } - /* ************************************* * Exceptions @@ -768,7 +760,7 @@ static int BMK_loadFiles(void* buffer, size_t bufferSize, } { FILE* const f = fopen(fileNamesTable[n], "rb"); if (f==NULL) RETURN_ERROR_INT(10, "impossible to open file %s", fileNamesTable[n]); - DISPLAYUPDATE(2, "Loading %s... \r", fileNamesTable[n]); + DISPLAYLEVEL(2, "Loading %s... \r", fileNamesTable[n]); if (fileSize > bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n; /* buffer too small - stop after this file */ { size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f); if (readSize != (size_t)fileSize) RETURN_ERROR_INT(11, "could not read %s", fileNamesTable[n]); diff --git a/programs/fileio.c b/programs/fileio.c index 571b3548..ec939083 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -93,10 +93,10 @@ struct FIO_display_prefs_s { int displayLevel; /* 0 : no display; 1: errors; 2: + result + interaction + warnings; 3: + progression; 4: + information */ - U32 noProgress; + FIO_progressSetting_e progressSetting; }; -static FIO_display_prefs_t g_display_prefs = {2, 0}; +static FIO_display_prefs_t g_display_prefs = {2, FIO_ps_auto}; #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) #define DISPLAYOUT(...) fprintf(stdout, __VA_ARGS__) @@ -105,10 +105,10 @@ static FIO_display_prefs_t g_display_prefs = {2, 0}; static const U64 g_refreshRate = SEC_TO_MICRO / 6; static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; -#define READY_FOR_UPDATE() (!g_display_prefs.noProgress && UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) +#define READY_FOR_UPDATE() ((g_display_prefs.progressSetting != FIO_ps_never) && UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) #define DELAY_NEXT_UPDATE() { g_displayClock = UTIL_getTime(); } #define DISPLAYUPDATE(l, ...) { \ - if (g_display_prefs.displayLevel>=l && !g_display_prefs.noProgress) { \ + if (g_display_prefs.displayLevel>=l && (g_display_prefs.progressSetting != FIO_ps_never)) { \ if (READY_FOR_UPDATE() || (g_display_prefs.displayLevel>=4)) { \ DELAY_NEXT_UPDATE(); \ DISPLAY(__VA_ARGS__); \ @@ -430,7 +430,7 @@ void FIO_freeContext(FIO_ctx_t* const fCtx) void FIO_setNotificationLevel(int level) { g_display_prefs.displayLevel=level; } -void FIO_setNoProgress(unsigned noProgress) { g_display_prefs.noProgress = noProgress; } +void FIO_setProgressSetting(FIO_progressSetting_e setting) { g_display_prefs.progressSetting = setting; } /*-************************************* @@ -1404,24 +1404,25 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx, (unsigned)(zfp.consumed >> 20), (unsigned)(zfp.produced >> 20), cShare ); - } else { /* summarized notifications if == 2 */ - DISPLAYLEVEL(2, "\r%79s\r", ""); /* Clear out the current displayed line */ + } else if (g_display_prefs.displayLevel >= 2 || g_display_prefs.progressSetting == FIO_ps_always) { + /* Require level 2 or forcibly displayed progress counter for summarized updates */ + DISPLAYLEVEL(1, "\r%79s\r", ""); /* Clear out the current displayed line */ if (fCtx->nbFilesTotal > 1) { size_t srcFileNameSize = strlen(srcFileName); /* Ensure that the string we print is roughly the same size each time */ if (srcFileNameSize > 18) { const char* truncatedSrcFileName = srcFileName + srcFileNameSize - 15; - DISPLAYLEVEL(2, "Compress: %u/%u files. Current: ...%s ", - fCtx->currFileIdx+1, fCtx->nbFilesTotal, truncatedSrcFileName); + DISPLAYLEVEL(1, "Compress: %u/%u files. Current: ...%s ", + fCtx->currFileIdx+1, fCtx->nbFilesTotal, truncatedSrcFileName); } else { - DISPLAYLEVEL(2, "Compress: %u/%u files. Current: %*s ", - fCtx->currFileIdx+1, fCtx->nbFilesTotal, (int)(18-srcFileNameSize), srcFileName); + DISPLAYLEVEL(1, "Compress: %u/%u files. Current: %*s ", + fCtx->currFileIdx+1, fCtx->nbFilesTotal, (int)(18-srcFileNameSize), srcFileName); } } - DISPLAYLEVEL(2, "Read : %2u ", (unsigned)(zfp.consumed >> 20)); + DISPLAYLEVEL(1, "Read : %2u ", (unsigned)(zfp.consumed >> 20)); if (fileSize != UTIL_FILESIZE_UNKNOWN) DISPLAYLEVEL(2, "/ %2u ", (unsigned)(fileSize >> 20)); - DISPLAYLEVEL(2, "MB ==> %2.f%%", cShare); + DISPLAYLEVEL(1, "MB ==> %2.f%%", cShare); DELAY_NEXT_UPDATE(); } @@ -2164,7 +2165,7 @@ FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress, FILE* finput, /* Write block */ storedSkips = FIO_fwriteSparse(ress->dstFile, ress->dstBuffer, outBuff.pos, prefs, storedSkips); frameSize += outBuff.pos; - if (!fCtx->hasStdoutOutput) { + if (!fCtx->hasStdoutOutput || g_display_prefs.progressSetting == FIO_ps_always) { if (fCtx->nbFilesTotal > 1) { size_t srcFileNameSize = strlen(srcFileName); if (srcFileNameSize > 18) { diff --git a/programs/fileio.h b/programs/fileio.h index 2a5591c3..9d97ec8b 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -68,6 +68,8 @@ void FIO_freeContext(FIO_ctx_t* const fCtx); typedef struct FIO_display_prefs_s FIO_display_prefs_t; +typedef enum { FIO_ps_auto, FIO_ps_never, FIO_ps_always } FIO_progressSetting_e; + /*-************************************* * Parameters ***************************************/ @@ -100,7 +102,7 @@ void FIO_setLiteralCompressionMode( FIO_prefs_t* const prefs, ZSTD_literalCompressionMode_e mode); -void FIO_setNoProgress(unsigned noProgress); +void FIO_setProgressSetting(FIO_progressSetting_e progressSetting); void FIO_setNotificationLevel(int level); void FIO_setExcludeCompressedFile(FIO_prefs_t* const prefs, int excludeCompressedFiles); void FIO_setAllowBlockDevices(FIO_prefs_t* const prefs, int allowBlockDevices); diff --git a/programs/util.c b/programs/util.c index ffda7866..8d190c62 100644 --- a/programs/util.c +++ b/programs/util.c @@ -323,9 +323,7 @@ U64 UTIL_getTotalFileSize(const char* const * fileNamesTable, unsigned nbFiles) static size_t readLineFromFile(char* buf, size_t len, FILE* file) { assert(!feof(file)); - /* Work around Cygwin problem when len == 1 it returns NULL. */ - if (len <= 1) return 0; - CONTROL( fgets(buf, (int) len, file) ); + if ( fgets(buf, (int) len, file) == NULL ) return 0; { size_t linelen = strlen(buf); if (strlen(buf)==0) return 0; if (buf[linelen-1] == '\n') linelen--; diff --git a/programs/zstdcli.c b/programs/zstdcli.c index b0ca7afc..239aaf40 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -167,7 +167,8 @@ static void usage_advanced(const char* programName) DISPLAYOUT( " -v : verbose mode; specify multiple times to increase verbosity \n"); DISPLAYOUT( " -q : suppress warnings; specify twice to suppress errors too \n"); - DISPLAYOUT( "--no-progress : do not display the progress counter \n"); + DISPLAYOUT( "--[no-]progress : forcibly display, or never display the progress counter.\n"); + DISPLAYOUT( " note: any (de)compressed output to terminal will mix with progress counter text. \n"); #ifdef UTIL_HAS_CREATEFILELIST DISPLAYOUT( " -r : operate recursively on directories \n"); @@ -879,7 +880,8 @@ int main(int const argCount, const char* argv[]) if (!strcmp(argument, "--rsyncable")) { rsyncable = 1; continue; } if (!strcmp(argument, "--compress-literals")) { literalCompressionMode = ZSTD_lcm_huffman; continue; } if (!strcmp(argument, "--no-compress-literals")) { literalCompressionMode = ZSTD_lcm_uncompressed; continue; } - if (!strcmp(argument, "--no-progress")) { FIO_setNoProgress(1); continue; } + if (!strcmp(argument, "--no-progress")) { FIO_setProgressSetting(FIO_ps_never); continue; } + if (!strcmp(argument, "--progress")) { FIO_setProgressSetting(FIO_ps_always); continue; } if (!strcmp(argument, "--exclude-compressed")) { FIO_setExcludeCompressedFile(prefs, 1); continue; } /* long commands with arguments */ diff --git a/tests/Makefile b/tests/Makefile index c4d1472d..85553007 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -24,7 +24,7 @@ PRGDIR = ../programs PYTHON ?= python3 TESTARTEFACT := versionsTest -DEBUGLEVEL ?= 1 +DEBUGLEVEL ?= 2 export DEBUGLEVEL # transmit value to sub-makefiles DEBUGFLAGS = -g -DDEBUGLEVEL=$(DEBUGLEVEL) CPPFLAGS += -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \ @@ -38,7 +38,7 @@ CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ -Wstrict-prototypes -Wundef \ -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \ - -Wredundant-decls -Wmissing-prototypes + -Wredundant-decls -Wmissing-prototypes -Wno-deprecated-declarations CFLAGS += $(DEBUGFLAGS) CPPFLAGS += $(MOREFLAGS) @@ -131,7 +131,7 @@ zstdmt_d_%.o : $(ZSTDDIR)/decompress/%.c $(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@ fullbench32: CPPFLAGS += -m32 -fullbench fullbench32 : CPPFLAGS += $(MULTITHREAD_CPP) +fullbench fullbench32 : CPPFLAGS += $(MULTITHREAD_CPP) -Wno-deprecated-declarations fullbench fullbench32 : LDFLAGS += $(MULTITHREAD_LD) fullbench fullbench32 : DEBUGFLAGS = -DNDEBUG # turn off assert() for speed measurements fullbench fullbench32 : $(ZSTD_FILES) @@ -147,7 +147,7 @@ fullbench-dll: $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/benchfn.c $(PRGDIR # $(CC) $(FLAGS) $(filter %.c,$^) -o $@$(EXT) -DZSTD_DLL_IMPORT=1 $(ZSTDDIR)/dll/libzstd.dll $(LINK.c) $^ $(LDLIBS) -o $@$(EXT) -fuzzer : CPPFLAGS += $(MULTITHREAD_CPP) +fuzzer : CPPFLAGS += $(MULTITHREAD_CPP) -Wno-deprecated-declarations fuzzer : LDFLAGS += $(MULTITHREAD_LD) fuzzer : $(ZSTDMT_OBJECTS) fuzzer fuzzer32 : $(ZDICT_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/datagen.c fuzzer.c diff --git a/tests/README.md b/tests/README.md index 1e40c46a..cd255e90 100644 --- a/tests/README.md +++ b/tests/README.md @@ -8,7 +8,6 @@ This directory contains the following programs and scripts: - `paramgrill` : parameter tester for zstd - `test-zstd-speed.py` : script for testing zstd speed difference between commits - `test-zstd-versions.py` : compatibility test between zstd versions stored on Github (v0.1+) -- `zbufftest` : Test tool to check ZBUFF (a buffered streaming API) integrity - `zstreamtest` : Fuzzer test tool for zstd streaming API - `legacy` : Test tool to test decoding of legacy zstd frames - `decodecorpus` : Tool to generate valid Zstandard frames, for verifying decoder implementations diff --git a/tests/fuzz/.gitignore b/tests/fuzz/.gitignore index 8ef3a3ef..93d935a8 100644 --- a/tests/fuzz/.gitignore +++ b/tests/fuzz/.gitignore @@ -16,6 +16,7 @@ zstd_frame_info decompress_dstSize_tooSmall fse_read_ncount sequence_compression_api +seekable_roundtrip fuzz-*.log rt_lib_* d_lib_* diff --git a/tests/fuzz/Makefile b/tests/fuzz/Makefile index 24b9f346..ccb574b7 100644 --- a/tests/fuzz/Makefile +++ b/tests/fuzz/Makefile @@ -25,10 +25,11 @@ CORPORA_URL_PREFIX:=https://github.com/facebook/zstd/releases/download/fuzz-corp ZSTDDIR = ../../lib PRGDIR = ../../programs +CONTRIBDIR = ../../contrib FUZZ_CPPFLAGS := -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \ -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(ZSTDDIR)/legacy \ - -I$(PRGDIR) -DZSTD_MULTITHREAD -DZSTD_LEGACY_SUPPORT=1 $(CPPFLAGS) + -I$(CONTRIBDIR)/seekable_format -I$(PRGDIR) -DZSTD_MULTITHREAD -DZSTD_LEGACY_SUPPORT=1 $(CPPFLAGS) FUZZ_EXTRA_FLAGS := -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ -Wstrict-prototypes -Wundef \ @@ -46,6 +47,9 @@ FUZZ_ROUND_TRIP_FLAGS := -DFUZZING_ASSERT_VALID_SEQUENCE FUZZ_HEADERS := fuzz_helpers.h fuzz.h zstd_helpers.h fuzz_data_producer.h FUZZ_SRC := $(PRGDIR)/util.c ./fuzz_helpers.c ./zstd_helpers.c ./fuzz_data_producer.c +SEEKABLE_HEADERS = $(CONTRIBDIR)/seekable_format/zstd_seekable.h +SEEKABLE_OBJS = $(CONTRIBDIR)/seekable_format/zstdseek_compress.c $(CONTRIBDIR)/seekable_format/zstdseek_decompress.c + ZSTDCOMMON_SRC := $(ZSTDDIR)/common/*.c ZSTDCOMP_SRC := $(ZSTDDIR)/compress/*.c ZSTDDECOMP_SRC := $(ZSTDDIR)/decompress/*.c @@ -98,7 +102,8 @@ FUZZ_TARGETS := \ dictionary_stream_round_trip \ decompress_dstSize_tooSmall \ fse_read_ncount \ - sequence_compression_api + sequence_compression_api \ + seekable_roundtrip all: libregression.a $(FUZZ_TARGETS) @@ -192,6 +197,9 @@ fse_read_ncount: $(FUZZ_HEADERS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_fse_read_ncount. sequence_compression_api: $(FUZZ_HEADERS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_sequence_compression_api.o $(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_sequence_compression_api.o $(LIB_FUZZING_ENGINE) -o $@ +seekable_roundtrip: $(FUZZ_HEADERS) $(SEEKABLE_HEADERS) $(FUZZ_ROUND_TRIP_OBJ) $(SEEKABLE_OBJS) rt_fuzz_seekable_roundtrip.o + $(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_ROUND_TRIP_OBJ) $(SEEKABLE_OBJS) rt_fuzz_seekable_roundtrip.o $(LIB_FUZZING_ENGINE) -o $@ + libregression.a: $(FUZZ_HEADERS) $(PRGDIR)/util.h $(PRGDIR)/util.c d_fuzz_regression_driver.o $(AR) $(FUZZ_ARFLAGS) $@ d_fuzz_regression_driver.o diff --git a/tests/fuzz/fuzz.py b/tests/fuzz/fuzz.py index a3431f36..d8dfa778 100755 --- a/tests/fuzz/fuzz.py +++ b/tests/fuzz/fuzz.py @@ -62,6 +62,7 @@ TARGET_INFO = { 'decompress_dstSize_tooSmall': TargetInfo(InputType.RAW_DATA), 'fse_read_ncount': TargetInfo(InputType.RAW_DATA), 'sequence_compression_api': TargetInfo(InputType.RAW_DATA), + 'seekable_roundtrip': TargetInfo(InputType.RAW_DATA), } TARGETS = list(TARGET_INFO.keys()) ALL_TARGETS = TARGETS + ['all'] diff --git a/tests/fuzz/seekable_roundtrip.c b/tests/fuzz/seekable_roundtrip.c new file mode 100644 index 00000000..dcdcaae1 --- /dev/null +++ b/tests/fuzz/seekable_roundtrip.c @@ -0,0 +1,88 @@ +/* + * Copyright (c) Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "zstd.h" +#include "zstd_seekable.h" +#include "fuzz_helpers.h" +#include "fuzz_data_producer.h" + +static ZSTD_seekable *stream = NULL; +static ZSTD_seekable_CStream *zscs = NULL; +static const size_t kSeekableOverheadSize = ZSTD_seekTableFooterSize; + +int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) +{ + /* Give a random portion of src data to the producer, to use for + parameter generation. The rest will be used for (de)compression */ + FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); + size = FUZZ_dataProducer_reserveDataPrefix(producer); + size_t const compressedBufferSize = ZSTD_compressBound(size) + kSeekableOverheadSize; + uint8_t* compressedBuffer = (uint8_t*)malloc(compressedBufferSize); + uint8_t* decompressedBuffer = (uint8_t*)malloc(size); + + int const cLevel = FUZZ_dataProducer_int32Range(producer, ZSTD_minCLevel(), ZSTD_maxCLevel()); + unsigned const checksumFlag = FUZZ_dataProducer_int32Range(producer, 0, 1); + size_t const uncompressedSize = FUZZ_dataProducer_uint32Range(producer, 0, size); + size_t const offset = FUZZ_dataProducer_uint32Range(producer, 0, size - uncompressedSize); + size_t seekSize; + + if (!zscs) { + zscs = ZSTD_seekable_createCStream(); + FUZZ_ASSERT(zscs); + } + if (!stream) { + stream = ZSTD_seekable_create(); + FUZZ_ASSERT(stream); + } + + { /* Perform a compression */ + size_t const initStatus = ZSTD_seekable_initCStream(zscs, cLevel, checksumFlag, size); + size_t endStatus; + ZSTD_outBuffer out = { .dst=compressedBuffer, .pos=0, .size=compressedBufferSize }; + ZSTD_inBuffer in = { .src=src, .pos=0, .size=size }; + FUZZ_ASSERT(!ZSTD_isError(initStatus)); + + do { + size_t cSize = ZSTD_seekable_compressStream(zscs, &out, &in); + FUZZ_ASSERT(!ZSTD_isError(cSize)); + } while (in.pos != in.size); + + FUZZ_ASSERT(in.pos == in.size); + endStatus = ZSTD_seekable_endStream(zscs, &out); + FUZZ_ASSERT(!ZSTD_isError(endStatus)); + seekSize = out.pos; + } + + { /* Decompress at an offset */ + size_t const initStatus = ZSTD_seekable_initBuff(stream, compressedBuffer, seekSize); + size_t decompressedBytesTotal = 0; + size_t dSize; + + FUZZ_ZASSERT(initStatus); + do { + dSize = ZSTD_seekable_decompress(stream, decompressedBuffer, uncompressedSize, offset); + FUZZ_ASSERT(!ZSTD_isError(dSize)); + decompressedBytesTotal += dSize; + } while (decompressedBytesTotal < uncompressedSize && dSize > 0); + FUZZ_ASSERT(decompressedBytesTotal == uncompressedSize); + } + + FUZZ_ASSERT_MSG(!FUZZ_memcmp(src+offset, decompressedBuffer, uncompressedSize), "Corruption!"); + + free(decompressedBuffer); + free(compressedBuffer); + FUZZ_dataProducer_free(producer); + +#ifndef STATEFUL_FUZZING + ZSTD_seekable_free(stream); stream = NULL; + ZSTD_seekable_freeCStream(zscs); zscs = NULL; +#endif + return 0; +} diff --git a/tests/fuzz/zstd_helpers.c b/tests/fuzz/zstd_helpers.c index 31bf4c20..4d889dee 100644 --- a/tests/fuzz/zstd_helpers.c +++ b/tests/fuzz/zstd_helpers.c @@ -97,6 +97,7 @@ void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, FUZZ_dataProducer setRand(cctx, ZSTD_c_literalCompressionMode, 0, 2, producer); setRand(cctx, ZSTD_c_forceAttachDict, 0, 2, producer); setRand(cctx, ZSTD_c_splitBlocks, 0, 1, producer); + setRand(cctx, ZSTD_c_deterministicRefPrefix, 0, 1, producer); if (FUZZ_dataProducer_uint32Range(producer, 0, 1) == 0) { setRand(cctx, ZSTD_c_srcSizeHint, ZSTD_SRCSIZEHINT_MIN, 2 * srcSize, producer); } diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 2c6c60d3..46147a24 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -766,6 +766,50 @@ static int basicUnitTests(U32 const seed, double compressibility) } DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "test%3i : testing dict compression for determinism : ", testNb++); + { + size_t const testSize = 1024; + ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + char* dict = (char*)malloc(2 * testSize); + int ldmEnabled, level; + + RDG_genBuffer(dict, testSize, 0.5, 0.5, seed); + RDG_genBuffer(CNBuffer, testSize, 0.6, 0.6, seed); + memcpy(dict + testSize, CNBuffer, testSize); + for (level = 1; level <= 5; ++level) { + for (ldmEnabled = 0; ldmEnabled <= 1; ++ldmEnabled) { + size_t cSize0; + XXH64_hash_t compressedChecksum0; + + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, level)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ldmEnabled)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_deterministicRefPrefix, 1)); + + CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, testSize)); + cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, testSize); + CHECK_Z(cSize); + CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, testSize, compressedBuffer, cSize, dict, testSize)); + + cSize0 = cSize; + compressedChecksum0 = XXH64(compressedBuffer, cSize, 0); + + CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, testSize)); + cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, dict + testSize, testSize); + CHECK_Z(cSize); + + if (cSize != cSize0) goto _output_error; + if (XXH64(compressedBuffer, cSize, 0) != compressedChecksum0) goto _output_error; + } + } + + ZSTD_freeCCtx(cctx); + ZSTD_freeDCtx(dctx); + free(dict); + } + DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "test%3i : LDM + opt parser with small uncompressible block ", testNb++); { ZSTD_CCtx* cctx = ZSTD_createCCtx(); ZSTD_DCtx* dctx = ZSTD_createDCtx(); @@ -1738,10 +1782,7 @@ static int basicUnitTests(U32 const seed, double compressibility) DISPLAYLEVEL(3, "test%3i : check content size on duplicated context : ", testNb++); { size_t const testSize = CNBuffSize / 3; - { ZSTD_parameters p = ZSTD_getParams(2, testSize, dictSize); - p.fParams.contentSizeFlag = 1; - CHECK( ZSTD_compressBegin_advanced(ctxOrig, CNBuffer, dictSize, p, testSize-1) ); - } + CHECK( ZSTD_compressBegin(ctxOrig, ZSTD_defaultCLevel()) ); CHECK( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, testSize) ); CHECK_VAR(cSize, ZSTD_compressEnd(ctxDuplicated, compressedBuffer, ZSTD_compressBound(testSize), @@ -2562,12 +2603,8 @@ static int basicUnitTests(U32 const seed, double compressibility) int const compressionLevel = -1; assert(cctx != NULL); - { ZSTD_parameters const params = ZSTD_getParams(compressionLevel, srcSize, 0); - size_t const cSize_1pass = ZSTD_compress_advanced(cctx, - compressedBuffer, compressedBufferSize, - CNBuffer, srcSize, - NULL, 0, - params); + { size_t const cSize_1pass = ZSTD_compress(compressedBuffer, compressedBufferSize, + CNBuffer, srcSize, compressionLevel); if (ZSTD_isError(cSize_1pass)) goto _output_error; CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) ); diff --git a/tests/regression/results.csv b/tests/regression/results.csv index 68150a61..b94d5501 100644 --- a/tests/regression/results.csv +++ b/tests/regression/results.csv @@ -97,11 +97,11 @@ github, uncompressed literals, compress github, uncompressed literals optimal, compress cctx, 134064 github, huffman literals, compress cctx, 175568 github, multithreaded with advanced params, compress cctx, 141102 -silesia, level -5, zstdcli, 6882553 -silesia, level -3, zstdcli, 6568424 -silesia, level -1, zstdcli, 6183451 +silesia, level -5, zstdcli, 6737655 +silesia, level -3, zstdcli, 6444725 +silesia, level -1, zstdcli, 6178508 silesia, level 0, zstdcli, 4849600 -silesia, level 1, zstdcli, 5314210 +silesia, level 1, zstdcli, 5313252 silesia, level 3, zstdcli, 4849600 silesia, level 4, zstdcli, 4787018 silesia, level 5, zstdcli, 4707842 @@ -111,16 +111,16 @@ silesia, level 9, zstdcli, silesia, level 13, zstdcli, 4482183 silesia, level 16, zstdcli, 4360299 silesia, level 19, zstdcli, 4283285 -silesia, long distance mode, zstdcli, 4840792 +silesia, long distance mode, zstdcli, 4840806 silesia, multithreaded, zstdcli, 4849600 -silesia, multithreaded long distance mode, zstdcli, 4840792 -silesia, small window log, zstdcli, 7111012 +silesia, multithreaded long distance mode, zstdcli, 4840806 +silesia, small window log, zstdcli, 7095967 silesia, small hash log, zstdcli, 6526189 silesia, small chain log, zstdcli, 4912245 -silesia, explicit params, zstdcli, 4795887 +silesia, explicit params, zstdcli, 4795856 silesia, uncompressed literals, zstdcli, 5128030 silesia, uncompressed literals optimal, zstdcli, 4317944 -silesia, huffman literals, zstdcli, 5331216 +silesia, huffman literals, zstdcli, 5326316 silesia, multithreaded with advanced params, zstdcli, 5128030 silesia.tar, level -5, zstdcli, 6738934 silesia.tar, level -3, zstdcli, 6448419 @@ -137,9 +137,9 @@ silesia.tar, level 13, zstdcli, silesia.tar, level 16, zstdcli, 4356831 silesia.tar, level 19, zstdcli, 4264491 silesia.tar, no source size, zstdcli, 4861508 -silesia.tar, long distance mode, zstdcli, 4853153 +silesia.tar, long distance mode, zstdcli, 4853226 silesia.tar, multithreaded, zstdcli, 4861512 -silesia.tar, multithreaded long distance mode, zstdcli, 4853153 +silesia.tar, multithreaded long distance mode, zstdcli, 4853226 silesia.tar, small window log, zstdcli, 7101576 silesia.tar, small hash log, zstdcli, 6529290 silesia.tar, small chain log, zstdcli, 4917022 @@ -187,16 +187,16 @@ github, uncompressed literals, zstdcli, github, uncompressed literals optimal, zstdcli, 159227 github, huffman literals, zstdcli, 144465 github, multithreaded with advanced params, zstdcli, 167915 -github.tar, level -5, zstdcli, 46751 -github.tar, level -5 with dict, zstdcli, 44444 -github.tar, level -3, zstdcli, 43541 -github.tar, level -3 with dict, zstdcli, 41116 -github.tar, level -1, zstdcli, 42469 -github.tar, level -1 with dict, zstdcli, 41200 +github.tar, level -5, zstdcli, 46860 +github.tar, level -5 with dict, zstdcli, 44575 +github.tar, level -3, zstdcli, 43758 +github.tar, level -3 with dict, zstdcli, 41451 +github.tar, level -1, zstdcli, 42494 +github.tar, level -1 with dict, zstdcli, 41135 github.tar, level 0, zstdcli, 38445 github.tar, level 0 with dict, zstdcli, 37999 -github.tar, level 1, zstdcli, 39346 -github.tar, level 1 with dict, zstdcli, 38297 +github.tar, level 1, zstdcli, 39269 +github.tar, level 1 with dict, zstdcli, 38284 github.tar, level 3, zstdcli, 38445 github.tar, level 3 with dict, zstdcli, 37999 github.tar, level 4, zstdcli, 38471 @@ -217,16 +217,16 @@ github.tar, level 19, zstdcli, github.tar, level 19 with dict, zstdcli, 32899 github.tar, no source size, zstdcli, 38442 github.tar, no source size with dict, zstdcli, 38004 -github.tar, long distance mode, zstdcli, 39726 +github.tar, long distance mode, zstdcli, 39730 github.tar, multithreaded, zstdcli, 38445 -github.tar, multithreaded long distance mode, zstdcli, 39726 -github.tar, small window log, zstdcli, 199432 +github.tar, multithreaded long distance mode, zstdcli, 39730 +github.tar, small window log, zstdcli, 198544 github.tar, small hash log, zstdcli, 129874 github.tar, small chain log, zstdcli, 41673 github.tar, explicit params, zstdcli, 41227 github.tar, uncompressed literals, zstdcli, 41126 github.tar, uncompressed literals optimal, zstdcli, 35392 -github.tar, huffman literals, zstdcli, 38804 +github.tar, huffman literals, zstdcli, 38781 github.tar, multithreaded with advanced params, zstdcli, 41126 silesia, level -5, advanced one pass, 6737607 silesia, level -3, advanced one pass, 6444677 @@ -249,9 +249,9 @@ silesia, level 13, advanced silesia, level 16, advanced one pass, 4360251 silesia, level 19, advanced one pass, 4283237 silesia, no source size, advanced one pass, 4849552 -silesia, long distance mode, advanced one pass, 4840744 +silesia, long distance mode, advanced one pass, 4840738 silesia, multithreaded, advanced one pass, 4849552 -silesia, multithreaded long distance mode, advanced one pass, 4840744 +silesia, multithreaded long distance mode, advanced one pass, 4840758 silesia, small window log, advanced one pass, 7095919 silesia, small hash log, advanced one pass, 6526141 silesia, small chain log, advanced one pass, 4912197 @@ -281,9 +281,9 @@ silesia.tar, level 13, advanced silesia.tar, level 16, advanced one pass, 4356827 silesia.tar, level 19, advanced one pass, 4264487 silesia.tar, no source size, advanced one pass, 4861425 -silesia.tar, long distance mode, advanced one pass, 4847735 +silesia.tar, long distance mode, advanced one pass, 4847754 silesia.tar, multithreaded, advanced one pass, 4861508 -silesia.tar, multithreaded long distance mode, advanced one pass, 4853149 +silesia.tar, multithreaded long distance mode, advanced one pass, 4853222 silesia.tar, small window log, advanced one pass, 7101530 silesia.tar, small hash log, advanced one pass, 6529232 silesia.tar, small chain log, advanced one pass, 4917041 @@ -511,9 +511,9 @@ github.tar, level 19 with dict copy, advanced github.tar, level 19 with dict load, advanced one pass, 32676 github.tar, no source size, advanced one pass, 38441 github.tar, no source size with dict, advanced one pass, 37995 -github.tar, long distance mode, advanced one pass, 39722 +github.tar, long distance mode, advanced one pass, 39757 github.tar, multithreaded, advanced one pass, 38441 -github.tar, multithreaded long distance mode, advanced one pass, 39722 +github.tar, multithreaded long distance mode, advanced one pass, 39726 github.tar, small window log, advanced one pass, 198540 github.tar, small hash log, advanced one pass, 129870 github.tar, small chain log, advanced one pass, 41669 @@ -543,9 +543,9 @@ silesia, level 13, advanced silesia, level 16, advanced one pass small out, 4360251 silesia, level 19, advanced one pass small out, 4283237 silesia, no source size, advanced one pass small out, 4849552 -silesia, long distance mode, advanced one pass small out, 4840744 +silesia, long distance mode, advanced one pass small out, 4840738 silesia, multithreaded, advanced one pass small out, 4849552 -silesia, multithreaded long distance mode, advanced one pass small out, 4840744 +silesia, multithreaded long distance mode, advanced one pass small out, 4840758 silesia, small window log, advanced one pass small out, 7095919 silesia, small hash log, advanced one pass small out, 6526141 silesia, small chain log, advanced one pass small out, 4912197 @@ -575,9 +575,9 @@ silesia.tar, level 13, advanced silesia.tar, level 16, advanced one pass small out, 4356827 silesia.tar, level 19, advanced one pass small out, 4264487 silesia.tar, no source size, advanced one pass small out, 4861425 -silesia.tar, long distance mode, advanced one pass small out, 4847735 +silesia.tar, long distance mode, advanced one pass small out, 4847754 silesia.tar, multithreaded, advanced one pass small out, 4861508 -silesia.tar, multithreaded long distance mode, advanced one pass small out, 4853149 +silesia.tar, multithreaded long distance mode, advanced one pass small out, 4853222 silesia.tar, small window log, advanced one pass small out, 7101530 silesia.tar, small hash log, advanced one pass small out, 6529232 silesia.tar, small chain log, advanced one pass small out, 4917041 @@ -805,9 +805,9 @@ github.tar, level 19 with dict copy, advanced github.tar, level 19 with dict load, advanced one pass small out, 32676 github.tar, no source size, advanced one pass small out, 38441 github.tar, no source size with dict, advanced one pass small out, 37995 -github.tar, long distance mode, advanced one pass small out, 39722 +github.tar, long distance mode, advanced one pass small out, 39757 github.tar, multithreaded, advanced one pass small out, 38441 -github.tar, multithreaded long distance mode, advanced one pass small out, 39722 +github.tar, multithreaded long distance mode, advanced one pass small out, 39726 github.tar, small window log, advanced one pass small out, 198540 github.tar, small hash log, advanced one pass small out, 129870 github.tar, small chain log, advanced one pass small out, 41669 @@ -837,9 +837,9 @@ silesia, level 13, advanced silesia, level 16, advanced streaming, 4360251 silesia, level 19, advanced streaming, 4283237 silesia, no source size, advanced streaming, 4849516 -silesia, long distance mode, advanced streaming, 4840744 +silesia, long distance mode, advanced streaming, 4840738 silesia, multithreaded, advanced streaming, 4849552 -silesia, multithreaded long distance mode, advanced streaming, 4840744 +silesia, multithreaded long distance mode, advanced streaming, 4840758 silesia, small window log, advanced streaming, 7112062 silesia, small hash log, advanced streaming, 6526141 silesia, small chain log, advanced streaming, 4912197 @@ -869,9 +869,9 @@ silesia.tar, level 13, advanced silesia.tar, level 16, advanced streaming, 4356834 silesia.tar, level 19, advanced streaming, 4264392 silesia.tar, no source size, advanced streaming, 4861423 -silesia.tar, long distance mode, advanced streaming, 4847735 +silesia.tar, long distance mode, advanced streaming, 4847754 silesia.tar, multithreaded, advanced streaming, 4861508 -silesia.tar, multithreaded long distance mode, advanced streaming, 4853149 +silesia.tar, multithreaded long distance mode, advanced streaming, 4853222 silesia.tar, small window log, advanced streaming, 7118769 silesia.tar, small hash log, advanced streaming, 6529235 silesia.tar, small chain log, advanced streaming, 4917021 @@ -1099,9 +1099,9 @@ github.tar, level 19 with dict copy, advanced github.tar, level 19 with dict load, advanced streaming, 32676 github.tar, no source size, advanced streaming, 38438 github.tar, no source size with dict, advanced streaming, 38000 -github.tar, long distance mode, advanced streaming, 39722 +github.tar, long distance mode, advanced streaming, 39757 github.tar, multithreaded, advanced streaming, 38441 -github.tar, multithreaded long distance mode, advanced streaming, 39722 +github.tar, multithreaded long distance mode, advanced streaming, 39726 github.tar, small window log, advanced streaming, 199558 github.tar, small hash log, advanced streaming, 129870 github.tar, small chain log, advanced streaming, 41669 diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index f2b16db2..56f77232 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -321,7 +321,9 @@ static int basicUnitTests(U32 seed, double compressibility) /* Basic compression test using dict */ DISPLAYLEVEL(3, "test%3i : skipframe + compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); - CHECK_Z( ZSTD_initCStream_usingDict(zc, CNBuffer, dictSize, 1 /* cLevel */) ); + CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) ); + CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 1) ); + CHECK_Z( ZSTD_CCtx_loadDictionary(zc, CNBuffer, dictSize) ); outBuff.dst = (char*)(compressedBuffer)+cSize; assert(compressedBufferSize > cSize); outBuff.size = compressedBufferSize - cSize; @@ -368,7 +370,7 @@ static int basicUnitTests(U32 seed, double compressibility) } /* Attempt bad compression parameters */ - DISPLAYLEVEL(3, "test%3i : use bad compression parameters : ", testNb++); + DISPLAYLEVEL(3, "test%3i : use bad compression parameters with ZSTD_initCStream_advanced : ", testNb++); { size_t r; ZSTD_parameters params = ZSTD_getParams(1, 0, 0); params.cParams.minMatch = 2; @@ -539,7 +541,10 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(3, "OK\n"); /* _srcSize compression test */ DISPLAYLEVEL(3, "test%3i : compress_srcSize %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); - CHECK_Z( ZSTD_initCStream_srcSize(zc, 1, CNBufferSize) ); + CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) ); + CHECK_Z( ZSTD_CCtx_refCDict(zc, NULL) ); + CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 1) ); + CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, CNBufferSize) ); outBuff.dst = (char*)(compressedBuffer); outBuff.size = compressedBufferSize; outBuff.pos = 0; @@ -559,7 +564,10 @@ static int basicUnitTests(U32 seed, double compressibility) /* wrong _srcSize compression test */ DISPLAYLEVEL(3, "test%3i : too large srcSize : %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH-1); - ZSTD_initCStream_srcSize(zc, 1, CNBufferSize+1); + CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) ); + CHECK_Z( ZSTD_CCtx_refCDict(zc, NULL) ); + CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 1) ); + CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, CNBufferSize+1) ); outBuff.dst = (char*)(compressedBuffer); outBuff.size = compressedBufferSize; outBuff.pos = 0; @@ -574,7 +582,10 @@ static int basicUnitTests(U32 seed, double compressibility) /* wrong _srcSize compression test */ DISPLAYLEVEL(3, "test%3i : too small srcSize : %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH-1); - ZSTD_initCStream_srcSize(zc, 1, CNBufferSize-1); + CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) ); + CHECK_Z( ZSTD_CCtx_refCDict(zc, NULL) ); + CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 1) ); + CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, CNBufferSize-1) ); outBuff.dst = (char*)(compressedBuffer); outBuff.size = compressedBufferSize; outBuff.pos = 0; @@ -587,9 +598,9 @@ static int basicUnitTests(U32 seed, double compressibility) } DISPLAYLEVEL(3, "test%3i : wrong srcSize !contentSizeFlag : %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH-1); - { ZSTD_parameters params = ZSTD_getParams(1, CNBufferSize, 0); - params.fParams.contentSizeFlag = 0; - CHECK_Z(ZSTD_initCStream_advanced(zc, NULL, 0, params, CNBufferSize - MIN(CNBufferSize, 200 KB))); + { CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) ); + CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_contentSizeFlag, 0) ); + CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, CNBufferSize - MIN(CNBufferSize, 200 KB)) ); outBuff.dst = (char*)compressedBuffer; outBuff.size = compressedBufferSize; outBuff.pos = 0; @@ -609,7 +620,9 @@ static int basicUnitTests(U32 seed, double compressibility) /* use 1 */ { size_t const inSize = 513; DISPLAYLEVEL(5, "use1 "); - ZSTD_initCStream_advanced(zc, NULL, 0, ZSTD_getParams(19, inSize, 0), inSize); /* needs btopt + search3 to trigger hashLog3 */ + CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) ); + CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 19) ); + CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, inSize) ); inBuff.src = CNBuffer; inBuff.size = inSize; inBuff.pos = 0; @@ -626,7 +639,9 @@ static int basicUnitTests(U32 seed, double compressibility) /* use 2 */ { size_t const inSize = 1025; /* will not continue, because tables auto-adjust and are therefore different size */ DISPLAYLEVEL(5, "use2 "); - ZSTD_initCStream_advanced(zc, NULL, 0, ZSTD_getParams(19, inSize, 0), inSize); /* needs btopt + search3 to trigger hashLog3 */ + CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) ); + CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 19) ); + CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, inSize) ); inBuff.src = CNBuffer; inBuff.size = inSize; inBuff.pos = 0; @@ -1274,7 +1289,7 @@ static int basicUnitTests(U32 seed, double compressibility) if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != 0) goto _output_error; DISPLAYLEVEL(3, "OK \n"); - DISPLAYLEVEL(3, "test%3i : pledgedSrcSize == 0 behaves properly : ", testNb++); + DISPLAYLEVEL(3, "test%3i : pledgedSrcSize == 0 behaves properly with ZSTD_initCStream_advanced : ", testNb++); { ZSTD_parameters params = ZSTD_getParams(5, 0, 0); params.fParams.contentSizeFlag = 1; CHECK_Z( ZSTD_initCStream_advanced(zc, NULL, 0, params, 0) ); @@ -1290,7 +1305,8 @@ static int basicUnitTests(U32 seed, double compressibility) cSize = outBuff.pos; if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != 0) goto _output_error; - ZSTD_resetCStream(zc, 0); /* resetCStream should treat 0 as unknown */ + CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) ); + CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, ZSTD_CONTENTSIZE_UNKNOWN) ); outBuff.dst = compressedBuffer; outBuff.size = compressedBufferSize; outBuff.pos = 0; @@ -1434,7 +1450,8 @@ static int basicUnitTests(U32 seed, double compressibility) CHECK_Z(ZSTD_initCStream_srcSize(zc, 11, ZSTD_CONTENTSIZE_UNKNOWN)); CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_compressionLevel, &level)); CHECK(level != 11, "Compression level does not match"); - ZSTD_resetCStream(zc, ZSTD_CONTENTSIZE_UNKNOWN); + CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) ); + CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, ZSTD_CONTENTSIZE_UNKNOWN) ); CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_compressionLevel, &level)); CHECK(level != 11, "Compression level does not match"); } @@ -1444,7 +1461,8 @@ static int basicUnitTests(U32 seed, double compressibility) { ZSTD_parameters const params = ZSTD_getParams(9, 0, 0); CHECK_Z(ZSTD_initCStream_advanced(zc, NULL, 0, params, ZSTD_CONTENTSIZE_UNKNOWN)); CHECK(badParameters(zc, params), "Compression parameters do not match"); - ZSTD_resetCStream(zc, ZSTD_CONTENTSIZE_UNKNOWN); + CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) ); + CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, ZSTD_CONTENTSIZE_UNKNOWN) ); CHECK(badParameters(zc, params), "Compression parameters do not match"); } DISPLAYLEVEL(3, "OK \n"); @@ -1836,8 +1854,9 @@ static int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double co && oldTestLog /* at least one test happened */ && resetAllowed) { maxTestSize = FUZ_randomLength(&lseed, oldTestLog+2); maxTestSize = MIN(maxTestSize, srcBufferSize-16); - { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? 0 : maxTestSize; - CHECK_Z( ZSTD_resetCStream(zc, pledgedSrcSize) ); + { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize; + CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) ); + CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) ); } } else { U32 const testLog = FUZ_rand(&lseed) % maxSrcLog; @@ -1855,11 +1874,13 @@ static int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double co dict = srcBuffer + dictStart; } { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize; - ZSTD_parameters params = ZSTD_getParams(cLevel, pledgedSrcSize, dictSize); - params.fParams.checksumFlag = FUZ_rand(&lseed) & 1; - params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1; - params.fParams.contentSizeFlag = FUZ_rand(&lseed) & 1; - CHECK_Z ( ZSTD_initCStream_advanced(zc, dict, dictSize, params, pledgedSrcSize) ); + CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) ); + CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, cLevel) ); + CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_checksumFlag, FUZ_rand(&lseed) & 1) ); + CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_contentSizeFlag, FUZ_rand(&lseed) & 1) ); + CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_dictIDFlag, FUZ_rand(&lseed) & 1) ); + CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) ); + CHECK_Z( ZSTD_CCtx_loadDictionary(zc, dict, dictSize) ); } } /* multi-segments compression test */ @@ -2215,6 +2236,7 @@ static int fuzzerTests_newAPI(U32 seed, int nbTests, int startTest, } if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_forceMaxWindow, FUZ_rand(&lseed) & 1, opaqueAPI) ); + if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_deterministicRefPrefix, FUZ_rand(&lseed) & 1, opaqueAPI) ); /* Apply parameters */ if (opaqueAPI) { diff --git a/zlibWrapper/examples/zwrapbench.c b/zlibWrapper/examples/zwrapbench.c index 127f6114..5993e51b 100644 --- a/zlibWrapper/examples/zwrapbench.c +++ b/zlibWrapper/examples/zwrapbench.c @@ -264,9 +264,22 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize, ZSTD_outBuffer outBuffer; ZSTD_CStream* zbc = ZSTD_createCStream(); size_t rSize; + ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams(); + + if (!cctxParams) EXM_THROW(1, "ZSTD_createCCtxParams() allocation failure"); if (zbc == NULL) EXM_THROW(1, "ZSTD_createCStream() allocation failure"); - rSize = ZSTD_initCStream_advanced(zbc, dictBuffer, dictBufferSize, zparams, avgSize); - if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_initCStream_advanced() failed : %s", ZSTD_getErrorName(rSize)); + + { int initErr = 0; + initErr |= ZSTD_isError(ZSTD_CCtx_reset(zbc, ZSTD_reset_session_only)); + initErr |= ZSTD_isError(ZSTD_CCtxParams_init_advanced(cctxParams, zparams)); + initErr |= ZSTD_isError(ZSTD_CCtx_setParametersUsingCCtxParams(zbc, cctxParams)); + initErr |= ZSTD_isError(ZSTD_CCtx_setPledgedSrcSize(zbc, avgSize)); + initErr |= ZSTD_isError(ZSTD_CCtx_loadDictionary(zbc, dictBuffer, dictBufferSize)); + + ZSTD_freeCCtxParams(cctxParams); + if (initErr) EXM_THROW(1, "CCtx init failed!"); + } + do { U32 blockNb; for (blockNb=0; blockNbzbc == NULL) return Z_STREAM_ERROR; if (!pledgedSrcSize) pledgedSrcSize = zwc->pledgedSrcSize; - { ZSTD_parameters const params = ZSTD_getParams(zwc->compressionLevel, pledgedSrcSize, dictSize); - size_t initErr; + { unsigned initErr = 0; + ZSTD_parameters const params = ZSTD_getParams(zwc->compressionLevel, pledgedSrcSize, dictSize); + ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams(); + if (!cctxParams) return Z_STREAM_ERROR; LOG_WRAPPERC("pledgedSrcSize=%d windowLog=%d chainLog=%d hashLog=%d searchLog=%d minMatch=%d strategy=%d\n", (int)pledgedSrcSize, params.cParams.windowLog, params.cParams.chainLog, params.cParams.hashLog, params.cParams.searchLog, params.cParams.minMatch, params.cParams.strategy); - initErr = ZSTD_initCStream_advanced(zwc->zbc, dict, dictSize, params, pledgedSrcSize); - if (ZSTD_isError(initErr)) return Z_STREAM_ERROR; + + initErr |= ZSTD_isError(ZSTD_CCtx_reset(zwc->zbc, ZSTD_reset_session_only)); + initErr |= ZSTD_isError(ZSTD_CCtxParams_init_advanced(cctxParams, params)); + initErr |= ZSTD_isError(ZSTD_CCtx_setParametersUsingCCtxParams(zwc->zbc, cctxParams)); + initErr |= ZSTD_isError(ZSTD_CCtx_setPledgedSrcSize(zwc->zbc, pledgedSrcSize)); + initErr |= ZSTD_isError(ZSTD_CCtx_loadDictionary(zwc->zbc, dict, dictSize)); + + ZSTD_freeCCtxParams(cctxParams); + if (initErr) return Z_STREAM_ERROR; } return Z_OK;