facebook/zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'dev' into fasterCygwin

Browse Source
This commit is contained in:
Yann Collet 2021-05-06 16:06:00 -07:00
commit ee65162655
60 changed files with 727 additions and 254 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Makefile
View File

@ -122,8 +122,8 @@ contrib: lib
$(MAKE) -C contrib/seekable_format/examples all
$(MAKE) -C contrib/seekable_format/tests test
$(MAKE) -C contrib/largeNbDicts all
cd contrib/single_file_libs/ ; ./build_decoder_test.sh
cd contrib/single_file_libs/ ; ./build_library_test.sh
cd build/single_file_libs/ ; ./build_decoder_test.sh
cd build/single_file_libs/ ; ./build_library_test.sh
.PHONY: cleanTabs
cleanTabs:

1
TESTING.md
View File

@ -40,5 +40,4 @@ They consist of the following tests:
- Versions test (ensuring `zstd` can decode files from all previous versions)
- `pzstd` with asan and tsan, as well as in 32-bits mode
- Testing `zstd` with legacy mode off
- Testing `zbuff` (old streaming API)
- Entire test suite and make install on macOS

36
appveyor.yml
View File

@ -52,6 +52,15 @@
PLATFORM: "Win32"
CONFIGURATION: "Release"
- COMPILER: "clang-cl"
HOST: "cmake-visual"
PLATFORM: "x64"
CONFIGURATION: "Release"
CMAKE_GENERATOR: "Visual Studio 15 2017"
CMAKE_GENERATOR_PLATFORM: "x64"
CMAKE_GENERATOR_TOOLSET: "LLVM"
APPVEYOR_BUILD_WORKER_IMAGE: "Visual Studio 2017"
install:
- ECHO Installing %COMPILER% %PLATFORM% %CONFIGURATION%
- SET PATH_ORIGINAL=%PATH%
@ -154,6 +163,15 @@
COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe &&
COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe tests\
)
- if [%HOST%]==[cmake-visual] (
ECHO *** &&
ECHO *** Building %CMAKE_GENERATOR% ^(%CMAKE_GENERATOR_TOOLSET%^) %PLATFORM%\%CONFIGURATION% &&
PUSHD build\cmake &&
cmake -DBUILD_TESTING=ON . &&
cmake --build . --config %CONFIGURATION% -j4 &&
POPD &&
ECHO ***
)
test_script:
- ECHO Testing %COMPILER% %PLATFORM% %CONFIGURATION%
@ -223,6 +241,15 @@
PLATFORM: "Win32"
CONFIGURATION: "Release"
- COMPILER: "clang-cl"
HOST: "cmake-visual"
PLATFORM: "x64"
CONFIGURATION: "Release"
CMAKE_GENERATOR: "Visual Studio 15 2017"
CMAKE_GENERATOR_PLATFORM: "x64"
CMAKE_GENERATOR_TOOLSET: "LLVM"
APPVEYOR_BUILD_WORKER_IMAGE: "Visual Studio 2017"
install:
- ECHO Installing %COMPILER% %PLATFORM% %CONFIGURATION%
- SET PATH_ORIGINAL=%PATH%
@ -281,6 +308,15 @@
COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe &&
COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe tests\
)
- if [%HOST%]==[cmake-visual] (
ECHO *** &&
ECHO *** Building %CMAKE_GENERATOR% ^(%CMAKE_GENERATOR_TOOLSET%^) %PLATFORM%\%CONFIGURATION% &&
PUSHD build\cmake &&
cmake -DBUILD_TESTING=ON . &&
cmake --build . --config %CONFIGURATION% -j4 &&
POPD &&
ECHO ***
)
test_script:

8
build/VS2008/fullbench/fullbench.vcproj
View File

@ -45,7 +45,7 @@
Name="VCCLCompilerTool"
Optimization="0"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\programs\legacy"
PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
PreprocessorDefinitions="ZSTD_DISABLE_DEPRECATE_WARNINGS=1;WIN32;_DEBUG;_CONSOLE"
MinimalRebuild="true"
BasicRuntimeChecks="3"
RuntimeLibrary="3"
@ -121,7 +121,7 @@
EnableIntrinsicFunctions="true"
OmitFramePointers="true"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\programs\legacy"
PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
PreprocessorDefinitions="ZSTD_DISABLE_DEPRECATE_WARNINGS=1;WIN32;NDEBUG;_CONSOLE"
RuntimeLibrary="0"
EnableFunctionLevelLinking="true"
UsePrecompiledHeader="0"
@ -195,7 +195,7 @@
Name="VCCLCompilerTool"
Optimization="0"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\programs\legacy"
PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
PreprocessorDefinitions="ZSTD_DISABLE_DEPRECATE_WARNINGS=1;WIN32;_DEBUG;_CONSOLE"
MinimalRebuild="true"
BasicRuntimeChecks="3"
RuntimeLibrary="3"
@ -272,7 +272,7 @@
EnableIntrinsicFunctions="true"
OmitFramePointers="true"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\programs\legacy"
PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
PreprocessorDefinitions="ZSTD_DISABLE_DEPRECATE_WARNINGS=1;WIN32;NDEBUG;_CONSOLE"
RuntimeLibrary="0"
EnableFunctionLevelLinking="true"
UsePrecompiledHeader="0"

8
build/VS2008/fuzzer/fuzzer.vcproj
View File

@ -45,7 +45,7 @@
Name="VCCLCompilerTool"
Optimization="0"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\compress;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs"
PreprocessorDefinitions="ZSTD_MULTITHREAD=1;WIN32;_DEBUG;_CONSOLE"
PreprocessorDefinitions="ZSTD_DISABLE_DEPRECATE_WARNINGS=1;ZSTD_MULTITHREAD=1;WIN32;_DEBUG;_CONSOLE"
MinimalRebuild="true"
BasicRuntimeChecks="3"
RuntimeLibrary="3"
@ -121,7 +121,7 @@
EnableIntrinsicFunctions="true"
OmitFramePointers="true"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\compress;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs"
PreprocessorDefinitions="ZSTD_MULTITHREAD=1;WIN32;NDEBUG;_CONSOLE"
PreprocessorDefinitions="ZSTD_DISABLE_DEPRECATE_WARNINGS=1;ZSTD_MULTITHREAD=1;WIN32;NDEBUG;_CONSOLE"
RuntimeLibrary="0"
EnableFunctionLevelLinking="true"
UsePrecompiledHeader="0"
@ -195,7 +195,7 @@
Name="VCCLCompilerTool"
Optimization="0"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\compress;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs"
PreprocessorDefinitions="ZSTD_MULTITHREAD=1;WIN32;_DEBUG;_CONSOLE"
PreprocessorDefinitions="ZSTD_DISABLE_DEPRECATE_WARNINGS=1;ZSTD_MULTITHREAD=1;WIN32;_DEBUG;_CONSOLE"
MinimalRebuild="true"
BasicRuntimeChecks="3"
RuntimeLibrary="3"
@ -272,7 +272,7 @@
EnableIntrinsicFunctions="true"
OmitFramePointers="true"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\compress;$(SolutionDir)..\..\programs"
PreprocessorDefinitions="ZSTD_MULTITHREAD=1;WIN32;NDEBUG;_CONSOLE"
PreprocessorDefinitions="ZSTD_DISABLE_DEPRECATE_WARNINGS=1;ZSTD_MULTITHREAD=1;WIN32;NDEBUG;_CONSOLE"
RuntimeLibrary="0"
EnableFunctionLevelLinking="true"
UsePrecompiledHeader="0"

8
build/VS2010/fullbench-dll/fullbench-dll.vcxproj
View File

@ -90,7 +90,7 @@
</PrecompiledHeader>
<WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;ZSTD_DLL_IMPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<PreprocessorDefinitions>ZSTD_DISABLE_DEPRECATE_WARNINGS=1;WIN32;_DEBUG;_CONSOLE;ZSTD_DLL_IMPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
<EnablePREfast>false</EnablePREfast>
</ClCompile>
@ -108,7 +108,7 @@
</PrecompiledHeader>
<WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;ZSTD_DLL_IMPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<PreprocessorDefinitions>ZSTD_DISABLE_DEPRECATE_WARNINGS=1;WIN32;_DEBUG;_CONSOLE;ZSTD_DLL_IMPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
<EnablePREfast>false</EnablePREfast>
</ClCompile>
@ -127,7 +127,7 @@
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;ZSTD_DLL_IMPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<PreprocessorDefinitions>ZSTD_DISABLE_DEPRECATE_WARNINGS=1;WIN32;_DEBUG;_CONSOLE;ZSTD_DLL_IMPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<EnablePREfast>false</EnablePREfast>
<TreatWarningAsError>false</TreatWarningAsError>
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
@ -150,7 +150,7 @@
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;ZSTD_DLL_IMPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<PreprocessorDefinitions>ZSTD_DISABLE_DEPRECATE_WARNINGS=1;WIN32;_DEBUG;_CONSOLE;ZSTD_DLL_IMPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>false</TreatWarningAsError>
<EnablePREfast>false</EnablePREfast>
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>

8
build/VS2010/fullbench/fullbench.vcxproj
View File

@ -90,7 +90,7 @@
</PrecompiledHeader>
<WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<PreprocessorDefinitions>ZSTD_DISABLE_DEPRECATE_WARNINGS=1;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
<EnablePREfast>false</EnablePREfast>
</ClCompile>
@ -105,7 +105,7 @@
</PrecompiledHeader>
<WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<PreprocessorDefinitions>ZSTD_DISABLE_DEPRECATE_WARNINGS=1;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
<EnablePREfast>false</EnablePREfast>
</ClCompile>
@ -122,7 +122,7 @@
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<PreprocessorDefinitions>ZSTD_DISABLE_DEPRECATE_WARNINGS=1;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<EnablePREfast>false</EnablePREfast>
<TreatWarningAsError>false</TreatWarningAsError>
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
@ -142,7 +142,7 @@
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<PreprocessorDefinitions>ZSTD_DISABLE_DEPRECATE_WARNINGS=1;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>false</TreatWarningAsError>
<EnablePREfast>false</EnablePREfast>
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>

8
build/VS2010/fuzzer/fuzzer.vcxproj
View File

@ -90,7 +90,7 @@
</PrecompiledHeader>
<WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<PreprocessorDefinitions>ZSTD_DISABLE_DEPRECATE_WARNINGS=1;ZSTD_MULTITHREAD=1;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
<EnablePREfast>false</EnablePREfast>
</ClCompile>
@ -105,7 +105,7 @@
</PrecompiledHeader>
<WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<PreprocessorDefinitions>ZSTD_DISABLE_DEPRECATE_WARNINGS=1;ZSTD_MULTITHREAD=1;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
<EnablePREfast>false</EnablePREfast>
</ClCompile>
@ -122,7 +122,7 @@
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<PreprocessorDefinitions>ZSTD_DISABLE_DEPRECATE_WARNINGS=1;ZSTD_MULTITHREAD=1;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<EnablePREfast>false</EnablePREfast>
<TreatWarningAsError>false</TreatWarningAsError>
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
@ -142,7 +142,7 @@
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<PreprocessorDefinitions>ZSTD_DISABLE_DEPRECATE_WARNINGS=1;ZSTD_MULTITHREAD=1;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>false</TreatWarningAsError>
<EnablePREfast>false</EnablePREfast>
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>

5
build/cmake/CMakeModules/AddZstdCompilationFlags.cmake
View File

@ -26,7 +26,12 @@ macro(ADD_ZSTD_COMPILATION_FLAGS)
EnableCompilerFlag("-std=c++11" false true)
#Set c99 by default
EnableCompilerFlag("-std=c99" true false)
if (CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND MSVC)
# clang-cl normally maps -Wall to -Weverything.
EnableCompilerFlag("/clang:-Wall" true true)
else ()
EnableCompilerFlag("-Wall" true true)
endif ()
EnableCompilerFlag("-Wextra" true true)
EnableCompilerFlag("-Wundef" true true)
EnableCompilerFlag("-Wshadow" true true)

8
build/cmake/lib/CMakeLists.txt
View File

@ -24,28 +24,24 @@ file(GLOB CommonSources ${LIBRARY_DIR}/common/*.c)
file(GLOB CompressSources ${LIBRARY_DIR}/compress/*.c)
file(GLOB DecompressSources ${LIBRARY_DIR}/decompress/*.c)
file(GLOB DictBuilderSources ${LIBRARY_DIR}/dictBuilder/*.c)
file(GLOB DeprecatedSources ${LIBRARY_DIR}/deprecated/*.c)
set(Sources
${CommonSources}
${CompressSources}
${DecompressSources}
${DictBuilderSources}
${DeprecatedSources})
${DictBuilderSources})
file(GLOB CommonHeaders ${LIBRARY_DIR}/common/*.h)
file(GLOB CompressHeaders ${LIBRARY_DIR}/compress/*.h)
file(GLOB DecompressHeaders ${LIBRARY_DIR}/decompress/*.h)
file(GLOB DictBuilderHeaders ${LIBRARY_DIR}/dictBuilder/*.h)
file(GLOB DeprecatedHeaders ${LIBRARY_DIR}/deprecated/*.h)
set(Headers
${LIBRARY_DIR}/zstd.h
${CommonHeaders}
${CompressHeaders}
${DecompressHeaders}
${DictBuilderHeaders}
${DeprecatedHeaders})
${DictBuilderHeaders})
if (ZSTD_LEGACY_SUPPORT)
set(LIBRARY_LEGACY_DIR ${LIBRARY_DIR}/legacy)

3
build/cmake/tests/CMakeLists.txt
View File

@ -57,6 +57,7 @@ target_link_libraries(datagen libzstd_static)
# fullbench
#
add_executable(fullbench ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${TESTS_DIR}/fullbench.c)
set_property(TARGET fullbench APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_DISABLE_DEPRECATE_WARNINGS")
target_link_libraries(fullbench libzstd_static)
add_test(NAME fullbench COMMAND fullbench ${ZSTD_FULLBENCH_FLAGS})
@ -64,6 +65,7 @@ add_test(NAME fullbench COMMAND fullbench ${ZSTD_FULLBENCH_FLAGS})
# fuzzer
#
add_executable(fuzzer ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/fuzzer.c)
set_property(TARGET fuzzer APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_DISABLE_DEPRECATE_WARNINGS")
target_link_libraries(fuzzer libzstd_static)
AddTestFlagsOption(ZSTD_FUZZER_FLAGS "$ENV{FUZZERTEST} $ENV{FUZZER_FLAGS}"
"Semicolon-separated list of flags to pass to the fuzzer test (see `fuzzer -h` for usage)")
@ -76,6 +78,7 @@ add_test(NAME fuzzer COMMAND fuzzer ${ZSTD_FUZZER_FLAGS})
# zstreamtest
#
add_executable(zstreamtest ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/seqgen.c ${TESTS_DIR}/zstreamtest.c)
set_property(TARGET zstreamtest APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_DISABLE_DEPRECATE_WARNINGS")
target_link_libraries(zstreamtest libzstd_static)
AddTestFlagsOption(ZSTD_ZSTREAM_FLAGS "$ENV{ZSTREAM_TESTTIME} $ENV{FUZZER_FLAGS}"
"Semicolon-separated list of flags to pass to the zstreamtest test (see `zstreamtest -h` for usage)")

3
build/meson/lib/meson.build
View File

@ -14,8 +14,7 @@ libzstd_includes = [include_directories(join_paths(zstd_rootdir,'lib'),
join_paths(zstd_rootdir, 'lib/common'),
join_paths(zstd_rootdir, 'lib/compress'),
join_paths(zstd_rootdir, 'lib/decompress'),
join_paths(zstd_rootdir, 'lib/dictBuilder'),
join_paths(zstd_rootdir, 'lib/deprecated'))]
join_paths(zstd_rootdir, 'lib/dictBuilder'))]
libzstd_sources = [join_paths(zstd_rootdir, 'lib/common/entropy_common.c'),
join_paths(zstd_rootdir, 'lib/common/fse_decompress.c'),

0
contrib/single_file_libs/.gitignore → build/single_file_libs/.gitignore vendored
View File

4
contrib/single_file_libs/README.md → build/single_file_libs/README.md
View File

@ -11,7 +11,7 @@ This is the most common use case. The decompression library is small, adding, fo
Create `zstddeclib.c` from the Zstd source using:
```
cd zstd/contrib/single_file_libs
cd zstd/build/single_file_libs
./combine.sh -r ../../lib -o zstddeclib.c zstddeclib-in.c
```
Then add the resulting file to your project (see the [example files](examples)).
@ -25,7 +25,7 @@ The same tool can amalgamate the entire Zstd library for ease of adding both com
Create `zstd.c` from the Zstd source using:
```
cd zstd/contrib/single_file_libs
cd zstd/build/single_file_libs
./combine.sh -r ../../lib -o zstd.c zstd-in.c
```
It's possible to create a compressor-only library but since the decompressor is so small in comparison this doesn't bring much of a gain (but for the curious, simply remove the files in the _decompress_ section at the end of `zstd-in.c`).

0
contrib/single_file_libs/build_decoder_test.sh → build/single_file_libs/build_decoder_test.sh
View File

0
contrib/single_file_libs/build_library_test.sh → build/single_file_libs/build_library_test.sh
View File

0
contrib/single_file_libs/combine.sh → build/single_file_libs/combine.sh
View File

0
contrib/single_file_libs/create_single_file_decoder.sh → build/single_file_libs/create_single_file_decoder.sh
View File

0
contrib/single_file_libs/create_single_file_library.sh → build/single_file_libs/create_single_file_library.sh
View File

0
contrib/single_file_libs/examples/README.md → build/single_file_libs/examples/README.md
View File

0
contrib/single_file_libs/examples/emscripten.c → build/single_file_libs/examples/emscripten.c
View File

0
contrib/single_file_libs/examples/roundtrip.c → build/single_file_libs/examples/roundtrip.c
View File

0
contrib/single_file_libs/examples/shell.html → build/single_file_libs/examples/shell.html
View File

0
contrib/single_file_libs/examples/simple.c → build/single_file_libs/examples/simple.c
View File

0
contrib/single_file_libs/examples/testcard-dxt1.inl → build/single_file_libs/examples/testcard-dxt1.inl
View File

0
contrib/single_file_libs/examples/testcard-zstd.inl → build/single_file_libs/examples/testcard-zstd.inl
View File

0
contrib/single_file_libs/examples/testcard.png → build/single_file_libs/examples/testcard.png
View File

Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 12 KiB

After

Width:  |  Height:  |  Size: 12 KiB

0
contrib/single_file_libs/zstd-in.c → build/single_file_libs/zstd-in.c
View File

0
contrib/single_file_libs/zstddeclib-in.c → build/single_file_libs/zstddeclib-in.c
View File

4
contrib/VS2005/zstdlib/zstdlib.vcproj
View File

@ -473,10 +473,6 @@
RelativePath="..\..\..\lib\common\xxhash.h"
>
</File>
<File
RelativePath="..\..\..\lib\common\zbuff.h"
>
</File>
<File
RelativePath="..\..\..\lib\zdict.h"
>

2
contrib/linux-kernel/test/Makefile
View File

@ -11,7 +11,7 @@
LINUX := ../linux
LINUX_ZSTDLIB := $(LINUX)/lib/zstd
CPPFLAGS += -I$(LINUX)/include -I$(LINUX_ZSTDLIB) -Iinclude -DNDEBUG
CPPFLAGS += -I$(LINUX)/include -I$(LINUX_ZSTDLIB) -Iinclude -DNDEBUG -Wno-deprecated-declarations
# Don't poison the workspace, it currently doesn't work with static allocation and workspace reuse
CPPFLAGS += -DZSTD_ASAN_DONT_POISON_WORKSPACE

3
contrib/pzstd/Makefile
View File

@ -30,6 +30,9 @@ CXXFLAGS ?= -O3 -Wall -Wextra -pedantic
CPPFLAGS ?=
LDFLAGS ?=
# PZstd uses legacy APIs
CFLAGS += -Wno-deprecated-declarations
# Include flags
PZSTD_INC = -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(PROGDIR) -I.
GTEST_INC = -isystem googletest/googletest/include

2
contrib/seekable_format/examples/Makefile
View File

@ -13,7 +13,7 @@ ZSTDLIB_PATH = ../../../lib
ZSTDLIB_NAME = libzstd.a
ZSTDLIB = $(ZSTDLIB_PATH)/$(ZSTDLIB_NAME)
CPPFLAGS += -I../ -I../../../lib -I../../../lib/common
CPPFLAGS += -DXXH_NAMESPACE=ZSTD_ -I../ -I../../../lib -I../../../lib/common
CFLAGS ?= -O3
CFLAGS += -g

1
contrib/seekable_format/examples/parallel_compression.c
View File

@ -21,7 +21,6 @@
# define SLEEP(x) usleep(x * 1000)
#endif
#define XXH_NAMESPACE ZSTD_
#include "xxhash.h"
#include "pool.h" // use zstd thread pool for demo

2
contrib/seekable_format/tests/Makefile
View File

@ -13,7 +13,7 @@ ZSTDLIB_PATH = ../../../lib
ZSTDLIB_NAME = libzstd.a
ZSTDLIB = $(ZSTDLIB_PATH)/$(ZSTDLIB_NAME)
CPPFLAGS += -I../ -I$(ZSTDLIB_PATH) -I$(ZSTDLIB_PATH)/common
CPPFLAGS += -DXXH_NAMESPACE=ZSTD_ -I../ -I$(ZSTDLIB_PATH) -I$(ZSTDLIB_PATH)/common
CFLAGS ?= -O3
CFLAGS += -g -Wall -Wextra -Wcast-qual -Wcast-align -Wconversion \

7
contrib/seekable_format/zstdseek_compress.c
View File

@ -12,7 +12,6 @@
#include <assert.h>
#define XXH_STATIC_LINKING_ONLY
#define XXH_NAMESPACE ZSTD_
#include "xxhash.h"
#define ZSTD_STATIC_LINKING_ONLY
@ -83,7 +82,7 @@ static size_t ZSTD_seekable_frameLog_freeVec(ZSTD_frameLog* fl)
ZSTD_frameLog* ZSTD_seekable_createFrameLog(int checksumFlag)
{
ZSTD_frameLog* const fl = malloc(sizeof(ZSTD_frameLog));
ZSTD_frameLog* const fl = (ZSTD_frameLog*)malloc(sizeof(ZSTD_frameLog));
if (fl == NULL) return NULL;
if (ZSTD_isError(ZSTD_seekable_frameLog_allocVec(fl))) {
@ -108,7 +107,7 @@ size_t ZSTD_seekable_freeFrameLog(ZSTD_frameLog* fl)
ZSTD_seekable_CStream* ZSTD_seekable_createCStream(void)
{
ZSTD_seekable_CStream* const zcs = malloc(sizeof(ZSTD_seekable_CStream));
ZSTD_seekable_CStream* const zcs = (ZSTD_seekable_CStream*)malloc(sizeof(ZSTD_seekable_CStream));
if (zcs == NULL) return NULL;
memset(zcs, 0, sizeof(*zcs));
@ -177,7 +176,7 @@ size_t ZSTD_seekable_logFrame(ZSTD_frameLog* fl,
if (fl->size == fl->capacity) {
/* exponential size increase for constant amortized runtime */
size_t const newCapacity = fl->capacity * 2;
framelogEntry_t* const newEntries = realloc(fl->entries,
framelogEntry_t* const newEntries = (framelogEntry_t*)realloc(fl->entries,
sizeof(framelogEntry_t) * newCapacity);
if (newEntries == NULL) return ERROR(memory_allocation);

5
contrib/seekable_format/zstdseek_decompress.c
View File

@ -60,7 +60,6 @@
#include <assert.h>
#define XXH_STATIC_LINKING_ONLY
#define XXH_NAMESPACE ZSTD_
#include "xxhash.h"
#define ZSTD_STATIC_LINKING_ONLY
@ -176,7 +175,7 @@ struct ZSTD_seekable_s {
ZSTD_seekable* ZSTD_seekable_create(void)
{
ZSTD_seekable* const zs = malloc(sizeof(ZSTD_seekable));
ZSTD_seekable* const zs = (ZSTD_seekable*)malloc(sizeof(ZSTD_seekable));
if (zs == NULL) return NULL;
/* also initializes stage to zsds_init */
@ -202,7 +201,7 @@ size_t ZSTD_seekable_free(ZSTD_seekable* zs)
ZSTD_seekTable* ZSTD_seekTable_create_fromSeekable(const ZSTD_seekable* zs)
{
ZSTD_seekTable* const st = malloc(sizeof(ZSTD_seekTable));
ZSTD_seekTable* const st = (ZSTD_seekTable*)malloc(sizeof(ZSTD_seekTable));
if (st==NULL) return NULL;
st->checksumFlag = zs->seekTable.checksumFlag;

4
doc/zstd_manual.html
View File

@ -1611,6 +1611,10 @@ ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
Note: ZSTD_resetCStream() interprets pledgedSrcSize == 0 as ZSTD_CONTENTSIZE_UNKNOWN, but
ZSTD_CCtx_setPledgedSrcSize() does not do the same, so ZSTD_CONTENTSIZE_UNKNOWN must be
explicitly specified.
start a new frame, using same parameters from previous frame.
This is typically useful to skip dictionary loading stage, since it will re-use it in-place.
Note that zcs must be init at least once before using ZSTD_resetCStream().

147
lib/compress/zstd_compress.c
View File

@ -559,6 +559,11 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param)
bounds.upperBound = (int)ZSTD_urm_enableRowMatchFinder;
return bounds;
case ZSTD_c_deterministicRefPrefix:
bounds.lowerBound = 0;
bounds.upperBound = 1;
return bounds;
default:
bounds.error = ERROR(parameter_unsupported);
return bounds;
@ -622,6 +627,7 @@ static int ZSTD_isUpdateAuthorized(ZSTD_cParameter param)
case ZSTD_c_validateSequences:
case ZSTD_c_splitBlocks:
case ZSTD_c_useRowMatchFinder:
case ZSTD_c_deterministicRefPrefix:
default:
return 0;
}
@ -676,6 +682,7 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value)
case ZSTD_c_validateSequences:
case ZSTD_c_splitBlocks:
case ZSTD_c_useRowMatchFinder:
case ZSTD_c_deterministicRefPrefix:
break;
default: RETURN_ERROR(parameter_unsupported, "unknown parameter");
@ -897,6 +904,11 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams,
CCtxParams->useRowMatchFinder = (ZSTD_useRowMatchFinderMode_e)value;
return CCtxParams->useRowMatchFinder;
case ZSTD_c_deterministicRefPrefix:
BOUNDCHECK(ZSTD_c_deterministicRefPrefix, value);
CCtxParams->deterministicRefPrefix = !!value;
return CCtxParams->deterministicRefPrefix;
default: RETURN_ERROR(parameter_unsupported, "unknown parameter");
}
}
@ -1026,6 +1038,9 @@ size_t ZSTD_CCtxParams_getParameter(
case ZSTD_c_useRowMatchFinder :
*value = (int)CCtxParams->useRowMatchFinder;
break;
case ZSTD_c_deterministicRefPrefix:
*value = (int)CCtxParams->deterministicRefPrefix;
break;
default: RETURN_ERROR(parameter_unsupported, "unknown parameter");
}
return 0;
@ -1802,7 +1817,7 @@ static int ZSTD_dictTooBig(size_t const loadedDictSize)
* note : `params` are assumed fully validated at this stage.
*/
static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
ZSTD_CCtx_params params,
ZSTD_CCtx_params const* params,
U64 const pledgedSrcSize,
size_t const loadedDictSize,
ZSTD_compResetPolicy_e const crp,
@ -1810,30 +1825,36 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
{
ZSTD_cwksp* const ws = &zc->workspace;
DEBUGLOG(4, "ZSTD_resetCCtx_internal: pledgedSrcSize=%u, wlog=%u, useRowMatchFinder=%d",
(U32)pledgedSrcSize, params.cParams.windowLog, (int)params.useRowMatchFinder);
assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
(U32)pledgedSrcSize, params->cParams.windowLog, (int)params->useRowMatchFinder);
assert(!ZSTD_isError(ZSTD_checkCParams(params->cParams)));
zc->isFirstBlock = 1;
assert(params.useRowMatchFinder != ZSTD_urm_auto);
if (params.ldmParams.enableLdm) {
/* Set applied params early so we can modify them for LDM,
* and point params at the applied params.
*/
zc->appliedParams = *params;
params = &zc->appliedParams;
assert(params->useRowMatchFinder != ZSTD_urm_auto);
if (params->ldmParams.enableLdm) {
/* Adjust long distance matching parameters */
ZSTD_ldm_adjustParameters(&params.ldmParams, &params.cParams);
assert(params.ldmParams.hashLog >= params.ldmParams.bucketSizeLog);
assert(params.ldmParams.hashRateLog < 32);
ZSTD_ldm_adjustParameters(&zc->appliedParams.ldmParams, &params->cParams);
assert(params->ldmParams.hashLog >= params->ldmParams.bucketSizeLog);
assert(params->ldmParams.hashRateLog < 32);
}
{ size_t const windowSize = MAX(1, (size_t)MIN(((U64)1 << params.cParams.windowLog), pledgedSrcSize));
{ size_t const windowSize = MAX(1, (size_t)MIN(((U64)1 << params->cParams.windowLog), pledgedSrcSize));
size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize);
U32 const divider = (params.cParams.minMatch==3) ? 3 : 4;
U32 const divider = (params->cParams.minMatch==3) ? 3 : 4;
size_t const maxNbSeq = blockSize / divider;
size_t const buffOutSize = (zbuff == ZSTDb_buffered && params.outBufferMode == ZSTD_bm_buffered)
size_t const buffOutSize = (zbuff == ZSTDb_buffered && params->outBufferMode == ZSTD_bm_buffered)
? ZSTD_compressBound(blockSize) + 1
: 0;
size_t const buffInSize = (zbuff == ZSTDb_buffered && params.inBufferMode == ZSTD_bm_buffered)
size_t const buffInSize = (zbuff == ZSTDb_buffered && params->inBufferMode == ZSTD_bm_buffered)
? windowSize + blockSize
: 0;
size_t const maxNbLdmSeq = ZSTD_ldm_getMaxNbSeq(params.ldmParams, blockSize);
size_t const maxNbLdmSeq = ZSTD_ldm_getMaxNbSeq(params->ldmParams, blockSize);
int const indexTooClose = ZSTD_indexTooCloseToMax(zc->blockState.matchState.window);
int const dictTooBig = ZSTD_dictTooBig(loadedDictSize);
@ -1842,7 +1863,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
size_t const neededSpace =
ZSTD_estimateCCtxSize_usingCCtxParams_internal(
&params.cParams, &params.ldmParams, zc->staticSize != 0, params.useRowMatchFinder,
&params->cParams, &params->ldmParams, zc->staticSize != 0, params->useRowMatchFinder,
buffInSize, buffOutSize, pledgedSrcSize);
int resizeWorkspace;
@ -1885,8 +1906,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
ZSTD_cwksp_clear(ws);
/* init params */
zc->appliedParams = params;
zc->blockState.matchState.cParams = params.cParams;
zc->blockState.matchState.cParams = params->cParams;
zc->pledgedSrcSizePlusOne = pledgedSrcSize+1;
zc->consumedSrcSize = 0;
zc->producedCSize = 0;
@ -1917,11 +1937,11 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
zc->outBuff = (char*)ZSTD_cwksp_reserve_buffer(ws, buffOutSize);
/* ldm bucketOffsets table */
if (params.ldmParams.enableLdm) {
if (params->ldmParams.enableLdm) {
/* TODO: avoid memset? */
size_t const numBuckets =
((size_t)1) << (params.ldmParams.hashLog -
params.ldmParams.bucketSizeLog);
((size_t)1) << (params->ldmParams.hashLog -
params->ldmParams.bucketSizeLog);
zc->ldmState.bucketOffsets = ZSTD_cwksp_reserve_buffer(ws, numBuckets);
ZSTD_memset(zc->ldmState.bucketOffsets, 0, numBuckets);
}
@ -1937,16 +1957,16 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
FORWARD_IF_ERROR(ZSTD_reset_matchState(
&zc->blockState.matchState,
ws,
&params.cParams,
params.useRowMatchFinder,
&params->cParams,
params->useRowMatchFinder,
crp,
needsIndexReset,
ZSTD_resetTarget_CCtx), "");
/* ldm hash table */
if (params.ldmParams.enableLdm) {
if (params->ldmParams.enableLdm) {
/* TODO: avoid memset? */
size_t const ldmHSize = ((size_t)1) << params.ldmParams.hashLog;
size_t const ldmHSize = ((size_t)1) << params->ldmParams.hashLog;
zc->ldmState.hashTable = (ldmEntry_t*)ZSTD_cwksp_reserve_aligned(ws, ldmHSize * sizeof(ldmEntry_t));
ZSTD_memset(zc->ldmState.hashTable, 0, ldmHSize * sizeof(ldmEntry_t));
zc->ldmSequences = (rawSeq*)ZSTD_cwksp_reserve_aligned(ws, maxNbLdmSeq * sizeof(rawSeq));
@ -2014,7 +2034,8 @@ ZSTD_resetCCtx_byAttachingCDict(ZSTD_CCtx* cctx,
U64 pledgedSrcSize,
ZSTD_buffered_policy_e zbuff)
{
DEBUGLOG(4, "ZSTD_resetCCtx_byAttachingCDict() pledgedSrcSize=%zu", pledgedSrcSize);
DEBUGLOG(4, "ZSTD_resetCCtx_byAttachingCDict() pledgedSrcSize=%llu",
(unsigned long long)pledgedSrcSize);
{
ZSTD_compressionParameters adjusted_cdict_cParams = cdict->matchState.cParams;
unsigned const windowLog = params.cParams.windowLog;
@ -2031,7 +2052,7 @@ ZSTD_resetCCtx_byAttachingCDict(ZSTD_CCtx* cctx,
cdict->dictContentSize, ZSTD_cpm_attachDict);
params.cParams.windowLog = windowLog;
params.useRowMatchFinder = cdict->useRowMatchFinder; /* cdict overrides */
FORWARD_IF_ERROR(ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize,
FORWARD_IF_ERROR(ZSTD_resetCCtx_internal(cctx, &params, pledgedSrcSize,
/* loadedDictSize */ 0,
ZSTDcrp_makeClean, zbuff), "");
assert(cctx->appliedParams.cParams.strategy == adjusted_cdict_cParams.strategy);
@ -2076,7 +2097,8 @@ static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx,
const ZSTD_compressionParameters *cdict_cParams = &cdict->matchState.cParams;
assert(!cdict->matchState.dedicatedDictSearch);
DEBUGLOG(4, "ZSTD_resetCCtx_byCopyingCDict() pledgedSrcSize=%zu", pledgedSrcSize);
DEBUGLOG(4, "ZSTD_resetCCtx_byCopyingCDict() pledgedSrcSize=%llu",
(unsigned long long)pledgedSrcSize);
{ unsigned const windowLog = params.cParams.windowLog;
assert(windowLog != 0);
@ -2084,7 +2106,7 @@ static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx,
params.cParams = *cdict_cParams;
params.cParams.windowLog = windowLog;
params.useRowMatchFinder = cdict->useRowMatchFinder;
FORWARD_IF_ERROR(ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize,
FORWARD_IF_ERROR(ZSTD_resetCCtx_internal(cctx, &params, pledgedSrcSize,
/* loadedDictSize */ 0,
ZSTDcrp_leaveDirty, zbuff), "");
assert(cctx->appliedParams.cParams.strategy == cdict_cParams->strategy);
@ -2190,7 +2212,7 @@ static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx,
assert(srcCCtx->appliedParams.useRowMatchFinder != ZSTD_urm_auto);
params.useRowMatchFinder = srcCCtx->appliedParams.useRowMatchFinder;
params.fParams = fParams;
ZSTD_resetCCtx_internal(dstCCtx, params, pledgedSrcSize,
ZSTD_resetCCtx_internal(dstCCtx, &params, pledgedSrcSize,
/* loadedDictSize */ 0,
ZSTDcrp_leaveDirty, zbuff);
assert(dstCCtx->appliedParams.cParams.windowLog == srcCCtx->appliedParams.cParams.windowLog);
@ -4045,11 +4067,12 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx,
if (!srcSize) return fhSize; /* do not generate an empty block if no input */
if (!ZSTD_window_update(&ms->window, src, srcSize)) {
if (!ZSTD_window_update(&ms->window, src, srcSize, ms->forceNonContiguous)) {
ms->forceNonContiguous = 0;
ms->nextToUpdate = ms->window.dictLimit;
}
if (cctx->appliedParams.ldmParams.enableLdm) {
ZSTD_window_update(&cctx->ldmState.window, src, srcSize);
ZSTD_window_update(&cctx->ldmState.window, src, srcSize, /* forceNonContiguous */ 0);
}
if (!frame) {
@ -4141,11 +4164,12 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms,
}
DEBUGLOG(4, "ZSTD_loadDictionaryContent(): useRowMatchFinder=%d", (int)params->useRowMatchFinder);
ZSTD_window_update(&ms->window, src, srcSize);
ZSTD_window_update(&ms->window, src, srcSize, /* forceNonContiguous */ 0);
ms->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ms->window.base);
ms->forceNonContiguous = params->deterministicRefPrefix;
if (loadLdmDict) {
ZSTD_window_update(&ls->window, src, srcSize);
ZSTD_window_update(&ls->window, src, srcSize, /* forceNonContiguous */ 0);
ls->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ls->window.base);
}
@ -4422,7 +4446,7 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
return ZSTD_resetCCtx_usingCDict(cctx, cdict, params, pledgedSrcSize, zbuff);
}
FORWARD_IF_ERROR( ZSTD_resetCCtx_internal(cctx, *params, pledgedSrcSize,
FORWARD_IF_ERROR( ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize,
dictContentSize,
ZSTDcrp_makeClean, zbuff) , "");
{ size_t const dictID = cdict ?
@ -4591,15 +4615,14 @@ size_t ZSTD_compress_advanced (ZSTD_CCtx* cctx,
const void* dict,size_t dictSize,
ZSTD_parameters params)
{
ZSTD_CCtx_params cctxParams;
DEBUGLOG(4, "ZSTD_compress_advanced");
FORWARD_IF_ERROR(ZSTD_checkCParams(params.cParams), "");
ZSTD_CCtxParams_init_internal(&cctxParams, &params, ZSTD_NO_CLEVEL);
ZSTD_CCtxParams_init_internal(&cctx->simpleApiParams, &params, ZSTD_NO_CLEVEL);
return ZSTD_compress_advanced_internal(cctx,
dst, dstCapacity,
src, srcSize,
dict, dictSize,
&cctxParams);
&cctx->simpleApiParams);
}
/* Internal */
@ -4623,14 +4646,13 @@ size_t ZSTD_compress_usingDict(ZSTD_CCtx* cctx,
const void* dict, size_t dictSize,
int compressionLevel)
{
ZSTD_CCtx_params cctxParams;
{
ZSTD_parameters const params = ZSTD_getParams_internal(compressionLevel, srcSize, dict ? dictSize : 0, ZSTD_cpm_noAttachDict);
assert(params.fParams.contentSizeFlag == 1);
ZSTD_CCtxParams_init_internal(&cctxParams, &params, (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT: compressionLevel);
ZSTD_CCtxParams_init_internal(&cctx->simpleApiParams, &params, (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT: compressionLevel);
}
DEBUGLOG(4, "ZSTD_compress_usingDict (srcSize=%u)", (unsigned)srcSize);
return ZSTD_compress_advanced_internal(cctx, dst, dstCapacity, src, srcSize, dict, dictSize, &cctxParams);
return ZSTD_compress_advanced_internal(cctx, dst, dstCapacity, src, srcSize, dict, dictSize, &cctx->simpleApiParams);
}
size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx,
@ -4963,15 +4985,15 @@ unsigned ZSTD_getDictID_fromCDict(const ZSTD_CDict* cdict)
return cdict->dictID;
}
/* ZSTD_compressBegin_usingCDict_advanced() :
* cdict must be != NULL */
size_t ZSTD_compressBegin_usingCDict_advanced(
/* ZSTD_compressBegin_usingCDict_internal() :
* Implementation of various ZSTD_compressBegin_usingCDict* functions.
*/
static size_t ZSTD_compressBegin_usingCDict_internal(
ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict,
ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize)
{
ZSTD_CCtx_params cctxParams;
DEBUGLOG(4, "ZSTD_compressBegin_usingCDict_advanced");
DEBUGLOG(4, "ZSTD_compressBegin_usingCDict_internal");
RETURN_ERROR_IF(cdict==NULL, dictionary_wrong, "NULL pointer!");
/* Initialize the cctxParams from the cdict */
{
@ -5003,23 +5025,46 @@ size_t ZSTD_compressBegin_usingCDict_advanced(
ZSTDb_not_buffered);
}
/* ZSTD_compressBegin_usingCDict_advanced() :
* This function is DEPRECATED.
* cdict must be != NULL */
size_t ZSTD_compressBegin_usingCDict_advanced(
ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict,
ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize)
{
return ZSTD_compressBegin_usingCDict_internal(cctx, cdict, fParams, pledgedSrcSize);
}
/* ZSTD_compressBegin_usingCDict() :
* pledgedSrcSize=0 means "unknown"
* if pledgedSrcSize>0, it will enable contentSizeFlag */
* cdict must be != NULL */
size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict)
{
ZSTD_frameParameters const fParams = { 0 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ };
DEBUGLOG(4, "ZSTD_compressBegin_usingCDict : dictIDFlag == %u", !fParams.noDictIDFlag);
return ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, ZSTD_CONTENTSIZE_UNKNOWN);
return ZSTD_compressBegin_usingCDict_internal(cctx, cdict, fParams, ZSTD_CONTENTSIZE_UNKNOWN);
}
/*! ZSTD_compress_usingCDict_internal():
* Implementation of various ZSTD_compress_usingCDict* functions.
*/
static size_t ZSTD_compress_usingCDict_internal(ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
const ZSTD_CDict* cdict, ZSTD_frameParameters fParams)
{
FORWARD_IF_ERROR(ZSTD_compressBegin_usingCDict_internal(cctx, cdict, fParams, srcSize), ""); /* will check if cdict != NULL */
return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize);
}
/*! ZSTD_compress_usingCDict_advanced():
* This function is DEPRECATED.
*/
size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
const ZSTD_CDict* cdict, ZSTD_frameParameters fParams)
{
FORWARD_IF_ERROR(ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, srcSize), ""); /* will check if cdict != NULL */
return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize);
return ZSTD_compress_usingCDict_internal(cctx, dst, dstCapacity, src, srcSize, cdict, fParams);
}
/*! ZSTD_compress_usingCDict() :
@ -5033,7 +5078,7 @@ size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
const ZSTD_CDict* cdict)
{
ZSTD_frameParameters const fParams = { 1 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ };
return ZSTD_compress_usingCDict_advanced(cctx, dst, dstCapacity, src, srcSize, cdict, fParams);
return ZSTD_compress_usingCDict_internal(cctx, dst, dstCapacity, src, srcSize, cdict, fParams);
}

11
lib/compress/zstd_compress_internal.h
View File

@ -219,6 +219,8 @@ struct ZSTD_matchState_t {
U32* hashTable3;
U32* chainTable;
U32 forceNonContiguous; /* Non-zero if we should force non-contiguous load for the next window update. */
int dedicatedDictSearch; /* Indicates whether this matchState is using the
* dedicated dictionary search structure.
*/
@ -317,6 +319,9 @@ struct ZSTD_CCtx_params_s {
/* Param for deciding whether to use row-based matchfinder */
ZSTD_useRowMatchFinderMode_e useRowMatchFinder;
/* Always load a dictionary in ext-dict mode (not prefix mode)? */
int deterministicRefPrefix;
/* Internal use, for createCCtxParams() and freeCCtxParams() only */
ZSTD_customMem customMem;
}; /* typedef'd to ZSTD_CCtx_params within "zstd.h" */
@ -340,6 +345,7 @@ struct ZSTD_CCtx_s {
int bmi2; /* == 1 if the CPU supports BMI2 and 0 otherwise. CPU support is determined dynamically once per context lifetime. */
ZSTD_CCtx_params requestedParams;
ZSTD_CCtx_params appliedParams;
ZSTD_CCtx_params simpleApiParams; /* Param storage used by the simple API - not sticky. Must only be used in top-level simple API functions for storage. */
U32 dictID;
size_t dictContentSize;
@ -1138,7 +1144,8 @@ MEM_STATIC void ZSTD_window_init(ZSTD_window_t* window) {
* Returns non-zero if the segment is contiguous.
*/
MEM_STATIC U32 ZSTD_window_update(ZSTD_window_t* window,
void const* src, size_t srcSize)
void const* src, size_t srcSize,
int forceNonContiguous)
{
BYTE const* const ip = (BYTE const*)src;
U32 contiguous = 1;
@ -1148,7 +1155,7 @@ MEM_STATIC U32 ZSTD_window_update(ZSTD_window_t* window,
assert(window->base != NULL);
assert(window->dictBase != NULL);
/* Check if blocks follow each other */
if (src != window->nextSrc) {
if (src != window->nextSrc || forceNonContiguous) {
/* not contiguous */
size_t const distanceFromBase = (size_t)(window->nextSrc - window->base);
DEBUGLOG(5, "Non contiguous blocks, new segment starts at %u", window->dictLimit);

14
lib/compress/zstdmt_compress.c
View File

@ -512,7 +512,7 @@ ZSTDMT_serialState_reset(serialState_t* serialState,
if (dictSize > 0) {
if (dictContentType == ZSTD_dct_rawContent) {
BYTE const* const dictEnd = (const BYTE*)dict + dictSize;
ZSTD_window_update(&serialState->ldmState.window, dict, dictSize);
ZSTD_window_update(&serialState->ldmState.window, dict, dictSize, /* forceNonContiguous */ 0);
ZSTD_ldm_fillHashTable(&serialState->ldmState, (const BYTE*)dict, dictEnd, &params.ldmParams);
serialState->ldmState.loadedDictEnd = params.forceWindow ? 0 : (U32)(dictEnd - serialState->ldmState.window.base);
} else {
@ -569,7 +569,7 @@ static void ZSTDMT_serialState_update(serialState_t* serialState,
assert(seqStore.seq != NULL && seqStore.pos == 0 &&
seqStore.size == 0 && seqStore.capacity > 0);
assert(src.size <= serialState->params.jobSize);
ZSTD_window_update(&serialState->ldmState.window, src.start, src.size);
ZSTD_window_update(&serialState->ldmState.window, src.start, src.size, /* forceNonContiguous */ 0);
error = ZSTD_ldm_generateSequences(
&serialState->ldmState, &seqStore,
&serialState->params.ldmParams, src.start, src.size);
@ -695,6 +695,10 @@ static void ZSTDMT_compressionJob(void* jobDescription)
{ size_t const forceWindowError = ZSTD_CCtxParams_setParameter(&jobParams, ZSTD_c_forceMaxWindow, !job->firstJob);
if (ZSTD_isError(forceWindowError)) JOB_ERROR(forceWindowError);
}
if (!job->firstJob) {
size_t const err = ZSTD_CCtxParams_setParameter(&jobParams, ZSTD_c_deterministicRefPrefix, 0);
if (ZSTD_isError(err)) JOB_ERROR(err);
}
{ size_t const initError = ZSTD_compressBegin_advanced_internal(cctx,
job->prefix.start, job->prefix.size, ZSTD_dct_rawContent, /* load dictionary in "content-only" mode (no header analysis) */
ZSTD_dtlm_fast,
@ -750,6 +754,12 @@ static void ZSTDMT_compressionJob(void* jobDescription)
if (ZSTD_isError(cSize)) JOB_ERROR(cSize);
lastCBlockSize = cSize;
} }
if (!job->firstJob) {
/* Double check that we don't have an ext-dict, because then our
* repcode invalidation doesn't work.
*/
assert(!ZSTD_window_hasExtDict(cctx->blockState.matchState.window));
}
ZSTD_CCtx_trace(cctx, 0);
_endJob:

148
lib/zdict.h
View File

@ -36,6 +36,145 @@ extern "C" {
# define ZDICTLIB_API ZDICTLIB_VISIBILITY
#endif
/*******************************************************************************
* Zstd dictionary builder
*
* FAQ
* ===
* Why should I use a dictionary?
* ------------------------------
*
* Zstd can use dictionaries to improve compression ratio of small data.
* Traditionally small files don't compress well because there is very little
* repetion in a single sample, since it is small. But, if you are compressing
* many similar files, like a bunch of JSON records that share the same
* structure, you can train a dictionary on ahead of time on some samples of
* these files. Then, zstd can use the dictionary to find repetitions that are
* present across samples. This can vastly improve compression ratio.
*
* When is a dictionary useful?
* ----------------------------
*
* Dictionaries are useful when compressing many small files that are similar.
* The larger a file is, the less benefit a dictionary will have. Generally,
* we don't expect dictionary compression to be effective past 100KB. And the
* smaller a file is, the more we would expect the dictionary to help.
*
* How do I use a dictionary?
* --------------------------
*
* Simply pass the dictionary to the zstd compressor with
* `ZSTD_CCtx_loadDictionary()`. The same dictionary must then be passed to
* the decompressor, using `ZSTD_DCtx_loadDictionary()`. There are other
* more advanced functions that allow selecting some options, see zstd.h for
* complete documentation.
*
* What is a zstd dictionary?
* --------------------------
*
* A zstd dictionary has two pieces: Its header, and its content. The header
* contains a magic number, the dictionary ID, and entropy tables. These
* entropy tables allow zstd to save on header costs in the compressed file,
* which really matters for small data. The content is just bytes, which are
* repeated content that is common across many samples.
*
* What is a raw content dictionary?
* ---------------------------------
*
* A raw content dictionary is just bytes. It doesn't have a zstd dictionary
* header, a dictionary ID, or entropy tables. Any buffer is a valid raw
* content dictionary.
*
* How do I train a dictionary?
* ----------------------------
*
* Gather samples from your use case. These samples should be similar to each
* other. If you have several use cases, you could try to train one dictionary
* per use case.
*
* Pass those samples to `ZDICT_trainFromBuffer()` and that will train your
* dictionary. There are a few advanced versions of this function, but this
* is a great starting point. If you want to further tune your dictionary
* you could try `ZDICT_optimizeTrainFromBuffer_cover()`. If that is too slow
* you can try `ZDICT_optimizeTrainFromBuffer_fastCover()`.
*
* If the dictionary training function fails, that is likely because you
* either passed too few samples, or a dictionary would not be effective
* for your data. Look at the messages that the dictionary trainer printed,
* if it doesn't say too few samples, then a dictionary would not be effective.
*
* How large should my dictionary be?
* ----------------------------------
*
* A reasonable dictionary size, the `dictBufferCapacity`, is about 100KB.
* The zstd CLI defaults to a 110KB dictionary. You likely don't need a
* dictionary larger than that. But, most use cases can get away with a
* smaller dictionary. The advanced dictionary builders can automatically
* shrink the dictionary for you, and select a the smallest size that
* doesn't hurt compression ratio too much. See the `shrinkDict` parameter.
* A smaller dictionary can save memory, and potentially speed up
* compression.
*
* How many samples should I provide to the dictionary builder?
* ------------------------------------------------------------
*
* We generally recommend passing ~100x the size of the dictionary
* in samples. A few thousand should suffice. Having too few samples
* can hurt the dictionaries effectiveness. Having more samples will
* only improve the dictionaries effectiveness. But having too many
* samples can slow down the dictionary builder.
*
* How do I determine if a dictionary will be effective?
* -----------------------------------------------------
*
* Simply train a dictionary and try it out. You can use zstd's built in
* benchmarking tool to test the dictionary effectiveness.
*
* # Benchmark levels 1-3 without a dictionary
* zstd -b1e3 -r /path/to/my/files
* # Benchmark levels 1-3 with a dictioanry
* zstd -b1e3 -r /path/to/my/files -D /path/to/my/dictionary
*
* When should I retrain a dictionary?
* -----------------------------------
*
* You should retrain a dictionary when its effectiveness drops. Dictionary
* effectiveness drops as the data you are compressing changes. Generally, we do
* expect dictionaries to "decay" over time, as your data changes, but the rate
* at which they decay depends on your use case. Internally, we regularly
* retrain dictionaries, and if the new dictionary performs significantly
* better than the old dictionary, we will ship the new dictionary.
*
* I have a raw content dictionary, how do I turn it into a zstd dictionary?
* -------------------------------------------------------------------------
*
* If you have a raw content dictionary, e.g. by manually constructing it, or
* using a third-party dictionary builder, you can turn it into a zstd
* dictionary by using `ZDICT_finalizeDictionary()`. You'll also have to
* provide some samples of the data. It will add the zstd header to the
* raw content, which contains a dictionary ID and entropy tables, which
* will improve compression ratio, and allow zstd to write the dictionary ID
* into the frame, if you so choose.
*
* Do I have to use zstd's dictionary builder?
* -------------------------------------------
*
* No! You can construct dictionary content however you please, it is just
* bytes. It will always be valid as a raw content dictionary. If you want
* a zstd dictionary, which can improve compression ratio, use
* `ZDICT_finalizeDictionary()`.
*
* What is the attack surface of a zstd dictionary?
* ------------------------------------------------
*
* Zstd is heavily fuzz tested, including loading fuzzed dictionaries, so
* zstd should never crash, or access out-of-bounds memory no matter what
* the dictionary is. However, if an attacker can control the dictionary
* during decompression, they can cause zstd to generate arbitrary bytes,
* just like if they controlled the compressed data.
*
******************************************************************************/
/*! ZDICT_trainFromBuffer():
* Train a dictionary from an array of samples.
@ -64,7 +203,14 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCap
typedef struct {
int compressionLevel; /*< optimize for a specific zstd compression level; 0 means default */
unsigned notificationLevel; /*< Write log to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */
unsigned dictID; /*< force dictID value; 0 means auto mode (32-bits random value) */
unsigned dictID; /*< force dictID value; 0 means auto mode (32-bits random value)
* NOTE: The zstd format reserves some dictionary IDs for future use.
* You may use them in private settings, but be warned that they
* may be used by zstd in a public dictionary registry in the future.
* These dictionary IDs are:
* - low range : <= 32767
* - high range : >= (2^31)
*/
} ZDICT_params_t;
/*! ZDICT_finalizeDictionary():

119
lib/zstd.h
View File

@ -449,7 +449,8 @@ typedef enum {
ZSTD_c_experimentalParam11=1008,
ZSTD_c_experimentalParam12=1009,
ZSTD_c_experimentalParam13=1010,
ZSTD_c_experimentalParam14=1011
ZSTD_c_experimentalParam14=1011,
ZSTD_c_experimentalParam15=1012
} ZSTD_cParameter;
typedef struct {
@ -1085,6 +1086,28 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
#if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY)
#define ZSTD_H_ZSTD_STATIC_LINKING_ONLY
/* Deprecation warnings :
* Should these warnings be a problem, it is generally possible to disable them,
* typically with -Wno-deprecated-declarations for gcc or _CRT_SECURE_NO_WARNINGS in Visual.
* Otherwise, it's also possible to define ZSTD_DISABLE_DEPRECATE_WARNINGS.
*/
#ifdef ZSTD_DISABLE_DEPRECATE_WARNINGS
# define ZSTD_DEPRECATED(message) ZSTDLIB_API /* disable deprecation warnings */
#else
# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
# define ZSTD_DEPRECATED(message) [[deprecated(message)]] ZSTDLIB_API
# elif (defined(GNUC) && (GNUC > 4 || (GNUC == 4 && GNUC_MINOR >= 5))) || defined(__clang__)
# define ZSTD_DEPRECATED(message) ZSTDLIB_API __attribute__((deprecated(message)))
# elif defined(__GNUC__) && (__GNUC__ >= 3)
# define ZSTD_DEPRECATED(message) ZSTDLIB_API __attribute__((deprecated))
# elif defined(_MSC_VER)
# define ZSTD_DEPRECATED(message) ZSTDLIB_API __declspec(deprecated(message))
# else
# pragma message("WARNING: You need to implement ZSTD_DEPRECATED for this compiler")
# define ZSTD_DEPRECATED(message) ZSTDLIB_API
# endif
#endif /* ZSTD_DISABLE_DEPRECATE_WARNINGS */
/****************************************************************************************
* experimental API (static linking only)
****************************************************************************************
@ -1620,18 +1643,20 @@ ZSTDLIB_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParame
/*! ZSTD_compress_advanced() :
* Note : this function is now DEPRECATED.
* It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_setParameter() and other parameter setters.
* This prototype will be marked as deprecated and generate compilation warning on reaching v1.5.x */
ZSTDLIB_API size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx,
* This prototype will generate compilation warnings. */
ZSTD_DEPRECATED("use ZSTD_compress2")
size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
const void* dict,size_t dictSize,
ZSTD_parameters params);
/*! ZSTD_compress_usingCDict_advanced() :
* Note : this function is now REDUNDANT.
* Note : this function is now DEPRECATED.
* It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_loadDictionary() and other parameter setters.
* This prototype will be marked as deprecated and generate compilation warning in some future version */
ZSTDLIB_API size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
* This prototype will generate compilation warnings. */
ZSTD_DEPRECATED("use ZSTD_compress2 with ZSTD_CCtx_loadDictionary")
size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
const ZSTD_CDict* cdict,
@ -1859,6 +1884,26 @@ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* pre
*/
#define ZSTD_c_useRowMatchFinder ZSTD_c_experimentalParam14
/* ZSTD_c_deterministicRefPrefix
* Default is 0 == disabled. Set to 1 to enable.
*
* Zstd produces different results for prefix compression when the prefix is
* directly adjacent to the data about to be compressed vs. when it isn't.
* This is because zstd detects that the two buffers are contiguous and it can
* use a more efficient match finding algorithm. However, this produces different
* results than when the two buffers are non-contiguous. This flag forces zstd
* to always load the prefix in non-contiguous mode, even if it happens to be
* adjacent to the data, to guarantee determinism.
*
* If you really care about determinism when using a dictionary or prefix,
* like when doing delta compression, you should select this option. It comes
* at a speed penalty of about ~2.5% if the dictionary and data happened to be
* contiguous, and is free if they weren't contiguous. We don't expect that
* intentionally making the dictionary and data contiguous will be worth the
* cost to memcpy() the data.
*/
#define ZSTD_c_deterministicRefPrefix ZSTD_c_experimentalParam15
/*! ZSTD_CCtx_getParameter() :
* Get the requested compression parameter value, selected by enum ZSTD_cParameter,
* and store it into int* value.
@ -2070,11 +2115,13 @@ ZSTDLIB_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param
/*! ZSTD_DCtx_setFormat() :
* This function is REDUNDANT. Prefer ZSTD_DCtx_setParameter().
* Instruct the decoder context about what kind of data to decode next.
* This instruction is mandatory to decode data without a fully-formed header,
* such ZSTD_f_zstd1_magicless for example.
* @return : 0, or an error code (which can be tested using ZSTD_isError()). */
ZSTDLIB_API size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format);
ZSTD_DEPRECATED("use ZSTD_DCtx_setParameter() instead")
size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format);
/*! ZSTD_decompressStream_simpleArgs() :
* Same as ZSTD_decompressStream(),
@ -2098,7 +2145,7 @@ ZSTDLIB_API size_t ZSTD_decompressStream_simpleArgs (
/*===== Advanced Streaming compression functions =====*/
/*! ZSTD_initCStream_srcSize() :
* This function is deprecated, and equivalent to:
* This function is DEPRECATED, and equivalent to:
* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
* ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any)
* ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
@ -2107,15 +2154,15 @@ ZSTDLIB_API size_t ZSTD_decompressStream_simpleArgs (
* pledgedSrcSize must be correct. If it is not known at init time, use
* ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs,
* "0" also disables frame content size field. It may be enabled in the future.
* Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
* This prototype will generate compilation warnings.
*/
ZSTDLIB_API size_t
ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
int compressionLevel,
unsigned long long pledgedSrcSize);
/*! ZSTD_initCStream_usingDict() :
* This function is deprecated, and is equivalent to:
* This function is DEPRECATED, and is equivalent to:
* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
* ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
* ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
@ -2124,15 +2171,15 @@ ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
* dict == NULL or dictSize < 8, in which case no dict is used.
* Note: dict is loaded with ZSTD_dct_auto (treated as a full zstd dictionary if
* it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.
* Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
* This prototype will generate compilation warnings.
*/
ZSTDLIB_API size_t
ZSTD_initCStream_usingDict(ZSTD_CStream* zcs,
ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs,
const void* dict, size_t dictSize,
int compressionLevel);
/*! ZSTD_initCStream_advanced() :
* This function is deprecated, and is approximately equivalent to:
* This function is DEPRECATED, and is approximately equivalent to:
* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
* // Pseudocode: Set each zstd parameter and leave the rest as-is.
* for ((param, value) : params) {
@ -2144,23 +2191,24 @@ ZSTD_initCStream_usingDict(ZSTD_CStream* zcs,
* dict is loaded with ZSTD_dct_auto and ZSTD_dlm_byCopy.
* pledgedSrcSize must be correct.
* If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN.
* Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
* This prototype will generate compilation warnings.
*/
ZSTDLIB_API size_t
ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
const void* dict, size_t dictSize,
ZSTD_parameters params,
unsigned long long pledgedSrcSize);
/*! ZSTD_initCStream_usingCDict() :
* This function is deprecated, and equivalent to:
* This function is DEPRECATED, and equivalent to:
* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
* ZSTD_CCtx_refCDict(zcs, cdict);
*
* note : cdict will just be referenced, and must outlive compression session
* Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
* This prototype will generate compilation warnings.
*/
ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);
ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions")
size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);
/*! ZSTD_initCStream_usingCDict_advanced() :
* This function is DEPRECATED, and is approximately equivalent to:
@ -2175,18 +2223,21 @@ ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDi
* same as ZSTD_initCStream_usingCDict(), with control over frame parameters.
* pledgedSrcSize must be correct. If srcSize is not known at init time, use
* value ZSTD_CONTENTSIZE_UNKNOWN.
* Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
* This prototype will generate compilation warnings.
*/
ZSTDLIB_API size_t
ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions")
size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
const ZSTD_CDict* cdict,
ZSTD_frameParameters fParams,
unsigned long long pledgedSrcSize);
/*! ZSTD_resetCStream() :
* This function is deprecated, and is equivalent to:
* This function is DEPRECATED, and is equivalent to:
* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
* ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
* Note: ZSTD_resetCStream() interprets pledgedSrcSize == 0 as ZSTD_CONTENTSIZE_UNKNOWN, but
* ZSTD_CCtx_setPledgedSrcSize() does not do the same, so ZSTD_CONTENTSIZE_UNKNOWN must be
* explicitly specified.
*
* start a new frame, using same parameters from previous frame.
* This is typically useful to skip dictionary loading stage, since it will re-use it in-place.
@ -2196,9 +2247,10 @@ ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
* For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs,
* but it will change to mean "empty" in future version, so use macro ZSTD_CONTENTSIZE_UNKNOWN instead.
* @return : 0, or an error code (which can be tested using ZSTD_isError())
* Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
* This prototype will generate compilation warnings.
*/
ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
typedef struct {
@ -2285,8 +2337,7 @@ ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);
ZSTD_CCtx object can be re-used multiple times within successive compression operations.
Start by initializing a context.
Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression,
or ZSTD_compressBegin_advanced(), for finer parameter control.
Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression.
It's also possible to duplicate a reference context which has already been initialized, using ZSTD_copyCCtx()
Then, consume your input using ZSTD_compressContinue().
@ -2311,15 +2362,17 @@ ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);
/*===== Buffer-less streaming compression functions =====*/
ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */
ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /**< note: fails if cdict==NULL */
ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */
ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */
ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
/* The ZSTD_compressBegin_advanced() and ZSTD_compressBegin_usingCDict_advanced() are now DEPRECATED and will generate a compiler warning */
ZSTD_DEPRECATED("use advanced API to access custom parameters")
size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */
ZSTD_DEPRECATED("use advanced API to access custom parameters")
size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */
/**
Buffer-less streaming decompression (synchronous mode)

12
programs/benchzstd.c
View File

@ -67,18 +67,10 @@ static const size_t maxMemory = (sizeof(size_t)==4) ?
/* *************************************
* console display
***************************************/
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
#define DISPLAY(...) { fprintf(stderr, __VA_ARGS__); fflush(NULL); }
#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
/* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */
static const U64 g_refreshRate = SEC_TO_MICRO / 6;
static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
#define DISPLAYUPDATE(l, ...) { if (displayLevel>=l) { \
if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (displayLevel>=4)) \
{ g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \
if (displayLevel>=4) fflush(stderr); } } }
/* *************************************
* Exceptions
@ -768,7 +760,7 @@ static int BMK_loadFiles(void* buffer, size_t bufferSize,
}
{ FILE* const f = fopen(fileNamesTable[n], "rb");
if (f==NULL) RETURN_ERROR_INT(10, "impossible to open file %s", fileNamesTable[n]);
DISPLAYUPDATE(2, "Loading %s... \r", fileNamesTable[n]);
DISPLAYLEVEL(2, "Loading %s... \r", fileNamesTable[n]);
if (fileSize > bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n; /* buffer too small - stop after this file */
{ size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f);
if (readSize != (size_t)fileSize) RETURN_ERROR_INT(11, "could not read %s", fileNamesTable[n]);

25
programs/fileio.c
View File

@ -93,10 +93,10 @@
struct FIO_display_prefs_s {
int displayLevel; /* 0 : no display; 1: errors; 2: + result + interaction + warnings; 3: + progression; 4: + information */
U32 noProgress;
FIO_progressSetting_e progressSetting;
};
static FIO_display_prefs_t g_display_prefs = {2, 0};
static FIO_display_prefs_t g_display_prefs = {2, FIO_ps_auto};
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
#define DISPLAYOUT(...) fprintf(stdout, __VA_ARGS__)
@ -105,10 +105,10 @@ static FIO_display_prefs_t g_display_prefs = {2, 0};
static const U64 g_refreshRate = SEC_TO_MICRO / 6;
static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
#define READY_FOR_UPDATE() (!g_display_prefs.noProgress && UTIL_clockSpanMicro(g_displayClock) > g_refreshRate)
#define READY_FOR_UPDATE() ((g_display_prefs.progressSetting != FIO_ps_never) && UTIL_clockSpanMicro(g_displayClock) > g_refreshRate)
#define DELAY_NEXT_UPDATE() { g_displayClock = UTIL_getTime(); }
#define DISPLAYUPDATE(l, ...) { \
if (g_display_prefs.displayLevel>=l && !g_display_prefs.noProgress) { \
if (g_display_prefs.displayLevel>=l && (g_display_prefs.progressSetting != FIO_ps_never)) { \
if (READY_FOR_UPDATE() || (g_display_prefs.displayLevel>=4)) { \
DELAY_NEXT_UPDATE(); \
DISPLAY(__VA_ARGS__); \
@ -430,7 +430,7 @@ void FIO_freeContext(FIO_ctx_t* const fCtx)
void FIO_setNotificationLevel(int level) { g_display_prefs.displayLevel=level; }
void FIO_setNoProgress(unsigned noProgress) { g_display_prefs.noProgress = noProgress; }
void FIO_setProgressSetting(FIO_progressSetting_e setting) { g_display_prefs.progressSetting = setting; }
/*-*************************************
@ -1404,24 +1404,25 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx,
(unsigned)(zfp.consumed >> 20),
(unsigned)(zfp.produced >> 20),
cShare );
} else { /* summarized notifications if == 2 */
DISPLAYLEVEL(2, "\r%79s\r", ""); /* Clear out the current displayed line */
} else if (g_display_prefs.displayLevel >= 2 || g_display_prefs.progressSetting == FIO_ps_always) {
/* Require level 2 or forcibly displayed progress counter for summarized updates */
DISPLAYLEVEL(1, "\r%79s\r", ""); /* Clear out the current displayed line */
if (fCtx->nbFilesTotal > 1) {
size_t srcFileNameSize = strlen(srcFileName);
/* Ensure that the string we print is roughly the same size each time */
if (srcFileNameSize > 18) {
const char* truncatedSrcFileName = srcFileName + srcFileNameSize - 15;
DISPLAYLEVEL(2, "Compress: %u/%u files. Current: ...%s ",
DISPLAYLEVEL(1, "Compress: %u/%u files. Current: ...%s ",
fCtx->currFileIdx+1, fCtx->nbFilesTotal, truncatedSrcFileName);
} else {
DISPLAYLEVEL(2, "Compress: %u/%u files. Current: %*s ",
DISPLAYLEVEL(1, "Compress: %u/%u files. Current: %*s ",
fCtx->currFileIdx+1, fCtx->nbFilesTotal, (int)(18-srcFileNameSize), srcFileName);
}
}
DISPLAYLEVEL(2, "Read : %2u ", (unsigned)(zfp.consumed >> 20));
DISPLAYLEVEL(1, "Read : %2u ", (unsigned)(zfp.consumed >> 20));
if (fileSize != UTIL_FILESIZE_UNKNOWN)
DISPLAYLEVEL(2, "/ %2u ", (unsigned)(fileSize >> 20));
DISPLAYLEVEL(2, "MB ==> %2.f%%", cShare);
DISPLAYLEVEL(1, "MB ==> %2.f%%", cShare);
DELAY_NEXT_UPDATE();
}
@ -2164,7 +2165,7 @@ FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress, FILE* finput,
/* Write block */
storedSkips = FIO_fwriteSparse(ress->dstFile, ress->dstBuffer, outBuff.pos, prefs, storedSkips);
frameSize += outBuff.pos;
if (!fCtx->hasStdoutOutput) {
if (!fCtx->hasStdoutOutput || g_display_prefs.progressSetting == FIO_ps_always) {
if (fCtx->nbFilesTotal > 1) {
size_t srcFileNameSize = strlen(srcFileName);
if (srcFileNameSize > 18) {

4
programs/fileio.h
View File

@ -68,6 +68,8 @@ void FIO_freeContext(FIO_ctx_t* const fCtx);
typedef struct FIO_display_prefs_s FIO_display_prefs_t;
typedef enum { FIO_ps_auto, FIO_ps_never, FIO_ps_always } FIO_progressSetting_e;
/*-*************************************
* Parameters
***************************************/
@ -100,7 +102,7 @@ void FIO_setLiteralCompressionMode(
FIO_prefs_t* const prefs,
ZSTD_literalCompressionMode_e mode);
void FIO_setNoProgress(unsigned noProgress);
void FIO_setProgressSetting(FIO_progressSetting_e progressSetting);
void FIO_setNotificationLevel(int level);
void FIO_setExcludeCompressedFile(FIO_prefs_t* const prefs, int excludeCompressedFiles);
void FIO_setAllowBlockDevices(FIO_prefs_t* const prefs, int allowBlockDevices);

4
programs/util.c
View File

@ -323,9 +323,7 @@ U64 UTIL_getTotalFileSize(const char* const * fileNamesTable, unsigned nbFiles)
static size_t readLineFromFile(char* buf, size_t len, FILE* file)
{
assert(!feof(file));
/* Work around Cygwin problem when len == 1 it returns NULL. */
if (len <= 1) return 0;
CONTROL( fgets(buf, (int) len, file) );
if ( fgets(buf, (int) len, file) == NULL ) return 0;
{ size_t linelen = strlen(buf);
if (strlen(buf)==0) return 0;
if (buf[linelen-1] == '\n') linelen--;

6
programs/zstdcli.c
View File

@ -167,7 +167,8 @@ static void usage_advanced(const char* programName)
DISPLAYOUT( " -v : verbose mode; specify multiple times to increase verbosity \n");
DISPLAYOUT( " -q : suppress warnings; specify twice to suppress errors too \n");
DISPLAYOUT( "--no-progress : do not display the progress counter \n");
DISPLAYOUT( "--[no-]progress : forcibly display, or never display the progress counter.\n");
DISPLAYOUT( " note: any (de)compressed output to terminal will mix with progress counter text. \n");
#ifdef UTIL_HAS_CREATEFILELIST
DISPLAYOUT( " -r : operate recursively on directories \n");
@ -879,7 +880,8 @@ int main(int const argCount, const char* argv[])
if (!strcmp(argument, "--rsyncable")) { rsyncable = 1; continue; }
if (!strcmp(argument, "--compress-literals")) { literalCompressionMode = ZSTD_lcm_huffman; continue; }
if (!strcmp(argument, "--no-compress-literals")) { literalCompressionMode = ZSTD_lcm_uncompressed; continue; }
if (!strcmp(argument, "--no-progress")) { FIO_setNoProgress(1); continue; }
if (!strcmp(argument, "--no-progress")) { FIO_setProgressSetting(FIO_ps_never); continue; }
if (!strcmp(argument, "--progress")) { FIO_setProgressSetting(FIO_ps_always); continue; }
if (!strcmp(argument, "--exclude-compressed")) { FIO_setExcludeCompressedFile(prefs, 1); continue; }
/* long commands with arguments */

8
tests/Makefile
View File

@ -24,7 +24,7 @@ PRGDIR = ../programs
PYTHON ?= python3
TESTARTEFACT := versionsTest
DEBUGLEVEL ?= 1
DEBUGLEVEL ?= 2
export DEBUGLEVEL # transmit value to sub-makefiles
DEBUGFLAGS = -g -DDEBUGLEVEL=$(DEBUGLEVEL)
CPPFLAGS += -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
@ -38,7 +38,7 @@ CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
-Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
-Wstrict-prototypes -Wundef \
-Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
-Wredundant-decls -Wmissing-prototypes
-Wredundant-decls -Wmissing-prototypes -Wno-deprecated-declarations
CFLAGS += $(DEBUGFLAGS)
CPPFLAGS += $(MOREFLAGS)
@ -131,7 +131,7 @@ zstdmt_d_%.o : $(ZSTDDIR)/decompress/%.c
$(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@
fullbench32: CPPFLAGS += -m32
fullbench fullbench32 : CPPFLAGS += $(MULTITHREAD_CPP)
fullbench fullbench32 : CPPFLAGS += $(MULTITHREAD_CPP) -Wno-deprecated-declarations
fullbench fullbench32 : LDFLAGS += $(MULTITHREAD_LD)
fullbench fullbench32 : DEBUGFLAGS = -DNDEBUG # turn off assert() for speed measurements
fullbench fullbench32 : $(ZSTD_FILES)
@ -147,7 +147,7 @@ fullbench-dll: $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/benchfn.c $(PRGDIR
# $(CC) $(FLAGS) $(filter %.c,$^) -o $@$(EXT) -DZSTD_DLL_IMPORT=1 $(ZSTDDIR)/dll/libzstd.dll
$(LINK.c) $^ $(LDLIBS) -o $@$(EXT)
fuzzer : CPPFLAGS += $(MULTITHREAD_CPP)
fuzzer : CPPFLAGS += $(MULTITHREAD_CPP) -Wno-deprecated-declarations
fuzzer : LDFLAGS += $(MULTITHREAD_LD)
fuzzer : $(ZSTDMT_OBJECTS)
fuzzer fuzzer32 : $(ZDICT_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/datagen.c fuzzer.c

1
tests/README.md
View File

@ -8,7 +8,6 @@ This directory contains the following programs and scripts:
- `paramgrill` : parameter tester for zstd
- `test-zstd-speed.py` : script for testing zstd speed difference between commits
- `test-zstd-versions.py` : compatibility test between zstd versions stored on Github (v0.1+)
- `zbufftest` : Test tool to check ZBUFF (a buffered streaming API) integrity
- `zstreamtest` : Fuzzer test tool for zstd streaming API
- `legacy` : Test tool to test decoding of legacy zstd frames
- `decodecorpus` : Tool to generate valid Zstandard frames, for verifying decoder implementations

1
tests/fuzz/.gitignore vendored
View File

@ -16,6 +16,7 @@ zstd_frame_info
decompress_dstSize_tooSmall
fse_read_ncount
sequence_compression_api
seekable_roundtrip
fuzz-*.log
rt_lib_*
d_lib_*

12
tests/fuzz/Makefile
View File

@ -25,10 +25,11 @@ CORPORA_URL_PREFIX:=https://github.com/facebook/zstd/releases/download/fuzz-corp
ZSTDDIR = ../../lib
PRGDIR = ../../programs
CONTRIBDIR = ../../contrib
FUZZ_CPPFLAGS := -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
-I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(ZSTDDIR)/legacy \
-I$(PRGDIR) -DZSTD_MULTITHREAD -DZSTD_LEGACY_SUPPORT=1 $(CPPFLAGS)
-I$(CONTRIBDIR)/seekable_format -I$(PRGDIR) -DZSTD_MULTITHREAD -DZSTD_LEGACY_SUPPORT=1 $(CPPFLAGS)
FUZZ_EXTRA_FLAGS := -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
-Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
-Wstrict-prototypes -Wundef \
@ -46,6 +47,9 @@ FUZZ_ROUND_TRIP_FLAGS := -DFUZZING_ASSERT_VALID_SEQUENCE
FUZZ_HEADERS := fuzz_helpers.h fuzz.h zstd_helpers.h fuzz_data_producer.h
FUZZ_SRC := $(PRGDIR)/util.c ./fuzz_helpers.c ./zstd_helpers.c ./fuzz_data_producer.c
SEEKABLE_HEADERS = $(CONTRIBDIR)/seekable_format/zstd_seekable.h
SEEKABLE_OBJS = $(CONTRIBDIR)/seekable_format/zstdseek_compress.c $(CONTRIBDIR)/seekable_format/zstdseek_decompress.c
ZSTDCOMMON_SRC := $(ZSTDDIR)/common/*.c
ZSTDCOMP_SRC := $(ZSTDDIR)/compress/*.c
ZSTDDECOMP_SRC := $(ZSTDDIR)/decompress/*.c
@ -98,7 +102,8 @@ FUZZ_TARGETS := \
dictionary_stream_round_trip \
decompress_dstSize_tooSmall \
fse_read_ncount \
sequence_compression_api
sequence_compression_api \
seekable_roundtrip
all: libregression.a $(FUZZ_TARGETS)
@ -192,6 +197,9 @@ fse_read_ncount: $(FUZZ_HEADERS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_fse_read_ncount.
sequence_compression_api: $(FUZZ_HEADERS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_sequence_compression_api.o
$(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_sequence_compression_api.o $(LIB_FUZZING_ENGINE) -o $@
seekable_roundtrip: $(FUZZ_HEADERS) $(SEEKABLE_HEADERS) $(FUZZ_ROUND_TRIP_OBJ) $(SEEKABLE_OBJS) rt_fuzz_seekable_roundtrip.o
$(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_ROUND_TRIP_OBJ) $(SEEKABLE_OBJS) rt_fuzz_seekable_roundtrip.o $(LIB_FUZZING_ENGINE) -o $@
libregression.a: $(FUZZ_HEADERS) $(PRGDIR)/util.h $(PRGDIR)/util.c d_fuzz_regression_driver.o
$(AR) $(FUZZ_ARFLAGS) $@ d_fuzz_regression_driver.o

1
tests/fuzz/fuzz.py
View File

@ -62,6 +62,7 @@ TARGET_INFO = {
'decompress_dstSize_tooSmall': TargetInfo(InputType.RAW_DATA),
'fse_read_ncount': TargetInfo(InputType.RAW_DATA),
'sequence_compression_api': TargetInfo(InputType.RAW_DATA),
'seekable_roundtrip': TargetInfo(InputType.RAW_DATA),
}
TARGETS = list(TARGET_INFO.keys())
ALL_TARGETS = TARGETS + ['all']

88
tests/fuzz/seekable_roundtrip.c Normal file
View File

@ -0,0 +1,88 @@
/*
* Copyright (c) Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
#include "zstd.h"
#include "zstd_seekable.h"
#include "fuzz_helpers.h"
#include "fuzz_data_producer.h"
static ZSTD_seekable *stream = NULL;
static ZSTD_seekable_CStream *zscs = NULL;
static const size_t kSeekableOverheadSize = ZSTD_seekTableFooterSize;
int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
{
/* Give a random portion of src data to the producer, to use for
parameter generation. The rest will be used for (de)compression */
FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size);
size = FUZZ_dataProducer_reserveDataPrefix(producer);
size_t const compressedBufferSize = ZSTD_compressBound(size) + kSeekableOverheadSize;
uint8_t* compressedBuffer = (uint8_t*)malloc(compressedBufferSize);
uint8_t* decompressedBuffer = (uint8_t*)malloc(size);
int const cLevel = FUZZ_dataProducer_int32Range(producer, ZSTD_minCLevel(), ZSTD_maxCLevel());
unsigned const checksumFlag = FUZZ_dataProducer_int32Range(producer, 0, 1);
size_t const uncompressedSize = FUZZ_dataProducer_uint32Range(producer, 0, size);
size_t const offset = FUZZ_dataProducer_uint32Range(producer, 0, size - uncompressedSize);
size_t seekSize;
if (!zscs) {
zscs = ZSTD_seekable_createCStream();
FUZZ_ASSERT(zscs);
}
if (!stream) {
stream = ZSTD_seekable_create();
FUZZ_ASSERT(stream);
}
{ /* Perform a compression */
size_t const initStatus = ZSTD_seekable_initCStream(zscs, cLevel, checksumFlag, size);
size_t endStatus;
ZSTD_outBuffer out = { .dst=compressedBuffer, .pos=0, .size=compressedBufferSize };
ZSTD_inBuffer in = { .src=src, .pos=0, .size=size };
FUZZ_ASSERT(!ZSTD_isError(initStatus));
do {
size_t cSize = ZSTD_seekable_compressStream(zscs, &out, &in);
FUZZ_ASSERT(!ZSTD_isError(cSize));
} while (in.pos != in.size);
FUZZ_ASSERT(in.pos == in.size);
endStatus = ZSTD_seekable_endStream(zscs, &out);
FUZZ_ASSERT(!ZSTD_isError(endStatus));
seekSize = out.pos;
}
{ /* Decompress at an offset */
size_t const initStatus = ZSTD_seekable_initBuff(stream, compressedBuffer, seekSize);
size_t decompressedBytesTotal = 0;
size_t dSize;
FUZZ_ZASSERT(initStatus);
do {
dSize = ZSTD_seekable_decompress(stream, decompressedBuffer, uncompressedSize, offset);
FUZZ_ASSERT(!ZSTD_isError(dSize));
decompressedBytesTotal += dSize;
} while (decompressedBytesTotal < uncompressedSize && dSize > 0);
FUZZ_ASSERT(decompressedBytesTotal == uncompressedSize);
}
FUZZ_ASSERT_MSG(!FUZZ_memcmp(src+offset, decompressedBuffer, uncompressedSize), "Corruption!");
free(decompressedBuffer);
free(compressedBuffer);
FUZZ_dataProducer_free(producer);
#ifndef STATEFUL_FUZZING
ZSTD_seekable_free(stream); stream = NULL;
ZSTD_seekable_freeCStream(zscs); zscs = NULL;
#endif
return 0;
}

1
tests/fuzz/zstd_helpers.c
View File

@ -97,6 +97,7 @@ void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, FUZZ_dataProducer
setRand(cctx, ZSTD_c_literalCompressionMode, 0, 2, producer);
setRand(cctx, ZSTD_c_forceAttachDict, 0, 2, producer);
setRand(cctx, ZSTD_c_splitBlocks, 0, 1, producer);
setRand(cctx, ZSTD_c_deterministicRefPrefix, 0, 1, producer);
if (FUZZ_dataProducer_uint32Range(producer, 0, 1) == 0) {
setRand(cctx, ZSTD_c_srcSizeHint, ZSTD_SRCSIZEHINT_MIN, 2 * srcSize, producer);
}

57
tests/fuzzer.c
View File

@ -766,6 +766,50 @@ static int basicUnitTests(U32 const seed, double compressibility)
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : testing dict compression for determinism : ", testNb++);
{
size_t const testSize = 1024;
ZSTD_CCtx* const cctx = ZSTD_createCCtx();
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
char* dict = (char*)malloc(2 * testSize);
int ldmEnabled, level;
RDG_genBuffer(dict, testSize, 0.5, 0.5, seed);
RDG_genBuffer(CNBuffer, testSize, 0.6, 0.6, seed);
memcpy(dict + testSize, CNBuffer, testSize);
for (level = 1; level <= 5; ++level) {
for (ldmEnabled = 0; ldmEnabled <= 1; ++ldmEnabled) {
size_t cSize0;
XXH64_hash_t compressedChecksum0;
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, level));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ldmEnabled));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_deterministicRefPrefix, 1));
CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, testSize));
cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, testSize);
CHECK_Z(cSize);
CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, testSize, compressedBuffer, cSize, dict, testSize));
cSize0 = cSize;
compressedChecksum0 = XXH64(compressedBuffer, cSize, 0);
CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, testSize));
cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, dict + testSize, testSize);
CHECK_Z(cSize);
if (cSize != cSize0) goto _output_error;
if (XXH64(compressedBuffer, cSize, 0) != compressedChecksum0) goto _output_error;
}
}
ZSTD_freeCCtx(cctx);
ZSTD_freeDCtx(dctx);
free(dict);
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : LDM + opt parser with small uncompressible block ", testNb++);
{ ZSTD_CCtx* cctx = ZSTD_createCCtx();
ZSTD_DCtx* dctx = ZSTD_createDCtx();
@ -1738,10 +1782,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
DISPLAYLEVEL(3, "test%3i : check content size on duplicated context : ", testNb++);
{ size_t const testSize = CNBuffSize / 3;
{ ZSTD_parameters p = ZSTD_getParams(2, testSize, dictSize);
p.fParams.contentSizeFlag = 1;
CHECK( ZSTD_compressBegin_advanced(ctxOrig, CNBuffer, dictSize, p, testSize-1) );
}
CHECK( ZSTD_compressBegin(ctxOrig, ZSTD_defaultCLevel()) );
CHECK( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, testSize) );
CHECK_VAR(cSize, ZSTD_compressEnd(ctxDuplicated, compressedBuffer, ZSTD_compressBound(testSize),
@ -2562,12 +2603,8 @@ static int basicUnitTests(U32 const seed, double compressibility)
int const compressionLevel = -1;
assert(cctx != NULL);
{ ZSTD_parameters const params = ZSTD_getParams(compressionLevel, srcSize, 0);
size_t const cSize_1pass = ZSTD_compress_advanced(cctx,
compressedBuffer, compressedBufferSize,
CNBuffer, srcSize,
NULL, 0,
params);
{ size_t const cSize_1pass = ZSTD_compress(compressedBuffer, compressedBufferSize,
CNBuffer, srcSize, compressionLevel);
if (ZSTD_isError(cSize_1pass)) goto _output_error;
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) );

82
tests/regression/results.csv
View File

@ -97,11 +97,11 @@ github, uncompressed literals, compress
github, uncompressed literals optimal, compress cctx, 134064
github, huffman literals, compress cctx, 175568
github, multithreaded with advanced params, compress cctx, 141102
silesia, level -5, zstdcli, 6882553
silesia, level -3, zstdcli, 6568424
silesia, level -1, zstdcli, 6183451
silesia, level -5, zstdcli, 6737655
silesia, level -3, zstdcli, 6444725
silesia, level -1, zstdcli, 6178508
silesia, level 0, zstdcli, 4849600
silesia, level 1, zstdcli, 5314210
silesia, level 1, zstdcli, 5313252
silesia, level 3, zstdcli, 4849600
silesia, level 4, zstdcli, 4787018
silesia, level 5, zstdcli, 4707842
@ -111,16 +111,16 @@ silesia, level 9, zstdcli,
silesia, level 13, zstdcli, 4482183
silesia, level 16, zstdcli, 4360299
silesia, level 19, zstdcli, 4283285
silesia, long distance mode, zstdcli, 4840792
silesia, long distance mode, zstdcli, 4840806
silesia, multithreaded, zstdcli, 4849600
silesia, multithreaded long distance mode, zstdcli, 4840792
silesia, small window log, zstdcli, 7111012
silesia, multithreaded long distance mode, zstdcli, 4840806
silesia, small window log, zstdcli, 7095967
silesia, small hash log, zstdcli, 6526189
silesia, small chain log, zstdcli, 4912245
silesia, explicit params, zstdcli, 4795887
silesia, explicit params, zstdcli, 4795856
silesia, uncompressed literals, zstdcli, 5128030
silesia, uncompressed literals optimal, zstdcli, 4317944
silesia, huffman literals, zstdcli, 5331216
silesia, huffman literals, zstdcli, 5326316
silesia, multithreaded with advanced params, zstdcli, 5128030
silesia.tar, level -5, zstdcli, 6738934
silesia.tar, level -3, zstdcli, 6448419
@ -137,9 +137,9 @@ silesia.tar, level 13, zstdcli,
silesia.tar, level 16, zstdcli, 4356831
silesia.tar, level 19, zstdcli, 4264491
silesia.tar, no source size, zstdcli, 4861508
silesia.tar, long distance mode, zstdcli, 4853153
silesia.tar, long distance mode, zstdcli, 4853226
silesia.tar, multithreaded, zstdcli, 4861512
silesia.tar, multithreaded long distance mode, zstdcli, 4853153
silesia.tar, multithreaded long distance mode, zstdcli, 4853226
silesia.tar, small window log, zstdcli, 7101576
silesia.tar, small hash log, zstdcli, 6529290
silesia.tar, small chain log, zstdcli, 4917022
@ -187,16 +187,16 @@ github, uncompressed literals, zstdcli,
github, uncompressed literals optimal, zstdcli, 159227
github, huffman literals, zstdcli, 144465
github, multithreaded with advanced params, zstdcli, 167915
github.tar, level -5, zstdcli, 46751
github.tar, level -5 with dict, zstdcli, 44444
github.tar, level -3, zstdcli, 43541
github.tar, level -3 with dict, zstdcli, 41116
github.tar, level -1, zstdcli, 42469
github.tar, level -1 with dict, zstdcli, 41200
github.tar, level -5, zstdcli, 46860
github.tar, level -5 with dict, zstdcli, 44575
github.tar, level -3, zstdcli, 43758
github.tar, level -3 with dict, zstdcli, 41451
github.tar, level -1, zstdcli, 42494
github.tar, level -1 with dict, zstdcli, 41135
github.tar, level 0, zstdcli, 38445
github.tar, level 0 with dict, zstdcli, 37999
github.tar, level 1, zstdcli, 39346
github.tar, level 1 with dict, zstdcli, 38297
github.tar, level 1, zstdcli, 39269
github.tar, level 1 with dict, zstdcli, 38284
github.tar, level 3, zstdcli, 38445
github.tar, level 3 with dict, zstdcli, 37999
github.tar, level 4, zstdcli, 38471
@ -217,16 +217,16 @@ github.tar, level 19, zstdcli,
github.tar, level 19 with dict, zstdcli, 32899
github.tar, no source size, zstdcli, 38442
github.tar, no source size with dict, zstdcli, 38004
github.tar, long distance mode, zstdcli, 39726
github.tar, long distance mode, zstdcli, 39730
github.tar, multithreaded, zstdcli, 38445
github.tar, multithreaded long distance mode, zstdcli, 39726
github.tar, small window log, zstdcli, 199432
github.tar, multithreaded long distance mode, zstdcli, 39730
github.tar, small window log, zstdcli, 198544
github.tar, small hash log, zstdcli, 129874
github.tar, small chain log, zstdcli, 41673
github.tar, explicit params, zstdcli, 41227
github.tar, uncompressed literals, zstdcli, 41126
github.tar, uncompressed literals optimal, zstdcli, 35392
github.tar, huffman literals, zstdcli, 38804
github.tar, huffman literals, zstdcli, 38781
github.tar, multithreaded with advanced params, zstdcli, 41126
silesia, level -5, advanced one pass, 6737607
silesia, level -3, advanced one pass, 6444677
@ -249,9 +249,9 @@ silesia, level 13, advanced
silesia, level 16, advanced one pass, 4360251
silesia, level 19, advanced one pass, 4283237
silesia, no source size, advanced one pass, 4849552
silesia, long distance mode, advanced one pass, 4840744
silesia, long distance mode, advanced one pass, 4840738
silesia, multithreaded, advanced one pass, 4849552
silesia, multithreaded long distance mode, advanced one pass, 4840744
silesia, multithreaded long distance mode, advanced one pass, 4840758
silesia, small window log, advanced one pass, 7095919
silesia, small hash log, advanced one pass, 6526141
silesia, small chain log, advanced one pass, 4912197
@ -281,9 +281,9 @@ silesia.tar, level 13, advanced
silesia.tar, level 16, advanced one pass, 4356827
silesia.tar, level 19, advanced one pass, 4264487
silesia.tar, no source size, advanced one pass, 4861425
silesia.tar, long distance mode, advanced one pass, 4847735
silesia.tar, long distance mode, advanced one pass, 4847754
silesia.tar, multithreaded, advanced one pass, 4861508
silesia.tar, multithreaded long distance mode, advanced one pass, 4853149
silesia.tar, multithreaded long distance mode, advanced one pass, 4853222
silesia.tar, small window log, advanced one pass, 7101530
silesia.tar, small hash log, advanced one pass, 6529232
silesia.tar, small chain log, advanced one pass, 4917041
@ -511,9 +511,9 @@ github.tar, level 19 with dict copy, advanced
github.tar, level 19 with dict load, advanced one pass, 32676
github.tar, no source size, advanced one pass, 38441
github.tar, no source size with dict, advanced one pass, 37995
github.tar, long distance mode, advanced one pass, 39722
github.tar, long distance mode, advanced one pass, 39757
github.tar, multithreaded, advanced one pass, 38441
github.tar, multithreaded long distance mode, advanced one pass, 39722
github.tar, multithreaded long distance mode, advanced one pass, 39726
github.tar, small window log, advanced one pass, 198540
github.tar, small hash log, advanced one pass, 129870
github.tar, small chain log, advanced one pass, 41669
@ -543,9 +543,9 @@ silesia, level 13, advanced
silesia, level 16, advanced one pass small out, 4360251
silesia, level 19, advanced one pass small out, 4283237
silesia, no source size, advanced one pass small out, 4849552
silesia, long distance mode, advanced one pass small out, 4840744
silesia, long distance mode, advanced one pass small out, 4840738
silesia, multithreaded, advanced one pass small out, 4849552
silesia, multithreaded long distance mode, advanced one pass small out, 4840744
silesia, multithreaded long distance mode, advanced one pass small out, 4840758
silesia, small window log, advanced one pass small out, 7095919
silesia, small hash log, advanced one pass small out, 6526141
silesia, small chain log, advanced one pass small out, 4912197
@ -575,9 +575,9 @@ silesia.tar, level 13, advanced
silesia.tar, level 16, advanced one pass small out, 4356827
silesia.tar, level 19, advanced one pass small out, 4264487
silesia.tar, no source size, advanced one pass small out, 4861425
silesia.tar, long distance mode, advanced one pass small out, 4847735
silesia.tar, long distance mode, advanced one pass small out, 4847754
silesia.tar, multithreaded, advanced one pass small out, 4861508
silesia.tar, multithreaded long distance mode, advanced one pass small out, 4853149
silesia.tar, multithreaded long distance mode, advanced one pass small out, 4853222
silesia.tar, small window log, advanced one pass small out, 7101530
silesia.tar, small hash log, advanced one pass small out, 6529232
silesia.tar, small chain log, advanced one pass small out, 4917041
@ -805,9 +805,9 @@ github.tar, level 19 with dict copy, advanced
github.tar, level 19 with dict load, advanced one pass small out, 32676
github.tar, no source size, advanced one pass small out, 38441
github.tar, no source size with dict, advanced one pass small out, 37995
github.tar, long distance mode, advanced one pass small out, 39722
github.tar, long distance mode, advanced one pass small out, 39757
github.tar, multithreaded, advanced one pass small out, 38441
github.tar, multithreaded long distance mode, advanced one pass small out, 39722
github.tar, multithreaded long distance mode, advanced one pass small out, 39726
github.tar, small window log, advanced one pass small out, 198540
github.tar, small hash log, advanced one pass small out, 129870
github.tar, small chain log, advanced one pass small out, 41669
@ -837,9 +837,9 @@ silesia, level 13, advanced
silesia, level 16, advanced streaming, 4360251
silesia, level 19, advanced streaming, 4283237
silesia, no source size, advanced streaming, 4849516
silesia, long distance mode, advanced streaming, 4840744
silesia, long distance mode, advanced streaming, 4840738
silesia, multithreaded, advanced streaming, 4849552
silesia, multithreaded long distance mode, advanced streaming, 4840744
silesia, multithreaded long distance mode, advanced streaming, 4840758
silesia, small window log, advanced streaming, 7112062
silesia, small hash log, advanced streaming, 6526141
silesia, small chain log, advanced streaming, 4912197
@ -869,9 +869,9 @@ silesia.tar, level 13, advanced
silesia.tar, level 16, advanced streaming, 4356834
silesia.tar, level 19, advanced streaming, 4264392
silesia.tar, no source size, advanced streaming, 4861423
silesia.tar, long distance mode, advanced streaming, 4847735
silesia.tar, long distance mode, advanced streaming, 4847754
silesia.tar, multithreaded, advanced streaming, 4861508
silesia.tar, multithreaded long distance mode, advanced streaming, 4853149
silesia.tar, multithreaded long distance mode, advanced streaming, 4853222
silesia.tar, small window log, advanced streaming, 7118769
silesia.tar, small hash log, advanced streaming, 6529235
silesia.tar, small chain log, advanced streaming, 4917021
@ -1099,9 +1099,9 @@ github.tar, level 19 with dict copy, advanced
github.tar, level 19 with dict load, advanced streaming, 32676
github.tar, no source size, advanced streaming, 38438
github.tar, no source size with dict, advanced streaming, 38000
github.tar, long distance mode, advanced streaming, 39722
github.tar, long distance mode, advanced streaming, 39757
github.tar, multithreaded, advanced streaming, 38441
github.tar, multithreaded long distance mode, advanced streaming, 39722
github.tar, multithreaded long distance mode, advanced streaming, 39726
github.tar, small window log, advanced streaming, 199558
github.tar, small hash log, advanced streaming, 129870
github.tar, small chain log, advanced streaming, 41669

1 Data Config Method Total compressed size
97 github uncompressed literals optimal compress cctx 134064
98 github huffman literals compress cctx 175568
99 github multithreaded with advanced params compress cctx 141102
100 silesia level -5 zstdcli 6882553 6737655
101 silesia level -3 zstdcli 6568424 6444725
102 silesia level -1 zstdcli 6183451 6178508
103 silesia level 0 zstdcli 4849600
104 silesia level 1 zstdcli 5314210 5313252
105 silesia level 3 zstdcli 4849600
106 silesia level 4 zstdcli 4787018
107 silesia level 5 zstdcli 4707842
111 silesia level 13 zstdcli 4482183
112 silesia level 16 zstdcli 4360299
113 silesia level 19 zstdcli 4283285
114 silesia long distance mode zstdcli 4840792 4840806
115 silesia multithreaded zstdcli 4849600
116 silesia multithreaded long distance mode zstdcli 4840792 4840806
117 silesia small window log zstdcli 7111012 7095967
118 silesia small hash log zstdcli 6526189
119 silesia small chain log zstdcli 4912245
120 silesia explicit params zstdcli 4795887 4795856
121 silesia uncompressed literals zstdcli 5128030
122 silesia uncompressed literals optimal zstdcli 4317944
123 silesia huffman literals zstdcli 5331216 5326316
124 silesia multithreaded with advanced params zstdcli 5128030
125 silesia.tar level -5 zstdcli 6738934
126 silesia.tar level -3 zstdcli 6448419
137 silesia.tar level 16 zstdcli 4356831
138 silesia.tar level 19 zstdcli 4264491
139 silesia.tar no source size zstdcli 4861508
140 silesia.tar long distance mode zstdcli 4853153 4853226
141 silesia.tar multithreaded zstdcli 4861512
142 silesia.tar multithreaded long distance mode zstdcli 4853153 4853226
143 silesia.tar small window log zstdcli 7101576
144 silesia.tar small hash log zstdcli 6529290
145 silesia.tar small chain log zstdcli 4917022
187 github uncompressed literals optimal zstdcli 159227
188 github huffman literals zstdcli 144465
189 github multithreaded with advanced params zstdcli 167915
190 github.tar level -5 zstdcli 46751 46860
191 github.tar level -5 with dict zstdcli 44444 44575
192 github.tar level -3 zstdcli 43541 43758
193 github.tar level -3 with dict zstdcli 41116 41451
194 github.tar level -1 zstdcli 42469 42494
195 github.tar level -1 with dict zstdcli 41200 41135
196 github.tar level 0 zstdcli 38445
197 github.tar level 0 with dict zstdcli 37999
198 github.tar level 1 zstdcli 39346 39269
199 github.tar level 1 with dict zstdcli 38297 38284
200 github.tar level 3 zstdcli 38445
201 github.tar level 3 with dict zstdcli 37999
202 github.tar level 4 zstdcli 38471
217 github.tar level 19 with dict zstdcli 32899
218 github.tar no source size zstdcli 38442
219 github.tar no source size with dict zstdcli 38004
220 github.tar long distance mode zstdcli 39726 39730
221 github.tar multithreaded zstdcli 38445
222 github.tar multithreaded long distance mode zstdcli 39726 39730
223 github.tar small window log zstdcli 199432 198544
224 github.tar small hash log zstdcli 129874
225 github.tar small chain log zstdcli 41673
226 github.tar explicit params zstdcli 41227
227 github.tar uncompressed literals zstdcli 41126
228 github.tar uncompressed literals optimal zstdcli 35392
229 github.tar huffman literals zstdcli 38804 38781
230 github.tar multithreaded with advanced params zstdcli 41126
231 silesia level -5 advanced one pass 6737607
232 silesia level -3 advanced one pass 6444677
249 silesia level 16 advanced one pass 4360251
250 silesia level 19 advanced one pass 4283237
251 silesia no source size advanced one pass 4849552
252 silesia long distance mode advanced one pass 4840744 4840738
253 silesia multithreaded advanced one pass 4849552
254 silesia multithreaded long distance mode advanced one pass 4840744 4840758
255 silesia small window log advanced one pass 7095919
256 silesia small hash log advanced one pass 6526141
257 silesia small chain log advanced one pass 4912197
281 silesia.tar level 16 advanced one pass 4356827
282 silesia.tar level 19 advanced one pass 4264487
283 silesia.tar no source size advanced one pass 4861425
284 silesia.tar long distance mode advanced one pass 4847735 4847754
285 silesia.tar multithreaded advanced one pass 4861508
286 silesia.tar multithreaded long distance mode advanced one pass 4853149 4853222
287 silesia.tar small window log advanced one pass 7101530
288 silesia.tar small hash log advanced one pass 6529232
289 silesia.tar small chain log advanced one pass 4917041
511 github.tar level 19 with dict load advanced one pass 32676
512 github.tar no source size advanced one pass 38441
513 github.tar no source size with dict advanced one pass 37995
514 github.tar long distance mode advanced one pass 39722 39757
515 github.tar multithreaded advanced one pass 38441
516 github.tar multithreaded long distance mode advanced one pass 39722 39726
517 github.tar small window log advanced one pass 198540
518 github.tar small hash log advanced one pass 129870
519 github.tar small chain log advanced one pass 41669
543 silesia level 16 advanced one pass small out 4360251
544 silesia level 19 advanced one pass small out 4283237
545 silesia no source size advanced one pass small out 4849552
546 silesia long distance mode advanced one pass small out 4840744 4840738
547 silesia multithreaded advanced one pass small out 4849552
548 silesia multithreaded long distance mode advanced one pass small out 4840744 4840758
549 silesia small window log advanced one pass small out 7095919
550 silesia small hash log advanced one pass small out 6526141
551 silesia small chain log advanced one pass small out 4912197
575 silesia.tar level 16 advanced one pass small out 4356827
576 silesia.tar level 19 advanced one pass small out 4264487
577 silesia.tar no source size advanced one pass small out 4861425
578 silesia.tar long distance mode advanced one pass small out 4847735 4847754
579 silesia.tar multithreaded advanced one pass small out 4861508
580 silesia.tar multithreaded long distance mode advanced one pass small out 4853149 4853222
581 silesia.tar small window log advanced one pass small out 7101530
582 silesia.tar small hash log advanced one pass small out 6529232
583 silesia.tar small chain log advanced one pass small out 4917041
805 github.tar level 19 with dict load advanced one pass small out 32676
806 github.tar no source size advanced one pass small out 38441
807 github.tar no source size with dict advanced one pass small out 37995
808 github.tar long distance mode advanced one pass small out 39722 39757
809 github.tar multithreaded advanced one pass small out 38441
810 github.tar multithreaded long distance mode advanced one pass small out 39722 39726
811 github.tar small window log advanced one pass small out 198540
812 github.tar small hash log advanced one pass small out 129870
813 github.tar small chain log advanced one pass small out 41669
837 silesia level 16 advanced streaming 4360251
838 silesia level 19 advanced streaming 4283237
839 silesia no source size advanced streaming 4849516
840 silesia long distance mode advanced streaming 4840744 4840738
841 silesia multithreaded advanced streaming 4849552
842 silesia multithreaded long distance mode advanced streaming 4840744 4840758
843 silesia small window log advanced streaming 7112062
844 silesia small hash log advanced streaming 6526141
845 silesia small chain log advanced streaming 4912197
869 silesia.tar level 16 advanced streaming 4356834
870 silesia.tar level 19 advanced streaming 4264392
871 silesia.tar no source size advanced streaming 4861423
872 silesia.tar long distance mode advanced streaming 4847735 4847754
873 silesia.tar multithreaded advanced streaming 4861508
874 silesia.tar multithreaded long distance mode advanced streaming 4853149 4853222
875 silesia.tar small window log advanced streaming 7118769
876 silesia.tar small hash log advanced streaming 6529235
877 silesia.tar small chain log advanced streaming 4917021
1099 github.tar level 19 with dict load advanced streaming 32676
1100 github.tar no source size advanced streaming 38438
1101 github.tar no source size with dict advanced streaming 38000
1102 github.tar long distance mode advanced streaming 39722 39757
1103 github.tar multithreaded advanced streaming 38441
1104 github.tar multithreaded long distance mode advanced streaming 39722 39726
1105 github.tar small window log advanced streaming 199558
1106 github.tar small hash log advanced streaming 129870
1107 github.tar small chain log advanced streaming 41669

64
tests/zstreamtest.c
View File

@ -321,7 +321,9 @@ static int basicUnitTests(U32 seed, double compressibility)
/* Basic compression test using dict */
DISPLAYLEVEL(3, "test%3i : skipframe + compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
CHECK_Z( ZSTD_initCStream_usingDict(zc, CNBuffer, dictSize, 1 /* cLevel */) );
CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 1) );
CHECK_Z( ZSTD_CCtx_loadDictionary(zc, CNBuffer, dictSize) );
outBuff.dst = (char*)(compressedBuffer)+cSize;
assert(compressedBufferSize > cSize);
outBuff.size = compressedBufferSize - cSize;
@ -368,7 +370,7 @@ static int basicUnitTests(U32 seed, double compressibility)
}
/* Attempt bad compression parameters */
DISPLAYLEVEL(3, "test%3i : use bad compression parameters : ", testNb++);
DISPLAYLEVEL(3, "test%3i : use bad compression parameters with ZSTD_initCStream_advanced : ", testNb++);
{ size_t r;
ZSTD_parameters params = ZSTD_getParams(1, 0, 0);
params.cParams.minMatch = 2;
@ -539,7 +541,10 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAYLEVEL(3, "OK\n");
/* _srcSize compression test */
DISPLAYLEVEL(3, "test%3i : compress_srcSize %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
CHECK_Z( ZSTD_initCStream_srcSize(zc, 1, CNBufferSize) );
CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
CHECK_Z( ZSTD_CCtx_refCDict(zc, NULL) );
CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 1) );
CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, CNBufferSize) );
outBuff.dst = (char*)(compressedBuffer);
outBuff.size = compressedBufferSize;
outBuff.pos = 0;
@ -559,7 +564,10 @@ static int basicUnitTests(U32 seed, double compressibility)
/* wrong _srcSize compression test */
DISPLAYLEVEL(3, "test%3i : too large srcSize : %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH-1);
ZSTD_initCStream_srcSize(zc, 1, CNBufferSize+1);
CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
CHECK_Z( ZSTD_CCtx_refCDict(zc, NULL) );
CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 1) );
CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, CNBufferSize+1) );
outBuff.dst = (char*)(compressedBuffer);
outBuff.size = compressedBufferSize;
outBuff.pos = 0;
@ -574,7 +582,10 @@ static int basicUnitTests(U32 seed, double compressibility)
/* wrong _srcSize compression test */
DISPLAYLEVEL(3, "test%3i : too small srcSize : %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH-1);
ZSTD_initCStream_srcSize(zc, 1, CNBufferSize-1);
CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
CHECK_Z( ZSTD_CCtx_refCDict(zc, NULL) );
CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 1) );
CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, CNBufferSize-1) );
outBuff.dst = (char*)(compressedBuffer);
outBuff.size = compressedBufferSize;
outBuff.pos = 0;
@ -587,9 +598,9 @@ static int basicUnitTests(U32 seed, double compressibility)
}
DISPLAYLEVEL(3, "test%3i : wrong srcSize !contentSizeFlag : %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH-1);
{ ZSTD_parameters params = ZSTD_getParams(1, CNBufferSize, 0);
params.fParams.contentSizeFlag = 0;
CHECK_Z(ZSTD_initCStream_advanced(zc, NULL, 0, params, CNBufferSize - MIN(CNBufferSize, 200 KB)));
{ CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_contentSizeFlag, 0) );
CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, CNBufferSize - MIN(CNBufferSize, 200 KB)) );
outBuff.dst = (char*)compressedBuffer;
outBuff.size = compressedBufferSize;
outBuff.pos = 0;
@ -609,7 +620,9 @@ static int basicUnitTests(U32 seed, double compressibility)
/* use 1 */
{ size_t const inSize = 513;
DISPLAYLEVEL(5, "use1 ");
ZSTD_initCStream_advanced(zc, NULL, 0, ZSTD_getParams(19, inSize, 0), inSize); /* needs btopt + search3 to trigger hashLog3 */
CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 19) );
CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, inSize) );
inBuff.src = CNBuffer;
inBuff.size = inSize;
inBuff.pos = 0;
@ -626,7 +639,9 @@ static int basicUnitTests(U32 seed, double compressibility)
/* use 2 */
{ size_t const inSize = 1025; /* will not continue, because tables auto-adjust and are therefore different size */
DISPLAYLEVEL(5, "use2 ");
ZSTD_initCStream_advanced(zc, NULL, 0, ZSTD_getParams(19, inSize, 0), inSize); /* needs btopt + search3 to trigger hashLog3 */
CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 19) );
CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, inSize) );
inBuff.src = CNBuffer;
inBuff.size = inSize;
inBuff.pos = 0;
@ -1274,7 +1289,7 @@ static int basicUnitTests(U32 seed, double compressibility)
if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != 0) goto _output_error;
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : pledgedSrcSize == 0 behaves properly : ", testNb++);
DISPLAYLEVEL(3, "test%3i : pledgedSrcSize == 0 behaves properly with ZSTD_initCStream_advanced : ", testNb++);
{ ZSTD_parameters params = ZSTD_getParams(5, 0, 0);
params.fParams.contentSizeFlag = 1;
CHECK_Z( ZSTD_initCStream_advanced(zc, NULL, 0, params, 0) );
@ -1290,7 +1305,8 @@ static int basicUnitTests(U32 seed, double compressibility)
cSize = outBuff.pos;
if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != 0) goto _output_error;
ZSTD_resetCStream(zc, 0); /* resetCStream should treat 0 as unknown */
CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, ZSTD_CONTENTSIZE_UNKNOWN) );
outBuff.dst = compressedBuffer;
outBuff.size = compressedBufferSize;
outBuff.pos = 0;
@ -1434,7 +1450,8 @@ static int basicUnitTests(U32 seed, double compressibility)
CHECK_Z(ZSTD_initCStream_srcSize(zc, 11, ZSTD_CONTENTSIZE_UNKNOWN));
CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_compressionLevel, &level));
CHECK(level != 11, "Compression level does not match");
ZSTD_resetCStream(zc, ZSTD_CONTENTSIZE_UNKNOWN);
CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, ZSTD_CONTENTSIZE_UNKNOWN) );
CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_compressionLevel, &level));
CHECK(level != 11, "Compression level does not match");
}
@ -1444,7 +1461,8 @@ static int basicUnitTests(U32 seed, double compressibility)
{ ZSTD_parameters const params = ZSTD_getParams(9, 0, 0);
CHECK_Z(ZSTD_initCStream_advanced(zc, NULL, 0, params, ZSTD_CONTENTSIZE_UNKNOWN));
CHECK(badParameters(zc, params), "Compression parameters do not match");
ZSTD_resetCStream(zc, ZSTD_CONTENTSIZE_UNKNOWN);
CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, ZSTD_CONTENTSIZE_UNKNOWN) );
CHECK(badParameters(zc, params), "Compression parameters do not match");
}
DISPLAYLEVEL(3, "OK \n");
@ -1836,8 +1854,9 @@ static int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double co
&& oldTestLog /* at least one test happened */ && resetAllowed) {
maxTestSize = FUZ_randomLength(&lseed, oldTestLog+2);
maxTestSize = MIN(maxTestSize, srcBufferSize-16);
{ U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? 0 : maxTestSize;
CHECK_Z( ZSTD_resetCStream(zc, pledgedSrcSize) );
{ U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize;
CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) );
}
} else {
U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
@ -1855,11 +1874,13 @@ static int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double co
dict = srcBuffer + dictStart;
}
{ U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize;
ZSTD_parameters params = ZSTD_getParams(cLevel, pledgedSrcSize, dictSize);
params.fParams.checksumFlag = FUZ_rand(&lseed) & 1;
params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1;
params.fParams.contentSizeFlag = FUZ_rand(&lseed) & 1;
CHECK_Z ( ZSTD_initCStream_advanced(zc, dict, dictSize, params, pledgedSrcSize) );
CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, cLevel) );
CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_checksumFlag, FUZ_rand(&lseed) & 1) );
CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_contentSizeFlag, FUZ_rand(&lseed) & 1) );
CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_dictIDFlag, FUZ_rand(&lseed) & 1) );
CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) );
CHECK_Z( ZSTD_CCtx_loadDictionary(zc, dict, dictSize) );
} }
/* multi-segments compression test */
@ -2215,6 +2236,7 @@ static int fuzzerTests_newAPI(U32 seed, int nbTests, int startTest,
}
if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_forceMaxWindow, FUZ_rand(&lseed) & 1, opaqueAPI) );
if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_deterministicRefPrefix, FUZ_rand(&lseed) & 1, opaqueAPI) );
/* Apply parameters */
if (opaqueAPI) {

17
zlibWrapper/examples/zwrapbench.c
View File

@ -264,9 +264,22 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize,
ZSTD_outBuffer outBuffer;
ZSTD_CStream* zbc = ZSTD_createCStream();
size_t rSize;
ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams();
if (!cctxParams) EXM_THROW(1, "ZSTD_createCCtxParams() allocation failure");
if (zbc == NULL) EXM_THROW(1, "ZSTD_createCStream() allocation failure");
rSize = ZSTD_initCStream_advanced(zbc, dictBuffer, dictBufferSize, zparams, avgSize);
if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_initCStream_advanced() failed : %s", ZSTD_getErrorName(rSize));
{ int initErr = 0;
initErr |= ZSTD_isError(ZSTD_CCtx_reset(zbc, ZSTD_reset_session_only));
initErr |= ZSTD_isError(ZSTD_CCtxParams_init_advanced(cctxParams, zparams));
initErr |= ZSTD_isError(ZSTD_CCtx_setParametersUsingCCtxParams(zbc, cctxParams));
initErr |= ZSTD_isError(ZSTD_CCtx_setPledgedSrcSize(zbc, avgSize));
initErr |= ZSTD_isError(ZSTD_CCtx_loadDictionary(zbc, dictBuffer, dictBufferSize));
ZSTD_freeCCtxParams(cctxParams);
if (initErr) EXM_THROW(1, "CCtx init failed!");
}
do {
U32 blockNb;
for (blockNb=0; blockNb<nbBlocks; blockNb++) {

17
zlibWrapper/zstd_zlibwrapper.c
View File

@ -205,12 +205,21 @@ static int ZWRAP_initializeCStream(ZWRAP_CCtx* zwc, const void* dict, size_t dic
if (zwc == NULL || zwc->zbc == NULL) return Z_STREAM_ERROR;
if (!pledgedSrcSize) pledgedSrcSize = zwc->pledgedSrcSize;
{ ZSTD_parameters const params = ZSTD_getParams(zwc->compressionLevel, pledgedSrcSize, dictSize);
size_t initErr;
{ unsigned initErr = 0;
ZSTD_parameters const params = ZSTD_getParams(zwc->compressionLevel, pledgedSrcSize, dictSize);
ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams();
if (!cctxParams) return Z_STREAM_ERROR;
LOG_WRAPPERC("pledgedSrcSize=%d windowLog=%d chainLog=%d hashLog=%d searchLog=%d minMatch=%d strategy=%d\n",
(int)pledgedSrcSize, params.cParams.windowLog, params.cParams.chainLog, params.cParams.hashLog, params.cParams.searchLog, params.cParams.minMatch, params.cParams.strategy);
initErr = ZSTD_initCStream_advanced(zwc->zbc, dict, dictSize, params, pledgedSrcSize);
if (ZSTD_isError(initErr)) return Z_STREAM_ERROR;
initErr |= ZSTD_isError(ZSTD_CCtx_reset(zwc->zbc, ZSTD_reset_session_only));
initErr |= ZSTD_isError(ZSTD_CCtxParams_init_advanced(cctxParams, params));
initErr |= ZSTD_isError(ZSTD_CCtx_setParametersUsingCCtxParams(zwc->zbc, cctxParams));
initErr |= ZSTD_isError(ZSTD_CCtx_setPledgedSrcSize(zwc->zbc, pledgedSrcSize));
initErr |= ZSTD_isError(ZSTD_CCtx_loadDictionary(zwc->zbc, dict, dictSize));
ZSTD_freeCCtxParams(cctxParams);
if (initErr) return Z_STREAM_ERROR;
}
return Z_OK;